diff --git a/cmd/cmd.go b/cmd/cmd.go index f59b08689..4141504a9 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -4,14 +4,15 @@ import ( "fmt" "os" + "github.com/gookit/color" + "github.com/spf13/cobra" "github.com/anchore/syft/syft/presenter" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/anchore/stereoscope" "github.com/anchore/syft/internal/config" - "github.com/anchore/syft/internal/format" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/logger" "github.com/anchore/syft/syft" @@ -49,8 +50,8 @@ func setGlobalCliOptions() { // scan options flag := "scope" rootCmd.Flags().StringP( - "scope", "s", scope.SquashedScope.String(), - fmt.Sprintf("selection of layers to catalog, options=%v", scope.Options)) + "scope", "s", source.SquashedScope.String(), + fmt.Sprintf("selection of layers to catalog, options=%v", source.AllScopes)) if err := viper.BindPFlag(flag, rootCmd.Flags().Lookup(flag)); err != nil { fmt.Printf("unable to bind flag '%s': %+v", flag, err) os.Exit(1) @@ -111,7 +112,7 @@ func logAppConfig() { if err != nil { log.Debugf("Could not display application config: %+v", err) } else { - log.Debugf("Application config:\n%+v", format.Magenta.Format(string(appCfgStr))) + log.Debugf("Application config:\n%+v", color.Magenta.Sprint(string(appCfgStr))) } } diff --git a/cmd/root.go b/cmd/root.go index ffa936f90..7b72fafdd 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -92,7 +92,7 @@ func startWorker(userInput string) <-chan error { } } - catalog, scope, distro, err := syft.Catalog(userInput, appConfig.ScopeOpt) + src, catalog, distro, err := syft.Catalog(userInput, appConfig.ScopeOpt) if err != nil { errs <- fmt.Errorf("failed to catalog input: %+v", err) return @@ -100,7 +100,7 @@ func startWorker(userInput string) <-chan error { bus.Publish(partybus.Event{ Type: event.CatalogerFinished, - Value: presenter.GetPresenter(appConfig.PresenterOpt, *scope, catalog, distro), + Value: presenter.GetPresenter(appConfig.PresenterOpt, src.Metadata, catalog, distro), }) }() return errs diff --git a/go.sum b/go.sum index 55021990b..e6ba371c3 100644 --- a/go.sum +++ b/go.sum @@ -124,16 +124,12 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e h1:kty6r0R2JeaNPeWKSYDC+HW3hkqwFh4PP5TQ8pUPYFw= -github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e/go.mod h1:iYuIG0Nai8dR0ri3LhZQKUyO1loxUWAGvoWhXDmjy1A= github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 h1:xbeIbn5F52JVx3RUIajxCj8b0y+9lywspql4sFhcxWQ= github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12/go.mod h1:juoyWXIj7sJ1IDl4E/KIfyLtovbs5XQVSIdaQifFQT8= github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0vW0nnNKJfJieyH/TZ9UYAnTZs5/gHTdAe8= github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E= -github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe h1:m4NSyTo2fVUoUHAV/ZVqE/PFMr/y8oz9HRrhWLk9It0= -github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E= github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409 h1:xKSpDRjmYrEFrdMeDh4AuSUAFc99pdro6YFBKxy2um0= github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= @@ -164,8 +160,6 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4= github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/bmatcuk/doublestar v1.3.1 h1:rT8rxDPsavp9G+4ZULzqhhUSaI/OPsTZNG88Z3i0xvY= -github.com/bmatcuk/doublestar v1.3.1/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/bmatcuk/doublestar v1.3.3 h1:pVP1d49CcQQaNOl+PI6sPybIrIOD/6sux31PFdmhTH0= github.com/bmatcuk/doublestar v1.3.3/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U= @@ -301,8 +295,6 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-test/deep v1.0.6 h1:UHSEyLZUwX9Qoi99vVwvewiMC8mM2bf7XEM2nqvzEn8= -github.com/go-test/deep v1.0.6/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g= diff --git a/internal/config/config.go b/internal/config/config.go index f0faa32f5..91a4fa770 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -8,7 +8,7 @@ import ( "github.com/adrg/xdg" "github.com/anchore/syft/internal" "github.com/anchore/syft/syft/presenter" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/mitchellh/go-homedir" "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -23,7 +23,7 @@ type Application struct { ConfigPath string PresenterOpt presenter.Option Output string `mapstructure:"output"` - ScopeOpt scope.Option + ScopeOpt source.Scope Scope string `mapstructure:"scope"` Quiet bool `mapstructure:"quiet"` Log Logging `mapstructure:"log"` @@ -79,9 +79,9 @@ func (cfg *Application) Build() error { } cfg.PresenterOpt = presenterOption - // set the scope - scopeOption := scope.ParseOption(cfg.Scope) - if scopeOption == scope.UnknownScope { + // set the source + scopeOption := source.ParseScope(cfg.Scope) + if scopeOption == source.UnknownScope { return fmt.Errorf("bad --scope value '%s'", cfg.Scope) } cfg.ScopeOpt = scopeOption diff --git a/internal/constants.go b/internal/constants.go index 249eaf148..61fc3a2e4 100644 --- a/internal/constants.go +++ b/internal/constants.go @@ -1,4 +1,4 @@ package internal -// note: do not change this +// ApplicationName is the non-capitalized name of the application (do not change this) const ApplicationName = "syft" diff --git a/internal/docs.go b/internal/docs.go new file mode 100644 index 000000000..d06d47f6a --- /dev/null +++ b/internal/docs.go @@ -0,0 +1,4 @@ +/* +Package internal contains miscellaneous functions and objects useful within syft but should not be used externally. +*/ +package internal diff --git a/internal/file/glob_match.go b/internal/file/glob_match.go index 81575de24..6befe32e4 100644 --- a/internal/file/glob_match.go +++ b/internal/file/glob_match.go @@ -1,5 +1,6 @@ package file +// GlobMatch evaluates the given glob pattern against the given "name" string, indicating if there is a match or not. // Source: https://research.swtch.com/glob.go func GlobMatch(pattern, name string) bool { px := 0 diff --git a/internal/file/opener.go b/internal/file/opener.go index c79e2a845..969930f11 100644 --- a/internal/file/opener.go +++ b/internal/file/opener.go @@ -5,10 +5,12 @@ import ( "os" ) +// Opener is an object that stores a path to later be opened as a file. type Opener struct { path string } +// Open the stored path as a io.ReadCloser. func (o Opener) Open() (io.ReadCloser, error) { return os.Open(o.path) } diff --git a/internal/file/zip_file_manifest.go b/internal/file/zip_file_manifest.go index 7b19c2070..d7b7b80be 100644 --- a/internal/file/zip_file_manifest.go +++ b/internal/file/zip_file_manifest.go @@ -12,16 +12,20 @@ import ( "github.com/anchore/syft/internal/log" ) +// ZipFileManifest is a collection of paths and their file metadata. type ZipFileManifest map[string]os.FileInfo +// newZipManifest creates an empty ZipFileManifest. func newZipManifest() ZipFileManifest { return make(ZipFileManifest) } +// Add a new path and it's file metadata to the collection. func (z ZipFileManifest) Add(entry string, info os.FileInfo) { z[entry] = info } +// GlobMatch returns the path keys that match the given value(s). func (z ZipFileManifest) GlobMatch(patterns ...string) []string { uniqueMatches := internal.NewStringSet() @@ -43,6 +47,7 @@ func (z ZipFileManifest) GlobMatch(patterns ...string) []string { return results } +// NewZipFileManifest creates and returns a new ZipFileManifest populated with path and metadata from the given zip archive path. func NewZipFileManifest(archivePath string) (ZipFileManifest, error) { zipReader, err := zip.OpenReader(archivePath) manifest := newZipManifest() @@ -62,6 +67,7 @@ func NewZipFileManifest(archivePath string) (ZipFileManifest, error) { return manifest, nil } +// normalizeZipEntryName takes the given path entry and ensures it is prefixed with "/". func normalizeZipEntryName(entry string) string { if !strings.HasPrefix(entry, "/") { return "/" + entry diff --git a/internal/file/zip_file_traversal.go b/internal/file/zip_file_traversal.go index 0f8fe149e..742a88dbc 100644 --- a/internal/file/zip_file_traversal.go +++ b/internal/file/zip_file_traversal.go @@ -15,6 +15,7 @@ import ( ) const ( + // represents the order of bytes _ = iota KB = 1 << (10 * iota) MB @@ -33,6 +34,7 @@ func newZipTraverseRequest(paths ...string) zipTraversalRequest { return results } +// TraverseFilesInZip enumerates all paths stored within a zip archive using the visitor pattern. func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths ...string) error { request := newZipTraverseRequest(paths...) @@ -63,6 +65,7 @@ func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths return nil } +// ExtractFromZipToUniqueTempFile extracts select paths for the given archive to a temporary directory, returning file openers for each file extracted. func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (map[string]Opener, error) { results := make(map[string]Opener) @@ -121,6 +124,7 @@ func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (m return results, TraverseFilesInZip(archivePath, visitor, paths...) } +// ContentsFromZip extracts select paths for the given archive and returns a set of string contents for each path. func ContentsFromZip(archivePath string, paths ...string) (map[string]string, error) { results := make(map[string]string) @@ -162,6 +166,7 @@ func ContentsFromZip(archivePath string, paths ...string) (map[string]string, er return results, TraverseFilesInZip(archivePath, visitor, paths...) } +// UnzipToDir extracts a zip archive to a target directory. func UnzipToDir(archivePath, targetDir string) error { visitor := func(file *zip.File) error { // the zip-slip attack protection is still being erroneously detected diff --git a/internal/format/color.go b/internal/format/color.go deleted file mode 100644 index fa1757c34..000000000 --- a/internal/format/color.go +++ /dev/null @@ -1,21 +0,0 @@ -package format - -import "fmt" - -const ( - DefaultColor Color = iota + 30 - Red - Green - Yellow - Blue - Magenta - Cyan - White -) - -type Color uint8 - -// TODO: not cross platform (windows...) -func (c Color) Format(s string) string { - return fmt.Sprintf("\x1b[%dm%s\x1b[0m", c, s) -} diff --git a/internal/log/log.go b/internal/log/log.go index 5ea65c839..0dd2199c6 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -1,37 +1,49 @@ +/* +Package log contains the singleton object and helper functions for facilitating logging within the syft library. +*/ package log import "github.com/anchore/syft/syft/logger" +// Log is the singleton used to facilitate logging internally within syft var Log logger.Logger = &nopLogger{} +// Errorf takes a formatted template string and template arguments for the error logging level. func Errorf(format string, args ...interface{}) { Log.Errorf(format, args...) } +// Error logs the given arguments at the error logging level. func Error(args ...interface{}) { Log.Error(args...) } +// Warnf takes a formatted template string and template arguments for the warning logging level. func Warnf(format string, args ...interface{}) { Log.Warnf(format, args...) } +// Warn logs the given arguments at the warning logging level. func Warn(args ...interface{}) { Log.Warn(args...) } +// Infof takes a formatted template string and template arguments for the info logging level. func Infof(format string, args ...interface{}) { Log.Infof(format, args...) } +// Info logs the given arguments at the info logging level. func Info(args ...interface{}) { Log.Info(args...) } +// Debugf takes a formatted template string and template arguments for the debug logging level. func Debugf(format string, args ...interface{}) { Log.Debugf(format, args...) } +// Debug logs the given arguments at the debug logging level. func Debug(args ...interface{}) { Log.Debug(args...) } diff --git a/internal/logger/doc.go b/internal/logger/doc.go new file mode 100644 index 000000000..59647a80c --- /dev/null +++ b/internal/logger/doc.go @@ -0,0 +1,4 @@ +/* +Package logger contains implementations for the syft.logger.Logger interface. +*/ +package logger diff --git a/internal/logger/logrus.go b/internal/logger/logrus.go index 006e5f9c9..96fad888c 100644 --- a/internal/logger/logrus.go +++ b/internal/logger/logrus.go @@ -10,6 +10,7 @@ import ( prefixed "github.com/x-cray/logrus-prefixed-formatter" ) +// LogrusConfig contains all configurable values for the Logrus logger type LogrusConfig struct { EnableConsole bool EnableFile bool @@ -18,16 +19,19 @@ type LogrusConfig struct { FileLocation string } +// LogrusLogger contains all runtime values for using Logrus with the configured output target and input configuration values. type LogrusLogger struct { Config LogrusConfig Logger *logrus.Logger Output io.Writer } +// LogrusNestedLogger is a wrapper for Logrus to enable nested logging configuration (loggers that always attach key-value pairs to all log entries) type LogrusNestedLogger struct { Logger *logrus.Entry } +// NewLogrusLogger creates a new LogrusLogger with the given configuration func NewLogrusLogger(cfg LogrusConfig) *LogrusLogger { appLogger := logrus.New() @@ -76,66 +80,82 @@ func NewLogrusLogger(cfg LogrusConfig) *LogrusLogger { } } +// Debugf takes a formatted template string and template arguments for the debug logging level. func (l *LogrusLogger) Debugf(format string, args ...interface{}) { l.Logger.Debugf(format, args...) } +// Infof takes a formatted template string and template arguments for the info logging level. func (l *LogrusLogger) Infof(format string, args ...interface{}) { l.Logger.Infof(format, args...) } +// Warnf takes a formatted template string and template arguments for the warning logging level. func (l *LogrusLogger) Warnf(format string, args ...interface{}) { l.Logger.Warnf(format, args...) } +// Errorf takes a formatted template string and template arguments for the error logging level. func (l *LogrusLogger) Errorf(format string, args ...interface{}) { l.Logger.Errorf(format, args...) } +// Debug logs the given arguments at the debug logging level. func (l *LogrusLogger) Debug(args ...interface{}) { l.Logger.Debug(args...) } +// Info logs the given arguments at the info logging level. func (l *LogrusLogger) Info(args ...interface{}) { l.Logger.Info(args...) } +// Warn logs the given arguments at the warning logging level. func (l *LogrusLogger) Warn(args ...interface{}) { l.Logger.Warn(args...) } +// Error logs the given arguments at the error logging level. func (l *LogrusLogger) Error(args ...interface{}) { l.Logger.Error(args...) } +// Debugf takes a formatted template string and template arguments for the debug logging level. func (l *LogrusNestedLogger) Debugf(format string, args ...interface{}) { l.Logger.Debugf(format, args...) } +// Infof takes a formatted template string and template arguments for the info logging level. func (l *LogrusNestedLogger) Infof(format string, args ...interface{}) { l.Logger.Infof(format, args...) } +// Warnf takes a formatted template string and template arguments for the warning logging level. func (l *LogrusNestedLogger) Warnf(format string, args ...interface{}) { l.Logger.Warnf(format, args...) } +// Errorf takes a formatted template string and template arguments for the error logging level. func (l *LogrusNestedLogger) Errorf(format string, args ...interface{}) { l.Logger.Errorf(format, args...) } +// Debug logs the given arguments at the debug logging level. func (l *LogrusNestedLogger) Debug(args ...interface{}) { l.Logger.Debug(args...) } +// Info logs the given arguments at the info logging level. func (l *LogrusNestedLogger) Info(args ...interface{}) { l.Logger.Info(args...) } +// Warn logs the given arguments at the warning logging level. func (l *LogrusNestedLogger) Warn(args ...interface{}) { l.Logger.Warn(args...) } +// Error logs the given arguments at the error logging level. func (l *LogrusNestedLogger) Error(args ...interface{}) { l.Logger.Error(args...) } diff --git a/internal/stringset.go b/internal/stringset.go index 41518aaad..2bd68a4b1 100644 --- a/internal/stringset.go +++ b/internal/stringset.go @@ -1,11 +1,16 @@ package internal +import "sort" + +// StringSet represents a set of string types. type StringSet map[string]struct{} +// NewStringSet creates a new empty StringSet. func NewStringSet() StringSet { return make(StringSet) } +// NewStringSetFromSlice creates a StringSet populated with values from the given slice. func NewStringSetFromSlice(start []string) StringSet { ret := make(StringSet) for _, s := range start { @@ -14,19 +19,23 @@ func NewStringSetFromSlice(start []string) StringSet { return ret } +// Add a string to the set. func (s StringSet) Add(i string) { s[i] = struct{}{} } +// Remove a string from the set. func (s StringSet) Remove(i string) { delete(s, i) } +// Contains indicates if the given string is contained within the set. func (s StringSet) Contains(i string) bool { _, ok := s[i] return ok } +// ToSlice returns a sorted slice of strings that are contained within the set. func (s StringSet) ToSlice() []string { ret := make([]string, len(s)) idx := 0 @@ -34,5 +43,6 @@ func (s StringSet) ToSlice() []string { ret[idx] = v idx++ } + sort.Strings(ret) return ret } diff --git a/internal/version/build.go b/internal/version/build.go index e0c960228..3d6702c0c 100644 --- a/internal/version/build.go +++ b/internal/version/build.go @@ -1,3 +1,6 @@ +/* +Package version contains all build time metadata (version, build time, git commit, etc). +*/ package version import ( diff --git a/internal/version/update.go b/internal/version/update.go index bdba75f4d..3aa13ef3a 100644 --- a/internal/version/update.go +++ b/internal/version/update.go @@ -18,6 +18,7 @@ var latestAppVersionURL = struct { path: fmt.Sprintf("/%s/releases/latest/VERSION", internal.ApplicationName), } +// IsUpdateAvailable indicates if there is a newer application version available, and if so, what the new version is. func IsUpdateAvailable() (bool, string, error) { currentVersionStr := FromBuild().Version currentVersion, err := hashiVersion.NewVersion(currentVersionStr) diff --git a/schema/json/schema.json b/schema/json/schema.json index 954c58d92..edfef558c 100644 --- a/schema/json/schema.json +++ b/schema/json/schema.json @@ -5,10 +5,10 @@ "items": { "properties": { "foundBy": { - "items": { - "type": "string" - }, - "type": "array" + "type": "string" + }, + "language": { + "type": "string" }, "licenses": { "anyOf": [ @@ -25,26 +25,18 @@ }, "locations": { "items": { - "anyOf": [ - { + "properties": { + "layerID": { "type": "string" }, - { - "properties": { - "layerIndex": { - "type": "integer" - }, - "path": { - "type": "string" - } - }, - "required": [ - "layerIndex", - "path" - ], - "type": "object" + "path": { + "type": "string" } - ] + }, + "required": [ + "path" + ], + "type": "object" }, "type": "array" }, @@ -315,6 +307,9 @@ }, "type": "object" }, + "metadataType": { + "type": "string" + }, "name": { "type": "string" }, @@ -327,8 +322,10 @@ }, "required": [ "foundBy", + "language", "licenses", "locations", + "metadataType", "name", "type", "version" @@ -337,6 +334,40 @@ }, "type": "array" }, + "descriptor": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + } + }, + "required": [ + "name", + "version" + ], + "type": "object" + }, + "distro": { + "properties": { + "idLike": { + "type": "string" + }, + "name": { + "type": "string" + }, + "version": { + "type": "string" + } + }, + "required": [ + "idLike", + "name", + "version" + ], + "type": "object" + }, "source": { "properties": { "target": { @@ -374,6 +405,9 @@ "mediaType": { "type": "string" }, + "scope": { + "type": "string" + }, "size": { "type": "integer" }, @@ -382,14 +416,19 @@ "type": "string" }, "type": "array" + }, + "userInput": { + "type": "string" } }, "required": [ "digest", "layers", "mediaType", + "scope", "size", - "tags" + "tags", + "userInput" ], "type": "object" } @@ -408,6 +447,8 @@ }, "required": [ "artifacts", + "descriptor", + "distro", "source" ], "type": "object" diff --git a/syft/cataloger/catalog.go b/syft/cataloger/catalog.go index 022f81855..262fbdbb3 100644 --- a/syft/cataloger/catalog.go +++ b/syft/cataloger/catalog.go @@ -5,7 +5,7 @@ import ( "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/hashicorp/go-multierror" "github.com/wagoodman/go-partybus" "github.com/wagoodman/go-progress" @@ -32,11 +32,11 @@ func newMonitor() (*progress.Manual, *progress.Manual) { return &filesProcessed, &packagesDiscovered } -// Catalog a given scope (container image or filesystem) with the given catalogers, returning all discovered packages. +// Catalog a given source (container image or filesystem) with the given catalogers, returning all discovered packages. // In order to efficiently retrieve contents from a underlying container image the content fetch requests are // done in bulk. Specifically, all files of interest are collected from each catalogers and accumulated into a single // request. -func Catalog(resolver scope.Resolver, catalogers ...Cataloger) (*pkg.Catalog, error) { +func Catalog(resolver source.Resolver, catalogers ...Cataloger) (*pkg.Catalog, error) { catalog := pkg.NewCatalog() filesProcessed, packagesDiscovered := newMonitor() diff --git a/syft/cataloger/cataloger.go b/syft/cataloger/cataloger.go index 0ee9f5be2..aba71dfb9 100644 --- a/syft/cataloger/cataloger.go +++ b/syft/cataloger/cataloger.go @@ -15,7 +15,7 @@ import ( "github.com/anchore/syft/syft/cataloger/rpmdb" "github.com/anchore/syft/syft/cataloger/ruby" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) // Cataloger describes behavior for an object to participate in parsing container image or file system @@ -25,7 +25,7 @@ type Cataloger interface { // Name returns a string that uniquely describes a cataloger Name() string // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source. - Catalog(resolver scope.Resolver) ([]pkg.Package, error) + Catalog(resolver source.Resolver) ([]pkg.Package, error) } // ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages. diff --git a/syft/cataloger/common/generic_cataloger.go b/syft/cataloger/common/generic_cataloger.go index e21270093..a35544af9 100644 --- a/syft/cataloger/common/generic_cataloger.go +++ b/syft/cataloger/common/generic_cataloger.go @@ -6,10 +6,9 @@ package common import ( "strings" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) // GenericCataloger implements the Catalog interface and is responsible for dispatching the proper parser function for @@ -17,8 +16,8 @@ import ( type GenericCataloger struct { globParsers map[string]ParserFn pathParsers map[string]ParserFn - selectedFiles []file.Reference - parsers map[file.Reference]ParserFn + selectedFiles []source.Location + parsers map[source.Location]ParserFn upstreamCataloger string } @@ -27,8 +26,8 @@ func NewGenericCataloger(pathParsers map[string]ParserFn, globParsers map[string return &GenericCataloger{ globParsers: globParsers, pathParsers: pathParsers, - selectedFiles: make([]file.Reference, 0), - parsers: make(map[file.Reference]ParserFn), + selectedFiles: make([]source.Location, 0), + parsers: make(map[source.Location]ParserFn), upstreamCataloger: upstreamCataloger, } } @@ -39,7 +38,7 @@ func (c *GenericCataloger) Name() string { } // register pairs a set of file references with a parser function for future cataloging (when the file contents are resolved) -func (c *GenericCataloger) register(files []file.Reference, parser ParserFn) { +func (c *GenericCataloger) register(files []source.Location, parser ParserFn) { c.selectedFiles = append(c.selectedFiles, files...) for _, f := range files { c.parsers[f] = parser @@ -48,14 +47,14 @@ func (c *GenericCataloger) register(files []file.Reference, parser ParserFn) { // clear deletes all registered file-reference-to-parser-function pairings from former SelectFiles() and register() calls func (c *GenericCataloger) clear() { - c.selectedFiles = make([]file.Reference, 0) - c.parsers = make(map[file.Reference]ParserFn) + c.selectedFiles = make([]source.Location, 0) + c.parsers = make(map[source.Location]ParserFn) } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source. -func (c *GenericCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) { +func (c *GenericCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { fileSelection := c.selectFiles(resolver) - contents, err := resolver.MultipleFileContentsByRef(fileSelection...) + contents, err := resolver.MultipleFileContentsByLocation(fileSelection) if err != nil { return nil, err } @@ -63,10 +62,10 @@ func (c *GenericCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, erro } // SelectFiles takes a set of file trees and resolves and file references of interest for future cataloging -func (c *GenericCataloger) selectFiles(resolver scope.FileResolver) []file.Reference { +func (c *GenericCataloger) selectFiles(resolver source.FileResolver) []source.Location { // select by exact path for path, parser := range c.pathParsers { - files, err := resolver.FilesByPath(file.Path(path)) + files, err := resolver.FilesByPath(path) if err != nil { log.Warnf("cataloger failed to select files by path: %+v", err) } @@ -90,28 +89,28 @@ func (c *GenericCataloger) selectFiles(resolver scope.FileResolver) []file.Refer } // catalog takes a set of file contents and uses any configured parser functions to resolve and return discovered packages -func (c *GenericCataloger) catalog(contents map[file.Reference]string) ([]pkg.Package, error) { +func (c *GenericCataloger) catalog(contents map[source.Location]string) ([]pkg.Package, error) { defer c.clear() packages := make([]pkg.Package, 0) - for reference, parser := range c.parsers { - content, ok := contents[reference] + for location, parser := range c.parsers { + content, ok := contents[location] if !ok { - log.Warnf("cataloger '%s' missing file content: %+v", c.upstreamCataloger, reference) + log.Warnf("cataloger '%s' missing file content: %+v", c.upstreamCataloger, location) continue } - entries, err := parser(string(reference.Path), strings.NewReader(content)) + entries, err := parser(location.Path, strings.NewReader(content)) if err != nil { // TODO: should we fail? or only log? - log.Warnf("cataloger '%s' failed to parse entries (reference=%+v): %+v", c.upstreamCataloger, reference, err) + log.Warnf("cataloger '%s' failed to parse entries (location=%+v): %+v", c.upstreamCataloger, location, err) continue } for _, entry := range entries { entry.FoundBy = c.upstreamCataloger - entry.Source = []file.Reference{reference} + entry.Locations = []source.Location{location} packages = append(packages, entry) } diff --git a/syft/cataloger/common/generic_cataloger_test.go b/syft/cataloger/common/generic_cataloger_test.go index 6083d390a..e82f507a8 100644 --- a/syft/cataloger/common/generic_cataloger_test.go +++ b/syft/cataloger/common/generic_cataloger_test.go @@ -6,48 +6,50 @@ import ( "io/ioutil" "testing" - "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/syft/pkg" ) type testResolverMock struct { - contents map[file.Reference]string + contents map[source.Location]string } func newTestResolver() *testResolverMock { return &testResolverMock{ - contents: make(map[file.Reference]string), + contents: make(map[source.Location]string), } } -func (r *testResolverMock) FileContentsByRef(_ file.Reference) (string, error) { +func (r *testResolverMock) FileContentsByLocation(_ source.Location) (string, error) { return "", fmt.Errorf("not implemented") } -func (r *testResolverMock) MultipleFileContentsByRef(_ ...file.Reference) (map[file.Reference]string, error) { +func (r *testResolverMock) MultipleFileContentsByLocation([]source.Location) (map[source.Location]string, error) { return r.contents, nil } -func (r *testResolverMock) FilesByPath(paths ...file.Path) ([]file.Reference, error) { - results := make([]file.Reference, len(paths)) +func (r *testResolverMock) FilesByPath(paths ...string) ([]source.Location, error) { + results := make([]source.Location, len(paths)) for idx, p := range paths { - results[idx] = file.NewFileReference(p) + results[idx] = source.NewLocation(p) r.contents[results[idx]] = fmt.Sprintf("%s file contents!", p) } return results, nil } -func (r *testResolverMock) FilesByGlob(_ ...string) ([]file.Reference, error) { +func (r *testResolverMock) FilesByGlob(_ ...string) ([]source.Location, error) { path := "/a-path.txt" - ref := file.NewFileReference(file.Path(path)) - r.contents[ref] = fmt.Sprintf("%s file contents!", path) - return []file.Reference{ref}, nil + location := source.NewLocation(path) + r.contents[location] = fmt.Sprintf("%s file contents!", path) + return []source.Location{location}, nil } -func (r *testResolverMock) RelativeFileByPath(_ file.Reference, _ string) (*file.Reference, error) { - return nil, fmt.Errorf("not implemented") +func (r *testResolverMock) RelativeFileByPath(_ source.Location, _ string) *source.Location { + panic(fmt.Errorf("not implemented")) + return nil } func parser(_ string, reader io.Reader) ([]pkg.Package, error) { @@ -94,8 +96,8 @@ func TestGenericCataloger(t *testing.T) { } for _, p := range actualPkgs { - ref := p.Source[0] - exP, ok := expectedPkgs[string(ref.Path)] + ref := p.Locations[0] + exP, ok := expectedPkgs[ref.Path] if !ok { t.Errorf("missing expected pkg: ref=%+v", ref) continue @@ -106,7 +108,7 @@ func TestGenericCataloger(t *testing.T) { } if exP.Name != p.Name { - t.Errorf("bad contents mapping: %+v", p.Source) + t.Errorf("bad contents mapping: %+v", p.Locations) } } } diff --git a/syft/cataloger/deb/cataloger.go b/syft/cataloger/deb/cataloger.go index c10f0dd71..cbd31c03b 100644 --- a/syft/cataloger/deb/cataloger.go +++ b/syft/cataloger/deb/cataloger.go @@ -7,11 +7,11 @@ import ( "fmt" "io" "path" + "path/filepath" "strings" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) const ( @@ -33,30 +33,30 @@ func (c *Cataloger) Name() string { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files. -func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) { +func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { dbFileMatches, err := resolver.FilesByGlob(dpkgStatusGlob) if err != nil { return nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err) } var pkgs []pkg.Package - for _, dbRef := range dbFileMatches { - dbContents, err := resolver.FileContentsByRef(dbRef) + for _, dbLocation := range dbFileMatches { + dbContents, err := resolver.FileContentsByLocation(dbLocation) if err != nil { return nil, err } pkgs, err = parseDpkgStatus(strings.NewReader(dbContents)) if err != nil { - return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbRef.Path, err) + return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.Path, err) } - md5ContentsByName, md5RefsByName, err := fetchMd5Contents(resolver, dbRef, pkgs) + md5ContentsByName, md5RefsByName, err := fetchMd5Contents(resolver, dbLocation, pkgs) if err != nil { return nil, fmt.Errorf("unable to find dpkg md5 contents: %w", err) } - copyrightContentsByName, copyrightRefsByName, err := fetchCopyrightContents(resolver, dbRef, pkgs) + copyrightContentsByName, copyrightRefsByName, err := fetchCopyrightContents(resolver, dbLocation, pkgs) if err != nil { return nil, fmt.Errorf("unable to find dpkg copyright contents: %w", err) } @@ -64,7 +64,7 @@ func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) { for i := range pkgs { p := &pkgs[i] p.FoundBy = c.Name() - p.Source = []file.Reference{dbRef} + p.Locations = []source.Location{dbLocation} if md5Reader, ok := md5ContentsByName[md5Key(*p)]; ok { // attach the file list @@ -74,7 +74,7 @@ func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) { // keep a record of the file where this was discovered if ref, ok := md5RefsByName[md5Key(*p)]; ok { - p.Source = append(p.Source, ref) + p.Locations = append(p.Locations, ref) } } @@ -85,7 +85,7 @@ func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) { // keep a record of the file where this was discovered if ref, ok := copyrightRefsByName[p.Name]; ok { - p.Source = append(p.Source, ref) + p.Locations = append(p.Locations, ref) } } } @@ -93,93 +93,82 @@ func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) { return pkgs, nil } -func fetchMd5Contents(resolver scope.Resolver, dbRef file.Reference, pkgs []pkg.Package) (map[string]io.Reader, map[string]file.Reference, error) { +func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) { // fetch all MD5 file contents. This approach is more efficient than fetching each MD5 file one at a time - var md5FileMatches []file.Reference - var nameByRef = make(map[file.Reference]string) - parentPath, err := dbRef.Path.ParentPath() - if err != nil { - return nil, nil, fmt.Errorf("unable to find parent of path=%+v: %w", dbRef.Path, err) - } + var md5FileMatches []source.Location + var nameByRef = make(map[source.Location]string) + parentPath := filepath.Dir(dbLocation.Path) + for _, p := range pkgs { // look for /var/lib/dpkg/info/NAME:ARCH.md5sums name := md5Key(p) - md5sumPath := path.Join(string(parentPath), "info", name+md5sumsExt) - md5SumRef, err := resolver.RelativeFileByPath(dbRef, md5sumPath) - if err != nil { - return nil, nil, fmt.Errorf("unable to find relative md5sum from path=%+v: %w", dbRef.Path, err) - } + md5sumPath := path.Join(parentPath, "info", name+md5sumsExt) + md5SumLocation := resolver.RelativeFileByPath(dbLocation, md5sumPath) - if md5SumRef == nil { + if md5SumLocation == nil { // the most specific key did not work, fallback to just the name // look for /var/lib/dpkg/info/NAME.md5sums name := p.Name - md5sumPath := path.Join(string(parentPath), "info", name+md5sumsExt) - md5SumRef, err = resolver.RelativeFileByPath(dbRef, md5sumPath) - if err != nil { - return nil, nil, fmt.Errorf("unable to find relative md5sum from path=%+v: %w", dbRef.Path, err) - } + md5sumPath := path.Join(parentPath, "info", name+md5sumsExt) + md5SumLocation = resolver.RelativeFileByPath(dbLocation, md5sumPath) } // we should have at least one reference - if md5SumRef != nil { - md5FileMatches = append(md5FileMatches, *md5SumRef) - nameByRef[*md5SumRef] = name + if md5SumLocation != nil { + md5FileMatches = append(md5FileMatches, *md5SumLocation) + nameByRef[*md5SumLocation] = name } } // fetch the md5 contents - md5ContentsByRef, err := resolver.MultipleFileContentsByRef(md5FileMatches...) + md5ContentsByLocation, err := resolver.MultipleFileContentsByLocation(md5FileMatches) if err != nil { return nil, nil, err } // organize content results and refs by a combination of name and architecture var contentsByName = make(map[string]io.Reader) - var refsByName = make(map[string]file.Reference) - for ref, contents := range md5ContentsByRef { - name := nameByRef[ref] + var refsByName = make(map[string]source.Location) + for location, contents := range md5ContentsByLocation { + name := nameByRef[location] contentsByName[name] = strings.NewReader(contents) - refsByName[name] = ref + refsByName[name] = location } return contentsByName, refsByName, nil } -func fetchCopyrightContents(resolver scope.Resolver, dbRef file.Reference, pkgs []pkg.Package) (map[string]io.Reader, map[string]file.Reference, error) { +func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) { // fetch all copyright file contents. This approach is more efficient than fetching each copyright file one at a time - var copyrightFileMatches []file.Reference - var nameByRef = make(map[file.Reference]string) + var copyrightFileMatches []source.Location + var nameByLocation = make(map[source.Location]string) for _, p := range pkgs { // look for /usr/share/docs/NAME/copyright files name := p.Name copyrightPath := path.Join(docsPath, name, "copyright") - copyrightRef, err := resolver.RelativeFileByPath(dbRef, copyrightPath) - if err != nil { - return nil, nil, fmt.Errorf("unable to find relative copyright from path=%+v: %w", dbRef.Path, err) - } + copyrightLocation := resolver.RelativeFileByPath(dbLocation, copyrightPath) // we may not have a copyright file for each package, ignore missing files - if copyrightRef != nil { - copyrightFileMatches = append(copyrightFileMatches, *copyrightRef) - nameByRef[*copyrightRef] = name + if copyrightLocation != nil { + copyrightFileMatches = append(copyrightFileMatches, *copyrightLocation) + nameByLocation[*copyrightLocation] = name } } // fetch the copyright contents - copyrightContentsByRef, err := resolver.MultipleFileContentsByRef(copyrightFileMatches...) + copyrightContentsByLocation, err := resolver.MultipleFileContentsByLocation(copyrightFileMatches) if err != nil { return nil, nil, err } // organize content results and refs by package name var contentsByName = make(map[string]io.Reader) - var refsByName = make(map[string]file.Reference) - for ref, contents := range copyrightContentsByRef { - name := nameByRef[ref] + var refsByName = make(map[string]source.Location) + for location, contents := range copyrightContentsByLocation { + name := nameByLocation[location] contentsByName[name] = strings.NewReader(contents) - refsByName[name] = ref + refsByName[name] = location } return contentsByName, refsByName, nil diff --git a/syft/cataloger/deb/cataloger_test.go b/syft/cataloger/deb/cataloger_test.go index fb36c59e5..9a3e390c7 100644 --- a/syft/cataloger/deb/cataloger_test.go +++ b/syft/cataloger/deb/cataloger_test.go @@ -5,7 +5,7 @@ import ( "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/go-test/deep" ) @@ -54,7 +54,7 @@ func TestDpkgCataloger(t *testing.T) { img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-dpkg") defer cleanup() - s, err := scope.NewScopeFromImage(img, scope.AllLayersScope) + s, err := source.NewFromImage(img, source.AllLayersScope, "") if err != nil { t.Fatal(err) } @@ -77,11 +77,11 @@ func TestDpkgCataloger(t *testing.T) { for idx := range actual { a := &actual[idx] // we will test the sources separately - var sourcesList = make([]string, len(a.Source)) - for i, s := range a.Source { - sourcesList[i] = string(s.Path) + var sourcesList = make([]string, len(a.Locations)) + for i, s := range a.Locations { + sourcesList[i] = s.Path } - a.Source = nil + a.Locations = nil for _, d := range deep.Equal(sourcesList, test.sources[a.Name]) { t.Errorf("diff: %+v", d) diff --git a/syft/cataloger/python/package_cataloger.go b/syft/cataloger/python/package_cataloger.go index 051933987..0d42a4473 100644 --- a/syft/cataloger/python/package_cataloger.go +++ b/syft/cataloger/python/package_cataloger.go @@ -8,11 +8,9 @@ import ( "github.com/anchore/syft/internal/log" - "github.com/anchore/stereoscope/pkg/file" - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) const ( @@ -33,9 +31,9 @@ func (c *PackageCataloger) Name() string { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations. -func (c *PackageCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) { +func (c *PackageCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { // nolint:prealloc - var fileMatches []file.Reference + var fileMatches []source.Location for _, glob := range []string{eggMetadataGlob, wheelMetadataGlob} { matches, err := resolver.FilesByGlob(glob) @@ -46,10 +44,10 @@ func (c *PackageCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, erro } var pkgs []pkg.Package - for _, ref := range fileMatches { - p, err := c.catalogEggOrWheel(resolver, ref) + for _, location := range fileMatches { + p, err := c.catalogEggOrWheel(resolver, location) if err != nil { - return nil, fmt.Errorf("unable to catalog python package=%+v: %w", ref.Path, err) + return nil, fmt.Errorf("unable to catalog python package=%+v: %w", location.Path, err) } if p != nil { pkgs = append(pkgs, *p) @@ -59,8 +57,8 @@ func (c *PackageCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, erro } // catalogEggOrWheel takes the primary metadata file reference and returns the python package it represents. -func (c *PackageCataloger) catalogEggOrWheel(resolver scope.Resolver, metadataRef file.Reference) (*pkg.Package, error) { - metadata, sources, err := c.assembleEggOrWheelMetadata(resolver, metadataRef) +func (c *PackageCataloger) catalogEggOrWheel(resolver source.Resolver, metadataLocation source.Location) (*pkg.Package, error) { + metadata, sources, err := c.assembleEggOrWheelMetadata(resolver, metadataLocation) if err != nil { return nil, err } @@ -74,7 +72,7 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver scope.Resolver, metadataRe Name: metadata.Name, Version: metadata.Version, FoundBy: c.Name(), - Source: sources, + Locations: sources, Licenses: licenses, Language: pkg.Python, Type: pkg.PythonPkg, @@ -84,22 +82,19 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver scope.Resolver, metadataRe } // fetchRecordFiles finds a corresponding RECORD file for the given python package metadata file and returns the set of file records contained. -func (c *PackageCataloger) fetchRecordFiles(resolver scope.Resolver, metadataRef file.Reference) (files []pkg.PythonFileRecord, sources []file.Reference, err error) { +func (c *PackageCataloger) fetchRecordFiles(resolver source.Resolver, metadataLocation source.Location) (files []pkg.PythonFileRecord, sources []source.Location, err error) { // we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory // or for an image... for an image the METADATA file may be present within multiple layers, so it is important // to reconcile the RECORD path to the same layer (or the next adjacent lower layer). // lets find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure) - recordPath := filepath.Join(filepath.Dir(string(metadataRef.Path)), "RECORD") - recordRef, err := resolver.RelativeFileByPath(metadataRef, recordPath) - if err != nil { - return nil, nil, err - } + recordPath := filepath.Join(filepath.Dir(metadataLocation.Path), "RECORD") + recordRef := resolver.RelativeFileByPath(metadataLocation, recordPath) if recordRef != nil { sources = append(sources, *recordRef) - recordContents, err := resolver.FileContentsByRef(*recordRef) + recordContents, err := resolver.FileContentsByLocation(*recordRef) if err != nil { return nil, nil, err } @@ -116,22 +111,20 @@ func (c *PackageCataloger) fetchRecordFiles(resolver scope.Resolver, metadataRef } // fetchTopLevelPackages finds a corresponding top_level.txt file for the given python package metadata file and returns the set of package names contained. -func (c *PackageCataloger) fetchTopLevelPackages(resolver scope.Resolver, metadataRef file.Reference) (pkgs []string, sources []file.Reference, err error) { +func (c *PackageCataloger) fetchTopLevelPackages(resolver source.Resolver, metadataLocation source.Location) (pkgs []string, sources []source.Location, err error) { // a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages - parentDir := filepath.Dir(string(metadataRef.Path)) + parentDir := filepath.Dir(metadataLocation.Path) topLevelPath := filepath.Join(parentDir, "top_level.txt") - topLevelRef, err := resolver.RelativeFileByPath(metadataRef, topLevelPath) - if err != nil { - return nil, nil, err - } + topLevelRef := resolver.RelativeFileByPath(metadataLocation, topLevelPath) + if topLevelRef == nil { - log.Warnf("missing python package top_level.txt (package=%q)", string(metadataRef.Path)) + log.Warnf("missing python package top_level.txt (package=%q)", metadataLocation.Path) return nil, nil, nil } sources = append(sources, *topLevelRef) - topLevelContents, err := resolver.FileContentsByRef(*topLevelRef) + topLevelContents, err := resolver.FileContentsByLocation(*topLevelRef) if err != nil { return nil, nil, err } @@ -149,21 +142,21 @@ func (c *PackageCataloger) fetchTopLevelPackages(resolver scope.Resolver, metada } // assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from. -func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver scope.Resolver, metadataRef file.Reference) (*pkg.PythonPackageMetadata, []file.Reference, error) { - var sources = []file.Reference{metadataRef} +func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.Resolver, metadataLocation source.Location) (*pkg.PythonPackageMetadata, []source.Location, error) { + var sources = []source.Location{metadataLocation} - metadataContents, err := resolver.FileContentsByRef(metadataRef) + metadataContents, err := resolver.FileContentsByLocation(metadataLocation) if err != nil { return nil, nil, err } - metadata, err := parseWheelOrEggMetadata(metadataRef.Path, strings.NewReader(metadataContents)) + metadata, err := parseWheelOrEggMetadata(metadataLocation.Path, strings.NewReader(metadataContents)) if err != nil { return nil, nil, err } // attach any python files found for the given wheel/egg installation - r, s, err := c.fetchRecordFiles(resolver, metadataRef) + r, s, err := c.fetchRecordFiles(resolver, metadataLocation) if err != nil { return nil, nil, err } @@ -171,7 +164,7 @@ func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver scope.Resolver, m metadata.Files = r // attach any top-level package names found for the given wheel/egg installation - p, s, err := c.fetchTopLevelPackages(resolver, metadataRef) + p, s, err := c.fetchTopLevelPackages(resolver, metadataLocation) if err != nil { return nil, nil, err } diff --git a/syft/cataloger/python/package_cataloger_test.go b/syft/cataloger/python/package_cataloger_test.go index 91081e8b1..2ad16da73 100644 --- a/syft/cataloger/python/package_cataloger_test.go +++ b/syft/cataloger/python/package_cataloger_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/pkg" "github.com/go-test/deep" @@ -18,10 +18,10 @@ type pythonTestResolverMock struct { metadataReader io.Reader recordReader io.Reader topLevelReader io.Reader - metadataRef *file.Reference - recordRef *file.Reference - topLevelRef *file.Reference - contents map[file.Reference]string + metadataRef *source.Location + recordRef *source.Location + topLevelRef *source.Location + contents map[source.Location]string } func newTestResolver(metaPath, recordPath, topPath string) *pythonTestResolverMock { @@ -46,17 +46,17 @@ func newTestResolver(metaPath, recordPath, topPath string) *pythonTestResolverMo } } - var recordRef *file.Reference + var recordRef *source.Location if recordReader != nil { - ref := file.NewFileReference("test-fixtures/dist-info/RECORD") + ref := source.NewLocation("test-fixtures/dist-info/RECORD") recordRef = &ref } - var topLevelRef *file.Reference + var topLevelRef *source.Location if topLevelReader != nil { - ref := file.NewFileReference("test-fixtures/dist-info/top_level.txt") + ref := source.NewLocation("test-fixtures/dist-info/top_level.txt") topLevelRef = &ref } - metadataRef := file.NewFileReference("test-fixtures/dist-info/METADATA") + metadataRef := source.NewLocation("test-fixtures/dist-info/METADATA") return &pythonTestResolverMock{ recordReader: recordReader, metadataReader: metadataReader, @@ -64,11 +64,11 @@ func newTestResolver(metaPath, recordPath, topPath string) *pythonTestResolverMo metadataRef: &metadataRef, recordRef: recordRef, topLevelRef: topLevelRef, - contents: make(map[file.Reference]string), + contents: make(map[source.Location]string), } } -func (r *pythonTestResolverMock) FileContentsByRef(ref file.Reference) (string, error) { +func (r *pythonTestResolverMock) FileContentsByLocation(ref source.Location) (string, error) { switch { case r.topLevelRef != nil && ref.Path == r.topLevelRef.Path: b, err := ioutil.ReadAll(r.topLevelReader) @@ -92,25 +92,25 @@ func (r *pythonTestResolverMock) FileContentsByRef(ref file.Reference) (string, return "", fmt.Errorf("invalid value given") } -func (r *pythonTestResolverMock) MultipleFileContentsByRef(_ ...file.Reference) (map[file.Reference]string, error) { +func (r *pythonTestResolverMock) MultipleFileContentsByLocation(_ []source.Location) (map[source.Location]string, error) { return nil, fmt.Errorf("not implemented") } -func (r *pythonTestResolverMock) FilesByPath(_ ...file.Path) ([]file.Reference, error) { +func (r *pythonTestResolverMock) FilesByPath(_ ...string) ([]source.Location, error) { return nil, fmt.Errorf("not implemented") } -func (r *pythonTestResolverMock) FilesByGlob(_ ...string) ([]file.Reference, error) { +func (r *pythonTestResolverMock) FilesByGlob(_ ...string) ([]source.Location, error) { return nil, fmt.Errorf("not implemented") } -func (r *pythonTestResolverMock) RelativeFileByPath(_ file.Reference, path string) (*file.Reference, error) { +func (r *pythonTestResolverMock) RelativeFileByPath(_ source.Location, path string) *source.Location { switch { case strings.Contains(path, "RECORD"): - return r.recordRef, nil + return r.recordRef case strings.Contains(path, "top_level.txt"): - return r.topLevelRef, nil + return r.topLevelRef default: - return nil, fmt.Errorf("invalid RelativeFileByPath value given: %q", path) + panic(fmt.Errorf("invalid RelativeFileByPath value given: %q", path)) } } @@ -142,12 +142,12 @@ func TestPythonPackageWheelCataloger(t *testing.T) { AuthorEmail: "me@kennethreitz.org", SitePackagesRootPath: "test-fixtures", Files: []pkg.PythonFileRecord{ - {Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.Digest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"}, - {Path: "requests/__init__.py", Digest: &pkg.Digest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"}, + {Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"}, + {Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"}, {Path: "requests/__pycache__/__version__.cpython-38.pyc"}, {Path: "requests/__pycache__/utils.cpython-38.pyc"}, - {Path: "requests/__version__.py", Digest: &pkg.Digest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"}, - {Path: "requests/utils.py", Digest: &pkg.Digest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"}, + {Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"}, + {Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"}, }, TopLevelPackages: []string{"requests"}, }, @@ -174,11 +174,11 @@ func TestPythonPackageWheelCataloger(t *testing.T) { AuthorEmail: "georg@python.org", SitePackagesRootPath: "test-fixtures", Files: []pkg.PythonFileRecord{ - {Path: "../../../bin/pygmentize", Digest: &pkg.Digest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, - {Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.Digest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, + {Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, + {Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, {Path: "Pygments-2.6.1.dist-info/RECORD"}, {Path: "pygments/__pycache__/__init__.cpython-38.pyc"}, - {Path: "pygments/util.py", Digest: &pkg.Digest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, + {Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, }, TopLevelPackages: []string{"pygments", "something_else"}, }, @@ -214,13 +214,13 @@ func TestPythonPackageWheelCataloger(t *testing.T) { resolver := newTestResolver(test.MetadataFixture, test.RecordFixture, test.TopLevelFixture) // note that the source is the record ref created by the resolver mock... attach the expected values - test.ExpectedPackage.Source = []file.Reference{*resolver.metadataRef} + test.ExpectedPackage.Locations = []source.Location{*resolver.metadataRef} if resolver.recordRef != nil { - test.ExpectedPackage.Source = append(test.ExpectedPackage.Source, *resolver.recordRef) + test.ExpectedPackage.Locations = append(test.ExpectedPackage.Locations, *resolver.recordRef) } if resolver.topLevelRef != nil { - test.ExpectedPackage.Source = append(test.ExpectedPackage.Source, *resolver.topLevelRef) + test.ExpectedPackage.Locations = append(test.ExpectedPackage.Locations, *resolver.topLevelRef) } // end patching expected values with runtime data... diff --git a/syft/cataloger/python/parse_wheel_egg_metadata.go b/syft/cataloger/python/parse_wheel_egg_metadata.go index dcb90a14c..ca6e40c9f 100644 --- a/syft/cataloger/python/parse_wheel_egg_metadata.go +++ b/syft/cataloger/python/parse_wheel_egg_metadata.go @@ -7,8 +7,6 @@ import ( "path/filepath" "strings" - "github.com/anchore/stereoscope/pkg/file" - "github.com/mitchellh/mapstructure" "github.com/anchore/syft/syft/pkg" @@ -16,7 +14,7 @@ import ( // parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes), // returning all Python packages listed. -func parseWheelOrEggMetadata(path file.Path, reader io.Reader) (pkg.PythonPackageMetadata, error) { +func parseWheelOrEggMetadata(path string, reader io.Reader) (pkg.PythonPackageMetadata, error) { fields := make(map[string]string) var key string @@ -73,7 +71,7 @@ func parseWheelOrEggMetadata(path file.Path, reader io.Reader) (pkg.PythonPackag // add additional metadata not stored in the egg/wheel metadata file - sitePackagesRoot := filepath.Clean(filepath.Join(filepath.Dir(string(path)), "..")) + sitePackagesRoot := filepath.Clean(filepath.Join(filepath.Dir(path), "..")) metadata.SitePackagesRootPath = sitePackagesRoot return metadata, nil diff --git a/syft/cataloger/python/parse_wheel_egg_metadata_test.go b/syft/cataloger/python/parse_wheel_egg_metadata_test.go index 98896eef2..3c04beba2 100644 --- a/syft/cataloger/python/parse_wheel_egg_metadata_test.go +++ b/syft/cataloger/python/parse_wheel_egg_metadata_test.go @@ -4,8 +4,6 @@ import ( "os" "testing" - "github.com/anchore/stereoscope/pkg/file" - "github.com/anchore/syft/syft/pkg" "github.com/go-test/deep" ) @@ -48,7 +46,7 @@ func TestParseWheelEggMetadata(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parseWheelOrEggMetadata(file.Path(test.Fixture), fixture) + actual, err := parseWheelOrEggMetadata(test.Fixture, fixture) if err != nil { t.Fatalf("failed to parse: %+v", err) } diff --git a/syft/cataloger/python/parse_wheel_egg_record.go b/syft/cataloger/python/parse_wheel_egg_record.go index 42faafa3f..4f3e828ac 100644 --- a/syft/cataloger/python/parse_wheel_egg_record.go +++ b/syft/cataloger/python/parse_wheel_egg_record.go @@ -44,7 +44,7 @@ func parseWheelOrEggRecord(reader io.Reader) ([]pkg.PythonFileRecord, error) { return nil, fmt.Errorf("unexpected python record digest: %q", item) } - record.Digest = &pkg.Digest{ + record.Digest = &pkg.PythonFileDigest{ Algorithm: fields[0], Value: fields[1], } diff --git a/syft/cataloger/python/parse_wheel_egg_record_test.go b/syft/cataloger/python/parse_wheel_egg_record_test.go index d14868e0f..c0cf578b5 100644 --- a/syft/cataloger/python/parse_wheel_egg_record_test.go +++ b/syft/cataloger/python/parse_wheel_egg_record_test.go @@ -16,22 +16,22 @@ func TestParseWheelEggRecord(t *testing.T) { { Fixture: "test-fixtures/egg-info/RECORD", ExpectedMetadata: []pkg.PythonFileRecord{ - {Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.Digest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"}, - {Path: "requests/__init__.py", Digest: &pkg.Digest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"}, + {Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"}, + {Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"}, {Path: "requests/__pycache__/__version__.cpython-38.pyc"}, {Path: "requests/__pycache__/utils.cpython-38.pyc"}, - {Path: "requests/__version__.py", Digest: &pkg.Digest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"}, - {Path: "requests/utils.py", Digest: &pkg.Digest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"}, + {Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"}, + {Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"}, }, }, { Fixture: "test-fixtures/dist-info/RECORD", ExpectedMetadata: []pkg.PythonFileRecord{ - {Path: "../../../bin/pygmentize", Digest: &pkg.Digest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, - {Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.Digest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, + {Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, + {Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, {Path: "Pygments-2.6.1.dist-info/RECORD"}, {Path: "pygments/__pycache__/__init__.cpython-38.pyc"}, - {Path: "pygments/util.py", Digest: &pkg.Digest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, + {Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, }, }, } diff --git a/syft/cataloger/rpmdb/cataloger.go b/syft/cataloger/rpmdb/cataloger.go index 6563f076e..c9234a1c7 100644 --- a/syft/cataloger/rpmdb/cataloger.go +++ b/syft/cataloger/rpmdb/cataloger.go @@ -8,11 +8,12 @@ import ( "strings" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) const ( - packagesGlob = "**/var/lib/rpm/Packages" + packagesGlob = "**/var/lib/rpm/Packages" + catalogerName = "rpmdb-cataloger" ) type Cataloger struct{} @@ -24,26 +25,26 @@ func NewRpmdbCataloger() *Cataloger { // Name returns a string that uniquely describes a cataloger func (c *Cataloger) Name() string { - return "rpmdb-cataloger" + return catalogerName } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation. -func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) { +func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { fileMatches, err := resolver.FilesByGlob(packagesGlob) if err != nil { return nil, fmt.Errorf("failed to find rpmdb's by glob: %w", err) } var pkgs []pkg.Package - for _, dbRef := range fileMatches { - dbContents, err := resolver.FileContentsByRef(dbRef) + for _, location := range fileMatches { + dbContents, err := resolver.FileContentsByLocation(location) if err != nil { return nil, err } - pkgs, err = parseRpmDB(resolver, dbRef, strings.NewReader(dbContents)) + pkgs, err = parseRpmDB(resolver, location, strings.NewReader(dbContents)) if err != nil { - return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", dbRef.Path, err) + return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.Path, err) } } return pkgs, nil diff --git a/syft/cataloger/rpmdb/parse_rpmdb.go b/syft/cataloger/rpmdb/parse_rpmdb.go index 5a88b45cc..9f0b3ed80 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb.go +++ b/syft/cataloger/rpmdb/parse_rpmdb.go @@ -6,18 +6,15 @@ import ( "io/ioutil" "os" - "github.com/anchore/stereoscope/pkg/file" - - "github.com/anchore/syft/syft/scope" - rpmdb "github.com/anchore/go-rpmdb/pkg" "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" ) // parseApkDb parses an "Packages" RPM DB and returns the Packages listed within it. -func parseRpmDB(resolver scope.FileResolver, dbRef file.Reference, reader io.Reader) ([]pkg.Package, error) { +func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader io.Reader) ([]pkg.Package, error) { f, err := ioutil.TempFile("", internal.ApplicationName+"-rpmdb") if err != nil { return nil, fmt.Errorf("failed to create temp rpmdb file: %w", err) @@ -54,10 +51,10 @@ func parseRpmDB(resolver scope.FileResolver, dbRef file.Reference, reader io.Rea } p := pkg.Package{ - Name: entry.Name, - Version: fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does - //Version: fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch), - Source: []file.Reference{dbRef}, + Name: entry.Name, + Version: fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does, instead of fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch) + Locations: []source.Location{dbLocation}, + FoundBy: catalogerName, Type: pkg.RpmPkg, MetadataType: pkg.RpmdbMetadataType, Metadata: pkg.RpmdbMetadata{ @@ -80,11 +77,11 @@ func parseRpmDB(resolver scope.FileResolver, dbRef file.Reference, reader io.Rea return allPkgs, nil } -func extractRpmdbFileRecords(resolver scope.FileResolver, entry *rpmdb.PackageInfo) ([]pkg.RpmdbFileRecord, error) { +func extractRpmdbFileRecords(resolver source.FileResolver, entry *rpmdb.PackageInfo) ([]pkg.RpmdbFileRecord, error) { var records = make([]pkg.RpmdbFileRecord, 0) for _, record := range entry.Files { - refs, err := resolver.FilesByPath(file.Path(record.Path)) + refs, err := resolver.FilesByPath(record.Path) if err != nil { return nil, fmt.Errorf("failed to resolve path=%+v: %w", record.Path, err) } diff --git a/syft/cataloger/rpmdb/parse_rpmdb_test.go b/syft/cataloger/rpmdb/parse_rpmdb_test.go index ae0f76e2e..a80fa2523 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb_test.go +++ b/syft/cataloger/rpmdb/parse_rpmdb_test.go @@ -5,7 +5,8 @@ import ( "os" "testing" - "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/syft/pkg" "github.com/go-test/deep" ) @@ -20,28 +21,29 @@ func newTestFileResolver(ignorePaths bool) *rpmdbTestFileResolverMock { } } -func (r *rpmdbTestFileResolverMock) FilesByPath(paths ...file.Path) ([]file.Reference, error) { +func (r *rpmdbTestFileResolverMock) FilesByPath(paths ...string) ([]source.Location, error) { if r.ignorePaths { // act as if no paths exist return nil, nil } // act as if all files exist - var refs = make([]file.Reference, len(paths)) + var locations = make([]source.Location, len(paths)) for i, p := range paths { - refs[i] = file.NewFileReference(p) + locations[i] = source.NewLocation(p) } - return refs, nil + return locations, nil } -func (r *rpmdbTestFileResolverMock) FilesByGlob(...string) ([]file.Reference, error) { +func (r *rpmdbTestFileResolverMock) FilesByGlob(...string) ([]source.Location, error) { return nil, fmt.Errorf("not implemented") } -func (r *rpmdbTestFileResolverMock) RelativeFileByPath(file.Reference, string) (*file.Reference, error) { - return nil, fmt.Errorf("not implemented") +func (r *rpmdbTestFileResolverMock) RelativeFileByPath(source.Location, string) *source.Location { + panic(fmt.Errorf("not implemented")) + return nil } func TestParseRpmDB(t *testing.T) { - dbRef := file.NewFileReference("test-path") + dbLocation := source.NewLocation("test-path") tests := []struct { fixture string @@ -56,7 +58,8 @@ func TestParseRpmDB(t *testing.T) { "dive": { Name: "dive", Version: "0.9.2-1", - Source: []file.Reference{dbRef}, + Locations: []source.Location{dbLocation}, + FoundBy: catalogerName, Type: pkg.RpmPkg, MetadataType: pkg.RpmdbMetadataType, Metadata: pkg.RpmdbMetadata{ @@ -82,7 +85,8 @@ func TestParseRpmDB(t *testing.T) { "dive": { Name: "dive", Version: "0.9.2-1", - Source: []file.Reference{dbRef}, + Locations: []source.Location{dbLocation}, + FoundBy: catalogerName, Type: pkg.RpmPkg, MetadataType: pkg.RpmdbMetadataType, Metadata: pkg.RpmdbMetadata{ @@ -118,7 +122,7 @@ func TestParseRpmDB(t *testing.T) { fileResolver := newTestFileResolver(test.ignorePaths) - actual, err := parseRpmDB(fileResolver, dbRef, fixture) + actual, err := parseRpmDB(fileResolver, dbLocation, fixture) if err != nil { t.Fatalf("failed to parse rpmdb: %+v", err) } diff --git a/syft/distro/distro.go b/syft/distro/distro.go index 7e01b2493..a78f5ab33 100644 --- a/syft/distro/distro.go +++ b/syft/distro/distro.go @@ -6,6 +6,7 @@ import ( hashiVer "github.com/hashicorp/go-version" ) +// Distro represents a Linux Distribution. type Distro struct { Type Type Version *hashiVer.Version @@ -20,6 +21,7 @@ func NewUnknownDistro() Distro { } } +// NewDistro creates a new Distro object populated with the given values. func NewDistro(t Type, ver, like string) (Distro, error) { if ver == "" { return Distro{Type: t}, nil @@ -36,6 +38,12 @@ func NewDistro(t Type, ver, like string) (Distro, error) { }, nil } +// Name provides a string repr of the distro +func (d Distro) Name() string { + return string(d.Type) +} + +// MajorVersion returns the major version value from the pseudo-semantically versioned distro version value. func (d Distro) MajorVersion() string { if d.Version == nil { return fmt.Sprint("(version unknown)") @@ -43,10 +51,12 @@ func (d Distro) MajorVersion() string { return fmt.Sprintf("%d", d.Version.Segments()[0]) } +// FullVersion returns the original user version value. func (d Distro) FullVersion() string { return d.RawVersion } +// String returns a human-friendly representation of the Linux distribution. func (d Distro) String() string { versionStr := "(version unknown)" if d.RawVersion != "" { @@ -54,8 +64,3 @@ func (d Distro) String() string { } return fmt.Sprintf("%s %s", d.Type, versionStr) } - -// Name provides a string repr of the distro -func (d Distro) Name() string { - return d.Type.String() -} diff --git a/syft/distro/identify.go b/syft/distro/identify.go index ec7cbcb2a..b029a20a3 100644 --- a/syft/distro/identify.go +++ b/syft/distro/identify.go @@ -4,21 +4,20 @@ import ( "regexp" "strings" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) // returns a distro or nil type parseFunc func(string) *Distro type parseEntry struct { - path file.Path + path string fn parseFunc } // Identify parses distro-specific files to determine distro metadata like version and release. -func Identify(resolver scope.Resolver) Distro { +func Identify(resolver source.Resolver) Distro { distro := NewUnknownDistro() identityFiles := []parseEntry{ @@ -41,25 +40,19 @@ func Identify(resolver scope.Resolver) Distro { identifyLoop: for _, entry := range identityFiles { - refs, err := resolver.FilesByPath(entry.path) + locations, err := resolver.FilesByPath(entry.path) if err != nil { - log.Errorf("unable to get path refs from %s: %s", entry.path, err) + log.Errorf("unable to get path locations from %s: %s", entry.path, err) break } - if len(refs) == 0 { + if len(locations) == 0 { log.Debugf("No Refs found from path: %s", entry.path) continue } - for _, ref := range refs { - contents, err := resolver.MultipleFileContentsByRef(ref) - content, ok := contents[ref] - - if !ok { - log.Infof("no content present for ref: %s", ref) - continue - } + for _, location := range locations { + content, err := resolver.FileContentsByLocation(location) if err != nil { log.Debugf("unable to get contents from %s: %s", entry.path, err) diff --git a/syft/distro/identify_test.go b/syft/distro/identify_test.go index fc61965c2..2ed9fbc46 100644 --- a/syft/distro/identify_test.go +++ b/syft/distro/identify_test.go @@ -8,7 +8,7 @@ import ( "github.com/anchore/syft/internal" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) func TestIdentifyDistro(t *testing.T) { @@ -78,15 +78,15 @@ func TestIdentifyDistro(t *testing.T) { observedDistros := internal.NewStringSet() definedDistros := internal.NewStringSet() - for _, d := range All { - definedDistros.Add(d.String()) + for _, distroType := range All { + definedDistros.Add(string(distroType)) } for _, test := range tests { t.Run(test.fixture, func(t *testing.T) { - s, err := scope.NewScopeFromDir(test.fixture) + s, err := source.NewFromDirectory(test.fixture) if err != nil { - t.Fatalf("unable to produce a new scope for testing: %s", test.fixture) + t.Fatalf("unable to produce a new source for testing: %s", test.fixture) } d := Identify(s.Resolver) diff --git a/syft/distro/type.go b/syft/distro/type.go index 0f42fca07..aac86384a 100644 --- a/syft/distro/type.go +++ b/syft/distro/type.go @@ -1,37 +1,25 @@ package distro +// Type represents the different Linux distribution options +type Type string + const ( - UnknownDistroType Type = iota - Debian - Ubuntu - RedHat - CentOS - Fedora - Alpine - Busybox - AmazonLinux - OracleLinux - ArchLinux - OpenSuseLeap + // represents the set of valid/supported Linux Distributions + UnknownDistroType Type = "UnknownDistroType" + Debian Type = "debian" + Ubuntu Type = "ubuntu" + RedHat Type = "redhat" + CentOS Type = "centos" + Fedora Type = "fedora" + Alpine Type = "alpine" + Busybox Type = "busybox" + AmazonLinux Type = "amazonlinux" + OracleLinux Type = "oraclelinux" + ArchLinux Type = "archlinux" + OpenSuseLeap Type = "opensuseleap" ) -type Type int - -var distroStr = []string{ - "UnknownDistroType", - "debian", - "ubuntu", - "redhat", - "centos", - "fedora", - "alpine", - "busybox", - "amazn", - "oraclelinux", - "archlinux", - "opensuse-leap", -} - +// All contains all Linux distribution options var All = []Type{ Debian, Ubuntu, @@ -46,14 +34,6 @@ var All = []Type{ OpenSuseLeap, } -func (t Type) String() string { - if int(t) >= len(distroStr) || t < 0 { - return distroStr[0] - } - - return distroStr[t] -} - // IDMapping connects a distro ID like "ubuntu" to a Distro type var IDMapping = map[string]Type{ "debian": Debian, @@ -68,3 +48,8 @@ var IDMapping = map[string]Type{ "arch": ArchLinux, "opensuse-leap": OpenSuseLeap, } + +// String returns the string representation of the given Linux distribution. +func (t Type) String() string { + return string(t) +} diff --git a/syft/event/event.go b/syft/event/event.go index e76ebd983..caf1e41f5 100644 --- a/syft/event/event.go +++ b/syft/event/event.go @@ -7,7 +7,12 @@ package event import "github.com/wagoodman/go-partybus" const ( + // AppUpdateAvailable is a partybus event that occurs when an application update is available AppUpdateAvailable partybus.EventType = "syft-app-update-available" - CatalogerStarted partybus.EventType = "syft-cataloger-started-event" - CatalogerFinished partybus.EventType = "syft-cataloger-finished-event" + + // CatalogerStarted is a partybus event that occurs when the package cataloging has begun + CatalogerStarted partybus.EventType = "syft-cataloger-started-event" + + // CatalogerFinished is a partybus event that occurs when the package cataloging has completed + CatalogerFinished partybus.EventType = "syft-cataloger-finished-event" ) diff --git a/syft/lib.go b/syft/lib.go index cb7dc66a1..931c3747b 100644 --- a/syft/lib.go +++ b/syft/lib.go @@ -7,8 +7,8 @@ Here is what the main execution path for syft does: 2. Invoke all catalogers to catalog the image, adding discovered packages to a single catalog object 3. Invoke a single presenter to show the contents of the catalog -A Scope object encapsulates the image object to be cataloged and the user options (catalog all layers vs. squashed layer), -providing a way to inspect paths and file content within the image. The Scope object, not the image object, is used +A Source object encapsulates the image object to be cataloged and the user options (catalog all layers vs. squashed layer), +providing a way to inspect paths and file content within the image. The Source object, not the image object, is used throughout the main execution path. This abstraction allows for decoupling of what is cataloged (a docker image, an OCI image, a filesystem, etc) and how it is cataloged (the individual catalogers). @@ -17,7 +17,9 @@ Similar to the cataloging process, Linux distribution identification is also per package syft import ( + "encoding/json" "fmt" + "io" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" @@ -25,33 +27,34 @@ import ( "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/logger" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + jsonPresenter "github.com/anchore/syft/syft/presenter/json" + "github.com/anchore/syft/syft/source" "github.com/wagoodman/go-partybus" ) -// Catalog the given image from a particular perspective (e.g. squashed scope, all-layers scope). Returns the discovered -// set of packages, the identified Linux distribution, and the scope object used to wrap the data source. -func Catalog(userInput string, scoptOpt scope.Option) (*pkg.Catalog, *scope.Scope, *distro.Distro, error) { +// Catalog the given image from a particular perspective (e.g. squashed source, all-layers source). Returns the discovered +// set of packages, the identified Linux distribution, and the source object used to wrap the data source. +func Catalog(userInput string, scope source.Scope) (source.Source, *pkg.Catalog, distro.Distro, error) { log.Info("cataloging image") - s, cleanup, err := scope.NewScope(userInput, scoptOpt) + s, cleanup, err := source.New(userInput, scope) defer cleanup() if err != nil { - return nil, nil, nil, err + return source.Source{}, nil, distro.Distro{}, err } d := IdentifyDistro(s) catalog, err := CatalogFromScope(s) if err != nil { - return nil, nil, nil, err + return source.Source{}, nil, distro.Distro{}, err } - return catalog, &s, &d, nil + return s, catalog, d, nil } // IdentifyDistro attempts to discover what the underlying Linux distribution may be from the available flat files -// provided by the given scope object. If results are inconclusive a "UnknownDistro" Type is returned. -func IdentifyDistro(s scope.Scope) distro.Distro { +// provided by the given source object. If results are inconclusive a "UnknownDistro" Type is returned. +func IdentifyDistro(s source.Source) distro.Distro { d := distro.Identify(s.Resolver) if d.Type != distro.UnknownDistroType { log.Infof("identified distro: %s", d.String()) @@ -61,24 +64,54 @@ func IdentifyDistro(s scope.Scope) distro.Distro { return d } -// Catalog the given scope, which may represent a container image or filesystem. Returns the discovered set of packages. -func CatalogFromScope(s scope.Scope) (*pkg.Catalog, error) { +// Catalog the given source, which may represent a container image or filesystem. Returns the discovered set of packages. +func CatalogFromScope(s source.Source) (*pkg.Catalog, error) { log.Info("building the catalog") // conditionally have two sets of catalogers var catalogers []cataloger.Cataloger - switch s.Scheme { - case scope.ImageScheme: + switch s.Metadata.Scheme { + case source.ImageScheme: catalogers = cataloger.ImageCatalogers() - case scope.DirectoryScheme: + case source.DirectoryScheme: catalogers = cataloger.DirectoryCatalogers() default: - return nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", s.Scheme) + return nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", s.Metadata.Scheme) } return cataloger.Catalog(s.Resolver, catalogers...) } +// CatalogFromJSON takes an existing syft report and generates native syft objects. +func CatalogFromJSON(reader io.Reader) (source.Metadata, *pkg.Catalog, distro.Distro, error) { + var doc jsonPresenter.Document + decoder := json.NewDecoder(reader) + if err := decoder.Decode(&doc); err != nil { + return source.Metadata{}, nil, distro.Distro{}, err + } + + var pkgs = make([]pkg.Package, len(doc.Artifacts)) + for i, a := range doc.Artifacts { + pkgs[i] = a.ToPackage() + } + + catalog := pkg.NewCatalog(pkgs...) + + var distroType distro.Type + if doc.Distro.Name == "" { + distroType = distro.UnknownDistroType + } else { + distroType = distro.Type(doc.Distro.Name) + } + + theDistro, err := distro.NewDistro(distroType, doc.Distro.Version, doc.Distro.IDLike) + if err != nil { + return source.Metadata{}, nil, distro.Distro{}, err + } + + return doc.Source.ToSourceMetadata(), catalog, theDistro, nil +} + // SetLogger sets the logger object used for all syft logging calls. func SetLogger(logger logger.Logger) { log.Log = logger diff --git a/syft/logger/logger.go b/syft/logger/logger.go index 3816296ff..3ace363f5 100644 --- a/syft/logger/logger.go +++ b/syft/logger/logger.go @@ -1,8 +1,9 @@ /* -Defines the logging interface which is used throughout the syft library. +Package logger defines the logging interface which is used throughout the syft library. */ package logger +// Logger represents the behavior for logging within the syft library. type Logger interface { Errorf(format string, args ...interface{}) Error(args ...interface{}) diff --git a/syft/pkg/apk_metadata.go b/syft/pkg/apk_metadata.go index ef036509a..705528fd0 100644 --- a/syft/pkg/apk_metadata.go +++ b/syft/pkg/apk_metadata.go @@ -35,6 +35,7 @@ type ApkFileRecord struct { Checksum string `json:"checksum,omitempty"` } +// PackageURL returns the PURL for the specific Alpine package (see https://github.com/package-url/purl-spec) func (m ApkMetadata) PackageURL() string { pURL := packageurl.NewPackageURL( // note: this is currently a candidate and not technically within spec diff --git a/syft/pkg/catalog.go b/syft/pkg/catalog.go index 5afde2998..81a1d4652 100644 --- a/syft/pkg/catalog.go +++ b/syft/pkg/catalog.go @@ -4,7 +4,8 @@ import ( "sort" "sync" - "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/internal/log" ) @@ -14,17 +15,23 @@ var nextPackageID int64 type Catalog struct { byID map[ID]*Package byType map[Type][]*Package - byFile map[file.Reference][]*Package + byFile map[source.Location][]*Package lock sync.RWMutex } // NewCatalog returns a new empty Catalog -func NewCatalog() *Catalog { - return &Catalog{ +func NewCatalog(pkgs ...Package) *Catalog { + catalog := Catalog{ byID: make(map[ID]*Package), byType: make(map[Type][]*Package), - byFile: make(map[file.Reference][]*Package), + byFile: make(map[source.Location][]*Package), } + + for _, p := range pkgs { + catalog.Add(p) + } + + return &catalog } // PackageCount returns the total number of packages that have been added. @@ -38,8 +45,8 @@ func (c *Catalog) Package(id ID) *Package { } // PackagesByFile returns all packages that were discovered from the given source file reference. -func (c *Catalog) PackagesByFile(ref file.Reference) []*Package { - return c.byFile[ref] +func (c *Catalog) PackagesByFile(location source.Location) []*Package { + return c.byFile[location] } // Add a package to the Catalog. @@ -65,7 +72,7 @@ func (c *Catalog) Add(p Package) { c.byType[p.Type] = append(c.byType[p.Type], &p) // store by file references - for _, s := range p.Source { + for _, s := range p.Locations { _, ok := c.byFile[s] if !ok { c.byFile[s] = make([]*Package, 0) @@ -111,6 +118,9 @@ func (c *Catalog) Sorted(types ...Type) []*Package { sort.SliceStable(pkgs, func(i, j int) bool { if pkgs[i].Name == pkgs[j].Name { + if pkgs[i].Version == pkgs[j].Version { + return pkgs[i].Type < pkgs[j].Type + } return pkgs[i].Version < pkgs[j].Version } return pkgs[i].Name < pkgs[j].Name diff --git a/syft/pkg/dpkg_metadata.go b/syft/pkg/dpkg_metadata.go index f1e80802e..c810a8198 100644 --- a/syft/pkg/dpkg_metadata.go +++ b/syft/pkg/dpkg_metadata.go @@ -17,11 +17,13 @@ type DpkgMetadata struct { Files []DpkgFileRecord `json:"files"` } +// DpkgFileRecord represents a single file attributed to a debian package. type DpkgFileRecord struct { Path string `json:"path"` MD5 string `json:"md5"` } +// PackageURL returns the PURL for the specific Debian package (see https://github.com/package-url/purl-spec) func (m DpkgMetadata) PackageURL(d distro.Distro) string { pURL := packageurl.NewPackageURL( // TODO: replace with `packageurl.TypeDebian` upon merge of https://github.com/package-url/packageurl-go/pull/21 diff --git a/syft/pkg/gem_metadata.go b/syft/pkg/gem_metadata.go index c06a244b2..51a07a148 100644 --- a/syft/pkg/gem_metadata.go +++ b/syft/pkg/gem_metadata.go @@ -1,5 +1,6 @@ package pkg +// GemMetadata represents all metadata parsed from the gemspec file type GemMetadata struct { Name string `mapstructure:"name" json:"name"` Version string `mapstructure:"version" json:"version"` diff --git a/syft/pkg/java_metadata.go b/syft/pkg/java_metadata.go index d4852eb55..beb03463f 100644 --- a/syft/pkg/java_metadata.go +++ b/syft/pkg/java_metadata.go @@ -26,6 +26,7 @@ type JavaManifest struct { NamedSections map[string]map[string]string `json:"namedSections,omitempty"` } +// PackageURL returns the PURL for the specific Alpine package (see https://github.com/package-url/purl-spec) func (m JavaMetadata) PackageURL() string { if m.PomProperties != nil { pURL := packageurl.NewPackageURL( diff --git a/syft/pkg/language.go b/syft/pkg/language.go index 75032937d..9f2ab7dae 100644 --- a/syft/pkg/language.go +++ b/syft/pkg/language.go @@ -1,25 +1,19 @@ package pkg +// Language represents a single programming language. +type Language string + const ( - UnknownLanguage Language = iota - Java - JavaScript - Python - Ruby - Go + // the full set of supported programming languages + UnknownLanguage Language = "UnknownLanguage" + Java Language = "java" + JavaScript Language = "javascript" + Python Language = "python" + Ruby Language = "ruby" + Go Language = "go" ) -type Language uint - -var languageStr = []string{ - "UnknownLanguage", - "java", - "javascript", - "python", - "ruby", - "go", -} - +// AllLanguages is a set of all programming languages detected by syft. var AllLanguages = []Language{ Java, JavaScript, @@ -28,9 +22,7 @@ var AllLanguages = []Language{ Go, } -func (t Language) String() string { - if int(t) >= len(languageStr) { - return languageStr[0] - } - return languageStr[t] +// String returns the string representation of the language. +func (l Language) String() string { + return string(l) } diff --git a/syft/pkg/metadata.go b/syft/pkg/metadata.go index c1e402934..65d58724f 100644 --- a/syft/pkg/metadata.go +++ b/syft/pkg/metadata.go @@ -1,14 +1,16 @@ package pkg +// MetadataType represents the data shape stored within pkg.Package.Metadata. type MetadataType string const ( + // this is the full set of data shapes that can be represented within the pkg.Package.Metadata field UnknownMetadataType MetadataType = "UnknownMetadata" - ApkMetadataType MetadataType = "apk-metadata" - DpkgMetadataType MetadataType = "dpkg-metadata" - GemMetadataType MetadataType = "gem-metadata" - JavaMetadataType MetadataType = "java-metadata" - NpmPackageJSONMetadataType MetadataType = "npm-package-json-metadata" - RpmdbMetadataType MetadataType = "rpmdb-metadata" - PythonPackageMetadataType MetadataType = "python-package-metadata" + ApkMetadataType MetadataType = "ApkMetadata" + DpkgMetadataType MetadataType = "DpkgMetadata" + GemMetadataType MetadataType = "GemMetadata" + JavaMetadataType MetadataType = "JavaMetadata" + NpmPackageJSONMetadataType MetadataType = "NpmPackageJsonMetadata" + RpmdbMetadataType MetadataType = "RpmdbMetadata" + PythonPackageMetadataType MetadataType = "PythonPackageMetadata" ) diff --git a/syft/pkg/package.go b/syft/pkg/package.go index c3224a07f..d2c83a363 100644 --- a/syft/pkg/package.go +++ b/syft/pkg/package.go @@ -8,26 +8,28 @@ import ( "regexp" "strings" - "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/syft/distro" "github.com/package-url/packageurl-go" ) +// ID represents a unique value for each package added to a package catalog. type ID int64 // Package represents an application or library that has been bundled into a distributable format. type Package struct { - id ID // uniquely identifies a package, set by the cataloger - Name string `json:"manifest"` // the package name - Version string `json:"version"` // the version of the package - FoundBy string `json:"foundBy"` // the specific cataloger that discovered this package - Source []file.Reference `json:"sources"` // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package) + id ID // uniquely identifies a package, set by the cataloger + Name string // the package name + Version string // the version of the package + FoundBy string // the specific cataloger that discovered this package + Locations []source.Location // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package) // TODO: should we move licenses into metadata? - Licenses []string `json:"licenses"` // licenses discovered with the package metadata - Language Language `json:"language"` // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc) - Type Type `json:"type"` // the package type (e.g. Npm, Yarn, Python, Rpm, Deb, etc) - MetadataType MetadataType `json:"metadataType"` // the shape of the additional data in the "metadata" field - Metadata interface{} `json:"metadata,omitempty"` // additional data found while parsing the package source + Licenses []string // licenses discovered with the package metadata + Language Language // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc) + Type Type // the package type (e.g. Npm, Yarn, Python, Rpm, Deb, etc) + MetadataType MetadataType // the shape of the additional data in the "metadata" field + Metadata interface{} // additional data found while parsing the package source } // ID returns the package ID, which is unique relative to a package catalog. diff --git a/syft/pkg/python_package_metadata.go b/syft/pkg/python_package_metadata.go index 637e6220c..e01771602 100644 --- a/syft/pkg/python_package_metadata.go +++ b/syft/pkg/python_package_metadata.go @@ -1,15 +1,16 @@ package pkg -type Digest struct { +// PythonFileDigest represents the file metadata for a single file attributed to a python package. +type PythonFileDigest struct { Algorithm string `json:"algorithm"` Value string `json:"value"` } // PythonFileRecord represents a single entry within a RECORD file for a python wheel or egg package type PythonFileRecord struct { - Path string `json:"path"` - Digest *Digest `json:"digest,omitempty"` - Size string `json:"size,omitempty"` + Path string `json:"path"` + Digest *PythonFileDigest `json:"digest,omitempty"` + Size string `json:"size,omitempty"` } // PythonPackageMetadata represents all captured data for a python egg or wheel package. diff --git a/syft/pkg/rpmdb_metadata.go b/syft/pkg/rpmdb_metadata.go index c9823c1a8..6b3ea723c 100644 --- a/syft/pkg/rpmdb_metadata.go +++ b/syft/pkg/rpmdb_metadata.go @@ -21,6 +21,7 @@ type RpmdbMetadata struct { Files []RpmdbFileRecord `json:"files"` } +// RpmdbFileRecord represents the file metadata for a single file attributed to a RPM package. type RpmdbFileRecord struct { Path string `json:"path"` Mode RpmdbFileMode `json:"mode"` @@ -28,8 +29,10 @@ type RpmdbFileRecord struct { SHA256 string `json:"sha256"` } +// RpmdbFileMode is the raw file mode for a single file. This can be interpreted as the linux stat.h mode (see https://pubs.opengroup.org/onlinepubs/007908799/xsh/sysstat.h.html) type RpmdbFileMode uint16 +// PackageURL returns the PURL for the specific RHEL package (see https://github.com/package-url/purl-spec) func (m RpmdbMetadata) PackageURL(d distro.Distro) string { pURL := packageurl.NewPackageURL( packageurl.TypeRPM, diff --git a/syft/pkg/type.go b/syft/pkg/type.go index 85dc83159..49d3d3b26 100644 --- a/syft/pkg/type.go +++ b/syft/pkg/type.go @@ -6,6 +6,7 @@ import "github.com/package-url/packageurl-go" type Type string const ( + // the full set of supported packages UnknownPkg Type = "UnknownPackage" ApkPkg Type = "apk" GemPkg Type = "gem" @@ -18,6 +19,7 @@ const ( GoModulePkg Type = "go-module" ) +// AllPkgs represents all supported package types var AllPkgs = []Type{ ApkPkg, GemPkg, @@ -30,6 +32,7 @@ var AllPkgs = []Type{ GoModulePkg, } +// PackageURLType returns the PURL package type for the current package. func (t Type) PackageURLType() string { switch t { case ApkPkg: diff --git a/syft/presenter/cyclonedx/bom-extension.go b/syft/presenter/cyclonedx/bom-extension.go index f25713734..7f99de637 100644 --- a/syft/presenter/cyclonedx/bom-extension.go +++ b/syft/presenter/cyclonedx/bom-extension.go @@ -4,8 +4,7 @@ import ( "encoding/xml" "time" - "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/version" + "github.com/anchore/syft/syft/source" ) // Source: https://cyclonedx.org/ext/bom-descriptor/ @@ -35,15 +34,34 @@ type BdComponent struct { } // NewBomDescriptor returns a new BomDescriptor tailored for the current time and "syft" tool details. -func NewBomDescriptor() *BomDescriptor { - versionInfo := version.FromBuild() - return &BomDescriptor{ +func NewBomDescriptor(name, version string, srcMetadata source.Metadata) *BomDescriptor { + descriptor := BomDescriptor{ XMLName: xml.Name{}, Timestamp: time.Now().Format(time.RFC3339), Tool: &BdTool{ Vendor: "anchore", - Name: internal.ApplicationName, - Version: versionInfo.Version, + Name: name, + Version: version, }, } + + switch srcMetadata.Scheme { + case source.ImageScheme: + descriptor.Component = &BdComponent{ + Component: Component{ + Type: "container", + Name: srcMetadata.ImageMetadata.UserInput, + Version: srcMetadata.ImageMetadata.Digest, + }, + } + case source.DirectoryScheme: + descriptor.Component = &BdComponent{ + Component: Component{ + Type: "file", + Name: srcMetadata.Path, + }, + } + } + + return &descriptor } diff --git a/syft/presenter/cyclonedx/component.go b/syft/presenter/cyclonedx/component.go index 2217bb5d1..e540ef9ea 100644 --- a/syft/presenter/cyclonedx/component.go +++ b/syft/presenter/cyclonedx/component.go @@ -15,7 +15,7 @@ type Component struct { Description string `xml:"description,omitempty"` // A description of the component Licenses *[]License `xml:"licenses>license"` // A node describing zero or more license names, SPDX license IDs or expressions PackageURL string `xml:"purl,omitempty"` // Specifies the package-url (PackageURL). The purl, if specified, must be valid and conform to the specification defined at: https://github.com/package-url/purl-spec - // TODO: scope, hashes, copyright, cpe, purl, swid, modified, pedigree, externalReferences + // TODO: source, hashes, copyright, cpe, purl, swid, modified, pedigree, externalReferences // TODO: add user-defined parameters for syft-specific values (image layer index, cataloger, location path, etc.) } diff --git a/syft/presenter/cyclonedx/document.go b/syft/presenter/cyclonedx/document.go index 4539ab73e..78752480e 100644 --- a/syft/presenter/cyclonedx/document.go +++ b/syft/presenter/cyclonedx/document.go @@ -3,9 +3,11 @@ package cyclonedx import ( "encoding/xml" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/version" "github.com/anchore/syft/syft/distro" - "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" "github.com/google/uuid" ) @@ -22,19 +24,19 @@ type Document struct { BomDescriptor *BomDescriptor `xml:"bd:metadata"` // The BOM descriptor extension } -// NewDocument returns an empty CycloneDX Document object. -func NewDocument() Document { - return Document{ - XMLNs: "http://cyclonedx.org/schema/bom/1.2", - XMLNsBd: "http://cyclonedx.org/schema/ext/bom-descriptor/1.0", - Version: 1, - SerialNumber: uuid.New().URN(), - } -} - // NewDocumentFromCatalog returns a CycloneDX Document object populated with the catalog contents. -func NewDocumentFromCatalog(catalog *pkg.Catalog, d distro.Distro) Document { - bom := NewDocument() +func NewDocument(catalog *pkg.Catalog, d distro.Distro, srcMetadata source.Metadata) Document { + versionInfo := version.FromBuild() + + doc := Document{ + XMLNs: "http://cyclonedx.org/schema/bom/1.2", + XMLNsBd: "http://cyclonedx.org/schema/ext/bom-descriptor/1.0", + Version: 1, + SerialNumber: uuid.New().URN(), + BomDescriptor: NewBomDescriptor(internal.ApplicationName, versionInfo.Version, srcMetadata), + } + + // attach components for p := range catalog.Enumerate() { component := Component{ Type: "library", // TODO: this is not accurate @@ -51,10 +53,8 @@ func NewDocumentFromCatalog(catalog *pkg.Catalog, d distro.Distro) Document { if len(licenses) > 0 { component.Licenses = &licenses } - bom.Components = append(bom.Components, component) + doc.Components = append(doc.Components, component) } - bom.BomDescriptor = NewBomDescriptor() - - return bom + return doc } diff --git a/syft/presenter/cyclonedx/presenter.go b/syft/presenter/cyclonedx/presenter.go index 1b8accebe..ad86ac947 100644 --- a/syft/presenter/cyclonedx/presenter.go +++ b/syft/presenter/cyclonedx/presenter.go @@ -5,63 +5,33 @@ package cyclonedx import ( "encoding/xml" - "fmt" "io" "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) -// Presenter writes a CycloneDX report from the given Catalog and Scope contents +// Presenter writes a CycloneDX report from the given Catalog and Locations contents type Presenter struct { - catalog *pkg.Catalog - scope scope.Scope - distro distro.Distro + catalog *pkg.Catalog + srcMetadata source.Metadata + distro distro.Distro } -// NewPresenter creates a CycloneDX presenter from the given Catalog and Scope objects. -func NewPresenter(catalog *pkg.Catalog, s scope.Scope, d distro.Distro) *Presenter { +// NewPresenter creates a CycloneDX presenter from the given Catalog and Locations objects. +func NewPresenter(catalog *pkg.Catalog, srcMetadata source.Metadata, d distro.Distro) *Presenter { return &Presenter{ - catalog: catalog, - scope: s, - distro: d, + catalog: catalog, + srcMetadata: srcMetadata, + distro: d, } } // Present writes the CycloneDX report to the given io.Writer. func (pres *Presenter) Present(output io.Writer) error { - bom := NewDocumentFromCatalog(pres.catalog, pres.distro) - - switch src := pres.scope.Source.(type) { - case scope.DirSource: - bom.BomDescriptor.Component = &BdComponent{ - Component: Component{ - Type: "file", - Name: src.Path, - Version: "", - }, - } - case scope.ImageSource: - var imageID string - var versionStr string - if len(src.Img.Metadata.Tags) > 0 { - imageID = src.Img.Metadata.Tags[0].Context().Name() - versionStr = src.Img.Metadata.Tags[0].TagStr() - } else { - imageID = src.Img.Metadata.Digest - } - bom.BomDescriptor.Component = &BdComponent{ - Component: Component{ - Type: "container", - Name: imageID, - Version: versionStr, - }, - } - default: - return fmt.Errorf("unsupported source: %T", src) - } + bom := NewDocument(pres.catalog, pres.distro, pres.srcMetadata) encoder := xml.NewEncoder(output) encoder.Indent("", " ") diff --git a/syft/presenter/cyclonedx/presenter_test.go b/syft/presenter/cyclonedx/presenter_test.go index 39778d3c1..8829b6ec5 100644 --- a/syft/presenter/cyclonedx/presenter_test.go +++ b/syft/presenter/cyclonedx/presenter_test.go @@ -10,9 +10,8 @@ import ( "github.com/anchore/syft/syft/distro" "github.com/anchore/go-testutils" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/sergi/go-diff/diffmatchpatch" ) @@ -29,7 +28,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) { Version: "1.0.1", Type: pkg.DebPkg, FoundBy: "the-cataloger-1", - Source: []file.Reference{ + Locations: []source.Location{ {Path: "/some/path/pkg1"}, }, Metadata: pkg.DpkgMetadata{ @@ -43,7 +42,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) { Version: "2.0.1", Type: pkg.DebPkg, FoundBy: "the-cataloger-2", - Source: []file.Reference{ + Locations: []source.Location{ {Path: "/some/path/pkg1"}, }, Licenses: []string{ @@ -57,7 +56,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) { }, }) - s, err := scope.NewScopeFromDir("/some/path") + s, err := source.NewFromDirectory("/some/path") if err != nil { t.Fatal(err) } @@ -67,7 +66,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) { t.Fatal(err) } - pres := NewPresenter(catalog, s, d) + pres := NewPresenter(catalog, s.Metadata, d) // run presenter err = pres.Present(&buffer) @@ -105,8 +104,8 @@ func TestCycloneDxImgsPresenter(t *testing.T) { catalog.Add(pkg.Package{ Name: "package1", Version: "1.0.1", - Source: []file.Reference{ - *img.SquashedTree().File("/somefile-1.txt"), + Locations: []source.Location{ + source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img), }, Type: pkg.RpmPkg, FoundBy: "the-cataloger-1", @@ -125,8 +124,8 @@ func TestCycloneDxImgsPresenter(t *testing.T) { catalog.Add(pkg.Package{ Name: "package2", Version: "2.0.1", - Source: []file.Reference{ - *img.SquashedTree().File("/somefile-2.txt"), + Locations: []source.Location{ + source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img), }, Type: pkg.RpmPkg, FoundBy: "the-cataloger-2", @@ -147,7 +146,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) { }, }) - s, err := scope.NewScopeFromImage(img, scope.AllLayersScope) + s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input") if err != nil { t.Fatal(err) } @@ -157,7 +156,15 @@ func TestCycloneDxImgsPresenter(t *testing.T) { t.Fatal(err) } - pres := NewPresenter(catalog, s, d) + // This accounts for the non-deterministic digest value that we end up with when + // we build a container image dynamically during testing. Ultimately, we should + // use a golden image as a test fixture in place of building this image during + // testing. At that time, this line will no longer be necessary. + // + // This value is sourced from the "version" node in "./test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden" + s.Metadata.ImageMetadata.Digest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368" + + pres := NewPresenter(catalog, s.Metadata, d) // run presenter err = pres.Present(&buffer) @@ -178,7 +185,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) { if !bytes.Equal(expected, actual) { dmp := diffmatchpatch.New() - diffs := dmp.DiffMain(string(actual), string(expected), true) + diffs := dmp.DiffMain(string(expected), string(actual), true) t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs)) } } diff --git a/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden b/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden index 68ca8ae8e..2e2d0d8ae 100644 --- a/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden +++ b/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden @@ -1,5 +1,5 @@ - + package1 @@ -21,7 +21,7 @@ - 2020-09-23T18:26:58-04:00 + 2020-11-16T08:45:54-05:00 anchore syft diff --git a/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden b/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden index 03f34889a..b4fcc0188 100644 --- a/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden +++ b/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden @@ -1,5 +1,5 @@ - + package1 @@ -21,15 +21,15 @@ - 2020-09-23T18:26:58-04:00 + 2020-11-16T08:45:54-05:00 anchore syft [not provided] - index.docker.io/library/stereoscope-fixture-image-simple - 04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7 + user-image-input + sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368 diff --git a/syft/presenter/json/artifact.go b/syft/presenter/json/artifact.go index 3c7957cef..2773ec57f 100644 --- a/syft/presenter/json/artifact.go +++ b/syft/presenter/json/artifact.go @@ -1,33 +1,141 @@ package json import ( + "encoding/json" + "fmt" + "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) -type Artifact struct { - Name string `json:"name"` - Version string `json:"version"` - Type string `json:"type"` - FoundBy []string `json:"foundBy"` - Locations Locations `json:"locations,omitempty"` - Licenses []string `json:"licenses"` - Metadata interface{} `json:"metadata,omitempty"` +// Package represents a pkg.Package object specialized for JSON marshaling and unmarshaling. +type Package struct { + packageBasicMetadata + packageCustomMetadata } -func NewArtifact(p *pkg.Package, s scope.Scope) (Artifact, error) { - locations, err := NewLocations(p, s) - if err != nil { - return Artifact{}, err - } +// packageBasicMetadata contains non-ambiguous values (type-wise) from pkg.Package. +type packageBasicMetadata struct { + Name string `json:"name"` + Version string `json:"version"` + Type pkg.Type `json:"type"` + FoundBy string `json:"foundBy"` + Locations []source.Location `json:"locations"` + Licenses []string `json:"licenses"` + Language pkg.Language `json:"language"` +} - return Artifact{ - Name: p.Name, - Version: p.Version, - Type: string(p.Type), - FoundBy: []string{p.FoundBy}, - Locations: locations, - Licenses: p.Licenses, - Metadata: p.Metadata, +// packageCustomMetadata contains ambiguous values (type-wise) from pkg.Package. +type packageCustomMetadata struct { + MetadataType pkg.MetadataType `json:"metadataType"` + Metadata interface{} `json:"metadata,omitempty"` +} + +// packageMetadataUnpacker is all values needed from Package to disambiguate ambiguous fields during json unmarshaling. +type packageMetadataUnpacker struct { + MetadataType string `json:"metadataType"` + Metadata json.RawMessage `json:"metadata"` +} + +// NewPackage crates a new Package from the given pkg.Package. +func NewPackage(p *pkg.Package) (Package, error) { + return Package{ + packageBasicMetadata: packageBasicMetadata{ + Name: p.Name, + Version: p.Version, + Type: p.Type, + FoundBy: p.FoundBy, + Locations: p.Locations, + Licenses: p.Licenses, + Language: p.Language, + }, + packageCustomMetadata: packageCustomMetadata{ + MetadataType: p.MetadataType, + Metadata: p.Metadata, + }, }, nil } + +// ToPackage generates a pkg.Package from the current Package. +func (a Package) ToPackage() pkg.Package { + return pkg.Package{ + // does not include found-by and locations + Name: a.Name, + Version: a.Version, + FoundBy: a.FoundBy, + Licenses: a.Licenses, + Language: a.Language, + Locations: a.Locations, + Type: a.Type, + MetadataType: a.MetadataType, + Metadata: a.Metadata, + } +} + +// UnmarshalJSON is a custom unmarshaller for handling basic values and values with ambiguous types. +// nolint:funlen +func (a *Package) UnmarshalJSON(b []byte) error { + var basic packageBasicMetadata + if err := json.Unmarshal(b, &basic); err != nil { + return err + } + a.packageBasicMetadata = basic + + var unpacker packageMetadataUnpacker + if err := json.Unmarshal(b, &unpacker); err != nil { + return err + } + + a.MetadataType = pkg.MetadataType(unpacker.MetadataType) + + switch a.MetadataType { + case pkg.RpmdbMetadataType: + var payload pkg.RpmdbMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + a.Metadata = payload + case pkg.PythonPackageMetadataType: + var payload pkg.PythonPackageMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + a.Metadata = payload + case pkg.DpkgMetadataType: + var payload pkg.DpkgMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + a.Metadata = payload + case pkg.ApkMetadataType: + var payload pkg.ApkMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + a.Metadata = payload + case pkg.JavaMetadataType: + var payload pkg.JavaMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + a.Metadata = payload + case pkg.NpmPackageJSONMetadataType: + var payload pkg.NpmPackageJSONMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + a.Metadata = payload + case pkg.GemMetadataType: + var payload pkg.GemMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + a.Metadata = payload + case "": + // there may be packages with no metadata, which is OK + default: + return fmt.Errorf("unsupported package metadata type: %+v", a.MetadataType) + } + + return nil +} diff --git a/syft/presenter/json/descriptor.go b/syft/presenter/json/descriptor.go new file mode 100644 index 000000000..de07721fc --- /dev/null +++ b/syft/presenter/json/descriptor.go @@ -0,0 +1,7 @@ +package json + +// Descriptor describes what created the document as well as surrounding metadata +type Descriptor struct { + Name string `json:"name"` + Version string `json:"version"` +} diff --git a/syft/presenter/json/distribution.go b/syft/presenter/json/distribution.go new file mode 100644 index 000000000..150334a4c --- /dev/null +++ b/syft/presenter/json/distribution.go @@ -0,0 +1,24 @@ +package json + +import "github.com/anchore/syft/syft/distro" + +// Distribution provides information about a detected Linux Distribution. +type Distribution struct { + Name string `json:"name"` // Name of the Linux distribution + Version string `json:"version"` // Version of the Linux distribution (major or major.minor version) + IDLike string `json:"idLike"` // the ID_LIKE field found within the /etc/os-release file +} + +// NewDistribution creates a struct with the Linux distribution to be represented in JSON. +func NewDistribution(d distro.Distro) Distribution { + distroName := d.Name() + if distroName == "UnknownDistroType" { + distroName = "" + } + + return Distribution{ + Name: distroName, + Version: d.FullVersion(), + IDLike: d.IDLike, + } +} diff --git a/syft/presenter/json/document.go b/syft/presenter/json/document.go index 713dc7d7a..27cd6e48e 100644 --- a/syft/presenter/json/document.go +++ b/syft/presenter/json/document.go @@ -1,46 +1,40 @@ package json import ( + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/version" "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) +// Document represents the syft cataloging findings as a JSON document type Document struct { - Artifacts []Artifact `json:"artifacts"` - Source Source `json:"source"` - Distro Distribution `json:"distro"` + Artifacts []Package `json:"artifacts"` // Artifacts is the list of packages discovered and placed into the catalog + Source Source `json:"source"` // Source represents the original object that was cataloged + Distro Distribution `json:"distro"` // Distro represents the Linux distribution that was detected from the source + Descriptor Descriptor `json:"descriptor"` // Descriptor is a block containing self-describing information about syft } -// Distritbution provides information about a detected Linux Distribution -type Distribution struct { - Name string `json:"name"` - Version string `json:"version"` - IDLike string `json:"idLike"` -} - -func NewDocument(catalog *pkg.Catalog, s scope.Scope, d distro.Distro) (Document, error) { - doc := Document{ - Artifacts: make([]Artifact, 0), - } - - src, err := NewSource(s) +// NewDocument creates and populates a new JSON document struct from the given cataloging results. +func NewDocument(catalog *pkg.Catalog, srcMetadata source.Metadata, d distro.Distro) (Document, error) { + src, err := NewSource(srcMetadata) if err != nil { return Document{}, nil } - doc.Source = src - distroName := d.Name() - if distroName == "UnknownDistroType" { - distroName = "" - } - doc.Distro = Distribution{ - Name: distroName, - Version: d.FullVersion(), - IDLike: d.IDLike, + + doc := Document{ + Artifacts: make([]Package, 0), + Source: src, + Distro: NewDistribution(d), + Descriptor: Descriptor{ + Name: internal.ApplicationName, + Version: version.FromBuild().Version, + }, } for _, p := range catalog.Sorted() { - art, err := NewArtifact(p, s) + art, err := NewPackage(p) if err != nil { return Document{}, err } diff --git a/syft/presenter/json/image.go b/syft/presenter/json/image.go deleted file mode 100644 index bffa999bf..000000000 --- a/syft/presenter/json/image.go +++ /dev/null @@ -1,44 +0,0 @@ -package json - -import ( - "github.com/anchore/syft/syft/scope" -) - -type Image struct { - Layers []Layer `json:"layers"` - Size int64 `json:"size"` - Digest string `json:"digest"` - MediaType string `json:"mediaType"` - Tags []string `json:"tags"` -} - -type Layer struct { - MediaType string `json:"mediaType"` - Digest string `json:"digest"` - Size int64 `json:"size"` -} - -func NewImage(src scope.ImageSource) *Image { - // populate artifacts... - tags := make([]string, len(src.Img.Metadata.Tags)) - for idx, tag := range src.Img.Metadata.Tags { - tags[idx] = tag.String() - } - img := Image{ - Digest: src.Img.Metadata.Digest, - Size: src.Img.Metadata.Size, - MediaType: string(src.Img.Metadata.MediaType), - Tags: tags, - Layers: make([]Layer, len(src.Img.Layers)), - } - - // populate image metadata - for idx, l := range src.Img.Layers { - img.Layers[idx] = Layer{ - MediaType: string(l.Metadata.MediaType), - Digest: l.Metadata.Digest, - Size: l.Metadata.Size, - } - } - return &img -} diff --git a/syft/presenter/json/location.go b/syft/presenter/json/location.go deleted file mode 100644 index 8a0fe1209..000000000 --- a/syft/presenter/json/location.go +++ /dev/null @@ -1,45 +0,0 @@ -package json - -import ( - "fmt" - - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" -) - -type Locations interface{} - -type ImageLocation struct { - Path string `json:"path"` - LayerIndex uint `json:"layerIndex"` -} - -func NewLocations(p *pkg.Package, s scope.Scope) (Locations, error) { - switch src := s.Source.(type) { - case scope.ImageSource: - locations := make([]ImageLocation, len(p.Source)) - for idx := range p.Source { - entry, err := src.Img.FileCatalog.Get(p.Source[idx]) - if err != nil { - return nil, fmt.Errorf("unable to find layer index for source-idx=%d package=%s", idx, p.Name) - } - - artifactSource := ImageLocation{ - LayerIndex: entry.Source.Metadata.Index, - Path: string(p.Source[idx].Path), - } - - locations[idx] = artifactSource - } - return locations, nil - - case scope.DirSource: - locations := make([]string, len(p.Source)) - for idx := range p.Source { - locations[idx] = string(p.Source[idx].Path) - } - return locations, nil - default: - return nil, fmt.Errorf("unable to determine source: %T", src) - } -} diff --git a/syft/presenter/json/presenter.go b/syft/presenter/json/presenter.go index 7c5ee5407..dd11381a0 100644 --- a/syft/presenter/json/presenter.go +++ b/syft/presenter/json/presenter.go @@ -6,25 +6,28 @@ import ( "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) +// Presenter is a JSON presentation object for the syft results type Presenter struct { - catalog *pkg.Catalog - scope scope.Scope - distro distro.Distro + catalog *pkg.Catalog + srcMetadata source.Metadata + distro distro.Distro } -func NewPresenter(catalog *pkg.Catalog, s scope.Scope, d distro.Distro) *Presenter { +// NewPresenter creates a new JSON presenter object for the given cataloging results. +func NewPresenter(catalog *pkg.Catalog, s source.Metadata, d distro.Distro) *Presenter { return &Presenter{ - catalog: catalog, - scope: s, - distro: d, + catalog: catalog, + srcMetadata: s, + distro: d, } } +// Present the catalog results to the given writer. func (pres *Presenter) Present(output io.Writer) error { - doc, err := NewDocument(pres.catalog, pres.scope, pres.distro) + doc, err := NewDocument(pres.catalog, pres.srcMetadata, pres.distro) if err != nil { return err } diff --git a/syft/presenter/json/presenter_test.go b/syft/presenter/json/presenter_test.go index b935050a2..57869e08e 100644 --- a/syft/presenter/json/presenter_test.go +++ b/syft/presenter/json/presenter_test.go @@ -6,11 +6,10 @@ import ( "testing" "github.com/anchore/go-testutils" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/sergi/go-diff/diffmatchpatch" ) @@ -25,27 +24,39 @@ func TestJsonDirsPresenter(t *testing.T) { catalog.Add(pkg.Package{ Name: "package-1", Version: "1.0.1", - Type: pkg.DebPkg, + Type: pkg.PythonPkg, FoundBy: "the-cataloger-1", - Source: []file.Reference{ + Locations: []source.Location{ {Path: "/some/path/pkg1"}, }, + Language: pkg.Python, + MetadataType: pkg.PythonPackageMetadataType, + Licenses: []string{"MIT"}, + Metadata: pkg.PythonPackageMetadata{ + Name: "package-1", + Version: "1.0.1", + }, }) catalog.Add(pkg.Package{ Name: "package-2", Version: "2.0.1", Type: pkg.DebPkg, FoundBy: "the-cataloger-2", - Source: []file.Reference{ + Locations: []source.Location{ {Path: "/some/path/pkg1"}, }, + MetadataType: pkg.DpkgMetadataType, + Metadata: pkg.DpkgMetadata{ + Package: "package-2", + Version: "2.0.1", + }, }) d := distro.NewUnknownDistro() - s, err := scope.NewScopeFromDir("/some/path") + s, err := source.NewFromDirectory("/some/path") if err != nil { t.Fatal(err) } - pres := NewPresenter(catalog, s, d) + pres := NewPresenter(catalog, s.Metadata, d) // run presenter err = pres.Present(&buffer) @@ -62,7 +73,7 @@ func TestJsonDirsPresenter(t *testing.T) { if !bytes.Equal(expected, actual) { dmp := diffmatchpatch.New() - diffs := dmp.DiffMain(string(actual), string(expected), true) + diffs := dmp.DiffMain(string(expected), string(actual), true) t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs)) } @@ -84,25 +95,37 @@ func TestJsonImgsPresenter(t *testing.T) { catalog.Add(pkg.Package{ Name: "package-1", Version: "1.0.1", - Source: []file.Reference{ - *img.SquashedTree().File("/somefile-1.txt"), + Locations: []source.Location{ + source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img), + }, + Type: pkg.PythonPkg, + FoundBy: "the-cataloger-1", + Language: pkg.Python, + MetadataType: pkg.PythonPackageMetadataType, + Licenses: []string{"MIT"}, + Metadata: pkg.PythonPackageMetadata{ + Name: "package-1", + Version: "1.0.1", }, - Type: pkg.DebPkg, - FoundBy: "the-cataloger-1", }) catalog.Add(pkg.Package{ Name: "package-2", Version: "2.0.1", - Source: []file.Reference{ - *img.SquashedTree().File("/somefile-2.txt"), + Locations: []source.Location{ + source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img), + }, + Type: pkg.DebPkg, + FoundBy: "the-cataloger-2", + MetadataType: pkg.DpkgMetadataType, + Metadata: pkg.DpkgMetadata{ + Package: "package-2", + Version: "2.0.1", }, - Type: pkg.DebPkg, - FoundBy: "the-cataloger-2", }) - s, err := scope.NewScopeFromImage(img, scope.AllLayersScope) + s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input") d := distro.NewUnknownDistro() - pres := NewPresenter(catalog, s, d) + pres := NewPresenter(catalog, s.Metadata, d) // run presenter err = pres.Present(&buffer) @@ -119,7 +142,7 @@ func TestJsonImgsPresenter(t *testing.T) { if !bytes.Equal(expected, actual) { dmp := diffmatchpatch.New() - diffs := dmp.DiffMain(string(actual), string(expected), true) + diffs := dmp.DiffMain(string(expected), string(actual), true) t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs)) } } diff --git a/syft/presenter/json/source.go b/syft/presenter/json/source.go index 471390999..7a5ad6ad2 100644 --- a/syft/presenter/json/source.go +++ b/syft/presenter/json/source.go @@ -1,24 +1,33 @@ package json import ( + "encoding/json" "fmt" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) +// Source object represents the thing that was cataloged type Source struct { Type string `json:"type"` Target interface{} `json:"target"` } -func NewSource(s scope.Scope) (Source, error) { - switch src := s.Source.(type) { - case scope.ImageSource: +// sourceUnpacker is used to unmarshal Source objects +type sourceUnpacker struct { + Type string `json:"type"` + Target json.RawMessage `json:"target"` +} + +// NewSource creates a new source object to be represented into JSON. +func NewSource(src source.Metadata) (Source, error) { + switch src.Scheme { + case source.ImageScheme: return Source{ Type: "image", - Target: NewImage(src), + Target: src.ImageMetadata, }, nil - case scope.DirSource: + case source.DirectoryScheme: return Source{ Type: "directory", Target: src.Path, @@ -27,3 +36,40 @@ func NewSource(s scope.Scope) (Source, error) { return Source{}, fmt.Errorf("unsupported source: %T", src) } } + +// UnmarshalJSON populates a source object from JSON bytes. +func (s *Source) UnmarshalJSON(b []byte) error { + var unpacker sourceUnpacker + if err := json.Unmarshal(b, &unpacker); err != nil { + return err + } + + s.Type = unpacker.Type + + switch s.Type { + case "image": + var payload source.ImageMetadata + if err := json.Unmarshal(unpacker.Target, &payload); err != nil { + return err + } + s.Target = payload + default: + return fmt.Errorf("unsupported package metadata type: %+v", s.Type) + } + + return nil +} + +// ToSourceMetadata takes a source object represented from JSON and creates a source.Metadata object. +func (s *Source) ToSourceMetadata() source.Metadata { + var metadata source.Metadata + switch s.Type { + case "directory": + metadata.Scheme = source.DirectoryScheme + metadata.Path = s.Target.(string) + case "image": + metadata.Scheme = source.ImageScheme + metadata.ImageMetadata = s.Target.(source.ImageMetadata) + } + return metadata +} diff --git a/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden b/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden index 2193e7875..4876f5949 100644 --- a/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden +++ b/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden @@ -3,26 +3,50 @@ { "name": "package-1", "version": "1.0.1", - "type": "deb", - "foundBy": [ - "the-cataloger-1" - ], + "type": "python", + "foundBy": "the-cataloger-1", "locations": [ - "/some/path/pkg1" + { + "path": "/some/path/pkg1" + } ], - "licenses": null + "licenses": [ + "MIT" + ], + "language": "python", + "metadataType": "PythonPackageMetadata", + "metadata": { + "name": "package-1", + "version": "1.0.1", + "license": "", + "author": "", + "authorEmail": "", + "platform": "", + "sitePackagesRootPath": "" + } }, { "name": "package-2", "version": "2.0.1", "type": "deb", - "foundBy": [ - "the-cataloger-2" - ], + "foundBy": "the-cataloger-2", "locations": [ - "/some/path/pkg1" + { + "path": "/some/path/pkg1" + } ], - "licenses": null + "licenses": null, + "language": "", + "metadataType": "DpkgMetadata", + "metadata": { + "package": "package-2", + "source": "", + "version": "2.0.1", + "architecture": "", + "maintainer": "", + "installedSize": 0, + "files": null + } } ], "source": { @@ -33,5 +57,9 @@ "name": "", "version": "", "idLike": "" + }, + "descriptor": { + "name": "syft", + "version": "[not provided]" } } diff --git a/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden b/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden index f3c7f56a2..81ba778fd 100644 --- a/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden +++ b/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden @@ -3,56 +3,78 @@ { "name": "package-1", "version": "1.0.1", - "type": "deb", - "foundBy": [ - "the-cataloger-1" - ], + "type": "python", + "foundBy": "the-cataloger-1", "locations": [ { "path": "/somefile-1.txt", - "layerIndex": 0 + "layerID": "sha256:e158b57d6f5a96ef5fd22f2fe76c70b5ba6ff5b2619f9d83125b2aad0492ac7b" } ], - "licenses": null + "licenses": [ + "MIT" + ], + "language": "python", + "metadataType": "PythonPackageMetadata", + "metadata": { + "name": "package-1", + "version": "1.0.1", + "license": "", + "author": "", + "authorEmail": "", + "platform": "", + "sitePackagesRootPath": "" + } }, { "name": "package-2", "version": "2.0.1", "type": "deb", - "foundBy": [ - "the-cataloger-2" - ], + "foundBy": "the-cataloger-2", "locations": [ { "path": "/somefile-2.txt", - "layerIndex": 1 + "layerID": "sha256:da21056e7bf4308ecea0c0836848a7fe92f38fdcf35bc09ee6d98e7ab7beeebf" } ], - "licenses": null + "licenses": null, + "language": "", + "metadataType": "DpkgMetadata", + "metadata": { + "package": "package-2", + "source": "", + "version": "2.0.1", + "architecture": "", + "maintainer": "", + "installedSize": 0, + "files": null + } } ], "source": { "type": "image", "target": { + "userInput": "user-image-input", + "scope": "AllLayers", "layers": [ { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "digest": "sha256:78783bfc74fef84f899b4977561ad1172f87753f82cc2157b06bf097e56dfbce", + "digest": "sha256:e158b57d6f5a96ef5fd22f2fe76c70b5ba6ff5b2619f9d83125b2aad0492ac7b", "size": 22 }, { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "digest": "sha256:54ec7f643dafbf9f27032a5e60afe06248c0e99b50aed54bb0fe28ea4825ccaf", + "digest": "sha256:da21056e7bf4308ecea0c0836848a7fe92f38fdcf35bc09ee6d98e7ab7beeebf", "size": 16 }, { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "digest": "sha256:ec4775a139c45b1ddf9ea8e1cb43385e92e5c0bf6ec2e3f4192372785b18c106", + "digest": "sha256:f0e18aa6032c24659a9c741fc36ca56f589782ea132061ccf6f52b952403da94", "size": 27 } ], "size": 65, - "digest": "sha256:fedd7bcc0b90f071501b662d8e7c9ac7548b88daba6b3deedfdf33f22ed8d95b", + "digest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368", "mediaType": "application/vnd.docker.distribution.manifest.v2+json", "tags": [ "stereoscope-fixture-image-simple:04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7" @@ -63,5 +85,9 @@ "name": "", "version": "", "idLike": "" + }, + "descriptor": { + "name": "syft", + "version": "[not provided]" } } diff --git a/syft/presenter/json/test-fixtures/snapshot/anchore-fixture-image-simple.golden b/syft/presenter/json/test-fixtures/snapshot/anchore-fixture-image-simple.golden deleted file mode 100644 index 3e1d2daaa..000000000 Binary files a/syft/presenter/json/test-fixtures/snapshot/anchore-fixture-image-simple.golden and /dev/null differ diff --git a/syft/presenter/json/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden b/syft/presenter/json/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden index c98a9207c..4e7ce36e0 100644 Binary files a/syft/presenter/json/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden and b/syft/presenter/json/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden differ diff --git a/syft/presenter/presenter.go b/syft/presenter/presenter.go index 65184a74e..6c32b60d0 100644 --- a/syft/presenter/presenter.go +++ b/syft/presenter/presenter.go @@ -15,7 +15,7 @@ import ( "github.com/anchore/syft/syft/presenter/json" "github.com/anchore/syft/syft/presenter/table" "github.com/anchore/syft/syft/presenter/text" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) // Presenter defines the expected behavior for an object responsible for displaying arbitrary input and processed data @@ -25,16 +25,16 @@ type Presenter interface { } // GetPresenter returns a presenter for images or directories -func GetPresenter(option Option, s scope.Scope, catalog *pkg.Catalog, d *distro.Distro) Presenter { +func GetPresenter(option Option, srcMetadata source.Metadata, catalog *pkg.Catalog, d distro.Distro) Presenter { switch option { case JSONPresenter: - return json.NewPresenter(catalog, s, *d) + return json.NewPresenter(catalog, srcMetadata, d) case TextPresenter: - return text.NewPresenter(catalog, s) + return text.NewPresenter(catalog, srcMetadata) case TablePresenter: - return table.NewPresenter(catalog, s) + return table.NewPresenter(catalog) case CycloneDxPresenter: - return cyclonedx.NewPresenter(catalog, s, *d) + return cyclonedx.NewPresenter(catalog, srcMetadata, d) default: return nil } diff --git a/syft/presenter/table/presenter.go b/syft/presenter/table/presenter.go index 29f4631c8..c7303e8a5 100644 --- a/syft/presenter/table/presenter.go +++ b/syft/presenter/table/presenter.go @@ -9,18 +9,15 @@ import ( "github.com/olekukonko/tablewriter" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" ) type Presenter struct { catalog *pkg.Catalog - scope scope.Scope } -func NewPresenter(catalog *pkg.Catalog, s scope.Scope) *Presenter { +func NewPresenter(catalog *pkg.Catalog) *Presenter { return &Presenter{ catalog: catalog, - scope: s, } } diff --git a/syft/presenter/table/presenter_test.go b/syft/presenter/table/presenter_test.go index 4958b1943..02dd2f5d4 100644 --- a/syft/presenter/table/presenter_test.go +++ b/syft/presenter/table/presenter_test.go @@ -3,14 +3,14 @@ package table import ( "bytes" "flag" - "github.com/go-test/deep" "testing" + "github.com/go-test/deep" + "github.com/anchore/go-testutils" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/sergi/go-diff/diffmatchpatch" ) @@ -29,25 +29,24 @@ func TestTablePresenter(t *testing.T) { catalog.Add(pkg.Package{ Name: "package-1", Version: "1.0.1", - Source: []file.Reference{ - *img.SquashedTree().File("/somefile-1.txt"), + Locations: []source.Location{ + source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img), }, Type: pkg.DebPkg, }) catalog.Add(pkg.Package{ Name: "package-2", Version: "2.0.1", - Source: []file.Reference{ - *img.SquashedTree().File("/somefile-2.txt"), + Locations: []source.Location{ + source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img), }, Type: pkg.DebPkg, }) - s, err := scope.NewScopeFromImage(img, scope.AllLayersScope) - pres := NewPresenter(catalog, s) + pres := NewPresenter(catalog) // run presenter - err = pres.Present(&buffer) + err := pres.Present(&buffer) if err != nil { t.Fatal(err) } diff --git a/syft/presenter/text/presenter.go b/syft/presenter/text/presenter.go index 3291370d9..8e86fea47 100644 --- a/syft/presenter/text/presenter.go +++ b/syft/presenter/text/presenter.go @@ -7,18 +7,20 @@ import ( "text/tabwriter" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) +// Presenter is a human-friendly text presenter to represent package and source data. type Presenter struct { - catalog *pkg.Catalog - scope scope.Scope + catalog *pkg.Catalog + srcMetadata source.Metadata } -func NewPresenter(catalog *pkg.Catalog, s scope.Scope) *Presenter { +// NewPresenter creates a new presenter for the given set of catalog and image data. +func NewPresenter(catalog *pkg.Catalog, srcMetadata source.Metadata) *Presenter { return &Presenter{ - catalog: catalog, - scope: s, + catalog: catalog, + srcMetadata: srcMetadata, } } @@ -28,22 +30,22 @@ func (pres *Presenter) Present(output io.Writer) error { w := new(tabwriter.Writer) w.Init(output, 0, 8, 0, '\t', tabwriter.AlignRight) - switch src := pres.scope.Source.(type) { - case scope.DirSource: - fmt.Fprintln(w, fmt.Sprintf("[Path: %s]", src.Path)) - case scope.ImageSource: + switch pres.srcMetadata.Scheme { + case source.DirectoryScheme: + fmt.Fprintln(w, fmt.Sprintf("[Path: %s]", pres.srcMetadata.Path)) + case source.ImageScheme: fmt.Fprintln(w, "[Image]") - for idx, l := range src.Img.Layers { + for idx, l := range pres.srcMetadata.ImageMetadata.Layers { fmt.Fprintln(w, " Layer:\t", idx) - fmt.Fprintln(w, " Digest:\t", l.Metadata.Digest) - fmt.Fprintln(w, " Size:\t", l.Metadata.Size) - fmt.Fprintln(w, " MediaType:\t", l.Metadata.MediaType) + fmt.Fprintln(w, " Digest:\t", l.Digest) + fmt.Fprintln(w, " Size:\t", l.Size) + fmt.Fprintln(w, " MediaType:\t", l.MediaType) fmt.Fprintln(w) w.Flush() } default: - return fmt.Errorf("unsupported source: %T", src) + return fmt.Errorf("unsupported source: %T", pres.srcMetadata.Scheme) } // populate artifacts... diff --git a/syft/presenter/text/presenter_test.go b/syft/presenter/text/presenter_test.go index 96dbd84c1..4ca2f756b 100644 --- a/syft/presenter/text/presenter_test.go +++ b/syft/presenter/text/presenter_test.go @@ -6,10 +6,9 @@ import ( "testing" "github.com/anchore/go-testutils" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/sergi/go-diff/diffmatchpatch" ) @@ -32,11 +31,11 @@ func TestTextDirPresenter(t *testing.T) { Type: pkg.DebPkg, }) - s, err := scope.NewScopeFromDir("/some/path") + s, err := source.NewFromDirectory("/some/path") if err != nil { - t.Fatalf("unable to create scope: %+v", err) + t.Fatalf("unable to create source: %+v", err) } - pres := NewPresenter(catalog, s) + pres := NewPresenter(catalog, s.Metadata) // run presenter err = pres.Present(&buffer) @@ -75,8 +74,8 @@ func TestTextImgPresenter(t *testing.T) { catalog.Add(pkg.Package{ Name: "package-1", Version: "1.0.1", - Source: []file.Reference{ - *img.SquashedTree().File("/somefile-1.txt"), + Locations: []source.Location{ + source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img), }, FoundBy: "dpkg", Type: pkg.DebPkg, @@ -84,8 +83,8 @@ func TestTextImgPresenter(t *testing.T) { catalog.Add(pkg.Package{ Name: "package-2", Version: "2.0.1", - Source: []file.Reference{ - *img.SquashedTree().File("/somefile-2.txt"), + Locations: []source.Location{ + source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img), }, FoundBy: "dpkg", Metadata: PackageInfo{Name: "package-2", Version: "1.0.2"}, @@ -98,11 +97,11 @@ func TestTextImgPresenter(t *testing.T) { l.Metadata.Digest = "sha256:ad8ecdc058976c07e7e347cb89fa9ad86a294b5ceaae6d09713fb035f84115abf3c4a2388a4af3aa60f13b94f4c6846930bdf53" } - s, err := scope.NewScopeFromImage(img, scope.AllLayersScope) + s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input") if err != nil { t.Fatal(err) } - pres := NewPresenter(catalog, s) + pres := NewPresenter(catalog, s.Metadata) // run presenter err = pres.Present(&buffer) if err != nil { diff --git a/syft/scope/option.go b/syft/scope/option.go deleted file mode 100644 index fcfd49302..000000000 --- a/syft/scope/option.go +++ /dev/null @@ -1,40 +0,0 @@ -package scope - -import "strings" - -const ( - UnknownScope Option = iota - SquashedScope - AllLayersScope -) - -type Option int - -var optionStr = []string{ - "UnknownScope", - "Squashed", - "AllLayers", -} - -var Options = []Option{ - SquashedScope, - AllLayersScope, -} - -func ParseOption(userStr string) Option { - switch strings.ToLower(userStr) { - case strings.ToLower(SquashedScope.String()): - return SquashedScope - case "all-layers", strings.ToLower(AllLayersScope.String()): - return AllLayersScope - } - return UnknownScope -} - -func (o Option) String() string { - if int(o) >= len(optionStr) || o < 0 { - return optionStr[0] - } - - return optionStr[o] -} diff --git a/syft/scope/option_test.go b/syft/scope/option_test.go deleted file mode 100644 index a684c2260..000000000 --- a/syft/scope/option_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package scope - -import ( - "fmt" - "testing" -) - -func TestOptionStringerBoundary(t *testing.T) { - var _ fmt.Stringer = Option(0) - - for _, c := range []int{-1, 0, 3} { - option := Option(c) - if option.String() != UnknownScope.String() { - t.Errorf("expected Option(%d) to be unknown, found '%+v'", c, option) - } - } -} diff --git a/syft/scope/resolvers/directory_resolver.go b/syft/scope/resolvers/directory_resolver.go deleted file mode 100644 index 5677c6dde..000000000 --- a/syft/scope/resolvers/directory_resolver.go +++ /dev/null @@ -1,127 +0,0 @@ -package resolvers - -import ( - "fmt" - "io/ioutil" - "os" - "path" - "path/filepath" - - "github.com/anchore/stereoscope/pkg/file" - "github.com/anchore/syft/internal/log" - "github.com/bmatcuk/doublestar" -) - -// DirectoryResolver implements path and content access for the directory data source. -type DirectoryResolver struct { - Path string -} - -// Stringer to represent a directory path data source -func (s DirectoryResolver) String() string { - return fmt.Sprintf("dir:%s", s.Path) -} - -// FilesByPath returns all file.References that match the given paths from the directory. -func (s DirectoryResolver) FilesByPath(userPaths ...file.Path) ([]file.Reference, error) { - var references = make([]file.Reference, 0) - - for _, userPath := range userPaths { - userStrPath := string(userPath) - - if filepath.IsAbs(userStrPath) { - // a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is - userStrPath = path.Join(s.Path, userStrPath) - } - fileMeta, err := os.Stat(userStrPath) - if os.IsNotExist(err) { - continue - } else if err != nil { - log.Errorf("path (%s) is not valid: %v", userStrPath, err) - } - - // don't consider directories - if fileMeta.IsDir() { - continue - } - - references = append(references, file.NewFileReference(file.Path(userStrPath))) - } - - return references, nil -} - -func fileContents(path file.Path) ([]byte, error) { - contents, err := ioutil.ReadFile(string(path)) - - if err != nil { - return nil, err - } - return contents, nil -} - -// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. -func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) { - result := make([]file.Reference, 0) - - for _, pattern := range patterns { - pathPattern := path.Join(s.Path, pattern) - matches, err := doublestar.Glob(pathPattern) - if err != nil { - return result, err - } - for _, match := range matches { - fileMeta, err := os.Stat(match) - if err != nil { - continue - } - - // don't consider directories - if fileMeta.IsDir() { - continue - } - - matchedPath := file.Path(match) - result = append(result, file.NewFileReference(matchedPath)) - } - } - - return result, nil -} - -func (s *DirectoryResolver) RelativeFileByPath(_ file.Reference, path string) (*file.Reference, error) { - paths, err := s.FilesByPath(file.Path(path)) - if err != nil { - return nil, err - } - if len(paths) == 0 { - return nil, nil - } - - return &paths[0], nil -} - -// MultipleFileContentsByRef returns the file contents for all file.References relative a directory. -func (s DirectoryResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) { - refContents := make(map[file.Reference]string) - for _, fileRef := range f { - contents, err := fileContents(fileRef.Path) - - if err != nil { - return nil, fmt.Errorf("could not read contents of file: %s", fileRef.Path) - } - refContents[fileRef] = string(contents) - } - return refContents, nil -} - -// FileContentsByRef fetches file contents for a single file reference relative to a directory. -// If the path does not exist an error is returned. -func (s DirectoryResolver) FileContentsByRef(reference file.Reference) (string, error) { - contents, err := fileContents(reference.Path) - if err != nil { - return "", fmt.Errorf("could not read contents of file: %s", reference.Path) - } - - return string(contents), nil -} diff --git a/syft/scope/resolvers/docs.go b/syft/scope/resolvers/docs.go deleted file mode 100644 index fe8f17e76..000000000 --- a/syft/scope/resolvers/docs.go +++ /dev/null @@ -1,4 +0,0 @@ -/* -Package resolvers provides concrete implementations for the scope.Resolver interface for all supported data sources and scope options. -*/ -package resolvers diff --git a/syft/scope/scope.go b/syft/scope/scope.go deleted file mode 100644 index bf2ed74f3..000000000 --- a/syft/scope/scope.go +++ /dev/null @@ -1,166 +0,0 @@ -/* -Package scope provides an abstraction to allow a user to loosely define a data source to catalog and expose a common interface that -catalogers and use explore and analyze data from the data source. All valid (cataloggable) data sources are defined -within this package. -*/ -package scope - -import ( - "fmt" - "strings" - - "github.com/mitchellh/go-homedir" - - "github.com/spf13/afero" - - "github.com/anchore/stereoscope" - - "github.com/anchore/stereoscope/pkg/image" - "github.com/anchore/syft/syft/scope/resolvers" -) - -const ( - UnknownScheme Scheme = "unknown-scheme" - DirectoryScheme Scheme = "directory-scheme" - ImageScheme Scheme = "image-scheme" -) - -type Scheme string - -// ImageSource represents a data source that is a container image -type ImageSource struct { - Img *image.Image // the image object to be cataloged -} - -// DirSource represents a data source that is a filesystem directory tree -type DirSource struct { - Path string // the root path to be cataloged -} - -// Scope is an object that captures the data source to be cataloged, configuration, and a specific resolver used -// in cataloging (based on the data source and configuration) -type Scope struct { - Option Option // specific perspective to catalog - Resolver Resolver // a Resolver object to use in file path/glob resolution and file contents resolution - Source interface{} // the specific source object to be cataloged - Scheme Scheme // the source data scheme type (directory or image) -} - -// NewScope produces a Scope based on userInput like dir: or image:tag -func NewScope(userInput string, o Option) (Scope, func(), error) { - fs := afero.NewOsFs() - parsedScheme, location, err := detectScheme(fs, image.DetectSource, userInput) - if err != nil { - return Scope{}, func() {}, fmt.Errorf("unable to parse input=%q: %w", userInput, err) - } - - switch parsedScheme { - case DirectoryScheme: - fileMeta, err := fs.Stat(location) - if err != nil { - return Scope{}, func() {}, fmt.Errorf("unable to stat dir=%q: %w", location, err) - } - - if !fileMeta.IsDir() { - return Scope{}, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", location, err) - } - - s, err := NewScopeFromDir(location) - if err != nil { - return Scope{}, func() {}, fmt.Errorf("could not populate scope from path=%q: %w", location, err) - } - return s, func() {}, nil - - case ImageScheme: - img, err := stereoscope.GetImage(location) - cleanup := func() { - stereoscope.Cleanup() - } - - if err != nil || img == nil { - return Scope{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", location, err) - } - - s, err := NewScopeFromImage(img, o) - if err != nil { - return Scope{}, cleanup, fmt.Errorf("could not populate scope with image: %w", err) - } - return s, cleanup, nil - } - - return Scope{}, func() {}, fmt.Errorf("unable to process input for scanning: '%s'", userInput) -} - -// NewScopeFromDir creates a new scope object tailored to catalog a given filesystem directory recursively. -func NewScopeFromDir(path string) (Scope, error) { - return Scope{ - Resolver: &resolvers.DirectoryResolver{ - Path: path, - }, - Source: DirSource{ - Path: path, - }, - Scheme: DirectoryScheme, - }, nil -} - -// NewScopeFromImage creates a new scope object tailored to catalog a given container image, relative to the -// option given (e.g. all-layers, squashed, etc) -func NewScopeFromImage(img *image.Image, option Option) (Scope, error) { - if img == nil { - return Scope{}, fmt.Errorf("no image given") - } - - resolver, err := getImageResolver(img, option) - if err != nil { - return Scope{}, fmt.Errorf("could not determine file resolver: %w", err) - } - - return Scope{ - Option: option, - Resolver: resolver, - Source: ImageSource{ - Img: img, - }, - Scheme: ImageScheme, - }, nil -} - -type sourceDetector func(string) (image.Source, string, error) - -func detectScheme(fs afero.Fs, imageDetector sourceDetector, userInput string) (Scheme, string, error) { - if strings.HasPrefix(userInput, "dir:") { - // blindly trust the user's scheme - dirLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "dir:")) - if err != nil { - return UnknownScheme, "", fmt.Errorf("unable to expand directory path: %w", err) - } - return DirectoryScheme, dirLocation, nil - } - - // we should attempt to let stereoscope determine what the source is first --just because the source is a valid directory - // doesn't mean we yet know if it is an OCI layout directory (to be treated as an image) or if it is a generic filesystem directory. - source, imageSpec, err := imageDetector(userInput) - if err != nil { - return UnknownScheme, "", fmt.Errorf("unable to detect the scheme from %q: %w", userInput, err) - } - - if source == image.UnknownSource { - dirLocation, err := homedir.Expand(userInput) - if err != nil { - return UnknownScheme, "", fmt.Errorf("unable to expand potential directory path: %w", err) - } - - fileMeta, err := fs.Stat(dirLocation) - if err != nil { - return UnknownScheme, "", nil - } - - if fileMeta.IsDir() { - return DirectoryScheme, dirLocation, nil - } - return UnknownScheme, "", nil - } - - return ImageScheme, imageSpec, nil -} diff --git a/syft/scope/resolvers/all_layers_resolver.go b/syft/source/all_layers_resolver.go similarity index 61% rename from syft/scope/resolvers/all_layers_resolver.go rename to syft/source/all_layers_resolver.go index a1c57ccda..4b99bda5b 100644 --- a/syft/scope/resolvers/all_layers_resolver.go +++ b/syft/source/all_layers_resolver.go @@ -1,4 +1,4 @@ -package resolvers +package source import ( "archive/tar" @@ -8,7 +8,9 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) -// AllLayersResolver implements path and content access for the AllLayers scope option for container image data sources. +var _ Resolver = (*AllLayersResolver)(nil) + +// AllLayersResolver implements path and content access for the AllLayers source option for container image data sources. type AllLayersResolver struct { img *image.Image layers []int @@ -41,7 +43,7 @@ func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.Ref if entry.Metadata.TypeFlag == tar.TypeLink || entry.Metadata.TypeFlag == tar.TypeSymlink { // a link may resolve in this layer or higher, assuming a squashed tree is used to search - // we should search all possible resolutions within the valid scope + // we should search all possible resolutions within the valid source for _, subLayerIdx := range r.layers[layerIdx:] { resolvedRef, err := r.img.ResolveLinkByLayerSquash(ref, subLayerIdx) if err != nil { @@ -61,14 +63,14 @@ func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.Ref } // FilesByPath returns all file.References that match the given paths from any layer in the image. -func (r *AllLayersResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) { +func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) { uniqueFileIDs := file.NewFileReferenceSet() - uniqueFiles := make([]file.Reference, 0) + uniqueLocations := make([]Location, 0) for _, path := range paths { for idx, layerIdx := range r.layers { tree := r.img.Layers[layerIdx].Tree - ref := tree.File(path) + ref := tree.File(file.Path(path)) if ref == nil { // no file found, keep looking through layers continue @@ -91,17 +93,19 @@ func (r *AllLayersResolver) FilesByPath(paths ...file.Path) ([]file.Reference, e if err != nil { return nil, err } - uniqueFiles = append(uniqueFiles, results...) + for _, result := range results { + uniqueLocations = append(uniqueLocations, NewLocationFromImage(result, r.img)) + } } } - - return uniqueFiles, nil + return uniqueLocations, nil } // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. -func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) { +// nolint:gocognit +func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error) { uniqueFileIDs := file.NewFileReferenceSet() - uniqueFiles := make([]file.Reference, 0) + uniqueLocations := make([]Location, 0) for _, pattern := range patterns { for idx, layerIdx := range r.layers { @@ -128,31 +132,65 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, e if err != nil { return nil, err } - uniqueFiles = append(uniqueFiles, results...) + for _, result := range results { + uniqueLocations = append(uniqueLocations, NewLocationFromImage(result, r.img)) + } } } } - return uniqueFiles, nil + return uniqueLocations, nil } -func (r *AllLayersResolver) RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) { - entry, err := r.img.FileCatalog.Get(reference) +// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. +// This is helpful when attempting to find a file that is in the same layer or lower as another file. +func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) *Location { + entry, err := r.img.FileCatalog.Get(location.ref) + if err != nil { + return nil + } + + relativeRef := entry.Source.SquashedTree.File(file.Path(path)) + if relativeRef == nil { + return nil + } + + relativeLocation := NewLocationFromImage(*relativeRef, r.img) + + return &relativeLocation +} + +// MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a +// file.Reference is a path relative to a particular layer. +func (r *AllLayersResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) { + return mapLocationRefs(r.img.MultipleFileContentsByRef, locations) +} + +// FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer. +// If the path does not exist an error is returned. +func (r *AllLayersResolver) FileContentsByLocation(location Location) (string, error) { + return r.img.FileContentsByRef(location.ref) +} + +type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]string, error) + +func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]string, error) { + var fileRefs = make([]file.Reference, len(locations)) + var locationByRefs = make(map[file.Reference]Location) + var results = make(map[Location]string) + + for i, location := range locations { + locationByRefs[location.ref] = location + fileRefs[i] = location.ref + } + + contentsByRef, err := callback(fileRefs...) if err != nil { return nil, err } - return entry.Source.SquashedTree.File(file.Path(path)), nil -} - -// MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a -// file.Reference is a path relative to a particular layer. -func (r *AllLayersResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) { - return r.img.MultipleFileContentsByRef(f...) -} - -// FileContentsByRef fetches file contents for a single file reference, irregardless of the source layer. -// If the path does not exist an error is returned. -func (r *AllLayersResolver) FileContentsByRef(ref file.Reference) (string, error) { - return r.img.FileContentsByRef(ref) + for ref, content := range contentsByRef { + results[locationByRefs[ref]] = content + } + return results, nil } diff --git a/syft/scope/resolvers/all_layers_resolver_test.go b/syft/source/all_layers_resolver_test.go similarity index 93% rename from syft/scope/resolvers/all_layers_resolver_test.go rename to syft/source/all_layers_resolver_test.go index e99ebd8a6..ac3ea4ebf 100644 --- a/syft/scope/resolvers/all_layers_resolver_test.go +++ b/syft/source/all_layers_resolver_test.go @@ -1,11 +1,9 @@ -package resolvers +package source import ( "testing" "github.com/anchore/stereoscope/pkg/imagetest" - - "github.com/anchore/stereoscope/pkg/file" ) type resolution struct { @@ -97,7 +95,7 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) { t.Fatalf("could not create resolver: %+v", err) } - refs, err := resolver.FilesByPath(file.Path(c.linkPath)) + refs, err := resolver.FilesByPath(c.linkPath) if err != nil { t.Fatalf("could not use resolver: %+v", err) } @@ -109,11 +107,11 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) { for idx, actual := range refs { expected := c.resolutions[idx] - if actual.Path != file.Path(expected.path) { + if actual.Path != expected.path { t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path) } - entry, err := img.FileCatalog.Get(actual) + entry, err := img.FileCatalog.Get(actual.ref) if err != nil { t.Fatalf("failed to get metadata: %+v", err) } @@ -222,11 +220,11 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) { for idx, actual := range refs { expected := c.resolutions[idx] - if actual.Path != file.Path(expected.path) { + if actual.Path != expected.path { t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path) } - entry, err := img.FileCatalog.Get(actual) + entry, err := img.FileCatalog.Get(actual.ref) if err != nil { t.Fatalf("failed to get metadata: %+v", err) } diff --git a/syft/source/directory_resolver.go b/syft/source/directory_resolver.go new file mode 100644 index 000000000..389914950 --- /dev/null +++ b/syft/source/directory_resolver.go @@ -0,0 +1,130 @@ +package source + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + + "github.com/anchore/syft/internal/log" + "github.com/bmatcuk/doublestar" +) + +var _ Resolver = (*DirectoryResolver)(nil) + +// DirectoryResolver implements path and content access for the directory data source. +type DirectoryResolver struct { + Path string +} + +// Stringer to represent a directory path data source +func (s DirectoryResolver) String() string { + return fmt.Sprintf("dir:%s", s.Path) +} + +// FilesByPath returns all file.References that match the given paths from the directory. +func (s DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error) { + var references = make([]Location, 0) + + for _, userPath := range userPaths { + userStrPath := userPath + + if filepath.IsAbs(userStrPath) { + // a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is + userStrPath = path.Join(s.Path, userStrPath) + } + fileMeta, err := os.Stat(userStrPath) + if os.IsNotExist(err) { + continue + } else if err != nil { + log.Errorf("path (%s) is not valid: %v", userStrPath, err) + } + + // don't consider directories + if fileMeta.IsDir() { + continue + } + + references = append(references, NewLocation(userStrPath)) + } + + return references, nil +} + +// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. +func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) { + result := make([]Location, 0) + + for _, pattern := range patterns { + pathPattern := path.Join(s.Path, pattern) + pathMatches, err := doublestar.Glob(pathPattern) + if err != nil { + return nil, err + } + for _, matchedPath := range pathMatches { + fileMeta, err := os.Stat(matchedPath) + if err != nil { + continue + } + + // don't consider directories + if fileMeta.IsDir() { + continue + } + + result = append(result, NewLocation(matchedPath)) + } + } + + return result, nil +} + +// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. +// This is helpful when attempting to find a file that is in the same layer or lower as another file. For the +// DirectoryResolver, this is a simple path lookup. +func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Location { + paths, err := s.FilesByPath(path) + if err != nil { + return nil + } + if len(paths) == 0 { + return nil + } + + return &paths[0] +} + +// MultipleFileContentsByLocation returns the file contents for all file.References relative a directory. +func (s DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) { + refContents := make(map[Location]string) + for _, location := range locations { + contents, err := fileContents(location.Path) + + if err != nil { + return nil, fmt.Errorf("could not read contents of file: %s", location.Path) + } + refContents[location] = string(contents) + } + return refContents, nil +} + +// FileContentsByLocation fetches file contents for a single file reference relative to a directory. +// If the path does not exist an error is returned. +func (s DirectoryResolver) FileContentsByLocation(location Location) (string, error) { + contents, err := fileContents(location.Path) + if err != nil { + return "", fmt.Errorf("could not read contents of file: %s", location.Path) + } + + return string(contents), nil +} + +func fileContents(path string) ([]byte, error) { + contents, err := ioutil.ReadFile(path) + + if err != nil { + return nil, err + } + return contents, nil +} diff --git a/syft/scope/resolvers/directory_resolver_test.go b/syft/source/directory_resolver_test.go similarity index 77% rename from syft/scope/resolvers/directory_resolver_test.go rename to syft/source/directory_resolver_test.go index 809148da8..f2bcad89f 100644 --- a/syft/scope/resolvers/directory_resolver_test.go +++ b/syft/source/directory_resolver_test.go @@ -1,9 +1,7 @@ -package resolvers +package source import ( "testing" - - "github.com/anchore/stereoscope/pkg/file" ) func TestDirectoryResolver_FilesByPath(t *testing.T) { @@ -58,7 +56,7 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { resolver := DirectoryResolver{c.root} - refs, err := resolver.FilesByPath(file.Path(c.input)) + refs, err := resolver.FilesByPath(c.input) if err != nil { t.Fatalf("could not use resolver: %+v, %+v", err, refs) } @@ -68,7 +66,7 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) { } for _, actual := range refs { - if actual.Path != file.Path(c.expected) { + if actual.Path != c.expected { t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.expected) } } @@ -79,22 +77,22 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) { func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) { cases := []struct { name string - input []file.Path + input []string refCount int }{ { name: "finds multiple files", - input: []file.Path{file.Path("test-fixtures/image-symlinks/file-1.txt"), file.Path("test-fixtures/image-symlinks/file-2.txt")}, + input: []string{"test-fixtures/image-symlinks/file-1.txt", "test-fixtures/image-symlinks/file-2.txt"}, refCount: 2, }, { name: "skips non-existing files", - input: []file.Path{file.Path("test-fixtures/image-symlinks/bogus.txt"), file.Path("test-fixtures/image-symlinks/file-1.txt")}, + input: []string{"test-fixtures/image-symlinks/bogus.txt", "test-fixtures/image-symlinks/file-1.txt"}, refCount: 1, }, { name: "does not return anything for non-existing directories", - input: []file.Path{file.Path("test-fixtures/non-existing/bogus.txt"), file.Path("test-fixtures/non-existing/file-1.txt")}, + input: []string{"test-fixtures/non-existing/bogus.txt", "test-fixtures/non-existing/file-1.txt"}, refCount: 0, }, } @@ -117,47 +115,47 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) { func TestDirectoryResolver_MultipleFileContentsByRef(t *testing.T) { cases := []struct { name string - input []file.Path + input []string refCount int contents []string }{ { name: "gets multiple file contents", - input: []file.Path{file.Path("test-fixtures/image-symlinks/file-1.txt"), file.Path("test-fixtures/image-symlinks/file-2.txt")}, + input: []string{"test-fixtures/image-symlinks/file-1.txt", "test-fixtures/image-symlinks/file-2.txt"}, refCount: 2, }, { name: "skips non-existing files", - input: []file.Path{file.Path("test-fixtures/image-symlinks/bogus.txt"), file.Path("test-fixtures/image-symlinks/file-1.txt")}, + input: []string{"test-fixtures/image-symlinks/bogus.txt", "test-fixtures/image-symlinks/file-1.txt"}, refCount: 1, }, { name: "does not return anything for non-existing directories", - input: []file.Path{file.Path("test-fixtures/non-existing/bogus.txt"), file.Path("test-fixtures/non-existing/file-1.txt")}, + input: []string{"test-fixtures/non-existing/bogus.txt", "test-fixtures/non-existing/file-1.txt"}, refCount: 0, }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - refs := make([]file.Reference, 0) + locations := make([]Location, 0) resolver := DirectoryResolver{"test-fixtures"} for _, p := range c.input { newRefs, err := resolver.FilesByPath(p) if err != nil { - t.Errorf("could not generate refs: %+v", err) + t.Errorf("could not generate locations: %+v", err) } for _, ref := range newRefs { - refs = append(refs, ref) + locations = append(locations, ref) } } - contents, err := resolver.MultipleFileContentsByRef(refs...) + contents, err := resolver.MultipleFileContentsByLocation(locations) if err != nil { t.Fatalf("unable to generate file contents by ref: %+v", err) } if len(contents) != c.refCount { - t.Errorf("unexpected number of refs produced: %d != %d", len(contents), c.refCount) + t.Errorf("unexpected number of locations produced: %d != %d", len(contents), c.refCount) } }) diff --git a/syft/source/image_metadata.go b/syft/source/image_metadata.go new file mode 100644 index 000000000..ce6c59223 --- /dev/null +++ b/syft/source/image_metadata.go @@ -0,0 +1,50 @@ +package source + +import "github.com/anchore/stereoscope/pkg/image" + +// ImageMetadata represents all static metadata that defines what a container image is. This is useful to later describe +// "what" was cataloged without needing the more complicated stereoscope Image objects or Resolver objects. +type ImageMetadata struct { + UserInput string `json:"userInput"` + Scope Scope `json:"scope"` // specific perspective to catalog + Layers []LayerMetadata `json:"layers"` + Size int64 `json:"size"` + Digest string `json:"digest"` + MediaType string `json:"mediaType"` + Tags []string `json:"tags"` +} + +// LayerMetadata represents all static metadata that defines what a container image layer is. +type LayerMetadata struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + Size int64 `json:"size"` +} + +// NewImageMetadata creates a new ImageMetadata object populated from the given stereoscope Image object and user configuration. +func NewImageMetadata(img *image.Image, userInput string, scope Scope) ImageMetadata { + // populate artifacts... + tags := make([]string, len(img.Metadata.Tags)) + for idx, tag := range img.Metadata.Tags { + tags[idx] = tag.String() + } + theImg := ImageMetadata{ + UserInput: userInput, + Scope: scope, + Digest: img.Metadata.Digest, + Size: img.Metadata.Size, + MediaType: string(img.Metadata.MediaType), + Tags: tags, + Layers: make([]LayerMetadata, len(img.Layers)), + } + + // populate image metadata + for idx, l := range img.Layers { + theImg.Layers[idx] = LayerMetadata{ + MediaType: string(l.Metadata.MediaType), + Digest: l.Metadata.Digest, + Size: l.Metadata.Size, + } + } + return theImg +} diff --git a/syft/scope/resolvers/image_squash_resolver.go b/syft/source/image_squash_resolver.go similarity index 61% rename from syft/scope/resolvers/image_squash_resolver.go rename to syft/source/image_squash_resolver.go index be1b7fe80..6be3422e9 100644 --- a/syft/scope/resolvers/image_squash_resolver.go +++ b/syft/source/image_squash_resolver.go @@ -1,4 +1,4 @@ -package resolvers +package source import ( "fmt" @@ -7,7 +7,9 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) -// ImageSquashResolver implements path and content access for the Squashed scope option for container image data sources. +var _ Resolver = (*ImageSquashResolver)(nil) + +// ImageSquashResolver implements path and content access for the Squashed source option for container image data sources. type ImageSquashResolver struct { img *image.Image } @@ -21,13 +23,13 @@ func NewImageSquashResolver(img *image.Image) (*ImageSquashResolver, error) { } // FilesByPath returns all file.References that match the given paths within the squashed representation of the image. -func (r *ImageSquashResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) { +func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) { uniqueFileIDs := file.NewFileReferenceSet() - uniqueFiles := make([]file.Reference, 0) + uniqueLocations := make([]Location, 0) for _, path := range paths { tree := r.img.SquashedTree() - ref := tree.File(path) + ref := tree.File(file.Path(path)) if ref == nil { // no file found, keep looking through layers continue @@ -54,17 +56,17 @@ func (r *ImageSquashResolver) FilesByPath(paths ...file.Path) ([]file.Reference, if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) { uniqueFileIDs.Add(*resolvedRef) - uniqueFiles = append(uniqueFiles, *resolvedRef) + uniqueLocations = append(uniqueLocations, NewLocationFromImage(*resolvedRef, r.img)) } } - return uniqueFiles, nil + return uniqueLocations, nil } // FilesByGlob returns all file.References that match the given path glob pattern within the squashed representation of the image. -func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) { +func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error) { uniqueFileIDs := file.NewFileReferenceSet() - uniqueFiles := make([]file.Reference, 0) + uniqueLocations := make([]Location, 0) for _, pattern := range patterns { refs, err := r.img.SquashedTree().FilesByGlob(pattern) @@ -86,42 +88,45 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference, } } - resolvedRefs, err := r.FilesByPath(ref.Path) + resolvedLocations, err := r.FilesByPath(string(ref.Path)) if err != nil { return nil, fmt.Errorf("failed to find files by path (ref=%+v): %w", ref, err) } - for _, resolvedRef := range resolvedRefs { - if !uniqueFileIDs.Contains(resolvedRef) { - uniqueFileIDs.Add(resolvedRef) - uniqueFiles = append(uniqueFiles, resolvedRef) + for _, resolvedLocation := range resolvedLocations { + if !uniqueFileIDs.Contains(resolvedLocation.ref) { + uniqueFileIDs.Add(resolvedLocation.ref) + uniqueLocations = append(uniqueLocations, resolvedLocation) } } } } - return uniqueFiles, nil + return uniqueLocations, nil } -func (r *ImageSquashResolver) RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) { - paths, err := r.FilesByPath(file.Path(path)) +// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. +// This is helpful when attempting to find a file that is in the same layer or lower as another file. For the +// ImageSquashResolver, this is a simple path lookup. +func (r *ImageSquashResolver) RelativeFileByPath(_ Location, path string) *Location { + paths, err := r.FilesByPath(path) if err != nil { - return nil, err + return nil } if len(paths) == 0 { - return nil, nil + return nil } - return &paths[0], nil + return &paths[0] } -// MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a +// MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a // file.Reference is a path relative to a particular layer, in this case only from the squashed representation. -func (r *ImageSquashResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) { - return r.img.MultipleFileContentsByRef(f...) +func (r *ImageSquashResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) { + return mapLocationRefs(r.img.MultipleFileContentsByRef, locations) } -// FileContentsByRef fetches file contents for a single file reference, irregardless of the source layer. +// FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer. // If the path does not exist an error is returned. -func (r *ImageSquashResolver) FileContentsByRef(ref file.Reference) (string, error) { - return r.img.FileContentsByRef(ref) +func (r *ImageSquashResolver) FileContentsByLocation(location Location) (string, error) { + return r.img.FileContentsByRef(location.ref) } diff --git a/syft/scope/resolvers/image_squash_resolver_test.go b/syft/source/image_squash_resolver_test.go similarity index 92% rename from syft/scope/resolvers/image_squash_resolver_test.go rename to syft/source/image_squash_resolver_test.go index dee432125..f5d9b2d9f 100644 --- a/syft/scope/resolvers/image_squash_resolver_test.go +++ b/syft/source/image_squash_resolver_test.go @@ -1,11 +1,9 @@ -package resolvers +package source import ( "testing" "github.com/anchore/stereoscope/pkg/imagetest" - - "github.com/anchore/stereoscope/pkg/file" ) func TestImageSquashResolver_FilesByPath(t *testing.T) { @@ -61,7 +59,7 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) { t.Fatalf("could not create resolver: %+v", err) } - refs, err := resolver.FilesByPath(file.Path(c.linkPath)) + refs, err := resolver.FilesByPath(c.linkPath) if err != nil { t.Fatalf("could not use resolver: %+v", err) } @@ -82,11 +80,11 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) { actual := refs[0] - if actual.Path != file.Path(c.resolvePath) { + if actual.Path != c.resolvePath { t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath) } - entry, err := img.FileCatalog.Get(actual) + entry, err := img.FileCatalog.Get(actual.ref) if err != nil { t.Fatalf("failed to get metadata: %+v", err) } @@ -172,11 +170,11 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) { actual := refs[0] - if actual.Path != file.Path(c.resolvePath) { + if actual.Path != c.resolvePath { t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath) } - entry, err := img.FileCatalog.Get(actual) + entry, err := img.FileCatalog.Get(actual.ref) if err != nil { t.Fatalf("failed to get metadata: %+v", err) } diff --git a/syft/source/location.go b/syft/source/location.go new file mode 100644 index 000000000..774fd6413 --- /dev/null +++ b/syft/source/location.go @@ -0,0 +1,40 @@ +package source + +import ( + "github.com/anchore/syft/internal/log" + + "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/stereoscope/pkg/image" +) + +// Location represents a path relative to a particular filesystem. +type Location struct { + Path string `json:"path"` // The string path of the location (e.g. /etc/hosts) + FileSystemID string `json:"layerID,omitempty"` // An ID representing the filesystem. For container images this is a layer digest, directories or root filesystem this is blank. + ref file.Reference // The file reference relative to the stereoscope.FileCatalog that has more information about this location. +} + +// NewLocation creates a new Location representing a path without denoting a filesystem or FileCatalog reference. +func NewLocation(path string) Location { + return Location{ + Path: path, + } +} + +// NewLocationFromImage creates a new Location representing the given path (extracted from the ref) relative to the given image. +func NewLocationFromImage(ref file.Reference, img *image.Image) Location { + entry, err := img.FileCatalog.Get(ref) + if err != nil { + log.Warnf("unable to find file catalog entry for ref=%+v", ref) + return Location{ + Path: string(ref.Path), + ref: ref, + } + } + + return Location{ + Path: string(ref.Path), + FileSystemID: entry.Source.Metadata.Digest, + ref: ref, + } +} diff --git a/syft/source/metadata.go b/syft/source/metadata.go new file mode 100644 index 000000000..b9747362e --- /dev/null +++ b/syft/source/metadata.go @@ -0,0 +1,8 @@ +package source + +// Metadata represents any static source data that helps describe "what" was cataloged. +type Metadata struct { + Scheme Scheme // the source data scheme type (directory or image) + ImageMetadata ImageMetadata // all image info (image only) + Path string // the root path to be cataloged (directory only) +} diff --git a/syft/scope/resolver.go b/syft/source/resolver.go similarity index 57% rename from syft/scope/resolver.go rename to syft/source/resolver.go index d32740f3b..0f22c2096 100644 --- a/syft/scope/resolver.go +++ b/syft/source/resolver.go @@ -1,11 +1,9 @@ -package scope +package source import ( "fmt" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/image" - "github.com/anchore/syft/syft/scope/resolvers" ) // Resolver is an interface that encompasses how to get specific file references and file contents for a generic data source. @@ -16,30 +14,30 @@ type Resolver interface { // ContentResolver knows how to get file content for given file.References type ContentResolver interface { - FileContentsByRef(ref file.Reference) (string, error) - MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) + FileContentsByLocation(Location) (string, error) + MultipleFileContentsByLocation([]Location) (map[Location]string, error) // TODO: we should consider refactoring to return a set of io.Readers or file.Openers instead of the full contents themselves (allow for optional buffering). } -// FileResolver knows how to get file.References for given string paths and globs +// FileResolver knows how to get file.References for given string paths and globs type FileResolver interface { // FilesByPath fetches a set of file references which have the given path (for an image, there may be multiple matches) - FilesByPath(paths ...file.Path) ([]file.Reference, error) + FilesByPath(paths ...string) ([]Location, error) // FilesByGlob fetches a set of file references which the given glob matches - FilesByGlob(patterns ...string) ([]file.Reference, error) + FilesByGlob(patterns ...string) ([]Location, error) // RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. // This is helpful when attempting to find a file that is in the same layer or lower as another file. - RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) + RelativeFileByPath(_ Location, path string) *Location } -// getImageResolver returns the appropriate resolve for a container image given the scope option -func getImageResolver(img *image.Image, option Option) (Resolver, error) { - switch option { +// getImageResolver returns the appropriate resolve for a container image given the source option +func getImageResolver(img *image.Image, scope Scope) (Resolver, error) { + switch scope { case SquashedScope: - return resolvers.NewImageSquashResolver(img) + return NewImageSquashResolver(img) case AllLayersScope: - return resolvers.NewAllLayersResolver(img) + return NewAllLayersResolver(img) default: - return nil, fmt.Errorf("bad option provided: %+v", option) + return nil, fmt.Errorf("bad scope provided: %+v", scope) } } diff --git a/syft/source/scheme.go b/syft/source/scheme.go new file mode 100644 index 000000000..d2f1c73eb --- /dev/null +++ b/syft/source/scheme.go @@ -0,0 +1,59 @@ +package source + +import ( + "fmt" + "strings" + + "github.com/anchore/stereoscope/pkg/image" + "github.com/mitchellh/go-homedir" + "github.com/spf13/afero" +) + +// Scheme represents the optional prefixed string at the beginning of a user request (e.g. "docker:"). +type Scheme string + +const ( + // UnknownScheme is the default scheme + UnknownScheme Scheme = "UnknownScheme" + // DirectoryScheme indicates the source being cataloged is a directory on the root filesystem + DirectoryScheme Scheme = "DirectoryScheme" + // ImageScheme indicates the source being cataloged is a container image + ImageScheme Scheme = "ImageScheme" +) + +func detectScheme(fs afero.Fs, imageDetector sourceDetector, userInput string) (Scheme, string, error) { + if strings.HasPrefix(userInput, "dir:") { + // blindly trust the user's scheme + dirLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "dir:")) + if err != nil { + return UnknownScheme, "", fmt.Errorf("unable to expand directory path: %w", err) + } + return DirectoryScheme, dirLocation, nil + } + + // we should attempt to let stereoscope determine what the source is first --just because the source is a valid directory + // doesn't mean we yet know if it is an OCI layout directory (to be treated as an image) or if it is a generic filesystem directory. + source, imageSpec, err := imageDetector(userInput) + if err != nil { + return UnknownScheme, "", fmt.Errorf("unable to detect the scheme from %q: %w", userInput, err) + } + + if source == image.UnknownSource { + dirLocation, err := homedir.Expand(userInput) + if err != nil { + return UnknownScheme, "", fmt.Errorf("unable to expand potential directory path: %w", err) + } + + fileMeta, err := fs.Stat(dirLocation) + if err != nil { + return UnknownScheme, "", nil + } + + if fileMeta.IsDir() { + return DirectoryScheme, dirLocation, nil + } + return UnknownScheme, "", nil + } + + return ImageScheme, imageSpec, nil +} diff --git a/syft/source/scope.go b/syft/source/scope.go new file mode 100644 index 000000000..e959d1a42 --- /dev/null +++ b/syft/source/scope.go @@ -0,0 +1,36 @@ +package source + +import "strings" + +// Scope indicates "how" or from "which perspectives" the source object should be cataloged from. +type Scope string + +const ( + // UnknownScope is the default scope + UnknownScope Scope = "UnknownScope" + // SquashedScope indicates to only catalog content visible from the squashed filesystem representation (what can be seen only within the container at runtime) + SquashedScope Scope = "Squashed" + // AllLayersScope indicates to catalog content on all layers, irregardless if it is visible from the container at runtime. + AllLayersScope Scope = "AllLayers" +) + +// AllScopes is a slice containing all possible scope options +var AllScopes = []Scope{ + SquashedScope, + AllLayersScope, +} + +// ParseScope returns a scope as indicated from the given string. +func ParseScope(userStr string) Scope { + switch strings.ToLower(userStr) { + case strings.ToLower(SquashedScope.String()): + return SquashedScope + case "all-layers", strings.ToLower(AllLayersScope.String()): + return AllLayersScope + } + return UnknownScope +} + +func (o Scope) String() string { + return string(o) +} diff --git a/syft/source/source.go b/syft/source/source.go new file mode 100644 index 000000000..182a87f71 --- /dev/null +++ b/syft/source/source.go @@ -0,0 +1,106 @@ +/* +Package source provides an abstraction to allow a user to loosely define a data source to catalog and expose a common interface that +catalogers and use explore and analyze data from the data source. All valid (cataloggable) data sources are defined +within this package. +*/ +package source + +import ( + "fmt" + + "github.com/spf13/afero" + + "github.com/anchore/stereoscope" + + "github.com/anchore/stereoscope/pkg/image" +) + +// Source is an object that captures the data source to be cataloged, configuration, and a specific resolver used +// in cataloging (based on the data source and configuration) +type Source struct { + Resolver Resolver // a Resolver object to use in file path/glob resolution and file contents resolution + Image *image.Image // the image object to be cataloged (image only) + Metadata Metadata +} + +type sourceDetector func(string) (image.Source, string, error) + +// New produces a Source based on userInput like dir: or image:tag +func New(userInput string, o Scope) (Source, func(), error) { + fs := afero.NewOsFs() + parsedScheme, location, err := detectScheme(fs, image.DetectSource, userInput) + if err != nil { + return Source{}, func() {}, fmt.Errorf("unable to parse input=%q: %w", userInput, err) + } + + switch parsedScheme { + case DirectoryScheme: + fileMeta, err := fs.Stat(location) + if err != nil { + return Source{}, func() {}, fmt.Errorf("unable to stat dir=%q: %w", location, err) + } + + if !fileMeta.IsDir() { + return Source{}, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", location, err) + } + + s, err := NewFromDirectory(location) + if err != nil { + return Source{}, func() {}, fmt.Errorf("could not populate source from path=%q: %w", location, err) + } + return s, func() {}, nil + + case ImageScheme: + img, err := stereoscope.GetImage(location) + cleanup := func() { + stereoscope.Cleanup() + } + + if err != nil || img == nil { + return Source{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", location, err) + } + + s, err := NewFromImage(img, o, location) + if err != nil { + return Source{}, cleanup, fmt.Errorf("could not populate source with image: %w", err) + } + return s, cleanup, nil + } + + return Source{}, func() {}, fmt.Errorf("unable to process input for scanning: '%s'", userInput) +} + +// NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively. +func NewFromDirectory(path string) (Source, error) { + return Source{ + Resolver: &DirectoryResolver{ + Path: path, + }, + Metadata: Metadata{ + Scheme: DirectoryScheme, + Path: path, + }, + }, nil +} + +// NewFromImage creates a new source object tailored to catalog a given container image, relative to the +// option given (e.g. all-layers, squashed, etc) +func NewFromImage(img *image.Image, scope Scope, userImageStr string) (Source, error) { + if img == nil { + return Source{}, fmt.Errorf("no image given") + } + + resolver, err := getImageResolver(img, scope) + if err != nil { + return Source{}, fmt.Errorf("could not determine file resolver: %w", err) + } + + return Source{ + Resolver: resolver, + Image: img, + Metadata: Metadata{ + Scheme: ImageScheme, + ImageMetadata: NewImageMetadata(img, userImageStr, scope), + }, + }, nil +} diff --git a/syft/scope/scope_test.go b/syft/source/source_test.go similarity index 84% rename from syft/scope/scope_test.go rename to syft/source/source_test.go index b6656fc07..49946cf3b 100644 --- a/syft/scope/scope_test.go +++ b/syft/source/source_test.go @@ -1,96 +1,95 @@ -package scope +package source import ( "os" "testing" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/image" "github.com/mitchellh/go-homedir" "github.com/spf13/afero" ) -func TestNewScopeFromImageFails(t *testing.T) { +func TestNewFromImageFails(t *testing.T) { t.Run("no image given", func(t *testing.T) { - _, err := NewScopeFromImage(nil, AllLayersScope) + _, err := NewFromImage(nil, AllLayersScope, "") if err == nil { t.Errorf("expected an error condition but none was given") } }) } -func TestNewScopeFromImageUnknownOption(t *testing.T) { +func TestNewFromImageUnknownOption(t *testing.T) { img := image.Image{} t.Run("unknown option is an error", func(t *testing.T) { - _, err := NewScopeFromImage(&img, UnknownScope) + _, err := NewFromImage(&img, UnknownScope, "") if err == nil { t.Errorf("expected an error condition but none was given") } }) } -func TestNewScopeFromImage(t *testing.T) { +func TestNewFromImage(t *testing.T) { layer := image.NewLayer(nil) img := image.Image{ Layers: []*image.Layer{layer}, } - t.Run("create a new Scope object from image", func(t *testing.T) { - _, err := NewScopeFromImage(&img, AllLayersScope) + t.Run("create a new source object from image", func(t *testing.T) { + _, err := NewFromImage(&img, AllLayersScope, "") if err != nil { - t.Errorf("unexpected error when creating a new Scope from img: %w", err) + t.Errorf("unexpected error when creating a new Locations from img: %+v", err) } }) } -func TestDirectoryScope(t *testing.T) { +func TestNewFromDirectory(t *testing.T) { testCases := []struct { desc string input string expString string - inputPaths []file.Path + inputPaths []string expRefs int }{ { desc: "no paths exist", input: "foobar/", - inputPaths: []file.Path{file.Path("/opt/"), file.Path("/other")}, + inputPaths: []string{"/opt/", "/other"}, expRefs: 0, }, { desc: "path detected", input: "test-fixtures", - inputPaths: []file.Path{file.Path("test-fixtures/path-detected/.vimrc")}, + inputPaths: []string{"test-fixtures/path-detected/.vimrc"}, expRefs: 1, }, { desc: "directory ignored", input: "test-fixtures", - inputPaths: []file.Path{file.Path("test-fixtures/path-detected")}, + inputPaths: []string{"test-fixtures/path-detected"}, expRefs: 0, }, { desc: "no files-by-path detected", input: "test-fixtures", - inputPaths: []file.Path{file.Path("test-fixtures/no-path-detected")}, + inputPaths: []string{"test-fixtures/no-path-detected"}, expRefs: 0, }, } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { - p, err := NewScopeFromDir(test.input) + src, err := NewFromDirectory(test.input) if err != nil { - t.Errorf("could not create NewDirScope: %w", err) + t.Errorf("could not create NewDirScope: %+v", err) } - if p.Source.(DirSource).Path != test.input { - t.Errorf("mismatched stringer: '%s' != '%s'", p.Source.(DirSource).Path, test.input) + if src.Metadata.Path != test.input { + t.Errorf("mismatched stringer: '%s' != '%s'", src.Metadata.Path, test.input) } - refs, err := p.Resolver.FilesByPath(test.inputPaths...) + refs, err := src.Resolver.FilesByPath(test.inputPaths...) if err != nil { - t.Errorf("FilesByPath call produced an error: %w", err) + t.Errorf("FilesByPath call produced an error: %+v", err) } if len(refs) != test.expRefs { t.Errorf("unexpected number of refs returned: %d != %d", len(refs), test.expRefs) @@ -101,7 +100,7 @@ func TestDirectoryScope(t *testing.T) { } } -func TestMultipleFileContentsByRefContents(t *testing.T) { +func TestMultipleFileContentsByLocation(t *testing.T) { testCases := []struct { desc string input string @@ -123,22 +122,22 @@ func TestMultipleFileContentsByRefContents(t *testing.T) { } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { - p, err := NewScopeFromDir(test.input) + p, err := NewFromDirectory(test.input) if err != nil { - t.Errorf("could not create NewDirScope: %w", err) + t.Errorf("could not create NewDirScope: %+v", err) } - refs, err := p.Resolver.FilesByPath(file.Path(test.path)) + locations, err := p.Resolver.FilesByPath(test.path) if err != nil { t.Errorf("could not get file references from path: %s, %v", test.path, err) } - if len(refs) != 1 { - t.Fatalf("expected a single ref to be generated but got: %d", len(refs)) + if len(locations) != 1 { + t.Fatalf("expected a single location to be generated but got: %d", len(locations)) } - ref := refs[0] + location := locations[0] - contents, err := p.Resolver.MultipleFileContentsByRef(ref) - content := contents[ref] + contents, err := p.Resolver.MultipleFileContentsByLocation([]Location{location}) + content := contents[location] if content != test.expected { t.Errorf("unexpected contents from file: '%s' != '%s'", content, test.expected) @@ -148,7 +147,7 @@ func TestMultipleFileContentsByRefContents(t *testing.T) { } } -func TestMultipleFileContentsByRefNoContents(t *testing.T) { +func TestFilesByPathDoesNotExist(t *testing.T) { testCases := []struct { desc string input string @@ -163,11 +162,11 @@ func TestMultipleFileContentsByRefNoContents(t *testing.T) { } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { - p, err := NewScopeFromDir(test.input) + p, err := NewFromDirectory(test.input) if err != nil { - t.Errorf("could not create NewDirScope: %w", err) + t.Errorf("could not create NewDirScope: %+v", err) } - refs, err := p.Resolver.FilesByPath(file.Path(test.path)) + refs, err := p.Resolver.FilesByPath(test.path) if err != nil { t.Errorf("could not get file references from path: %s, %v", test.path, err) } @@ -208,9 +207,9 @@ func TestFilesByGlob(t *testing.T) { } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { - p, err := NewScopeFromDir(test.input) + p, err := NewFromDirectory(test.input) if err != nil { - t.Errorf("could not create NewDirScope: %w", err) + t.Errorf("could not create NewDirScope: %+v", err) } contents, err := p.Resolver.FilesByGlob(test.glob) diff --git a/syft/scope/resolvers/test-fixtures/image-symlinks/Dockerfile b/syft/source/test-fixtures/image-symlinks/Dockerfile similarity index 100% rename from syft/scope/resolvers/test-fixtures/image-symlinks/Dockerfile rename to syft/source/test-fixtures/image-symlinks/Dockerfile diff --git a/syft/scope/resolvers/test-fixtures/image-symlinks/file-1.txt b/syft/source/test-fixtures/image-symlinks/file-1.txt similarity index 100% rename from syft/scope/resolvers/test-fixtures/image-symlinks/file-1.txt rename to syft/source/test-fixtures/image-symlinks/file-1.txt diff --git a/syft/scope/resolvers/test-fixtures/image-symlinks/file-2.txt b/syft/source/test-fixtures/image-symlinks/file-2.txt similarity index 100% rename from syft/scope/resolvers/test-fixtures/image-symlinks/file-2.txt rename to syft/source/test-fixtures/image-symlinks/file-2.txt diff --git a/syft/scope/resolvers/test-fixtures/image-symlinks/nested/nested/file-3.txt b/syft/source/test-fixtures/image-symlinks/nested/nested/file-3.txt similarity index 100% rename from syft/scope/resolvers/test-fixtures/image-symlinks/nested/nested/file-3.txt rename to syft/source/test-fixtures/image-symlinks/nested/nested/file-3.txt diff --git a/syft/scope/resolvers/test-fixtures/image-symlinks/new-file-2.txt b/syft/source/test-fixtures/image-symlinks/new-file-2.txt similarity index 100% rename from syft/scope/resolvers/test-fixtures/image-symlinks/new-file-2.txt rename to syft/source/test-fixtures/image-symlinks/new-file-2.txt diff --git a/syft/scope/test-fixtures/path-detected/.vimrc b/syft/source/test-fixtures/path-detected/.vimrc similarity index 100% rename from syft/scope/test-fixtures/path-detected/.vimrc rename to syft/source/test-fixtures/path-detected/.vimrc diff --git a/syft/scope/test-fixtures/path-detected/empty b/syft/source/test-fixtures/path-detected/empty similarity index 100% rename from syft/scope/test-fixtures/path-detected/empty rename to syft/source/test-fixtures/path-detected/empty diff --git a/test/integration/distro_test.go b/test/integration/distro_test.go index fdbb5a824..e6a9ce297 100644 --- a/test/integration/distro_test.go +++ b/test/integration/distro_test.go @@ -6,7 +6,7 @@ import ( "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft" "github.com/anchore/syft/syft/distro" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/go-test/deep" ) @@ -16,24 +16,18 @@ func TestDistroImage(t *testing.T) { tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) defer cleanup() - _, _, actualDistro, err := syft.Catalog("docker-archive:"+tarPath, scope.AllLayersScope) + _, _, actualDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } - if actualDistro == nil { - t.Fatalf("could not find distro") - } expected, err := distro.NewDistro(distro.Busybox, "1.31.1", "") if err != nil { t.Fatalf("could not create distro: %+v", err) } - diffs := deep.Equal(*actualDistro, expected) - if len(diffs) != 0 { - for _, d := range diffs { - t.Errorf("found distro difference: %+v", d) - } + for _, d := range deep.Equal(actualDistro, expected) { + t.Errorf("found distro difference: %+v", d) } } diff --git a/test/integration/document_import_test.go b/test/integration/document_import_test.go new file mode 100644 index 000000000..86c07a68f --- /dev/null +++ b/test/integration/document_import_test.go @@ -0,0 +1,92 @@ +package integration + +import ( + "bytes" + "testing" + + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/anchore/syft/syft" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/presenter/json" + "github.com/anchore/syft/syft/source" + "github.com/go-test/deep" +) + +func TestCatalogFromJSON(t *testing.T) { + + // ensure each of our fixture images results in roughly the same shape when: + // generate json -> import json -> assert packages and distro are the same (except for select fields) + + tests := []struct { + fixture string + }{ + { + fixture: "image-pkg-coverage", + }, + } + + for _, test := range tests { + t.Run(test.fixture, func(t *testing.T) { + _, cleanup := imagetest.GetFixtureImage(t, "docker-archive", test.fixture) + tarPath := imagetest.GetFixtureImageTarPath(t, test.fixture) + defer cleanup() + + expectedSource, expectedCatalog, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) + if err != nil { + t.Fatalf("failed to catalog image: %+v", err) + } + + var buf bytes.Buffer + jsonPres := json.NewPresenter(expectedCatalog, expectedSource.Metadata, expectedDistro) + if err = jsonPres.Present(&buf); err != nil { + t.Fatalf("failed to write to presenter: %+v", err) + } + + sourceMetadata, actualCatalog, actualDistro, err := syft.CatalogFromJSON(&buf) + if err != nil { + t.Fatalf("failed to import document: %+v", err) + } + + for _, d := range deep.Equal(sourceMetadata, expectedSource.Metadata) { + t.Errorf(" image metadata diff: %+v", d) + } + + for _, d := range deep.Equal(actualDistro, expectedDistro) { + t.Errorf(" distro diff: %+v", d) + } + + var actualPackages, expectedPackages []*pkg.Package + + // TODO: take out pkg.RpmdbMetadataType filter + + for _, p := range expectedCatalog.Sorted() { + expectedPackages = append(expectedPackages, p) + } + + for _, p := range actualCatalog.Sorted() { + actualPackages = append(actualPackages, p) + } + + if len(actualPackages) != len(expectedPackages) { + t.Fatalf("mismatched package length: %d != %d", len(actualPackages), len(expectedPackages)) + } + + for i, e := range expectedPackages { + a := actualPackages[i] + + // omit fields that should be missing + if e.MetadataType == pkg.JavaMetadataType { + metadata := e.Metadata.(pkg.JavaMetadata) + metadata.Parent = nil + e.Metadata = metadata + } + + for _, d := range deep.Equal(a, e) { + t.Errorf(" package %d (name=%s) diff: %+v", i, e.Name, d) + } + } + + }) + } + +} diff --git a/test/integration/json_schema_test.go b/test/integration/json_schema_test.go index 279b74337..07b1a32cb 100644 --- a/test/integration/json_schema_test.go +++ b/test/integration/json_schema_test.go @@ -15,7 +15,7 @@ import ( "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/presenter" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/xeipuuv/gojsonschema" ) @@ -53,7 +53,7 @@ func validateAgainstV1Schema(t *testing.T, json string) { } } -func testJsonSchema(t *testing.T, catalog *pkg.Catalog, theScope *scope.Scope, prefix string) { +func testJsonSchema(t *testing.T, catalog *pkg.Catalog, theScope source.Source, prefix string) { // make the json output example dir if it does not exist absJsonSchemaExamplesPath := path.Join(repoRoot(t), jsonSchemaExamplesPath) if _, err := os.Stat(absJsonSchemaExamplesPath); os.IsNotExist(err) { @@ -67,7 +67,7 @@ func testJsonSchema(t *testing.T, catalog *pkg.Catalog, theScope *scope.Scope, p t.Fatalf("bad distro: %+v", err) } - p := presenter.GetPresenter(presenter.JSONPresenter, *theScope, catalog, &d) + p := presenter.GetPresenter(presenter.JSONPresenter, theScope.Metadata, catalog, d) if p == nil { t.Fatal("unable to get presenter") } @@ -101,7 +101,7 @@ func TestJsonSchemaImg(t *testing.T) { tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) defer cleanup() - catalog, theScope, _, err := syft.Catalog("docker-archive:"+tarPath, scope.AllLayersScope) + src, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } @@ -112,15 +112,15 @@ func TestJsonSchemaImg(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { - testJsonSchema(t, catalog, theScope, "img") + testJsonSchema(t, catalog, src, "img") }) } } func TestJsonSchemaDirs(t *testing.T) { - catalog, theScope, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", scope.AllLayersScope) + src, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope) if err != nil { - t.Errorf("unable to create scope from dir: %+v", err) + t.Errorf("unable to create source from dir: %+v", err) } var cases []testCase @@ -129,7 +129,7 @@ func TestJsonSchemaDirs(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { - testJsonSchema(t, catalog, theScope, "dir") + testJsonSchema(t, catalog, src, "dir") }) } } diff --git a/test/integration/pkg_coverage_test.go b/test/integration/pkg_coverage_test.go index 13a5af513..cc25f391a 100644 --- a/test/integration/pkg_coverage_test.go +++ b/test/integration/pkg_coverage_test.go @@ -9,7 +9,7 @@ import ( "github.com/anchore/syft/internal" "github.com/anchore/syft/syft" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) func TestPkgCoverageImage(t *testing.T) { @@ -18,7 +18,7 @@ func TestPkgCoverageImage(t *testing.T) { tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) defer cleanup() - catalog, _, _, err := syft.Catalog("docker-archive:"+tarPath, scope.AllLayersScope) + _, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } @@ -100,10 +100,10 @@ func TestPkgCoverageImage(t *testing.T) { } func TestPkgCoverageDirectory(t *testing.T) { - catalog, _, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", scope.AllLayersScope) + _, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope) if err != nil { - t.Errorf("unable to create scope from dir: %+v", err) + t.Errorf("unable to create source from dir: %+v", err) } observedLanguages := internal.NewStringSet() diff --git a/test/integration/regression_test.go b/test/integration/regression_test.go index 572ddae8d..d018e85b0 100644 --- a/test/integration/regression_test.go +++ b/test/integration/regression_test.go @@ -9,7 +9,7 @@ import ( "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) func TestRegression212ApkBufferSize(t *testing.T) { @@ -21,7 +21,7 @@ func TestRegression212ApkBufferSize(t *testing.T) { tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) defer cleanup() - catalog, _, _, err := syft.Catalog("docker-archive:"+tarPath, scope.SquashedScope) + catalog, _, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) }