diff --git a/.golangci.yaml b/.golangci.yaml index 1445395ff..238fb91fa 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,3 +1,9 @@ +# TODO: enable this when we have coverage on docstring comments +#issues: +# # The list of ids of default excludes to include or disable. +# include: +# - EXC0002 # disable excluding of issues about comments from golint + linters: # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint disable-all: true diff --git a/go.mod b/go.mod index cd209428c..667d6eba2 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,6 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.3.1 github.com/olekukonko/tablewriter v0.0.4 - github.com/opencontainers/runc v0.1.1 // indirect github.com/pelletier/go-toml v1.8.0 github.com/rogpeppe/go-internal v1.5.2 github.com/sergi/go-diff v1.1.0 @@ -32,7 +31,9 @@ require ( github.com/wagoodman/jotframe v0.0.0-20200730190914-3517092dd163 github.com/x-cray/logrus-prefixed-formatter v0.5.2 github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/net v0.0.0-20200625001655-4c5254603344 // indirect + golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 // indirect golang.org/x/sys v0.0.0-20200610111108-226ff32320da // indirect google.golang.org/genproto v0.0.0-20200615140333-fd031eab31e7 // indirect gopkg.in/ini.v1 v1.57.0 // indirect diff --git a/go.sum b/go.sum index e71aa3d3b..5142a4956 100644 --- a/go.sum +++ b/go.sum @@ -131,8 +131,6 @@ github.com/anchore/go-testutils v0.0.0-20200624184116-66aa578126db/go.mod h1:D3r github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E= github.com/anchore/stereoscope v0.0.0-20200520221116-025e07f1c93e/go.mod h1:bkyLl5VITnrmgErv4S1vDfVz/TGAZ5il6161IQo7w2g= -github.com/anchore/stereoscope v0.0.0-20200803190343-146f38f8cc19 h1:iJ6du/cA9KJ0ctP905u2zCcpJubsy6kxLZBvG4XG+uY= -github.com/anchore/stereoscope v0.0.0-20200803190343-146f38f8cc19/go.mod h1:WntReQTI/I27FOQ87UgLVVzWgku6+ZsqfOTLxpIZFCs= github.com/anchore/stereoscope v0.0.0-20200813152757-548b22c8a0b3 h1:pl+txuYlhK8Mmio4d+4zQI/1xg8X6BtNErTASrx23Wk= github.com/anchore/stereoscope v0.0.0-20200813152757-548b22c8a0b3/go.mod h1:WntReQTI/I27FOQ87UgLVVzWgku6+ZsqfOTLxpIZFCs= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= @@ -886,8 +884,8 @@ golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM= -golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -962,6 +960,8 @@ golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -978,6 +978,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/main.go b/main.go index b512bb116..df0522d5b 100644 --- a/main.go +++ b/main.go @@ -1,3 +1,8 @@ +/* +Syft is a CLI tool and go library for generating a Software Bill of Materials (SBOM) from container images and filesystems. + +Note that Syft is both a command line tool as well as a library. See the syft/ child package for library functionality. +*/ package main import ( diff --git a/syft/cataloger/apkdb/cataloger.go b/syft/cataloger/apkdb/cataloger.go index dbe809da0..8f33709f6 100644 --- a/syft/cataloger/apkdb/cataloger.go +++ b/syft/cataloger/apkdb/cataloger.go @@ -1,3 +1,6 @@ +/* +Package apkdb provides a concrete Cataloger implementation for Alpine DB files. +*/ package apkdb import ( diff --git a/syft/cataloger/bundler/cataloger.go b/syft/cataloger/bundler/cataloger.go index 3f3a37b33..265702659 100644 --- a/syft/cataloger/bundler/cataloger.go +++ b/syft/cataloger/bundler/cataloger.go @@ -1,3 +1,6 @@ +/* +Package bundler provides a concrete Cataloger implementation for Ruby Gemfile.lock bundler files. +*/ package bundler import ( diff --git a/syft/cataloger/catalog.go b/syft/cataloger/catalog.go index ae4e4239d..e1399a381 100644 --- a/syft/cataloger/catalog.go +++ b/syft/cataloger/catalog.go @@ -37,7 +37,7 @@ func newMonitor() (*progress.Manual, *progress.Manual) { // In order to efficiently retrieve contents from a underlying container image the content fetch requests are // done in bulk. Specifically, all files of interest are collected from each catalogers and accumulated into a single // request. -func Catalog(s scope.Resolver, catalogers ...Cataloger) (*pkg.Catalog, error) { +func Catalog(resolver scope.Resolver, catalogers ...Cataloger) (*pkg.Catalog, error) { catalog := pkg.NewCatalog() fileSelection := make([]file.Reference, 0) @@ -45,14 +45,14 @@ func Catalog(s scope.Resolver, catalogers ...Cataloger) (*pkg.Catalog, error) { // ask catalogers for files to extract from the image tar for _, a := range catalogers { - fileSelection = append(fileSelection, a.SelectFiles(s)...) + fileSelection = append(fileSelection, a.SelectFiles(resolver)...) log.Debugf("cataloger '%s' selected '%d' files", a.Name(), len(fileSelection)) filesProcessed.N += int64(len(fileSelection)) } // fetch contents for requested selection by catalogers // TODO: we should consider refactoring to return a set of io.Readers instead of the full contents themselves (allow for optional buffering). - contents, err := s.MultipleFileContentsByRef(fileSelection...) + contents, err := resolver.MultipleFileContentsByRef(fileSelection...) if err != nil { return nil, err } diff --git a/syft/cataloger/cataloger.go b/syft/cataloger/cataloger.go index 14621c491..098445abb 100644 --- a/syft/cataloger/cataloger.go +++ b/syft/cataloger/cataloger.go @@ -1,6 +1,6 @@ /* Package cataloger provides the ability to process files from a container image or file system and discover packages -(e.g. gems, wheels, jars, rpms, debs, etc.). Specifically, this package contains both a catalog function to utilize all +(gems, wheels, jars, rpms, debs, etc). Specifically, this package contains both a catalog function to utilize all catalogers defined in child packages as well as the interface definition to implement a cataloger. */ package cataloger diff --git a/syft/cataloger/common/generic_cataloger.go b/syft/cataloger/common/generic_cataloger.go index 1a048540e..0aacedf3a 100644 --- a/syft/cataloger/common/generic_cataloger.go +++ b/syft/cataloger/common/generic_cataloger.go @@ -1,3 +1,6 @@ +/* +Package common provides generic utilities used by multiple catalogers. +*/ package common import ( diff --git a/syft/cataloger/dpkg/cataloger.go b/syft/cataloger/dpkg/cataloger.go index 3a053d566..e45ab5aa6 100644 --- a/syft/cataloger/dpkg/cataloger.go +++ b/syft/cataloger/dpkg/cataloger.go @@ -1,3 +1,6 @@ +/* +Package dpkg provides a concrete Cataloger implementation for Debian package DB status files. +*/ package dpkg import ( diff --git a/syft/cataloger/golang/cataloger.go b/syft/cataloger/golang/cataloger.go index 2b44da5ee..8616f4468 100644 --- a/syft/cataloger/golang/cataloger.go +++ b/syft/cataloger/golang/cataloger.go @@ -1,3 +1,6 @@ +/* +Package golang provides a concrete Cataloger implementation for go.mod files. +*/ package golang import ( diff --git a/syft/cataloger/java/cataloger.go b/syft/cataloger/java/cataloger.go index 97da1e702..eaa5b19f9 100644 --- a/syft/cataloger/java/cataloger.go +++ b/syft/cataloger/java/cataloger.go @@ -1,3 +1,6 @@ +/* +Package java provides a concrete Cataloger implementation for Java archives (jar, war, ear, jpi, hpi formats). +*/ package java import ( diff --git a/syft/cataloger/javascript/cataloger.go b/syft/cataloger/javascript/cataloger.go index 723a42a94..e09bdf485 100644 --- a/syft/cataloger/javascript/cataloger.go +++ b/syft/cataloger/javascript/cataloger.go @@ -1,3 +1,6 @@ +/* +Package javascript provides a concrete Cataloger implementation for JavaScript ecosystem files (yarn and npm). +*/ package javascript import ( diff --git a/syft/cataloger/python/cataloger.go b/syft/cataloger/python/cataloger.go index e92716413..133a90fe0 100644 --- a/syft/cataloger/python/cataloger.go +++ b/syft/cataloger/python/cataloger.go @@ -1,3 +1,6 @@ +/* +Package python provides a concrete Cataloger implementation for Python ecosystem files (egg, wheel, requirements.txt). +*/ package python import ( diff --git a/syft/cataloger/rpmdb/cataloger.go b/syft/cataloger/rpmdb/cataloger.go index 44b5c9758..db9a5e867 100644 --- a/syft/cataloger/rpmdb/cataloger.go +++ b/syft/cataloger/rpmdb/cataloger.go @@ -1,3 +1,6 @@ +/* +Package rpmdb provides a concrete Cataloger implementation for RPM "Package" DB files. +*/ package rpmdb import ( diff --git a/syft/distro/identify.go b/syft/distro/identify.go index 2199577af..b1088a2c1 100644 --- a/syft/distro/identify.go +++ b/syft/distro/identify.go @@ -17,8 +17,8 @@ type parseEntry struct { fn parseFunc } -// Identify parses distro-specific files to determine distro metadata like version and release -func Identify(s scope.Scope) Distro { +// Identify parses distro-specific files to determine distro metadata like version and release. +func Identify(resolver scope.Resolver) Distro { distro := NewUnknownDistro() identityFiles := []parseEntry{ @@ -41,7 +41,7 @@ func Identify(s scope.Scope) Distro { identifyLoop: for _, entry := range identityFiles { - refs, err := s.FilesByPath(entry.path) + refs, err := resolver.FilesByPath(entry.path) if err != nil { log.Errorf("unable to get path refs from %s: %s", entry.path, err) break @@ -52,7 +52,7 @@ identifyLoop: } for _, ref := range refs { - contents, err := s.MultipleFileContentsByRef(ref) + contents, err := resolver.MultipleFileContentsByRef(ref) content, ok := contents[ref] if !ok { diff --git a/syft/distro/identify_test.go b/syft/distro/identify_test.go index 2dcbd9ce9..b5f272804 100644 --- a/syft/distro/identify_test.go +++ b/syft/distro/identify_test.go @@ -75,12 +75,12 @@ func TestIdentifyDistro(t *testing.T) { for _, test := range tests { t.Run(test.fixture, func(t *testing.T) { - s, err := scope.NewScopeFromDir(test.fixture, scope.AllLayersScope) + s, err := scope.NewScopeFromDir(test.fixture) if err != nil { t.Fatalf("unable to produce a new scope for testing: %s", test.fixture) } - d := Identify(s) + d := Identify(s.Resolver) observedDistros.Add(d.String()) if d.Type != test.Type { diff --git a/syft/event/event.go b/syft/event/event.go index ab4e60ffc..e76ebd983 100644 --- a/syft/event/event.go +++ b/syft/event/event.go @@ -1,3 +1,7 @@ +/* +Package event provides event types for all events that the syft library published onto the event bus. By convention, for each event +defined here there should be a corresponding event parser defined in the parsers/ child package. +*/ package event import "github.com/wagoodman/go-partybus" diff --git a/syft/event/parsers/parsers.go b/syft/event/parsers/parsers.go index f39ab9b6e..623d040cc 100644 --- a/syft/event/parsers/parsers.go +++ b/syft/event/parsers/parsers.go @@ -1,3 +1,6 @@ +/* +Package parsers provides parser helpers to extract payloads for each event type that the syft library publishes onto the event bus. +*/ package parsers import ( diff --git a/syft/lib.go b/syft/lib.go index 3d2810815..b74bf3efe 100644 --- a/syft/lib.go +++ b/syft/lib.go @@ -1,3 +1,6 @@ +/* +A "one-stop-shop" for helper utilities for all major functionality provided by child packages of the syft library. +*/ package syft import ( @@ -11,6 +14,8 @@ import ( "github.com/wagoodman/go-partybus" ) +// Catalog the given image from a particular perspective (e.g. squashed scope, all-layers scope). Returns the discovered +// set of packages, the identified Linux distribution, and the scope object used to wrap the data source. func Catalog(userInput string, scoptOpt scope.Option) (*pkg.Catalog, *scope.Scope, *distro.Distro, error) { log.Info("cataloging image") s, cleanup, err := scope.NewScope(userInput, scoptOpt) @@ -29,8 +34,10 @@ func Catalog(userInput string, scoptOpt scope.Option) (*pkg.Catalog, *scope.Scop return catalog, &s, &d, nil } +// IdentifyDistro attempts to discover what the underlying Linux distribution may be from the available flat files +// provided by the given scope object. If results are inconclusive a "UnknownDistro" Type is returned. func IdentifyDistro(s scope.Scope) distro.Distro { - d := distro.Identify(s) + d := distro.Identify(s.Resolver) if d.Type != distro.UnknownDistroType { log.Infof("identified distro: %s", d.String()) } else { @@ -39,15 +46,18 @@ func IdentifyDistro(s scope.Scope) distro.Distro { return d } +// Catalog the given scope, which may represent a container image or filesystem. Returns the discovered set of packages. func CatalogFromScope(s scope.Scope) (*pkg.Catalog, error) { log.Info("building the catalog") - return cataloger.Catalog(s, cataloger.All()...) + return cataloger.Catalog(s.Resolver, cataloger.All()...) } +// SetLogger sets the logger object used for all syft logging calls. func SetLogger(logger logger.Logger) { log.Log = logger } +// SetBus sets the event bus for all syft library bus publish events onto (in-library subscriptions are not allowed). func SetBus(b *partybus.Bus) { bus.SetPublisher(b) } diff --git a/syft/logger/logger.go b/syft/logger/logger.go index c76b9776a..bb8f9db06 100644 --- a/syft/logger/logger.go +++ b/syft/logger/logger.go @@ -1,3 +1,6 @@ +/* +Defines the logging interface which is used throughout the syft library. +*/ package logger type Logger interface { diff --git a/syft/pkg/catalog.go b/syft/pkg/catalog.go index 5d143c220..97bf53a76 100644 --- a/syft/pkg/catalog.go +++ b/syft/pkg/catalog.go @@ -12,6 +12,7 @@ import ( var nextPackageID int64 +// Catalog represents a collection of Packages. type Catalog struct { byID map[ID]*Package byType map[Type][]*Package @@ -19,6 +20,7 @@ type Catalog struct { lock sync.RWMutex } +// NewCatalog returns a new empty Catalog func NewCatalog() *Catalog { return &Catalog{ byID: make(map[ID]*Package), @@ -27,18 +29,22 @@ func NewCatalog() *Catalog { } } +// PackageCount returns the total number of packages that have been added. func (c *Catalog) PackageCount() int { return len(c.byID) } +// Package returns the package with the given ID. func (c *Catalog) Package(id ID) *Package { return c.byID[id] } +// PackagesByFile returns all packages that were discovered from the given source file reference. func (c *Catalog) PackagesByFile(ref file.Reference) []*Package { return c.byFile[ref] } +// Add a package to the Catalog. func (c *Catalog) Add(p Package) { if p.id != 0 { log.Errorf("package already added to catalog: %s", p) @@ -70,6 +76,7 @@ func (c *Catalog) Add(p Package) { } } +// Enumerate all packages for the given type(s), enumerating all packages if no type is specified. func (c *Catalog) Enumerate(types ...Type) <-chan *Package { channel := make(chan *Package) go func() { @@ -96,6 +103,8 @@ func (c *Catalog) Enumerate(types ...Type) <-chan *Package { return channel } +// Sorted enumerates all packages for the given types sorted by package name. Enumerates all packages if no type +// is specified. func (c *Catalog) Sorted(types ...Type) []*Package { pkgs := make([]*Package, 0) for p := range c.Enumerate(types...) { diff --git a/syft/pkg/metadata.go b/syft/pkg/metadata.go index 697ef3b5f..283a66c95 100644 --- a/syft/pkg/metadata.go +++ b/syft/pkg/metadata.go @@ -9,7 +9,7 @@ type DpkgMetadata struct { // TODO: consider keeping the remaining values as an embedded map } -// RpmMetadata represents all captured data for a RHEL package DB entry. +// RpmMetadata represents all captured data for a RPM DB package entry. type RpmMetadata struct { Version string `mapstructure:"Version" json:"version"` Epoch int `mapstructure:"Epoch" json:"epoch"` @@ -21,6 +21,7 @@ type RpmMetadata struct { Vendor string `mapstructure:"Vendor" json:"vendor"` } +// JavaManifest represents the fields of interest extracted from a Java archive's META-INF/MANIFEST.MF file. type JavaManifest struct { Name string `mapstructure:"Name" json:"name"` ManifestVersion string `mapstructure:"Manifest-Version" json:"manifest-version"` @@ -33,6 +34,7 @@ type JavaManifest struct { Extra map[string]string `mapstructure:",remain" json:"extra-fields"` } +// PomProperties represents the fields of interest extracted from a Java archive's pom.xml file. type PomProperties struct { Path string Name string `mapstructure:"name" json:"name"` @@ -42,13 +44,14 @@ type PomProperties struct { Extra map[string]string `mapstructure:",remain" json:"extra-fields"` } +// JavaMetadata encapsulates all Java ecosystem metadata for a package as well as an (optional) parent relationship. type JavaMetadata struct { Manifest *JavaManifest `mapstructure:"Manifest" json:"manifest"` PomProperties *PomProperties `mapstructure:"PomProperties" json:"pom-properties"` Parent *Package `json:"parent-package"` } -// source: https://wiki.alpinelinux.org/wiki/Apk_spec +// ApkMetadata represents all captured data for a Alpine DB package entry. See https://wiki.alpinelinux.org/wiki/Apk_spec for more information. type ApkMetadata struct { Package string `mapstructure:"P" json:"package"` OriginPackage string `mapstructure:"o" json:"origin-package"` @@ -66,6 +69,7 @@ type ApkMetadata struct { Files []ApkFileRecord `json:"files"` } +// ApkFileRecord represents a single file listing and metadata from a APK DB entry (which may have many of these file records). type ApkFileRecord struct { Path string `json:"path"` OwnerUID string `json:"owner-uid"` diff --git a/syft/pkg/package.go b/syft/pkg/package.go index 85b7bc432..ea16fd6ec 100644 --- a/syft/pkg/package.go +++ b/syft/pkg/package.go @@ -1,3 +1,6 @@ +/* +Package pkg provides the data structures for a package, a package catalog, package types, and domain-specific metadata. +*/ package pkg import ( @@ -8,25 +11,26 @@ import ( type ID int64 -// TODO: add field to trace which cataloger detected this - -// Package represents an application or library that has been bundled into a distributable format +// Package represents an application or library that has been bundled into a distributable format. type Package struct { - id ID // this is set when a package is added to the catalog - Name string `json:"manifest"` - Version string `json:"version"` - FoundBy string `json:"found-by"` // FoundBy is the cataloger that discovered this package - Source []file.Reference `json:"sources"` - Licenses []string `json:"licenses"` // TODO: should we move this into metadata? - Language Language `json:"language"` // TODO: should this support multiple languages as a slice? - Type Type `json:"type"` - Metadata interface{} `json:"metadata,omitempty"` + id ID // uniquely identifies a package, set by the cataloger + Name string `json:"manifest"` // the package name + Version string `json:"version"` // the version of the package + FoundBy string `json:"found-by"` // the specific cataloger that discovered this package + Source []file.Reference `json:"sources"` // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package) + // TODO: should we move licenses into metadata? + Licenses []string `json:"licenses"` // licenses discovered with the package metadata + Language Language `json:"language"` // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc) + Type Type `json:"type"` // the package type (e.g. Npm, Yarn, Egg, Wheel, Rpm, Deb, etc) + Metadata interface{} `json:"metadata,omitempty"` // additional data found while parsing the package source } +// ID returns the package ID, which is unique relative to a package catalog. func (p Package) ID() ID { return p.id } +// Stringer to represent a package. func (p Package) String() string { return fmt.Sprintf("Pkg(type=%s, name=%s, version=%s)", p.Type, p.Name, p.Version) } diff --git a/syft/pkg/type.go b/syft/pkg/type.go index 27ef5d271..e57be7836 100644 --- a/syft/pkg/type.go +++ b/syft/pkg/type.go @@ -1,5 +1,6 @@ package pkg +// Type represents a Package Type for or within a language ecosystem (there may be multiple package types within a language ecosystem) type Type string const ( diff --git a/syft/presenter/json/presenter_test.go b/syft/presenter/json/presenter_test.go index 537df1d29..ff149e1e7 100644 --- a/syft/presenter/json/presenter_test.go +++ b/syft/presenter/json/presenter_test.go @@ -39,7 +39,7 @@ func TestJsonDirsPresenter(t *testing.T) { }, }) - s, err := scope.NewScopeFromDir("/some/path", scope.AllLayersScope) + s, err := scope.NewScopeFromDir("/some/path") if err != nil { t.Fatal(err) } diff --git a/syft/presenter/presenter.go b/syft/presenter/presenter.go index e6a52d3ef..bf37a21ae 100644 --- a/syft/presenter/presenter.go +++ b/syft/presenter/presenter.go @@ -1,3 +1,7 @@ +/* +Defines a Presenter interface for displaying catalog results to an io.Writer as well as a helper utility to obtain +a specific Presenter implementation given user configuration. +*/ package presenter import ( @@ -10,6 +14,8 @@ import ( "github.com/anchore/syft/syft/scope" ) +// Presenter defines the expected behavior for an object responsible for displaying arbitrary input and processed data +// to a given io.Writer. type Presenter interface { Present(io.Writer) error } diff --git a/syft/presenter/text/presenter_test.go b/syft/presenter/text/presenter_test.go index 9df07730e..0c16b0d38 100644 --- a/syft/presenter/text/presenter_test.go +++ b/syft/presenter/text/presenter_test.go @@ -31,7 +31,7 @@ func TestTextDirPresenter(t *testing.T) { Type: pkg.DebPkg, }) - s, err := scope.NewScopeFromDir("/some/path", scope.AllLayersScope) + s, err := scope.NewScopeFromDir("/some/path") if err != nil { t.Fatalf("unable to create scope: %+v", err) } diff --git a/syft/scope/resolver.go b/syft/scope/resolver.go index ab0a9752b..4a965d30e 100644 --- a/syft/scope/resolver.go +++ b/syft/scope/resolver.go @@ -8,22 +8,24 @@ import ( "github.com/anchore/syft/syft/scope/resolvers" ) +// Resolver is an interface that encompasses how to get specific file references and file contents for a generic data source. type Resolver interface { ContentResolver FileResolver } -// ContentResolver knows how to get content from file.References +// ContentResolver knows how to get file content for given file.References type ContentResolver interface { MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) } -// FileResolver knows how to get file.References from string paths and globs +// FileResolver knows how to get file.References for given string paths and globs type FileResolver interface { FilesByPath(paths ...file.Path) ([]file.Reference, error) FilesByGlob(patterns ...string) ([]file.Reference, error) } +// getImageResolver returns the appropriate resolve for a container image given the scope option func getImageResolver(img *image.Image, option Option) (Resolver, error) { switch option { case SquashedScope: diff --git a/syft/scope/resolvers/all_layers_resolver.go b/syft/scope/resolvers/all_layers_resolver.go index fdff7a966..e14eae8fd 100644 --- a/syft/scope/resolvers/all_layers_resolver.go +++ b/syft/scope/resolvers/all_layers_resolver.go @@ -8,11 +8,13 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) +// AllLayersResolver implements path and content access for the AllLayers scope option for container image data sources. type AllLayersResolver struct { img *image.Image layers []int } +// NewAllLayersResolver returns a new resolver from the perspective of all image layers for the given image. func NewAllLayersResolver(img *image.Image) (*AllLayersResolver, error) { if len(img.Layers) == 0 { return nil, fmt.Errorf("the image does not contain any layers") @@ -58,6 +60,7 @@ func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.Ref return uniqueFiles, nil } +// FilesByPath returns all file.References that match the given paths from any layer in the image. func (r *AllLayersResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) { uniqueFileIDs := file.NewFileReferenceSet() uniqueFiles := make([]file.Reference, 0) @@ -81,6 +84,7 @@ func (r *AllLayersResolver) FilesByPath(paths ...file.Path) ([]file.Reference, e return uniqueFiles, nil } +// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) { uniqueFileIDs := file.NewFileReferenceSet() uniqueFiles := make([]file.Reference, 0) @@ -105,6 +109,8 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, e return uniqueFiles, nil } +// MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a +// file.Reference is a path relative to a particular layer. func (r *AllLayersResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) { return r.img.MultipleFileContentsByRef(f...) } diff --git a/syft/scope/resolvers/directory_resolver.go b/syft/scope/resolvers/directory_resolver.go index 697384f1c..b8ed280bc 100644 --- a/syft/scope/resolvers/directory_resolver.go +++ b/syft/scope/resolvers/directory_resolver.go @@ -11,14 +11,17 @@ import ( "github.com/bmatcuk/doublestar" ) +// DirectoryResolver implements path and content access for the directory data source. type DirectoryResolver struct { Path string } +// Stringer to represent a directory path data source func (s DirectoryResolver) String() string { return fmt.Sprintf("dir://%s", s.Path) } +// FilesByPath returns all file.References that match the given paths from the directory. func (s DirectoryResolver) FilesByPath(userPaths ...file.Path) ([]file.Reference, error) { var references = make([]file.Reference, 0) @@ -46,6 +49,7 @@ func fileContents(path file.Path) ([]byte, error) { return contents, nil } +// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) { result := make([]file.Reference, 0) @@ -71,6 +75,7 @@ func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]file.Reference, er return result, nil } +// MultipleFileContentsByRef returns the file contents for all file.References relative a directory. func (s DirectoryResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) { refContents := make(map[file.Reference]string) for _, fileRef := range f { diff --git a/syft/scope/resolvers/docs.go b/syft/scope/resolvers/docs.go new file mode 100644 index 000000000..fe8f17e76 --- /dev/null +++ b/syft/scope/resolvers/docs.go @@ -0,0 +1,4 @@ +/* +Package resolvers provides concrete implementations for the scope.Resolver interface for all supported data sources and scope options. +*/ +package resolvers diff --git a/syft/scope/resolvers/image_squash_resolver.go b/syft/scope/resolvers/image_squash_resolver.go index cad6528da..3df7efb88 100644 --- a/syft/scope/resolvers/image_squash_resolver.go +++ b/syft/scope/resolvers/image_squash_resolver.go @@ -7,10 +7,12 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) +// ImageSquashResolver implements path and content access for the Squashed scope option for container image data sources. type ImageSquashResolver struct { img *image.Image } +// NewImageSquashResolver returns a new resolver from the perspective of the squashed representation for the given image. func NewImageSquashResolver(img *image.Image) (*ImageSquashResolver, error) { if img.SquashedTree() == nil { return nil, fmt.Errorf("the image does not have have a squashed tree") @@ -18,6 +20,7 @@ func NewImageSquashResolver(img *image.Image) (*ImageSquashResolver, error) { return &ImageSquashResolver{img: img}, nil } +// FilesByPath returns all file.References that match the given paths within the squashed representation of the image. func (r *ImageSquashResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) { uniqueFileIDs := file.NewFileReferenceSet() uniqueFiles := make([]file.Reference, 0) @@ -42,6 +45,7 @@ func (r *ImageSquashResolver) FilesByPath(paths ...file.Path) ([]file.Reference, return uniqueFiles, nil } +// FilesByGlob returns all file.References that match the given path glob pattern within the squashed representation of the image. func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) { uniqueFileIDs := file.NewFileReferenceSet() uniqueFiles := make([]file.Reference, 0) @@ -69,6 +73,8 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference, return uniqueFiles, nil } +// MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a +// file.Reference is a path relative to a particular layer, in this case only from the squashed representation. func (r *ImageSquashResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) { return r.img.MultipleFileContentsByRef(f...) } diff --git a/syft/scope/scope.go b/syft/scope/scope.go index 4ac7c6b3d..ddb63b36f 100644 --- a/syft/scope/scope.go +++ b/syft/scope/scope.go @@ -1,3 +1,8 @@ +/* +Package scope provides an abstraction to allow a user to loosely define a data source to catalog and expose a common interface that +catalogers and use explore and analyze data from the data source. All valid (cataloggable) data sources are defined +within this package. +*/ package scope import ( @@ -6,24 +11,27 @@ import ( "github.com/anchore/stereoscope" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/image" "github.com/anchore/syft/syft/scope/resolvers" ) +// ImageSource represents a data source that is a container image type ImageSource struct { - Img *image.Image + Img *image.Image // the image object to be cataloged } +// DirSource represents a data source that is a filesystem directory tree type DirSource struct { - Path string + Path string // the root path to be cataloged } +// Scope is an object that captures the data source to be cataloged, configuration, and a specific resolver used +// in cataloging (based on the data source and configuration) type Scope struct { - Option Option - resolver Resolver - ImgSrc ImageSource - DirSrc DirSource + Option Option // specific perspective to catalog + Resolver Resolver // a Resolver object to use in file path/glob resolution and file contents resolution + ImgSrc ImageSource // the specific image to be cataloged + DirSrc DirSource // the specific directory to be cataloged } // NewScope produces a Scope based on userInput like dir:// or image:tag @@ -37,7 +45,7 @@ func NewScope(userInput string, o Option) (Scope, func(), error) { return Scope{}, func() {}, fmt.Errorf("unable to process path, must exist and be a directory: %w", err) } - s, err := NewScopeFromDir(protocol.Value, o) + s, err := NewScopeFromDir(protocol.Value) if err != nil { return Scope{}, func() {}, fmt.Errorf("could not populate scope from path (%s): %w", protocol.Value, err) } @@ -64,10 +72,10 @@ func NewScope(userInput string, o Option) (Scope, func(), error) { } } -func NewScopeFromDir(path string, option Option) (Scope, error) { +// NewScopeFromDir creates a new scope object tailored to catalog a given filesystem directory recursively. +func NewScopeFromDir(path string) (Scope, error) { return Scope{ - Option: option, - resolver: &resolvers.DirectoryResolver{ + Resolver: &resolvers.DirectoryResolver{ Path: path, }, DirSrc: DirSource{ @@ -76,6 +84,8 @@ func NewScopeFromDir(path string, option Option) (Scope, error) { }, nil } +// NewScopeFromImage creates a new scope object tailored to catalog a given container image, relative to the +// option given (e.g. all-layers, squashed, etc) func NewScopeFromImage(img *image.Image, option Option) (Scope, error) { if img == nil { return Scope{}, fmt.Errorf("no image given") @@ -88,26 +98,14 @@ func NewScopeFromImage(img *image.Image, option Option) (Scope, error) { return Scope{ Option: option, - resolver: resolver, + Resolver: resolver, ImgSrc: ImageSource{ Img: img, }, }, nil } -func (s Scope) FilesByPath(paths ...file.Path) ([]file.Reference, error) { - return s.resolver.FilesByPath(paths...) -} - -func (s Scope) FilesByGlob(patterns ...string) ([]file.Reference, error) { - return s.resolver.FilesByGlob(patterns...) -} - -func (s Scope) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) { - return s.resolver.MultipleFileContentsByRef(f...) -} - -// return either a dir source or img source +// Source returns the configured data source (either a dir source or container image source) func (s Scope) Source() interface{} { if s.ImgSrc != (ImageSource{}) { return s.ImgSrc @@ -119,8 +117,7 @@ func (s Scope) Source() interface{} { return nil } -// isValidPath ensures that the user-provided input will correspond to a path -// that exists and is a directory +// isValidPath ensures that the user-provided input will correspond to a path that exists and is a directory func isValidPath(userInput string) error { fileMeta, err := os.Stat(userInput) if err != nil { diff --git a/syft/scope/scope_test.go b/syft/scope/scope_test.go index 67e3d1b12..b67d6440c 100644 --- a/syft/scope/scope_test.go +++ b/syft/scope/scope_test.go @@ -70,7 +70,7 @@ func TestDirectoryScope(t *testing.T) { } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { - p, err := NewScopeFromDir(test.input, AllLayersScope) + p, err := NewScopeFromDir(test.input) if err != nil { t.Errorf("could not create NewDirScope: %w", err) @@ -79,7 +79,7 @@ func TestDirectoryScope(t *testing.T) { t.Errorf("mismatched stringer: '%s' != '%s'", p.DirSrc.Path, test.input) } - refs, err := p.FilesByPath(test.inputPaths...) + refs, err := p.Resolver.FilesByPath(test.inputPaths...) if err != nil { t.Errorf("FilesByPath call produced an error: %w", err) } @@ -114,11 +114,11 @@ func TestMultipleFileContentsByRefContents(t *testing.T) { } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { - p, err := NewScopeFromDir(test.input, AllLayersScope) + p, err := NewScopeFromDir(test.input) if err != nil { t.Errorf("could not create NewDirScope: %w", err) } - refs, err := p.FilesByPath(file.Path(test.path)) + refs, err := p.Resolver.FilesByPath(file.Path(test.path)) if err != nil { t.Errorf("could not get file references from path: %s, %v", test.path, err) } @@ -128,7 +128,7 @@ func TestMultipleFileContentsByRefContents(t *testing.T) { } ref := refs[0] - contents, err := p.MultipleFileContentsByRef(ref) + contents, err := p.Resolver.MultipleFileContentsByRef(ref) content := contents[ref] if content != test.expected { @@ -154,11 +154,11 @@ func TestMultipleFileContentsByRefNoContents(t *testing.T) { } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { - p, err := NewScopeFromDir(test.input, AllLayersScope) + p, err := NewScopeFromDir(test.input) if err != nil { t.Errorf("could not create NewDirScope: %w", err) } - refs, err := p.FilesByPath(file.Path(test.path)) + refs, err := p.Resolver.FilesByPath(file.Path(test.path)) if err != nil { t.Errorf("could not get file references from path: %s, %v", test.path, err) } @@ -199,12 +199,12 @@ func TestFilesByGlob(t *testing.T) { } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { - p, err := NewScopeFromDir(test.input, AllLayersScope) + p, err := NewScopeFromDir(test.input) if err != nil { t.Errorf("could not create NewDirScope: %w", err) } - contents, err := p.FilesByGlob(test.glob) + contents, err := p.Resolver.FilesByGlob(test.glob) if len(contents) != test.expected { t.Errorf("unexpected number of files found by glob (%s): %d != %d", test.glob, len(contents), test.expected)