diff --git a/internal/err_helper.go b/internal/err_helper.go new file mode 100644 index 000000000..823f2912d --- /dev/null +++ b/internal/err_helper.go @@ -0,0 +1,14 @@ +package internal + +import ( + "io" + + "github.com/anchore/syft/internal/log" +) + +// CloseAndLogError closes the given io.Closer and reports any errors found as a warning in the log +func CloseAndLogError(closer io.Closer, location string) { + if err := closer.Close(); err != nil { + log.Warnf("unable to close file for location=%q: %+v", location, err) + } +} diff --git a/syft/distro/identify.go b/syft/distro/identify.go index 72be74b97..d322582e6 100644 --- a/syft/distro/identify.go +++ b/syft/distro/identify.go @@ -5,6 +5,8 @@ import ( "regexp" "strings" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/source" ) @@ -54,13 +56,13 @@ identifyLoop: for _, location := range locations { contentReader, err := resolver.FileContentsByLocation(location) - if err != nil { log.Debugf("unable to get contents from %s: %s", entry.path, err) continue } content, err := ioutil.ReadAll(contentReader) + internal.CloseAndLogError(contentReader, location.VirtualPath) if err != nil { log.Errorf("unable to read %q: %+v", location.RealPath, err) break diff --git a/syft/file/classifier.go b/syft/file/classifier.go index b3aa144cd..d4efa538a 100644 --- a/syft/file/classifier.go +++ b/syft/file/classifier.go @@ -81,7 +81,7 @@ func (c Classifier) Classify(resolver source.FileResolver, location source.Locat if err != nil { return nil, err } - defer contentReader.Close() + defer internal.CloseAndLogError(contentReader, location.VirtualPath) // TODO: there is room for improvement here, as this may use an excessive amount of memory. Alternate approach is to leverage a RuneReader. contents, err := ioutil.ReadAll(contentReader) diff --git a/syft/file/contents_cataloger.go b/syft/file/contents_cataloger.go index 97652868c..34d060fa8 100644 --- a/syft/file/contents_cataloger.go +++ b/syft/file/contents_cataloger.go @@ -6,6 +6,8 @@ import ( "fmt" "io" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/source" ) @@ -57,7 +59,7 @@ func (i *ContentsCataloger) catalogLocation(resolver source.FileResolver, locati if err != nil { return "", err } - defer contentReader.Close() + defer internal.CloseAndLogError(contentReader, location.VirtualPath) buf := &bytes.Buffer{} if _, err = io.Copy(base64.NewEncoder(base64.StdEncoding, buf), contentReader); err != nil { diff --git a/syft/file/digest_cataloger.go b/syft/file/digest_cataloger.go index e0696329c..b9808e76e 100644 --- a/syft/file/digest_cataloger.go +++ b/syft/file/digest_cataloger.go @@ -7,6 +7,8 @@ import ( "io" "strings" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/bus" @@ -53,7 +55,7 @@ func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, locatio if err != nil { return nil, err } - defer contentReader.Close() + defer internal.CloseAndLogError(contentReader, location.VirtualPath) // create a set of hasher objects tied together with a single writer to feed content into hashers := make([]hash.Hash, len(i.hashes)) diff --git a/syft/file/secrets_cataloger.go b/syft/file/secrets_cataloger.go index c83ef1167..95b1a11f7 100644 --- a/syft/file/secrets_cataloger.go +++ b/syft/file/secrets_cataloger.go @@ -8,6 +8,8 @@ import ( "regexp" "sort" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/event" @@ -101,7 +103,7 @@ func extractValue(resolver source.FileResolver, location source.Location, start, if err != nil { return "", fmt.Errorf("unable to fetch reader for location=%q : %w", location, err) } - defer readCloser.Close() + defer internal.CloseAndLogError(readCloser, location.VirtualPath) n, err := io.CopyN(ioutil.Discard, readCloser, start) if err != nil { diff --git a/syft/file/secrets_search_by_line_strategy.go b/syft/file/secrets_search_by_line_strategy.go index 4ee93a24d..0fd87ff63 100644 --- a/syft/file/secrets_search_by_line_strategy.go +++ b/syft/file/secrets_search_by_line_strategy.go @@ -8,6 +8,8 @@ import ( "io/ioutil" "regexp" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/source" ) @@ -16,7 +18,7 @@ func catalogLocationByLine(resolver source.FileResolver, location source.Locatio if err != nil { return nil, fmt.Errorf("unable to fetch reader for location=%q : %w", location, err) } - defer readCloser.Close() + defer internal.CloseAndLogError(readCloser, location.VirtualPath) var scanner = bufio.NewReader(readCloser) var position int64 @@ -65,6 +67,7 @@ func searchForSecretsWithinLine(resolver source.FileResolver, location source.Lo if secret != nil { secrets = append(secrets, *secret) } + internal.CloseAndLogError(reader, location.VirtualPath) } } diff --git a/syft/pkg/cataloger/common/generic_cataloger.go b/syft/pkg/cataloger/common/generic_cataloger.go index 5471a9129..84ef0f865 100644 --- a/syft/pkg/cataloger/common/generic_cataloger.go +++ b/syft/pkg/cataloger/common/generic_cataloger.go @@ -6,6 +6,7 @@ package common import ( "fmt" + "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" @@ -39,13 +40,14 @@ func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, parserByLocation := c.selectFiles(resolver) for location, parser := range parserByLocation { - content, err := resolver.FileContentsByLocation(location) + contentReader, err := resolver.FileContentsByLocation(location) if err != nil { // TODO: fail or log? return nil, fmt.Errorf("unable to fetch contents for location=%v : %w", location, err) } - entries, err := parser(location.RealPath, content) + entries, err := parser(location.RealPath, contentReader) + internal.CloseAndLogError(contentReader, location.VirtualPath) if err != nil { // TODO: should we fail? or only log? log.Warnf("cataloger '%s' failed to parse entries (location=%+v): %+v", c.upstreamCataloger, location, err) diff --git a/syft/pkg/cataloger/deb/cataloger.go b/syft/pkg/cataloger/deb/cataloger.go index 905d08727..1cd000707 100644 --- a/syft/pkg/cataloger/deb/cataloger.go +++ b/syft/pkg/cataloger/deb/cataloger.go @@ -10,6 +10,8 @@ import ( "path/filepath" "sort" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" @@ -50,6 +52,7 @@ func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) } pkgs, err = parseDpkgStatus(dbContents) + internal.CloseAndLogError(dbContents, dbLocation.VirtualPath) if err != nil { return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.RealPath, err) } @@ -78,6 +81,7 @@ func addLicenses(resolver source.FileResolver, dbLocation source.Location, p *pk copyrightReader, copyrightLocation := fetchCopyrightContents(resolver, dbLocation, p) if copyrightReader != nil { + defer internal.CloseAndLogError(copyrightReader, copyrightLocation.VirtualPath) // attach the licenses p.Licenses = parseLicensesFromCopyright(copyrightReader) @@ -124,6 +128,7 @@ func getAdditionalFileListing(resolver source.FileResolver, dbLocation source.Lo md5Reader, md5Location := fetchMd5Contents(resolver, dbLocation, p) if md5Reader != nil { + defer internal.CloseAndLogError(md5Reader, md5Location.VirtualPath) // attach the file list files = append(files, parseDpkgMD5Info(md5Reader)...) @@ -136,6 +141,7 @@ func getAdditionalFileListing(resolver source.FileResolver, dbLocation source.Lo conffilesReader, conffilesLocation := fetchConffileContents(resolver, dbLocation, p) if conffilesReader != nil { + defer internal.CloseAndLogError(conffilesReader, conffilesLocation.VirtualPath) // attach the file list files = append(files, parseDpkgConffileInfo(md5Reader)...) diff --git a/syft/pkg/cataloger/javascript/parse_package_json.go b/syft/pkg/cataloger/javascript/parse_package_json.go index 9cc2f5953..9d4050328 100644 --- a/syft/pkg/cataloger/javascript/parse_package_json.go +++ b/syft/pkg/cataloger/javascript/parse_package_json.go @@ -18,7 +18,7 @@ import ( ) // integrity check -var _ common.ParserFn = parsePackageLock +var _ common.ParserFn = parsePackageJSON // PackageJSON represents a JavaScript package.json file type PackageJSON struct { diff --git a/syft/pkg/cataloger/python/package_cataloger.go b/syft/pkg/cataloger/python/package_cataloger.go index 980c4b699..b49512125 100644 --- a/syft/pkg/cataloger/python/package_cataloger.go +++ b/syft/pkg/cataloger/python/package_cataloger.go @@ -5,6 +5,8 @@ import ( "fmt" "path/filepath" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" @@ -101,6 +103,7 @@ func (c *PackageCataloger) fetchRecordFiles(resolver source.FileResolver, metada if err != nil { return nil, nil, err } + defer internal.CloseAndLogError(recordContents, recordPath) // parse the record contents records, err := parseWheelOrEggRecord(recordContents) @@ -130,6 +133,7 @@ func (c *PackageCataloger) fetchTopLevelPackages(resolver source.FileResolver, m if err != nil { return nil, nil, err } + defer internal.CloseAndLogError(topLevelContents, topLevelLocation.VirtualPath) scanner := bufio.NewScanner(topLevelContents) for scanner.Scan() { @@ -151,6 +155,7 @@ func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.FileResolv if err != nil { return nil, nil, err } + defer internal.CloseAndLogError(metadataContents, metadataLocation.VirtualPath) metadata, err := parseWheelOrEggMetadata(metadataLocation.RealPath, metadataContents) if err != nil { diff --git a/syft/pkg/cataloger/rpmdb/cataloger.go b/syft/pkg/cataloger/rpmdb/cataloger.go index eca4c60c7..50e3697a3 100644 --- a/syft/pkg/cataloger/rpmdb/cataloger.go +++ b/syft/pkg/cataloger/rpmdb/cataloger.go @@ -6,6 +6,8 @@ package rpmdb import ( "fmt" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" ) @@ -39,6 +41,7 @@ func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) } pkgs, err = parseRpmDB(resolver, location, dbContentReader) + internal.CloseAndLogError(dbContentReader, location.VirtualPath) if err != nil { return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.RealPath, err) }