Merge pull request #414 from anchore/fix-leaky-fds

Close all ReadClosers explicitly retrieved from resolvers

internal/err_helper.go

@@ -0,0 +1,14 @@
+package internal
+
+import (
+	"io"
+
+	"github.com/anchore/syft/internal/log"
+)
+
+// CloseAndLogError closes the given io.Closer and reports any errors found as a warning in the log
+func CloseAndLogError(closer io.Closer, location string) {
+	if err := closer.Close(); err != nil {
+		log.Warnf("unable to close file for location=%q: %+v", location, err)
+	}
+}

syft/distro/identify.go

@@ -5,6 +5,8 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/anchore/syft/internal"
+
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/source"
 )
@@ -54,13 +56,13 @@ identifyLoop:
 
 		for _, location := range locations {
 			contentReader, err := resolver.FileContentsByLocation(location)
-
 			if err != nil {
 				log.Debugf("unable to get contents from %s: %s", entry.path, err)
 				continue
 			}
 
 			content, err := ioutil.ReadAll(contentReader)
+			internal.CloseAndLogError(contentReader, location.VirtualPath)
 			if err != nil {
 				log.Errorf("unable to read %q: %+v", location.RealPath, err)
 				break

syft/file/classifier.go

@@ -81,7 +81,7 @@ func (c Classifier) Classify(resolver source.FileResolver, location source.Locat
 	if err != nil {
 		return nil, err
 	}
-	defer contentReader.Close()
+	defer internal.CloseAndLogError(contentReader, location.VirtualPath)
 
 	// TODO: there is room for improvement here, as this may use an excessive amount of memory. Alternate approach is to leverage a RuneReader.
 	contents, err := ioutil.ReadAll(contentReader)

syft/file/contents_cataloger.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/anchore/syft/internal"
+
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/source"
 )
@@ -57,7 +59,7 @@ func (i *ContentsCataloger) catalogLocation(resolver source.FileResolver, locati
 	if err != nil {
 		return "", err
 	}
-	defer contentReader.Close()
+	defer internal.CloseAndLogError(contentReader, location.VirtualPath)
 
 	buf := &bytes.Buffer{}
 	if _, err = io.Copy(base64.NewEncoder(base64.StdEncoding, buf), contentReader); err != nil {

syft/file/digest_cataloger.go

@@ -7,6 +7,8 @@ import (
 	"io"
 	"strings"
 
+	"github.com/anchore/syft/internal"
+
 	"github.com/anchore/syft/internal/log"
 
 	"github.com/anchore/syft/internal/bus"
@@ -53,7 +55,7 @@ func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, locatio
 	if err != nil {
 		return nil, err
 	}
-	defer contentReader.Close()
+	defer internal.CloseAndLogError(contentReader, location.VirtualPath)
 
 	// create a set of hasher objects tied together with a single writer to feed content into
 	hashers := make([]hash.Hash, len(i.hashes))

syft/file/secrets_cataloger.go

@@ -8,6 +8,8 @@ import (
 	"regexp"
 	"sort"
 
+	"github.com/anchore/syft/internal"
+
 	"github.com/anchore/syft/internal/bus"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/event"
@@ -101,7 +103,7 @@ func extractValue(resolver source.FileResolver, location source.Location, start,
 	if err != nil {
 		return "", fmt.Errorf("unable to fetch reader for location=%q : %w", location, err)
 	}
-	defer readCloser.Close()
+	defer internal.CloseAndLogError(readCloser, location.VirtualPath)
 
 	n, err := io.CopyN(ioutil.Discard, readCloser, start)
 	if err != nil {

syft/file/secrets_search_by_line_strategy.go

@@ -8,6 +8,8 @@ import (
 	"io/ioutil"
 	"regexp"
 
+	"github.com/anchore/syft/internal"
+
 	"github.com/anchore/syft/syft/source"
 )
 
@@ -16,7 +18,7 @@ func catalogLocationByLine(resolver source.FileResolver, location source.Locatio
 	if err != nil {
 		return nil, fmt.Errorf("unable to fetch reader for location=%q : %w", location, err)
 	}
-	defer readCloser.Close()
+	defer internal.CloseAndLogError(readCloser, location.VirtualPath)
 
 	var scanner = bufio.NewReader(readCloser)
 	var position int64
@@ -65,6 +67,7 @@ func searchForSecretsWithinLine(resolver source.FileResolver, location source.Lo
 			if secret != nil {
 				secrets = append(secrets, *secret)
 			}
+			internal.CloseAndLogError(reader, location.VirtualPath)
 		}
 	}
 

syft/pkg/cataloger/common/generic_cataloger.go

@@ -6,6 +6,7 @@ package common
 import (
 	"fmt"
 
+	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
@@ -39,13 +40,14 @@ func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package,
 	parserByLocation := c.selectFiles(resolver)
 
 	for location, parser := range parserByLocation {
-		content, err := resolver.FileContentsByLocation(location)
+		contentReader, err := resolver.FileContentsByLocation(location)
 		if err != nil {
 			// TODO: fail or log?
 			return nil, fmt.Errorf("unable to fetch contents for location=%v : %w", location, err)
 		}
 
-		entries, err := parser(location.RealPath, content)
+		entries, err := parser(location.RealPath, contentReader)
+		internal.CloseAndLogError(contentReader, location.VirtualPath)
 		if err != nil {
 			// TODO: should we fail? or only log?
 			log.Warnf("cataloger '%s' failed to parse entries (location=%+v): %+v", c.upstreamCataloger, location, err)

syft/pkg/cataloger/deb/cataloger.go

@@ -10,6 +10,8 @@ import (
 	"path/filepath"
 	"sort"
 
+	"github.com/anchore/syft/internal"
+
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
@@ -50,6 +52,7 @@ func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error)
 		}
 
 		pkgs, err = parseDpkgStatus(dbContents)
+		internal.CloseAndLogError(dbContents, dbLocation.VirtualPath)
 		if err != nil {
 			return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.RealPath, err)
 		}
@@ -78,6 +81,7 @@ func addLicenses(resolver source.FileResolver, dbLocation source.Location, p *pk
 	copyrightReader, copyrightLocation := fetchCopyrightContents(resolver, dbLocation, p)
 
 	if copyrightReader != nil {
+		defer internal.CloseAndLogError(copyrightReader, copyrightLocation.VirtualPath)
 		// attach the licenses
 		p.Licenses = parseLicensesFromCopyright(copyrightReader)
 
@@ -124,6 +128,7 @@ func getAdditionalFileListing(resolver source.FileResolver, dbLocation source.Lo
 	md5Reader, md5Location := fetchMd5Contents(resolver, dbLocation, p)
 
 	if md5Reader != nil {
+		defer internal.CloseAndLogError(md5Reader, md5Location.VirtualPath)
 		// attach the file list
 		files = append(files, parseDpkgMD5Info(md5Reader)...)
 
@@ -136,6 +141,7 @@ func getAdditionalFileListing(resolver source.FileResolver, dbLocation source.Lo
 	conffilesReader, conffilesLocation := fetchConffileContents(resolver, dbLocation, p)
 
 	if conffilesReader != nil {
+		defer internal.CloseAndLogError(conffilesReader, conffilesLocation.VirtualPath)
 		// attach the file list
 		files = append(files, parseDpkgConffileInfo(md5Reader)...)
 

syft/pkg/cataloger/javascript/parse_package_json.go

@@ -18,7 +18,7 @@ import (
 )
 
 // integrity check
-var _ common.ParserFn = parsePackageLock
+var _ common.ParserFn = parsePackageJSON
 
 // PackageJSON represents a JavaScript package.json file
 type PackageJSON struct {

syft/pkg/cataloger/python/package_cataloger.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"path/filepath"
 
+	"github.com/anchore/syft/internal"
+
 	"github.com/anchore/syft/syft/pkg"
 
 	"github.com/anchore/syft/syft/source"
@@ -101,6 +103,7 @@ func (c *PackageCataloger) fetchRecordFiles(resolver source.FileResolver, metada
 		if err != nil {
 			return nil, nil, err
 		}
+		defer internal.CloseAndLogError(recordContents, recordPath)
 
 		// parse the record contents
 		records, err := parseWheelOrEggRecord(recordContents)
@@ -130,6 +133,7 @@ func (c *PackageCataloger) fetchTopLevelPackages(resolver source.FileResolver, m
 	if err != nil {
 		return nil, nil, err
 	}
+	defer internal.CloseAndLogError(topLevelContents, topLevelLocation.VirtualPath)
 
 	scanner := bufio.NewScanner(topLevelContents)
 	for scanner.Scan() {
@@ -151,6 +155,7 @@ func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.FileResolv
 	if err != nil {
 		return nil, nil, err
 	}
+	defer internal.CloseAndLogError(metadataContents, metadataLocation.VirtualPath)
 
 	metadata, err := parseWheelOrEggMetadata(metadataLocation.RealPath, metadataContents)
 	if err != nil {

syft/pkg/cataloger/rpmdb/cataloger.go

@@ -6,6 +6,8 @@ package rpmdb
 import (
 	"fmt"
 
+	"github.com/anchore/syft/internal"
+
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
 )
@@ -39,6 +41,7 @@ func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error)
 		}
 
 		pkgs, err = parseRpmDB(resolver, location, dbContentReader)
+		internal.CloseAndLogError(dbContentReader, location.VirtualPath)
 		if err != nil {
 			return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.RealPath, err)
 		}