Add support for searching for jars within archives (#734)

* add support for searching jars within archives

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add package cataloger config options

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* address review comments + factor out safeCopy helper

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update config docs regarding package archive search options

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* show that unindexed archive cataloging defaults to false

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* remove lies about -s

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* address review comments

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update search archive note about java

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

internal/config/application.go

@@ -38,7 +38,7 @@ type Application struct {
 	CliOptions         CliOnlyOptions     `yaml:"-" json:"-"`                                                                           // all options only available through the CLI (not via env vars or config)
 	Dev                development        `yaml:"dev" json:"dev" mapstructure:"dev"`
 	Log                logging            `yaml:"log" json:"log" mapstructure:"log"` // all logging-related options
-	Package            packages           `yaml:"package" json:"package" mapstructure:"package"`
+	Package            pkg                `yaml:"package" json:"package" mapstructure:"package"`
 	FileMetadata       FileMetadata       `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"`
 	FileClassification fileClassification `yaml:"file-classification" json:"file-classification" mapstructure:"file-classification"`
 	FileContents       fileContents       `yaml:"file-contents" json:"file-contents" mapstructure:"file-contents"`

internal/config/cataloger_options.go

@@ -3,6 +3,8 @@ package config
 import (
 	"fmt"
 
+	"github.com/spf13/viper"
+
 	"github.com/anchore/syft/syft/source"
 )
 
@@ -12,6 +14,10 @@ type catalogerOptions struct {
 	ScopeOpt source.Scope `yaml:"-" json:"-"`
 }
 
+func (cfg catalogerOptions) loadDefaultValues(v *viper.Viper) {
+	v.SetDefault("package.cataloger.enabled", true)
+}
+
 func (cfg *catalogerOptions) parseConfigValues() error {
 	scopeOption := source.ParseScope(cfg.Scope)
 	if scopeOption == source.UnknownScope {

internal/config/packages.go

@@ -1,15 +0,0 @@
-package config
-
-import "github.com/spf13/viper"
-
-type packages struct {
-	Cataloger catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"`
-}
-
-func (cfg packages) loadDefaultValues(v *viper.Viper) {
-	v.SetDefault("package.cataloger.enabled", true)
-}
-
-func (cfg *packages) parseConfigValues() error {
-	return cfg.Cataloger.parseConfigValues()
-}

internal/config/pkg.go

@@ -0,0 +1,33 @@
+package config
+
+import (
+	"github.com/anchore/syft/syft/pkg/cataloger"
+	"github.com/spf13/viper"
+)
+
+type pkg struct {
+	Cataloger               catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"`
+	SearchUnindexedArchives bool             `yaml:"search-unindexed-archives" json:"search-unindexed-archives" mapstructure:"search-unindexed-archives"`
+	SearchIndexedArchives   bool             `yaml:"search-indexed-archives" json:"search-indexed-archives" mapstructure:"search-indexed-archives"`
+}
+
+func (cfg pkg) loadDefaultValues(v *viper.Viper) {
+	cfg.Cataloger.loadDefaultValues(v)
+	c := cataloger.DefaultSearchConfig()
+	v.SetDefault("package.search-unindexed-archives", c.IncludeUnindexedArchives)
+	v.SetDefault("package.search-indexed-archives", c.IncludeIndexedArchives)
+}
+
+func (cfg *pkg) parseConfigValues() error {
+	return cfg.Cataloger.parseConfigValues()
+}
+
+func (cfg pkg) ToConfig() cataloger.Config {
+	return cataloger.Config{
+		Search: cataloger.SearchConfig{
+			IncludeIndexedArchives:   cfg.SearchIndexedArchives,
+			IncludeUnindexedArchives: cfg.SearchUnindexedArchives,
+			Scope:                    cfg.Cataloger.ScopeOpt,
+		},
+	}
+}

internal/file/copy.go

@@ -0,0 +1,19 @@
+package file
+
+import (
+	"errors"
+	"fmt"
+	"io"
+)
+
+const perFileReadLimit = 2 * GB
+
+// safeCopy limits the copy from the reader. This is useful when extracting files from archives to
+// protect against decompression bomb attacks.
+func safeCopy(writer io.Writer, reader io.Reader) error {
+	numBytes, err := io.Copy(writer, io.LimitReader(reader, perFileReadLimit))
+	if numBytes >= perFileReadLimit || errors.Is(err, io.EOF) {
+		return fmt.Errorf("zip read limit hit (potential decompression bomb attack)")
+	}
+	return nil
+}

internal/file/tar_file_traversal.go

@@ -0,0 +1,64 @@
+package file
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/bmatcuk/doublestar/v4"
+	"github.com/mholt/archiver/v3"
+)
+
+// ExtractGlobsFromTarToUniqueTempFile extracts paths matching the given globs within the given archive to a temporary directory, returning file openers for each file extracted.
+func ExtractGlobsFromTarToUniqueTempFile(archivePath, dir string, globs ...string) (map[string]Opener, error) {
+	results := make(map[string]Opener)
+
+	// don't allow for full traversal, only select traversal from given paths
+	if len(globs) == 0 {
+		return results, nil
+	}
+
+	visitor := func(file archiver.File) error {
+		defer file.Close()
+
+		// ignore directories
+		if file.FileInfo.IsDir() {
+			return nil
+		}
+
+		// ignore any filename that doesn't match the given globs...
+		if !matchesAnyGlob(file.Name(), globs...) {
+			return nil
+		}
+
+		// we have a file we want to extract....
+		tempfilePrefix := filepath.Base(filepath.Clean(file.Name())) + "-"
+		tempFile, err := ioutil.TempFile(dir, tempfilePrefix)
+		if err != nil {
+			return fmt.Errorf("unable to create temp file: %w", err)
+		}
+		// we shouldn't try and keep the tempfile open as the returned result may have several files, which takes up
+		// resources (leading to "too many open files"). Instead we'll return a file opener to the caller which
+		// provides a ReadCloser. It is up to the caller to handle closing the file explicitly.
+		defer tempFile.Close()
+
+		if err := safeCopy(tempFile, file.ReadCloser); err != nil {
+			return fmt.Errorf("unable to copy source=%q for tar=%q: %w", file.Name(), archivePath, err)
+		}
+
+		results[file.Name()] = Opener{path: tempFile.Name()}
+
+		return nil
+	}
+
+	return results, archiver.Walk(archivePath, visitor)
+}
+
+func matchesAnyGlob(name string, globs ...string) bool {
+	for _, glob := range globs {
+		if matches, err := doublestar.PathMatch(glob, name); err == nil && matches {
+			return true
+		}
+	}
+	return false
+}

internal/file/zip_file_traversal.go

@@ -3,9 +3,7 @@ package file
 import (
 	"archive/zip"
 	"bytes"
-	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -22,8 +20,6 @@ const (
 	GB
 )
 
-const perFileReadLimit = 2 * GB
-
 type errZipSlipDetected struct {
 	Prefix   string
 	JoinArgs []string
@@ -110,21 +106,10 @@ func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (m
 			return fmt.Errorf("unable to extract directories, only files: %s", file.Name)
 		}
 
-		// limit the zip reader on each file read to prevent decompression bomb attacks
-		numBytes, err := io.Copy(tempFile, io.LimitReader(zippedFile, perFileReadLimit))
-		if numBytes >= perFileReadLimit || errors.Is(err, io.EOF) {
-			return fmt.Errorf("zip read limit hit (potential decompression bomb attack)")
-		}
-		if err != nil {
+		if err := safeCopy(tempFile, zippedFile); err != nil {
 			return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.Name, archivePath, err)
 		}
 
-		// the file pointer is at the end due to the copy operation, reset back to the beginning
-		_, err = tempFile.Seek(0, io.SeekStart)
-		if err != nil {
-			return fmt.Errorf("unable to reset file pointer (%s): %w", tempFile.Name(), err)
-		}
-
 		results[file.Name] = Opener{path: tempFile.Name()}
 
 		return nil
@@ -153,13 +138,7 @@ func ContentsFromZip(archivePath string, paths ...string) (map[string]string, er
 		}
 
 		var buffer bytes.Buffer
-
-		// limit the zip reader on each file read to prevent decompression bomb attacks
-		numBytes, err := io.Copy(&buffer, io.LimitReader(zippedFile, perFileReadLimit))
-		if numBytes >= perFileReadLimit || errors.Is(err, io.EOF) {
-			return fmt.Errorf("zip read limit hit (potential decompression bomb attack)")
-		}
-		if err != nil {
+		if err := safeCopy(&buffer, zippedFile); err != nil {
 			return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.Name, archivePath, err)
 		}
 
@@ -228,12 +207,7 @@ func extractSingleFile(file *zip.File, expandedFilePath, archivePath string) err
 			return fmt.Errorf("unable to create dest file=%q from zip=%q: %w", expandedFilePath, archivePath, err)
 		}
 
-		// limit the zip reader on each file read to prevent decompression bomb attacks
-		numBytes, err := io.Copy(outputFile, io.LimitReader(zippedFile, perFileReadLimit))
-		if numBytes >= perFileReadLimit || errors.Is(err, io.EOF) {
-			return fmt.Errorf("zip read limit hit (potential decompression bomb attack)")
-		}
-		if err != nil {
+		if err := safeCopy(outputFile, zippedFile); err != nil {
 			return fmt.Errorf("unable to copy source=%q to dest=%q for zip=%q: %w", file.Name, outputFile.Name(), archivePath, err)
 		}