Catalog archive contents for single-file input (#637)

* add first-level archive processing when input is a file

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add license exception for github.com/xi2/xz

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* always return cleanup function

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* change source.NewFromFile log entry to warn

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* ensure file source always has cleanup function

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* ensure we are always preferring the unarchive cleanup function for source

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

syft/source/source.go

@@ -7,11 +7,14 @@ package source
 
 import (
 	"fmt"
+	"io/ioutil"
+	"os"
 	"sync"
 
 	"github.com/anchore/stereoscope"
 	"github.com/anchore/stereoscope/pkg/image"
 	"github.com/anchore/syft/internal/log"
+	"github.com/mholt/archiver/v3"
 	"github.com/spf13/afero"
 )
 
@@ -19,9 +22,10 @@ import (
 // in cataloging (based on the data source and configuration)
 type Source struct {
 	Image             *image.Image // the image object to be cataloged (image only)
-	DirectoryResolver *directoryResolver
 	Metadata          Metadata
-	Mutex             *sync.Mutex
+	directoryResolver *directoryResolver
+	path              string
+	mutex             *sync.Mutex
 }
 
 type sourceDetector func(string) (image.Source, string, error)
@@ -97,34 +101,61 @@ func generateFileSource(fs afero.Fs, location string) (*Source, func(), error) {
 		return &Source{}, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", location, err)
 	}
 
-	s, err := NewFromFile(location)
-	if err != nil {
-		return &Source{}, func() {}, fmt.Errorf("could not populate source from path=%q: %w", location, err)
-	}
+	s, cleanupFn := NewFromFile(location)
 
-	return &s, func() {}, nil
+	return &s, cleanupFn, nil
 }
 
 // NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively.
 func NewFromDirectory(path string) (Source, error) {
 	return Source{
-		Mutex: &sync.Mutex{},
+		mutex: &sync.Mutex{},
 		Metadata: Metadata{
 			Scheme: DirectoryScheme,
 			Path:   path,
 		},
+		path: path,
 	}, nil
 }
 
-// NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively.
-func NewFromFile(path string) (Source, error) {
+// NewFromFile creates a new source object tailored to catalog a file.
+func NewFromFile(path string) (Source, func()) {
+	analysisPath, cleanupFn := fileAnalysisPath(path)
+
 	return Source{
-		Mutex: &sync.Mutex{},
+		mutex: &sync.Mutex{},
 		Metadata: Metadata{
 			Scheme: FileScheme,
 			Path:   path,
 		},
-	}, nil
+		path: analysisPath,
+	}, cleanupFn
+}
+
+// fileAnalysisPath returns the path given, or in the case the path is an archive, the location where the archive
+// contents have been made available. A cleanup function is provided for any temp files created (if any).
+func fileAnalysisPath(path string) (string, func()) {
+	var analysisPath = path
+	var cleanupFn = func() {}
+
+	// if the given file is an archive (as indicated by the file extension and not MIME type) then unarchive it and
+	// use the contents as the source. Note: this does NOT recursively unarchive contents, only the given path is
+	// unarchived.
+	envelopedUnarchiver, err := archiver.ByExtension(path)
+	if unarchiver, ok := envelopedUnarchiver.(archiver.Unarchiver); err == nil && ok {
+		unarchivedPath, tmpCleanup, err := unarchiveToTmp(path, unarchiver)
+		if err != nil {
+			log.Warnf("file could not be unarchived: %+v", err)
+		} else {
+			log.Debugf("source path is an archive")
+			analysisPath = unarchivedPath
+		}
+		if tmpCleanup != nil {
+			cleanupFn = tmpCleanup
+		}
+	}
+
+	return analysisPath, cleanupFn
 }
 
 // NewFromImage creates a new source object tailored to catalog a given container image, relative to the
@@ -146,16 +177,16 @@ func NewFromImage(img *image.Image, userImageStr string) (Source, error) {
 func (s *Source) FileResolver(scope Scope) (FileResolver, error) {
 	switch s.Metadata.Scheme {
 	case DirectoryScheme, FileScheme:
-		s.Mutex.Lock()
-		defer s.Mutex.Unlock()
-		if s.DirectoryResolver == nil {
-			resolver, err := newDirectoryResolver(s.Metadata.Path)
+		s.mutex.Lock()
+		defer s.mutex.Unlock()
+		if s.directoryResolver == nil {
+			resolver, err := newDirectoryResolver(s.path)
 			if err != nil {
 				return nil, err
 			}
-			s.DirectoryResolver = resolver
+			s.directoryResolver = resolver
 		}
-		return s.DirectoryResolver, nil
+		return s.directoryResolver, nil
 	case ImageScheme:
 		switch scope {
 		case SquashedScope:
@@ -168,3 +199,18 @@ func (s *Source) FileResolver(scope Scope) (FileResolver, error) {
 	}
 	return nil, fmt.Errorf("unable to determine FilePathResolver with current scheme=%q", s.Metadata.Scheme)
 }
+
+func unarchiveToTmp(path string, unarchiver archiver.Unarchiver) (string, func(), error) {
+	tempDir, err := ioutil.TempDir("", "syft-archive-contents-")
+	if err != nil {
+		return "", func() {}, fmt.Errorf("unable to create tempdir for archive processing: %w", err)
+	}
+
+	cleanupFn := func() {
+		if err := os.RemoveAll(tempDir); err != nil {
+			log.Warnf("unable to cleanup archive tempdir: %+v", err)
+		}
+	}
+
+	return tempDir, cleanupFn, unarchiver.Unarchive(path, tempDir)
+}

syft/source/source_test.go

@@ -1,8 +1,16 @@
 package source
 
 import (
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"syscall"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/stretchr/testify/assert"
 
 	"github.com/anchore/stereoscope/pkg/image"
@@ -89,6 +97,80 @@ func TestNewFromDirectory(t *testing.T) {
 	}
 }
 
+func TestNewFromFile(t *testing.T) {
+	testCases := []struct {
+		desc       string
+		input      string
+		expString  string
+		inputPaths []string
+		expRefs    int
+	}{
+		{
+			desc:       "path detected",
+			input:      "test-fixtures/path-detected",
+			inputPaths: []string{"/.vimrc"},
+			expRefs:    1,
+		},
+	}
+	for _, test := range testCases {
+		t.Run(test.desc, func(t *testing.T) {
+			src, cleanup := NewFromFile(test.input)
+			if cleanup != nil {
+				t.Cleanup(cleanup)
+			}
+
+			assert.Equal(t, test.input, src.Metadata.Path)
+			assert.Equal(t, src.Metadata.Path, src.path)
+
+			resolver, err := src.FileResolver(SquashedScope)
+			require.NoError(t, err)
+
+			refs, err := resolver.FilesByPath(test.inputPaths...)
+			require.NoError(t, err)
+			assert.Len(t, refs, test.expRefs)
+
+		})
+	}
+}
+
+func TestNewFromFile_WithArchive(t *testing.T) {
+	testCases := []struct {
+		desc       string
+		input      string
+		expString  string
+		inputPaths []string
+		expRefs    int
+	}{
+		{
+			desc:       "path detected",
+			input:      "test-fixtures/path-detected",
+			inputPaths: []string{"/.vimrc"},
+			expRefs:    1,
+		},
+	}
+	for _, test := range testCases {
+		t.Run(test.desc, func(t *testing.T) {
+			archivePath := setupArchiveTest(t, test.input)
+
+			src, cleanup := NewFromFile(archivePath)
+			if cleanup != nil {
+				t.Cleanup(cleanup)
+			}
+
+			assert.Equal(t, archivePath, src.Metadata.Path)
+			assert.NotEqual(t, src.Metadata.Path, src.path)
+
+			resolver, err := src.FileResolver(SquashedScope)
+			require.NoError(t, err)
+
+			refs, err := resolver.FilesByPath(test.inputPaths...)
+			require.NoError(t, err)
+			assert.Len(t, refs, test.expRefs)
+
+		})
+	}
+}
+
 func TestNewFromDirectoryShared(t *testing.T) {
 	testCases := []struct {
 		desc       string
@@ -232,3 +314,83 @@ func TestFilesByGlob(t *testing.T) {
 		})
 	}
 }
+
+// createArchive creates a new archive file at destinationArchivePath based on the directory found at sourceDirPath.
+func createArchive(t testing.TB, sourceDirPath, destinationArchivePath string) {
+	t.Helper()
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("unable to get cwd: %+v", err)
+	}
+
+	cmd := exec.Command("./generate-tar-fixture-from-source-dir.sh", destinationArchivePath, path.Base(sourceDirPath))
+	cmd.Dir = filepath.Join(cwd, "test-fixtures")
+
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("unable to start generate zip fixture script: %+v", err)
+	}
+
+	if err := cmd.Wait(); err != nil {
+		if exiterr, ok := err.(*exec.ExitError); ok {
+			// The program has exited with an exit code != 0
+
+			// This works on both Unix and Windows. Although package
+			// syscall is generally platform dependent, WaitStatus is
+			// defined for both Unix and Windows and in both cases has
+			// an ExitStatus() method with the same signature.
+			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
+				if status.ExitStatus() != 0 {
+					t.Fatalf("failed to generate fixture: rc=%d", status.ExitStatus())
+				}
+			}
+		} else {
+			t.Fatalf("unable to get generate fixture script result: %+v", err)
+		}
+	}
+
+}
+
+// setupArchiveTest encapsulates common test setup work for tar file tests. It returns a cleanup function,
+// which should be called (typically deferred) by the caller, the path of the created tar archive, and an error,
+// which should trigger a fatal test failure in the consuming test. The returned cleanup function will never be nil
+// (even if there's an error), and it should always be called.
+func setupArchiveTest(t testing.TB, sourceDirPath string) string {
+	t.Helper()
+
+	archivePrefix, err := ioutil.TempFile("", "syft-archive-TEST-")
+	require.NoError(t, err)
+
+	t.Cleanup(
+		assertNoError(t,
+			func() error {
+				return os.Remove(archivePrefix.Name())
+			},
+		),
+	)
+
+	destinationArchiveFilePath := archivePrefix.Name() + ".tar"
+	t.Logf("archive path: %s", destinationArchiveFilePath)
+	createArchive(t, sourceDirPath, destinationArchiveFilePath)
+
+	t.Cleanup(
+		assertNoError(t,
+			func() error {
+				return os.Remove(destinationArchiveFilePath)
+			},
+		),
+	)
+
+	cwd, err := os.Getwd()
+	require.NoError(t, err)
+
+	t.Logf("running from: %s", cwd)
+
+	return destinationArchiveFilePath
+}
+
+func assertNoError(t testing.TB, fn func() error) func() {
+	return func() {
+		assert.NoError(t, fn())
+	}
+}

syft/source/test-fixtures/generate-tar-fixture-from-source-dir.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -eux
+
+# $1 —— absolute path to destination file, should end with .tar
+# $2 —— absolute path to directory from which to add entries to the archive
+
+pushd "$2"
+  tar -cvf "$1" .
+popd