Merge pull request #428 from anchore/self-extracting-jar

Add support for processing files with prepended bytes before the zip archive

.github/ISSUE_TEMPLATE/config.yml

@@ -1,11 +1,6 @@
 contact_links:
 
-  - name: Help & Support Slack Channel ⛑️
-    # link to toolbox-help channel
-    url: https://anchorecommunity.slack.com/archives/C019BUXV7R6
-    about: Ask a questions and get answers
-
-  - name: Developer Slack Channel 💬
-    # link to toolbox-dev channel
-    url: https://anchorecommunity.slack.com/archives/C0190NF9TSM
-    about: Participate in design discussions and feature development
+  - name: Join the Slack community 💬
+    # link to our community Slack registration page
+    url: https://anchore.com/slack
+    about: 'Be sure to check out the channels #toolbox-help (ask us questions) and #toolbox-dev (collaborate with us on design and development)!'

.github/scripts/go-mod-tidy-check.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -u
+
+TMP_GOMOD=$(mktemp)
+TMP_GOSUM=$(mktemp)
+
+trap "rm -f ${TMP_GOSUM} ${TMP_GOMOD}" EXIT
+
+cp go.mod "${TMP_GOMOD}"
+cp go.sum "${TMP_GOSUM}"
+
+go mod tidy
+
+DIFF_MOD=$(diff -u "${TMP_GOMOD}" go.mod)
+DIFF_SUM=$(diff -u "${TMP_GOSUM}" go.sum)
+
+cp "${TMP_GOMOD}" go.mod
+cp "${TMP_GOSUM}" go.sum
+
+if [[ -n "${DIFF_MOD}" || -n "${DIFF_SUM}" ]]; then
+    echo "go.mod and/or go.sum are not tidy; please run go mod tidy"
+    echo "go.mod diff:"
+    echo "${DIFF_MOD}"
+    echo "go.sum diff:"
+    echo "${DIFF_SUM}"
+    exit 1
+fi

Makefile

@@ -115,7 +115,7 @@ bootstrap: $(RESULTSDIR) bootstrap-go bootstrap-tools ## Download and install al
 	$(call title,Bootstrapping dependencies)
 
 .PHONY: static-analysis
-static-analysis: lint check-licenses
+static-analysis: lint check-go-mod-tidy check-licenses
 
 .PHONY: lint
 lint: ## Run gofmt + golangci lint checks
@@ -136,11 +136,15 @@ lint-fix: ## Auto-format all source code + run golangci lint fixers
 	$(call title,Running lint fixers)
 	gofmt -w -s .
 	$(LINTCMD) --fix
+	go mod tidy
 
 .PHONY: check-licenses
 check-licenses:
 	$(TEMPDIR)/bouncer check
 
+check-go-mod-tidy:
+	@ .github/scripts/go-mod-tidy-check.sh && echo "go.mod and go.sum are tidy!"
+
 .PHONY: validate-cyclonedx-schema
 validate-cyclonedx-schema:
 	cd schema/cyclonedx && make
@@ -346,4 +350,4 @@ show-test-image-cache: ## Show all docker and image tar cache
 .PHONY: show-test-snapshots
 show-test-snapshots: ## Show all test snapshots
 	$(call title,Test snapshots)
-	@find . -type f -wholename "**/test-fixtures/snapshot/*" | sort
+	@find . -type f -wholename "**/test-fixtures/snapshot/*" | sort

cmd/packages.go

@@ -137,6 +137,11 @@ func setPackageFlags(flags *pflag.FlagSet) {
 		"overwrite-existing-image", false,
 		"overwrite an existing image during the upload to Anchore Enterprise",
 	)
+
+	flags.Uint(
+		"import-timeout", 30,
+		"set a timeout duration (in seconds) for the upload to Anchore Enterprise",
+	)
 }
 
 func bindPackagesConfigOptions(flags *pflag.FlagSet) error {
@@ -172,6 +177,10 @@ func bindPackagesConfigOptions(flags *pflag.FlagSet) error {
 		return err
 	}
 
+	if err := viper.BindPFlag("anchore.import-timeout", flags.Lookup("import-timeout")); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -262,6 +271,7 @@ func runPackageSbomUpload(src source.Source, s source.Metadata, catalog *pkg.Cat
 		Dockerfile:              dockerfileContents,
 		OverwriteExistingUpload: appConfig.Anchore.OverwriteExistingImage,
 		Scope:                   scope,
+		Timeout:                 appConfig.Anchore.ImportTimeout,
 	}
 
 	if err := c.Import(context.Background(), importCfg); err != nil {

go.mod

@@ -7,10 +7,10 @@ require (
 	github.com/adrg/xdg v0.2.1
 	github.com/alecthomas/jsonschema v0.0.0-20210301060011-54c507b6f074
 	github.com/anchore/client-go v0.0.0-20210222170800-9c70f9b80bcf
-	github.com/anchore/go-rpmdb v0.0.0-20210415132930-2460011e83c6
+	github.com/anchore/go-rpmdb v0.0.0-20210602151223-1f0f707a2894
 	github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
 	github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
-	github.com/anchore/stereoscope v0.0.0-20210413221244-d577f30b19e6
+	github.com/anchore/stereoscope v0.0.0-20210524175238-3b7662f3a66f
 	github.com/antihax/optional v1.0.0
 	github.com/bmatcuk/doublestar/v2 v2.0.4
 	github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible

go.sum

@@ -109,14 +109,14 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/anchore/client-go v0.0.0-20210222170800-9c70f9b80bcf h1:DYssiUV1pBmKqzKsm4mqXx8artqC0Q8HgZsVI3lMsAg=
 github.com/anchore/client-go v0.0.0-20210222170800-9c70f9b80bcf/go.mod h1:FaODhIA06mxO1E6R32JE0TL1JWZZkmjRIAd4ULvHUKk=
-github.com/anchore/go-rpmdb v0.0.0-20210415132930-2460011e83c6 h1:wEN3HXc3VuC4wo7Cz27YCpeQ4gaB5ASKwMwM5GdFsew=
-github.com/anchore/go-rpmdb v0.0.0-20210415132930-2460011e83c6/go.mod h1:8jNYOxCJC5kyD/Ct4MbzsDN2hOhRoCAzQcb/7KdYYGw=
+github.com/anchore/go-rpmdb v0.0.0-20210602151223-1f0f707a2894 h1:VvCq7fFNU8dwWkCTGUykm4p64nVaDCYkKrj87x350Sk=
+github.com/anchore/go-rpmdb v0.0.0-20210602151223-1f0f707a2894/go.mod h1:8jNYOxCJC5kyD/Ct4MbzsDN2hOhRoCAzQcb/7KdYYGw=
 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0vW0nnNKJfJieyH/TZ9UYAnTZs5/gHTdAe8=
 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ=
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods=
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
-github.com/anchore/stereoscope v0.0.0-20210413221244-d577f30b19e6 h1:g9ZS2V/T0wxseccI4t1hQTqWBek5DVOQZOzzdWBjwnU=
-github.com/anchore/stereoscope v0.0.0-20210413221244-d577f30b19e6/go.mod h1:vhh1M99rfWx5ejMvz1lkQiFZUrC5wu32V12R4JXH+ZI=
+github.com/anchore/stereoscope v0.0.0-20210524175238-3b7662f3a66f h1:bFadyOLOkzME3BrZFZ5m8cf/b2hsn3aMSS9s+SKubRk=
+github.com/anchore/stereoscope v0.0.0-20210524175238-3b7662f3a66f/go.mod h1:vhh1M99rfWx5ejMvz1lkQiFZUrC5wu32V12R4JXH+ZI=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=

internal/anchore/import.go

@@ -27,6 +27,7 @@ type ImportConfig struct {
 	Dockerfile              []byte
 	OverwriteExistingUpload bool
 	Scope                   source.Scope
+	Timeout                 uint
 }
 
 func importProgress(source string) (*progress.Stage, *progress.Manual) {
@@ -54,7 +55,8 @@ func importProgress(source string) (*progress.Stage, *progress.Manual) {
 func (c *Client) Import(ctx context.Context, cfg ImportConfig) error {
 	stage, prog := importProgress(c.config.BaseURL)
 
-	ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Second*30)
+	timeout := time.Duration(cfg.Timeout) * time.Second
+	ctxWithTimeout, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
 
 	authedCtx := c.newRequestContext(ctxWithTimeout)

internal/config/anchore.go

@@ -12,6 +12,8 @@ type anchore struct {
 	Password               string `yaml:"-" json:"-" mapstructure:"password"`                                                               // -p , password to authenticate upload
 	Dockerfile             string `yaml:"dockerfile" json:"dockerfile" mapstructure:"dockerfile"`                                           // -d , dockerfile to attach for upload
 	OverwriteExistingImage bool   `yaml:"overwrite-existing-image" json:"overwrite-existing-image" mapstructure:"overwrite-existing-image"` // --overwrite-existing-image , if any of the SBOM components have already been uploaded this flag will ensure they are overwritten with the current upload
+	ImportTimeout          uint   `yaml:"import-timeout" json:"import-timeout" mapstructure:"import-timeout"`                               // --import-timeout
+	// , customize the number of seconds within which the SBOM import must be completed or canceled
 }
 
 func (cfg anchore) loadDefaultValues(v *viper.Viper) {

internal/config/registry.go

@@ -37,7 +37,7 @@ func (cfg *registry) parseConfigValues() error {
 		os.Getenv("SYFT_REGISTRY_AUTH_PASSWORD"),
 		os.Getenv("SYFT_REGISTRY_AUTH_TOKEN")
 
-	if hasNonEmptyCredentials(authority, username, password, token) {
+	if hasNonEmptyCredentials(username, password, token) {
 		// note: we prepend the credentials such that the environment variables take precedence over on-disk configuration.
 		cfg.Auth = append([]RegistryCredentials{
 			{
@@ -51,8 +51,8 @@ func (cfg *registry) parseConfigValues() error {
 	return nil
 }
 
-func hasNonEmptyCredentials(authority, username, password, token string) bool {
-	return authority != "" && password != "" && username != "" || authority != "" && token != ""
+func hasNonEmptyCredentials(username, password, token string) bool {
+	return password != "" && username != "" || token != ""
 }
 
 func (cfg *registry) ToOptions() *image.RegistryOptions {

internal/config/registry_test.go

@@ -9,51 +9,48 @@ import (
 
 func TestHasNonEmptyCredentials(t *testing.T) {
 	tests := []struct {
-		auth, username, password, token string
-		expected                        bool
+		username, password, token string
+		expected                  bool
 	}{
+
 		{
-			"", "", "", "",
+			"", "", "",
 			false,
 		},
 		{
-			"auth", "", "", "",
+			"user", "", "",
 			false,
 		},
 		{
-			"auth", "user", "", "",
+			"", "pass", "",
 			false,
 		},
 		{
-			"auth", "", "pass", "",
-			false,
-		},
-		{
-			"auth", "", "pass", "tok",
+			"", "pass", "tok",
 			true,
 		},
 		{
-			"auth", "user", "", "tok",
+			"user", "", "tok",
 			true,
 		},
 		{
-			"auth", "", "", "tok",
+			"", "", "tok",
 			true,
 		},
 		{
-			"auth", "user", "pass", "tok",
+			"user", "pass", "tok",
 			true,
 		},
 
 		{
-			"auth", "user", "pass", "",
+			"user", "pass", "",
 			true,
 		},
 	}
 
 	for _, test := range tests {
 		t.Run(fmt.Sprintf("%+v", test), func(t *testing.T) {
-			assert.Equal(t, test.expected, hasNonEmptyCredentials(test.auth, test.username, test.password, test.token))
+			assert.Equal(t, test.expected, hasNonEmptyCredentials(test.username, test.password, test.token))
 		})
 	}
 }

internal/file/zip_file_helpers_test.go

@@ -1,7 +1,6 @@
 package file
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -9,6 +8,8 @@ import (
 	"path/filepath"
 	"syscall"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 var expectedZipArchiveEntries = []string{
@@ -18,35 +19,21 @@ var expectedZipArchiveEntries = []string{
 	"nested.zip",
 }
 
-// fatalIfError calls the supplied function. If the function returns a non-nil error, t.Fatal(err) is called.
-func fatalIfError(t *testing.T, fn func() error) {
-	t.Helper()
-
-	if fn == nil {
-		return
-	}
-
-	err := fn()
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
 // createZipArchive creates a new ZIP archive file at destinationArchivePath based on the directory found at
 // sourceDirPath.
-func createZipArchive(t *testing.T, sourceDirPath, destinationArchivePath string) error {
+func createZipArchive(t testing.TB, sourceDirPath, destinationArchivePath string) {
 	t.Helper()
 
 	cwd, err := os.Getwd()
 	if err != nil {
-		return fmt.Errorf("unable to get cwd: %+v", err)
+		t.Fatalf("unable to get cwd: %+v", err)
 	}
 
-	cmd := exec.Command("./generate-zip-fixture.sh", destinationArchivePath, path.Base(sourceDirPath))
+	cmd := exec.Command("./generate-zip-fixture-from-source-dir.sh", destinationArchivePath, path.Base(sourceDirPath))
 	cmd.Dir = filepath.Join(cwd, "test-fixtures")
 
 	if err := cmd.Start(); err != nil {
-		return fmt.Errorf("unable to start generate zip fixture script: %+v", err)
+		t.Fatalf("unable to start generate zip fixture script: %+v", err)
 	}
 
 	if err := cmd.Wait(); err != nil {
@@ -59,61 +46,62 @@ func createZipArchive(t *testing.T, sourceDirPath, destinationArchivePath string
 			// an ExitStatus() method with the same signature.
 			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
 				if status.ExitStatus() != 0 {
-					return fmt.Errorf("failed to generate fixture: rc=%d", status.ExitStatus())
+					t.Fatalf("failed to generate fixture: rc=%d", status.ExitStatus())
 				}
 			}
 		} else {
-			return fmt.Errorf("unable to get generate fixture script result: %+v", err)
+			t.Fatalf("unable to get generate fixture script result: %+v", err)
 		}
 	}
 
-	return nil
+}
+
+func assertNoError(t testing.TB, fn func() error) func() {
+	return func() {
+		assert.NoError(t, fn())
+	}
 }
 
 // setupZipFileTest encapsulates common test setup work for zip file tests. It returns a cleanup function,
 // which should be called (typically deferred) by the caller, the path of the created zip archive, and an error,
 // which should trigger a fatal test failure in the consuming test. The returned cleanup function will never be nil
 // (even if there's an error), and it should always be called.
-func setupZipFileTest(t *testing.T, sourceDirPath string) (func() error, string, error) {
+func setupZipFileTest(t testing.TB, sourceDirPath string) string {
 	t.Helper()
 
-	// Keep track of any needed cleanup work as we go
-	var cleanupFns []func() error
-	cleanup := func(fns []func() error) func() error {
-		return func() error {
-			for _, fn := range fns {
-				err := fn()
-				if err != nil {
-					return err
-				}
-			}
-
-			return nil
-		}
-	}
-
 	archivePrefix, err := ioutil.TempFile("", "syft-ziputil-archive-TEST-")
 	if err != nil {
-		return cleanup(cleanupFns), "", fmt.Errorf("unable to create tempfile: %+v", err)
+		t.Fatalf("unable to create tempfile: %+v", err)
 	}
-	cleanupFns = append(cleanupFns, func() error { return os.Remove(archivePrefix.Name()) })
+
+	t.Cleanup(
+		assertNoError(t,
+			func() error {
+				return os.Remove(archivePrefix.Name())
+			},
+		),
+	)
 
 	destinationArchiveFilePath := archivePrefix.Name() + ".zip"
 	t.Logf("archive path: %s", destinationArchiveFilePath)
-	err = createZipArchive(t, sourceDirPath, destinationArchiveFilePath)
-	cleanupFns = append(cleanupFns, func() error { return os.Remove(destinationArchiveFilePath) })
-	if err != nil {
-		return cleanup(cleanupFns), "", err
-	}
+	createZipArchive(t, sourceDirPath, destinationArchiveFilePath)
+
+	t.Cleanup(
+		assertNoError(t,
+			func() error {
+				return os.Remove(destinationArchiveFilePath)
+			},
+		),
+	)
 
 	cwd, err := os.Getwd()
 	if err != nil {
-		return cleanup(cleanupFns), "", fmt.Errorf("unable to get cwd: %+v", err)
+		t.Fatalf("unable to get cwd: %+v", err)
 	}
 
 	t.Logf("running from: %s", cwd)
 
-	return cleanup(cleanupFns), destinationArchiveFilePath, nil
+	return destinationArchiveFilePath
 }
 
 // TODO: Consider moving any non-git asset generation to a task (e.g. make) that's run ahead of running go tests.
@@ -121,11 +109,7 @@ func ensureNestedZipExists(t *testing.T, sourceDirPath string) error {
 	t.Helper()
 
 	nestedArchiveFilePath := path.Join(sourceDirPath, "nested.zip")
-	err := createZipArchive(t, sourceDirPath, nestedArchiveFilePath)
-
-	if err != nil {
-		return fmt.Errorf("unable to create nested archive for test fixture: %+v", err)
-	}
+	createZipArchive(t, sourceDirPath, nestedArchiveFilePath)
 
 	return nil
 }

internal/file/zip_file_manifest.go

@@ -1,23 +1,36 @@
 package file
 
 import (
-	"archive/zip"
 	"fmt"
 	"os"
 	"sort"
 	"strings"
 
 	"github.com/anchore/syft/internal"
-
 	"github.com/anchore/syft/internal/log"
 )
 
 // ZipFileManifest is a collection of paths and their file metadata.
 type ZipFileManifest map[string]os.FileInfo
 
-// newZipManifest creates an empty ZipFileManifest.
-func newZipManifest() ZipFileManifest {
-	return make(ZipFileManifest)
+// NewZipFileManifest creates and returns a new ZipFileManifest populated with path and metadata from the given zip archive path.
+func NewZipFileManifest(archivePath string) (ZipFileManifest, error) {
+	zipReader, err := OpenZip(archivePath)
+	manifest := make(ZipFileManifest)
+	if err != nil {
+		return manifest, fmt.Errorf("unable to open zip archive (%s): %w", archivePath, err)
+	}
+	defer func() {
+		err = zipReader.Close()
+		if err != nil {
+			log.Warnf("unable to close zip archive (%s): %+v", archivePath, err)
+		}
+	}()
+
+	for _, file := range zipReader.Reader.File {
+		manifest.Add(file.Name, file.FileInfo())
+	}
+	return manifest, nil
 }
 
 // Add a new path and it's file metadata to the collection.
@@ -47,26 +60,6 @@ func (z ZipFileManifest) GlobMatch(patterns ...string) []string {
 	return results
 }
 
-// NewZipFileManifest creates and returns a new ZipFileManifest populated with path and metadata from the given zip archive path.
-func NewZipFileManifest(archivePath string) (ZipFileManifest, error) {
-	zipReader, err := zip.OpenReader(archivePath)
-	manifest := newZipManifest()
-	if err != nil {
-		return manifest, fmt.Errorf("unable to open zip archive (%s): %w", archivePath, err)
-	}
-	defer func() {
-		err = zipReader.Close()
-		if err != nil {
-			log.Errorf("unable to close zip archive (%s): %+v", archivePath, err)
-		}
-	}()
-
-	for _, file := range zipReader.Reader.File {
-		manifest.Add(file.Name, file.FileInfo())
-	}
-	return manifest, nil
-}
-
 // normalizeZipEntryName takes the given path entry and ensures it is prefixed with "/".
 func normalizeZipEntryName(entry string) string {
 	if !strings.HasPrefix(entry, "/") {

internal/file/zip_file_manifest_test.go

@@ -19,11 +19,7 @@ func TestNewZipFileManifest(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cleanup, archiveFilePath, err := setupZipFileTest(t, sourceDirPath)
-	defer fatalIfError(t, cleanup)
-	if err != nil {
-		t.Fatal(err)
-	}
+	archiveFilePath := setupZipFileTest(t, sourceDirPath)
 
 	actual, err := NewZipFileManifest(archiveFilePath)
 	if err != nil {
@@ -63,12 +59,7 @@ func TestZipFileManifest_GlobMatch(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cleanup, archiveFilePath, err := setupZipFileTest(t, sourceDirPath)
-	//goland:noinspection GoNilness
-	defer fatalIfError(t, cleanup)
-	if err != nil {
-		t.Fatal(err)
-	}
+	archiveFilePath := setupZipFileTest(t, sourceDirPath)
 
 	z, err := NewZipFileManifest(archiveFilePath)
 	if err != nil {

internal/file/zip_file_traversal.go

@@ -47,7 +47,7 @@ func newZipTraverseRequest(paths ...string) zipTraversalRequest {
 func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths ...string) error {
 	request := newZipTraverseRequest(paths...)
 
-	zipReader, err := zip.OpenReader(archivePath)
+	zipReader, err := OpenZip(archivePath)
 	if err != nil {
 		return fmt.Errorf("unable to open zip archive (%s): %w", archivePath, err)
 	}

internal/file/zip_file_traversal_test.go

@@ -44,14 +44,12 @@ func TestUnzipToDir(t *testing.T) {
 
 	goldenRootDir := filepath.Join(cwd, "test-fixtures")
 	sourceDirPath := path.Join(goldenRootDir, "zip-source")
-	cleanup, archiveFilePath, err := setupZipFileTest(t, sourceDirPath)
-	defer fatalIfError(t, cleanup)
-	if err != nil {
-		t.Fatal(err)
-	}
+	archiveFilePath := setupZipFileTest(t, sourceDirPath)
 
 	unzipDestinationDir, err := ioutil.TempDir("", "syft-ziputil-contents-TEST-")
-	defer os.RemoveAll(unzipDestinationDir)
+	t.Cleanup(assertNoError(t, func() error {
+		return os.RemoveAll(unzipDestinationDir)
+	}))
 	if err != nil {
 		t.Fatalf("unable to create tempdir: %+v", err)
 	}
@@ -127,41 +125,119 @@ func TestUnzipToDir(t *testing.T) {
 }
 
 func TestContentsFromZip(t *testing.T) {
-	archivePrefix, err := ioutil.TempFile("", "syft-ziputil-archive-TEST-")
+	tests := []struct {
+		name        string
+		archivePrep func(tb testing.TB) string
+	}{
+		{
+			name:        "standard, non-nested zip",
+			archivePrep: prepZipSourceFixture,
+		},
+		{
+			name:        "zip with prepended bytes",
+			archivePrep: prependZipSourceFixtureWithString(t, "junk at the beginning of the file..."),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			archivePath := test.archivePrep(t)
+			expected := zipSourceFixtureExpectedContents()
+
+			var paths []string
+			for p := range expected {
+				paths = append(paths, p)
+			}
+
+			actual, err := ContentsFromZip(archivePath, paths...)
+			if err != nil {
+				t.Fatalf("unable to extract from unzip archive: %+v", err)
+			}
+
+			assertZipSourceFixtureContents(t, actual, expected)
+		})
+	}
+}
+
+func prependZipSourceFixtureWithString(tb testing.TB, value string) func(tb testing.TB) string {
+	if len(value) == 0 {
+		tb.Fatalf("no bytes given to prefix")
+	}
+	return func(t testing.TB) string {
+		archivePath := prepZipSourceFixture(t)
+
+		// create a temp file
+		tmpFile, err := ioutil.TempFile("", "syft-ziputil-prependZipSourceFixtureWithString-")
+		if err != nil {
+			t.Fatalf("unable to create tempfile: %+v", err)
+		}
+		defer tmpFile.Close()
+
+		// write value to the temp file
+		if _, err := tmpFile.WriteString(value); err != nil {
+			t.Fatalf("unable to write to tempfile: %+v", err)
+		}
+
+		// open the original archive
+		sourceFile, err := os.Open(archivePath)
+		if err != nil {
+			t.Fatalf("unable to read source file: %+v", err)
+		}
+
+		// copy all contents from the archive to the temp file
+		if _, err := io.Copy(tmpFile, sourceFile); err != nil {
+			t.Fatalf("unable to copy source to dest: %+v", err)
+		}
+
+		sourceFile.Close()
+
+		// remove the original archive and replace it with the temp file
+		if err := os.Remove(archivePath); err != nil {
+			t.Fatalf("unable to remove original source archive (%q): %+v", archivePath, err)
+		}
+
+		if err := os.Rename(tmpFile.Name(), archivePath); err != nil {
+			t.Fatalf("unable to move new archive to old path (%q): %+v", tmpFile.Name(), err)
+		}
+
+		return archivePath
+	}
+}
+
+func prepZipSourceFixture(t testing.TB) string {
+	t.Helper()
+	archivePrefix, err := ioutil.TempFile("", "syft-ziputil-prepZipSourceFixture-")
 	if err != nil {
 		t.Fatalf("unable to create tempfile: %+v", err)
 	}
-	defer os.Remove(archivePrefix.Name())
+
+	t.Cleanup(func() {
+		assert.NoError(t, os.Remove(archivePrefix.Name()))
+	})
+
 	// the zip utility will add ".zip" to the end of the given name
 	archivePath := archivePrefix.Name() + ".zip"
-	defer os.Remove(archivePath)
+
+	t.Cleanup(func() {
+		assert.NoError(t, os.Remove(archivePath))
+	})
+
 	t.Logf("archive path: %s", archivePath)
 
-	err = createZipArchive(t, "zip-source", archivePrefix.Name())
-	if err != nil {
-		t.Fatal(err)
-	}
+	createZipArchive(t, "zip-source", archivePrefix.Name())
 
-	cwd, err := os.Getwd()
-	if err != nil {
-		t.Errorf("unable to get cwd: %+v", err)
-	}
+	return archivePath
+}
 
-	t.Logf("running from: %s", cwd)
-
-	aFilePath := filepath.Join("some-dir", "a-file.txt")
-	bFilePath := filepath.Join("b-file.txt")
-
-	expected := map[string]string{
-		aFilePath: "A file! nice!",
-		bFilePath: "B file...",
-	}
-
-	actual, err := ContentsFromZip(archivePath, aFilePath, bFilePath)
-	if err != nil {
-		t.Fatalf("unable to extract from unzip archive: %+v", err)
+func zipSourceFixtureExpectedContents() map[string]string {
+	return map[string]string{
+		filepath.Join("some-dir", "a-file.txt"): "A file! nice!",
+		filepath.Join("b-file.txt"):             "B file...",
 	}
+}
 
+func assertZipSourceFixtureContents(t testing.TB, actual map[string]string, expected map[string]string) {
+	t.Helper()
 	diffs := deep.Equal(actual, expected)
 	if len(diffs) > 0 {
 		for _, d := range diffs {

internal/file/zip_read_closer.go

@@ -0,0 +1,149 @@
+package file
+
+import (
+	"archive/zip"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"os"
+)
+
+// directoryEndLen, readByf, directoryEnd, and findSignatureInBlock were copied from the golang stdlib, specifically:
+// - https://github.com/golang/go/blob/go1.16.4/src/archive/zip/struct.go
+// - https://github.com/golang/go/blob/go1.16.4/src/archive/zip/reader.go
+// findArchiveStartOffset is derived from the same stdlib utils, specifically the readDirectoryEnd function.
+
+const directoryEndLen = 22
+
+// ZipReadCloser is a drop-in replacement for zip.ReadCloser (from zip.OpenReader) that additionally considers zips
+// that have bytes prefixed to the front of the archive (common with self-extracting jars).
+type ZipReadCloser struct {
+	*zip.Reader
+	io.Closer
+}
+
+// OpenZip provides a ZipReadCloser for the given filepath.
+func OpenZip(filepath string) (*ZipReadCloser, error) {
+	f, err := os.Open(filepath)
+	if err != nil {
+		return nil, err
+	}
+	fi, err := f.Stat()
+	if err != nil {
+		f.Close()
+		return nil, err
+	}
+
+	// some archives may have bytes prepended to the front of the archive, such as with self executing JARs. We first
+	// need to find the start of the archive and keep track of this offset.
+	offset, err := findArchiveStartOffset(f, fi.Size())
+	if err != nil {
+		return nil, fmt.Errorf("cannot find beginning of zip archive=%q : %w", filepath, err)
+	}
+
+	if _, err := f.Seek(0, io.SeekStart); err != nil {
+		return nil, fmt.Errorf("unable to seek to beginning of archive: %w", err)
+	}
+
+	size := fi.Size() - int64(offset)
+
+	r, err := zip.NewReader(io.NewSectionReader(f, int64(offset), size), size)
+	if err != nil {
+		return nil, fmt.Errorf("unable to open ZipReadCloser @ %q: %w", filepath, err)
+	}
+
+	return &ZipReadCloser{
+		Reader: r,
+		Closer: f,
+	}, nil
+}
+
+type readBuf []byte
+
+func (b *readBuf) uint16() uint16 {
+	v := binary.LittleEndian.Uint16(*b)
+	*b = (*b)[2:]
+	return v
+}
+
+func (b *readBuf) uint32() uint32 {
+	v := binary.LittleEndian.Uint32(*b)
+	*b = (*b)[4:]
+	return v
+}
+
+type directoryEnd struct {
+	diskNbr            uint32 // unused
+	dirDiskNbr         uint32 // unused
+	dirRecordsThisDisk uint64 // unused
+	directoryRecords   uint64
+	directorySize      uint64
+	directoryOffset    uint64 // relative to file
+}
+
+// note: this is derived from readDirectoryEnd within the archive/zip package
+func findArchiveStartOffset(r io.ReaderAt, size int64) (startOfArchive uint64, err error) {
+	// look for directoryEndSignature in the last 1k, then in the last 65k
+	var buf []byte
+	var directoryEndOffset int64
+	for i, bLen := range []int64{1024, 65 * 1024} {
+		if bLen > size {
+			bLen = size
+		}
+		buf = make([]byte, int(bLen))
+		if _, err := r.ReadAt(buf, size-bLen); err != nil && err != io.EOF {
+			return 0, err
+		}
+		if p := findSignatureInBlock(buf); p >= 0 {
+			buf = buf[p:]
+			directoryEndOffset = size - bLen + int64(p)
+			break
+		}
+		if i == 1 || bLen == size {
+			return 0, zip.ErrFormat
+		}
+	}
+
+	if buf == nil {
+		// we were unable to find the directoryEndSignature block
+		return 0, zip.ErrFormat
+	}
+
+	// read header into struct
+	b := readBuf(buf[4:]) // skip signature
+	d := &directoryEnd{
+		diskNbr:            uint32(b.uint16()),
+		dirDiskNbr:         uint32(b.uint16()),
+		dirRecordsThisDisk: uint64(b.uint16()),
+		directoryRecords:   uint64(b.uint16()),
+		directorySize:      uint64(b.uint32()),
+		directoryOffset:    uint64(b.uint32()),
+	}
+	// Calculate where the zip data actually begins
+	startOfArchive = uint64(directoryEndOffset) - d.directorySize - d.directoryOffset
+
+	// These values mean that the file can be a zip64 file
+	if d.directoryRecords == 0xffff || d.directorySize == 0xffff || d.directoryOffset == 0xffffffff {
+		startOfArchive = 0 // Prefixed data not supported
+	}
+
+	// Make sure directoryOffset points to somewhere in our file.
+	if o := int64(d.directoryOffset); o < 0 || o >= size {
+		return 0, zip.ErrFormat
+	}
+	return startOfArchive, nil
+}
+
+func findSignatureInBlock(b []byte) int {
+	for i := len(b) - directoryEndLen; i >= 0; i-- {
+		// defined from directoryEndSignature
+		if b[i] == 'P' && b[i+1] == 'K' && b[i+2] == 0x05 && b[i+3] == 0x06 {
+			// n is length of comment
+			n := int(b[i+directoryEndLen-2]) | int(b[i+directoryEndLen-1])<<8
+			if n+directoryEndLen+i <= len(b) {
+				return i
+			}
+		}
+	}
+	return -1
+}

internal/file/zip_read_closer_test.go

@@ -0,0 +1,47 @@
+package file
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFindArchiveStartOffset(t *testing.T) {
+	tests := []struct {
+		name        string
+		archivePrep func(tb testing.TB) string
+		expected    uint64
+	}{
+		{
+			name:        "standard, non-nested zip",
+			archivePrep: prepZipSourceFixture,
+			expected:    0,
+		},
+		{
+			name:        "zip with prepended bytes",
+			archivePrep: prependZipSourceFixtureWithString(t, "junk at the beginning of the file..."),
+			expected:    36,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			archivePath := test.archivePrep(t)
+			f, err := os.Open(archivePath)
+			if err != nil {
+				t.Fatalf("could not open archive %q: %+v", archivePath, err)
+			}
+			fi, err := os.Stat(f.Name())
+			if err != nil {
+				t.Fatalf("unable to stat archive: %+v", err)
+			}
+
+			actual, err := findArchiveStartOffset(f, fi.Size())
+			if err != nil {
+				t.Fatalf("unable to find offset: %+v", err)
+			}
+			assert.Equal(t, test.expected, actual)
+		})
+	}
+}

internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden

@@ -174,7 +174,8 @@
     "host": "",
     "path": "",
     "dockerfile": "",
-    "overwrite-existing-image": false
+    "overwrite-existing-image": false,
+    "import-timeout": 0
    },
    "dev": {
     "profile-cpu": false,

syft/pkg/cataloger/java/test-fixtures/java-builds/example-sb-app/pom.xml

@@ -52,10 +52,12 @@
 			<plugin>
 				<groupId>org.springframework.boot</groupId>
 				<artifactId>spring-boot-maven-plugin</artifactId>
-				<version>1.0.2.RELEASE</version>
+				<version>2.1.5.RELEASE</version>
 				<configuration>
 					<mainClass>${start-class}</mainClass>
 					<layout>ZIP</layout>
+					<!-- this gives us a specific build example with a self-executing jar (jar with a shell script prefixed to the archive) -->
+					<executable>true</executable>
 				</configuration>
 				<executions>
 					<execution>

syft/pkg/cataloger/package_url_test.go

@@ -30,30 +30,6 @@ func TestPackageURL(t *testing.T) {
 			},
 			expected: "pkg:pypi/name@v0.1.0",
 		},
-		{
-			pkg: pkg.Package{
-				Name:    "name",
-				Version: "v0.1.0",
-				Type:    pkg.PythonPkg,
-			},
-			expected: "pkg:pypi/name@v0.1.0",
-		},
-		{
-			pkg: pkg.Package{
-				Name:    "name",
-				Version: "v0.1.0",
-				Type:    pkg.PythonPkg,
-			},
-			expected: "pkg:pypi/name@v0.1.0",
-		},
-		{
-			pkg: pkg.Package{
-				Name:    "name",
-				Version: "v0.1.0",
-				Type:    pkg.PythonPkg,
-			},
-			expected: "pkg:pypi/name@v0.1.0",
-		},
 		{
 			pkg: pkg.Package{
 				Name:    "name",
@@ -96,13 +72,31 @@ func TestPackageURL(t *testing.T) {
 				Type:    pkg.RpmPkg,
 				Metadata: pkg.RpmdbMetadata{
 					Name:    "name",
-					Version: "v0.1.0",
-					Epoch:   2,
+					Version: "0.1.0",
+					Epoch:   intRef(2),
 					Arch:    "amd64",
 					Release: "3",
 				},
 			},
-			expected: "pkg:rpm/centos/name@2:v0.1.0-3?arch=amd64",
+			expected: "pkg:rpm/centos/name@0.1.0-3?arch=amd64&epoch=2",
+		},
+		{
+			distro: &distro.Distro{
+				Type: distro.CentOS,
+			},
+			pkg: pkg.Package{
+				Name:    "bad-name",
+				Version: "bad-v0.1.0",
+				Type:    pkg.RpmPkg,
+				Metadata: pkg.RpmdbMetadata{
+					Name:    "name",
+					Version: "0.1.0",
+					Epoch:   nil,
+					Arch:    "amd64",
+					Release: "3",
+				},
+			},
+			expected: "pkg:rpm/centos/name@0.1.0-3?arch=amd64",
 		},
 		{
 			distro: &distro.Distro{
@@ -136,3 +130,7 @@ func TestPackageURL(t *testing.T) {
 		})
 	}
 }
+
+func intRef(i int) *int {
+	return &i
+}

syft/pkg/cataloger/rpmdb/parse_rpmdb.go

@@ -47,25 +47,27 @@ func parseRpmDB(resolver source.FilePathResolver, dbLocation source.Location, re
 	allPkgs := make([]pkg.Package, 0)
 
 	for _, entry := range pkgList {
+		metadata := pkg.RpmdbMetadata{
+			Name:      entry.Name,
+			Version:   entry.Version,
+			Epoch:     entry.Epoch,
+			Arch:      entry.Arch,
+			Release:   entry.Release,
+			SourceRpm: entry.SourceRpm,
+			Vendor:    entry.Vendor,
+			License:   entry.License,
+			Size:      entry.Size,
+			Files:     extractRpmdbFileRecords(resolver, entry),
+		}
+
 		p := pkg.Package{
 			Name:         entry.Name,
-			Version:      fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does, instead of fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch)
+			Version:      toELVersion(metadata),
 			Locations:    []source.Location{dbLocation},
 			FoundBy:      catalogerName,
 			Type:         pkg.RpmPkg,
 			MetadataType: pkg.RpmdbMetadataType,
-			Metadata: pkg.RpmdbMetadata{
-				Name:      entry.Name,
-				Version:   entry.Version,
-				Epoch:     entry.Epoch,
-				Arch:      entry.Arch,
-				Release:   entry.Release,
-				SourceRpm: entry.SourceRpm,
-				Vendor:    entry.Vendor,
-				License:   entry.License,
-				Size:      entry.Size,
-				Files:     extractRpmdbFileRecords(resolver, entry),
-			},
+			Metadata:     metadata,
 		}
 
 		allPkgs = append(allPkgs, p)
@@ -74,6 +76,19 @@ func parseRpmDB(resolver source.FilePathResolver, dbLocation source.Location, re
 	return allPkgs, nil
 }
 
+// The RPM naming scheme is [name]-[version]-[release]-[arch], where version is implicitly expands to [epoch]:[version].
+// RPM version comparison depends on comparing at least the version and release fields together as a subset of the
+// naming scheme. This toELVersion function takes a RPM DB package information and converts it into a minimally comparable
+// version string, containing epoch (optional), version, and release information. Epoch is an optional field and can be
+// assumed to be 0 when not provided for comparison purposes, however, if the underlying RPM DB entry does not have
+// an epoch specified it would be slightly disingenuous to display a value of 0.
+func toELVersion(metadata pkg.RpmdbMetadata) string {
+	if metadata.Epoch != nil {
+		return fmt.Sprintf("%d:%s-%s", *metadata.Epoch, metadata.Version, metadata.Release)
+	}
+	return fmt.Sprintf("%s-%s", metadata.Version, metadata.Release)
+}
+
 func extractRpmdbFileRecords(resolver source.FilePathResolver, entry *rpmdb.PackageInfo) []pkg.RpmdbFileRecord {
 	var records = make([]pkg.RpmdbFileRecord, 0)
 

syft/pkg/cataloger/rpmdb/parse_rpmdb_test.go

@@ -5,6 +5,8 @@ import (
 	"os"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/anchore/syft/syft/file"
 
 	"github.com/anchore/syft/syft/source"
@@ -70,7 +72,7 @@ func TestParseRpmDB(t *testing.T) {
 					MetadataType: pkg.RpmdbMetadataType,
 					Metadata: pkg.RpmdbMetadata{
 						Name:      "dive",
-						Epoch:     0,
+						Epoch:     nil,
 						Arch:      "x86_64",
 						Release:   "1",
 						Version:   "0.9.2",
@@ -97,7 +99,7 @@ func TestParseRpmDB(t *testing.T) {
 					MetadataType: pkg.RpmdbMetadataType,
 					Metadata: pkg.RpmdbMetadata{
 						Name:      "dive",
-						Epoch:     0,
+						Epoch:     nil,
 						Arch:      "x86_64",
 						Release:   "1",
 						Version:   "0.9.2",
@@ -157,3 +159,51 @@ func TestParseRpmDB(t *testing.T) {
 	}
 
 }
+
+func TestToElVersion(t *testing.T) {
+	tests := []struct {
+		name     string
+		entry    pkg.RpmdbMetadata
+		expected string
+	}{
+		{
+			name: "no epoch",
+			entry: pkg.RpmdbMetadata{
+				Version: "1.2.3-4",
+				Release: "el7",
+				Arch:    "x86-64",
+			},
+			expected: "1.2.3-4-el7",
+		},
+		{
+			name: "with 0 epoch",
+			entry: pkg.RpmdbMetadata{
+				Version: "1.2.3-4",
+				Release: "el7",
+				Arch:    "x86-64",
+				Epoch:   intRef(0),
+			},
+			expected: "0:1.2.3-4-el7",
+		},
+		{
+			name: "with non-zero epoch",
+			entry: pkg.RpmdbMetadata{
+				Version: "1.2.3-4",
+				Release: "el7",
+				Arch:    "x86-64",
+				Epoch:   intRef(12),
+			},
+			expected: "12:1.2.3-4-el7",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			assert.Equal(t, test.expected, toELVersion(test.entry))
+		})
+	}
+}
+
+func intRef(i int) *int {
+	return &i
+}

syft/pkg/metadata.go

@@ -16,3 +16,15 @@ const (
 	RustCargoPackageMetadataType MetadataType = "RustCargoPackageMetadata"
 	KbPackageMetadataType        MetadataType = "KbPackageMetadata"
 )
+
+var AllMetadataTypes = []MetadataType{
+	ApkMetadataType,
+	DpkgMetadataType,
+	GemMetadataType,
+	JavaMetadataType,
+	NpmPackageJSONMetadataType,
+	RpmdbMetadataType,
+	PythonPackageMetadataType,
+	RustCargoPackageMetadataType,
+	KbPackageMetadataType,
+}

syft/pkg/rpmdb_metadata.go

@@ -3,6 +3,7 @@ package pkg
 import (
 	"fmt"
 	"sort"
+	"strconv"
 
 	"github.com/anchore/syft/syft/file"
 
@@ -20,7 +21,7 @@ var _ fileOwner = (*RpmdbMetadata)(nil)
 type RpmdbMetadata struct {
 	Name      string            `json:"name"`
 	Version   string            `json:"version"`
-	Epoch     int               `json:"epoch"`
+	Epoch     *int              `json:"epoch"`
 	Arch      string            `json:"architecture"`
 	Release   string            `json:"release"`
 	SourceRpm string            `json:"sourceRpm"`
@@ -50,17 +51,30 @@ func (m RpmdbMetadata) PackageURL(d *distro.Distro) string {
 		return ""
 	}
 
+	qualifiers := packageurl.Qualifiers{
+		{
+			Key:   "arch",
+			Value: m.Arch,
+		},
+	}
+
+	if m.Epoch != nil {
+		qualifiers = append(qualifiers,
+			packageurl.Qualifier{
+				Key:   "epoch",
+				Value: strconv.Itoa(*m.Epoch),
+			},
+		)
+	}
+
 	pURL := packageurl.NewPackageURL(
 		packageurl.TypeRPM,
 		d.Type.String(),
 		m.Name,
-		fmt.Sprintf("%d:%s-%s", m.Epoch, m.Version, m.Release),
-		packageurl.Qualifiers{
-			{
-				Key:   "arch",
-				Value: m.Arch,
-			},
-		},
+		// for purl the epoch is a qualifier, not part of the version
+		// see https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst under the RPM section
+		fmt.Sprintf("%s-%s", m.Version, m.Release),
+		qualifiers,
 		"")
 	return pURL.ToString()
 }

syft/pkg/rpmdb_metadata_test.go

@@ -25,9 +25,9 @@ func TestRpmMetadata_pURL(t *testing.T) {
 				Version: "v",
 				Arch:    "a",
 				Release: "r",
-				Epoch:   1,
+				Epoch:   intRef(1),
 			},
-			expected: "pkg:rpm/centos/p@1:v-r?arch=a",
+			expected: "pkg:rpm/centos/p@v-r?arch=a&epoch=1",
 		},
 		{
 			distro: distro.Distro{
@@ -38,9 +38,9 @@ func TestRpmMetadata_pURL(t *testing.T) {
 				Version: "v",
 				Arch:    "a",
 				Release: "r",
-				Epoch:   1,
+				Epoch:   nil,
 			},
-			expected: "pkg:rpm/redhat/p@1:v-r?arch=a",
+			expected: "pkg:rpm/redhat/p@v-r?arch=a",
 		},
 	}
 
@@ -97,3 +97,7 @@ func TestRpmMetadata_fileOwner(t *testing.T) {
 		})
 	}
 }
+
+func intRef(i int) *int {
+	return &i
+}

test/cli/packages_cmd_test.go

@@ -165,7 +165,7 @@ func TestRegistryAuth(t *testing.T) {
 			assertions: []traitAssertion{
 				assertInOutput("source=OciRegistry"),
 				assertInOutput("localhost:5000/something:latest"),
-				assertInOutput(`using registry credentials for "localhost:5000"`),
+				assertInOutput(`using basic auth for registry "localhost:5000"`),
 			},
 		},
 		{
@@ -178,7 +178,7 @@ func TestRegistryAuth(t *testing.T) {
 			assertions: []traitAssertion{
 				assertInOutput("source=OciRegistry"),
 				assertInOutput("localhost:5000/something:latest"),
-				assertInOutput(`using registry token for "localhost:5000"`),
+				assertInOutput(`using token for registry "localhost:5000"`),
 			},
 		},
 		{