diff --git a/.gitignore b/.gitignore index 157a35422..f3fb01c94 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ coverage.txt # Output of the go coverage tool, specifically when used with LiteIDE *.out + +# macOS Finder metadata +.DS_STORE diff --git a/internal/file/test-fixtures/generate-zip-fixture.sh b/internal/file/test-fixtures/generate-zip-fixture.sh index 3c6d829e7..9cba4e09c 100755 --- a/internal/file/test-fixtures/generate-zip-fixture.sh +++ b/internal/file/test-fixtures/generate-zip-fixture.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash set -eux -zip -r "$1" zip-source \ No newline at end of file +# $1 —— absolute path to destination file, should end with .zip, ideally +# $2 —— absolute path to directory from which to add entries to the archive + +pushd "$2" && find . -print | zip "$1" -@ && popd diff --git a/internal/file/zip_file_helpers_test.go b/internal/file/zip_file_helpers_test.go new file mode 100644 index 000000000..78855a212 --- /dev/null +++ b/internal/file/zip_file_helpers_test.go @@ -0,0 +1,131 @@ +package file + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "syscall" + "testing" +) + +var expectedZipArchiveEntries = []string{ + "some-dir" + string(os.PathSeparator), + filepath.Join("some-dir", "a-file.txt"), + "b-file.txt", + "nested.zip", +} + +// fatalIfError calls the supplied function. If the function returns a non-nil error, t.Fatal(err) is called. +func fatalIfError(t *testing.T, fn func() error) { + t.Helper() + + if fn == nil { + return + } + + err := fn() + if err != nil { + t.Fatal(err) + } +} + +// createZipArchive creates a new ZIP archive file at destinationArchivePath based on the directory found at +// sourceDirPath. +func createZipArchive(t *testing.T, sourceDirPath, destinationArchivePath string) error { + t.Helper() + + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("unable to get cwd: %+v", err) + } + + cmd := exec.Command("./generate-zip-fixture.sh", destinationArchivePath, path.Base(sourceDirPath)) + cmd.Dir = filepath.Join(cwd, "test-fixtures") + + if err := cmd.Start(); err != nil { + return fmt.Errorf("unable to start generate zip fixture script: %+v", err) + } + + if err := cmd.Wait(); err != nil { + if exiterr, ok := err.(*exec.ExitError); ok { + // The program has exited with an exit code != 0 + + // This works on both Unix and Windows. Although package + // syscall is generally platform dependent, WaitStatus is + // defined for both Unix and Windows and in both cases has + // an ExitStatus() method with the same signature. + if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { + if status.ExitStatus() != 0 { + return fmt.Errorf("failed to generate fixture: rc=%d", status.ExitStatus()) + } + } + } else { + return fmt.Errorf("unable to get generate fixture script result: %+v", err) + } + } + + return nil +} + +// setupZipFileTest encapsulates common test setup work for zip file tests. It returns a cleanup function, +// which should be called (typically deferred) by the caller, the path of the created zip archive, and an error, +// which should trigger a fatal test failure in the consuming test. The returned cleanup function will never be nil +// (even if there's an error), and it should always be called. +func setupZipFileTest(t *testing.T, sourceDirPath string) (func() error, string, error) { + t.Helper() + + // Keep track of any needed cleanup work as we go + var cleanupFns []func() error + cleanup := func(fns []func() error) func() error { + return func() error { + for _, fn := range fns { + err := fn() + if err != nil { + return err + } + } + + return nil + } + } + + archivePrefix, err := ioutil.TempFile("", "syft-ziputil-archive-TEST-") + if err != nil { + return cleanup(cleanupFns), "", fmt.Errorf("unable to create tempfile: %+v", err) + } + cleanupFns = append(cleanupFns, func() error { return os.Remove(archivePrefix.Name()) }) + + destinationArchiveFilePath := archivePrefix.Name() + ".zip" + t.Logf("archive path: %s", destinationArchiveFilePath) + err = createZipArchive(t, sourceDirPath, destinationArchiveFilePath) + cleanupFns = append(cleanupFns, func() error { return os.Remove(destinationArchiveFilePath) }) + if err != nil { + return cleanup(cleanupFns), "", err + } + + cwd, err := os.Getwd() + if err != nil { + return cleanup(cleanupFns), "", fmt.Errorf("unable to get cwd: %+v", err) + } + + t.Logf("running from: %s", cwd) + + return cleanup(cleanupFns), destinationArchiveFilePath, nil +} + +// TODO: Consider moving any non-git asset generation to a task (e.g. make) that's run ahead of running go tests. +func ensureNestedZipExists(t *testing.T, sourceDirPath string) error { + t.Helper() + + nestedArchiveFilePath := path.Join(sourceDirPath, "nested.zip") + err := createZipArchive(t, sourceDirPath, nestedArchiveFilePath) + + if err != nil { + return fmt.Errorf("unable to create nested archive for test fixture: %+v", err) + } + + return nil +} diff --git a/internal/file/zip_file_manifest.go b/internal/file/zip_file_manifest.go index 35a27f2d8..7b19c2070 100644 --- a/internal/file/zip_file_manifest.go +++ b/internal/file/zip_file_manifest.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "sort" + "strings" "github.com/anchore/syft/internal" @@ -26,7 +27,11 @@ func (z ZipFileManifest) GlobMatch(patterns ...string) []string { for _, pattern := range patterns { for entry := range z { - if GlobMatch(pattern, entry) { + // We want to match globs as if entries begin with a leading slash (akin to an absolute path) + // so that glob logic is consistent inside and outside of ZIP archives + normalizedEntry := normalizeZipEntryName(entry) + + if GlobMatch(pattern, normalizedEntry) { uniqueMatches.Add(entry) } } @@ -56,3 +61,11 @@ func NewZipFileManifest(archivePath string) (ZipFileManifest, error) { } return manifest, nil } + +func normalizeZipEntryName(entry string) string { + if !strings.HasPrefix(entry, "/") { + return "/" + entry + } + + return entry +} diff --git a/internal/file/zip_file_manifest_test.go b/internal/file/zip_file_manifest_test.go new file mode 100644 index 000000000..5596a5dbc --- /dev/null +++ b/internal/file/zip_file_manifest_test.go @@ -0,0 +1,109 @@ +package file + +import ( + "encoding/json" + "os" + "path" + "testing" +) + +func TestNewZipFileManifest(t *testing.T) { + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + sourceDirPath := path.Join(cwd, "test-fixtures", "zip-source") + err = ensureNestedZipExists(t, sourceDirPath) + if err != nil { + t.Fatal(err) + } + + cleanup, archiveFilePath, err := setupZipFileTest(t, sourceDirPath) + defer fatalIfError(t, cleanup) + if err != nil { + t.Fatal(err) + } + + actual, err := NewZipFileManifest(archiveFilePath) + if err != nil { + t.Fatalf("unable to extract from unzip archive: %+v", err) + } + + if len(expectedZipArchiveEntries) != len(actual) { + t.Fatalf("mismatched manifest: %d != %d", len(actual), len(expectedZipArchiveEntries)) + } + + for _, e := range expectedZipArchiveEntries { + _, ok := actual[e] + if !ok { + t.Errorf("missing path: %s", e) + } + } + + if t.Failed() { + b, err := json.MarshalIndent(actual, "", " ") + if err != nil { + t.Fatalf("can't show results: %+v", err) + } + + t.Errorf("full result: %s", string(b)) + } +} + +func TestZipFileManifest_GlobMatch(t *testing.T) { + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + sourceDirPath := path.Join(cwd, "test-fixtures", "zip-source") + err = ensureNestedZipExists(t, sourceDirPath) + if err != nil { + t.Fatal(err) + } + + cleanup, archiveFilePath, err := setupZipFileTest(t, sourceDirPath) + //goland:noinspection GoNilness + defer fatalIfError(t, cleanup) + if err != nil { + t.Fatal(err) + } + + z, err := NewZipFileManifest(archiveFilePath) + if err != nil { + t.Fatalf("unable to extract from unzip archive: %+v", err) + } + + cases := []struct { + glob string + expected string + }{ + { + "/b*", + "b-file.txt", + }, + { + "*/a-file.txt", + "some-dir/a-file.txt", + }, + { + "**/*.zip", + "nested.zip", + }, + } + + for _, tc := range cases { + t.Run(tc.glob, func(t *testing.T) { + glob := tc.glob + + results := z.GlobMatch(glob) + + if len(results) == 1 && results[0] == tc.expected { + return + } + + t.Errorf("unexpected results for glob '%s': %+v", glob, results) + }) + } +} diff --git a/internal/file/zip_file_traversal_test.go b/internal/file/zip_file_traversal_test.go index 037d5466a..7da9e9f5f 100644 --- a/internal/file/zip_file_traversal_test.go +++ b/internal/file/zip_file_traversal_test.go @@ -6,47 +6,14 @@ import ( "io" "io/ioutil" "os" - "os/exec" + "path" "path/filepath" "strings" - "syscall" "testing" "github.com/go-test/deep" ) -func generateFixture(t *testing.T, archivePath string) { - cwd, err := os.Getwd() - if err != nil { - t.Errorf("unable to get cwd: %+v", err) - } - - cmd := exec.Command("./generate-zip-fixture.sh", archivePath) - cmd.Dir = filepath.Join(cwd, "test-fixtures") - - if err := cmd.Start(); err != nil { - t.Fatalf("unable to start generate zip fixture script: %+v", err) - } - - if err := cmd.Wait(); err != nil { - if exiterr, ok := err.(*exec.ExitError); ok { - // The program has exited with an exit code != 0 - - // This works on both Unix and Windows. Although package - // syscall is generally platform dependent, WaitStatus is - // defined for both Unix and Windows and in both cases has - // an ExitStatus() method with the same signature. - if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { - if status.ExitStatus() != 0 { - t.Fatalf("failed to generate fixture: rc=%d", status.ExitStatus()) - } - } - } else { - t.Fatalf("unable to get generate fixture script result: %+v", err) - } - } -} - func equal(r1, r2 io.Reader) (bool, error) { w1 := sha256.New() w2 := sha256.New() @@ -67,54 +34,50 @@ func equal(r1, r2 io.Reader) (bool, error) { } func TestUnzipToDir(t *testing.T) { - archivePrefix, err := ioutil.TempFile("", "syft-ziputil-archive-TEST-") + cwd, err := os.Getwd() if err != nil { - t.Fatalf("unable to create tempfile: %+v", err) + t.Fatal(err) } - defer os.Remove(archivePrefix.Name()) - // the zip utility will add ".zip" to the end of the given name - archivePath := archivePrefix.Name() + ".zip" - defer os.Remove(archivePath) - t.Logf("archive path: %s", archivePath) - generateFixture(t, archivePrefix.Name()) + goldenRootDir := filepath.Join(cwd, "test-fixtures") + sourceDirPath := path.Join(goldenRootDir, "zip-source") + cleanup, archiveFilePath, err := setupZipFileTest(t, sourceDirPath) + defer fatalIfError(t, cleanup) + if err != nil { + t.Fatal(err) + } - contentsDir, err := ioutil.TempDir("", "syft-ziputil-contents-TEST-") + unzipDestinationDir, err := ioutil.TempDir("", "syft-ziputil-contents-TEST-") + defer os.RemoveAll(unzipDestinationDir) if err != nil { t.Fatalf("unable to create tempdir: %+v", err) } - defer os.RemoveAll(contentsDir) - t.Logf("content path: %s", contentsDir) + t.Logf("content path: %s", unzipDestinationDir) - cwd, err := os.Getwd() - if err != nil { - t.Errorf("unable to get cwd: %+v", err) - } - - t.Logf("running from: %s", cwd) - - // note: zip utility already includes "zip-source" as a parent dir for all contained files - goldenRootDir := filepath.Join(cwd, "test-fixtures") - expectedPaths := 4 + expectedPaths := len(expectedZipArchiveEntries) observedPaths := 0 - err = UnzipToDir(archivePath, contentsDir) + err = UnzipToDir(archiveFilePath, unzipDestinationDir) if err != nil { t.Fatalf("unable to unzip archive: %+v", err) } // compare the source dir tree and the unzipped tree - err = filepath.Walk(filepath.Join(contentsDir, "zip-source"), + err = filepath.Walk(unzipDestinationDir, func(path string, info os.FileInfo, err error) error { - t.Logf("unzipped path: %s", path) - observedPaths++ + // We don't unzip the root archive dir, since there's no archive entry for it + if path != unzipDestinationDir { + t.Logf("unzipped path: %s", path) + observedPaths++ + } + if err != nil { t.Fatalf("this should not happen") return err } - goldenPath := filepath.Join(goldenRootDir, strings.TrimPrefix(path, contentsDir)) + goldenPath := filepath.Join(sourceDirPath, strings.TrimPrefix(path, unzipDestinationDir)) if info.IsDir() { i, err := os.Stat(goldenPath) @@ -156,12 +119,11 @@ func TestUnzipToDir(t *testing.T) { } if observedPaths != expectedPaths { - t.Errorf("missed test paths: %d!=%d", observedPaths, expectedPaths) + t.Errorf("missed test paths: %d != %d", observedPaths, expectedPaths) } - } -func TestExtractFilesFromZipFile(t *testing.T) { +func TestContentsFromZip(t *testing.T) { archivePrefix, err := ioutil.TempFile("", "syft-ziputil-archive-TEST-") if err != nil { t.Fatalf("unable to create tempfile: %+v", err) @@ -172,7 +134,10 @@ func TestExtractFilesFromZipFile(t *testing.T) { defer os.Remove(archivePath) t.Logf("archive path: %s", archivePath) - generateFixture(t, archivePrefix.Name()) + err = createZipArchive(t, "zip-source", archivePrefix.Name()) + if err != nil { + t.Fatal(err) + } cwd, err := os.Getwd() if err != nil { @@ -181,8 +146,8 @@ func TestExtractFilesFromZipFile(t *testing.T) { t.Logf("running from: %s", cwd) - aFilePath := filepath.Join("zip-source", "some-dir", "a-file.txt") - bFilePath := filepath.Join("zip-source", "b-file.txt") + aFilePath := filepath.Join("some-dir", "a-file.txt") + bFilePath := filepath.Join("b-file.txt") expected := map[string]string{ aFilePath: "A file! nice!", @@ -207,60 +172,4 @@ func TestExtractFilesFromZipFile(t *testing.T) { t.Errorf("full result: %s", string(b)) } - -} - -func TestZipFileManifest(t *testing.T) { - archivePrefix, err := ioutil.TempFile("", "syft-ziputil-archive-TEST-") - if err != nil { - t.Fatalf("unable to create tempfile: %+v", err) - } - defer os.Remove(archivePrefix.Name()) - // the zip utility will add ".zip" to the end of the given name - archivePath := archivePrefix.Name() + ".zip" - defer os.Remove(archivePath) - t.Logf("archive path: %s", archivePath) - - generateFixture(t, archivePrefix.Name()) - - cwd, err := os.Getwd() - if err != nil { - t.Errorf("unable to get cwd: %+v", err) - } - - t.Logf("running from: %s", cwd) - - expected := []string{ - filepath.Join("zip-source") + string(os.PathSeparator), - filepath.Join("zip-source", "some-dir") + string(os.PathSeparator), - filepath.Join("zip-source", "some-dir", "a-file.txt"), - filepath.Join("zip-source", "b-file.txt"), - } - - actual, err := NewZipFileManifest(archivePath) - if err != nil { - t.Fatalf("unable to extract from unzip archive: %+v", err) - } - - if len(expected) != len(actual) { - t.Fatalf("mismatched manifest: %d != %d", len(actual), len(expected)) - } - - for _, e := range expected { - _, ok := actual[e] - if !ok { - t.Errorf("missing path: %s", e) - } - } - - if t.Failed() { - - b, err := json.MarshalIndent(actual, "", " ") - if err != nil { - t.Fatalf("can't show results: %+v", err) - } - - t.Errorf("full result: %s", string(b)) - } - } diff --git a/syft/cataloger/java/archive_parser.go b/syft/cataloger/java/archive_parser.go index b64c43adf..453433463 100644 --- a/syft/cataloger/java/archive_parser.go +++ b/syft/cataloger/java/archive_parser.go @@ -120,7 +120,7 @@ func (j *archiveParser) parse() ([]pkg.Package, error) { // discoverMainPackage parses the root Java manifest used as the parent package to all discovered nested packages. func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) { // search and parse java manifest files - manifestMatches := j.fileManifest.GlobMatch(manifestPath) + manifestMatches := j.fileManifest.GlobMatch(manifestGlob) if len(manifestMatches) > 1 { return nil, fmt.Errorf("found multiple manifests in the jar: %+v", manifestMatches) } else if len(manifestMatches) == 0 { diff --git a/syft/cataloger/java/java_manifest.go b/syft/cataloger/java/java_manifest.go index 5afec3f27..819f5c748 100644 --- a/syft/cataloger/java/java_manifest.go +++ b/syft/cataloger/java/java_manifest.go @@ -10,7 +10,7 @@ import ( "github.com/mitchellh/mapstructure" ) -const manifestPath = "META-INF/MANIFEST.MF" +const manifestGlob = "/META-INF/MANIFEST.MF" func parseJavaManifest(reader io.Reader) (*pkg.JavaManifest, error) { var manifest pkg.JavaManifest