mirror of
https://github.com/anchore/syft.git
synced 2026-11-02 15:36:41 +01:00
keep file catalogers separate from file-related definitions
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
const perFileReadLimit = 2 * file.GB
|
||||
|
||||
// safeCopy limits the copy from the reader. This is useful when extracting files from archives to
|
||||
// protect against decompression bomb attacks.
|
||||
func safeCopy(writer io.Writer, reader io.Reader) error {
|
||||
numBytes, err := io.Copy(writer, io.LimitReader(reader, perFileReadLimit))
|
||||
if numBytes >= perFileReadLimit || errors.Is(err, io.EOF) {
|
||||
return fmt.Errorf("zip read limit hit (potential decompression bomb attack)")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
"github.com/mholt/archiver/v3"
|
||||
)
|
||||
|
||||
// ExtractGlobsFromTarToUniqueTempFile extracts paths matching the given globs within the given archive to a temporary directory, returning file openers for each file extracted.
|
||||
func ExtractGlobsFromTarToUniqueTempFile(archivePath, dir string, globs ...string) (map[string]file.Opener, error) {
|
||||
results := make(map[string]file.Opener)
|
||||
|
||||
// don't allow for full traversal, only select traversal from given paths
|
||||
if len(globs) == 0 {
|
||||
return results, nil
|
||||
}
|
||||
|
||||
visitor := func(f archiver.File) error {
|
||||
defer f.Close()
|
||||
|
||||
// ignore directories
|
||||
if f.FileInfo.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ignore any filename that doesn't match the given globs...
|
||||
if !matchesAnyGlob(f.Name(), globs...) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// we have a file we want to extract....
|
||||
tempfilePrefix := filepath.Base(filepath.Clean(f.Name())) + "-"
|
||||
tempFile, err := ioutil.TempFile(dir, tempfilePrefix)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create temp file: %w", err)
|
||||
}
|
||||
// we shouldn't try and keep the tempfile open as the returned result may have several files, which takes up
|
||||
// resources (leading to "too many open files"). Instead we'll return a file opener to the caller which
|
||||
// provides a ReadCloser. It is up to the caller to handle closing the file explicitly.
|
||||
defer tempFile.Close()
|
||||
|
||||
if err := safeCopy(tempFile, f.ReadCloser); err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q for tar=%q: %w", f.Name(), archivePath, err)
|
||||
}
|
||||
|
||||
results[f.Name()] = file.NewOpener(tempFile.Name())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return results, archiver.Walk(archivePath, visitor)
|
||||
}
|
||||
|
||||
func matchesAnyGlob(name string, globs ...string) bool {
|
||||
for _, glob := range globs {
|
||||
if matches, err := doublestar.PathMatch(glob, name); err == nil && matches {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eux
|
||||
|
||||
# $1 —— absolute path to destination file, should end with .zip, ideally
|
||||
# $2 —— absolute path to directory from which to add entries to the archive
|
||||
|
||||
pushd "$2" && find . -print | zip "$1" -@ && popd
|
||||
@@ -0,0 +1 @@
|
||||
B file...
|
||||
@@ -0,0 +1 @@
|
||||
A file! nice!
|
||||
@@ -0,0 +1,115 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
var expectedZipArchiveEntries = []string{
|
||||
"some-dir" + string(os.PathSeparator),
|
||||
filepath.Join("some-dir", "a-file.txt"),
|
||||
"b-file.txt",
|
||||
"nested.zip",
|
||||
}
|
||||
|
||||
// createZipArchive creates a new ZIP archive file at destinationArchivePath based on the directory found at
|
||||
// sourceDirPath.
|
||||
func createZipArchive(t testing.TB, sourceDirPath, destinationArchivePath string) {
|
||||
t.Helper()
|
||||
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("unable to get cwd: %+v", err)
|
||||
}
|
||||
|
||||
cmd := exec.Command("./generate-zip-fixture-from-source-dir.sh", destinationArchivePath, path.Base(sourceDirPath))
|
||||
cmd.Dir = filepath.Join(cwd, "test-fixtures")
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatalf("unable to start generate zip fixture script: %+v", err)
|
||||
}
|
||||
|
||||
if err := cmd.Wait(); err != nil {
|
||||
if exiterr, ok := err.(*exec.ExitError); ok {
|
||||
// The program has exited with an exit code != 0
|
||||
|
||||
// This works on both Unix and Windows. Although package
|
||||
// syscall is generally platform dependent, WaitStatus is
|
||||
// defined for both Unix and Windows and in both cases has
|
||||
// an ExitStatus() method with the same signature.
|
||||
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
|
||||
if status.ExitStatus() != 0 {
|
||||
t.Fatalf("failed to generate fixture: rc=%d", status.ExitStatus())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
t.Fatalf("unable to get generate fixture script result: %+v", err)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func assertNoError(t testing.TB, fn func() error) func() {
|
||||
return func() {
|
||||
assert.NoError(t, fn())
|
||||
}
|
||||
}
|
||||
|
||||
// setupZipFileTest encapsulates common test setup work for zip file tests. It returns a cleanup function,
|
||||
// which should be called (typically deferred) by the caller, the path of the created zip archive, and an error,
|
||||
// which should trigger a fatal test failure in the consuming test. The returned cleanup function will never be nil
|
||||
// (even if there's an error), and it should always be called.
|
||||
func setupZipFileTest(t testing.TB, sourceDirPath string) string {
|
||||
t.Helper()
|
||||
|
||||
archivePrefix, err := ioutil.TempFile("", "syft-ziputil-archive-TEST-")
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create tempfile: %+v", err)
|
||||
}
|
||||
|
||||
t.Cleanup(
|
||||
assertNoError(t,
|
||||
func() error {
|
||||
return os.Remove(archivePrefix.Name())
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
destinationArchiveFilePath := archivePrefix.Name() + ".zip"
|
||||
t.Logf("archive path: %s", destinationArchiveFilePath)
|
||||
createZipArchive(t, sourceDirPath, destinationArchiveFilePath)
|
||||
|
||||
t.Cleanup(
|
||||
assertNoError(t,
|
||||
func() error {
|
||||
return os.Remove(destinationArchiveFilePath)
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("unable to get cwd: %+v", err)
|
||||
}
|
||||
|
||||
t.Logf("running from: %s", cwd)
|
||||
|
||||
return destinationArchiveFilePath
|
||||
}
|
||||
|
||||
// TODO: Consider moving any non-git asset generation to a task (e.g. make) that's run ahead of running go tests.
|
||||
func ensureNestedZipExists(t *testing.T, sourceDirPath string) error {
|
||||
t.Helper()
|
||||
|
||||
nestedArchiveFilePath := path.Join(sourceDirPath, "nested.zip")
|
||||
createZipArchive(t, sourceDirPath, nestedArchiveFilePath)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
)
|
||||
|
||||
// ZipFileManifest is a collection of paths and their file metadata.
|
||||
type ZipFileManifest map[string]os.FileInfo
|
||||
|
||||
// NewZipFileManifest creates and returns a new ZipFileManifest populated with path and metadata from the given zip archive path.
|
||||
func NewZipFileManifest(archivePath string) (ZipFileManifest, error) {
|
||||
zipReader, err := OpenZip(archivePath)
|
||||
manifest := make(ZipFileManifest)
|
||||
if err != nil {
|
||||
return manifest, fmt.Errorf("unable to open zip archive (%s): %w", archivePath, err)
|
||||
}
|
||||
defer func() {
|
||||
err = zipReader.Close()
|
||||
if err != nil {
|
||||
log.Warnf("unable to close zip archive (%s): %+v", archivePath, err)
|
||||
}
|
||||
}()
|
||||
|
||||
for _, file := range zipReader.Reader.File {
|
||||
manifest.Add(file.Name, file.FileInfo())
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
// Add a new path and it's file metadata to the collection.
|
||||
func (z ZipFileManifest) Add(entry string, info os.FileInfo) {
|
||||
z[entry] = info
|
||||
}
|
||||
|
||||
// GlobMatch returns the path keys that match the given value(s).
|
||||
func (z ZipFileManifest) GlobMatch(patterns ...string) []string {
|
||||
uniqueMatches := internal.NewStringSet()
|
||||
|
||||
for _, pattern := range patterns {
|
||||
for entry := range z {
|
||||
// We want to match globs as if entries begin with a leading slash (akin to an absolute path)
|
||||
// so that glob logic is consistent inside and outside of ZIP archives
|
||||
normalizedEntry := normalizeZipEntryName(entry)
|
||||
|
||||
if file.GlobMatch(pattern, normalizedEntry) {
|
||||
uniqueMatches.Add(entry)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results := uniqueMatches.ToSlice()
|
||||
sort.Strings(results)
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
// normalizeZipEntryName takes the given path entry and ensures it is prefixed with "/".
|
||||
func normalizeZipEntryName(entry string) string {
|
||||
if !strings.HasPrefix(entry, "/") {
|
||||
return "/" + entry
|
||||
}
|
||||
|
||||
return entry
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
//go:build !windows
|
||||
// +build !windows
|
||||
|
||||
package archive
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewZipFileManifest(t *testing.T) {
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sourceDirPath := path.Join(cwd, "test-fixtures", "zip-source")
|
||||
err = ensureNestedZipExists(t, sourceDirPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
archiveFilePath := setupZipFileTest(t, sourceDirPath)
|
||||
|
||||
actual, err := NewZipFileManifest(archiveFilePath)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to extract from unzip archive: %+v", err)
|
||||
}
|
||||
|
||||
if len(expectedZipArchiveEntries) != len(actual) {
|
||||
t.Fatalf("mismatched manifest: %d != %d", len(actual), len(expectedZipArchiveEntries))
|
||||
}
|
||||
|
||||
for _, e := range expectedZipArchiveEntries {
|
||||
_, ok := actual[e]
|
||||
if !ok {
|
||||
t.Errorf("missing path: %s", e)
|
||||
}
|
||||
}
|
||||
|
||||
if t.Failed() {
|
||||
b, err := json.MarshalIndent(actual, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("can't show results: %+v", err)
|
||||
}
|
||||
|
||||
t.Errorf("full result: %s", string(b))
|
||||
}
|
||||
}
|
||||
|
||||
func TestZipFileManifest_GlobMatch(t *testing.T) {
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sourceDirPath := path.Join(cwd, "test-fixtures", "zip-source")
|
||||
err = ensureNestedZipExists(t, sourceDirPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
archiveFilePath := setupZipFileTest(t, sourceDirPath)
|
||||
|
||||
z, err := NewZipFileManifest(archiveFilePath)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to extract from unzip archive: %+v", err)
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
glob string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
"/b*",
|
||||
"b-file.txt",
|
||||
},
|
||||
{
|
||||
"*/a-file.txt",
|
||||
"some-dir/a-file.txt",
|
||||
},
|
||||
{
|
||||
"**/*.zip",
|
||||
"nested.zip",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.glob, func(t *testing.T) {
|
||||
glob := tc.glob
|
||||
|
||||
results := z.GlobMatch(glob)
|
||||
|
||||
if len(results) == 1 && results[0] == tc.expected {
|
||||
return
|
||||
}
|
||||
|
||||
t.Errorf("unexpected results for glob '%s': %+v", glob, results)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
)
|
||||
|
||||
type errZipSlipDetected struct {
|
||||
Prefix string
|
||||
JoinArgs []string
|
||||
}
|
||||
|
||||
func (e *errZipSlipDetected) Error() string {
|
||||
return fmt.Sprintf("paths are not allowed to resolve outside of the root prefix (%q). Destination: %q", e.Prefix, e.JoinArgs)
|
||||
}
|
||||
|
||||
type zipTraversalRequest map[string]struct{}
|
||||
|
||||
func newZipTraverseRequest(paths ...string) zipTraversalRequest {
|
||||
results := make(zipTraversalRequest)
|
||||
for _, p := range paths {
|
||||
results[p] = struct{}{}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
// TraverseFilesInZip enumerates all paths stored within a zip archive using the visitor pattern.
|
||||
func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths ...string) error {
|
||||
request := newZipTraverseRequest(paths...)
|
||||
|
||||
zipReader, err := OpenZip(archivePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to open zip archive (%s): %w", archivePath, err)
|
||||
}
|
||||
defer func() {
|
||||
err = zipReader.Close()
|
||||
if err != nil {
|
||||
log.Errorf("unable to close zip archive (%s): %+v", archivePath, err)
|
||||
}
|
||||
}()
|
||||
|
||||
for _, file := range zipReader.Reader.File {
|
||||
// if no paths are given then assume that all files should be traversed
|
||||
if len(paths) > 0 {
|
||||
if _, ok := request[file.Name]; !ok {
|
||||
// this file path is not of interest
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if err = visitor(file); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ExtractFromZipToUniqueTempFile extracts select paths for the given archive to a temporary directory, returning file openers for each file extracted.
|
||||
func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (map[string]file.Opener, error) {
|
||||
results := make(map[string]file.Opener)
|
||||
|
||||
// don't allow for full traversal, only select traversal from given paths
|
||||
if len(paths) == 0 {
|
||||
return results, nil
|
||||
}
|
||||
|
||||
visitor := func(f *zip.File) error {
|
||||
tempfilePrefix := filepath.Base(filepath.Clean(f.Name)) + "-"
|
||||
|
||||
tempFile, err := ioutil.TempFile(dir, tempfilePrefix)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create temp file: %w", err)
|
||||
}
|
||||
// we shouldn't try and keep the tempfile open as the returned result may have several files, which takes up
|
||||
// resources (leading to "too many open files"). Instead we'll return a file opener to the caller which
|
||||
// provides a ReadCloser. It is up to the caller to handle closing the file explicitly.
|
||||
defer tempFile.Close()
|
||||
|
||||
zippedFile, err := f.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read file=%q from zip=%q: %w", f.Name, archivePath, err)
|
||||
}
|
||||
defer func() {
|
||||
err := zippedFile.Close()
|
||||
if err != nil {
|
||||
log.Errorf("unable to close source file=%q from zip=%q: %+v", f.Name, archivePath, err)
|
||||
}
|
||||
}()
|
||||
|
||||
if f.FileInfo().IsDir() {
|
||||
return fmt.Errorf("unable to extract directories, only files: %s", f.Name)
|
||||
}
|
||||
|
||||
if err := safeCopy(tempFile, zippedFile); err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q for zip=%q: %w", f.Name, archivePath, err)
|
||||
}
|
||||
|
||||
results[f.Name] = file.NewOpener(tempFile.Name())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return results, TraverseFilesInZip(archivePath, visitor, paths...)
|
||||
}
|
||||
|
||||
// ContentsFromZip extracts select paths for the given archive and returns a set of string contents for each path.
|
||||
func ContentsFromZip(archivePath string, paths ...string) (map[string]string, error) {
|
||||
results := make(map[string]string)
|
||||
|
||||
// don't allow for full traversal, only select traversal from given paths
|
||||
if len(paths) == 0 {
|
||||
return results, nil
|
||||
}
|
||||
|
||||
visitor := func(file *zip.File) error {
|
||||
zippedFile, err := file.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
|
||||
if file.FileInfo().IsDir() {
|
||||
return fmt.Errorf("unable to extract directories, only files: %s", file.Name)
|
||||
}
|
||||
|
||||
var buffer bytes.Buffer
|
||||
if err := safeCopy(&buffer, zippedFile); err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
|
||||
results[file.Name] = buffer.String()
|
||||
|
||||
err = zippedFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to close source file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return results, TraverseFilesInZip(archivePath, visitor, paths...)
|
||||
}
|
||||
|
||||
// UnzipToDir extracts a zip archive to a target directory.
|
||||
func UnzipToDir(archivePath, targetDir string) error {
|
||||
visitor := func(file *zip.File) error {
|
||||
joinedPath, err := safeJoin(targetDir, file.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = extractSingleFile(file, joinedPath, archivePath); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return TraverseFilesInZip(archivePath, visitor)
|
||||
}
|
||||
|
||||
// safeJoin ensures that any destinations do not resolve to a path above the prefix path.
|
||||
func safeJoin(prefix string, dest ...string) (string, error) {
|
||||
joinResult := filepath.Join(append([]string{prefix}, dest...)...)
|
||||
cleanJoinResult := filepath.Clean(joinResult)
|
||||
if !strings.HasPrefix(cleanJoinResult, filepath.Clean(prefix)) {
|
||||
return "", &errZipSlipDetected{
|
||||
Prefix: prefix,
|
||||
JoinArgs: dest,
|
||||
}
|
||||
}
|
||||
// why not return the clean path? the called may not be expected it from what should only be a join operation.
|
||||
return joinResult, nil
|
||||
}
|
||||
|
||||
func extractSingleFile(file *zip.File, expandedFilePath, archivePath string) error {
|
||||
zippedFile, err := file.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
|
||||
if file.FileInfo().IsDir() {
|
||||
err = os.MkdirAll(expandedFilePath, file.Mode())
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create dir=%q from zip=%q: %w", expandedFilePath, archivePath, err)
|
||||
}
|
||||
} else {
|
||||
// Open an output file for writing
|
||||
outputFile, err := os.OpenFile(
|
||||
expandedFilePath,
|
||||
os.O_WRONLY|os.O_CREATE|os.O_TRUNC,
|
||||
file.Mode(),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create dest file=%q from zip=%q: %w", expandedFilePath, archivePath, err)
|
||||
}
|
||||
|
||||
if err := safeCopy(outputFile, zippedFile); err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q to dest=%q for zip=%q: %w", file.Name, outputFile.Name(), archivePath, err)
|
||||
}
|
||||
|
||||
err = outputFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to close dest file=%q from zip=%q: %w", outputFile.Name(), archivePath, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = zippedFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to close source file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
//go:build !windows
|
||||
// +build !windows
|
||||
|
||||
package archive
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-test/deep"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func equal(r1, r2 io.Reader) (bool, error) {
|
||||
w1 := sha256.New()
|
||||
w2 := sha256.New()
|
||||
n1, err1 := io.Copy(w1, r1)
|
||||
if err1 != nil {
|
||||
return false, err1
|
||||
}
|
||||
n2, err2 := io.Copy(w2, r2)
|
||||
if err2 != nil {
|
||||
return false, err2
|
||||
}
|
||||
|
||||
var b1, b2 [sha256.Size]byte
|
||||
copy(b1[:], w1.Sum(nil))
|
||||
copy(b2[:], w2.Sum(nil))
|
||||
|
||||
return n1 != n2 || b1 == b2, nil
|
||||
}
|
||||
|
||||
func TestUnzipToDir(t *testing.T) {
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
goldenRootDir := filepath.Join(cwd, "test-fixtures")
|
||||
sourceDirPath := path.Join(goldenRootDir, "zip-source")
|
||||
archiveFilePath := setupZipFileTest(t, sourceDirPath)
|
||||
|
||||
unzipDestinationDir, err := ioutil.TempDir("", "syft-ziputil-contents-TEST-")
|
||||
t.Cleanup(assertNoError(t, func() error {
|
||||
return os.RemoveAll(unzipDestinationDir)
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create tempdir: %+v", err)
|
||||
}
|
||||
|
||||
t.Logf("content path: %s", unzipDestinationDir)
|
||||
|
||||
expectedPaths := len(expectedZipArchiveEntries)
|
||||
observedPaths := 0
|
||||
|
||||
err = UnzipToDir(archiveFilePath, unzipDestinationDir)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to unzip archive: %+v", err)
|
||||
}
|
||||
|
||||
// compare the source dir tree and the unzipped tree
|
||||
err = filepath.Walk(unzipDestinationDir,
|
||||
func(path string, info os.FileInfo, err error) error {
|
||||
// We don't unzip the root archive dir, since there's no archive entry for it
|
||||
if path != unzipDestinationDir {
|
||||
t.Logf("unzipped path: %s", path)
|
||||
observedPaths++
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("this should not happen")
|
||||
return err
|
||||
}
|
||||
|
||||
goldenPath := filepath.Join(sourceDirPath, strings.TrimPrefix(path, unzipDestinationDir))
|
||||
|
||||
if info.IsDir() {
|
||||
i, err := os.Stat(goldenPath)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to stat golden path: %+v", err)
|
||||
}
|
||||
if !i.IsDir() {
|
||||
t.Fatalf("mismatched file types: %s", goldenPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// this is a file, not a dir...
|
||||
|
||||
testFile, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to open test file=%s :%+v", path, err)
|
||||
}
|
||||
|
||||
goldenFile, err := os.Open(goldenPath)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to open golden file=%s :%+v", goldenPath, err)
|
||||
}
|
||||
|
||||
same, err := equal(testFile, goldenFile)
|
||||
if err != nil {
|
||||
t.Fatalf("could not compare files (%s, %s): %+v", goldenPath, path, err)
|
||||
}
|
||||
|
||||
if !same {
|
||||
t.Errorf("paths are not the same (%s, %s)", goldenPath, path)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("failed to walk dir: %+v", err)
|
||||
}
|
||||
|
||||
if observedPaths != expectedPaths {
|
||||
t.Errorf("missed test paths: %d != %d", observedPaths, expectedPaths)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentsFromZip(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
archivePrep func(tb testing.TB) string
|
||||
}{
|
||||
{
|
||||
name: "standard, non-nested zip",
|
||||
archivePrep: prepZipSourceFixture,
|
||||
},
|
||||
{
|
||||
name: "zip with prepended bytes",
|
||||
archivePrep: prependZipSourceFixtureWithString(t, "junk at the beginning of the file..."),
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
archivePath := test.archivePrep(t)
|
||||
expected := zipSourceFixtureExpectedContents()
|
||||
|
||||
var paths []string
|
||||
for p := range expected {
|
||||
paths = append(paths, p)
|
||||
}
|
||||
|
||||
actual, err := ContentsFromZip(archivePath, paths...)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to extract from unzip archive: %+v", err)
|
||||
}
|
||||
|
||||
assertZipSourceFixtureContents(t, actual, expected)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func prependZipSourceFixtureWithString(tb testing.TB, value string) func(tb testing.TB) string {
|
||||
if len(value) == 0 {
|
||||
tb.Fatalf("no bytes given to prefix")
|
||||
}
|
||||
return func(t testing.TB) string {
|
||||
archivePath := prepZipSourceFixture(t)
|
||||
|
||||
// create a temp file
|
||||
tmpFile, err := ioutil.TempFile("", "syft-ziputil-prependZipSourceFixtureWithString-")
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create tempfile: %+v", err)
|
||||
}
|
||||
defer tmpFile.Close()
|
||||
|
||||
// write value to the temp file
|
||||
if _, err := tmpFile.WriteString(value); err != nil {
|
||||
t.Fatalf("unable to write to tempfile: %+v", err)
|
||||
}
|
||||
|
||||
// open the original archive
|
||||
sourceFile, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to read source file: %+v", err)
|
||||
}
|
||||
|
||||
// copy all contents from the archive to the temp file
|
||||
if _, err := io.Copy(tmpFile, sourceFile); err != nil {
|
||||
t.Fatalf("unable to copy source to dest: %+v", err)
|
||||
}
|
||||
|
||||
sourceFile.Close()
|
||||
|
||||
// remove the original archive and replace it with the temp file
|
||||
if err := os.Remove(archivePath); err != nil {
|
||||
t.Fatalf("unable to remove original source archive (%q): %+v", archivePath, err)
|
||||
}
|
||||
|
||||
if err := os.Rename(tmpFile.Name(), archivePath); err != nil {
|
||||
t.Fatalf("unable to move new archive to old path (%q): %+v", tmpFile.Name(), err)
|
||||
}
|
||||
|
||||
return archivePath
|
||||
}
|
||||
}
|
||||
|
||||
func prepZipSourceFixture(t testing.TB) string {
|
||||
t.Helper()
|
||||
archivePrefix, err := ioutil.TempFile("", "syft-ziputil-prepZipSourceFixture-")
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create tempfile: %+v", err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() {
|
||||
assert.NoError(t, os.Remove(archivePrefix.Name()))
|
||||
})
|
||||
|
||||
// the zip utility will add ".zip" to the end of the given name
|
||||
archivePath := archivePrefix.Name() + ".zip"
|
||||
|
||||
t.Cleanup(func() {
|
||||
assert.NoError(t, os.Remove(archivePath))
|
||||
})
|
||||
|
||||
t.Logf("archive path: %s", archivePath)
|
||||
|
||||
createZipArchive(t, "zip-source", archivePrefix.Name())
|
||||
|
||||
return archivePath
|
||||
}
|
||||
|
||||
func zipSourceFixtureExpectedContents() map[string]string {
|
||||
return map[string]string{
|
||||
filepath.Join("some-dir", "a-file.txt"): "A file! nice!",
|
||||
filepath.Join("b-file.txt"): "B file...",
|
||||
}
|
||||
}
|
||||
|
||||
func assertZipSourceFixtureContents(t testing.TB, actual map[string]string, expected map[string]string) {
|
||||
t.Helper()
|
||||
diffs := deep.Equal(actual, expected)
|
||||
if len(diffs) > 0 {
|
||||
for _, d := range diffs {
|
||||
t.Errorf("diff: %+v", d)
|
||||
}
|
||||
|
||||
b, err := json.MarshalIndent(actual, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("can't show results: %+v", err)
|
||||
}
|
||||
|
||||
t.Errorf("full result: %s", string(b))
|
||||
}
|
||||
}
|
||||
|
||||
// looks like there isn't a helper for this yet? https://github.com/stretchr/testify/issues/497
|
||||
func assertErrorAs(expectedErr interface{}) assert.ErrorAssertionFunc {
|
||||
return func(t assert.TestingT, actualErr error, i ...interface{}) bool {
|
||||
return errors.As(actualErr, &expectedErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeJoin(t *testing.T) {
|
||||
tests := []struct {
|
||||
prefix string
|
||||
args []string
|
||||
expected string
|
||||
errAssertion assert.ErrorAssertionFunc
|
||||
}{
|
||||
// go cases...
|
||||
{
|
||||
prefix: "/a/place",
|
||||
args: []string{
|
||||
"somewhere/else",
|
||||
},
|
||||
expected: "/a/place/somewhere/else",
|
||||
errAssertion: assert.NoError,
|
||||
},
|
||||
{
|
||||
prefix: "/a/place",
|
||||
args: []string{
|
||||
"somewhere/../else",
|
||||
},
|
||||
expected: "/a/place/else",
|
||||
errAssertion: assert.NoError,
|
||||
},
|
||||
{
|
||||
prefix: "/a/../place",
|
||||
args: []string{
|
||||
"somewhere/else",
|
||||
},
|
||||
expected: "/place/somewhere/else",
|
||||
errAssertion: assert.NoError,
|
||||
},
|
||||
// zip slip examples....
|
||||
{
|
||||
prefix: "/a/place",
|
||||
args: []string{
|
||||
"../../../etc/passwd",
|
||||
},
|
||||
expected: "",
|
||||
errAssertion: assertErrorAs(&errZipSlipDetected{}),
|
||||
},
|
||||
{
|
||||
prefix: "/a/place",
|
||||
args: []string{
|
||||
"../",
|
||||
"../",
|
||||
},
|
||||
expected: "",
|
||||
errAssertion: assertErrorAs(&errZipSlipDetected{}),
|
||||
},
|
||||
{
|
||||
prefix: "/a/place",
|
||||
args: []string{
|
||||
"../",
|
||||
},
|
||||
expected: "",
|
||||
errAssertion: assertErrorAs(&errZipSlipDetected{}),
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(fmt.Sprintf("%+v:%+v", test.prefix, test.args), func(t *testing.T) {
|
||||
actual, err := safeJoin(test.prefix, test.args...)
|
||||
test.errAssertion(t, err)
|
||||
assert.Equal(t, test.expected, actual)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
// directoryEndLen, readByf, directoryEnd, and findSignatureInBlock were copied from the golang stdlib, specifically:
|
||||
// - https://github.com/golang/go/blob/go1.16.4/src/archive/zip/struct.go
|
||||
// - https://github.com/golang/go/blob/go1.16.4/src/archive/zip/reader.go
|
||||
// findArchiveStartOffset is derived from the same stdlib utils, specifically the readDirectoryEnd function.
|
||||
|
||||
const directoryEndLen = 22
|
||||
|
||||
// ZipReadCloser is a drop-in replacement for zip.ReadCloser (from zip.OpenReader) that additionally considers zips
|
||||
// that have bytes prefixed to the front of the archive (common with self-extracting jars).
|
||||
type ZipReadCloser struct {
|
||||
*zip.Reader
|
||||
io.Closer
|
||||
}
|
||||
|
||||
// OpenZip provides a ZipReadCloser for the given filepath.
|
||||
func OpenZip(filepath string) (*ZipReadCloser, error) {
|
||||
f, err := os.Open(filepath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// some archives may have bytes prepended to the front of the archive, such as with self executing JARs. We first
|
||||
// need to find the start of the archive and keep track of this offset.
|
||||
offset, err := findArchiveStartOffset(f, fi.Size())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot find beginning of zip archive=%q : %w", filepath, err)
|
||||
}
|
||||
|
||||
if _, err := f.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, fmt.Errorf("unable to seek to beginning of archive: %w", err)
|
||||
}
|
||||
|
||||
size := fi.Size() - int64(offset)
|
||||
|
||||
r, err := zip.NewReader(io.NewSectionReader(f, int64(offset), size), size)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to open ZipReadCloser @ %q: %w", filepath, err)
|
||||
}
|
||||
|
||||
return &ZipReadCloser{
|
||||
Reader: r,
|
||||
Closer: f,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type readBuf []byte
|
||||
|
||||
func (b *readBuf) uint16() uint16 {
|
||||
v := binary.LittleEndian.Uint16(*b)
|
||||
*b = (*b)[2:]
|
||||
return v
|
||||
}
|
||||
|
||||
func (b *readBuf) uint32() uint32 {
|
||||
v := binary.LittleEndian.Uint32(*b)
|
||||
*b = (*b)[4:]
|
||||
return v
|
||||
}
|
||||
|
||||
type directoryEnd struct {
|
||||
diskNbr uint32 // unused
|
||||
dirDiskNbr uint32 // unused
|
||||
dirRecordsThisDisk uint64 // unused
|
||||
directoryRecords uint64
|
||||
directorySize uint64
|
||||
directoryOffset uint64 // relative to file
|
||||
}
|
||||
|
||||
// note: this is derived from readDirectoryEnd within the archive/zip package
|
||||
func findArchiveStartOffset(r io.ReaderAt, size int64) (startOfArchive uint64, err error) {
|
||||
// look for directoryEndSignature in the last 1k, then in the last 65k
|
||||
var buf []byte
|
||||
var directoryEndOffset int64
|
||||
for i, bLen := range []int64{1024, 65 * 1024} {
|
||||
if bLen > size {
|
||||
bLen = size
|
||||
}
|
||||
buf = make([]byte, int(bLen))
|
||||
if _, err := r.ReadAt(buf, size-bLen); err != nil && err != io.EOF {
|
||||
return 0, err
|
||||
}
|
||||
if p := findSignatureInBlock(buf); p >= 0 {
|
||||
buf = buf[p:]
|
||||
directoryEndOffset = size - bLen + int64(p)
|
||||
break
|
||||
}
|
||||
if i == 1 || bLen == size {
|
||||
return 0, zip.ErrFormat
|
||||
}
|
||||
}
|
||||
|
||||
if buf == nil {
|
||||
// we were unable to find the directoryEndSignature block
|
||||
return 0, zip.ErrFormat
|
||||
}
|
||||
|
||||
// read header into struct
|
||||
b := readBuf(buf[4:]) // skip signature
|
||||
d := &directoryEnd{
|
||||
diskNbr: uint32(b.uint16()),
|
||||
dirDiskNbr: uint32(b.uint16()),
|
||||
dirRecordsThisDisk: uint64(b.uint16()),
|
||||
directoryRecords: uint64(b.uint16()),
|
||||
directorySize: uint64(b.uint32()),
|
||||
directoryOffset: uint64(b.uint32()),
|
||||
}
|
||||
// Calculate where the zip data actually begins
|
||||
startOfArchive = uint64(directoryEndOffset) - d.directorySize - d.directoryOffset
|
||||
|
||||
// These values mean that the file can be a zip64 file
|
||||
if d.directoryRecords == 0xffff || d.directorySize == 0xffff || d.directoryOffset == 0xffffffff {
|
||||
startOfArchive = 0 // Prefixed data not supported
|
||||
}
|
||||
|
||||
// Make sure directoryOffset points to somewhere in our file.
|
||||
if o := int64(d.directoryOffset); o < 0 || o >= size {
|
||||
return 0, zip.ErrFormat
|
||||
}
|
||||
return startOfArchive, nil
|
||||
}
|
||||
|
||||
func findSignatureInBlock(b []byte) int {
|
||||
for i := len(b) - directoryEndLen; i >= 0; i-- {
|
||||
// defined from directoryEndSignature
|
||||
if b[i] == 'P' && b[i+1] == 'K' && b[i+2] == 0x05 && b[i+3] == 0x06 {
|
||||
// n is length of comment
|
||||
n := int(b[i+directoryEndLen-2]) | int(b[i+directoryEndLen-1])<<8
|
||||
if n+directoryEndLen+i <= len(b) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
//go:build !windows
|
||||
// +build !windows
|
||||
|
||||
package archive
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestFindArchiveStartOffset(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
archivePrep func(tb testing.TB) string
|
||||
expected uint64
|
||||
}{
|
||||
{
|
||||
name: "standard, non-nested zip",
|
||||
archivePrep: prepZipSourceFixture,
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "zip with prepended bytes",
|
||||
archivePrep: prependZipSourceFixtureWithString(t, "junk at the beginning of the file..."),
|
||||
expected: 36,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
archivePath := test.archivePrep(t)
|
||||
f, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
t.Fatalf("could not open archive %q: %+v", archivePath, err)
|
||||
}
|
||||
fi, err := os.Stat(f.Name())
|
||||
if err != nil {
|
||||
t.Fatalf("unable to stat archive: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := findArchiveStartOffset(f, fi.Size())
|
||||
if err != nil {
|
||||
t.Fatalf("unable to find offset: %+v", err)
|
||||
}
|
||||
assert.Equal(t, test.expected, actual)
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user