mirror of
https://github.com/anchore/syft.git
synced 2026-11-03 16:06:36 +01:00
feat: initial recursive archive cataloging impl
Signed-off-by: Keith Zantow <keith.zantow@anchore.com>
This commit is contained in:
Keith Zantow
+386
-159
@@ -1,7 +1,9 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
@@ -18,31 +20,135 @@ import (
|
||||
|
||||
// Extractor extracts archive contents to a destination directory.
|
||||
type Extractor interface {
|
||||
// CanExtract returns true if this extractor can handle the given file.
|
||||
CanExtract(ctx context.Context, path string, reader io.ReadSeeker) bool
|
||||
// CanExtract returns true if this extractor can handle the given archive content.
|
||||
CanExtract(ctx context.Context, content Content) bool
|
||||
|
||||
// Extract extracts the archive contents to destDir, respecting the given limits.
|
||||
// Returns the number of files extracted and total bytes written.
|
||||
Extract(ctx context.Context, path string, destDir string, limits ExtractionLimits) (ExtractionResult, error)
|
||||
// Extract writes the archive's entries into sink - one store per archive, not one file per entry -
|
||||
// respecting the given limits. Returns how many entries were written and how many bytes that took.
|
||||
Extract(ctx context.Context, content Content, sink EntrySink, limits ExtractionLimits) (ExtractionResult, error)
|
||||
}
|
||||
|
||||
// ExtractionLimits defines safety limits for archive extraction.
|
||||
// EntrySink is where an extractor puts the entries it reads out of an archive.
|
||||
//
|
||||
// Two implementations exist because they store an archive differently and the difference is
|
||||
// measurable: OverflowTar writes every entry into one tar on disk and is indexed by seek offset,
|
||||
// while EntryStore holds entries in memory until a bound says otherwise and is indexed by the entries
|
||||
// themselves. The extractors do not care which they were given, with one exception - see
|
||||
// VerbatimSink.
|
||||
type EntrySink interface {
|
||||
// AddEntry stores one entry of the archive.
|
||||
AddEntry(f archives.FileInfo, result *ExtractionResult, limits ExtractionLimits) error
|
||||
|
||||
// Finish is called once the last entry has been offered, for whatever a sink has to write after
|
||||
// its content.
|
||||
Finish(result *ExtractionResult, limits ExtractionLimits)
|
||||
}
|
||||
|
||||
// VerbatimSink is an EntrySink that stores archives in tar form, so a tar-family archive can be
|
||||
// copied into it as it stands rather than decoded and re-encoded entry by entry.
|
||||
//
|
||||
// This is the one place the storage format shows through the extractor interface, and it is worth the
|
||||
// leak: a tar.gz copied verbatim is one streamed decompression, where walking it costs a read per
|
||||
// entry. A sink that does not store tars does not implement this and gets the entry-by-entry path.
|
||||
type VerbatimSink interface {
|
||||
EntrySink
|
||||
CopyVerbatim(r io.Reader, result *ExtractionResult, limits ExtractionLimits) error
|
||||
}
|
||||
|
||||
// ExtractionLimits defines safety limits for extracting one archive.
|
||||
type ExtractionLimits struct {
|
||||
MaxExtractionSizeBytes int64
|
||||
MaxFileCount int
|
||||
|
||||
// Charge is this archive's draw on the scan's resource limiter. Entry bytes are charged to it as
|
||||
// they land, never from the size the entry declared, and a refused charge truncates the
|
||||
// extraction. A nil Charge enforces no byte bound.
|
||||
//
|
||||
// It is a handle rather than a remaining-bytes value because what it charges has to be given back
|
||||
// when the archive is released, and a value copied into a call cannot be.
|
||||
Charge *Charge
|
||||
}
|
||||
|
||||
// TruncationReason names the configured bound that stopped an extraction early. The empty value
|
||||
// means the archive was extracted in full.
|
||||
type TruncationReason string
|
||||
|
||||
const (
|
||||
// TruncatedByDiskLimit means placing more of this archive's content would have taken the scan's
|
||||
// disk usage past the disk limit. Named rather than reported as a bare truncation because a breach
|
||||
// has to be attributed to the archive that triggered it, and saying which bound was reached is
|
||||
// half of that attribution.
|
||||
TruncatedByDiskLimit TruncationReason = "disk limit"
|
||||
|
||||
// TruncatedByLimit means a bound stopped the extraction but the walker replaced the error that
|
||||
// said which one. Defensive: both bounds record their own reason on the result before stopping.
|
||||
TruncatedByLimit TruncationReason = "configured limit"
|
||||
)
|
||||
|
||||
// errTruncated stops an in-progress archive walk when a configured limit is reached. It never
|
||||
// escapes this package: the extractors convert it into a successful, truncated ExtractionResult.
|
||||
var errTruncated = errors.New("archive extraction stopped at a configured limit")
|
||||
|
||||
// ExtractionResult holds the result of an extraction operation.
|
||||
//
|
||||
// Reaching a configured limit is NOT an error: it is a truncation. Whatever was written before the
|
||||
// limit is left on disk and is usable, so callers can still catalog what was extracted rather than
|
||||
// discarding the whole archive. Check Truncated to tell a complete extraction from a partial one;
|
||||
// a non-nil error from Extract means the extraction failed and nothing usable was produced.
|
||||
type ExtractionResult struct {
|
||||
// FilesExtracted counts the entries this archive's filesystem holds - the records its overflow tar
|
||||
// was indexed into: regular files, symlinks, and directory entries. It is the count after indexing
|
||||
// rather than the count written, so an entry dropped by an exclusion pattern is not counted as
|
||||
// present. Reported, not bounded.
|
||||
FilesExtracted int
|
||||
BytesWritten int64
|
||||
|
||||
// BytesWritten is what this archive put on disk: every byte of the overflow tar, headers and block
|
||||
// padding included, because that is what the disk limit was charged for.
|
||||
BytesWritten int64
|
||||
|
||||
// Truncation names the limit that stopped extraction early, or "" when the archive was
|
||||
// extracted in full.
|
||||
Truncation TruncationReason
|
||||
}
|
||||
|
||||
// DefaultExtractionLimits returns limits from the given config.
|
||||
func DefaultExtractionLimits(cfg cataloging.ArchiveSearchConfig) ExtractionLimits {
|
||||
return ExtractionLimits{
|
||||
MaxExtractionSizeBytes: cfg.MaxExtractionSizeBytes,
|
||||
MaxFileCount: cfg.MaxFileCount,
|
||||
// Truncated reports whether a configured limit stopped this extraction before the end of the
|
||||
// archive.
|
||||
func (r ExtractionResult) Truncated() bool {
|
||||
return r.Truncation != ""
|
||||
}
|
||||
|
||||
// finishExtraction turns a walk that stopped at a configured limit into a successful, truncated
|
||||
// result.
|
||||
//
|
||||
// The limit signal is carried on the result rather than only in the returned error because the
|
||||
// underlying archive walkers are free to wrap or replace a handler's error, in which case
|
||||
// errors.Is(err, errTruncated) would not match and a truncation would be reported as a failure -
|
||||
// discarding an archive syft could have cataloged. The error is still checked, so a walker that
|
||||
// passes it through cleanly is handled too.
|
||||
func finishExtraction(result ExtractionResult, err error) (ExtractionResult, error) {
|
||||
if result.Truncated() || errors.Is(err, errTruncated) {
|
||||
if !result.Truncated() {
|
||||
// the walker swallowed the reason but passed the sentinel through
|
||||
result.Truncation = TruncatedByLimit
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
|
||||
// DefaultExtractionLimits returns the per-archive limits from the given config.
|
||||
//
|
||||
// Nothing is read from the config: every bound this scan enforces is an in-use limit held by the
|
||||
// scan's limiter, and the only per-archive field left is the Charge the caller draws against it.
|
||||
// The parameter stays so the two Default* constructors are called the same way, and so a per-archive
|
||||
// bound that is genuinely per-archive has somewhere to land.
|
||||
func DefaultExtractionLimits(_ cataloging.ArchiveSearchConfig) ExtractionLimits {
|
||||
return ExtractionLimits{}
|
||||
}
|
||||
|
||||
// DefaultLimits returns the scan-wide in-use limits from the given config.
|
||||
func DefaultLimits(cfg cataloging.ArchiveSearchConfig) Limits {
|
||||
return Limits{
|
||||
MaxMemoryBytes: cfg.MaxMemoryBytes,
|
||||
MaxDiskBytes: cfg.MaxDiskBytes,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,90 +160,82 @@ func DefaultExtractors() []Extractor {
|
||||
}
|
||||
}
|
||||
|
||||
// FindExtractor finds an extractor that can handle the given file, or nil if none can.
|
||||
func FindExtractor(ctx context.Context, extractors []Extractor, path string) Extractor {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
log.Tracef("unable to open file for archive detection: %v", err)
|
||||
return nil
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// FindExtractor finds an extractor that can handle the given archive content, or nil if none can.
|
||||
func FindExtractor(ctx context.Context, extractors []Extractor, content Content) Extractor {
|
||||
for _, ext := range extractors {
|
||||
if ext.CanExtract(ctx, path, f) {
|
||||
return ext
|
||||
}
|
||||
// reset for next extractor
|
||||
if _, err := f.Seek(0, io.SeekStart); err != nil {
|
||||
log.Tracef("unable to seek file for archive detection: %v", err)
|
||||
// every extractor sniffs from the start, whether the content is a file or a buffer
|
||||
if _, err := content.Reader.Seek(0, io.SeekStart); err != nil {
|
||||
log.Tracef("unable to seek archive content for detection: %v", err)
|
||||
return nil
|
||||
}
|
||||
if ext.CanExtract(ctx, content) {
|
||||
return ext
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsExcludedExtension checks if a file path has an excluded extension.
|
||||
func IsExcludedExtension(path string, excludeExtensions []string) bool {
|
||||
for _, ext := range excludeExtensions {
|
||||
if strings.HasSuffix(strings.ToLower(path), strings.ToLower(ext)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ZipExtractor extracts zip-based archives (zip, jar, war, ear, etc.).
|
||||
type ZipExtractor struct{}
|
||||
|
||||
func (z *ZipExtractor) CanExtract(ctx context.Context, path string, reader io.ReadSeeker) bool {
|
||||
format, _, err := intFile.IdentifyArchive(ctx, path, reader)
|
||||
if err != nil {
|
||||
// CanExtract reports whether this content is a zip, asking two questions in an order that matters.
|
||||
//
|
||||
// First, what does the content say it is, reading from the start? A format that identifies itself at
|
||||
// offset zero is what this content is, and if that format is not zip then it is not a zip - a tar
|
||||
// holding one jar ends in that jar's end-of-central-directory record, and reading it as a zip
|
||||
// instead of as a tar would lose everything else the tar holds.
|
||||
//
|
||||
// Only when nothing identifies from the head is the end-of-central-directory record consulted. That
|
||||
// record is what defines a zip - a reader locates the central directory through it - and it is the
|
||||
// only thing that can see an archive whose head is something else entirely. A self-extracting
|
||||
// archive is a launcher stub with a zip concatenated onto it, so no magic at offset zero can find
|
||||
// it; the canonical case is a Spring Boot executable jar, a shell script followed by a whole zip.
|
||||
//
|
||||
// Identification is intFile.IdentifyArchive, which matches on the filename as well as the content,
|
||||
// and it is left that way. Dropping the name makes the answer content's alone, which reads better -
|
||||
// but archives.Identify with no path has to try every format's content matcher against every
|
||||
// candidate, and measured on a stock ruby image that took the scan from 17 seconds to 45. The name
|
||||
// cannot admit a non-archive in practice: a file only reaches an extractor after its sniffed content
|
||||
// type put it in the candidate set, so the prose named notes.zip that this would wrongly identify is
|
||||
// never offered here at all. `#content-based-archive-detection` is upheld by the candidate set, and
|
||||
// Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError is where that is asserted.
|
||||
func (z *ZipExtractor) CanExtract(ctx context.Context, content Content) bool {
|
||||
format, _, err := intFile.IdentifyArchive(ctx, content.Name, content.Reader)
|
||||
if err == nil {
|
||||
_, isZip := format.(archives.Zip)
|
||||
return isZip
|
||||
}
|
||||
|
||||
if _, seekErr := content.Reader.Seek(0, io.SeekStart); seekErr != nil {
|
||||
return false
|
||||
}
|
||||
_, ok := format.(archives.Zip)
|
||||
return ok
|
||||
return HasZipEndOfCentralDirectory(content.Reader)
|
||||
}
|
||||
|
||||
func (z *ZipExtractor) Extract(ctx context.Context, path string, destDir string, limits ExtractionLimits) (ExtractionResult, error) {
|
||||
func (z *ZipExtractor) Extract(ctx context.Context, content Content, sink EntrySink, limits ExtractionLimits) (ExtractionResult, error) {
|
||||
var result ExtractionResult
|
||||
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("unable to open zip archive %q: %w", path, err)
|
||||
if _, err := content.Reader.Seek(0, io.SeekStart); err != nil {
|
||||
return result, fmt.Errorf("unable to seek zip archive %q: %w", content.Name, err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
return result, archives.Zip{}.Extract(ctx, f, func(_ context.Context, file archives.FileInfo) error {
|
||||
if file.IsDir() {
|
||||
joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.MkdirAll(joinedPath, 0o755)
|
||||
}
|
||||
|
||||
if limits.MaxFileCount > 0 && result.FilesExtracted >= limits.MaxFileCount {
|
||||
return fmt.Errorf("archive file count limit reached (%d files)", limits.MaxFileCount)
|
||||
}
|
||||
|
||||
joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(joinedPath), 0o755); err != nil {
|
||||
return fmt.Errorf("unable to create parent dir: %w", err)
|
||||
}
|
||||
|
||||
return extractFileWithLimits(file, joinedPath, destDir, &result, limits)
|
||||
// mholt/archives imposes no extraction target: the FileHandler is the caller's, so an entry can
|
||||
// be written into a tar header rather than onto the filesystem. Nothing about the library had to
|
||||
// be customized - the old code wrote files because the handler wrote files.
|
||||
err := archives.Zip{}.Extract(ctx, content.Reader, func(_ context.Context, file archives.FileInfo) error {
|
||||
return sink.AddEntry(file, &result, limits)
|
||||
})
|
||||
|
||||
sink.Finish(&result, limits)
|
||||
|
||||
return finishExtraction(result, err)
|
||||
}
|
||||
|
||||
// TarExtractor extracts tar-based archives (tar, tar.gz, tar.bz2, tar.xz, tar.zst).
|
||||
type TarExtractor struct{}
|
||||
|
||||
func (t *TarExtractor) CanExtract(ctx context.Context, path string, reader io.ReadSeeker) bool {
|
||||
format, _, err := intFile.IdentifyArchive(ctx, path, reader)
|
||||
func (t *TarExtractor) CanExtract(ctx context.Context, content Content) bool {
|
||||
format, _, err := intFile.IdentifyArchive(ctx, content.Name, content.Reader)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
@@ -149,125 +247,166 @@ func (t *TarExtractor) CanExtract(ctx context.Context, path string, reader io.Re
|
||||
return ok
|
||||
}
|
||||
|
||||
func (t *TarExtractor) Extract(ctx context.Context, path string, destDir string, limits ExtractionLimits) (ExtractionResult, error) {
|
||||
func (t *TarExtractor) Extract(ctx context.Context, content Content, sink EntrySink, limits ExtractionLimits) (ExtractionResult, error) {
|
||||
var result ExtractionResult
|
||||
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("unable to open tar archive %q: %w", path, err)
|
||||
if _, err := content.Reader.Seek(0, io.SeekStart); err != nil {
|
||||
return result, fmt.Errorf("unable to seek tar archive %q: %w", content.Name, err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
format, readerAfterIdentify, err := intFile.IdentifyArchive(ctx, path, f)
|
||||
format, readerAfterIdentify, err := intFile.IdentifyArchive(ctx, content.Name, content.Reader)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("unable to identify archive format for %q: %w", path, err)
|
||||
return result, fmt.Errorf("unable to identify archive format for %q: %w", content.Name, err)
|
||||
}
|
||||
|
||||
extractor, ok := format.(archives.Extractor)
|
||||
if !ok {
|
||||
return result, fmt.Errorf("file format does not support extraction: %s", path)
|
||||
return result, fmt.Errorf("file format does not support extraction: %s", content.Name)
|
||||
}
|
||||
|
||||
// use the reader returned by IdentifyArchive since it may have consumed some bytes
|
||||
extractReader := readerAfterIdentify
|
||||
var extractReader = readerAfterIdentify
|
||||
if extractReader == nil {
|
||||
extractReader = f
|
||||
extractReader = content.Reader
|
||||
}
|
||||
|
||||
return result, extractor.Extract(ctx, extractReader, func(_ context.Context, file archives.FileInfo) error {
|
||||
if file.IsDir() {
|
||||
joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive)
|
||||
if err != nil {
|
||||
return err
|
||||
// a tar handed to a sink that stores tars is not walked at all: a compressed tar is decompressed
|
||||
// once into the sink's file and a plain one is copied, and the entries are first looked at when
|
||||
// the file is indexed. Where the sink stores entries rather than a tar there is nothing to copy
|
||||
// into, so the tar is walked like any other archive.
|
||||
if verbatim, isVerbatim := sink.(VerbatimSink); isVerbatim {
|
||||
if plain, ok := tarStream(format, extractReader); ok {
|
||||
err = verbatim.CopyVerbatim(plain, &result, limits)
|
||||
if closeErr := plain.Close(); closeErr != nil {
|
||||
log.WithFields("archive", content.Name, "error", closeErr).Trace("unable to close decompressed tar stream")
|
||||
}
|
||||
return os.MkdirAll(joinedPath, 0o755)
|
||||
return finishExtraction(result, err)
|
||||
}
|
||||
}
|
||||
|
||||
if limits.MaxFileCount > 0 && result.FilesExtracted >= limits.MaxFileCount {
|
||||
return fmt.Errorf("archive file count limit reached (%d files)", limits.MaxFileCount)
|
||||
}
|
||||
|
||||
joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(joinedPath), 0o755); err != nil {
|
||||
return fmt.Errorf("unable to create parent dir: %w", err)
|
||||
}
|
||||
|
||||
return extractFileWithLimits(file, joinedPath, destDir, &result, limits)
|
||||
// anything else - 7z, rar, or a tar going into a sink that does not store tars - is walked entry
|
||||
// by entry
|
||||
err = extractor.Extract(ctx, extractReader, func(_ context.Context, file archives.FileInfo) error {
|
||||
return sink.AddEntry(file, &result, limits)
|
||||
})
|
||||
|
||||
sink.Finish(&result, limits)
|
||||
|
||||
return finishExtraction(result, err)
|
||||
}
|
||||
|
||||
// extractFileWithLimits extracts a single file from an archive, respecting size limits.
|
||||
// Symlinks are written as real symlinks (only when the target stays inside destDir);
|
||||
// non-regular file types other than symlinks are skipped to avoid writing bogus content.
|
||||
func extractFileWithLimits(file archives.FileInfo, destPath, destDir string, result *ExtractionResult, limits ExtractionLimits) error {
|
||||
mode := file.Mode()
|
||||
if mode.Type()&fs.ModeSymlink != 0 {
|
||||
if err := writeSafeSymlink(file.LinkTarget, destPath, destDir); err != nil {
|
||||
log.WithFields("entry", file.NameInArchive, "target", file.LinkTarget, "error", err).Debug("skipping unsafe symlink in archive")
|
||||
return nil
|
||||
}
|
||||
result.FilesExtracted++
|
||||
return nil
|
||||
}
|
||||
if !mode.IsRegular() {
|
||||
// skip device nodes, fifos, sockets, etc. — they can't safely round-trip
|
||||
log.WithFields("entry", file.NameInArchive, "mode", mode).Debug("skipping non-regular archive entry")
|
||||
return nil
|
||||
}
|
||||
|
||||
src, err := file.Open()
|
||||
// resolvesInsideRoot reports whether writing to path really lands inside root, following any symlink
|
||||
// that already exists along the way the same way the filesystem will.
|
||||
//
|
||||
// The one filesystem write left in this package is the archive's own bytes overflowing into its work
|
||||
// directory, under a name derived from where the archive was found - which inside another archive is
|
||||
// attacker-controlled text. That is what this now guards; entries are tar headers and traverse no
|
||||
// path at all. See overflowContent.
|
||||
//
|
||||
// It walks path one component at a time rather than calling filepath.EvalSymlinks on the whole
|
||||
// thing, because the leaf of an entry being extracted does not exist yet, which makes EvalSymlinks
|
||||
// fail rather than answer. The first component that does not exist ends the walk: nothing below it
|
||||
// exists either, so whatever is created there is created fresh under a path already known to be
|
||||
// inside the root.
|
||||
func resolvesInsideRoot(root, path string) (bool, error) {
|
||||
// the root itself may sit under a symlinked directory (a temp dir on macOS, for one), so
|
||||
// containment is judged against its real location
|
||||
realRoot, err := filepath.EvalSymlinks(root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to open archive entry %q: %w", file.NameInArchive, err)
|
||||
return false, fmt.Errorf("unable to resolve extraction root: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
dst, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create dest file %q: %w", destPath, err)
|
||||
return false, err
|
||||
}
|
||||
if !isWithin(filepath.Join(realRoot, rel), realRoot) {
|
||||
return false, nil
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
var reader io.Reader = src
|
||||
|
||||
// enforce per-archive total size limit
|
||||
if limits.MaxExtractionSizeBytes > 0 {
|
||||
remaining := limits.MaxExtractionSizeBytes - result.BytesWritten
|
||||
if remaining <= 0 {
|
||||
return fmt.Errorf("archive extraction size limit reached (%d bytes)", limits.MaxExtractionSizeBytes)
|
||||
current := realRoot
|
||||
for _, part := range strings.Split(filepath.ToSlash(rel), "/") {
|
||||
if part == "" || part == "." {
|
||||
continue
|
||||
}
|
||||
reader = io.LimitReader(src, remaining+1) // +1 to detect exceeding the limit
|
||||
current = filepath.Join(current, part)
|
||||
|
||||
info, err := os.Lstat(current)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
break
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
if info.Mode()&fs.ModeSymlink == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
resolved, err := resolveLink(current)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !isWithin(resolved, realRoot) {
|
||||
return false, nil
|
||||
}
|
||||
current = resolved
|
||||
}
|
||||
|
||||
n, err := io.Copy(dst, reader)
|
||||
result.BytesWritten += n
|
||||
result.FilesExtracted++
|
||||
|
||||
if limits.MaxExtractionSizeBytes > 0 && result.BytesWritten > limits.MaxExtractionSizeBytes {
|
||||
return fmt.Errorf("archive extraction size limit reached (%d bytes)", limits.MaxExtractionSizeBytes)
|
||||
}
|
||||
|
||||
return err
|
||||
return isWithin(current, realRoot), nil
|
||||
}
|
||||
|
||||
// writeSafeSymlink creates a symlink at destPath only if target is relative and
|
||||
// resolves — relative to destPath's directory — to a location inside destDir.
|
||||
// resolveLink returns where a symlink really points, falling back to the lexically cleaned target
|
||||
// when the target does not exist yet - a dangling link still redirects a write.
|
||||
func resolveLink(path string) (string, error) {
|
||||
target, err := os.Readlink(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !filepath.IsAbs(target) {
|
||||
target = filepath.Join(filepath.Dir(path), target)
|
||||
}
|
||||
resolved, err := filepath.EvalSymlinks(target)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Clean(target), nil
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
// isWithin reports whether path is root or sits beneath it, compared on a path boundary rather than
|
||||
// as a string prefix so a sibling sharing the prefix does not pass.
|
||||
func isWithin(path, root string) bool {
|
||||
rel, err := filepath.Rel(filepath.Clean(root), filepath.Clean(path))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)))
|
||||
}
|
||||
|
||||
// writeSafeSymlink creates a symlink at destPath only if target is relative and really resolves,
|
||||
// relative to destPath's directory, to a location inside destDir.
|
||||
//
|
||||
// Absolute targets are rejected outright: os.Symlink writes the literal target
|
||||
// string, so an absolute target like "/etc/passwd" would resolve on the host
|
||||
// filesystem when read, regardless of any safety check we performed against
|
||||
// destDir at extraction time.
|
||||
//
|
||||
// NOTE: this guards against the single-symlink-escape pattern, but a malicious
|
||||
// archive can still attempt a multi-step chain where two cooperating link
|
||||
// entries together resolve outside destDir (e.g., one entry creates a benign
|
||||
// link, a later entry writes through it). Fully preventing that requires
|
||||
// either openat/O_NOFOLLOW per path component during extraction or
|
||||
// transitively resolving every newly-created symlink against all prior ones
|
||||
// — neither is implemented here.
|
||||
// A relative target is resolved against the REAL location of destPath's directory, not against its
|
||||
// lexical one, because two cooperating entries can otherwise each pass on their own and together
|
||||
// point outside: a link to "." followed by a link to ".." inside it resolves lexically to the root
|
||||
// while really landing above it, and repeating the pair climbs a level each time. The parent of
|
||||
// every write is checked the same way before the write happens, so a link that slipped through here
|
||||
// still could not be written through - this is the earlier of two refusals, not the only one.
|
||||
//
|
||||
// It has no caller since an archive's entries became headers in one tar rather than files on disk: a
|
||||
// symlink entry is now a header write, which creates nothing and traverses nothing, and where the
|
||||
// link points is decided inside the archive's own filetree. It is kept rather than retired because
|
||||
// that is a judgement about every path that could write a link, not about this one call site, and
|
||||
// because the reasoning above is the expensive part to reconstruct.
|
||||
//
|
||||
//nolint:unused // retained deliberately; see the note above
|
||||
func writeSafeSymlink(target, destPath, destDir string) error {
|
||||
if target == "" {
|
||||
return fmt.Errorf("empty link target")
|
||||
@@ -275,11 +414,99 @@ func writeSafeSymlink(target, destPath, destDir string) error {
|
||||
if filepath.IsAbs(target) {
|
||||
return fmt.Errorf("absolute symlink target not allowed")
|
||||
}
|
||||
resolved := filepath.Join(filepath.Dir(destPath), target)
|
||||
cleanRoot := filepath.Clean(destDir)
|
||||
cleanResolved := filepath.Clean(resolved)
|
||||
if cleanResolved != cleanRoot && !strings.HasPrefix(cleanResolved, cleanRoot+string(filepath.Separator)) {
|
||||
|
||||
realRoot, err := filepath.EvalSymlinks(destDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to resolve extraction root: %w", err)
|
||||
}
|
||||
realParent, err := filepath.EvalSymlinks(filepath.Dir(destPath))
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to resolve link parent: %w", err)
|
||||
}
|
||||
if !isWithin(realParent, realRoot) {
|
||||
return fmt.Errorf("symlink parent escapes extraction root")
|
||||
}
|
||||
|
||||
resolved := filepath.Join(realParent, target)
|
||||
if !isWithin(resolved, realRoot) {
|
||||
return fmt.Errorf("symlink target escapes extraction root")
|
||||
}
|
||||
// the target itself may run through links planted by earlier entries
|
||||
if existing, err := filepath.EvalSymlinks(resolved); err == nil && !isWithin(existing, realRoot) {
|
||||
return fmt.Errorf("symlink target resolves outside extraction root")
|
||||
}
|
||||
|
||||
return os.Symlink(target, destPath)
|
||||
}
|
||||
|
||||
// zipEOCDSignature is the end-of-central-directory record's signature. A zip is defined by this
|
||||
// record: it is what a reader seeks to in order to find the central directory, which is why a zip
|
||||
// with something concatenated in front of it still opens.
|
||||
var zipEOCDSignature = []byte{'P', 'K', 0x05, 0x06}
|
||||
|
||||
// maxZipEOCDSearch is how far back the record can be. The EOCD is 22 bytes and ends with a comment
|
||||
// of up to 65535 bytes, so it starts no earlier than this many bytes from the end.
|
||||
const maxZipEOCDSearch = 22 + 65535
|
||||
|
||||
// zipLocalFileHeaderSignature starts every entry in a zip. In a self-extracting archive the first
|
||||
// one follows the stub, so it is what a bounded read of the head can find.
|
||||
var zipLocalFileHeaderSignature = []byte{'P', 'K', 0x03, 0x04}
|
||||
|
||||
// MaxAppendedArchiveStubBytes is how far into a file a hidden archive is looked for. A Spring Boot
|
||||
// launcher stub is about 9 KB, so this is generous by a wide margin; a stub longer than this hides
|
||||
// its archive from the walk, which is the cost of not reading every candidate to the end.
|
||||
const MaxAppendedArchiveStubBytes = 64 * 1024
|
||||
|
||||
// MayHideAnAppendedArchive reports whether the head of this stream contains a zip local file header,
|
||||
// and returns a reader that replays what it consumed.
|
||||
//
|
||||
// It exists to make the candidate set affordable. A file whose head sniffs as a script, an
|
||||
// executable, or as nothing recognizable may still be an archive wearing a stub, but almost none of
|
||||
// them are - and the authoritative test, the end-of-central-directory record, is at the far end of
|
||||
// the file, so applying it to every such candidate means reading every executable in a scan to its
|
||||
// end. Measured on a stock ruby image that doubled the scan.
|
||||
//
|
||||
// So the head is checked first, cheaply and with a bound, and only a file that shows an entry
|
||||
// signature there is read in full and put to the real test. This narrows what is examined; it never
|
||||
// admits anything, because a file that passes here is still refused by the extractor unless it ends
|
||||
// in a valid central directory.
|
||||
func MayHideAnAppendedArchive(r io.Reader) (io.Reader, bool) {
|
||||
head := make([]byte, MaxAppendedArchiveStubBytes)
|
||||
n, err := io.ReadFull(r, head)
|
||||
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return io.MultiReader(bytes.NewReader(head[:n]), r), false
|
||||
}
|
||||
head = head[:n]
|
||||
return io.MultiReader(bytes.NewReader(head), r), bytes.Contains(head, zipLocalFileHeaderSignature)
|
||||
}
|
||||
|
||||
// HasZipEndOfCentralDirectory reports whether the content ends in a zip end-of-central-directory
|
||||
// record, which is the definition of a zip and the only part of one whose position is knowable
|
||||
// without parsing.
|
||||
//
|
||||
// This is what identifies an archive that content sniffing types by its head: a self-extracting
|
||||
// archive is a launcher stub with a zip concatenated onto it, so its first bytes are the stub's and
|
||||
// no magic-at-offset-zero test can see the archive. The record at the end is unambiguous, and
|
||||
// mholt/archives extracts such a file correctly once it is asked to - it reads through
|
||||
// archive/zip, which locates the central directory from the end.
|
||||
//
|
||||
// It is a content test, so it cannot make a non-archive extractable: a file named `notes.zip` that
|
||||
// is prose has no such record and is refused here exactly as it is refused by format identification.
|
||||
func HasZipEndOfCentralDirectory(r ReaderAtSeeker) bool {
|
||||
size, err := r.Seek(0, io.SeekEnd)
|
||||
if err != nil || size < int64(len(zipEOCDSignature)) {
|
||||
return false
|
||||
}
|
||||
|
||||
window := int64(maxZipEOCDSearch)
|
||||
if size < window {
|
||||
window = size
|
||||
}
|
||||
|
||||
buf := make([]byte, window)
|
||||
if _, err := r.ReadAt(buf, size-window); err != nil && !errors.Is(err, io.EOF) {
|
||||
return false
|
||||
}
|
||||
|
||||
return bytes.LastIndex(buf, zipEOCDSignature) >= 0
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user