package archive import ( "archive/tar" "archive/zip" "bytes" "compress/gzip" "context" "errors" "fmt" "io" "io/fs" "os" "path/filepath" "sort" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/anchore/syft/syft/cataloging" ) // fileContent opens an archive already on disk as Content, which is what the extractors take now // that an archive's bytes may be held in memory instead. Registered for close so a test that opens // several does not leak handles. func fileContent(t *testing.T, path string) Content { t.Helper() c, err := OpenFileContent(path) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, c.Close()) }) return c } // diskCharge returns a charge against a disk limit of the given size, for a test that wants the byte // bound to bite without standing up a whole scan. func diskCharge(max int64) *Charge { return NewLimiter(Limits{MaxDiskBytes: max}).Charge() } // sortedNames fixes the entry order of a test archive. Limits are enforced as the walk proceeds, so // which entries land before a truncation is a function of entry order and a map's order is random. func sortedNames(files map[string]string) []string { names := make([]string, 0, len(files)) for name := range files { names = append(names, name) } sort.Strings(names) return names } // overflowEntry is one entry read back out of an overflow tar. Assertions are made against these rather // than against files in a directory, because an archive's entries are now headers in one file. type overflowEntry struct { name string body string linkTarget string mode fs.FileMode typeflag byte } // overflowFor makes the one tar an archive's entries are written into, in the test's own temp space. func overflowFor(t *testing.T) *OverflowTar { t.Helper() s, err := newOverflowTar(filepath.Join(t.TempDir(), overflowTarName)) require.NoError(t, err) t.Cleanup(func() { _ = s.Close() }) return s } // readOverflow reads an overflow tar back, keyed by entry name. It closes the tar first: the writer holds // it open for the header rewrites that give each entry the size that actually arrived. func readOverflow(t *testing.T, s *OverflowTar) map[string]overflowEntry { t.Helper() require.NoError(t, s.Close()) f, err := os.Open(s.Path()) require.NoError(t, err) defer f.Close() out := map[string]overflowEntry{} tr := tar.NewReader(f) for { hdr, err := tr.Next() if errors.Is(err, io.EOF) { break } require.NoError(t, err) body, err := io.ReadAll(tr) require.NoError(t, err) out[hdr.Name] = overflowEntry{ name: hdr.Name, body: string(body), linkTarget: hdr.Linkname, mode: hdr.FileInfo().Mode(), typeflag: hdr.Typeflag, } } return out } // overflowNames is the entry names an overflow tar holds, sorted. func overflowNames(t *testing.T, s *OverflowTar) []string { t.Helper() entries := readOverflow(t, s) names := make([]string, 0, len(entries)) for name := range entries { names = append(names, name) } sort.Strings(names) return names } func createTestZip(t *testing.T, dir string, files map[string]string) string { t.Helper() zipPath := filepath.Join(dir, "test.zip") f, err := os.Create(zipPath) require.NoError(t, err) defer f.Close() w := zip.NewWriter(f) for _, name := range sortedNames(files) { fw, err := w.Create(name) require.NoError(t, err) _, err = fw.Write([]byte(files[name])) require.NoError(t, err) } require.NoError(t, w.Close()) return zipPath } func createTestTarGz(t *testing.T, dir string, files map[string]string) string { t.Helper() tarPath := filepath.Join(dir, "test.tar.gz") f, err := os.Create(tarPath) require.NoError(t, err) defer f.Close() gw := gzip.NewWriter(f) tw := tar.NewWriter(gw) for _, name := range sortedNames(files) { content := files[name] hdr := &tar.Header{ Name: name, Mode: 0o644, Size: int64(len(content)), } require.NoError(t, tw.WriteHeader(hdr)) _, err = tw.Write([]byte(content)) require.NoError(t, err) } require.NoError(t, tw.Close()) require.NoError(t, gw.Close()) return tarPath } func TestZipExtractor_CanExtract(t *testing.T) { dir := t.TempDir() ctx := context.Background() zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) ext := &ZipExtractor{} assert.True(t, ext.CanExtract(ctx, fileContent(t, zipPath))) // non-zip file nonZipPath := filepath.Join(dir, "notazip.txt") require.NoError(t, os.WriteFile(nonZipPath, []byte("just text"), 0o644)) assert.False(t, ext.CanExtract(ctx, fileContent(t, nonZipPath))) } func TestZipExtractor_Extract(t *testing.T) { dir := t.TempDir() ctx := context.Background() files := map[string]string{ "file1.txt": "content1", "dir/file2.txt": "content2", } zipPath := createTestZip(t, dir, files) ext := &ZipExtractor{} overflow := overflowFor(t) result, err := ext.Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{}) require.NoError(t, err) assert.Equal(t, 2, result.FilesExtracted) // both entries are in the one tar, under their own names and with their own content entries := readOverflow(t, overflow) assert.Equal(t, "content1", entries["file1.txt"].body) assert.Equal(t, "content2", entries["dir/file2.txt"].body) // and the archive occupies exactly one filesystem entry, which is the whole point info, err := os.Stat(overflow.Path()) require.NoError(t, err) assert.True(t, info.Mode().IsRegular()) } func TestZipExtractor_Extract_FileLimitReached(t *testing.T) { dir := t.TempDir() ctx := context.Background() files := map[string]string{ "file1.txt": "a", "file2.txt": "b", "file3.txt": "c", } zipPath := createTestZip(t, dir, files) ext := &ZipExtractor{} overflow := overflowFor(t) // reaching a limit is a truncation, not a failure: what was written before the limit stays in the // tar so the caller can still catalog it. Two entries of one byte each cost two header blocks and // two data blocks, so four blocks of room admits two entries and refuses the third. result, err := ext.Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{ Charge: diskCharge(4 * 512), }) require.NoError(t, err) assert.True(t, result.Truncated()) assert.Equal(t, TruncatedByDiskLimit, result.Truncation) assert.Equal(t, 2, result.FilesExtracted) assert.Equal(t, []string{"file1.txt", "file2.txt"}, overflowNames(t, overflow), "the two entries written before the limit must still be readable") } func TestZipExtractor_Extract_SizeLimitReached(t *testing.T) { dir := t.TempDir() ctx := context.Background() files := map[string]string{ "file1.txt": strings.Repeat("x", 100), } zipPath := createTestZip(t, dir, files) ext := &ZipExtractor{} overflow := overflowFor(t) result, err := ext.Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{Charge: diskCharge(50)}) require.NoError(t, err) assert.True(t, result.Truncated()) assert.Equal(t, TruncatedByDiskLimit, result.Truncation) // the single entry blew the budget on its own, so it is dropped rather than left half-written: // a partial jar or pom that a cataloger parses anyway is worse than an absent one. The budget is // spent on the entry's header before any of its content, because a header is bytes on disk too. assert.Zero(t, result.FilesExtracted) assert.Zero(t, result.BytesWritten) assert.Empty(t, overflowNames(t, overflow), "the over-limit entry must not be left behind") } func TestTarExtractor_CanExtract(t *testing.T) { dir := t.TempDir() ctx := context.Background() tarPath := createTestTarGz(t, dir, map[string]string{"hello.txt": "world"}) ext := &TarExtractor{} assert.True(t, ext.CanExtract(ctx, fileContent(t, tarPath))) } func TestTarExtractor_Extract(t *testing.T) { dir := t.TempDir() ctx := context.Background() files := map[string]string{ "file1.txt": "content1", "dir/file2.txt": "content2", } tarPath := createTestTarGz(t, dir, files) ext := &TarExtractor{} overflow := overflowFor(t) result, err := ext.Extract(ctx, fileContent(t, tarPath), overflow, ExtractionLimits{}) require.NoError(t, err) // a compressed tar is decompressed once into the overflow file and never walked, so the extractor // counts nothing: how many entries this archive contributes is the index's answer, reported by // ExtractToResolver. What the extractor promises is that the file it produced is that tar. assert.Zero(t, result.FilesExtracted, "a tar is copied, not walked; the index does the counting") assert.NotZero(t, result.BytesWritten) entries := readOverflow(t, overflow) assert.Equal(t, "content1", entries["file1.txt"].body) assert.Equal(t, "content2", entries["dir/file2.txt"].body) } func TestFindExtractor(t *testing.T) { dir := t.TempDir() ctx := context.Background() extractors := DefaultExtractors() // zip file should match zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) ext := FindExtractor(ctx, extractors, fileContent(t, zipPath)) require.NotNil(t, ext) assert.IsType(t, &ZipExtractor{}, ext) // tar.gz should match tarPath := createTestTarGz(t, dir, map[string]string{"hello.txt": "world"}) ext = FindExtractor(ctx, extractors, fileContent(t, tarPath)) require.NotNil(t, ext) assert.IsType(t, &TarExtractor{}, ext) // non-archive should return nil textPath := filepath.Join(dir, "plain.txt") require.NoError(t, os.WriteFile(textPath, []byte("just text"), 0o644)) ext = FindExtractor(ctx, extractors, fileContent(t, textPath)) assert.Nil(t, ext) } func TestFindExtractor_NoReaders(t *testing.T) { // content no extractor claims yields nil rather than an error, which is how a file that matched a // broad mime filter but is not an archive gets skipped ctx := context.Background() dir := t.TempDir() path := filepath.Join(dir, "plain.txt") require.NoError(t, os.WriteFile(path, []byte("just text"), 0o644)) ext := FindExtractor(ctx, DefaultExtractors(), fileContent(t, path)) assert.Nil(t, ext) } func TestTarExtractor_CanExtract_NonTarFile(t *testing.T) { dir := t.TempDir() ctx := context.Background() // create a plain text file path := filepath.Join(dir, "plain.txt") require.NoError(t, os.WriteFile(path, []byte("not a tar"), 0o644)) ext := &TarExtractor{} assert.False(t, ext.CanExtract(ctx, fileContent(t, path))) } func TestTarExtractor_CanExtract_ZipFileReturnsFalse(t *testing.T) { dir := t.TempDir() ctx := context.Background() zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) ext := &TarExtractor{} // tar extractor should NOT claim to handle zip files assert.False(t, ext.CanExtract(ctx, fileContent(t, zipPath))) } func TestDefaultExtractionLimits(t *testing.T) { limits := DefaultExtractionLimits(cataloging.ArchiveSearchConfig{}) assert.Nil(t, limits.Charge, "the charge belongs to one archive, so the config cannot supply it") } func TestDefaultLimits(t *testing.T) { cfg := cataloging.ArchiveSearchConfig{ MaxMemoryBytes: 111, MaxDiskBytes: 222, } limits := DefaultLimits(cfg) assert.Equal(t, int64(111), limits.MaxMemoryBytes) assert.Equal(t, int64(222), limits.MaxDiskBytes) } func TestZipExtractor_Extract_ZipSlipPrevented(t *testing.T) { // An entry named "../../etc/passwd" no longer has anywhere to go: it becomes a field in a tar // header, and writing a header traverses no path. So the extraction succeeds and nothing is // created anywhere, where before the entry was refused outright and the archive failed with it. // Where the name still matters is the path the SBOM reports, and that is clamped when the tar is // indexed - see the archive resolver's own tests. dir := t.TempDir() ctx := context.Background() zipPath := filepath.Join(dir, "evil.zip") f, err := os.Create(zipPath) require.NoError(t, err) w := zip.NewWriter(f) fw, err := w.Create("../../etc/passwd") require.NoError(t, err) _, err = fw.Write([]byte("evil")) require.NoError(t, err) require.NoError(t, w.Close()) require.NoError(t, f.Close()) ext := &ZipExtractor{} overflow := overflowFor(t) result, err := ext.Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{}) require.NoError(t, err) assert.Equal(t, 1, result.FilesExtracted) // nothing is created outside the tar, at the traversed path or anywhere else _, statErr := os.Stat(filepath.Join(dir, "etc", "passwd")) assert.True(t, os.IsNotExist(statErr)) _, statErr = os.Stat(filepath.Join(filepath.Dir(overflow.Path()), "etc")) assert.True(t, os.IsNotExist(statErr)) // the name is carried through unsanitized, because this is the archive's own record of what it // held; sanitizing happens once, where the logical filetree is built assert.Equal(t, []string{"../../etc/passwd"}, overflowNames(t, overflow)) } func TestTarExtractor_Extract_SymlinkIsRecordedAsALink(t *testing.T) { // A symlink entry is recorded as a symlink header carrying its target, not as a regular file // containing the target as bytes and not as a link on the real filesystem. Nothing is created, so // there is nothing for a target to redirect; where the link points is decided inside the archive's // own filetree when the tar is indexed. dir := t.TempDir() ctx := context.Background() tarPath := filepath.Join(dir, "with-link.tar.gz") f, err := os.Create(tarPath) require.NoError(t, err) gw := gzip.NewWriter(f) tw := tar.NewWriter(gw) target := []byte("real content") require.NoError(t, tw.WriteHeader(&tar.Header{ Name: "real.txt", Mode: 0o644, Size: int64(len(target)), })) _, err = tw.Write(target) require.NoError(t, err) require.NoError(t, tw.WriteHeader(&tar.Header{ Name: "link.txt", Linkname: "real.txt", Typeflag: tar.TypeSymlink, Mode: 0o777, })) require.NoError(t, tw.Close()) require.NoError(t, gw.Close()) require.NoError(t, f.Close()) ext := &TarExtractor{} overflow := overflowFor(t) _, err = ext.Extract(ctx, fileContent(t, tarPath), overflow, ExtractionLimits{}) require.NoError(t, err) entries := readOverflow(t, overflow) link := entries["link.txt"] assert.Equal(t, byte(tar.TypeSymlink), link.typeflag, "link.txt must stay a link, not become a file") assert.Equal(t, "real.txt", link.linkTarget) assert.Empty(t, link.body, "a link carries a target, not content") assert.Equal(t, "real content", entries["real.txt"].body) // and no link exists on the real filesystem beside the tar _, statErr := os.Lstat(filepath.Join(filepath.Dir(overflow.Path()), "link.txt")) assert.True(t, os.IsNotExist(statErr)) } func TestTarExtractor_Extract_SymlinkEscapingRootCreatesNothing(t *testing.T) { // A link target climbing out of the archive used to be refused at extraction, because extraction // created a real link that a later entry could be written through. Now the entry is a header: the // target is recorded as the archive gave it and nothing is created, so there is no write to // redirect. The target is clamped to the archive's own root when the filetree is built, which is // what keeps it from naming the host - see the archive resolver's own tests. dir := t.TempDir() ctx := context.Background() tarPath := filepath.Join(dir, "evil-link.tar.gz") f, err := os.Create(tarPath) require.NoError(t, err) gw := gzip.NewWriter(f) tw := tar.NewWriter(gw) require.NoError(t, tw.WriteHeader(&tar.Header{ Name: "passwd", Linkname: "../../../../etc/passwd", Typeflag: tar.TypeSymlink, Mode: 0o777, })) require.NoError(t, tw.Close()) require.NoError(t, gw.Close()) require.NoError(t, f.Close()) ext := &TarExtractor{} overflow := overflowFor(t) _, err = ext.Extract(ctx, fileContent(t, tarPath), overflow, ExtractionLimits{}) require.NoError(t, err) // nothing on the real filesystem beside the tar _, statErr := os.Lstat(filepath.Join(filepath.Dir(overflow.Path()), "passwd")) assert.True(t, os.IsNotExist(statErr), "no link may be created for an escaping symlink") entries := readOverflow(t, overflow) assert.Equal(t, byte(tar.TypeSymlink), entries["passwd"].typeflag) assert.Equal(t, "../../../../etc/passwd", entries["passwd"].linkTarget) } func TestTarExtractor_Extract_AbsoluteSymlinkTargetCreatesNothing(t *testing.T) { // An absolute target had to be rejected while extraction created real links: os.Symlink writes the // literal string, so reading the link resolved on the host filesystem whatever was checked at // extraction time. A header write creates no link, so there is nothing that resolves anywhere; // the target is re-rooted inside the archive when the filetree is built. dir := t.TempDir() ctx := context.Background() tarPath := filepath.Join(dir, "absolute-link.tar.gz") f, err := os.Create(tarPath) require.NoError(t, err) gw := gzip.NewWriter(f) tw := tar.NewWriter(gw) require.NoError(t, tw.WriteHeader(&tar.Header{ Name: "shadow", Linkname: "/etc/shadow", Typeflag: tar.TypeSymlink, Mode: 0o777, })) require.NoError(t, tw.Close()) require.NoError(t, gw.Close()) require.NoError(t, f.Close()) ext := &TarExtractor{} overflow := overflowFor(t) _, err = ext.Extract(ctx, fileContent(t, tarPath), overflow, ExtractionLimits{}) require.NoError(t, err) _, statErr := os.Lstat(filepath.Join(filepath.Dir(overflow.Path()), "shadow")) assert.True(t, os.IsNotExist(statErr), "no link may be created for an absolute target") entries := readOverflow(t, overflow) assert.Equal(t, "/etc/shadow", entries["shadow"].linkTarget) } func TestZipExtractor_Extract_ReadOnlyDirectoryPermissions(t *testing.T) { // JARs may contain directory entries with read-only permissions (e.g., META-INF/ with mode 0o555). // The extractor must still be able to write files into those directories. dir := t.TempDir() ctx := context.Background() zipPath := filepath.Join(dir, "readonly-dirs.zip") f, err := os.Create(zipPath) require.NoError(t, err) w := zip.NewWriter(f) // create a directory entry with read-only permissions (0o555) dirHeader := &zip.FileHeader{ Name: "META-INF/", } dirHeader.SetMode(0o555) _, err = w.CreateHeader(dirHeader) require.NoError(t, err) // create a file inside that directory fileHeader := &zip.FileHeader{ Name: "META-INF/MANIFEST.MF", } fileHeader.SetMode(0o644) fw, err := w.CreateHeader(fileHeader) require.NoError(t, err) _, err = fw.Write([]byte("Manifest-Version: 1.0")) require.NoError(t, err) require.NoError(t, w.Close()) require.NoError(t, f.Close()) ext := &ZipExtractor{} overflow := overflowFor(t) result, err := ext.Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{}) require.NoError(t, err) // two entries: the directory and the file. A directory carries no bytes of its own, but its header // and padding are charged to the disk limit, which is what bounds an archive of nothing but // directories. assert.Equal(t, 2, result.FilesExtracted) // the directory's mode no longer decides whether the file beside it can be written - nothing is // written into a directory - but both entries must be recorded, modes and all entries := readOverflow(t, overflow) assert.Equal(t, "Manifest-Version: 1.0", entries["META-INF/MANIFEST.MF"].body) assert.Equal(t, fs.FileMode(0o555), entries["META-INF/"].mode.Perm()) assert.Equal(t, byte(tar.TypeDir), entries["META-INF/"].typeflag) } func TestTarExtractor_Extract_ReadOnlyDirectoryPermissions(t *testing.T) { // Tar archives may contain directory entries with read-only permissions. // The extractor must still be able to write files into those directories. dir := t.TempDir() ctx := context.Background() tarPath := filepath.Join(dir, "readonly-dirs.tar.gz") f, err := os.Create(tarPath) require.NoError(t, err) gw := gzip.NewWriter(f) tw := tar.NewWriter(gw) // write a directory entry with read-only permissions require.NoError(t, tw.WriteHeader(&tar.Header{ Name: "META-INF/", Typeflag: tar.TypeDir, Mode: 0o555, })) // write a file inside that directory content := []byte("Manifest-Version: 1.0") require.NoError(t, tw.WriteHeader(&tar.Header{ Name: "META-INF/MANIFEST.MF", Mode: 0o644, Size: int64(len(content)), })) _, err = tw.Write(content) require.NoError(t, err) require.NoError(t, tw.Close()) require.NoError(t, gw.Close()) require.NoError(t, f.Close()) ext := &TarExtractor{} overflow := overflowFor(t) _, err = ext.Extract(ctx, fileContent(t, tarPath), overflow, ExtractionLimits{}) require.NoError(t, err) entries := readOverflow(t, overflow) assert.Equal(t, "Manifest-Version: 1.0", entries["META-INF/MANIFEST.MF"].body) assert.Equal(t, fs.FileMode(0o555), entries["META-INF/"].mode.Perm()) } func TestZipExtractor_Extract_nonPositiveLimitDisablesOnlyThatLimit(t *testing.T) { // <= 0 means "no limit" per limit, so a caller can opt out of one without opting out of all of // them. Substituting a default here would silently re-impose a limit the user turned off. dir := t.TempDir() ctx := context.Background() files := map[string]string{} for i := range 20 { files[fmt.Sprintf("file%02d.txt", i)] = "x" } files["big.txt"] = strings.Repeat("y", 500) zipPath := createTestZip(t, dir, files) t.Run("the disk limit bites", func(t *testing.T) { result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflowFor(t), ExtractionLimits{ Charge: diskCharge(100), }) require.NoError(t, err) assert.Equal(t, TruncatedByDiskLimit, result.Truncation, "the disk limit must still bite") }) t.Run("unbounded extracts everything", func(t *testing.T) { result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflowFor(t), ExtractionLimits{}) require.NoError(t, err) assert.False(t, result.Truncated()) assert.Equal(t, len(files), result.FilesExtracted) }) } func TestZipExtractor_Extract_dataEdgeCases(t *testing.T) { ctx := context.Background() t.Run("empty archive is a non-event", func(t *testing.T) { dir := t.TempDir() zipPath := createTestZip(t, dir, map[string]string{}) result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflowFor(t), ExtractionLimits{}) require.NoError(t, err, "a valid archive containing nothing is not a failure") assert.Zero(t, result.FilesExtracted) assert.False(t, result.Truncated()) }) t.Run("zero-byte entry is recorded", func(t *testing.T) { dir := t.TempDir() zipPath := createTestZip(t, dir, map[string]string{"empty.txt": ""}) overflow := overflowFor(t) result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{}) require.NoError(t, err) assert.Equal(t, 1, result.FilesExtracted) // its header and the end-of-archive marker are bytes on disk even though the entry carries // none, which is exactly why the disk limit counts what is written rather than entry sizes assert.Equal(t, int64(3*tarBlockSize), result.BytesWritten) entries := readOverflow(t, overflow) require.Contains(t, entries, "empty.txt") assert.Empty(t, entries["empty.txt"].body) }) t.Run("absolute entry path is contained", func(t *testing.T) { // distinct from ../ traversal: an entry literally named /etc/hosts must land inside the // extraction dir, never at the host path dir := t.TempDir() var buf bytes.Buffer zw := zip.NewWriter(&buf) w, err := zw.Create("/etc/hosts") require.NoError(t, err) _, err = w.Write([]byte("pwned")) require.NoError(t, err) require.NoError(t, zw.Close()) zipPath := filepath.Join(dir, "abs.zip") require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0o600)) overflow := overflowFor(t) _, err = (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{}) require.NoError(t, err) // it is one entry in the tar and nothing at the host path; the leading slash is dropped when // the filetree is built, so it reads as a path inside this archive assert.Equal(t, "pwned", readOverflow(t, overflow)["/etc/hosts"].body) _, statErr := os.Stat(filepath.Join(dir, "etc", "hosts")) assert.True(t, os.IsNotExist(statErr)) }) t.Run("unicode and non-ascii entry names survive", func(t *testing.T) { dir := t.TempDir() names := map[string]string{ "café/naïve.txt": "accents", "日本語/ファイル.txt": "japanese", "emoji-🎉.txt": "emoji", } zipPath := createTestZip(t, dir, names) overflow := overflowFor(t) result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{}) require.NoError(t, err) assert.Equal(t, len(names), result.FilesExtracted) entries := readOverflow(t, overflow) for name, want := range names { require.Contains(t, entries, name, "entry %q must be recorded under its own name", name) assert.Equal(t, want, entries[name].body) } }) } func TestExtract_diskLimitTruncatesOneArchive(t *testing.T) { // the disk limit is enforced as the entries land, not between archives: a bound checked only // between archives lets one archive overshoot by everything it writes. // // The budget is expressed in tar blocks because that is now what an entry costs: a 512-byte header // plus its content padded up to the next block. Two entries here cost one block of header and one // of data each, so a budget of two and a half entries' worth admits the first whole and refuses // the second at its header. ctx := context.Background() files := map[string]string{ "a/small.txt": strings.Repeat("s", 50), "b/big.txt": strings.Repeat("b", 500), } dir := t.TempDir() archivePath := createTestZip(t, dir, files) overflow := overflowFor(t) charge := diskCharge(3 * tarBlockSize) result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, archivePath), overflow, ExtractionLimits{Charge: charge}) require.NoError(t, err, "reaching a limit is a truncation, not a failure") assert.Equal(t, TruncatedByDiskLimit, result.Truncation, "a limit breach reports its own reason, so the log can name which bound was reached") assert.Equal(t, 1, result.FilesExtracted) assert.Equal(t, int64(2*tarBlockSize), result.BytesWritten, "one header and one padded data block") _, disk := charge.Held() assert.Equal(t, int64(2*tarBlockSize), disk, "the limiter holds exactly what landed: the dropped entry's header is refunded") // the partial tar is still a readable tar: what was written before the limit filled stays, and the // entry that did not fit is absent rather than short entries := readOverflow(t, overflow) assert.Len(t, entries["a/small.txt"].body, 50) assert.NotContains(t, entries, "b/big.txt") } func TestTarExtractor_Extract_diskLimitTruncatesTheCopy(t *testing.T) { // a tar-family archive is copied rather than walked, so the disk limit stops it at a copy chunk // rather than on an entry boundary. What is left is a tar that ends part way through an entry, and // keeping the entries that precede that point is the indexer's job - see the archive resolver's // tests. Here the assertion is that the copy stops, says why, and holds only what it wrote. ctx := context.Background() dir := t.TempDir() // two entries either side of a copy chunk, so the limit can bite in the middle of the stream // rather than refusing the first chunk and writing nothing at all files := map[string]string{ "a/first.txt": strings.Repeat("a", 40*1024), "b/second.txt": strings.Repeat("b", 40*1024), } archivePath := createTestTarGz(t, dir, files) overflow := overflowFor(t) charge := diskCharge(64 * 1024) result, err := (&TarExtractor{}).Extract(ctx, fileContent(t, archivePath), overflow, ExtractionLimits{Charge: charge}) require.NoError(t, err) assert.Equal(t, TruncatedByDiskLimit, result.Truncation) assert.Equal(t, int64(64*1024), result.BytesWritten, "every byte written was charged, and no more") _, disk := charge.Held() assert.Equal(t, result.BytesWritten, disk) info, err := os.Stat(overflow.Path()) require.NoError(t, err) assert.Equal(t, result.BytesWritten, info.Size(), "the tar holds exactly what was charged for") } func TestExtract_diskLimitFallsWhenAnArchiveIsReleased(t *testing.T) { // the whole point of a limiter over a counter: two archives of the same size both extract in full // when the first is released between them, and the second is bounded when it is not ctx := context.Background() dir := t.TempDir() zipPath := createTestZip(t, dir, map[string]string{"a.txt": strings.Repeat("a", 200)}) // one entry of 200 bytes costs a header block, a padded data block and the end-of-archive marker: // four blocks. A limit of five leaves no room for a second archive while the first is held. const oneArchive = 4 * tarBlockSize limiter := NewLimiter(Limits{MaxDiskBytes: 5 * tarBlockSize}) first := limiter.Charge() result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflowFor(t), ExtractionLimits{Charge: first}) require.NoError(t, err) require.False(t, result.Truncated()) require.Equal(t, int64(oneArchive), result.BytesWritten) t.Run("a second archive is bounded while the first is still held", func(t *testing.T) { second := limiter.Charge() result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflowFor(t), ExtractionLimits{Charge: second}) require.NoError(t, err) assert.Equal(t, TruncatedByDiskLimit, result.Truncation) second.Release() }) t.Run("and extracts in full once the first is released", func(t *testing.T) { first.Release() _, disk := limiter.InUse() require.Zero(t, disk, "releasing every charge must take the limiter back to nothing") third := limiter.Charge() result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflowFor(t), ExtractionLimits{Charge: third}) require.NoError(t, err) assert.False(t, result.Truncated(), "a counter would have refused this; a limiter does not") assert.Equal(t, int64(oneArchive), result.BytesWritten) }) } // createDirOnlyZip writes a zip of nothing but directory entries. func createDirOnlyZip(t *testing.T, dir string, count int) string { t.Helper() zipPath := filepath.Join(dir, "dirs.zip") f, err := os.Create(zipPath) require.NoError(t, err) defer f.Close() w := zip.NewWriter(f) for i := range count { hdr := &zip.FileHeader{Name: fmt.Sprintf("d%04d/", i)} hdr.SetMode(fs.ModeDir | 0o755) _, err := w.CreateHeader(hdr) require.NoError(t, err) } require.NoError(t, w.Close()) return zipPath } // createDirOnlyTarGz writes a gzipped tar of nothing but directory entries. func createDirOnlyTarGz(t *testing.T, dir string, count int) string { t.Helper() tarPath := filepath.Join(dir, "dirs.tar.gz") f, err := os.Create(tarPath) require.NoError(t, err) defer f.Close() gw := gzip.NewWriter(f) tw := tar.NewWriter(gw) for i := range count { require.NoError(t, tw.WriteHeader(&tar.Header{ Name: fmt.Sprintf("d%04d/", i), Typeflag: tar.TypeDir, Mode: 0o755, })) } require.NoError(t, tw.Close()) require.NoError(t, gw.Close()) return tarPath } func TestExtract_directoryEntriesAreBoundedByDisk(t *testing.T) { // an archive of directory entries alone carries no content, so what bounds it is the disk limit // charging each entry's header and the tar's padding. That is the resource such an archive // actually consumes, and it is the only bound left now that the entry count is gone. ctx := context.Background() t.Run("zip", func(t *testing.T) { dir := t.TempDir() zipPath := createDirOnlyZip(t, dir, 50) overflow := overflowFor(t) // each directory entry is one header block; room for twelve blocks admits ten entries plus the // two end-of-archive blocks result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{ Charge: diskCharge(12 * tarBlockSize), }) require.NoError(t, err, "a bound is a truncation, not a failure") assert.Equal(t, TruncatedByDiskLimit, result.Truncation) assert.Less(t, result.FilesExtracted, 50, "the bound must stop the archive short") assert.NotEmpty(t, overflowNames(t, overflow), "what was written before the bound stays readable") }) t.Run("tar.gz is copied whole, so the disk limit is what stops it", func(t *testing.T) { // a tar is not walked entry by entry: the copy takes the whole archive, and the disk limit is // charged as those bytes land dir := t.TempDir() tarPath := createDirOnlyTarGz(t, dir, 50) overflow := overflowFor(t) result, err := (&TarExtractor{}).Extract(ctx, fileContent(t, tarPath), overflow, ExtractionLimits{ Charge: diskCharge(-1), }) require.NoError(t, err) assert.False(t, result.Truncated()) assert.Len(t, overflowNames(t, overflow), 50) }) } // symlinkEntry describes one archive entry for the escape tests: a symlink when linkTarget is set, // a regular file otherwise. type symlinkEntry struct { name string linkTarget string body string } func createSymlinkChainTarGz(t *testing.T, dir, name string, entries []symlinkEntry) string { t.Helper() path := filepath.Join(dir, name) f, err := os.Create(path) require.NoError(t, err) defer f.Close() gw := gzip.NewWriter(f) tw := tar.NewWriter(gw) for _, e := range entries { if e.linkTarget != "" { require.NoError(t, tw.WriteHeader(&tar.Header{ Name: e.name, Typeflag: tar.TypeSymlink, Linkname: e.linkTarget, Mode: 0o777, })) continue } require.NoError(t, tw.WriteHeader(&tar.Header{ Name: e.name, Mode: 0o644, Size: int64(len(e.body)), })) _, err := tw.Write([]byte(e.body)) require.NoError(t, err) } require.NoError(t, tw.Close()) require.NoError(t, gw.Close()) return path } func createSymlinkChainZip(t *testing.T, dir, name string, entries []symlinkEntry) string { t.Helper() path := filepath.Join(dir, name) f, err := os.Create(path) require.NoError(t, err) defer f.Close() w := zip.NewWriter(f) for _, e := range entries { hdr := &zip.FileHeader{Name: e.name} body := e.body if e.linkTarget != "" { // a zip stores a symlink as an entry whose mode carries the symlink bit and whose // content is the link target hdr.SetMode(fs.ModeSymlink | 0o777) body = e.linkTarget } else { hdr.SetMode(0o644) } fw, err := w.CreateHeader(hdr) require.NoError(t, err) _, err = fw.Write([]byte(body)) require.NoError(t, err) } require.NoError(t, w.Close()) return path } // findEscapedFiles returns every path under base, outside destDir, whose basename is name. func findEscapedFiles(t *testing.T, base, destDir, name string) []string { t.Helper() var found []string require.NoError(t, filepath.WalkDir(base, func(path string, entry fs.DirEntry, err error) error { if err != nil { return nil // an escaping symlink can make a walk stumble; that is not the assertion } if entry.IsDir() { return nil } if filepath.Base(path) != name { return nil } if rel, relErr := filepath.Rel(destDir, path); relErr == nil && !strings.HasPrefix(rel, "..") { return nil } found = append(found, path) return nil })) return found } func TestExtract_symlinkChainEscapeWritesNothing(t *testing.T) { // two cooperating link entries used to each pass a lexical check and together resolve outside the // extraction directory: "d" -> "." landed on the root, so "d/up" -> ".." was created AS // "/up" -> ".." and pointed above it, and a third entry was written through it. // // The chain needed real links on a real filesystem to work. Entries are headers now, so the whole // class is gone rather than closed: no link is created, so there is nothing for a later entry to // be written through, and no entry is written anywhere in the first place. The archive is kept in // the suite because "nothing outside the tar is created" is the property to keep asserting, and // this is the input that would have violated it. chain := []symlinkEntry{ {name: "d", linkTarget: "."}, {name: "d/up", linkTarget: ".."}, {name: "up/PWNED", body: "arbitrary write"}, } // three link pairs, each climbing one more level. A fix that only walked one level would pass a // one-level test and still let this through. escalating := []symlinkEntry{ {name: "a", linkTarget: "."}, {name: "a/b", linkTarget: ".."}, {name: "b/c", linkTarget: "."}, {name: "b/c/d", linkTarget: ".."}, {name: "b/d/e", linkTarget: "."}, {name: "b/d/e/f", linkTarget: ".."}, {name: "b/d/f/PWNED", body: "arbitrary write, three levels up"}, } for _, tc := range []struct { name string entries []symlinkEntry }{ {name: "one pair", entries: chain}, {name: "three pairs escalating", entries: escalating}, } { t.Run(tc.name, func(t *testing.T) { t.Run("tar.gz", func(t *testing.T) { assertNoEscape(t, createSymlinkChainTarGz, func() Extractor { return &TarExtractor{} }, "chain.tar.gz", tc.entries) }) t.Run("zip", func(t *testing.T) { assertNoEscape(t, createSymlinkChainZip, func() Extractor { return &ZipExtractor{} }, "chain.zip", tc.entries) }) }) } } func assertNoEscape(t *testing.T, build func(*testing.T, string, string, []symlinkEntry) string, newExtractor func() Extractor, archiveName string, entries []symlinkEntry) { t.Helper() // several levels of headroom above the extraction directory, all inside the test's own temp // tree, so an escape has somewhere to land and is still cleaned up base := t.TempDir() destDir := filepath.Join(base, "one", "two", "three", "contents") require.NoError(t, os.MkdirAll(destDir, 0o755)) archivePath := build(t, base, archiveName, entries) overflow, err := newOverflowTar(filepath.Join(destDir, overflowTarName)) require.NoError(t, err) t.Cleanup(func() { _ = overflow.Close() }) _, err = newExtractor().Extract(context.Background(), fileContent(t, archivePath), overflow, ExtractionLimits{}) require.NoError(t, err, "an unsafe entry is skipped, not a failure of the whole archive") assert.Empty(t, findEscapedFiles(t, base, destDir, "PWNED"), "nothing may be written outside the extraction directory") // and nothing was created inside it either, other than the one tar created, err := os.ReadDir(destDir) require.NoError(t, err) require.Len(t, created, 1) assert.Equal(t, overflowTarName, created[0].Name()) } func TestTarExtractor_Extract_legitimateRelativeSymlinkIsCarriedThrough(t *testing.T) { // containing an escape must not cost what real archives contain: a relative link into a sibling // directory of the archive's own tree keeps its target verbatim, so the filetree can resolve it. // That it then resolves is asserted where resolution happens, in the archive resolver's tests. dir := t.TempDir() archivePath := createSymlinkChainTarGz(t, dir, "legit.tar.gz", []symlinkEntry{ {name: "lib/real.txt", body: "real content"}, {name: "bin/link.txt", linkTarget: "../lib/real.txt"}, {name: "lib/sub/deep.txt", body: "deep"}, {name: "bin/deep.txt", linkTarget: "../lib/sub/deep.txt"}, }) overflow := overflowFor(t) _, err := (&TarExtractor{}).Extract(context.Background(), fileContent(t, archivePath), overflow, ExtractionLimits{}) require.NoError(t, err) entries := readOverflow(t, overflow) assert.Equal(t, "../lib/real.txt", entries["bin/link.txt"].linkTarget) assert.Equal(t, "../lib/sub/deep.txt", entries["bin/deep.txt"].linkTarget) assert.Equal(t, "real content", entries["lib/real.txt"].body) assert.Equal(t, "deep", entries["lib/sub/deep.txt"].body) } func TestExtract_siblingOfExtractionDirectoryWritesNothing(t *testing.T) { // the realistic form of the string-prefix hole SafeJoin was fixed for: an entry naming // "../contents-evil/..." shares the extraction directory's prefix and is outside it. // // Both extractors used to fail the archive here, because SafeJoin refused the entry. Nothing joins // an entry name to a path any more, so the entry is recorded as the name it claims and no // filesystem path is derived from it at all. The assertion that matters is unchanged: nothing is // written beside the extraction directory. ctx := context.Background() for _, tc := range []struct { name string build func(t *testing.T, dir string, files map[string]string) string extract func(path string, overflow *OverflowTar) (ExtractionResult, error) }{ { name: "zip", build: createTestZip, extract: func(path string, overflow *OverflowTar) (ExtractionResult, error) { return (&ZipExtractor{}).Extract(ctx, fileContent(t, path), overflow, ExtractionLimits{}) }, }, { name: "tar.gz", build: createTestTarGz, extract: func(path string, overflow *OverflowTar) (ExtractionResult, error) { return (&TarExtractor{}).Extract(ctx, fileContent(t, path), overflow, ExtractionLimits{}) }, }, } { t.Run(tc.name, func(t *testing.T) { base := t.TempDir() destDir := filepath.Join(base, "contents") require.NoError(t, os.MkdirAll(destDir, 0o755)) archivePath := tc.build(t, base, map[string]string{"../contents-evil/x.txt": "escaped"}) overflow, err := newOverflowTar(filepath.Join(base, overflowTarName)) require.NoError(t, err) t.Cleanup(func() { _ = overflow.Close() }) _, err = tc.extract(archivePath, overflow) require.NoError(t, err, "a hostile name is a header field, not a failure") _, statErr := os.Stat(filepath.Join(base, "contents-evil", "x.txt")) assert.True(t, os.IsNotExist(statErr), "nothing may be written to a sibling of the extraction directory") entries, err := os.ReadDir(destDir) require.NoError(t, err) assert.Empty(t, entries, "the extraction directory stays empty: entries live in the tar") assert.Equal(t, "escaped", readOverflow(t, overflow)["../contents-evil/x.txt"].body) }) } } // scriptPrefixedZip is a self-extracting archive in the shape Spring Boot produces: a launcher // script with a complete zip concatenated onto it. Built rather than committed, so what makes the // case is visible in the test. func scriptPrefixedZip(t *testing.T, prefix string, files map[string]string) []byte { t.Helper() var zbuf bytes.Buffer w := zip.NewWriter(&zbuf) for _, name := range sortedNames(files) { f, err := w.Create(name) require.NoError(t, err) _, err = f.Write([]byte(files[name])) require.NoError(t, err) } require.NoError(t, w.Close()) return append([]byte(prefix), zbuf.Bytes()...) } func bytesContent(name string, data []byte) Content { return Content{Name: name, Reader: bytes.NewReader(data)} } func Test_HasZipEndOfCentralDirectory(t *testing.T) { var plainZip bytes.Buffer zw := zip.NewWriter(&plainZip) f, err := zw.Create("a.txt") require.NoError(t, err) _, err = f.Write([]byte("hello")) require.NoError(t, err) require.NoError(t, zw.Close()) var tarGz bytes.Buffer gw := gzip.NewWriter(&tarGz) tw := tar.NewWriter(gw) require.NoError(t, tw.WriteHeader(&tar.Header{Name: "a.txt", Size: 5, Mode: 0o600})) _, err = tw.Write([]byte("hello")) require.NoError(t, err) require.NoError(t, tw.Close()) require.NoError(t, gw.Close()) tests := []struct { name string data []byte want bool }{ {"a plain zip", plainZip.Bytes(), true}, { "a zip behind a launcher script - the case content sniffing types by its head", scriptPrefixedZip(t, "#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n", map[string]string{"META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n"}), true, }, {"prose, whatever it is named", []byte("this is not an archive, it is a sentence"), false}, {"a tar.gz, which is an archive but not a zip", tarGz.Bytes(), false}, {"empty", nil, false}, {"shorter than the signature", []byte("PK"), false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { assert.Equal(t, tt.want, HasZipEndOfCentralDirectory(bytes.NewReader(tt.data))) }) } } func Test_ZipExtractor_canExtractAZipBehindALauncherScript(t *testing.T) { // the regression this closes: a Spring Boot executable jar sniffs as text/x-shellscript, so // format identification refuses it and every package inside is lost data := scriptPrefixedZip(t, "#!/bin/bash\necho launcher\nexit 0\n", map[string]string{ "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n", "BOOT-INF/lib/dep.jar": "not really a jar, just bytes", }) extractor := &ZipExtractor{} require.True(t, extractor.CanExtract(context.Background(), bytesContent("app.jar", data)), "a zip behind a launcher script must be recognized") // and it extracts, because archive/zip finds the central directory from the end overflow, err := newOverflowTar(filepath.Join(t.TempDir(), "app.jar.tar")) require.NoError(t, err) result, err := extractor.Extract(context.Background(), bytesContent("app.jar", data), overflow, ExtractionLimits{Charge: diskCharge(-1)}) require.NoError(t, err) assert.Positive(t, result.BytesWritten, "the entries behind the script prefix must actually be written") assert.Empty(t, string(result.Truncation)) } func Test_ZipExtractor_refusesANonArchiveThatNothingIdentifies(t *testing.T) { // the probe must not turn a file with no central directory into an archive. Note the name matters // here in a way it does not in a scan: identification matches on it, so `notes.zip` is claimed by // name before the probe is reached. That is why the scenario `misnamed non-archive is not // extracted` is asserted at the task level, where the candidate set decides - see // Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError. extractor := &ZipExtractor{} for _, name := range []string{"app.jar", "bundle", "launcher"} { t.Run(name, func(t *testing.T) { content := bytesContent(name, []byte("plain text, no central directory anywhere in it")) assert.False(t, extractor.CanExtract(context.Background(), content)) }) } }