Add nested archive cataloging (#5260)

Signed-off-by: Kudryavcev Nikolay <kydry.nikolau@gmail.com>
Signed-off-by: Thomas Bechtold <thomasbechtold@jpberlin.de>
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
Signed-off-by: Keith Zantow <kzantow@gmail.com>
Co-authored-by: Kudryavcev Nikolay <kydry.nikolau@gmail.com>
Co-authored-by: Thomas Bechtold <thomasbechtold@jpberlin.de>
Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>

internal/file/copy.go

@@ -6,13 +6,13 @@ import (
 	"io"
 )
 
-const perFileReadLimit = 2 * GB
+// PerFileReadLimit is the most read from any one archive entry, guarding against decompression bombs.
+const PerFileReadLimit = 2 * GB
 
-// safeCopy limits the copy from the reader. This is useful when extracting files from archives to
-// protect against decompression bomb attacks.
-func safeCopy(writer io.Writer, reader io.Reader) error {
-	numBytes, err := io.Copy(writer, io.LimitReader(reader, perFileReadLimit))
-	if numBytes >= perFileReadLimit {
+// SafeCopy bounds the copy to guard against decompression bombs when extracting from archives.
+func SafeCopy(writer io.Writer, reader io.Reader) error {
+	numBytes, err := io.Copy(writer, io.LimitReader(reader, PerFileReadLimit))
+	if numBytes >= PerFileReadLimit {
 		return fmt.Errorf("zip read limit hit (potential decompression bomb attack)")
 	}
 	// Propagate decompression / read errors up to the caller. io.Copy

internal/file/copy_test.go

@@ -29,7 +29,7 @@ func (r *errReader) Read(p []byte) (int, error) {
 func TestSafeCopy(t *testing.T) {
 	t.Run("clean copy returns nil", func(t *testing.T) {
 		var buf bytes.Buffer
-		if err := safeCopy(&buf, strings.NewReader("hello")); err != nil {
+		if err := SafeCopy(&buf, strings.NewReader("hello")); err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}
 		if got := buf.String(); got != "hello" {
@@ -38,12 +38,12 @@ func TestSafeCopy(t *testing.T) {
 	})
 
 	t.Run("propagates decompression error", func(t *testing.T) {
-		// #4806: safeCopy used to drop non-EOF errors, so the caller
+		// #4806: SafeCopy used to drop non-EOF errors, so the caller
 		// would persist a partial buffer as a successful extract and
 		// downstream catalogers silently read empty manifests.
 		sentinel := errors.New("flate: corrupt input before offset 42")
 		var buf bytes.Buffer
-		err := safeCopy(&buf, &errReader{data: []byte("partial"), err: sentinel})
+		err := SafeCopy(&buf, &errReader{data: []byte("partial"), err: sentinel})
 		if err == nil {
 			t.Fatalf("expected error to be returned, got nil")
 		}
@@ -56,7 +56,7 @@ func TestSafeCopy(t *testing.T) {
 		// The old code had a dead io.EOF branch that labelled clean
 		// reads as decompression bombs; keep the happy path clean.
 		var buf bytes.Buffer
-		err := safeCopy(&buf, io.LimitReader(strings.NewReader("abc"), 3))
+		err := SafeCopy(&buf, io.LimitReader(strings.NewReader("abc"), 3))
 		if err != nil {
 			t.Fatalf("unexpected error on clean copy: %v", err)
 		}

internal/file/tar_file_traversal.go

@@ -70,7 +70,7 @@ func ExtractGlobsFromTarToUniqueTempFile(ctx context.Context, archivePath, dir s
 		}
 		defer internal.CloseAndLogError(packedFile, archivePath)
 
-		if err := safeCopy(tempFile, packedFile); err != nil {
+		if err := SafeCopy(tempFile, packedFile); err != nil {
 			return fmt.Errorf("unable to copy source=%q for tar=%q: %w", file.Name(), archivePath, err)
 		}
 

internal/file/zip_file_traversal.go

@@ -101,7 +101,7 @@ func ExtractFromZipToUniqueTempFile(ctx context.Context, archivePath, dir string
 			return fmt.Errorf("unable to extract directories, only files: %s", file.NameInArchive)
 		}
 
-		if err := safeCopy(tempFile, zippedFile); err != nil {
+		if err := SafeCopy(tempFile, zippedFile); err != nil {
 			return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.NameInArchive, archivePath, err)
 		}
 
@@ -138,7 +138,7 @@ func ContentsFromZip(ctx context.Context, archivePath string, paths ...string) (
 		}
 
 		var buffer bytes.Buffer
-		if err := safeCopy(&buffer, zippedFile); err != nil {
+		if err := SafeCopy(&buffer, zippedFile); err != nil {
 			return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.NameInArchive, archivePath, err)
 		}
 
@@ -210,7 +210,7 @@ func extractSingleFile(file archives.FileInfo, expandedFilePath, archivePath str
 			}
 		}()
 
-		if err := safeCopy(outputFile, zippedFile); err != nil {
+		if err := SafeCopy(outputFile, zippedFile); err != nil {
 			return fmt.Errorf("unable to copy source=%q to dest=%q for zip=%q: %w", file.NameInArchive, outputFile.Name(), archivePath, err)
 		}
 	}