From 3e4b036e223ea6e0f2c049d346a354a943b843a1 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Mon, 19 Oct 2026 15:44:37 -0400 Subject: [PATCH] fix: keep RealPath as top-level, use FullPath and utility functions FullNestedPath, LocalPath, Path Signed-off-by: Keith Zantow --- internal/archive/extract.go | 18 +- internal/archive/extract_test.go | 131 +++--- internal/archive/helpers_test.go | 39 +- internal/archive/resolver.go | 121 +++-- internal/archive/resolver_test.go | 122 ++--- internal/archive/storage.go | 4 +- internal/archive/storage_test.go | 4 +- internal/archive/traversal.go | 68 ++- internal/archive/traversal_test.go | 64 +-- internal/cmptest/common_options.go | 2 +- internal/cmptest/location.go | 2 +- internal/os/feature_detection.go | 12 +- .../binary/binary_dependencies.go | 4 +- .../binary/shared_library_index.go | 31 +- internal/relationship/by_file_ownership.go | 22 +- .../relationship/by_file_ownership_test.go | 55 +++ internal/relationship/evident_by_test.go | 20 +- internal/relationship/index_test.go | 32 +- internal/task/archive_nesting_test.go | 233 +++------- internal/task/archive_tasks.go | 14 +- internal/task/archive_tasks_test.go | 317 ++----------- internal/task/file_tasks.go | 4 +- internal/task/file_tasks_test.go | 24 +- internal/task/package_task_factory.go | 4 +- internal/task/package_task_factory_test.go | 4 +- internal/task/scope_tasks.go | 7 +- internal/task/unknowns_tasks.go | 2 +- internal/testutils/archives.go | 84 ++++ internal/unknown/coordinate_error.go | 4 +- internal/unknown/coordinate_error_test.go | 4 +- internal/unknown/path_error_test.go | 18 +- schema/json/schema-16.1.12.json | 14 +- schema/json/schema-latest.json | 14 +- syft/archive_cataloger_test.go | 228 ++++------ syft/file/cataloger/executable/cataloger.go | 22 +- syft/file/cataloger/filecontent/cataloger.go | 4 +- syft/file/cataloger/filedigest/cataloger.go | 12 +- syft/file/cataloger/filemetadata/cataloger.go | 4 +- .../cataloger/internal/all_regular_files.go | 2 +- syft/file/coordinate_set.go | 2 +- syft/file/coordinates.go | 38 +- syft/file/location.go | 18 +- syft/file/location_set.go | 10 +- syft/file/location_set_test.go | 27 ++ syft/file/location_test.go | 58 ++- syft/file/locations.go | 4 +- syft/file/locations_test.go | 14 +- syft/file/mock_resolver.go | 14 +- .../common/spdxhelpers/to_format_model.go | 15 +- .../spdxhelpers/to_format_model_test.go | 43 +- .../common/spdxhelpers/to_syft_model.go | 14 +- .../common/spdxhelpers/to_syft_model_test.go | 12 +- .../common/spdxhelpers/to_syft_model_v3.go | 5 +- .../spdxhelpers/to_syft_model_v3_test.go | 4 +- syft/format/github/internal/model/model.go | 6 +- .../github/internal/model/model_test.go | 15 +- .../cyclonedxutil/helpers/component_test.go | 2 +- .../internal/spdxutil/helpers/source_info.go | 2 +- .../internal/testutil/file_relationships.go | 2 +- syft/format/spdxjson/encoder_test.go | 5 +- syft/format/syftjson/decoder_test.go | 10 +- syft/format/syftjson/encoder_test.go | 10 +- syft/format/syftjson/model/package.go | 8 +- syft/format/syftjson/to_format_model_test.go | 427 +++++------------- syft/format/syftjson/to_syft_model.go | 4 +- syft/format/syftjson/to_syft_model_test.go | 5 +- .../container_image_all_layers.go | 2 +- .../container_image_deep_squash_test.go | 10 +- .../fileresolver/container_image_squash.go | 2 +- syft/internal/fileresolver/excluding_file.go | 6 +- .../fileresolver/unindexed_directory.go | 2 +- syft/linux/identify_release.go | 6 +- syft/linux/supplement_release.go | 6 +- syft/pkg/cataloger/ai/identity_oci.go | 8 +- syft/pkg/cataloger/ai/parse_gguf_model.go | 2 +- .../cataloger/ai/parse_safetensors_model.go | 2 +- .../pkg/cataloger/ai/parse_safetensors_oci.go | 4 +- syft/pkg/cataloger/ai/processor.go | 4 +- syft/pkg/cataloger/alpine/parse_apk_db.go | 6 +- syft/pkg/cataloger/arch/parse_alpm_db.go | 6 +- .../binary/classifier_cataloger_test.go | 4 +- .../cataloger/binary/elf_package_cataloger.go | 8 +- syft/pkg/cataloger/binary/pe_package.go | 2 +- .../cataloger/binary/pe_package_cataloger.go | 2 +- syft/pkg/cataloger/bitnami/cataloger.go | 12 +- syft/pkg/cataloger/bitnami/package.go | 2 +- .../pkg/cataloger/bitnami/parse_components.go | 2 +- syft/pkg/cataloger/conda/cataloger.go | 2 +- syft/pkg/cataloger/cpp/parse_conaninfo.go | 5 +- syft/pkg/cataloger/cpp/parse_vcpkgmanifest.go | 4 +- syft/pkg/cataloger/debian/package.go | 8 +- syft/pkg/cataloger/debian/parse_dpkg_db.go | 4 +- .../cataloger/dotnet/deps_binary_cataloger.go | 20 +- syft/pkg/cataloger/dotnet/deps_json.go | 2 +- syft/pkg/cataloger/dotnet/libman_json.go | 4 +- syft/pkg/cataloger/dotnet/package.go | 2 +- syft/pkg/cataloger/elixir/parse_mix_lock.go | 2 +- syft/pkg/cataloger/generic/cataloger.go | 8 +- syft/pkg/cataloger/generic/cataloger_test.go | 8 +- syft/pkg/cataloger/gentoo/license.go | 8 +- .../gentoo/parse_portage_contents.go | 26 +- syft/pkg/cataloger/githubactions/package.go | 2 +- syft/pkg/cataloger/golang/licenses.go | 4 +- syft/pkg/cataloger/golang/parse_go_binary.go | 2 +- .../cataloger/golang/parse_go_binary_test.go | 111 +---- syft/pkg/cataloger/golang/parse_go_mod.go | 8 +- syft/pkg/cataloger/golang/scan_binary.go | 8 +- .../pkg/cataloger/haskell/parse_stack_lock.go | 2 +- .../pkg/cataloger/haskell/parse_stack_yaml.go | 2 +- .../homebrew/parse_homebrew_formula.go | 6 +- .../internal/binutils/branching_matcher.go | 2 +- .../cataloger/internal/binutils/classifier.go | 20 +- syft/pkg/cataloger/internal/dotnet/pe/pe.go | 2 +- .../internal/licenses/find_licenses.go | 4 +- .../internal/pkgtest/observing_resolver.go | 6 +- .../internal/pkgtest/test_generic_parser.go | 2 +- syft/pkg/cataloger/java/archive_entries.go | 4 +- syft/pkg/cataloger/java/archive_parser.go | 15 +- .../java/archive_parser_traversal_test.go | 57 +-- .../pkg/cataloger/java/archive_source_test.go | 18 +- syft/pkg/cataloger/java/cataloger.go | 3 +- .../java/graalvm_native_image_cataloger.go | 4 +- .../cataloger/java/internal/maven/resolver.go | 2 +- syft/pkg/cataloger/java/parse_jvm_release.go | 8 +- syft/pkg/cataloger/java/parse_pom_xml.go | 4 +- .../java/zip_wrapped_archive_parser.go | 2 +- syft/pkg/cataloger/javascript/package.go | 4 +- .../cataloger/javascript/parse_yarn_lock.go | 8 +- .../kernel/parse_linux_kernel_module_file.go | 6 +- syft/pkg/cataloger/lua/parse_rockspec.go | 2 +- syft/pkg/cataloger/nix/db_cataloger.go | 16 +- syft/pkg/cataloger/nix/db_cataloger_v10.go | 2 +- syft/pkg/cataloger/nix/derivation.go | 6 +- syft/pkg/cataloger/nix/package.go | 4 +- syft/pkg/cataloger/nix/store_cataloger.go | 16 +- syft/pkg/cataloger/perl/meta.go | 2 +- syft/pkg/cataloger/perl/parse_install_json.go | 4 +- syft/pkg/cataloger/perl/parse_packlist.go | 2 +- syft/pkg/cataloger/perl/parse_release_meta.go | 4 +- syft/pkg/cataloger/perl/perllocal.go | 4 +- syft/pkg/cataloger/perl/perllocal_test.go | 2 +- syft/pkg/cataloger/perl/processor.go | 2 +- .../cataloger/php/interpreter_cataloger.go | 4 +- syft/pkg/cataloger/python/dependency.go | 2 +- .../cataloger/python/parse_requirements.go | 4 +- syft/pkg/cataloger/python/parse_setup.go | 2 +- syft/pkg/cataloger/python/parse_wheel_egg.go | 10 +- .../python/parse_wheel_egg_metadata.go | 4 +- syft/pkg/cataloger/python/virtual_env.go | 6 +- .../pkg/cataloger/redhat/parse_rpm_archive.go | 4 +- syft/pkg/cataloger/redhat/parse_rpm_db.go | 4 +- syft/pkg/cataloger/rust/parse_audit_binary.go | 2 +- syft/pkg/cataloger/sbom/cataloger.go | 4 +- syft/pkg/cataloger/sbom/cataloger_test.go | 24 +- syft/pkg/cataloger/swipl/parse_pack.go | 2 +- syft/pkg/cataloger/terraform/parse_tf_lock.go | 2 +- syft/pkg/cataloger/wordpress/parse_plugin.go | 4 +- syft/pkg/collection.go | 38 +- syft/pkg/collection_test.go | 20 +- syft/pkg/license.go | 2 +- syft/pkg/license_set_test.go | 18 +- syft/pkg/package.go | 4 +- syft/pkg/package_test.go | 28 +- syft/source/snapsource/manifest.go | 2 +- test/rules/rules.go | 2 +- 165 files changed, 1349 insertions(+), 1970 deletions(-) create mode 100644 internal/testutils/archives.go diff --git a/internal/archive/extract.go b/internal/archive/extract.go index 1f33e88c0..592092d67 100644 --- a/internal/archive/extract.go +++ b/internal/archive/extract.go @@ -8,7 +8,6 @@ import ( "fmt" "io" "path" - "strings" "github.com/mholt/archives" @@ -22,11 +21,12 @@ import ( // ErrDiskLimitReached when the archive's own bytes cannot be placed within the disk limit. The caller // must Cleanup the result. // -// fileSystemID and archivePath are stamped onto every location the resolver returns: the filesystem -// the archive was found on, and the colon-delimited chain of archives from the scan root to this one. -func Extract(ctx context.Context, r io.Reader, fileSystemID, archivePath string, limiter *Limiter, exclusions Exclusions) (*Resolver, error) { +// archiveCoordinates are those of the archive file itself; every location the resolver returns +// extends them with its entry path (see file.Coordinates). +func Extract(ctx context.Context, r io.Reader, archiveCoordinates file.Coordinates, limiter *Limiter, exclusions Exclusions) (*Resolver, error) { charge := limiter.charge() - resolver := newResolver(ctx, fileSystemID, archivePath, charge) + resolver := newResolver(ctx, archiveCoordinates, charge) + archivePath := archiveCoordinates.FullNestedPath() // deferred so that a panic in a decoder still gives back the charge and removes any spill file succeeded := false @@ -43,7 +43,7 @@ func Extract(ctx context.Context, r io.Reader, fileSystemID, archivePath string, // the archive's own bytes are needed only until they are extracted and digested defer release() - format := identifyFormat(ctx, archiveFileName(archivePath), content) + format := identifyFormat(ctx, path.Base(archiveCoordinates.LocalPath()), content) if format == nil { return nil, nil } @@ -75,12 +75,6 @@ func (r *Resolver) acquire(content io.Reader) (ReaderAtSeeker, func(), error) { return r.open(&b), func() { r.discard(&b) }, nil } -// archiveFileName returns the archive's own file name from the end of its archive path, which format -// identification reads for its extension. -func archiveFileName(archivePath string) string { - return path.Base(archivePath[strings.LastIndexByte(archivePath, ':')+1:]) -} - // identifyFormat returns the format able to extract this content, or nil when it is not an archive. // // Identification from the head of the stream wins: a tar holding one jar ends in that jar's central diff --git a/internal/archive/extract_test.go b/internal/archive/extract_test.go index d2e092d07..488203c9b 100644 --- a/internal/archive/extract_test.go +++ b/internal/archive/extract_test.go @@ -118,28 +118,6 @@ func TestResolver_extract_diskLimitBitesMidEntry(t *testing.T) { assert.Equal(t, r.written, info.Size()) } -func TestResolver_extract_diskLimitFallsWhenAnArchiveIsReleased(t *testing.T) { - data := zipBytes(t, map[string]string{"a.txt": strings.Repeat("a", 200)}) - oneArchive := approxIndexBytes(tar.Header{Name: "a.txt"}) + 200 - limiter := NewLimiter(Limits{MaxDiskBytes: oneArchive * 3 / 2}) - - first, _ := extractBytes(t, data, "test.zip", limiter.charge()) - require.NoError(t, first.Truncated) - require.Equal(t, int64(200), first.written) - - second, _ := extractBytes(t, data, "test.zip", limiter.charge()) - assert.Error(t, second.Truncated, "a second archive is bounded while the first is still held") - second.Cleanup() - - first.Cleanup() - _, disk := limiter.InUse() - require.Zero(t, disk) - - third, _ := extractBytes(t, data, "test.zip", limiter.charge()) - assert.NoError(t, third.Truncated, "and extracts in full once the first is released") - assert.Equal(t, int64(200), third.written) -} - func TestResolver_extract_directoryOnlyArchivesAreBoundedByTheIndexEstimate(t *testing.T) { var entries []testEntry for i := range 50 { @@ -262,7 +240,7 @@ func TestResolver_extract_excludedEntriesAreNeverStored(t *testing.T) { func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) { data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"}) - extracted, err := Extract(context.Background(), bytes.NewReader(data), "parentFS", "app.war:some/path/app.zip", nil, nil) + extracted, err := Extract(context.Background(), bytes.NewReader(data), file.Coordinates{RealPath: "app.war", FileSystemID: "parentFS", FullPath: "app.war!/some/path/app.zip"}, nil, nil) require.NoError(t, err) require.NotNil(t, extracted) t.Cleanup(extracted.Cleanup) @@ -274,7 +252,7 @@ func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) { locations, err := extracted.FilesByGlob("**/*.txt") require.NoError(t, err) require.Len(t, locations, 1) - assert.Equal(t, file.Coordinates{RealPath: "dir/hello.txt", FileSystemID: "parentFS", ArchivePath: "app.war:some/path/app.zip"}, + assert.Equal(t, file.Coordinates{RealPath: "app.war", FileSystemID: "parentFS", FullPath: "app.war!/some/path/app.zip!/dir/hello.txt"}, locations[0].Coordinates) reader, err := extracted.FileContentsByLocation(locations[0]) @@ -284,39 +262,13 @@ func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) { assert.Equal(t, "hello world", string(body)) } -func TestExtract_notAnArchive(t *testing.T) { - extracted, err := Extract(context.Background(), strings.NewReader("this is not an archive"), "", "notes.txt", nil, nil) - require.NoError(t, err) - assert.Nil(t, extracted) -} - -func TestExtract_spillsUnderTheScansTempRootAndCleansUp(t *testing.T) { - ctx, root := scanContext(t) - limiter := NewLimiter(spillingLimits) - - extracted, err := Extract(ctx, bytes.NewReader(zipBytes(t, sampleFiles)), "", "app.zip", limiter, nil) - require.NoError(t, err) - require.NotNil(t, extracted) - t.Cleanup(extracted.Cleanup) - - require.Len(t, filesIn(t, root), 1) - _, disk := limiter.InUse() - assert.Positive(t, disk) - - extracted.Cleanup() - assert.Empty(t, filesIn(t, root)) - _, disk = limiter.InUse() - assert.Zero(t, disk, "cleanup releases everything the archive held") - extracted.Cleanup() -} - func TestExtract_heldContentIsReleasedOnceExtracted(t *testing.T) { // the archive's own bytes are only needed to extract and digest it; afterwards the limiter holds // the entries and the index estimate alone data := zipBytes(t, map[string]string{"hello.txt": "hi"}) limiter := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) - extracted, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil) + extracted, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil) require.NoError(t, err) t.Cleanup(extracted.Cleanup) @@ -332,7 +284,7 @@ func TestExtract_heldContentWrittenToDiskStaysUntilCleanup(t *testing.T) { data := zipBytes(t, map[string]string{"hello.txt": "hi"}) limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) - extracted, err := Extract(ctx, unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil) + extracted, err := Extract(ctx, unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil) require.NoError(t, err) t.Cleanup(extracted.Cleanup) @@ -354,7 +306,7 @@ func TestExtract_aTruncatedStreamKeepsWhatWasRead(t *testing.T) { data := tarGzBytes(t, map[string]string{"a.txt": "first", "b.bin": string(noise)}) cut := data[:len(data)/2] - extracted, err := Extract(context.Background(), bytes.NewReader(cut), "", "app.tar.gz", NewLimiter(unboundedLimits), nil) + extracted, err := Extract(context.Background(), bytes.NewReader(cut), file.NewCoordinates("app.tar.gz", ""), NewLimiter(unboundedLimits), nil) require.NoError(t, err) require.NotNil(t, extracted) t.Cleanup(extracted.Cleanup) @@ -362,7 +314,7 @@ func TestExtract_aTruncatedStreamKeepsWhatWasRead(t *testing.T) { assert.Error(t, extracted.Truncated) locations, err := extracted.FilesByGlob("**") require.NoError(t, err) - assert.Equal(t, []string{"a.txt"}, realPaths(locations)) + assert.Equal(t, []string{"a.txt"}, localPaths(locations)) } func TestExtract_aPanicStillReleasesEverything(t *testing.T) { @@ -371,7 +323,7 @@ func TestExtract_aPanicStillReleasesEverything(t *testing.T) { limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) require.Panics(t, func() { - _, _ = Extract(ctx, &panicsAfter{n: 4096}, "", "app.zip", limiter, nil) + _, _ = Extract(ctx, &panicsAfter{n: 4096}, file.NewCoordinates("app.zip", ""), limiter, nil) }) mem, disk := limiter.InUse() @@ -397,11 +349,11 @@ func TestExtract_archiveBytesThatCannotBePlacedAreSkipped(t *testing.T) { data := zipBytes(t, sampleFiles) limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}) - _, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil) + _, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil) assert.ErrorIs(t, err, ErrDiskLimitReached) // read in place, the same bytes cost nothing; only the entries are refused - extracted, err := Extract(context.Background(), bytes.NewReader(data), "", "app.zip", limiter, nil) + extracted, err := Extract(context.Background(), bytes.NewReader(data), file.NewCoordinates("app.zip", ""), limiter, nil) require.NoError(t, err) require.NotNil(t, extracted) t.Cleanup(extracted.Cleanup) @@ -411,24 +363,6 @@ func TestExtract_archiveBytesThatCannotBePlacedAreSkipped(t *testing.T) { assert.Empty(t, locations) } -func TestExtract_aZipBehindALauncherScript(t *testing.T) { - // a Spring Boot executable jar sniffs as a shell script; archive/zip still finds the central directory - data := append([]byte("#!/bin/bash\necho launcher\nexit 0\n"), zipBytes(t, map[string]string{ - "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n", - "BOOT-INF/lib/dep.jar": "not really a jar, just bytes", - })...) - - extracted, err := Extract(context.Background(), bytes.NewReader(data), "", "/opt/app.jar", nil, nil) - require.NoError(t, err) - require.NotNil(t, extracted) - t.Cleanup(extracted.Cleanup) - - assert.NoError(t, extracted.Truncated) - locations, err := extracted.FilesByGlob("**/*") - require.NoError(t, err) - assert.Len(t, locations, 2) -} - func Test_HasZipEndOfCentralDirectory(t *testing.T) { tests := []struct { name string @@ -521,7 +455,7 @@ func TestExtract_decompressionBombIsBoundedByTheLimits(t *testing.T) { ctx, root := scanContext(t) limiter := NewLimiter(limits) - r, err := Extract(ctx, bytes.NewReader(data), "", name, limiter, nil) + r, err := Extract(ctx, bytes.NewReader(data), file.NewCoordinates(name, ""), limiter, nil) require.NoError(t, err) require.NotNil(t, r) t.Cleanup(r.Cleanup) @@ -547,7 +481,7 @@ func TestExtract_entryCountBombIsBoundedByTheMemoryLimit(t *testing.T) { limits := Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1} limiter := NewLimiter(limits) - r, err := Extract(context.Background(), bytes.NewReader(manyEntryZip(t, 20_000)), "", "many.zip", limiter, nil) + r, err := Extract(context.Background(), bytes.NewReader(manyEntryZip(t, 20_000)), file.NewCoordinates("many.zip", ""), limiter, nil) require.NoError(t, err) t.Cleanup(r.Cleanup) @@ -575,7 +509,7 @@ func TestExtract_deepPathBombIsBoundedByTheMemoryLimit(t *testing.T) { limits := Limits{MaxMemoryBytes: 8 << 20, MaxDiskBytes: -1} limiter := NewLimiter(limits) - r, err := Extract(context.Background(), bytes.NewReader(zipBytes(t, files)), "", "deep.zip", limiter, nil) + r, err := Extract(context.Background(), bytes.NewReader(zipBytes(t, files)), file.NewCoordinates("deep.zip", ""), limiter, nil) require.NoError(t, err) t.Cleanup(r.Cleanup) @@ -584,3 +518,44 @@ func TestExtract_deepPathBombIsBoundedByTheMemoryLimit(t *testing.T) { peakMemory, _ := limiter.Peak() assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes) } + +func TestExtract_locationsKeepTheTopLevelRealPath(t *testing.T) { + data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"}) + + // a top-level archive's coordinates, with FullPath unset or set to RealPath, name entries alike + for _, archive := range []file.Coordinates{ + {RealPath: "/app.zip"}, + {RealPath: "/app.zip", FullPath: "/app.zip"}, + } { + extracted, err := Extract(context.Background(), bytes.NewReader(data), archive, nil, nil) + require.NoError(t, err) + require.NotNil(t, extracted) + t.Cleanup(extracted.Cleanup) + + locations, err := extracted.FilesByPath("dir/hello.txt") + require.NoError(t, err) + require.Len(t, locations, 1) + assert.Equal(t, file.Coordinates{RealPath: "/app.zip", FullPath: "/app.zip!/dir/hello.txt"}, locations[0].Coordinates) + assert.Equal(t, "dir/hello.txt", locations[0].LocalPath()) + + for _, loc := range []file.Location{ + locations[0], + file.NewLocation("dir/hello.txt"), + file.NewLocation("/dir/hello.txt"), + file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/app.zip", FullPath: "/app.zip!/dir/hello.txt"}), + } { + reader, err := extracted.FileContentsByLocation(loc) + require.NoError(t, err, "location %s", loc) + body, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "hello world", string(body)) + + metadata, err := extracted.FileMetadataByLocation(loc) + require.NoError(t, err, "location %s", loc) + assert.Equal(t, "/app.zip!/dir/hello.txt", metadata.Path) + } + + _, err = extracted.FileContentsByLocation(file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/other.zip", FullPath: "/other.zip!/dir/hello.txt"})) + assert.Error(t, err, "an entry of another archive is not read from this one") + } +} diff --git a/internal/archive/helpers_test.go b/internal/archive/helpers_test.go index 2cf950b0b..387bc2869 100644 --- a/internal/archive/helpers_test.go +++ b/internal/archive/helpers_test.go @@ -10,7 +10,6 @@ import ( "io" "io/fs" "os" - "path" "sort" "testing" @@ -179,7 +178,7 @@ func resolverIn(t testing.TB, charge *charge) (*Resolver, string) { t.Helper() root := t.TempDir() ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(root)) - r := newResolver(ctx, "", "test-archive", charge) + r := newResolver(ctx, file.NewCoordinates("test-archive", ""), charge) t.Cleanup(r.Cleanup) return r, root } @@ -197,7 +196,7 @@ func addAll(t testing.TB, r *Resolver, entries []testEntry) { func resolverOver(t testing.TB, maxInMemory int64, files map[string]string) *Resolver { t.Helper() r, _ := resolverIn(t, memCharge(maxInMemory)) - r.archivePath = "outer.jar" + r.archiveCoordinates = file.NewCoordinates("outer.jar", "") addAll(t, r, fileEntries(files)) return r } @@ -207,35 +206,11 @@ func resolverOver(t testing.TB, maxInMemory int64, files map[string]string) *Res func resolverFrom(t testing.TB, entries ...testEntry) *Resolver { t.Helper() r, _ := resolverIn(t, memCharge(1<<20)) - r.archivePath = "outer.jar" + r.archiveCoordinates = file.NewCoordinates("outer.jar", "") addAll(t, r, entries) return r } -// indexCostOf is what add charges for the given entries in name order: every listed node and every -// directory their paths imply. -func indexCostOf(entries map[string]string) int64 { - names := make([]string, 0, len(entries)) - for name := range entries { - names = append(names, name) - } - sort.Strings(names) - seen := map[string]bool{} - var total int64 - for _, name := range names { - p := path.Clean("/" + name) - for i := 1; i < len(p); i++ { - if p[i] == '/' && !seen[p[:i]] { - seen[p[:i]] = true - total += approxIndexBytesPerEntry - } - } - seen[p] = true - total += approxIndexBytes(tar.Header{Name: name}) - } - return total -} - // extractBytes extracts an archive into a resolver built over the given charge. func extractBytes(t testing.TB, data []byte, name string, charge *charge) (*Resolver, string) { t.Helper() @@ -316,11 +291,11 @@ func diskCharge(maxDisk int64) *charge { return NewLimiter(Limits{MaxDiskBytes: maxDisk}).charge() } -// realPaths lists the real path of every location, sorted. -func realPaths(locations []file.Location) []string { +// localPaths lists the local entry path of every location (rather than full nested archive path), sorted. +func localPaths(locations []file.Location) []string { var out []string for _, loc := range locations { - out = append(out, loc.RealPath) + out = append(out, loc.LocalPath()) } sort.Strings(out) return out @@ -342,7 +317,7 @@ func extractedResolver(t testing.TB, data []byte, limits Limits) (*Resolver, str t.Helper() ctx, root := scanContext(t) - r, err := Extract(ctx, bytes.NewReader(data), "", "app.zip", NewLimiter(limits), nil) + r, err := Extract(ctx, bytes.NewReader(data), file.NewCoordinates("app.zip", ""), NewLimiter(limits), nil) require.NoError(t, err) require.NotNil(t, r) t.Cleanup(r.Cleanup) diff --git a/internal/archive/resolver.go b/internal/archive/resolver.go index 64a0afd0c..81a93951b 100644 --- a/internal/archive/resolver.go +++ b/internal/archive/resolver.go @@ -28,7 +28,8 @@ import ( // into it, holding their content within the scan's limits. Entries are indexed in a stereoscope file // tree, the same index the directory and image resolvers search. // -// Paths are archive-relative with no leading slash. +// Locations it returns keep the outermost archive's RealPath and carry the entry's FullPath. Queries +// may name an entry either by its full path or by its path within the archive (see rootedPath). type Resolver struct { // Digests are of the archive file itself. Digests []file.Digest @@ -37,10 +38,9 @@ type Resolver struct { // or the decompression budget stopped extraction early, or an entry was too large or unreadable. Truncated error - fileSystemID string - archivePath string - charge *charge - budget *Budget + archiveCoordinates file.Coordinates + charge *charge + budget *Budget // content is held in memory while the memory limit admits it and in one spill file once it does // not. The file is created on first use, so an archive that stays in memory never touches the @@ -61,18 +61,17 @@ type Resolver struct { var _ file.Resolver = (*Resolver)(nil) -// newResolver returns an empty resolver whose locations carry the given fileSystemID and archivePath, -// spilling into the scan's temp root when ctx carries one (see internal/tmpdir). -func newResolver(ctx context.Context, fileSystemID, archivePath string, charge *charge) *Resolver { +// newResolver returns an empty resolver whose locations extend the archive's coordinates, spilling into +// the scan's temp root when ctx carries one (see internal/tmpdir). +func newResolver(ctx context.Context, archiveCoordinates file.Coordinates, charge *charge) *Resolver { return &Resolver{ - fileSystemID: fileSystemID, - archivePath: archivePath, - charge: charge, - budget: budgetFromContext(ctx), - tempDir: tmpdir.FromContext(ctx), - tree: filetree.New(), - index: filetree.NewIndex(), - content: map[stereoscopeFile.ID]*blob{}, + archiveCoordinates: archiveCoordinates, + charge: charge, + budget: budgetFromContext(ctx), + tempDir: tmpdir.FromContext(ctx), + tree: filetree.New(), + index: filetree.NewIndex(), + content: map[stereoscopeFile.ID]*blob{}, } } @@ -160,13 +159,13 @@ func (r *Resolver) add(hdr tar.Header, content io.Reader) error { return nil } -func (r *Resolver) store(entryPath string, hdr tar.Header, b *blob) error { - ref, err := r.addToTree(entryPath, hdr) +func (r *Resolver) store(rooted string, hdr tar.Header, b *blob) error { + ref, err := r.addToTree(rooted, hdr) if err != nil { return err } metadata := stereoscopeFile.NewMetadata(hdr, r.open(b)) - metadata.Path = reportedPath(entryPath) + metadata.Path = entryPath(rooted) r.index.Add(*ref, metadata) r.content[ref.ID()] = b return nil @@ -217,34 +216,63 @@ func (r *Resolver) finish() { r.search = filetree.NewSearchContext(r.tree, r.index) } -// reportedPath is how paths leave the resolver: archive-relative with no leading slash. -func reportedPath(rooted string) string { +// linkDestination is the rooted path a link points at inside the archive. A hard link names its +// target from the archive root, as does an absolute symlink, so "/etc/passwd" is an entry of this +// archive and never the host's. A relative symlink is joined onto the link's own directory, and +// cleaning against the root keeps "../../x" inside the archive. +func linkDestination(linkPath, linkname string, hard bool) string { + if hard || path.IsAbs(linkname) { + return path.Join("/", linkname) + } + return path.Join(path.Dir(linkPath), linkname) +} + +// entryPath is a node's path within the archive: archive-relative with no leading slash. +func entryPath(rooted string) string { return strings.TrimPrefix(rooted, "/") } -func rootedPath(userPath string) string { - return path.Clean("/" + userPath) +// fullPath is how paths leave the resolver: the archive's own full path, then the entry path. +func (r *Resolver) fullPath(rooted string) string { + return r.archiveCoordinates.FullNestedPath() + "!/" + entryPath(rooted) } -// location names a resolution: a link is reported at its target's path with the link's own path as -// the access path. -func (r *Resolver) location(res stereoscopeFile.Resolution) file.Location { - coordinates := file.Coordinates{ - RealPath: reportedPath(string(res.RealPath)), - FileSystemID: r.fileSystemID, - ArchivePath: r.archivePath, +// rootedPath is the tree path a query names, which is either form a path takes: a full path is trimmed to +// the entry path within this archive, and anything else (a LocalPath) is taken as archive-relative +// already. The archive's full path with a trailing "!" (what path.Dir gives for an entry at the root) +// names the root. +func (r *Resolver) rootedPath(userPath string) string { + archivePath := r.archiveCoordinates.FullNestedPath() + "!" + if userPath == archivePath { + return "/" } - accessPath := reportedPath(string(res.RequestPath)) - if accessPath == "" || accessPath == coordinates.RealPath { + return path.Clean("/" + strings.TrimPrefix(userPath, archivePath+"/")) +} + +// coordinates name an entry by the outermost archive's real path, which never includes a path inside an +// archive, and its full path through every archive. +func (r *Resolver) coordinates(rooted string) file.Coordinates { + return file.Coordinates{ + RealPath: r.archiveCoordinates.RealPath, + FileSystemID: r.archiveCoordinates.FileSystemID, + FullPath: r.fullPath(rooted), + } +} + +// location names a resolution: a link is reported at its target's path with the link's own full path +// as the access path. +func (r *Resolver) location(res stereoscopeFile.Resolution) file.Location { + coordinates := r.coordinates(string(res.RealPath)) + if res.RequestPath == "" || res.RequestPath == res.RealPath { return file.NewLocationFromCoordinates(coordinates) } - return file.NewVirtualLocationFromCoordinates(coordinates, accessPath) + return file.NewVirtualLocationFromCoordinates(coordinates, r.fullPath(string(res.RequestPath))) } // resolve finds the file a path names, following links, and reports false for directories and // paths the archive never listed. func (r *Resolver) resolve(userPath string) (*stereoscopeFile.Resolution, bool) { - res, err := r.search.SearchByPath(rootedPath(userPath), filetree.FollowBasenameLinks) + res, err := r.search.SearchByPath(r.rootedPath(userPath), filetree.FollowBasenameLinks) if err != nil || res == nil || !res.HasReference() { return nil, false } @@ -256,27 +284,34 @@ func (r *Resolver) resolve(userPath string) (*stereoscopeFile.Resolution, bool) } func (r *Resolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { - res, ok := r.resolve(location.RealPath) + res, ok := r.resolve(location.FullNestedPath()) if !ok { - return nil, fmt.Errorf("no archive entry for path=%q", location.RealPath) + return nil, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath()) } return entryReader{r.open(r.content[res.ID()])}, nil } func (r *Resolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) { - exists, res, err := r.tree.File(stereoscopeFile.Path(rootedPath(location.RealPath))) + rooted := r.rootedPath(location.FullNestedPath()) + exists, res, err := r.tree.File(stereoscopeFile.Path(rooted)) if err != nil || !exists || !res.HasReference() { - return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.RealPath) + return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath()) } entry, err := r.index.Get(*res.Reference) if err != nil { return file.Metadata{}, err } - return entry.Metadata, nil + // the index holds entry paths; callers get full paths, as with locations + metadata := entry.Metadata + metadata.Path = r.fullPath(rooted) + if metadata.LinkDestination != "" { + metadata.LinkDestination = r.fullPath(linkDestination(rooted, metadata.LinkDestination, metadata.Type == stereoscopeFile.TypeHardLink)) + } + return metadata, nil } func (r *Resolver) HasPath(userPath string) bool { - return r.tree.HasPath(stereoscopeFile.Path(rootedPath(userPath))) + return r.tree.HasPath(stereoscopeFile.Path(r.rootedPath(userPath))) } func (r *Resolver) FilesByPath(paths ...string) ([]file.Location, error) { @@ -297,7 +332,7 @@ func (r *Resolver) FilesByGlob(patterns ...string) ([]file.Location, error) { var out []file.Location seen := stereoscopeFile.NewFileReferenceSet() for _, pattern := range patterns { - results, err := r.searchByGlob(rootedPath(pattern)) + results, err := r.searchByGlob(r.rootedPath(pattern)) if err != nil { return nil, fmt.Errorf("invalid glob %q: %w", pattern, err) } @@ -330,8 +365,8 @@ func (r *Resolver) searchByGlob(pattern string) ([]stereoscopeFile.Resolution, e // merge into the SBOM. func sortLocations(locations []file.Location) { sort.Slice(locations, func(i, j int) bool { - if locations[i].RealPath != locations[j].RealPath { - return locations[i].RealPath < locations[j].RealPath + if a, b := locations[i].FullNestedPath(), locations[j].FullNestedPath(); a != b { + return a < b } return locations[i].AccessPath < locations[j].AccessPath }) diff --git a/internal/archive/resolver_test.go b/internal/archive/resolver_test.go index 8b4f98c08..70dd24fab 100644 --- a/internal/archive/resolver_test.go +++ b/internal/archive/resolver_test.go @@ -4,8 +4,8 @@ import ( "archive/tar" "bytes" "context" - "fmt" "io" + "path" "sort" "strings" "testing" @@ -55,31 +55,6 @@ func TestResolver_add_directoryEntriesHoldNoContent(t *testing.T) { assert.Empty(t, readStore(t, r)["lib"].body) } -func TestResolver_add_indexCostIsChargedEvenForEmptyContent(t *testing.T) { - // the index estimate is what bounds an archive of many empty entries - limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1}) - r, _ := resolverIn(t, limiter.charge()) - - hdr := regularHeader("a.txt", 0) - require.NoError(t, r.add(hdr, bytes.NewReader(nil))) - - mem, _ := limiter.InUse() - assert.Equal(t, approxIndexBytes(hdr), mem) - assert.Zero(t, r.heldInMemory()) -} - -func TestResolver_add_indexCostFallsBackToDiskWhenMemoryIsZero(t *testing.T) { - limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) - r, _ := resolverIn(t, limiter.charge()) - - hdr := regularHeader("a.txt", 0) - require.NoError(t, r.add(hdr, bytes.NewReader(nil))) - - mem, disk := limiter.InUse() - assert.Zero(t, mem) - assert.Equal(t, approxIndexBytes(hdr), disk) -} - func TestResolver_add_indexDoesNotFallBackToDiskWhenMemoryIsBounded(t *testing.T) { // the index lives in memory, so a bounded memory limit bounds it; only a zero limit defers to disk limiter := NewLimiter(Limits{MaxMemoryBytes: 1024, MaxDiskBytes: -1}) @@ -108,21 +83,6 @@ func TestResolver_add_indexMakesRoomBySpillingHeldContent(t *testing.T) { assert.Equal(t, strings.Repeat("a", 100), readStore(t, r)["a.txt"].body) } -func TestResolver_add_refusesEntriesWhenNoBudgetAdmitsTheIndex(t *testing.T) { - room := approxIndexBytes(regularHeader("x0", 0)) * 3 - r, _ := resolverIn(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: room}).charge()) - - var err error - admitted := 0 - for i := 0; i < 100 && err == nil; i++ { - if err = r.add(regularHeader(fmt.Sprintf("x%d", i), 0), bytes.NewReader(nil)); err == nil { - admitted++ - } - } - require.ErrorIs(t, err, ErrDiskLimitReached) - assert.Equal(t, 3, admitted) -} - func TestResolver_add_firstEntryAtAPathWins(t *testing.T) { r, _ := resolverIn(t, memCharge(1<<20)) @@ -153,22 +113,6 @@ func TestResolver_pathsAndContents(t *testing.T) { assert.Equal(t, "Manifest-Version: 1.0\n", string(body)) } -func TestResolver_contentsSurviveTheMoveToDisk(t *testing.T) { - // room for the index but only a few bytes of content, so the rest moves to disk - r := resolverOver(t, indexCostOf(jarEntriesFixture)+8, jarEntriesFixture) - - locations, err := r.FilesByGlob("**/*.jar") - require.NoError(t, err) - require.Len(t, locations, 1) - - reader, err := r.FileContentsByLocation(locations[0]) - require.NoError(t, err) - body, err := io.ReadAll(reader) - require.NoError(t, err) - require.NoError(t, reader.Close()) - assert.Equal(t, "PK\x03\x04 not really", string(body)) -} - func TestResolver_mimeTypesAreSniffedFromContent(t *testing.T) { r := resolverOver(t, 1<<20, map[string]string{ "notes.txt": "just some text", @@ -177,26 +121,7 @@ func TestResolver_mimeTypesAreSniffedFromContent(t *testing.T) { locations, err := r.FilesByMIMEType("text/x-shellscript") require.NoError(t, err) - assert.Equal(t, []string{"run.sh"}, realPaths(locations)) -} - -func TestResolver_allLocationsCoversEveryFile(t *testing.T) { - r := resolverOver(t, 1<<20, jarEntriesFixture) - - var got []string - for loc := range r.AllLocations(context.Background()) { - got = append(got, loc.RealPath) - } - sort.Strings(got) - - assert.Equal(t, []string{ - "BOOT-INF/lib/inner.jar", - "LICENSE", - "META-INF/MANIFEST.MF", - "META-INF/maven/com.example/lib/pom.properties", - "META-INF/maven/com.example/lib/pom.xml", - "com/example/Thing.class", - }, got) + assert.Equal(t, []string{"run.sh"}, localPaths(locations)) } func TestResolver_answersAreOrdered(t *testing.T) { @@ -227,14 +152,14 @@ func TestResolver_answersAreOrdered(t *testing.T) { var all []string for loc := range r.AllLocations(context.Background()) { - all = append(all, loc.RealPath) + all = append(all, loc.LocalPath()) } assert.True(t, sort.StringsAreSorted(all)) } var paths []string for _, loc := range first { - paths = append(paths, loc.RealPath) + paths = append(paths, loc.LocalPath()) } assert.True(t, sort.StringsAreSorted(paths), "ordered by path, not merely stable") } @@ -279,7 +204,7 @@ func globPaths(t *testing.T, r *Resolver, pattern string) []string { t.Helper() locations, err := r.FilesByGlob(pattern) require.NoError(t, err) - return realPaths(locations) + return localPaths(locations) } // several paths to one file answer as one location: the file's own path when it matched, otherwise @@ -399,8 +324,8 @@ func TestResolver_collapsedLocationReadsTheTarget(t *testing.T) { require.NoError(t, err) require.Len(t, locations, 1) - assert.Equal(t, "opt/real.bin", locations[0].RealPath, "content comes from the file") - assert.Equal(t, "opt/a-sym.jar", locations[0].AccessPath, "the path searched by is the one matched") + assert.Equal(t, "opt/real.bin", locations[0].LocalPath(), "content comes from the file") + assert.Equal(t, "outer.jar!/opt/a-sym.jar", locations[0].AccessPath, "the path searched by is the one matched") reader, err := r.FileContentsByLocation(locations[0]) require.NoError(t, err) @@ -412,6 +337,28 @@ func TestResolver_collapsedLocationReadsTheTarget(t *testing.T) { "a location standing for a link must still read the file's bytes, not the link's none") } +func TestResolver_fullPathsAreTrimmedToEntryPaths(t *testing.T) { + // catalogers build queries from the full paths the resolver hands out, e.g. path.Dir plus a sibling + r := resolverOver(t, 1<<20, map[string]string{"pom.xml": "a", "lib/dep.jar": "b"}) + + locations, err := r.FilesByPath("outer.jar!/lib/dep.jar") + require.NoError(t, err) + require.Len(t, locations, 1) + assert.Equal(t, "outer.jar!/lib/dep.jar", locations[0].AccessPath) + + locations, err = r.FilesByPath(path.Join(path.Dir(locations[0].AccessPath), "../pom.xml")) + require.NoError(t, err) + require.Len(t, locations, 1, "a sibling at the archive root is reached through the full path") + assert.Equal(t, "outer.jar!/pom.xml", locations[0].FullNestedPath()) + + assert.True(t, r.HasPath(path.Dir("outer.jar!/pom.xml")), "the directory of a root entry is the root") + assert.True(t, r.HasPath("lib/dep.jar"), "an entry path is still accepted") + + locations, err = r.FilesByGlob("outer.jar!/**/*.jar") + require.NoError(t, err) + require.Len(t, locations, 1) +} + func TestResolver_danglingLinkHasNoContent(t *testing.T) { r := resolverFrom(t, testEntry{name: "opt/gone.jar", link: "missing.jar"}) @@ -420,14 +367,15 @@ func TestResolver_danglingLinkHasNoContent(t *testing.T) { locations = append(locations, l) } require.Len(t, locations, 1) - assert.Equal(t, "opt/gone.jar", locations[0].RealPath) + assert.Equal(t, "opt/gone.jar", locations[0].LocalPath()) _, err := r.FileContentsByLocation(locations[0]) assert.Error(t, err, "a link to nothing has no content to read") metadata, err := r.FileMetadataByLocation(locations[0]) require.NoError(t, err) - assert.Equal(t, "missing.jar", metadata.LinkDestination, "where it pointed is still reported, as the archive named it") + assert.Equal(t, "outer.jar!/opt/missing.jar", metadata.LinkDestination, + "where it pointed is still reported, resolved against the link's own directory") } func TestResolver_mimeTypeAnswersOnlyWithFiles(t *testing.T) { @@ -443,7 +391,7 @@ func TestResolver_mimeTypeAnswersOnlyWithFiles(t *testing.T) { t.Helper() locations, err := r.FilesByMIMEType(types...) require.NoError(t, err) - return realPaths(locations) + return localPaths(locations) } assert.Equal(t, []string{"opt/real.jar"}, byMIME("application/zip"), @@ -465,7 +413,7 @@ func TestResolver_allLocationsKeepsEveryPath(t *testing.T) { var got []string for l := range r.AllLocations(context.Background()) { - got = append(got, l.RealPath) + got = append(got, l.LocalPath()) } assert.Equal(t, []string{"opt/gone.jar", "opt/real.jar", "opt/sym.jar"}, got, "every path the archive names, links included") @@ -489,7 +437,7 @@ func TestResolver_add_anEntryOverTheCapIsSkippedAndReadNoFurther(t *testing.T) { assert.ErrorContains(t, r.Truncated, "per-entry size cap of 1.0 KiB") locations, err := r.FilesByGlob("**") require.NoError(t, err) - assert.Equal(t, []string{"ok.txt"}, realPaths(locations)) + assert.Equal(t, []string{"ok.txt"}, localPaths(locations)) } type countingReader struct { diff --git a/internal/archive/storage.go b/internal/archive/storage.go index 26ea65650..bf77777bc 100644 --- a/internal/archive/storage.go +++ b/internal/archive/storage.go @@ -109,7 +109,7 @@ func (r *Resolver) spill() error { if err := r.openFile(); err != nil { return err } - log.WithFields("archive", r.archivePath, "held", len(r.held), "bytes", r.written). + log.WithFields("archive", r.archiveCoordinates.FullNestedPath(), "held", len(r.held), "bytes", r.written). Debug("archive content does not fit in memory; writing it to disk") for len(r.held) > 0 { @@ -188,7 +188,7 @@ func (r *Resolver) releaseStorage() { return } if err := r.file.Close(); err != nil { - log.WithFields("archive", r.archivePath, "error", err).Trace("unable to close archive temp file") + log.WithFields("archive", r.archiveCoordinates.FullNestedPath(), "error", err).Trace("unable to close archive temp file") } r.remove() r.charge.refundDisk(r.written) diff --git a/internal/archive/storage_test.go b/internal/archive/storage_test.go index 6dafe0c44..67faa1035 100644 --- a/internal/archive/storage_test.go +++ b/internal/archive/storage_test.go @@ -223,7 +223,7 @@ func TestResolver_put_discardRefundsWhatIsHeldInMemory(t *testing.T) { assert.Equal(t, "keep", readEntry(t, s.open(&keep))) } -func TestStorage_filesystemCostDoesNotTrackEntryCount(t *testing.T) { +func TestResolver_filesystemCostDoesNotTrackEntryCount(t *testing.T) { small, smallRoot := extractedResolver(t, manyEntryZip(t, 2), spillingLimits) large, largeRoot := extractedResolver(t, manyEntryZip(t, 5000), spillingLimits) @@ -238,7 +238,7 @@ func TestStorage_filesystemCostDoesNotTrackEntryCount(t *testing.T) { assert.Equal(t, int64(5000), large.written) } -func TestStorage_anEntryAtTheEndIsReadBySeeking(t *testing.T) { +func TestResolver_anEntryAtTheEndIsReadBySeeking(t *testing.T) { // the reader over an entry is a bounded window at its offset, not a re-read of what precedes it const filler = 64 * 1024 diff --git a/internal/archive/traversal.go b/internal/archive/traversal.go index 6d9d63705..87e61b62f 100644 --- a/internal/archive/traversal.go +++ b/internal/archive/traversal.go @@ -13,9 +13,8 @@ import ( // such as java can describe the archive it is inside without owning the recursion. type Traversal struct { // Location is the archive file's location within its parent filesystem, including the access path - // it was reached by. Its ArchivePath names the archives above it, so VirtualPath(Location) is the - // full chain from the scan root as seen through that access path. The ArchivePath of the files - // inside it is built from the real path instead. + // it was reached by, so VirtualPath(Location) is the chain from the scan root as seen through that + // access path. The files inside it extend its real coordinates instead (see ContainerPath). Location file.Location // Digests are of the archive file itself, taken during extraction; a cataloger running inside the @@ -31,41 +30,64 @@ type Traversal struct { Packages []pkg.Package } -// ContentsArchivePath is the ArchivePath every file inside the archive carries: the chain to it, built -// from real paths. Outside any archive (a nil Traversal) it is blank, as for files at the scan root. -func (t *Traversal) ContentsArchivePath() string { +// FullTraversalArchivePath is the archive's own full path, which is the ContainerPath of every file inside it. Outside +// any archive (a nil Traversal) it is blank, as for files on the scanned filesystem. +func (t *Traversal) FullTraversalArchivePath() string { if t == nil { return "" } - return VirtualPath(file.NewLocationFromCoordinates(t.Location.Coordinates)) + return t.Location.FullNestedPath() +} + +// ContainerPath returns the full path of the archive directly holding the file at c, or blank for a +// file on the scanned filesystem, whether or not its FullPath is set. +func ContainerPath(c file.Coordinates) string { + i := strings.LastIndex(c.FullPath, file.ArchivePathSeparator) + if i < 0 { + return "" + } + return c.FullPath[:i] +} + +// archiveEntries is the "!/"-delimited chain of entry paths after RealPath, reporting false for a file +// not inside an archive. RealPath is split off first, so a "!/" in it is not taken for a boundary. +func archiveEntries(c file.Coordinates) (string, bool) { + remain, ok := strings.CutPrefix(c.FullNestedPath(), c.RealPath) + if !ok { + return "", false + } + return strings.CutPrefix(remain, file.ArchivePathSeparator) } // VirtualPath returns the colon-delimited chain of archives from the scan root to loc, ending in loc -// itself: "app.war:WEB-INF/lib/dep.jar". Outside any archive it is the location's path unchanged. In an -// entry name '%' is escaped as "%25" and then ':' as "%3A", so splitting on ':' recovers the archive -// boundaries and no two entry names share an escaped form. The leading slash is dropped to match the -// java cataloger, which joins slash-less zip entry names onto the containing archive's path. +// itself as accessed: "app.war:WEB-INF/lib/dep.jar". Outside any archive it is the location's path +// unchanged. In an entry name '%' is escaped as "%25" and then ':' as "%3A", so splitting on ':' +// recovers the archive boundaries and no two entry names share an escaped form. Leading slashes are +// dropped to match the java cataloger, which joins slash-less zip entry names onto the containing +// archive's path. func VirtualPath(loc file.Location) string { - if loc.ArchivePath == "" { - return loc.Path() + accessPath := loc.AccessPath + if accessPath == "" { + accessPath = loc.FullNestedPath() } - return loc.ArchivePath + ":" + strings.TrimPrefix(escapeEntryPath(loc.Path()), "/") + entries, ok := archiveEntries(loc.Coordinates) + if !ok { + return accessPath + } + segments := strings.Split(entries, file.ArchivePathSeparator) + segments[len(segments)-1] = strings.TrimPrefix(accessPath, ContainerPath(loc.Coordinates)+file.ArchivePathSeparator) + for i, s := range segments { + segments[i] = strings.TrimPrefix(escapeEntryPath(s), "/") + } + return loc.RealPath + ":" + strings.Join(segments, ":") } -var ( - entryPathEscaper = strings.NewReplacer("%", "%25", ":", "%3A") - entryPathUnescaper = strings.NewReplacer("%3A", ":", "%25", "%") -) +var entryPathEscaper = strings.NewReplacer("%", "%25", ":", "%3A") func escapeEntryPath(p string) string { return entryPathEscaper.Replace(p) } -// UnescapeEntryPath reverses the escaping VirtualPath applies to one segment of the chain. -func UnescapeEntryPath(p string) string { - return entryPathUnescaper.Replace(p) -} - type traversalCtxKey struct{} func WithTraversal(ctx context.Context, t *Traversal) context.Context { diff --git a/internal/archive/traversal_test.go b/internal/archive/traversal_test.go index 62f48881c..15010c697 100644 --- a/internal/archive/traversal_test.go +++ b/internal/archive/traversal_test.go @@ -2,7 +2,6 @@ package archive import ( "context" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -18,16 +17,24 @@ func TestTraversalContextRoundTrip(t *testing.T) { ctx := WithTraversal(context.Background(), trav) require.Same(t, trav, TraversalFromContext(ctx)) - child := &Traversal{Location: locationIn("app.war", "/WEB-INF/lib/dep.jar")} + child := &Traversal{Location: locationIn("app.war", "WEB-INF/lib/dep.jar")} ctx = WithTraversal(ctx, child) require.Same(t, child, TraversalFromContext(ctx)) } -// locationIn is a location as the Index returns it: an entry path stamped with the containing chain. -func locationIn(archivePath, entryPath string) file.Location { - loc := file.NewLocation(entryPath) - loc.ArchivePath = archivePath - return loc +// locationIn is a location as the Resolver returns it: the outermost archive's real path, and the chain +// of entry paths within it. +func locationIn(realPath, archivePath string) file.Location { + return file.NewLocationFromCoordinates(file.Coordinates{RealPath: realPath, FullPath: realPath + file.ArchivePathSeparator + archivePath}) +} + +func TestContainerPath(t *testing.T) { + assert.Equal(t, "", ContainerPath(file.NewCoordinates("/app.war", ""))) + assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/app.war", FullPath: "/app.war"}), "a top-level file whose FullPath is set") + assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/odd!/name"}), "a separator in a top-level real path is not an archive boundary") + assert.Equal(t, "/app.war", ContainerPath(locationIn("/app.war", "WEB-INF/lib/dep.jar").Coordinates)) + assert.Equal(t, "/app.war!/WEB-INF/lib/dep.jar", + ContainerPath(locationIn("/app.war", "WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF").Coordinates)) } func TestVirtualPath(t *testing.T) { @@ -46,6 +53,11 @@ func TestVirtualPath(t *testing.T) { location: file.NewLocation("some/dir/example.jar"), expected: "some/dir/example.jar", }, + { + name: "outside an archive with FullPath set returns the path unchanged", + location: file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/odd!/name.jar", FullPath: "/odd!/name.jar"}), + expected: "/odd!/name.jar", + }, { name: "single level trims leading slash", location: locationIn("app.war", "/WEB-INF/lib/dep.jar"), @@ -58,7 +70,7 @@ func TestVirtualPath(t *testing.T) { }, { name: "multi-level chain", - location: locationIn("app.war:WEB-INF/lib/dep.jar", "/inner/most.jar"), + location: locationIn("app.war", "WEB-INF/lib/dep.jar!/inner/most.jar"), expected: "app.war:WEB-INF/lib/dep.jar:inner/most.jar", }, { @@ -72,11 +84,23 @@ func TestVirtualPath(t *testing.T) { location: locationIn("app.war", "lib/a:b:c.jar"), expected: "app.war:lib/a%3Ab%3Ac.jar", }, + { + // or the literal text %3A would collide with an escaped colon + name: "the escape character is itself escaped", + location: locationIn("outer.zip", "a%3Ab.jar"), + expected: "outer.zip:a%253Ab.jar", + }, { name: "access path is preferred over real path", location: file.NewVirtualLocation("/real/dep.jar", "/link/dep.jar"), expected: "/link/dep.jar", }, + { + name: "access path within an archive replaces the last entry", + location: file.NewVirtualLocationFromCoordinates( + file.Coordinates{RealPath: "app.war", FullPath: "app.war!/WEB-INF/lib/real.jar"}, "app.war!/WEB-INF/lib/link.jar"), + expected: "app.war:WEB-INF/lib/link.jar", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -84,27 +108,3 @@ func TestVirtualPath(t *testing.T) { }) } } - -func TestVirtualPath_theEscapeCharacterIsItselfEscaped(t *testing.T) { - // an entry path holding the literal text %3A must not collide with one holding a colon, or their - // contents share an ArchivePath and overwrite each other's file metadata - fromColon := VirtualPath(locationIn("outer.zip", "a:b.jar")) - fromLiteral := VirtualPath(locationIn("outer.zip", "a%3Ab.jar")) - - assert.Equal(t, "outer.zip:a%3Ab.jar", fromColon) - assert.Equal(t, "outer.zip:a%253Ab.jar", fromLiteral) -} - -func TestVirtualPath_escapingComposesAcrossNestingLevels(t *testing.T) { - level2 := VirtualPath(locationIn("outer.zip", "lib/mid:dle.war")) - require.Equal(t, "outer.zip:lib/mid%3Adle.war", level2) - - final := VirtualPath(locationIn(level2, "WEB-INF/lib/inn:er.jar")) - require.Equal(t, "outer.zip:lib/mid%3Adle.war:WEB-INF/lib/inn%3Aer.jar", final) - - assert.Equal(t, - []string{"outer.zip", "lib/mid%3Adle.war", "WEB-INF/lib/inn%3Aer.jar"}, - strings.Split(final, ":"), - "splitting on ':' must recover exactly the three real archive boundaries", - ) -} diff --git a/internal/cmptest/common_options.go b/internal/cmptest/common_options.go index 8e8c2c3ea..f9ea242b5 100644 --- a/internal/cmptest/common_options.go +++ b/internal/cmptest/common_options.go @@ -69,5 +69,5 @@ func locationSorter(a, b file.Location) int { return strings.Compare(a.AccessPath, b.AccessPath) } - return strings.Compare(a.RealPath, b.RealPath) + return strings.Compare(a.FullNestedPath(), b.FullNestedPath()) } diff --git a/internal/cmptest/location.go b/internal/cmptest/location.go index 1959926f0..2cc5dec0a 100644 --- a/internal/cmptest/location.go +++ b/internal/cmptest/location.go @@ -14,5 +14,5 @@ func DefaultLocationComparer(x, y file.Location) bool { } func LocationComparerWithoutLayer(x, y file.Location) bool { - return cmp.Equal(x.RealPath, y.RealPath) && cmp.Equal(x.AccessPath, y.AccessPath) + return cmp.Equal(x.FullNestedPath(), y.FullNestedPath()) && cmp.Equal(x.AccessPath, y.AccessPath) } diff --git a/internal/os/feature_detection.go b/internal/os/feature_detection.go index a836a2191..14405dda7 100644 --- a/internal/os/feature_detection.go +++ b/internal/os/feature_detection.go @@ -49,7 +49,7 @@ func findRhelFeatures(resolver file.Resolver, release *linux.Release) error { for _, contentManifestFile := range contentManifestFiles { found, err := fileMatchesAny(resolver, contentManifestFile, rhelEUSPatterns...) if err != nil { - return fmt.Errorf("unable to read content manifest from %s: %w", contentManifestFile.RealPath, err) + return fmt.Errorf("unable to read content manifest from %s: %w", contentManifestFile.FullNestedPath(), err) } if found { release.ExtendedSupport = true @@ -163,9 +163,9 @@ func hasActiveUbuntuESMService(resolver file.Resolver) (bool, error) { func hasEnabledESMServiceInStatus(resolver file.Resolver, location file.Location) (bool, error) { contents, err := resolver.FileContentsByLocation(location) if err != nil { - return false, fmt.Errorf("unable to read ubuntu-advantage status from %s: %w", location.RealPath, err) + return false, fmt.Errorf("unable to read ubuntu-advantage status from %s: %w", location.FullNestedPath(), err) } - defer internal.CloseAndLogError(contents, location.RealPath) + defer internal.CloseAndLogError(contents, location.FullNestedPath()) var status struct { Services []struct { @@ -174,7 +174,7 @@ func hasEnabledESMServiceInStatus(resolver file.Resolver, location file.Location } `json:"services"` } if err := json.NewDecoder(contents).Decode(&status); err != nil { - return false, fmt.Errorf("unable to parse ubuntu-advantage status from %s: %w", location.RealPath, err) + return false, fmt.Errorf("unable to parse ubuntu-advantage status from %s: %w", location.FullNestedPath(), err) } // pro status values are enabled/disabled/n-a/warning/—; only "enabled" proves the stream is active. @@ -200,8 +200,8 @@ func hasInstalledESMPackage(packages *pkg.Collection) bool { func fileMatchesAny(resolver file.Resolver, location file.Location, patterns ...*regexp.Regexp) (bool, error) { contents, err := resolver.FileContentsByLocation(location) if err != nil { - return false, fmt.Errorf("unable to read %s: %w", location.RealPath, err) + return false, fmt.Errorf("unable to read %s: %w", location.FullNestedPath(), err) } - defer internal.CloseAndLogError(contents, location.RealPath) + defer internal.CloseAndLogError(contents, location.FullNestedPath()) return internal.MatchAnyFromReader(contents, patterns...) } diff --git a/internal/relationship/binary/binary_dependencies.go b/internal/relationship/binary/binary_dependencies.go index 552f767e7..1b595f052 100644 --- a/internal/relationship/binary/binary_dependencies.go +++ b/internal/relationship/binary/binary_dependencies.go @@ -71,7 +71,7 @@ func compareElfBinaryPackages(s *sbom.SBOM) []artifact.ID { pkgsToDelete := make([]artifact.ID, 0) for _, elfPkg := range allElfPackages(s) { for _, loc := range onlyPrimaryEvidenceLocations(elfPkg) { - for _, otherPkg := range s.Artifacts.Packages.PackagesByPath(loc.RealPath) { + for _, otherPkg := range s.Artifacts.Packages.PackagesByPath(loc.FullNestedPath()) { // we only care about comparing binary packages to each other (not other types) if otherPkg.Type != pkg.BinaryPkg { continue @@ -130,7 +130,7 @@ func populateRelationships(exec file.Executable, parentPkg pkg.Package, resolver for _, loc := range libLocations { // are you in our index? - realBaseName := path.Base(loc.RealPath) + realBaseName := path.Base(loc.FullNestedPath()) pkgCollection := index.owningLibraryPackage(realBaseName) if pkgCollection.PackageCount() < 1 { addRelationship( diff --git a/internal/relationship/binary/shared_library_index.go b/internal/relationship/binary/shared_library_index.go index aa93c0db0..af780ceca 100644 --- a/internal/relationship/binary/shared_library_index.go +++ b/internal/relationship/binary/shared_library_index.go @@ -3,6 +3,7 @@ package binary import ( "path" + "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/sbomsync" "github.com/anchore/syft/syft/file" @@ -63,7 +64,7 @@ func locationsThatProvideLibraries(accessor sbomsync.Accessor) (map[string]file. continue } - basename := path.Base(coord.RealPath) + basename := path.Base(coord.FullNestedPath()) set := libLocationsByBasename[basename] set.Add(coord) allLibLocations.Add(coord) @@ -82,6 +83,11 @@ func packagesWithLibraryOwnership(resolver file.Resolver, accessor sbomsync.Acce var ownedFilePaths []string if p.Type == pkg.BinaryPkg { for _, loc := range p.Locations.ToSlice() { + if archive.ContainerPath(loc.Coordinates) != "" { + // a path within an archive cannot be resolved from the scan root; the location is the library + addLibraryOwner(packagesByLibraryPath, allLibLocations, loc.Coordinates, p) + continue + } ownedFilePaths = append(ownedFilePaths, loc.Path()) } } else { @@ -114,18 +120,19 @@ func populatePackagesByLibraryPath( } for _, loc := range ownedLocation { - // if the location is a library, add the package to the set of packages that own the library - if !allLibLocations.Contains(loc.Coordinates) { - continue - } - - if _, ok := packagesByLibraryPath[loc.Coordinates]; !ok { - packagesByLibraryPath[loc.Coordinates] = pkg.NewCollection() - } - - // we have a library path, add the package to the set of packages that own the library - packagesByLibraryPath[loc.Coordinates].Add(p) + addLibraryOwner(packagesByLibraryPath, allLibLocations, loc.Coordinates, p) } } return packagesByLibraryPath } + +// addLibraryOwner records p as an owner of the file at coordinates when that file is a library. +func addLibraryOwner(packagesByLibraryPath map[file.Coordinates]*pkg.Collection, allLibLocations file.CoordinateSet, coordinates file.Coordinates, p pkg.Package) { + if !allLibLocations.Contains(coordinates) { + return + } + if _, ok := packagesByLibraryPath[coordinates]; !ok { + packagesByLibraryPath[coordinates] = pkg.NewCollection() + } + packagesByLibraryPath[coordinates].Add(p) +} diff --git a/internal/relationship/by_file_ownership.go b/internal/relationship/by_file_ownership.go index 92b2ddc49..ce8f9a3a2 100644 --- a/internal/relationship/by_file_ownership.go +++ b/internal/relationship/by_file_ownership.go @@ -6,6 +6,7 @@ import ( "github.com/bmatcuk/doublestar/v4" "github.com/scylladb/go-set/strset" + "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/sbomsync" "github.com/anchore/syft/syft/artifact" @@ -100,7 +101,7 @@ func findOwnershipByFilesRelationships(resolver file.Resolver, catalog *pkg.Coll // Now establish relationships, considering symlinks for _, candidateOwnerPkg := range catalog.Sorted() { id := candidateOwnerPkg.ID() - if candidateOwnerPkg.Metadata == nil { + if candidateOwnerPkg.Metadata == nil || withinArchive(candidateOwnerPkg) { continue } @@ -164,7 +165,7 @@ func directOwnersByPath(catalog *pkg.Collection) map[pkg.Type]*strset.Set { // First, identify direct ownership of all files for _, p := range catalog.Sorted() { - if p.Metadata == nil { + if p.Metadata == nil || withinArchive(p) { continue } @@ -192,6 +193,21 @@ func directOwnersByPath(catalog *pkg.Collection) map[pkg.Type]*strset.Set { return directOwnership } +// withinArchive reports whether every location of p is inside an archive. Such a package's owned files +// are paths within its archive, not the scan root, so it is not treated as a file owner. +func withinArchive(p pkg.Package) bool { + locations := p.Locations.ToSlice() + if len(locations) == 0 { + return false + } + for _, l := range locations { + if archive.ContainerPath(l.Coordinates) == "" { + return false + } + } + return true +} + func resolvePaths(ownedFilePath string, resolver file.Resolver) []string { // though we have a string path, we need to resolve symlinks and other filesystem oddities since we cannot assume this is a real path var locs []file.Location @@ -206,6 +222,8 @@ func resolvePaths(ownedFilePath string, resolver file.Resolver) []string { ownedFilePaths := strset.New(ownedFilePath) for _, loc := range locs { + // Owned file paths are currently based on the top-level resolver, which is always the RealPath and includes + // all files found within archives at that location ownedFilePaths.Add(loc.RealPath) } return ownedFilePaths.List() diff --git a/internal/relationship/by_file_ownership_test.go b/internal/relationship/by_file_ownership_test.go index a69f99c99..1e83a2c76 100644 --- a/internal/relationship/by_file_ownership_test.go +++ b/internal/relationship/by_file_ownership_test.go @@ -78,6 +78,61 @@ func TestOwnershipByFilesRelationship(t *testing.T) { return []pkg.Package{parent, child}, []artifact.Relationship{relationship} }, }, + { + name: "package-within-archive-is-not-a-file-owner", + setup: func(t testing.TB) ([]pkg.Package, []artifact.Relationship) { + nestedOwner := pkg.Package{ + Locations: file.NewLocationSet(file.NewLocationFromCoordinates(file.Coordinates{ + RealPath: "/rootfs.tar", + FullPath: "/rootfs.tar!/var/lib/rpm/Packages", + })), + Type: pkg.RpmPkg, + Metadata: pkg.RpmDBEntry{ + Files: []pkg.RpmFileRecord{{Path: "/d/path"}}, + }, + } + nestedOwner.SetID() + + child := pkg.Package{ + Locations: file.NewLocationSet(file.NewLocation("/d/path")), + Type: pkg.BinaryPkg, + } + child.SetID() + + return []pkg.Package{nestedOwner, child}, nil + }, + }, + { + name: "top-level-owner-of-an-archive-owns-packages-within-it", + setup: func(t testing.TB) ([]pkg.Package, []artifact.Relationship) { + parent := pkg.Package{ + Locations: file.NewLocationSet(file.NewLocation("/var/lib/rpm/Packages")), + Type: pkg.RpmPkg, + Metadata: pkg.RpmDBEntry{ + Files: []pkg.RpmFileRecord{{Path: "/opt/bundle.tar"}}, + }, + } + parent.SetID() + + child := pkg.Package{ + Locations: file.NewLocationSet(file.NewLocationFromCoordinates(file.Coordinates{ + RealPath: "/opt/bundle.tar", + FullPath: "/opt/bundle.tar!/lib/thing.jar", + })), + Type: pkg.JavaPkg, + } + child.SetID() + + relationship := artifact.Relationship{ + From: parent, + To: child, + Type: artifact.OwnershipByFileOverlapRelationship, + Data: ownershipByFilesMetadata{Files: []string{"/opt/bundle.tar"}}, + } + + return []pkg.Package{parent, child}, []artifact.Relationship{relationship} + }, + }, { name: "only-real-file-owner-gets-ownership-relationship-not-symlink-owner", resolver: mockFR{ diff --git a/internal/relationship/evident_by_test.go b/internal/relationship/evident_by_test.go index 9e08d967b..bd001696d 100644 --- a/internal/relationship/evident_by_test.go +++ b/internal/relationship/evident_by_test.go @@ -15,18 +15,9 @@ func TestRelationshipsEvidentBy(t *testing.T) { c := pkg.NewCollection() - coordA := file.Coordinates{ - RealPath: "/somewhere/real", - FileSystemID: "abc", - } - coordC := file.Coordinates{ - RealPath: "/somewhere/real", - FileSystemID: "abc", - } - coordD := file.Coordinates{ - RealPath: "/somewhere/real", - FileSystemID: "abc", - } + coordA := file.NewCoordinates("/somewhere/real", "abc") + coordC := file.NewCoordinates("/somewhere/real", "abc") + coordD := file.NewCoordinates("/somewhere/real", "abc") pkgA := pkg.Package{ Locations: file.NewLocationSet( // added! @@ -39,10 +30,7 @@ func TestRelationshipsEvidentBy(t *testing.T) { pkgA.SetID() c.Add(pkgA) - coordB := file.Coordinates{ - RealPath: "/somewhere-else/real", - FileSystemID: "def", - } + coordB := file.NewCoordinates("/somewhere-else/real", "def") pkgB := pkg.Package{ Locations: file.NewLocationSet( // added! diff --git a/internal/relationship/index_test.go b/internal/relationship/index_test.go index 12c933346..d856d184a 100644 --- a/internal/relationship/index_test.go +++ b/internal/relationship/index_test.go @@ -21,12 +21,8 @@ func Test_Index(t *testing.T) { p3 := pkg.Package{ Name: "pkg-3", } - c1 := file.Coordinates{ - RealPath: "/coords/1", - } - c2 := file.Coordinates{ - RealPath: "/coords/2", - } + c1 := file.NewCoordinates("/coords/1", "") + c2 := file.NewCoordinates("/coords/2", "") for _, p := range []*pkg.Package{&p1, &p2, &p3} { p.SetID() @@ -156,18 +152,10 @@ func Test_Coordinates(t *testing.T) { p3 := pkg.Package{ Name: "pkg-3", } - c1 := file.Coordinates{ - RealPath: "/coords/1", - } - c2 := file.Coordinates{ - RealPath: "/coords/2", - } - c3 := file.Coordinates{ - RealPath: "/coords/3", - } - c4 := file.Coordinates{ - RealPath: "/coords/4", - } + c1 := file.NewCoordinates("/coords/1", "") + c2 := file.NewCoordinates("/coords/2", "") + c3 := file.NewCoordinates("/coords/3", "") + c4 := file.NewCoordinates("/coords/4", "") for _, p := range []*pkg.Package{&p1, &p2, &p3} { p.SetID() @@ -237,10 +225,10 @@ func TestRemove(t *testing.T) { p1 := pkg.Package{Name: "pkg-1"} p2 := pkg.Package{Name: "pkg-2"} p3 := pkg.Package{Name: "pkg-3"} - c1 := file.Coordinates{RealPath: "/coords/1"} - c2 := file.Coordinates{RealPath: "/coords/2"} - c3 := file.Coordinates{RealPath: "/coords/3"} - c4 := file.Coordinates{RealPath: "/coords/4"} + c1 := file.NewCoordinates("/coords/1", "") + c2 := file.NewCoordinates("/coords/2", "") + c3 := file.NewCoordinates("/coords/3", "") + c4 := file.NewCoordinates("/coords/4", "") for _, p := range []*pkg.Package{&p1, &p2, &p3} { p.SetID() diff --git a/internal/task/archive_nesting_test.go b/internal/task/archive_nesting_test.go index f5df06159..16952b6dd 100644 --- a/internal/task/archive_nesting_test.go +++ b/internal/task/archive_nesting_test.go @@ -18,6 +18,7 @@ import ( "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/sbomsync" + "github.com/anchore/syft/internal/testutils" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/file" @@ -28,7 +29,7 @@ import ( // This file covers nesting across archive families. The other nested-archive fixtures are all zips, // so a zip-specific mechanism would pass them; the requirement here is that an archive of one family -// inside another behaves as it would inside its own family - same packages, same filesystem-id chain, +// inside another behaves as it would inside its own family - same packages, same container-path chain, // same CONTAINS edges, same depth accounting. // // Chains are built in the test rather than committed as binaries, so each case can vary which family @@ -71,8 +72,8 @@ type nestedArchive struct { // paths is each archive's path within its parent, outermost first; paths[0] is the outermost archive // as the scan's resolver reports it. paths []string - // fileSystemIDs is the composed chain each archive's own contents carry, outermost first. - fileSystemIDs []string + // containerPaths is each archive's full path, the container path its own contents carry, outermost first. + containerPaths []string // virtualPaths is the same chain in java's colon-joined form, outermost first. virtualPaths []string } @@ -87,6 +88,7 @@ type nestingRow struct { // nestingVisit is one filesystem the archive cataloger's sub-pipeline was run against. type nestingVisit struct { virtualPath string + fullPath string depth int // heldFiles are the spill files live at this moment, sampled only when the probe was given a temp // dir to look in. A nested archive's own bytes are read where they lie, so one appears per archive @@ -120,12 +122,12 @@ func Test_mixedFamilyNesting_leafIsCatalogedOnceWithTheFullChain(t *testing.T) { locs := leafLocations(s) require.Len(t, locs, 1, "the leaf must be reached by exactly one path") // exact, never a substring: a partially composed chain passes a substring check - assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath, + assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates), "the chain must name all three archives outer-to-inner") - assert.Equal(t, "nested/marker.txt", locs[0].RealPath, + assert.Equal(t, nested.leafContainerPath()+"!/nested/marker.txt", locs[0].FullNestedPath(), "the path stays relative to the archive the file came from") - assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs(), + assert.Equal(t, nested.containerPaths, probe.containerPaths(), "every level must be cataloged, outer to inner, whatever families they are") }) } @@ -145,11 +147,7 @@ func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) { // directory, so it carries no filesystem id of its own archiveCoords := make([]file.Coordinates, len(row.families)) for i := range row.families { - var parentFSID string - if i > 0 { - parentFSID = nested.fileSystemIDs[i-1] - } - archiveCoords[i] = file.Coordinates{RealPath: nested.paths[i], ArchivePath: parentFSID} + archiveCoords[i] = nested.coordinates(i) } fileEdges := strset.New() @@ -185,47 +183,25 @@ func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) { for _, rel := range s.Relationships { from, _ := rel.From.(file.Coordinates) to, _ := rel.To.(file.Coordinates) - leafEdge = leafEdge || (from == innermost && to == file.Coordinates{RealPath: "nested/marker.txt", ArchivePath: nested.leafFileSystemID()}) + leafEdge = leafEdge || (from == innermost && to == file.Coordinates{RealPath: innermost.RealPath, FullPath: innermost.FullNestedPath() + "!/nested/marker.txt"}) } assert.True(t, leafEdge, "expected a CONTAINS edge from the innermost archive to the leaf file") }) } } -func Test_mixedFamilyNesting_javaLeafVirtualPathIsTheColonJoinedChain(t *testing.T) { - // these colon-delimited strings are in already-published SBOMs, so a jar reached through three - // levels of mixed families must still be reported under the chain of all three - for _, row := range nestingRows() { - if row.families[len(row.families)-1] != familyJar { - continue - } - t.Run(row.name, func(t *testing.T) { - scanDir := t.TempDir() - nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()}) - - s := runNesting(t, scanDir, 3, defaultLimits(), markerTask(), javaTask()) - - // the innermost archive is the jar, so its own package is the java leaf and the chain of - // all three archives is exactly its virtual path - leaf := fmt.Sprintf("level%d", len(row.families)-1) - assert.Equal(t, 1, packageCount(s, leaf), "the leaf jar must be cataloged exactly once") - assert.Equal(t, nested.virtualPaths[len(row.families)-1], javaVirtualPaths(s)[leaf]) - }) - } -} - -func Test_mixedFamilyNesting_jarIsExercisedAsAContainer(t *testing.T) { +func Test_mixedFamilyNesting_everyJarIsCatalogedOnceUnderItsChain(t *testing.T) { // two mechanisms meet a jar: the java cataloger catalogs it as a package, the archive task descends - // into it. A jar mid-chain must do both, exactly once each; the failure mode is a duplicate, not an - // error. + // into it. A jar at any level must do both, exactly once each; the failure mode is a duplicate, not an + // error. The colon-joined virtual paths are in already-published SBOMs, so they must name the chain. for _, row := range nestingRows() { - var containerLevels []int + var jarLevels []int for i, family := range row.families { - if family == familyJar && i < len(row.families)-1 { - containerLevels = append(containerLevels, i) + if family == familyJar { + jarLevels = append(jarLevels, i) } } - if len(containerLevels) == 0 { + if len(jarLevels) == 0 { continue } @@ -236,19 +212,18 @@ func Test_mixedFamilyNesting_jarIsExercisedAsAContainer(t *testing.T) { probe := &nestingProbe{} s := runNesting(t, scanDir, 3, defaultLimits(), markerTask(), javaTask(), probe.task()) - for _, level := range containerLevels { + for _, level := range jarLevels { name := fmt.Sprintf("level%d", level) - assert.Equal(t, 1, packageCount(s, name), - "the container jar's own package must be cataloged exactly once") + assert.Equal(t, 1, packageCount(s, name), "the jar's own package must be cataloged exactly once") assert.Equal(t, nested.virtualPaths[level], javaVirtualPaths(s)[name], - "the container jar's virtual path must be its own place in the chain") + "the jar's virtual path must be its own place in the chain") } - // and the archive task still descended through it, so the chain is complete below it - assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs()) + // and the archive task still descended through every level + assert.Equal(t, nested.containerPaths, probe.containerPaths()) locs := leafLocations(s) require.Len(t, locs, 1) - assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath) + assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates)) }) } } @@ -264,16 +239,13 @@ func Test_mixedFamilyNesting_depthCountsLevelsNotFamilies(t *testing.T) { probe := &nestingProbe{} s := runNesting(t, scanDir, 2, defaultLimits(), markerTask(), fileMetadataTask(), probe.task()) - assert.Equal(t, nested.fileSystemIDs[:2], probe.archiveFileSystemIDs(), + assert.Equal(t, nested.containerPaths[:2], probe.containerPaths(), "at depth 2 the outer and middle archives are cataloged and the innermost is not") assert.Zero(t, packageCount(s, "marker-pkg"), "the leaf is one level past the bound, so its package must be absent") // the innermost archive is still cataloged as a file, inside the middle archive - innermost := file.Coordinates{ - RealPath: nested.paths[2], - ArchivePath: nested.fileSystemIDs[1], - } + innermost := nested.coordinates(2) _, ok := s.Artifacts.FileMetadata[innermost] assert.True(t, ok, "the undescended archive must still be recorded as a file at %s", coordKey(innermost)) @@ -285,64 +257,6 @@ func Test_mixedFamilyNesting_depthCountsLevelsNotFamilies(t *testing.T) { } } -func Test_mixedFamilyNesting_defaultBoundsCatalogEveryRow(t *testing.T) { - // the default bounds, pinned along with the fact that a three-level chain of any family mix sits - // well inside them - bounds := cataloging.DefaultArchiveSearchConfig() - require.Equal(t, int64(2*1024*1024*1024), bounds.MaxMemoryBytes) - require.Equal(t, int64(100*1024*1024*1024), bounds.MaxDiskBytes) - - for _, row := range nestingRows() { - t.Run(row.name, func(t *testing.T) { - scanDir := t.TempDir() - nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()}) - - probe := &nestingProbe{} - s := runNesting(t, scanDir, 3, archiveLimits(bounds), markerTask(), probe.task()) - - assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs(), - "every level of the chain must be cataloged at the default bounds") - locs := leafLocations(s) - require.Len(t, locs, 1) - assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath) - }) - } -} - -func Test_mixedFamilyNesting_memoryPressureOverflowsRatherThanFailing(t *testing.T) { - // entries that will not fit in memory are written to disk rather than refused - scanDir := t.TempDir() - tempDir := isolatedTempDir(t) - nested := writeNestedArchive(t, scanDir, gradedChain()) - - t.Run("with a generous memory limit nothing is written to disk", func(t *testing.T) { - probe := &nestingProbe{tempDir: tempDir} - bounds := defaultLimits() - bounds.MaxMemoryBytes = int64(2 * (nested.sizes[0] + nested.sizes[1] + nested.sizes[2])) - runNesting(t, scanDir, 3, bounds, markerTask(), probe.task()) - - deepest := probe.deepestVisit() - require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held") - assert.Empty(t, deepest.heldFiles) - }) - - t.Run("with a memory limit the outermost archive's entries do not fit in", func(t *testing.T) { - // room for every level's index estimate, which must stay in memory, but not the outer entries - probe := &nestingProbe{tempDir: tempDir} - bounds := archive.Limits{MaxMemoryBytes: int64(nested.sizes[2] + 32*1024), MaxDiskBytes: -1} - s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task()) - - deepest := probe.deepestVisit() - require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held") - assert.Contains(t, deepest.heldFiles, "archive-spill", "entries that do not fit in memory are written to disk") - - assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs(), "and every level is still cataloged") - locs := leafLocations(s) - require.Len(t, locs, 1) - assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath) - }) -} - func Test_mixedFamilyNesting_aNestedArchiveIsReadWhereItLies(t *testing.T) { // an archive's own bytes are read where they lie: a scanned file from disk, a nested archive from // its parent's entries. Nothing copies them, so at most one spill file per level appears whatever @@ -376,10 +290,10 @@ func Test_mixedFamilyNesting_aNestedArchiveIsReadWhereItLies(t *testing.T) { require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held") assert.Equal(t, tc.wantHeldFiles, deepest.heldFiles) - assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs()) + assert.Equal(t, nested.containerPaths, probe.containerPaths()) locs := leafLocations(s) require.Len(t, locs, 1) - assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath) + assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates)) }) } } @@ -395,7 +309,7 @@ func Test_mixedFamilyNesting_diskLimitTruncatesTheLevelItCannotAdmit(t *testing. // a sibling of a different family, processed once the chain has been released, so cataloged in full require.NoError(t, os.WriteFile( filepath.Join(scanDir, "sibling.tgz"), - makeTarGz(t, map[string][]byte{"nested/marker.txt": []byte("sibling")}), + testutils.TarGz(t, map[string][]byte{"nested/marker.txt": []byte("sibling")}), 0o644, )) @@ -408,50 +322,29 @@ func Test_mixedFamilyNesting_diskLimitTruncatesTheLevelItCannotAdmit(t *testing. s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task()) - want := append(append([]string{}, nested.fileSystemIDs...), "/sibling.tgz") - assert.ElementsMatch(t, want, probe.archiveFileSystemIDs(), + want := append(append([]string{}, nested.containerPaths...), "/sibling.tgz") + assert.ElementsMatch(t, want, probe.containerPaths(), "every level is visited, the truncated one with nothing inside it") - fsIDs := strset.New() + containers := strset.New() for _, loc := range leafLocations(s) { - fsIDs.Add(loc.ArchivePath) + containers.Add(archive.ContainerPath(loc.Coordinates)) } - assert.False(t, fsIDs.Has(nested.leafFileSystemID()), "the truncated level's contents must be absent") - assert.True(t, fsIDs.Has("/sibling.tgz"), "the sibling of a different family must be cataloged fully") + assert.False(t, containers.Has(nested.leafContainerPath()), "the truncated level's contents must be absent") + assert.True(t, containers.Has("/sibling.tgz"), "the sibling of a different family must be cataloged fully") var truncated []string for coords, reasons := range s.Artifacts.Unknowns { for _, reason := range reasons { if strings.Contains(reason, "disk limit") { - truncated = append(truncated, coords.ArchivePath+":"+coords.RealPath) + truncated = append(truncated, coords.FullNestedPath()) } } } - assert.Equal(t, []string{nested.fileSystemIDs[1] + ":" + nested.paths[2]}, truncated, + assert.Equal(t, []string{nested.coordinates(2).FullNestedPath()}, truncated, "the truncation is recorded against the archive that reached the limit") } -func Test_mixedFamilyNesting_unboundedLimitsEnforceNothing(t *testing.T) { - // a caller can opt out of one bound without the other, and opting out of both must still find the - // deepest leaf. Opting out is a negative value; zero forbids the resource. - for _, row := range nestingRows() { - t.Run(row.name, func(t *testing.T) { - scanDir := t.TempDir() - nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()}) - - probe := &nestingProbe{} - bounds := archive.Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1} - - s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task()) - - assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs()) - locs := leafLocations(s) - require.Len(t, locs, 1) - assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath) - }) - } -} - func defaultLimits() archive.Limits { return archiveLimits(cataloging.DefaultArchiveSearchConfig()) } @@ -524,37 +417,30 @@ func (f archiveFamily) pack(t *testing.T, base string, entries map[string][]byte t.Helper() switch f { case familyZip: - return makeZip(t, entries) + return testutils.Zip(t, entries) case familyJar: // a manifest makes it a real jar to the java cataloger at any level; the package java reports is // named after the file, not the manifest title - return makeJar(t, base, "1.0", entries) + return testutils.Jar(t, base, "1.0", entries) case familyTar: - return makeTar(t, entries) + return testutils.Tar(t, entries) case familyTarGz, familyTgz: - return makeTarGz(t, entries) + return testutils.TarGz(t, entries) } t.Fatalf("unknown archive family %q", f) return nil } -func makeJar(t *testing.T, title, version string, extra map[string][]byte) []byte { - t.Helper() - entries := map[string][]byte{ - "META-INF/MANIFEST.MF": []byte( - "Manifest-Version: 1.0\nImplementation-Title: " + title + "\nImplementation-Version: " + version + "\n", - ), - } - for k, v := range extra { - entries[k] = v - } - return makeZip(t, entries) +// leafContainerPath is the container path the innermost archive's contents carry: every archive, outer to +// inner. Assert it exactly, never as a substring: a partially composed chain passes a substring check. +func (n nestedArchive) leafContainerPath() string { + return n.containerPaths[len(n.containerPaths)-1] } -// leafFileSystemID is the chain the innermost archive's contents carry: every archive, outer to -// inner. Assert it exactly, never as a substring: a partially composed chain passes a substring check. -func (n nestedArchive) leafFileSystemID() string { - return n.fileSystemIDs[len(n.fileSystemIDs)-1] +// coordinates are those of the archive at the given level: the outermost archive's path, and the chain +// of entries within it. +func (n nestedArchive) coordinates(level int) file.Coordinates { + return file.Coordinates{RealPath: n.paths[0], FullPath: strings.Join(n.paths[:level+1], "!/")} } // buildNestedArchive builds one archive per family given, each holding the next, with leaf inside @@ -564,10 +450,10 @@ func buildNestedArchive(t *testing.T, plan nestPlan) nestedArchive { require.NotEmpty(t, plan.families, "a chain needs at least one level") out := nestedArchive{ - sizes: make([]int, len(plan.families)), - paths: make([]string, len(plan.families)), - fileSystemIDs: make([]string, len(plan.families)), - virtualPaths: make([]string, len(plan.families)), + sizes: make([]int, len(plan.families)), + paths: make([]string, len(plan.families)), + containerPaths: make([]string, len(plan.families)), + virtualPaths: make([]string, len(plan.families)), } // the level index goes in every name and path segment, so no two levels share a path and a chain @@ -585,11 +471,11 @@ func buildNestedArchive(t *testing.T, plan nestPlan) nestedArchive { for i := range plan.families { if i == 0 { - out.fileSystemIDs[i] = out.paths[0] + out.containerPaths[i] = out.paths[0] out.virtualPaths[i] = out.paths[0] continue } - out.fileSystemIDs[i] = out.fileSystemIDs[i-1] + ":" + out.paths[i] + out.containerPaths[i] = out.containerPaths[i-1] + "!/" + out.paths[i] out.virtualPaths[i] = out.virtualPaths[i-1] + ":" + out.paths[i] } @@ -667,6 +553,7 @@ func (p *nestingProbe) task() Task { visit := nestingVisit{} if trav != nil { visit.virtualPath = archive.VirtualPath(trav.Location) + visit.fullPath = trav.FullTraversalArchivePath() visit.depth = strings.Count(archive.VirtualPath(trav.Location), ":") + 1 } if p.tempDir != "" { @@ -680,16 +567,16 @@ func (p *nestingProbe) task() Task { }) } -// archiveFileSystemIDs is the chain of every archive whose contents were cataloged, in visit order. -func (p *nestingProbe) archiveFileSystemIDs() []string { +// containerPaths is the chain of every archive whose contents were cataloged, in visit order. +func (p *nestingProbe) containerPaths() []string { p.mu.Lock() defer p.mu.Unlock() var out []string for _, v := range p.visits { - if v.virtualPath == "" { + if v.fullPath == "" { continue } - out = append(out, v.virtualPath) + out = append(out, v.fullPath) } return out } @@ -770,7 +657,7 @@ func packageCount(s *sbom.SBOM, name string) int { } func coordKey(c file.Coordinates) string { - return c.ArchivePath + "|" + c.RealPath + return c.FullNestedPath() } // gradedChain is the headline row with each level padded so the three archives have clearly different diff --git a/internal/task/archive_tasks.go b/internal/task/archive_tasks.go index fe98f9143..fa2e1e430 100644 --- a/internal/task/archive_tasks.go +++ b/internal/task/archive_tasks.go @@ -194,12 +194,12 @@ func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver fi ctx = archive.WithBudget(ctx, archive.NewBudget(archiveTreeBudget(parentResolver, location))) } - // extracted files keep the filesystem ID of where the archive was found; the nesting chain is - // carried separately as the archive path. The chain names real paths, not the link the archive was + // extracted files keep the real path and filesystem ID of the outermost archive; the chain of entries + // to each is carried as its archive path. The chain names real paths, not the link the archive was // reached by, so it is stable and joins to the archive's package location by prefix. traversal := &archive.Traversal{Location: location, Parent: archive.TraversalFromContext(ctx)} - archivePath := traversal.ContentsArchivePath() - extracted, err := archive.Extract(ctx, archiveContent, location.FileSystemID, archivePath, c.limiter, c.exclusions) + archivePath := location.FullNestedPath() + extracted, err := archive.Extract(ctx, archiveContent, location.Coordinates, c.limiter, c.exclusions) if errors.Is(err, archive.ErrDiskLimitReached) { // nothing is released while this archive waits, so skip it rather than block the scan recordArchiveUnknown(builder, location.Coordinates, "archive skipped: its content would exceed the disk limit") @@ -370,7 +370,7 @@ func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCan if i, dup := seen[loc.Coordinates]; dup { // a MIME hit is the bare real path; the name it is reached by (a link such as // lib/dep.jar -> ../store/3f9a1c) is what name-based catalogers like java key off - if prev := candidates[i].location; prev.AccessPath == "" || prev.AccessPath == prev.RealPath { + if prev := candidates[i].location; prev.AccessPath == "" || prev.AccessPath == prev.FullNestedPath() { candidates[i].location = loc } continue @@ -384,7 +384,7 @@ func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCan // mergeArchiveResults copies the throwaway SBOM's packages, files and relationships into the shared // SBOM, adding a CONTAINS relationship from the archive to everything found inside it. // -// Files inside the archive are those whose ArchivePath is the archive's own chain, which leaves out the +// Files inside the archive are those whose container is the archive's own full path, which leaves out the // archive itself (a java package or an unknown recorded at its coordinates would otherwise become a // self-loop). The nested archives are always included, so the chain of archives is navigable whatever // file selection is configured. @@ -409,7 +409,7 @@ func mergeArchiveResults(archiveCoordinates file.Coordinates, archivePath string contained.Add(child.location.Coordinates) } for _, coordinates := range scratch.AllCoordinates() { - if coordinates.ArchivePath == archivePath { + if archive.ContainerPath(coordinates) == archivePath { contained.Add(coordinates) } } diff --git a/internal/task/archive_tasks_test.go b/internal/task/archive_tasks_test.go index c0f05e7e3..70beefac5 100644 --- a/internal/task/archive_tasks_test.go +++ b/internal/task/archive_tasks_test.go @@ -1,10 +1,7 @@ package task import ( - "archive/tar" - "archive/zip" "bytes" - "compress/gzip" "context" "crypto/sha1" "encoding/hex" @@ -14,7 +11,6 @@ import ( "io/fs" "os" "path/filepath" - "sort" "strings" "testing" "time" @@ -31,6 +27,7 @@ import ( "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/mimetype" "github.com/anchore/syft/internal/sbomsync" + "github.com/anchore/syft/internal/testutils" "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/cataloging" @@ -87,7 +84,7 @@ func (panicReader) Read([]byte) (int, error) { panic("decoder blew up") } func Test_archiveCataloger_aPanicCostsOnlyThatArchive(t *testing.T) { rootDir := t.TempDir() for _, name := range []string{"bad.zip", "good.zip"} { - require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), makeZip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600)) } var seen []string @@ -147,35 +144,10 @@ func capturingTask(t *testing.T, seen *[]string, fileCounts *map[string]int) Tas }) } -func Test_archiveCataloger_traversalThreading(t *testing.T) { - manifest := []byte("Manifest-Version: 1.0\nImplementation-Title: example\nImplementation-Version: 1.0.0\n") - innerJar := makeZip(t, map[string][]byte{"META-INF/MANIFEST.MF": manifest}) - innerZip := makeZip(t, map[string][]byte{"README.txt": []byte("hi")}) - outerZip := makeZip(t, map[string][]byte{ - "lib/inner.jar": innerJar, - "nested/inner.zip": innerZip, - }) - - rootDir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) - - var seen []string - tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3), capturingTask(t, &seen, nil)) - - require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM()))) - - // java archives are traversed like any other archive when the task is enabled - assert.ElementsMatch(t, []string{ - "/outer.zip", - "/outer.zip:lib/inner.jar", - "/outer.zip:nested/inner.zip", - }, seen) -} - func Test_archiveCataloger_truncationStillCatalogs(t *testing.T) { // reaching a limit is a truncation, not a failure: the sub-pipeline still runs over what was stored big := bytes.Repeat([]byte("x"), 4096) - outerZip := makeZip(t, map[string][]byte{ + outerZip := testutils.Zip(t, map[string][]byte{ "a/small.txt": []byte("small"), "b/big.txt": big, }) @@ -197,7 +169,7 @@ func Test_archiveCataloger_truncationStillCatalogs(t *testing.T) { assert.Equal(t, []string{"/outer.zip"}, seen, "the sub-pipeline must run over the partial contents") assert.Equal(t, 1, fileCounts["/outer.zip"], "only the entry stored before the limit is visible") - reasons, ok := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}] + reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")] require.True(t, ok, "the truncation must be recorded against the archive; got %v", s.Artifacts.Unknowns) require.Len(t, reasons, 1) assert.Contains(t, reasons[0], "archive cataloged from part of its contents") @@ -208,7 +180,7 @@ func Test_archiveCataloger_aDamagedEntryCostsOnlyThatEntry(t *testing.T) { rootDir := t.TempDir() // entry data overwritten but headers and end-of-central-directory intact: detected as an archive, // and data.bin fails when it is read. ok.txt comes after it and is intact. - corrupt := makeZip(t, map[string][]byte{"data.bin": incompressible(4096), "ok.txt": []byte("ok")}) + corrupt := testutils.Zip(t, map[string][]byte{"data.bin": incompressible(4096), "ok.txt": []byte("ok")}) require.Greater(t, len(corrupt), 1024) for i := 100; i < 600; i++ { corrupt[i] ^= 0xff @@ -224,7 +196,7 @@ func Test_archiveCataloger_aDamagedEntryCostsOnlyThatEntry(t *testing.T) { assert.Equal(t, []string{"/corrupt.zip"}, seen, "the archive is still cataloged") assert.Equal(t, 1, fileCounts["/corrupt.zip"], "from the entries that could be read") - reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/corrupt.zip"}] + reasons := s.Artifacts.Unknowns[file.NewCoordinates("/corrupt.zip", "")] require.Len(t, reasons, 1) assert.Contains(t, reasons[0], `entry "data.bin" could not be read`) } @@ -275,31 +247,11 @@ func Test_NewArchiveCatalogerTask_gating(t *testing.T) { } } -func Test_archiveCataloger_depthBoundIsExact(t *testing.T) { - innerZip := makeZip(t, map[string][]byte{"deep.txt": []byte("deep")}) - outerZip := makeZip(t, map[string][]byte{"nested/inner.zip": innerZip}) - - rootDir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) - - run := func(t *testing.T, depth int) []string { - t.Helper() - var seen []string - tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth), capturingTask(t, &seen, nil)) - require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM())), - "reaching the depth bound must not be an error") - return seen - } - - assert.Equal(t, []string{"/outer.zip"}, run(t, 1), "depth 1 catalogs the top-level archive and does not descend") - assert.ElementsMatch(t, []string{"/outer.zip", "/outer.zip:nested/inner.zip"}, run(t, 2), "depth 2 descends one level") -} - func Test_archiveCataloger_negativeDepthStopsAtTheCeiling(t *testing.T) { // "unbounded" must still end, or a zip quine or recursive bomb runs forever - chain := makeZip(t, map[string][]byte{"deep.txt": []byte("deep")}) + chain := testutils.Zip(t, map[string][]byte{"deep.txt": []byte("deep")}) for range unboundedArchiveDepth + 4 { - chain = makeZip(t, map[string][]byte{"n.zip": chain}) + chain = testutils.Zip(t, map[string][]byte{"n.zip": chain}) } rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), chain, 0o600)) @@ -314,7 +266,7 @@ func Test_archiveCataloger_negativeDepthStopsAtTheCeiling(t *testing.T) { func Test_archiveCataloger_anArchiveIdenticalToAnAncestorIsNotEntered(t *testing.T) { // a zip quine contains itself. Building one is impractical, so the walk is started as if it were // already inside an archive with these exact bytes, which is the state a quine reaches one level down - outerZip := makeZip(t, map[string][]byte{"a.txt": []byte("a")}) + outerZip := testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")}) rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) @@ -327,7 +279,7 @@ func Test_archiveCataloger_anArchiveIdenticalToAnAncestorIsNotEntered(t *testing require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(s))) assert.Zero(t, ran, "the sub-pipeline must not run inside a repeat of an ancestor") - reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}] + reasons := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")] require.Len(t, reasons, 1) assert.Contains(t, reasons[0], "identical to an archive that contains it") assert.Len(t, c.ancestors, 1, "the walk leaves the ancestor stack as it found it") @@ -335,7 +287,7 @@ func Test_archiveCataloger_anArchiveIdenticalToAnAncestorIsNotEntered(t *testing func Test_archiveCataloger_stopsWhenTheContextIsCancelled(t *testing.T) { rootDir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), makeZip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600)) ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -356,10 +308,10 @@ func Test_archiveCataloger_decompressionBudgetIsSharedByATreeAndFreshPerTopLevel t.Cleanup(func() { archiveTreeRatio, archiveTreeFloorBytes = origRatio, origFloor }) zeros := make([]byte, 60_000) - nested := makeZip(t, map[string][]byte{"zeros.bin": zeros}) + nested := testutils.Zip(t, map[string][]byte{"zeros.bin": zeros}) rootDir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(rootDir, "bomb.zip"), makeZip(t, map[string][]byte{"a.zip": nested, "b.zip": nested}), 0o600)) - require.NoError(t, os.WriteFile(filepath.Join(rootDir, "sibling.zip"), makeZip(t, map[string][]byte{"zeros.bin": zeros}), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "bomb.zip"), testutils.Zip(t, map[string][]byte{"a.zip": nested, "b.zip": nested}), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "sibling.zip"), testutils.Zip(t, map[string][]byte{"zeros.bin": zeros}), 0o600)) var seen []string fileCounts := map[string]int{} @@ -369,7 +321,7 @@ func Test_archiveCataloger_decompressionBudgetIsSharedByATreeAndFreshPerTopLevel assert.Equal(t, 1, fileCounts["/bomb.zip:a.zip"], "the first nested archive fits the tree's budget") assert.Equal(t, 0, fileCounts["/bomb.zip:b.zip"], "the second does not: the budget is shared, not per archive") - reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "b.zip", ArchivePath: "/bomb.zip"}] + reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/bomb.zip", FullPath: "/bomb.zip!/b.zip"}] require.Len(t, reasons, 1) assert.Contains(t, reasons[0], "possible decompression bomb") assert.Equal(t, 1, fileCounts["/sibling.zip"], "another top-level archive starts with a fresh budget") @@ -379,14 +331,14 @@ func Test_archiveCataloger_noSelfContainsEdge(t *testing.T) { // a jar java cannot identify records an unknown at its own coordinates; that must not turn into // "x.jar CONTAINS x.jar", which a graph walker would loop on rootDir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(rootDir, "x.jar"), makeZip(t, map[string][]byte{"readme.txt": []byte("no manifest")}), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "x.jar"), testutils.Zip(t, map[string][]byte{"readme.txt": []byte("no manifest")}), 0o600)) tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), javaTask(), fileMetadataTask()) s := newTestSBOM() s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{} _ = tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)) - require.Contains(t, s.Artifacts.Unknowns, file.Coordinates{RealPath: "/x.jar"}, "the fixture must reproduce the unknown") + require.Contains(t, s.Artifacts.Unknowns, file.NewCoordinates("/x.jar", ""), "the fixture must reproduce the unknown") var contains int for _, rel := range s.Relationships { if rel.Type != artifact.ContainsRelationship { @@ -402,8 +354,8 @@ func Test_NewArchiveCatalogerTask_dropsItselfFromSubPipeline(t *testing.T) { // a copy of itself in the sub-pipeline would process every nesting level twice cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2) - innerZip := makeZip(t, map[string][]byte{"leaf.txt": []byte("leaf")}) - outerZip := makeZip(t, map[string][]byte{"nested/inner.zip": innerZip}) + innerZip := testutils.Zip(t, map[string][]byte{"leaf.txt": []byte("leaf")}) + outerZip := testutils.Zip(t, map[string][]byte{"nested/inner.zip": innerZip}) rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) @@ -424,11 +376,11 @@ func Test_NewArchiveCatalogerTask_dropsItselfFromSubPipeline(t *testing.T) { }) } -func Test_archiveCataloger_chainStartsAtTheArchivesOwnFileSystemID(t *testing.T) { +func Test_archiveCataloger_nestedArchivesKeepTheLayerFileSystemID(t *testing.T) { // extracted files keep the layer digest as their FileSystemID at every level; the nesting chain - // rides on the traversal's VirtualPath and the coordinate's ArchivePath - innerZip := makeZip(t, map[string][]byte{"leaf.txt": []byte("leaf")}) - outerZip := makeZip(t, map[string][]byte{"nested/inner.zip": innerZip}) + // rides on the traversal's VirtualPath and the coordinate's FullPath + innerZip := testutils.Zip(t, map[string][]byte{"leaf.txt": []byte("leaf")}) + outerZip := testutils.Zip(t, map[string][]byte{"nested/inner.zip": innerZip}) rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) @@ -463,8 +415,8 @@ func Test_archiveCataloger_sameArchivePathInTwoLayersDoesNotCollide(t *testing.T resolver := twoLayerResolver{ path: "/app/bundle.zip", contentBy: map[string][]byte{ - "layer-one": makeZip(t, map[string][]byte{"lib/config.json": []byte(`{"v":1}`)}), - "layer-two": makeZip(t, map[string][]byte{"lib/config.json": []byte(`{"v":2}`)}), + "layer-one": testutils.Zip(t, map[string][]byte{"lib/config.json": []byte(`{"v":1}`)}), + "layer-two": testutils.Zip(t, map[string][]byte{"lib/config.json": []byte(`{"v":2}`)}), }, } @@ -501,7 +453,7 @@ func Test_archiveCataloger_sameArchivePathInTwoLayersDoesNotCollide(t *testing.T func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) { // runSubPipeline bypasses RunTask, where the coordinate-error to unknown conversion lives - outerZip := makeZip(t, map[string][]byte{"lib/broken.json": []byte("{")}) + outerZip := testutils.Zip(t, map[string][]byte{"lib/broken.json": []byte("{")}) rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) @@ -526,8 +478,8 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) require.Len(t, s.Artifacts.Unknowns, 1) for coords, reasons := range s.Artifacts.Unknowns { - assert.Equal(t, "lib/broken.json", coords.RealPath, "the inner coordinates must survive the merge") - assert.Equal(t, "/outer.zip", coords.ArchivePath, "and must be addressable per archive") + assert.Equal(t, "/outer.zip!/lib/broken.json", coords.FullNestedPath(), "the inner coordinates must survive the merge") + assert.Equal(t, "/outer.zip", coords.RealPath, "and must be addressable per archive") assert.Equal(t, []string{"failing-cataloger: unable to parse"}, reasons) } }) @@ -542,7 +494,7 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) require.Len(t, s.Artifacts.Unknowns, 1) - reasons, ok := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}] + reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")] require.True(t, ok, "an error with no location of its own belongs to the containing archive") assert.Equal(t, []string{"bare-error-cataloger: cataloger blew up"}, reasons) }) @@ -557,7 +509,7 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)), "a panic inside an archive must not fail the scan") - reasons, ok := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}] + reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")] require.True(t, ok) require.Len(t, reasons, 1) assert.Equal(t, "panicking-cataloger: panic: nope", reasons[0]) @@ -565,11 +517,11 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) } func Test_archiveCataloger_subPipelineUnknownsAreAddedNotReplaced(t *testing.T) { - outerZip := makeZip(t, map[string][]byte{"lib/thing.json": []byte("{}")}) + outerZip := testutils.Zip(t, map[string][]byte{"lib/thing.json": []byte("{}")}) rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) - preexisting := file.Coordinates{RealPath: "lib/thing.json", ArchivePath: "/outer.zip"} + preexisting := file.Coordinates{RealPath: "/outer.zip", FullPath: "/outer.zip!/lib/thing.json"} writer := NewTask("scratch-writer", func(_ context.Context, _ file.Resolver, builder sbomsync.Builder) error { builder.(sbomsync.Accessor).WriteToSBOM(func(s *sbom.SBOM) { if s.Artifacts.Unknowns == nil { @@ -588,39 +540,7 @@ func Test_archiveCataloger_subPipelineUnknownsAreAddedNotReplaced(t *testing.T) require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) assert.Equal(t, []string{"recorded by the cataloger itself"}, s.Artifacts.Unknowns[preexisting]) - assert.Equal(t, []string{"failing-cataloger: boom"}, s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}]) -} - -func Test_archiveCataloger_manySmallArchivesAreEachCataloged(t *testing.T) { - // every archive is released before the next begins, so a scan whose archives sum well past a limit - // never holds more than one at a time - entry := bytes.Repeat([]byte("z"), 400) - entries := map[string][]byte{} - for i := range 5 { - entries[fmt.Sprintf("e%d.bin", i)] = entry - } - - rootDir := t.TempDir() - names := []string{"one.zip", "two.zip", "three.zip", "four.zip", "five.zip"} - for _, name := range names { - require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), makeZip(t, entries), 0o600)) - } - - // room for one archive's entries and their index cost, not for two - limits := archive.Limits{MaxMemoryBytes: 0, MaxDiskBytes: 20000} - - var seen []string - fileCounts := map[string]int{} - tsk := newLimitedTestTask(t, 1, limits, capturingTask(t, &seen, &fileCounts)) - - s := newTestSBOM() - require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) - - assert.Len(t, seen, len(names), "every archive must be cataloged: no two were ever held at once") - for _, archivePath := range seen { - assert.Equal(t, 5, fileCounts[archivePath], "%s must extract in full", archivePath) - } - assert.Empty(t, s.Artifacts.Unknowns) + assert.Equal(t, []string{"failing-cataloger: boom"}, s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]) } func Test_archiveCataloger_limitsFallWhenAnArchiveIsReleased(t *testing.T) { @@ -632,7 +552,7 @@ func Test_archiveCataloger_limitsFallWhenAnArchiveIsReleased(t *testing.T) { rootDir := t.TempDir() for _, name := range []string{"one.zip", "two.zip"} { - require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), makeZip(t, entries), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), testutils.Zip(t, entries), 0o600)) } // room in memory for one archive's entries and their index cost, not for two at once @@ -675,8 +595,8 @@ func newTestCataloger(maxDepth int, limiter *archive.Limiter, subPipeline ...Tas func Test_archiveCataloger_archiveWhoseOwnBytesExceedTheDiskLimitIsSkipped(t *testing.T) { // a streamed archive's own bytes must be placed before it can be opened; when they do not fit on // disk the archive is skipped rather than waited for, and a sibling that fits is cataloged in full - oversized := makeZip(t, map[string][]byte{"payload.bin": incompressible(8000)}) - small := makeZip(t, map[string][]byte{"ok.txt": []byte("ok")}) + oversized := testutils.Zip(t, map[string][]byte{"payload.bin": incompressible(8000)}) + small := testutils.Zip(t, map[string][]byte{"ok.txt": []byte("ok")}) rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "oversized.zip"), oversized, 0o600)) @@ -699,84 +619,20 @@ func Test_archiveCataloger_archiveWhoseOwnBytesExceedTheDiskLimitIsSkipped(t *te assert.Equal(t, []string{"/small.zip"}, seen, "the oversized archive is skipped and the sibling is still cataloged") assert.Equal(t, 1, fileCounts["/small.zip"]) - reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/oversized.zip"}] + reasons := s.Artifacts.Unknowns[file.NewCoordinates("/oversized.zip", "")] require.Len(t, reasons, 1, "the skip must be recorded against the archive; got %v", s.Artifacts.Unknowns) assert.Contains(t, reasons[0], "archive skipped") } -func Test_archiveCataloger_zeroDiskLimitTruncatesWhatWillNotFitInMemory(t *testing.T) { - // with a zero disk limit there is nowhere to overflow, so an archive's entries are stored exactly - // while they fit in memory - oversized := makeZip(t, map[string][]byte{"payload.bin": incompressible(8000)}) - small := makeZip(t, map[string][]byte{"ok.txt": []byte("ok")}) - - rootDir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(rootDir, "oversized.zip"), oversized, 0o600)) - require.NoError(t, os.WriteFile(filepath.Join(rootDir, "small.zip"), small, 0o600)) - - // room for the small archive's entry and its index cost, not for the oversized one's - limits := archive.Limits{MaxMemoryBytes: 3000, MaxDiskBytes: 0} - - var seen []string - fileCounts := map[string]int{} - tsk := newLimitedTestTask(t, 1, limits, capturingTask(t, &seen, &fileCounts)) - - s := newTestSBOM() - require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)), - "reaching the disk limit must not fail the scan") - - assert.ElementsMatch(t, []string{"/oversized.zip", "/small.zip"}, seen) - assert.Equal(t, 1, fileCounts["/small.zip"], "the archive that fits in memory needs no disk at all") - assert.Equal(t, 0, fileCounts["/oversized.zip"], "the entry that does not fit is refused") - - assert.Contains(t, s.Artifacts.Unknowns, file.Coordinates{RealPath: "/oversized.zip"}) - assert.NotContains(t, s.Artifacts.Unknowns, file.Coordinates{RealPath: "/small.zip"}) -} - -func Test_archiveCataloger_nestedArchiveExceedingALimitIsTruncatedNotBlocked(t *testing.T) { - // the walk descends before it unwinds, so nothing is released while a child waits; a child that does - // not fit is truncated and the scan finishes - inner := makeZip(t, map[string][]byte{"payload.bin": incompressible(3000)}) - outer := makeZip(t, map[string][]byte{"inner.zip": inner}) - - rootDir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outer, 0o600)) - - // enough for the outer archive's one entry and its index cost, plus the inner's index cost, and not - // the inner's payload on top - diskLimit := int64(len(inner)) + 2*2100 + 500 - require.Greater(t, 3000, 500, "the inner payload must not fit in the slack") - - limits := archive.Limits{MaxMemoryBytes: 0, MaxDiskBytes: diskLimit} - - var seen []string - fileCounts := map[string]int{} - tsk := newLimitedTestTask(t, 2, limits, capturingTask(t, &seen, &fileCounts)) - - s := newTestSBOM() - require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) - - assert.Equal(t, []string{"/outer.zip", "/outer.zip:inner.zip"}, seen) - assert.Equal(t, 1, fileCounts["/outer.zip"], "the parent is cataloged in full") - assert.Equal(t, 0, fileCounts["/outer.zip:inner.zip"], "the child it cannot afford is truncated") - assert.Contains(t, s.Artifacts.Unknowns, file.Coordinates{RealPath: "inner.zip", ArchivePath: "/outer.zip"}) -} - -// allFilesResolver is dirTestResolver without a filesystem id. -type allFilesResolver struct { - file.Resolver - dir string -} - func Test_archiveCataloger_discoverArchivesSelectsTarFamily(t *testing.T) { marker := map[string][]byte{"nested/marker.txt": []byte("hello")} scanDir := t.TempDir() for name, body := range map[string][]byte{ - "bundle.tar.gz": makeTarGz(t, marker), - "bundle.tgz": makeTarGz(t, marker), - "bundle.tar": makeTar(t, marker), - "bundle.zip": makeZip(t, marker), + "bundle.tar.gz": testutils.TarGz(t, marker), + "bundle.tgz": testutils.TarGz(t, marker), + "bundle.tar": testutils.Tar(t, marker), + "bundle.zip": testutils.Zip(t, marker), "notes.txt": []byte("not an archive at all"), } { require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), body, 0o600)) @@ -785,7 +641,7 @@ func Test_archiveCataloger_discoverArchivesSelectsTarFamily(t *testing.T) { var seen []string fileCounts := map[string]int{} tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), capturingTask(t, &seen, &fileCounts)) - require.NoError(t, tsk.Execute(context.Background(), allFilesResolver{dir: scanDir}, sbomsync.NewBuilder(newTestSBOM()))) + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(newTestSBOM()))) assert.ElementsMatch(t, []string{"/bundle.tar.gz", "/bundle.tgz", "/bundle.tar", "/bundle.zip"}, seen, "every tar-family archive must be reached, and the plain text file must not be") @@ -807,7 +663,7 @@ type recordingPublisher struct { func Test_archiveCatalogerTask_publishesOneProgressRowPerCataloger(t *testing.T) { // a consumer keying rows by ID (as the syft CLI does) replaces a republished row and never sees it // complete, so each cataloger keeps to the one row it already owns - inner := makeZip(t, map[string][]byte{"inner/file.txt": []byte("hello")}) + inner := testutils.Zip(t, map[string][]byte{"inner/file.txt": []byte("hello")}) scanDir := t.TempDir() for _, name := range []string{"one.zip", "two.zip"} { require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), inner, 0o600)) @@ -865,8 +721,8 @@ func Test_archiveCatalogerTask_publishesOneProgressRowPerCataloger(t *testing.T) func Test_archiveCataloger_reportsPeakUsageAndTheSlowestArchive(t *testing.T) { // self time excludes nested archives, or the outer archive would be named every scan - innerJar := makeZip(t, map[string][]byte{"README.txt": []byte("inner")}) - outerZip := makeZip(t, map[string][]byte{"lib/inner.jar": innerJar}) + innerJar := testutils.Zip(t, map[string][]byte{"README.txt": []byte("inner")}) + outerZip := testutils.Zip(t, map[string][]byte{"lib/inner.jar": innerJar}) rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) @@ -883,7 +739,7 @@ func Test_archiveCataloger_reportsPeakUsageAndTheSlowestArchive(t *testing.T) { c := newTestCataloger(3, limiter, slow) require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(newTestSBOM()))) - assert.Equal(t, "/outer.zip:lib/inner.jar", c.slowestPath, "the slowest archive is named by its full chain") + assert.Equal(t, "/outer.zip!/lib/inner.jar", c.slowestPath, "the slowest archive is named by its full chain") assert.GreaterOrEqual(t, c.slowest, innerWork) assert.Equal(t, int64(2), c.progress.Current()) @@ -926,9 +782,9 @@ func Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError(t *testing.T) { func Test_archiveCataloger_catalogsAZipBehindALauncherScript(t *testing.T) { // the Spring Boot executable jar shape: content sniffing types it as a shell script, so it reaches // the walk only by name and qualifies only by its end-of-central-directory record - inner := makeZip(t, map[string][]byte{"nested/marker.txt": []byte("found me")}) + inner := testutils.Zip(t, map[string][]byte{"nested/marker.txt": []byte("found me")}) prefixed := append([]byte("#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n"), - makeZip(t, map[string][]byte{"BOOT-INF/lib/inner.jar": inner})...) + testutils.Zip(t, map[string][]byte{"BOOT-INF/lib/inner.jar": inner})...) rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "app.jar"), prefixed, 0o600)) @@ -983,28 +839,6 @@ func (d dirTestResolver) FileMetadataByLocation(loc file.Location) (file.Metadat return file.Metadata{FileInfo: info}, err } -// makeZip builds a zip in sorted entry-name order: limits are enforced as the walk proceeds, so entry -// order decides which entries land before a truncation. -func makeZip(t *testing.T, entries map[string][]byte) []byte { - t.Helper() - names := make([]string, 0, len(entries)) - for name := range entries { - names = append(names, name) - } - sort.Strings(names) - - var buf bytes.Buffer - zw := zip.NewWriter(&buf) - for _, name := range names { - w, err := zw.Create(name) - require.NoError(t, err) - _, err = w.Write(entries[name]) - require.NoError(t, err) - } - require.NoError(t, zw.Close()) - return buf.Bytes() -} - // matchesAnyGlob matches the resolver-relative path with a leading slash, the form a real resolver // globs over. No patterns matches everything. func matchesAnyGlob(root, path string, patterns []string) bool { @@ -1024,10 +858,6 @@ func matchesAnyGlob(root, path string, patterns []string) bool { return false } -func (a allFilesResolver) FilesByGlob(patterns ...string) ([]file.Location, error) { - return dirTestResolver{dir: a.dir}.FilesByGlob(patterns...) -} - func (r twoLayerResolver) FilesByGlob(_ ...string) ([]file.Location, error) { return nil, nil } @@ -1055,7 +885,7 @@ func (d dirTestResolver) FilesByGlob(patterns ...string) ([]file.Location, error func (r twoLayerResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) { var locs []file.Location for _, fsid := range []string{"layer-one", "layer-two"} { - locs = append(locs, file.NewLocationFromCoordinates(file.Coordinates{RealPath: r.path, FileSystemID: fsid})) + locs = append(locs, file.NewLocationFromCoordinates(file.NewCoordinates(r.path, fsid))) } return locs, nil } @@ -1085,55 +915,6 @@ func incompressible(n int) []byte { return b } -func makeTarGz(t *testing.T, entries map[string][]byte) []byte { - t.Helper() - var buf bytes.Buffer - gw := gzip.NewWriter(&buf) - require.NoError(t, writeTar(t, gw, entries)) - require.NoError(t, gw.Close()) - return buf.Bytes() -} - -func makeTar(t *testing.T, entries map[string][]byte) []byte { - t.Helper() - var buf bytes.Buffer - require.NoError(t, writeTar(t, &buf, entries)) - return buf.Bytes() -} - -func writeTar(t *testing.T, w io.Writer, entries map[string][]byte) error { - t.Helper() - names := make([]string, 0, len(entries)) - for name := range entries { - names = append(names, name) - } - sort.Strings(names) - - tw := tar.NewWriter(w) - for _, name := range names { - body := entries[name] - if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil { - return err - } - if _, err := tw.Write(body); err != nil { - return err - } - } - return tw.Close() -} - -func (a allFilesResolver) FilesByMIMEType(types ...string) ([]file.Location, error) { - return filesByMIMEType(a.dir, "", types...) -} - -func (a allFilesResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) { - return os.Open(filepath.Join(a.dir, filepath.FromSlash(loc.RealPath))) -} - -func (a allFilesResolver) FileMetadataByLocation(loc file.Location) (file.Metadata, error) { - return dirTestResolver{dir: a.dir}.FileMetadataByLocation(loc) -} - func (p *recordingPublisher) Publish(e partybus.Event) { p.events = append(p.events, e) } diff --git a/internal/task/file_tasks.go b/internal/task/file_tasks.go index 53a776674..758799cba 100644 --- a/internal/task/file_tasks.go +++ b/internal/task/file_tasks.go @@ -175,10 +175,10 @@ func coordinatesForSelection(ctx context.Context, selection file.Selection, acce } }) - frame := archive.TraversalFromContext(ctx).ContentsArchivePath() + frame := archive.TraversalFromContext(ctx).FullTraversalArchivePath() var coords []file.Coordinates for _, c := range coordinates.ToSlice() { - if c.ArchivePath == frame { + if archive.ContainerPath(c) == frame { coords = append(coords, c) } } diff --git a/internal/task/file_tasks_test.go b/internal/task/file_tasks_test.go index 06641fb49..8e7af9cbb 100644 --- a/internal/task/file_tasks_test.go +++ b/internal/task/file_tasks_test.go @@ -43,19 +43,13 @@ func Test_coordinatesForSelection(t *testing.T) { Relationships: []artifact.Relationship{ { From: pkg.Package{}, - To: file.Coordinates{ - RealPath: "path", - FileSystemID: "fs", - }, + To: file.NewCoordinates("path", "fs"), Type: artifact.ContainsRelationship, }, }, }, files: []file.Coordinates{ - { - RealPath: "path", - FileSystemID: "fs", - }, + file.NewCoordinates("path", "fs"), }, ok: true, }, @@ -66,10 +60,7 @@ func Test_coordinatesForSelection(t *testing.T) { Relationships: []artifact.Relationship{ { From: pkg.Package{}, - To: file.Coordinates{ - RealPath: "path", - FileSystemID: "fs", - }, + To: file.NewCoordinates("path", "fs"), // wrong type Type: artifact.DependencyOfRelationship, }, @@ -85,10 +76,7 @@ func Test_coordinatesForSelection(t *testing.T) { Relationships: []artifact.Relationship{ { From: file.Coordinates{}, // wrong type - To: file.Coordinates{ - RealPath: "path", - FileSystemID: "fs", - }, + To: file.NewCoordinates("path", "fs"), Type: artifact.ContainsRelationship, }, }, @@ -109,8 +97,8 @@ func Test_coordinatesForSelection(t *testing.T) { func Test_coordinatesForSelection_onlyTheResolversFrame(t *testing.T) { // a package located at an archive in its parent's frame must not send the file catalogers looking up // that path inside the archive's own resolver - atRoot := file.Coordinates{RealPath: "/app.war"} - inWar := file.Coordinates{RealPath: "WEB-INF/lib/dep.jar", ArchivePath: "/app.war"} + atRoot := file.NewCoordinates("/app.war", "") + inWar := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/dep.jar"} s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection( pkg.Package{Name: "app", Locations: file.NewLocationSet(file.NewLocationFromCoordinates(atRoot))}, pkg.Package{Name: "dep", Locations: file.NewLocationSet(file.NewLocationFromCoordinates(inWar))}, diff --git a/internal/task/package_task_factory.go b/internal/task/package_task_factory.go index a3995ce6a..26e0b3db3 100644 --- a/internal/task/package_task_factory.go +++ b/internal/task/package_task_factory.go @@ -167,7 +167,7 @@ func applyComplianceRules(p *pkg.Package, cfg cataloging.ComplianceConfig) (bool loc := "unknown" locs := p.Locations.ToSlice() if len(locs) > 0 { - loc = locs[0].Path() + loc = locs[0].FullNestedPath() } switch action { case cataloging.ComplianceActionDrop: @@ -251,7 +251,7 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver for _, ref := range pathRefs { if oldRef, ok := locations[ref.ID()]; ok { - log.Debugf("found path duplicate of %s", oldRef.RealPath) + log.Debugf("found path duplicate of %s", oldRef.FullNestedPath()) } locations[ref.ID()] = ref } diff --git a/internal/task/package_task_factory_test.go b/internal/task/package_task_factory_test.go index f1b8a217c..a13650782 100644 --- a/internal/task/package_task_factory_test.go +++ b/internal/task/package_task_factory_test.go @@ -64,8 +64,8 @@ func TestApplyCompliance(t *testing.T) { p2 := pkg.Package{Name: "", Version: "1.0"} // missing name p3 := pkg.Package{Name: "pkg-3", Version: ""} // missing version p4 := pkg.Package{Name: "pkg-4", Version: ""} // missing version - c1 := file.Coordinates{RealPath: "/coords/1"} - c2 := file.Coordinates{RealPath: "/coords/2"} + c1 := file.NewCoordinates("/coords/1", "") + c2 := file.NewCoordinates("/coords/2", "") for _, p := range []*pkg.Package{&p1, &p2, &p3, &p4} { p.SetID() diff --git a/internal/task/scope_tasks.go b/internal/task/scope_tasks.go index 8127829ab..11e9e8016 100644 --- a/internal/task/scope_tasks.go +++ b/internal/task/scope_tasks.go @@ -40,10 +40,11 @@ func packagesToRemove(accessor sbomsync.Accessor) []artifact.ID { } case "": if isPrimaryEvidence { - if exists := filterDuplicates[getKey(p, l)]; exists { + k := getKey(p, l) + if exists := filterDuplicates[k]; exists { break } - filterDuplicates[getKey(p, l)] = true + filterDuplicates[k] = true noPrimary = false break } @@ -59,5 +60,5 @@ func packagesToRemove(accessor sbomsync.Accessor) []artifact.ID { } func getKey(pkg pkg.Package, loc file.Location) string { - return fmt.Sprintf("%s-%s-%s-%s", pkg.Name, pkg.Version, loc.RealPath, loc.AccessPath) + return fmt.Sprintf("%s-%s-%s-%s", pkg.Name, pkg.Version, loc.FullNestedPath(), loc.AccessPath) } diff --git a/internal/task/unknowns_tasks.go b/internal/task/unknowns_tasks.go index 8761a06cd..01ecfcefe 100644 --- a/internal/task/unknowns_tasks.go +++ b/internal/task/unknowns_tasks.go @@ -59,7 +59,7 @@ func (c unknownsLabelerTask) finalize(resolver file.Resolver, s *sbom.SBOM) { if c.IncludeUnexpandedArchives { ctx := context.Background() for coords := range s.Artifacts.FileMetadata { - format, _, notArchiveErr := intFile.IdentifyArchive(ctx, coords.RealPath, nil) + format, _, notArchiveErr := intFile.IdentifyArchive(ctx, coords.FullNestedPath(), nil) if format != nil && notArchiveErr == nil && !hasPackageReference(coords) { s.Artifacts.Unknowns[coords] = append(s.Artifacts.Unknowns[coords], "archive not cataloged") } diff --git a/internal/testutils/archives.go b/internal/testutils/archives.go new file mode 100644 index 000000000..df6524ae2 --- /dev/null +++ b/internal/testutils/archives.go @@ -0,0 +1,84 @@ +package testutils + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "fmt" + "io" + "sort" + "testing" + + "github.com/stretchr/testify/require" +) + +// Zip builds a zip of the entries in sorted name order: archive limits bite as entries are read, so +// order decides which land before a truncation. +func Zip[T ~string | ~[]byte](t testing.TB, entries map[string]T) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, name := range sortedNames(entries) { + w, err := zw.Create(name) + require.NoError(t, err) + _, err = w.Write([]byte(entries[name])) + require.NoError(t, err) + } + require.NoError(t, zw.Close()) + return buf.Bytes() +} + +// Tar builds a tar of the entries in sorted name order. +func Tar[T ~string | ~[]byte](t testing.TB, entries map[string]T) []byte { + t.Helper() + var buf bytes.Buffer + writeTar(t, &buf, entries) + return buf.Bytes() +} + +// TarGz builds a gzipped tar of the entries in sorted name order. +func TarGz[T ~string | ~[]byte](t testing.TB, entries map[string]T) []byte { + t.Helper() + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + writeTar(t, gw, entries) + require.NoError(t, gw.Close()) + return buf.Bytes() +} + +// Jar is the smallest jar the java cataloger identifies, a manifest naming the artifact, plus extra. +func Jar(t testing.TB, title, version string, extra map[string][]byte) []byte { + t.Helper() + entries := map[string][]byte{"META-INF/MANIFEST.MF": Manifest(title, version)} + for name, body := range extra { + entries[name] = body + } + return Zip(t, entries) +} + +// Manifest is a jar manifest naming the artifact. +func Manifest(title, version string) []byte { + return fmt.Appendf(nil, "Manifest-Version: 1.0\nImplementation-Title: %s\nImplementation-Version: %s\n", title, version) +} + +func writeTar[T ~string | ~[]byte](t testing.TB, w io.Writer, entries map[string]T) { + t.Helper() + tw := tar.NewWriter(w) + for _, name := range sortedNames(entries) { + body := []byte(entries[name]) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg})) + _, err := tw.Write(body) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) +} + +func sortedNames[T any](entries map[string]T) []string { + names := make([]string, 0, len(entries)) + for name := range entries { + names = append(names, name) + } + sort.Strings(names) + return names +} diff --git a/internal/unknown/coordinate_error.go b/internal/unknown/coordinate_error.go index 2db6caf7e..3b8824ee5 100644 --- a/internal/unknown/coordinate_error.go +++ b/internal/unknown/coordinate_error.go @@ -23,9 +23,9 @@ var _ error = (*CoordinateError)(nil) func (u *CoordinateError) Error() string { if u.Coordinates.FileSystemID == "" { - return fmt.Sprintf("%s: %v", u.Coordinates.RealPath, u.Reason) + return fmt.Sprintf("%s: %v", u.Coordinates.FullNestedPath(), u.Reason) } - return fmt.Sprintf("%s (%s): %v", u.Coordinates.RealPath, u.Coordinates.FileSystemID, u.Reason) + return fmt.Sprintf("%s (%s): %v", u.Coordinates.FullNestedPath(), u.Coordinates.FileSystemID, u.Reason) } // New returns a new CoordinateError unless the reason is a CoordinateError itself, in which case diff --git a/internal/unknown/coordinate_error_test.go b/internal/unknown/coordinate_error_test.go index e863e6359..53adb14fe 100644 --- a/internal/unknown/coordinate_error_test.go +++ b/internal/unknown/coordinate_error_test.go @@ -169,9 +169,7 @@ func Test_Join(t *testing.T) { } func Test_flatten(t *testing.T) { - coords := file.Coordinates{ - RealPath: "real/path", - } + coords := file.NewCoordinates("real/path", "") e1 := fmt.Errorf("e1") e2 := fmt.Errorf("e2") c1 := New(coords, fmt.Errorf("c1")) diff --git a/internal/unknown/path_error_test.go b/internal/unknown/path_error_test.go index d25133a7f..53a82c6b2 100644 --- a/internal/unknown/path_error_test.go +++ b/internal/unknown/path_error_test.go @@ -17,28 +17,22 @@ func Test_ProcessPathErrors(t *testing.T) { { errorText: `prefix path="/var/lib/thing" suffix`, expected: &CoordinateError{ - Coordinates: file.Coordinates{ - RealPath: "/var/lib/thing", - }, - Reason: fmt.Errorf(`prefix path="/var/lib/thing" suffix`), + Coordinates: file.NewCoordinates("/var/lib/thing", ""), + Reason: fmt.Errorf(`prefix path="/var/lib/thing" suffix`), }, }, { errorText: `prefix path="/var/lib/thing"`, expected: &CoordinateError{ - Coordinates: file.Coordinates{ - RealPath: "/var/lib/thing", - }, - Reason: fmt.Errorf(`prefix path="/var/lib/thing"`), + Coordinates: file.NewCoordinates("/var/lib/thing", ""), + Reason: fmt.Errorf(`prefix path="/var/lib/thing"`), }, }, { errorText: `path="/var/lib/thing" suffix`, expected: &CoordinateError{ - Coordinates: file.Coordinates{ - RealPath: "/var/lib/thing", - }, - Reason: fmt.Errorf(`path="/var/lib/thing" suffix`), + Coordinates: file.NewCoordinates("/var/lib/thing", ""), + Reason: fmt.Errorf(`path="/var/lib/thing" suffix`), }, }, { diff --git a/schema/json/schema-16.1.12.json b/schema/json/schema-16.1.12.json index 84049aa4a..bc9ce1d73 100644 --- a/schema/json/schema-16.1.12.json +++ b/schema/json/schema-16.1.12.json @@ -716,15 +716,15 @@ "properties": { "path": { "type": "string", - "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed)." + "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem." }, "layerID": { "type": "string", "description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in." }, - "archivePath": { + "fullPath": { "type": "string", - "description": "ArchivePath is the colon-delimited chain of archive paths, from the scan root, traversed to reach this file (e.g. \"app.war:WEB-INF/lib/dep.jar\"). Blank for files not found within an archive. It disambiguates identically-named files in different archives on the same filesystem. Within each segment after the first, '%' is escaped as \"%25\" and ':' as \"%3A\", so splitting on ':' recovers the archive boundaries. Each segment is an archive's real path (links resolved), relative to the segment before it." + "description": "RealFullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it equals RealPath; it may be blank on coordinates not built by a constructor, such as those decoded from an older SBOM." } }, "type": "object", @@ -2548,19 +2548,19 @@ "properties": { "path": { "type": "string", - "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed)." + "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem." }, "layerID": { "type": "string", "description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in." }, - "archivePath": { + "fullPath": { "type": "string", - "description": "ArchivePath is the colon-delimited chain of archive paths, from the scan root, traversed to reach this file (e.g. \"app.war:WEB-INF/lib/dep.jar\"). Blank for files not found within an archive. It disambiguates identically-named files in different archives on the same filesystem. Within each segment after the first, '%' is escaped as \"%25\" and ':' as \"%3A\", so splitting on ':' recovers the archive boundaries. Each segment is an archive's real path (links resolved), relative to the segment before it." + "description": "RealFullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it equals RealPath; it may be blank on coordinates not built by a constructor, such as those decoded from an older SBOM." }, "accessPath": { "type": "string", - "description": "AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path)" + "description": "AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path). For a file within an archive, it is a full path (see Coordinates.RealFullPath)." }, "annotations": { "additionalProperties": { diff --git a/schema/json/schema-latest.json b/schema/json/schema-latest.json index 84049aa4a..bc9ce1d73 100644 --- a/schema/json/schema-latest.json +++ b/schema/json/schema-latest.json @@ -716,15 +716,15 @@ "properties": { "path": { "type": "string", - "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed)." + "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem." }, "layerID": { "type": "string", "description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in." }, - "archivePath": { + "fullPath": { "type": "string", - "description": "ArchivePath is the colon-delimited chain of archive paths, from the scan root, traversed to reach this file (e.g. \"app.war:WEB-INF/lib/dep.jar\"). Blank for files not found within an archive. It disambiguates identically-named files in different archives on the same filesystem. Within each segment after the first, '%' is escaped as \"%25\" and ':' as \"%3A\", so splitting on ':' recovers the archive boundaries. Each segment is an archive's real path (links resolved), relative to the segment before it." + "description": "RealFullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it equals RealPath; it may be blank on coordinates not built by a constructor, such as those decoded from an older SBOM." } }, "type": "object", @@ -2548,19 +2548,19 @@ "properties": { "path": { "type": "string", - "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed)." + "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem." }, "layerID": { "type": "string", "description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in." }, - "archivePath": { + "fullPath": { "type": "string", - "description": "ArchivePath is the colon-delimited chain of archive paths, from the scan root, traversed to reach this file (e.g. \"app.war:WEB-INF/lib/dep.jar\"). Blank for files not found within an archive. It disambiguates identically-named files in different archives on the same filesystem. Within each segment after the first, '%' is escaped as \"%25\" and ':' as \"%3A\", so splitting on ':' recovers the archive boundaries. Each segment is an archive's real path (links resolved), relative to the segment before it." + "description": "RealFullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it equals RealPath; it may be blank on coordinates not built by a constructor, such as those decoded from an older SBOM." }, "accessPath": { "type": "string", - "description": "AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path)" + "description": "AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path). For a file within an archive, it is a full path (see Coordinates.RealFullPath)." }, "annotations": { "additionalProperties": { diff --git a/syft/archive_cataloger_test.go b/syft/archive_cataloger_test.go index 6a4bdc276..27adbdbf4 100644 --- a/syft/archive_cataloger_test.go +++ b/syft/archive_cataloger_test.go @@ -2,7 +2,6 @@ package syft import ( "archive/tar" - "archive/zip" "bytes" "compress/gzip" "context" @@ -18,8 +17,10 @@ import ( "github.com/wagoodman/go-partybus" "github.com/wagoodman/go-progress" + "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/task" + "github.com/anchore/syft/internal/testutils" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/cataloging/filecataloging" @@ -59,38 +60,37 @@ var fixtureExclusions = []string{"./nested/marker.txt", "**/excluded.txt", "**/i func writeFixture(t *testing.T) string { t.Helper() dir := t.TempDir() - marker := map[string]string{"nested/marker.txt": "hello"} - markerZip := buildZipBytes(t, marker) + write := func(name string, data []byte) { + require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, name), data, 0o644)) + } + markerZip := testutils.Zip(t, map[string]string{"nested/marker.txt": "hello"}) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "nested"), 0o755)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "app"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "loose.txt"), []byte("outside"), 0o600)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "nested", "marker.txt"), []byte("at the root"), 0o644)) - - writeTestZip(t, filepath.Join(dir, "app.zip"), map[string]string{ + write("loose.txt", []byte("outside")) + write("nested/marker.txt", []byte("at the root")) + write("app.zip", testutils.Zip(t, map[string]string{ "nested/marker.txt": "hello", "nested/keep.txt": "hello", "nested/excluded.txt": "hello", - }) - writeTestZipRaw(t, filepath.Join(dir, "app", "data.zip"), map[string][]byte{"nested/marker.txt": []byte("hello")}) - writeTestZipRaw(t, filepath.Join(dir, "outer.zip"), map[string][]byte{ + })) + write("app/data.zip", markerZip) + write("outer.zip", testutils.Zip(t, map[string][]byte{ "outer/marker.txt": []byte("hello"), "deeper/inner.zip": markerZip, "skip/inner-excluded.zip": markerZip, "日本語/café-🎉/unicode.zip": markerZip, - }) - require.NoError(t, os.WriteFile(filepath.Join(dir, "pkg.rpm"), markerZip, 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "KEPT.RPM"), markerZip, 0o644)) - - writeTestZipRaw(t, filepath.Join(dir, "app.war"), map[string][]byte{ - "WEB-INF/lib/war-lib-2.0.jar": jarBytes(t, "war-lib", "2.0"), - }) - writeTestZipRaw(t, filepath.Join(dir, "bundle.zip"), map[string][]byte{ - "lib/zip-lib-1.2.jar": jarBytes(t, "zip-lib", "1.2"), - }) - require.NoError(t, os.WriteFile(filepath.Join(dir, "bundle.tar.gz"), buildTarGzBytes(t, map[string][]byte{ - "lib/tgz-lib-1.2.jar": jarBytes(t, "tgz-lib", "1.2"), - }), 0o644)) + })) + write("pkg.rpm", markerZip) + write("KEPT.RPM", markerZip) + write("app.war", testutils.Zip(t, map[string][]byte{ + "WEB-INF/lib/war-lib-2.0.jar": testutils.Jar(t, "war-lib", "2.0", nil), + })) + write("bundle.zip", testutils.Zip(t, map[string][]byte{ + "lib/zip-lib-1.2.jar": testutils.Jar(t, "zip-lib", "1.2", nil), + })) + write("bundle.tar.gz", testutils.TarGz(t, map[string][]byte{ + "lib/tgz-lib-1.2.jar": testutils.Jar(t, "tgz-lib", "1.2", nil), + })) return dir } @@ -134,21 +134,21 @@ func TestArchiveCataloger(t *testing.T) { s := scanDirWithExclusions(t, scanDir, cfg, fixtureExclusions...) t.Run("markers are found at every depth under archive-relative paths", func(t *testing.T) { - // the location's ArchivePath is the chain of archives; RealPath is relative to the innermost + // the location's RealPath is the outermost archive; FullPath adds the chain of entries within it assert.Equal(t, []string{ - "KEPT.RPM|nested/marker.txt", - "app.zip|nested/marker.txt", - "app/data.zip|nested/marker.txt", - "outer.zip:deeper/inner.zip|nested/marker.txt", - "outer.zip:日本語/café-🎉/unicode.zip|nested/marker.txt", - "outer.zip|outer/marker.txt", + "KEPT.RPM!/nested/marker.txt", + "app.zip!/nested/marker.txt", + "app/data.zip!/nested/marker.txt", + "outer.zip!/deeper/inner.zip!/nested/marker.txt", + "outer.zip!/outer/marker.txt", + "outer.zip!/日本語/café-🎉/unicode.zip!/nested/marker.txt", }, packageLocations(s, "marker-pkg")) }) t.Run("archive contents do not leak into the parent tree", func(t *testing.T) { for _, loc := range packageLocations(s, "marker-pkg") { assert.NotContains(t, loc, ".zip/", "a joined path smears the archive's contents into the parent's tree") - assert.NotContains(t, loc, "|/", "a nested path must not be rooted") + assert.NotContains(t, loc, "!//", "a nested path must not be rooted") } }) @@ -169,18 +169,18 @@ func TestArchiveCataloger(t *testing.T) { continue } if coords, ok := rel.From.(file.Coordinates); ok { - containers = append(containers, coords.ArchivePath+"|"+coords.RealPath) + containers = append(containers, coords.FullNestedPath()) } } } sort.Strings(containers) assert.Equal(t, []string{ - "outer.zip|deeper/inner.zip", - "outer.zip|日本語/café-🎉/unicode.zip", - "|KEPT.RPM", - "|app.zip", - "|app/data.zip", - "|outer.zip", + "KEPT.RPM", + "app.zip", + "app/data.zip", + "outer.zip", + "outer.zip!/deeper/inner.zip", + "outer.zip!/日本語/café-🎉/unicode.zip", }, containers, "the archive file, named in its own parent, contains the package") }) @@ -189,16 +189,16 @@ func TestArchiveCataloger(t *testing.T) { // reaches inside an archive, and an excluded entry is absent from the archive's filesystem // rather than present and skipped markers := packageLocations(s, "marker-pkg") - assert.NotContains(t, markers, "|nested/marker.txt", "a root-anchored pattern excludes the file at the scan root") - assert.Contains(t, markers, "app.zip|nested/marker.txt", "and says nothing about the same path inside an archive") - assert.NotContains(t, markers, "pkg.rpm|nested/marker.txt", "an archive whose own path is excluded is not extracted") - assert.Contains(t, markers, "KEPT.RPM|nested/marker.txt", "a lower-case pattern does not match an upper-case name") - assert.NotContains(t, markers, "outer.zip:skip/inner-excluded.zip|nested/marker.txt", "a nested archive is excluded at its own level") + assert.NotContains(t, markers, "nested/marker.txt", "a root-anchored pattern excludes the file at the scan root") + assert.Contains(t, markers, "app.zip!/nested/marker.txt", "and says nothing about the same path inside an archive") + assert.NotContains(t, markers, "pkg.rpm!/nested/marker.txt", "an archive whose own path is excluded is not extracted") + assert.Contains(t, markers, "KEPT.RPM!/nested/marker.txt", "a lower-case pattern does not match an upper-case name") + assert.NotContains(t, markers, "outer.zip!/skip/inner-excluded.zip!/nested/marker.txt", "a nested archive is excluded at its own level") - assert.Contains(t, seen, "nested/keep.txt") - assert.NotContains(t, seen, "nested/excluded.txt", "a **/ pattern excludes inside an archive, from the index itself") - assert.Contains(t, seen, "deeper/inner.zip", "the probe ran inside outer.zip") - assert.NotContains(t, seen, "skip/inner-excluded.zip", "the excluded inner archive is absent from outer's filesystem") + assert.Contains(t, seen, "app.zip!/nested/keep.txt") + assert.NotContains(t, seen, "app.zip!/nested/excluded.txt", "a **/ pattern excludes inside an archive, from the index itself") + assert.Contains(t, seen, "outer.zip!/deeper/inner.zip", "the probe ran inside outer.zip") + assert.NotContains(t, seen, "outer.zip!/skip/inner-excluded.zip", "the excluded inner archive is absent from outer's filesystem") }) t.Run("a jar inside an archive is cataloged exactly once, under the virtual path java gives it", func(t *testing.T) { @@ -249,28 +249,28 @@ func TestArchiveCataloger(t *testing.T) { // output would fail it var got []string for coords := range s.Artifacts.FileMetadata { - if coords.ArchivePath == "" { + if archive.ContainerPath(coords) == "" { continue } - got = append(got, coords.ArchivePath+"|"+coords.RealPath) + got = append(got, coords.FullNestedPath()) } sort.Strings(got) assert.Equal(t, []string{ - "KEPT.RPM|nested/marker.txt", - "app.war:WEB-INF/lib/war-lib-2.0.jar|META-INF/MANIFEST.MF", - "app.war|WEB-INF/lib/war-lib-2.0.jar", - "app.zip|nested/keep.txt", - "app.zip|nested/marker.txt", - "app/data.zip|nested/marker.txt", - "bundle.tar.gz:lib/tgz-lib-1.2.jar|META-INF/MANIFEST.MF", - "bundle.tar.gz|lib/tgz-lib-1.2.jar", - "bundle.zip:lib/zip-lib-1.2.jar|META-INF/MANIFEST.MF", - "bundle.zip|lib/zip-lib-1.2.jar", - "outer.zip:deeper/inner.zip|nested/marker.txt", - "outer.zip:日本語/café-🎉/unicode.zip|nested/marker.txt", - "outer.zip|deeper/inner.zip", - "outer.zip|outer/marker.txt", - "outer.zip|日本語/café-🎉/unicode.zip", + "KEPT.RPM!/nested/marker.txt", + "app.war!/WEB-INF/lib/war-lib-2.0.jar", + "app.war!/WEB-INF/lib/war-lib-2.0.jar!/META-INF/MANIFEST.MF", + "app.zip!/nested/keep.txt", + "app.zip!/nested/marker.txt", + "app/data.zip!/nested/marker.txt", + "bundle.tar.gz!/lib/tgz-lib-1.2.jar", + "bundle.tar.gz!/lib/tgz-lib-1.2.jar!/META-INF/MANIFEST.MF", + "bundle.zip!/lib/zip-lib-1.2.jar", + "bundle.zip!/lib/zip-lib-1.2.jar!/META-INF/MANIFEST.MF", + "outer.zip!/deeper/inner.zip", + "outer.zip!/deeper/inner.zip!/nested/marker.txt", + "outer.zip!/outer/marker.txt", + "outer.zip!/日本語/café-🎉/unicode.zip", + "outer.zip!/日本語/café-🎉/unicode.zip!/nested/marker.txt", }, got) }) @@ -338,7 +338,7 @@ func TestArchiveCataloger_jarReachedThroughALinkIsIdentified(t *testing.T) { var buf bytes.Buffer gw := gzip.NewWriter(&buf) tw := tar.NewWriter(gw) - jar := jarBytes(t, "json-simple", "1.1.1") + jar := testutils.Jar(t, "json-simple", "1.1.1", nil) require.NoError(t, tw.WriteHeader(&tar.Header{Name: "store/3f9a1c", Mode: 0o644, Size: int64(len(jar))})) _, err := tw.Write(jar) require.NoError(t, err) @@ -355,10 +355,10 @@ func TestArchiveCataloger_jarReachedThroughALinkIsIdentified(t *testing.T) { WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2)) s := scanDirWithExclusions(t, dir, cfg) - assert.Equal(t, []string{"bundle.tar.gz|store/3f9a1c"}, packageLocations(s, "json-simple")) + assert.Equal(t, []string{"bundle.tar.gz!/store/3f9a1c"}, packageLocations(s, "json-simple")) // the jar's own files are addressed by its real path, the same one its package location names - manifest := file.Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "bundle.tar.gz:store/3f9a1c"} + manifest := file.Coordinates{RealPath: "bundle.tar.gz", FullPath: "bundle.tar.gz!/store/3f9a1c!/META-INF/MANIFEST.MF"} assert.Contains(t, s.Artifacts.FileMetadata, manifest) } @@ -366,7 +366,7 @@ func TestArchiveCataloger_defaultFileSelectionCoversRootJars(t *testing.T) { // with the default owned-by-package selection, a jar at the root is described by the archive task, // so the root file catalogers must run after it or the jar gets no file entries at all dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "app.jar"), jarBytes(t, "app", "1.0"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "app.jar"), testutils.Jar(t, "app", "1.0", nil), 0o644)) for name, depth := range map[string]int{"feature off": 0, "feature on": 1} { t.Run(name, func(t *testing.T) { @@ -375,8 +375,8 @@ func TestArchiveCataloger_defaultFileSelectionCoversRootJars(t *testing.T) { WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth)) s := scanDirWithExclusions(t, dir, cfg) - require.Equal(t, []string{"|app.jar"}, packageLocations(s, "app")) - jar := file.Coordinates{RealPath: "app.jar"} + require.Equal(t, []string{"app.jar"}, packageLocations(s, "app")) + jar := file.NewCoordinates("app.jar", "") assert.Contains(t, s.Artifacts.FileMetadata, jar) assert.Contains(t, s.Artifacts.FileDigests, jar) }) @@ -387,9 +387,8 @@ func TestArchiveCataloger_javaParentLinkageMatchesTheLegacyPath(t *testing.T) { // a consumer walking java DEPENDENCY_OF edges (the only structure SPDX and CycloneDX keep) must see // the same war-to-jar linkage whether or not the archive task does the recursion dir := t.TempDir() - war := buildZipBytesRaw(t, map[string][]byte{ - "META-INF/MANIFEST.MF": []byte("Manifest-Version: 1.0\nImplementation-Title: app\nImplementation-Version: 1.0\n"), - "WEB-INF/lib/lib-2.0.jar": jarBytes(t, "lib", "2.0"), + war := testutils.Jar(t, "app", "1.0", map[string][]byte{ + "WEB-INF/lib/lib-2.0.jar": testutils.Jar(t, "lib", "2.0", nil), }) require.NoError(t, os.WriteFile(filepath.Join(dir, "app.war"), war, 0o644)) @@ -458,7 +457,7 @@ func TestArchiveCataloger_javaArchiveDepthIsAConfigError(t *testing.T) { } } -// packageLocations lists "archivePath|realPath" for every location of every package with the name. +// packageLocations lists the full path of every location of every package with the name. func packageLocations(s *sbom.SBOM, name string) []string { var out []string for _, p := range s.Artifacts.Packages.Sorted() { @@ -466,7 +465,7 @@ func packageLocations(s *sbom.SBOM, name string) []string { continue } for _, loc := range p.Locations.ToSlice() { - out = append(out, loc.ArchivePath+"|"+loc.RealPath) + out = append(out, loc.FullNestedPath()) } } sort.Strings(out) @@ -526,7 +525,7 @@ func (c resolverProbeCataloger) Catalog(_ context.Context, resolver file.Resolve } var paths []string for _, loc := range locations { - paths = append(paths, loc.RealPath) + paths = append(paths, loc.FullNestedPath()) } c.record(paths) return nil, nil, nil @@ -559,76 +558,3 @@ func scanDirWithExclusions(t *testing.T, dir string, cfg *CreateSBOMConfig, excl require.NoError(t, err) return s } - -func writeTestZip(t *testing.T, path string, files map[string]string) { - t.Helper() - require.NoError(t, os.WriteFile(path, buildZipBytes(t, files), 0o644)) -} - -func writeTestZipRaw(t *testing.T, path string, files map[string][]byte) { - t.Helper() - require.NoError(t, os.WriteFile(path, buildZipBytesRaw(t, files), 0o644)) -} - -func buildZipBytes(t *testing.T, files map[string]string) []byte { - t.Helper() - raw := make(map[string][]byte, len(files)) - for k, v := range files { - raw[k] = []byte(v) - } - return buildZipBytesRaw(t, raw) -} - -// buildZipBytesRaw writes entries in sorted name order rather than map order: limits are enforced as -// the walk proceeds, so entry order decides which entries land before a truncation. -func buildZipBytesRaw(t *testing.T, files map[string][]byte) []byte { - t.Helper() - names := make([]string, 0, len(files)) - for name := range files { - names = append(names, name) - } - sort.Strings(names) - - var buf bytes.Buffer - zw := zip.NewWriter(&buf) - for _, name := range names { - w, err := zw.Create(name) - require.NoError(t, err) - _, err = w.Write(files[name]) - require.NoError(t, err) - } - require.NoError(t, zw.Close()) - return buf.Bytes() -} - -func buildTarGzBytes(t *testing.T, files map[string][]byte) []byte { - t.Helper() - names := make([]string, 0, len(files)) - for name := range files { - names = append(names, name) - } - sort.Strings(names) - - var buf bytes.Buffer - gw := gzip.NewWriter(&buf) - tw := tar.NewWriter(gw) - for _, name := range names { - body := files[name] - require.NoError(t, tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))})) - _, err := tw.Write(body) - require.NoError(t, err) - } - require.NoError(t, tw.Close()) - require.NoError(t, gw.Close()) - return buf.Bytes() -} - -// jarBytes is the smallest jar the java cataloger identifies: a manifest naming the artifact. -func jarBytes(t *testing.T, title, version string) []byte { - t.Helper() - return buildZipBytesRaw(t, map[string][]byte{ - "META-INF/MANIFEST.MF": []byte( - "Manifest-Version: 1.0\nImplementation-Title: " + title + "\nImplementation-Version: " + version + "\n", - ), - }) -} diff --git a/syft/file/cataloger/executable/cataloger.go b/syft/file/cataloger/executable/cataloger.go index cb73dc8f4..7929a80eb 100644 --- a/syft/file/cataloger/executable/cataloger.go +++ b/syft/file/cataloger/executable/cataloger.go @@ -68,7 +68,7 @@ func (i *Cataloger) CatalogCtx(ctx context.Context, resolver file.Resolver) (map results := make(map[file.Coordinates]file.Executable) errs := sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(locs), func(loc file.Location) (*file.Executable, error) { - prog.AtomicStage.Set(loc.Path()) + prog.AtomicStage.Set(loc.FullNestedPath()) exec, err := processExecutableLocation(loc, resolver) if err != nil { @@ -93,15 +93,15 @@ func (i *Cataloger) CatalogCtx(ctx context.Context, resolver file.Resolver) (map func processExecutableLocation(loc file.Location, resolver file.Resolver) (exec *file.Executable, err error) { reader, err := resolver.FileContentsByLocation(loc) if err != nil { - log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get file contents") + log.WithFields("error", err, "path", loc.FullNestedPath()).Debug("unable to get file contents") return nil, fmt.Errorf("unable to get file contents: %w", err) } - defer internal.CloseAndLogError(reader, loc.RealPath) + defer internal.CloseAndLogError(reader, loc.FullNestedPath()) defer func() { if r := recover(); r != nil { // a malformed binary must not take down the whole SBOM; the caller records this as an unknown - log.WithFields("path", loc.RealPath, "panic", r).Debug("recovered from panic while reading executable") + log.WithFields("path", loc.FullNestedPath(), "panic", r).Debug("recovered from panic while reading executable") log.Tracef("executable panic stack:\n%s", debug.Stack()) exec, err = nil, fmt.Errorf("recovered from panic while reading executable: %v", r) } @@ -109,7 +109,7 @@ func processExecutableLocation(loc file.Location, resolver file.Resolver) (exec uReader, err := unionreader.GetUnionReader(reader) if err != nil { - log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get union reader") + log.WithFields("error", err, "path", loc.FullNestedPath()).Debug("unable to get union reader") return nil, fmt.Errorf("unable to get union reader: %w", err) } @@ -146,7 +146,7 @@ func filterByGlobs(locs []file.Location, globs []string) ([]file.Location, error func locationMatchesGlob(loc file.Location, globs []string) (bool, error) { for _, glob := range globs { - for _, path := range []string{loc.RealPath, loc.AccessPath} { + for _, path := range []string{loc.LocalPath(), loc.AccessPath} { if path == "" { continue } @@ -169,13 +169,13 @@ func processExecutable(loc file.Location, reader unionreader.UnionReader) (*file format, err := findExecutableFormat(reader) if err != nil { - log.Debugf("unable to determine executable kind for %v: %v", loc.RealPath, err) + log.Debugf("unable to determine executable kind for %v: %v", loc.FullNestedPath(), err) return nil, fmt.Errorf("unable to determine executable kind: %w", err) } if format == "" { // this is not an "unknown", so just log -- this binary does not have parseable data in it - log.Debugf("unable to determine executable format for %q", loc.RealPath) + log.Debugf("unable to determine executable format for %q", loc.FullNestedPath()) return nil, nil } @@ -184,17 +184,17 @@ func processExecutable(loc file.Location, reader unionreader.UnionReader) (*file switch format { case file.ELF: if err = findELFFeatures(&data, reader); err != nil { - log.WithFields("error", err, "path", loc.RealPath).Trace("unable to determine ELF features") + log.WithFields("error", err, "path", loc.FullNestedPath()).Trace("unable to determine ELF features") err = fmt.Errorf("unable to determine ELF features: %w", err) } case file.PE: if err = findPEFeatures(&data, reader); err != nil { - log.WithFields("error", err, "path", loc.RealPath).Trace("unable to determine PE features") + log.WithFields("error", err, "path", loc.FullNestedPath()).Trace("unable to determine PE features") err = fmt.Errorf("unable to determine PE features: %w", err) } case file.MachO: if err = findMachoFeatures(&data, reader); err != nil { - log.WithFields("error", err, "path", loc.RealPath).Trace("unable to determine Macho features") + log.WithFields("error", err, "path", loc.FullNestedPath()).Trace("unable to determine Macho features") err = fmt.Errorf("unable to determine Macho features: %w", err) } } diff --git a/syft/file/cataloger/filecontent/cataloger.go b/syft/file/cataloger/filecontent/cataloger.go index 2e3417d4f..3d5ebe85d 100644 --- a/syft/file/cataloger/filecontent/cataloger.go +++ b/syft/file/cataloger/filecontent/cataloger.go @@ -57,7 +57,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) (map[fi prog := catalogingProgress(ctx, int64(len(locations))) for _, location := range locations { - prog.AtomicStage.Set(location.Path()) + prog.AtomicStage.Set(location.FullNestedPath()) metadata, err := resolver.FileMetadataByLocation(location) if err != nil { @@ -103,7 +103,7 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati buf := &bytes.Buffer{} encoder := base64.NewEncoder(base64.StdEncoding, buf) if _, err = io.Copy(encoder, contentReader); err != nil { - return "", internal.ErrPath{Context: "content-cataloger", Path: location.RealPath, Err: err} + return "", internal.ErrPath{Context: "content-cataloger", Path: location.FullNestedPath(), Err: err} } // note: it's important to close the reader before reading from the buffer since closing will flush the remaining bytes if err := encoder.Close(); err != nil { diff --git a/syft/file/cataloger/filedigest/cataloger.go b/syft/file/cataloger/filedigest/cataloger.go index 447405648..5617c7eaa 100644 --- a/syft/file/cataloger/filedigest/cataloger.go +++ b/syft/file/cataloger/filedigest/cataloger.go @@ -42,9 +42,9 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin locations = intCataloger.AllRegularFiles(ctx, resolver) } else { for _, c := range coordinates { - locs, err := resolver.FilesByPath(c.RealPath) + locs, err := resolver.FilesByPath(c.LocalPath()) if err != nil { - return nil, fmt.Errorf("unable to get file locations for path %q: %w", c.RealPath, err) + return nil, fmt.Errorf("unable to get file locations for path %q: %w", c.FullNestedPath(), err) } locations = append(locations, locs...) } @@ -59,10 +59,10 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin return nil, nil } - prog.AtomicStage.Set(location.Path()) + prog.AtomicStage.Set(location.FullNestedPath()) if internal.IsErrPathPermission(err) { - log.Debugf("file digests cataloger skipping %q: %+v", location.RealPath, err) + log.Debugf("file digests cataloger skipping %q: %+v", location.FullNestedPath(), err) return nil, unknown.New(location, err) } @@ -108,7 +108,7 @@ func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver, defer func() { if r := recover(); r != nil { // the caller records this as an unknown for the location instead of failing the whole SBOM - log.WithFields("path", location.RealPath, "panic", r).Debug("recovered from panic while digesting file") + log.WithFields("path", location.FullNestedPath(), "panic", r).Debug("recovered from panic while digesting file") log.Tracef("file digest panic stack:\n%s", debug.Stack()) digests, err = nil, fmt.Errorf("recovered from panic while digesting file: %v", r) } @@ -116,7 +116,7 @@ func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver, digests, err = intFile.NewDigestsFromFile(ctx, contentReader, i.hashes) if err != nil { - return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err} + return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.FullNestedPath(), Err: err} } return digests, nil diff --git a/syft/file/cataloger/filemetadata/cataloger.go b/syft/file/cataloger/filemetadata/cataloger.go index b8e7d64a2..ab24a1f40 100644 --- a/syft/file/cataloger/filemetadata/cataloger.go +++ b/syft/file/cataloger/filemetadata/cataloger.go @@ -34,7 +34,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin go func() { defer close(ch) for _, c := range coordinates { - locs, err := resolver.FilesByPath(c.RealPath) + locs, err := resolver.FilesByPath(c.LocalPath()) if err != nil { errs = unknown.Append(errs, c, err) continue @@ -55,7 +55,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin prog := catalogingProgress(ctx, -1) for location := range locations { - prog.AtomicStage.Set(location.Path()) + prog.AtomicStage.Set(location.FullNestedPath()) metadata, err := resolver.FileMetadataByLocation(location) if err != nil { diff --git a/syft/file/cataloger/internal/all_regular_files.go b/syft/file/cataloger/internal/all_regular_files.go index d653132fa..1ea1d80a1 100644 --- a/syft/file/cataloger/internal/all_regular_files.go +++ b/syft/file/cataloger/internal/all_regular_files.go @@ -12,7 +12,7 @@ func AllRegularFiles(ctx context.Context, resolver file.Resolver) (locations []f ctx, cancel := context.WithCancel(ctx) defer cancel() for location := range resolver.AllLocations(ctx) { - resolvedLocations, err := resolver.FilesByPath(location.RealPath) + resolvedLocations, err := resolver.FilesByPath(location.LocalPath()) if err != nil { log.Debugf("unable to resolve %+v: %+v", location, err) continue diff --git a/syft/file/coordinate_set.go b/syft/file/coordinate_set.go index 086184e30..b275cf6ad 100644 --- a/syft/file/coordinate_set.go +++ b/syft/file/coordinate_set.go @@ -79,7 +79,7 @@ func (s CoordinateSet) ToSlice(sorters ...func(a, b Coordinates) int) []Coordina sort.SliceStable(coordinates, func(i, j int) bool { if coordinates[i].FileSystemID == coordinates[j].FileSystemID { if coordinates[i].RealPath == coordinates[j].RealPath { - return coordinates[i].ArchivePath < coordinates[j].ArchivePath + return coordinates[i].FullNestedPath() < coordinates[j].FullNestedPath() } return coordinates[i].RealPath < coordinates[j].RealPath } diff --git a/syft/file/coordinates.go b/syft/file/coordinates.go index 792d1c4e9..48b86c5bd 100644 --- a/syft/file/coordinates.go +++ b/syft/file/coordinates.go @@ -2,21 +2,42 @@ package file import ( "fmt" + "strings" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" ) +const ArchivePathSeparator = "!/" + // Coordinates contains the minimal information needed to describe how to find a file within any possible source object (e.g. image and directory sources) type Coordinates struct { - // RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). + // RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem. RealPath string `json:"path" cyclonedx:"path"` // FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in. FileSystemID string `json:"layerID,omitempty" cyclonedx:"layerID"` - // ArchivePath is the colon-delimited chain of archive paths, from the scan root, traversed to reach this file (e.g. "app.war:WEB-INF/lib/dep.jar"). Blank for files not found within an archive. It disambiguates identically-named files in different archives on the same filesystem. Within each segment after the first, '%' is escaped as "%25" and ':' as "%3A", so splitting on ':' recovers the archive boundaries. Each segment is an archive's real path (links resolved), relative to the segment before it. - ArchivePath string `json:"archivePath,omitempty" cyclonedx:"archivePath"` + // FullPath is the full path to the file: RealPath, then for a file within an archive a "!/"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. "/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it is blank, so coordinates outside archives are unchanged by the field (in JSON, equality and ID). + FullPath string `json:"fullPath,omitempty" cyclonedx:"fullPath"` +} + +// FullNestedPath is FullPath, or RealPath where FullPath is blank. +func (c Coordinates) FullNestedPath() string { + if c.FullPath == "" { + return c.RealPath + } + return c.FullPath +} + +// LocalPath is the path within the resolver holding the file: the segment of FullNestedPath after the +// last archive separator, or RealPath for a file not within an archive. +func (c Coordinates) LocalPath() string { + fullPath := c.FullNestedPath() + if i := strings.LastIndex(fullPath, ArchivePathSeparator); i >= 0 { + return fullPath[i+len(ArchivePathSeparator):] + } + return fullPath } func NewCoordinates(realPath, fsID string) Coordinates { @@ -43,8 +64,9 @@ func (c Coordinates) String() string { if c.FileSystemID != "" { str += fmt.Sprintf(" Layer=%q", c.FileSystemID) } - if c.ArchivePath != "" { - str += fmt.Sprintf(" Archive=%q", c.ArchivePath) + + if c.FullNestedPath() != c.RealPath { + str += fmt.Sprintf(" FullPath=%q", c.FullNestedPath()) } return fmt.Sprintf("Location<%s>", str) } @@ -53,10 +75,10 @@ func (c Coordinates) GetCoordinates() Coordinates { return c } -// HashInclude keeps an empty ArchivePath out of the artifact ID hash (see artifact.IDByHash), so -// coordinates outside archives keep the IDs they had before the field existed. +// HashInclude keeps a FullPath that adds nothing to RealPath out of the artifact ID hash (see +// artifact.IDByHash), so coordinates outside archives keep the IDs they had before the field existed. func (c Coordinates) HashInclude(field string, _ any) (bool, error) { - if field == "ArchivePath" && c.ArchivePath == "" { + if field == "FullPath" && c.FullNestedPath() == c.RealPath { return false, nil } return true, nil diff --git a/syft/file/location.go b/syft/file/location.go index 0cd3c6998..18651ee71 100644 --- a/syft/file/location.go +++ b/syft/file/location.go @@ -33,7 +33,7 @@ type LocationData struct { // note: it is IMPORTANT to ignore anything but the coordinates for a Location when considering the ID (hash value) // since the coordinates are the minimally correct ID for a location (symlinks should not come into play) - // AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path) + // AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path). For a file within an archive, it is a full path (see Coordinates.FullPath). AccessPath string `hash:"ignore" json:"accessPath"` // ref is the stereoscope file reference relative to the stereoscope.FileCatalog that has more information about this location. @@ -110,12 +110,12 @@ func NewVirtualLocation(realPath, accessPath string) Location { }} } -// NewLocationFromCoordinates creates a new location for the given Coordinates. +// NewLocationFromCoordinates creates a new location for the given Coordinates, accessed by their full path. func NewLocationFromCoordinates(coordinates Coordinates) Location { return Location{ LocationData: LocationData{ Coordinates: coordinates, - AccessPath: coordinates.RealPath, + AccessPath: coordinates.FullNestedPath(), }, LocationMetadata: LocationMetadata{ Annotations: map[string]string{}, @@ -205,11 +205,12 @@ func NewVirtualLocationFromDirectory(responsePath, responseAccessPath string, re } } +// Path is the path this location was accessed by, or its local path (see Coordinates.LocalPath). func (l Location) Path() string { if l.AccessPath != "" { return l.AccessPath } - return l.RealPath + return l.LocalPath() } func (l Location) String() string { @@ -220,12 +221,12 @@ func (l Location) String() string { str += fmt.Sprintf("RealPath=%q", l.RealPath) - if l.AccessPath != "" && l.AccessPath != l.RealPath { + if l.AccessPath != "" { str += fmt.Sprintf(" AccessPath=%q", l.AccessPath) } - if l.ArchivePath != "" { - str += fmt.Sprintf(" ArchivePath=%q", l.ArchivePath) + if l.FullNestedPath() != l.RealPath { + str += fmt.Sprintf(" FullPath=%q", l.FullNestedPath()) } if l.FileSystemID != "" { @@ -235,8 +236,7 @@ func (l Location) String() string { } func (l Location) Equals(other Location) bool { - return l.RealPath == other.RealPath && + return l.FullNestedPath() == other.FullNestedPath() && l.AccessPath == other.AccessPath && - l.ArchivePath == other.ArchivePath && l.FileSystemID == other.FileSystemID } diff --git a/syft/file/location_set.go b/syft/file/location_set.go index 52beb9e51..fc6603c8a 100644 --- a/syft/file/location_set.go +++ b/syft/file/location_set.go @@ -5,6 +5,7 @@ import ( "sort" "github.com/gohugoio/hashstructure" + "github.com/scylladb/go-set/strset" "github.com/anchore/syft/internal/log" ) @@ -112,8 +113,13 @@ func (s *LocationSet) Empty() bool { } func (s LocationSet) Hash() (uint64, error) { - // access paths and filesystem IDs are not considered when hashing a location set, only the real paths - return hashstructure.Hash(s.CoordinateSet().Paths(), &hashstructure.HashOptions{ + // access paths and filesystem IDs are not considered when hashing a location set, only the full paths + // (unlike CoordinateSet.Paths, which ownership-by-overlap needs as real paths only) + paths := strset.New() + for _, c := range s.CoordinateSet().ToUnorderedSlice() { + paths.Add(c.FullNestedPath()) + } + return hashstructure.Hash(paths.List(), &hashstructure.HashOptions{ ZeroNil: true, SlicesAsSets: true, }) diff --git a/syft/file/location_set_test.go b/syft/file/location_set_test.go index b80028325..37f62a13f 100644 --- a/syft/file/location_set_test.go +++ b/syft/file/location_set_test.go @@ -4,6 +4,7 @@ import ( "sort" "testing" + "github.com/gohugoio/hashstructure" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -255,6 +256,10 @@ func TestLocationSet_Hash(t *testing.T) { }, } + inA := NewLocationFromCoordinates(Coordinates{RealPath: "/app.war", FullPath: "/app.war!/a.jar!/package.json"}) + inB := NewLocationFromCoordinates(Coordinates{RealPath: "/app.war", FullPath: "/app.war!/b.jar!/package.json"}) + etcFull := Location{LocationData: LocationData{Coordinates: Coordinates{RealPath: "/etc/hosts", FileSystemID: "a", FullPath: "/etc/hosts"}}} + tests := []struct { name string setA LocationSet @@ -297,6 +302,18 @@ func TestLocationSet_Hash(t *testing.T) { setB: NewLocationSet(binB, etcB), want: assert.Equal, }, + { + name: "same outer archive but different nested entries have different hashes", + setA: NewLocationSet(inA), + setB: NewLocationSet(inB), + want: assert.NotEqual, + }, + { + name: "a full path equal to the real path has the same hash as a blank one", + setA: NewLocationSet(etcA), + setB: NewLocationSet(etcFull), + want: assert.Equal, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -308,3 +325,13 @@ func TestLocationSet_Hash(t *testing.T) { }) } } + +// IDs outside archives must not change from when only real paths were hashed +func TestLocationSet_Hash_unchangedOutsideArchives(t *testing.T) { + want, err := hashstructure.Hash([]string{"/etc/hosts", "/bin"}, &hashstructure.HashOptions{ZeroNil: true, SlicesAsSets: true}) + require.NoError(t, err) + + got, err := NewLocationSet(NewLocation("/etc/hosts"), NewVirtualLocation("/bin", "/usr/bin")).Hash() + require.NoError(t, err) + assert.Equal(t, want, got) +} diff --git a/syft/file/location_test.go b/syft/file/location_test.go index 7bf999637..8ad3ebe20 100644 --- a/syft/file/location_test.go +++ b/syft/file/location_test.go @@ -1,9 +1,11 @@ package file import ( + "encoding/json" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/anchore/stereoscope/pkg/file" ) @@ -49,11 +51,59 @@ func TestLocation_ID(t *testing.T) { } -func TestLocation_archivePathIsIdentity(t *testing.T) { - a := NewLocationFromCoordinates(Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "/a.jar"}) - b := NewLocationFromCoordinates(Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "/b.jar"}) +func TestLocation_fullPathIsIdentity(t *testing.T) { + a := NewLocationFromCoordinates(Coordinates{RealPath: "/a.jar", FullPath: "/a.jar!/META-INF/MANIFEST.MF"}) + b := NewLocationFromCoordinates(Coordinates{RealPath: "/b.jar", FullPath: "/b.jar!/META-INF/MANIFEST.MF"}) assert.False(t, a.Equals(b)) assert.NotEqual(t, a.String(), b.String()) - assert.Contains(t, a.String(), `ArchivePath="/a.jar"`) + assert.Contains(t, a.String(), `RealPath="/a.jar"`) +} + +func TestLocation_JSON(t *testing.T) { + tests := []struct { + name string + location Location + expected string + }{ + { + name: "a file outside an archive writes no fullPath", + location: NewVirtualLocationFromDirectory("/etc/os-release", "/etc/link", file.Reference{}).WithAnnotation("k", "v"), + expected: `{"path":"/etc/os-release","accessPath":"/etc/link","annotations":{"k":"v"}}`, + }, + { + name: "a file within an archive writes its full path", + location: NewLocationFromCoordinates(Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: "/app.war!/META-INF/MANIFEST.MF"}), + expected: `{"path":"/app.war","layerID":"layer","fullPath":"/app.war!/META-INF/MANIFEST.MF","accessPath":"/app.war!/META-INF/MANIFEST.MF"}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := json.Marshal(tt.location) + require.NoError(t, err) + assert.JSONEq(t, tt.expected, string(data)) + + var decoded Location + require.NoError(t, json.Unmarshal(data, &decoded)) + assert.Equal(t, tt.location.Coordinates, decoded.Coordinates, "decoding fills FullPath back in") + assert.Equal(t, tt.location.AccessPath, decoded.AccessPath) + + data, err = json.Marshal(tt.location.Coordinates) + require.NoError(t, err) + var coordinates Coordinates + require.NoError(t, json.Unmarshal(data, &coordinates)) + assert.Equal(t, tt.location.Coordinates, coordinates) + }) + } +} + +func TestCoordinates_ID_unchangedByTopLevelFullRealPath(t *testing.T) { + assert.Equal(t, Coordinates{RealPath: "/a"}.ID(), NewCoordinates("/a", "").ID()) + assert.NotEqual(t, NewCoordinates("/a", "").ID(), Coordinates{RealPath: "/a", FullPath: "/a!/b"}.ID()) +} + +func TestCoordinates_LocalPath(t *testing.T) { + assert.Equal(t, "/a.jar", NewCoordinates("/a.jar", "").LocalPath()) + assert.Equal(t, "/a.jar", Coordinates{RealPath: "/a.jar"}.LocalPath()) + assert.Equal(t, "META-INF/MANIFEST.MF", Coordinates{RealPath: "/a.war", FullPath: "/a.war!/lib/b.jar!/META-INF/MANIFEST.MF"}.LocalPath()) } diff --git a/syft/file/locations.go b/syft/file/locations.go index df5a281bb..7cc8c46c9 100644 --- a/syft/file/locations.go +++ b/syft/file/locations.go @@ -88,7 +88,7 @@ func LocationSorter(layers []string) func(a, b Location) int { //nolint:gocognit } // the same path in different archives is a different file - return strings.Compare(a.ArchivePath, b.ArchivePath) + return strings.Compare(a.FullNestedPath(), b.FullNestedPath()) } } @@ -128,6 +128,6 @@ func CoordinatesSorter(layers []string) func(a, b Coordinates) int { } // the same path in different archives is a different file - return strings.Compare(a.ArchivePath, b.ArchivePath) + return strings.Compare(a.FullNestedPath(), b.FullNestedPath()) } } diff --git a/syft/file/locations_test.go b/syft/file/locations_test.go index 6c57753c7..64b917dcb 100644 --- a/syft/file/locations_test.go +++ b/syft/file/locations_test.go @@ -557,23 +557,23 @@ func TestLocationAndCoordinatesSorters(t *testing.T) { } } -func TestSorters_archivePathBreaksTies(t *testing.T) { +func TestSorters_fullPathBreaksTies(t *testing.T) { // the same path in two archives must sort the same way whatever order the inputs arrive in, or // output built from map iteration is not reproducible - a := Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "/app.war:WEB-INF/lib/a.jar"} - b := Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "/app.war:WEB-INF/lib/b.jar"} - want := []string{a.ArchivePath, b.ArchivePath} + a := Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"} + b := Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/META-INF/MANIFEST.MF"} + want := []string{a.FullNestedPath(), b.FullNestedPath()} for _, in := range [][]Coordinates{{a, b}, {b, a}} { coords := slices.Clone(in) slices.SortFunc(coords, CoordinatesSorter(nil)) - assert.Equal(t, want, []string{coords[0].ArchivePath, coords[1].ArchivePath}, "CoordinatesSorter") + assert.Equal(t, want, []string{coords[0].FullNestedPath(), coords[1].FullNestedPath()}, "CoordinatesSorter") locs := []Location{NewLocationFromCoordinates(in[0]), NewLocationFromCoordinates(in[1])} slices.SortFunc(locs, LocationSorter(nil)) - assert.Equal(t, want, []string{locs[0].ArchivePath, locs[1].ArchivePath}, "LocationSorter") + assert.Equal(t, want, []string{locs[0].FullNestedPath(), locs[1].FullNestedPath()}, "LocationSorter") got := NewCoordinateSet(in...).ToSlice() - assert.Equal(t, want, []string{got[0].ArchivePath, got[1].ArchivePath}, "CoordinateSet.ToSlice") + assert.Equal(t, want, []string{got[0].FullNestedPath(), got[1].FullNestedPath()}, "CoordinateSet.ToSlice") } } diff --git a/syft/file/mock_resolver.go b/syft/file/mock_resolver.go index 67388796e..5341bd7e6 100644 --- a/syft/file/mock_resolver.go +++ b/syft/file/mock_resolver.go @@ -60,9 +60,9 @@ func NewMockResolverForPathsWithMetadata(metadata map[Coordinates]Metadata) *Moc l := NewLocationFromCoordinates(c) locations = append(locations, l) mimeTypeIndex[m.MIMEType] = append(mimeTypeIndex[m.MIMEType], l) - ext := path.Ext(l.RealPath) + ext := path.Ext(l.FullNestedPath()) extension[ext] = append(extension[ext], l) - bn := path.Base(l.RealPath) + bn := path.Base(l.FullNestedPath()) basename[bn] = append(basename[bn], l) } @@ -97,9 +97,9 @@ func NewMockResolverForMediaTypes(mediaTypes map[string][]Location) *MockResolve mediaTypeIndex[mediaType] = append(mediaTypeIndex[mediaType], locs...) for _, l := range locs { locations = append(locations, l) - ext := path.Ext(l.RealPath) + ext := path.Ext(l.FullNestedPath()) extension[ext] = append(extension[ext], l) - bn := path.Base(l.RealPath) + bn := path.Base(l.FullNestedPath()) basename[bn] = append(basename[bn], l) } } @@ -121,7 +121,7 @@ func (r MockResolver) ImageReference() string { // HasPath indicates if the given path exists in the underlying source. func (r MockResolver) HasPath(path string) bool { for _, l := range r.locations { - if l.RealPath == path { + if l.FullNestedPath() == path { return true } } @@ -150,7 +150,7 @@ func (r MockResolver) FilesByPath(paths ...string) ([]Location, error) { var results []Location for _, p := range paths { for _, location := range r.locations { - if p == location.RealPath { + if p == location.LocalPath() { results = append(results, NewLocation(p)) } } @@ -164,7 +164,7 @@ func (r MockResolver) FilesByGlob(patterns ...string) ([]Location, error) { var results []Location for _, pattern := range patterns { for _, location := range r.locations { - matches, err := doublestar.Match(pattern, location.RealPath) + matches, err := doublestar.Match(pattern, location.LocalPath()) if err != nil { return nil, err } diff --git a/syft/format/common/spdxhelpers/to_format_model.go b/syft/format/common/spdxhelpers/to_format_model.go index a86193aef..991e28025 100644 --- a/syft/format/common/spdxhelpers/to_format_model.go +++ b/syft/format/common/spdxhelpers/to_format_model.go @@ -15,7 +15,6 @@ import ( "github.com/spdx/tools-golang/spdx" "github.com/anchore/packageurl-go" - "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/mimetype" "github.com/anchore/syft/internal/relationship" @@ -356,7 +355,7 @@ func toSPDXID(identifiable artifact.Identifiable) spdx.ElementID { return spdx.ElementID(helpers.SanitizeElementID(id)) } p := "" - parts := strings.Split(it.RealPath, "/") + parts := strings.Split(it.FullNestedPath(), "/") for i := len(parts); i > 0; i-- { part := parts[i-1] if len(part) == 0 { @@ -748,13 +747,13 @@ func toFiles(s sbom.SBOM) (results []*spdx.File) { comments = append(comments, layerIDCommentPrefix+c.FileSystemID) } - // a file inside an archive is named by its full chain ("app.war:WEB-INF/lib/a.jar:META-INF/MANIFEST.MF"), + // a file inside an archive is named by its full path ("/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"), // so a consumer reading only the name does not mistake it for a file at that path in the scan - // root. The archive path is also kept in the comment so decoding need not guess where it ends. - name := c.RealPath - if c.ArchivePath != "" { - name = archive.VirtualPath(file.NewLocationFromCoordinates(c)) - comments = append(comments, archivePathCommentPrefix+c.ArchivePath) + // root. The entry chain after the real path is also kept in the comment so decoding need not guess + // where it starts. + name := c.FullNestedPath() + if entries, ok := strings.CutPrefix(c.FullNestedPath(), c.RealPath+file.ArchivePathSeparator); ok { + comments = append(comments, archivePathCommentPrefix+entries) } relativePath, err := convertAbsoluteToRelative(name) diff --git a/syft/format/common/spdxhelpers/to_format_model_test.go b/syft/format/common/spdxhelpers/to_format_model_test.go index c756995a0..0692cfe51 100644 --- a/syft/format/common/spdxhelpers/to_format_model_test.go +++ b/syft/format/common/spdxhelpers/to_format_model_test.go @@ -762,10 +762,7 @@ func Test_fileIDsForPackage(t *testing.T) { Name: "bogus", } - c := file.Coordinates{ - RealPath: "/path", - FileSystemID: "nowhere", - } + c := file.NewCoordinates("/path", "nowhere") docElementId := func(identifiable artifact.Identifiable) spdx.DocElementID { return spdx.DocElementID{ @@ -1027,17 +1024,13 @@ func Test_toSPDXID(t *testing.T) { expected string }{ { - name: "short filename", - it: file.Coordinates{ - RealPath: "/short/path/file.txt", - }, + name: "short filename", + it: file.NewCoordinates("/short/path/file.txt", ""), expected: "File-short-path-file.txt", }, { - name: "long filename", - it: file.Coordinates{ - RealPath: "/some/long/path/with/a/lot/of-text/that-contains-a/file.txt", - }, + name: "long filename", + it: file.NewCoordinates("/some/long/path/with/a/lot/of-text/that-contains-a/file.txt", ""), expected: "File-...a-lot-of-text-that-contains-a-file.txt", }, { @@ -1166,9 +1159,9 @@ func Test_otherLicenses(t *testing.T) { func Test_toFiles_nestedFilesAreNamedByTheirChainAndRoundTrip(t *testing.T) { // two archives' META-INF/MANIFEST.MF must not come out as two files with the same name and nothing // saying which archive each is in - root := file.Coordinates{RealPath: "/etc/os-release", FileSystemID: "sha256:layer"} - inA := file.Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "/app.war:WEB-INF/lib/a.jar", FileSystemID: "sha256:layer"} - inB := file.Coordinates{RealPath: "odd:na%me.txt", ArchivePath: "/app.war:WEB-INF/lib/b.jar"} + root := file.NewCoordinates("/etc/os-release", "sha256:layer") + inA := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", FileSystemID: "sha256:layer"} + inB := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt"} s := sbom.SBOM{Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{root: {}, inA: {}, inB: {}}}} byName := map[string]*spdx.File{} @@ -1178,15 +1171,23 @@ func Test_toFiles_nestedFilesAreNamedByTheirChainAndRoundTrip(t *testing.T) { require.Len(t, byName, 3) assert.Equal(t, "layerID: sha256:layer", byName["etc/os-release"].FileComment, "files outside archives are unchanged") + assert.Equal(t, file.NewCoordinates("etc/os-release", "sha256:layer"), toSyftCoordinates(byName["etc/os-release"])) - a := byName["app.war:WEB-INF/lib/a.jar:META-INF/MANIFEST.MF"] - require.NotNil(t, a) - assert.Equal(t, "layerID: sha256:layer\narchivePath: /app.war:WEB-INF/lib/a.jar", a.FileComment) - assert.Equal(t, inA, toSyftCoordinates(a)) + // the name is made relative like any other file's, so the decoded top path is too + a := byName["app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"] + require.NotNil(t, a, "names: %v", byName) + assert.Equal(t, "layerID: sha256:layer\narchivePath: WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", a.FileComment) + wantA := inA + wantA.RealPath = "app.war" + wantA.FullPath = "app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF" + assert.Equal(t, wantA, toSyftCoordinates(a)) - b := byName["app.war:WEB-INF/lib/b.jar:odd%3Ana%25me.txt"] + b := byName["app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt"] require.NotNil(t, b, "names: %v", byName) - assert.Equal(t, inB, toSyftCoordinates(b), "the entry path's escaping is undone on decode") + wantB := inB + wantB.RealPath = "app.war" + wantB.FullPath = "app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt" + assert.Equal(t, wantB, toSyftCoordinates(b)) } func Test_toRootPackage_name(t *testing.T) { diff --git a/syft/format/common/spdxhelpers/to_syft_model.go b/syft/format/common/spdxhelpers/to_syft_model.go index 325aacd67..db7905dbb 100644 --- a/syft/format/common/spdxhelpers/to_syft_model.go +++ b/syft/format/common/spdxhelpers/to_syft_model.go @@ -15,7 +15,6 @@ import ( "github.com/spdx/tools-golang/spdx/v2/common" "github.com/anchore/packageurl-go" - "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/spdxlicense" "github.com/anchore/syft/syft/artifact" @@ -531,28 +530,25 @@ func collectPackageFileRelationships(spdxIDMap map[string]any, doc *spdx.Documen } func toSyftCoordinates(f *spdx.File) file.Coordinates { - c := file.Coordinates{RealPath: f.FileName} + c := file.NewCoordinates(f.FileName, "") for _, line := range strings.Split(f.FileComment, "\n") { if v, ok := strings.CutPrefix(line, layerIDCommentPrefix); ok { c.FileSystemID = v } if v, ok := strings.CutPrefix(line, archivePathCommentPrefix); ok { - c.ArchivePath = v + // the name is the full path; the real path is what precedes the entry chain + c.RealPath = strings.TrimSuffix(f.FileName, file.ArchivePathSeparator+v) + c.FullPath = f.FileName } } if v, ok := strings.CutPrefix(string(f.FileSPDXIdentifier), layerIDCommentPrefix); ok { c.FileSystemID = v } - if c.ArchivePath != "" { - // the name is the chain; the file's own path is its last, escaped, segment - c.RealPath = archive.UnescapeEntryPath(f.FileName[strings.LastIndexByte(f.FileName, ':')+1:]) - } return c } func toSyftLocation(f *spdx.File) file.Location { - c := toSyftCoordinates(f) - return file.NewVirtualLocationFromCoordinates(c, c.RealPath) + return file.NewLocationFromCoordinates(toSyftCoordinates(f)) } func requireAndTrimPrefix(val any, prefix string) string { diff --git a/syft/format/common/spdxhelpers/to_syft_model_test.go b/syft/format/common/spdxhelpers/to_syft_model_test.go index 23a8177ca..ca94f9cc8 100644 --- a/syft/format/common/spdxhelpers/to_syft_model_test.go +++ b/syft/format/common/spdxhelpers/to_syft_model_test.go @@ -344,10 +344,7 @@ func Test_toSyftRelationships(t *testing.T) { } pkg3.SetID() - loc1 := file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/somewhere/real", - FileSystemID: "abc", - }) + loc1 := file.NewLocationFromCoordinates(file.NewCoordinates("/somewhere/real", "abc")) tests := []struct { name string @@ -694,11 +691,8 @@ func Test_directPackageFiles(t *testing.T) { p.OverrideID("1") // the same as the spdxID on the package element f := file.Location{ LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "some-file", - FileSystemID: "", - }, - AccessPath: "some-file", + Coordinates: file.NewCoordinates("some-file", ""), + AccessPath: "some-file", }, LocationMetadata: file.LocationMetadata{ Annotations: map[string]string{}, diff --git a/syft/format/common/spdxhelpers/to_syft_model_v3.go b/syft/format/common/spdxhelpers/to_syft_model_v3.go index 336c2d30b..cd86bf4ca 100644 --- a/syft/format/common/spdxhelpers/to_syft_model_v3.go +++ b/syft/format/common/spdxhelpers/to_syft_model_v3.go @@ -401,10 +401,7 @@ func v3toSyftCoordinates(f spdx.AnyFile) file.Coordinates { if strings.HasPrefix(f.GetID(), layerIDPrefix) { fileSystemID = strings.TrimPrefix(f.GetID(), layerIDPrefix) } - return file.Coordinates{ - RealPath: f.GetName(), - FileSystemID: fileSystemID, - } + return file.NewCoordinates(f.GetName(), fileSystemID) } func v3toSyftLocation(f spdx.AnyFile) file.Location { diff --git a/syft/format/common/spdxhelpers/to_syft_model_v3_test.go b/syft/format/common/spdxhelpers/to_syft_model_v3_test.go index f00399b21..9c9dec7cd 100644 --- a/syft/format/common/spdxhelpers/to_syft_model_v3_test.go +++ b/syft/format/common/spdxhelpers/to_syft_model_v3_test.go @@ -193,14 +193,14 @@ func TestToSyftModelV3_documentConversion(t *testing.T) { assert.Equal(t, "BSD-3-Clause AND GPL-2.0+", pkg2Concluded[0].Value) // Verify files - coords1 := file.Coordinates{RealPath: "/src/main.go"} + coords1 := file.NewCoordinates("/src/main.go", "") digests, ok := result.Artifacts.FileDigests[coords1] require.True(t, ok, "file1 digests not found") require.Len(t, digests, 1) assert.Equal(t, "sha256", digests[0].Algorithm) assert.Equal(t, "abc123", digests[0].Value) - coords2 := file.Coordinates{RealPath: "/src/util.go"} + coords2 := file.NewCoordinates("/src/util.go", "") _, ok = result.Artifacts.FileMetadata[coords2] assert.True(t, ok, "file2 metadata not found") diff --git a/syft/format/github/internal/model/model.go b/syft/format/github/internal/model/model.go index 874bb88a7..5096f77a0 100644 --- a/syft/format/github/internal/model/model.go +++ b/syft/format/github/internal/model/model.go @@ -113,9 +113,9 @@ func toPath(s source.Description, p pkg.Package) string { locations := p.Locations.ToSlice() if len(locations) > 0 { location := locations[0] - packagePath := location.RealPath - if location.AccessPath != "" { - packagePath = location.AccessPath + packagePath := location.AccessPath + if packagePath == "" { + packagePath = location.FullNestedPath() } packagePath = strings.TrimPrefix(packagePath, "/") switch metadata := s.Metadata.(type) { diff --git a/syft/format/github/internal/model/model_test.go b/syft/format/github/internal/model/model_test.go index 0ea36695c..a2a14df55 100644 --- a/syft/format/github/internal/model/model_test.go +++ b/syft/format/github/internal/model/model_test.go @@ -41,30 +41,21 @@ func sbomFixture() sbom.SBOM { Name: "pkg-1", Version: "1.0.1", Locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/usr/lib", - FileSystemID: "fsid-1", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/usr/lib", "fsid-1")), ), }, { Name: "pkg-2", Version: "2.0.2", Locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/usr/lib", - FileSystemID: "fsid-1", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/usr/lib", "fsid-1")), ), }, { Name: "pkg-3", Version: "3.0.3", Locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/etc", - FileSystemID: "fsid-1", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/etc", "fsid-1")), ), }, } { diff --git a/syft/format/internal/cyclonedxutil/helpers/component_test.go b/syft/format/internal/cyclonedxutil/helpers/component_test.go index 2b1f44421..12ec1485d 100644 --- a/syft/format/internal/cyclonedxutil/helpers/component_test.go +++ b/syft/format/internal/cyclonedxutil/helpers/component_test.go @@ -29,7 +29,7 @@ func Test_encodeComponentProperties(t *testing.T) { input: pkg.Package{ FoundBy: "cataloger", Locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{RealPath: "test"}), + file.NewLocationFromCoordinates(file.NewCoordinates("test", "")), ), Metadata: pkg.ApkDBEntry{ Package: "libc-utils", diff --git a/syft/format/internal/spdxutil/helpers/source_info.go b/syft/format/internal/spdxutil/helpers/source_info.go index 52bc6479a..7bab77699 100644 --- a/syft/format/internal/spdxutil/helpers/source_info.go +++ b/syft/format/internal/spdxutil/helpers/source_info.go @@ -98,7 +98,7 @@ func SourceInfo(p pkg.Package) string { } var paths []string for _, l := range p.Locations.ToSlice() { - paths = append(paths, l.RealPath) + paths = append(paths, l.FullNestedPath()) } return answer + ": " + strings.Join(paths, ", ") diff --git a/syft/format/internal/testutil/file_relationships.go b/syft/format/internal/testutil/file_relationships.go index 5a522a17a..5c9923329 100644 --- a/syft/format/internal/testutil/file_relationships.go +++ b/syft/format/internal/testutil/file_relationships.go @@ -20,7 +20,7 @@ func AddSampleFileRelationships(s *sbom.SBOM) { for _, f := range files { meta := file.Metadata{} - coords := file.Coordinates{RealPath: f} + coords := file.NewCoordinates(f, "") s.Artifacts.FileMetadata[coords] = meta s.Relationships = append(s.Relationships, artifact.Relationship{ diff --git a/syft/format/spdxjson/encoder_test.go b/syft/format/spdxjson/encoder_test.go index 42912d5d5..bb85b008b 100644 --- a/syft/format/spdxjson/encoder_test.go +++ b/syft/format/spdxjson/encoder_test.go @@ -123,10 +123,7 @@ func TestSPDX22JSONRequredProperties(t *testing.T) { enc, err := NewFormatEncoderWithConfig(cfg) require.NoError(t, err) - coords := file.Coordinates{ - RealPath: "/some/file", - FileSystemID: "ac897d978b6c38749a1", - } + coords := file.NewCoordinates("/some/file", "ac897d978b6c38749a1") p1 := pkg.Package{ Name: "files-analyzed-true", diff --git a/syft/format/syftjson/decoder_test.go b/syft/format/syftjson/decoder_test.go index 9436da0c0..23a6822d2 100644 --- a/syft/format/syftjson/decoder_test.go +++ b/syft/format/syftjson/decoder_test.go @@ -137,10 +137,7 @@ func Test_encodeDecodeFileMetadata(t *testing.T) { FoundBy: "something", Locations: file.NewLocationSet(file.Location{ LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/somewhere", - FileSystemID: "id", - }, + Coordinates: file.NewCoordinates("/somewhere", "id"), }, LocationMetadata: file.LocationMetadata{ Annotations: map[string]string{ @@ -174,10 +171,7 @@ func Test_encodeDecodeFileMetadata(t *testing.T) { } p.SetID() - c := file.Coordinates{ - RealPath: "some-file", - FileSystemID: "some-fs-id", - } + c := file.NewCoordinates("some-file", "some-fs-id") s := sbom.SBOM{ Artifacts: sbom.Artifacts{ diff --git a/syft/format/syftjson/encoder_test.go b/syft/format/syftjson/encoder_test.go index b8b4ad71b..3b0161b2d 100644 --- a/syft/format/syftjson/encoder_test.go +++ b/syft/format/syftjson/encoder_test.go @@ -98,7 +98,7 @@ func TestEncode_sameFileInManyArchivesIsReproducible(t *testing.T) { s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{} p := pkg.Package{Name: "shared", Version: "1.0"} for i := range 20 { - c := file.Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: fmt.Sprintf("/app.war:WEB-INF/lib/%02d.jar", i)} + c := file.Coordinates{RealPath: "/app.war", FullPath: fmt.Sprintf("/app.war!/WEB-INF/lib/%02d.jar!/META-INF/MANIFEST.MF", i)} s.Artifacts.FileMetadata[c] = file.Metadata{} p.Locations.Add(file.NewLocationFromCoordinates(c)) } @@ -168,9 +168,7 @@ func TestEncodeFullJSONDocument(t *testing.T) { Name: "package-1", Version: "1.0.1", Locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/a/place/a", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/a/place/a", "")), ), Type: pkg.PythonPkg, FoundBy: "the-cataloger-1", @@ -191,9 +189,7 @@ func TestEncodeFullJSONDocument(t *testing.T) { Name: "package-2", Version: "2.0.1", Locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/b/place/b", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/b/place/b", "")), ), Type: pkg.DebPkg, FoundBy: "the-cataloger-2", diff --git a/syft/format/syftjson/model/package.go b/syft/format/syftjson/model/package.go index e2fb53722..33f2b7099 100644 --- a/syft/format/syftjson/model/package.go +++ b/syft/format/syftjson/model/package.go @@ -172,17 +172,17 @@ func unpackPkgMetadata(p *Package, unpacker packageMetadataUnpacker) error { switch unpacker.MetadataType { case "HackageMetadataType": for _, l := range p.Locations { - if strings.HasSuffix(l.RealPath, ".yaml.lock") { + if strings.HasSuffix(l.FullNestedPath(), ".yaml.lock") { ty = "haskell-hackage-stack-lock-entry" break - } else if strings.HasSuffix(l.RealPath, ".yaml") { + } else if strings.HasSuffix(l.FullNestedPath(), ".yaml") { ty = "haskell-hackage-stack-entry" break } } case "RpmMetadata": for _, l := range p.Locations { - if strings.HasSuffix(l.RealPath, ".rpm") { + if strings.HasSuffix(l.FullNestedPath(), ".rpm") { ty = "rpm-archive" break } @@ -190,7 +190,7 @@ func unpackPkgMetadata(p *Package, unpacker packageMetadataUnpacker) error { case "RustCargoPackageMetadata": var found bool for _, l := range p.Locations { - if strings.HasSuffix(strings.ToLower(l.RealPath), "cargo.lock") { + if strings.HasSuffix(strings.ToLower(l.FullNestedPath()), "cargo.lock") { ty = "rust-cargo-lock-entry" found = true break diff --git a/syft/format/syftjson/to_format_model_test.go b/syft/format/syftjson/to_format_model_test.go index 8ff3fdcc5..dc29c49e1 100644 --- a/syft/format/syftjson/to_format_model_test.go +++ b/syft/format/syftjson/to_format_model_test.go @@ -352,10 +352,7 @@ func Test_toFileType(t *testing.T) { } func Test_toFileMetadataEntry(t *testing.T) { - coords := file.Coordinates{ - RealPath: "/path", - FileSystemID: "x", - } + coords := file.NewCoordinates("/path", "x") tests := []struct { name string metadata *file.Metadata @@ -453,41 +450,17 @@ func Test_toPackageModel_layerOrdering(t *testing.T) { Licenses: pkg.NewLicenseSet(pkg.License{ Value: "MIT", Locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/lic-a", - FileSystemID: "fsid-3", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/lic-a", - FileSystemID: "fsid-1", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/lic-b", - FileSystemID: "fsid-0", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/lic-a", - FileSystemID: "fsid-2", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/lic-a", "fsid-3")), + file.NewLocationFromCoordinates(file.NewCoordinates("/lic-a", "fsid-1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/lic-b", "fsid-0")), + file.NewLocationFromCoordinates(file.NewCoordinates("/lic-a", "fsid-2")), ), }), Locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-3", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-0", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-2", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-3")), + file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-0")), + file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-2")), ), }, layerOrder: []string{ @@ -505,38 +478,26 @@ func Test_toPackageModel_layerOrdering(t *testing.T) { Locations: []file.Location{ { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/lic-b", - FileSystemID: "fsid-0", // important! - }, - AccessPath: "/lic-b", + Coordinates: file.NewCoordinates("/lic-b", "fsid-0"), // important! + AccessPath: "/lic-b", }, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/lic-a", - FileSystemID: "fsid-1", // important! - }, - AccessPath: "/lic-a", + Coordinates: file.NewCoordinates("/lic-a", "fsid-1"), // important! + AccessPath: "/lic-a", }, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/lic-a", - FileSystemID: "fsid-2", // important! - }, - AccessPath: "/lic-a", + Coordinates: file.NewCoordinates("/lic-a", "fsid-2"), // important! + AccessPath: "/lic-a", }, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/lic-a", - FileSystemID: "fsid-3", // important! - }, - AccessPath: "/lic-a", + Coordinates: file.NewCoordinates("/lic-a", "fsid-3"), // important! + AccessPath: "/lic-a", }, }, }, @@ -545,38 +506,26 @@ func Test_toPackageModel_layerOrdering(t *testing.T) { Locations: []file.Location{ { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-0", // important! - }, - AccessPath: "/b", + Coordinates: file.NewCoordinates("/b", "fsid-0"), // important! + AccessPath: "/b", }, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", // important! - }, - AccessPath: "/a", + Coordinates: file.NewCoordinates("/a", "fsid-1"), // important! + AccessPath: "/a", }, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-2", // important! - }, - AccessPath: "/a", + Coordinates: file.NewCoordinates("/a", "fsid-2"), // important! + AccessPath: "/a", }, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-3", // important! - }, - AccessPath: "/a", + Coordinates: file.NewCoordinates("/a", "fsid-3"), // important! + AccessPath: "/a", }, }, }, @@ -609,34 +558,22 @@ func Test_toLocationModel(t *testing.T) { { name: "nil layer order map", locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-2", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-2")), ), layers: nil, // please don't panic! want: []file.Location{ { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, - AccessPath: "/a", + Coordinates: file.NewCoordinates("/a", "fsid-1"), + AccessPath: "/a", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-2", - }, - AccessPath: "/b", + Coordinates: file.NewCoordinates("/b", "fsid-2"), + AccessPath: "/b", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, @@ -645,18 +582,9 @@ func Test_toLocationModel(t *testing.T) { { name: "go case", locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-3", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-2", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-3")), + file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/c", "fsid-2")), ), layers: []string{ "fsid-1", @@ -666,31 +594,22 @@ func Test_toLocationModel(t *testing.T) { want: []file.Location{ { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, - AccessPath: "/b", + Coordinates: file.NewCoordinates("/b", "fsid-1"), + AccessPath: "/b", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-2", - }, - AccessPath: "/c", + Coordinates: file.NewCoordinates("/c", "fsid-2"), + AccessPath: "/c", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-3", - }, - AccessPath: "/a", + Coordinates: file.NewCoordinates("/a", "fsid-3"), + AccessPath: "/a", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, @@ -699,18 +618,9 @@ func Test_toLocationModel(t *testing.T) { { name: "same layer different paths", // prove we can sort by path irrespective of layer locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-1", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/c", "fsid-1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-1")), ), layers: []string{ "fsid-1", @@ -718,31 +628,22 @@ func Test_toLocationModel(t *testing.T) { want: []file.Location{ { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, - AccessPath: "/a", + Coordinates: file.NewCoordinates("/a", "fsid-1"), + AccessPath: "/a", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, - AccessPath: "/b", + Coordinates: file.NewCoordinates("/b", "fsid-1"), + AccessPath: "/b", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-1", - }, - AccessPath: "/c", + Coordinates: file.NewCoordinates("/c", "fsid-1"), + AccessPath: "/c", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, @@ -751,22 +652,10 @@ func Test_toLocationModel(t *testing.T) { { name: "mixed layers and paths", locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/z", - FileSystemID: "fsid-3", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-2", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-2", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/z", "fsid-3")), + file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-2")), + file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/c", "fsid-2")), ), layers: []string{ "fsid-1", @@ -776,41 +665,29 @@ func Test_toLocationModel(t *testing.T) { want: []file.Location{ { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, - AccessPath: "/b", + Coordinates: file.NewCoordinates("/b", "fsid-1"), + AccessPath: "/b", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-2", - }, - AccessPath: "/a", + Coordinates: file.NewCoordinates("/a", "fsid-2"), + AccessPath: "/a", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-2", - }, - AccessPath: "/c", + Coordinates: file.NewCoordinates("/c", "fsid-2"), + AccessPath: "/c", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/z", - FileSystemID: "fsid-3", - }, - AccessPath: "/z", + Coordinates: file.NewCoordinates("/z", "fsid-3"), + AccessPath: "/z", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, @@ -845,35 +722,23 @@ func Test_sortFiles(t *testing.T) { name: "nil layer order map", files: []model.File{ { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-1", + Location: file.NewCoordinates("/a", "fsid-1"), }, { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-2", - }, + ID: "file-2", + Location: file.NewCoordinates("/b", "fsid-2"), }, }, layers: nil, want: []model.File{ { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-1", + Location: file.NewCoordinates("/a", "fsid-1"), }, { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-2", - }, + ID: "file-2", + Location: file.NewCoordinates("/b", "fsid-2"), }, }, }, @@ -881,25 +746,16 @@ func Test_sortFiles(t *testing.T) { name: "layer ordering", files: []model.File{ { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-3", - }, + ID: "file-1", + Location: file.NewCoordinates("/a", "fsid-3"), }, { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, + ID: "file-2", + Location: file.NewCoordinates("/b", "fsid-1"), }, { - ID: "file-3", - Location: file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-2", - }, + ID: "file-3", + Location: file.NewCoordinates("/c", "fsid-2"), }, }, layers: []string{ @@ -909,25 +765,16 @@ func Test_sortFiles(t *testing.T) { }, want: []model.File{ { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, + ID: "file-2", + Location: file.NewCoordinates("/b", "fsid-1"), }, { - ID: "file-3", - Location: file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-2", - }, + ID: "file-3", + Location: file.NewCoordinates("/c", "fsid-2"), }, { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-3", - }, + ID: "file-1", + Location: file.NewCoordinates("/a", "fsid-3"), }, }, }, @@ -935,25 +782,16 @@ func Test_sortFiles(t *testing.T) { name: "same layer different paths", files: []model.File{ { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-1", - }, + ID: "file-1", + Location: file.NewCoordinates("/c", "fsid-1"), }, { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-2", + Location: file.NewCoordinates("/a", "fsid-1"), }, { - ID: "file-3", - Location: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, + ID: "file-3", + Location: file.NewCoordinates("/b", "fsid-1"), }, }, layers: []string{ @@ -961,25 +799,16 @@ func Test_sortFiles(t *testing.T) { }, want: []model.File{ { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-2", + Location: file.NewCoordinates("/a", "fsid-1"), }, { - ID: "file-3", - Location: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, + ID: "file-3", + Location: file.NewCoordinates("/b", "fsid-1"), }, { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-1", - }, + ID: "file-1", + Location: file.NewCoordinates("/c", "fsid-1"), }, }, }, @@ -987,25 +816,16 @@ func Test_sortFiles(t *testing.T) { name: "stability test - preserve original order for equivalent items", files: []model.File{ { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-1", + Location: file.NewCoordinates("/a", "fsid-1"), }, { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-2", + Location: file.NewCoordinates("/a", "fsid-1"), }, { - ID: "file-3", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-3", + Location: file.NewCoordinates("/a", "fsid-1"), }, }, layers: []string{ @@ -1013,25 +833,16 @@ func Test_sortFiles(t *testing.T) { }, want: []model.File{ { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-1", + Location: file.NewCoordinates("/a", "fsid-1"), }, { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-2", + Location: file.NewCoordinates("/a", "fsid-1"), }, { - ID: "file-3", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-3", + Location: file.NewCoordinates("/a", "fsid-1"), }, }, }, @@ -1039,11 +850,8 @@ func Test_sortFiles(t *testing.T) { name: "complex file metadata doesn't affect sorting", files: []model.File{ { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-2", - }, + ID: "file-1", + Location: file.NewCoordinates("/a", "fsid-2"), Metadata: &model.FileMetadataEntry{ Mode: 0644, Type: "file", @@ -1061,11 +869,8 @@ func Test_sortFiles(t *testing.T) { }, }, { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, + ID: "file-2", + Location: file.NewCoordinates("/b", "fsid-1"), Metadata: &model.FileMetadataEntry{ Mode: 0755, Type: "directory", @@ -1082,11 +887,8 @@ func Test_sortFiles(t *testing.T) { }, want: []model.File{ { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, + ID: "file-2", + Location: file.NewCoordinates("/b", "fsid-1"), Metadata: &model.FileMetadataEntry{ Mode: 0755, Type: "directory", @@ -1097,11 +899,8 @@ func Test_sortFiles(t *testing.T) { }, }, { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-2", - }, + ID: "file-1", + Location: file.NewCoordinates("/a", "fsid-2"), Metadata: &model.FileMetadataEntry{ Mode: 0644, Type: "file", diff --git a/syft/format/syftjson/to_syft_model.go b/syft/format/syftjson/to_syft_model.go index 651f8bba9..a66db17d0 100644 --- a/syft/format/syftjson/to_syft_model.go +++ b/syft/format/syftjson/to_syft_model.go @@ -90,11 +90,11 @@ func toSyftFiles(files []model.File) sbom.Artifacts { ret.FileMetadata[coord] = file.Metadata{ FileInfo: stereoscopeFile.ManualInfo{ - NameValue: path.Base(coord.RealPath), + NameValue: path.Base(coord.FullNestedPath()), SizeValue: f.Metadata.Size, ModeValue: fm, }, - Path: coord.RealPath, + Path: coord.FullNestedPath(), LinkDestination: f.Metadata.LinkDestination, UserID: f.Metadata.UserID, GroupID: f.Metadata.GroupID, diff --git a/syft/format/syftjson/to_syft_model_test.go b/syft/format/syftjson/to_syft_model_test.go index ed4b7b59b..6e4b4cc68 100644 --- a/syft/format/syftjson/to_syft_model_test.go +++ b/syft/format/syftjson/to_syft_model_test.go @@ -295,10 +295,7 @@ func Test_idsHaveChanged(t *testing.T) { } func Test_toSyftFiles(t *testing.T) { - coord := file.Coordinates{ - RealPath: "/somerwhere/place", - FileSystemID: "abc", - } + coord := file.NewCoordinates("/somerwhere/place", "abc") tests := []struct { name string diff --git a/syft/internal/fileresolver/container_image_all_layers.go b/syft/internal/fileresolver/container_image_all_layers.go index 1a3151e8c..ca2351785 100644 --- a/syft/internal/fileresolver/container_image_all_layers.go +++ b/syft/internal/fileresolver/container_image_all_layers.go @@ -197,7 +197,7 @@ func (r *ContainerImageAllLayers) RelativeFileByPath(location file.Location, pat func (r *ContainerImageAllLayers) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { entry, err := r.img.FileCatalog.Get(location.Reference()) if err != nil { - return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err) + return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.FullNestedPath(), err) } switch entry.Type { diff --git a/syft/internal/fileresolver/container_image_deep_squash_test.go b/syft/internal/fileresolver/container_image_deep_squash_test.go index f88afdf13..6bd9e4d9b 100644 --- a/syft/internal/fileresolver/container_image_deep_squash_test.go +++ b/syft/internal/fileresolver/container_image_deep_squash_test.go @@ -1104,15 +1104,9 @@ func TestContainerImageDeepSquash_MergeLocationStreams_FunCases(t *testing.T) { } func makeLocation(path string, layer int) file.Location { - return file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: path, - FileSystemID: fmt.Sprintf("layer-%d", layer), - }) + return file.NewLocationFromCoordinates(file.NewCoordinates(path, fmt.Sprintf("layer-%d", layer))) } func makeVirtualLocation(path, access string, layer int) file.Location { - return file.NewVirtualLocationFromCoordinates(file.Coordinates{ - RealPath: path, - FileSystemID: fmt.Sprintf("layer-%d", layer), - }, access) + return file.NewVirtualLocationFromCoordinates(file.NewCoordinates(path, fmt.Sprintf("layer-%d", layer)), access) } diff --git a/syft/internal/fileresolver/container_image_squash.go b/syft/internal/fileresolver/container_image_squash.go index efcd2e8e4..b953a3583 100644 --- a/syft/internal/fileresolver/container_image_squash.go +++ b/syft/internal/fileresolver/container_image_squash.go @@ -148,7 +148,7 @@ func (r *ContainerImageSquash) RelativeFileByPath(_ file.Location, path string) func (r *ContainerImageSquash) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { entry, err := r.img.FileCatalog.Get(location.Reference()) if err != nil { - return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err) + return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.FullNestedPath(), err) } switch entry.Type { diff --git a/syft/internal/fileresolver/excluding_file.go b/syft/internal/fileresolver/excluding_file.go index 535f46671..88c706a78 100644 --- a/syft/internal/fileresolver/excluding_file.go +++ b/syft/internal/fileresolver/excluding_file.go @@ -28,14 +28,14 @@ func NewExcludingDecorator(delegate file.Resolver, excludeFn excludeFn) file.Res func (r *excluding) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { if locationMatches(&location, r.excludeFn) { - return nil, fmt.Errorf("no such location: %+v", location.RealPath) + return nil, fmt.Errorf("no such location: %+v", location.FullNestedPath()) } return r.delegate.FileContentsByLocation(location) } func (r *excluding) FileMetadataByLocation(location file.Location) (file.Metadata, error) { if locationMatches(&location, r.excludeFn) { - return file.Metadata{}, fmt.Errorf("no such location: %+v", location.RealPath) + return file.Metadata{}, fmt.Errorf("no such location: %+v", location.FullNestedPath()) } return r.delegate.FileMetadataByLocation(location) } @@ -89,7 +89,7 @@ func (r *excluding) AllLocations(ctx context.Context) <-chan file.Location { } func locationMatches(location *file.Location, exclusionFn excludeFn) bool { - return exclusionFn(location.RealPath) || exclusionFn(location.AccessPath) + return exclusionFn(location.LocalPath()) || exclusionFn(location.AccessPath) } func filterLocations(locations []file.Location, err error, exclusionFn excludeFn) ([]file.Location, error) { diff --git a/syft/internal/fileresolver/unindexed_directory.go b/syft/internal/fileresolver/unindexed_directory.go index f2435ff0d..fbc1c75c6 100644 --- a/syft/internal/fileresolver/unindexed_directory.go +++ b/syft/internal/fileresolver/unindexed_directory.go @@ -89,7 +89,7 @@ func (u UnindexedDirectory) FileContentsByLocation(location file.Location) (io.R return nil, err } if fi.IsDir() { - return nil, fmt.Errorf("unable to get contents of directory: %s", location.RealPath) + return nil, fmt.Errorf("unable to get contents of directory: %s", location.FullNestedPath()) } return f, nil } diff --git a/syft/linux/identify_release.go b/syft/linux/identify_release.go index de45c2db7..ac082e5fc 100644 --- a/syft/linux/identify_release.go +++ b/syft/linux/identify_release.go @@ -87,20 +87,20 @@ func IdentifyRelease(resolver file.Resolver) *Release { func tryParseReleaseInfo(resolver file.Resolver, location file.Location, logger logger.Logger, entry parseEntry) *Release { contentReader, err := resolver.FileContentsByLocation(location) if err != nil { - logger.WithFields("error", err, "path", location.RealPath).Trace("unable to get contents") + logger.WithFields("error", err, "path", location.FullNestedPath()).Trace("unable to get contents") return nil } defer internal.CloseAndLogError(contentReader, location.AccessPath) content, err := io.ReadAll(io.LimitReader(contentReader, 5*1024*1024)) if err != nil { - logger.WithFields("error", err, "path", location.RealPath).Trace("unable to read contents") + logger.WithFields("error", err, "path", location.FullNestedPath()).Trace("unable to read contents") return nil } release, err := entry.fn(string(content)) if err != nil { - logger.WithFields("error", err, "path", location.RealPath).Trace("unable to parse contents") + logger.WithFields("error", err, "path", location.FullNestedPath()).Trace("unable to parse contents") return nil } diff --git a/syft/linux/supplement_release.go b/syft/linux/supplement_release.go index 33a6815f6..f4366d48a 100644 --- a/syft/linux/supplement_release.go +++ b/syft/linux/supplement_release.go @@ -35,13 +35,13 @@ func supplementDebianVersion(resolver file.Resolver, release *Release) { func readDebianVersionFile(resolver file.Resolver, location file.Location) string { rdr, err := resolver.FileContentsByLocation(location) if err != nil { - log.Debugf("error getting contents for %s: %v", location.RealPath, err) + log.Debugf("error getting contents for %s: %v", location.FullNestedPath(), err) return "" } - defer internal.CloseAndLogError(rdr, location.RealPath) + defer internal.CloseAndLogError(rdr, location.FullNestedPath()) contents, err := io.ReadAll(io.LimitReader(rdr, 5*1024*1024)) if err != nil { - log.Debugf("error reading %s: %v", location.RealPath, err) + log.Debugf("error reading %s: %v", location.FullNestedPath(), err) return "" } version := strings.TrimSpace(string(contents)) diff --git a/syft/pkg/cataloger/ai/identity_oci.go b/syft/pkg/cataloger/ai/identity_oci.go index 97d58aa6b..d26001f39 100644 --- a/syft/pkg/cataloger/ai/identity_oci.go +++ b/syft/pkg/cataloger/ai/identity_oci.go @@ -32,8 +32,8 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver, // seen wins below, so sort for a stable result across runs. For OCI // artifacts every RealPath is "/", so in practice this is digest order. sort.Slice(modelFileLocs, func(i, j int) bool { - if modelFileLocs[i].RealPath != modelFileLocs[j].RealPath { - return modelFileLocs[i].RealPath < modelFileLocs[j].RealPath + if modelFileLocs[i].FullNestedPath() != modelFileLocs[j].FullNestedPath() { + return modelFileLocs[i].FullNestedPath() < modelFileLocs[j].FullNestedPath() } return modelFileLocs[i].FileSystemID < modelFileLocs[j].FileSystemID }) @@ -128,7 +128,7 @@ func classifyOCIModelFileLayer(resolver file.Resolver, loc file.Location) (*hfCo if err != nil { return nil, nil } - defer internal.CloseAndLogError(rc, loc.RealPath) + defer internal.CloseAndLogError(rc, loc.FullNestedPath()) buf, err := readBounded(rc, maxModelFileLayerSize) if err != nil { @@ -174,7 +174,7 @@ func readLicenseSPDXIDFromFrontmatter(resolver file.Resolver, loc file.Location) if err != nil { return "" } - defer internal.CloseAndLogError(rc, loc.RealPath) + defer internal.CloseAndLogError(rc, loc.FullNestedPath()) buf, err := readPrefix(rc, maxLicensePrefixSize) if err != nil { diff --git a/syft/pkg/cataloger/ai/parse_gguf_model.go b/syft/pkg/cataloger/ai/parse_gguf_model.go index 6b9a89259..39d409303 100644 --- a/syft/pkg/cataloger/ai/parse_gguf_model.go +++ b/syft/pkg/cataloger/ai/parse_gguf_model.go @@ -23,7 +23,7 @@ import ( // parseGGUFModel parses a GGUF model file and returns the discovered package. // This implementation only reads the header portion of the file, not the entire model. func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - defer internal.CloseAndLogError(reader, reader.Path()) + defer internal.CloseAndLogError(reader, reader.FullNestedPath()) td := tmpdir.FromContext(ctx) if td == nil { diff --git a/syft/pkg/cataloger/ai/parse_safetensors_model.go b/syft/pkg/cataloger/ai/parse_safetensors_model.go index da3234240..b90a6cae5 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors_model.go +++ b/syft/pkg/cataloger/ai/parse_safetensors_model.go @@ -18,7 +18,7 @@ import ( // resolution, sibling enrichment, and cross-shard rollup are all handled by // safeTensorsMergeProcessor. func parseSafeTensorsFile(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - defer internal.CloseAndLogError(reader, reader.Path()) + defer internal.CloseAndLogError(reader, reader.FullNestedPath()) header, err := readSafeTensorsHeader(&io.LimitedReader{R: reader, N: maxSafeTensorsHeaderSize + 8}) if err != nil { diff --git a/syft/pkg/cataloger/ai/parse_safetensors_oci.go b/syft/pkg/cataloger/ai/parse_safetensors_oci.go index 8aa04e5b0..519532c0e 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors_oci.go +++ b/syft/pkg/cataloger/ai/parse_safetensors_oci.go @@ -47,7 +47,7 @@ type dockerAIModelConfig struct { // parseSafeTensorsOCIConfig decodes the Docker AI model-config blob func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - defer internal.CloseAndLogError(reader, reader.Path()) + defer internal.CloseAndLogError(reader, reader.FullNestedPath()) body, err := readBounded(reader, maxModelConfigBlobSize) if err != nil { @@ -86,7 +86,7 @@ func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.En // parseSafeTensorsOCILayer decodes the JSON header of a SafeTensors weight // layer fetched from an OCI model artifact func parseSafeTensorsOCILayer(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - defer internal.CloseAndLogError(reader, reader.Path()) + defer internal.CloseAndLogError(reader, reader.FullNestedPath()) header, err := readSafeTensorsHeader(&io.LimitedReader{R: reader, N: maxSafeTensorsHeaderSize + 8}) if err != nil { diff --git a/syft/pkg/cataloger/ai/processor.go b/syft/pkg/cataloger/ai/processor.go index 5cb31c3e7..f15f3c778 100644 --- a/syft/pkg/cataloger/ai/processor.go +++ b/syft/pkg/cataloger/ai/processor.go @@ -56,7 +56,7 @@ func partitionSafeTensorsPackages(pkgs []pkg.Package) (safeTensors, other []pkg. // single scan is one source, so the first package is representative of the rest. func fromOCIArtifact(pkgs []pkg.Package) bool { loc := primaryEvidenceLocation(pkgs[0]) - return loc != nil && loc.RealPath == "/" + return loc != nil && loc.LocalPath() == "/" } // mergeOCIModel treats the whole OCI artifact as a single model: every layer @@ -120,7 +120,7 @@ func groupByParentDir(pkgs []pkg.Package) map[string][]pkg.Package { if loc == nil { continue } - dir := path.Dir(loc.RealPath) + dir := path.Dir(loc.LocalPath()) out[dir] = append(out[dir], p) } return out diff --git a/syft/pkg/cataloger/alpine/parse_apk_db.go b/syft/pkg/cataloger/alpine/parse_apk_db.go index bf721c68d..a17ac2ddc 100644 --- a/syft/pkg/cataloger/alpine/parse_apk_db.go +++ b/syft/pkg/cataloger/alpine/parse_apk_db.go @@ -126,7 +126,7 @@ func parseApkDB(ctx context.Context, resolver file.Resolver, env *generic.Enviro // This should get fixed with https://gitlab.alpinelinux.org/alpine/apk-tools/-/issues/10875 if r == nil { // find the repositories file from the relative directory of the DB file - releases := findReleases(resolver, reader.RealPath) + releases := findReleases(resolver, reader.LocalPath()) if len(releases) > 0 { r = &releases[0] @@ -163,7 +163,7 @@ func findReleases(resolver file.Resolver, dbPath string) []linux.Release { log.Tracef("unable to fetch contents for APK repositories file %q: %+v", reposLocation, err) return nil } - defer internal.CloseAndLogError(reposReader, location.RealPath) + defer internal.CloseAndLogError(reposReader, location.FullNestedPath()) return parseReleasesFromAPKRepository(file.LocationReadCloser{ Location: location, @@ -176,7 +176,7 @@ func parseReleasesFromAPKRepository(reader file.LocationReadCloser) []linux.Rele reposB, err := io.ReadAll(reader) //nolint:gocritic // regex matching requires full buffer if err != nil { - log.Tracef("unable to read APK repositories file %q: %+v", reader.RealPath, err) + log.Tracef("unable to read APK repositories file %q: %+v", reader.FullNestedPath(), err) return nil } diff --git a/syft/pkg/cataloger/arch/parse_alpm_db.go b/syft/pkg/cataloger/arch/parse_alpm_db.go index ac63a4141..ef1c34316 100644 --- a/syft/pkg/cataloger/arch/parse_alpm_db.go +++ b/syft/pkg/cataloger/arch/parse_alpm_db.go @@ -55,7 +55,7 @@ func parseAlpmDB(ctx context.Context, resolver file.Resolver, env *generic.Envir return nil, nil, nil } - base := path.Dir(reader.RealPath) + base := path.Dir(reader.LocalPath()) var locs []file.Location @@ -102,7 +102,7 @@ func fetchPkgFiles(base string, resolver file.Resolver) ([]pkg.AlpmFileRecord, f if err != nil { return []pkg.AlpmFileRecord{}, loc, unknown.New(loc, fmt.Errorf("failed to get contents: %w", err)) } - defer internal.CloseAndLogError(reader, loc.RealPath) + defer internal.CloseAndLogError(reader, loc.FullNestedPath()) pkgFiles, err := parseMtree(reader) if err != nil { @@ -126,7 +126,7 @@ func fetchBackupFiles(base string, resolver file.Resolver) ([]pkg.AlpmFileRecord if err != nil { return []pkg.AlpmFileRecord{}, loc, unknown.New(loc, fmt.Errorf("failed to get contents: %w", err)) } - defer internal.CloseAndLogError(reader, loc.RealPath) + defer internal.CloseAndLogError(reader, loc.FullNestedPath()) filesMetadata, err := parseAlpmDBEntry(reader) if err != nil { diff --git a/syft/pkg/cataloger/binary/classifier_cataloger_test.go b/syft/pkg/cataloger/binary/classifier_cataloger_test.go index a9d0a676e..28bb33626 100644 --- a/syft/pkg/cataloger/binary/classifier_cataloger_test.go +++ b/syft/pkg/cataloger/binary/classifier_cataloger_test.go @@ -3410,9 +3410,7 @@ func match(classifier string, paths ...string) pkg.ClassifierMatch { return pkg.ClassifierMatch{ Classifier: classifier, Location: file.NewVirtualLocationFromCoordinates( - file.Coordinates{ - RealPath: realPath, - }, + file.NewCoordinates(realPath, ""), virtualPath, ), } diff --git a/syft/pkg/cataloger/binary/elf_package_cataloger.go b/syft/pkg/cataloger/binary/elf_package_cataloger.go index 943550d2e..32238aeaf 100644 --- a/syft/pkg/cataloger/binary/elf_package_cataloger.go +++ b/syft/pkg/cataloger/binary/elf_package_cataloger.go @@ -99,7 +99,7 @@ func (c *elfPackageCataloger) Catalog(ctx context.Context, resolver file.Resolve func parseElfPackageNotes(resolver file.Resolver, location file.Location, c *elfPackageCataloger) (*elfBinaryPackageNotes, elfPackageKey, error) { reader, err := resolver.FileContentsByLocation(location) if err != nil { - return nil, elfPackageKey{}, fmt.Errorf("unable to get binary contents %q: %w", location.Path(), err) + return nil, elfPackageKey{}, fmt.Errorf("unable to get binary contents %q: %w", location.FullNestedPath(), err) } defer internal.CloseAndLogError(reader, location.AccessPath) @@ -109,7 +109,7 @@ func parseElfPackageNotes(resolver file.Resolver, location file.Location, c *elf }) if err != nil { - log.WithFields("file", location.Path(), "error", err).Trace("unable to parse ELF notes") + log.WithFields("file", location.FullNestedPath(), "error", err).Trace("unable to parse ELF notes") return nil, elfPackageKey{}, err } @@ -148,7 +148,7 @@ func getELFNotes(r file.LocationReadCloser) (*elfBinaryPackageNotes, error) { f, err := elfutil.NewFile(unionReader) if f == nil || err != nil { - log.WithFields("file", r.Location.Path(), "error", err).Trace("unable to parse binary as ELF") + log.WithFields("file", r.Location.FullNestedPath(), "error", err).Trace("unable to parse binary as ELF") return nil, nil } @@ -182,7 +182,7 @@ func getELFNotes(r file.LocationReadCloser) (*elfBinaryPackageNotes, error) { if metadata, err = unmarshalELFPackageNotesPayload(newPayload); err == nil { return metadata, nil } - log.WithFields("file", r.Location.Path(), "error", err).Trace("unable to unmarshal ELF package notes as JSON") + log.WithFields("file", r.Location.FullNestedPath(), "error", err).Trace("unable to unmarshal ELF package notes as JSON") } } diff --git a/syft/pkg/cataloger/binary/pe_package.go b/syft/pkg/cataloger/binary/pe_package.go index a7cab30e2..9d25b8487 100644 --- a/syft/pkg/cataloger/binary/pe_package.go +++ b/syft/pkg/cataloger/binary/pe_package.go @@ -22,7 +22,7 @@ func newPEPackage(versionResources map[string]string, f file.Location) pkg.Packa if name == "" { // it's possible that the version resources are empty, so we fall back to the file name - name = strings.TrimSuffix(strings.TrimSuffix(path.Base(f.RealPath), ".exe"), ".dll") + name = strings.TrimSuffix(strings.TrimSuffix(path.Base(f.LocalPath()), ".exe"), ".dll") } p := pkg.Package{ diff --git a/syft/pkg/cataloger/binary/pe_package_cataloger.go b/syft/pkg/cataloger/binary/pe_package_cataloger.go index 05410ab97..2d77c24bf 100644 --- a/syft/pkg/cataloger/binary/pe_package_cataloger.go +++ b/syft/pkg/cataloger/binary/pe_package_cataloger.go @@ -22,7 +22,7 @@ func NewPEPackageCataloger() pkg.Cataloger { func parsePE(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { f, err := pe.Read(reader) if err != nil { - return nil, nil, fmt.Errorf("failed to parse PE file %q: %w", reader.RealPath, err) + return nil, nil, fmt.Errorf("failed to parse PE file %q: %w", reader.FullNestedPath(), err) } if f == nil { diff --git a/syft/pkg/cataloger/bitnami/cataloger.go b/syft/pkg/cataloger/bitnami/cataloger.go index 2d845d6d1..d505088a7 100644 --- a/syft/pkg/cataloger/bitnami/cataloger.go +++ b/syft/pkg/cataloger/bitnami/cataloger.go @@ -36,13 +36,13 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment } if s == nil { - log.WithFields("path", reader.RealPath).Trace("file is not an SBOM") + log.WithFields("path", reader.FullNestedPath()).Trace("file is not an SBOM") return nil, nil, nil } // Bitnami exclusively uses SPDX JSON SBOMs if sFormat != "spdx-json" { - log.WithFields("path", reader.RealPath).Trace("file is not an SPDX JSON SBOM") + log.WithFields("path", reader.FullNestedPath()).Trace("file is not an SPDX JSON SBOM") return nil, nil, nil } @@ -73,7 +73,7 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment // Bitnami packages reported in a SPDX file are shipped under the same directory // as the SPDX file itself. - metadata.Path = path.Dir(reader.RealPath) + metadata.Path = path.Dir(reader.LocalPath()) if p.ID() != mainPkgID { metadata.Files = packageFiles(s.Relationships, p, metadata.Path) secondaryPkgsFiles = append(secondaryPkgsFiles, metadata.Files...) @@ -89,12 +89,12 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment } // Resolve all files owned by the main package in the SBOM and update the metadata - if mainPkgFiles, err := mainPkgFiles(resolver, reader.RealPath, secondaryPkgsFiles); err == nil { + if mainPkgFiles, err := mainPkgFiles(resolver, reader.LocalPath(), secondaryPkgsFiles); err == nil { for i, p := range pkgs { if p.ID() == mainPkgID { metadata, ok := p.Metadata.(*pkg.BitnamiSBOMEntry) if !ok { - log.WithFields("spdx-filepath", reader.RealPath).Trace("main package in SBOM does not have Bitnami metadata") + log.WithFields("spdx-filepath", reader.FullNestedPath()).Trace("main package in SBOM does not have Bitnami metadata") continue } @@ -103,7 +103,7 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment } } } else { - log.WithFields("spdx-filepath", reader.RealPath, "error", err).Trace("unable to resolve owned files for main package in SBOM") + log.WithFields("spdx-filepath", reader.FullNestedPath(), "error", err).Trace("unable to resolve owned files for main package in SBOM") } return pkgs, filterRelationships(s.Relationships, pkgs), nil diff --git a/syft/pkg/cataloger/bitnami/package.go b/syft/pkg/cataloger/bitnami/package.go index cb46d5453..f4eb402e1 100644 --- a/syft/pkg/cataloger/bitnami/package.go +++ b/syft/pkg/cataloger/bitnami/package.go @@ -61,7 +61,7 @@ func packageFiles(relationships []artifact.Relationship, p pkg.Package, baseDire // note: the file.Location is from the SBOM, and all files within the Bitnami SBOM by convention // are relative to the /opt/bitnami/PRODUCT directory, so we need to prepend the base directory // so that it's relative to the path found within the container image. - result = append(result, path.Join(baseDirectory, value.RealPath)) + result = append(result, path.Join(baseDirectory, value.LocalPath())) } } } diff --git a/syft/pkg/cataloger/bitnami/parse_components.go b/syft/pkg/cataloger/bitnami/parse_components.go index d2eb6d592..d22424ca4 100644 --- a/syft/pkg/cataloger/bitnami/parse_components.go +++ b/syft/pkg/cataloger/bitnami/parse_components.go @@ -67,7 +67,7 @@ func parseComponentsJSON(_ context.Context, _ file.Resolver, _ *generic.Environm Revision: revision, Architecture: entry.Arch, Distro: entry.Distro, - Path: path.Join(path.Dir(reader.RealPath), name), + Path: path.Join(path.Dir(reader.LocalPath()), name), } p := pkg.Package{ diff --git a/syft/pkg/cataloger/conda/cataloger.go b/syft/pkg/cataloger/conda/cataloger.go index 39667c8c0..61fc009db 100644 --- a/syft/pkg/cataloger/conda/cataloger.go +++ b/syft/pkg/cataloger/conda/cataloger.go @@ -24,7 +24,7 @@ func parseCondaMetaJSON(ctx context.Context, _ file.Resolver, _ *generic.Environ dec := json.NewDecoder(reader) var meta pkg.CondaMetaPackage if err := dec.Decode(&meta); err != nil { - return nil, nil, fmt.Errorf("failed to parse conda-meta package file at %s: %w", reader.RealPath, err) + return nil, nil, fmt.Errorf("failed to parse conda-meta package file at %s: %w", reader.FullNestedPath(), err) } p := pkg.Package{ diff --git a/syft/pkg/cataloger/cpp/parse_conaninfo.go b/syft/pkg/cataloger/cpp/parse_conaninfo.go index 895497174..e8586d19c 100644 --- a/syft/pkg/cataloger/cpp/parse_conaninfo.go +++ b/syft/pkg/cataloger/cpp/parse_conaninfo.go @@ -81,10 +81,7 @@ func parseFullRequiresLine(line string, reader file.LocationReadCloser, pkgs *[] // - recipe revision (recipe_hash) func parseConaninfo(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { // First set the base package info by checking the relative path - fullFilePath := string(reader.Location.LocationData.Reference().RealPath) - if len(fullFilePath) == 0 { - fullFilePath = reader.RealPath - } + fullFilePath := reader.LocalPath() mainMetadata, err := parseConanMetadataFromFilePath(fullFilePath) if err != nil { diff --git a/syft/pkg/cataloger/cpp/parse_vcpkgmanifest.go b/syft/pkg/cataloger/cpp/parse_vcpkgmanifest.go index cf11d15bb..fdd86f1d6 100644 --- a/syft/pkg/cataloger/cpp/parse_vcpkgmanifest.go +++ b/syft/pkg/cataloger/cpp/parse_vcpkgmanifest.go @@ -132,7 +132,7 @@ func findManAtLoc(loc file.Location, resolver file.Resolver) (*vcpkg.Vcpkg, erro if err != nil { return nil, err } - defer internal.CloseAndLogError(manCont, loc.RealPath) + defer internal.CloseAndLogError(manCont, loc.FullNestedPath()) manBytes, err := io.ReadAll(io.LimitReader(manCont, maxVcpkgManifestSize)) if err != nil { return nil, err @@ -166,7 +166,7 @@ func findVcpkgConfig(resolver file.Resolver) (*vcpkg.Config, error) { if err != nil { return nil, err } - defer internal.CloseAndLogError(cfgCont, locs[0].RealPath) + defer internal.CloseAndLogError(cfgCont, locs[0].FullNestedPath()) cfgBytes, err := io.ReadAll(io.LimitReader(cfgCont, maxVcpkgManifestSize)) if err != nil { return nil, err diff --git a/syft/pkg/cataloger/debian/package.go b/syft/pkg/cataloger/debian/package.go index c21dc6452..967434c14 100644 --- a/syft/pkg/cataloger/debian/package.go +++ b/syft/pkg/cataloger/debian/package.go @@ -250,7 +250,7 @@ func getAdditionalFileListing(resolver file.Resolver, dbLocation file.Location, // attach the file list records, err := parseDpkgMD5Info(md5Reader) if err != nil { - log.Debugf("dpkg md5sums file %q: %v", md5Location.RealPath, err) + log.Debugf("dpkg md5sums file %q: %v", md5Location.FullNestedPath(), err) } files = append(files, records...) @@ -265,7 +265,7 @@ func getAdditionalFileListing(resolver file.Resolver, dbLocation file.Location, // attach the file list records, err := parseDpkgConffileInfo(conffilesReader) if err != nil { - log.Debugf("dpkg conffiles file %q: %v", conffilesLocation.RealPath, err) + log.Debugf("dpkg conffiles file %q: %v", conffilesLocation.FullNestedPath(), err) } files = append(files, records...) @@ -288,7 +288,7 @@ func fetchMd5Contents(resolver file.Resolver, dbLocation file.Location, m pkg.Dp // and the md5sum information is under /var/lib/dpkg/info/; however, for distroless the installed // package info is across multiple files under /var/lib/dpkg/status.d/ and the md5sums are contained in // the same directory - searchPath := filepath.Dir(dbLocation.RealPath) + searchPath := filepath.Dir(dbLocation.LocalPath()) if !strings.HasSuffix(searchPath, "status.d") { searchPath = path.Join(searchPath, "info") @@ -327,7 +327,7 @@ func fetchConffileContents(resolver file.Resolver, dbLocation file.Location, m p return nil, nil } - parentPath := filepath.Dir(dbLocation.RealPath) + parentPath := filepath.Dir(dbLocation.LocalPath()) // look for /var/lib/dpkg/info/NAME:ARCH.conffiles name := md5Key(m) diff --git a/syft/pkg/cataloger/debian/parse_dpkg_db.go b/syft/pkg/cataloger/debian/parse_dpkg_db.go index 0168e5fb5..8ab89c5ca 100644 --- a/syft/pkg/cataloger/debian/parse_dpkg_db.go +++ b/syft/pkg/cataloger/debian/parse_dpkg_db.go @@ -37,7 +37,7 @@ var ( func parseDpkgDB(ctx context.Context, resolver file.Resolver, env *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { metadata, err := parseDpkgStatus(reader) if err != nil { - return nil, nil, fmt.Errorf("unable to catalog dpkg DB=%q: %w", reader.RealPath, err) + return nil, nil, fmt.Errorf("unable to catalog dpkg DB=%q: %w", reader.FullNestedPath(), err) } dbLoc := reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation) @@ -61,7 +61,7 @@ func findDpkgInfoFiles(name string, resolver file.Resolver, dbLocation file.Loca // and the md5sum information is under /var/lib/dpkg/info/; however, for distroless the installed // package info is across multiple files under /var/lib/dpkg/status.d/ and the md5sums are contained in // the same directory - searchPath := path.Dir(dbLocation.RealPath) + searchPath := path.Dir(dbLocation.LocalPath()) if !strings.HasSuffix(searchPath, "status.d") { searchPath = path.Join(searchPath, "info") diff --git a/syft/pkg/cataloger/dotnet/deps_binary_cataloger.go b/syft/pkg/cataloger/dotnet/deps_binary_cataloger.go index cde56fd44..d4f4d69ea 100644 --- a/syft/pkg/cataloger/dotnet/deps_binary_cataloger.go +++ b/syft/pkg/cataloger/dotnet/deps_binary_cataloger.go @@ -150,7 +150,7 @@ var runtimeDLLPathPattern = regexp.MustCompile(`/Microsoft\.NETCore\.App/(?P depJsons[j].Location.RealPath + return depJsons[i].Location.FullNestedPath() > depJsons[j].Location.FullNestedPath() }) // we should be processing PE files in a stable order sort.Slice(peFiles, func(i, j int) bool { - return peFiles[i].Location.RealPath > peFiles[j].Location.RealPath + return peFiles[i].Location.FullNestedPath() > peFiles[j].Location.FullNestedPath() }) peFilesByPath := make(map[file.Coordinates][]logicalPE) @@ -191,7 +191,7 @@ func partitionPEs(depJsons []logicalDepsJSON, peFiles []logicalPE) ([]logicalDep var found bool for i := range depJsons { dep := &depJsons[i] - if isParentOf(dep.Location.RealPath, pe.Location.RealPath) && attachAssociatedExecutables(dep, pe) { + if isParentOf(dep.Location.LocalPath(), pe.Location.LocalPath()) && attachAssociatedExecutables(dep, pe) { peFilesByPath[dep.Location.Coordinates] = append(peFilesByPath[dep.Location.Coordinates], pe) found = true // note: we cannot break from the dep JSON search since the same binary could be associated with multiple packages @@ -222,8 +222,8 @@ func partitionPEs(depJsons []logicalDepsJSON, peFiles []logicalPE) ([]logicalDep // attachAssociatedExecutables looks for PE files matching runtime or resource entries // and attaches them to the appropriate package. func attachAssociatedExecutables(dep *logicalDepsJSON, pe logicalPE) bool { - appDir := path.Dir(dep.Location.RealPath) - relativeDllPath := strings.TrimPrefix(strings.TrimPrefix(pe.Location.RealPath, appDir), "/") + appDir := path.Dir(dep.Location.LocalPath()) + relativeDllPath := strings.TrimPrefix(strings.TrimPrefix(pe.Location.LocalPath(), appDir), "/") var found bool for key, p := range dep.PackagesByNameVersion { @@ -471,7 +471,7 @@ func readDepsJSON(resolver file.Resolver, loc file.Location) (*depsJSON, error) if err != nil { return nil, unknown.New(loc, fmt.Errorf("unable to read deps.json file: %w", err)) } - defer internal.CloseAndLogError(reader, loc.RealPath) + defer internal.CloseAndLogError(reader, loc.FullNestedPath()) dj, err := newDepsJSON(file.NewLocationReadCloser(loc, reader)) if err != nil { @@ -479,7 +479,7 @@ func readDepsJSON(resolver file.Resolver, loc file.Location) (*depsJSON, error) } if dj == nil { - return nil, unknown.New(loc, fmt.Errorf("expected to find packages in deps.json but did not: %q", loc.RealPath)) + return nil, unknown.New(loc, fmt.Errorf("expected to find packages in deps.json but did not: %q", loc.FullNestedPath())) } return dj, nil @@ -520,7 +520,7 @@ func readPEFile(resolver file.Resolver, loc file.Location) (*logicalPE, error) { if err != nil { return nil, unknown.New(loc, fmt.Errorf("unable to read PE file: %w", err)) } - defer internal.CloseAndLogError(reader, loc.RealPath) + defer internal.CloseAndLogError(reader, loc.FullNestedPath()) ldpe, err := readLogicalPE(file.NewLocationReadCloser(loc, reader)) if err != nil { @@ -566,7 +566,7 @@ func readELFBundledDepsJSON(resolver file.Resolver, loc file.Location) (*logical if err != nil { return nil, err } - defer internal.CloseAndLogError(reader, loc.RealPath) + defer internal.CloseAndLogError(reader, loc.FullNestedPath()) header := make([]byte, 4) if _, err := io.ReadFull(reader, header); err != nil { diff --git a/syft/pkg/cataloger/dotnet/deps_json.go b/syft/pkg/cataloger/dotnet/deps_json.go index b0a8d2af8..e036d2fe5 100644 --- a/syft/pkg/cataloger/dotnet/deps_json.go +++ b/syft/pkg/cataloger/dotnet/deps_json.go @@ -161,7 +161,7 @@ type logicalDepsJSON struct { } func (l logicalDepsJSON) RootPackage() (logicalDepsJSONPackage, bool) { - rootName := getDepsJSONFilePrefix(l.Location.RealPath) + rootName := getDepsJSONFilePrefix(l.Location.FullNestedPath()) if rootName == "" { return logicalDepsJSONPackage{}, false } diff --git a/syft/pkg/cataloger/dotnet/libman_json.go b/syft/pkg/cataloger/dotnet/libman_json.go index 7f585eb08..f1e8d58a3 100644 --- a/syft/pkg/cataloger/dotnet/libman_json.go +++ b/syft/pkg/cataloger/dotnet/libman_json.go @@ -87,14 +87,14 @@ func newLibmanJSON(reader file.LocationReadCloser) (*libmanJSON, error) { } func findLibmanJSON(resolver file.Resolver, depsJSON file.Location) (*libmanJSON, error) { - parent := path.Dir(depsJSON.RealPath) + parent := path.Dir(depsJSON.LocalPath()) loc := resolver.RelativeFileByPath(depsJSON, path.Join(parent, "libman.json")) if loc == nil { return nil, nil } reader, err := resolver.FileContentsByLocation(*loc) - defer internal.CloseAndLogError(reader, loc.RealPath) + defer internal.CloseAndLogError(reader, loc.FullNestedPath()) if err != nil { return nil, err } diff --git a/syft/pkg/cataloger/dotnet/package.go b/syft/pkg/cataloger/dotnet/package.go index ac2361d60..e8e3f63a3 100644 --- a/syft/pkg/cataloger/dotnet/package.go +++ b/syft/pkg/cataloger/dotnet/package.go @@ -265,7 +265,7 @@ func newDotnetBinaryPackage(versionResources map[string]string, f file.Location) if name == "" { // older .NET runtime dlls may not have any version resources - name = strings.TrimSuffix(strings.TrimSuffix(path.Base(f.RealPath), ".exe"), ".dll") + name = strings.TrimSuffix(strings.TrimSuffix(path.Base(f.FullNestedPath()), ".exe"), ".dll") } ver := findVersionFromVersionResources(versionResources) diff --git a/syft/pkg/cataloger/elixir/parse_mix_lock.go b/syft/pkg/cataloger/elixir/parse_mix_lock.go index e44aaa8c0..ed836293f 100644 --- a/syft/pkg/cataloger/elixir/parse_mix_lock.go +++ b/syft/pkg/cataloger/elixir/parse_mix_lock.go @@ -83,7 +83,7 @@ func parseMixLock(_ context.Context, _ file.Resolver, _ *generic.Environment, re } if name == "" { - log.WithFields("path", reader.RealPath).Debug("skipping empty package name from mix.lock file") + log.WithFields("path", reader.FullNestedPath()).Debug("skipping empty package name from mix.lock file") errs = unknown.Appendf(errs, reader, "skipping empty package name from mix.lock file, for line: %d: %s", lineNum, line) continue } diff --git a/syft/pkg/cataloger/generic/cataloger.go b/syft/pkg/cataloger/generic/cataloger.go index ec4a5384a..2ac703be3 100644 --- a/syft/pkg/cataloger/generic/cataloger.go +++ b/syft/pkg/cataloger/generic/cataloger.go @@ -203,7 +203,7 @@ func (c *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg. errs := sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(c.selectFiles(resolver)), func(req request) (result, error) { location, parser := req.Location, req.Parser - log.WithFields("path", location.RealPath).Trace("parsing file contents") + log.WithFields("path", location.FullNestedPath()).Trace("parsing file contents") discoveredPackages, discoveredRelationships, err := invokeParser(ctx, resolver, location, lgr, parser, &env) if err != nil { @@ -243,7 +243,7 @@ func (c *Cataloger) processSafely(ctx context.Context, p processExecutor, resolv func invokeParser(ctx context.Context, resolver file.Resolver, location file.Location, logger logger.Logger, parser Parser, env *Environment) (pkgs []pkg.Package, rels []artifact.Relationship, err error) { contentReader, err := resolver.FileContentsByLocation(location) if err != nil { - logger.WithFields("location", location.RealPath, "error", err).Debug("unable to fetch contents") + logger.WithFields("location", location.FullNestedPath(), "error", err).Debug("unable to fetch contents") return nil, nil, err } defer internal.CloseAndLogError(contentReader, location.AccessPath) @@ -251,7 +251,7 @@ func invokeParser(ctx context.Context, resolver file.Resolver, location file.Loc defer func() { if r := recover(); r != nil { // a malformed file must not take down the whole SBOM; the caller records this as an unknown - logger.WithFields("location", location.RealPath, "panic", r).Debug("recovered from panic while parsing file") + logger.WithFields("location", location.FullNestedPath(), "panic", r).Debug("recovered from panic while parsing file") logger.Tracef("parser panic stack:\n%s", debug.Stack()) pkgs, rels, err = nil, nil, fmt.Errorf("recovered from panic while parsing file: %v", r) } @@ -260,7 +260,7 @@ func invokeParser(ctx context.Context, resolver file.Resolver, location file.Loc discoveredPackages, discoveredRelationships, err := parser(ctx, resolver, env, file.NewLocationReadCloser(location, contentReader)) if err != nil { // these errors are propagated up, and are likely to be coordinate errors - logger.WithFields("location", location.RealPath, "error", err).Trace("cataloger returned errors") + logger.WithFields("location", location.FullNestedPath(), "error", err).Trace("cataloger returned errors") } return discoveredPackages, discoveredRelationships, err diff --git a/syft/pkg/cataloger/generic/cataloger_test.go b/syft/pkg/cataloger/generic/cataloger_test.go index 93f7813d0..57df02893 100644 --- a/syft/pkg/cataloger/generic/cataloger_test.go +++ b/syft/pkg/cataloger/generic/cataloger_test.go @@ -19,7 +19,7 @@ import ( func Test_Cataloger(t *testing.T) { allParsedPaths := make(map[string]bool) parser := func(_ context.Context, resolver file.Resolver, env *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - allParsedPaths[reader.Path()] = true + allParsedPaths[reader.FullNestedPath()] = true contents, err := io.ReadAll(reader) require.NoError(t, err) @@ -219,7 +219,7 @@ func TestParserPanicBecomesUnknown(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { parser := func(_ context.Context, _ file.Resolver, _ *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - if reader.Path() == "testdata/another-path.txt" { + if reader.FullNestedPath() == "testdata/another-path.txt" { tt.doPanic() } return []pkg.Package{{Name: "good", Locations: file.NewLocationSet(reader.Location)}}, nil, nil @@ -308,7 +308,7 @@ func TestProcessorPanicKeepsResults(t *testing.T) { func Test_CatalogerWithParserByMediaType(t *testing.T) { allParsedPaths := make(map[string]bool) parser := func(_ context.Context, resolver file.Resolver, env *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - allParsedPaths[reader.Path()] = true + allParsedPaths[reader.FullNestedPath()] = true contents, err := io.ReadAll(reader) require.NoError(t, err) @@ -358,7 +358,7 @@ func Test_genericCatalogerReturnsErrors(t *testing.T) { genericErrorReturning := NewCataloger("error returning").WithParserByGlobs(func(ctx context.Context, resolver file.Resolver, environment *Environment, locationReader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { return []pkg.Package{ { - Name: "some-package-" + locationReader.Path(), + Name: "some-package-" + locationReader.FullNestedPath(), }, }, nil, unknown.Newf(locationReader, "unable to read") }, "**/*") diff --git a/syft/pkg/cataloger/gentoo/license.go b/syft/pkg/cataloger/gentoo/license.go index 4fedf5938..cb294fce5 100644 --- a/syft/pkg/cataloger/gentoo/license.go +++ b/syft/pkg/cataloger/gentoo/license.go @@ -71,7 +71,7 @@ func extractLicenses(resolver file.Resolver, closestLocation *file.Location, rea if err := scanner.Err(); err != nil { fields := []any{"error", err} if closestLocation != nil { - fields = append(fields, "path", closestLocation.RealPath) + fields = append(fields, "path", closestLocation.FullNestedPath()) } log.WithFields(fields...).Debug("failed to fully read portage LICENSE") } @@ -115,9 +115,9 @@ func readLicenseGroups(resolver file.Resolver, closestLocation *file.Location) m } groupReader, err := resolver.FileContentsByLocation(*groupLocation) - defer internal.CloseAndLogError(groupReader, groupLocation.RealPath) + defer internal.CloseAndLogError(groupReader, groupLocation.FullNestedPath()) if err != nil { - log.WithFields("path", groupLocation.RealPath, "error", err).Debug("failed to fetch portage LICENSE") + log.WithFields("path", groupLocation.FullNestedPath(), "error", err).Debug("failed to fetch portage LICENSE") return nil } @@ -127,7 +127,7 @@ func readLicenseGroups(resolver file.Resolver, closestLocation *file.Location) m licenseGroups, err = parseLicenseGroups(groupReader) if err != nil { - log.WithFields("path", groupLocation.RealPath, "error", err).Debug("failed to parse portage LICENSE") + log.WithFields("path", groupLocation.FullNestedPath(), "error", err).Debug("failed to parse portage LICENSE") } return licenseGroups diff --git a/syft/pkg/cataloger/gentoo/parse_portage_contents.go b/syft/pkg/cataloger/gentoo/parse_portage_contents.go index db9d68035..f8476a4fb 100644 --- a/syft/pkg/cataloger/gentoo/parse_portage_contents.go +++ b/syft/pkg/cataloger/gentoo/parse_portage_contents.go @@ -24,14 +24,14 @@ var ( // parses individual CONTENTS files from the portage flat-file store (e.g. /var/db/pkg/*/*/CONTENTS). func parsePortageContents(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - cpvMatch := cpvRe.FindStringSubmatch(reader.RealPath) + cpvMatch := cpvRe.FindStringSubmatch(reader.LocalPath()) if cpvMatch == nil { - return nil, nil, fmt.Errorf("failed to match package and version in %s", reader.RealPath) + return nil, nil, fmt.Errorf("failed to match package and version in %s", reader.FullNestedPath()) } name, version := cpvMatch[1], cpvMatch[2] if name == "" || version == "" { - log.WithFields("path", reader.RealPath).Debug("failed to parse portage name and version") + log.WithFields("path", reader.FullNestedPath()).Debug("failed to parse portage name and version") return nil, nil, fmt.Errorf("failed to parse portage name and version") } @@ -64,10 +64,10 @@ func parsePortageContents(ctx context.Context, resolver file.Resolver, _ *generi func addFiles(resolver file.Resolver, dbLocation file.Location, entry *pkg.PortageEntry) { contentsReader, err := resolver.FileContentsByLocation(dbLocation) if err != nil { - log.WithFields("path", dbLocation.RealPath, "error", err).Debug("failed to fetch portage contents") + log.WithFields("path", dbLocation.FullNestedPath(), "error", err).Debug("failed to fetch portage contents") return } - defer internal.CloseAndLogError(contentsReader, dbLocation.RealPath) + defer internal.CloseAndLogError(contentsReader, dbLocation.FullNestedPath()) scanner := internal.NewLineScanner(contentsReader) for scanner.Scan() { @@ -87,12 +87,12 @@ func addFiles(resolver file.Resolver, dbLocation file.Location, entry *pkg.Porta } } if err := scanner.Err(); err != nil { - log.WithFields("path", dbLocation.RealPath, "error", err).Debug("failed to fully read portage contents") + log.WithFields("path", dbLocation.FullNestedPath(), "error", err).Debug("failed to fully read portage contents") } } func addLicenses(ctx context.Context, resolver file.Resolver, dbLocation file.Location, entry *pkg.PortageEntry) (pkg.LicenseSet, []file.Location) { - parentPath := filepath.Dir(dbLocation.RealPath) + parentPath := filepath.Dir(dbLocation.LocalPath()) location := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "LICENSE")) @@ -102,10 +102,10 @@ func addLicenses(ctx context.Context, resolver file.Resolver, dbLocation file.Lo licenseReader, err := resolver.FileContentsByLocation(*location) if err != nil { - log.WithFields("path", dbLocation.RealPath, "error", err).Debug("failed to fetch portage LICENSE") + log.WithFields("path", dbLocation.FullNestedPath(), "error", err).Debug("failed to fetch portage LICENSE") return pkg.NewLicenseSet(), nil } - defer internal.CloseAndLogError(licenseReader, location.RealPath) + defer internal.CloseAndLogError(licenseReader, location.FullNestedPath()) og, spdxExpression := extractLicenses(resolver, location, licenseReader) entry.Licenses = og @@ -115,7 +115,7 @@ func addLicenses(ctx context.Context, resolver file.Resolver, dbLocation file.Lo } func addSize(resolver file.Resolver, dbLocation file.Location, entry *pkg.PortageEntry) []file.Location { - parentPath := filepath.Dir(dbLocation.RealPath) + parentPath := filepath.Dir(dbLocation.LocalPath()) location := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "SIZE")) @@ -125,10 +125,10 @@ func addSize(resolver file.Resolver, dbLocation file.Location, entry *pkg.Portag sizeReader, err := resolver.FileContentsByLocation(*location) if err != nil { - log.WithFields("path", dbLocation.RealPath, "error", err).Debug("failed to fetch portage SIZE") + log.WithFields("path", dbLocation.FullNestedPath(), "error", err).Debug("failed to fetch portage SIZE") return nil } - defer internal.CloseAndLogError(sizeReader, location.RealPath) + defer internal.CloseAndLogError(sizeReader, location.FullNestedPath()) scanner := internal.NewLineScanner(sizeReader) for scanner.Scan() { @@ -139,7 +139,7 @@ func addSize(resolver file.Resolver, dbLocation file.Location, entry *pkg.Portag } } if err := scanner.Err(); err != nil { - log.WithFields("path", location.RealPath, "error", err).Debug("failed to fully read portage SIZE") + log.WithFields("path", location.FullNestedPath(), "error", err).Debug("failed to fully read portage SIZE") } return []file.Location{location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.SupportingEvidenceAnnotation)} diff --git a/syft/pkg/cataloger/githubactions/package.go b/syft/pkg/cataloger/githubactions/package.go index 49c5c83e4..f7f36863b 100644 --- a/syft/pkg/cataloger/githubactions/package.go +++ b/syft/pkg/cataloger/githubactions/package.go @@ -15,7 +15,7 @@ func newPackageFromUsageStatement(use, comment string, location file.Location) ( name, version := parseStepUsageStatement(use, comment) if name == "" { - log.WithFields("file", location.RealPath, "statement", use).Trace("unable to parse github action usage statement") + log.WithFields("file", location.FullNestedPath(), "statement", use).Trace("unable to parse github action usage statement") return nil, fmt.Errorf("unable to parse github action usage statement") } diff --git a/syft/pkg/cataloger/golang/licenses.go b/syft/pkg/cataloger/golang/licenses.go index e3c40af5d..2fdcb8889 100644 --- a/syft/pkg/cataloger/golang/licenses.go +++ b/syft/pkg/cataloger/golang/licenses.go @@ -239,13 +239,13 @@ func (c *goLicenseResolver) findLicensesInSource(ctx context.Context, resolver f func (c *goLicenseResolver) parseLicenseFromLocation(ctx context.Context, l file.Location, resolver file.Resolver) ([]pkg.License, error) { var out []pkg.License - fileName := path.Base(l.RealPath) + fileName := path.Base(l.FullNestedPath()) if licenses.IsLicenseFile(fileName) { contents, err := resolver.FileContentsByLocation(l) if err != nil { return nil, err } - defer internal.CloseAndLogError(contents, l.RealPath) + defer internal.CloseAndLogError(contents, l.FullNestedPath()) out = pkg.NewLicensesFromReadCloserWithContext(ctx, file.NewLocationReadCloser(l, contents)) } return out, nil diff --git a/syft/pkg/cataloger/golang/parse_go_binary.go b/syft/pkg/cataloger/golang/parse_go_binary.go index e33874800..002e44d43 100644 --- a/syft/pkg/cataloger/golang/parse_go_binary.go +++ b/syft/pkg/cataloger/golang/parse_go_binary.go @@ -119,7 +119,7 @@ func (c *goBinaryCataloger) parseGoBinary(ctx context.Context, resolver file.Res if err != nil { return nil, nil, err } - defer internal.CloseAndLogError(reader.ReadCloser, reader.RealPath) + defer internal.CloseAndLogError(reader.ReadCloser, reader.FullNestedPath()) mods, errs := scanFile(ctx, reader.Location, unionReader, c.symbolSelector.enabled()) // scanFile hands back the reconstruction for each binary that turned out to be packed. The readers diff --git a/syft/pkg/cataloger/golang/parse_go_binary_test.go b/syft/pkg/cataloger/golang/parse_go_binary_test.go index 7e33b0155..1b5b58f23 100644 --- a/syft/pkg/cataloger/golang/parse_go_binary_test.go +++ b/syft/pkg/cataloger/golang/parse_go_binary_test.go @@ -163,10 +163,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -209,10 +206,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{}, @@ -254,10 +248,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -292,10 +283,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -366,10 +354,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft@v0.0.0-20221014195457-41bc6bb41035", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -433,10 +418,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft@v0.79.0", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -498,10 +480,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft@v0.79.0", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -555,10 +534,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft@v0.79.0", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -612,10 +588,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft@v0.79.0", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -670,10 +643,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft@v0.0.0-20221014195457-41bc6bb41035", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -742,10 +712,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -763,10 +730,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -820,10 +784,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -840,10 +801,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -895,10 +853,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -916,10 +871,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -971,10 +923,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft@v1.0.0-somethingelse%2Bincompatible", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -1030,10 +979,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -1082,10 +1028,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/kuskoman/something-else@v1.2.3", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -1102,10 +1045,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/kuskoman/logstash-exporter@v1.7.0", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -1143,10 +1083,7 @@ func TestBuildGoPkgInfo(t *testing.T) { p.SetID() } location := file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ) if test.cfg == nil { @@ -1433,7 +1370,7 @@ func (alwaysErrorReader) Read(_ []byte) (int, error) { } func Test_buildGoPkgInfo_symbolScope(t *testing.T) { - location := file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/a-path", FileSystemID: "layer-id"}) + location := file.NewLocationFromCoordinates(file.NewCoordinates("/a-path", "layer-id")) // the symbols a binary would carry once scanFile has extracted them: one main-package symbol, two // dependency symbols from distinct packages within the same module (so the per-package keying of the @@ -1615,7 +1552,7 @@ var ( // symbols more than once. This happens in production for universal/fat Mach-O binaries: scanFile yields // one build info per architecture and each is recorded under the same location coordinates. func Test_recordStdlibSymbols_merge(t *testing.T) { - coord := file.Coordinates{RealPath: "/a-path", FileSystemID: "layer-id"} + coord := file.NewCoordinates("/a-path", "layer-id") c := newGoBinaryCataloger(CatalogerConfig{}) c.recordStdlibSymbols(coord, map[string][]string{ @@ -1638,7 +1575,7 @@ func Test_recordStdlibSymbols_merge(t *testing.T) { // Test_stdlibSymbolsFor_isolation asserts the returned map is a deep copy: mutating it (or its slices) // must not corrupt the cataloger's guarded internal state. func Test_stdlibSymbolsFor_isolation(t *testing.T) { - coord := file.Coordinates{RealPath: "/a-path", FileSystemID: "layer-id"} + coord := file.NewCoordinates("/a-path", "layer-id") c := newGoBinaryCataloger(CatalogerConfig{}) c.recordStdlibSymbols(coord, map[string][]string{"net/http": {"Get"}}) diff --git a/syft/pkg/cataloger/golang/parse_go_mod.go b/syft/pkg/cataloger/golang/parse_go_mod.go index c4a727987..78bf9dcb5 100644 --- a/syft/pkg/cataloger/golang/parse_go_mod.go +++ b/syft/pkg/cataloger/golang/parse_go_mod.go @@ -40,6 +40,10 @@ func newGoModCataloger(opts CatalogerConfig) *goModCataloger { // parseGoModFile takes a go.mod and tries to resolve and lists all packages discovered. func (c *goModCataloger) parseGoModFile(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) (pkgs []pkg.Package, relationships []artifact.Relationship, err error) { realPath := string(reader.Location.Reference().RealPath) + if realPath == "" { + // the Reference().RealPath is used to find full paths when using a directory scan, it does not provide correct paths for archive scanning + realPath = reader.LocalPath() + } if windows.HostRunningOnWindows() { // directory resolvers hand back volume-encoded posix paths (/c/...), the go toolchain needs the native path realPath = windows.FromPosix(realPath) @@ -334,7 +338,7 @@ func (c *goModCataloger) parseModFileContents(reader file.LocationReadCloser) (* return nil, fmt.Errorf("failed to read go module: %w", err) } - f, err := modfile.Parse(reader.RealPath, contents, nil) + f, err := modfile.Parse(reader.LocalPath(), contents, nil) if err != nil { return nil, fmt.Errorf("failed to parse go module: %w", err) } @@ -427,7 +431,7 @@ func parseGoSumFile(resolver file.Resolver, reader file.LocationReadCloser) (map return out, fmt.Errorf("no resolver provided") } - goSumPath := strings.TrimSuffix(reader.RealPath, ".mod") + ".sum" + goSumPath := strings.TrimSuffix(reader.LocalPath(), ".mod") + ".sum" goSumLocation := resolver.RelativeFileByPath(reader.Location, goSumPath) if goSumLocation == nil { return nil, fmt.Errorf("unable to resolve: %s", goSumPath) diff --git a/syft/pkg/cataloger/golang/scan_binary.go b/syft/pkg/cataloger/golang/scan_binary.go index d017223ba..abb1e1e72 100644 --- a/syft/pkg/cataloger/golang/scan_binary.go +++ b/syft/pkg/cataloger/golang/scan_binary.go @@ -96,7 +96,7 @@ func scanReader(ctx context.Context, location file.Location, r io.ReaderAt, capt if isCancelled(err) { return nil, err } - log.WithFields("file", location.RealPath, "error", err).Trace("unable to fully read golang buildinfo") + log.WithFields("file", location.FullNestedPath(), "error", err).Trace("unable to fully read golang buildinfo") if reportableGap(err) { // the build info is either missing or it is not, and the same err covers both: a partial // reconstruction still hands back everything .go.buildinfo carried, and what it lost is the @@ -128,7 +128,7 @@ func scanReader(ctx context.Context, location file.Location, r io.ReaderAt, capt // bounds only the section elf.NewFile expands as it parses. v, err := getCryptoInformation(contents) if err != nil { - log.WithFields("file", location.RealPath, "error", err).Trace("unable to read golang version info") + log.WithFields("file", location.FullNestedPath(), "error", err).Trace("unable to read golang version info") // don't skip this build info. // we can still catalog packages, even if we can't get the crypto information errs = unknown.Appendf(errs, location, "unable to read golang version info: %w", err) @@ -138,7 +138,7 @@ func scanReader(ctx context.Context, location file.Location, r io.ReaderAt, capt if arch == "" { arch, err = getGOARCHFromBin(contents) if err != nil { - log.WithFields("file", location.RealPath, "error", err).Trace("unable to read golang arch info") + log.WithFields("file", location.FullNestedPath(), "error", err).Trace("unable to read golang arch info") // don't skip this build info. // we can still catalog packages, even if we can't get the arch information errs = unknown.Appendf(errs, location, "unable to read golang arch info: %w", err) @@ -149,7 +149,7 @@ func scanReader(ctx context.Context, location file.Location, r io.ReaderAt, capt if captureSymbols { symbols, err = getSymbols(contents) if err != nil { - log.WithFields("file", location.RealPath, "error", err).Trace("unable to read golang symbol info") + log.WithFields("file", location.FullNestedPath(), "error", err).Trace("unable to read golang symbol info") // don't skip this build info. // we can still catalog packages, even if we can't get the symbol information errs = unknown.Appendf(errs, location, "unable to read golang symbol info: %w", err) diff --git a/syft/pkg/cataloger/haskell/parse_stack_lock.go b/syft/pkg/cataloger/haskell/parse_stack_lock.go index 1f36ea424..66c6c0201 100644 --- a/syft/pkg/cataloger/haskell/parse_stack_lock.go +++ b/syft/pkg/cataloger/haskell/parse_stack_lock.go @@ -44,7 +44,7 @@ func parseStackLock(_ context.Context, _ file.Resolver, _ *generic.Environment, var lockFile stackLock if err := yaml.NewDecoder(reader).Decode(&lockFile); err != nil { - log.WithFields("error", err, "path", reader.RealPath).Trace("failed to parse stack.yaml.lock") + log.WithFields("error", err, "path", reader.FullNestedPath()).Trace("failed to parse stack.yaml.lock") return nil, nil, fmt.Errorf("failed to parse stack.yaml.lock file: %w", err) } diff --git a/syft/pkg/cataloger/haskell/parse_stack_yaml.go b/syft/pkg/cataloger/haskell/parse_stack_yaml.go index cd0fc4167..109625ce7 100644 --- a/syft/pkg/cataloger/haskell/parse_stack_yaml.go +++ b/syft/pkg/cataloger/haskell/parse_stack_yaml.go @@ -25,7 +25,7 @@ func parseStackYaml(_ context.Context, _ file.Resolver, _ *generic.Environment, var stackFile stackYaml if err := yaml.NewDecoder(reader).Decode(&stackFile); err != nil { - log.WithFields("error", err, "path", reader.RealPath).Trace("failed to parse stack.yaml") + log.WithFields("error", err, "path", reader.FullNestedPath()).Trace("failed to parse stack.yaml") return nil, nil, fmt.Errorf("failed to parse stack.yaml file: %w", err) } diff --git a/syft/pkg/cataloger/homebrew/parse_homebrew_formula.go b/syft/pkg/cataloger/homebrew/parse_homebrew_formula.go index 93d866746..8a928cc0e 100644 --- a/syft/pkg/cataloger/homebrew/parse_homebrew_formula.go +++ b/syft/pkg/cataloger/homebrew/parse_homebrew_formula.go @@ -25,7 +25,7 @@ type parsedHomebrewData struct { func parseHomebrewFormula(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { pd, err := parseFormulaFile(reader) if err != nil { - log.WithFields("path", reader.RealPath).Trace("failed to parse formula") + log.WithFields("path", reader.FullNestedPath()).Trace("failed to parse formula") return nil, nil, err } @@ -69,7 +69,7 @@ func parseFormulaFile(reader file.LocationReadCloser) (*parsedHomebrewData, erro } } - pd.Tap = getTapFromPath(reader.RealPath) + pd.Tap = getTapFromPath(reader.LocalPath()) if err := scanner.Err(); err != nil { return nil, err @@ -79,7 +79,7 @@ func parseFormulaFile(reader file.LocationReadCloser) (*parsedHomebrewData, erro return &pd, nil } - pd.Name, pd.Version = getNameAndVersionFromPath(reader.RealPath) + pd.Name, pd.Version = getNameAndVersionFromPath(reader.LocalPath()) return &pd, nil } diff --git a/syft/pkg/cataloger/internal/binutils/branching_matcher.go b/syft/pkg/cataloger/internal/binutils/branching_matcher.go index c18e433e1..d75298b0c 100644 --- a/syft/pkg/cataloger/internal/binutils/branching_matcher.go +++ b/syft/pkg/cataloger/internal/binutils/branching_matcher.go @@ -18,7 +18,7 @@ func BranchingEvidenceMatcher(classifiers ...Classifier) EvidenceMatcher { if err != nil { return nil, err } - defer internal.CloseAndLogError(rdr, context.Location.RealPath) + defer internal.CloseAndLogError(rdr, context.Location.FullNestedPath()) if err != nil { return nil, err } diff --git a/syft/pkg/cataloger/internal/binutils/classifier.go b/syft/pkg/cataloger/internal/binutils/classifier.go index 05b62897e..4dc5f61dd 100644 --- a/syft/pkg/cataloger/internal/binutils/classifier.go +++ b/syft/pkg/cataloger/internal/binutils/classifier.go @@ -162,11 +162,11 @@ func (c ContextualEvidenceMatchers) FileContentsVersionMatcher(patterns ...strin func FileNameTemplateVersionMatcher(fileNamePattern, contentTemplate, catalogerName string) EvidenceMatcher { pat := regexp.MustCompile(fileNamePattern) return func(classifier Classifier, context MatcherContext) ([]pkg.Package, error) { - if !pat.MatchString(context.Location.RealPath) { + if !pat.MatchString(context.Location.LocalPath()) { return nil, nil } - filepathNamedGroupValues := internal.MatchNamedCaptureGroups(pat, context.Location.RealPath) + filepathNamedGroupValues := internal.MatchNamedCaptureGroups(pat, context.Location.LocalPath()) // versions like 3.5 should not match any character, but explicit dot for k, v := range filepathNamedGroupValues { @@ -328,7 +328,7 @@ func MatchPath(path string) EvidenceMatcher { panic("invalid pattern") } return func(_ Classifier, context MatcherContext) ([]pkg.Package, error) { - if doublestar.MatchUnvalidated(path, context.Location.RealPath) { + if doublestar.MatchUnvalidated(path, context.Location.LocalPath()) { return []pkg.Package{}, nil // return non-nil } return nil, nil @@ -339,7 +339,7 @@ func MatchPath(path string) EvidenceMatcher { // relative to a primary file, for example: a VERSION file in the same or a parent directory to another binary func SupportingEvidenceMatcher(relativePathGlob string, evidenceMatcher EvidenceMatcher) EvidenceMatcher { return func(classifier Classifier, context MatcherContext) ([]pkg.Package, error) { - f := path.Dir(context.Location.RealPath) + f := path.Dir(context.Location.LocalPath()) f = path.Join(f, relativePathGlob) f = path.Clean(f) // this would ideally be RelativeFileByPath but with a glob search: @@ -376,7 +376,7 @@ func sharedLibraries(context MatcherContext) ([]string, error) { if err != nil { return nil, err } - defer internal.CloseAndLogError(contents, context.Location.RealPath) + defer internal.CloseAndLogError(contents, context.Location.FullNestedPath()) e, err := elfutil.NewFile(contents) if err != nil { @@ -386,13 +386,13 @@ func sharedLibraries(context MatcherContext) ([]string, error) { // dropped, and nothing downstream would report it. var fmtErr *elf.FormatError if !errors.As(err, &fmtErr) && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { - log.WithFields("file", context.Location.RealPath, "error", err).Debug("unable to parse ELF binary") + log.WithFields("file", context.Location.FullNestedPath(), "error", err).Debug("unable to parse ELF binary") } } if e != nil { symbols, err := e.ImportedLibraries() if err != nil { - log.Debugf("unable to read elf binary at: %s -- %s", context.Location.RealPath, err) + log.Debugf("unable to read elf binary at: %s -- %s", context.Location.FullNestedPath(), err) } return symbols, nil } @@ -404,7 +404,7 @@ func sharedLibraries(context MatcherContext) ([]string, error) { if m != nil { symbols, err := m.ImportedLibraries() if err != nil { - log.Debugf("unable to read macho binary at: %s -- %s", context.Location.RealPath, err) + log.Debugf("unable to read macho binary at: %s -- %s", context.Location.FullNestedPath(), err) } return symbols, nil } @@ -416,7 +416,7 @@ func sharedLibraries(context MatcherContext) ([]string, error) { if p != nil { symbols, err := p.ImportedLibraries() if err != nil { - log.Debugf("unable to read pe binary at: %s -- %s", context.Location.RealPath, err) + log.Debugf("unable to read pe binary at: %s -- %s", context.Location.FullNestedPath(), err) } return symbols, nil } @@ -443,7 +443,7 @@ func collectSupportingEvidence(classifier Classifier, context MatcherContext, re if err != nil { return nil, err } - defer internal.CloseAndLogError(rdr, relativeFile.Path()) + defer internal.CloseAndLogError(rdr, relativeFile.FullNestedPath()) ur, err := unionreader.GetUnionReader(rdr) if err != nil { return nil, err diff --git a/syft/pkg/cataloger/internal/dotnet/pe/pe.go b/syft/pkg/cataloger/internal/dotnet/pe/pe.go index 9939e153e..d527ba6db 100644 --- a/syft/pkg/cataloger/internal/dotnet/pe/pe.go +++ b/syft/pkg/cataloger/internal/dotnet/pe/pe.go @@ -330,7 +330,7 @@ func Read(f file.LocationReadCloser) (*File, error) { parseErr := errors.Join(parseErrs...) if parseErr != nil { - log.Tracef("partially parsed PE file %s: %v", f.RealPath, parseErr) + log.Tracef("partially parsed PE file %s: %v", f.FullNestedPath(), parseErr) } return &File{ diff --git a/syft/pkg/cataloger/internal/licenses/find_licenses.go b/syft/pkg/cataloger/internal/licenses/find_licenses.go index 8a6e4865e..4852febbf 100644 --- a/syft/pkg/cataloger/internal/licenses/find_licenses.go +++ b/syft/pkg/cataloger/internal/licenses/find_licenses.go @@ -115,9 +115,9 @@ func NewFromValues(ctx context.Context, locations []file.Location, values ...str func readFromResolver(ctx context.Context, resolver file.Resolver, location file.Location) []pkg.License { metadataContents, err := resolver.FileContentsByLocation(location) if err != nil || metadataContents == nil { - log.WithFields("error", err, "path", location.Path()).Trace("unable to license file contents") + log.WithFields("error", err, "path", location.FullNestedPath()).Trace("unable to license file contents") return nil } - defer internal.CloseAndLogError(metadataContents, location.Path()) + defer internal.CloseAndLogError(metadataContents, location.FullNestedPath()) return pkg.NewLicensesFromReadCloserWithContext(ctx, file.NewLocationReadCloser(location, metadataContents)) } diff --git a/syft/pkg/cataloger/internal/pkgtest/observing_resolver.go b/syft/pkg/cataloger/internal/pkgtest/observing_resolver.go index 8fca2ce58..61aaa0b37 100644 --- a/syft/pkg/cataloger/internal/pkgtest/observing_resolver.go +++ b/syft/pkg/cataloger/internal/pkgtest/observing_resolver.go @@ -54,7 +54,7 @@ func (r *ObservingResolver) ObservedPathQuery(input string) bool { // ObservedPathResponses checks if a specific path was returned in any response. func (r *ObservingResolver) ObservedPathResponses(path string) bool { for _, loc := range r.pathResponses { - if loc.RealPath == path { + if loc.FullNestedPath() == path { return true } } @@ -64,7 +64,7 @@ func (r *ObservingResolver) ObservedPathResponses(path string) bool { // ObservedContentQueries checks if a specific file's content was read. func (r *ObservingResolver) ObservedContentQueries(path string) bool { for _, loc := range r.contentQueries { - if loc.RealPath == path { + if loc.FullNestedPath() == path { return true } } @@ -75,7 +75,7 @@ func (r *ObservingResolver) ObservedContentQueries(path string) bool { func (r *ObservingResolver) AllContentQueries() []string { observed := strset.New() for _, loc := range r.contentQueries { - observed.Add(loc.RealPath) + observed.Add(loc.FullNestedPath()) } return observed.List() } diff --git a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go index 22042fcf1..6cc30c14a 100644 --- a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go +++ b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go @@ -495,7 +495,7 @@ func stringPackage(p pkg.Package) string { locs := p.Locations.ToSlice() var loc string if len(locs) > 0 { - loc = p.Locations.ToSlice()[0].RealPath + loc = p.Locations.ToSlice()[0].FullNestedPath() } return fmt.Sprintf("%s @ %s (%s)", p.Name, p.Version, loc) diff --git a/syft/pkg/cataloger/java/archive_entries.go b/syft/pkg/cataloger/java/archive_entries.go index a24b0a31b..f1f22d486 100644 --- a/syft/pkg/cataloger/java/archive_entries.go +++ b/syft/pkg/cataloger/java/archive_entries.go @@ -62,7 +62,7 @@ func (r *resolverEntries) glob(patterns ...string) []string { continue } for _, location := range locations { - name := strings.TrimPrefix(location.Path(), "/") + name := strings.TrimPrefix(location.LocalPath(), "/") if seen.Has(name) { continue } @@ -98,7 +98,7 @@ func (r *resolverEntries) read(location file.Location) (string, error) { if err != nil { return "", err } - defer internal.CloseAndLogError(reader, location.RealPath) + defer internal.CloseAndLogError(reader, location.FullNestedPath()) var contents strings.Builder if err := intFile.SafeCopy(&contents, reader); err != nil { diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index 2e520594b..99d7010ff 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -156,9 +156,10 @@ func newExtractedArchiveParser(trav *archive.Traversal, resolver file.Resolver, return nil, fmt.Errorf("no resolver for the contents of %q", archive.VirtualPath(trav.Location)) } return &archiveParser{ - entries: &resolverEntries{resolver: resolver}, - location: trav.Location, - fileInfo: newJavaArchiveFilename(path.Base(trav.Location.Path())), + entries: &resolverEntries{resolver: resolver}, + location: trav.Location, + // the name it was reached by, as for isJavaArchiveName + fileInfo: newJavaArchiveFilename(path.Base(trav.Location.AccessPath)), archiveDigests: trav.Digests, cfg: cfg, maven: maven.NewResolver(nil, cfg.mavenConfig()), @@ -715,7 +716,7 @@ func discoverPkgsFromOpeners(ctx context.Context, location file.Location, opener for pathWithinArchive, archiveOpener := range sortedIter(openers) { nestedPkgs, nestedRelationships, err := discoverPkgsFromOpener(ctx, location, pathWithinArchive, archiveOpener, cfg, parentPkg) if err != nil { - log.WithFields("location", location.Path(), "error", err).Debug("unable to discover java packages from opener") + log.WithFields("location", location.FullNestedPath(), "error", err).Debug("unable to discover java packages from opener") continue } @@ -748,7 +749,7 @@ func discoverPkgsFromOpener(ctx context.Context, location file.Location, pathWit } }() - nestedPath := fmt.Sprintf("%s:%s", location.Path(), pathWithinArchive) + nestedPath := fmt.Sprintf("%s:%s", location.AccessPath, pathWithinArchive) nestedLocation := file.NewLocationFromCoordinates(location.Coordinates) nestedLocation.AccessPath = nestedPath gap := newGenericArchiveParserAdapter(cfg) @@ -773,7 +774,7 @@ func pomPropertiesByParentPath(ctx context.Context, entries archiveEntries, loca for filePath, fileContents := range sortedIter(contentsOfMavenPropertiesFiles) { pomProperties, err := parsePomProperties(filePath, strings.NewReader(fileContents)) if err != nil { - log.WithFields("contents-path", filePath, "location", location.Path(), "error", err).Debug("failed to parse pom.properties") + log.WithFields("contents-path", filePath, "location", location.FullNestedPath(), "error", err).Debug("failed to parse pom.properties") continue } @@ -803,7 +804,7 @@ func pomProjectByParentPath(ctx context.Context, entries archiveEntries, locatio // TODO: when we support locations of paths within archives we should start passing the specific pom.xml location object instead of the top jar pom, err := maven.ParsePomXML(strings.NewReader(fileContents)) if err != nil { - log.WithFields("contents-path", filePath, "location", location.Path(), "error", err).Debug("failed to parse pom.xml") + log.WithFields("contents-path", filePath, "location", location.FullNestedPath(), "error", err).Debug("failed to parse pom.xml") continue } if pom == nil { diff --git a/syft/pkg/cataloger/java/archive_parser_traversal_test.go b/syft/pkg/cataloger/java/archive_parser_traversal_test.go index ba8c0305c..4adeb8d3d 100644 --- a/syft/pkg/cataloger/java/archive_parser_traversal_test.go +++ b/syft/pkg/cataloger/java/archive_parser_traversal_test.go @@ -21,7 +21,7 @@ import ( "github.com/anchore/syft/syft/source/directorysource" ) -// Test_virtualPathFromArchiveTraversal_colonInEntryPath is the java-cataloger-level counterpart of +// Test_virtualPathFromArchiveLocation_colonInEntryPath is the java-cataloger-level counterpart of // internal/archive's escaping tests: a package reached through an entry whose path contains a colon // must carry it escaped as %3A end to end in its virtual path, while the maven-derived // ":groupID:artifactID" suffix that distinguishes a pom.properties package from its containing archive @@ -35,8 +35,7 @@ func Test_virtualPathFromArchiveLocation_colonInEntryPath(t *testing.T) { }) // as the archive cataloger task's resolver would present an entry of outer.zip - location := file.NewLocation("/lib/dep:special.jar") - location.ArchivePath = "outer.zip" + location := file.NewLocationFromCoordinates(file.Coordinates{RealPath: "outer.zip", FullPath: "outer.zip!/lib/dep:special.jar"}) gap := newGenericArchiveParserAdapter(ArchiveCatalogerConfig{}) pkgs, _, err := gap.processJavaArchive(pkgtest.Context(t), file.LocationReadCloser{ @@ -172,58 +171,6 @@ func Test_NewArchiveCataloger_wrappedParserRegistration(t *testing.T) { }) } -func Test_mavenVirtualPathSuffixUnderTraversal(t *testing.T) { - // a package derived from a pom.properties whose artifact does not match the containing archive gets - // a trailing "::" appended to the archive's virtual path. That suffix form is in - // published SBOMs, and once the archive cataloger drives the parser the base carries the location's - // archive path - so the two must compose rather than one clobbering the other. - jarBytes := makeTestZip(t, map[string][]byte{ - "META-INF/MANIFEST.MF": testManifest("example-app", "1.0.0"), - "META-INF/maven/com.example.other/other-lib/pom.properties": []byte( - "groupId=com.example.other\nartifactId=other-lib\nversion=2.5.0\n", - ), - }) - - parse := func(t *testing.T, archivePath, readerPath string) []pkg.Package { - t.Helper() - location := file.NewLocation(readerPath) - location.ArchivePath = archivePath - gap := newGenericArchiveParserAdapter(ArchiveCatalogerConfig{}) - pkgs, _, err := gap.processJavaArchive(pkgtest.Context(t), file.LocationReadCloser{ - Location: location, - ReadCloser: io.NopCloser(bytes.NewReader(jarBytes)), - }, nil) - require.NoError(t, err) - return pkgs - } - - virtualPathOf := func(t *testing.T, pkgs []pkg.Package, name string) string { - t.Helper() - for _, p := range pkgs { - if p.Name != name { - continue - } - metadata, ok := p.Metadata.(pkg.JavaArchive) - require.True(t, ok) - return metadata.VirtualPath - } - t.Fatalf("no package named %q in %v", name, pkgs) - return "" - } - - t.Run("outside an archive: suffix hangs off the reader path", func(t *testing.T) { - pkgs := parse(t, "", "/example-app-1.0.0.jar") - assert.Equal(t, "/example-app-1.0.0.jar:com.example.other:other-lib", - virtualPathOf(t, pkgs, "other-lib")) - }) - - t.Run("inside an archive: suffix hangs off the archive chain", func(t *testing.T) { - pkgs := parse(t, "app.war", "/WEB-INF/lib/example-app-1.0.0.jar") - assert.Equal(t, "app.war:WEB-INF/lib/example-app-1.0.0.jar:com.example.other:other-lib", - virtualPathOf(t, pkgs, "other-lib")) - }) -} - // makeTestZip builds a zip in memory so the parser can be exercised without the java-toolchain-built // fixtures. func makeTestZip(t *testing.T, entries map[string][]byte) []byte { diff --git a/syft/pkg/cataloger/java/archive_source_test.go b/syft/pkg/cataloger/java/archive_source_test.go index fbfcb31cf..11e206603 100644 --- a/syft/pkg/cataloger/java/archive_source_test.go +++ b/syft/pkg/cataloger/java/archive_source_test.go @@ -51,7 +51,7 @@ func Test_archiveContentsCataloger_agreesWithTheArchiveReadingCataloger(t *testi jarLocations, err := archiveResolver.FilesByGlob("**/*.jar") require.NoError(t, err) require.Len(t, jarLocations, 1) - archivePath := jarLocations[0].Path() + archivePath := jarLocations[0].FullNestedPath() digests := digestsOfFile(t, jarPath) trav := &archive.Traversal{ @@ -65,22 +65,6 @@ func Test_archiveContentsCataloger_agreesWithTheArchiveReadingCataloger(t *testi assert.Equal(t, summarize(fromArchive), summarize(fromContents)) } -func Test_archiveContentsCataloger_reportsNothingOutsideAnArchive(t *testing.T) { - // with the archive cataloger task owning extraction, every archive comes back with a traversal - // naming it. Outside one there is nothing to describe: opening archive files here as well would - // catalog each of them a second time. - dir := t.TempDir() - jarPath := writeJar(t, dir, "example-lib-1.2.3.jar", jarEntries()) - require.FileExists(t, jarPath) - resolver, err := fileresolver.NewFromDirectory(dir, "") - require.NoError(t, err) - - ctx := archive.WithNestedCataloging(context.Background()) - pkgs, _, err := NewArchiveCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver) - require.NoError(t, err) - assert.Empty(t, pkgs) -} - func Test_archiveContentsCataloger_reportsNothingForAnArchiveThatIsNotJava(t *testing.T) { // a .zip is an archive and its contents are not a jar's; the archive-reading cataloger draws that // line with its globs and this one draws it with the same list diff --git a/syft/pkg/cataloger/java/cataloger.go b/syft/pkg/cataloger/java/cataloger.go index 88170be4f..d2b437e0e 100644 --- a/syft/pkg/cataloger/java/cataloger.go +++ b/syft/pkg/cataloger/java/cataloger.go @@ -73,7 +73,8 @@ func (c *archiveCataloger) Catalog(ctx context.Context, resolver file.Resolver) // catalogExtracted reports the java package of the archive whose contents the resolver holds, if it // is a java archive. Archives nested inside it are left to the archive cataloger task. func (c *archiveCataloger) catalogExtracted(ctx context.Context, trav *archive.Traversal, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { - if !isJavaArchiveName(trav.Location.Path()) { + // by the name it was reached by: a content-addressed jar is named like one only by its link + if !isJavaArchiveName(trav.Location.AccessPath) { return nil, nil, nil } diff --git a/syft/pkg/cataloger/java/graalvm_native_image_cataloger.go b/syft/pkg/cataloger/java/graalvm_native_image_cataloger.go index ab0845360..65a8648bc 100644 --- a/syft/pkg/cataloger/java/graalvm_native_image_cataloger.go +++ b/syft/pkg/cataloger/java/graalvm_native_image_cataloger.go @@ -586,7 +586,7 @@ func fetchPkgs(reader unionreader.UnionReader, location file.Location) ([]pkg.Pa log.Debugf("failed to open the java native-image binary: %v", err) return nil, nil } - filename := location.RealPath + filename := location.FullNestedPath() for _, r := range readers { for _, makeNativeImage := range imageFormats { ni, err := makeNativeImage(filename, r) @@ -645,7 +645,7 @@ func processLocation(location file.Location, resolver file.Resolver) ([]pkg.Pack log.Debugf("error opening file: %v", err) return nil, nil, nil } - defer internal.CloseAndLogError(readerCloser, location.RealPath) + defer internal.CloseAndLogError(readerCloser, location.FullNestedPath()) reader, err := unionreader.GetUnionReader(readerCloser) if err != nil { diff --git a/syft/pkg/cataloger/java/internal/maven/resolver.go b/syft/pkg/cataloger/java/internal/maven/resolver.go index b7d3d8ced..67f3b25bb 100644 --- a/syft/pkg/cataloger/java/internal/maven/resolver.go +++ b/syft/pkg/cataloger/java/internal/maven/resolver.go @@ -668,7 +668,7 @@ func (r *Resolver) findParentPomByRelativePath(ctx context.Context, pom *Project Debug("unable to get contents of parent pom by relative path") return nil } - defer internal.CloseAndLogError(parentContents, parentLocation.RealPath) + defer internal.CloseAndLogError(parentContents, parentLocation.FullNestedPath()) parentPom, err := ParsePomXML(parentContents) if err != nil || parentPom == nil { log.WithFields("error", err, "mavenID", r.resolveID(ctx, resolvingProperties, pom), "parentID", parentID, "parentLocation", parentLocation). diff --git a/syft/pkg/cataloger/java/parse_jvm_release.go b/syft/pkg/cataloger/java/parse_jvm_release.go index c5460ed3b..4a639695f 100644 --- a/syft/pkg/cataloger/java/parse_jvm_release.go +++ b/syft/pkg/cataloger/java/parse_jvm_release.go @@ -62,7 +62,7 @@ type jvmCpeInfo struct { func parseJVMRelease(_ context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { ri, err := parseJvmReleaseInfo(reader) if err != nil { - return nil, nil, fmt.Errorf("unable to parse JVM release info %q: %w", reader.Path(), err) + return nil, nil, fmt.Errorf("unable to parse JVM release info %q: %w", reader.FullNestedPath(), err) } if ri == nil { @@ -87,7 +87,11 @@ func parseJVMRelease(_ context.Context, resolver file.Resolver, _ *generic.Envir // the reason we use the reference to get the real path is in cases where the file cataloger is involved // (thus the real path is not available except for in the original file reference). This is important // since the path is critical for distinguishing between different JVM vendors. - vendor, product := jvmPrimaryVendorProduct(ri, string(reader.Reference().RealPath), hasJdk) + jdkPath := string(reader.Reference().RealPath) + if jdkPath == "" { + jdkPath = reader.Path() + } + vendor, product := jvmPrimaryVendorProduct(ri, jdkPath, hasJdk) p := pkg.Package{ Name: product, diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index 5b5e06070..92726ef62 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -94,7 +94,7 @@ func readPomFromLocation(fileResolver file.Resolver, pomLocation file.Location) if err != nil { return nil, err } - defer internal.CloseAndLogError(contents, pomLocation.RealPath) + defer internal.CloseAndLogError(contents, pomLocation.FullNestedPath()) return maven.ParsePomXML(contents) } @@ -313,7 +313,7 @@ func pomParent(ctx context.Context, r *maven.Resolver, pom *maven.Project) *pkg. // src/test/resources, a tarball of a source tree) nothing else reads these poms, so they are cataloged. func isArchiveMetaPom(ctx context.Context, location file.Location) bool { trav := archive.TraversalFromContext(ctx) - if trav == nil || !isJavaArchiveName(trav.Location.Path()) { + if trav == nil || !isJavaArchiveName(trav.Location.AccessPath) { return false } return strings.Contains(location.Path(), "META-INF/maven/") diff --git a/syft/pkg/cataloger/java/zip_wrapped_archive_parser.go b/syft/pkg/cataloger/java/zip_wrapped_archive_parser.go index e5a7b6aec..54cc310b5 100644 --- a/syft/pkg/cataloger/java/zip_wrapped_archive_parser.go +++ b/syft/pkg/cataloger/java/zip_wrapped_archive_parser.go @@ -35,7 +35,7 @@ func (gzp genericZipWrappedJavaArchiveParser) parseZipWrappedJavaArchive(ctx con if td == nil { return nil, nil, fmt.Errorf("no temp dir factory in context") } - contentPath, archivePath, cleanupFn, err := saveArchiveToTmp(td, reader.Path(), reader) + contentPath, archivePath, cleanupFn, err := saveArchiveToTmp(td, reader.FullNestedPath(), reader) // note: even on error, we should always run cleanup functions defer cleanupFn() if err != nil { diff --git a/syft/pkg/cataloger/javascript/package.go b/syft/pkg/cataloger/javascript/package.go index 085f186dd..11ca8431f 100644 --- a/syft/pkg/cataloger/javascript/package.go +++ b/syft/pkg/cataloger/javascript/package.go @@ -276,7 +276,7 @@ func addLicenses(name string, resolver file.Resolver, location file.Location) (a return allLicenses } - dir := path.Dir(location.RealPath) + dir := path.Dir(location.LocalPath()) pkgPath := []string{dir, "node_modules"} pkgPath = append(pkgPath, strings.Split(name, "/")...) pkgPath = append(pkgPath, "package.json") @@ -308,7 +308,7 @@ func parseLicensesFromLocation(l file.Location, resolver file.Resolver, pkgFile log.Debugf("error getting file content reader for %s: %v", pkgFile, err) return nil, err } - defer internal.CloseAndLogError(contentReader, l.RealPath) + defer internal.CloseAndLogError(contentReader, l.FullNestedPath()) var pkgJSON packageJSON err = json.NewDecoder(contentReader).Decode(&pkgJSON) diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock.go b/syft/pkg/cataloger/javascript/parse_yarn_lock.go index 7e65acf68..12638e1d8 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock.go @@ -93,20 +93,20 @@ func readPackageJSONDeps(resolver file.Resolver, lockfileLocation file.Location) pkgJSONLocation := resolver.RelativeFileByPath(lockfileLocation, "package.json") if pkgJSONLocation == nil { - log.WithFields("lockfile", lockfileLocation.RealPath).Debug("could not find package.json for dev dependency detection") + log.WithFields("lockfile", lockfileLocation.FullNestedPath()).Debug("could not find package.json for dev dependency detection") return prod, dev } reader, err := resolver.FileContentsByLocation(*pkgJSONLocation) if err != nil { - log.WithFields("location", pkgJSONLocation.RealPath, "error", err).Debug("could not read package.json for dev dependency detection") + log.WithFields("location", pkgJSONLocation.FullNestedPath(), "error", err).Debug("could not read package.json for dev dependency detection") return prod, dev } - defer internal.CloseAndLogError(reader, pkgJSONLocation.RealPath) + defer internal.CloseAndLogError(reader, pkgJSONLocation.FullNestedPath()) var pkgJSON packageJSON if err := json.NewDecoder(reader).Decode(&pkgJSON); err != nil { - log.WithFields("location", pkgJSONLocation.RealPath, "error", err).Debug("could not parse package.json for dev dependency detection") + log.WithFields("location", pkgJSONLocation.FullNestedPath(), "error", err).Debug("could not parse package.json for dev dependency detection") return prod, dev } diff --git a/syft/pkg/cataloger/kernel/parse_linux_kernel_module_file.go b/syft/pkg/cataloger/kernel/parse_linux_kernel_module_file.go index de92128a2..a323aba6a 100644 --- a/syft/pkg/cataloger/kernel/parse_linux_kernel_module_file.go +++ b/syft/pkg/cataloger/kernel/parse_linux_kernel_module_file.go @@ -34,9 +34,9 @@ func parseLinuxKernelModuleFile(ctx context.Context, _ file.Resolver, _ *generic return nil, nil, fmt.Errorf("unable to get union reader for file: %w", err) } - moduleReader, err := decompressedModuleReader(ctx, reader.RealPath, unionReader) + moduleReader, err := decompressedModuleReader(ctx, reader.FullNestedPath(), unionReader) if err != nil { - return nil, nil, fmt.Errorf("unable to decompress kernel module %q: %w", reader.RealPath, err) + return nil, nil, fmt.Errorf("unable to decompress kernel module %q: %w", reader.FullNestedPath(), err) } defer moduleReader.Close() @@ -48,7 +48,7 @@ func parseLinuxKernelModuleFile(ctx context.Context, _ file.Resolver, _ *generic return nil, nil, nil } - metadata.Path = reader.RealPath + metadata.Path = reader.LocalPath() return []pkg.Package{ newLinuxKernelModulePackage( diff --git a/syft/pkg/cataloger/lua/parse_rockspec.go b/syft/pkg/cataloger/lua/parse_rockspec.go index 82646aa67..847a476f7 100644 --- a/syft/pkg/cataloger/lua/parse_rockspec.go +++ b/syft/pkg/cataloger/lua/parse_rockspec.go @@ -75,7 +75,7 @@ func parseRockspec(ctx context.Context, resolver file.Resolver, _ *generic.Envir } if name == "" { - log.WithFields("path", reader.Path()).Trace("rockspec has no package name, skipping") + log.WithFields("path", reader.FullNestedPath()).Trace("rockspec has no package name, skipping") return nil, nil, nil } diff --git a/syft/pkg/cataloger/nix/db_cataloger.go b/syft/pkg/cataloger/nix/db_cataloger.go index 825895b69..09f9581c7 100644 --- a/syft/pkg/cataloger/nix/db_cataloger.go +++ b/syft/pkg/cataloger/nix/db_cataloger.go @@ -62,7 +62,7 @@ func (c dbCataloger) catalog(ctx context.Context, resolver file.Resolver) ([]pkg for _, dbLoc := range dbLocs { parser, schema := c.selectDBParser(dbLoc, resolver) if parser == nil { - errs = unknown.Appendf(errs, dbLoc.Coordinates, "unsupported Nix database schema for schema=%d at %q", schema, dbLoc.RealPath) + errs = unknown.Appendf(errs, dbLoc.Coordinates, "unsupported Nix database schema for schema=%d at %q", schema, dbLoc.FullNestedPath()) continue } @@ -80,28 +80,28 @@ func (c dbCataloger) catalog(ctx context.Context, resolver file.Resolver) ([]pkg } func (c dbCataloger) selectDBParser(dbLocation file.Location, resolver file.Resolver) (dbProcessor, int) { - loc := resolver.RelativeFileByPath(dbLocation, path.Join(path.Dir(dbLocation.RealPath), "schema")) + loc := resolver.RelativeFileByPath(dbLocation, path.Join(path.Dir(dbLocation.LocalPath()), "schema")) if loc == nil { - log.WithFields("path", dbLocation.RealPath).Tracef("failed to detect Nix database schema, assuming %d", defaultSchema) + log.WithFields("path", dbLocation.FullNestedPath()).Tracef("failed to detect Nix database schema, assuming %d", defaultSchema) return c.schemaProcessor[defaultSchema], 0 } schemaContents, err := resolver.FileContentsByLocation(*loc) - defer internal.CloseAndLogError(schemaContents, loc.RealPath) + defer internal.CloseAndLogError(schemaContents, loc.FullNestedPath()) if err != nil { - log.WithFields("path", loc.RealPath).Tracef("failed to open Nix database schema file, assuming %d", defaultSchema) + log.WithFields("path", loc.FullNestedPath()).Tracef("failed to open Nix database schema file, assuming %d", defaultSchema) return c.schemaProcessor[defaultSchema], 0 } scanner := bufio.NewScanner(schemaContents) if !scanner.Scan() { - log.WithFields("path", loc.RealPath, "error", scanner.Err()).Tracef("failed to read Nix database schema file, assuming %d", defaultSchema) + log.WithFields("path", loc.FullNestedPath(), "error", scanner.Err()).Tracef("failed to read Nix database schema file, assuming %d", defaultSchema) return c.schemaProcessor[defaultSchema], 0 } schema, err := strconv.Atoi(strings.TrimSpace(scanner.Text())) if err != nil { - log.WithFields("path", loc.RealPath).Tracef("failed to parse Nix database schema file, assuming %d", defaultSchema) + log.WithFields("path", loc.FullNestedPath()).Tracef("failed to parse Nix database schema file, assuming %d", defaultSchema) return c.schemaProcessor[defaultSchema], 0 } @@ -113,7 +113,7 @@ func (c dbCataloger) selectDBParser(dbLocation file.Location, resolver file.Reso schema = defaultSchema } processor = c.schemaProcessor[closestSchema] - log.WithFields("path", loc.RealPath).Tracef("unsupported Nix database schema (%d), treating as closest available schema (%d)", schema, closestSchema) + log.WithFields("path", loc.FullNestedPath()).Tracef("unsupported Nix database schema (%d), treating as closest available schema (%d)", schema, closestSchema) } return processor, schema diff --git a/syft/pkg/cataloger/nix/db_cataloger_v10.go b/syft/pkg/cataloger/nix/db_cataloger_v10.go index 849faffa4..3d8850150 100644 --- a/syft/pkg/cataloger/nix/db_cataloger_v10.go +++ b/syft/pkg/cataloger/nix/db_cataloger_v10.go @@ -19,7 +19,7 @@ var _ dbProcessor = processV10DB func processV10DB(ctx context.Context, config Config, dbLocation file.Location, resolver file.Resolver, catalogerName string) ([]pkg.Package, []artifact.Relationship, error) { dbContents, err := resolver.FileContentsByLocation(dbLocation) - defer internal.CloseAndLogError(dbContents, dbLocation.RealPath) + defer internal.CloseAndLogError(dbContents, dbLocation.FullNestedPath()) if err != nil { return nil, nil, fmt.Errorf("unable to read Nix database: %w", err) } diff --git a/syft/pkg/cataloger/nix/derivation.go b/syft/pkg/cataloger/nix/derivation.go index 97d163c9a..12b740497 100644 --- a/syft/pkg/cataloger/nix/derivation.go +++ b/syft/pkg/cataloger/nix/derivation.go @@ -34,7 +34,7 @@ func newDerivationFromLocation(loc file.Location, resolver file.Resolver) (*deri if err != nil { return nil, fmt.Errorf("failed to read derivation: %w", err) } - defer internal.CloseAndLogError(reader, loc.RealPath) + defer internal.CloseAndLogError(reader, loc.FullNestedPath()) d, err := derivation.ReadDerivation(reader) if err != nil { @@ -60,12 +60,12 @@ func newDerivations() *derivations { } func (c *derivations) add(df derivationFile) { - c.derivationsByDrvPath[df.Location.RealPath] = df + c.derivationsByDrvPath[df.Location.LocalPath()] = df for _, output := range df.Outputs { if output == nil || output.Path == "" { continue } - c.drvPathByOutputPath[output.Path] = df.Location.RealPath + c.drvPathByOutputPath[output.Path] = df.Location.LocalPath() } } diff --git a/syft/pkg/cataloger/nix/package.go b/syft/pkg/cataloger/nix/package.go index 36e8294ed..b214383a5 100644 --- a/syft/pkg/cataloger/nix/package.go +++ b/syft/pkg/cataloger/nix/package.go @@ -21,7 +21,7 @@ func newNixStorePackage(pp nixStorePackage, catalogerName string) pkg.Package { var derivationPath string if pp.derivationFile != nil { - derivationPath = pp.derivationFile.Location.RealPath + derivationPath = pp.derivationFile.Location.LocalPath() locations.Add(pp.derivationFile.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.SupportingEvidenceAnnotation)) } @@ -97,7 +97,7 @@ func newDerivation(df *derivationFile) pkg.NixDerivation { sort.Strings(sources) return pkg.NixDerivation{ - Path: df.Location.RealPath, + Path: df.Location.LocalPath(), System: df.Platform, InputDerivations: inputDerivations, InputSources: sources, diff --git a/syft/pkg/cataloger/nix/store_cataloger.go b/syft/pkg/cataloger/nix/store_cataloger.go index e239330ef..c212b9ffd 100644 --- a/syft/pkg/cataloger/nix/store_cataloger.go +++ b/syft/pkg/cataloger/nix/store_cataloger.go @@ -75,7 +75,7 @@ func (c storeCataloger) finalizeStorePackages(ctx context.Context, resolver file p := newNixStorePackage(pp, c.name) p = licenses.RelativeToPackage(ctx, resolver, p) pkgs = append(pkgs, p) - pkgByStorePath[pp.Location.RealPath] = p + pkgByStorePath[pp.Location.LocalPath()] = p } var relationships []artifact.Relationship @@ -117,7 +117,7 @@ func (c storeCataloger) findDerivationsFromStore(resolver file.Resolver, pkgProt // attach derivations to the packages they belong to for i := range pkgPrototypes { p := &pkgPrototypes[i] - p.derivationFile = dvs.findDerivationForOutputPath(p.Location.RealPath) + p.derivationFile = dvs.findDerivationForOutputPath(p.Location.LocalPath()) } return dvs, errs @@ -130,16 +130,16 @@ func (c storeCataloger) findPackagesFromStore(ctx context.Context, resolver file ctx, cancel := context.WithCancel(ctx) defer cancel() for location := range resolver.AllLocations(ctx) { - matchesStorePath, err := doublestar.Match("**/nix/store/*", location.RealPath) + matchesStorePath, err := doublestar.Match("**/nix/store/*", location.LocalPath()) if err != nil { return nil, fmt.Errorf("failed to match nix store path: %w", err) } - parentStorePath := findParentNixStorePath(location.RealPath) + parentStorePath := findParentNixStorePath(location.LocalPath()) if c.config.CaptureOwnedFiles && parentStorePath != "" { fileInfo, err := resolver.FileMetadataByLocation(location) if err != nil { - log.WithFields("path", location.RealPath).Trace("failed to get file metadata") + log.WithFields("path", location.FullNestedPath()).Trace("failed to get file metadata") continue } @@ -159,7 +159,7 @@ func (c storeCataloger) findPackagesFromStore(ctx context.Context, resolver file continue } - storePath := parseNixStorePath(location.RealPath) + storePath := parseNixStorePath(location.LocalPath()) if storePath == nil || !storePath.isValidPackage() { continue @@ -178,7 +178,7 @@ func (c storeCataloger) findPackagesFromStore(ctx context.Context, resolver file log.WithFields("package", p.nixStorePath.Name).Debug("nix package has no evidence locations associated") continue } - parentStorePath := p.Location.RealPath + parentStorePath := p.Location.LocalPath() files, ok := filesByStorePath[parentStorePath] if !ok { log.WithFields("path", parentStorePath, "nix-store-path", parentStorePath).Debug("found a nix store file for a non-existent package") @@ -193,7 +193,7 @@ func (c storeCataloger) findPackagesFromStore(ctx context.Context, resolver file func filePaths(files []file.Location) []string { var relativePaths []string for _, f := range files { - relativePaths = append(relativePaths, f.RealPath) + relativePaths = append(relativePaths, f.LocalPath()) } return relativePaths } diff --git a/syft/pkg/cataloger/perl/meta.go b/syft/pkg/cataloger/perl/meta.go index a4ca5f5bd..2d9a29dae 100644 --- a/syft/pkg/cataloger/perl/meta.go +++ b/syft/pkg/cataloger/perl/meta.go @@ -145,7 +145,7 @@ func decode(resolver file.Resolver, location file.Location, into func(io.Reader, if err != nil { return nil, err } - defer internal.CloseAndLogError(reader, location.Path()) + defer internal.CloseAndLogError(reader, location.FullNestedPath()) var m cpanMeta if err := into(reader, &m); err != nil { diff --git a/syft/pkg/cataloger/perl/parse_install_json.go b/syft/pkg/cataloger/perl/parse_install_json.go index 15e5e2abb..937597f34 100644 --- a/syft/pkg/cataloger/perl/parse_install_json.go +++ b/syft/pkg/cataloger/perl/parse_install_json.go @@ -28,13 +28,13 @@ type installJSON struct { func parseInstallJSON(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { var doc installJSON if err := json.NewDecoder(reader).Decode(&doc); err != nil { - log.WithFields("path", reader.Path(), "error", err).Debug("unable to parse CPAN install.json") + log.WithFields("path", reader.FullNestedPath(), "error", err).Debug("unable to parse CPAN install.json") return nil, nil, nil } name, version := distributionNameVersion(doc) if name == "" || version == "" { - log.WithFields("path", reader.Path()).Debug("CPAN install.json is missing a distribution name or version") + log.WithFields("path", reader.FullNestedPath()).Debug("CPAN install.json is missing a distribution name or version") return nil, nil, nil } diff --git a/syft/pkg/cataloger/perl/parse_packlist.go b/syft/pkg/cataloger/perl/parse_packlist.go index 22cb50240..ea00a5c6e 100644 --- a/syft/pkg/cataloger/perl/parse_packlist.go +++ b/syft/pkg/cataloger/perl/parse_packlist.go @@ -124,7 +124,7 @@ func readVersionFromPM(resolver file.Resolver, location file.Location) string { if err != nil { return "" } - defer internal.CloseAndLogError(reader, location.Path()) + defer internal.CloseAndLogError(reader, location.FullNestedPath()) return versionFromPM(reader) } diff --git a/syft/pkg/cataloger/perl/parse_release_meta.go b/syft/pkg/cataloger/perl/parse_release_meta.go index 42224a842..185f36fbe 100644 --- a/syft/pkg/cataloger/perl/parse_release_meta.go +++ b/syft/pkg/cataloger/perl/parse_release_meta.go @@ -42,12 +42,12 @@ func parseReleaseMeta(ctx context.Context, resolver file.Resolver, _ *generic.En if err != nil { // a META.yml a strict parser rejects is routine rather than exceptional, so this skips the // directory and never fails the scan - log.WithFields("path", reader.Path(), "error", err).Debug("unable to parse CPAN meta file") + log.WithFields("path", reader.FullNestedPath(), "error", err).Debug("unable to parse CPAN meta file") return nil, nil, nil } if m.Name == "" || m.Version == "" { - log.WithFields("path", reader.Path()).Debug("CPAN meta file is missing a name or version") + log.WithFields("path", reader.FullNestedPath()).Debug("CPAN meta file is missing a name or version") return nil, nil, nil } diff --git a/syft/pkg/cataloger/perl/perllocal.go b/syft/pkg/cataloger/perl/perllocal.go index 21d892bc4..d46c62613 100644 --- a/syft/pkg/cataloger/perl/perllocal.go +++ b/syft/pkg/cataloger/perl/perllocal.go @@ -176,10 +176,10 @@ func perllocalLocations(resolver file.Resolver) []file.Location { func readPerllocal(resolver file.Resolver, location file.Location) []perllocalEntry { reader, err := resolver.FileContentsByLocation(location) if err != nil { - log.WithFields("path", location.Path(), "error", err).Debug("unable to read perllocal.pod") + log.WithFields("path", location.FullNestedPath(), "error", err).Debug("unable to read perllocal.pod") return nil } - defer internal.CloseAndLogError(reader, location.Path()) + defer internal.CloseAndLogError(reader, location.FullNestedPath()) return parsePerllocal(reader) } diff --git a/syft/pkg/cataloger/perl/perllocal_test.go b/syft/pkg/cataloger/perl/perllocal_test.go index ebf159a34..93f5c399e 100644 --- a/syft/pkg/cataloger/perl/perllocal_test.go +++ b/syft/pkg/cataloger/perl/perllocal_test.go @@ -164,7 +164,7 @@ func Test_perllocalVersion(t *testing.T) { return } require.NotNil(t, evidence) - assert.Equal(t, "/usr/local/lib/perl5/5.40.4/aarch64-linux-gnu/perllocal.pod", evidence.Path()) + assert.Equal(t, "/usr/local/lib/perl5/5.40.4/aarch64-linux-gnu/perllocal.pod", evidence.FullNestedPath()) }) } } diff --git a/syft/pkg/cataloger/perl/processor.go b/syft/pkg/cataloger/perl/processor.go index ee871e516..068ed7c08 100644 --- a/syft/pkg/cataloger/perl/processor.go +++ b/syft/pkg/cataloger/perl/processor.go @@ -212,7 +212,7 @@ func providedModules(p pkg.Package) []string { // belong in relationships and not in the metadata struct. func prereqModules(resolver file.Resolver, p pkg.Package) []string { for _, location := range p.Locations.ToSlice() { - if path.Base(location.Path()) != "MYMETA.json" { + if path.Base(location.FullNestedPath()) != "MYMETA.json" { continue } m, err := decodeMeta(resolver, location) diff --git a/syft/pkg/cataloger/php/interpreter_cataloger.go b/syft/pkg/cataloger/php/interpreter_cataloger.go index f9022d358..4cea616a2 100644 --- a/syft/pkg/cataloger/php/interpreter_cataloger.go +++ b/syft/pkg/cataloger/php/interpreter_cataloger.go @@ -182,12 +182,12 @@ func (p interpreterCataloger) catalogExtensions(resolver file.Resolver) ([]pkg.P func (p interpreterCataloger) catalogExtension(resolver file.Resolver, location file.Location) ([]pkg.Package, error) { reader, err := resolver.FileContentsByLocation(location) - defer internal.CloseAndLogError(reader, location.RealPath) + defer internal.CloseAndLogError(reader, location.FullNestedPath()) if err != nil { return nil, unknown.ProcessPathErrors(err) } - name, cls := p.getClassifier(location.RealPath) + name, cls := p.getClassifier(location.FullNestedPath()) if name == "" || cls == nil { return nil, nil } diff --git a/syft/pkg/cataloger/python/dependency.go b/syft/pkg/cataloger/python/dependency.go index 29671ed59..336cd5921 100644 --- a/syft/pkg/cataloger/python/dependency.go +++ b/syft/pkg/cataloger/python/dependency.go @@ -287,7 +287,7 @@ func collectPackages(pkgsBySitePackageAndName map[string]map[string]pkg.Package, func deriveSitePackageDir(p pkg.Package) string { for _, l := range packagePrimaryLocations(p) { - sitePackageDir := extractSitePackageDir(l.RealPath) + sitePackageDir := extractSitePackageDir(l.LocalPath()) if sitePackageDir != "" { return sitePackageDir } diff --git a/syft/pkg/cataloger/python/parse_requirements.go b/syft/pkg/cataloger/python/parse_requirements.go index 6622496fd..ed3536d0f 100644 --- a/syft/pkg/cataloger/python/parse_requirements.go +++ b/syft/pkg/cataloger/python/parse_requirements.go @@ -133,7 +133,7 @@ func (rp requirementsParser) parseRequirementsTxt(ctx context.Context, _ file.Re req := newRequirement(line) if req == nil { - log.WithFields("path", reader.RealPath, "line", line).Debug("unable to parse requirements.txt line") + log.WithFields("path", reader.FullNestedPath(), "line", line).Debug("unable to parse requirements.txt line") errs = unknown.Appendf(errs, reader, "unable to parse requirements.txt line: %q", line) continue } @@ -142,7 +142,7 @@ func (rp requirementsParser) parseRequirementsTxt(ctx context.Context, _ file.Re version := parseVersion(req.VersionConstraint, rp.cfg.GuessUnpinnedRequirements) if version == "" { - log.WithFields("path", reader.RealPath, "line", line).Trace("unable to determine package version in requirements.txt line") + log.WithFields("path", reader.FullNestedPath(), "line", line).Trace("unable to determine package version in requirements.txt line") errs = unknown.Appendf(errs, reader, "unable to determine package version in requirements.txt line: %q", line) continue } diff --git a/syft/pkg/cataloger/python/parse_setup.go b/syft/pkg/cataloger/python/parse_setup.go index d8cff06de..b487e7de3 100644 --- a/syft/pkg/cataloger/python/parse_setup.go +++ b/syft/pkg/cataloger/python/parse_setup.go @@ -108,7 +108,7 @@ func (sp setupFileParser) validateAndCreatePackage(ctx context.Context, name, ve if name == "" || version == "" { // the raw line is not logged since this runs once per match, which is quadratic on a long line - log.WithFields("path", reader.RealPath, "name", name, "version", version).Debug("unable to parse package in setup.py") + log.WithFields("path", reader.FullNestedPath(), "name", name, "version", version).Debug("unable to parse package in setup.py") return pkg.Package{}, false } diff --git a/syft/pkg/cataloger/python/parse_wheel_egg.go b/syft/pkg/cataloger/python/parse_wheel_egg.go index 33d2d0161..7a1f17672 100644 --- a/syft/pkg/cataloger/python/parse_wheel_egg.go +++ b/syft/pkg/cataloger/python/parse_wheel_egg.go @@ -52,7 +52,7 @@ func fetchInstalledFiles(resolver file.Resolver, metadataLocation file.Location, // to reconcile the installed-files.txt path to the same layer (or the next adjacent lower layer). // find the installed-files.txt file relative to the directory where the METADATA file resides (in path AND layer structure) - installedFilesPath := path.Join(path.Dir(metadataLocation.RealPath), "installed-files.txt") + installedFilesPath := path.Join(path.Dir(metadataLocation.LocalPath()), "installed-files.txt") installedFilesRef := resolver.RelativeFileByPath(metadataLocation, installedFilesPath) if installedFilesRef != nil { @@ -65,7 +65,7 @@ func fetchInstalledFiles(resolver file.Resolver, metadataLocation file.Location, defer internal.CloseAndLogError(installedFilesContents, installedFilesPath) // parse the installed-files contents - installedFiles, err := parseInstalledFiles(installedFilesContents, metadataLocation.RealPath, sitePackagesRootPath) + installedFiles, err := parseInstalledFiles(installedFilesContents, metadataLocation.LocalPath(), sitePackagesRootPath) if err != nil { retErr = unknown.Newf(*installedFilesRef, "unable to parse installed-files.txt for python package: %w", retErr) } @@ -82,7 +82,7 @@ func fetchRecordFiles(resolver file.Resolver, metadataLocation file.Location) (f // to reconcile the RECORD path to the same layer (or the next adjacent lower layer). // find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure) - recordPath := path.Join(path.Dir(metadataLocation.RealPath), "RECORD") + recordPath := path.Join(path.Dir(metadataLocation.LocalPath()), "RECORD") recordRef := resolver.RelativeFileByPath(metadataLocation, recordPath) if recordRef != nil { @@ -106,7 +106,7 @@ func fetchRecordFiles(resolver file.Resolver, metadataLocation file.Location) (f // fetchTopLevelPackages finds a corresponding top_level.txt file for the given python package metadata file and returns the set of package names contained. func fetchTopLevelPackages(resolver file.Resolver, metadataLocation file.Location) (pkgs []string, sources []file.Location, err error) { // a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages - parentDir := path.Dir(metadataLocation.RealPath) + parentDir := path.Dir(metadataLocation.LocalPath()) topLevelPath := path.Join(parentDir, "top_level.txt") topLevelLocation := resolver.RelativeFileByPath(metadataLocation, topLevelPath) @@ -156,7 +156,7 @@ type vcsInfo struct { } func fetchDirectURLData(resolver file.Resolver, metadataLocation file.Location) (d *pkg.PythonDirectURLOriginInfo, sources []file.Location, err error) { - parentDir := path.Dir(metadataLocation.RealPath) + parentDir := path.Dir(metadataLocation.LocalPath()) directURLPath := path.Join(parentDir, "direct_url.json") directURLLocation := resolver.RelativeFileByPath(metadataLocation, directURLPath) diff --git a/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go b/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go index 05c49f5f4..8a0c10cb6 100644 --- a/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go +++ b/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go @@ -104,13 +104,13 @@ func extractRFC5322Fields(locationReader file.LocationReadCloser) (map[string]an fields[key] = handleSingleOrMultiField(fields[key], val) } else { - log.Debugf("cannot parse field from path: %q from line: %q", locationReader.Path(), line) + log.Debugf("cannot parse field from path: %q from line: %q", locationReader.FullNestedPath(), line) } } } if err := scanner.Err(); err != nil { // the caller discards everything on error, so keep the partial fields and log instead - log.WithFields("path", locationReader.Path(), "error", err).Debug("failed to fully read python wheel/egg metadata") + log.WithFields("path", locationReader.FullNestedPath(), "error", err).Debug("failed to fully read python wheel/egg metadata") } return fields, nil } diff --git a/syft/pkg/cataloger/python/virtual_env.go b/syft/pkg/cataloger/python/virtual_env.go index f6b48b602..d7d5b083c 100644 --- a/syft/pkg/cataloger/python/virtual_env.go +++ b/syft/pkg/cataloger/python/virtual_env.go @@ -97,7 +97,7 @@ func (v virtualEnvInfo) matchVirtualEnvSitePackagesPath(sitePackagePaths []strin sitePackagePathsSet := strset.New(sitePackagePaths...) // the parent directory of the venv config is the top-level directory of the virtualenv // e.g. /app/project1/venv/pyvenv.cfg -> /app/project1/venv - parent := strings.TrimLeft(path.Dir(v.Location.RealPath), "/") + parent := strings.TrimLeft(path.Dir(v.Location.LocalPath()), "/") // we are searchin for the site-packages directory within the virtualenv search := parent + "/lib/python" + v.majorMinorVersion() + "/site-packages" @@ -120,9 +120,9 @@ func (v virtualEnvInfo) matchVirtualEnvSitePackagesPath(sitePackagePaths []strin func parsePyvenvCfg(_ context.Context, resolver file.Resolver, location file.Location) (*virtualEnvInfo, error) { reader, err := resolver.FileContentsByLocation(location) if err != nil { - return nil, fmt.Errorf("unable to read file %q: %w", location.Path(), err) + return nil, fmt.Errorf("unable to read file %q: %w", location.FullNestedPath(), err) } - defer internal.CloseAndLogError(reader, location.Path()) + defer internal.CloseAndLogError(reader, location.FullNestedPath()) cfg, err := parsePyvenvCfgReader(file.NewLocationReadCloser(location, reader)) if err != nil { diff --git a/syft/pkg/cataloger/redhat/parse_rpm_archive.go b/syft/pkg/cataloger/redhat/parse_rpm_archive.go index 5013825b8..9215b305e 100644 --- a/syft/pkg/cataloger/redhat/parse_rpm_archive.go +++ b/syft/pkg/cataloger/redhat/parse_rpm_archive.go @@ -60,7 +60,7 @@ var hashLookup = map[uint8]string{ func parseRpmArchive(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { rpm, err := rpmutils.ReadRpm(reader) if err != nil { - return nil, nil, fmt.Errorf("RPM file found but unable to read: %s (%w)", reader.RealPath, err) + return nil, nil, fmt.Errorf("RPM file found but unable to read: %s (%w)", reader.FullNestedPath(), err) } nevra, err := rpm.Header.GetNEVRA() @@ -269,6 +269,6 @@ func parseEpoch(epoch string) *int { func logRpmArchiveErr(location file.Location, operation string, err error) { if err != nil { - log.WithFields("error", err, "operation", operation, "path", location.RealPath).Trace("unable to parse RPM archive") + log.WithFields("error", err, "operation", operation, "path", location.FullNestedPath()).Trace("unable to parse RPM archive") } } diff --git a/syft/pkg/cataloger/redhat/parse_rpm_db.go b/syft/pkg/cataloger/redhat/parse_rpm_db.go index 22da431b5..971911aeb 100644 --- a/syft/pkg/cataloger/redhat/parse_rpm_db.go +++ b/syft/pkg/cataloger/redhat/parse_rpm_db.go @@ -69,7 +69,7 @@ func parseRpmDB(ctx context.Context, resolver file.Resolver, env *generic.Enviro // This appears to be a holdover, where nowadays only the RSA Header is used. sigs, err := parseSignatures(strings.TrimSpace(entry.PGP), strings.TrimSpace(entry.RSAHeader)) if err != nil { - log.WithFields("error", err, "location", reader.RealPath, "pkg", fmt.Sprintf("%s@%s", entry.Name, entry.Version)).Trace("unable to parse signatures for package %s", entry.Name) + log.WithFields("error", err, "location", reader.FullNestedPath(), "pkg", fmt.Sprintf("%s@%s", entry.Name, entry.Version)).Trace("unable to parse signatures for package %s", entry.Name) sigs = nil } @@ -98,7 +98,7 @@ func parseRpmDB(ctx context.Context, resolver file.Resolver, env *generic.Enviro ) if !pkg.IsValid(&p) { - log.WithFields("location", reader.RealPath, "pkg", fmt.Sprintf("%s@%s", entry.Name, entry.Version)). + log.WithFields("location", reader.FullNestedPath(), "pkg", fmt.Sprintf("%s@%s", entry.Name, entry.Version)). Warn("ignoring invalid package found in RPM DB") errs = unknown.Appendf(errs, reader, "invalild package found; name: %s, version: %s", entry.Name, entry.Version) continue diff --git a/syft/pkg/cataloger/rust/parse_audit_binary.go b/syft/pkg/cataloger/rust/parse_audit_binary.go index e93bae25a..9d1357624 100644 --- a/syft/pkg/cataloger/rust/parse_audit_binary.go +++ b/syft/pkg/cataloger/rust/parse_audit_binary.go @@ -26,7 +26,7 @@ func parseAuditBinary(_ context.Context, _ file.Resolver, _ *generic.Environment return nil, nil, err } - infos, err := parseAuditBinaryEntry(unionReader, reader.RealPath) + infos, err := parseAuditBinaryEntry(unionReader, reader.FullNestedPath()) for _, versionInfo := range infos { auditPkgs, auditRelationships := processAuditVersionInfo(reader.Location, versionInfo) pkgs = append(pkgs, auditPkgs...) diff --git a/syft/pkg/cataloger/sbom/cataloger.go b/syft/pkg/cataloger/sbom/cataloger.go index 0ea8483cd..550464890 100644 --- a/syft/pkg/cataloger/sbom/cataloger.go +++ b/syft/pkg/cataloger/sbom/cataloger.go @@ -40,7 +40,7 @@ func NewCataloger() pkg.Cataloger { func parseSBOM(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { readSeeker, err := adaptToReadSeeker(reader) if err != nil { - return nil, nil, fmt.Errorf("unable to read SBOM file %q: %w", reader.RealPath, err) + return nil, nil, fmt.Errorf("unable to read SBOM file %q: %w", reader.FullNestedPath(), err) } s, _, _, err := format.Decode(readSeeker) if err != nil { @@ -48,7 +48,7 @@ func parseSBOM(_ context.Context, _ file.Resolver, _ *generic.Environment, reade } if s == nil { - log.WithFields("path", reader.RealPath).Trace("file is not an SBOM") + log.WithFields("path", reader.FullNestedPath()).Trace("file is not an SBOM") return nil, nil, nil } diff --git a/syft/pkg/cataloger/sbom/cataloger_test.go b/syft/pkg/cataloger/sbom/cataloger_test.go index 7617e9e9b..620d2e711 100644 --- a/syft/pkg/cataloger/sbom/cataloger_test.go +++ b/syft/pkg/cataloger/sbom/cataloger_test.go @@ -259,10 +259,7 @@ func Test_parseSBOM(t *testing.T) { apkgdbLocation := file.NewLocationSet(file.Location{ LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/lib/apk/db/installed", - FileSystemID: "sha256:e5e13b0c77cbb769548077189c3da2f0a764ceca06af49d8d558e759f5c232bd", - }, + Coordinates: file.NewCoordinates("/lib/apk/db/installed", "sha256:e5e13b0c77cbb769548077189c3da2f0a764ceca06af49d8d558e759f5c232bd"), }, }) @@ -349,10 +346,7 @@ func Test_parseSBOM(t *testing.T) { }, { From: libSSL, - To: file.Coordinates{ - RealPath: "/lib/libssl.so.1.1", - FileSystemID: "sha256:e5e13b0c77cbb769548077189c3da2f0a764ceca06af49d8d558e759f5c232bd", - }, + To: file.NewCoordinates("/lib/libssl.so.1.1", "sha256:e5e13b0c77cbb769548077189c3da2f0a764ceca06af49d8d558e759f5c232bd"), Type: artifact.ContainsRelationship, }, { @@ -362,18 +356,12 @@ func Test_parseSBOM(t *testing.T) { }, { From: baseLayout, - To: file.Coordinates{ - RealPath: "/etc/profile.d/color_prompt.sh.disabled", - FileSystemID: "sha256:e5e13b0c77cbb769548077189c3da2f0a764ceca06af49d8d558e759f5c232bd", - }, + To: file.NewCoordinates("/etc/profile.d/color_prompt.sh.disabled", "sha256:e5e13b0c77cbb769548077189c3da2f0a764ceca06af49d8d558e759f5c232bd"), Type: artifact.ContainsRelationship, }, { From: baseLayout, - To: file.Coordinates{ - RealPath: "/etc/modprobe.d/kms.conf", - FileSystemID: "sha256:e5e13b0c77cbb769548077189c3da2f0a764ceca06af49d8d558e759f5c232bd", - }, + To: file.NewCoordinates("/etc/modprobe.d/kms.conf", "sha256:e5e13b0c77cbb769548077189c3da2f0a764ceca06af49d8d558e759f5c232bd"), Type: artifact.ContainsRelationship, }, { @@ -386,9 +374,7 @@ func Test_parseSBOM(t *testing.T) { for _, p := range expectedPkgs { expectedRelationships = append(expectedRelationships, artifact.Relationship{ From: p, - To: file.Coordinates{ - RealPath: "sbom.syft.json", - }, + To: file.NewCoordinates("sbom.syft.json", ""), Type: artifact.DescribedByRelationship, }) } diff --git a/syft/pkg/cataloger/swipl/parse_pack.go b/syft/pkg/cataloger/swipl/parse_pack.go index f4fbe5d73..e50be797e 100644 --- a/syft/pkg/cataloger/swipl/parse_pack.go +++ b/syft/pkg/cataloger/swipl/parse_pack.go @@ -30,7 +30,7 @@ func parsePackPackage(ctx context.Context, resolver file.Resolver, _ *generic.En version := versionRe.FindSubmatch(data) if name == nil || version == nil { - log.Debugf("encountered pack.pl file without a name and/or version field, ignoring (path=%q)", reader.Path()) + log.WithFields("path", reader.FullNestedPath()).Debug("encountered pack.pl file without a name and/or version field, ignoring") return nil, nil, nil } diff --git a/syft/pkg/cataloger/terraform/parse_tf_lock.go b/syft/pkg/cataloger/terraform/parse_tf_lock.go index 8056c8e2b..064746577 100644 --- a/syft/pkg/cataloger/terraform/parse_tf_lock.go +++ b/syft/pkg/cataloger/terraform/parse_tf_lock.go @@ -25,7 +25,7 @@ func parseTerraformLock(_ context.Context, _ file.Resolver, _ *generic.Environme return nil, nil, fmt.Errorf("failed to read terraform lock file: %w", err) } - err = hclsimple.Decode(reader.RealPath, contents, nil, &lockFile) + err = hclsimple.Decode(reader.FullNestedPath(), contents, nil, &lockFile) if err != nil { return nil, nil, fmt.Errorf("failed to decode terraform lock file: %w", err) } diff --git a/syft/pkg/cataloger/wordpress/parse_plugin.go b/syft/pkg/cataloger/wordpress/parse_plugin.go index a3de5060c..0f776d3ee 100644 --- a/syft/pkg/cataloger/wordpress/parse_plugin.go +++ b/syft/pkg/cataloger/wordpress/parse_plugin.go @@ -47,7 +47,7 @@ func parseWordpressPluginFiles(ctx context.Context, resolver file.Resolver, _ *g // a file shorter than the buffer is expected, so only the header it does have is parsed n, err := io.ReadFull(reader, buffer) if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { - return nil, nil, fmt.Errorf("failed to read %s file: %w", reader.Path(), err) + return nil, nil, fmt.Errorf("failed to read %s file: %w", reader.FullNestedPath(), err) } fields := extractFields(string(buffer[:n])) @@ -56,7 +56,7 @@ func parseWordpressPluginFiles(ctx context.Context, resolver file.Resolver, _ *g version, versionOk := fields["version"] // get a plugin name from a plugin's directory name - pluginInstallDirectory := filepath.Base(filepath.Dir(reader.RealPath)) + pluginInstallDirectory := filepath.Base(filepath.Dir(reader.LocalPath())) if nameOk && name != "" && versionOk && version != "" { var metadata pluginData diff --git a/syft/pkg/collection.go b/syft/pkg/collection.go index 215e7eedf..b808edf56 100644 --- a/syft/pkg/collection.go +++ b/syft/pkg/collection.go @@ -61,7 +61,8 @@ func (c *Collection) Package(id artifact.ID) *Package { return &p } -// PackagesByPath returns all packages that were discovered from the given path. +// PackagesByPath returns all packages that were discovered from the given path: a file's full path or +// access path, or the top path of an archive, which answers with every package found within it. func (c *Collection) PackagesByPath(path string) []Package { c.lock.RLock() defer c.lock.RUnlock() @@ -158,19 +159,22 @@ func (c *Collection) addTypeToIndex(p Package) { func (c *Collection) addPathsToIndex(p Package) { // note: write lock must be held by caller - observedPaths := strset.New() - for _, l := range p.Locations.ToSlice() { - if l.RealPath != "" && !observedPaths.Has(l.RealPath) { - c.addPathToIndex(p.id, l.RealPath) - observedPaths.Add(l.RealPath) - } - if l.AccessPath != "" && l.RealPath != l.AccessPath && !observedPaths.Has(l.AccessPath) { - c.addPathToIndex(p.id, l.AccessPath) - observedPaths.Add(l.AccessPath) - } + for _, path := range indexedPaths(p) { + c.addPathToIndex(p.id, path) } } +// indexedPaths are the paths a package is found by: each location's full path and access path, and +// the top path of an archive it was found within. +func indexedPaths(p Package) []string { + paths := strset.New() + for _, l := range p.Locations.ToSlice() { + paths.Add(l.RealPath, l.AccessPath, l.FullNestedPath()) + } + paths.Remove("") + return paths.List() +} + func (c *Collection) addPathToIndex(id artifact.ID, path string) { // note: write lock must be held by caller @@ -223,16 +227,8 @@ func (c *Collection) deleteTypeFromIndex(p Package) { func (c *Collection) deletePathsFromIndex(p Package) { // note: write lock must be held by caller - observedPaths := strset.New() - for _, l := range p.Locations.ToSlice() { - if l.RealPath != "" && !observedPaths.Has(l.RealPath) { - c.deletePathFromIndex(p.id, l.RealPath) - observedPaths.Add(l.RealPath) - } - if l.AccessPath != "" && l.RealPath != l.AccessPath && !observedPaths.Has(l.AccessPath) { - c.deletePathFromIndex(p.id, l.AccessPath) - observedPaths.Add(l.AccessPath) - } + for _, path := range indexedPaths(p) { + c.deletePathFromIndex(p.id, path) } } diff --git a/syft/pkg/collection_test.go b/syft/pkg/collection_test.go index 0d8c510ec..e1f08fd91 100644 --- a/syft/pkg/collection_test.go +++ b/syft/pkg/collection_test.go @@ -392,10 +392,7 @@ func TestCatalog_MergeRecords(t *testing.T) { CPEs: []cpe.CPE{cpe.Must("cpe:2.3:a:package:1:1:*:*:*:*:*:*:*", cpe.GeneratedSource)}, Locations: file.NewLocationSet( file.NewVirtualLocationFromCoordinates( - file.Coordinates{ - RealPath: "/b/path", - FileSystemID: "a", - }, + file.NewCoordinates("/b/path", "a"), "/another/path", ), ), @@ -405,10 +402,7 @@ func TestCatalog_MergeRecords(t *testing.T) { CPEs: []cpe.CPE{cpe.Must("cpe:2.3:a:package:2:2:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource)}, Locations: file.NewLocationSet( file.NewVirtualLocationFromCoordinates( - file.Coordinates{ - RealPath: "/b/path", - FileSystemID: "b", - }, + file.NewCoordinates("/b/path", "b"), "/another/path", ), ), @@ -417,17 +411,11 @@ func TestCatalog_MergeRecords(t *testing.T) { }, expectedLocations: []file.Location{ file.NewVirtualLocationFromCoordinates( - file.Coordinates{ - RealPath: "/b/path", - FileSystemID: "a", - }, + file.NewCoordinates("/b/path", "a"), "/another/path", ), file.NewVirtualLocationFromCoordinates( - file.Coordinates{ - RealPath: "/b/path", - FileSystemID: "b", - }, + file.NewCoordinates("/b/path", "b"), "/another/path", ), }, diff --git a/syft/pkg/license.go b/syft/pkg/license.go index df7fe0f78..30389a94f 100644 --- a/syft/pkg/license.go +++ b/syft/pkg/license.go @@ -257,7 +257,7 @@ func (b *licenseBuilder) WithURLs(urls ...string) *licenseBuilder { func (b *licenseBuilder) WithLocations(locations ...file.Location) *licenseBuilder { for _, loc := range locations { - if loc.Path() != "" { + if loc.RealPath != "" { b.locations = append(b.locations, loc) } } diff --git a/syft/pkg/license_set_test.go b/syft/pkg/license_set_test.go index c6e420a3d..fa9692fed 100644 --- a/syft/pkg/license_set_test.go +++ b/syft/pkg/license_set_test.go @@ -64,16 +64,16 @@ func TestLicenseSet_Add(t *testing.T) { { name: "deduplicate licenses with locations", licenses: []License{ - NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "1"})), - NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "1"})), - NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "2"})), + NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.NewCoordinates("/place", "1"))), + NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.NewCoordinates("/place", "1"))), + NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.NewCoordinates("/place", "2"))), }, want: []License{ NewLicenseFromLocationsWithContext( ctx, "MIT", - file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "1"}), - file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "2"}), + file.NewLocationFromCoordinates(file.NewCoordinates("/place", "1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/place", "2")), ), }, }, @@ -81,15 +81,15 @@ func TestLicenseSet_Add(t *testing.T) { name: "same licenses with different locations", licenses: []License{ NewLicenseWithContext(ctx, "MIT"), - NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "2"})), - NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "1"})), + NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.NewCoordinates("/place", "2"))), + NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.NewCoordinates("/place", "1"))), }, want: []License{ NewLicenseFromLocationsWithContext( ctx, "MIT", - file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "1"}), - file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "2"}), + file.NewLocationFromCoordinates(file.NewCoordinates("/place", "1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/place", "2")), ), }, }, diff --git a/syft/pkg/package.go b/syft/pkg/package.go index d649db292..79f9089c2 100644 --- a/syft/pkg/package.go +++ b/syft/pkg/package.go @@ -119,10 +119,10 @@ func Less(i, j Package) bool { } return len(iLocations) < len(jLocations) } - if iLocations[l].RealPath == jLocations[l].RealPath { + if iLocations[l].FullNestedPath() == jLocations[l].FullNestedPath() { continue } - return iLocations[l].RealPath < jLocations[l].RealPath + return iLocations[l].FullNestedPath() < jLocations[l].FullNestedPath() } // compare remaining metadata as a final fallback // note: we cannot guarantee that IDs (which digests the metadata) are stable enough to sort on diff --git a/syft/pkg/package_test.go b/syft/pkg/package_test.go index 4c2ad5249..9a4e94a7b 100644 --- a/syft/pkg/package_test.go +++ b/syft/pkg/package_test.go @@ -15,10 +15,7 @@ import ( func TestIDUniqueness(t *testing.T) { ctx := context.TODO() originalLocation := file.NewVirtualLocationFromCoordinates( - file.Coordinates{ - RealPath: "39.0742° N, 21.8243° E", - FileSystemID: "Earth", - }, + file.NewCoordinates("39.0742° N, 21.8243° E", "Earth"), "/Ancient-Greece", ) @@ -230,10 +227,7 @@ func TestIDUniqueness(t *testing.T) { func TestPackage_Merge(t *testing.T) { originalLocation := file.NewVirtualLocationFromCoordinates( - file.Coordinates{ - RealPath: "39.0742° N, 21.8243° E", - FileSystemID: "Earth", - }, + file.NewCoordinates("39.0742° N, 21.8243° E", "Earth"), "/Ancient-Greece", ) @@ -461,3 +455,21 @@ func TestIsValid(t *testing.T) { require.Equal(t, c.want, IsValid(c.given), "when package: %s", c.name) } } + +func TestID_nestedArchiveLocations(t *testing.T) { + id := func(c file.Coordinates) string { + p := Package{Name: "x", Version: "1", Type: NpmPkg, Locations: file.NewLocationSet(file.NewLocationFromCoordinates(c))} + p.SetID() + return string(p.ID()) + } + + assert.NotEqual(t, + id(file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/a.jar!/package.json"}), + id(file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/b.jar!/package.json"}), + "packages in different entries of the same outer archive must not share an ID") + + assert.Equal(t, + id(file.Coordinates{RealPath: "/package.json"}), + id(file.NewCoordinates("/package.json", "")), + "a full path equal to the real path must not change the ID") +} diff --git a/syft/source/snapsource/manifest.go b/syft/source/snapsource/manifest.go index a2d0b04d2..86f1c79fe 100644 --- a/syft/source/snapsource/manifest.go +++ b/syft/source/snapsource/manifest.go @@ -47,7 +47,7 @@ func parseManifest(resolver file.Resolver) (*snapManifest, error) { if err != nil { return nil, fmt.Errorf("unable to read snap manifest file: %w", err) } - defer internal.CloseAndLogError(reader, manifestFile.RealPath) + defer internal.CloseAndLogError(reader, manifestFile.FullNestedPath()) var manifest snapManifest if err := yaml.NewDecoder(reader).Decode(&manifest); err != nil { diff --git a/test/rules/rules.go b/test/rules/rules.go index e001a12bd..cb66f9c9f 100644 --- a/test/rules/rules.go +++ b/test/rules/rules.go @@ -16,7 +16,7 @@ func resourceCleanup(m dsl.Matcher) { m["res"].Type.Implements(`io.Reader`) && m["err"].Type.Implements(`error`) && !m["next"].Text.Matches(`defer internal.CloseAndLogError`)). - Report(`please call "defer internal.CloseAndLogError($res, $loc.RealPath)" right after checking the error returned from $resolver.FileContentsByLocation.`) + Report(`please call "defer internal.CloseAndLogError($res, $loc.FullNestedPath())" right after checking the error returned from $resolver.FileContentsByLocation.`) } // nolint:unused