fix: move image hardlink handling upstream during image indexing (#5196)

* chore(deps): pin stereoscope to the hardlink-inode-adoption branch

temporary pin to anchore/stereoscope#670 so the hardlink changes can be
exercised end to end. needs re-pinning to a release tag before merge.

pulls transitive bumps along with it: docker/cli, docker/go-connections and
gabriel-vasile/mimetype.

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix: drop the image resolver hardlink workaround

stereoscope now adopts a hardlink's target at index time, so a hardlinked name
arrives already described as a regular file with the target's size, mime type
and content. the resolver-side fix from #5029 sat on top of that doing the same
job a second time, gated on a file type that no longer shows up, so both
`resolveHardLinkTarget` implementations and the extra tree walk they cost on
every `FilesByPath` are gone.

`file.NewVirtualLocationFromImage` stays as-is. it is exported and syft is v1,
so it keeps working for anyone using it, it just has no callers in syft now.

this picks up `FilesByMIMEType`, which #5029 explicitly could not fix, so
hardlinked names now reach mime-driven catalogers. on images built around
multi-call binaries (busybox and friends) that is a lot more file and executable
entries than before; distro and toolchain images move by about one entry.

one shape regresses: a hardlink stereoscope could not adopt (a link name that is
absent, empty, names a directory, names another un-adopted link, or points into
a lower layer) keeps `TypeHardLink`, and syft goes back to collapsing it onto
its target's path. no mainstream builder emits the cross-layer case.

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* bump stereoscope to main

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

syft/file/location.go

@@ -153,14 +153,9 @@ func NewLocationFromImage(accessPath string, ref file.Reference, img *image.Imag
 }
 
 // NewVirtualLocationFromImage creates a new Location surfaced at realPath but whose contents and metadata are
-// described by the given reference. This is used to present a hardlink as the underlying type it points to (at the
-// hardlink's own path) so that image results are in parity with directory results, which cannot distinguish a
-// hardlink from a regular file. Note that RealPath (the hardlink's own path) intentionally differs from
-// ref.RealPath (the target's path); callers must not assume the two agree for these locations.
+// described by the given reference. Note that RealPath may differ from ref.RealPath; callers must not assume the two
+// agree for these locations. The FileSystemID comes from the given ref's layer.
 func NewVirtualLocationFromImage(realPath, accessPath string, ref file.Reference, img *image.Image) Location {
-	// the FileSystemID comes from the target ref's layer. a hardlink and its target are always materialized in the
-	// same layer tar (a tar hardlink entry can only reference a file within the same archive), so the target's layer
-	// digest is also the hardlink path's layer.
 	layer := img.FileCatalog.Layer(ref)
 	return Location{
 		LocationData: LocationData{

syft/format/syftjson/to_syft_model.go

@@ -175,6 +175,8 @@ func toSyftFileType(ty string) stereoscopeFile.Type {
 	case "SymbolicLink":
 		return stereoscopeFile.TypeSymLink
 	case "HardLink":
+		// kept for reading SBOMs written before stereoscope adopted hardlinks at index time (they are emitted as
+		// regular files now, except for malformed link entries)
 		return stereoscopeFile.TypeHardLink
 	case "Directory":
 		return stereoscopeFile.TypeDirectory

syft/internal/fileresolver/container_image_all_layers.go

@@ -56,18 +56,6 @@ func (r *ContainerImageAllLayers) HasPath(path string) bool {
 func (r *ContainerImageAllLayers) locationsByRef(ref stereoscopeFile.Reference, accessPath string, uniqueFileIDs stereoscopeFile.ReferenceSet, layerPos int) ([]file.Location, error) {
 	uniqueLocations := make([]file.Location, 0)
 
-	// if the access path is itself a hardlink, surface it as the underlying type it points to, at its own path
-	// (bound to the target's content), so that image results are in parity with directory results (which cannot
-	// tell a hardlink from a regular file). the path-based lookup is required because the search that produced ref
-	// already followed the basename link, collapsing the hardlink onto its target.
-	if ownRef, targetRef, ok := r.hardLinkAtPath(accessPath, r.layers[layerPos]); ok {
-		if !uniqueFileIDs.Contains(ownRef) {
-			uniqueFileIDs.Add(ownRef)
-			uniqueLocations = append(uniqueLocations, file.NewVirtualLocationFromImage(string(ownRef.RealPath), accessPath, targetRef, r.img))
-		}
-		return uniqueLocations, nil
-	}
-
 	// since there is potentially considerable work for each symlink/hardlink that needs to be resolved, let's check to see if this is a symlink/hardlink first
 	entry, err := r.img.FileCatalog.Get(ref)
 	if err != nil {
@@ -95,25 +83,6 @@ func (r *ContainerImageAllLayers) locationsByRef(ref stereoscopeFile.Reference,
 	return uniqueLocations, nil
 }
 
-// hardLinkAtPath returns the hardlink's own reference and its resolved target reference when the basename of path is a
-// hardlink within the given layer. ok is false when path does not exist there or is not a hardlink. The lookup does
-// not follow the basename link so that the hardlink's own path is preserved. This adds a tree walk per
-// matched ref (path x layer); if it shows up in profiles, fold the hardlink check into the existing search resolution.
-func (r *ContainerImageAllLayers) hardLinkAtPath(path string, layerIdx int) (stereoscopeFile.Reference, stereoscopeFile.Reference, bool) {
-	var own stereoscopeFile.Reference
-	// use the squashed-to-layer view (not just this layer's additions) so a hardlink added in a lower layer is still
-	// detected when the path is searched at a higher layer; otherwise it would collapse onto its target there.
-	exists, resolution, err := r.img.Layers[layerIdx].SquashedTree.File(stereoscopeFile.Path(path))
-	if err != nil || !exists || !resolution.HasReference() {
-		return own, own, false
-	}
-	target, ok := r.resolveHardLinkTarget(*resolution.Reference, layerIdx)
-	if !ok {
-		return own, own, false
-	}
-	return *resolution.Reference, target, true
-}
-
 // FilesByPath returns all file.References that match the given paths from any layer in the image.
 func (r *ContainerImageAllLayers) FilesByPath(paths ...string) ([]file.Location, error) {
 	uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
@@ -284,11 +253,6 @@ func (r *ContainerImageAllLayers) AllLocations(ctx context.Context) <-chan file.
 			tree := r.img.Layers[layerIdx].Tree
 			for _, ref := range tree.AllFiles(stereoscopeFile.AllTypes()...) {
 				l := file.NewLocationFromImage(string(ref.RealPath), ref, r.img)
-				// surface a hardlink as the underlying type it points to (at its own path) so image results match
-				// directory results, which cannot distinguish a hardlink from a regular file.
-				if targetRef, ok := r.resolveHardLinkTarget(ref, layerIdx); ok {
-					l = file.NewVirtualLocationFromImage(string(ref.RealPath), string(ref.RealPath), targetRef, r.img)
-				}
 				r.annotateLocation(&l)
 				select {
 				case <-ctx.Done():
@@ -306,21 +270,6 @@ func (r *ContainerImageAllLayers) FileMetadataByLocation(location file.Location)
 	return fileMetadataByLocation(r.img, location)
 }
 
-// resolveHardLinkTarget returns the reference of a hardlink's underlying target (resolved relative to the given layer)
-// when ref is a hardlink; ok is false otherwise. No resolution is performed for non-hardlinks (symlinks keep their
-// existing resolution semantics).
-func (r *ContainerImageAllLayers) resolveHardLinkTarget(ref stereoscopeFile.Reference, layerIdx int) (stereoscopeFile.Reference, bool) {
-	metadata, err := r.img.FileCatalog.Get(ref)
-	if err != nil || metadata.Type != stereoscopeFile.TypeHardLink {
-		return ref, false
-	}
-	resolved, err := r.img.ResolveLinkByLayerSquash(ref, layerIdx)
-	if err != nil || !resolved.HasReference() {
-		return ref, false
-	}
-	return *resolved.Reference, true
-}
-
 func (r *ContainerImageAllLayers) annotateLocation(l *file.Location) {
 	if !r.markVisibility || l == nil {
 		return
@@ -345,8 +294,6 @@ func (r *ContainerImageAllLayers) annotateLocation(l *file.Location) {
 }
 
 // pathResolvesToRef reports whether the given path in the squashed tree resolves to the given reference.
-// SearchByPath always follows basename links, so a hardlink surfaced at its own path (whose reference is its
-// target) still resolves to the target here and is correctly considered visible.
 func (r *ContainerImageAllLayers) pathResolvesToRef(path string, target stereoscopeFile.Reference) bool {
 	ref, err := r.img.SquashedSearchContext.SearchByPath(path, filetree.DoNotFollowDeadBasenameLinks)
 	return err == nil && ref.HasReference() && ref.ID() == target.ID()

syft/internal/fileresolver/container_image_squash.go

@@ -40,17 +40,6 @@ func (r *ContainerImageSquash) FilesByPath(paths ...string) ([]file.Location, er
 	uniqueLocations := make([]file.Location, 0)
 
 	for _, path := range paths {
-		// if the requested path is itself a hardlink, surface it at its own path (bound to its target's content)
-		// rather than collapsing it onto the target's path, so that image results are in parity with directory
-		// results (which cannot tell a hardlink from a regular file).
-		if ownRef, targetRef, ok := r.hardLinkAtPath(path); ok {
-			if !uniqueFileIDs.Contains(ownRef) {
-				uniqueFileIDs.Add(ownRef)
-				uniqueLocations = append(uniqueLocations, file.NewVirtualLocationFromImage(string(ownRef.RealPath), path, targetRef, r.img))
-			}
-			continue
-		}
-
 		ref, err := r.img.SquashedSearchContext.SearchByPath(path, filetree.FollowBasenameLinks)
 		if err != nil {
 			return nil, err
@@ -93,7 +82,7 @@ func (r *ContainerImageSquash) FilesByPath(paths ...string) ([]file.Location, er
 //
 //nolint:gocognit
 func (r *ContainerImageSquash) FilesByGlob(patterns ...string) ([]file.Location, error) {
-	uniqueCoordinates := file.NewCoordinateSet()
+	uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
 	uniqueLocations := make([]file.Location, 0)
 
 	for _, pattern := range patterns {
@@ -127,14 +116,10 @@ func (r *ContainerImageSquash) FilesByGlob(patterns ...string) ([]file.Location,
 				return nil, fmt.Errorf("failed to find files by path (result=%+v): %w", result, err)
 			}
 			for _, resolvedLocation := range resolvedLocations {
-				// dedup on the surfaced coordinate rather than the underlying reference: distinct hardlinks share a
-				// single target reference but each has its own real path, so a reference-based dedup would collapse
-				// them back onto one entry (the exact behavior this parity fix removes). symlink resolutions keep
-				// their target's real path, so they still collapse as before.
-				if uniqueCoordinates.Contains(resolvedLocation.Coordinates) {
+				if uniqueFileIDs.Contains(resolvedLocation.Reference()) {
 					continue
 				}
-				uniqueCoordinates.Add(resolvedLocation.Coordinates)
+				uniqueFileIDs.Add(resolvedLocation.Reference())
 				uniqueLocations = append(uniqueLocations, resolvedLocation)
 			}
 		}
@@ -194,16 +179,10 @@ func (r *ContainerImageSquash) AllLocations(ctx context.Context) <-chan file.Loc
 	go func() {
 		defer close(results)
 		for _, ref := range r.img.SquashedTree().AllFiles(stereoscopeFile.AllTypes()...) {
-			loc := file.NewLocationFromImage(string(ref.RealPath), ref, r.img)
-			// surface a hardlink as the underlying type it points to (at its own path) so image results match
-			// directory results, which cannot distinguish a hardlink from a regular file.
-			if targetRef, ok := r.resolveHardLinkTarget(ref); ok {
-				loc = file.NewVirtualLocationFromImage(string(ref.RealPath), string(ref.RealPath), targetRef, r.img)
-			}
 			select {
 			case <-ctx.Done():
 				return
-			case results <- loc:
+			case results <- file.NewLocationFromImage(string(ref.RealPath), ref, r.img):
 				continue
 			}
 		}
@@ -211,37 +190,6 @@ func (r *ContainerImageSquash) AllLocations(ctx context.Context) <-chan file.Loc
 	return results
 }
 
-// hardLinkAtPath returns the hardlink's own reference and its resolved target reference when the basename of path is a
-// hardlink. ok is false when path does not exist or is not a hardlink. The lookup does not follow the basename link so
-// that the hardlink's own path is preserved. This adds a tree walk per FilesByPath path; if it shows up in
-// profiles, fold the hardlink check into the existing SearchByPath resolution.
-func (r *ContainerImageSquash) hardLinkAtPath(path string) (stereoscopeFile.Reference, stereoscopeFile.Reference, bool) {
-	var own stereoscopeFile.Reference
-	exists, resolution, err := r.img.SquashedTree().File(stereoscopeFile.Path(path))
-	if err != nil || !exists || !resolution.HasReference() {
-		return own, own, false
-	}
-	target, ok := r.resolveHardLinkTarget(*resolution.Reference)
-	if !ok {
-		return own, own, false
-	}
-	return *resolution.Reference, target, true
-}
-
-// resolveHardLinkTarget returns the reference of a hardlink's underlying target when ref is a hardlink; ok is false
-// otherwise. No resolution is performed for non-hardlinks (symlinks keep their existing resolution semantics).
-func (r *ContainerImageSquash) resolveHardLinkTarget(ref stereoscopeFile.Reference) (stereoscopeFile.Reference, bool) {
-	metadata, err := r.img.FileCatalog.Get(ref)
-	if err != nil || metadata.Type != stereoscopeFile.TypeHardLink {
-		return ref, false
-	}
-	resolved, err := r.img.ResolveLinkByImageSquash(ref)
-	if err != nil || !resolved.HasReference() {
-		return ref, false
-	}
-	return *resolved.Reference, true
-}
-
 func (r *ContainerImageSquash) FilesByMIMEType(types ...string) ([]file.Location, error) {
 	refs, err := r.img.SquashedSearchContext.SearchByMIMEType(types...)
 	if err != nil {

syft/internal/fileresolver/container_image_squash_test.go

@@ -642,6 +642,24 @@ func TestImageResolvers_Hardlinks(t *testing.T) {
 				}
 			})
 
+			t.Run("FilesByMIMEType surfaces every hardlink name", func(t *testing.T) {
+				// this is the surface #5029 could not fix and stereoscope's index-time adoption does: MIME-driven
+				// catalogers see each name. it also pins that the names are separately addressable (distinct
+				// references), since this resolver dedups on reference and would return one entry if they shared it.
+				locs, err := resolver.FilesByMIMEType("text/plain")
+				require.NoError(t, err)
+
+				byPath := map[string]file.Location{}
+				for _, loc := range locs {
+					byPath[loc.RealPath] = loc
+				}
+				for _, path := range []string{"/file.txt", "/hardlink-a", "/hardlink-b"} {
+					loc, ok := byPath[path]
+					require.Truef(t, ok, "expected MIME search to surface %s (got %v)", path, byPath)
+					assertHardlink(t, resolver, path, loc)
+				}
+			})
+
 			t.Run("AllLocations includes every hardlink path as a regular file exactly once", func(t *testing.T) {
 				counts := map[string]int{}
 				locsByPath := map[string]file.Location{}