diff --git a/cmd/syft/internal/options/catalog.go b/cmd/syft/internal/options/catalog.go index 3ff68ecbd..4fe7cbff0 100644 --- a/cmd/syft/internal/options/catalog.go +++ b/cmd/syft/internal/options/catalog.go @@ -102,6 +102,7 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig { WithComplianceConfig(cfg.ToComplianceConfig()). WithUnknownsConfig(cfg.ToUnknownsConfig()). WithSearchConfig(cfg.ToSearchConfig()). + WithArchiveConfig(cfg.ToArchiveConfig()). WithPackagesConfig(cfg.ToPackagesConfig()). WithLicenseConfig(cfg.ToLicenseConfig()). WithFilesConfig(cfg.ToFilesConfig()). @@ -118,6 +119,15 @@ func (cfg Catalog) ToSearchConfig() cataloging.SearchConfig { } } +func (cfg Catalog) ToArchiveConfig() cataloging.ArchiveSearchConfig { + return cataloging.DefaultArchiveSearchConfig(). + WithIncludeIndexedArchives(cfg.Package.SearchIndexedArchives). + WithIncludeUnindexedArchives(cfg.Package.SearchUnindexedArchives). + WithMaxDepth(cfg.Package.NestedArchiveMaxDepth). + WithMaxMemoryBytes(cfg.Package.NestedArchiveMaxMemoryBytes). + WithMaxDiskBytes(cfg.Package.NestedArchiveMaxDiskBytes) +} + func (cfg Catalog) ToRelationshipsConfig() cataloging.RelationshipsConfig { return cataloging.RelationshipsConfig{ PackageFileOwnership: cfg.Relationships.PackageFileOwnership, @@ -136,6 +146,7 @@ func (cfg Catalog) ToComplianceConfig() cataloging.ComplianceConfig { func (cfg Catalog) ToUnknownsConfig() cataloging.UnknownsConfig { return cataloging.UnknownsConfig{ + RemoveWhenPackagesDefined: cfg.Unknowns.RemoveWhenPackagesDefined, IncludeExecutablesWithoutPackages: cfg.Unknowns.ExecutablesWithoutPackages, IncludeUnexpandedArchives: cfg.Unknowns.UnexpandedArchives, } @@ -169,10 +180,6 @@ func (cfg Catalog) ToLicenseConfig() cataloging.LicenseConfig { } func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config { - archiveSearch := cataloging.ArchiveSearchConfig{ - IncludeIndexedArchives: cfg.Package.SearchIndexedArchives, - IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives, - } return pkgcataloging.Config{ Binary: binary.DefaultClassifierCatalogerConfig(), Cpp: cpp.DefaultCatalogerConfig(). @@ -219,7 +226,7 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config { WithMavenLocalRepositoryDir(cfg.Java.MavenLocalRepositoryDir). WithUseNetwork(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.Java, task.Maven), cfg.Java.UseNetwork)). WithMavenBaseURL(cfg.Java.MavenURL). - WithArchiveTraversal(archiveSearch, cfg.Java.MaxParentRecursiveDepth). + WithMaxParentRecursiveDepth(cfg.Java.MaxParentRecursiveDepth). WithResolveTransitiveDependencies(cfg.Java.ResolveTransitiveDependencies), } } diff --git a/cmd/syft/internal/options/pkg.go b/cmd/syft/internal/options/pkg.go index 14ea68882..6d424e1c8 100644 --- a/cmd/syft/internal/options/pkg.go +++ b/cmd/syft/internal/options/pkg.go @@ -6,9 +6,12 @@ import ( ) type packageConfig struct { - SearchUnindexedArchives bool `yaml:"search-unindexed-archives" json:"search-unindexed-archives" mapstructure:"search-unindexed-archives"` - SearchIndexedArchives bool `yaml:"search-indexed-archives" json:"search-indexed-archives" mapstructure:"search-indexed-archives"` - ExcludeBinaryOverlapByOwnership bool `yaml:"exclude-binary-overlap-by-ownership" json:"exclude-binary-overlap-by-ownership" mapstructure:"exclude-binary-overlap-by-ownership"` // exclude synthetic binary packages owned by os package files + SearchUnindexedArchives bool `yaml:"search-unindexed-archives" json:"search-unindexed-archives" mapstructure:"search-unindexed-archives"` + SearchIndexedArchives bool `yaml:"search-indexed-archives" json:"search-indexed-archives" mapstructure:"search-indexed-archives"` + ExcludeBinaryOverlapByOwnership bool `yaml:"exclude-binary-overlap-by-ownership" json:"exclude-binary-overlap-by-ownership" mapstructure:"exclude-binary-overlap-by-ownership"` // exclude synthetic binary packages owned by os package files + NestedArchiveMaxDepth int `yaml:"nested-archive-max-depth" json:"nested-archive-max-depth" mapstructure:"nested-archive-max-depth"` + NestedArchiveMaxMemoryBytes int64 `yaml:"nested-archive-max-memory-bytes" json:"nested-archive-max-memory-bytes" mapstructure:"nested-archive-max-memory-bytes"` + NestedArchiveMaxDiskBytes int64 `yaml:"nested-archive-max-disk-bytes" json:"nested-archive-max-disk-bytes" mapstructure:"nested-archive-max-disk-bytes"` } var _ interface { @@ -17,12 +20,17 @@ var _ interface { func (o *packageConfig) DescribeFields(descriptions clio.FieldDescriptionSet) { descriptions.Add(&o.SearchIndexedArchives, `search within archives that do contain a file index to search against (zip) -note: for now this only applies to the java package cataloger`) +note: this only applies to the java package cataloger, and is ignored when nested-archive-max-depth is not 0 (every archive is searched)`) descriptions.Add(&o.SearchUnindexedArchives, `search within archives that do not contain a file index to search against (tar, tar.gz, tar.bz2, etc) note: enabling this may result in a performance impact since all discovered compressed tars will be decompressed -note: for now this only applies to the java package cataloger`) +note: this only applies to the java package cataloger, and is ignored when nested-archive-max-depth is not 0 (every archive is searched)`) descriptions.Add(&o.ExcludeBinaryOverlapByOwnership, `allows users to exclude synthetic binary packages from the sbom these packages are removed if an overlap with a non-synthetic package is found`) + descriptions.Add(&o.NestedArchiveMaxDepth, `maximum depth to recursively catalog nested archives (0 = disabled, -1 = as deep as 16 levels, the most allowed) +note: when enabled this replaces the java cataloger's own unbounded jar-in-jar recursion, so archives deeper than this are reported as unknowns rather than cataloged`) + descriptions.Add(&o.NestedArchiveMaxMemoryBytes, `maximum bytes of nested archive content held in memory at once (0 = none, -1 = unbounded)`) + descriptions.Add(&o.NestedArchiveMaxDiskBytes, `maximum bytes of nested archive content on disk at once (0 = none, -1 = unbounded) +note: this and nested-archive-max-memory-bytes cannot both be 0 when nested-archive-max-depth is not 0`) } func defaultPackageConfig() packageConfig { @@ -31,5 +39,8 @@ func defaultPackageConfig() packageConfig { SearchIndexedArchives: c.IncludeIndexedArchives, SearchUnindexedArchives: c.IncludeUnindexedArchives, ExcludeBinaryOverlapByOwnership: true, + NestedArchiveMaxDepth: c.MaxDepth, + NestedArchiveMaxMemoryBytes: c.MaxMemoryBytes, + NestedArchiveMaxDiskBytes: c.MaxDiskBytes, } } diff --git a/cmd/syft/internal/options/pkg_test.go b/cmd/syft/internal/options/pkg_test.go new file mode 100644 index 000000000..37a39d7e0 --- /dev/null +++ b/cmd/syft/internal/options/pkg_test.go @@ -0,0 +1,53 @@ +package options + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/cataloging" +) + +func Test_packageConfigDefaultsMatchArchiveSearchConfig(t *testing.T) { + want := cataloging.DefaultArchiveSearchConfig() + got := defaultPackageConfig() + + assert.Equal(t, want.MaxDepth, got.NestedArchiveMaxDepth) + assert.Equal(t, want.MaxMemoryBytes, got.NestedArchiveMaxMemoryBytes) + assert.Equal(t, want.MaxDiskBytes, got.NestedArchiveMaxDiskBytes) + + require.Zero(t, got.NestedArchiveMaxDepth, "nested archive cataloging must be off by default") +} + +func Test_ToArchiveConfig(t *testing.T) { + defaults := cataloging.DefaultArchiveSearchConfig() + + t.Run("defaults round trip unchanged", func(t *testing.T) { + cfg := Catalog{Package: defaultPackageConfig()} + got := cfg.ToArchiveConfig() + + assert.Equal(t, defaults.MaxDepth, got.MaxDepth) + assert.Equal(t, defaults.MaxMemoryBytes, got.MaxMemoryBytes) + assert.Equal(t, defaults.MaxDiskBytes, got.MaxDiskBytes) + }) + + t.Run("every setting is threaded through", func(t *testing.T) { + // a key that parses but is not carried into the archive config is invisible in a config + // dump, so assert each one arrives + cfg := Catalog{Package: packageConfig{ + NestedArchiveMaxDepth: 3, + NestedArchiveMaxMemoryBytes: 111, + NestedArchiveMaxDiskBytes: 3333, + SearchIndexedArchives: true, + SearchUnindexedArchives: true, + }} + got := cfg.ToArchiveConfig() + + assert.Equal(t, 3, got.MaxDepth) + assert.Equal(t, int64(111), got.MaxMemoryBytes) + assert.Equal(t, int64(3333), got.MaxDiskBytes) + assert.True(t, got.IncludeIndexedArchives) + assert.True(t, got.IncludeUnindexedArchives) + }) +} diff --git a/cmd/syft/internal/ui/ui.go b/cmd/syft/internal/ui/ui.go index 4673a2917..878558367 100644 --- a/cmd/syft/internal/ui/ui.go +++ b/cmd/syft/internal/ui/ui.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "os" + "strings" "sync" "time" @@ -186,7 +187,12 @@ func (m *UI) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } func (m UI) View() string { - return m.frame.View() + v := m.frame.View() + // bubbletea erases the line the cursor rests on at exit, which would drop the frame's last line + if !strings.HasSuffix(v, "\n") { + return v + "\n" + } + return v } var ( diff --git a/cmd/syft/internal/ui/ui_test.go b/cmd/syft/internal/ui/ui_test.go new file mode 100644 index 000000000..23e344814 --- /dev/null +++ b/cmd/syft/internal/ui/ui_test.go @@ -0,0 +1,31 @@ +package ui + +import ( + "io" + "testing" + + tea "github.com/charmbracelet/bubbletea" + "github.com/stretchr/testify/require" + + "github.com/anchore/bubbly/bubbles/frame" +) + +var _ tea.Model = (*staticModel)(nil) + +type staticModel string + +func Test_UI_viewEndsOnAnEmptyLine(t *testing.T) { + // bubbletea erases the line the cursor rests on when the program stops, so a view whose last line + // carries content loses it. Unless a log line happened to be in the footer, that dropped the last row + // of the cataloging tree on any source that logs nothing. + subject := New(io.Discard, false) + subject.frame.(*frame.Frame).AppendModel(staticModel("first row\nlast row")) + + require.Equal(t, "first row\nlast row\n", subject.View()) +} + +func (m staticModel) Init() tea.Cmd { return nil } + +func (m staticModel) Update(tea.Msg) (tea.Model, tea.Cmd) { return m, nil } + +func (m staticModel) View() string { return string(m) } diff --git a/internal/archive/README.md b/internal/archive/README.md new file mode 100644 index 000000000..cb79a2524 --- /dev/null +++ b/internal/archive/README.md @@ -0,0 +1,106 @@ +# internal/archive + +Opens one archive as its own filesystem so the ordinary catalogers can run against its contents. +The archive cataloger task (`internal/task/archive_tasks.go`) drives the recursion; this package +does everything for a single archive. + +## Flow + +``` +Extract(ctx, reader, archiveCoordinates, limiter, exclusions) -> *Resolver + acquire read in place if the reader is already seekable, else hold in memory / spill to disk + identifyFormat internal/file.IdentifyArchive first, end-of-central-directory fallback for stub-prefixed zips + Resolver.extract every entry -> entryHeader -> Resolver.add (excluded entries are never stored) + digestsOf SHA-1 of the archive itself +``` + +`Extract` returns `nil` for content that is not an archive (a bare compression format such as a +gzipped file that is not a tar counts as not an archive), `ErrDiskLimitReached` when the archive's +own bytes cannot be placed, and otherwise a `Resolver` whose `Cleanup` releases everything. + +## Resolver + +One type does the work: `Resolver` is fed entries by `Extract`, holds their content, indexes them, +and implements `file.Resolver` over the result. Each entry is a `node` carrying its header and a +`blob`; the archive's own bytes, when they have to be held at all, are one more blob. + +Every location it returns extends the archive's coordinates: `RealPath` and `FileSystemID` stay +those of the outermost archive on the scanned filesystem, and `FullPath` is the `!/`-delimited +chain down to the entry (`/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF`). `LocalPath` is the +entry's path within this archive. + +## Where bytes live + +Blobs are held in memory while the memory limit admits them and in one spill file per archive +(`archive-spill-*`) once it does not; when a chunk is refused, everything held in memory moves to +the file first. The file is created on first use, so most archives never touch the filesystem, and +blobs are located in it by offset with no framing of their own. It goes under the scan's temp root +when the context carries one (`internal/tmpdir`), otherwise the system temp directory. + +Anything already random-access (an `*os.File`, or a nested archive read out of its parent's +resolver) is used in place and costs nothing, so only a top-level stream is ever copied. Its memory +is refunded once it is extracted and digested; bytes it spilled wait for `Cleanup` with the entries. + +## Limits + +`Limits` bound what one scan holds *at once*; each falls as archives are released. For both limits, +positive is the bound, zero forbids the resource, negative is unbounded. + +- `Limiter` is scan-wide; a `charge` is one archive's draw on it. Bytes are charged as they land, + never from declared sizes. +- Memory refused → content moves to disk. Disk refused → the archive's own bytes cannot be placed + (`ErrDiskLimitReached`, archive skipped) or an entry cannot be stored (`Resolver.Truncated`, the + entries stored so far are still cataloged). +- Index bookkeeping is approximated as `approxIndexBytesPerEntry` (2 KiB) per node, listed or + implied by an entry's path, plus name length. It lives in memory, so it is charged to the memory + limit, spilling held content to make room; only a zero memory limit lets it fall back to the disk + budget so that configuration still indexes. An entry whose index cost is refused is dropped like + one whose content is, and marks the resolver `Truncated`. +- `Budget` bounds the bytes decompressed under one top-level archive, summed over everything nested + in it. Unlike `Limits` it never refills, so a recursive bomb whose every level fits the limits + still ends (`errBudgetSpent`, the resolver is `Truncated`). +- `Limiter.Peak` reports the most held at once over the scan, for reporting. + +## Hostile archives + +Nothing an archive declares is trusted. Content is charged chunk by chunk as it decompresses, so a +deflate or gzip bomb stops at the disk limit with the entry dropped whole. Every node is charged +against the memory limit, including directories implied by deep paths, so an archive of millions of +tiny entries or of a few entries with 4 KiB names stops at the memory limit. A recursive bomb stops +at the `Budget`. Names longer than PATH_MAX are refused, and nesting stops at the task's `MaxDepth`. +See the bomb tests in `extract_test.go`. + +## Lookup + +Entries are indexed in a stereoscope `filetree.FileTree` + `filetree.Index` and searched with +`filetree.SearchContext`, the same index the directory and image resolvers use. A query may name an +entry by its full path or by its `LocalPath` (leading slash optional); a full path into another +archive in the scan names nothing. Directories implied by entry paths exist for `HasPath` but are +never files. The last entry at a path wins over earlier +duplicates, as it does on extraction; a file and a directory at one path keep the first. + +Links resolve inside the archive only, as the file tree resolves them: a hard link names its target +from the archive root. A lookup matching a link and its target returns one location, with +`AccessPath` the full path matched and `FullPath` that of the file read. Dangling and cyclic links are not +files. `AllLocations` enumerates every non-directory path, links included. Answers are sorted by path. + +## Other pieces + +- `Exclusions`: the scan's exclusion patterns, of which only the `**/` shape means the same thing + re-rooted at an archive. Matched against each entry path and its parent directories at extraction + time. `syft.CreateSBOM` passes the source's patterns to the task, which builds one `Exclusions` + for the scan. +- `Traversal`: put on the context by the task so a cataloger (java) knows which archive it is inside, + what its digests are, and which packages describe its parent. `VirtualPath(location)` converts a + location's `FullPath` to the java cataloger's colon-delimited `app.war:WEB-INF/lib/inner.jar` + form, escaping `%` and `:` in entry names. `Container` and `EntryCoordinates` split and build the + coordinates of files within archives (the SPDX and CycloneDX formats use them too); `OwnerContainers` and + `OwnedCoordinates` map package-owned file paths to coordinates in the right archive. +- `WithNestedCataloging` / `NestedCatalogingEnabled`: `syft.CreateSBOM` marks the context when the + archive task is running, so the java cataloger leaves nested archives to the task instead of + opening them itself. +- `entryHeader`: keeps regular files, directories and links only (no devices or fifos), cleans entry names to + archive-relative paths, refuses names longer than PATH_MAX or containing control characters, line + separators or the `!/` archive path separator (marking the resolver truncated), and drops header + fields nothing reads (PAX records, user and group names, sub-second and access times) since the + header is held for the life of the archive. diff --git a/internal/archive/exclusions.go b/internal/archive/exclusions.go new file mode 100644 index 000000000..b8cef0681 --- /dev/null +++ b/internal/archive/exclusions.go @@ -0,0 +1,38 @@ +package archive + +import ( + "strings" + + "github.com/bmatcuk/doublestar/v4" +) + +// Exclusions are the scan's exclusion patterns that apply inside archives. +// +// A scan accepts three pattern shapes: `./x` anchored at the scan root, `*/x` exactly one level below +// it, and `**/x` at any depth. Only the last means the same thing when re-rooted at an archive, so +// only it reaches inside one. +type Exclusions []string + +func NewExclusions(scanPatterns []string) Exclusions { + var out Exclusions + for _, pattern := range scanPatterns { + if strings.HasPrefix(pattern, "**/") { + // a trailing slash reads as "a directory" but makes doublestar match nothing (issue #4839) + out = append(out, strings.TrimSuffix(pattern, "/")) + } + } + return out +} + +// Excludes reports whether an archive-relative path, or any directory above it, matches a pattern. +// Directories are checked because an archive need not list a directory before the files in it. +func (e Exclusions) Excludes(entryPath string) bool { + for _, pattern := range e { + for end := len(entryPath); end > 0; end = strings.LastIndexByte(entryPath[:end], '/') { + if matched, _ := doublestar.Match(pattern, entryPath[:end]); matched { + return true + } + } + } + return false +} diff --git a/internal/archive/exclusions_test.go b/internal/archive/exclusions_test.go new file mode 100644 index 000000000..e2e31893b --- /dev/null +++ b/internal/archive/exclusions_test.go @@ -0,0 +1,56 @@ +package archive + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewExclusions(t *testing.T) { + tests := []struct { + name string + given []string + want Exclusions + }{ + {name: "nothing configured"}, + {name: "root-anchored does not reach inside", given: []string{"./nested/marker.txt"}}, + {name: "one level down is anchored too", given: []string{"*/vendor"}}, + {name: "any depth reaches inside", given: []string{"**/*.rpm"}, want: Exclusions{"**/*.rpm"}}, + {name: "mixed keeps only the any-depth patterns", given: []string{"./a", "**/b", "*/c", "**/d"}, want: Exclusions{"**/b", "**/d"}}, + // a trailing slash makes doublestar match nothing (issue #4839) + {name: "a trailing slash is trimmed", given: []string{"**/vendor/"}, want: Exclusions{"**/vendor"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, NewExclusions(tt.given)) + }) + } +} + +func TestExclusions_Excludes(t *testing.T) { + exclusions := NewExclusions([]string{"**/vendor", "**/*.rpm"}) + + tests := []struct { + path string + want bool + }{ + {"vendor", true}, + {"a/vendor", true}, + {"vendor/x/y.go", true}, + {"a/vendor/y.go", true}, + {"pkg.rpm", true}, + {"deep/down/pkg.rpm", true}, + {"keep/vendored.txt", false}, + {"vendor-ish", false}, + {"keep.txt", false}, + {"PKG.RPM", false}, + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + assert.Equal(t, tt.want, exclusions.Excludes(tt.path)) + }) + } + + assert.False(t, NewExclusions(nil).Excludes("anything")) + assert.False(t, Exclusions{"["}.Excludes("keep.txt"), "a malformed pattern excludes nothing") +} diff --git a/internal/archive/extract.go b/internal/archive/extract.go new file mode 100644 index 000000000..60cd93fd3 --- /dev/null +++ b/internal/archive/extract.go @@ -0,0 +1,263 @@ +package archive + +import ( + "bytes" + "context" + "crypto" + "errors" + "fmt" + "io" + "path" + + "github.com/mholt/archives" + + "github.com/anchore/syft/internal" + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/file" +) + +// ErrUnsupportedFormat is returned for an archive in a format that is not extracted (see supportedExtractor). +var ErrUnsupportedFormat = errors.New("archive format is not supported") + +// Extract opens the archive read from r as its own filesystem, holding its bytes and entries within +// the limiter's bounds. It returns nil when the content is not an archive, ErrUnsupportedFormat when it +// is one that is not extracted, and ErrDiskLimitReached when the archive's own bytes cannot be placed +// within the disk limit. The caller must Cleanup the result. +// +// archiveCoordinates are those of the archive file itself; every location the resolver returns +// extends them with its entry path (see file.Coordinates). +func Extract(ctx context.Context, r io.Reader, archiveCoordinates file.Coordinates, limiter *Limiter, exclusions Exclusions) (*Resolver, error) { + charge := limiter.charge() + resolver := newResolver(ctx, archiveCoordinates, charge) + archivePath := archiveCoordinates.FullNestedPath() + + // deferred so that a panic in a decoder still gives back the charge and removes any spill file + succeeded := false + defer func() { + if !succeeded { + resolver.Cleanup() + } + }() + + content, release, err := resolver.acquire(r) + if err != nil { + return nil, err + } + // the archive's own bytes are needed only until they are extracted and digested + defer release() + + format, err := identifyFormat(ctx, path.Base(archiveCoordinates.LocalPath()), content) + if format == nil || err != nil { + return nil, err + } + + if err := resolver.extract(ctx, format, content, exclusions); err != nil { + return nil, fmt.Errorf("unable to extract archive %q: %w", archivePath, err) + } + resolver.Digests = digestsOf(ctx, content, archivePath) + succeeded = true + return resolver, nil +} + +// acquire makes the archive's own bytes random-access, which identifying, extracting and digesting +// them needs. Bytes that already are, such as a file on disk or an entry of a parent archive, are +// read in place and cost nothing. Anything else is held within the limits like an entry, and the +// returned release gives back what memory it takes; bytes spilled to disk stay until Cleanup. +// Reaching the disk limit yields ErrDiskLimitReached. +func (r *Resolver) acquire(content io.Reader) (ReaderAtSeeker, func(), error) { + if ras, ok := content.(ReaderAtSeeker); ok { + return ras, func() {}, nil + } + var b blob + if err := r.put(&b, content); err != nil { + if errors.Is(err, ErrDiskLimitReached) { + return nil, nil, err + } + return nil, nil, fmt.Errorf("unable to read archive content: %w", err) + } + return r.open(&b), func() { r.discard(&b) }, nil +} + +// identifyFormat returns the format able to extract this content, or nil when it is not an archive. +// +// Identification from the head of the stream wins: a tar holding one jar ends in that jar's central +// directory, and reading the tar as a zip would lose the rest. The end of the stream is consulted +// only when the head identifies nothing supported, which is how a zip appended to a launcher script (a +// Spring Boot executable jar) is found, and how a java resource adapter (a zip named .rar) is read. +func identifyFormat(ctx context.Context, name string, content ReaderAtSeeker) (archives.Extractor, error) { + if _, err := content.Seek(0, io.SeekStart); err != nil { + return nil, nil + } + format, _, err := intFile.IdentifyArchive(ctx, name, content) + if err == nil { + if extractor := supportedExtractor(format); extractor != nil { + return extractor, nil + } + } + if HasZipEndOfCentralDirectory(content) { + return archives.Zip{}, nil + } + if err == nil && holdsFiles(format) { + return nil, fmt.Errorf("%w: %s", ErrUnsupportedFormat, format.MediaType()) + } + return nil, nil +} + +// holdsFiles reports whether the format is an archive of files rather than a bare compression format, +// such as a gzipped man page, which holds one stream and is not cataloged as an archive. +func holdsFiles(format archives.Format) bool { + switch f := format.(type) { + case archives.CompressedArchive: + return f.Extraction != nil + case archives.Extractor: + return true + } + return false +} + +// supportedExtractor admits only the zip and tar families, the tar family optionally compressed. Other +// formats mholt identifies, such as 7z and rar, pull in decoders that have not been exercised against +// untrusted input here, so they are not opened until they are deliberately supported. A bare +// compression format, such as a gzipped file that is not a tar, is not extractable either. +func supportedExtractor(format archives.Format) archives.Extractor { + switch f := format.(type) { + case archives.Zip: + return f + case archives.Tar: + return f + case archives.CompressedArchive: + if _, ok := f.Extraction.(archives.Tar); ok { + return f + } + } + return nil +} + +// extract adds every entry of the archive. Reaching the disk limit stops the walk and marks the +// resolver truncated rather than failing. +func (r *Resolver) extract(ctx context.Context, format archives.Extractor, content ReaderAtSeeker, exclusions Exclusions) error { + if _, err := content.Seek(0, io.SeekStart); err != nil { + return err + } + var source io.Reader = content + if ca, ok := format.(archives.CompressedArchive); ok && ca.Compression != nil { + // a compressed tar decompresses everything to reach each header, including entries skipped whole + // and whatever an entry leaves unread, so the budget is charged for every byte the decoder + // produces rather than for what is kept. A zip or a plain tar seeks past what it skips + rc, err := ca.OpenReader(content) + if err != nil { + return err + } + defer internal.CloseAndLogError(rc, "decompressor") + source, format, r.metered = &meteredReader{ctx: ctx, reader: rc, budget: r.budget}, ca.Extraction, true + } + err := format.Extract(ctx, source, func(_ context.Context, f archives.FileInfo) error { + hdr, ok := r.entryHeader(f) + if !ok || exclusions.Excludes(hdr.Name) { + return nil + } + if !f.Mode().IsRegular() { + return r.add(hdr, nil) + } + entry, err := f.Open() + if err != nil { + log.WithFields("entry", hdr.Name, "error", err).Trace("unable to open archive entry, skipping it") + return nil + } + defer func() { + if err := entry.Close(); err != nil { + log.WithFields("entry", hdr.Name, "error", err).Trace("unable to close archive entry") + } + }() + return r.add(hdr, entry) + }) + r.finish() + if errors.Is(err, ErrDiskLimitReached) { + r.truncate(fmt.Errorf("extraction stopped: %w", err)) + return nil + } + if errors.Is(err, errBudgetSpent) { + r.truncate(fmt.Errorf("extraction stopped, possible decompression bomb: %w", err)) + return nil + } + if err != nil && ctx.Err() == nil && r.entries > 0 { + // the stream broke partway (a truncated .tar.gz); what was read before that is still good + r.truncate(fmt.Errorf("extraction stopped early: %w", err)) + return nil + } + return err +} + +// archiveDigestHashes are the digests taken of an archive as a whole: SHA-1, matching what the java +// cataloger reports for a jar. +var archiveDigestHashes = []crypto.Hash{crypto.SHA1} + +// digestsOf digests the archive's own bytes while they are in hand. A failure is logged and returns +// nothing rather than discarding an archive that extracted fine. +func digestsOf(ctx context.Context, content ReaderAtSeeker, archivePath string) []file.Digest { + if _, err := content.Seek(0, io.SeekStart); err != nil { + log.WithFields("archive", archivePath, "error", err).Trace("unable to rewind archive content for digests") + return nil + } + digests, err := intFile.NewDigestsFromFile(ctx, io.NopCloser(content), archiveDigestHashes) + if err != nil { + log.WithFields("archive", archivePath, "error", err).Trace("unable to digest archive") + return nil + } + return digests +} + +var ( + // zipLocalFileHeaderSignature starts every entry in a zip. + zipLocalFileHeaderSignature = []byte{'P', 'K', 0x03, 0x04} + + // zipEOCDSignature starts the end-of-central-directory record that closes every zip. Readers + // locate the central directory from it, which is why a zip with data prepended still opens. + zipEOCDSignature = []byte{'P', 'K', 0x05, 0x06} +) + +const ( + // maxAppendedArchiveStubBytes is how far into a file an appended zip's first entry is looked for. A + // Spring Boot launcher stub is about 9 KB. + maxAppendedArchiveStubBytes = 64 * 1024 + + // maxZipEOCDSearch is how far from the end the end-of-central-directory record can start: the 22 + // byte record plus a comment of up to 65535 bytes. + maxZipEOCDSearch = 22 + 65535 +) + +// MayHideAnAppendedArchive reports whether a zip entry signature appears in the head of the stream, +// which a zip appended to a launcher stub does and a file that merely has an archive's extension +// almost never does. The returned reader replays what was consumed. +func MayHideAnAppendedArchive(r io.Reader) (io.Reader, bool) { + head := make([]byte, maxAppendedArchiveStubBytes) + n, err := io.ReadFull(r, head) + if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { + return r, false + } + head = head[:n] + found := bytes.Contains(head, zipLocalFileHeaderSignature) + + if seeker, ok := r.(io.Seeker); ok { + if _, err := seeker.Seek(0, io.SeekStart); err == nil { + return r, found + } + } + return io.MultiReader(bytes.NewReader(head), r), found +} + +// HasZipEndOfCentralDirectory reports whether the content ends in a zip end-of-central-directory +// record, the one part of a zip whose position is known without parsing it. +func HasZipEndOfCentralDirectory(r ReaderAtSeeker) bool { + size, err := r.Seek(0, io.SeekEnd) + if err != nil || size < int64(len(zipEOCDSignature)) { + return false + } + window := min(size, int64(maxZipEOCDSearch)) + buf := make([]byte, window) + if _, err := r.ReadAt(buf, size-window); err != nil && !errors.Is(err, io.EOF) { + return false + } + return bytes.Contains(buf, zipEOCDSignature) +} diff --git a/internal/archive/extract_test.go b/internal/archive/extract_test.go new file mode 100644 index 000000000..de9c4bfa9 --- /dev/null +++ b/internal/archive/extract_test.go @@ -0,0 +1,706 @@ +package archive + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "context" + "fmt" + "io" + "io/fs" + "math/rand" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/mholt/archives" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + stereoscopeFile "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/file" +) + +var sampleFiles = map[string]string{ + "file1.txt": "content1", + "dir/file2.txt": "content2", +} + +func Test_identifyFormat(t *testing.T) { + ctx := context.Background() + launcher := append([]byte("#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n"), zipBytes(t, sampleFiles)...) + + tests := []struct { + name string + data []byte + want archives.Extractor + unsupported bool + }{ + {"test.zip", zipBytes(t, sampleFiles), archives.Zip{}, false}, + {"test.tar.gz", tarGzBytes(t, sampleFiles), archives.CompressedArchive{}, false}, + {"plain.txt", []byte("just text"), nil, false}, + // identified by mholt, but not opened until their decoders are deliberately supported + {"test.7z", []byte("7z\xbc\xaf\x27\x1c\x00\x04"), nil, true}, + {"test.rar", []byte("Rar!\x1a\x07\x01\x00"), nil, true}, + // a bare compression format holds one stream, not files + {"page.1.gz", gzBytes(t, "a man page"), nil, false}, + // a java resource adapter archive is a zip that shares rar's extension + {"adapter.rar", zipBytes(t, sampleFiles), archives.Zip{}, false}, + // the name can be claimed by identification, so the probe alone must not admit prose + {"app.jar", []byte("plain text, no central directory anywhere in it"), nil, false}, + {"bundle", []byte("plain text, no central directory anywhere in it"), nil, false}, + // content sniffing types this as a shell script; only its tail says zip + {"app.jar", launcher, archives.Zip{}, false}, + } + for _, tt := range tests { + t.Run(fmt.Sprintf("%s/%d bytes", tt.name, len(tt.data)), func(t *testing.T) { + got, err := identifyFormat(ctx, tt.name, bytes.NewReader(tt.data)) + if tt.unsupported { + assert.ErrorIs(t, err, ErrUnsupportedFormat) + } else { + assert.NoError(t, err) + } + if tt.want == nil { + assert.Nil(t, got) + return + } + assert.IsType(t, tt.want, got) + }) + } +} + +func TestResolver_extract_storesEveryEntryInMemoryWhenUnbounded(t *testing.T) { + for name, data := range map[string][]byte{ + "test.zip": zipBytes(t, sampleFiles), + "test.tar.gz": tarGzBytes(t, sampleFiles), + } { + t.Run(name, func(t *testing.T) { + r, root := extractBytes(t, data, name, nil) + assert.NoError(t, r.Truncated) + assert.Zero(t, r.written) + assert.Empty(t, filesIn(t, root)) + + entries := readStore(t, r) + assert.Equal(t, "content1", entries["file1.txt"].body) + assert.Equal(t, "content2", entries["dir/file2.txt"].body) + }) + } +} + +func TestResolver_extract_diskLimitTruncatesAtTheEntryThatDoesNotFit(t *testing.T) { + // each entry costs its index estimate plus its content on disk + record := approxIndexBytes(tar.Header{Name: "file1.txt"}) + data := zipBytes(t, map[string]string{"file1.txt": "a", "file2.txt": "b", "file3.txt": "c"}) + + r, _ := extractBytes(t, data, "test.zip", diskCharge(2*(record+1)+record)) + + assert.Error(t, r.Truncated) + assert.Equal(t, []string{"file1.txt", "file2.txt"}, storedNames(t, r), "the refused entry is not stored") + entries := readStore(t, r) + assert.Equal(t, "a", entries["file1.txt"].body) + assert.Equal(t, "b", entries["file2.txt"].body) +} + +func TestResolver_extract_diskLimitBitesMidEntry(t *testing.T) { + // a tar is walked entry by entry, so the limit can bite part way through an entry + data := tarGzBytes(t, map[string]string{ + "a/first.txt": strings.Repeat("a", 40*1024), + "b/second.txt": strings.Repeat("b", 40*1024), + }) + const budget = 80 * 1024 + charge := diskCharge(budget) + + r, root := extractBytes(t, data, "test.tar.gz", charge) + + assert.Error(t, r.Truncated) + assert.Equal(t, []string{"a", "a/first.txt", "b"}, storedNames(t, r), "the directory a refused entry implied stays") + assert.Greater(t, r.written, int64(40*1024), "the first entry landed whole and the second part way") + _, disk := charge.held() + assert.LessOrEqual(t, disk, int64(budget)) + assert.Greater(t, disk, r.written, "the index estimate is charged to disk too") + + info, err := os.Stat(filepath.Join(root, spillFile(t, root))) + require.NoError(t, err) + assert.Equal(t, r.written, info.Size()) +} + +func TestResolver_extract_directoryOnlyArchivesAreBoundedByTheIndexEstimate(t *testing.T) { + var entries []testEntry + for i := range 50 { + entries = append(entries, testEntry{name: fmt.Sprintf("d%04d/", i), dir: true}) + } + room := approxIndexBytes(tar.Header{Name: "d0000"}) * 10 + + for name, data := range map[string][]byte{ + "dirs.zip": zipFromEntries(t, entries), + "dirs.tar.gz": tarGzFromEntries(t, entries), + } { + t.Run(name, func(t *testing.T) { + r, _ := extractBytes(t, data, name, diskCharge(room)) + assert.Error(t, r.Truncated) + assert.Len(t, storedNames(t, r), 10) + }) + } +} + +func TestResolver_extract_entryNamesAreCleanedAndNothingIsWrittenUnderThem(t *testing.T) { + entries := []testEntry{ + {name: "../../etc/passwd", body: "evil"}, + {name: "/etc/hosts", body: "pwned"}, + {name: "META-INF/", dir: true, mode: 0o555}, + {name: "META-INF/MANIFEST.MF", body: "Manifest-Version: 1.0"}, + {name: "passwd", link: "../../../../etc/passwd"}, + {name: "shadow", link: "/etc/shadow"}, + {name: "bin/link.txt", link: "../lib/real.txt"}, + } + + for name, data := range map[string][]byte{ + "evil.zip": zipFromEntries(t, entries), + "evil.tar.gz": tarGzFromEntries(t, entries), + } { + t.Run(name, func(t *testing.T) { + r, root := extractBytes(t, data, name, nil) + require.NoError(t, r.Truncated) + + assert.Equal(t, []string{"META-INF", "META-INF/MANIFEST.MF", "bin", "bin/link.txt", "etc", "etc/hosts", "etc/passwd", "passwd", "shadow"}, + storedNames(t, r)) + assert.Empty(t, filesIn(t, root), "entries live in the store, never under their own names") + assert.NoFileExists(t, filepath.Join(t.TempDir(), "..", "etc", "passwd")) + + got := readStore(t, r) + assert.Equal(t, "evil", got["etc/passwd"].body) + assert.Equal(t, "pwned", got["etc/hosts"].body) + assert.Equal(t, stereoscopeFile.TypeDirectory, got["META-INF"].kind) + assert.Equal(t, fs.FileMode(0o555), got["META-INF"].mode.Perm()) + assert.Equal(t, "Manifest-Version: 1.0", got["META-INF/MANIFEST.MF"].body) + + // links are headers carrying their target verbatim; the index resolves them inside the archive + for _, link := range []string{"passwd", "shadow", "bin/link.txt"} { + assert.Equal(t, stereoscopeFile.TypeSymLink, got[link].kind) + assert.Empty(t, got[link].body) + } + assert.Equal(t, "../../../../etc/passwd", got["passwd"].linkTarget) + assert.Equal(t, "/etc/shadow", got["shadow"].linkTarget) + assert.Equal(t, "../lib/real.txt", got["bin/link.txt"].linkTarget) + }) + } +} + +func TestResolver_extract_dataEdgeCases(t *testing.T) { + t.Run("an empty archive is not an error", func(t *testing.T) { + r, _ := extractBytes(t, zipBytes(t, map[string]string{}), "test.zip", nil) + assert.Zero(t, r.entries) + assert.NoError(t, r.Truncated) + }) + + t.Run("a zero-byte entry costs no disk", func(t *testing.T) { + r, _ := extractBytes(t, zipBytes(t, map[string]string{"empty.txt": ""}), "test.zip", memCharge(0)) + assert.Equal(t, []string{"empty.txt"}, storedNames(t, r)) + assert.Zero(t, r.written) + assert.Empty(t, readStore(t, r)["empty.txt"].body) + }) + + t.Run("unicode entry names survive", func(t *testing.T) { + names := map[string]string{ + "café/naïve.txt": "accents", + "日本語/ファイル.txt": "japanese", + "emoji-🎉.txt": "emoji", + } + r, _ := extractBytes(t, zipBytes(t, names), "test.zip", nil) + entries := readStore(t, r) + for name, want := range names { + assert.Equal(t, want, entries[name].body, name) + } + }) + + t.Run("an entry name with too many path components is skipped and reported", func(t *testing.T) { + atCap := strings.Repeat("a/", maxEntryNameDepth-1) + "f" + data := zipBytes(t, map[string]string{ + atCap: "at the cap", + strings.Repeat("b/", maxEntryNameDepth) + "f": "over the cap", + }) + r, _ := extractBytes(t, data, "test.zip", nil) + names := storedNames(t, r) + assert.Contains(t, names, atCap) + assert.Len(t, names, maxEntryNameDepth, "only the entry at the cap and its implied directories") + assert.ErrorContains(t, r.Truncated, "path components") + }) + + t.Run("an entry whose chain is longer than a path is skipped and reported", func(t *testing.T) { + // the chain is the archive's own path, the separator, then the name + fits := strings.Repeat("a", MaxPathBytes-len("test-archive!/")) + data := zipBytes(t, map[string]string{ + fits: "at the cap", + fits + "b": "over the cap", + }) + r, _ := extractBytes(t, data, "test.zip", nil) + assert.Equal(t, []string{fits}, storedNames(t, r)) + assert.ErrorContains(t, r.Truncated, "archive chain") + }) + + t.Run("an over-long entry name is skipped", func(t *testing.T) { + data := zipBytes(t, map[string]string{ + strings.Repeat("a/", maxEntryNameBytes) + "f": "deep", + "ok.txt": "fine", + }) + r, _ := extractBytes(t, data, "test.zip", nil) + assert.Equal(t, []string{"ok.txt"}, storedNames(t, r)) + }) +} + +func TestResolver_extract_unsafeEntryNamesAreRefusedAndReported(t *testing.T) { + jar := string(zipBytes(t, map[string]string{"META-INF/MANIFEST.MF": "real"})) + tests := []struct { + name string + forged string + }{ + {name: "archive path separator", forged: "a.jar!/META-INF/MANIFEST.MF"}, + {name: "line break", forged: "x\nlayerID: sha256:forged"}, + {name: "unicode line separator", forged: "x\u2028y"}, + {name: "C1 control", forged: "x\u0085y"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data := zipBytes(t, map[string]string{"a.jar": jar, tt.forged: "forged"}) + r, _ := extractBytes(t, data, "outer.zip", nil) + assert.Equal(t, []string{"a.jar"}, storedNames(t, r)) + require.Error(t, r.Truncated) + assert.Contains(t, r.Truncated.Error(), fmt.Sprintf("%q", tt.forged)) + }) + } +} + +func TestResolver_extract_excludedEntriesAreNeverStored(t *testing.T) { + data := zipBytes(t, map[string]string{ + "keep.txt": "kept", + "vendor/lib.go": "excluded by directory", + "deep/down/pkg.rpm": "excluded by name", + }) + limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1}) + r, _ := resolverIn(t, limiter.charge()) + content := bytes.NewReader(data) + + format, err := identifyFormat(context.Background(), "test.zip", content) + require.NoError(t, err) + err = r.extract(context.Background(), format, content, + NewExclusions([]string{"**/vendor", "**/*.rpm"})) + require.NoError(t, err) + require.NoError(t, r.Truncated) + + assert.Equal(t, []string{"keep.txt"}, storedNames(t, r)) + mem, _ := limiter.InUse() + assert.Equal(t, approxIndexBytes(tar.Header{Name: "keep.txt"})+4, mem, "an excluded entry costs nothing") +} + +// a compressed tar reader decompresses a skipped entry to reach the next header, so skipping must still +// spend the budget or an excluded or oversized entry would decompress without bound +func TestResolver_extract_skippedCompressedTarBytesSpendTheBudget(t *testing.T) { + big := strings.Repeat("\x00", 4<<20) + tests := []struct { + name string + exclusions []string + entryCap int64 + }{ + {name: "excluded entry", exclusions: []string{"**/*.bin"}, entryCap: maxEntryBytes}, + {name: "entry over the size cap", entryCap: 1 << 10}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer func(prev int64) { maxEntryBytes = prev }(maxEntryBytes) + maxEntryBytes = tt.entryCap + + data := tarGzFromEntries(t, []testEntry{{name: "bomb.bin", body: big}, {name: "after.txt", body: "after"}}) + r, _ := resolverIn(t, memCharge(1<<30)) + r.budget = NewBudget(1 << 20) + content := bytes.NewReader(data) + format, err := identifyFormat(context.Background(), "test.tar.gz", content) + require.NoError(t, err) + + require.NoError(t, r.extract(context.Background(), format, content, NewExclusions(tt.exclusions))) + require.Error(t, r.Truncated) + assert.Empty(t, storedNames(t, r), "the walk stopped at the entry that spent the budget") + assert.Zero(t, r.budget.remaining) + }) + } +} + +// a body behind a header that doesn't read as a regular file (here a socket per its mode bits) is never +// opened, but the tar reader still decompresses it to reach the next header +func TestResolver_extract_bodiesOfNonRegularTarEntriesSpendTheBudget(t *testing.T) { + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + body := strings.Repeat("\x00", 4<<20) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "sock", Typeflag: tar.TypeReg, Mode: 0o140644, Size: int64(len(body))})) + _, err := tw.Write([]byte(body)) + require.NoError(t, err) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "after.txt", Typeflag: tar.TypeReg, Mode: 0o644, Size: 5})) + _, err = tw.Write([]byte("after")) + require.NoError(t, err) + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + + r, _ := resolverIn(t, memCharge(1<<30)) + r.budget = NewBudget(1 << 20) + content := bytes.NewReader(buf.Bytes()) + format, err := identifyFormat(context.Background(), "test.tar.gz", content) + require.NoError(t, err) + + require.NoError(t, r.extract(context.Background(), format, content, nil)) + assert.ErrorIs(t, r.Truncated, errBudgetSpent) + assert.Empty(t, storedNames(t, r)) + assert.Zero(t, r.budget.remaining) +} + +// a zip or a plain tar seeks past what it skips, so an excluded entry there costs nothing +func TestResolver_extract_skippedSeekableBytesAreFree(t *testing.T) { + entries := []testEntry{{name: "bomb.bin", body: strings.Repeat("\x00", 4<<20)}, {name: "after.txt", body: "after"}} + var plainTar bytes.Buffer + tw := tar.NewWriter(&plainTar) + for _, e := range entries { + hdr := e.header() + require.NoError(t, tw.WriteHeader(&hdr)) + _, err := tw.Write([]byte(e.body)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + + for name, data := range map[string][]byte{"test.zip": zipFromEntries(t, entries), "test.tar": plainTar.Bytes()} { + t.Run(name, func(t *testing.T) { + r, _ := resolverIn(t, memCharge(1<<30)) + r.budget = NewBudget(1 << 20) + content := bytes.NewReader(data) + format, err := identifyFormat(context.Background(), name, content) + require.NoError(t, err) + + require.NoError(t, r.extract(context.Background(), format, content, NewExclusions([]string{"**/*.bin"}))) + require.NoError(t, r.Truncated) + assert.Equal(t, []string{"after.txt"}, storedNames(t, r)) + }) + } +} + +func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) { + data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"}) + + extracted, err := Extract(context.Background(), bytes.NewReader(data), file.Coordinates{RealPath: "app.war", FileSystemID: "parentFS", FullPath: "app.war!/some/path/app.zip"}, nil, nil) + require.NoError(t, err) + require.NotNil(t, extracted) + t.Cleanup(extracted.Cleanup) + + assert.NoError(t, extracted.Truncated) + require.Len(t, extracted.Digests, 1) + assert.Equal(t, "sha1", extracted.Digests[0].Algorithm) + + locations, err := extracted.FilesByGlob("**/*.txt") + require.NoError(t, err) + require.Len(t, locations, 1) + assert.Equal(t, file.Coordinates{RealPath: "app.war", FileSystemID: "parentFS", FullPath: "app.war!/some/path/app.zip!/dir/hello.txt"}, + locations[0].Coordinates) + + reader, err := extracted.FileContentsByLocation(locations[0]) + require.NoError(t, err) + body, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "hello world", string(body)) +} + +func TestExtract_heldContentIsReleasedOnceExtracted(t *testing.T) { + // the archive's own bytes are only needed to extract and digest it; afterwards the limiter holds + // the entries and the index estimate alone + data := zipBytes(t, map[string]string{"hello.txt": "hi"}) + limiter := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) + + extracted, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil) + require.NoError(t, err) + t.Cleanup(extracted.Cleanup) + + mem, disk := limiter.InUse() + assert.Equal(t, approxIndexBytes(tar.Header{Name: "hello.txt"})+2, mem) + assert.Zero(t, disk) +} + +func TestExtract_heldContentWrittenToDiskStaysUntilCleanup(t *testing.T) { + // only a top-level stream is ever held; a nested archive is read out of its parent, which is held + // for the whole descent anyway, so spilled bytes wait for Cleanup with the entries + ctx, root := scanContext(t) + data := zipBytes(t, map[string]string{"hello.txt": "hi"}) + limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) + + extracted, err := Extract(ctx, unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil) + require.NoError(t, err) + t.Cleanup(extracted.Cleanup) + + require.Len(t, filesIn(t, root), 1, "the archive's bytes and its entries share one file") + _, disk := limiter.InUse() + assert.Equal(t, approxIndexBytes(tar.Header{Name: "hello.txt"})+2+int64(len(data)), disk) + + extracted.Cleanup() + assert.Empty(t, filesIn(t, root)) + _, disk = limiter.InUse() + assert.Zero(t, disk) +} + +func TestExtract_aTruncatedStreamKeepsWhatWasRead(t *testing.T) { + // a .tar.gz cut off partway breaks the stream for every entry after the cut; the ones before it are + // still good and must not be thrown away + noise := make([]byte, 64*1024) + rand.New(rand.NewSource(1)).Read(noise) + data := tarGzBytes(t, map[string]string{"a.txt": "first", "b.bin": string(noise)}) + cut := data[:len(data)/2] + + extracted, err := Extract(context.Background(), bytes.NewReader(cut), file.NewCoordinates("app.tar.gz", ""), NewLimiter(unboundedLimits), nil) + require.NoError(t, err) + require.NotNil(t, extracted) + t.Cleanup(extracted.Cleanup) + + assert.Error(t, extracted.Truncated) + locations, err := extracted.FilesByGlob("**") + require.NoError(t, err) + assert.Equal(t, []string{"/a.txt"}, localPaths(locations)) +} + +func TestExtract_aPanicStillReleasesEverything(t *testing.T) { + // a decoder panicking on hostile bytes must not leak the charge or the spill file + ctx, root := scanContext(t) + limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) + + require.Panics(t, func() { + _, _ = Extract(ctx, &panicsAfter{n: 4096}, file.NewCoordinates("app.zip", ""), limiter, nil) + }) + + mem, disk := limiter.InUse() + assert.Zero(t, mem) + assert.Zero(t, disk) + assert.Empty(t, filesIn(t, root)) +} + +// panicsAfter hands out n zero bytes, then panics. +type panicsAfter struct{ n int } + +func (p *panicsAfter) Read(b []byte) (int, error) { + if p.n <= 0 { + panic("decoder blew up") + } + n := min(len(b), p.n) + clear(b[:n]) + p.n -= n + return n, nil +} + +func TestExtract_archiveBytesThatCannotBePlacedAreSkipped(t *testing.T) { + data := zipBytes(t, sampleFiles) + limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}) + + _, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil) + assert.ErrorIs(t, err, ErrDiskLimitReached) + + // read in place, the same bytes cost nothing; only the entries are refused + extracted, err := Extract(context.Background(), bytes.NewReader(data), file.NewCoordinates("app.zip", ""), limiter, nil) + require.NoError(t, err) + require.NotNil(t, extracted) + t.Cleanup(extracted.Cleanup) + assert.Error(t, extracted.Truncated) + locations, err := extracted.FilesByGlob("**/*") + require.NoError(t, err) + assert.Empty(t, locations) +} + +func Test_HasZipEndOfCentralDirectory(t *testing.T) { + tests := []struct { + name string + data []byte + want bool + }{ + {"a plain zip", zipBytes(t, sampleFiles), true}, + {"a zip behind a launcher script", append([]byte("#!/bin/bash\nexit 0\n"), zipBytes(t, sampleFiles)...), true}, + {"prose", []byte("this is not an archive, it is a sentence"), false}, + {"a tar.gz", tarGzBytes(t, sampleFiles), false}, + {"empty", nil, false}, + {"shorter than the signature", []byte("PK"), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, HasZipEndOfCentralDirectory(bytes.NewReader(tt.data))) + }) + } +} + +func Test_MayHideAnAppendedArchive(t *testing.T) { + launcher := append([]byte("#!/bin/bash\nexit 0\n"), zipBytes(t, sampleFiles)...) + prose := []byte("plain text, whatever the extension claims") + + t.Run("a seekable reader is rewound and returned as is", func(t *testing.T) { + reader := bytes.NewReader(launcher) + got, found := MayHideAnAppendedArchive(reader) + assert.True(t, found) + assert.Same(t, reader, got) + rest, err := io.ReadAll(got) + require.NoError(t, err) + assert.Equal(t, launcher, rest) + }) + + t.Run("an unseekable reader replays what was consumed", func(t *testing.T) { + got, found := MayHideAnAppendedArchive(unseekable{bytes.NewReader(launcher)}) + assert.True(t, found) + rest, err := io.ReadAll(got) + require.NoError(t, err) + assert.Equal(t, launcher, rest) + }) + + t.Run("prose hides nothing", func(t *testing.T) { + got, found := MayHideAnAppendedArchive(unseekable{bytes.NewReader(prose)}) + assert.False(t, found) + rest, err := io.ReadAll(got) + require.NoError(t, err) + assert.Equal(t, prose, rest) + }) +} + +// zeros is a reader of n zero bytes: the most compressible content there is. +func zeros(n int64) io.Reader { + return io.LimitReader(zeroSource{}, n) +} + +type zeroSource struct{} + +func (zeroSource) Read(p []byte) (int, error) { + clear(p) + return len(p), nil +} + +func TestExtract_decompressionBombIsBoundedByTheLimits(t *testing.T) { + // 64 MiB of zeros compresses to a few hundred KiB. The declared size is never trusted: bytes are + // charged as they land, so the limits hold whatever the archive claims + const inflated = 64 << 20 + limits := Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: 4 << 20} + + var zipBomb bytes.Buffer + zw := zip.NewWriter(&zipBomb) + w, err := zw.Create("bomb.bin") + require.NoError(t, err) + _, err = io.Copy(w, zeros(inflated)) + require.NoError(t, err) + require.NoError(t, zw.Close()) + + var tarBomb bytes.Buffer + gw := gzip.NewWriter(&tarBomb) + tw := tar.NewWriter(gw) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "bomb.bin", Mode: 0o644, Size: inflated, Typeflag: tar.TypeReg})) + _, err = io.Copy(tw, zeros(inflated)) + require.NoError(t, err) + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + + for name, data := range map[string][]byte{"bomb.zip": zipBomb.Bytes(), "bomb.tar.gz": tarBomb.Bytes()} { + t.Run(name, func(t *testing.T) { + require.Less(t, len(data), 1<<20, "the fixture must be a bomb: small on disk, huge inflated") + ctx, root := scanContext(t) + limiter := NewLimiter(limits) + + r, err := Extract(ctx, bytes.NewReader(data), file.NewCoordinates(name, ""), limiter, nil) + require.NoError(t, err) + require.NotNil(t, r) + t.Cleanup(r.Cleanup) + + assert.Error(t, r.Truncated) + locations, err := r.FilesByGlob("**/*") + require.NoError(t, err) + assert.Empty(t, locations, "the entry that did not fit is not cataloged in part") + + peakMemory, peakDisk := limiter.Peak() + assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes) + assert.LessOrEqual(t, peakDisk, limits.MaxDiskBytes) + + r.Cleanup() + assert.Empty(t, filesIn(t, root)) + }) + } +} + +func TestExtract_entryCountBombIsBoundedByTheMemoryLimit(t *testing.T) { + // every entry is a node in memory whatever the disk limit says, so a tiny archive of many entries + // is bounded by the memory limit alone + limits := Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1} + limiter := NewLimiter(limits) + + r, err := Extract(context.Background(), bytes.NewReader(manyEntryZip(t, 20_000)), file.NewCoordinates("many.zip", ""), limiter, nil) + require.NoError(t, err) + t.Cleanup(r.Cleanup) + + assert.Error(t, r.Truncated) + assert.Less(t, r.entries, 20_000) + assert.LessOrEqual(t, int64(nodeCount(r))*approxIndexBytesPerEntry, limits.MaxMemoryBytes+approxIndexBytesPerEntry, + "nodes held must not exceed what the memory limit paid for") + peakMemory, _ := limiter.Peak() + assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes) +} + +func TestExtract_deepPathBombIsBoundedByTheMemoryLimit(t *testing.T) { + // 100 entries whose names each imply 250 distinct directories would be 25,000 nodes; the implied + // directories are charged as nodes, so the memory limit bounds them too + files := map[string]string{} + for i := range 100 { + var b strings.Builder + for j := range maxEntryNameDepth - 6 { + fmt.Fprintf(&b, "%03d-%03d/", i, j) + } + b.WriteString("f") + require.LessOrEqual(t, b.Len(), maxEntryNameBytes) + files[b.String()] = "x" + } + limits := Limits{MaxMemoryBytes: 8 << 20, MaxDiskBytes: -1} + limiter := NewLimiter(limits) + + r, err := Extract(context.Background(), bytes.NewReader(zipBytes(t, files)), file.NewCoordinates("deep.zip", ""), limiter, nil) + require.NoError(t, err) + t.Cleanup(r.Cleanup) + + assert.Error(t, r.Truncated) + assert.LessOrEqual(t, int64(nodeCount(r))*approxIndexBytesPerEntry, limits.MaxMemoryBytes+approxIndexBytesPerEntry) + peakMemory, _ := limiter.Peak() + assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes) +} + +func TestExtract_locationsKeepTheTopLevelRealPath(t *testing.T) { + data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"}) + + // a top-level archive's coordinates, with FullPath unset or set to RealPath, name entries alike + for _, archive := range []file.Coordinates{ + {RealPath: "/app.zip"}, + {RealPath: "/app.zip", FullPath: "/app.zip"}, + } { + extracted, err := Extract(context.Background(), bytes.NewReader(data), archive, nil, nil) + require.NoError(t, err) + require.NotNil(t, extracted) + t.Cleanup(extracted.Cleanup) + + locations, err := extracted.FilesByPath("dir/hello.txt") + require.NoError(t, err) + require.Len(t, locations, 1) + assert.Equal(t, file.Coordinates{RealPath: "/app.zip", FullPath: "/app.zip!/dir/hello.txt"}, locations[0].Coordinates) + assert.Equal(t, "/dir/hello.txt", locations[0].LocalPath()) + + for _, loc := range []file.Location{ + locations[0], + file.NewLocation("dir/hello.txt"), + file.NewLocation("/dir/hello.txt"), + file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/app.zip", FullPath: "/app.zip!/dir/hello.txt"}), + } { + reader, err := extracted.FileContentsByLocation(loc) + require.NoError(t, err, "location %s", loc) + body, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "hello world", string(body)) + + metadata, err := extracted.FileMetadataByLocation(loc) + require.NoError(t, err, "location %s", loc) + assert.Equal(t, "/app.zip!/dir/hello.txt", metadata.Path) + } + + _, err = extracted.FileContentsByLocation(file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/other.zip", FullPath: "/other.zip!/dir/hello.txt"})) + assert.Error(t, err, "an entry of another archive is not read from this one") + } +} diff --git a/internal/archive/helpers_test.go b/internal/archive/helpers_test.go new file mode 100644 index 000000000..bd37a94f6 --- /dev/null +++ b/internal/archive/helpers_test.go @@ -0,0 +1,344 @@ +package archive + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "context" + "fmt" + "io" + "io/fs" + "os" + "sort" + "testing" + + "github.com/stretchr/testify/require" + + stereoscopeFile "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/internal/tmpdir" + "github.com/anchore/syft/syft/file" +) + +// held reports what this charge is currently holding, in memory and on disk. +func (c *charge) held() (memory, disk int64) { + if c == nil { + return 0, 0 + } + c.limiter.mu.Lock() + defer c.limiter.mu.Unlock() + return c.inMemory, c.onDisk +} + +// heldInMemory reports the content the resolver is still holding in memory. +func (r *Resolver) heldInMemory() int64 { + var total int64 + for _, b := range r.held { + total += int64(len(b.mem)) + } + return total +} + +// unseekable hides a reader's Seek and ReadAt, forcing content through the held path. +type unseekable struct{ io.Reader } + +// testEntry is one entry of a fixture archive: a directory when dir is set, a link when link is set +// (hard when hard is set, otherwise symbolic), otherwise a regular file. +type testEntry struct { + name string + body string + link string + hard bool + dir bool + mode fs.FileMode +} + +// header is the tar header this entry would carry. +func (e testEntry) header() tar.Header { + switch { + case e.dir: + return tar.Header{Name: e.name, Mode: int64(e.modeOr(0o755)), Typeflag: tar.TypeDir} + case e.link != "" && e.hard: + return tar.Header{Name: e.name, Mode: 0o644, Typeflag: tar.TypeLink, Linkname: e.link} + case e.link != "": + return tar.Header{Name: e.name, Mode: 0o777, Typeflag: tar.TypeSymlink, Linkname: e.link} + } + return tar.Header{Name: e.name, Mode: int64(e.modeOr(0o644)), Size: int64(len(e.body)), Typeflag: tar.TypeReg} +} + +func fileEntries(files map[string]string) []testEntry { + names := make([]string, 0, len(files)) + for name := range files { + names = append(names, name) + } + // limits bite as the walk proceeds, so entry order decides what lands before a truncation + sort.Strings(names) + entries := make([]testEntry, 0, len(names)) + for _, name := range names { + entries = append(entries, testEntry{name: name, body: files[name]}) + } + return entries +} + +func zipBytes(t testing.TB, files map[string]string) []byte { + return zipFromEntries(t, fileEntries(files)) +} + +func zipFromEntries(t testing.TB, entries []testEntry) []byte { + t.Helper() + var buf bytes.Buffer + w := zip.NewWriter(&buf) + for _, e := range entries { + hdr := &zip.FileHeader{Name: e.name} + body := e.body + switch { + case e.dir: + hdr.SetMode(fs.ModeDir | e.modeOr(0o755)) + case e.link != "": + hdr.SetMode(fs.ModeSymlink | 0o777) + body = e.link + default: + hdr.SetMode(e.modeOr(0o644)) + } + fw, err := w.CreateHeader(hdr) + require.NoError(t, err) + _, err = fw.Write([]byte(body)) + require.NoError(t, err) + } + require.NoError(t, w.Close()) + return buf.Bytes() +} + +func gzBytes(t testing.TB, body string) []byte { + t.Helper() + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + _, err := gw.Write([]byte(body)) + require.NoError(t, err) + require.NoError(t, gw.Close()) + return buf.Bytes() +} + +func tarGzBytes(t testing.TB, files map[string]string) []byte { + return tarGzFromEntries(t, fileEntries(files)) +} + +func tarGzFromEntries(t testing.TB, entries []testEntry) []byte { + t.Helper() + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + for _, e := range entries { + hdr := e.header() + require.NoError(t, tw.WriteHeader(&hdr)) + _, err := tw.Write([]byte(e.body)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + return buf.Bytes() +} + +func (e testEntry) modeOr(fallback fs.FileMode) fs.FileMode { + if e.mode != 0 { + return e.mode + } + return fallback +} + +func regularHeader(name string, size int64) tar.Header { + return tar.Header{Name: name, Size: size, Mode: 0o600, Typeflag: tar.TypeReg} +} + +// scanContext returns a context whose temp root is a fresh directory, and that directory. +func scanContext(t testing.TB) (context.Context, string) { + t.Helper() + root := t.TempDir() + return tmpdir.WithValue(context.Background(), tmpdir.FromPath(root)), root +} + +// filesIn lists the regular files directly in dir. +func filesIn(t testing.TB, dir string) []string { + t.Helper() + entries, err := os.ReadDir(dir) + require.NoError(t, err) + var names []string + for _, e := range entries { + if !e.IsDir() { + names = append(names, e.Name()) + } + } + return names +} + +// spillFile is the one file a resolver has written, or "" when it has not. +func spillFile(t testing.TB, dir string) string { + t.Helper() + files := filesIn(t, dir) + require.LessOrEqual(t, len(files), 1) + if len(files) == 0 { + return "" + } + return files[0] +} + +// resolverIn makes an empty resolver over the given charge that spills into dir, along with that +// directory. Its entries are added by the test. +func resolverIn(t testing.TB, charge *charge) (*Resolver, string) { + t.Helper() + root := t.TempDir() + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(root)) + r := newResolver(ctx, file.NewCoordinates("test-archive", ""), charge) + t.Cleanup(r.Cleanup) + return r, root +} + +// addAll adds entries in the order given and finishes the resolver. +func addAll(t testing.TB, r *Resolver, entries []testEntry) { + t.Helper() + for _, e := range entries { + require.NoError(t, r.add(e.header(), bytes.NewReader([]byte(e.body)))) + } + r.finish() +} + +// resolverOver builds a resolver holding the given files within a memory bound. +func resolverOver(t testing.TB, maxInMemory int64, files map[string]string) *Resolver { + t.Helper() + r, _ := resolverIn(t, memCharge(maxInMemory)) + r.archiveCoordinates = file.NewCoordinates("outer.jar", "") + addAll(t, r, fileEntries(files)) + return r +} + +// resolverFrom builds a resolver holding the given entries, links and directories included, with +// ample memory. +func resolverFrom(t testing.TB, entries ...testEntry) *Resolver { + t.Helper() + r, _ := resolverIn(t, memCharge(1<<20)) + r.archiveCoordinates = file.NewCoordinates("outer.jar", "") + addAll(t, r, entries) + return r +} + +// extractBytes extracts an archive into a resolver built over the given charge. +func extractBytes(t testing.TB, data []byte, name string, charge *charge) (*Resolver, string) { + t.Helper() + r, root := resolverIn(t, charge) + content := bytes.NewReader(data) + format, err := identifyFormat(context.Background(), name, content) + require.NoError(t, err) + require.NotNil(t, format, "fixture must identify as an archive") + require.NoError(t, r.extract(context.Background(), format, content, nil)) + return r, root +} + +func readEntry(t testing.TB, r ReaderAtSeeker) string { + t.Helper() + _, err := r.Seek(0, io.SeekStart) + require.NoError(t, err) + b, err := io.ReadAll(r) + require.NoError(t, err) + return string(b) +} + +// storedEntry is one entry read back out of a resolver. +type storedEntry struct { + body string + linkTarget string + mode fs.FileMode + kind stereoscopeFile.Type +} + +// readStore reads every stored entry by name: files, links, and directories listed or implied. +func readStore(t testing.TB, r *Resolver) map[string]storedEntry { + t.Helper() + out := map[string]storedEntry{} + for _, ref := range r.tree.AllFiles(stereoscopeFile.AllTypes()...) { + entry, err := r.index.Get(ref) + require.NoError(t, err) + out[entry.Metadata.Path] = storedEntry{ + body: readEntry(t, r.open(r.content[ref.ID()])), + linkTarget: entry.Metadata.LinkDestination, + mode: entry.Metadata.FileInfo.Mode(), + kind: entry.Metadata.Type, + } + } + return out +} + +// storedNames lists every listed entry's name, files and directories alike. +func storedNames(t testing.TB, r *Resolver) []string { + t.Helper() + var names []string + for name := range readStore(t, r) { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// nodeCount is every path the tree holds, including directories the archive never listed. +func nodeCount(r *Resolver) int { + return len(r.tree.AllRealPaths()) +} + +// contentAt is the blob held for the entry at a rooted path. +func contentAt(t testing.TB, r *Resolver, p string) *blob { + t.Helper() + _, res, err := r.tree.File(stereoscopeFile.Path(p)) + require.NoError(t, err) + require.NotNil(t, res) + return r.content[res.Reference.ID()] +} + +// memCharge is a charge against a memory bound with disk unbounded. +func memCharge(maxMemory int64) *charge { + return NewLimiter(Limits{MaxMemoryBytes: maxMemory, MaxDiskBytes: -1}).charge() +} + +// diskCharge is a charge against a disk bound with memory refusing everything. +func diskCharge(maxDisk int64) *charge { + return NewLimiter(Limits{MaxDiskBytes: maxDisk}).charge() +} + +// localPaths lists the local entry path of every location (rather than full nested archive path), sorted. +func localPaths(locations []file.Location) []string { + var out []string + for _, loc := range locations { + out = append(out, loc.LocalPath()) + } + sort.Strings(out) + return out +} + +// manyEntryZip is a zip of count one-byte files. +func manyEntryZip(t testing.TB, count int) []byte { + t.Helper() + files := make(map[string]string, count) + for i := range count { + files[fmt.Sprintf("e%06d.txt", i)] = "x" + } + return zipBytes(t, files) +} + +// extractedResolver extracts one archive through Extract and returns the resolver, along with the scan +// temp root its spill file would appear under. +func extractedResolver(t testing.TB, data []byte, limits Limits) (*Resolver, string) { + t.Helper() + ctx, root := scanContext(t) + + r, err := Extract(ctx, bytes.NewReader(data), file.NewCoordinates("app.zip", ""), NewLimiter(limits), nil) + require.NoError(t, err) + require.NotNil(t, r) + t.Cleanup(r.Cleanup) + + return r, root +} + +var ( + unboundedLimits = Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1} + + // spillingLimits hold nothing in memory, so every entry lands in the spill file + spillingLimits = Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1} +) diff --git a/internal/archive/limits.go b/internal/archive/limits.go new file mode 100644 index 000000000..dd807d07f --- /dev/null +++ b/internal/archive/limits.go @@ -0,0 +1,202 @@ +package archive + +import ( + "errors" + "sync" +) + +// Budget bounds the bytes decompressed under one top-level archive, summed over every archive nested in +// it. Unlike Limits it never refills as archives are released: it bounds work, not what is held at once, +// which is what stops a recursive bomb (42.zip) whose every level fits the limits on its own. A nil +// *Budget admits everything. +// +// ponytail: no lock, since the archive walk is sequential; guard it if the walk ever runs concurrently. +type Budget struct { + remaining int64 +} + +func NewBudget(bytes int64) *Budget { + return &Budget{remaining: bytes} +} + +// take spends n bytes, or reports false (and spends the rest) when fewer than n remain. +func (b *Budget) take(n int64) bool { + if b == nil { + return true + } + if n > b.remaining { + b.remaining = 0 + return false + } + b.remaining -= n + return true +} + +var errBudgetSpent = errors.New("archive tree decompression budget spent") + +// ErrDiskLimitReached reports that placing an archive's content would exceed the disk limit. +var ErrDiskLimitReached = errors.New("archive content would exceed the disk limit") + +// Limits bound how much archive content one scan holds at once. Each limit falls as archives are +// released, so it bounds peak usage rather than the total ever written. +// +// For both limits: positive is the bound, zero forbids the resource, negative is unbounded. +type Limits struct { + // MaxMemoryBytes bounds archive content held in memory. Content that does not fit is written to + // disk instead. + MaxMemoryBytes int64 + + // MaxDiskBytes bounds archive content written to disk. Content that does not fit is dropped and + // the archive is skipped or truncated. + MaxDiskBytes int64 +} + +// Limiter tracks how much archive content a scan holds right now against its Limits. A nil *Limiter +// enforces nothing. +type Limiter struct { + limits Limits + + mu sync.Mutex + memory int64 + disk int64 + + peakMemory int64 + peakDisk int64 +} + +func NewLimiter(limits Limits) *Limiter { + return &Limiter{limits: limits} +} + +// Peak reports the most memory and disk held at any one moment. +func (l *Limiter) Peak() (memory, disk int64) { + if l == nil { + return 0, 0 + } + l.mu.Lock() + defer l.mu.Unlock() + return l.peakMemory, l.peakDisk +} + +// InUse reports the memory and disk held right now. +func (l *Limiter) InUse() (memory, disk int64) { + if l == nil { + return 0, 0 + } + l.mu.Lock() + defer l.mu.Unlock() + return l.memory, l.disk +} + +// charge starts accounting for one archive. Releasing the charge gives back everything it took. +func (l *Limiter) charge() *charge { + if l == nil { + return nil + } + return &charge{limiter: l} +} + +// charge is one archive's draw on a Limiter. A nil *charge admits everything. +// +// Bytes are charged as they land, never from the sizes an archive declares, since those can lie. +type charge struct { + limiter *Limiter + inMemory int64 + onDisk int64 +} + +func admits(limit, held, n int64) bool { + switch { + case limit < 0: + return true + case limit == 0: + return false + default: + return held+n <= limit + } +} + +// memory charges n bytes held in memory, or reports false (charging nothing) when the memory limit +// does not admit them. +func (c *charge) memory(n int64) bool { + if c == nil || n <= 0 { + return true + } + l := c.limiter + l.mu.Lock() + defer l.mu.Unlock() + if !admits(l.limits.MaxMemoryBytes, l.memory, n) { + return false + } + l.memory += n + c.inMemory += n + l.peakMemory = max(l.peakMemory, l.memory) + return true +} + +// disk charges n bytes written to disk, or reports false (charging nothing) when the disk limit does +// not admit them. +func (c *charge) disk(n int64) bool { + if c == nil || n <= 0 { + return true + } + l := c.limiter + l.mu.Lock() + defer l.mu.Unlock() + if !admits(l.limits.MaxDiskBytes, l.disk, n) { + return false + } + l.disk += n + c.onDisk += n + l.peakDisk = max(l.peakDisk, l.disk) + return true +} + +// index charges n bytes of index bookkeeping, which lives in memory whatever the limits say. A zero +// memory limit is a valid configuration that must still index archives, so only then does the cost +// fall back to the disk budget; a bounded memory limit bounds the index too. +func (c *charge) index(n int64) bool { + if c.memory(n) { + return true + } + return c != nil && c.limiter.limits.MaxMemoryBytes == 0 && c.disk(n) +} + +// refundMemory gives back n bytes no longer held in memory. +func (c *charge) refundMemory(n int64) { + if c == nil || n <= 0 { + return + } + l := c.limiter + l.mu.Lock() + defer l.mu.Unlock() + n = min(n, c.inMemory) + l.memory -= n + c.inMemory -= n +} + +// refundDisk gives back n bytes no longer on disk. +func (c *charge) refundDisk(n int64) { + if c == nil || n <= 0 { + return + } + l := c.limiter + l.mu.Lock() + defer l.mu.Unlock() + n = min(n, c.onDisk) + l.disk -= n + c.onDisk -= n +} + +// release gives back everything this charge took. Safe to call more than once. +func (c *charge) release() { + if c == nil { + return + } + l := c.limiter + l.mu.Lock() + defer l.mu.Unlock() + l.memory -= c.inMemory + l.disk -= c.onDisk + c.inMemory, c.onDisk = 0, 0 +} diff --git a/internal/archive/limits_test.go b/internal/archive/limits_test.go new file mode 100644 index 000000000..7cdc85c86 --- /dev/null +++ b/internal/archive/limits_test.go @@ -0,0 +1,178 @@ +package archive + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLimiter_chargeAndRelease(t *testing.T) { + limiter := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 200}) + + first := limiter.charge() + require.True(t, first.memory(60)) + require.True(t, first.disk(150)) + + mem, disk := limiter.InUse() + assert.Equal(t, int64(60), mem) + assert.Equal(t, int64(150), disk) + + second := limiter.charge() + assert.False(t, second.memory(50), "60 + 50 is over the memory limit") + assert.False(t, second.disk(60), "150 + 60 is over the disk limit") + assert.True(t, second.memory(40), "and exactly at the limit is admitted") + + mem, _ = limiter.InUse() + assert.Equal(t, int64(100), mem, "a refused charge takes nothing") + + first.release() + mem, disk = limiter.InUse() + assert.Equal(t, int64(40), mem, "release gives back exactly what that charge took") + assert.Zero(t, disk) + + first.release() + mem, _ = limiter.InUse() + assert.Equal(t, int64(40), mem, "releasing twice must not give back what was never taken") +} + +func TestLimiter_refund(t *testing.T) { + limiter := NewLimiter(Limits{MaxDiskBytes: 100}) + charge := limiter.charge() + + require.True(t, charge.disk(80)) + charge.refundDisk(80) + + _, disk := limiter.InUse() + assert.Zero(t, disk) + assert.True(t, charge.disk(100), "the refunded room is available again") + + charge.refundDisk(500) + _, disk = limiter.InUse() + assert.Zero(t, disk, "a refund larger than what was charged cannot drive the limiter negative") +} + +func TestLimiter_threeStateReadingIsUniform(t *testing.T) { + t.Run("positive is the limit", func(t *testing.T) { + c := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 100}).charge() + assert.True(t, c.memory(100)) + assert.False(t, c.memory(1)) + assert.True(t, c.disk(100)) + assert.False(t, c.disk(1)) + }) + + t.Run("zero refuses unconditionally on both", func(t *testing.T) { + c := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).charge() + assert.False(t, c.memory(1)) + assert.False(t, c.disk(1)) + }) + + t.Run("negative admits unconditionally on both", func(t *testing.T) { + c := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}).charge() + assert.True(t, c.memory(1_000_000_000)) + assert.True(t, c.disk(1_000_000_000)) + }) +} + +func TestLimiter_nilIsUnbounded(t *testing.T) { + var limiter *Limiter + charge := limiter.charge() + assert.Nil(t, charge) + + assert.True(t, charge.memory(1_000_000)) + assert.True(t, charge.disk(1_000_000)) + charge.refundDisk(10) + charge.release() + + mem, disk := charge.held() + assert.Zero(t, mem) + assert.Zero(t, disk) + + mem, disk = limiter.InUse() + assert.Zero(t, mem) + assert.Zero(t, disk) + + mem, disk = limiter.Peak() + assert.Zero(t, mem) + assert.Zero(t, disk) +} + +func TestLimiter_concurrentChargesAgree(t *testing.T) { + limiter := NewLimiter(Limits{MaxMemoryBytes: 1_000_000, MaxDiskBytes: -1}) + + var wg sync.WaitGroup + charges := make([]*charge, 50) + for i := range charges { + charges[i] = limiter.charge() + wg.Add(1) + go func(c *charge) { + defer wg.Done() + for range 100 { + c.memory(1) + c.disk(2) + } + }(charges[i]) + } + wg.Wait() + + mem, disk := limiter.InUse() + assert.Equal(t, int64(50*100), mem) + assert.Equal(t, int64(50*100*2), disk) + + for _, c := range charges { + c.release() + } + mem, disk = limiter.InUse() + assert.Zero(t, mem) + assert.Zero(t, disk) +} + +func TestLimiter_peak(t *testing.T) { + t.Run("is a high-water mark", func(t *testing.T) { + l := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) + + first := l.charge() + require.True(t, first.memory(100)) + require.True(t, first.disk(300)) + first.release() + + second := l.charge() + require.True(t, second.memory(40)) + require.True(t, second.disk(50)) + + memory, disk := l.InUse() + assert.Equal(t, int64(40), memory) + assert.Equal(t, int64(50), disk) + + peakMemory, peakDisk := l.Peak() + assert.Equal(t, int64(100), peakMemory, "the peak is the most held at once, not the most recently held") + assert.Equal(t, int64(300), peakDisk) + }) + + t.Run("counts concurrent holders together", func(t *testing.T) { + l := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) + + outer := l.charge() + require.True(t, outer.memory(100)) + inner := l.charge() + require.True(t, inner.memory(60)) + inner.release() + outer.release() + + peakMemory, _ := l.Peak() + assert.Equal(t, int64(160), peakMemory) + }) + + t.Run("a refused charge does not move it", func(t *testing.T) { + l := NewLimiter(Limits{MaxMemoryBytes: 50, MaxDiskBytes: 50}) + + c := l.charge() + assert.False(t, c.memory(500)) + assert.False(t, c.disk(500)) + + peakMemory, peakDisk := l.Peak() + assert.Zero(t, peakMemory) + assert.Zero(t, peakDisk) + }) +} diff --git a/internal/archive/resolver.go b/internal/archive/resolver.go new file mode 100644 index 000000000..7ec0dc305 --- /dev/null +++ b/internal/archive/resolver.go @@ -0,0 +1,575 @@ +package archive + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "slices" + "sort" + "strings" + "time" + "unicode" + + "github.com/dustin/go-humanize" + "github.com/mholt/archives" + + stereoscopeFile "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/stereoscope/pkg/filetree" + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/internal/tmpdir" + "github.com/anchore/syft/syft/file" +) + +// Resolver is one archive opened as its own filesystem: a file.Resolver over the entries Extract fed +// into it, holding their content within the scan's limits. Entries are indexed in a stereoscope file +// tree, the same index the directory and image resolvers search. +// +// Locations it returns keep the outermost archive's RealPath and carry the entry's FullPath. Queries +// may name an entry either by its full path or by its path within the archive (see rootedPath). +type Resolver struct { + // Digests are of the archive file itself. + Digests []file.Digest + + // Truncated is non-nil when the resolver covers only part of the archive, and says why: the disk limit + // or the decompression budget stopped extraction early, or an entry was too large or unreadable. + Truncated error + + archiveCoordinates file.Coordinates + charge *charge + budget *Budget + metered bool // the budget is charged as the archive is decoded (see meteredReader), not in put + + // content is held in memory while the memory limit admits it and in one spill file once it does + // not. The file is created on first use, so an archive that stays in memory never touches the + // filesystem, and blobs are located in it by offset with no framing of their own. + tempDir *tmpdir.TempDir + held []*blob // blobs in memory, in the order they were added + file *os.File + remove func() + written int64 + chunk []byte + + tree *filetree.FileTree + index filetree.Index + search filetree.Searcher // built by finish, once every entry is in the tree + content map[stereoscopeFile.ID]*blob + entries int // non-directory entries added +} + +var _ file.Resolver = (*Resolver)(nil) + +// newResolver returns an empty resolver whose locations extend the archive's coordinates, spilling into +// the scan's temp root when ctx carries one (see internal/tmpdir). +func newResolver(ctx context.Context, archiveCoordinates file.Coordinates, charge *charge) *Resolver { + return &Resolver{ + archiveCoordinates: archiveCoordinates, + charge: charge, + budget: budgetFromContext(ctx), + tempDir: tmpdir.FromContext(ctx), + tree: filetree.New(), + index: filetree.NewIndex(), + content: map[stereoscopeFile.ID]*blob{}, + } +} + +// Cleanup removes everything the archive holds on disk and releases its charge against the limiter. +// Readers handed out before are no longer valid. Safe to call more than once. +func (r *Resolver) Cleanup() { + if r == nil { + return + } + r.releaseStorage() + r.charge.release() +} + +// approxIndexBytesPerEntry estimates what holding one tree node costs beyond its content: the node, its +// header and metadata, and its index slots. It is charged for every node an entry creates, including +// the directories its path implies, so an archive of many tiny entries or of a few entries with very +// deep paths is still bounded by the limits. +const approxIndexBytesPerEntry = 2 * 1024 + +func approxIndexBytes(hdr tar.Header) int64 { + return approxIndexBytesPerEntry + int64(len(hdr.Name)+len(hdr.Linkname)) +} + +// add stores and indexes one entry, reading its content in full now since archive readers are +// sequential. Reaching a limit stores nothing for this entry and returns ErrDiskLimitReached; entries +// added before it remain usable. The last entry at a path wins, as it does when the archive is +// extracted (tar appends updates, an OCI layer applies in order, and java's legacy zip reading +// overwrites too), so a stale earlier copy, such as an older pom.properties, can't stand in for what is +// really there. +func (r *Resolver) add(hdr tar.Header, content io.Reader) error { + entryPath := path.Clean("/" + hdr.Name) + if entryPath == "/" { + return nil + } + replacing, ok := r.replace(entryPath, hdr) + if !ok { + return nil + } + + // directories the path implies that do not exist yet are nodes this entry pays for. Walking from the + // deepest parent up stops at the first that exists, since its ancestors exist too + var newDirs []string + for i := strings.LastIndexByte(entryPath, '/'); i > 0; i = strings.LastIndexByte(entryPath[:i], '/') { + if r.tree.HasPath(stereoscopeFile.Path(entryPath[:i])) { + break + } + newDirs = append(newDirs, entryPath[:i]) + } + slices.Reverse(newDirs) + if !r.chargeIndex(approxIndexBytes(hdr) + int64(len(newDirs))*approxIndexBytesPerEntry) { + return ErrDiskLimitReached + } + // indexed as if listed, since a search for a directory's children looks the directory up by name + for _, dir := range newDirs { + if err := r.store(dir, tar.Header{Name: dir, Typeflag: tar.TypeDir, Mode: 0o755}, &blob{}); err != nil { + log.WithFields("entry", hdr.Name, "error", err).Trace("skipping archive entry") + return nil + } + } + + b := &blob{} + if hdr.Typeflag == tar.TypeReg && content != nil { + // reading one byte past the cap is how an entry over it is told apart from one exactly at it + err := r.put(b, io.LimitReader(content, maxEntryBytes+1)) + if err == nil && b.size > maxEntryBytes { + err = errEntryTooLarge + } + if err != nil { + r.discard(b) + if errors.Is(err, errEntryTooLarge) { + r.truncate(fmt.Errorf("an entry was skipped: %w of %s", err, humanize.IBytes(uint64(maxEntryBytes)))) + return nil + } + if errors.Is(err, ErrDiskLimitReached) || errors.Is(err, errBudgetSpent) { + return err + } + // a damaged entry (a CRC mismatch, a corrupt deflate stream) costs only that entry + r.truncate(fmt.Errorf("entry %q could not be read: %w", hdr.Name, err)) + return nil + } + } + + if err := r.store(entryPath, hdr, b); err != nil { + // a path the tree cannot hold, such as a file beneath another file's name + r.discard(b) + log.WithFields("entry", hdr.Name, "error", err).Trace("skipping archive entry") + return nil + } + if hdr.Typeflag != tar.TypeDir && !replacing { + r.entries++ + } + return nil +} + +// replace clears the way for an entry at a path that is already held, reporting whether it did and +// false when the entry should be dropped. A later non-directory entry replaces an earlier one, refunding +// what it held. A file and a directory at one path keep the first, since swapping kinds would mean +// re-parenting what is already beneath it. +func (r *Resolver) replace(entryPath string, hdr tar.Header) (replacing, ok bool) { + exists, res, _ := r.tree.File(stereoscopeFile.Path(entryPath)) + if !exists || !res.HasReference() { + return false, true + } + earlier, err := r.index.Get(*res.Reference) + if err != nil || earlier.IsDir() || hdr.Typeflag == tar.TypeDir { + return false, false + } + // a removed node's index entry is left behind, but searches only answer with references in the tree + if err := r.tree.RemovePath(stereoscopeFile.Path(entryPath)); err != nil { + log.WithFields("entry", hdr.Name, "error", err).Trace("unable to replace archive entry") + return false, false + } + if b := r.content[res.ID()]; b != nil { + r.discard(b) + } + delete(r.content, res.ID()) + return true, true +} + +func (r *Resolver) store(rooted string, hdr tar.Header, b *blob) error { + ref, err := r.addToTree(rooted, hdr) + if err != nil { + return err + } + metadata := stereoscopeFile.NewMetadata(hdr, r.open(b)) + metadata.Path = entryPath(rooted) + r.index.Add(*ref, metadata) + r.content[ref.ID()] = b + return nil +} + +func (r *Resolver) addToTree(entryPath string, hdr tar.Header) (*stereoscopeFile.Reference, error) { + p := stereoscopeFile.Path(entryPath) + switch hdr.Typeflag { + case tar.TypeDir: + return r.tree.AddDir(p) + case tar.TypeSymlink: + return r.tree.AddSymLink(p, stereoscopeFile.Path(hdr.Linkname)) + case tar.TypeLink: + // a hard link names its target from the archive root + return r.tree.AddHardLink(p, stereoscopeFile.Path(path.Join("/", hdr.Linkname))) + default: + return r.tree.AddFile(p) + } +} + +// maxEntryBytes caps any one entry, as the legacy java extraction did (see intFile.SafeCopy): without it +// a single entry of an overlapping-entry zip bomb can decompress up to the disk limit. +var maxEntryBytes int64 = intFile.PerFileReadLimit + +var errEntryTooLarge = errors.New("archive entry exceeds the per-entry size cap") + +// truncate marks the resolver as covering part of the archive, keeping the first reason given. +func (r *Resolver) truncate(err error) { + if r.Truncated == nil { + r.Truncated = err + } +} + +// chargeIndex charges index bookkeeping, which lives in memory. Content held in memory can move to +// disk to make room for it; the index cannot, so when it is refused the entries are. +func (r *Resolver) chargeIndex(n int64) bool { + if r.charge.index(n) { + return true + } + if len(r.held) == 0 || r.spill() != nil { + return false + } + return r.charge.index(n) +} + +// finish builds the search context once every entry is added. +func (r *Resolver) finish() { + r.search = filetree.NewSearchContext(r.tree, r.index) +} + +// linkDestination is the rooted path a link points at inside the archive. A hard link names its +// target from the archive root, as does an absolute symlink, so "/etc/passwd" is an entry of this +// archive and never the host's. A relative symlink is joined onto the link's own directory, and +// cleaning against the root keeps "../../x" inside the archive. +func linkDestination(linkPath, linkname string, hard bool) string { + if hard || path.IsAbs(linkname) { + return path.Join("/", linkname) + } + return path.Join(path.Dir(linkPath), linkname) +} + +// entryPath is a node's path within the archive: archive-relative with no leading slash. +func entryPath(rooted string) string { + return strings.TrimPrefix(rooted, "/") +} + +// fullPath is how paths leave the resolver: the archive's own full path, then the entry path. +func (r *Resolver) fullPath(rooted string) string { + return r.archiveCoordinates.FullNestedPath() + file.ArchivePathSeparator + entryPath(rooted) +} + +// rootedPath is the tree path a query names, which is either form a path takes: a full path is trimmed to +// the entry path within this archive, and anything else (a LocalPath, with or without its leading slash) +// is taken as archive-relative already, since the tree looks a relative path up from its root. The archive's full path with a trailing "!" (what path.Dir gives for an entry at the root) +// names the root. A full path into another archive in the scan names nothing here and reports false. +func (r *Resolver) rootedPath(userPath string) (string, bool) { + if rest, ok := strings.CutPrefix(userPath, r.archiveCoordinates.FullNestedPath()); ok { + if rest == file.ArchivePathSeparator[:1] { + return "/", true + } + if strings.HasPrefix(rest, file.ArchivePathSeparator) { + // keep the separator's slash as the rooted path's + return path.Clean(rest[len(file.ArchivePathSeparator)-1:]), true + } + } + if r.outside(userPath) { + return "", false + } + return path.Clean(userPath), true +} + +// outside reports whether a path not under this archive's full path is a full path from the scan +// nonetheless: every full path in this archive tree starts with the outermost archive's real path and a +// separator. The real path alone is taken as an entry path, so an entry named like the outermost archive +// is found by its LocalPath. +func (r *Resolver) outside(userPath string) bool { + rest, ok := strings.CutPrefix(userPath, r.archiveCoordinates.RealPath) + return ok && strings.HasPrefix(rest, file.ArchivePathSeparator) +} + +// coordinates name an entry by the outermost archive's real path, which never includes a path inside an +// archive, and its full path through every archive. +func (r *Resolver) coordinates(rooted string) file.Coordinates { + return file.Coordinates{ + RealPath: r.archiveCoordinates.RealPath, + FileSystemID: r.archiveCoordinates.FileSystemID, + FullPath: r.fullPath(rooted), + } +} + +// location names a resolution: a link is reported at its target's path with the link's own full path +// as the access path. +func (r *Resolver) location(res stereoscopeFile.Resolution) file.Location { + coordinates := r.coordinates(string(res.RealPath)) + if res.RequestPath == "" || res.RequestPath == res.RealPath { + return file.NewLocationFromCoordinates(coordinates) + } + return file.NewVirtualLocationFromCoordinates(coordinates, r.fullPath(string(res.RequestPath))) +} + +// resolve finds the file a path names, following links, and reports false for directories and +// paths the archive never listed. +func (r *Resolver) resolve(userPath string) (*stereoscopeFile.Resolution, bool) { + rooted, ok := r.rootedPath(userPath) + if !ok { + return nil, false + } + res, err := r.search.SearchByPath(rooted, filetree.FollowBasenameLinks) + if err != nil || res == nil || !res.HasReference() { + return nil, false + } + entry, err := r.index.Get(*res.Reference) + if err != nil || entry.IsDir() { + return nil, false + } + return res, true +} + +func (r *Resolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { + res, ok := r.resolve(location.FullNestedPath()) + if !ok { + return nil, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath()) + } + return entryReader{r.open(r.content[res.ID()])}, nil +} + +func (r *Resolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) { + rooted, ok := r.rootedPath(location.FullNestedPath()) + if !ok { + return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath()) + } + exists, res, err := r.tree.File(stereoscopeFile.Path(rooted)) + if err != nil || !exists || !res.HasReference() { + return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath()) + } + entry, err := r.index.Get(*res.Reference) + if err != nil { + return file.Metadata{}, err + } + // the index holds entry paths; callers get full paths, as with locations + metadata := entry.Metadata + metadata.Path = r.fullPath(rooted) + if metadata.LinkDestination != "" { + metadata.LinkDestination = r.fullPath(linkDestination(rooted, metadata.LinkDestination, metadata.Type == stereoscopeFile.TypeHardLink)) + } + return metadata, nil +} + +func (r *Resolver) HasPath(userPath string) bool { + rooted, ok := r.rootedPath(userPath) + return ok && r.tree.HasPath(stereoscopeFile.Path(rooted)) +} + +func (r *Resolver) FilesByPath(paths ...string) ([]file.Location, error) { + var out []file.Location + var seen []stereoscopeFile.ID // few paths are asked for at once + for _, userPath := range paths { + res, ok := r.resolve(userPath) + if !ok || slices.Contains(seen, res.ID()) { + continue + } + seen = append(seen, res.ID()) + out = append(out, r.location(*res)) + } + return out, nil +} + +func (r *Resolver) FilesByGlob(patterns ...string) ([]file.Location, error) { + var out []file.Location + seen := stereoscopeFile.NewFileReferenceSet() + for _, pattern := range patterns { + rooted, ok := r.rootedPath(pattern) + if !ok { + continue + } + results, err := r.searchByGlob(rooted) + if err != nil { + return nil, fmt.Errorf("invalid glob %q: %w", pattern, err) + } + for _, res := range results { + if !res.HasReference() || seen.Contains(*res.Reference) { + continue + } + entry, err := r.index.Get(*res.Reference) + if err != nil || entry.IsDir() { + continue + } + seen.Add(*res.Reference) + out = append(out, r.location(res)) + } + } + sortLocations(out) + return out, nil +} + +// searchByGlob is the index search, except that stereoscope answers "/*" by looking the directory +// up by name and the root has none, so "/*" (or "*") walks the tree instead. +func (r *Resolver) searchByGlob(pattern string) ([]stereoscopeFile.Resolution, error) { + if strings.TrimPrefix(pattern, "/") == "*" { + return r.tree.FilesByGlob("/*", filetree.FollowBasenameLinks) + } + return r.search.SearchByGlob(pattern, filetree.FollowBasenameLinks) +} + +// sortLocations orders answers by path, which decides how locations group into packages when results +// merge into the SBOM. +func sortLocations(locations []file.Location) { + sort.Slice(locations, func(i, j int) bool { + if a, b := locations[i].FullNestedPath(), locations[j].FullNestedPath(); a != b { + return a < b + } + return locations[i].AccessPath < locations[j].AccessPath + }) +} + +func (r *Resolver) FilesByMIMEType(types ...string) ([]file.Location, error) { + results, err := r.search.SearchByMIMEType(types...) + if err != nil { + return nil, err + } + var out []file.Location + for _, res := range results { + if res.HasReference() { + out = append(out, r.location(res)) + } + } + sortLocations(out) + return out, nil +} + +// RelativeFileByPath resolves against the archive root: an archive has no layers to be relative to. +func (r *Resolver) RelativeFileByPath(_ file.Location, userPath string) *file.Location { + locations, _ := r.FilesByPath(userPath) + if len(locations) == 0 { + return nil + } + return &locations[0] +} + +func (r *Resolver) AllLocations(ctx context.Context) <-chan file.Location { + out := make(chan file.Location) + go func() { + defer close(out) + refs := r.tree.AllFiles(stereoscopeFile.TypeRegular, stereoscopeFile.TypeSymLink, stereoscopeFile.TypeHardLink) + sort.Slice(refs, func(i, j int) bool { return refs[i].RealPath < refs[j].RealPath }) + for _, ref := range refs { + select { + case out <- r.location(stereoscopeFile.Resolution{RequestPath: ref.RealPath, Reference: &ref}): + case <-ctx.Done(): + return + } + } + }() + return out +} + +// entryReader gives an entry's reader the Close the resolver interface requires. The resolver owns +// the bytes, so there is nothing to close. +type entryReader struct { + ReaderAtSeeker +} + +func (entryReader) Close() error { return nil } + +// maxEntryNameBytes is the longest entry name stored: PATH_MAX. Every path component becomes a node, +// so a longer name is a cheap way to inflate the index and no filesystem could hold it anyway. +const maxEntryNameBytes = MaxPathBytes + +// MaxPathBytes is PATH_MAX, the longest whole archive chain a scan stores or a decoded document may build. +// Each nested file copies its container's chain, so without it a long name shared by many small entries +// costs memory quadratic in the document. +const MaxPathBytes = 4096 + +// MaxDepth is the deepest archives nest: a scan stops there whatever depth is configured, and a decoded +// document is held to it. No real nesting comes close, and a hostile archive (a zip quine, a recursive bomb) +// must still end. +const MaxDepth = 16 + +// maxEntryNameDepth is the most path components an entry name may have. Adding an entry costs work per +// component, and no real archive nests anywhere near this deep. +const maxEntryNameDepth = 256 + +// entryHeader builds the header stored for one archive entry, or reports false for an entry not worth +// carrying: device nodes, fifos and sockets, which no cataloger reads, and names that cannot be a path +// inside the archive. A name refused as unsafe marks the resolver truncated. +// +// The name is cleaned to an archive-relative path with no leading slash, so "../x" names x at the +// archive root rather than anything outside it. Symlinks are stored as headers, never followed on the +// host; the resolver resolves their targets inside the archive. +func (r *Resolver) entryHeader(f archives.FileInfo) (tar.Header, bool) { + mode := f.Mode() + if !mode.IsRegular() && !mode.IsDir() && mode.Type()&fs.ModeSymlink == 0 { + log.WithFields("entry", f.NameInArchive, "mode", mode).Debug("skipping non-regular archive entry") + return tar.Header{}, false + } + + name, err := EntryName(f.NameInArchive) + if err != nil { + r.truncate(fmt.Errorf("an entry was skipped: %w", err)) + return tar.Header{}, false + } + if name == "" { + return tar.Header{}, false + } + // held to the chain length a decoder accepts back, so the SBOM round-trips + if chain := len(r.archiveCoordinates.FullNestedPath()) + len(file.ArchivePathSeparator) + len(name); chain > MaxPathBytes { + r.truncate(fmt.Errorf("an entry was skipped: its path of %d bytes within the archive chain exceeds the %d byte cap", chain, MaxPathBytes)) + return tar.Header{}, false + } + + hdr, err := tar.FileInfoHeader(f.FileInfo, f.LinkTarget) + if err != nil { + log.WithFields("entry", f.NameInArchive, "error", err).Debug("skipping archive entry with an unusable header") + return tar.Header{}, false + } + hdr.Name = name + hdr.Linkname = f.LinkTarget + + // nothing reads these, and the header is held for the life of the archive + hdr.PAXRecords = nil + hdr.Uname, hdr.Gname = "", "" + hdr.AccessTime, hdr.ChangeTime = time.Time{}, time.Time{} + hdr.ModTime = hdr.ModTime.Truncate(time.Second) + + return *hdr, true +} + +// EntryName cleans an archive entry's name to a path relative to the archive root, returning "" when it +// names nothing inside the archive. This is the trust boundary for names: every one ends up in FullPath +// and the encoders, so a name that could forge an archive boundary ("!/") or a line in a line-oriented +// format (control characters, unicode line separators) is refused. Decoders building nested coordinates +// from a document hold its names to the same rules. +func EntryName(name string) (string, error) { + if len(name) > maxEntryNameBytes { + return "", fmt.Errorf("entry name of %d bytes exceeds the %d byte cap", len(name), maxEntryNameBytes) + } + if strings.IndexFunc(name, isUnsafeNameRune) >= 0 { + return "", fmt.Errorf("entry name %q contains a control character", name) + } + cleaned := strings.TrimPrefix(path.Clean("/"+name), "/") + if depth := strings.Count(cleaned, "/") + 1; depth > maxEntryNameDepth { + return "", fmt.Errorf("entry name of %d path components exceeds the %d component cap", depth, maxEntryNameDepth) + } + if strings.Contains(cleaned, file.ArchivePathSeparator) { + return "", fmt.Errorf("entry name %q contains the archive path separator %q", name, file.ArchivePathSeparator) + } + return cleaned, nil +} + +func isUnsafeNameRune(c rune) bool { + return unicode.IsControl(c) || c == '\u2028' || c == '\u2029' +} diff --git a/internal/archive/resolver_test.go b/internal/archive/resolver_test.go new file mode 100644 index 000000000..e39331f4f --- /dev/null +++ b/internal/archive/resolver_test.go @@ -0,0 +1,506 @@ +package archive + +import ( + "archive/tar" + "bytes" + "context" + "io" + "path" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + stereoscopeFile "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/file" +) + +var jarEntriesFixture = map[string]string{ + "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n", + "META-INF/maven/com.example/lib/pom.properties": "groupId=com.example\n", + "META-INF/maven/com.example/lib/pom.xml": "", + "LICENSE": "Apache License 2.0", + "BOOT-INF/lib/inner.jar": "PK\x03\x04 not really", + "com/example/Thing.class": "bytecode", +} + +func TestResolver_add_refusedEntryIsNotStored(t *testing.T) { + // a half-stored pom a cataloger parses is worse than an absent one + r, _ := resolverIn(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).charge()) + + err := r.add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc"))) + assert.ErrorIs(t, err, ErrDiskLimitReached) + assert.Empty(t, storedNames(t, r)) + assert.False(t, r.HasPath("a.txt")) +} + +func TestResolver_add_refusedContentLeavesNoNodeBehind(t *testing.T) { + // the index estimate is admitted, then the content is refused + record := approxIndexBytes(regularHeader("dir/a.txt", 0)) + approxIndexBytesPerEntry // the entry and the directory it implies + r, _ := resolverIn(t, diskCharge(record+2)) + + err := r.add(regularHeader("dir/a.txt", 3), bytes.NewReader([]byte("abc"))) + assert.ErrorIs(t, err, ErrDiskLimitReached) + assert.False(t, r.HasPath("dir/a.txt")) + assert.True(t, r.HasPath("dir"), "the directories it implied stay") +} + +func TestResolver_add_directoryEntriesHoldNoContent(t *testing.T) { + r, _ := resolverIn(t, memCharge(1<<20)) + + require.NoError(t, r.add(tar.Header{Name: "lib", Typeflag: tar.TypeDir, Mode: 0o755}, nil)) + + assert.Zero(t, r.heldInMemory()) + assert.Empty(t, readStore(t, r)["lib"].body) +} + +func TestResolver_add_indexDoesNotFallBackToDiskWhenMemoryIsBounded(t *testing.T) { + // the index lives in memory, so a bounded memory limit bounds it; only a zero limit defers to disk + limiter := NewLimiter(Limits{MaxMemoryBytes: 1024, MaxDiskBytes: -1}) + r, _ := resolverIn(t, limiter.charge()) + + err := r.add(regularHeader("a.txt", 0), bytes.NewReader(nil)) + assert.ErrorIs(t, err, ErrDiskLimitReached) + _, disk := limiter.InUse() + assert.Zero(t, disk) +} + +func TestResolver_add_indexMakesRoomBySpillingHeldContent(t *testing.T) { + record := approxIndexBytes(regularHeader("a.txt", 0)) + r, dir := resolverIn(t, memCharge(2*record+50)) + + require.NoError(t, r.add(regularHeader("a.txt", 100), bytes.NewReader(bytes.Repeat([]byte("a"), 100)))) + require.Equal(t, int64(100), r.heldInMemory()) + + // b's index does not fit beside a's content, so a's content moves to disk and b is indexed + require.NoError(t, r.add(regularHeader("b.txt", 3), bytes.NewReader([]byte("bbb")))) + r.finish() + + assert.Equal(t, int64(3), r.heldInMemory()) + assert.NotEmpty(t, spillFile(t, dir)) + assert.Equal(t, []string{"a.txt", "b.txt"}, storedNames(t, r)) + assert.Equal(t, strings.Repeat("a", 100), readStore(t, r)["a.txt"].body) +} + +func TestResolver_add_lastEntryAtAPathWins(t *testing.T) { + r, _ := resolverIn(t, memCharge(1<<20)) + + require.NoError(t, r.add(regularHeader("a.txt", 5), bytes.NewReader([]byte("first")))) + require.NoError(t, r.add(regularHeader("a.txt", 6), bytes.NewReader([]byte("second")))) + require.NoError(t, r.add(regularHeader("b.txt", 1), bytes.NewReader([]byte("b")))) + require.NoError(t, r.add(tar.Header{Name: "b.txt", Typeflag: tar.TypeSymlink, Linkname: "a.txt"}, nil)) + r.finish() + + assert.Equal(t, "second", readStore(t, r)["a.txt"].body) + assert.Equal(t, "a.txt", readStore(t, r)["b.txt"].linkTarget, "a later entry may change kind") + assert.Equal(t, 2, r.entries) + assert.Equal(t, int64(len("second")), r.heldInMemory(), "the earlier copy is refunded") + + locations, err := r.FilesByGlob("**/a.txt") + require.NoError(t, err) + require.Len(t, locations, 1) + reader, err := r.FileContentsByLocation(locations[0]) + require.NoError(t, err) + assert.Equal(t, "second", readEntry(t, reader.(entryReader).ReaderAtSeeker)) +} + +func TestResolver_add_fileAndDirectoryAtOnePathKeepsTheFirst(t *testing.T) { + r, _ := resolverIn(t, memCharge(1<<20)) + + require.NoError(t, r.add(regularHeader("a/b", 1), bytes.NewReader([]byte("x")))) + require.NoError(t, r.add(regularHeader("a", 4), bytes.NewReader([]byte("file")))) + require.NoError(t, r.add(regularHeader("c", 4), bytes.NewReader([]byte("file")))) + require.NoError(t, r.add(tar.Header{Name: "c", Typeflag: tar.TypeDir, Mode: 0o755}, nil)) + r.finish() + + got := readStore(t, r) + assert.Equal(t, stereoscopeFile.TypeDirectory, got["a"].kind) + assert.Equal(t, "file", got["c"].body) +} + +func TestResolver_pathsAndContents(t *testing.T) { + r := resolverOver(t, 1<<20, jarEntriesFixture) + + locations, err := r.FilesByPath("/META-INF/MANIFEST.MF") + require.NoError(t, err) + require.Len(t, locations, 1) + + assert.True(t, r.HasPath("/META-INF/MANIFEST.MF")) + assert.True(t, r.HasPath("META-INF/MANIFEST.MF"), "a path may arrive with or without the leading slash") + assert.False(t, r.HasPath("/nope")) + + reader, err := r.FileContentsByLocation(locations[0]) + require.NoError(t, err) + body, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + assert.Equal(t, "Manifest-Version: 1.0\n", string(body)) +} + +func TestResolver_fullPathsOutsideTheArchiveNameNothing(t *testing.T) { + r, _ := resolverIn(t, memCharge(1<<20)) + r.archiveCoordinates = file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/dep.jar"} + addAll(t, r, fileEntries(map[string]string{"/app.war": "same name", "/META-INF/MANIFEST.MF": "m"})) + + locations, err := r.FilesByPath("/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF") + require.NoError(t, err) + assert.Len(t, locations, 1, "a full path in this archive") + + for _, p := range []string{"/app.war!/WEB-INF/lib/dep.jar", "/app.war!/WEB-INF/lib/other.jar!/META-INF/MANIFEST.MF"} { + locations, err = r.FilesByPath(p) + require.NoError(t, err) + assert.Empty(t, locations, "%s is outside the archive", p) + assert.False(t, r.HasPath(p), p) + } + + for _, p := range []string{"app.war", "/app.war"} { + locations, err = r.FilesByPath(p) + require.NoError(t, err) + require.Len(t, locations, 1, "%s is an entry path named like the outermost archive", p) + assert.Equal(t, "/app.war!/WEB-INF/lib/dep.jar!/app.war", locations[0].FullPath, p) + } + + again, err := r.FilesByPath(locations[0].LocalPath()) + require.NoError(t, err) + assert.Equal(t, locations, again, "a LocalPath finds its own entry") +} + +func TestResolver_mimeTypesAreSniffedFromContent(t *testing.T) { + r := resolverOver(t, 1<<20, map[string]string{ + "notes.txt": "just some text", + "run.sh": "#!/bin/bash\necho hello\n", + }) + + locations, err := r.FilesByMIMEType("text/x-shellscript") + require.NoError(t, err) + assert.Equal(t, []string{"/run.sh"}, localPaths(locations)) +} + +func TestResolver_answersAreOrdered(t *testing.T) { + // order decides which locations group into which package when results merge into the SBOM + entries := map[string]string{} + for _, name := range []string{ + "b/z.jar", "a/y.jar", "c/x.jar", "a/b/c/w.jar", "z.jar", "a.jar", + } { + entries[name] = "PK\x03\x04 not really" + } + r := resolverOver(t, 1<<20, entries) + + first, err := r.FilesByGlob("**/*.jar") + require.NoError(t, err) + require.Len(t, first, len(entries)) + + byMIME, err := r.FilesByMIMEType("application/octet-stream") + require.NoError(t, err) + + for range 8 { + again, err := r.FilesByGlob("**/*.jar") + require.NoError(t, err) + assert.Equal(t, first, again) + + mimeAgain, err := r.FilesByMIMEType("application/octet-stream") + require.NoError(t, err) + assert.Equal(t, byMIME, mimeAgain) + + var all []string + for loc := range r.AllLocations(context.Background()) { + all = append(all, loc.LocalPath()) + } + assert.True(t, sort.StringsAreSorted(all)) + } + + var paths []string + for _, loc := range first { + paths = append(paths, loc.LocalPath()) + } + assert.True(t, sort.StringsAreSorted(paths), "ordered by path, not merely stable") +} + +func TestResolver_impliedDirectoriesArePathsButNotFiles(t *testing.T) { + r := resolverOver(t, 1<<20, jarEntriesFixture) + + assert.True(t, r.HasPath("META-INF")) + assert.True(t, r.HasPath("/META-INF/maven")) + + locations, err := r.FilesByPath("META-INF") + require.NoError(t, err) + assert.Empty(t, locations) + locations, err = r.FilesByGlob("**/maven") + require.NoError(t, err) + assert.Empty(t, locations) +} + +// stereoscope finds a directory's children by looking the directory up by name: directories the archive +// never listed must be indexed, and the root, which has no name, answered another way +func TestResolver_globsForADirectorysChildren(t *testing.T) { + r := resolverOver(t, 1<<20, map[string]string{ + "LICENSE": "root", + "META-INF/LICENSE.txt": "nested", + "META-INF/maven/g/a/pom.xml": "deeper", + }) + + assert.Equal(t, []string{"/LICENSE"}, globPaths(t, r, "/*")) + assert.Equal(t, []string{"/LICENSE"}, globPaths(t, r, "*")) + assert.Equal(t, []string{"/META-INF/LICENSE.txt"}, globPaths(t, r, "/META-INF/*")) + assert.Equal(t, []string{"/META-INF/LICENSE.txt"}, globPaths(t, r, "META-INF/*")) + assert.Equal(t, []string{"/META-INF/maven/g/a/pom.xml"}, globPaths(t, r, "**/a/*")) +} + +func TestResolver_invalidGlobIsAnError(t *testing.T) { + r := resolverOver(t, 1<<20, jarEntriesFixture) + _, err := r.FilesByGlob("**/[") + assert.Error(t, err) +} + +func globPaths(t *testing.T, r *Resolver, pattern string) []string { + t.Helper() + locations, err := r.FilesByGlob(pattern) + require.NoError(t, err) + return localPaths(locations) +} + +// several paths to one file answer as one location: the file's own path when it matched, otherwise +// the lowest-sorting link +func TestResolver_linksCollapseToOnePath(t *testing.T) { + tests := []struct { + name string + entries []testEntry + pattern string + want []string + reason string + }{ + { + name: "the file itself beats every link to it", + entries: []testEntry{ + {name: "opt/real.jar", body: "PK\x03\x04"}, + {name: "opt/a-sym.jar", link: "real.jar"}, + {name: "opt/z-sym.jar", link: "real.jar"}, + {name: "opt/hard.jar", link: "opt/real.jar", hard: true}, + }, + pattern: "**/*.jar", + want: []string{"/opt/real.jar"}, + reason: "one file, so one location - and it is named by the path that holds it", + }, + { + name: "with no candidate naming the file, the lowest-sorting link wins", + entries: []testEntry{ + {name: "opt/real.bin", body: "PK\x03\x04"}, + {name: "opt/z-sym.jar", link: "real.bin"}, + {name: "opt/a-sym.jar", link: "real.bin"}, + {name: "opt/m-sym.jar", link: "real.bin"}, + }, + pattern: "**/*.jar", + want: []string{"/opt/real.bin"}, + reason: "the content is the target's, so that is the real path the answer carries", + }, + { + name: "distinct files stay distinct", + entries: []testEntry{ + {name: "opt/one.jar", body: "one"}, + {name: "opt/two.jar", body: "two"}, + {name: "opt/one-sym.jar", link: "one.jar"}, + }, + pattern: "**/*.jar", + want: []string{"/opt/one.jar", "/opt/two.jar"}, + reason: "collapsing is per file, not per pattern", + }, + { + name: "a dangling link is not a file", + entries: []testEntry{ + {name: "opt/real.jar", body: "PK\x03\x04"}, + {name: "opt/gone.jar", link: "missing.jar"}, + }, + pattern: "**/*.jar", + want: []string{"/opt/real.jar"}, + reason: "it names no content", + }, + { + name: "two dangling links are two paths, not one", + entries: []testEntry{ + {name: "opt/a.jar", link: "missing.jar"}, + {name: "opt/b.jar", link: "missing.jar"}, + }, + pattern: "**/*.jar", + want: nil, + reason: "a link to nothing is not a file", + }, + { + name: "a chain of links collapses onto the file at its end", + entries: []testEntry{ + {name: "opt/real.jar", body: "PK\x03\x04"}, + {name: "opt/mid.jar", link: "real.jar"}, + {name: "opt/outer.jar", link: "mid.jar"}, + }, + pattern: "**/*.jar", + want: []string{"/opt/real.jar"}, + reason: "following one hop at a time reaches the same file from every path", + }, + { + name: "a cycle resolves to nothing rather than looping", + entries: []testEntry{ + {name: "opt/a.jar", link: "b.jar"}, + {name: "opt/b.jar", link: "a.jar"}, + }, + pattern: "**/*.jar", + want: nil, + reason: "neither reaches a file", + }, + { + name: "a hard link resolves from the archive root, not from its own directory", + entries: []testEntry{ + {name: "opt/real.jar", body: "PK\x03\x04"}, + {name: "opt/nested/hard.jar", link: "opt/real.jar", hard: true}, + }, + pattern: "**/*.jar", + want: []string{"/opt/real.jar"}, + reason: "read as relative it would resolve to opt/nested/opt/real.jar and dangle", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := resolverFrom(t, tt.entries...) + assert.Equal(t, tt.want, globPaths(t, r, tt.pattern), tt.reason) + }) + } +} + +func TestResolver_collapsedLocationReadsTheTarget(t *testing.T) { + r := resolverFrom(t, + testEntry{name: "opt/real.bin", body: "PK\x03\x04 real contents"}, + testEntry{name: "opt/a-sym.jar", link: "real.bin"}, + testEntry{name: "opt/z-sym.jar", link: "real.bin"}, + ) + + locations, err := r.FilesByGlob("**/*.jar") + require.NoError(t, err) + require.Len(t, locations, 1) + + assert.Equal(t, "/opt/real.bin", locations[0].LocalPath(), "content comes from the file") + assert.Equal(t, "outer.jar!/opt/a-sym.jar", locations[0].AccessPath, "the path searched by is the one matched") + + reader, err := r.FileContentsByLocation(locations[0]) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reader.Close()) }) + + body, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "PK\x03\x04 real contents", string(body), + "a location standing for a link must still read the file's bytes, not the link's none") +} + +func TestResolver_fullPathsAreTrimmedToEntryPaths(t *testing.T) { + // catalogers build queries from the full paths the resolver hands out, e.g. path.Dir plus a sibling + r := resolverOver(t, 1<<20, map[string]string{"pom.xml": "a", "lib/dep.jar": "b"}) + + locations, err := r.FilesByPath("outer.jar!/lib/dep.jar") + require.NoError(t, err) + require.Len(t, locations, 1) + assert.Equal(t, "outer.jar!/lib/dep.jar", locations[0].AccessPath) + + locations, err = r.FilesByPath(path.Join(path.Dir(locations[0].AccessPath), "../pom.xml")) + require.NoError(t, err) + require.Len(t, locations, 1, "a sibling at the archive root is reached through the full path") + assert.Equal(t, "outer.jar!/pom.xml", locations[0].FullNestedPath()) + + assert.True(t, r.HasPath(path.Dir("outer.jar!/pom.xml")), "the directory of a root entry is the root") + assert.True(t, r.HasPath("lib/dep.jar"), "an entry path is still accepted") + + locations, err = r.FilesByGlob("outer.jar!/**/*.jar") + require.NoError(t, err) + require.Len(t, locations, 1) +} + +func TestResolver_danglingLinkHasNoContent(t *testing.T) { + r := resolverFrom(t, testEntry{name: "opt/gone.jar", link: "missing.jar"}) + + var locations []file.Location + for l := range r.AllLocations(context.Background()) { + locations = append(locations, l) + } + require.Len(t, locations, 1) + assert.Equal(t, "/opt/gone.jar", locations[0].LocalPath()) + + _, err := r.FileContentsByLocation(locations[0]) + assert.Error(t, err, "a link to nothing has no content to read") + + metadata, err := r.FileMetadataByLocation(locations[0]) + require.NoError(t, err) + assert.Equal(t, "outer.jar!/opt/missing.jar", metadata.LinkDestination, + "where it pointed is still reported, resolved against the link's own directory") +} + +func TestResolver_mimeTypeAnswersOnlyWithFiles(t *testing.T) { + r := resolverFrom(t, + testEntry{name: "opt/real.jar", body: "PK\x03\x04 zip contents here"}, + testEntry{name: "opt/sym.jar", link: "real.jar"}, + testEntry{name: "opt/hard.jar", link: "opt/real.jar", hard: true}, + testEntry{name: "opt/gone.jar", link: "missing.jar"}, + testEntry{name: "opt/notes.txt", body: "hello world"}, + ) + + byMIME := func(types ...string) []string { + t.Helper() + locations, err := r.FilesByMIMEType(types...) + require.NoError(t, err) + return localPaths(locations) + } + + assert.Equal(t, []string{"/opt/real.jar"}, byMIME("application/zip"), + "the file answers, not the two links to it") + assert.Equal(t, []string{"/opt/notes.txt"}, byMIME("text/plain")) + assert.Empty(t, byMIME(""), + "a link sniffs to no type, and must not be reachable by asking for that") + assert.Equal(t, []string{"/opt/notes.txt", "/opt/real.jar"}, byMIME("application/zip", "text/plain")) +} + +// AllLocations enumerates what the archive holds rather than searching for files: the file metadata +// cataloger records a row per path, links included +func TestResolver_allLocationsKeepsEveryPath(t *testing.T) { + r := resolverFrom(t, + testEntry{name: "opt/real.jar", body: "PK\x03\x04"}, + testEntry{name: "opt/sym.jar", link: "real.jar"}, + testEntry{name: "opt/gone.jar", link: "missing.jar"}, + ) + + var got []string + for l := range r.AllLocations(context.Background()) { + got = append(got, l.LocalPath()) + } + assert.Equal(t, []string{"/opt/gone.jar", "/opt/real.jar", "/opt/sym.jar"}, got, + "every path the archive names, links included") +} + +func TestResolver_add_anEntryOverTheCapIsSkippedAndReadNoFurther(t *testing.T) { + // one entry of an overlapping-entry bomb can inflate without bound; it must stop at the cap, cost + // the archive only that entry, and say so + orig := maxEntryBytes + maxEntryBytes = 1024 + t.Cleanup(func() { maxEntryBytes = orig }) + + r, _ := resolverIn(t, memCharge(-1)) + endless := &countingReader{r: zeroSource{}} + require.NoError(t, r.add(tar.Header{Name: "bomb.bin", Typeflag: tar.TypeReg, Mode: 0o644}, endless)) + require.NoError(t, r.add(tar.Header{Name: "ok.txt", Typeflag: tar.TypeReg, Mode: 0o644}, strings.NewReader("ok"))) + r.finish() + + assert.LessOrEqual(t, endless.n, 1024+copyChunkSize, "reading stops just past the cap") + assert.ErrorIs(t, r.Truncated, errEntryTooLarge) + assert.ErrorContains(t, r.Truncated, "per-entry size cap of 1.0 KiB") + locations, err := r.FilesByGlob("**") + require.NoError(t, err) + assert.Equal(t, []string{"/ok.txt"}, localPaths(locations)) +} + +type countingReader struct { + r io.Reader + n int +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += n + return n, err +} diff --git a/internal/archive/storage.go b/internal/archive/storage.go new file mode 100644 index 000000000..c654d0f3c --- /dev/null +++ b/internal/archive/storage.go @@ -0,0 +1,217 @@ +package archive + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + + "github.com/anchore/syft/internal/log" +) + +// copyChunkSize is how many bytes are read and charged at a time, bounding how far a charge runs +// ahead of what is actually held. +const copyChunkSize = 32 * 1024 + +// spillFilePattern names the one file an archive's content spills into, as in os.CreateTemp. +const spillFilePattern = "archive-spill-*" + +// ReaderAtSeeker is the random access an archive format needs: a zip is read from its central +// directory at the end of the stream. +type ReaderAtSeeker interface { + io.Reader + io.ReaderAt + io.Seeker +} + +// blob is one piece of content the resolver holds: in memory, or at an offset in its spill file once +// memory refused it. Moving changes nothing that points at the blob. +type blob struct { + mem []byte // content while held in memory + off int64 // where content starts in the spill file once written there + size int64 +} + +// put reads content in full into b, since archive readers are sequential. Each chunk is charged before it +// is held, so a stream larger than the memory limit is never held in full. Reaching the disk limit +// returns ErrDiskLimitReached; what landed in the file before that stays there and stays charged. +func (r *Resolver) put(b *blob, content io.Reader) error { + if r.chunk == nil { + r.chunk = make([]byte, copyChunkSize) + } + for { + n, readErr := content.Read(r.chunk) + if n > 0 { + // content read through a meteredReader was charged as it was decoded + if !r.metered && !r.budget.take(int64(n)) { + return errBudgetSpent + } + if err := r.write(b, r.chunk[:n]); err != nil { + return err + } + } + if readErr == nil { + continue + } + if errors.Is(readErr, io.EOF) { + break + } + return readErr + } + return nil +} + +// meteredReader charges the decompression budget for every byte read through it and stops once ctx is +// done, so nothing a decoder produces gets around the budget. +type meteredReader struct { + ctx context.Context + reader io.Reader + budget *Budget +} + +func (m *meteredReader) Read(p []byte) (int, error) { + if err := m.ctx.Err(); err != nil { + return 0, err + } + n, err := m.reader.Read(p) + if n > 0 && !m.budget.take(int64(n)) { + return 0, errBudgetSpent + } + return n, err +} + +// write appends p to b: into memory while the memory limit admits it, otherwise into the spill file +// after everything held in memory is moved there first. +func (r *Resolver) write(b *blob, p []byte) error { + n := int64(len(p)) + onDisk := b.mem == nil && b.size > 0 + if !onDisk { + if r.charge.memory(n) { + if b.mem == nil { + r.held = append(r.held, b) + } + b.mem = append(b.mem, p...) + b.size += n + return nil + } + if err := r.spill(); err != nil { + return err + } + } + + if err := r.openFile(); err != nil { + return err + } + if !r.charge.disk(n) { + return ErrDiskLimitReached + } + if b.size == 0 { + b.off = r.written + } + // b is always the blob most recently written, so it ends where the file does + w, err := r.file.WriteAt(p, r.written) + r.written += int64(w) + b.size += int64(w) + r.charge.refundDisk(n - int64(w)) + if err != nil { + return fmt.Errorf("unable to write archive content to disk: %w", err) + } + return nil +} + +// spill moves every blob held in memory into the spill file, charging the disk limit as bytes land +// and refunding the memory they leave. +func (r *Resolver) spill() error { + if len(r.held) == 0 { + return nil + } + if err := r.openFile(); err != nil { + return err + } + log.WithFields("archive", r.archiveCoordinates.FullNestedPath(), "held", len(r.held), "bytes", r.written). + Debug("archive content does not fit in memory; writing it to disk") + + for len(r.held) > 0 { + b := r.held[0] + n := int64(len(b.mem)) + if !r.charge.disk(n) { + return ErrDiskLimitReached + } + if _, err := r.file.WriteAt(b.mem, r.written); err != nil { + return fmt.Errorf("unable to write archive content to disk: %w", err) + } + b.off, b.mem = r.written, nil + r.written += n + r.charge.refundMemory(n) + r.held = r.held[1:] + } + return nil +} + +func (r *Resolver) openFile() error { + if r.file != nil { + return nil + } + var err error + if r.tempDir != nil { + r.file, r.remove, err = r.tempDir.NewFile(spillFilePattern) + } else { + r.file, err = os.CreateTemp("", spillFilePattern) + r.remove = func() { + if err := os.Remove(r.file.Name()); err != nil { + log.WithFields("path", r.file.Name(), "error", err).Trace("unable to remove archive temp file") + } + } + } + if err != nil { + return fmt.Errorf("unable to create archive temp file: %w", err) + } + return nil +} + +// discard drops a blob's bytes held in memory and refunds them. Bytes it holds on disk stay until +// Cleanup. +func (r *Resolver) discard(b *blob) { + for i, held := range r.held { + if held == b { + r.held = append(r.held[:i], r.held[i+1:]...) + break + } + } + r.charge.refundMemory(int64(len(b.mem))) + b.mem, b.size = nil, 0 +} + +// open returns a reader over a blob's content, wherever it is. The reader supports Read, Seek and +// ReadAt, so a nested archive can be read where it lies. +func (r *Resolver) open(b *blob) ReaderAtSeeker { + switch { + case b.mem != nil: + return bytes.NewReader(b.mem) + case b.size == 0: + return bytes.NewReader(nil) + default: + return io.NewSectionReader(r.file, b.off, b.size) + } +} + +// releaseStorage refunds every blob held in memory and closes, removes and refunds the spill file. +// Readers from open must not be used after this. Safe to call more than once. +func (r *Resolver) releaseStorage() { + for _, b := range r.held { + r.charge.refundMemory(int64(len(b.mem))) + b.mem = nil + } + r.held = nil + if r.file == nil { + return + } + if err := r.file.Close(); err != nil { + log.WithFields("archive", r.archiveCoordinates.FullNestedPath(), "error", err).Trace("unable to close archive temp file") + } + r.remove() + r.charge.refundDisk(r.written) + r.file, r.remove, r.written = nil, nil, 0 +} diff --git a/internal/archive/storage_test.go b/internal/archive/storage_test.go new file mode 100644 index 000000000..67faa1035 --- /dev/null +++ b/internal/archive/storage_test.go @@ -0,0 +1,276 @@ +package archive + +import ( + "archive/zip" + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolver_put_holdsSmallContentInMemory(t *testing.T) { + s, dir := resolverIn(t, memCharge(1024)) + + var b blob + require.NoError(t, s.put(&b, bytes.NewReader([]byte("hello")))) + + assert.Equal(t, int64(5), s.heldInMemory()) + assert.Empty(t, filesIn(t, dir)) + assert.Equal(t, "hello", readEntry(t, s.open(&b))) +} + +func TestResolver_put_spillsToDiskAndKeepsTheSameBlobs(t *testing.T) { + // nodes pointing at a blob are not rebuilt when its content moves + s, dir := resolverIn(t, memCharge(8)) + + var first, second blob + require.NoError(t, s.put(&first, bytes.NewReader([]byte("aaaa")))) + require.Equal(t, int64(4), s.heldInMemory()) + + require.NoError(t, s.put(&second, bytes.NewReader([]byte("bbbbbb")))) + + assert.Zero(t, s.heldInMemory(), "everything moves out, not just the blob that did not fit") + assert.NotEmpty(t, spillFile(t, dir)) + assert.Equal(t, int64(10), s.written) + assert.Equal(t, "aaaa", readEntry(t, s.open(&first))) + assert.Equal(t, "bbbbbb", readEntry(t, s.open(&second))) +} + +func TestResolver_put_threeStateLimits(t *testing.T) { + body := []byte(strings.Repeat("e", 500)) + + t.Run("a zero memory limit sends everything to disk", func(t *testing.T) { + charge := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 10_000}).charge() + s, dir := resolverIn(t, charge) + + var b blob + require.NoError(t, s.put(&b, bytes.NewReader(body))) + + assert.Zero(t, s.heldInMemory()) + assert.NotEmpty(t, spillFile(t, dir)) + assert.Equal(t, body, []byte(readEntry(t, s.open(&b)))) + mem, disk := charge.held() + assert.Zero(t, mem) + assert.Equal(t, int64(500), disk) + }) + + t.Run("a negative memory limit holds content regardless of how much is already held", func(t *testing.T) { + charge := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: 10_000}).charge() + require.True(t, charge.memory(1_000_000)) + s, dir := resolverIn(t, charge) + + var b blob + require.NoError(t, s.put(&b, bytes.NewReader(body))) + + assert.Empty(t, filesIn(t, dir)) + mem, disk := charge.held() + assert.Equal(t, int64(1_000_000+len(body)), mem) + assert.Zero(t, disk) + }) + + t.Run("a zero disk limit still holds what memory admits", func(t *testing.T) { + charge := NewLimiter(Limits{MaxMemoryBytes: 10_000, MaxDiskBytes: 0}).charge() + s, dir := resolverIn(t, charge) + + var b blob + require.NoError(t, s.put(&b, bytes.NewReader(body))) + + assert.Empty(t, filesIn(t, dir)) + mem, disk := charge.held() + assert.Equal(t, int64(len(body)), mem) + assert.Zero(t, disk) + }) + + t.Run("a negative disk limit writes with no ceiling", func(t *testing.T) { + s, dir := resolverIn(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}).charge()) + + var b blob + require.NoError(t, s.put(&b, bytes.NewReader(body))) + + assert.NotEmpty(t, spillFile(t, dir)) + assert.Equal(t, body, []byte(readEntry(t, s.open(&b)))) + }) + + t.Run("a nil charge holds everything in memory", func(t *testing.T) { + s, dir := resolverIn(t, nil) + + var b blob + require.NoError(t, s.put(&b, bytes.NewReader(body))) + + assert.Empty(t, filesIn(t, dir)) + assert.Equal(t, body, []byte(readEntry(t, s.open(&b)))) + }) +} + +func TestResolver_put_bytesReadBeforeTheLimitRefusedAreNotLost(t *testing.T) { + body := []byte(strings.Repeat("b", 250)) + charge := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 10_000}).charge() + s, _ := resolverIn(t, charge) + + var b blob + require.NoError(t, s.put(&b, bytes.NewReader(body))) + + assert.Equal(t, body, []byte(readEntry(t, s.open(&b)))) + mem, disk := charge.held() + assert.Zero(t, mem, "nothing stays charged to a limit that refused") + assert.Equal(t, int64(250), disk) +} + +// peakTrackingReader records the highest memory charge observed while being read. +type peakTrackingReader struct { + r io.Reader + charge *charge + peak int64 +} + +func (p *peakTrackingReader) Read(b []byte) (int, error) { + n, err := p.r.Read(b) + if mem, _ := p.charge.held(); mem > p.peak { + p.peak = mem + } + return n, err +} + +func TestResolver_put_neverBuffersBeyondTheMemoryLimit(t *testing.T) { + const limit = int64(3 * copyChunkSize) + body := []byte(strings.Repeat("c", 10*copyChunkSize)) + charge := NewLimiter(Limits{MaxMemoryBytes: limit, MaxDiskBytes: int64(len(body)) + 1}).charge() + tracked := &peakTrackingReader{r: bytes.NewReader(body), charge: charge} + s, _ := resolverIn(t, charge) + + var b blob + require.NoError(t, s.put(&b, tracked)) + + assert.Equal(t, body, []byte(readEntry(t, s.open(&b)))) + assert.LessOrEqual(t, tracked.peak, limit, "memory charged must never exceed the limit, even transiently") + mem, disk := charge.held() + assert.Zero(t, mem) + assert.Equal(t, int64(len(body)), disk) +} + +func TestResolver_put_diskLimitIsTerminal(t *testing.T) { + tests := []struct { + name string + limits Limits + }{ + {"a disk limit that is exceeded", Limits{MaxDiskBytes: 100}}, + {"a zero disk limit", Limits{MaxMemoryBytes: 10, MaxDiskBytes: 0}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + charge := NewLimiter(tt.limits).charge() + s, dir := resolverIn(t, charge) + + var b blob + err := s.put(&b, bytes.NewReader([]byte(strings.Repeat("d", 500)))) + require.ErrorIs(t, err, ErrDiskLimitReached) + + s.Cleanup() + assert.Empty(t, filesIn(t, dir), "cleanup removes the partial file") + mem, disk := charge.held() + assert.Zero(t, mem) + assert.Zero(t, disk, "and refunds it") + }) + } +} + +func TestResolver_cleanupRefundsEverythingAndIsIdempotent(t *testing.T) { + charge := NewLimiter(Limits{MaxMemoryBytes: 8, MaxDiskBytes: -1}).charge() + s, dir := resolverIn(t, charge) + + var spilled, held blob + require.NoError(t, s.put(&spilled, bytes.NewReader([]byte("spilled to disk")))) + require.NoError(t, s.put(&held, bytes.NewReader([]byte("held")))) + mem, disk := charge.held() + require.Equal(t, int64(4), mem) + require.Equal(t, int64(15), disk) + require.Equal(t, "held", readEntry(t, s.open(&held))) + + s.Cleanup() + + assert.Empty(t, filesIn(t, dir)) + mem, disk = charge.held() + assert.Zero(t, mem) + assert.Zero(t, disk) + _, err := io.ReadAll(s.open(&spilled)) + assert.Error(t, err, "readers over the spill file are no longer valid") + s.Cleanup() + mem, disk = charge.held() + assert.Zero(t, mem, "a second cleanup refunds nothing twice") + assert.Zero(t, disk) +} + +func TestResolver_put_discardRefundsWhatIsHeldInMemory(t *testing.T) { + charge := memCharge(1024) + s, _ := resolverIn(t, charge) + + var keep, drop blob + require.NoError(t, s.put(&keep, bytes.NewReader([]byte("keep")))) + require.NoError(t, s.put(&drop, bytes.NewReader([]byte("dropped")))) + + s.discard(&drop) + + mem, _ := charge.held() + assert.Equal(t, int64(4), mem) + assert.Equal(t, int64(4), s.heldInMemory()) + assert.Empty(t, readEntry(t, s.open(&drop))) + assert.Equal(t, "keep", readEntry(t, s.open(&keep))) +} + +func TestResolver_filesystemCostDoesNotTrackEntryCount(t *testing.T) { + small, smallRoot := extractedResolver(t, manyEntryZip(t, 2), spillingLimits) + large, largeRoot := extractedResolver(t, manyEntryZip(t, 5000), spillingLimits) + + assert.Len(t, filesIn(t, smallRoot), 1) + assert.Len(t, filesIn(t, largeRoot), 1, "5000 entries cost the same filesystem entries as 2") + + assert.Equal(t, 2, small.entries) + assert.Equal(t, 5000, large.entries) + + // entries cost their content and nothing else on disk: no headers, no padding + assert.Equal(t, int64(2), small.written) + assert.Equal(t, int64(5000), large.written) +} + +func TestResolver_anEntryAtTheEndIsReadBySeeking(t *testing.T) { + // the reader over an entry is a bounded window at its offset, not a re-read of what precedes it + const filler = 64 * 1024 + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for i := range 8 { + w, err := zw.Create(fmt.Sprintf("filler%d.bin", i)) + require.NoError(t, err) + _, err = w.Write(bytes.Repeat([]byte{byte(i)}, filler)) + require.NoError(t, err) + } + last, err := zw.Create("zz-last.txt") + require.NoError(t, err) + wanted := strings.Repeat("last", 64) + _, err = last.Write([]byte(wanted)) + require.NoError(t, err) + require.NoError(t, zw.Close()) + + r, root := extractedResolver(t, buf.Bytes(), spillingLimits) + + body, err := os.ReadFile(filepath.Join(root, spillFile(t, root))) + require.NoError(t, err) + offset := int64(bytes.Index(body, []byte(wanted))) + require.Greater(t, offset, int64(len(body))*9/10, "the last entry must really be at the end of the file") + + reader := r.open(contentAt(t, r, "/zz-last.txt")) + end, err := reader.Seek(0, io.SeekEnd) + require.NoError(t, err) + assert.Equal(t, int64(len(wanted)), end) + + head := make([]byte, 8) + _, err = reader.ReadAt(head, 0) + require.NoError(t, err) + assert.Equal(t, wanted[:8], string(head)) +} diff --git a/internal/archive/traversal.go b/internal/archive/traversal.go new file mode 100644 index 000000000..dd79d126a --- /dev/null +++ b/internal/archive/traversal.go @@ -0,0 +1,194 @@ +package archive + +import ( + "context" + "path" + "slices" + "strings" + + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" +) + +// Traversal describes the archive whose contents a cataloger is running against. The archive +// cataloger task puts one on the context before running catalogers inside an archive, so a cataloger +// such as java can describe the archive it is inside without owning the recursion. +type Traversal struct { + // Location is the archive file's location within its parent filesystem, including the access path + // it was reached by, so VirtualPath(Location) is the chain from the scan root as seen through that + // access path. The files inside it extend its real coordinates instead (see ContainerPath). + Location file.Location + + // Digests are of the archive file itself, taken during extraction; a cataloger running inside the + // archive cannot compute them since the archive is not inside itself. + Digests []file.Digest + + // Parent is the traversal of the archive this one is nested in, nil for an archive at the scan root. + Parent *Traversal + + // Packages are those the catalogers found describing this archive (located at it), filled in once + // they have run. Catalogers inside a nested archive read them through Parent, as java does to link a + // nested jar to the jar that holds it. + Packages []pkg.Package +} + +// FullPathIn is the full path of the entry at entryPath inside the archive whose full path is +// containerPath, or entryPath itself for a blank containerPath (the scan root). +func FullPathIn(containerPath, entryPath string) string { + if containerPath == "" { + return entryPath + } + // cleaning an already clean path allocates nothing + return containerPath + file.ArchivePathSeparator + strings.TrimPrefix(path.Clean(entryPath), "/") +} + +// OwnerContainers are the containers a package owns files within: the container of each of its +// locations, each once, as coordinates whose FullPath is the container's full path (blank at the scan +// root). A package without locations owns files at the scan root. See OwnedCoordinates. +func OwnerContainers(p pkg.Package) []file.Coordinates { + var out []file.Coordinates + for _, l := range p.Locations.ToUnorderedSlice() { + c := file.Coordinates{FileSystemID: l.FileSystemID} + if container := ContainerPath(l.Coordinates); container != "" { + c.RealPath, c.FullPath = l.RealPath, container + } + if !slices.Contains(out, c) { + out = append(out, c) + } + } + if len(out) == 0 { + out = append(out, file.Coordinates{}) + } + return out +} + +// OwnedCoordinates are the coordinates of the file at ownedPath within container, one of +// OwnerContainers. At the scan root the path may still need resolving (it may pass through links). +func OwnedCoordinates(container file.Coordinates, ownedPath string) file.Coordinates { + if container.FullPath == "" { + return file.Coordinates{RealPath: ownedPath, FileSystemID: container.FileSystemID} + } + return file.Coordinates{ + RealPath: container.RealPath, + FileSystemID: container.FileSystemID, + FullPath: FullPathIn(container.FullPath, ownedPath), + } +} + +// FullTraversalArchivePath is the archive's own full path, which is the ContainerPath of every file inside it. Outside +// any archive (a nil Traversal) it is blank, as for files on the scanned filesystem. +func (t *Traversal) FullTraversalArchivePath() string { + if t == nil { + return "" + } + return t.Location.FullNestedPath() +} + +// ContainerPath returns the full path of the archive directly holding the file at c, or blank for a +// file on the scanned filesystem, whether or not its FullPath is set. +func ContainerPath(c file.Coordinates) string { + entries, ok := archiveEntries(c) + if !ok { + return "" + } + if i := strings.LastIndex(entries, file.ArchivePathSeparator); i >= 0 { + return c.RealPath + file.ArchivePathSeparator + entries[:i] + } + return c.RealPath +} + +// ContainingArchive returns the coordinates of the archive directly holding the file at c, reporting false +// for a file on the scanned filesystem. +func ContainingArchive(c file.Coordinates) (file.Coordinates, bool) { + containerPath := ContainerPath(c) + if containerPath == "" { + return file.Coordinates{}, false + } + container := file.Coordinates{RealPath: c.RealPath, FileSystemID: c.FileSystemID} + if containerPath != c.RealPath { + container.FullPath = containerPath + } + return container, true +} + +// archiveEntries is the "!/"-delimited chain of entry paths after RealPath, reporting false for a file +// not inside an archive. RealPath is split off first, so a "!/" in it is not taken for a boundary. +func archiveEntries(c file.Coordinates) (string, bool) { + remain, ok := strings.CutPrefix(c.FullNestedPath(), c.RealPath) + if !ok { + return "", false + } + return strings.CutPrefix(remain, file.ArchivePathSeparator) +} + +// VirtualPath returns the colon-delimited chain of archives from the scan root to loc, ending in loc +// itself as accessed: "app.war:WEB-INF/lib/dep.jar". Outside any archive it is the location's path +// unchanged. In an entry name '%' is escaped as "%25" and then ':' as "%3A", so splitting on ':' +// recovers the archive boundaries and no two entry names share an escaped form. Leading slashes are +// dropped to match the java cataloger, which joins slash-less zip entry names onto the containing +// archive's path. +func VirtualPath(loc file.Location) string { + accessPath := loc.Path() + entries, ok := archiveEntries(loc.Coordinates) + if !ok { + return accessPath + } + segments := strings.Split(entries, file.ArchivePathSeparator) + if rest, ok := strings.CutPrefix(accessPath, ContainerPath(loc.Coordinates)); ok && strings.HasPrefix(rest, file.ArchivePathSeparator) { + accessPath = rest[len(file.ArchivePathSeparator):] + } + segments[len(segments)-1] = accessPath + for i, s := range segments { + segments[i] = strings.TrimPrefix(escapeEntryPath(s), "/") + } + return loc.RealPath + ":" + strings.Join(segments, ":") +} + +var entryPathEscaper = strings.NewReplacer("%", "%25", ":", "%3A") + +func escapeEntryPath(p string) string { + if !strings.ContainsAny(p, "%:") { + return p + } + return entryPathEscaper.Replace(p) +} + +type traversalCtxKey struct{} + +func WithTraversal(ctx context.Context, t *Traversal) context.Context { + return context.WithValue(ctx, traversalCtxKey{}, t) +} + +// TraversalFromContext returns the archive traversal on the context, or nil outside an archive. +func TraversalFromContext(ctx context.Context) *Traversal { + t, _ := ctx.Value(traversalCtxKey{}).(*Traversal) + return t +} + +type budgetCtxKey struct{} + +// WithBudget sets the decompression budget that Extract spends from, for this archive and every archive +// extracted beneath it. +func WithBudget(ctx context.Context, b *Budget) context.Context { + return context.WithValue(ctx, budgetCtxKey{}, b) +} + +func budgetFromContext(ctx context.Context) *Budget { + b, _ := ctx.Value(budgetCtxKey{}).(*Budget) + return b +} + +type nestedCatalogingCtxKey struct{} + +// WithNestedCataloging marks the context of a scan whose archive cataloger task will extract every +// archive and run the catalogers inside it. syft.CreateSBOM sets it before any task runs, so a +// cataloger that would otherwise open archives itself can leave them to the task instead. +func WithNestedCataloging(ctx context.Context) context.Context { + return context.WithValue(ctx, nestedCatalogingCtxKey{}, true) +} + +// NestedCatalogingEnabled reports whether the archive cataloger task is extracting archives in this scan. +func NestedCatalogingEnabled(ctx context.Context) bool { + enabled, _ := ctx.Value(nestedCatalogingCtxKey{}).(bool) + return enabled +} diff --git a/internal/archive/traversal_test.go b/internal/archive/traversal_test.go new file mode 100644 index 000000000..73d0306b0 --- /dev/null +++ b/internal/archive/traversal_test.go @@ -0,0 +1,113 @@ +package archive + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/file" +) + +func TestTraversalContextRoundTrip(t *testing.T) { + assert.Nil(t, TraversalFromContext(context.Background())) + + trav := &Traversal{Location: file.NewLocation("app.war")} + ctx := WithTraversal(context.Background(), trav) + require.Same(t, trav, TraversalFromContext(ctx)) + + child := &Traversal{Location: locationIn("app.war", "WEB-INF/lib/dep.jar")} + ctx = WithTraversal(ctx, child) + require.Same(t, child, TraversalFromContext(ctx)) +} + +// locationIn is a location as the Resolver returns it: the outermost archive's real path, and the chain +// of entry paths within it. +func locationIn(realPath, archivePath string) file.Location { + return file.NewLocationFromCoordinates(file.Coordinates{RealPath: realPath, FullPath: realPath + file.ArchivePathSeparator + archivePath}) +} + +func TestContainerPath(t *testing.T) { + assert.Equal(t, "", ContainerPath(file.NewCoordinates("/app.war", ""))) + assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/app.war", FullPath: "/app.war"}), "a top-level file whose FullPath is set") + assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/odd!/name"}), "a separator in a top-level real path is not an archive boundary") + assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/odd!/name", FullPath: "/odd!/name"}), "nor when FullPath is set") + assert.Equal(t, "/odd!/name.zip", ContainerPath(locationIn("/odd!/name.zip", "a.txt").Coordinates)) + assert.Equal(t, "/odd!/name.zip!/b.jar", ContainerPath(locationIn("/odd!/name.zip", "b.jar!/a.txt").Coordinates)) + assert.Equal(t, "/app.war", ContainerPath(locationIn("/app.war", "WEB-INF/lib/dep.jar").Coordinates)) + assert.Equal(t, "/app.war!/WEB-INF/lib/dep.jar", + ContainerPath(locationIn("/app.war", "WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF").Coordinates)) +} + +func TestVirtualPath(t *testing.T) { + tests := []struct { + name string + location file.Location + expected string + }{ + { + name: "outside an archive returns the path unchanged", + location: file.NewLocation("/WEB-INF/lib/dep.jar"), + expected: "/WEB-INF/lib/dep.jar", + }, + { + name: "outside an archive preserves a relative path", + location: file.NewLocation("some/dir/example.jar"), + expected: "some/dir/example.jar", + }, + { + name: "outside an archive with FullPath set returns the path unchanged", + location: file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/odd!/name.jar", FullPath: "/odd!/name.jar"}), + expected: "/odd!/name.jar", + }, + { + name: "single level trims leading slash", + location: locationIn("app.war", "/WEB-INF/lib/dep.jar"), + expected: "app.war:WEB-INF/lib/dep.jar", + }, + { + name: "single level with slash-less entry", + location: locationIn("app.war", "WEB-INF/lib/dep.jar"), + expected: "app.war:WEB-INF/lib/dep.jar", + }, + { + name: "multi-level chain", + location: locationIn("app.war", "WEB-INF/lib/dep.jar!/inner/most.jar"), + expected: "app.war:WEB-INF/lib/dep.jar:inner/most.jar", + }, + { + // a colon in the entry name must not be mistaken for a chain delimiter + name: "colon in entry path is escaped", + location: locationIn("app.war", "WEB-INF/lib/dep:special.jar"), + expected: "app.war:WEB-INF/lib/dep%3Aspecial.jar", + }, + { + name: "every colon in the entry path is escaped", + location: locationIn("app.war", "lib/a:b:c.jar"), + expected: "app.war:lib/a%3Ab%3Ac.jar", + }, + { + // or the literal text %3A would collide with an escaped colon + name: "the escape character is itself escaped", + location: locationIn("outer.zip", "a%3Ab.jar"), + expected: "outer.zip:a%253Ab.jar", + }, + { + name: "access path is preferred over real path", + location: file.NewVirtualLocation("/real/dep.jar", "/link/dep.jar"), + expected: "/link/dep.jar", + }, + { + name: "access path within an archive replaces the last entry", + location: file.NewVirtualLocationFromCoordinates( + file.Coordinates{RealPath: "app.war", FullPath: "app.war!/WEB-INF/lib/real.jar"}, "app.war!/WEB-INF/lib/link.jar"), + expected: "app.war:WEB-INF/lib/link.jar", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, VirtualPath(tt.location)) + }) + } +} diff --git a/internal/bus/helpers.go b/internal/bus/helpers.go index f13364c2e..2867aeff2 100644 --- a/internal/bus/helpers.go +++ b/internal/bus/helpers.go @@ -1,6 +1,9 @@ package bus import ( + "context" + "sync" + "github.com/wagoodman/go-partybus" "github.com/wagoodman/go-progress" @@ -36,7 +39,82 @@ func Notify(message string) { }) } -func StartCatalogerTask(info monitor.GenericTask, size int64, initialStage string) *monitor.TaskProgress { +type catalogerTaskRegistryKey struct{} + +// catalogerTaskRegistry remembers the progress row each cataloger publishes. A cataloger runs once +// over the scan root and again inside every archive the archive cataloger walks; a re-run reports +// into the row it already owns rather than publishing another, and republishing an ID would replace +// the live row with one the UI never sees completed. +type catalogerTaskRegistry struct { + mu sync.Mutex + rows map[string]*monitor.TaskProgress +} + +// WithCatalogerTaskRegistry returns a context under which each cataloger publishes one progress row +// however many times it runs. A context that already carries a registry is returned unchanged. +func WithCatalogerTaskRegistry(ctx context.Context) context.Context { + if catalogerTaskRegistryFromContext(ctx) != nil { + return ctx + } + return context.WithValue(ctx, catalogerTaskRegistryKey{}, &catalogerTaskRegistry{ + rows: make(map[string]*monitor.TaskProgress), + }) +} + +func catalogerTaskRegistryFromContext(ctx context.Context) *catalogerTaskRegistry { + reg, _ := ctx.Value(catalogerTaskRegistryKey{}).(*catalogerTaskRegistry) + return reg +} + +// catalogerTaskKey identifies the row a task owns: its ID when it has one, otherwise its parent and +// title (file catalogers publish no ID). +func catalogerTaskKey(info monitor.GenericTask) string { + if info.ID != "" { + return "id:" + info.ID + } + return "title:" + info.ParentID + "/" + info.Title.Default +} + +func StartCatalogerTask(ctx context.Context, info monitor.GenericTask, size int64, initialStage string) *monitor.TaskProgress { + reg := catalogerTaskRegistryFromContext(ctx) + if reg == nil { + return publishCatalogerTask(info, size, initialStage) + } + + reg.mu.Lock() + defer reg.mu.Unlock() + + key := catalogerTaskKey(info) + if row, ok := reg.rows[key]; ok { + resumeCatalogerTask(row, size, initialStage) + return row + } + + t := publishCatalogerTask(info, size, initialStage) + reg.rows[key] = t + + return t +} + +// resumeCatalogerTask reopens a row an earlier run completed and grows its total by what this run +// adds, so the row does not read as finished mid-walk. A row carrying a real failure keeps it. +func resumeCatalogerTask(row *monitor.TaskProgress, size int64, initialStage string) { + if size < 0 || row.Size() < 0 { + row.SetTotal(-1) + } else { + row.SetTotal(row.Size() + size) + } + + if progress.IsErrCompleted(row.Error()) { + row.SetError(nil) + } + + if initialStage != "" { + row.AtomicStage.Set(initialStage) + } +} + +func publishCatalogerTask(info monitor.GenericTask, size int64, initialStage string) *monitor.TaskProgress { t := &monitor.TaskProgress{ AtomicStage: progress.NewAtomicStage(initialStage), Manual: progress.NewManual(size), diff --git a/internal/bus/helpers_test.go b/internal/bus/helpers_test.go new file mode 100644 index 000000000..4df218c24 --- /dev/null +++ b/internal/bus/helpers_test.go @@ -0,0 +1,152 @@ +package bus + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-progress" + + "github.com/anchore/syft/syft/event/monitor" +) + +func digestsTask() monitor.GenericTask { + return monitor.GenericTask{ + Title: monitor.Title{Default: "File digests"}, + ParentID: monitor.TopLevelCatalogingTaskID, + } +} + +func TestStartCatalogerTask_withoutARegistryEveryRunPublishes(t *testing.T) { + // the right answer for a caller driving a single pass: nothing re-runs, so nothing should be reused + ctx := context.Background() + + first := StartCatalogerTask(ctx, digestsTask(), 2, "") + second := StartCatalogerTask(ctx, digestsTask(), 3, "") + + assert.NotSame(t, first, second) + assert.Equal(t, int64(2), first.Size()) + assert.Equal(t, int64(3), second.Size()) +} + +func TestStartCatalogerTask_reRunReportsIntoTheRowItAlreadyOwns(t *testing.T) { + ctx := WithCatalogerTaskRegistry(context.Background()) + + first := StartCatalogerTask(ctx, digestsTask(), 2, "") + first.Increment() + first.Increment() + first.SetCompleted() + + // the archive walk reaches the same cataloger again, over an archive's contents this time + second := StartCatalogerTask(ctx, digestsTask(), 3, "") + require.Same(t, first, second, "a re-run must find the row it already published") + + assert.Equal(t, int64(5), second.Size(), "the total must cover both runs, not just the first") + assert.False(t, progress.IsCompleted(second), "a row with work still to do must not read as complete") + + for i := 0; i < 3; i++ { + second.Increment() + } + second.SetCompleted() + + assert.Equal(t, int64(5), second.Current()) + assert.Equal(t, int64(5), second.Size(), "the count must not run past the total") + assert.True(t, progress.IsCompleted(second)) +} + +func TestStartCatalogerTask_aTotalResolvedOnCompletionIsAddable(t *testing.T) { + // a run that could not say how much it would do says so when it signs off (Manual.SetCompleted + // backfills the total from the count), so by the time a re-run arrives there is a real number to + // add to. This is the file metadata cataloger's shape: it publishes -1 every run. + ctx := WithCatalogerTaskRegistry(context.Background()) + + row := StartCatalogerTask(ctx, digestsTask(), -1, "") + row.Increment() + row.SetCompleted() + require.Equal(t, int64(1), row.Size(), "signing off must resolve the total to what the run did") + + require.Same(t, row, StartCatalogerTask(ctx, digestsTask(), -1, "")) + assert.Equal(t, int64(-1), row.Size(), "a re-run that cannot say its own size makes the row indeterminate again") + + row.Increment() + row.SetCompleted() + assert.Equal(t, int64(2), row.Current()) + assert.Equal(t, int64(2), row.Size(), "and signing off resolves it again, over both runs") +} + +func TestStartCatalogerTask_indeterminateTotalsStayIndeterminate(t *testing.T) { + // a run whose size is unknown cannot be added to one that is known + tests := []struct { + name string + first, next int64 + complete func(row *monitor.TaskProgress) + }{ + { + name: "the re-run cannot say its own size", + first: 2, next: -1, + complete: func(row *monitor.TaskProgress) { row.Increment(); row.SetCompleted() }, + }, + { + name: "the first run did nothing, so completing resolved nothing", + first: -1, next: 3, + complete: func(row *monitor.TaskProgress) { row.SetCompleted() }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := WithCatalogerTaskRegistry(context.Background()) + + row := StartCatalogerTask(ctx, digestsTask(), tt.first, "") + tt.complete(row) + + require.Same(t, row, StartCatalogerTask(ctx, digestsTask(), tt.next, "")) + assert.Equal(t, int64(-1), row.Size()) + assert.False(t, progress.IsCompleted(row), "an indeterminate row is not complete until it says so") + }) + } +} + +func TestStartCatalogerTask_aFailedRowKeepsItsFailure(t *testing.T) { + // a later run finding nothing wrong does not mean the earlier one succeeded + ctx := WithCatalogerTaskRegistry(context.Background()) + failure := errors.New("unable to read the thing") + + row := StartCatalogerTask(ctx, digestsTask(), 2, "") + row.SetError(failure) + + require.Same(t, row, StartCatalogerTask(ctx, digestsTask(), 3, "")) + assert.ErrorIs(t, row.Error(), failure) +} + +func TestStartCatalogerTask_rowsAreKeyedByTaskIdentity(t *testing.T) { + ctx := WithCatalogerTaskRegistry(context.Background()) + + byID := func(id string) monitor.GenericTask { + return monitor.GenericTask{ID: id, Title: monitor.Title{Default: "shared title"}} + } + + assert.Same(t, StartCatalogerTask(ctx, byID("a"), -1, ""), StartCatalogerTask(ctx, byID("a"), -1, "")) + assert.NotSame(t, StartCatalogerTask(ctx, byID("a"), -1, ""), StartCatalogerTask(ctx, byID("b"), -1, ""), + "an ID is the identity where there is one, even when two tasks share a title") + + // a task publishing without an ID, as every file cataloger does, is keyed by where it sits and what + // it is called + underCataloging := monitor.GenericTask{Title: monitor.Title{Default: "Files"}, ParentID: "cataloging"} + underSomethingElse := monitor.GenericTask{Title: monitor.Title{Default: "Files"}, ParentID: "other"} + + assert.Same(t, StartCatalogerTask(ctx, underCataloging, -1, ""), StartCatalogerTask(ctx, underCataloging, -1, "")) + assert.NotSame(t, StartCatalogerTask(ctx, underCataloging, -1, ""), StartCatalogerTask(ctx, underSomethingElse, -1, "")) +} + +func TestWithCatalogerTaskRegistry_isIdempotent(t *testing.T) { + // anything that re-runs catalogers may ask for a registry; a second one would hold none of the rows + // already published and every re-run would publish again + ctx := WithCatalogerTaskRegistry(context.Background()) + row := StartCatalogerTask(ctx, digestsTask(), 2, "") + + nested := WithCatalogerTaskRegistry(ctx) + assert.Same(t, row, StartCatalogerTask(nested, digestsTask(), 2, "")) +} diff --git a/internal/capabilities/generate/discover_cataloger_configs.go b/internal/capabilities/generate/discover_cataloger_configs.go index a6d1b7e34..f7570f6b3 100644 --- a/internal/capabilities/generate/discover_cataloger_configs.go +++ b/internal/capabilities/generate/discover_cataloger_configs.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "regexp" + "slices" "strings" "github.com/anchore/syft/internal/capabilities/internal" @@ -31,6 +32,10 @@ type ConfigInfo struct { var appConfigAnnotationPattern = regexp.MustCompile(`^//\s*app-config:\s*(.+)$`) +// embeddedFieldsAnnotationPattern matches "// capabilities-fields: A, B" on an embedded struct, which limits +// the embedded fields listed to the ones named (for a shared type the cataloger reads only part of). +var embeddedFieldsAnnotationPattern = regexp.MustCompile(`^//\s*capabilities-fields:\s*(.+)$`) + // DiscoverConfigs walks the cataloger directory and discovers all configuration structs // Returns map where key is "packageName.StructName" (e.g., "golang.CatalogerConfig") func DiscoverConfigs(repoRoot string) (map[string]ConfigInfo, error) { @@ -160,6 +165,9 @@ func extractCatalogerConfigFieldsRecursive(structType *ast.StructType, visitedTy if len(field.Names) == 0 { // this is an embedded field - resolve and extract its fields embeddedFields := resolveEmbeddedStructFields(field.Type, visitedTypes) + if only := embeddedFieldAllowlist(field.Doc); only != nil { + embeddedFields = slices.DeleteFunc(embeddedFields, func(f ConfigField) bool { return !only[f.Name] }) + } fields = append(fields, embeddedFields...) continue } @@ -195,6 +203,26 @@ func extractCatalogerConfigFieldsRecursive(structType *ast.StructType, visitedTy return fields } +// embeddedFieldAllowlist returns the field names from a "capabilities-fields:" annotation, or nil when there +// is none (every exported embedded field is listed). +func embeddedFieldAllowlist(commentGroup *ast.CommentGroup) map[string]bool { + if commentGroup == nil { + return nil + } + for _, comment := range commentGroup.List { + matches := embeddedFieldsAnnotationPattern.FindStringSubmatch(comment.Text) + if len(matches) < 2 { + continue + } + only := make(map[string]bool) + for _, name := range strings.Split(matches[1], ",") { + only[strings.TrimSpace(name)] = true + } + return only + } + return nil +} + // resolveEmbeddedStructFields resolves an embedded struct type and extracts its fields recursively func resolveEmbeddedStructFields(fieldType ast.Expr, visitedTypes map[string]bool) []ConfigField { // extract the type name from the expression diff --git a/internal/capabilities/generate/discover_cataloger_configs_test.go b/internal/capabilities/generate/discover_cataloger_configs_test.go index f22bf7b9f..9add0a439 100644 --- a/internal/capabilities/generate/discover_cataloger_configs_test.go +++ b/internal/capabilities/generate/discover_cataloger_configs_test.go @@ -457,3 +457,32 @@ func TestDiscoverAllowedConfigStructs(t *testing.T) { t.Logf(" - %s", configType) } } + +func TestEmbeddedFieldAllowlist(t *testing.T) { + comments := func(lines ...string) *ast.CommentGroup { + g := &ast.CommentGroup{} + for _, l := range lines { + g.List = append(g.List, &ast.Comment{Text: l}) + } + return g + } + + tests := []struct { + name string + doc *ast.CommentGroup + want map[string]bool + }{ + {name: "no doc lists every field", doc: nil, want: nil}, + {name: "no annotation lists every field", doc: comments("// Deprecated: use something else."), want: nil}, + { + name: "annotation limits to the named fields", + doc: comments("// Deprecated: use something else.", "// capabilities-fields: IncludeIndexedArchives, IncludeUnindexedArchives"), + want: map[string]bool{"IncludeIndexedArchives": true, "IncludeUnindexedArchives": true}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, embeddedFieldAllowlist(tt.doc)) + }) + } +} diff --git a/internal/cmptest/common_options.go b/internal/cmptest/common_options.go index 8e8c2c3ea..f9ea242b5 100644 --- a/internal/cmptest/common_options.go +++ b/internal/cmptest/common_options.go @@ -69,5 +69,5 @@ func locationSorter(a, b file.Location) int { return strings.Compare(a.AccessPath, b.AccessPath) } - return strings.Compare(a.RealPath, b.RealPath) + return strings.Compare(a.FullNestedPath(), b.FullNestedPath()) } diff --git a/internal/cmptest/location.go b/internal/cmptest/location.go index 1959926f0..2cc5dec0a 100644 --- a/internal/cmptest/location.go +++ b/internal/cmptest/location.go @@ -14,5 +14,5 @@ func DefaultLocationComparer(x, y file.Location) bool { } func LocationComparerWithoutLayer(x, y file.Location) bool { - return cmp.Equal(x.RealPath, y.RealPath) && cmp.Equal(x.AccessPath, y.AccessPath) + return cmp.Equal(x.FullNestedPath(), y.FullNestedPath()) && cmp.Equal(x.AccessPath, y.AccessPath) } diff --git a/internal/constants.go b/internal/constants.go index 0d2fe0820..2ce680d60 100644 --- a/internal/constants.go +++ b/internal/constants.go @@ -3,7 +3,7 @@ package internal const ( // JSONSchemaVersion is the current schema version output by the JSON encoder // This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment. - JSONSchemaVersion = "16.1.11" + JSONSchemaVersion = "16.1.12" // Changelog // 16.1.0 - reformulated the python pdm fields (added "URL" and removed the unused "path" field). @@ -18,4 +18,5 @@ const ( // 16.1.9 - add Symbols (grouped by owning package import path) to GolangBinaryBuildinfoEntry metadata // 16.1.10 - add SafeTensorsModelInfo metadata type for the safetensors AI model cataloger // 16.1.11 - add CpanDistribution and CpanUnpackedRelease metadata types for the perl catalogers + // 16.1.12 - add fullPath to file coordinates, naming the nested archive chain a file was found in (blank for files not within an archive) ) diff --git a/internal/file/copy.go b/internal/file/copy.go index 058fe27eb..636b3b431 100644 --- a/internal/file/copy.go +++ b/internal/file/copy.go @@ -6,13 +6,13 @@ import ( "io" ) -const perFileReadLimit = 2 * GB +// PerFileReadLimit is the most read from any one archive entry, guarding against decompression bombs. +const PerFileReadLimit = 2 * GB -// safeCopy limits the copy from the reader. This is useful when extracting files from archives to -// protect against decompression bomb attacks. -func safeCopy(writer io.Writer, reader io.Reader) error { - numBytes, err := io.Copy(writer, io.LimitReader(reader, perFileReadLimit)) - if numBytes >= perFileReadLimit { +// SafeCopy bounds the copy to guard against decompression bombs when extracting from archives. +func SafeCopy(writer io.Writer, reader io.Reader) error { + numBytes, err := io.Copy(writer, io.LimitReader(reader, PerFileReadLimit)) + if numBytes >= PerFileReadLimit { return fmt.Errorf("zip read limit hit (potential decompression bomb attack)") } // Propagate decompression / read errors up to the caller. io.Copy diff --git a/internal/file/copy_test.go b/internal/file/copy_test.go index 0c83d266f..f5d42a720 100644 --- a/internal/file/copy_test.go +++ b/internal/file/copy_test.go @@ -29,7 +29,7 @@ func (r *errReader) Read(p []byte) (int, error) { func TestSafeCopy(t *testing.T) { t.Run("clean copy returns nil", func(t *testing.T) { var buf bytes.Buffer - if err := safeCopy(&buf, strings.NewReader("hello")); err != nil { + if err := SafeCopy(&buf, strings.NewReader("hello")); err != nil { t.Fatalf("unexpected error: %v", err) } if got := buf.String(); got != "hello" { @@ -38,12 +38,12 @@ func TestSafeCopy(t *testing.T) { }) t.Run("propagates decompression error", func(t *testing.T) { - // #4806: safeCopy used to drop non-EOF errors, so the caller + // #4806: SafeCopy used to drop non-EOF errors, so the caller // would persist a partial buffer as a successful extract and // downstream catalogers silently read empty manifests. sentinel := errors.New("flate: corrupt input before offset 42") var buf bytes.Buffer - err := safeCopy(&buf, &errReader{data: []byte("partial"), err: sentinel}) + err := SafeCopy(&buf, &errReader{data: []byte("partial"), err: sentinel}) if err == nil { t.Fatalf("expected error to be returned, got nil") } @@ -56,7 +56,7 @@ func TestSafeCopy(t *testing.T) { // The old code had a dead io.EOF branch that labelled clean // reads as decompression bombs; keep the happy path clean. var buf bytes.Buffer - err := safeCopy(&buf, io.LimitReader(strings.NewReader("abc"), 3)) + err := SafeCopy(&buf, io.LimitReader(strings.NewReader("abc"), 3)) if err != nil { t.Fatalf("unexpected error on clean copy: %v", err) } diff --git a/internal/file/tar_file_traversal.go b/internal/file/tar_file_traversal.go index 54f10448d..a0c083e4c 100644 --- a/internal/file/tar_file_traversal.go +++ b/internal/file/tar_file_traversal.go @@ -70,7 +70,7 @@ func ExtractGlobsFromTarToUniqueTempFile(ctx context.Context, archivePath, dir s } defer internal.CloseAndLogError(packedFile, archivePath) - if err := safeCopy(tempFile, packedFile); err != nil { + if err := SafeCopy(tempFile, packedFile); err != nil { return fmt.Errorf("unable to copy source=%q for tar=%q: %w", file.Name(), archivePath, err) } diff --git a/internal/file/zip_file_traversal.go b/internal/file/zip_file_traversal.go index 5fc26a220..211527037 100644 --- a/internal/file/zip_file_traversal.go +++ b/internal/file/zip_file_traversal.go @@ -101,7 +101,7 @@ func ExtractFromZipToUniqueTempFile(ctx context.Context, archivePath, dir string return fmt.Errorf("unable to extract directories, only files: %s", file.NameInArchive) } - if err := safeCopy(tempFile, zippedFile); err != nil { + if err := SafeCopy(tempFile, zippedFile); err != nil { return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.NameInArchive, archivePath, err) } @@ -138,7 +138,7 @@ func ContentsFromZip(ctx context.Context, archivePath string, paths ...string) ( } var buffer bytes.Buffer - if err := safeCopy(&buffer, zippedFile); err != nil { + if err := SafeCopy(&buffer, zippedFile); err != nil { return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.NameInArchive, archivePath, err) } @@ -210,7 +210,7 @@ func extractSingleFile(file archives.FileInfo, expandedFilePath, archivePath str } }() - if err := safeCopy(outputFile, zippedFile); err != nil { + if err := SafeCopy(outputFile, zippedFile); err != nil { return fmt.Errorf("unable to copy source=%q to dest=%q for zip=%q: %w", file.NameInArchive, outputFile.Name(), archivePath, err) } } diff --git a/internal/os/feature_detection.go b/internal/os/feature_detection.go index a836a2191..14405dda7 100644 --- a/internal/os/feature_detection.go +++ b/internal/os/feature_detection.go @@ -49,7 +49,7 @@ func findRhelFeatures(resolver file.Resolver, release *linux.Release) error { for _, contentManifestFile := range contentManifestFiles { found, err := fileMatchesAny(resolver, contentManifestFile, rhelEUSPatterns...) if err != nil { - return fmt.Errorf("unable to read content manifest from %s: %w", contentManifestFile.RealPath, err) + return fmt.Errorf("unable to read content manifest from %s: %w", contentManifestFile.FullNestedPath(), err) } if found { release.ExtendedSupport = true @@ -163,9 +163,9 @@ func hasActiveUbuntuESMService(resolver file.Resolver) (bool, error) { func hasEnabledESMServiceInStatus(resolver file.Resolver, location file.Location) (bool, error) { contents, err := resolver.FileContentsByLocation(location) if err != nil { - return false, fmt.Errorf("unable to read ubuntu-advantage status from %s: %w", location.RealPath, err) + return false, fmt.Errorf("unable to read ubuntu-advantage status from %s: %w", location.FullNestedPath(), err) } - defer internal.CloseAndLogError(contents, location.RealPath) + defer internal.CloseAndLogError(contents, location.FullNestedPath()) var status struct { Services []struct { @@ -174,7 +174,7 @@ func hasEnabledESMServiceInStatus(resolver file.Resolver, location file.Location } `json:"services"` } if err := json.NewDecoder(contents).Decode(&status); err != nil { - return false, fmt.Errorf("unable to parse ubuntu-advantage status from %s: %w", location.RealPath, err) + return false, fmt.Errorf("unable to parse ubuntu-advantage status from %s: %w", location.FullNestedPath(), err) } // pro status values are enabled/disabled/n-a/warning/—; only "enabled" proves the stream is active. @@ -200,8 +200,8 @@ func hasInstalledESMPackage(packages *pkg.Collection) bool { func fileMatchesAny(resolver file.Resolver, location file.Location, patterns ...*regexp.Regexp) (bool, error) { contents, err := resolver.FileContentsByLocation(location) if err != nil { - return false, fmt.Errorf("unable to read %s: %w", location.RealPath, err) + return false, fmt.Errorf("unable to read %s: %w", location.FullNestedPath(), err) } - defer internal.CloseAndLogError(contents, location.RealPath) + defer internal.CloseAndLogError(contents, location.FullNestedPath()) return internal.MatchAnyFromReader(contents, patterns...) } diff --git a/internal/relationship/binary/binary_dependencies.go b/internal/relationship/binary/binary_dependencies.go index 552f767e7..1b595f052 100644 --- a/internal/relationship/binary/binary_dependencies.go +++ b/internal/relationship/binary/binary_dependencies.go @@ -71,7 +71,7 @@ func compareElfBinaryPackages(s *sbom.SBOM) []artifact.ID { pkgsToDelete := make([]artifact.ID, 0) for _, elfPkg := range allElfPackages(s) { for _, loc := range onlyPrimaryEvidenceLocations(elfPkg) { - for _, otherPkg := range s.Artifacts.Packages.PackagesByPath(loc.RealPath) { + for _, otherPkg := range s.Artifacts.Packages.PackagesByPath(loc.FullNestedPath()) { // we only care about comparing binary packages to each other (not other types) if otherPkg.Type != pkg.BinaryPkg { continue @@ -130,7 +130,7 @@ func populateRelationships(exec file.Executable, parentPkg pkg.Package, resolver for _, loc := range libLocations { // are you in our index? - realBaseName := path.Base(loc.RealPath) + realBaseName := path.Base(loc.FullNestedPath()) pkgCollection := index.owningLibraryPackage(realBaseName) if pkgCollection.PackageCount() < 1 { addRelationship( diff --git a/internal/relationship/binary/shared_library_index.go b/internal/relationship/binary/shared_library_index.go index aa93c0db0..af780ceca 100644 --- a/internal/relationship/binary/shared_library_index.go +++ b/internal/relationship/binary/shared_library_index.go @@ -3,6 +3,7 @@ package binary import ( "path" + "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/sbomsync" "github.com/anchore/syft/syft/file" @@ -63,7 +64,7 @@ func locationsThatProvideLibraries(accessor sbomsync.Accessor) (map[string]file. continue } - basename := path.Base(coord.RealPath) + basename := path.Base(coord.FullNestedPath()) set := libLocationsByBasename[basename] set.Add(coord) allLibLocations.Add(coord) @@ -82,6 +83,11 @@ func packagesWithLibraryOwnership(resolver file.Resolver, accessor sbomsync.Acce var ownedFilePaths []string if p.Type == pkg.BinaryPkg { for _, loc := range p.Locations.ToSlice() { + if archive.ContainerPath(loc.Coordinates) != "" { + // a path within an archive cannot be resolved from the scan root; the location is the library + addLibraryOwner(packagesByLibraryPath, allLibLocations, loc.Coordinates, p) + continue + } ownedFilePaths = append(ownedFilePaths, loc.Path()) } } else { @@ -114,18 +120,19 @@ func populatePackagesByLibraryPath( } for _, loc := range ownedLocation { - // if the location is a library, add the package to the set of packages that own the library - if !allLibLocations.Contains(loc.Coordinates) { - continue - } - - if _, ok := packagesByLibraryPath[loc.Coordinates]; !ok { - packagesByLibraryPath[loc.Coordinates] = pkg.NewCollection() - } - - // we have a library path, add the package to the set of packages that own the library - packagesByLibraryPath[loc.Coordinates].Add(p) + addLibraryOwner(packagesByLibraryPath, allLibLocations, loc.Coordinates, p) } } return packagesByLibraryPath } + +// addLibraryOwner records p as an owner of the file at coordinates when that file is a library. +func addLibraryOwner(packagesByLibraryPath map[file.Coordinates]*pkg.Collection, allLibLocations file.CoordinateSet, coordinates file.Coordinates, p pkg.Package) { + if !allLibLocations.Contains(coordinates) { + return + } + if _, ok := packagesByLibraryPath[coordinates]; !ok { + packagesByLibraryPath[coordinates] = pkg.NewCollection() + } + packagesByLibraryPath[coordinates].Add(p) +} diff --git a/internal/relationship/by_file_ownership.go b/internal/relationship/by_file_ownership.go index 92b2ddc49..ce8f9a3a2 100644 --- a/internal/relationship/by_file_ownership.go +++ b/internal/relationship/by_file_ownership.go @@ -6,6 +6,7 @@ import ( "github.com/bmatcuk/doublestar/v4" "github.com/scylladb/go-set/strset" + "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/sbomsync" "github.com/anchore/syft/syft/artifact" @@ -100,7 +101,7 @@ func findOwnershipByFilesRelationships(resolver file.Resolver, catalog *pkg.Coll // Now establish relationships, considering symlinks for _, candidateOwnerPkg := range catalog.Sorted() { id := candidateOwnerPkg.ID() - if candidateOwnerPkg.Metadata == nil { + if candidateOwnerPkg.Metadata == nil || withinArchive(candidateOwnerPkg) { continue } @@ -164,7 +165,7 @@ func directOwnersByPath(catalog *pkg.Collection) map[pkg.Type]*strset.Set { // First, identify direct ownership of all files for _, p := range catalog.Sorted() { - if p.Metadata == nil { + if p.Metadata == nil || withinArchive(p) { continue } @@ -192,6 +193,21 @@ func directOwnersByPath(catalog *pkg.Collection) map[pkg.Type]*strset.Set { return directOwnership } +// withinArchive reports whether every location of p is inside an archive. Such a package's owned files +// are paths within its archive, not the scan root, so it is not treated as a file owner. +func withinArchive(p pkg.Package) bool { + locations := p.Locations.ToSlice() + if len(locations) == 0 { + return false + } + for _, l := range locations { + if archive.ContainerPath(l.Coordinates) == "" { + return false + } + } + return true +} + func resolvePaths(ownedFilePath string, resolver file.Resolver) []string { // though we have a string path, we need to resolve symlinks and other filesystem oddities since we cannot assume this is a real path var locs []file.Location @@ -206,6 +222,8 @@ func resolvePaths(ownedFilePath string, resolver file.Resolver) []string { ownedFilePaths := strset.New(ownedFilePath) for _, loc := range locs { + // Owned file paths are currently based on the top-level resolver, which is always the RealPath and includes + // all files found within archives at that location ownedFilePaths.Add(loc.RealPath) } return ownedFilePaths.List() diff --git a/internal/relationship/by_file_ownership_test.go b/internal/relationship/by_file_ownership_test.go index a69f99c99..ed8282eb9 100644 --- a/internal/relationship/by_file_ownership_test.go +++ b/internal/relationship/by_file_ownership_test.go @@ -78,6 +78,56 @@ func TestOwnershipByFilesRelationship(t *testing.T) { return []pkg.Package{parent, child}, []artifact.Relationship{relationship} }, }, + { + name: "package-within-archive-is-not-a-file-owner", + setup: func(t testing.TB) ([]pkg.Package, []artifact.Relationship) { + nestedOwner := pkg.Package{ + Locations: file.NewLocationSet(file.NewLocationFromCoordinates(file.Coordinates{ + RealPath: "/rootfs.tar", + FullPath: "/rootfs.tar!/var/lib/rpm/Packages", + })), + Type: pkg.RpmPkg, + Metadata: pkg.RpmDBEntry{ + Files: []pkg.RpmFileRecord{{Path: "/d/path"}}, + }, + } + nestedOwner.SetID() + + child := pkg.Package{ + Locations: file.NewLocationSet(file.NewLocation("/d/path")), + Type: pkg.BinaryPkg, + } + child.SetID() + + return []pkg.Package{nestedOwner, child}, nil + }, + }, + { + // the archive's own package (when it has one) is found by its path; what is inside it is + // linked to that package by CONTAINS, not owned by whoever owns the archive file + name: "top-level-owner-of-an-archive-does-not-own-packages-within-it", + setup: func(t testing.TB) ([]pkg.Package, []artifact.Relationship) { + parent := pkg.Package{ + Locations: file.NewLocationSet(file.NewLocation("/var/lib/rpm/Packages")), + Type: pkg.RpmPkg, + Metadata: pkg.RpmDBEntry{ + Files: []pkg.RpmFileRecord{{Path: "/opt/bundle.tar"}}, + }, + } + parent.SetID() + + child := pkg.Package{ + Locations: file.NewLocationSet(file.NewLocationFromCoordinates(file.Coordinates{ + RealPath: "/opt/bundle.tar", + FullPath: "/opt/bundle.tar!/lib/thing.jar", + })), + Type: pkg.JavaPkg, + } + child.SetID() + + return []pkg.Package{parent, child}, nil + }, + }, { name: "only-real-file-owner-gets-ownership-relationship-not-symlink-owner", resolver: mockFR{ diff --git a/internal/relationship/evident_by_test.go b/internal/relationship/evident_by_test.go index 9e08d967b..bd001696d 100644 --- a/internal/relationship/evident_by_test.go +++ b/internal/relationship/evident_by_test.go @@ -15,18 +15,9 @@ func TestRelationshipsEvidentBy(t *testing.T) { c := pkg.NewCollection() - coordA := file.Coordinates{ - RealPath: "/somewhere/real", - FileSystemID: "abc", - } - coordC := file.Coordinates{ - RealPath: "/somewhere/real", - FileSystemID: "abc", - } - coordD := file.Coordinates{ - RealPath: "/somewhere/real", - FileSystemID: "abc", - } + coordA := file.NewCoordinates("/somewhere/real", "abc") + coordC := file.NewCoordinates("/somewhere/real", "abc") + coordD := file.NewCoordinates("/somewhere/real", "abc") pkgA := pkg.Package{ Locations: file.NewLocationSet( // added! @@ -39,10 +30,7 @@ func TestRelationshipsEvidentBy(t *testing.T) { pkgA.SetID() c.Add(pkgA) - coordB := file.Coordinates{ - RealPath: "/somewhere-else/real", - FileSystemID: "def", - } + coordB := file.NewCoordinates("/somewhere-else/real", "def") pkgB := pkg.Package{ Locations: file.NewLocationSet( // added! diff --git a/internal/relationship/index_test.go b/internal/relationship/index_test.go index 12c933346..d856d184a 100644 --- a/internal/relationship/index_test.go +++ b/internal/relationship/index_test.go @@ -21,12 +21,8 @@ func Test_Index(t *testing.T) { p3 := pkg.Package{ Name: "pkg-3", } - c1 := file.Coordinates{ - RealPath: "/coords/1", - } - c2 := file.Coordinates{ - RealPath: "/coords/2", - } + c1 := file.NewCoordinates("/coords/1", "") + c2 := file.NewCoordinates("/coords/2", "") for _, p := range []*pkg.Package{&p1, &p2, &p3} { p.SetID() @@ -156,18 +152,10 @@ func Test_Coordinates(t *testing.T) { p3 := pkg.Package{ Name: "pkg-3", } - c1 := file.Coordinates{ - RealPath: "/coords/1", - } - c2 := file.Coordinates{ - RealPath: "/coords/2", - } - c3 := file.Coordinates{ - RealPath: "/coords/3", - } - c4 := file.Coordinates{ - RealPath: "/coords/4", - } + c1 := file.NewCoordinates("/coords/1", "") + c2 := file.NewCoordinates("/coords/2", "") + c3 := file.NewCoordinates("/coords/3", "") + c4 := file.NewCoordinates("/coords/4", "") for _, p := range []*pkg.Package{&p1, &p2, &p3} { p.SetID() @@ -237,10 +225,10 @@ func TestRemove(t *testing.T) { p1 := pkg.Package{Name: "pkg-1"} p2 := pkg.Package{Name: "pkg-2"} p3 := pkg.Package{Name: "pkg-3"} - c1 := file.Coordinates{RealPath: "/coords/1"} - c2 := file.Coordinates{RealPath: "/coords/2"} - c3 := file.Coordinates{RealPath: "/coords/3"} - c4 := file.Coordinates{RealPath: "/coords/4"} + c1 := file.NewCoordinates("/coords/1", "") + c2 := file.NewCoordinates("/coords/2", "") + c3 := file.NewCoordinates("/coords/3", "") + c4 := file.NewCoordinates("/coords/4", "") for _, p := range []*pkg.Package{&p1, &p2, &p3} { p.SetID() diff --git a/internal/task/archive_nesting_test.go b/internal/task/archive_nesting_test.go new file mode 100644 index 000000000..314ef1d8c --- /dev/null +++ b/internal/task/archive_nesting_test.go @@ -0,0 +1,699 @@ +package task + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/scylladb/go-set/strset" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/archive" + "github.com/anchore/syft/internal/sbomsync" + "github.com/anchore/syft/internal/testutils" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/cataloging" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/java" + "github.com/anchore/syft/syft/sbom" +) + +// This file covers nesting across archive families. The other nested-archive fixtures are all zips, +// so a zip-specific mechanism would pass them; the requirement here is that an archive of one family +// inside another behaves as it would inside its own family - same packages, same container-path chain, +// same CONTAINS edges, same depth accounting. +// +// Chains are built in the test rather than committed as binaries, so each case can vary which family +// sits at which level. + +// archiveFamily is one archive format buildNestedArchive can produce. +// +// ".jar" is its own family rather than a flavour of zip: the java cataloger meets it as a jar wherever +// it sits, while the archive task descends into it like any other zip. That differs mid-chain from at +// the leaf, so the matrix below uses a jar as container and as payload. +type archiveFamily string + +const ( + familyZip archiveFamily = ".zip" + familyJar archiveFamily = ".jar" + familyTar archiveFamily = ".tar" + familyTarGz archiveFamily = ".tar.gz" + familyTgz archiveFamily = ".tgz" +) + +// nestPlan describes one chain to build: a family per level (outermost first), the entries for the +// innermost archive, and extras to add beside the nested archive at a given level. The extras grade +// the levels' sizes or entry counts, which is how the limit cases place a bound between two levels. +type nestPlan struct { + families []archiveFamily + leaf map[string][]byte + extra map[int]map[string][]byte +} + +// nestedArchive is one built chain: the outermost archive's bytes, and the addresses that chain +// composes to at each level. +type nestedArchive struct { + // name is the outermost archive's file name, as written into the scanned directory. + name string + // bytes is the outermost archive. + bytes []byte + // sizes is each archive's size in bytes, outermost first. A container is never smaller than what it + // holds, so these decrease, and the limit cases place a bound between two of them. + sizes []int + // paths is each archive's path within its parent, outermost first; paths[0] is the outermost archive + // as the scan's resolver reports it. + paths []string + // containerPaths is each archive's full path, the container path its own contents carry, outermost first. + containerPaths []string + // virtualPaths is the same chain in java's colon-joined form, outermost first. + virtualPaths []string +} + +// nestingRow is one line of the depth-3 matrix. +type nestingRow struct { + name string + families []archiveFamily + why string +} + +// nestingVisit is one filesystem the archive cataloger's sub-pipeline was run against. +type nestingVisit struct { + virtualPath string + fullPath string + depth int + // heldFiles are the spill files live at this moment, sampled only when the probe was given a temp + // dir to look in. A nested archive's own bytes are read where they lie, so one appears per archive + // whose entries did not fit in memory. + heldFiles []string +} + +// nestingProbe records one visit per filesystem the archive cataloger runs its sub-pipeline against, +// reading the chain off the traversal on the context. +type nestingProbe struct { + // tempDir, when set, is where extraction work directories live + tempDir string + + mu sync.Mutex + visits []nestingVisit +} + +func Test_mixedFamilyNesting_leafIsCatalogedOnceWithTheFullChain(t *testing.T) { + // family order must not change the result: the same package, addressed by a chain naming its own + // containers in its own order + for _, row := range nestingRows() { + t.Run(row.name, func(t *testing.T) { + scanDir := t.TempDir() + nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()}) + + probe := &nestingProbe{} + s := runNesting(t, scanDir, 3, defaultLimits(), markerTask(), probe.task()) + + assert.Equal(t, 1, packageCount(s, "marker-pkg"), "%s: the leaf package must be cataloged exactly once", row.why) + + locs := leafLocations(s) + require.Len(t, locs, 1, "the leaf must be reached by exactly one path") + // exact, never a substring: a partially composed chain passes a substring check + assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates), + "the chain must name all three archives outer-to-inner") + assert.Equal(t, nested.leafContainerPath()+"!/nested/marker.txt", locs[0].FullNestedPath(), + "the path stays relative to the archive the file came from") + + assert.Equal(t, nested.containerPaths, probe.containerPaths(), + "every level must be cataloged, outer to inner, whatever families they are") + }) + } +} + +func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) { + // the edges make the chain navigable in the SBOM. The archive-to-archive edges must not depend on + // the file catalogers having recorded the inner archives, since the default file selection does not + for _, row := range nestingRows() { + t.Run(row.name, func(t *testing.T) { + scanDir := t.TempDir() + nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()}) + + s := withArchiveContains(t, runNesting(t, scanDir, 3, defaultLimits(), markerTask())) + + // each archive's coordinates as seen in its parent; the outermost sits in the scanned + // directory, so it carries no filesystem id of its own + archiveCoords := make([]file.Coordinates, len(row.families)) + for i := range row.families { + archiveCoords[i] = nested.coordinates(i) + } + + fileEdges := strset.New() + pkgEdges := strset.New() + for _, rel := range s.Relationships { + if rel.Type != artifact.ContainsRelationship { + continue + } + from, ok := rel.From.(file.Coordinates) + if !ok { + continue + } + switch to := rel.To.(type) { + case file.Coordinates: + fileEdges.Add(coordKey(from) + " -> " + coordKey(to)) + case pkg.Package: + pkgEdges.Add(coordKey(from) + " -> " + to.Name) + } + } + + for i := 0; i < len(archiveCoords)-1; i++ { + assert.True(t, fileEdges.Has(coordKey(archiveCoords[i])+" -> "+coordKey(archiveCoords[i+1])), + "expected a CONTAINS edge from %s to %s", nested.paths[i], nested.paths[i+1]) + } + + innermost := archiveCoords[len(archiveCoords)-1] + assert.True(t, pkgEdges.Has(coordKey(innermost)+" -> marker-pkg"), + "expected a CONTAINS edge from %s to the leaf package", nested.paths[len(nested.paths)-1]) + + // with file cataloging on, files inside an archive get an edge from it too + s = withArchiveContains(t, runNesting(t, scanDir, 3, defaultLimits(), markerTask(), fileMetadataTask())) + var leafEdge bool + for _, rel := range s.Relationships { + from, _ := rel.From.(file.Coordinates) + to, _ := rel.To.(file.Coordinates) + leafEdge = leafEdge || (from == innermost && to == file.Coordinates{RealPath: innermost.RealPath, FullPath: innermost.FullNestedPath() + "!/nested/marker.txt"}) + } + assert.True(t, leafEdge, "expected a CONTAINS edge from the innermost archive to the leaf file") + }) + } +} + +func Test_mixedFamilyNesting_everyJarIsCatalogedOnceUnderItsChain(t *testing.T) { + // two mechanisms meet a jar: the java cataloger catalogs it as a package, the archive task descends + // into it. A jar at any level must do both, exactly once each; the failure mode is a duplicate, not an + // error. The colon-joined virtual paths are in already-published SBOMs, so they must name the chain. + for _, row := range nestingRows() { + var jarLevels []int + for i, family := range row.families { + if family == familyJar { + jarLevels = append(jarLevels, i) + } + } + if len(jarLevels) == 0 { + continue + } + + t.Run(row.name, func(t *testing.T) { + scanDir := t.TempDir() + nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()}) + + probe := &nestingProbe{} + s := runNesting(t, scanDir, 3, defaultLimits(), markerTask(), javaTask(), probe.task()) + + for _, level := range jarLevels { + name := fmt.Sprintf("level%d", level) + assert.Equal(t, 1, packageCount(s, name), "the jar's own package must be cataloged exactly once") + assert.Equal(t, nested.virtualPaths[level], javaVirtualPaths(s)[name], + "the jar's virtual path must be its own place in the chain") + } + + // and the archive task still descended through every level + assert.Equal(t, nested.containerPaths, probe.containerPaths()) + locs := leafLocations(s) + require.Len(t, locs, 1) + assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates)) + }) + } +} + +func Test_mixedFamilyNesting_depthCountsLevelsNotFamilies(t *testing.T) { + // two rows with different families at every level, so the bound is shown not to turn on which + // families are involved + for _, row := range nestingRows()[:2] { + t.Run(row.name, func(t *testing.T) { + scanDir := t.TempDir() + nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()}) + + probe := &nestingProbe{} + s := runNesting(t, scanDir, 2, defaultLimits(), markerTask(), fileMetadataTask(), probe.task()) + + assert.Equal(t, nested.containerPaths[:2], probe.containerPaths(), + "at depth 2 the outer and middle archives are cataloged and the innermost is not") + assert.Zero(t, packageCount(s, "marker-pkg"), + "the leaf is one level past the bound, so its package must be absent") + + // the innermost archive is still cataloged as a file, inside the middle archive + innermost := nested.coordinates(2) + _, ok := s.Artifacts.FileMetadata[innermost] + assert.True(t, ok, "the undescended archive must still be recorded as a file at %s", coordKey(innermost)) + + // and what was left unexplored is said out loud, not silently dropped + reasons := s.Artifacts.Unknowns[innermost] + require.Len(t, reasons, 1, "unknowns: %v", s.Artifacts.Unknowns) + assert.Contains(t, reasons[0], "depth limit was reached") + }) + } +} + +func Test_mixedFamilyNesting_aNestedArchiveIsReadWhereItLies(t *testing.T) { + // an archive's own bytes are read where they lie: a scanned file from disk, a nested archive from + // its parent's entries. Nothing copies them, so at most one spill file per level appears whatever + // the memory limit; only entries are held, and only they can be written out. + scanDir := t.TempDir() + tempDir := isolatedTempDir(t) + nested := writeNestedArchive(t, scanDir, gradedChain()) + + for _, tc := range []struct { + name string + memoryBytes int64 + wantHeldFiles []string + }{ + { + name: "with room in memory for every level's entries", + memoryBytes: int64(2 * (nested.sizes[0] + nested.sizes[1] + nested.sizes[2])), + }, + { + name: "with a memory limit of zero, so every level's entries are written out", + memoryBytes: 0, + wantHeldFiles: []string{"archive-spill", "archive-spill", "archive-spill"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + probe := &nestingProbe{tempDir: tempDir} + bounds := archive.Limits{MaxMemoryBytes: tc.memoryBytes, MaxDiskBytes: -1} + + s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task()) + + deepest := probe.deepestVisit() + require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held") + assert.Equal(t, tc.wantHeldFiles, deepest.heldFiles) + + assert.Equal(t, nested.containerPaths, probe.containerPaths()) + locs := leafLocations(s) + require.Len(t, locs, 1) + assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates)) + }) + } +} + +func Test_mixedFamilyNesting_diskLimitTruncatesTheLevelItCannotAdmit(t *testing.T) { + // the walk descends before it unwinds, so the outer and middle archives still hold their entries + // when the innermost level's do not fit in memory. With nothing admitted on disk that level is + // cataloged from nothing, the shallower ones keep what they found, and the scan finishes. An + // implementation that waited for capacity would deadlock, hence the timeout. + scanDir := t.TempDir() + nested := writeNestedArchive(t, scanDir, deepChain()) + + // a sibling of a different family, processed once the chain has been released, so cataloged in full + require.NoError(t, os.WriteFile( + filepath.Join(scanDir, "sibling.tgz"), + testutils.TarGz(t, map[string][]byte{"nested/marker.txt": []byte("sibling")}), + 0o644, + )) + + // room for the outer two levels' entries (each about the size of the archive it holds) but not the + // innermost's payload, which is half an archive larger than the slack + memoryLimit := int64(nested.sizes[0] + nested.sizes[1] + deepChainPayloadBytes/2) + + probe := &nestingProbe{} + bounds := archive.Limits{MaxMemoryBytes: memoryLimit, MaxDiskBytes: 0} + + s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task()) + + want := append(append([]string{}, nested.containerPaths...), "/sibling.tgz") + assert.ElementsMatch(t, want, probe.containerPaths(), + "every level is visited, the truncated one with nothing inside it") + + containers := strset.New() + for _, loc := range leafLocations(s) { + containers.Add(archive.ContainerPath(loc.Coordinates)) + } + assert.False(t, containers.Has(nested.leafContainerPath()), "the truncated level's contents must be absent") + assert.True(t, containers.Has("/sibling.tgz"), "the sibling of a different family must be cataloged fully") + + var truncated []string + for coords, reasons := range s.Artifacts.Unknowns { + for _, reason := range reasons { + if strings.Contains(reason, "disk limit") { + truncated = append(truncated, coords.FullNestedPath()) + } + } + } + assert.Equal(t, []string{nested.coordinates(2).FullNestedPath()}, truncated, + "the truncation is recorded against the archive that reached the limit") +} + +func defaultLimits() archive.Limits { + return archiveLimits(cataloging.DefaultArchiveSearchConfig()) +} + +// runNesting runs the archive cataloger task over scanDir with the given sub-pipeline and returns the +// SBOM it built. It runs on its own goroutine and fails if the task has not finished in time. +// +// The depth-3 limit cases need the timeout: the walk descends before it unwinds, so the outer and +// middle archives still hold their content when the inner one is refused. An implementation that +// waited for capacity would deadlock, and a test asserting only the result would hang rather than fail. +func runNesting(t *testing.T, scanDir string, depth int, bounds archive.Limits, subPipeline ...Task) *sbom.SBOM { + t.Helper() + tsk := newLimitedTestTask(t, depth, bounds, subPipeline...) + s := &sbom.SBOM{Artifacts: sbom.Artifacts{ + Packages: pkg.NewCollection(), + FileMetadata: map[file.Coordinates]file.Metadata{}, + }} + + done := make(chan error, 1) + go func() { + done <- tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(s)) + }() + + select { + case err := <-done: + require.NoError(t, err, "reaching a bound must not fail the scan") + return s + case <-time.After(60 * time.Second): + t.Fatal("the scan did not finish in time: reaching a limit must skip the archive that " + + "reached it, never wait for capacity that nothing in the chain will release") + return nil + } +} + +// withArchiveContains runs the task that follows every scan with archive cataloging on. +func withArchiveContains(t *testing.T, s *sbom.SBOM) *sbom.SBOM { + t.Helper() + require.NoError(t, NewArchiveContainsTask().Execute(context.Background(), nil, sbomsync.NewBuilder(s))) + return s +} + +// markerTask emits one package per "marker.txt" found, so the matrix asserts behavior without +// depending on any real cataloger's fixture format. +func markerTask() Task { + return NewTask("marker-cataloger", func(_ context.Context, resolver file.Resolver, builder sbomsync.Builder) error { + locations, err := resolver.FilesByGlob("**/marker.txt") + if err != nil { + return err + } + for _, loc := range locations { + p := pkg.Package{ + Name: "marker-pkg", + Version: "1.0.0", + Type: pkg.BinaryPkg, + Locations: file.NewLocationSet(loc), + } + p.SetID() + builder.AddPackages(p) + } + return nil + }) +} + +// javaTask is the java archive cataloger as the cataloging factory builds it, so a jar is met as a jar +// wherever it sits in a chain. +func javaTask() Task { + return NewPackageTask(CatalogingFactoryConfig{}, java.NewArchiveCataloger(java.DefaultArchiveCatalogerConfig())) +} + +// fileMetadataTask records every file, so each file inside an archive gets an archive-to-file edge. +func fileMetadataTask() Task { + return newFileMetadataCatalogerTask(file.AllFilesSelection) +} + +// pack builds one archive of this family holding the given entries. +func (f archiveFamily) pack(t *testing.T, base string, entries map[string][]byte) []byte { + t.Helper() + switch f { + case familyZip: + return testutils.Zip(t, entries) + case familyJar: + // a manifest makes it a real jar to the java cataloger at any level; the package java reports is + // named after the file, not the manifest title + return testutils.Jar(t, base, "1.0", entries) + case familyTar: + return testutils.Tar(t, entries) + case familyTarGz, familyTgz: + return testutils.TarGz(t, entries) + } + t.Fatalf("unknown archive family %q", f) + return nil +} + +// leafContainerPath is the container path the innermost archive's contents carry: every archive, outer to +// inner. Assert it exactly, never as a substring: a partially composed chain passes a substring check. +func (n nestedArchive) leafContainerPath() string { + return n.containerPaths[len(n.containerPaths)-1] +} + +// coordinates are those of the archive at the given level: the outermost archive's path, and the chain +// of entries within it. +func (n nestedArchive) coordinates(level int) file.Coordinates { + return file.Coordinates{RealPath: n.paths[0], FullPath: strings.Join(n.paths[:level+1], "!/")} +} + +// buildNestedArchive builds one archive per family given, each holding the next, with leaf inside +// the innermost. +func buildNestedArchive(t *testing.T, plan nestPlan) nestedArchive { + t.Helper() + require.NotEmpty(t, plan.families, "a chain needs at least one level") + + out := nestedArchive{ + sizes: make([]int, len(plan.families)), + paths: make([]string, len(plan.families)), + containerPaths: make([]string, len(plan.families)), + virtualPaths: make([]string, len(plan.families)), + } + + // the level index goes in every name and path segment, so no two levels share a path and a chain + // assertion cannot pass on a value composed from the wrong level + for i, family := range plan.families { + base := fmt.Sprintf("level%d", i) + if i == 0 { + // the outermost archive is a file in the scanned directory, which the resolver reports rooted + out.name = base + string(family) + out.paths[i] = "/" + out.name + continue + } + out.paths[i] = fmt.Sprintf("nest%d/%s%s", i, base, family) + } + + for i := range plan.families { + if i == 0 { + out.containerPaths[i] = out.paths[0] + out.virtualPaths[i] = out.paths[0] + continue + } + out.containerPaths[i] = out.containerPaths[i-1] + "!/" + out.paths[i] + out.virtualPaths[i] = out.virtualPaths[i-1] + ":" + out.paths[i] + } + + // built innermost first, because each level's bytes are an entry of the level above it + var packed []byte + for i := len(plan.families) - 1; i >= 0; i-- { + entries := map[string][]byte{} + if i == len(plan.families)-1 { + for name, body := range plan.leaf { + entries[name] = body + } + } else { + entries[out.paths[i+1]] = packed + } + for name, body := range plan.extra[i] { + entries[name] = body + } + packed = plan.families[i].pack(t, fmt.Sprintf("level%d", i), entries) + out.sizes[i] = len(packed) + } + out.bytes = packed + + return out +} + +// writeNestedArchive writes the chain's outermost archive into dir and returns the chain. +func writeNestedArchive(t *testing.T, dir string, plan nestPlan) nestedArchive { + t.Helper() + nested := buildNestedArchive(t, plan) + require.NoError(t, os.WriteFile(filepath.Join(dir, nested.name), nested.bytes, 0o644)) + return nested +} + +// markerLeaf is the family-agnostic leaf payload: one file no archive format has an opinion about, +// found by markerTask. It keeps the rows comparable - the same package must come out whichever +// families the chain is made of. +func markerLeaf() map[string][]byte { + return map[string][]byte{"nested/marker.txt": []byte("leaf")} +} + +// nestingRows is the depth-3 matrix. Each row covers a case the others do not. +func nestingRows() []nestingRow { + return []nestingRow{ + { + name: "tar.gz/zip/jar", + families: []archiveFamily{familyTarGz, familyZip, familyJar}, + why: "the headline case: compressed tar outermost, jar innermost", + }, + { + name: "zip/tar.gz/jar", + families: []archiveFamily{familyZip, familyTarGz, familyJar}, + why: "compressed tar in the middle, where the stream-only format is not the entry point", + }, + { + name: "jar/tar/zip", + families: []archiveFamily{familyJar, familyTar, familyZip}, + why: "jar as a container rather than a payload", + }, + { + name: "tgz/tar/jar", + families: []archiveFamily{familyTgz, familyTar, familyJar}, + why: "the .tgz alias resolving at a non-outermost boundary", + }, + { + name: "tar/jar/tar.gz", + families: []archiveFamily{familyTar, familyJar, familyTarGz}, + why: "compressed tar as the innermost, which nothing else covers", + }, + } +} + +func (p *nestingProbe) task() Task { + return NewTask("nesting-probe", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error { + trav := archive.TraversalFromContext(ctx) + visit := nestingVisit{} + if trav != nil { + visit.virtualPath = archive.VirtualPath(trav.Location) + visit.fullPath = trav.FullTraversalArchivePath() + visit.depth = strings.Count(archive.VirtualPath(trav.Location), ":") + 1 + } + if p.tempDir != "" { + visit.heldFiles = heldFiles(p.tempDir) + } + + p.mu.Lock() + defer p.mu.Unlock() + p.visits = append(p.visits, visit) + return nil + }) +} + +// containerPaths is the chain of every archive whose contents were cataloged, in visit order. +func (p *nestingProbe) containerPaths() []string { + p.mu.Lock() + defer p.mu.Unlock() + var out []string + for _, v := range p.visits { + if v.fullPath == "" { + continue + } + out = append(out, v.fullPath) + } + return out +} + +// deepestVisit is the visit at the innermost level reached, the only moment at which every level of a +// chain is still held and its routing observable. It returns the whole visit so a case can also assert +// it got as deep as it meant to: an empty overflow sample from a visit that never left the scan root +// would otherwise read as "nothing overflowed". +func (p *nestingProbe) deepestVisit() nestingVisit { + p.mu.Lock() + defer p.mu.Unlock() + var best nestingVisit + for _, v := range p.visits { + if v.depth > best.depth { + best = v + } + } + return best +} + +// heldFiles names the spill files live under tempDir, one "archive-spill" per archive holding content +// that did not fit in memory. Meaningful only while the archives are still held, so it is sampled from +// inside the sub-pipeline rather than after the scan. +func heldFiles(tempDir string) (names []string) { + _ = filepath.WalkDir(tempDir, func(_ string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasPrefix(d.Name(), "archive-") { + return nil //nolint:nilerr // an unreadable directory holds nothing this probe can report + } + // the name is the pattern with os.CreateTemp's random suffix in place of the "*" + names = append(names, d.Name()[:strings.LastIndexByte(d.Name(), '-')]) + return nil + }) + sort.Strings(names) + return names +} + +// isolatedTempDir points archive extraction at a directory of this test's own, so an overflow sample +// sees only this scan's work directories. +func isolatedTempDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + t.Setenv("TMPDIR", dir) // unix + t.Setenv("TMP", dir) // windows + t.Setenv("TEMP", dir) // windows fallback + require.Equal(t, dir, os.TempDir(), "extraction must be writing where the probe is looking") + return dir +} + +func leafLocations(s *sbom.SBOM) []file.Location { + var out []file.Location + for _, p := range s.Artifacts.Packages.Sorted() { + if p.Name != "marker-pkg" { + continue + } + out = append(out, p.Locations.ToSlice()...) + } + return out +} + +func javaVirtualPaths(s *sbom.SBOM) map[string]string { + out := map[string]string{} + for _, p := range s.Artifacts.Packages.Sorted() { + if metadata, ok := p.Metadata.(pkg.JavaArchive); ok { + out[p.Name] = metadata.VirtualPath + } + } + return out +} + +func packageCount(s *sbom.SBOM, name string) int { + var n int + for _, p := range s.Artifacts.Packages.Sorted() { + if p.Name == name { + n++ + } + } + return n +} + +func coordKey(c file.Coordinates) string { + return c.FullNestedPath() +} + +// gradedChain is the headline row with each level padded so the three archives have clearly different +// sizes: a container is never smaller than what it holds, so a bound can only be placed between two +// levels if they are graded on purpose. +func gradedChain() nestPlan { + return nestPlan{ + families: []archiveFamily{familyTarGz, familyZip, familyJar}, + leaf: markerLeaf(), + extra: map[int]map[string][]byte{ + 0: {"pad0.bin": incompressible(24 * 1024)}, + 1: {"pad1.bin": incompressible(24 * 1024)}, + 2: {"pad2.bin": incompressible(8 * 1024)}, + }, + } +} + +// deepChainPayloadBytes is the size of deepChain's leaf payload. +const deepChainPayloadBytes = 64 * 1024 + +// deepChain is the headline row with one large, incompressible payload at the leaf and no other +// padding. The payload dominates every level, so the three archives are about the same size. It sorts +// before the marker so it is the first entry refused when the leaf archive's entries do not fit. +func deepChain() nestPlan { + return nestPlan{ + families: []archiveFamily{familyTarGz, familyZip, familyJar}, + leaf: map[string][]byte{ + "0-payload.bin": incompressible(deepChainPayloadBytes), + "nested/marker.txt": []byte("leaf"), + }, + } +} diff --git a/internal/task/archive_tasks.go b/internal/task/archive_tasks.go new file mode 100644 index 000000000..50e805204 --- /dev/null +++ b/internal/task/archive_tasks.go @@ -0,0 +1,501 @@ +package task + +import ( + "context" + "errors" + "fmt" + "io" + "maps" + "runtime/debug" + "slices" + "time" + + "github.com/dustin/go-humanize" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/archive" + "github.com/anchore/syft/internal/bus" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/internal/mimetype" + "github.com/anchore/syft/internal/sbomsync" + "github.com/anchore/syft/internal/unknown" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/cataloging" + "github.com/anchore/syft/syft/cataloging/pkgcataloging" + "github.com/anchore/syft/syft/event/monitor" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/sbom" +) + +const ArchiveCatalogerTaskName = "archive-cataloger" + +// NewArchiveCatalogerTask returns a task that extracts every archive it finds, runs the same tasks +// against the archive's contents as a standalone filesystem, and records +// CONTAINS relationships from the archive to the packages found directly inside it. It recurses into the archives +// it extracts up to cfg.MaxDepth; the task drops itself from the tasks so that is the only recursion. +// +// Each archive is handled as the scan root is: pkgTasks, then the archives nested in it, then fileTasks, +// which see the packages found in the nested archives (see Traversal.PackageFiles). +// +// The task is the authority on which archives were not cataloged and why; with +// unknowns.IncludeUnexpandedArchives it also reports archives in formats it does not extract. +// +// exclusions are the scan's exclusion patterns, applied inside each archive. Returns nil when archive +// cataloging is disabled (MaxDepth == 0; negative or beyond archive.MaxDepth means as deep as archive.MaxDepth). +func NewArchiveCatalogerTask(cfg cataloging.ArchiveSearchConfig, unknowns cataloging.UnknownsConfig, pkgTasks, fileTasks []Task, exclusions []string) Task { + return newArchiveCatalogerTask(cfg.MaxDepth, archiveLimits(cfg), unknowns.IncludeUnexpandedArchives, pkgTasks, fileTasks, exclusions) +} + +func archiveLimits(cfg cataloging.ArchiveSearchConfig) archive.Limits { + return archive.Limits{MaxMemoryBytes: cfg.MaxMemoryBytes, MaxDiskBytes: cfg.MaxDiskBytes} +} + +func newArchiveCatalogerTask(maxDepth int, limits archive.Limits, reportUnsupportedUnknowns bool, pkgTasks, fileTasks []Task, exclusions []string) Task { + if maxDepth == 0 { + return nil + } + pkgTasks, fileTasks = withoutArchiveCataloger(pkgTasks), withoutArchiveCataloger(fileTasks) + if len(pkgTasks)+len(fileTasks) == 0 { + return nil + } + + fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error { + // catalogers run once per archive, and each must report into the one progress row it owns + ctx = bus.WithCatalogerTaskRegistry(ctx) + + c := &archiveCataloger{ + maxDepth: maxDepth, + reportUnsupportedUnknowns: reportUnsupportedUnknowns, + pkgTasks: pkgTasks, + fileTasks: fileTasks, + limiter: archive.NewLimiter(limits), + exclusions: archive.NewExclusions(exclusions), + progress: bus.StartCatalogerTask(ctx, archiveCatalogerProgressInfo(), -1, ""), + } + err := c.catalog(ctx, resolver, 0, builder) + c.progress.AtomicStage.Set(fmt.Sprintf("%s archives", humanize.Comma(c.progress.Current()))) + c.progress.SetCompleted() + c.logStats() + return err + } + return NewTask(ArchiveCatalogerTaskName, fn, pkgcataloging.PackageTag, "archive") +} + +func archiveCatalogerProgressInfo() monitor.GenericTask { + return monitor.GenericTask{ + Title: monitor.Title{ + Default: "Archives", + }, + ID: ArchiveCatalogerTaskName, + ParentID: monitor.TopLevelCatalogingTaskID, + } +} + +func withoutArchiveCataloger(tasks []Task) []Task { + var out []Task + for _, t := range tasks { + if t == nil || t.Name() == ArchiveCatalogerTaskName { + continue + } + out = append(out, t) + } + return out +} + +type archiveCataloger struct { + maxDepth int + reportUnsupportedUnknowns bool + pkgTasks []Task + fileTasks []Task + limiter *archive.Limiter + exclusions archive.Exclusions + + // progress is the one row for the whole walk: its count is how many archives have been entered + progress *monitor.TaskProgress + + // ancestors are the digests of the archives the walk is currently inside. The walk is depth-first + // and sequential, so this is a stack. + ancestors []file.Digest + + // the archive that spent the longest on its own extraction and cataloging, excluding nested archives + slowest time.Duration + slowestPath string +} + +// logStats reports what the walk cost, once. The peaks are the most held against each limit at any +// one moment, comparable to the configured bounds. +func (c *archiveCataloger) logStats() { + peakMemory, peakDisk := c.limiter.Peak() + fields := []any{ + "archives", c.progress.Current(), + "peak-memory", humanize.Bytes(uint64(peakMemory)), + "peak-disk", humanize.Bytes(uint64(peakDisk)), + } + if c.slowestPath != "" { + fields = append(fields, "slowest", c.slowestPath, "slowest-took", c.slowest.Round(time.Millisecond)) + } + log.WithFields(fields...).Info("nested archive cataloging complete") +} + +// catalog processes every archive in the resolver, recursing into each up to maxDepth. +func (c *archiveCataloger) catalog(ctx context.Context, resolver file.Resolver, depth int, builder sbomsync.Builder) error { + return c.catalogCandidates(ctx, resolver, c.discoverArchives(resolver), depth, builder) +} + +func (c *archiveCataloger) catalogCandidates(ctx context.Context, resolver file.Resolver, candidates []archiveCandidate, depth int, builder sbomsync.Builder) error { + maxDepth := c.maxDepth + // a deeper scan would write chains no decoder accepts back + if maxDepth < 0 || maxDepth > archive.MaxDepth { + maxDepth = archive.MaxDepth + } + if depth >= maxDepth { + // java's own recursion is off while this task runs, so an archive left here is otherwise lost + // without a trace + for _, candidate := range candidates { + if candidate.sniffedAsArchive { + recordArchiveUnknown(builder, candidate.location.Coordinates, "nested archive not cataloged: the nested archive depth limit was reached") + } + } + return nil + } + var errs error + for _, candidate := range candidates { + if err := ctx.Err(); err != nil { + return err + } + if err := c.processArchive(ctx, resolver, candidate, depth, builder); err != nil { + errs = unknown.Append(errs, candidate.location, err) + } + } + return errs +} + +// processArchive processes the given candidate archive, catching panics +// +//nolint:funlen +func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver file.Resolver, candidate archiveCandidate, depth int, builder sbomsync.Builder) (err error) { + // the decoders run on untrusted bytes; a panic in one costs this archive, not the scan + defer func() { + if r := recover(); r != nil { + // the stack goes to the log, not the reason: the reason lands in the published SBOM's unknowns + log.WithFields("path", candidate.location.FullNestedPath()).Debugf("archive cataloging panic stack:\n%s", debug.Stack()) + err = fmt.Errorf("panic while cataloging archive: %v", r) + } + }() + + location := candidate.location + content, err := parentResolver.FileContentsByLocation(location) + if err != nil { + return err + } + defer internal.CloseAndLogError(content, location.FullNestedPath()) + + var archiveContent io.Reader = content + if !candidate.sniffedAsArchive { + var mayHide bool + if archiveContent, mayHide = archive.MayHideAnAppendedArchive(content); !mayHide { + return nil + } + } + + started := time.Now() + + if depth == 0 { + // one budget for this archive and everything nested in it, however deep + ctx = archive.WithBudget(ctx, archive.NewBudget(archiveTreeBudget(parentResolver, location))) + } + + // extracted files keep the real path and filesystem ID of the outermost archive; the chain of entries + // to each is carried as its archive path. The chain names real paths, not the link the archive was + // reached by, so it is stable and joins to the archive's package location by prefix. + traversal := &archive.Traversal{Location: location, Parent: archive.TraversalFromContext(ctx)} + archivePath := location.FullNestedPath() + extracted, err := archive.Extract(ctx, archiveContent, location.Coordinates, c.limiter, c.exclusions) + if errors.Is(err, archive.ErrUnsupportedFormat) { + if c.reportUnsupportedUnknowns { + recordArchiveUnknown(builder, location.Coordinates, "archive not cataloged: "+err.Error()) + } + return nil + } + if errors.Is(err, archive.ErrDiskLimitReached) { + // nothing is released while this archive waits, so skip it rather than block the scan + recordArchiveUnknown(builder, location.Coordinates, "archive skipped: its content would exceed the disk limit") + return nil + } + if err != nil { + return err + } + if extracted == nil { + // a format with no extractor, or a candidate whose head only looked like an appended archive. + // Logged rather than returned: an image can hold thousands of these + log.WithFields("archive", archivePath, "sniffed-as-archive", candidate.sniffedAsArchive). + Trace("no extractor could read this candidate; skipping") + return nil + } + defer extracted.Cleanup() + + // an archive whose bytes match one it is nested in (a zip quine) would recurse forever + if c.isAncestor(extracted.Digests) { + recordArchiveUnknown(builder, location.Coordinates, "archive not cataloged: it is identical to an archive that contains it") + return nil + } + c.ancestors = append(c.ancestors, extracted.Digests...) + defer func() { c.ancestors = c.ancestors[:len(c.ancestors)-len(extracted.Digests)] }() + + if extracted.Truncated != nil { + recordArchiveUnknown(builder, location.Coordinates, "archive cataloged from part of its contents: "+extracted.Truncated.Error()) + } + + traversal.Digests = extracted.Digests + ctx = archive.WithTraversal(ctx, traversal) + + c.progress.Increment() + + // everything found in this archive and the archives nested in it, merged into the parent's SBOM once complete + found := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} + foundBuilder := sbomsync.NewBuilder(found) + runArchiveTasks(ctx, c.pkgTasks, extracted, location.Coordinates, foundBuilder) + direct := found.Artifacts.Packages.Sorted() + traversal.Packages = packagesAt(direct, location.Coordinates) + + // self time excludes descending, so a containing archive is not charged for what it contains + selfTime := time.Since(started) + + // as at the scan root, nested archives are cataloged before this archive's files, so the packages + // describing them (located in this archive) select their files here + errs := c.catalogCandidates(ctx, extracted, c.discoverArchives(extracted), depth+1, foundBuilder) + + started = time.Now() + runArchiveTasks(ctx, c.fileTasks, extracted, location.Coordinates, foundBuilder) + // the files within it are related once the scan has decided which it describes (see NewArchiveContainsTask) + for _, p := range direct { + found.Relationships = append(found.Relationships, artifact.Relationship{From: location.Coordinates, To: p, Type: artifact.ContainsRelationship}) + } + mergeArchiveResults(found, builder) + c.progress.AtomicStage.Set(humanize.Comma(c.progress.Current()) + " archives (" + archivePath + ")") + + if took := selfTime + time.Since(started); took > c.slowest { + c.slowest, c.slowestPath = took, archivePath + } + + return errs +} + +// a top-level archive and everything nested in it may decompress to archiveTreeRatio times its size, or +// archiveTreeFloorBytes if that is more. Real trees (a war of jars, a layer of tarballs) expand a few +// times over; a recursive bomb expands by orders of magnitude at every level. +var ( + archiveTreeRatio int64 = 100 + archiveTreeFloorBytes int64 = 1 << 30 +) + +func archiveTreeBudget(resolver file.Resolver, location file.Location) int64 { + var size int64 + if m, err := resolver.FileMetadataByLocation(location); err == nil && m.FileInfo != nil { + size = m.Size() + } + return max(archiveTreeRatio*size, archiveTreeFloorBytes) +} + +func (c *archiveCataloger) isAncestor(digests []file.Digest) bool { + for _, d := range digests { + if slices.Contains(c.ancestors, d) { + return true + } + } + return false +} + +// packagesAt returns the packages located at the given coordinates, i.e. those describing the file there. +func packagesAt(pkgs []pkg.Package, coordinates file.Coordinates) []pkg.Package { + var out []pkg.Package + for _, p := range pkgs { + if locatedAt(p, coordinates) { + out = append(out, p) + } + } + return out +} + +// locatedAt reports whether one of p's locations is at the given coordinates. +func locatedAt(p pkg.Package, coordinates file.Coordinates) bool { + for _, l := range p.Locations.ToUnorderedSlice() { + if l.Coordinates == coordinates { + return true + } + } + return false +} + +func recordArchiveUnknown(builder sbomsync.Builder, coordinates file.Coordinates, reason string) { + appendUnknowns(builder, ArchiveCatalogerTaskName, []unknown.CoordinateError{{Coordinates: coordinates, Reason: errors.New(reason)}}) +} + +// runArchiveTasks runs each task against the archive's contents. +// +// Failures are recorded as RunTask records them at the top level: errors carrying coordinates keep +// them, anything else is attributed to the archive. A failure never fails the archive or the scan. +func runArchiveTasks(ctx context.Context, tasks []Task, resolver file.Resolver, archiveCoordinates file.Coordinates, builder sbomsync.Builder) { + for _, t := range tasks { + err := runSubTaskSafely(ctx, t, resolver, builder) + if err == nil { + continue + } + log.WithFields("task", t.Name(), "error", err).Trace("archive sub-cataloger task reported an error") + + unknowns, remaining := unknown.ExtractCoordinateErrors(err) + if remaining != nil { + unknowns = append(unknowns, unknown.CoordinateError{Coordinates: archiveCoordinates, Reason: remaining}) + } + appendUnknowns(builder, t.Name(), unknowns) + } +} + +// runSubTaskSafely is runTaskSafely without the task name in a panic's error, since appendUnknowns +// already prefixes it. +func runSubTaskSafely(ctx context.Context, t Task, resolver file.Resolver, builder sbomsync.Builder) (err error) { + defer func() { + if r := recover(); r != nil { + log.WithFields("task", t.Name()).Debugf("archive sub-cataloger task panic stack:\n%s", debug.Stack()) + err = fmt.Errorf("panic: %v", r) + } + }() + return t.Execute(ctx, resolver, builder) +} + +// archiveCandidate is one file the walk will try to open. +type archiveCandidate struct { + location file.Location + + // sniffedAsArchive is true when the file's content types it as an archive. Such a file that cannot + // be extracted is a broken archive and is reported; a file that is a candidate only by name is + // passed over silently. + sniffedAsArchive bool +} + +// zipArchiveGlobs find files named like zip archives whose content did not sniff as one, such as a +// Spring Boot executable jar: a launcher script with a zip appended. The name only decides what gets +// opened; the end-of-central-directory record decides whether it is an archive. +// +// Globs rather than a MIME lookup, because asking for every executable, script and unrecognized file +// returned thousands of locations that were all discarded. +var zipArchiveGlobs = []string{ + "**/*.jar", "**/*.war", "**/*.ear", "**/*.par", "**/*.sar", "**/*.nar", "**/*.kar", + "**/*.hpi", "**/*.jpi", "**/*.far", "**/*.rar", "**/*.zip", "**/*.apk", "**/*.aar", + "**/*.egg", "**/*.whl", "**/*.lpkg", "**/*.zap", "**/*.exe", +} + +var archiveMIMETypes = slices.Sorted(slices.Values(mimetype.ArchiveMIMETypeSet.List())) + +// discoverArchives returns every file worth opening: those whose content is an archive, plus those +// named like a zip whose tail may still hold one. +func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCandidate { + archives, err := resolver.FilesByMIMEType(archiveMIMETypes...) + if err != nil { + log.WithFields("error", err).Debug("unable to list archive files for recursive cataloging") + return nil + } + + var candidates []archiveCandidate + seen := map[file.Coordinates]int{} + for _, loc := range archives { + seen[loc.Coordinates] = len(candidates) + candidates = append(candidates, archiveCandidate{location: loc, sniffedAsArchive: true}) + } + + namedLikeZips, err := resolver.FilesByGlob(zipArchiveGlobs...) + if err != nil { + log.WithFields("error", err).Debug("unable to list files that may carry an appended archive") + return candidates + } + for _, loc := range namedLikeZips { + if i, dup := seen[loc.Coordinates]; dup { + // a MIME hit is the bare real path; the name it is reached by (a link such as + // lib/dep.jar -> ../store/3f9a1c) is what name-based catalogers like java key off + if prev := candidates[i].location; prev.AccessPath == "" || prev.AccessPath == prev.FullNestedPath() { + candidates[i].location = loc + } + continue + } + seen[loc.Coordinates] = len(candidates) + candidates = append(candidates, archiveCandidate{location: loc}) + } + return candidates +} + +// mergeArchiveResults copies everything found in an archive into its parent's SBOM. +func mergeArchiveResults(found *sbom.SBOM, builder sbomsync.Builder) { + if pkgs := found.Artifacts.Packages.Sorted(); len(pkgs) > 0 { + builder.AddPackages(pkgs...) + } + if accessor, ok := builder.(sbomsync.Accessor); ok { + accessor.WriteToSBOM(func(s *sbom.SBOM) { + mergeFileArtifacts(&s.Artifacts, &found.Artifacts) + }) + } + if len(found.Relationships) > 0 { + builder.AddRelationships(found.Relationships...) + } +} + +// NewArchiveContainsTask relates each archive to every file the SBOM describes directly within it. It must +// run after every other task, so the files within archives are those the scan root's configuration +// selects (file selection, unknowns, package evidence) and no others. +func NewArchiveContainsTask() Task { + fn := func(_ context.Context, _ file.Resolver, builder sbomsync.Builder) error { + var rels []artifact.Relationship + builder.(sbomsync.Accessor).ReadFromSBOM(func(s *sbom.SBOM) { + rels = archiveContains(s.AllCoordinates()) + }) + builder.AddRelationships(rels...) + return nil + } + return NewTask("archive-contains", fn) +} + +// archiveContains relates the archive holding each file within one to that file, and each such archive to +// the archive holding it, so an archive the SBOM does not otherwise describe still links what is within it. +func archiveContains(coordinates []file.Coordinates) []artifact.Relationship { + related := file.NewCoordinateSet(coordinates...) + var rels []artifact.Relationship + for _, c := range coordinates { + for { + container, ok := archive.ContainingArchive(c) + if !ok { + break + } + rels = append(rels, artifact.Relationship{From: container, To: c, Type: artifact.ContainsRelationship}) + if related.Contains(container) { + break + } + related.Add(container) + c = container + } + } + return rels +} + +func mergeFileArtifacts(dst, src *sbom.Artifacts) { + mergeInto(&dst.FileMetadata, src.FileMetadata) + mergeInto(&dst.FileDigests, src.FileDigests) + mergeInto(&dst.FileContents, src.FileContents) + mergeInto(&dst.FileLicenses, src.FileLicenses) + mergeInto(&dst.Executables, src.Executables) + if len(src.Unknowns) > 0 && dst.Unknowns == nil { + dst.Unknowns = map[file.Coordinates][]string{} + } + for coordinates, reasons := range src.Unknowns { + dst.Unknowns[coordinates] = append(dst.Unknowns[coordinates], reasons...) + } +} + +func mergeInto[K comparable, V any](dst *map[K]V, src map[K]V) { + if len(src) == 0 { + return + } + if *dst == nil { + *dst = make(map[K]V, len(src)) + } + maps.Copy(*dst, src) +} diff --git a/internal/task/archive_tasks_test.go b/internal/task/archive_tasks_test.go new file mode 100644 index 000000000..1b4e6d35c --- /dev/null +++ b/internal/task/archive_tasks_test.go @@ -0,0 +1,957 @@ +package task + +import ( + "bytes" + "context" + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/bmatcuk/doublestar/v4" + "github.com/scylladb/go-set/strset" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + + stereoscopeFile "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/internal/archive" + "github.com/anchore/syft/internal/bus" + "github.com/anchore/syft/internal/mimetype" + "github.com/anchore/syft/internal/sbomsync" + "github.com/anchore/syft/internal/testutils" + "github.com/anchore/syft/internal/unknown" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/cataloging" + "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/event/monitor" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/sbom" +) + +// dirTestResolver is a minimal file.Resolver over a directory. FilesByMIMEType sniffs each file's +// content as a real resolver does; FileContentsByLocation opens the file, which is random-access. +type dirTestResolver struct { + file.Resolver + dir string + fsid string +} + +// streamingDirResolver is dirTestResolver handing out plain streams, so an archive's own bytes must be +// held in memory or written to disk rather than read in place. +type streamingDirResolver struct { + dirTestResolver +} + +func (d streamingDirResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) { + f, err := d.dirTestResolver.FileContentsByLocation(loc) + if err != nil { + return nil, err + } + return struct { + io.Reader + io.Closer + }{f, f}, nil +} + +// panickingDirResolver is dirTestResolver whose reads of one file panic, standing in for a decoder +// that panics on hostile bytes. +type panickingDirResolver struct { + dirTestResolver + path string +} + +func (d panickingDirResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) { + if loc.RealPath == d.path { + return io.NopCloser(panicReader{}), nil + } + return d.dirTestResolver.FileContentsByLocation(loc) +} + +type panicReader struct{} + +func (panicReader) Read([]byte) (int, error) { panic("decoder blew up") } + +func Test_archiveCataloger_aPanicCostsOnlyThatArchive(t *testing.T) { + rootDir := t.TempDir() + for _, name := range []string{"bad.zip", "good.zip"} { + require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600)) + } + + var seen []string + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), capturingTask(t, &seen, nil)) + + resolver := panickingDirResolver{dirTestResolver: dirTestResolver{dir: rootDir}, path: "/bad.zip"} + err := tsk.Execute(context.Background(), resolver, sbomsync.NewBuilder(newTestSBOM())) + + // a coordinate error becomes an unknown; a plain error (or an escaped panic) would fail the scan + coordErrs, remaining := unknown.ExtractCoordinateErrors(err) + require.NoError(t, remaining) + require.Len(t, coordErrs, 1) + assert.Equal(t, "/bad.zip", coordErrs[0].Coordinates.RealPath) + assert.EqualError(t, coordErrs[0].Reason, "panic while cataloging archive: decoder blew up", "no stack trace in the SBOM-bound reason") + assert.Equal(t, []string{"/good.zip"}, seen, "the sibling is still cataloged") +} + +func newTestTask(t *testing.T, cfg cataloging.ArchiveSearchConfig, subPipeline ...Task) Task { + t.Helper() + tsk := NewArchiveCatalogerTask(cfg, cataloging.UnknownsConfig{}, subPipeline, nil, nil) + require.NotNil(t, tsk) + return tsk +} + +// newLimitedTestTask builds the task with the limiter's own limits, where zero forbids the resource. +func newLimitedTestTask(t *testing.T, maxDepth int, limits archive.Limits, subPipeline ...Task) Task { + t.Helper() + tsk := newArchiveCatalogerTask(maxDepth, limits, false, subPipeline, nil, nil) + require.NotNil(t, tsk) + return tsk +} + +func newTestSBOM() *sbom.SBOM { + return &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} +} + +func countingTask(ran *int) Task { + return NewTask("count-runs", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { + *ran++ + return nil + }) +} + +// capturingTask records the virtual path of every archive the sub-pipeline is run against and how many +// files its resolver holds. +func capturingTask(t *testing.T, seen *[]string, fileCounts *map[string]int) Task { + return NewTask("capture", func(ctx context.Context, r file.Resolver, _ sbomsync.Builder) error { + trav := archive.TraversalFromContext(ctx) + require.NotNil(t, trav) + *seen = append(*seen, archive.VirtualPath(trav.Location)) + if fileCounts != nil { + locs, err := r.FilesByGlob("**") + require.NoError(t, err) + (*fileCounts)[archive.VirtualPath(trav.Location)] = len(locs) + } + return nil + }) +} + +func Test_archiveCataloger_truncationStillCatalogs(t *testing.T) { + // reaching a limit is a truncation, not a failure: the sub-pipeline still runs over what was stored + big := bytes.Repeat([]byte("x"), 4096) + outerZip := testutils.Zip(t, map[string][]byte{ + "a/small.txt": []byte("small"), + "b/big.txt": big, + }) + + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) + + // nothing is held in memory, so each entry is charged to disk: small.txt lands, big.txt is refused + limits := archive.Limits{MaxMemoryBytes: 0, MaxDiskBytes: 6000} + + var seen []string + fileCounts := map[string]int{} + tsk := newLimitedTestTask(t, 1, limits, capturingTask(t, &seen, &fileCounts)) + + s := newTestSBOM() + err := tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)) + + require.NoError(t, err, "a truncated extraction must not fail the scan") + assert.Equal(t, []string{"/outer.zip"}, seen, "the sub-pipeline must run over the partial contents") + assert.Equal(t, 1, fileCounts["/outer.zip"], "only the entry stored before the limit is visible") + + reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")] + require.True(t, ok, "the truncation must be recorded against the archive; got %v", s.Artifacts.Unknowns) + require.Len(t, reasons, 1) + assert.Contains(t, reasons[0], "archive cataloged from part of its contents") + assert.Contains(t, reasons[0], ArchiveCatalogerTaskName, "the unknown names the task that recorded it") +} + +func Test_archiveCataloger_aDamagedEntryCostsOnlyThatEntry(t *testing.T) { + rootDir := t.TempDir() + // entry data overwritten but headers and end-of-central-directory intact: detected as an archive, + // and data.bin fails when it is read. ok.txt comes after it and is intact. + corrupt := testutils.Zip(t, map[string][]byte{"data.bin": incompressible(4096), "ok.txt": []byte("ok")}) + require.Greater(t, len(corrupt), 1024) + for i := 100; i < 600; i++ { + corrupt[i] ^= 0xff + } + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "corrupt.zip"), corrupt, 0o600)) + + var seen []string + fileCounts := map[string]int{} + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), capturingTask(t, &seen, &fileCounts)) + + s := newTestSBOM() + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) + + assert.Equal(t, []string{"/corrupt.zip"}, seen, "the archive is still cataloged") + assert.Equal(t, 1, fileCounts["/corrupt.zip"], "from the entries that could be read") + reasons := s.Artifacts.Unknowns[file.NewCoordinates("/corrupt.zip", "")] + require.Len(t, reasons, 1) + assert.Contains(t, reasons[0], `entry "data.bin" could not be read`) +} + +func Test_NewArchiveCatalogerTask_gating(t *testing.T) { + someTask := NewTask("noop", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { return nil }) + + tests := []struct { + name string + cfg cataloging.ArchiveSearchConfig + subPipeline []Task + wantTask bool + }{ + { + name: "disabled at depth 0", + cfg: cataloging.DefaultArchiveSearchConfig(), + subPipeline: []Task{someTask}, + }, + { + name: "no sub-pipeline", + cfg: cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2), + subPipeline: nil, + }, + { + name: "enabled", + cfg: cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2), + subPipeline: []Task{someTask}, + wantTask: true, + }, + { + name: "enabled at negative depth", + cfg: cataloging.DefaultArchiveSearchConfig().WithMaxDepth(-1), + subPipeline: []Task{someTask}, + wantTask: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NewArchiveCatalogerTask(tt.cfg, cataloging.UnknownsConfig{}, tt.subPipeline, nil, nil) + if tt.wantTask { + require.NotNil(t, got) + assert.Equal(t, ArchiveCatalogerTaskName, got.Name()) + return + } + assert.Nil(t, got) + }) + } +} + +func Test_archiveCataloger_depthStopsAtTheCeiling(t *testing.T) { + // "unbounded" must still end, or a zip quine or recursive bomb runs forever, and a configured depth + // past the ceiling must too, or the SBOM holds chains no decoder accepts back + chain := testutils.Zip(t, map[string][]byte{"deep.txt": []byte("deep")}) + for range archive.MaxDepth + 4 { + chain = testutils.Zip(t, map[string][]byte{"n.zip": chain}) + } + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), chain, 0o600)) + + for _, depth := range []int{-1, archive.MaxDepth + 2} { + t.Run(fmt.Sprint(depth), func(t *testing.T) { + var seen []string + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth), capturingTask(t, &seen, nil)) + _ = tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM())) + + assert.Len(t, seen, archive.MaxDepth) + }) + } +} + +func Test_archiveCataloger_anArchiveIdenticalToAnAncestorIsNotEntered(t *testing.T) { + // a zip quine contains itself. Building one is impractical, so the walk is started as if it were + // already inside an archive with these exact bytes, which is the state a quine reaches one level down + outerZip := testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")}) + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) + + var ran int + c := newTestCataloger(-1, archive.NewLimiter(archive.Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}), countingTask(&ran)) + sum := sha1.Sum(outerZip) + c.ancestors = []file.Digest{{Algorithm: "sha1", Value: hex.EncodeToString(sum[:])}} + + s := newTestSBOM() + require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(s))) + + assert.Zero(t, ran, "the sub-pipeline must not run inside a repeat of an ancestor") + reasons := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")] + require.Len(t, reasons, 1) + assert.Contains(t, reasons[0], "identical to an archive that contains it") + assert.Len(t, c.ancestors, 1, "the walk leaves the ancestor stack as it found it") +} + +func Test_archiveCataloger_stopsWhenTheContextIsCancelled(t *testing.T) { + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600)) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + var ran int + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), countingTask(&ran)) + err := tsk.Execute(ctx, dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM())) + + require.ErrorIs(t, err, context.Canceled) + assert.Zero(t, ran) +} + +func Test_archiveCataloger_decompressionBudgetIsSharedByATreeAndFreshPerTopLevelArchive(t *testing.T) { + // a recursive bomb fits the limits one level at a time, so what bounds it is the total decompressed + // under one top-level archive + origRatio, origFloor := archiveTreeRatio, archiveTreeFloorBytes + archiveTreeRatio, archiveTreeFloorBytes = 0, 100_000 + t.Cleanup(func() { archiveTreeRatio, archiveTreeFloorBytes = origRatio, origFloor }) + + zeros := make([]byte, 60_000) + nested := testutils.Zip(t, map[string][]byte{"zeros.bin": zeros}) + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "bomb.zip"), testutils.Zip(t, map[string][]byte{"a.zip": nested, "b.zip": nested}), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "sibling.zip"), testutils.Zip(t, map[string][]byte{"zeros.bin": zeros}), 0o600)) + + var seen []string + fileCounts := map[string]int{} + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2), capturingTask(t, &seen, &fileCounts)) + s := newTestSBOM() + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) + + assert.Equal(t, 1, fileCounts["/bomb.zip:a.zip"], "the first nested archive fits the tree's budget") + assert.Equal(t, 0, fileCounts["/bomb.zip:b.zip"], "the second does not: the budget is shared, not per archive") + reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/bomb.zip", FullPath: "/bomb.zip!/b.zip"}] + require.Len(t, reasons, 1) + assert.Contains(t, reasons[0], "possible decompression bomb") + assert.Equal(t, 1, fileCounts["/sibling.zip"], "another top-level archive starts with a fresh budget") +} + +func Test_archiveCataloger_noSelfContainsEdge(t *testing.T) { + // a jar java cannot identify records an unknown at its own coordinates; that must not turn into + // "x.jar CONTAINS x.jar", which a graph walker would loop on + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "x.jar"), testutils.Zip(t, map[string][]byte{"readme.txt": []byte("no manifest")}), 0o600)) + + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), javaTask(), fileMetadataTask()) + s := newTestSBOM() + s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{} + _ = tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)) + withArchiveContains(t, s) + + require.Contains(t, s.Artifacts.Unknowns, file.NewCoordinates("/x.jar", ""), "the fixture must reproduce the unknown") + var contains int + for _, rel := range s.Relationships { + if rel.Type != artifact.ContainsRelationship { + continue + } + contains++ + assert.NotEqual(t, rel.From.ID(), rel.To.ID(), "self-referential CONTAINS edge: %+v", rel) + } + assert.Positive(t, contains, "the archive still contains its files") +} + +func Test_NewArchiveContainsTask(t *testing.T) { + // a file within an archive is contained only when the SBOM describes it, as a file at the scan root is + // listed only then; an executable no package owns is recorded whatever the file selection + in := func(fullPath string) file.Coordinates { + return file.Coordinates{RealPath: "/outer.tar", FileSystemID: "layer", FullPath: fullPath} + } + outer := file.NewCoordinates("/outer.tar", "layer") + jar := in("/outer.tar!/lib/app.jar") + manifest := in("/outer.tar!/lib/app.jar!/META-INF/MANIFEST.MF") + unknownExe := in("/outer.tar!/bin/setup.exe") + + s := newTestSBOM() + s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{manifest: {}, file.NewCoordinates("/loose.txt", "layer"): {}} + s.Artifacts.Executables = map[file.Coordinates]file.Executable{in("/outer.tar!/bin/tool"): {}} + s.Artifacts.Unknowns = map[file.Coordinates][]string{unknownExe: {"no package identified in executable file"}} + + withArchiveContains(t, s) + + type edge struct{ from, to file.Coordinates } + var edges []edge + for _, r := range s.Relationships { + assert.Equal(t, artifact.ContainsRelationship, r.Type) + edges = append(edges, edge{r.From.(file.Coordinates), r.To.(file.Coordinates)}) + } + assert.ElementsMatch(t, []edge{ + {jar, manifest}, + {outer, jar}, // the jar is not described itself, but is on the way to what is + {outer, unknownExe}, + }, edges) +} + +func Test_NewArchiveCatalogerTask_dropsItselfFromSubPipeline(t *testing.T) { + // a copy of itself in the sub-pipeline would process every nesting level twice + cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2) + + innerZip := testutils.Zip(t, map[string][]byte{"leaf.txt": []byte("leaf")}) + outerZip := testutils.Zip(t, map[string][]byte{"nested/inner.zip": innerZip}) + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) + + var ran int + countTask := countingTask(&ran) + + poisoned := []Task{countTask, NewArchiveCatalogerTask(cfg, cataloging.UnknownsConfig{}, []Task{countTask}, nil, nil)} + require.NotNil(t, poisoned[1]) + + tsk := newTestTask(t, cfg, poisoned...) + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM()))) + + assert.Equal(t, 2, ran, "two archives, one sub-pipeline task each") + + t.Run("a sub-pipeline of nothing but itself yields no task", func(t *testing.T) { + only := []Task{NewArchiveCatalogerTask(cfg, cataloging.UnknownsConfig{}, []Task{countTask}, nil, nil)} + assert.Nil(t, NewArchiveCatalogerTask(cfg, cataloging.UnknownsConfig{}, only, nil, nil)) + }) +} + +func Test_archiveCataloger_nestedArchivesKeepTheLayerFileSystemID(t *testing.T) { + // extracted files keep the layer digest as their FileSystemID at every level; the nesting chain + // rides on the traversal's VirtualPath and the coordinate's FullPath + innerZip := testutils.Zip(t, map[string][]byte{"leaf.txt": []byte("leaf")}) + outerZip := testutils.Zip(t, map[string][]byte{"nested/inner.zip": innerZip}) + + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) + + const layerDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111" + + var chains, fsids []string + captureTask := NewTask("capture-fsid", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error { + trav := archive.TraversalFromContext(ctx) + require.NotNil(t, trav) + chains = append(chains, archive.VirtualPath(trav.Location)) + fsids = append(fsids, trav.Location.FileSystemID) + return nil + }) + + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2), captureTask) + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir, fsid: layerDigest}, sbomsync.NewBuilder(newTestSBOM()))) + + assert.Equal(t, []string{layerDigest, layerDigest}, fsids) + assert.ElementsMatch(t, []string{"/outer.zip", "/outer.zip:nested/inner.zip"}, chains) +} + +// twoLayerResolver reports the same archive path under two filesystem ids with different content, as an +// all-layers image scan does for an archive rewritten in a later layer. +type twoLayerResolver struct { + file.Resolver + path string + contentBy map[string][]byte +} + +func Test_archiveCataloger_sameArchivePathInTwoLayersDoesNotCollide(t *testing.T) { + resolver := twoLayerResolver{ + path: "/app/bundle.zip", + contentBy: map[string][]byte{ + "layer-one": testutils.Zip(t, map[string][]byte{"lib/config.json": []byte(`{"v":1}`)}), + "layer-two": testutils.Zip(t, map[string][]byte{"lib/config.json": []byte(`{"v":2}`)}), + }, + } + + var chains []string + fileTask := NewTask("record-files", func(ctx context.Context, r file.Resolver, builder sbomsync.Builder) error { + trav := archive.TraversalFromContext(ctx) + require.NotNil(t, trav) + chains = append(chains, trav.Location.FileSystemID) + + locs, err := r.FilesByGlob("**") + require.NoError(t, err) + metadata := map[file.Coordinates]file.Metadata{} + for _, loc := range locs { + metadata[loc.Coordinates] = file.Metadata{Path: loc.RealPath} + } + builder.(sbomsync.Accessor).WriteToSBOM(func(s *sbom.SBOM) { + s.Artifacts.FileMetadata = metadata + }) + return nil + }) + + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), fileTask) + + s := &sbom.SBOM{Artifacts: sbom.Artifacts{ + Packages: pkg.NewCollection(), + FileMetadata: map[file.Coordinates]file.Metadata{}, + }} + require.NoError(t, tsk.Execute(context.Background(), resolver, sbomsync.NewBuilder(s))) + + assert.ElementsMatch(t, []string{"layer-one", "layer-two"}, chains) + // the tables are keyed by Coordinates, so a collision is a silently missing entry + assert.Len(t, s.Artifacts.FileMetadata, 2) +} + +func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) { + // runSubPipeline bypasses RunTask, where the coordinate-error to unknown conversion lives + outerZip := testutils.Zip(t, map[string][]byte{"lib/broken.json": []byte("{")}) + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) + + cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1) + + t.Run("a coordinate error keeps the coordinates the cataloger named", func(t *testing.T) { + failing := NewTask("failing-cataloger", func(_ context.Context, r file.Resolver, _ sbomsync.Builder) error { + locs, err := r.FilesByGlob("**") + require.NoError(t, err) + require.NotEmpty(t, locs) + var errs error + for _, loc := range locs { + errs = unknown.Append(errs, loc, errors.New("unable to parse")) + } + return errs + }) + + tsk := newTestTask(t, cfg, failing) + s := newTestSBOM() + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)), + "a cataloger failing inside an archive must not fail the archive or the scan") + + require.Len(t, s.Artifacts.Unknowns, 1) + for coords, reasons := range s.Artifacts.Unknowns { + assert.Equal(t, "/outer.zip!/lib/broken.json", coords.FullNestedPath(), "the inner coordinates must survive the merge") + assert.Equal(t, "/outer.zip", coords.RealPath, "and must be addressable per archive") + assert.Equal(t, []string{"failing-cataloger: unable to parse"}, reasons) + } + }) + + t.Run("an error with no coordinates is attributed to the archive", func(t *testing.T) { + failing := NewTask("bare-error-cataloger", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { + return errors.New("cataloger blew up") + }) + + tsk := newTestTask(t, cfg, failing) + s := newTestSBOM() + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) + + require.Len(t, s.Artifacts.Unknowns, 1) + reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")] + require.True(t, ok, "an error with no location of its own belongs to the containing archive") + assert.Equal(t, []string{"bare-error-cataloger: cataloger blew up"}, reasons) + }) + + t.Run("a panicking cataloger is recovered and recorded", func(t *testing.T) { + panicking := NewTask("panicking-cataloger", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { + panic("nope") + }) + + tsk := newTestTask(t, cfg, panicking) + s := newTestSBOM() + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)), + "a panic inside an archive must not fail the scan") + + reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")] + require.True(t, ok) + require.Len(t, reasons, 1) + assert.Equal(t, "panicking-cataloger: panic: nope", reasons[0]) + }) +} + +func Test_archiveCataloger_subPipelineUnknownsAreAddedNotReplaced(t *testing.T) { + outerZip := testutils.Zip(t, map[string][]byte{"lib/thing.json": []byte("{}")}) + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) + + preexisting := file.Coordinates{RealPath: "/outer.zip", FullPath: "/outer.zip!/lib/thing.json"} + writer := NewTask("scratch-writer", func(_ context.Context, _ file.Resolver, builder sbomsync.Builder) error { + builder.(sbomsync.Accessor).WriteToSBOM(func(s *sbom.SBOM) { + if s.Artifacts.Unknowns == nil { + s.Artifacts.Unknowns = map[file.Coordinates][]string{} + } + s.Artifacts.Unknowns[preexisting] = append(s.Artifacts.Unknowns[preexisting], "recorded by the cataloger itself") + }) + return nil + }) + failing := NewTask("failing-cataloger", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { + return errors.New("boom") + }) + + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), writer, failing) + s := newTestSBOM() + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) + + assert.Equal(t, []string{"recorded by the cataloger itself"}, s.Artifacts.Unknowns[preexisting]) + assert.Equal(t, []string{"failing-cataloger: boom"}, s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]) +} + +func Test_archiveCataloger_limitsFallWhenAnArchiveIsReleased(t *testing.T) { + entry := bytes.Repeat([]byte("z"), 400) + entries := map[string][]byte{} + for i := range 5 { + entries[fmt.Sprintf("e%d.bin", i)] = entry + } + + rootDir := t.TempDir() + for _, name := range []string{"one.zip", "two.zip"} { + require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), testutils.Zip(t, entries), 0o600)) + } + + // room in memory for one archive's entries and their index cost, not for two at once + limiter := archive.NewLimiter(archive.Limits{MaxMemoryBytes: 20000, MaxDiskBytes: 0}) + + // sampled while the archive is still held + var extracted []int + var peakMemory int64 + sample := NewTask("sample", func(_ context.Context, r file.Resolver, _ sbomsync.Builder) error { + locs, err := r.FilesByGlob("**") + require.NoError(t, err) + extracted = append(extracted, len(locs)) + if mem, _ := limiter.InUse(); mem > peakMemory { + peakMemory = mem + } + return nil + }) + + c := newTestCataloger(1, limiter, sample) + require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(newTestSBOM()))) + + assert.Equal(t, []int{5, 5}, extracted, + "both archives must extract in full: the first is released before the second is admitted") + assert.Positive(t, peakMemory, "entries within the memory limit are held in memory") + + mem, disk := limiter.InUse() + assert.Zero(t, mem, "the memory limit must fall back to nothing once the walk unwinds") + assert.Zero(t, disk, "and so must the disk limit") +} + +func newTestCataloger(maxDepth int, limiter *archive.Limiter, subPipeline ...Task) *archiveCataloger { + return &archiveCataloger{ + maxDepth: maxDepth, + pkgTasks: subPipeline, + limiter: limiter, + progress: bus.StartCatalogerTask(context.Background(), archiveCatalogerProgressInfo(), -1, ""), + } +} + +func Test_archiveCataloger_archiveWhoseOwnBytesExceedTheDiskLimitIsSkipped(t *testing.T) { + // a streamed archive's own bytes must be placed before it can be opened; when they do not fit on + // disk the archive is skipped rather than waited for, and a sibling that fits is cataloged in full + oversized := testutils.Zip(t, map[string][]byte{"payload.bin": incompressible(8000)}) + small := testutils.Zip(t, map[string][]byte{"ok.txt": []byte("ok")}) + + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "oversized.zip"), oversized, 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "small.zip"), small, 0o600)) + + // room for the small archive's bytes, its one entry and that entry's index cost + diskLimit := int64(len(small)) + 3000 + require.Greater(t, int64(len(oversized)), diskLimit, "the fixture must actually exceed the limit") + + limits := archive.Limits{MaxMemoryBytes: 0, MaxDiskBytes: diskLimit} + + var seen []string + fileCounts := map[string]int{} + tsk := newLimitedTestTask(t, 1, limits, capturingTask(t, &seen, &fileCounts)) + + s := newTestSBOM() + require.NoError(t, tsk.Execute(context.Background(), streamingDirResolver{dirTestResolver{dir: rootDir}}, sbomsync.NewBuilder(s)), + "reaching a limit must not fail the scan") + + assert.Equal(t, []string{"/small.zip"}, seen, "the oversized archive is skipped and the sibling is still cataloged") + assert.Equal(t, 1, fileCounts["/small.zip"]) + + reasons := s.Artifacts.Unknowns[file.NewCoordinates("/oversized.zip", "")] + require.Len(t, reasons, 1, "the skip must be recorded against the archive; got %v", s.Artifacts.Unknowns) + assert.Contains(t, reasons[0], "archive skipped") +} + +func Test_archiveCataloger_discoverArchivesSelectsTarFamily(t *testing.T) { + marker := map[string][]byte{"nested/marker.txt": []byte("hello")} + + scanDir := t.TempDir() + for name, body := range map[string][]byte{ + "bundle.tar.gz": testutils.TarGz(t, marker), + "bundle.tgz": testutils.TarGz(t, marker), + "bundle.tar": testutils.Tar(t, marker), + "bundle.zip": testutils.Zip(t, marker), + "notes.txt": []byte("not an archive at all"), + } { + require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), body, 0o600)) + } + + var seen []string + fileCounts := map[string]int{} + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), capturingTask(t, &seen, &fileCounts)) + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(newTestSBOM()))) + + assert.ElementsMatch(t, []string{"/bundle.tar.gz", "/bundle.tgz", "/bundle.tar", "/bundle.zip"}, seen, + "every tar-family archive must be reached, and the plain text file must not be") + for _, archivePath := range seen { + assert.Equal(t, 1, fileCounts[archivePath], "%s must have been extracted", archivePath) + } + + t.Run("the MIME types the tar family is sniffed as are in the set discovery asks for", func(t *testing.T) { + for _, mt := range []string{"application/x-tar", "application/gzip", "application/x-gtar", "application/zip"} { + assert.True(t, mimetype.ArchiveMIMETypeSet.Has(mt), "%s must be a candidate archive type", mt) + } + }) +} + +type recordingPublisher struct { + events []partybus.Event +} + +func Test_archiveCatalogerTask_publishesOneProgressRowPerCataloger(t *testing.T) { + // a consumer keying rows by ID (as the syft CLI does) replaces a republished row and never sees it + // complete, so each cataloger keeps to the one row it already owns + inner := testutils.Zip(t, map[string][]byte{"inner/file.txt": []byte("hello")}) + scanDir := t.TempDir() + for _, name := range []string{"one.zip", "two.zip"} { + require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), inner, 0o600)) + } + + publisher := &recordingPublisher{} + bus.Set(publisher) + t.Cleanup(func() { bus.Set(nil) }) + + // the archive row's stage as each archive's sub-pipeline starts: the previous archive is done by then, + // so the row must read the walk's own count rather than anything a nested cataloger left behind + var rows []*monitor.TaskProgress + var stageAtEntry []string + noisy := NewTask("noisy", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error { + stageAtEntry = append(stageAtEntry, publisher.events[0].Value.(progress.StagedProgressable).Stage()) + p := bus.StartCatalogerTask(ctx, monitor.GenericTask{ID: "noisy"}, 3, "") + rows = append(rows, p) + p.AtomicStage.Set("working on something") + p.Add(3) + p.SetCompleted() + return nil + }) + + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), noisy) + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(newTestSBOM()))) + + var started []string + for _, e := range publisher.events { + if e.Type != event.CatalogerTaskStarted { + continue + } + info, ok := e.Source.(monitor.GenericTask) + require.True(t, ok, "expected a generic task as the event source") + started = append(started, info.ID) + } + + assert.Equal(t, []string{ArchiveCatalogerTaskName, "noisy"}, started, + "each cataloger publishes one row, however many archives the walk enters") + + require.Len(t, rows, 2, "the sub-pipeline ran once per archive") + assert.Same(t, rows[0], rows[1], "both runs must report into the one row that cataloger owns") + assert.Equal(t, int64(6), rows[0].Current(), "what every run found is counted onto that row") + + require.Len(t, publisher.events, 2) + prog, ok := publisher.events[0].Value.(progress.StagedProgressable) + require.True(t, ok) + assert.ErrorIs(t, prog.Error(), progress.ErrCompleted, "the row must be completed, or a waiting consumer never finishes") + assert.Equal(t, int64(2), prog.Current(), "the archive row counts every archive the walk entered") + assert.Equal(t, "2 archives", prog.Stage(), "and says so once the walk is done") + + // one.zip sorts before two.zip, and the resolver walks the scan directory in lexical order + assert.Equal(t, []string{"", "1 archives (/one.zip)"}, stageAtEntry, + "the row must take its stage back from the sub-pipeline once an archive is done") +} + +func Test_archiveCataloger_reportsPeakUsageAndTheSlowestArchive(t *testing.T) { + // self time excludes nested archives, or the outer archive would be named every scan + innerJar := testutils.Zip(t, map[string][]byte{"README.txt": []byte("inner")}) + outerZip := testutils.Zip(t, map[string][]byte{"lib/inner.jar": innerJar}) + + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) + + const innerWork = 40 * time.Millisecond + slow := NewTask("slow-inside-the-inner-archive", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error { + if trav := archive.TraversalFromContext(ctx); trav != nil && strings.HasSuffix(archive.VirtualPath(trav.Location), "inner.jar") { + time.Sleep(innerWork) + } + return nil + }) + + limiter := archive.NewLimiter(archive.Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) + c := newTestCataloger(3, limiter, slow) + require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(newTestSBOM()))) + + assert.Equal(t, "/outer.zip!/lib/inner.jar", c.slowestPath, "the slowest archive is named by its full chain") + assert.GreaterOrEqual(t, c.slowest, innerWork) + assert.Equal(t, int64(2), c.progress.Current()) + + // the peaks outlive the walk that set them + peakMemory, _ := limiter.Peak() + assert.Positive(t, peakMemory, "entries held in memory must show on the peak") + memory, disk := limiter.InUse() + assert.Zero(t, memory) + assert.Zero(t, disk) +} + +func Test_archiveCataloger_slowestIsUnsetWhenNoArchiveIsCataloged(t *testing.T) { + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "notes.txt"), []byte("not an archive"), 0o600)) + + noop := NewTask("noop", func(context.Context, file.Resolver, sbomsync.Builder) error { return nil }) + c := newTestCataloger(2, archive.NewLimiter(archive.Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}), noop) + require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(newTestSBOM()))) + + assert.Empty(t, c.slowestPath) + assert.Zero(t, c.progress.Current()) + c.logStats() // must not panic with nothing to report +} + +func Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError(t *testing.T) { + // the name says archive, the content does not: it is never opened + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "notes.zip"), + []byte("this is prose, not an archive, whatever the extension claims\n"), 0o600)) + + var ran int + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), countingTask(&ran)) + + err := tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM())) + + assert.NoError(t, err, "a file that is not an archive is not a failure to report") + assert.Zero(t, ran, "and nothing is cataloged inside it") +} + +func Test_archiveCataloger_catalogsAZipBehindALauncherScript(t *testing.T) { + // the Spring Boot executable jar shape: content sniffing types it as a shell script, so it reaches + // the walk only by name and qualifies only by its end-of-central-directory record + inner := testutils.Zip(t, map[string][]byte{"nested/marker.txt": []byte("found me")}) + prefixed := append([]byte("#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n"), + testutils.Zip(t, map[string][]byte{"BOOT-INF/lib/inner.jar": inner})...) + + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "app.jar"), prefixed, 0o600)) + + var seen []string + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3), capturingTask(t, &seen, nil)) + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM()))) + + assert.Equal(t, []string{"/app.jar", "/app.jar:BOOT-INF/lib/inner.jar"}, seen, + "the stub must not hide the archive, and the walk must carry on into what it holds") +} + +func (d dirTestResolver) FilesByMIMEType(types ...string) ([]file.Location, error) { + return filesByMIMEType(d.dir, d.fsid, types...) +} + +func filesByMIMEType(dir, fsid string, types ...string) ([]file.Location, error) { + wanted := strset.New(types...) + var locs []file.Location + err := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return err + } + f, err := os.Open(path) + if err != nil { + return err + } + mimeType := stereoscopeFile.MIMEType(f) + if closeErr := f.Close(); closeErr != nil { + return closeErr + } + if !wanted.Has(mimeType) { + return nil + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + coords := file.Coordinates{RealPath: "/" + filepath.ToSlash(rel), FileSystemID: fsid} + locs = append(locs, file.NewLocationFromCoordinates(coords)) + return nil + }) + return locs, err +} + +func (d dirTestResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) { + return os.Open(filepath.Join(d.dir, filepath.FromSlash(loc.RealPath))) +} + +func (d dirTestResolver) FileMetadataByLocation(loc file.Location) (file.Metadata, error) { + info, err := os.Stat(filepath.Join(d.dir, filepath.FromSlash(loc.RealPath))) + return file.Metadata{FileInfo: info}, err +} + +// matchesAnyGlob matches the resolver-relative path with a leading slash, the form a real resolver +// globs over. No patterns matches everything. +func matchesAnyGlob(root, path string, patterns []string) bool { + if len(patterns) == 0 { + return true + } + rel, err := filepath.Rel(root, path) + if err != nil { + return false + } + rel = "/" + filepath.ToSlash(rel) + for _, pattern := range patterns { + if ok, err := doublestar.Match(pattern, rel); err == nil && ok { + return true + } + } + return false +} + +func (r twoLayerResolver) FilesByGlob(_ ...string) ([]file.Location, error) { + return nil, nil +} + +func (d dirTestResolver) FilesByGlob(patterns ...string) ([]file.Location, error) { + var locs []file.Location + err := filepath.WalkDir(d.dir, func(path string, entry fs.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return err + } + if !matchesAnyGlob(d.dir, path, patterns) { + return nil + } + rel, err := filepath.Rel(d.dir, path) + if err != nil { + return err + } + coords := file.Coordinates{RealPath: "/" + filepath.ToSlash(rel), FileSystemID: d.fsid} + locs = append(locs, file.NewLocationFromCoordinates(coords)) + return nil + }) + return locs, err +} + +func (r twoLayerResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) { + var locs []file.Location + for _, fsid := range []string{"layer-one", "layer-two"} { + locs = append(locs, file.NewLocationFromCoordinates(file.NewCoordinates(r.path, fsid))) + } + return locs, nil +} + +func (r twoLayerResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) { + body, ok := r.contentBy[loc.FileSystemID] + if !ok { + return nil, os.ErrNotExist + } + return io.NopCloser(bytes.NewReader(body)), nil +} + +// FileMetadataByLocation reports no metadata, so the archive falls back to the minimum budget. +func (r twoLayerResolver) FileMetadataByLocation(file.Location) (file.Metadata, error) { + return file.Metadata{}, os.ErrNotExist +} + +// incompressible returns n bytes deflate cannot shrink, so an archive built from them has a +// predictable size to set a limit against. +func incompressible(n int) []byte { + b := make([]byte, n) + x := uint32(12345) + for i := range b { + x = x*1664525 + 1013904223 + b[i] = byte(x >> 24) + } + return b +} + +func (p *recordingPublisher) Publish(e partybus.Event) { + p.events = append(p.events, e) +} diff --git a/internal/task/file_tasks.go b/internal/task/file_tasks.go index 7af11fcd1..e82b0ae1e 100644 --- a/internal/task/file_tasks.go +++ b/internal/task/file_tasks.go @@ -47,7 +47,7 @@ func newFileDigestCatalogerTask(selection file.Selection, hashers []crypto.Hash, result, err := filedigest.NewCataloger(hashers).Catalog(ctx, resolver, coordinates...) accessor.WriteToSBOM(func(sbom *sbom.SBOM) { - sbom.Artifacts.FileDigests = result + mergeInto(&sbom.Artifacts.FileDigests, result) }) return err @@ -78,7 +78,7 @@ func newFileMetadataCatalogerTask(selection file.Selection, tags ...string) Task result, err := filemetadata.NewCataloger().Catalog(ctx, resolver, coordinates...) accessor.WriteToSBOM(func(sbom *sbom.SBOM) { - sbom.Artifacts.FileMetadata = result + mergeInto(&sbom.Artifacts.FileMetadata, result) }) return err @@ -104,7 +104,7 @@ func newFileContentCatalogerTask(cfg filecontent.Config, tags ...string) Task { result, err := filecontent.NewCataloger(cfg).Catalog(ctx, resolver) accessor.WriteToSBOM(func(sbom *sbom.SBOM) { - sbom.Artifacts.FileContents = result + mergeInto(&sbom.Artifacts.FileContents, result) }) return err @@ -130,7 +130,7 @@ func newExecutableCatalogerTask(selection file.Selection, cfg executable.Config, result, err := executable.NewCataloger(cfg).CatalogCtx(ctx, resolver) accessor.WriteToSBOM(func(sbom *sbom.SBOM) { - sbom.Artifacts.Executables = result + mergeInto(&sbom.Artifacts.Executables, result) }) return err diff --git a/internal/task/file_tasks_test.go b/internal/task/file_tasks_test.go index b24e7e254..33e81f2ac 100644 --- a/internal/task/file_tasks_test.go +++ b/internal/task/file_tasks_test.go @@ -40,19 +40,13 @@ func Test_coordinatesForSelection(t *testing.T) { Relationships: []artifact.Relationship{ { From: pkg.Package{}, - To: file.Coordinates{ - RealPath: "path", - FileSystemID: "fs", - }, + To: file.NewCoordinates("path", "fs"), Type: artifact.ContainsRelationship, }, }, }, files: []file.Coordinates{ - { - RealPath: "path", - FileSystemID: "fs", - }, + file.NewCoordinates("path", "fs"), }, ok: true, }, @@ -63,10 +57,7 @@ func Test_coordinatesForSelection(t *testing.T) { Relationships: []artifact.Relationship{ { From: pkg.Package{}, - To: file.Coordinates{ - RealPath: "path", - FileSystemID: "fs", - }, + To: file.NewCoordinates("path", "fs"), // wrong type Type: artifact.DependencyOfRelationship, }, @@ -82,10 +73,7 @@ func Test_coordinatesForSelection(t *testing.T) { Relationships: []artifact.Relationship{ { From: file.Coordinates{}, // wrong type - To: file.Coordinates{ - RealPath: "path", - FileSystemID: "fs", - }, + To: file.NewCoordinates("path", "fs"), Type: artifact.ContainsRelationship, }, }, diff --git a/internal/task/package_task_factory.go b/internal/task/package_task_factory.go index 0e83bf2ca..6ba80554a 100644 --- a/internal/task/package_task_factory.go +++ b/internal/task/package_task_factory.go @@ -3,9 +3,11 @@ package task import ( "context" "fmt" + "path" "strings" "unicode" + "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/relationship" @@ -48,7 +50,7 @@ func NewPackageTask(cfg CatalogingFactoryConfig, c pkg.Cataloger, tags ...string HideOnSuccess: true, } - t := bus.StartCatalogerTask(info, -1, "") + t := bus.StartCatalogerTask(ctx, info, -1, "") pkgs, relationships, err := c.Catalog(ctx, resolver) @@ -167,7 +169,7 @@ func applyComplianceRules(p *pkg.Package, cfg cataloging.ComplianceConfig) (bool loc := "unknown" locs := p.Locations.ToSlice() if len(locs) > 0 { - loc = locs[0].Path() + loc = locs[0].FullNestedPath() } switch action { case cataloging.ComplianceActionDrop: @@ -236,10 +238,10 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver locations := map[artifact.ID]file.Location{} - for _, path := range fileOwner.OwnedFiles() { - pathRefs, err := resolver.FilesByPath(path) + for _, ownedPath := range fileOwner.OwnedFiles() { + pathRefs, err := resolver.FilesByPath(ownedPath) if err != nil { - return nil, fmt.Errorf("unable to find path for path=%q: %w", path, err) + return nil, fmt.Errorf("unable to find path for path=%q: %w", ownedPath, err) } if len(pathRefs) == 0 { @@ -250,8 +252,13 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver } for _, ref := range pathRefs { + // an archive looks a relative path up from its root, but an owned path may be relative to + // anything (a python RECORD is relative to site-packages), so only an absolute one owns a file in one + if archive.ContainerPath(ref.Coordinates) != "" && !path.IsAbs(ownedPath) { + continue + } if oldRef, ok := locations[ref.ID()]; ok { - log.Debugf("found path duplicate of %s", oldRef.RealPath) + log.Debugf("found path duplicate of %s", oldRef.FullNestedPath()) } locations[ref.ID()] = ref } diff --git a/internal/task/package_task_factory_test.go b/internal/task/package_task_factory_test.go index f1b8a217c..04f7d7e11 100644 --- a/internal/task/package_task_factory_test.go +++ b/internal/task/package_task_factory_test.go @@ -1,11 +1,15 @@ package task import ( + "bytes" + "context" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/anchore/syft/internal/archive" + "github.com/anchore/syft/internal/testutils" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/cpe" @@ -64,8 +68,8 @@ func TestApplyCompliance(t *testing.T) { p2 := pkg.Package{Name: "", Version: "1.0"} // missing name p3 := pkg.Package{Name: "pkg-3", Version: ""} // missing version p4 := pkg.Package{Name: "pkg-4", Version: ""} // missing version - c1 := file.Coordinates{RealPath: "/coords/1"} - c2 := file.Coordinates{RealPath: "/coords/2"} + c1 := file.NewCoordinates("/coords/1", "") + c2 := file.NewCoordinates("/coords/2", "") for _, p := range []*pkg.Package{&p1, &p2, &p3, &p4} { p.SetID() @@ -281,3 +285,28 @@ func TestApplyComplianceRules_DropAndStub(t *testing.T) { assert.False(t, isCompliant) assert.Equal(t, cataloging.UnknownStubValue, p.Version) } + +func Test_packageFileOwnershipRelationships_exactPathsWithinAnArchive(t *testing.T) { + // a wheel's RECORD is relative to site-packages: looked up from the wheel's root it finds the file, but + // only an exact path owns one, as at the scan root + data := testutils.Zip(t, map[string]string{"pip/__init__.py": "", "bin/tool": ""}) + whl, err := archive.Extract(context.Background(), bytes.NewReader(data), file.NewCoordinates("/pip.whl", ""), + archive.NewLimiter(archive.Limits{MaxMemoryBytes: 1 << 20}), archive.NewExclusions(nil)) + require.NoError(t, err) + t.Cleanup(whl.Cleanup) + + found, err := whl.FilesByPath("pip/__init__.py") + require.NoError(t, err) + require.Len(t, found, 1, "a relative path still finds the file") + + p := pkg.Package{Name: "pip", Metadata: pkg.PythonPackage{Files: []pkg.PythonFileRecord{{Path: "pip/__init__.py"}, {Path: "/bin/tool"}}}} + p.SetID() + rels, err := packageFileOwnershipRelationships(p, whl) + require.NoError(t, err) + + var owned []string + for _, r := range rels { + owned = append(owned, r.To.(file.Coordinates).FullNestedPath()) + } + assert.Equal(t, []string{"/pip.whl!/bin/tool"}, owned) +} diff --git a/internal/task/scope_tasks.go b/internal/task/scope_tasks.go index 8127829ab..11e9e8016 100644 --- a/internal/task/scope_tasks.go +++ b/internal/task/scope_tasks.go @@ -40,10 +40,11 @@ func packagesToRemove(accessor sbomsync.Accessor) []artifact.ID { } case "": if isPrimaryEvidence { - if exists := filterDuplicates[getKey(p, l)]; exists { + k := getKey(p, l) + if exists := filterDuplicates[k]; exists { break } - filterDuplicates[getKey(p, l)] = true + filterDuplicates[k] = true noPrimary = false break } @@ -59,5 +60,5 @@ func packagesToRemove(accessor sbomsync.Accessor) []artifact.ID { } func getKey(pkg pkg.Package, loc file.Location) string { - return fmt.Sprintf("%s-%s-%s-%s", pkg.Name, pkg.Version, loc.RealPath, loc.AccessPath) + return fmt.Sprintf("%s-%s-%s-%s", pkg.Name, pkg.Version, loc.FullNestedPath(), loc.AccessPath) } diff --git a/internal/task/unknowns_tasks.go b/internal/task/unknowns_tasks.go index 8761a06cd..fd874b73a 100644 --- a/internal/task/unknowns_tasks.go +++ b/internal/task/unknowns_tasks.go @@ -2,8 +2,11 @@ package task import ( "context" + "path" + "slices" "strings" + "github.com/anchore/syft/internal/archive" intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/sbomsync" @@ -36,11 +39,20 @@ func (c unknownsLabelerTask) finalize(resolver file.Resolver, s *sbom.SBOM) { hasPackageReference := coordinateReferenceLookup(resolver, s) if c.RemoveWhenPackagesDefined { - for coords := range s.Artifacts.Unknowns { + for coords, reasons := range s.Artifacts.Unknowns { if !hasPackageReference(coords) { continue } - delete(s.Artifacts.Unknowns, coords) + // what the archive cataloger reports is about the archive's contents (left out, cut short), + // which a package describing the archive does not account for + kept := slices.DeleteFunc(reasons, func(r string) bool { + return !strings.HasPrefix(r, formatUnknown("", ArchiveCatalogerTaskName)) + }) + if len(kept) == 0 { + delete(s.Artifacts.Unknowns, coords) + continue + } + s.Artifacts.Unknowns[coords] = kept } } @@ -59,7 +71,7 @@ func (c unknownsLabelerTask) finalize(resolver file.Resolver, s *sbom.SBOM) { if c.IncludeUnexpandedArchives { ctx := context.Background() for coords := range s.Artifacts.FileMetadata { - format, _, notArchiveErr := intFile.IdentifyArchive(ctx, coords.RealPath, nil) + format, _, notArchiveErr := intFile.IdentifyArchive(ctx, coords.FullNestedPath(), nil) if format != nil && notArchiveErr == nil && !hasPackageReference(coords) { s.Artifacts.Unknowns[coords] = append(s.Artifacts.Unknowns[coords], "archive not cataloged") } @@ -71,6 +83,7 @@ func formatUnknown(err string, task ...string) string { return strings.Join(task, "/") + ": " + err } +//nolint:gocognit func coordinateReferenceLookup(resolver file.Resolver, s *sbom.SBOM) func(coords file.Coordinates) bool { allPackageCoords := file.NewCoordinateSet() @@ -83,7 +96,28 @@ func coordinateReferenceLookup(resolver file.Resolver, s *sbom.SBOM) func(coords // relationships for these owned files may be disabled, but we always want to include them for p := range s.Artifacts.Packages.Enumerate() { if f, ok := p.Metadata.(pkg.FileOwner); ok { + // owned paths are within the containers the package was found in + var archives []file.Coordinates + atRoot := false + for _, container := range archive.OwnerContainers(p) { + if container.FullPath == "" { + atRoot = true + } else { + archives = append(archives, container) + } + } for _, ownedFilePath := range f.OwnedFiles() { + // an archive's contents are not in this resolver to follow links in, and only an absolute owned + // path names a file in one (see packageFileOwnershipRelationships) + if path.IsAbs(ownedFilePath) { + for _, container := range archives { + allPackageCoords.Add(archive.OwnedCoordinates(container, ownedFilePath)) + } + } + if !atRoot { + continue + } + // resolve these owned files, as they may have symlinks // but coordinates we will test against are always absolute paths locations, err := resolver.FilesByPath(ownedFilePath) diff --git a/internal/task/unknowns_tasks_test.go b/internal/task/unknowns_tasks_test.go new file mode 100644 index 000000000..12c0c74f7 --- /dev/null +++ b/internal/task/unknowns_tasks_test.go @@ -0,0 +1,120 @@ +package task + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/cataloging" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/sbom" +) + +func Test_unknownsLabeler_archiveReasonsSurvivePackages(t *testing.T) { + jar := file.NewCoordinates("/app.jar", "") + p := pkg.Package{Name: "app", Locations: file.NewLocationSet(file.NewLocationFromCoordinates(jar))} + p.SetID() + s := &sbom.SBOM{Artifacts: sbom.Artifacts{ + Packages: pkg.NewCollection(p), + Unknowns: map[file.Coordinates][]string{jar: { + formatUnknown("archive cataloged from part of its contents: too big", ArchiveCatalogerTaskName), + formatUnknown("unable to read manifest", "java-archive-cataloger"), + }}, + }} + + unknownsLabelerTask{cataloging.UnknownsConfig{RemoveWhenPackagesDefined: true}}.finalize(file.NewMockResolverForPaths(), s) + + assert.Equal(t, []string{formatUnknown("archive cataloged from part of its contents: too big", ArchiveCatalogerTaskName)}, s.Artifacts.Unknowns[jar], + "a package describing the archive explains a parse failure, not contents left out") +} + +func Test_unknownsLabeler_archiveContentsAreReportedByFullPath(t *testing.T) { + // a package describing an archive does not account for the files within it + war := file.NewCoordinates("/app.war", "layer") + in := func(fullPath string) file.Coordinates { + return file.Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: fullPath} + } + nativeLib := in("/app.war!/WEB-INF/lib/jna.jar!/linux-x86-64/libjnidispatch.so") + unidentifiedJar := in("/app.war!/WEB-INF/lib/javax.inject-1.jar") + p := pkg.Package{Name: "app", Locations: file.NewLocationSet(file.NewLocationFromCoordinates(war))} + p.SetID() + noPackage := formatUnknown("no package identified in archive", "java-archive-cataloger") + + s := &sbom.SBOM{ + Artifacts: sbom.Artifacts{ + Packages: pkg.NewCollection(p), + Executables: map[file.Coordinates]file.Executable{nativeLib: {}}, + Unknowns: map[file.Coordinates][]string{unidentifiedJar: {noPackage}}, + }, + Relationships: []artifact.Relationship{{From: war, To: unidentifiedJar, Type: artifact.ContainsRelationship}}, + } + + unknownsLabelerTask{cataloging.UnknownsConfig{RemoveWhenPackagesDefined: true, IncludeExecutablesWithoutPackages: true}}.finalize(file.NewMockResolverForPaths(), s) + + assert.Equal(t, map[file.Coordinates][]string{ + nativeLib: {formatUnknown("no package identified in executable file", unknownsLabelerTaskName)}, + unidentifiedJar: {noPackage}, + }, s.Artifacts.Unknowns) +} + +func Test_unknownsLabeler_relativeOwnedPathsNameNothingInAnArchive(t *testing.T) { + // a python RECORD lists paths relative to site-packages, which resolve nowhere at the scan root either + whl := func(fullPath string) file.Coordinates { + return file.Coordinates{RealPath: "/pip.whl", FullPath: fullPath} + } + tool := whl("/pip.whl!/pip/tool") + p := pkg.Package{ + Name: "pip", + Locations: file.NewLocationSet(file.NewLocationFromCoordinates(whl("/pip.whl!/pip.dist-info/METADATA"))), + Metadata: pkg.PythonPackage{Files: []pkg.PythonFileRecord{{Path: "pip/tool"}}}, + } + p.SetID() + s := &sbom.SBOM{Artifacts: sbom.Artifacts{ + Packages: pkg.NewCollection(p), + Executables: map[file.Coordinates]file.Executable{tool: {}}, + }} + + unknownsLabelerTask{cataloging.UnknownsConfig{IncludeExecutablesWithoutPackages: true}}.finalize(file.NewMockResolverForPaths(), s) + + assert.NotEmpty(t, s.Artifacts.Unknowns[tool]) +} + +func Test_unknownsLabeler_ownedFilesAreWithinTheOwnersContainer(t *testing.T) { + atRoot := file.NewCoordinates("/usr/bin/foo", "") + inArchive := file.Coordinates{RealPath: "/rootfs.tar", FullPath: "/rootfs.tar!/usr/bin/foo"} + owner := func(at file.Coordinates) pkg.Package { + p := pkg.Package{ + Name: "owner", + Type: pkg.RpmPkg, + Locations: file.NewLocationSet(file.NewLocationFromCoordinates(at)), + Metadata: pkg.RpmDBEntry{Files: []pkg.RpmFileRecord{{Path: "/usr/bin/foo"}}}, + } + p.SetID() + return p + } + + tests := []struct { + name string + owner file.Coordinates + referenced file.Coordinates + }{ + {name: "an owner at the scan root", owner: file.NewCoordinates("/var/lib/rpm/Packages", ""), referenced: atRoot}, + {name: "an owner inside an archive", owner: file.Coordinates{RealPath: "/rootfs.tar", FullPath: "/rootfs.tar!/var/lib/rpm/Packages"}, referenced: inArchive}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &sbom.SBOM{Artifacts: sbom.Artifacts{ + Packages: pkg.NewCollection(owner(tt.owner)), + Executables: map[file.Coordinates]file.Executable{atRoot: {}, inArchive: {}}, + }} + + unknownsLabelerTask{cataloging.UnknownsConfig{IncludeExecutablesWithoutPackages: true}}.finalize(file.NewMockResolverForPaths("/usr/bin/foo"), s) + + for _, exe := range []file.Coordinates{atRoot, inArchive} { + assert.Equal(t, exe != tt.referenced, len(s.Artifacts.Unknowns[exe]) > 0, exe.FullNestedPath()) + } + }) + } +} diff --git a/internal/testutils/archives.go b/internal/testutils/archives.go new file mode 100644 index 000000000..df6524ae2 --- /dev/null +++ b/internal/testutils/archives.go @@ -0,0 +1,84 @@ +package testutils + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "fmt" + "io" + "sort" + "testing" + + "github.com/stretchr/testify/require" +) + +// Zip builds a zip of the entries in sorted name order: archive limits bite as entries are read, so +// order decides which land before a truncation. +func Zip[T ~string | ~[]byte](t testing.TB, entries map[string]T) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, name := range sortedNames(entries) { + w, err := zw.Create(name) + require.NoError(t, err) + _, err = w.Write([]byte(entries[name])) + require.NoError(t, err) + } + require.NoError(t, zw.Close()) + return buf.Bytes() +} + +// Tar builds a tar of the entries in sorted name order. +func Tar[T ~string | ~[]byte](t testing.TB, entries map[string]T) []byte { + t.Helper() + var buf bytes.Buffer + writeTar(t, &buf, entries) + return buf.Bytes() +} + +// TarGz builds a gzipped tar of the entries in sorted name order. +func TarGz[T ~string | ~[]byte](t testing.TB, entries map[string]T) []byte { + t.Helper() + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + writeTar(t, gw, entries) + require.NoError(t, gw.Close()) + return buf.Bytes() +} + +// Jar is the smallest jar the java cataloger identifies, a manifest naming the artifact, plus extra. +func Jar(t testing.TB, title, version string, extra map[string][]byte) []byte { + t.Helper() + entries := map[string][]byte{"META-INF/MANIFEST.MF": Manifest(title, version)} + for name, body := range extra { + entries[name] = body + } + return Zip(t, entries) +} + +// Manifest is a jar manifest naming the artifact. +func Manifest(title, version string) []byte { + return fmt.Appendf(nil, "Manifest-Version: 1.0\nImplementation-Title: %s\nImplementation-Version: %s\n", title, version) +} + +func writeTar[T ~string | ~[]byte](t testing.TB, w io.Writer, entries map[string]T) { + t.Helper() + tw := tar.NewWriter(w) + for _, name := range sortedNames(entries) { + body := []byte(entries[name]) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg})) + _, err := tw.Write(body) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) +} + +func sortedNames[T any](entries map[string]T) []string { + names := make([]string, 0, len(entries)) + for name := range entries { + names = append(names, name) + } + sort.Strings(names) + return names +} diff --git a/internal/unknown/coordinate_error.go b/internal/unknown/coordinate_error.go index 2db6caf7e..3b8824ee5 100644 --- a/internal/unknown/coordinate_error.go +++ b/internal/unknown/coordinate_error.go @@ -23,9 +23,9 @@ var _ error = (*CoordinateError)(nil) func (u *CoordinateError) Error() string { if u.Coordinates.FileSystemID == "" { - return fmt.Sprintf("%s: %v", u.Coordinates.RealPath, u.Reason) + return fmt.Sprintf("%s: %v", u.Coordinates.FullNestedPath(), u.Reason) } - return fmt.Sprintf("%s (%s): %v", u.Coordinates.RealPath, u.Coordinates.FileSystemID, u.Reason) + return fmt.Sprintf("%s (%s): %v", u.Coordinates.FullNestedPath(), u.Coordinates.FileSystemID, u.Reason) } // New returns a new CoordinateError unless the reason is a CoordinateError itself, in which case diff --git a/internal/unknown/coordinate_error_test.go b/internal/unknown/coordinate_error_test.go index e863e6359..53adb14fe 100644 --- a/internal/unknown/coordinate_error_test.go +++ b/internal/unknown/coordinate_error_test.go @@ -169,9 +169,7 @@ func Test_Join(t *testing.T) { } func Test_flatten(t *testing.T) { - coords := file.Coordinates{ - RealPath: "real/path", - } + coords := file.NewCoordinates("real/path", "") e1 := fmt.Errorf("e1") e2 := fmt.Errorf("e2") c1 := New(coords, fmt.Errorf("c1")) diff --git a/internal/unknown/path_error.go b/internal/unknown/path_error.go index dce059563..05b8cf085 100644 --- a/internal/unknown/path_error.go +++ b/internal/unknown/path_error.go @@ -19,6 +19,8 @@ func ProcessPathErrors(err error) error { if pathErrorRegex.MatchString(errText) { foundPath := pathErrorRegex.ReplaceAllString(err.Error(), "$1") if foundPath != "" { + // a path in an error text is taken as is: nesting is never inferred from "!/" in it, since a real + // path can hold one too. A caller with the location in hand should use New instead return New(file.NewLocation(foundPath), err) } } diff --git a/internal/unknown/path_error_test.go b/internal/unknown/path_error_test.go index d25133a7f..c6bbe688a 100644 --- a/internal/unknown/path_error_test.go +++ b/internal/unknown/path_error_test.go @@ -17,28 +17,29 @@ func Test_ProcessPathErrors(t *testing.T) { { errorText: `prefix path="/var/lib/thing" suffix`, expected: &CoordinateError{ - Coordinates: file.Coordinates{ - RealPath: "/var/lib/thing", - }, - Reason: fmt.Errorf(`prefix path="/var/lib/thing" suffix`), + Coordinates: file.NewCoordinates("/var/lib/thing", ""), + Reason: fmt.Errorf(`prefix path="/var/lib/thing" suffix`), }, }, { errorText: `prefix path="/var/lib/thing"`, expected: &CoordinateError{ - Coordinates: file.Coordinates{ - RealPath: "/var/lib/thing", - }, - Reason: fmt.Errorf(`prefix path="/var/lib/thing"`), + Coordinates: file.NewCoordinates("/var/lib/thing", ""), + Reason: fmt.Errorf(`prefix path="/var/lib/thing"`), }, }, { errorText: `path="/var/lib/thing" suffix`, expected: &CoordinateError{ - Coordinates: file.Coordinates{ - RealPath: "/var/lib/thing", - }, - Reason: fmt.Errorf(`path="/var/lib/thing" suffix`), + Coordinates: file.NewCoordinates("/var/lib/thing", ""), + Reason: fmt.Errorf(`path="/var/lib/thing" suffix`), + }, + }, + { + errorText: `path="/data/Hello!/world.txt"`, + expected: &CoordinateError{ + Coordinates: file.NewCoordinates("/data/Hello!/world.txt", ""), + Reason: fmt.Errorf(`path="/data/Hello!/world.txt"`), }, }, { diff --git a/schema/json/schema-16.1.12.json b/schema/json/schema-16.1.12.json new file mode 100644 index 000000000..7428b8d31 --- /dev/null +++ b/schema/json/schema-16.1.12.json @@ -0,0 +1,4655 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "anchore.io/schema/syft/json/16.1.12/document", + "$ref": "#/$defs/Document", + "$defs": { + "AlpmDbEntry": { + "properties": { + "basepackage": { + "type": "string", + "description": "BasePackage is the base package name this package was built from (source package in Arch build system)" + }, + "package": { + "type": "string", + "description": "Package is the package name as found in the desc file" + }, + "version": { + "type": "string", + "description": "Version is the package version as found in the desc file" + }, + "description": { + "type": "string", + "description": "Description is a human-readable package description" + }, + "architecture": { + "type": "string", + "description": "Architecture is the target CPU architecture as defined in Arch architecture spec (e.g. x86_64, aarch64, or \"any\" for arch-independent packages)" + }, + "size": { + "type": "integer", + "description": "Size is the installed size in bytes" + }, + "packager": { + "type": "string", + "description": "Packager is the name and email of the person who packaged this (RFC822 format)" + }, + "url": { + "type": "string", + "description": "URL is the upstream project URL" + }, + "validation": { + "type": "string", + "description": "Validation is the validation method used for package integrity (e.g. pgp signature, sha256 checksum)" + }, + "reason": { + "type": "integer", + "description": "Reason is the installation reason tracked by pacman (0=explicitly installed by user, 1=installed as dependency)" + }, + "files": { + "items": { + "$ref": "#/$defs/AlpmFileRecord" + }, + "type": "array", + "description": "Files are the files installed by this package" + }, + "backup": { + "items": { + "$ref": "#/$defs/AlpmFileRecord" + }, + "type": "array", + "description": "Backup is the list of configuration files that pacman backs up before upgrades" + }, + "provides": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Provides are virtual packages provided by this package (allows other packages to depend on capabilities rather than specific packages)" + }, + "depends": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Depends are the runtime dependencies required by this package" + } + }, + "type": "object", + "required": [ + "basepackage", + "package", + "version", + "description", + "architecture", + "size", + "packager", + "url", + "validation", + "reason", + "files", + "backup" + ], + "description": "AlpmDBEntry is a struct that represents the package data stored in the pacman flat-file stores for arch linux." + }, + "AlpmFileRecord": { + "properties": { + "path": { + "type": "string", + "description": "Path is the file path relative to the filesystem root" + }, + "type": { + "type": "string", + "description": "Type is the file type (e.g. regular file, directory, symlink)" + }, + "uid": { + "type": "string", + "description": "UID is the file owner user ID as recorded by pacman" + }, + "gid": { + "type": "string", + "description": "GID is the file owner group ID as recorded by pacman" + }, + "time": { + "type": "string", + "format": "date-time", + "description": "Time is the file modification timestamp" + }, + "size": { + "type": "string", + "description": "Size is the file size in bytes" + }, + "link": { + "type": "string", + "description": "Link is the symlink target path if this is a symlink" + }, + "digest": { + "items": { + "$ref": "#/$defs/Digest" + }, + "type": "array", + "description": "Digests contains file content hashes for integrity verification" + } + }, + "type": "object", + "description": "AlpmFileRecord represents a single file entry within an Arch Linux package with its associated metadata tracked by pacman." + }, + "ApkDbEntry": { + "properties": { + "package": { + "type": "string", + "description": "Package is the package name as found in the installed file" + }, + "originPackage": { + "type": "string", + "description": "OriginPackage is the original source package name this binary was built from (used to track which aport/source built this)" + }, + "maintainer": { + "type": "string", + "description": "Maintainer is the package maintainer name and email" + }, + "version": { + "type": "string", + "description": "Version is the package version as found in the installed file" + }, + "architecture": { + "type": "string", + "description": "Architecture is the target CPU architecture" + }, + "url": { + "type": "string", + "description": "URL is the upstream project URL" + }, + "description": { + "type": "string", + "description": "Description is a human-readable package description" + }, + "size": { + "type": "integer", + "description": "Size is the package archive size in bytes (.apk file size)" + }, + "installedSize": { + "type": "integer", + "description": "InstalledSize is the total size of installed files in bytes" + }, + "pullDependencies": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Dependencies are the runtime dependencies required by this package" + }, + "provides": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Provides are virtual packages provided by this package (for capability-based dependencies)" + }, + "pullChecksum": { + "type": "string", + "description": "Checksum is the package content checksum for integrity verification" + }, + "gitCommitOfApkPort": { + "type": "string", + "description": "GitCommit is the git commit hash of the APK port definition in Alpine's aports repository" + }, + "files": { + "items": { + "$ref": "#/$defs/ApkFileRecord" + }, + "type": "array", + "description": "Files are the files installed by this package" + } + }, + "type": "object", + "required": [ + "package", + "originPackage", + "maintainer", + "version", + "architecture", + "url", + "description", + "size", + "installedSize", + "pullDependencies", + "provides", + "pullChecksum", + "gitCommitOfApkPort", + "files" + ], + "description": "ApkDBEntry represents all captured data for the alpine linux package manager flat-file store." + }, + "ApkFileRecord": { + "properties": { + "path": { + "type": "string", + "description": "Path is the file path relative to the filesystem root" + }, + "ownerUid": { + "type": "string", + "description": "OwnerUID is the file owner user ID" + }, + "ownerGid": { + "type": "string", + "description": "OwnerGID is the file owner group ID" + }, + "permissions": { + "type": "string", + "description": "Permissions is the file permission mode string (e.g. \"0755\", \"0644\")" + }, + "digest": { + "$ref": "#/$defs/Digest", + "description": "Digest is the file content hash for integrity verification" + } + }, + "type": "object", + "required": [ + "path" + ], + "description": "ApkFileRecord represents a single file listing and metadata from a APK DB entry (which may have many of these file records)." + }, + "AppleAppBundleEntry": { + "properties": { + "bundleIdentifier": { + "type": "string", + "description": "BundleIdentifier is the unique identifier for the bundle (e.g. \"com.apple.Safari\")" + }, + "name": { + "type": "string", + "description": "Name is the short name of the bundle" + }, + "displayName": { + "type": "string", + "description": "DisplayName is the user-facing name of the bundle" + }, + "executable": { + "type": "string", + "description": "Executable is the name of the executable within the bundle" + }, + "shortVersion": { + "type": "string", + "description": "ShortVersion is the release (marketing) version of the bundle" + }, + "version": { + "type": "string", + "description": "Version is the build version of the bundle, which often differs from the short version" + }, + "packageType": { + "type": "string", + "description": "PackageType is the four-letter type code (e.g. \"APPL\" for apps, \"FMWK\" for frameworks, \"BNDL\" for bundles)" + }, + "supportedPlatforms": { + "items": { + "type": "string" + }, + "type": "array", + "description": "SupportedPlatforms lists the platforms the bundle targets (e.g. \"MacOSX\", \"iPhoneOS\")" + }, + "minimumSystemVersion": { + "type": "string", + "description": "MinimumSystemVersion is the minimum macOS version required to run the bundle" + }, + "minimumOSVersion": { + "type": "string", + "description": "MinimumOSVersion is the minimum OS version required for non-macOS platforms (e.g. iOS)" + }, + "copyright": { + "type": "string", + "description": "Copyright is the human-readable copyright notice for the bundle" + }, + "platformName": { + "type": "string", + "description": "PlatformName is the platform name of the SDK used to build the bundle (e.g. \"macosx\")" + }, + "sdkName": { + "type": "string", + "description": "SDKName is the name of the SDK used to build the bundle (e.g. \"macosx14.0\")" + } + }, + "type": "object", + "description": "AppleAppBundleEntry represents metadata about an Apple application bundle (CFBundle) extracted from Info.plist files." + }, + "BinarySignature": { + "properties": { + "matches": { + "items": { + "$ref": "#/$defs/ClassifierMatch" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "matches" + ], + "description": "BinarySignature represents a set of matched values within a binary file." + }, + "BitnamiSbomEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is the package name as found in the Bitnami SPDX file" + }, + "arch": { + "type": "string", + "description": "Architecture is the target CPU architecture (amd64 or arm64 in Bitnami images)" + }, + "distro": { + "type": "string", + "description": "Distro is the distribution name this package is for (base OS like debian, ubuntu, etc.)" + }, + "revision": { + "type": "string", + "description": "Revision is the Bitnami-specific package revision number (incremented for Bitnami rebuilds of same upstream version)" + }, + "version": { + "type": "string", + "description": "Version is the package version as found in the Bitnami SPDX file" + }, + "path": { + "type": "string", + "description": "Path is the installation path in the filesystem where the package is located" + }, + "files": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Files are the file paths owned by this package (tracked via SPDX relationships)" + } + }, + "type": "object", + "required": [ + "name", + "arch", + "distro", + "revision", + "version", + "path", + "files" + ], + "description": "BitnamiSBOMEntry represents all captured data from Bitnami packages described in Bitnami' SPDX files." + }, + "CConanFileEntry": { + "properties": { + "ref": { + "type": "string", + "description": "Ref is the package reference string in format name/version@user/channel" + } + }, + "type": "object", + "required": [ + "ref" + ], + "description": "ConanfileEntry represents a single \"Requires\" entry from a conanfile.txt." + }, + "CConanInfoEntry": { + "properties": { + "ref": { + "type": "string", + "description": "Ref is the package reference string in format name/version@user/channel" + }, + "package_id": { + "type": "string", + "description": "PackageID is a unique package variant identifier" + } + }, + "type": "object", + "required": [ + "ref" + ], + "description": "ConaninfoEntry represents a single \"full_requires\" entry from a conaninfo.txt." + }, + "CConanLockEntry": { + "properties": { + "ref": { + "type": "string", + "description": "Ref is the package reference string in format name/version@user/channel" + }, + "package_id": { + "type": "string", + "description": "PackageID is a unique package variant identifier computed from settings/options (static hash in Conan 1.x, can have collisions with complex dependency graphs)" + }, + "prev": { + "type": "string", + "description": "Prev is the previous lock entry reference for versioning" + }, + "requires": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Requires are the runtime package dependencies" + }, + "build_requires": { + "items": { + "type": "string" + }, + "type": "array", + "description": "BuildRequires are the build-time dependencies (e.g. cmake, compilers)" + }, + "py_requires": { + "items": { + "type": "string" + }, + "type": "array", + "description": "PythonRequires are the Python dependencies needed for Conan recipes" + }, + "options": { + "$ref": "#/$defs/KeyValues", + "description": "Options are package configuration options as key-value pairs (e.g. shared=True, fPIC=True)" + }, + "path": { + "type": "string", + "description": "Path is the filesystem path to the package in Conan cache" + }, + "context": { + "type": "string", + "description": "Context is the build context information" + } + }, + "type": "object", + "required": [ + "ref" + ], + "description": "ConanV1LockEntry represents a single \"node\" entry from a conan.lock V1 file." + }, + "CConanLockV2Entry": { + "properties": { + "ref": { + "type": "string", + "description": "Ref is the package reference string in format name/version@user/channel" + }, + "packageID": { + "type": "string", + "description": "PackageID is a unique package variant identifier (dynamic in Conan 2.0, more accurate than V1)" + }, + "username": { + "type": "string", + "description": "Username is the Conan user/organization name" + }, + "channel": { + "type": "string", + "description": "Channel is the Conan channel name indicating stability/purpose (e.g. stable, testing, experimental)" + }, + "recipeRevision": { + "type": "string", + "description": "RecipeRevision is a git-like revision hash (RREV) of the recipe" + }, + "packageRevision": { + "type": "string", + "description": "PackageRevision is a git-like revision hash of the built binary package" + }, + "timestamp": { + "type": "string", + "description": "TimeStamp is when this package was built/locked" + } + }, + "type": "object", + "required": [ + "ref" + ], + "description": "ConanV2LockEntry represents a single \"node\" entry from a conan.lock V2 file." + }, + "CPE": { + "properties": { + "cpe": { + "type": "string", + "description": "Value is the CPE string identifier." + }, + "source": { + "type": "string", + "description": "Source is the source where this CPE was obtained or generated from." + } + }, + "type": "object", + "required": [ + "cpe" + ], + "description": "CPE represents a Common Platform Enumeration identifier used for matching packages to known vulnerabilities in security databases." + }, + "ClassifierMatch": { + "properties": { + "classifier": { + "type": "string" + }, + "location": { + "$ref": "#/$defs/Location" + } + }, + "type": "object", + "required": [ + "classifier", + "location" + ], + "description": "ClassifierMatch represents a single matched value within a binary file and the \"class\" name the search pattern represents." + }, + "CocoaPodfileLockEntry": { + "properties": { + "checksum": { + "type": "string", + "description": "Checksum is the SHA-1 hash of the podspec file for integrity verification (generated via `pod ipc spec ... | openssl sha1`), ensuring all team members use the same pod specification version" + } + }, + "type": "object", + "required": [ + "checksum" + ], + "description": "CocoaPodfileLockEntry represents a single entry from the \"Pods\" section of a Podfile.lock file." + }, + "CondaLink": { + "properties": { + "source": { + "type": "string", + "description": "Source is the original path where the package was extracted from cache." + }, + "type": { + "type": "integer", + "description": "Type indicates the link type (1 for hard link, 2 for soft link, 3 for copy)." + } + }, + "type": "object", + "required": [ + "source", + "type" + ], + "description": "CondaLink represents link metadata from a Conda package's link.json file describing package installation source." + }, + "CondaMetadataEntry": { + "properties": { + "arch": { + "type": "string", + "description": "Arch is the target CPU architecture for the package (e.g., \"arm64\", \"x86_64\")." + }, + "name": { + "type": "string", + "description": "Name is the package name as found in the conda-meta JSON file." + }, + "version": { + "type": "string", + "description": "Version is the package version as found in the conda-meta JSON file." + }, + "build": { + "type": "string", + "description": "Build is the build string identifier (e.g., \"h90dfc92_1014\")." + }, + "build_number": { + "type": "integer", + "description": "BuildNumber is the sequential build number for this version." + }, + "channel": { + "type": "string", + "description": "Channel is the Conda channel URL where the package was retrieved from." + }, + "subdir": { + "type": "string", + "description": "Subdir is the subdirectory within the channel (e.g., \"osx-arm64\", \"linux-64\")." + }, + "noarch": { + "type": "string", + "description": "Noarch indicates if the package is platform-independent (e.g., \"python\", \"generic\")." + }, + "license": { + "type": "string", + "description": "License is the package license identifier." + }, + "license_family": { + "type": "string", + "description": "LicenseFamily is the general license category (e.g., \"MIT\", \"Apache\", \"GPL\")." + }, + "md5": { + "type": "string", + "description": "MD5 is the MD5 hash of the package archive." + }, + "sha256": { + "type": "string", + "description": "SHA256 is the SHA-256 hash of the package archive." + }, + "size": { + "type": "integer", + "description": "Size is the package archive size in bytes." + }, + "timestamp": { + "type": "integer", + "description": "Timestamp is the Unix timestamp when the package was built." + }, + "fn": { + "type": "string", + "description": "Filename is the original package archive filename (e.g., \"zlib-1.2.11-h90dfc92_1014.tar.bz2\")." + }, + "url": { + "type": "string", + "description": "URL is the full download URL for the package archive." + }, + "extracted_package_dir": { + "type": "string", + "description": "ExtractedPackageDir is the local cache directory where the package was extracted." + }, + "depends": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Depends is the list of runtime dependencies with version constraints." + }, + "files": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Files is the list of files installed by this package." + }, + "paths_data": { + "$ref": "#/$defs/CondaPathsData", + "description": "PathsData contains detailed file metadata from the paths.json file." + }, + "link": { + "$ref": "#/$defs/CondaLink", + "description": "Link contains installation source metadata from the link.json file." + } + }, + "type": "object", + "required": [ + "name", + "version", + "build", + "build_number" + ], + "description": "CondaMetaPackage represents metadata for a Conda package extracted from the conda-meta/*.json files." + }, + "CondaPathData": { + "properties": { + "_path": { + "type": "string", + "description": "Path is the file path relative to the Conda environment root." + }, + "path_type": { + "type": "string", + "description": "PathType indicates the link type for the file (e.g., \"hardlink\", \"softlink\", \"directory\")." + }, + "sha256": { + "type": "string", + "description": "SHA256 is the SHA-256 hash of the file contents." + }, + "sha256_in_prefix": { + "type": "string", + "description": "SHA256InPrefix is the SHA-256 hash of the file after prefix replacement during installation." + }, + "size_in_bytes": { + "type": "integer", + "description": "SizeInBytes is the file size in bytes." + } + }, + "type": "object", + "required": [ + "_path", + "path_type", + "sha256", + "sha256_in_prefix", + "size_in_bytes" + ], + "description": "CondaPathData represents metadata for a single file within a Conda package from the paths.json file." + }, + "CondaPathsData": { + "properties": { + "paths_version": { + "type": "integer", + "description": "PathsVersion is the schema version of the paths data format." + }, + "paths": { + "items": { + "$ref": "#/$defs/CondaPathData" + }, + "type": "array", + "description": "Paths is the list of file metadata entries for all files in the package." + } + }, + "type": "object", + "required": [ + "paths_version", + "paths" + ], + "description": "CondaPathsData represents the paths.json file structure from a Conda package containing file metadata." + }, + "Coordinates": { + "properties": { + "path": { + "type": "string", + "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem." + }, + "layerID": { + "type": "string", + "description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in." + }, + "fullPath": { + "type": "string", + "description": "FullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it is blank, so coordinates outside archives are unchanged by the field (in JSON, equality and ID)." + } + }, + "type": "object", + "required": [ + "path" + ], + "description": "Coordinates contains the minimal information needed to describe how to find a file within any possible source object (e.g." + }, + "CpanDistribution": { + "properties": { + "dist": { + "type": "string", + "description": "Dist is the distvname install.json recorded (e.g. URI-5.35), the raw value the package name and\nversion were recovered from" + }, + "mainModule": { + "type": "string", + "description": "MainModule is the module name recovered from the .packlist path (e.g. LWP for libwww-perl).\nEmpty when the name came from install.json, which is authoritative. Set, it means the name is the\ninstaller's NAME and may not be the distribution name." + }, + "author": { + "type": "string", + "description": "Author is the PAUSE ID of the distribution author (e.g. OALDERS), parsed out of Path" + }, + "path": { + "type": "string", + "description": "Path is the raw PAUSE path of the release archive (e.g. O/OA/OALDERS/URI-5.35.tar.gz)" + }, + "modules": { + "items": { + "$ref": "#/$defs/CpanModule" + }, + "type": "array", + "description": "Modules are the modules this distribution provides, sorted by name" + }, + "files": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Files are the paths the .packlist recorded as installed by this distribution, kept in the\npacklist's own order and unfiltered: a path the packlist claims and the filesystem lacks means the\nfile was removed or overwritten out from under the installer, which is signal rather than noise" + } + }, + "type": "object", + "description": "CpanDistribution describes a CPAN distribution installed on disk, as recorded by a CPAN client's install.json and by the .packlist an installer wrote." + }, + "CpanModule": { + "properties": { + "name": { + "type": "string", + "description": "Name is the module name as it would be used in a perl `use` statement (e.g. URI::Escape)" + }, + "version": { + "type": "string", + "description": "Version is the module version, which may differ from the distribution version" + } + }, + "type": "object", + "required": [ + "name" + ], + "description": "CpanModule is a single perl module provided by a CPAN distribution." + }, + "CpanUnpackedRelease": { + "properties": { + "modules": { + "items": { + "$ref": "#/$defs/CpanModule" + }, + "type": "array", + "description": "Modules are the modules the release declares it provides, sorted by name" + } + }, + "type": "object", + "description": "CpanUnpackedRelease describes an unpacked CPAN release found on disk, read from the release's own META.json or META.yml." + }, + "DartPubspec": { + "properties": { + "homepage": { + "type": "string", + "description": "Homepage is the package homepage URL" + }, + "repository": { + "type": "string", + "description": "Repository is the source code repository URL" + }, + "documentation": { + "type": "string", + "description": "Documentation is the documentation site URL" + }, + "publish_to": { + "type": "string", + "description": "PublishTo is the package repository to publish to, or \"none\" to prevent accidental publishing" + }, + "environment": { + "$ref": "#/$defs/DartPubspecEnvironment", + "description": "Environment is SDK version constraints for Dart and Flutter" + }, + "platforms": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Platforms are the supported platforms (Android, iOS, web, etc.)" + }, + "ignored_advisories": { + "items": { + "type": "string" + }, + "type": "array", + "description": "IgnoredAdvisories are the security advisories to explicitly ignore for this package" + } + }, + "type": "object", + "description": "DartPubspec is a struct that represents a package described in a pubspec.yaml file" + }, + "DartPubspecEnvironment": { + "properties": { + "sdk": { + "type": "string", + "description": "SDK is the Dart SDK version constraint (e.g. \"\u003e=2.12.0 \u003c3.0.0\")" + }, + "flutter": { + "type": "string", + "description": "Flutter is the Flutter SDK version constraint if this is a Flutter package" + } + }, + "type": "object", + "description": "DartPubspecEnvironment represents SDK version constraints from the environment section of pubspec.yaml." + }, + "DartPubspecLockEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is the package name as found in the pubspec.lock file" + }, + "version": { + "type": "string", + "description": "Version is the package version as found in the pubspec.lock file" + }, + "hosted_url": { + "type": "string", + "description": "HostedURL is the URL of the package repository for hosted packages (typically pub.dev, but can be custom repository identified by hosted-url). When PUB_HOSTED_URL environment variable changes, lockfile tracks the source." + }, + "vcs_url": { + "type": "string", + "description": "VcsURL is the URL of the VCS repository for git/path dependencies (for packages fetched from version control systems like Git)" + } + }, + "type": "object", + "required": [ + "name", + "version" + ], + "description": "DartPubspecLockEntry is a struct that represents a single entry found in the \"packages\" section in a Dart pubspec.lock file." + }, + "DenoLockEntry": { + "properties": { + "integrity": { + "type": "string", + "description": "Integrity is the crpto hash of the package content for verification" + }, + "dependencies": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Dependencies is the list of package specifiers that this package depends on" + } + }, + "type": "object", + "required": [ + "integrity", + "dependencies" + ], + "description": "DenoLockEntry is a struct that rep a single entry found in the \"packages\" section of a Deno deno.lock file" + }, + "DenoRemoteLockEntry": { + "properties": { + "url": { + "type": "string", + "description": "URL is the remote URL from which the module fetcef" + }, + "integrity": { + "type": "string", + "description": "Integrity is the crpto hash of the package content for verification" + } + }, + "type": "object", + "required": [ + "url", + "integrity" + ], + "description": "DenoRemoteLockEntry is a struct that rep a single entry found in the \"remote\" section of a Deno deno.lock file" + }, + "Descriptor": { + "properties": { + "name": { + "type": "string", + "description": "Name is the name of the tool that generated this SBOM (e.g., \"syft\")." + }, + "version": { + "type": "string", + "description": "Version is the version of the tool that generated this SBOM." + }, + "configuration": { + "description": "Configuration contains the tool configuration used during SBOM generation." + } + }, + "type": "object", + "required": [ + "name", + "version" + ], + "description": "Descriptor identifies the tool that generated this SBOM document, including its name, version, and configuration used during catalog generation." + }, + "Digest": { + "properties": { + "algorithm": { + "type": "string", + "description": "Algorithm specifies the hash algorithm used (e.g., \"sha256\", \"md5\")." + }, + "value": { + "type": "string", + "description": "Value is the hexadecimal string representation of the hash." + } + }, + "type": "object", + "required": [ + "algorithm", + "value" + ], + "description": "Digest represents a cryptographic hash of file contents." + }, + "Document": { + "properties": { + "artifacts": { + "items": { + "$ref": "#/$defs/Package" + }, + "type": "array", + "description": "Artifacts is the list of packages discovered and placed into the catalog" + }, + "artifactRelationships": { + "items": { + "$ref": "#/$defs/Relationship" + }, + "type": "array" + }, + "files": { + "items": { + "$ref": "#/$defs/File" + }, + "type": "array", + "description": "note: must have omitempty" + }, + "source": { + "$ref": "#/$defs/Source", + "description": "Source represents the original object that was cataloged" + }, + "distro": { + "$ref": "#/$defs/LinuxRelease", + "description": "Distro represents the Linux distribution that was detected from the source" + }, + "descriptor": { + "$ref": "#/$defs/Descriptor", + "description": "Descriptor is a block containing self-describing information about syft" + }, + "schema": { + "$ref": "#/$defs/Schema", + "description": "Schema is a block reserved for defining the version for the shape of this JSON document and where to find the schema document to validate the shape" + } + }, + "type": "object", + "required": [ + "artifacts", + "artifactRelationships", + "source", + "distro", + "descriptor", + "schema" + ], + "description": "Document represents the syft cataloging findings as a JSON document" + }, + "DotnetDepsEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is the package name as found in the deps.json file" + }, + "version": { + "type": "string", + "description": "Version is the package version as found in the deps.json file" + }, + "path": { + "type": "string", + "description": "Path is the relative path to the package within the deps structure (e.g. \"app.metrics/3.0.0\")" + }, + "sha512": { + "type": "string", + "description": "Sha512 is the SHA-512 hash of the NuGet package content WITHOUT the signed content for verification (won't match hash from NuGet API or manual calculation of .nupkg file)" + }, + "hashPath": { + "type": "string", + "description": "HashPath is the relative path to the .nupkg.sha512 hash file (e.g. \"app.metrics.3.0.0.nupkg.sha512\")" + }, + "type": { + "type": "string", + "description": "Type is type of entry could be package or project for internal refs" + }, + "executables": { + "additionalProperties": { + "$ref": "#/$defs/DotnetPortableExecutableEntry" + }, + "type": "object", + "description": "Executables are the map of .NET Portable Executable files within this package with their version resources" + } + }, + "type": "object", + "required": [ + "name", + "version", + "path", + "sha512", + "hashPath" + ], + "description": "DotnetDepsEntry is a struct that represents a single entry found in the \"libraries\" section in a .NET [*.]deps.json file." + }, + "DotnetPackagesLockEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is the package name as found in the packages.lock.json file" + }, + "version": { + "type": "string", + "description": "Version is the package version as found in the packages.lock.json file" + }, + "contentHash": { + "type": "string", + "description": "ContentHash is the hash of the package content for verification" + }, + "type": { + "type": "string", + "description": "Type is the dependency type indicating how this dependency was added (Direct=explicit in project file, Transitive=pulled in by another package, Project=project reference)" + } + }, + "type": "object", + "required": [ + "name", + "version", + "contentHash", + "type" + ], + "description": "DotnetPackagesLockEntry is a struct that represents a single entry found in the \"dependencies\" section in a .NET packages.lock.json file." + }, + "DotnetPortableExecutableEntry": { + "properties": { + "assemblyVersion": { + "type": "string", + "description": "AssemblyVersion is the .NET assembly version number (strong-named version)" + }, + "legalCopyright": { + "type": "string", + "description": "LegalCopyright is the copyright notice string" + }, + "comments": { + "type": "string", + "description": "Comments are additional comments or description embedded in PE resources" + }, + "internalName": { + "type": "string", + "description": "InternalName is the internal name of the file" + }, + "companyName": { + "type": "string", + "description": "CompanyName is the company that produced the file" + }, + "productName": { + "type": "string", + "description": "ProductName is the name of the product this file is part of" + }, + "productVersion": { + "type": "string", + "description": "ProductVersion is the version of the product (may differ from AssemblyVersion)" + } + }, + "type": "object", + "required": [ + "assemblyVersion", + "legalCopyright", + "companyName", + "productName", + "productVersion" + ], + "description": "DotnetPortableExecutableEntry is a struct that represents a single entry found within \"VersionResources\" section of a .NET Portable Executable binary file." + }, + "DpkgArchiveEntry": { + "properties": { + "package": { + "type": "string", + "description": "Package is the package name as found in the status file" + }, + "source": { + "type": "string", + "description": "Source is the source package name this binary was built from (one source can produce multiple binary packages)" + }, + "version": { + "type": "string", + "description": "Version is the binary package version as found in the status file" + }, + "sourceVersion": { + "type": "string", + "description": "SourceVersion is the source package version (may differ from binary version when binNMU rebuilds occur)" + }, + "architecture": { + "type": "string", + "description": "Architecture is the target architecture per Debian spec (specific arch like amd64/arm64, wildcard like any, architecture-independent \"all\", or \"source\" for source packages)" + }, + "maintainer": { + "type": "string", + "description": "Maintainer is the package maintainer's name and email in RFC822 format (name must come first, then email in angle brackets)" + }, + "installedSize": { + "type": "integer", + "description": "InstalledSize is the total size of installed files in kilobytes" + }, + "provides": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Provides are the virtual packages provided by this package (allows other packages to depend on capabilities. Can include versioned provides like \"libdigest-md5-perl (= 2.55.01)\")" + }, + "depends": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Depends are the packages required for this package to function (will not be installed unless these requirements are met, creates strict ordering constraint)" + }, + "preDepends": { + "items": { + "type": "string" + }, + "type": "array", + "description": "PreDepends are the packages that must be installed and configured BEFORE even starting installation of this package (stronger than Depends, discouraged unless absolutely necessary as it adds strict constraints for apt)" + }, + "files": { + "items": { + "$ref": "#/$defs/DpkgFileRecord" + }, + "type": "array", + "description": "Files are the files installed by this package" + } + }, + "type": "object", + "required": [ + "package", + "source", + "version", + "sourceVersion", + "architecture", + "maintainer", + "installedSize", + "files" + ], + "description": "DpkgArchiveEntry represents package metadata extracted from a .deb archive file." + }, + "DpkgDbEntry": { + "properties": { + "package": { + "type": "string", + "description": "Package is the package name as found in the status file" + }, + "source": { + "type": "string", + "description": "Source is the source package name this binary was built from (one source can produce multiple binary packages)" + }, + "version": { + "type": "string", + "description": "Version is the binary package version as found in the status file" + }, + "sourceVersion": { + "type": "string", + "description": "SourceVersion is the source package version (may differ from binary version when binNMU rebuilds occur)" + }, + "architecture": { + "type": "string", + "description": "Architecture is the target architecture per Debian spec (specific arch like amd64/arm64, wildcard like any, architecture-independent \"all\", or \"source\" for source packages)" + }, + "maintainer": { + "type": "string", + "description": "Maintainer is the package maintainer's name and email in RFC822 format (name must come first, then email in angle brackets)" + }, + "installedSize": { + "type": "integer", + "description": "InstalledSize is the total size of installed files in kilobytes" + }, + "provides": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Provides are the virtual packages provided by this package (allows other packages to depend on capabilities. Can include versioned provides like \"libdigest-md5-perl (= 2.55.01)\")" + }, + "depends": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Depends are the packages required for this package to function (will not be installed unless these requirements are met, creates strict ordering constraint)" + }, + "preDepends": { + "items": { + "type": "string" + }, + "type": "array", + "description": "PreDepends are the packages that must be installed and configured BEFORE even starting installation of this package (stronger than Depends, discouraged unless absolutely necessary as it adds strict constraints for apt)" + }, + "files": { + "items": { + "$ref": "#/$defs/DpkgFileRecord" + }, + "type": "array", + "description": "Files are the files installed by this package" + } + }, + "type": "object", + "required": [ + "package", + "source", + "version", + "sourceVersion", + "architecture", + "maintainer", + "installedSize", + "files" + ], + "description": "DpkgDBEntry represents all captured data for a Debian package DB entry; available fields are described at http://manpages.ubuntu.com/manpages/xenial/man1/dpkg-query.1.html in the --showformat section." + }, + "DpkgFileRecord": { + "properties": { + "path": { + "type": "string", + "description": "Path is the file path relative to the filesystem root" + }, + "digest": { + "$ref": "#/$defs/Digest", + "description": "Digest is the file content hash (typically MD5 for dpkg compatibility with legacy systems)" + }, + "isConfigFile": { + "type": "boolean", + "description": "IsConfigFile is whether this file is marked as a configuration file (dpkg will preserve user modifications during upgrades)" + } + }, + "type": "object", + "required": [ + "path", + "isConfigFile" + ], + "description": "DpkgFileRecord represents a single file attributed to a debian package." + }, + "ELFSecurityFeatures": { + "properties": { + "symbolTableStripped": { + "type": "boolean", + "description": "SymbolTableStripped indicates whether debugging symbols have been removed." + }, + "stackCanary": { + "type": "boolean", + "description": "StackCanary indicates whether stack smashing protection is enabled." + }, + "nx": { + "type": "boolean", + "description": "NoExecutable indicates whether NX (no-execute) protection is enabled for the stack." + }, + "relRO": { + "type": "string", + "description": "RelocationReadOnly indicates the RELRO protection level." + }, + "pie": { + "type": "boolean", + "description": "PositionIndependentExecutable indicates whether the binary is compiled as PIE." + }, + "dso": { + "type": "boolean", + "description": "DynamicSharedObject indicates whether the binary is a shared library." + }, + "safeStack": { + "type": "boolean", + "description": "LlvmSafeStack represents a compiler-based security mechanism that separates the stack into a safe stack for storing return addresses and other critical data, and an unsafe stack for everything else, to mitigate stack-based memory corruption errors\nsee https://clang.llvm.org/docs/SafeStack.html" + }, + "cfi": { + "type": "boolean", + "description": "ControlFlowIntegrity represents runtime checks to ensure a program's control flow adheres to the legal paths determined at compile time, thus protecting against various types of control-flow hijacking attacks\nsee https://clang.llvm.org/docs/ControlFlowIntegrity.html" + }, + "fortify": { + "type": "boolean", + "description": "ClangFortifySource is a broad suite of extensions to libc aimed at catching misuses of common library functions\nsee https://android.googlesource.com/platform//bionic/+/d192dbecf0b2a371eb127c0871f77a9caf81c4d2/docs/clang_fortify_anatomy.md" + } + }, + "type": "object", + "required": [ + "symbolTableStripped", + "nx", + "relRO", + "pie", + "dso" + ], + "description": "ELFSecurityFeatures captures security hardening and protection mechanisms in ELF binaries." + }, + "ElfBinaryPackageNoteJsonPayload": { + "properties": { + "type": { + "type": "string", + "description": "Type is the type of the package (e.g. \"rpm\", \"deb\", \"apk\", etc.)" + }, + "architecture": { + "type": "string", + "description": "Architecture of the binary package (e.g. \"amd64\", \"arm\", etc.)" + }, + "osCPE": { + "type": "string", + "description": "OSCPE is a CPE name for the OS, typically corresponding to CPE_NAME in os-release (e.g. cpe:/o:fedoraproject:fedora:33)\n\nDeprecated: in Syft 2.0 the struct tag will be corrected to `osCpe` to match the systemd spec casing." + }, + "appCpe": { + "type": "string", + "description": "AppCpe is a CPE name for the upstream Application, as found in NVD CPE search (e.g. cpe:2.3:a:gnu:coreutils:5.0)" + }, + "os": { + "type": "string", + "description": "OS is the OS name, typically corresponding to ID in os-release (e.g. \"fedora\")" + }, + "osVersion": { + "type": "string", + "description": "osVersion is the version of the OS, typically corresponding to VERSION_ID in os-release (e.g. \"33\")" + }, + "system": { + "type": "string", + "description": "System is a context-specific name for the system that the binary package is intended to run on or a part of" + }, + "vendor": { + "type": "string", + "description": "Vendor is the individual or organization that produced the source code for the binary" + }, + "sourceRepo": { + "type": "string", + "description": "SourceRepo is the URL to the source repository for which the binary was built from" + }, + "commit": { + "type": "string", + "description": "Commit is the commit hash of the source repository for which the binary was built from" + } + }, + "type": "object", + "description": "ELFBinaryPackageNoteJSONPayload Represents metadata captured from the .note.package section of an ELF-formatted binary" + }, + "ElixirMixLockEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is the package name as found in the mix.lock file" + }, + "version": { + "type": "string", + "description": "Version is the package version as found in the mix.lock file" + }, + "pkgHash": { + "type": "string", + "description": "PkgHash is the outer checksum (SHA-256) of the entire Hex package tarball for integrity verification (preferred method, replaces deprecated inner checksum)" + }, + "pkgHashExt": { + "type": "string", + "description": "PkgHashExt is the extended package hash format (inner checksum is deprecated - SHA-256 of concatenated file contents excluding CHECKSUM file, now replaced by outer checksum)" + }, + "dependencies": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Dependencies are the names of the packages this entry depends on, as\ndeclared in the entry's dependency list within mix.lock. Used to derive\ndependency-of relationships between locked packages." + } + }, + "type": "object", + "required": [ + "name", + "version", + "pkgHash", + "pkgHashExt" + ], + "description": "ElixirMixLockEntry is a struct that represents a single entry in a mix.lock file" + }, + "ErlangRebarLockEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is the package name as found in the rebar.lock file" + }, + "version": { + "type": "string", + "description": "Version is the package version as found in the rebar.lock file" + }, + "pkgHash": { + "type": "string", + "description": "PkgHash is the outer checksum (SHA-256) of the entire Hex package tarball for integrity verification (preferred method over deprecated inner checksum)" + }, + "pkgHashExt": { + "type": "string", + "description": "PkgHashExt is the extended package hash format (inner checksum deprecated - was SHA-256 of concatenated file contents)" + } + }, + "type": "object", + "required": [ + "name", + "version", + "pkgHash", + "pkgHashExt" + ], + "description": "ErlangRebarLockEntry represents a single package entry from the \"deps\" section within an Erlang rebar.lock file." + }, + "Executable": { + "properties": { + "format": { + "type": "string", + "description": "Format denotes either ELF, Mach-O, or PE" + }, + "hasExports": { + "type": "boolean", + "description": "HasExports indicates whether the binary exports symbols." + }, + "hasEntrypoint": { + "type": "boolean", + "description": "HasEntrypoint indicates whether the binary has an entry point function." + }, + "importedLibraries": { + "items": { + "type": "string" + }, + "type": "array", + "description": "ImportedLibraries lists the shared libraries required by this executable." + }, + "elfSecurityFeatures": { + "$ref": "#/$defs/ELFSecurityFeatures", + "description": "ELFSecurityFeatures contains ELF-specific security hardening information when Format is ELF." + } + }, + "type": "object", + "required": [ + "format", + "hasExports", + "hasEntrypoint", + "importedLibraries" + ], + "description": "Executable contains metadata about binary files and their security features." + }, + "File": { + "properties": { + "id": { + "type": "string", + "description": "ID is a unique identifier for this file within the SBOM." + }, + "location": { + "$ref": "#/$defs/Coordinates", + "description": "Location is the file path and layer information where this file was found." + }, + "metadata": { + "$ref": "#/$defs/FileMetadataEntry", + "description": "Metadata contains filesystem metadata such as permissions, ownership, and file type." + }, + "contents": { + "type": "string", + "description": "Contents is the file contents for small files." + }, + "digests": { + "items": { + "$ref": "#/$defs/Digest" + }, + "type": "array", + "description": "Digests contains cryptographic hashes of the file contents." + }, + "licenses": { + "items": { + "$ref": "#/$defs/FileLicense" + }, + "type": "array", + "description": "Licenses contains license information discovered within this file." + }, + "executable": { + "$ref": "#/$defs/Executable", + "description": "Executable contains executable metadata if this file is a binary." + }, + "unknowns": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Unknowns contains unknown fields for forward compatibility." + } + }, + "type": "object", + "required": [ + "id", + "location" + ], + "description": "File represents a file discovered during cataloging with its metadata, content digests, licenses, and relationships to packages." + }, + "FileLicense": { + "properties": { + "value": { + "type": "string", + "description": "Value is the raw license identifier or text as found in the file." + }, + "spdxExpression": { + "type": "string", + "description": "SPDXExpression is the parsed SPDX license expression." + }, + "type": { + "type": "string", + "description": "Type is the license type classification (e.g., declared, concluded, discovered)." + }, + "evidence": { + "$ref": "#/$defs/FileLicenseEvidence", + "description": "Evidence contains supporting evidence for this license detection." + } + }, + "type": "object", + "required": [ + "value", + "spdxExpression", + "type" + ], + "description": "FileLicense represents license information discovered within a file's contents or metadata, including the matched license text and SPDX expression." + }, + "FileLicenseEvidence": { + "properties": { + "confidence": { + "type": "integer", + "description": "Confidence is the confidence score for this license detection (0-100)." + }, + "offset": { + "type": "integer", + "description": "Offset is the byte offset where the license text starts in the file." + }, + "extent": { + "type": "integer", + "description": "Extent is the length of the license text in bytes." + } + }, + "type": "object", + "required": [ + "confidence", + "offset", + "extent" + ], + "description": "FileLicenseEvidence contains supporting evidence for a license detection in a file, including the byte offset, extent, and confidence level." + }, + "FileMetadataEntry": { + "properties": { + "mode": { + "type": "integer", + "description": "Mode is the Unix file permission mode in octal format." + }, + "type": { + "type": "string", + "description": "Type is the file type (e.g., \"RegularFile\", \"Directory\", \"SymbolicLink\")." + }, + "linkDestination": { + "type": "string", + "description": "LinkDestination is the target path for symbolic links." + }, + "userID": { + "type": "integer", + "description": "UserID is the file owner user ID." + }, + "groupID": { + "type": "integer", + "description": "GroupID is the file owner group ID." + }, + "mimeType": { + "type": "string", + "description": "MIMEType is the MIME type of the file contents." + }, + "size": { + "type": "integer", + "description": "Size is the file size in bytes." + } + }, + "type": "object", + "required": [ + "mode", + "type", + "userID", + "groupID", + "mimeType", + "size" + ], + "description": "FileMetadataEntry contains filesystem-level metadata attributes such as permissions, ownership, type, and size for a cataloged file." + }, + "GgufFileHeader": { + "properties": { + "ggufVersion": { + "type": "integer", + "description": "GGUFVersion is the GGUF format version (e.g., 3)" + }, + "fileSize": { + "type": "integer", + "description": "FileSize is the size of the GGUF file in bytes (best-effort if available from resolver)" + }, + "architecture": { + "type": "string", + "description": "Architecture is the model architecture (from general.architecture, e.g., \"qwen3moe\", \"llama\")" + }, + "quantization": { + "type": "string", + "description": "Quantization is the quantization type (e.g., \"IQ4_NL\", \"Q4_K_M\")" + }, + "parameters": { + "type": "integer", + "description": "Parameters is the number of model parameters (if present in header)" + }, + "tensorCount": { + "type": "integer", + "description": "TensorCount is the number of tensors in the model" + }, + "header": { + "type": "object", + "description": "RemainingKeyValues contains the remaining key-value pairs from the GGUF header that are not already\nrepresented as typed fields above. This preserves additional metadata fields for reference\n(namespaced with general.*, llama.*, etc.) while avoiding duplication." + }, + "metadataHash": { + "type": "string", + "description": "MetadataKeyValuesHash is a xx64 hash of all key-value pairs from the GGUF header metadata.\nThis hash is computed over the complete header metadata (including the fields extracted\ninto typed fields above) and provides a stable identifier for the model configuration\nacross different file locations or remotes. It allows matching identical models even\nwhen stored in different repositories or with different filenames." + }, + "parts": { + "items": { + "$ref": "#/$defs/GgufFileHeader" + }, + "type": "array", + "description": "Parts contains headers from additional GGUF files that were merged\ninto this package during post-processing (e.g., from OCI layers without model names)." + } + }, + "type": "object", + "required": [ + "ggufVersion", + "tensorCount" + ], + "description": "GGUFFileHeader represents metadata extracted from a GGUF (GPT-Generated Unified Format) model file." + }, + "GithubActionsUseStatement": { + "properties": { + "value": { + "type": "string", + "description": "Value is the action reference (e.g. \"actions/checkout@v3\")" + }, + "comment": { + "type": "string", + "description": "Comment is the inline comment associated with this uses statement" + } + }, + "type": "object", + "required": [ + "value" + ], + "description": "GitHubActionsUseStatement represents a single 'uses' statement in a GitHub Actions workflow file referencing an action or reusable workflow." + }, + "GoModuleBuildinfoEntry": { + "properties": { + "goBuildSettings": { + "$ref": "#/$defs/KeyValues", + "description": "BuildSettings contains the Go build settings and flags used to compile the binary (e.g., GOARCH, GOOS, CGO_ENABLED)." + }, + "goCompiledVersion": { + "type": "string", + "description": "GoCompiledVersion is the version of Go used to compile the binary." + }, + "architecture": { + "type": "string", + "description": "Architecture is the target CPU architecture for the binary (extracted from GOARCH build setting)." + }, + "h1Digest": { + "type": "string", + "description": "H1Digest is the Go module hash in h1: format for the main module from go.sum." + }, + "mainModule": { + "type": "string", + "description": "MainModule is the main module path for the binary (e.g., \"github.com/anchore/syft\")." + }, + "goCryptoSettings": { + "items": { + "type": "string" + }, + "type": "array", + "description": "GoCryptoSettings contains FIPS and cryptographic configuration settings if present." + }, + "goExperiments": { + "items": { + "type": "string" + }, + "type": "array", + "description": "GoExperiments lists experimental Go features enabled during compilation (e.g., \"arenas\", \"cgocheck2\")." + }, + "symbols": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": "object", + "description": "Symbols are the function symbols from this module that are compiled into the binary, extracted from\nthe binary symbol table (pclntab) and grouped by the import path of the package that owns them. Each\nvalue is the sorted, deduplicated list of symbol names local to that package, i.e. with the import\npath prefix stripped (e.g. import path \"github.com/foo/bar\" -\u003e \"(*Type).Method\"). The fully qualified\nname is the import path, a \".\", and the local name. One exception: the binary's main package appears\nunder the key \"main\" (the name the linker assigns), not its original source import path, which is not\nrecoverable from the binary. Populated only when the golang cataloger's capture-symbols scope covers\nthis package: the \"all\" scope populates every module package plus the synthetic stdlib package, the\n\"extended-stdlib\" scope populates the stdlib package plus every module under golang.org/x/, and the\n\"stdlib\" scope populates only the stdlib package. The capture-symbols-modules glob patterns populate\nany additional modules they match." + } + }, + "type": "object", + "required": [ + "goCompiledVersion", + "architecture" + ], + "description": "GolangBinaryBuildinfoEntry represents all captured data for a Golang binary" + }, + "GoModuleEntry": { + "properties": { + "h1Digest": { + "type": "string", + "description": "H1Digest is the Go module hash in h1: format from go.sum for verifying module contents." + } + }, + "type": "object", + "description": "GolangModuleEntry represents all captured data for a Golang source scan with go.mod/go.sum" + }, + "GoSourceEntry": { + "properties": { + "h1Digest": { + "type": "string", + "description": "H1Digest is the Go module hash in h1: format from go.sum for verifying module contents." + }, + "os": { + "type": "string", + "description": "OperatingSystem is the target OS for build constraints (e.g., \"linux\", \"darwin\", \"windows\")." + }, + "architecture": { + "type": "string", + "description": "Architecture is the target CPU architecture for build constraints (e.g., \"amd64\", \"arm64\")." + }, + "buildTags": { + "type": "string", + "description": "BuildTags are the build tags used to conditionally compile code (e.g., \"integration,debug\")." + }, + "cgoEnabled": { + "type": "boolean", + "description": "CgoEnabled indicates whether CGO was enabled for this package." + } + }, + "type": "object", + "required": [ + "cgoEnabled" + ], + "description": "GolangSourceEntry represents all captured data for a Golang package found through source analysis" + }, + "HaskellHackageStackEntry": { + "properties": { + "pkgHash": { + "type": "string", + "description": "PkgHash is the package content hash for verification" + } + }, + "type": "object", + "description": "HackageStackYamlEntry represents a single entry from the \"extra-deps\" section of a stack.yaml file." + }, + "HaskellHackageStackLockEntry": { + "properties": { + "pkgHash": { + "type": "string", + "description": "PkgHash is the package content hash for verification" + }, + "snapshotURL": { + "type": "string", + "description": "SnapshotURL is the URL to the Stack snapshot this package came from" + } + }, + "type": "object", + "description": "HackageStackYamlLockEntry represents a single entry from the \"packages\" section of a stack.yaml.lock file." + }, + "HomebrewFormula": { + "properties": { + "tap": { + "type": "string", + "description": "Tap is Homebrew tap this formula belongs to (e.g. \"homebrew/core\")" + }, + "homepage": { + "type": "string", + "description": "Homepage is the upstream project homepage URL" + }, + "description": { + "type": "string", + "description": "Description is a human-readable formula description" + } + }, + "type": "object", + "description": "HomebrewFormula represents metadata about a Homebrew formula package extracted from formula JSON files." + }, + "IDLikes": { + "items": { + "type": "string" + }, + "type": "array", + "description": "IDLikes represents a list of distribution IDs that this Linux distribution is similar to or derived from, as defined in os-release ID_LIKE field." + }, + "JavaArchive": { + "properties": { + "virtualPath": { + "type": "string", + "description": "VirtualPath is path within the archive hierarchy, where nested entries are delimited with ':' (for nested JARs)" + }, + "manifest": { + "$ref": "#/$defs/JavaManifest", + "description": "Manifest is parsed META-INF/MANIFEST.MF contents" + }, + "pomProperties": { + "$ref": "#/$defs/JavaPomProperties", + "description": "PomProperties is parsed pom.properties file contents" + }, + "pomProject": { + "$ref": "#/$defs/JavaPomProject", + "description": "PomProject is parsed pom.xml file contents" + }, + "digest": { + "items": { + "$ref": "#/$defs/Digest" + }, + "type": "array", + "description": "ArchiveDigests is cryptographic hashes of the archive file" + } + }, + "type": "object", + "required": [ + "virtualPath" + ], + "description": "JavaArchive encapsulates all Java ecosystem metadata for a package as well as an (optional) parent relationship." + }, + "JavaJvmInstallation": { + "properties": { + "release": { + "$ref": "#/$defs/JavaVMRelease", + "description": "Release is JVM release information and version details" + }, + "files": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Files are the list of files that are part of this JVM installation" + } + }, + "type": "object", + "required": [ + "release", + "files" + ], + "description": "JavaVMInstallation represents a Java Virtual Machine installation discovered on the system with its release information and file list." + }, + "JavaManifest": { + "properties": { + "main": { + "$ref": "#/$defs/KeyValues", + "description": "Main is main manifest attributes as key-value pairs" + }, + "sections": { + "items": { + "$ref": "#/$defs/KeyValues" + }, + "type": "array", + "description": "Sections are the named sections from the manifest (e.g. per-entry attributes)" + } + }, + "type": "object", + "description": "JavaManifest represents the fields of interest extracted from a Java archive's META-INF/MANIFEST.MF file." + }, + "JavaPomParent": { + "properties": { + "groupId": { + "type": "string", + "description": "GroupID is the parent Maven group identifier" + }, + "artifactId": { + "type": "string", + "description": "ArtifactID is the parent Maven artifact identifier" + }, + "version": { + "type": "string", + "description": "Version is the parent version (child inherits configuration from this specific version of parent POM)" + } + }, + "type": "object", + "required": [ + "groupId", + "artifactId", + "version" + ], + "description": "JavaPomParent contains the fields within the \u003cparent\u003e tag in a pom.xml file" + }, + "JavaPomProject": { + "properties": { + "path": { + "type": "string", + "description": "Path is path to the pom.xml file within the archive" + }, + "parent": { + "$ref": "#/$defs/JavaPomParent", + "description": "Parent is the parent POM reference for inheritance (child POMs inherit configuration from parent)" + }, + "groupId": { + "type": "string", + "description": "GroupID is Maven group identifier (reversed domain name like org.apache.maven)" + }, + "artifactId": { + "type": "string", + "description": "ArtifactID is Maven artifact identifier (project name)" + }, + "version": { + "type": "string", + "description": "Version is project version (together with groupId and artifactId forms Maven coordinates groupId:artifactId:version)" + }, + "name": { + "type": "string", + "description": "Name is a human-readable project name (displayed in Maven-generated documentation)" + }, + "description": { + "type": "string", + "description": "Description is detailed project description" + }, + "url": { + "type": "string", + "description": "URL is the project URL (typically project website or repository)" + } + }, + "type": "object", + "required": [ + "path", + "groupId", + "artifactId", + "version", + "name" + ], + "description": "JavaPomProject represents fields of interest extracted from a Java archive's pom.xml file." + }, + "JavaPomProperties": { + "properties": { + "path": { + "type": "string", + "description": "Path is path to the pom.properties file within the archive" + }, + "name": { + "type": "string", + "description": "Name is the project name" + }, + "groupId": { + "type": "string", + "description": "GroupID is Maven group identifier uniquely identifying the project across all projects (follows reversed domain name convention like com.company.project)" + }, + "artifactId": { + "type": "string", + "description": "ArtifactID is Maven artifact identifier, the name of the jar/artifact (unique within the groupId scope)" + }, + "version": { + "type": "string", + "description": "Version is artifact version" + }, + "scope": { + "type": "string", + "description": "Scope is dependency scope determining when dependency is available (compile=default all phases, test=test compilation/execution only, runtime=runtime and test not compile, provided=expected from JDK or container)" + }, + "extraFields": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Extra is additional custom properties not in standard Maven coordinates" + } + }, + "type": "object", + "required": [ + "path", + "name", + "groupId", + "artifactId", + "version" + ], + "description": "JavaPomProperties represents the fields of interest extracted from a Java archive's pom.properties file." + }, + "JavaVMRelease": { + "properties": { + "implementor": { + "type": "string", + "description": "Implementor is extracted with the `java.vendor` JVM property" + }, + "implementorVersion": { + "type": "string", + "description": "ImplementorVersion is extracted with the `java.vendor.version` JVM property" + }, + "javaRuntimeVersion": { + "type": "string", + "description": "JavaRuntimeVersion is extracted from the 'java.runtime.version' JVM property" + }, + "javaVersion": { + "type": "string", + "description": "JavaVersion matches that from `java -version` command output" + }, + "javaVersionDate": { + "type": "string", + "description": "JavaVersionDate is extracted from the 'java.version.date' JVM property" + }, + "libc": { + "type": "string", + "description": "Libc can either be 'glibc' or 'musl'" + }, + "modules": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Modules is a list of JVM modules that are packaged" + }, + "osArch": { + "type": "string", + "description": "OsArch is the target CPU architecture" + }, + "osName": { + "type": "string", + "description": "OsName is the name of the target runtime operating system environment" + }, + "osVersion": { + "type": "string", + "description": "OsVersion is the version of the target runtime operating system environment" + }, + "source": { + "type": "string", + "description": "Source refers to the origin repository of OpenJDK source" + }, + "buildSource": { + "type": "string", + "description": "BuildSource Git SHA of the build repository" + }, + "buildSourceRepo": { + "type": "string", + "description": "BuildSourceRepo refers to rhe repository URL for the build source" + }, + "sourceRepo": { + "type": "string", + "description": "SourceRepo refers to the OpenJDK repository URL" + }, + "fullVersion": { + "type": "string", + "description": "FullVersion is extracted from the 'java.runtime.version' JVM property" + }, + "semanticVersion": { + "type": "string", + "description": "SemanticVersion is derived from the OpenJDK version" + }, + "buildInfo": { + "type": "string", + "description": "BuildInfo contains additional build information" + }, + "jvmVariant": { + "type": "string", + "description": "JvmVariant specifies the JVM variant (e.g., Hotspot or OpenJ9)" + }, + "jvmVersion": { + "type": "string", + "description": "JvmVersion is extracted from the 'java.vm.version' JVM property" + }, + "imageType": { + "type": "string", + "description": "ImageType can be 'JDK' or 'JRE'" + }, + "buildType": { + "type": "string", + "description": "BuildType can be 'commercial' (used in some older oracle JDK distributions)" + } + }, + "type": "object", + "description": "JavaVMRelease represents JVM version and build information extracted from the release file in a Java installation." + }, + "JavascriptBunLockEntry": { + "properties": { + "integrity": { + "type": "string", + "description": "Integrity is Subresource Integrity hash for verification (SRI format)" + }, + "dependencies": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Dependencies is a map of runtime dependencies and their version specifiers" + }, + "optionalDependencies": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "OptionalDependencies is a map of optional dependencies and their version specifiers" + }, + "peerDependencies": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "PeerDependencies is a map of peer dependencies and their version specifiers" + }, + "bin": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Bin is a map of binary names to the paths they are installed to" + }, + "os": { + "type": "string", + "description": "OS is the operating system constraint for the package (e.g. \"darwin\")" + }, + "cpu": { + "type": "string", + "description": "CPU is the CPU architecture constraint for the package (e.g. \"arm64\")" + } + }, + "type": "object", + "required": [ + "integrity", + "dependencies", + "optionalDependencies", + "peerDependencies", + "bin", + "os", + "cpu" + ], + "description": "BunLockEntry represents a single entry in the \"packages\" section of a bun.lock file" + }, + "JavascriptNpmPackage": { + "properties": { + "name": { + "type": "string", + "description": "Name is the package name as found in package.json" + }, + "version": { + "type": "string", + "description": "Version is the package version as found in package.json" + }, + "author": { + "type": "string", + "description": "Author is package author name" + }, + "homepage": { + "type": "string", + "description": "Homepage is project homepage URL" + }, + "description": { + "type": "string", + "description": "Description is a human-readable package description" + }, + "url": { + "type": "string", + "description": "URL is repository or project URL" + }, + "private": { + "type": "boolean", + "description": "Private is whether this is a private package" + } + }, + "type": "object", + "required": [ + "name", + "version", + "author", + "homepage", + "description", + "url", + "private" + ], + "description": "NpmPackage represents the contents of a javascript package.json file." + }, + "JavascriptNpmPackageLockEntry": { + "properties": { + "resolved": { + "type": "string", + "description": "Resolved is URL where this package was downloaded from (registry source)" + }, + "integrity": { + "type": "string", + "description": "Integrity is Subresource Integrity hash for verification using standard SRI format (sha512-... or sha1-...). npm changed from SHA-1 to SHA-512 in newer versions. For registry sources this is the integrity from registry, for remote tarballs it's SHA-512 of the file. npm verifies tarball matches this hash before unpacking, throwing EINTEGRITY error if mismatch detected." + }, + "dependencies": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Dependencies is a map of dependencies and their version markers, i.e. \"lodash\": \"^1.0.0\"" + } + }, + "type": "object", + "required": [ + "resolved", + "integrity", + "dependencies" + ], + "description": "NpmPackageLockEntry represents a single entry within the \"packages\" section of a package-lock.json file." + }, + "JavascriptPnpmLockEntry": { + "properties": { + "resolution": { + "$ref": "#/$defs/PnpmLockResolution", + "description": "Resolution is the resolution information for the package" + }, + "dependencies": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Dependencies is a map of dependencies and their versions" + } + }, + "type": "object", + "required": [ + "resolution", + "dependencies" + ], + "description": "PnpmLockEntry represents a single entry in the \"packages\" section of a pnpm-lock.yaml file." + }, + "JavascriptYarnLockEntry": { + "properties": { + "resolved": { + "type": "string", + "description": "Resolved is URL where this package was downloaded from" + }, + "integrity": { + "type": "string", + "description": "Integrity is Subresource Integrity hash for verification (SRI format)" + }, + "dependencies": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Dependencies is a map of dependencies and their versions" + } + }, + "type": "object", + "required": [ + "resolved", + "integrity", + "dependencies" + ], + "description": "YarnLockEntry represents a single entry section of a yarn.lock file." + }, + "KeyValue": { + "properties": { + "key": { + "type": "string", + "description": "Key is the key name" + }, + "value": { + "type": "string", + "description": "Value is the value associated with the key" + } + }, + "type": "object", + "required": [ + "key", + "value" + ], + "description": "KeyValue represents a single key-value pair." + }, + "KeyValues": { + "items": { + "$ref": "#/$defs/KeyValue" + }, + "type": "array", + "description": "KeyValues represents an ordered collection of key-value pairs that preserves insertion order." + }, + "License": { + "properties": { + "value": { + "type": "string", + "description": "Value is the raw license identifier or expression as found." + }, + "spdxExpression": { + "type": "string", + "description": "SPDXExpression is the parsed SPDX license expression." + }, + "type": { + "type": "string", + "description": "Type is the license type classification (e.g., declared, concluded, discovered)." + }, + "urls": { + "items": { + "type": "string" + }, + "type": "array", + "description": "URLs are URLs where license text or information can be found." + }, + "locations": { + "items": { + "$ref": "#/$defs/Location" + }, + "type": "array", + "description": "Locations are file locations where this license was discovered." + }, + "contents": { + "type": "string", + "description": "Contents is the full license text content." + } + }, + "type": "object", + "required": [ + "value", + "spdxExpression", + "type", + "urls", + "locations" + ], + "description": "License represents software license information discovered for a package, including SPDX expressions and supporting evidence locations." + }, + "LinuxKernelArchive": { + "properties": { + "name": { + "type": "string", + "description": "Name is kernel name (typically \"Linux\")" + }, + "architecture": { + "type": "string", + "description": "Architecture is the target CPU architecture" + }, + "version": { + "type": "string", + "description": "Version is kernel version string" + }, + "extendedVersion": { + "type": "string", + "description": "ExtendedVersion is additional version information" + }, + "buildTime": { + "type": "string", + "description": "BuildTime is when the kernel was built" + }, + "author": { + "type": "string", + "description": "Author is who built the kernel" + }, + "format": { + "type": "string", + "description": "Format is kernel image format (e.g. bzImage, zImage)" + }, + "rwRootFS": { + "type": "boolean", + "description": "RWRootFS is whether root filesystem is mounted read-write" + }, + "swapDevice": { + "type": "integer", + "description": "SwapDevice is swap device number" + }, + "rootDevice": { + "type": "integer", + "description": "RootDevice is root device number" + }, + "videoMode": { + "type": "string", + "description": "VideoMode is default video mode setting" + } + }, + "type": "object", + "required": [ + "name", + "architecture", + "version" + ], + "description": "LinuxKernel represents all captured data for a Linux kernel" + }, + "LinuxKernelModule": { + "properties": { + "name": { + "type": "string", + "description": "Name is module name" + }, + "version": { + "type": "string", + "description": "Version is module version string" + }, + "sourceVersion": { + "type": "string", + "description": "SourceVersion is the source code version identifier" + }, + "path": { + "type": "string", + "description": "Path is the filesystem path to the .ko kernel object file (absolute path)" + }, + "description": { + "type": "string", + "description": "Description is a human-readable module description" + }, + "author": { + "type": "string", + "description": "Author is module author name and email" + }, + "license": { + "type": "string", + "description": "License is module license (e.g. GPL, BSD) which must be compatible with kernel" + }, + "kernelVersion": { + "type": "string", + "description": "KernelVersion is kernel version this module was built for" + }, + "versionMagic": { + "type": "string", + "description": "VersionMagic is version magic string for compatibility checking (includes kernel version, SMP status, module loading capabilities like \"3.17.4-302.fc21.x86_64 SMP mod_unload modversions\"). Module will NOT load if vermagic doesn't match running kernel." + }, + "parameters": { + "additionalProperties": { + "$ref": "#/$defs/LinuxKernelModuleParameter" + }, + "type": "object", + "description": "Parameters are the module parameters that can be configured at load time (user-settable values like module options)" + } + }, + "type": "object", + "description": "LinuxKernelModule represents a loadable kernel module (.ko file) with its metadata, parameters, and dependencies." + }, + "LinuxKernelModuleParameter": { + "properties": { + "type": { + "type": "string", + "description": "Type is parameter data type (e.g. int, string, bool, array types)" + }, + "description": { + "type": "string", + "description": "Description is a human-readable parameter description explaining what the parameter controls" + } + }, + "type": "object", + "description": "LinuxKernelModuleParameter represents a configurable parameter for a kernel module with its type and description." + }, + "LinuxRelease": { + "properties": { + "prettyName": { + "type": "string", + "description": "PrettyName is a human-readable operating system name with version." + }, + "name": { + "type": "string", + "description": "Name is the operating system name without version information." + }, + "id": { + "type": "string", + "description": "ID is the lower-case operating system identifier (e.g., \"ubuntu\", \"rhel\")." + }, + "idLike": { + "$ref": "#/$defs/IDLikes", + "description": "IDLike is a list of operating system IDs this distribution is similar to or derived from." + }, + "version": { + "type": "string", + "description": "Version is the operating system version including codename if available." + }, + "versionID": { + "type": "string", + "description": "VersionID is the operating system version number or identifier." + }, + "versionCodename": { + "type": "string", + "description": "VersionCodename is the operating system release codename (e.g., \"jammy\", \"bullseye\")." + }, + "buildID": { + "type": "string", + "description": "BuildID is a build identifier for the operating system." + }, + "imageID": { + "type": "string", + "description": "ImageID is an identifier for container or cloud images." + }, + "imageVersion": { + "type": "string", + "description": "ImageVersion is the version for container or cloud images." + }, + "variant": { + "type": "string", + "description": "Variant is the operating system variant name (e.g., \"Server\", \"Workstation\")." + }, + "variantID": { + "type": "string", + "description": "VariantID is the lower-case operating system variant identifier." + }, + "homeURL": { + "type": "string", + "description": "HomeURL is the homepage URL for the operating system." + }, + "supportURL": { + "type": "string", + "description": "SupportURL is the support or help URL for the operating system." + }, + "bugReportURL": { + "type": "string", + "description": "BugReportURL is the bug reporting URL for the operating system." + }, + "privacyPolicyURL": { + "type": "string", + "description": "PrivacyPolicyURL is the privacy policy URL for the operating system." + }, + "cpeName": { + "type": "string", + "description": "CPEName is the Common Platform Enumeration name for the operating system." + }, + "supportEnd": { + "type": "string", + "description": "SupportEnd is the end of support date or version identifier." + }, + "extendedSupport": { + "type": "boolean", + "description": "ExtendedSupport indicates whether extended security or support is available." + } + }, + "type": "object", + "description": "LinuxRelease contains Linux distribution identification and version information extracted from /etc/os-release or similar system files." + }, + "Location": { + "properties": { + "path": { + "type": "string", + "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem." + }, + "layerID": { + "type": "string", + "description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in." + }, + "fullPath": { + "type": "string", + "description": "FullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it is blank, so coordinates outside archives are unchanged by the field (in JSON, equality and ID)." + }, + "accessPath": { + "type": "string", + "description": "AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path). For a file within an archive, it is a full path (see Coordinates.FullPath)." + }, + "annotations": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Arbitrary key-value pairs that can be used to annotate a location" + } + }, + "type": "object", + "required": [ + "path", + "accessPath" + ], + "description": "Location represents a path relative to a particular filesystem resolved to a specific file.Reference." + }, + "LuarocksPackage": { + "properties": { + "name": { + "type": "string", + "description": "Name is the package name as found in the .rockspec file" + }, + "version": { + "type": "string", + "description": "Version is the package version as found in the .rockspec file" + }, + "license": { + "type": "string", + "description": "License is license identifier" + }, + "homepage": { + "type": "string", + "description": "Homepage is project homepage URL" + }, + "description": { + "type": "string", + "description": "Description is a human-readable package description" + }, + "url": { + "type": "string", + "description": "URL is the source download URL" + }, + "dependencies": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Dependencies are the map of dependency names to version constraints" + } + }, + "type": "object", + "required": [ + "name", + "version", + "license", + "homepage", + "description", + "url", + "dependencies" + ], + "description": "LuaRocksPackage represents a Lua package managed by the LuaRocks package manager with metadata from .rockspec files." + }, + "MicrosoftKbPatch": { + "properties": { + "product_id": { + "type": "string", + "description": "ProductID is MSRC Product ID (e.g. \"Windows 10 Version 1703 for 32-bit Systems\")" + }, + "kb": { + "type": "string", + "description": "Kb is Knowledge Base article number (e.g. \"5001028\")" + } + }, + "type": "object", + "required": [ + "product_id", + "kb" + ], + "description": "MicrosoftKbPatch represents a Windows Knowledge Base patch identifier associated with a specific Microsoft product from the MSRC (Microsoft Security Response Center)." + }, + "NixDerivation": { + "properties": { + "path": { + "type": "string", + "description": "Path is path to the .drv file in Nix store" + }, + "system": { + "type": "string", + "description": "System is target system string indicating where derivation can be built (e.g. \"x86_64-linux\", \"aarch64-darwin\"). Must match current system for local builds." + }, + "inputDerivations": { + "items": { + "$ref": "#/$defs/NixDerivationReference" + }, + "type": "array", + "description": "InputDerivations are the list of other derivations that were inputs to this build (dependencies)" + }, + "inputSources": { + "items": { + "type": "string" + }, + "type": "array", + "description": "InputSources are the list of source file paths that were inputs to this build" + } + }, + "type": "object", + "description": "NixDerivation represents a Nix .drv file that describes how to build a package including inputs, outputs, and build instructions." + }, + "NixDerivationReference": { + "properties": { + "path": { + "type": "string", + "description": "Path is path to the referenced .drv file" + }, + "outputs": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Outputs are which outputs of the referenced derivation were used (e.g. [\"out\"], [\"bin\", \"dev\"])" + } + }, + "type": "object", + "description": "NixDerivationReference represents a reference to another derivation used as a build input or runtime dependency." + }, + "NixStoreEntry": { + "properties": { + "path": { + "type": "string", + "description": "Path is full store path for this output (e.g. /nix/store/abc123...-package-1.0)" + }, + "output": { + "type": "string", + "description": "Output is the specific output name for multi-output packages (empty string for default \"out\" output, can be \"bin\", \"dev\", \"doc\", etc.)" + }, + "outputHash": { + "type": "string", + "description": "OutputHash is hash prefix of the store path basename (first part before the dash)" + }, + "derivation": { + "$ref": "#/$defs/NixDerivation", + "description": "Derivation is information about the .drv file that describes how this package was built" + }, + "files": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Files are the list of files under the nix/store path for this package" + } + }, + "type": "object", + "required": [ + "outputHash" + ], + "description": "NixStoreEntry represents a package in the Nix store (/nix/store) with its derivation information and metadata." + }, + "OpamPackage": { + "properties": { + "name": { + "type": "string", + "description": "Name is the package name as found in the .opam file" + }, + "version": { + "type": "string", + "description": "Version is the package version as found in the .opam file" + }, + "licenses": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Licenses are the list of applicable licenses" + }, + "url": { + "type": "string", + "description": "URL is download URL for the package source" + }, + "checksum": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Checksums are the list of checksums for verification" + }, + "homepage": { + "type": "string", + "description": "Homepage is project homepage URL" + }, + "dependencies": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Dependencies are the list of required dependencies" + } + }, + "type": "object", + "required": [ + "name", + "version", + "licenses", + "url", + "checksum", + "homepage", + "dependencies" + ], + "description": "OpamPackage represents an OCaml package managed by the OPAM package manager with metadata from .opam files." + }, + "Package": { + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "type": { + "type": "string" + }, + "foundBy": { + "type": "string" + }, + "locations": { + "items": { + "$ref": "#/$defs/Location" + }, + "type": "array" + }, + "licenses": { + "$ref": "#/$defs/licenses" + }, + "language": { + "type": "string" + }, + "cpes": { + "$ref": "#/$defs/cpes" + }, + "purl": { + "type": "string" + }, + "metadataType": { + "type": "string" + }, + "metadata": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/AlpmDbEntry" + }, + { + "$ref": "#/$defs/ApkDbEntry" + }, + { + "$ref": "#/$defs/AppleAppBundleEntry" + }, + { + "$ref": "#/$defs/BinarySignature" + }, + { + "$ref": "#/$defs/BitnamiSbomEntry" + }, + { + "$ref": "#/$defs/CConanFileEntry" + }, + { + "$ref": "#/$defs/CConanInfoEntry" + }, + { + "$ref": "#/$defs/CConanLockEntry" + }, + { + "$ref": "#/$defs/CConanLockV2Entry" + }, + { + "$ref": "#/$defs/CocoaPodfileLockEntry" + }, + { + "$ref": "#/$defs/CondaMetadataEntry" + }, + { + "$ref": "#/$defs/CpanDistribution" + }, + { + "$ref": "#/$defs/CpanUnpackedRelease" + }, + { + "$ref": "#/$defs/DartPubspec" + }, + { + "$ref": "#/$defs/DartPubspecLockEntry" + }, + { + "$ref": "#/$defs/DenoLockEntry" + }, + { + "$ref": "#/$defs/DenoRemoteLockEntry" + }, + { + "$ref": "#/$defs/DotnetDepsEntry" + }, + { + "$ref": "#/$defs/DotnetPackagesLockEntry" + }, + { + "$ref": "#/$defs/DotnetPortableExecutableEntry" + }, + { + "$ref": "#/$defs/DpkgArchiveEntry" + }, + { + "$ref": "#/$defs/DpkgDbEntry" + }, + { + "$ref": "#/$defs/ElfBinaryPackageNoteJsonPayload" + }, + { + "$ref": "#/$defs/ElixirMixLockEntry" + }, + { + "$ref": "#/$defs/ErlangRebarLockEntry" + }, + { + "$ref": "#/$defs/GgufFileHeader" + }, + { + "$ref": "#/$defs/GithubActionsUseStatement" + }, + { + "$ref": "#/$defs/GoModuleBuildinfoEntry" + }, + { + "$ref": "#/$defs/GoModuleEntry" + }, + { + "$ref": "#/$defs/GoSourceEntry" + }, + { + "$ref": "#/$defs/HaskellHackageStackEntry" + }, + { + "$ref": "#/$defs/HaskellHackageStackLockEntry" + }, + { + "$ref": "#/$defs/HomebrewFormula" + }, + { + "$ref": "#/$defs/JavaArchive" + }, + { + "$ref": "#/$defs/JavaJvmInstallation" + }, + { + "$ref": "#/$defs/JavascriptBunLockEntry" + }, + { + "$ref": "#/$defs/JavascriptNpmPackage" + }, + { + "$ref": "#/$defs/JavascriptNpmPackageLockEntry" + }, + { + "$ref": "#/$defs/JavascriptPnpmLockEntry" + }, + { + "$ref": "#/$defs/JavascriptYarnLockEntry" + }, + { + "$ref": "#/$defs/LinuxKernelArchive" + }, + { + "$ref": "#/$defs/LinuxKernelModule" + }, + { + "$ref": "#/$defs/LuarocksPackage" + }, + { + "$ref": "#/$defs/MicrosoftKbPatch" + }, + { + "$ref": "#/$defs/NixStoreEntry" + }, + { + "$ref": "#/$defs/OpamPackage" + }, + { + "$ref": "#/$defs/PeBinary" + }, + { + "$ref": "#/$defs/PhpComposerInstalledEntry" + }, + { + "$ref": "#/$defs/PhpComposerLockEntry" + }, + { + "$ref": "#/$defs/PhpPearEntry" + }, + { + "$ref": "#/$defs/PhpPeclEntry" + }, + { + "$ref": "#/$defs/PortageDbEntry" + }, + { + "$ref": "#/$defs/PythonPackage" + }, + { + "$ref": "#/$defs/PythonPdmLockEntry" + }, + { + "$ref": "#/$defs/PythonPipRequirementsEntry" + }, + { + "$ref": "#/$defs/PythonPipfileLockEntry" + }, + { + "$ref": "#/$defs/PythonPoetryLockEntry" + }, + { + "$ref": "#/$defs/PythonUvLockEntry" + }, + { + "$ref": "#/$defs/RDescription" + }, + { + "$ref": "#/$defs/RpmArchive" + }, + { + "$ref": "#/$defs/RpmDbEntry" + }, + { + "$ref": "#/$defs/RubyGemspec" + }, + { + "$ref": "#/$defs/RustCargoAuditEntry" + }, + { + "$ref": "#/$defs/RustCargoLockEntry" + }, + { + "$ref": "#/$defs/SafetensorsModelInfo" + }, + { + "$ref": "#/$defs/SnapEntry" + }, + { + "$ref": "#/$defs/SwiftPackageManagerLockEntry" + }, + { + "$ref": "#/$defs/SwiplpackPackage" + }, + { + "$ref": "#/$defs/TerraformLockProviderEntry" + }, + { + "$ref": "#/$defs/VcpkgManifest" + }, + { + "$ref": "#/$defs/WordpressPluginEntry" + } + ] + } + }, + "type": "object", + "required": [ + "id", + "name", + "version", + "type", + "foundBy", + "locations", + "licenses", + "language", + "cpes", + "purl" + ], + "description": "Package represents a pkg.Package object specialized for JSON marshaling and unmarshalling." + }, + "PeBinary": { + "properties": { + "VersionResources": { + "$ref": "#/$defs/KeyValues", + "description": "VersionResources contains key-value pairs extracted from the PE file's version resource section (e.g., FileVersion, ProductName, CompanyName)." + } + }, + "type": "object", + "required": [ + "VersionResources" + ], + "description": "PEBinary represents metadata captured from a Portable Executable formatted binary (dll, exe, etc.)" + }, + "PhpComposerAuthors": { + "properties": { + "name": { + "type": "string", + "description": "Name is author's full name" + }, + "email": { + "type": "string", + "description": "Email is author's email address" + }, + "homepage": { + "type": "string", + "description": "Homepage is author's personal or company website" + } + }, + "type": "object", + "required": [ + "name" + ], + "description": "PhpComposerAuthors represents author information for a PHP Composer package from the authors field in composer.json." + }, + "PhpComposerExternalReference": { + "properties": { + "type": { + "type": "string", + "description": "Type is reference type (git for source VCS, zip/tar for dist archives)" + }, + "url": { + "type": "string", + "description": "URL is the URL to the resource (git repository URL or archive download URL)" + }, + "reference": { + "type": "string", + "description": "Reference is git commit hash or version tag for source, or archive version for dist" + }, + "shasum": { + "type": "string", + "description": "Shasum is SHA hash of the archive file for integrity verification (dist only)" + } + }, + "type": "object", + "required": [ + "type", + "url", + "reference" + ], + "description": "PhpComposerExternalReference represents source or distribution information for a PHP package, indicating where the package code is retrieved from." + }, + "PhpComposerInstalledEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is package name in vendor/package format (e.g. symfony/console)" + }, + "version": { + "type": "string", + "description": "Version is the package version" + }, + "source": { + "$ref": "#/$defs/PhpComposerExternalReference", + "description": "Source is the source repository information for development (typically git repo, used when passing --prefer-source). Originates from source code repository." + }, + "dist": { + "$ref": "#/$defs/PhpComposerExternalReference", + "description": "Dist is distribution archive information for production (typically zip/tar, default install method). Packaged version of released code." + }, + "require": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Require is runtime dependencies with version constraints (package will not install unless these requirements can be met)" + }, + "provide": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Provide is virtual packages/functionality provided by this package (allows other packages to depend on capabilities)" + }, + "require-dev": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "RequireDev is development-only dependencies (not installed in production, only when developing this package or running tests)" + }, + "suggest": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Suggest is optional but recommended dependencies (suggestions for packages that would extend functionality)" + }, + "license": { + "items": { + "type": "string" + }, + "type": "array", + "description": "License is the list of license identifiers (SPDX format)" + }, + "type": { + "type": "string", + "description": "Type is package type indicating purpose (library=reusable code, project=application, metapackage=aggregates dependencies, etc.)" + }, + "notification-url": { + "type": "string", + "description": "NotificationURL is the URL to notify when package is installed (for tracking/statistics)" + }, + "bin": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Bin is the list of binary/executable files that should be added to PATH" + }, + "authors": { + "items": { + "$ref": "#/$defs/PhpComposerAuthors" + }, + "type": "array", + "description": "Authors are the list of package authors with name/email/homepage" + }, + "description": { + "type": "string", + "description": "Description is a human-readable package description" + }, + "homepage": { + "type": "string", + "description": "Homepage is project homepage URL" + }, + "keywords": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Keywords are the list of keywords for package discovery/search" + }, + "time": { + "type": "string", + "description": "Time is timestamp when this package version was released" + } + }, + "type": "object", + "required": [ + "name", + "version", + "source", + "dist" + ], + "description": "PhpComposerInstalledEntry represents a single package entry from a composer v1/v2 \"installed.json\" files (very similar to composer.lock files)." + }, + "PhpComposerLockEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is package name in vendor/package format (e.g. symfony/console)" + }, + "version": { + "type": "string", + "description": "Version is the package version" + }, + "source": { + "$ref": "#/$defs/PhpComposerExternalReference", + "description": "Source is the source repository information for development (typically git repo, used when passing --prefer-source). Originates from source code repository." + }, + "dist": { + "$ref": "#/$defs/PhpComposerExternalReference", + "description": "Dist is distribution archive information for production (typically zip/tar, default install method). Packaged version of released code." + }, + "require": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Require is runtime dependencies with version constraints (package will not install unless these requirements can be met)" + }, + "provide": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Provide is virtual packages/functionality provided by this package (allows other packages to depend on capabilities)" + }, + "require-dev": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "RequireDev is development-only dependencies (not installed in production, only when developing this package or running tests)" + }, + "suggest": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Suggest is optional but recommended dependencies (suggestions for packages that would extend functionality)" + }, + "license": { + "items": { + "type": "string" + }, + "type": "array", + "description": "License is the list of license identifiers (SPDX format)" + }, + "type": { + "type": "string", + "description": "Type is package type indicating purpose (library=reusable code, project=application, metapackage=aggregates dependencies, etc.)" + }, + "notification-url": { + "type": "string", + "description": "NotificationURL is the URL to notify when package is installed (for tracking/statistics)" + }, + "bin": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Bin is the list of binary/executable files that should be added to PATH" + }, + "authors": { + "items": { + "$ref": "#/$defs/PhpComposerAuthors" + }, + "type": "array", + "description": "Authors are the list of package authors with name/email/homepage" + }, + "description": { + "type": "string", + "description": "Description is a human-readable package description" + }, + "homepage": { + "type": "string", + "description": "Homepage is project homepage URL" + }, + "keywords": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Keywords are the list of keywords for package discovery/search" + }, + "time": { + "type": "string", + "description": "Time is timestamp when this package version was released" + } + }, + "type": "object", + "required": [ + "name", + "version", + "source", + "dist" + ], + "description": "PhpComposerLockEntry represents a single package entry found from a composer.lock file." + }, + "PhpPearEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is the package name" + }, + "channel": { + "type": "string", + "description": "Channel is PEAR channel this package is from" + }, + "version": { + "type": "string", + "description": "Version is the package version" + }, + "license": { + "items": { + "type": "string" + }, + "type": "array", + "description": "License is the list of applicable licenses" + } + }, + "type": "object", + "required": [ + "name", + "version" + ], + "description": "PhpPearEntry represents a single package entry found within php pear metadata files." + }, + "PhpPeclEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is the package name" + }, + "channel": { + "type": "string", + "description": "Channel is PEAR channel this package is from" + }, + "version": { + "type": "string", + "description": "Version is the package version" + }, + "license": { + "items": { + "type": "string" + }, + "type": "array", + "description": "License is the list of applicable licenses" + } + }, + "type": "object", + "required": [ + "name", + "version" + ], + "description": "PhpPeclEntry represents a single package entry found within php pecl metadata files." + }, + "PnpmLockResolution": { + "properties": { + "integrity": { + "type": "string", + "description": "Integrity is Subresource Integrity hash for verification (SRI format)" + } + }, + "type": "object", + "required": [ + "integrity" + ], + "description": "PnpmLockResolution contains package resolution metadata from pnpm lockfiles, including the integrity hash used for verification." + }, + "PortageDbEntry": { + "properties": { + "installedSize": { + "type": "integer", + "description": "InstalledSize is total size of installed files in bytes" + }, + "licenses": { + "type": "string", + "description": "Licenses is license string which may be an expression (e.g. \"GPL-2 OR Apache-2.0\")" + }, + "files": { + "items": { + "$ref": "#/$defs/PortageFileRecord" + }, + "type": "array", + "description": "Files are the files installed by this package (tracked in CONTENTS file)" + } + }, + "type": "object", + "required": [ + "installedSize", + "files" + ], + "description": "PortageEntry represents a single package entry in the portage DB flat-file store." + }, + "PortageFileRecord": { + "properties": { + "path": { + "type": "string", + "description": "Path is the file path relative to the filesystem root" + }, + "digest": { + "$ref": "#/$defs/Digest", + "description": "Digest is file content hash (MD5 for regular files in CONTENTS format: \"obj filename md5hash mtime\")" + } + }, + "type": "object", + "required": [ + "path" + ], + "description": "PortageFileRecord represents a single file attributed to a portage package." + }, + "PythonDirectURLOriginInfo": { + "properties": { + "url": { + "type": "string", + "description": "URL is the source URL from which the package was installed." + }, + "commitId": { + "type": "string", + "description": "CommitID is the VCS commit hash if installed from version control." + }, + "vcs": { + "type": "string", + "description": "VCS is the version control system type (e.g., \"git\", \"hg\")." + } + }, + "type": "object", + "required": [ + "url" + ], + "description": "PythonDirectURLOriginInfo represents installation source metadata from direct_url.json for packages installed from VCS or direct URLs." + }, + "PythonFileDigest": { + "properties": { + "algorithm": { + "type": "string", + "description": "Algorithm is the hash algorithm used (e.g., \"sha256\")." + }, + "value": { + "type": "string", + "description": "Value is the hex-encoded hash digest value." + } + }, + "type": "object", + "required": [ + "algorithm", + "value" + ], + "description": "PythonFileDigest represents the file metadata for a single file attributed to a python package." + }, + "PythonFileRecord": { + "properties": { + "path": { + "type": "string", + "description": "Path is the installed file path from the RECORD file." + }, + "digest": { + "$ref": "#/$defs/PythonFileDigest", + "description": "Digest contains the hash algorithm and value for file integrity verification." + }, + "size": { + "type": "string", + "description": "Size is the file size in bytes as a string." + } + }, + "type": "object", + "required": [ + "path" + ], + "description": "PythonFileRecord represents a single entry within a RECORD file for a python wheel or egg package" + }, + "PythonPackage": { + "properties": { + "name": { + "type": "string", + "description": "Name is the package name from the Name field in PKG-INFO or METADATA." + }, + "version": { + "type": "string", + "description": "Version is the package version from the Version field in PKG-INFO or METADATA." + }, + "author": { + "type": "string", + "description": "Author is the package author name from the Author field." + }, + "authorEmail": { + "type": "string", + "description": "AuthorEmail is the package author's email address from the Author-Email field." + }, + "platform": { + "type": "string", + "description": "Platform indicates the target platform for the package (e.g., \"any\", \"linux\", \"win32\")." + }, + "files": { + "items": { + "$ref": "#/$defs/PythonFileRecord" + }, + "type": "array", + "description": "Files are the installed files listed in the RECORD file for wheels or installed-files.txt for eggs." + }, + "sitePackagesRootPath": { + "type": "string", + "description": "SitePackagesRootPath is the root directory path containing the package (e.g., \"/usr/lib/python3.9/site-packages\")." + }, + "topLevelPackages": { + "items": { + "type": "string" + }, + "type": "array", + "description": "TopLevelPackages are the top-level Python module names from top_level.txt file." + }, + "directUrlOrigin": { + "$ref": "#/$defs/PythonDirectURLOriginInfo", + "description": "DirectURLOrigin contains VCS or direct URL installation information from direct_url.json." + }, + "requiresPython": { + "type": "string", + "description": "RequiresPython specifies the Python version requirement (e.g., \"\u003e=3.6\")." + }, + "requiresDist": { + "items": { + "type": "string" + }, + "type": "array", + "description": "RequiresDist lists the package dependencies with version specifiers from Requires-Dist fields." + }, + "providesExtra": { + "items": { + "type": "string" + }, + "type": "array", + "description": "ProvidesExtra lists optional feature names that can be installed via extras (e.g., \"dev\", \"test\")." + } + }, + "type": "object", + "required": [ + "name", + "version", + "author", + "authorEmail", + "platform", + "sitePackagesRootPath" + ], + "description": "PythonPackage represents all captured data for a python egg or wheel package (specifically as outlined in the PyPA core metadata specification https://packaging.python.org/en/latest/specifications/core-metadata/)." + }, + "PythonPdmFileEntry": { + "properties": { + "url": { + "type": "string", + "description": "URL is the file download URL" + }, + "digest": { + "$ref": "#/$defs/PythonFileDigest", + "description": "Digest is the hash digest of the file hosted at the URL" + } + }, + "type": "object", + "required": [ + "url", + "digest" + ] + }, + "PythonPdmLockEntry": { + "properties": { + "summary": { + "type": "string", + "description": "Summary provides a description of the package" + }, + "files": { + "items": { + "$ref": "#/$defs/PythonPdmFileEntry" + }, + "type": "array", + "description": "Files are the package files with their paths and hash digests (for the base package without extras)" + }, + "marker": { + "type": "string", + "description": "Marker is the \"environment\" --conditional expressions that determine whether a package should be installed based on the runtime environment" + }, + "requiresPython": { + "type": "string", + "description": "RequiresPython specifies the Python version requirement (e.g., \"\u003e=3.6\")." + }, + "dependencies": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Dependencies are the dependency specifications for the base package (without extras)" + }, + "extras": { + "items": { + "$ref": "#/$defs/PythonPdmLockExtraVariant" + }, + "type": "array", + "description": "Extras contains variants for different extras combinations (PDM may have multiple entries per package)" + } + }, + "type": "object", + "required": [ + "summary", + "files" + ], + "description": "PythonPdmLockEntry represents a single package entry within a pdm.lock file." + }, + "PythonPdmLockExtraVariant": { + "properties": { + "extras": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Extras are the optional extras enabled for this variant (e.g., [\"toml\"], [\"dev\"], or [\"toml\", \"dev\"])" + }, + "dependencies": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Dependencies are the dependencies specific to this extras variant" + }, + "files": { + "items": { + "$ref": "#/$defs/PythonPdmFileEntry" + }, + "type": "array", + "description": "Files are the package files specific to this variant (only populated if different from base)" + }, + "marker": { + "type": "string", + "description": "Marker is the environment conditional expression for this variant (e.g., \"python_version \u003c \\\"3.11\\\"\")" + } + }, + "type": "object", + "required": [ + "extras" + ], + "description": "PythonPdmLockExtraVariant represents a specific extras combination variant within a PDM lock file." + }, + "PythonPipRequirementsEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is the package name from the requirements file." + }, + "extras": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Extras are the optional features to install from the package (e.g., package[dev,test])." + }, + "versionConstraint": { + "type": "string", + "description": "VersionConstraint specifies version requirements (e.g., \"\u003e=1.0,\u003c2.0\")." + }, + "url": { + "type": "string", + "description": "URL is the direct download URL or VCS URL if specified instead of a PyPI package." + }, + "markers": { + "type": "string", + "description": "Markers are environment marker expressions for conditional installation (e.g., \"python_version \u003e= '3.8'\")." + } + }, + "type": "object", + "required": [ + "name", + "versionConstraint" + ], + "description": "PythonRequirementsEntry represents a single entry within a [*-]requirements.txt file." + }, + "PythonPipfileLockEntry": { + "properties": { + "hashes": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Hashes are the package file hash values in the format \"algorithm:digest\" for integrity verification." + }, + "index": { + "type": "string", + "description": "Index is the PyPI index name where the package should be fetched from." + } + }, + "type": "object", + "required": [ + "hashes", + "index" + ], + "description": "PythonPipfileLockEntry represents a single package entry within a Pipfile.lock file." + }, + "PythonPoetryLockDependencyEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is the dependency package name." + }, + "version": { + "type": "string", + "description": "Version is the locked version or version constraint for the dependency." + }, + "optional": { + "type": "boolean", + "description": "Optional indicates whether this dependency is optional (only needed for certain extras)." + }, + "markers": { + "type": "string", + "description": "Markers are environment marker expressions that conditionally enable the dependency (e.g., \"python_version \u003e= '3.8'\")." + }, + "extras": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Extras are the optional feature names from the dependency that should be installed." + } + }, + "type": "object", + "required": [ + "name", + "version", + "optional" + ], + "description": "PythonPoetryLockDependencyEntry represents a single dependency entry within a Poetry lock file." + }, + "PythonPoetryLockEntry": { + "properties": { + "index": { + "type": "string", + "description": "Index is the package repository name where the package should be fetched from." + }, + "dependencies": { + "items": { + "$ref": "#/$defs/PythonPoetryLockDependencyEntry" + }, + "type": "array", + "description": "Dependencies are the package's runtime dependencies with version constraints." + }, + "extras": { + "items": { + "$ref": "#/$defs/PythonPoetryLockExtraEntry" + }, + "type": "array", + "description": "Extras are optional feature groups that include additional dependencies." + } + }, + "type": "object", + "required": [ + "index", + "dependencies" + ], + "description": "PythonPoetryLockEntry represents a single package entry within a Pipfile.lock file." + }, + "PythonPoetryLockExtraEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is the optional feature name (e.g., \"dev\", \"test\")." + }, + "dependencies": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Dependencies are the package names required when this extra is installed." + } + }, + "type": "object", + "required": [ + "name", + "dependencies" + ], + "description": "PythonPoetryLockExtraEntry represents an optional feature group in a Poetry lock file." + }, + "PythonUvLockDependencyEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is the dependency package name." + }, + "optional": { + "type": "boolean", + "description": "Optional indicates whether this dependency is optional (only needed for certain extras)." + }, + "markers": { + "type": "string", + "description": "Markers are environment marker expressions that conditionally enable the dependency (e.g., \"python_version \u003e= '3.8'\")." + }, + "extras": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Extras are the optional feature names from the dependency that should be installed." + } + }, + "type": "object", + "required": [ + "name", + "optional" + ], + "description": "PythonUvLockDependencyEntry represents a single dependency entry within a uv lock file." + }, + "PythonUvLockEntry": { + "properties": { + "index": { + "type": "string", + "description": "Index is the package repository name where the package should be fetched from." + }, + "dependencies": { + "items": { + "$ref": "#/$defs/PythonUvLockDependencyEntry" + }, + "type": "array", + "description": "Dependencies are the package's runtime dependencies with version constraints." + }, + "extras": { + "items": { + "$ref": "#/$defs/PythonUvLockExtraEntry" + }, + "type": "array", + "description": "Extras are optional feature groups that include additional dependencies." + } + }, + "type": "object", + "required": [ + "index", + "dependencies" + ], + "description": "PythonUvLockEntry represents a single package entry within a uv.lock file." + }, + "PythonUvLockExtraEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is the optional feature name (e.g., \"dev\", \"test\")." + }, + "dependencies": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Dependencies are the package names required when this extra is installed." + } + }, + "type": "object", + "required": [ + "name", + "dependencies" + ], + "description": "PythonUvLockExtraEntry represents an optional feature group in a uv lock file." + }, + "RDescription": { + "properties": { + "title": { + "type": "string", + "description": "Title is short one-line package title" + }, + "description": { + "type": "string", + "description": "Description is detailed package description" + }, + "author": { + "type": "string", + "description": "Author is package author(s)" + }, + "maintainer": { + "type": "string", + "description": "Maintainer is current package maintainer" + }, + "url": { + "items": { + "type": "string" + }, + "type": "array", + "description": "URL is the list of related URLs" + }, + "repository": { + "type": "string", + "description": "Repository is CRAN or other repository name" + }, + "built": { + "type": "string", + "description": "Built is R version and platform this was built with" + }, + "needsCompilation": { + "type": "boolean", + "description": "NeedsCompilation is whether this package requires compilation" + }, + "imports": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Imports are the packages imported in the NAMESPACE" + }, + "depends": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Depends are the packages this package depends on" + }, + "suggests": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Suggests are the optional packages that extend functionality" + } + }, + "type": "object", + "description": "RDescription represents metadata from an R package DESCRIPTION file containing package information, dependencies, and author details." + }, + "Relationship": { + "properties": { + "parent": { + "type": "string", + "description": "Parent is the ID of the parent artifact in this relationship." + }, + "child": { + "type": "string", + "description": "Child is the ID of the child artifact in this relationship." + }, + "type": { + "type": "string", + "description": "Type is the relationship type (e.g., \"contains\", \"dependency-of\", \"ancestor-of\")." + }, + "metadata": { + "description": "Metadata contains additional relationship-specific metadata." + } + }, + "type": "object", + "required": [ + "parent", + "child", + "type" + ], + "description": "Relationship represents a directed relationship between two artifacts in the SBOM, such as package-contains-file or package-depends-on-package." + }, + "RpmArchive": { + "properties": { + "name": { + "type": "string", + "description": "Name is the RPM package name as found in the RPM database." + }, + "version": { + "type": "string", + "description": "Version is the upstream version of the package." + }, + "epoch": { + "oneOf": [ + { + "type": "integer", + "description": "Epoch is the version epoch used to force upgrade ordering (null if not set)." + }, + { + "type": "null" + } + ] + }, + "architecture": { + "type": "string", + "description": "Arch is the target CPU architecture (e.g., \"x86_64\", \"aarch64\", \"noarch\")." + }, + "release": { + "type": "string", + "description": "Release is the package release number or distribution-specific version suffix." + }, + "sourceRpm": { + "type": "string", + "description": "SourceRpm is the source RPM filename that was used to build this package." + }, + "signatures": { + "items": { + "$ref": "#/$defs/RpmSignature" + }, + "type": "array", + "description": "Signatures contains GPG signature metadata for package verification." + }, + "size": { + "type": "integer", + "description": "Size is the total installed size of the package in bytes." + }, + "vendor": { + "type": "string", + "description": "Vendor is the organization that packaged the software." + }, + "modularityLabel": { + "type": "string", + "description": "ModularityLabel identifies the module stream for modular RPM packages (e.g., \"nodejs:12:20200101\")." + }, + "provides": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Provides lists the virtual packages and capabilities this package provides." + }, + "requires": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Requires lists the dependencies required by this package." + }, + "files": { + "items": { + "$ref": "#/$defs/RpmFileRecord" + }, + "type": "array", + "description": "Files are the file records for all files owned by this package." + } + }, + "type": "object", + "required": [ + "name", + "version", + "epoch", + "architecture", + "release", + "sourceRpm", + "size", + "vendor", + "files" + ], + "description": "RpmArchive represents package metadata extracted directly from a .rpm archive file, containing the same information as an RPM database entry." + }, + "RpmDbEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is the RPM package name as found in the RPM database." + }, + "version": { + "type": "string", + "description": "Version is the upstream version of the package." + }, + "epoch": { + "oneOf": [ + { + "type": "integer", + "description": "Epoch is the version epoch used to force upgrade ordering (null if not set)." + }, + { + "type": "null" + } + ] + }, + "architecture": { + "type": "string", + "description": "Arch is the target CPU architecture (e.g., \"x86_64\", \"aarch64\", \"noarch\")." + }, + "release": { + "type": "string", + "description": "Release is the package release number or distribution-specific version suffix." + }, + "sourceRpm": { + "type": "string", + "description": "SourceRpm is the source RPM filename that was used to build this package." + }, + "signatures": { + "items": { + "$ref": "#/$defs/RpmSignature" + }, + "type": "array", + "description": "Signatures contains GPG signature metadata for package verification." + }, + "size": { + "type": "integer", + "description": "Size is the total installed size of the package in bytes." + }, + "vendor": { + "type": "string", + "description": "Vendor is the organization that packaged the software." + }, + "modularityLabel": { + "type": "string", + "description": "ModularityLabel identifies the module stream for modular RPM packages (e.g., \"nodejs:12:20200101\")." + }, + "provides": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Provides lists the virtual packages and capabilities this package provides." + }, + "requires": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Requires lists the dependencies required by this package." + }, + "files": { + "items": { + "$ref": "#/$defs/RpmFileRecord" + }, + "type": "array", + "description": "Files are the file records for all files owned by this package." + } + }, + "type": "object", + "required": [ + "name", + "version", + "epoch", + "architecture", + "release", + "sourceRpm", + "size", + "vendor", + "files" + ], + "description": "RpmDBEntry represents all captured data from a RPM DB package entry." + }, + "RpmFileRecord": { + "properties": { + "path": { + "type": "string", + "description": "Path is the absolute file path where the file is installed." + }, + "mode": { + "type": "integer", + "description": "Mode is the file permission mode bits following Unix stat.h conventions." + }, + "size": { + "type": "integer", + "description": "Size is the file size in bytes." + }, + "digest": { + "$ref": "#/$defs/Digest", + "description": "Digest contains the hash algorithm and value for file integrity verification." + }, + "userName": { + "type": "string", + "description": "UserName is the owner username for the file." + }, + "groupName": { + "type": "string", + "description": "GroupName is the group name for the file." + }, + "flags": { + "type": "string", + "description": "Flags indicates the file type (e.g., \"%config\", \"%doc\", \"%ghost\")." + } + }, + "type": "object", + "required": [ + "path", + "mode", + "size", + "digest", + "userName", + "groupName", + "flags" + ], + "description": "RpmFileRecord represents the file metadata for a single file attributed to a RPM package." + }, + "RpmSignature": { + "properties": { + "algo": { + "type": "string", + "description": "PublicKeyAlgorithm is the public key algorithm used for signing (e.g., \"RSA\")." + }, + "hash": { + "type": "string", + "description": "HashAlgorithm is the hash algorithm used for the signature (e.g., \"SHA256\")." + }, + "created": { + "type": "string", + "description": "Created is the timestamp when the signature was created." + }, + "issuer": { + "type": "string", + "description": "IssuerKeyID is the GPG key ID that created the signature." + } + }, + "type": "object", + "required": [ + "algo", + "hash", + "created", + "issuer" + ], + "description": "RpmSignature represents a GPG signature for an RPM package used for authenticity verification." + }, + "RubyGemspec": { + "properties": { + "name": { + "type": "string", + "description": "Name is gem name as specified in the gemspec" + }, + "version": { + "type": "string", + "description": "Version is gem version as specified in the gemspec" + }, + "files": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Files is logical list of files in the gem (NOT directly usable as filesystem paths. Example: bundler gem lists \"lib/bundler/vendor/uri/lib/uri/ldap.rb\" but actual path is \"/usr/local/lib/ruby/3.2.0/bundler/vendor/uri/lib/uri/ldap.rb\". Would need gem installation path, ruby version, and env vars like GEM_HOME to resolve actual paths.)" + }, + "authors": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Authors are the list of gem authors (stored as array regardless of using `author` or `authors` method in gemspec)" + }, + "homepage": { + "type": "string", + "description": "Homepage is project homepage URL" + } + }, + "type": "object", + "required": [ + "name", + "version" + ], + "description": "RubyGemspec represents all metadata parsed from the *.gemspec file" + }, + "RustCargoAuditEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is crate name as specified in audit section of the build binary" + }, + "version": { + "type": "string", + "description": "Version is crate version as specified in audit section of the build binary" + }, + "source": { + "type": "string", + "description": "Source is the source registry or repository where this crate came from" + } + }, + "type": "object", + "required": [ + "name", + "version", + "source" + ], + "description": "RustBinaryAuditEntry represents Rust crate metadata extracted from a compiled binary using cargo-auditable format." + }, + "RustCargoLockEntry": { + "properties": { + "name": { + "type": "string", + "description": "Name is crate name as specified in Cargo.toml" + }, + "version": { + "type": "string", + "description": "Version is crate version as specified in Cargo.toml" + }, + "source": { + "type": "string", + "description": "Source is the source registry or repository URL in format \"registry+https://github.com/rust-lang/crates.io-index\" for registry packages" + }, + "checksum": { + "type": "string", + "description": "Checksum is content checksum for registry packages only (hexadecimal string). Cargo doesn't require or include checksums for git dependencies. Used to detect MITM attacks by verifying downloaded crate matches lockfile checksum." + }, + "dependencies": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Dependencies are the list of dependencies with version constraints" + } + }, + "type": "object", + "required": [ + "name", + "version", + "source", + "checksum", + "dependencies" + ], + "description": "RustCargoLockEntry represents a locked dependency from a Cargo.lock file with precise version and checksum information." + }, + "SafetensorsModelInfo": { + "properties": { + "format": { + "type": "string", + "description": "Format is the source format label (always \"safetensors\" for this metadata type).\nPresent because the Docker AI model config blob carries an explicit format field" + }, + "architecture": { + "type": "string", + "description": "Architecture is the model architecture (e.g., \"LlamaForCausalLM\",\n\"Qwen3MoeForConditionalGeneration\"). It is not present in the SafeTensors\nheader itself; it is enriched from the companion config.json\n\"architectures\" array when one is found alongside the model." + }, + "quantization": { + "type": "string", + "description": "Quantization describes tensor precision (e.g., \"BF16\", \"F16\", \"F32\", \"INT8\")." + }, + "parameters": { + "type": "integer", + "description": "Parameters is the total number of model parameters, computed from the tensor\nshapes in the SafeTensors header(s). For a sharded model it is the sum across\nevery shard." + }, + "tensorCount": { + "type": "integer", + "description": "TensorCount is the number of tensor entries in the file header." + }, + "totalSize": { + "type": "string", + "description": "TotalSize is the total byte size of tensor data across all shards when known\n(from the Docker AI model config \"size\" field)." + }, + "shardCount": { + "type": "integer", + "description": "ShardCount is the number of .safetensors shards for a sharded model (1 for a\nsingle-file model)." + }, + "userMetadata": { + "$ref": "#/$defs/KeyValues", + "description": "UserMetadata is the optional \"__metadata__\" map from a .safetensors file header\n(string-to-string key/values set by the producer)." + }, + "metadataHash": { + "type": "string", + "description": "MetadataHash is an xxhash over the on-disk SafeTensors header (sorted tensor\nentries + __metadata__). It is derived ONLY from the safetensors file bytes." + }, + "parts": { + "items": { + "$ref": "#/$defs/SafetensorsModelInfo" + }, + "type": "array", + "description": "Parts contains metadata from additional SafeTensors shards or OCI layers that\nwere merged into this package during post-processing." + } + }, + "type": "object", + "description": "SafeTensorsModelInfo holds the model details extracted from SafeTensors content." + }, + "Schema": { + "properties": { + "version": { + "type": "string", + "description": "Version is the JSON schema version for this document format." + }, + "url": { + "type": "string", + "description": "URL is the URL to the JSON schema definition document." + } + }, + "type": "object", + "required": [ + "version", + "url" + ], + "description": "Schema specifies the JSON schema version and URL reference that defines the structure and validation rules for this document format." + }, + "SnapEntry": { + "properties": { + "snapType": { + "type": "string", + "description": "SnapType indicates the snap type (base, kernel, app, gadget, or snapd)." + }, + "base": { + "type": "string", + "description": "Base is the base snap name that this snap depends on (e.g., \"core20\", \"core22\")." + }, + "snapName": { + "type": "string", + "description": "SnapName is the snap package name." + }, + "snapVersion": { + "type": "string", + "description": "SnapVersion is the snap package version." + }, + "architecture": { + "type": "string", + "description": "Architecture is the target CPU architecture (e.g., \"amd64\", \"arm64\")." + } + }, + "type": "object", + "required": [ + "snapType", + "base", + "snapName", + "snapVersion", + "architecture" + ], + "description": "SnapEntry represents metadata for a Snap package extracted from snap.yaml or snapcraft.yaml files." + }, + "Source": { + "properties": { + "id": { + "type": "string", + "description": "ID is a unique identifier for the analyzed source artifact." + }, + "name": { + "type": "string", + "description": "Name is the name of the analyzed artifact (e.g., image name, directory path)." + }, + "version": { + "type": "string", + "description": "Version is the version of the analyzed artifact (e.g., image tag)." + }, + "supplier": { + "type": "string", + "description": "Supplier is supplier information, which can be user-provided for NTIA minimum elements compliance." + }, + "type": { + "type": "string", + "description": "Type is the source type (e.g., \"image\", \"directory\", \"file\")." + }, + "metadata": { + "description": "Metadata contains additional source-specific metadata." + } + }, + "type": "object", + "required": [ + "id", + "name", + "version", + "type", + "metadata" + ], + "description": "Source represents the artifact that was analyzed to generate this SBOM, such as a container image, directory, or file archive." + }, + "SwiftPackageManagerLockEntry": { + "properties": { + "revision": { + "type": "string", + "description": "Revision is git commit hash of the resolved package" + } + }, + "type": "object", + "required": [ + "revision" + ], + "description": "SwiftPackageManagerResolvedEntry represents a resolved dependency from a Package.resolved file with its locked version and source location." + }, + "SwiplpackPackage": { + "properties": { + "name": { + "type": "string", + "description": "Name is the package name as found in the .toml file" + }, + "version": { + "type": "string", + "description": "Version is the package version as found in the .toml file" + }, + "author": { + "type": "string", + "description": "Author is author name" + }, + "authorEmail": { + "type": "string", + "description": "AuthorEmail is author email address" + }, + "packager": { + "type": "string", + "description": "Packager is packager name (if different from author)" + }, + "packagerEmail": { + "type": "string", + "description": "PackagerEmail is packager email address" + }, + "homepage": { + "type": "string", + "description": "Homepage is project homepage URL" + }, + "dependencies": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Dependencies are the list of required dependencies" + } + }, + "type": "object", + "required": [ + "name", + "version", + "author", + "authorEmail", + "packager", + "packagerEmail", + "homepage", + "dependencies" + ], + "description": "SwiplPackEntry represents a SWI-Prolog package from the pack system with metadata about the package and its dependencies." + }, + "TerraformLockProviderEntry": { + "properties": { + "url": { + "type": "string", + "description": "URL is the provider source address (e.g., \"registry.terraform.io/hashicorp/aws\")." + }, + "constraints": { + "type": "string", + "description": "Constraints specifies the version constraints for the provider (e.g., \"~\u003e 4.0\")." + }, + "version": { + "type": "string", + "description": "Version is the locked provider version selected during terraform init." + }, + "hashes": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Hashes are cryptographic checksums for the provider plugin archives across different platforms." + } + }, + "type": "object", + "required": [ + "url", + "constraints", + "version", + "hashes" + ], + "description": "TerraformLockProviderEntry represents a single provider entry in a Terraform dependency lock file (.terraform.lock.hcl)." + }, + "VcpkgManifest": { + "properties": { + "description": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Description is the human-readable description of the package (may be a single string or a list of paragraphs in the manifest)." + }, + "documentation": { + "type": "string", + "description": "Documentation is the URL to the package's documentation." + }, + "full-version": { + "type": "string", + "description": "FullVersion is the complete version string including the port-version suffix (e.g. \"1.2.3#2\")." + }, + "version": { + "type": "string", + "description": "Version is the upstream package version without the port-version suffix (e.g. \"1.2.3\")." + }, + "port-version": { + "type": "integer", + "description": "PortVersion is the vcpkg-specific packaging revision for a given upstream version." + }, + "maintainers": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Maintainers are the people responsible for maintaining the vcpkg port." + }, + "name": { + "type": "string", + "description": "Name is the package name as declared in the manifest." + }, + "supports": { + "type": "string", + "description": "Supports is the platform expression describing which triplets the package can be built for (e.g. \"!windows\")." + }, + "registry": { + "$ref": "#/$defs/VcpkgRegistryEntry", + "description": "Registry indicates where the package definition came from." + }, + "triplet": { + "type": "string", + "description": "Triplet is the build target discovered from the build folder (e.g. \"x64-linux\")." + } + }, + "type": "object", + "required": [ + "full-version", + "version", + "port-version", + "name" + ], + "description": "VcpkgManifest summarizes the data found in a vcpkg manifest (vcpkg.json) relevant to a single vcpkg package." + }, + "VcpkgRegistryEntry": { + "properties": { + "baseline": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "packages": { + "items": { + "type": "string" + }, + "type": "array" + }, + "path": { + "type": "string" + }, + "reference": { + "type": "string" + }, + "repository": { + "type": "string" + } + }, + "type": "object", + "required": [ + "kind" + ], + "description": "Matches definition of Vcpkg \"Registry\"." + }, + "WordpressPluginEntry": { + "properties": { + "pluginInstallDirectory": { + "type": "string", + "description": "PluginInstallDirectory is directory name where the plugin is installed" + }, + "author": { + "type": "string", + "description": "Author is plugin author name" + }, + "authorUri": { + "type": "string", + "description": "AuthorURI is author's website URL" + } + }, + "type": "object", + "required": [ + "pluginInstallDirectory" + ], + "description": "WordpressPluginEntry represents all metadata parsed from the wordpress plugin file" + }, + "cpes": { + "items": { + "$ref": "#/$defs/CPE" + }, + "type": "array" + }, + "licenses": { + "items": { + "$ref": "#/$defs/License" + }, + "type": "array" + } + } +} diff --git a/schema/json/schema-latest.json b/schema/json/schema-latest.json index 3af3fa8ca..7428b8d31 100644 --- a/schema/json/schema-latest.json +++ b/schema/json/schema-latest.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "anchore.io/schema/syft/json/16.1.11/document", + "$id": "anchore.io/schema/syft/json/16.1.12/document", "$ref": "#/$defs/Document", "$defs": { "AlpmDbEntry": { @@ -716,11 +716,15 @@ "properties": { "path": { "type": "string", - "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed)." + "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem." }, "layerID": { "type": "string", - "description": "FileSystemID is an ID representing and entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank." + "description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in." + }, + "fullPath": { + "type": "string", + "description": "FullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it is blank, so coordinates outside archives are unchanged by the field (in JSON, equality and ID)." } }, "type": "object", @@ -2544,15 +2548,19 @@ "properties": { "path": { "type": "string", - "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed)." + "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem." }, "layerID": { "type": "string", - "description": "FileSystemID is an ID representing and entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank." + "description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in." + }, + "fullPath": { + "type": "string", + "description": "FullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it is blank, so coordinates outside archives are unchanged by the field (in JSON, equality and ID)." }, "accessPath": { "type": "string", - "description": "AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path)" + "description": "AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path). For a file within an archive, it is a full path (see Coordinates.FullPath)." }, "annotations": { "additionalProperties": { diff --git a/syft/archive_cataloger_test.go b/syft/archive_cataloger_test.go new file mode 100644 index 000000000..e23399c97 --- /dev/null +++ b/syft/archive_cataloger_test.go @@ -0,0 +1,678 @@ +package syft + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "os" + "path/filepath" + "sort" + "sync" + "testing" + "unicode/utf8" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + + "github.com/anchore/syft/internal/archive" + "github.com/anchore/syft/internal/bus" + "github.com/anchore/syft/internal/task" + "github.com/anchore/syft/internal/testutils" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/cataloging" + "github.com/anchore/syft/syft/cataloging/filecataloging" + "github.com/anchore/syft/syft/cataloging/pkgcataloging" + "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/event/monitor" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/java" + "github.com/anchore/syft/syft/sbom" + "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/syft/source/directorysource" +) + +// Every CreateSBOM call pays to compile the license scanner, so the end-to-end behavior is asserted +// over one scan of one fixture directory rather than a scan per claim. The fixture holds every shape +// the claims need; the subtests read the one SBOM. +// +// scan root +// +// loose.txt an ordinary file, counted by the file catalogers +// nested/marker.txt excluded by the root-anchored pattern ./nested/marker.txt +// app.zip nested/marker.txt, nested/keep.txt, nested/excluded.txt (**/excluded.txt) +// app/data.zip nested/marker.txt, under a directory +// outer.zip outer/marker.txt +// deeper/inner.zip nested/marker.txt, two levels down +// skip/inner-excluded.zip nested/marker.txt, excluded by **/inner-excluded.zip +// 日本語/café-🎉/unicode.zip nested/marker.txt, under a non-ASCII path +// pkg.rpm a zip holding a marker, excluded by **/*.rpm +// KEPT.RPM the same zip, kept: the pattern is lower-case (a distinct base +// name, so the two coexist on a case-insensitive filesystem) +// app.war WEB-INF/lib/war-lib-2.0.jar +// bundle.zip lib/zip-lib-1.2.jar +// bundle.tar.gz lib/tgz-lib-1.2.jar +var fixtureExclusions = []string{"./nested/marker.txt", "**/excluded.txt", "**/inner-excluded.zip", "**/*.rpm"} + +func writeFixture(t *testing.T) string { + t.Helper() + dir := t.TempDir() + write := func(name string, data []byte) { + require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, name), data, 0o644)) + } + markerZip := testutils.Zip(t, map[string]string{"nested/marker.txt": "hello"}) + + write("loose.txt", []byte("outside")) + write("nested/marker.txt", []byte("at the root")) + write("app.zip", testutils.Zip(t, map[string]string{ + "nested/marker.txt": "hello", + "nested/keep.txt": "hello", + "nested/excluded.txt": "hello", + })) + write("app/data.zip", markerZip) + write("outer.zip", testutils.Zip(t, map[string][]byte{ + "outer/marker.txt": []byte("hello"), + "deeper/inner.zip": markerZip, + "skip/inner-excluded.zip": markerZip, + "日本語/café-🎉/unicode.zip": markerZip, + })) + write("pkg.rpm", markerZip) + write("KEPT.RPM", markerZip) + write("app.war", testutils.Zip(t, map[string][]byte{ + "WEB-INF/lib/war-lib-2.0.jar": testutils.Jar(t, "war-lib", "2.0", nil), + })) + write("bundle.zip", testutils.Zip(t, map[string][]byte{ + "lib/zip-lib-1.2.jar": testutils.Jar(t, "zip-lib", "1.2", nil), + })) + write("bundle.tar.gz", testutils.TarGz(t, map[string][]byte{ + "lib/tgz-lib-1.2.jar": testutils.Jar(t, "tgz-lib", "1.2", nil), + })) + return dir +} + +func TestArchiveCataloger(t *testing.T) { + scanDir := writeFixture(t) + resolvedScanDir, err := filepath.EvalSymlinks(scanDir) + require.NoError(t, err) + + resolvedScanDir = filepath.ToSlash(resolvedScanDir) + + recorder := &recordingBus{} + bus.Set(recorder) + t.Cleanup(func() { bus.Set(nil) }) + + var ( + probeMu sync.Mutex + seen []string // every .txt and .zip path any resolver handed a cataloger could see + ) + probe := resolverProbeCataloger{globs: []string{"**/*.txt", "**/*.zip"}, record: func(paths []string) { + probeMu.Lock() + defer probeMu.Unlock() + seen = append(seen, paths...) + }} + + // the java catalogers come from the factory, as the CLI builds them: the flag handing recursion + // to the archive task is derived onto the factory's config. Both legacy search settings are off, + // the cell of the matrix that looks like a bug: with the feature on they no longer gate anything + packages := pkgcataloging.DefaultConfig() + packages.JavaArchive.IncludeIndexedArchives = false + packages.JavaArchive.IncludeUnindexedArchives = false + cfg := DefaultCreateSBOMConfig(). + WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")). + WithCatalogers( + pkgcataloging.NewAlwaysEnabledCatalogerReference(markerCataloger{}), + pkgcataloging.NewAlwaysEnabledCatalogerReference(probe), + ). + WithPackagesConfig(packages). + WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.AllFilesSelection)). + WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3)) + + s := scanDirWithExclusions(t, scanDir, cfg, fixtureExclusions...) + + t.Run("markers are found at every depth under archive-relative paths", func(t *testing.T) { + // the location's RealPath is the outermost archive; FullPath adds the chain of entries within it + assert.Equal(t, []string{ + "KEPT.RPM!/nested/marker.txt", + "app.zip!/nested/marker.txt", + "app/data.zip!/nested/marker.txt", + "outer.zip!/deeper/inner.zip!/nested/marker.txt", + "outer.zip!/outer/marker.txt", + "outer.zip!/日本語/café-🎉/unicode.zip!/nested/marker.txt", + }, packageLocations(s, "marker-pkg")) + }) + + t.Run("archive contents do not leak into the parent tree", func(t *testing.T) { + for _, loc := range packageLocations(s, "marker-pkg") { + assert.NotContains(t, loc, ".zip/", "a joined path smears the archive's contents into the parent's tree") + assert.NotContains(t, loc, "!//", "a nested path must not be rooted") + } + }) + + t.Run("unicode entry names survive to output", func(t *testing.T) { + for _, loc := range packageLocations(s, "marker-pkg") { + assert.True(t, utf8.ValidString(loc)) + } + }) + + t.Run("a CONTAINS relationship links each archive to what was found inside", func(t *testing.T) { + var containers []string + for _, p := range s.Artifacts.Packages.Sorted() { + if p.Name != "marker-pkg" { + continue + } + for _, rel := range s.Relationships { + if rel.Type != artifact.ContainsRelationship || rel.To.ID() != p.ID() { + continue + } + if coords, ok := rel.From.(file.Coordinates); ok { + containers = append(containers, coords.FullNestedPath()) + } + } + } + sort.Strings(containers) + assert.Equal(t, []string{ + "KEPT.RPM", + "app.zip", + "app/data.zip", + "outer.zip", + "outer.zip!/deeper/inner.zip", + "outer.zip!/日本語/café-🎉/unicode.zip", + }, containers, "the archive file, named in its own parent, contains the package") + }) + + t.Run("each file within an archive is contained by that archive", func(t *testing.T) { + containers := fileContainers(s) + assert.Equal(t, "app.zip", containers["app.zip!/nested/keep.txt"]) + assert.Equal(t, "outer.zip!/deeper/inner.zip", containers["outer.zip!/deeper/inner.zip!/nested/marker.txt"]) + assert.Equal(t, "outer.zip", containers["outer.zip!/deeper/inner.zip"]) + }) + + t.Run("exclusions are the scan's own", func(t *testing.T) { + // there is no archive-specific exclusion setting: a scan pattern's shape decides whether it + // reaches inside an archive, and an excluded entry is absent from the archive's filesystem + // rather than present and skipped + markers := packageLocations(s, "marker-pkg") + assert.NotContains(t, markers, "nested/marker.txt", "a root-anchored pattern excludes the file at the scan root") + assert.Contains(t, markers, "app.zip!/nested/marker.txt", "and says nothing about the same path inside an archive") + assert.NotContains(t, markers, "pkg.rpm!/nested/marker.txt", "an archive whose own path is excluded is not extracted") + assert.Contains(t, markers, "KEPT.RPM!/nested/marker.txt", "a lower-case pattern does not match an upper-case name") + assert.NotContains(t, markers, "outer.zip!/skip/inner-excluded.zip!/nested/marker.txt", "a nested archive is excluded at its own level") + + assert.Contains(t, seen, "app.zip!/nested/keep.txt") + assert.NotContains(t, seen, "app.zip!/nested/excluded.txt", "a **/ pattern excludes inside an archive, from the index itself") + assert.Contains(t, seen, "outer.zip!/deeper/inner.zip", "the probe ran inside outer.zip") + assert.NotContains(t, seen, "outer.zip!/skip/inner-excluded.zip", "the excluded inner archive is absent from outer's filesystem") + }) + + t.Run("a jar inside an archive is cataloged exactly once, under the virtual path java gives it", func(t *testing.T) { + // the failure mode is a duplicate: a jar inside a zip can be reached by the task's walk and by + // the java cataloger's own unarchiving. The colon-delimited virtual path appears in published + // SBOMs, so it must not change with the mechanism; for a directory source the archive's path + // is relative, so the chain has no leading separator + assert.Equal(t, map[string][]string{ + "war-lib": {"app.war:WEB-INF/lib/war-lib-2.0.jar"}, + "zip-lib": {"bundle.zip:lib/zip-lib-1.2.jar"}, + "tgz-lib": {"bundle.tar.gz:lib/tgz-lib-1.2.jar"}, + }, javaVirtualPaths(s), "with both legacy search settings off, every container is still descended") + }) + + t.Run("no extraction path reaches the SBOM", func(t *testing.T) { + // covers syft/archive-content-identity#nested-paths-are-archive-relative: file cataloging is + // the one path that enumerates every location rather than searching for them, so a resolver + // root that leaked would land in FileMetadata and gain a CONTAINS edge + var paths []string + for coords := range s.Artifacts.FileMetadata { + paths = append(paths, coords.RealPath) + } + for _, p := range s.Artifacts.Packages.Sorted() { + for _, loc := range p.Locations.ToSlice() { + paths = append(paths, loc.RealPath, loc.AccessPath) + } + } + for _, rel := range s.Relationships { + for _, end := range []artifact.Identifiable{rel.From, rel.To} { + if coords, ok := end.(file.Coordinates); ok { + paths = append(paths, coords.RealPath) + } + } + } + require.NotEmpty(t, paths) + for _, p := range paths { + assert.NotContains(t, p, "archive-spill-", "the spill file reached the SBOM at %q", p) + if filepath.IsAbs(p) { + // the directory resolver has always reported the scan root absolute: it is the path the + // user asked for. Anything else absolute is a filesystem this feature invented + assert.Equal(t, resolvedScanDir, p, "only the scanned directory itself may be reported absolute") + } + } + }) + + t.Run("file metadata covers every entry of every archive, and nothing else", func(t *testing.T) { + // an exact set is also the reproducibility check: anything of a per-run temp directory reaching + // output would fail it + var got []string + for coords := range s.Artifacts.FileMetadata { + if archive.ContainerPath(coords) == "" { + continue + } + got = append(got, coords.FullNestedPath()) + } + sort.Strings(got) + assert.Equal(t, []string{ + "KEPT.RPM!/nested/marker.txt", + "app.war!/WEB-INF/lib/war-lib-2.0.jar", + "app.war!/WEB-INF/lib/war-lib-2.0.jar!/META-INF/MANIFEST.MF", + "app.zip!/nested/keep.txt", + "app.zip!/nested/marker.txt", + "app/data.zip!/nested/marker.txt", + "bundle.tar.gz!/lib/tgz-lib-1.2.jar", + "bundle.tar.gz!/lib/tgz-lib-1.2.jar!/META-INF/MANIFEST.MF", + "bundle.zip!/lib/zip-lib-1.2.jar", + "bundle.zip!/lib/zip-lib-1.2.jar!/META-INF/MANIFEST.MF", + "outer.zip!/deeper/inner.zip", + "outer.zip!/deeper/inner.zip!/nested/marker.txt", + "outer.zip!/outer/marker.txt", + "outer.zip!/日本語/café-🎉/unicode.zip", + "outer.zip!/日本語/café-🎉/unicode.zip!/nested/marker.txt", + }, got) + }) + + t.Run("file catalogers publish one progress row counting archive contents", func(t *testing.T) { + // the file catalogers run once over the scan root and again over every archive; each must + // report into one row carrying the total of every run, or the count runs past the total and + // renders as an overshot bar that reads as complete mid-walk + rows := map[string][]*progress.Manual{} + for _, e := range recorder.events { + if e.Type != event.CatalogerTaskStarted { + continue + } + info, ok := e.Source.(monitor.GenericTask) + require.True(t, ok) + tp, ok := e.Value.(*monitor.TaskProgress) + require.True(t, ok) + rows[info.Title.Default] = append(rows[info.Title.Default], tp.Manual) + } + for _, title := range []string{"File digests", "File metadata"} { + published := rows[title] + require.Len(t, published, 1, "%s must publish exactly one row for the whole scan", title) + row := published[0] + assert.Greater(t, row.Current(), int64(len(s.Artifacts.FileMetadata))/2, + "%s must count the files inside the archives, got %d", title, row.Current()) + if row.Size() >= 0 { + assert.Equal(t, row.Current(), row.Size(), "%s total must cover every run that reported into it", title) + } + } + }) +} + +// TestArchiveCataloger_featureOff is the one comparison that needs a second scan: with the task +// disabled, java's own unarchiving reaches the jars its legacy settings admit, under the same virtual +// path the task gives them. +func TestArchiveCataloger_featureOff(t *testing.T) { + scanDir := writeFixture(t) + + cfg := DefaultCreateSBOMConfig(). + WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")) + + s := scanDirWithExclusions(t, scanDir, cfg, fixtureExclusions...) + + // indexed archives (zip, war) are searched by default and unindexed ones (tar.gz) are not, so + // what a user gets without the feature depends on settings the feature makes irrelevant + assert.Equal(t, map[string][]string{ + "war-lib": {"app.war:WEB-INF/lib/war-lib-2.0.jar"}, + "zip-lib": {"bundle.zip:lib/zip-lib-1.2.jar"}, + }, javaVirtualPaths(s), "java's own recursion finds each once, under the virtual path the task also reports") + + t.Run("the top-level archive search booleans reach java", func(t *testing.T) { + cfg := DefaultCreateSBOMConfig(). + WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")). + WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithIncludeUnindexedArchives(true)) + + s := scanDirWithExclusions(t, scanDir, cfg, fixtureExclusions...) + + assert.Contains(t, javaVirtualPaths(s), "tgz-lib") + assert.False(t, cfg.Packages.JavaArchive.IncludeUnindexedArchives, "the caller's config is not written to") + }) +} + +func TestArchiveCataloger_jarReachedThroughALinkIsIdentified(t *testing.T) { + // a content-addressed layout (pnpm, nix, bazel): the jar is stored under a hash and only the link + // to it is named like a jar + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + jar := testutils.Jar(t, "json-simple", "1.1.1", nil) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "store/3f9a1c", Mode: 0o644, Size: int64(len(jar))})) + _, err := tw.Write(jar) + require.NoError(t, err) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "app/lib/json-simple-1.1.1.jar", Typeflag: tar.TypeSymlink, Linkname: "../../store/3f9a1c"})) + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "bundle.tar.gz"), buf.Bytes(), 0o644)) + + cfg := DefaultCreateSBOMConfig(). + WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")). + WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.AllFilesSelection)). + WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2)) + s := scanDirWithExclusions(t, dir, cfg) + + assert.Equal(t, []string{"bundle.tar.gz!/store/3f9a1c"}, packageLocations(s, "json-simple")) + + // the jar's own files are addressed by its real path, the same one its package location names + manifest := file.Coordinates{RealPath: "bundle.tar.gz", FullPath: "bundle.tar.gz!/store/3f9a1c!/META-INF/MANIFEST.MF"} + assert.Contains(t, s.Artifacts.FileMetadata, manifest) +} + +func TestArchiveCataloger_defaultFileSelectionCoversRootJars(t *testing.T) { + // with the default owned-by-package selection, a jar at the root is described by the archive task, + // so the root file catalogers must run after it or the jar gets no file entries at all + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "app.jar"), testutils.Jar(t, "app", "1.0", nil), 0o644)) + + for name, depth := range map[string]int{"feature off": 0, "feature on": 1} { + t.Run(name, func(t *testing.T) { + cfg := DefaultCreateSBOMConfig(). + WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")). + WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth)) + s := scanDirWithExclusions(t, dir, cfg) + + require.Equal(t, []string{"app.jar"}, packageLocations(s, "app")) + jar := file.NewCoordinates("app.jar", "") + assert.Contains(t, s.Artifacts.FileMetadata, jar) + assert.Contains(t, s.Artifacts.FileDigests, jar) + }) + } +} + +func TestArchiveCataloger_defaultFileSelectionCoversNestedJars(t *testing.T) { + // a nested jar's package is found inside the jar but located in the archive holding it, so that + // archive's file catalogers must run after it, as the root's do + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "outer.zip"), testutils.Zip(t, map[string][]byte{ + "lib/inner.jar": testutils.Jar(t, "inner", "1.0", nil), + "readme.txt": []byte("hello"), + "empty.zip": testutils.Zip(t, map[string]string{"a.txt": "hello"}), + }), 0o644)) + + cfg := DefaultCreateSBOMConfig(). + WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")). + WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2)) + s := scanDirWithExclusions(t, dir, cfg) + + require.Equal(t, []string{"outer.zip!/lib/inner.jar"}, packageLocations(s, "inner")) + jar := file.Coordinates{RealPath: "outer.zip", FullPath: "outer.zip!/lib/inner.jar"} + assert.Contains(t, s.Artifacts.FileMetadata, jar) + assert.Contains(t, s.Artifacts.FileDigests, jar) + assert.Equal(t, "outer.zip", fileContainers(s)["outer.zip!/lib/inner.jar"]) + + // as at the root, a file no package owns is not described, nor is an archive in which nothing was found + var described []string + for _, c := range s.AllCoordinates() { + described = append(described, c.FullNestedPath()) + } + assert.NotContains(t, described, "outer.zip!/readme.txt") + assert.NotContains(t, described, "outer.zip!/empty.zip") + assert.NotContains(t, described, "outer.zip!/empty.zip!/a.txt") +} + +func TestArchiveCataloger_archiveUnknowns(t *testing.T) { + dir := t.TempDir() + plain := testutils.Zip(t, map[string]string{"readme.txt": "hi"}) + require.NoError(t, os.WriteFile(filepath.Join(dir, "outer.zip"), testutils.Zip(t, map[string][]byte{ + "lib/inner.jar": testutils.Jar(t, "inner", "1.0", nil), + "lib/plain.zip": plain, + }), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "plain.zip"), plain, 0o644)) + + scan := func(depth int) map[string][]string { + cfg := DefaultCreateSBOMConfig(). + WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")). + WithUnknownsConfig(cataloging.DefaultUnknownsConfig()). + WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.AllFilesSelection)). + WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth)) + out := map[string][]string{} + for c, reasons := range scanDirWithExclusions(t, dir, cfg).Artifacts.Unknowns { + out[c.FullNestedPath()] = reasons + } + return out + } + + t.Run("an expanded archive is not reported as not cataloged", func(t *testing.T) { + assert.Empty(t, scan(3)) + }) + + t.Run("an archive past the depth limit is reported once, by the archive cataloger", func(t *testing.T) { + unknowns := scan(1) + assert.NotContains(t, unknowns, "outer.zip") + for _, nested := range []string{"outer.zip!/lib/inner.jar", "outer.zip!/lib/plain.zip"} { + require.Len(t, unknowns[nested], 1, nested) + assert.Contains(t, unknowns[nested][0], "depth limit was reached") + } + }) + + t.Run("an archive is reported as not cataloged when nesting is off", func(t *testing.T) { + assert.Equal(t, []string{"archive not cataloged"}, scan(0)["plain.zip"]) + }) +} + +func TestArchiveCataloger_unsupportedArchiveFormatsAreReported(t *testing.T) { + dir := t.TempDir() + var page bytes.Buffer + gw := gzip.NewWriter(&page) + _, err := gw.Write([]byte("a man page")) + require.NoError(t, err) + require.NoError(t, gw.Close()) + require.NoError(t, os.WriteFile(filepath.Join(dir, "outer.zip"), testutils.Zip(t, map[string][]byte{ + "legacy.7z": []byte("7z\xbc\xaf\x27\x1c\x00\x04"), + "page.1.gz": page.Bytes(), + }), 0o644)) + + cfg := DefaultCreateSBOMConfig(). + WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")). + WithUnknownsConfig(cataloging.DefaultUnknownsConfig()). + WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.AllFilesSelection)). + WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3)) + unknowns := map[string][]string{} + for c, reasons := range scanDirWithExclusions(t, dir, cfg).Artifacts.Unknowns { + unknowns[c.FullNestedPath()] = reasons + } + + assert.Equal(t, map[string][]string{ + "outer.zip!/legacy.7z": {"archive-cataloger: archive not cataloged: archive format is not supported: application/x-7z-compressed"}, + }, unknowns, "a bare compressed file holds no files, so it is not an archive left uncataloged") +} + +func TestArchiveCataloger_javaParentLinkageMatchesTheLegacyPath(t *testing.T) { + // a consumer walking java DEPENDENCY_OF edges (the only structure SPDX and CycloneDX keep) must see + // the same war-to-jar linkage whether or not the archive task does the recursion + dir := t.TempDir() + war := testutils.Jar(t, "app", "1.0", map[string][]byte{ + "WEB-INF/lib/lib-2.0.jar": testutils.Jar(t, "lib", "2.0", nil), + }) + require.NoError(t, os.WriteFile(filepath.Join(dir, "app.war"), war, 0o644)) + + for name, depth := range map[string]int{"feature off": 0, "feature on": 2} { + t.Run(name, func(t *testing.T) { + cfg := DefaultCreateSBOMConfig(). + WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")). + WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth)) + s := scanDirWithExclusions(t, dir, cfg) + + var edges []string + for _, rel := range s.Relationships { + from, fromOK := rel.From.(pkg.Package) + to, toOK := rel.To.(pkg.Package) + if fromOK && toOK && rel.Type == artifact.DependencyOfRelationship { + edges = append(edges, from.Name+" -> "+to.Name) + } + } + assert.Equal(t, []string{"lib -> app"}, edges) + + var parent string + for _, p := range s.Artifacts.Packages.Sorted() { + if m, ok := p.Metadata.(pkg.JavaArchive); ok && p.Name == "lib" && m.Parent != nil { + parent = m.Parent.Name + } + } + assert.Equal(t, "app", parent) + }) + } +} + +func TestArchiveCataloger_descriptorRecordsArchiveConfig(t *testing.T) { + // a consumer must be able to tell from the SBOM alone how deep archive contents were searched + cfg := DefaultCreateSBOMConfig(). + WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")). + WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3)) + s := scanDirWithExclusions(t, t.TempDir(), cfg) + + trail, ok := s.Descriptor.Configuration.(configurationAuditTrail) + require.True(t, ok) + assert.Equal(t, 3, trail.Archive.MaxDepth) + assert.Contains(t, trail.Catalogers.Used, task.ArchiveCatalogerTaskName) +} + +func TestArchiveCataloger_javaArchiveDepthIsAConfigError(t *testing.T) { + // depth and limits are not java settings; setting them on java's deprecated copy must fail fast + // rather than be silently ignored + for name, set := range map[string]func(*cataloging.ArchiveSearchConfig){ + "max depth": func(c *cataloging.ArchiveSearchConfig) { c.MaxDepth = 3 }, + "max memory bytes": func(c *cataloging.ArchiveSearchConfig) { c.MaxMemoryBytes = 1 }, + "max disk bytes": func(c *cataloging.ArchiveSearchConfig) { c.MaxDiskBytes = 1 }, + } { + t.Run(name, func(t *testing.T) { + supplied := java.DefaultArchiveCatalogerConfig() + supplied.ArchiveSearchConfig = cataloging.ArchiveSearchConfig{} + set(&supplied.ArchiveSearchConfig) + cfg := DefaultCreateSBOMConfig().WithPackagesConfig(pkgcataloging.Config{JavaArchive: supplied}) + + src, err := directorysource.New(directorysource.Config{Path: t.TempDir()}) + require.NoError(t, err) + t.Cleanup(func() { _ = src.Close() }) + + _, err = cfg.Create(context.Background(), src) + require.ErrorContains(t, err, "set them with CreateSBOMConfig.Archive") + }) + } +} + +// packageLocations lists the full path of every location of every package with the name. +func packageLocations(s *sbom.SBOM, name string) []string { + var out []string + for _, p := range s.Artifacts.Packages.Sorted() { + if p.Name != name { + continue + } + for _, loc := range p.Locations.ToSlice() { + out = append(out, loc.FullNestedPath()) + } + } + sort.Strings(out) + return out +} + +// javaVirtualPaths maps each java package name to the virtual paths it was reported under; a name +// with two entries was cataloged twice. +func javaVirtualPaths(s *sbom.SBOM) map[string][]string { + out := map[string][]string{} + for _, p := range s.Artifacts.Packages.Sorted() { + if metadata, ok := p.Metadata.(pkg.JavaArchive); ok { + out[p.Name] = append(out[p.Name], metadata.VirtualPath) + } + } + return out +} + +// markerCataloger is a stub package cataloger emitting one package per "marker.txt" it finds, so the +// end-to-end tests assert behavior without depending on any real cataloger's fixture format. +type markerCataloger struct{} + +func (markerCataloger) Name() string { return "marker-cataloger" } + +func (markerCataloger) Catalog(_ context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { + locations, err := resolver.FilesByGlob("**/marker.txt") + if err != nil { + return nil, nil, err + } + var pkgs []pkg.Package + for _, loc := range locations { + p := pkg.Package{ + Name: "marker-pkg", + Version: "1.0.0", + Type: pkg.BinaryPkg, + Locations: file.NewLocationSet(loc), + } + p.SetID() + pkgs = append(pkgs, p) + } + return pkgs, nil, nil +} + +// resolverProbeCataloger reports what the resolver it was handed can see, which is how "absent from +// the index" is told from "present and skipped" within a scan. +type resolverProbeCataloger struct { + globs []string + record func([]string) +} + +func (resolverProbeCataloger) Name() string { return "resolver-probe-cataloger" } + +func (c resolverProbeCataloger) Catalog(_ context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { + locations, err := resolver.FilesByGlob(c.globs...) + if err != nil { + return nil, nil, err + } + var paths []string + for _, loc := range locations { + paths = append(paths, loc.FullNestedPath()) + } + c.record(paths) + return nil, nil, nil +} + +// recordingBus captures the progress rows a scan publishes. +type recordingBus struct { + mu sync.Mutex + events []partybus.Event +} + +func (p *recordingBus) Publish(e partybus.Event) { + p.mu.Lock() + defer p.mu.Unlock() + p.events = append(p.events, e) +} + +// fileContainers maps the full path of each file an archive CONTAINS to the archive's full path. +func fileContainers(s *sbom.SBOM) map[string]string { + out := map[string]string{} + for _, r := range s.Relationships { + from, fromFile := r.From.(file.Coordinates) + to, toFile := r.To.(file.Coordinates) + if r.Type == artifact.ContainsRelationship && fromFile && toFile { + out[to.FullNestedPath()] = from.FullNestedPath() + } + } + return out +} + +// scanDirWithExclusions scans with the given exclusion patterns configured on the source, where +// exclusions live; archive cataloging has no exclusion setting of its own. +func scanDirWithExclusions(t *testing.T, dir string, cfg *CreateSBOMConfig, exclude ...string) *sbom.SBOM { + t.Helper() + src, err := directorysource.New(directorysource.Config{ + Path: dir, + Exclude: source.ExcludeConfig{Paths: exclude}, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = src.Close() }) + + s, err := cfg.Create(context.Background(), src) + require.NoError(t, err) + return s +} diff --git a/syft/cataloging/archive_search.go b/syft/cataloging/archive_search.go index 56dff6cc6..a404e513b 100644 --- a/syft/cataloging/archive_search.go +++ b/syft/cataloging/archive_search.go @@ -1,17 +1,49 @@ package cataloging +import "github.com/anchore/stereoscope/pkg/file" + +const ( + // DefaultArchiveMaxDepth is 0: nested archives are not cataloged unless asked for. + DefaultArchiveMaxDepth = 0 + + // DefaultArchiveMaxMemoryBytes bounds the archive content held in memory at once. It decides only + // whether an archive costs a write: content that does not fit goes to disk and is cataloged anyway. + DefaultArchiveMaxMemoryBytes = 2 * file.GB + + // DefaultArchiveMaxDiskBytes bounds the archive content on disk at once. Content that does not fit + // has nowhere to go, so the archive holding it is skipped or truncated. Set high as a backstop + // against a pathological archive filling the disk. + DefaultArchiveMaxDiskBytes = 100 * file.GB +) + type ArchiveSearchConfig struct { // IncludeIndexedArchives indicates whether to search within indexed archive files (e.g., .zip). IncludeIndexedArchives bool `yaml:"include-indexed-archives" json:"include-indexed-archives" mapstructure:"include-indexed-archives"` // IncludeUnindexedArchives indicates whether to search within unindexed archive files (e.g., .tar*). IncludeUnindexedArchives bool `yaml:"include-unindexed-archives" json:"include-unindexed-archives" mapstructure:"include-unindexed-archives"` + + // MaxDepth is how many levels of nested archives to catalog: 0 disables it, and no setting goes deeper than 16 levels + // (negative goes that deep). + MaxDepth int `yaml:"max-depth" json:"max-depth" mapstructure:"max-depth"` + + // MaxMemoryBytes bounds the archive content held in memory at once; content that does not fit is + // written to disk. Zero holds nothing in memory, negative is unbounded. + MaxMemoryBytes int64 `yaml:"max-memory-bytes" json:"max-memory-bytes" mapstructure:"max-memory-bytes"` + + // MaxDiskBytes bounds the archive content written to disk at once; an archive whose content does + // not fit is skipped or truncated. Zero writes nothing to disk, negative is unbounded. MaxMemoryBytes + // and MaxDiskBytes cannot both be zero while MaxDepth is not. + MaxDiskBytes int64 `yaml:"max-disk-bytes" json:"max-disk-bytes" mapstructure:"max-disk-bytes"` } func DefaultArchiveSearchConfig() ArchiveSearchConfig { return ArchiveSearchConfig{ IncludeIndexedArchives: true, IncludeUnindexedArchives: false, + MaxDepth: DefaultArchiveMaxDepth, + MaxMemoryBytes: DefaultArchiveMaxMemoryBytes, + MaxDiskBytes: DefaultArchiveMaxDiskBytes, } } @@ -24,3 +56,18 @@ func (c ArchiveSearchConfig) WithIncludeUnindexedArchives(include bool) ArchiveS c.IncludeUnindexedArchives = include return c } + +func (c ArchiveSearchConfig) WithMaxDepth(depth int) ArchiveSearchConfig { + c.MaxDepth = depth + return c +} + +func (c ArchiveSearchConfig) WithMaxMemoryBytes(size int64) ArchiveSearchConfig { + c.MaxMemoryBytes = size + return c +} + +func (c ArchiveSearchConfig) WithMaxDiskBytes(size int64) ArchiveSearchConfig { + c.MaxDiskBytes = size + return c +} diff --git a/syft/cataloging/archive_search_test.go b/syft/cataloging/archive_search_test.go new file mode 100644 index 000000000..46d88feb3 --- /dev/null +++ b/syft/cataloging/archive_search_test.go @@ -0,0 +1,36 @@ +package cataloging + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDefaultArchiveSearchConfig(t *testing.T) { + cfg := DefaultArchiveSearchConfig() + + assert.True(t, cfg.IncludeIndexedArchives) + assert.False(t, cfg.IncludeUnindexedArchives) + assert.Equal(t, DefaultArchiveMaxDepth, cfg.MaxDepth) + assert.Equal(t, int64(DefaultArchiveMaxMemoryBytes), cfg.MaxMemoryBytes) + assert.Equal(t, int64(DefaultArchiveMaxDiskBytes), cfg.MaxDiskBytes) +} + +func TestArchiveSearchConfig_WithMethods(t *testing.T) { + cfg := DefaultArchiveSearchConfig() + + cfg = cfg.WithMaxDepth(5) + assert.Equal(t, 5, cfg.MaxDepth) + + cfg = cfg.WithMaxMemoryBytes(1024) + assert.Equal(t, int64(1024), cfg.MaxMemoryBytes) + + cfg = cfg.WithMaxDiskBytes(2048) + assert.Equal(t, int64(2048), cfg.MaxDiskBytes) + + cfg = cfg.WithIncludeIndexedArchives(false) + assert.False(t, cfg.IncludeIndexedArchives) + + cfg = cfg.WithIncludeUnindexedArchives(true) + assert.True(t, cfg.IncludeUnindexedArchives) +} diff --git a/syft/configuration_audit_trail.go b/syft/configuration_audit_trail.go index aa7a51f4d..f5f456689 100644 --- a/syft/configuration_audit_trail.go +++ b/syft/configuration_audit_trail.go @@ -16,6 +16,7 @@ type configurationAuditTrail struct { DataGeneration cataloging.DataGenerationConfig `json:"data-generation" yaml:"data-generation" mapstructure:"data-generation"` Packages pkgcataloging.Config `json:"packages" yaml:"packages" mapstructure:"packages"` Files filecataloging.Config `json:"files" yaml:"files" mapstructure:"files"` + Archive cataloging.ArchiveSearchConfig `json:"archive" yaml:"archive" mapstructure:"archive"` Licenses cataloging.LicenseConfig `json:"licenses" yaml:"licenses" mapstructure:"licenses"` Catalogers catalogerManifest `json:"catalogers" yaml:"catalogers" mapstructure:"catalogers"` ExtraConfigs any `json:"extra,omitempty" yaml:"extra" mapstructure:"extra"` diff --git a/syft/create_sbom.go b/syft/create_sbom.go index 817e698c6..22c35e26f 100644 --- a/syft/create_sbom.go +++ b/syft/create_sbom.go @@ -10,6 +10,7 @@ import ( "github.com/scylladb/go-set/strset" "github.com/anchore/go-sync" + "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/licenses" "github.com/anchore/syft/internal/log" @@ -38,7 +39,7 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) ( srcMetadata := src.Describe() - taskGroups, audit, err := cfg.makeTaskGroups(srcMetadata) + taskGroups, audit, err := cfg.makeTaskGroups(srcMetadata, sourceExclusions(src)) if err != nil { return nil, err } @@ -57,8 +58,9 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) ( Search: cfg.Search, Relationships: cfg.Relationships, DataGeneration: cfg.DataGeneration, - Packages: cfg.Packages, + Packages: cfg.packagesWithArchiveSearch(), Files: cfg.Files, + Archive: cfg.Archive, Licenses: cfg.Licenses, Catalogers: *audit, ExtraConfigs: cfg.ToolConfiguration, @@ -88,8 +90,8 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) ( } } - catalogingProgress := monitorCatalogingTask(src.ID(), taskGroups) - packageCatalogingProgress := monitorPackageCatalogingTask() + catalogingProgress := monitorCatalogingTask(ctx, src.ID(), taskGroups) + packageCatalogingProgress := monitorPackageCatalogingTask(ctx) builder := sbomsync.NewBuilder(&s, monitorPackageCount(packageCatalogingProgress)) for i := range taskGroups { @@ -112,6 +114,14 @@ func setupContext(ctx context.Context, cfg *CreateSBOMConfig) (context.Context, // configure parallel executors ctx = setContextExecutors(ctx, cfg) + // one progress row per cataloger for the whole scan, however many archives it is run against + ctx = bus.WithCatalogerTaskRegistry(ctx) + + // the archive cataloger task recurses into jars too, so the java cataloger must not unarchive them itself + if cfg.Archive.MaxDepth != 0 { + ctx = archive.WithNestedCataloging(ctx) + } + // configure temp dir factory for catalogers (if not already set) if tmpdir.FromContext(ctx) == nil { ctx, _ = tmpdir.Root(ctx, "syft-cataloger") @@ -173,7 +183,7 @@ func monitorPackageCount(prog *monitor.TaskProgress) func(s *sbom.SBOM) { } } -func monitorPackageCatalogingTask() *monitor.TaskProgress { +func monitorPackageCatalogingTask(ctx context.Context) *monitor.TaskProgress { info := monitor.GenericTask{ Title: monitor.Title{ Default: "Packages", @@ -183,10 +193,10 @@ func monitorPackageCatalogingTask() *monitor.TaskProgress { ParentID: monitor.TopLevelCatalogingTaskID, } - return bus.StartCatalogerTask(info, -1, "") + return bus.StartCatalogerTask(ctx, info, -1, "") } -func monitorCatalogingTask(srcID artifact.ID, tasks [][]task.Task) *monitor.TaskProgress { +func monitorCatalogingTask(ctx context.Context, srcID artifact.ID, tasks [][]task.Task) *monitor.TaskProgress { info := monitor.GenericTask{ Title: monitor.Title{ Default: "Catalog contents", @@ -203,7 +213,7 @@ func monitorCatalogingTask(srcID artifact.ID, tasks [][]task.Task) *monitor.Task length += int64(len(tg)) } - return bus.StartCatalogerTask(info, length, "") + return bus.StartCatalogerTask(ctx, info, length, "") } func formatTaskNames(tasks []task.Task) []string { @@ -218,3 +228,12 @@ func formatTaskNames(tasks []task.Task) []string { sort.Strings(list) return list } + +// sourceExclusions returns the exclusion patterns the source was configured with, so the same +// exclusions apply inside the archives found in it. +func sourceExclusions(src source.Source) []string { + if excluder, ok := src.(source.PathExcluder); ok { + return excluder.ExcludedPaths() + } + return nil +} diff --git a/syft/create_sbom_config.go b/syft/create_sbom_config.go index 1a84c14b6..b3137f238 100644 --- a/syft/create_sbom_config.go +++ b/syft/create_sbom_config.go @@ -28,6 +28,7 @@ type CreateSBOMConfig struct { Packages pkgcataloging.Config Licenses cataloging.LicenseConfig Files filecataloging.Config + Archive cataloging.ArchiveSearchConfig Parallelism int CatalogerSelection cataloging.SelectionRequest @@ -49,6 +50,7 @@ func DefaultCreateSBOMConfig() *CreateSBOMConfig { Packages: pkgcataloging.DefaultConfig(), Licenses: cataloging.DefaultLicenseConfig(), Files: filecataloging.DefaultConfig(), + Archive: cataloging.DefaultArchiveSearchConfig(), Parallelism: 0, // use default: run in parallel based on number of CPUs packageTaskFactories: task.DefaultPackageTaskFactories(), @@ -144,6 +146,13 @@ func (c *CreateSBOMConfig) WithFilesConfig(cfg filecataloging.Config) *CreateSBO return c } +// WithArchiveConfig sets how archives are cataloged: how deep to recurse into nested archives (MaxDepth +// 0, the default, does not recurse) and how much extracted content the scan may hold at once. +func (c *CreateSBOMConfig) WithArchiveConfig(cfg cataloging.ArchiveSearchConfig) *CreateSBOMConfig { + c.Archive = cfg + return c +} + // WithoutFiles allows for disabling file cataloging altogether. func (c *CreateSBOMConfig) WithoutFiles() *CreateSBOMConfig { c.Files = filecataloging.Config{ @@ -183,14 +192,15 @@ func (c *CreateSBOMConfig) WithCatalogers(catalogerRefs ...pkgcataloging.Catalog // groups, where each task in a group can be run concurrently, while tasks in different groups must be run serially. // The final set of task groups is returned along with a cataloger manifest that describes the catalogers that were // selected and the tokens that were sensitive to this selection (both for adding and removing from the final set). -func (c *CreateSBOMConfig) makeTaskGroups(src source.Description) ([][]task.Task, *catalogerManifest, error) { +// +// exclusions are the source's exclusion patterns, which the archive cataloger applies inside each archive. +func (c *CreateSBOMConfig) makeTaskGroups(src source.Description, exclusions []string) ([][]task.Task, *catalogerManifest, error) { var taskGroups [][]task.Task // generate package and file tasks based on the configuration environmentTasks := c.environmentTasks() scopeTasks := c.scopeTasks() relationshipsTasks := c.relationshipTasks(src) - unknownTasks := c.unknownsTasks() osFeatureDetectionTasks := c.osFeatureDetectionTasks() pkgTasks, fileTasks, selectionEvidence, err := c.selectTasks(src) @@ -198,11 +208,29 @@ func (c *CreateSBOMConfig) makeTaskGroups(src source.Description) ([][]task.Task return nil, nil, err } + // the same package and file catalogers run against the contents of every archive, after the scan + // root and before relationship and unknowns post-processing + archiveTask := task.NewArchiveCatalogerTask(c.Archive, c.Unknowns, pkgTasks, fileTasks, exclusions) + + // the archive cataloger reports the archives it does not expand, which the labeler can only guess at + unknowns := c.Unknowns + if archiveTask != nil { + unknowns.IncludeUnexpandedArchives = false + } + unknownTasks := unknownsTasks(unknowns) + // combine the user-provided and configured tasks - if c.Files.Selection == file.FilesOwnedByPackageSelection { + switch { + case c.Files.Selection == file.FilesOwnedByPackageSelection && archiveTask != nil: + // the archive task describes archives at the root too (java leaves them to it), so the root file + // catalogers must run after it to see those packages + taskGroups = append(taskGroups, pkgTasks, []task.Task{archiveTask}, fileTasks) + case c.Files.Selection == file.FilesOwnedByPackageSelection: // special case: we need the package info when we are cataloging files owned by packages taskGroups = append(taskGroups, pkgTasks, fileTasks) - } else { + case archiveTask != nil: + taskGroups = append(taskGroups, append(pkgTasks, fileTasks...), []task.Task{archiveTask}) + default: taskGroups = append(taskGroups, append(pkgTasks, fileTasks...)) } @@ -226,6 +254,11 @@ func (c *CreateSBOMConfig) makeTaskGroups(src source.Description) ([][]task.Task taskGroups = append(taskGroups, osFeatureDetectionTasks) } + // archives are related to the files within them only once every task has decided which files are described + if archiveTask != nil { + taskGroups = append(taskGroups, []task.Task{task.NewArchiveContainsTask()}) + } + // identifying the environment (i.e. the linux release) must be done first as this is required for package cataloging taskGroups = append( [][]task.Task{ @@ -237,6 +270,9 @@ func (c *CreateSBOMConfig) makeTaskGroups(src source.Description) ([][]task.Task var allTasks []task.Task allTasks = append(allTasks, pkgTasks...) allTasks = append(allTasks, fileTasks...) + if archiveTask != nil { + allTasks = append(allTasks, archiveTask) + } return taskGroups, &catalogerManifest{ Requested: selectionEvidence.Request, @@ -260,7 +296,7 @@ func (c *CreateSBOMConfig) selectTasks(src source.Description) ([]task.Task, []t SearchConfig: c.Search, RelationshipsConfig: c.Relationships, DataGenerationConfig: c.DataGeneration, - PackagesConfig: c.Packages, + PackagesConfig: c.packagesWithArchiveSearch(), LicenseConfig: c.Licenses, ComplianceConfig: c.Compliance, FilesConfig: c.Files, @@ -440,10 +476,10 @@ func (c *CreateSBOMConfig) environmentTasks() []task.Task { // unknownsTasks returns a set of tasks that perform any necessary post-processing // to identify SBOM elements as unknowns -func (c *CreateSBOMConfig) unknownsTasks() []task.Task { +func unknownsTasks(cfg cataloging.UnknownsConfig) []task.Task { var tasks []task.Task - if t := task.NewUnknownsLabelerTask(c.Unknowns); t != nil { + if t := task.NewUnknownsLabelerTask(cfg); t != nil { tasks = append(tasks, t) } @@ -461,12 +497,36 @@ func (c *CreateSBOMConfig) osFeatureDetectionTasks() []task.Task { return tasks } +// packagesWithArchiveSearch returns a copy of the packages config with the top-level archive search +// booleans applied to java, which is still the only cataloger that reads them. +// +// ponytail: a top-level boolean wins only when it differs from the default, so callers who set just +// java's deprecated copy see no change. Delete this along with that field. +func (c *CreateSBOMConfig) packagesWithArchiveSearch() pkgcataloging.Config { + pkgs := c.Packages + def := cataloging.DefaultArchiveSearchConfig() + java := &pkgs.JavaArchive.ArchiveSearchConfig + if c.Archive.IncludeIndexedArchives != def.IncludeIndexedArchives { + java.IncludeIndexedArchives = c.Archive.IncludeIndexedArchives + } + if c.Archive.IncludeUnindexedArchives != def.IncludeUnindexedArchives { + java.IncludeUnindexedArchives = c.Archive.IncludeUnindexedArchives + } + return pkgs +} + func (c *CreateSBOMConfig) validate() error { if c.Relationships.ExcludeBinaryPackagesWithFileOwnershipOverlap { if !c.Relationships.PackageFileOwnershipOverlap { return fmt.Errorf("invalid configuration: to exclude binary packages based on file ownership overlap relationships, cataloging file ownership overlap relationships must be enabled") } } + if j := c.Packages.JavaArchive.ArchiveSearchConfig; j.MaxDepth != 0 || j.MaxMemoryBytes != 0 || j.MaxDiskBytes != 0 { + return fmt.Errorf("invalid configuration: nested archive depth and limits are not java cataloger settings, set them with CreateSBOMConfig.Archive") + } + if a := c.Archive; a.MaxDepth != 0 && a.MaxMemoryBytes == 0 && a.MaxDiskBytes == 0 { + return fmt.Errorf("invalid configuration: nested archive cataloging is enabled but its memory and disk limits are both 0, leaving nowhere to extract archives") + } return nil } diff --git a/syft/create_sbom_config_test.go b/syft/create_sbom_config_test.go index 0c12922f7..4b2dd9a46 100644 --- a/syft/create_sbom_config_test.go +++ b/syft/create_sbom_config_test.go @@ -18,6 +18,7 @@ import ( "github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/java" "github.com/anchore/syft/syft/source" ) @@ -329,7 +330,7 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) { require.NotEmpty(t, tt.wantTaskNames) // test the subject - gotTasks, gotManifest, err := tt.cfg.makeTaskGroups(tt.src) + gotTasks, gotManifest, err := tt.cfg.makeTaskGroups(tt.src, nil) tt.wantErr(t, err) if err != nil { return @@ -554,6 +555,31 @@ func TestCreateSBOMConfig_validate(t *testing.T) { ), wantErr: assert.Error, }, + { + name: "nested archives enabled with no memory or disk", + cfg: DefaultCreateSBOMConfig().WithArchiveConfig( + cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1).WithMaxMemoryBytes(0).WithMaxDiskBytes(0), + ), + wantErr: assert.Error, + }, + { + name: "nested archives enabled with memory only", + cfg: DefaultCreateSBOMConfig().WithArchiveConfig( + cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1).WithMaxDiskBytes(0), + ), + }, + { + name: "nested archives enabled with disk only", + cfg: DefaultCreateSBOMConfig().WithArchiveConfig( + cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1).WithMaxMemoryBytes(0), + ), + }, + { + name: "nested archives disabled with no memory or disk", + cfg: DefaultCreateSBOMConfig().WithArchiveConfig( + cataloging.DefaultArchiveSearchConfig().WithMaxMemoryBytes(0).WithMaxDiskBytes(0), + ), + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -564,3 +590,64 @@ func TestCreateSBOMConfig_validate(t *testing.T) { }) } } + +func TestCreateSBOMConfig_archiveTaskGroup(t *testing.T) { + src := source.Description{Metadata: source.DirectoryMetadata{}} + + archiveTaskIn := func(groups [][]task.Task) bool { + for _, group := range groups { + for _, tsk := range group { + if tsk.Name() == task.ArchiveCatalogerTaskName { + return true + } + } + } + return false + } + + t.Run("no archive task by default", func(t *testing.T) { + // the feature is off by default, and off must mean no task at all rather than a registered + // task that does nothing: a registered task is reported as a selected cataloger + groups, manifest, err := DefaultCreateSBOMConfig().makeTaskGroups(src, nil) + require.NoError(t, err) + assert.False(t, archiveTaskIn(groups)) + assert.NotContains(t, manifest.Used, task.ArchiveCatalogerTaskName) + }) + + t.Run("archive task is present at negative depth", func(t *testing.T) { + cfg := DefaultCreateSBOMConfig(). + WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(-1)) + groups, _, err := cfg.makeTaskGroups(src, nil) + require.NoError(t, err) + assert.True(t, archiveTaskIn(groups)) + }) + + t.Run("archive task is present when enabled", func(t *testing.T) { + cfg := DefaultCreateSBOMConfig(). + WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2)) + groups, manifest, err := cfg.makeTaskGroups(src, nil) + require.NoError(t, err) + assert.True(t, archiveTaskIn(groups)) + assert.Contains(t, manifest.Used, task.ArchiveCatalogerTaskName, "the manifest must say archive contents were searched") + }) + + t.Run("enabling the feature does not mutate the caller's package config", func(t *testing.T) { + // the java side learns the archive task is running from the context, not from its config: a + // consumer who handed us a package config gets it back exactly as supplied. + supplied := java.DefaultArchiveCatalogerConfig().WithUseNetwork(true) + + cfg := DefaultCreateSBOMConfig(). + WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2)). + WithPackagesConfig(pkgcataloging.Config{JavaArchive: supplied}) + + _, _, err := cfg.makeTaskGroups(src, nil) + require.NoError(t, err) + + assert.Equal(t, supplied, cfg.Packages.JavaArchive, + "the caller's java config must come back byte-identical") + assert.True(t, cfg.Packages.JavaArchive.UseNetwork, + "a caller-supplied field must survive untouched") + assert.Zero(t, cfg.Packages.JavaArchive.MaxDepth, + "java's own squash-inlined depth must not be written to either") + }) +} diff --git a/syft/file/cataloger/executable/cataloger.go b/syft/file/cataloger/executable/cataloger.go index 552698f48..7929a80eb 100644 --- a/syft/file/cataloger/executable/cataloger.go +++ b/syft/file/cataloger/executable/cataloger.go @@ -64,11 +64,11 @@ func (i *Cataloger) CatalogCtx(ctx context.Context, resolver file.Resolver) (map return nil, err } - prog := catalogingProgress(int64(len(locs))) + prog := catalogingProgress(ctx, int64(len(locs))) results := make(map[file.Coordinates]file.Executable) errs := sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(locs), func(loc file.Location) (*file.Executable, error) { - prog.AtomicStage.Set(loc.Path()) + prog.AtomicStage.Set(loc.FullNestedPath()) exec, err := processExecutableLocation(loc, resolver) if err != nil { @@ -93,15 +93,15 @@ func (i *Cataloger) CatalogCtx(ctx context.Context, resolver file.Resolver) (map func processExecutableLocation(loc file.Location, resolver file.Resolver) (exec *file.Executable, err error) { reader, err := resolver.FileContentsByLocation(loc) if err != nil { - log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get file contents") + log.WithFields("error", err, "path", loc.FullNestedPath()).Debug("unable to get file contents") return nil, fmt.Errorf("unable to get file contents: %w", err) } - defer internal.CloseAndLogError(reader, loc.RealPath) + defer internal.CloseAndLogError(reader, loc.FullNestedPath()) defer func() { if r := recover(); r != nil { // a malformed binary must not take down the whole SBOM; the caller records this as an unknown - log.WithFields("path", loc.RealPath, "panic", r).Debug("recovered from panic while reading executable") + log.WithFields("path", loc.FullNestedPath(), "panic", r).Debug("recovered from panic while reading executable") log.Tracef("executable panic stack:\n%s", debug.Stack()) exec, err = nil, fmt.Errorf("recovered from panic while reading executable: %v", r) } @@ -109,14 +109,14 @@ func processExecutableLocation(loc file.Location, resolver file.Resolver) (exec uReader, err := unionreader.GetUnionReader(reader) if err != nil { - log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get union reader") + log.WithFields("error", err, "path", loc.FullNestedPath()).Debug("unable to get union reader") return nil, fmt.Errorf("unable to get union reader: %w", err) } return processExecutable(loc, uReader) } -func catalogingProgress(locations int64) *monitor.TaskProgress { +func catalogingProgress(ctx context.Context, locations int64) *monitor.TaskProgress { info := monitor.GenericTask{ Title: monitor.Title{ Default: "Executables", @@ -124,7 +124,7 @@ func catalogingProgress(locations int64) *monitor.TaskProgress { ParentID: monitor.TopLevelCatalogingTaskID, } - return bus.StartCatalogerTask(info, locations, "") + return bus.StartCatalogerTask(ctx, info, locations, "") } func filterByGlobs(locs []file.Location, globs []string) ([]file.Location, error) { @@ -146,7 +146,7 @@ func filterByGlobs(locs []file.Location, globs []string) ([]file.Location, error func locationMatchesGlob(loc file.Location, globs []string) (bool, error) { for _, glob := range globs { - for _, path := range []string{loc.RealPath, loc.AccessPath} { + for _, path := range []string{loc.LocalPath(), loc.AccessPath} { if path == "" { continue } @@ -169,13 +169,13 @@ func processExecutable(loc file.Location, reader unionreader.UnionReader) (*file format, err := findExecutableFormat(reader) if err != nil { - log.Debugf("unable to determine executable kind for %v: %v", loc.RealPath, err) + log.Debugf("unable to determine executable kind for %v: %v", loc.FullNestedPath(), err) return nil, fmt.Errorf("unable to determine executable kind: %w", err) } if format == "" { // this is not an "unknown", so just log -- this binary does not have parseable data in it - log.Debugf("unable to determine executable format for %q", loc.RealPath) + log.Debugf("unable to determine executable format for %q", loc.FullNestedPath()) return nil, nil } @@ -184,17 +184,17 @@ func processExecutable(loc file.Location, reader unionreader.UnionReader) (*file switch format { case file.ELF: if err = findELFFeatures(&data, reader); err != nil { - log.WithFields("error", err, "path", loc.RealPath).Trace("unable to determine ELF features") + log.WithFields("error", err, "path", loc.FullNestedPath()).Trace("unable to determine ELF features") err = fmt.Errorf("unable to determine ELF features: %w", err) } case file.PE: if err = findPEFeatures(&data, reader); err != nil { - log.WithFields("error", err, "path", loc.RealPath).Trace("unable to determine PE features") + log.WithFields("error", err, "path", loc.FullNestedPath()).Trace("unable to determine PE features") err = fmt.Errorf("unable to determine PE features: %w", err) } case file.MachO: if err = findMachoFeatures(&data, reader); err != nil { - log.WithFields("error", err, "path", loc.RealPath).Trace("unable to determine Macho features") + log.WithFields("error", err, "path", loc.FullNestedPath()).Trace("unable to determine Macho features") err = fmt.Errorf("unable to determine Macho features: %w", err) } } diff --git a/syft/file/cataloger/filecontent/cataloger.go b/syft/file/cataloger/filecontent/cataloger.go index 5e57a94e8..4b70492da 100644 --- a/syft/file/cataloger/filecontent/cataloger.go +++ b/syft/file/cataloger/filecontent/cataloger.go @@ -44,7 +44,7 @@ func NewCataloger(cfg Config) *Cataloger { } } -func (i *Cataloger) Catalog(_ context.Context, resolver file.Resolver) (map[file.Coordinates]string, error) { +func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) (map[file.Coordinates]string, error) { results := make(map[file.Coordinates]string) var locations []file.Location var errs error @@ -54,10 +54,10 @@ func (i *Cataloger) Catalog(_ context.Context, resolver file.Resolver) (map[file return nil, err } - prog := catalogingProgress(int64(len(locations))) + prog := catalogingProgress(ctx, int64(len(locations))) for _, location := range locations { - prog.AtomicStage.Set(location.Path()) + prog.AtomicStage.Set(location.FullNestedPath()) metadata, err := resolver.FileMetadataByLocation(location) if err != nil { @@ -98,12 +98,12 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati if err != nil { return "", err } - defer internal.CloseAndLogError(contentReader, location.AccessPath) + defer internal.CloseAndLogError(contentReader, location.FullNestedPath()) buf := &bytes.Buffer{} encoder := base64.NewEncoder(base64.StdEncoding, buf) if _, err = io.Copy(encoder, contentReader); err != nil { - return "", internal.ErrPath{Context: "content-cataloger", Path: location.RealPath, Err: err} + return "", internal.ErrPath{Context: "content-cataloger", Path: location.FullNestedPath(), Err: err} } // note: it's important to close the reader before reading from the buffer since closing will flush the remaining bytes if err := encoder.Close(); err != nil { @@ -113,7 +113,7 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati return buf.String(), nil } -func catalogingProgress(locations int64) *monitor.TaskProgress { +func catalogingProgress(ctx context.Context, locations int64) *monitor.TaskProgress { info := monitor.GenericTask{ Title: monitor.Title{ Default: "File contents", @@ -121,5 +121,5 @@ func catalogingProgress(locations int64) *monitor.TaskProgress { ParentID: monitor.TopLevelCatalogingTaskID, } - return bus.StartCatalogerTask(info, locations, "") + return bus.StartCatalogerTask(ctx, info, locations, "") } diff --git a/syft/file/cataloger/filedigest/cataloger.go b/syft/file/cataloger/filedigest/cataloger.go index bee56a9e1..47f6e7e26 100644 --- a/syft/file/cataloger/filedigest/cataloger.go +++ b/syft/file/cataloger/filedigest/cataloger.go @@ -42,9 +42,9 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin locations = intCataloger.AllRegularFiles(ctx, resolver) } else { for _, c := range coordinates { - locs, err := resolver.FilesByPath(c.RealPath) + locs, err := resolver.FilesByPath(c.FullNestedPath()) if err != nil { - return nil, fmt.Errorf("unable to get file locations for path %q: %w", c.RealPath, err) + return nil, fmt.Errorf("unable to get file locations for path %q: %w", c.FullNestedPath(), err) } locations = append(locations, locs...) } @@ -62,7 +62,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin } locations = unique - prog := catalogingProgress(int64(len(locations))) + prog := catalogingProgress(ctx, int64(len(locations))) err := sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(locations), func(location file.Location) ([]file.Digest, error) { result, err := i.catalogLocation(ctx, resolver, location) @@ -71,10 +71,10 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin return nil, nil } - prog.AtomicStage.Set(location.Path()) + prog.AtomicStage.Set(location.FullNestedPath()) if internal.IsErrPathPermission(err) { - log.Debugf("file digests cataloger skipping %q: %+v", location.RealPath, err) + log.Debugf("file digests cataloger skipping %q: %+v", location.FullNestedPath(), err) return nil, unknown.New(location, err) } @@ -115,12 +115,12 @@ func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver, if err != nil { return nil, err } - defer internal.CloseAndLogError(contentReader, location.AccessPath) + defer internal.CloseAndLogError(contentReader, location.FullNestedPath()) defer func() { if r := recover(); r != nil { // the caller records this as an unknown for the location instead of failing the whole SBOM - log.WithFields("path", location.RealPath, "panic", r).Debug("recovered from panic while digesting file") + log.WithFields("path", location.FullNestedPath(), "panic", r).Debug("recovered from panic while digesting file") log.Tracef("file digest panic stack:\n%s", debug.Stack()) digests, err = nil, fmt.Errorf("recovered from panic while digesting file: %v", r) } @@ -128,13 +128,13 @@ func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver, digests, err = intFile.NewDigestsFromFile(ctx, contentReader, i.hashes) if err != nil { - return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err} + return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.FullNestedPath(), Err: err} } return digests, nil } -func catalogingProgress(locations int64) *monitor.TaskProgress { +func catalogingProgress(ctx context.Context, locations int64) *monitor.TaskProgress { info := monitor.GenericTask{ Title: monitor.Title{ Default: "File digests", @@ -142,5 +142,5 @@ func catalogingProgress(locations int64) *monitor.TaskProgress { ParentID: monitor.TopLevelCatalogingTaskID, } - return bus.StartCatalogerTask(info, locations, "") + return bus.StartCatalogerTask(ctx, info, locations, "") } diff --git a/syft/file/cataloger/filemetadata/cataloger.go b/syft/file/cataloger/filemetadata/cataloger.go index f22383a6e..76dff8d83 100644 --- a/syft/file/cataloger/filemetadata/cataloger.go +++ b/syft/file/cataloger/filemetadata/cataloger.go @@ -34,7 +34,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin go func() { defer close(ch) for _, c := range coordinates { - locs, err := resolver.FilesByPath(c.RealPath) + locs, err := resolver.FilesByPath(c.FullNestedPath()) if err != nil { errs = unknown.Append(errs, c, err) continue @@ -53,9 +53,9 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin }() } - prog := catalogingProgress(-1) + prog := catalogingProgress(ctx, -1) for location := range locations { - prog.AtomicStage.Set(location.Path()) + prog.AtomicStage.Set(location.FullNestedPath()) metadata, err := resolver.FileMetadataByLocation(location) if err != nil { @@ -76,7 +76,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin return results, errs } -func catalogingProgress(locations int64) *monitor.TaskProgress { +func catalogingProgress(ctx context.Context, locations int64) *monitor.TaskProgress { info := monitor.GenericTask{ Title: monitor.Title{ Default: "File metadata", @@ -84,5 +84,5 @@ func catalogingProgress(locations int64) *monitor.TaskProgress { ParentID: monitor.TopLevelCatalogingTaskID, } - return bus.StartCatalogerTask(info, locations, "") + return bus.StartCatalogerTask(ctx, info, locations, "") } diff --git a/syft/file/cataloger/internal/all_regular_files.go b/syft/file/cataloger/internal/all_regular_files.go index d653132fa..3c55a0f1c 100644 --- a/syft/file/cataloger/internal/all_regular_files.go +++ b/syft/file/cataloger/internal/all_regular_files.go @@ -12,16 +12,16 @@ func AllRegularFiles(ctx context.Context, resolver file.Resolver) (locations []f ctx, cancel := context.WithCancel(ctx) defer cancel() for location := range resolver.AllLocations(ctx) { - resolvedLocations, err := resolver.FilesByPath(location.RealPath) + resolvedLocations, err := resolver.FilesByPath(location.LocalPath()) if err != nil { - log.Debugf("unable to resolve %+v: %+v", location, err) + log.Debugf("unable to resolve %q: %+v", location.FullNestedPath(), err) continue } for _, resolvedLocation := range resolvedLocations { metadata, err := resolver.FileMetadataByLocation(resolvedLocation) if err != nil { - log.Debugf("unable to get metadata for %+v: %+v", location, err) + log.Debugf("unable to get metadata for %q: %+v", resolvedLocation.FullNestedPath(), err) continue } diff --git a/syft/file/coordinate_set.go b/syft/file/coordinate_set.go index 5342f6de2..b275cf6ad 100644 --- a/syft/file/coordinate_set.go +++ b/syft/file/coordinate_set.go @@ -78,6 +78,9 @@ func (s CoordinateSet) ToSlice(sorters ...func(a, b Coordinates) int) []Coordina if !sorted { sort.SliceStable(coordinates, func(i, j int) bool { if coordinates[i].FileSystemID == coordinates[j].FileSystemID { + if coordinates[i].RealPath == coordinates[j].RealPath { + return coordinates[i].FullNestedPath() < coordinates[j].FullNestedPath() + } return coordinates[i].RealPath < coordinates[j].RealPath } return coordinates[i].FileSystemID < coordinates[j].FileSystemID diff --git a/syft/file/coordinates.go b/syft/file/coordinates.go index afeda5b13..3645c534d 100644 --- a/syft/file/coordinates.go +++ b/syft/file/coordinates.go @@ -2,18 +2,46 @@ package file import ( "fmt" + "strings" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" ) +const ArchivePathSeparator = "!/" + // Coordinates contains the minimal information needed to describe how to find a file within any possible source object (e.g. image and directory sources) type Coordinates struct { - // RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). + // RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem. RealPath string `json:"path" cyclonedx:"path"` - // FileSystemID is an ID representing and entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. + // FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in. FileSystemID string `json:"layerID,omitempty" cyclonedx:"layerID"` + + // FullPath is the full path to the file: RealPath, then for a file within an archive a "!/"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. "/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it is blank, so coordinates outside archives are unchanged by the field (in JSON, equality and ID). + FullPath string `json:"fullPath,omitempty" cyclonedx:"fullPath"` +} + +// FullNestedPath is FullPath, or RealPath where FullPath is blank. +func (c Coordinates) FullNestedPath() string { + if c.FullPath == "" { + return c.RealPath + } + return c.FullPath +} + +// LocalPath is the rooted path within the resolver holding the file: the last entry path in FullPath, or +// RealPath for a file not within an archive. RealPath is cut off before looking for a separator, so a +// "!/" in a real path is never taken for an archive boundary. +func (c Coordinates) LocalPath() string { + entries, ok := strings.CutPrefix(c.FullPath, c.RealPath+ArchivePathSeparator) + if !ok { + return c.RealPath + } + if i := strings.LastIndex(entries, ArchivePathSeparator); i >= 0 { + entries = entries[i+len(ArchivePathSeparator):] + } + return "/" + entries } func NewCoordinates(realPath, fsID string) Coordinates { @@ -40,9 +68,22 @@ func (c Coordinates) String() string { if c.FileSystemID != "" { str += fmt.Sprintf(" Layer=%q", c.FileSystemID) } + + if c.FullNestedPath() != c.RealPath { + str += fmt.Sprintf(" FullPath=%q", c.FullNestedPath()) + } return fmt.Sprintf("Location<%s>", str) } func (c Coordinates) GetCoordinates() Coordinates { return c } + +// HashInclude keeps a FullPath that adds nothing to RealPath out of the artifact ID hash (see +// artifact.IDByHash), so coordinates outside archives keep the IDs they had before the field existed. +func (c Coordinates) HashInclude(field string, _ any) (bool, error) { + if field == "FullPath" && c.FullNestedPath() == c.RealPath { + return false, nil + } + return true, nil +} diff --git a/syft/file/location.go b/syft/file/location.go index 76678c40f..14256ffa3 100644 --- a/syft/file/location.go +++ b/syft/file/location.go @@ -33,7 +33,7 @@ type LocationData struct { // note: it is IMPORTANT to ignore anything but the coordinates for a Location when considering the ID (hash value) // since the coordinates are the minimally correct ID for a location (symlinks should not come into play) - // AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path) + // AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path). For a file within an archive, it is a full path (see Coordinates.FullPath). AccessPath string `hash:"ignore" json:"accessPath"` // ref is the stereoscope file reference relative to the stereoscope.FileCatalog that has more information about this location. @@ -110,12 +110,12 @@ func NewVirtualLocation(realPath, accessPath string) Location { }} } -// NewLocationFromCoordinates creates a new location for the given Coordinates. +// NewLocationFromCoordinates creates a new location for the given Coordinates, accessed by their full path. func NewLocationFromCoordinates(coordinates Coordinates) Location { return Location{ LocationData: LocationData{ Coordinates: coordinates, - AccessPath: coordinates.RealPath, + AccessPath: coordinates.FullNestedPath(), }, LocationMetadata: LocationMetadata{ Annotations: map[string]string{}, @@ -205,11 +205,13 @@ func NewVirtualLocationFromDirectory(responsePath, responseAccessPath string, re } } +// Path is the path this location was accessed by, or its full path (see Coordinates.FullNestedPath). For a +// file within an archive both are full paths, so use LocalPath for path arithmetic against the resolver. func (l Location) Path() string { if l.AccessPath != "" { return l.AccessPath } - return l.RealPath + return l.FullNestedPath() } func (l Location) String() string { @@ -220,10 +222,14 @@ func (l Location) String() string { str += fmt.Sprintf("RealPath=%q", l.RealPath) - if l.AccessPath != "" && l.AccessPath != l.RealPath { + if l.AccessPath != "" { str += fmt.Sprintf(" AccessPath=%q", l.AccessPath) } + if l.FullNestedPath() != l.RealPath { + str += fmt.Sprintf(" FullPath=%q", l.FullNestedPath()) + } + if l.FileSystemID != "" { str += fmt.Sprintf(" Layer=%q", l.FileSystemID) } @@ -231,7 +237,7 @@ func (l Location) String() string { } func (l Location) Equals(other Location) bool { - return l.RealPath == other.RealPath && + return l.FullNestedPath() == other.FullNestedPath() && l.AccessPath == other.AccessPath && l.FileSystemID == other.FileSystemID } diff --git a/syft/file/location_set.go b/syft/file/location_set.go index 52beb9e51..fc6603c8a 100644 --- a/syft/file/location_set.go +++ b/syft/file/location_set.go @@ -5,6 +5,7 @@ import ( "sort" "github.com/gohugoio/hashstructure" + "github.com/scylladb/go-set/strset" "github.com/anchore/syft/internal/log" ) @@ -112,8 +113,13 @@ func (s *LocationSet) Empty() bool { } func (s LocationSet) Hash() (uint64, error) { - // access paths and filesystem IDs are not considered when hashing a location set, only the real paths - return hashstructure.Hash(s.CoordinateSet().Paths(), &hashstructure.HashOptions{ + // access paths and filesystem IDs are not considered when hashing a location set, only the full paths + // (unlike CoordinateSet.Paths, which ownership-by-overlap needs as real paths only) + paths := strset.New() + for _, c := range s.CoordinateSet().ToUnorderedSlice() { + paths.Add(c.FullNestedPath()) + } + return hashstructure.Hash(paths.List(), &hashstructure.HashOptions{ ZeroNil: true, SlicesAsSets: true, }) diff --git a/syft/file/location_set_test.go b/syft/file/location_set_test.go index b80028325..37f62a13f 100644 --- a/syft/file/location_set_test.go +++ b/syft/file/location_set_test.go @@ -4,6 +4,7 @@ import ( "sort" "testing" + "github.com/gohugoio/hashstructure" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -255,6 +256,10 @@ func TestLocationSet_Hash(t *testing.T) { }, } + inA := NewLocationFromCoordinates(Coordinates{RealPath: "/app.war", FullPath: "/app.war!/a.jar!/package.json"}) + inB := NewLocationFromCoordinates(Coordinates{RealPath: "/app.war", FullPath: "/app.war!/b.jar!/package.json"}) + etcFull := Location{LocationData: LocationData{Coordinates: Coordinates{RealPath: "/etc/hosts", FileSystemID: "a", FullPath: "/etc/hosts"}}} + tests := []struct { name string setA LocationSet @@ -297,6 +302,18 @@ func TestLocationSet_Hash(t *testing.T) { setB: NewLocationSet(binB, etcB), want: assert.Equal, }, + { + name: "same outer archive but different nested entries have different hashes", + setA: NewLocationSet(inA), + setB: NewLocationSet(inB), + want: assert.NotEqual, + }, + { + name: "a full path equal to the real path has the same hash as a blank one", + setA: NewLocationSet(etcA), + setB: NewLocationSet(etcFull), + want: assert.Equal, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -308,3 +325,13 @@ func TestLocationSet_Hash(t *testing.T) { }) } } + +// IDs outside archives must not change from when only real paths were hashed +func TestLocationSet_Hash_unchangedOutsideArchives(t *testing.T) { + want, err := hashstructure.Hash([]string{"/etc/hosts", "/bin"}, &hashstructure.HashOptions{ZeroNil: true, SlicesAsSets: true}) + require.NoError(t, err) + + got, err := NewLocationSet(NewLocation("/etc/hosts"), NewVirtualLocation("/bin", "/usr/bin")).Hash() + require.NoError(t, err) + assert.Equal(t, want, got) +} diff --git a/syft/file/location_test.go b/syft/file/location_test.go index 36626955b..5302c951a 100644 --- a/syft/file/location_test.go +++ b/syft/file/location_test.go @@ -1,9 +1,11 @@ package file import ( + "encoding/json" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/anchore/stereoscope/pkg/file" ) @@ -48,3 +50,73 @@ func TestLocation_ID(t *testing.T) { } } + +func TestLocation_fullPathIsIdentity(t *testing.T) { + a := NewLocationFromCoordinates(Coordinates{RealPath: "/a.jar", FullPath: "/a.jar!/META-INF/MANIFEST.MF"}) + b := NewLocationFromCoordinates(Coordinates{RealPath: "/b.jar", FullPath: "/b.jar!/META-INF/MANIFEST.MF"}) + + assert.False(t, a.Equals(b)) + assert.NotEqual(t, a.String(), b.String()) + assert.Contains(t, a.String(), `RealPath="/a.jar"`) +} + +func TestLocation_JSON(t *testing.T) { + tests := []struct { + name string + location Location + expected string + }{ + { + name: "a file outside an archive writes no fullPath", + location: NewVirtualLocationFromDirectory("/etc/os-release", "/etc/link", file.Reference{}).WithAnnotation("k", "v"), + expected: `{"path":"/etc/os-release","accessPath":"/etc/link","annotations":{"k":"v"}}`, + }, + { + name: "a file within an archive writes its full path", + location: NewLocationFromCoordinates(Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: "/app.war!/META-INF/MANIFEST.MF"}), + expected: `{"path":"/app.war","layerID":"layer","fullPath":"/app.war!/META-INF/MANIFEST.MF","accessPath":"/app.war!/META-INF/MANIFEST.MF"}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := json.Marshal(tt.location) + require.NoError(t, err) + assert.JSONEq(t, tt.expected, string(data)) + + var decoded Location + require.NoError(t, json.Unmarshal(data, &decoded)) + assert.Equal(t, tt.location.Coordinates, decoded.Coordinates, "decoding fills FullPath back in") + assert.Equal(t, tt.location.AccessPath, decoded.AccessPath) + + data, err = json.Marshal(tt.location.Coordinates) + require.NoError(t, err) + var coordinates Coordinates + require.NoError(t, json.Unmarshal(data, &coordinates)) + assert.Equal(t, tt.location.Coordinates, coordinates) + }) + } +} + +func TestCoordinates_ID_unchangedByTopLevelFullRealPath(t *testing.T) { + assert.Equal(t, Coordinates{RealPath: "/a"}.ID(), NewCoordinates("/a", "").ID()) + assert.NotEqual(t, NewCoordinates("/a", "").ID(), Coordinates{RealPath: "/a", FullPath: "/a!/b"}.ID()) +} + +func TestCoordinates_LocalPath(t *testing.T) { + assert.Equal(t, "/a.jar", NewCoordinates("/a.jar", "").LocalPath()) + assert.Equal(t, "/a.jar", Coordinates{RealPath: "/a.jar"}.LocalPath()) + assert.Equal(t, "/META-INF/MANIFEST.MF", Coordinates{RealPath: "/a.war", FullPath: "/a.war!/lib/b.jar!/META-INF/MANIFEST.MF"}.LocalPath()) + assert.Equal(t, "/data/Hello!/world.txt", NewCoordinates("/data/Hello!/world.txt", "").LocalPath(), "a separator in a real path is not an archive boundary") + assert.Equal(t, "/data/Hello!/world.txt", Coordinates{RealPath: "/data/Hello!/world.txt", FullPath: "/data/Hello!/world.txt"}.LocalPath()) + assert.Equal(t, "/x.txt", Coordinates{RealPath: "/data/Hello!/a.zip", FullPath: "/data/Hello!/a.zip!/x.txt"}.LocalPath()) + assert.Equal(t, "/x.txt", Coordinates{RealPath: "/data/Hello!/a.zip", FullPath: "/data/Hello!/a.zip!/b.jar!/x.txt"}.LocalPath()) +} + +func TestLocation_Path(t *testing.T) { + nested := Coordinates{RealPath: "/a.war", FullPath: "/a.war!/lib/b.jar!/META-INF/MANIFEST.MF"} + assert.Equal(t, "/a.jar", NewLocation("/a.jar").Path()) + assert.Equal(t, "/link", NewVirtualLocation("/a.jar", "/link").Path()) + assert.Equal(t, nested.FullPath, NewLocationFromCoordinates(nested).Path()) + assert.Equal(t, nested.FullPath, Location{LocationData: LocationData{Coordinates: nested}}.Path(), "a nested location without an access path is still full") + assert.Equal(t, nested.FullPath, NewVirtualLocationFromCoordinates(nested, nested.FullPath).Path()) +} diff --git a/syft/file/locations.go b/syft/file/locations.go index 3f7be072d..7cc8c46c9 100644 --- a/syft/file/locations.go +++ b/syft/file/locations.go @@ -83,7 +83,12 @@ func LocationSorter(layers []string) func(a, b Location) int { //nolint:gocognit return strings.Compare(a.AccessPath, b.AccessPath) } - return strings.Compare(a.RealPath, b.RealPath) + if a.RealPath != b.RealPath { + return strings.Compare(a.RealPath, b.RealPath) + } + + // the same path in different archives is a different file + return strings.Compare(a.FullNestedPath(), b.FullNestedPath()) } } @@ -118,6 +123,11 @@ func CoordinatesSorter(layers []string) func(a, b Coordinates) int { } } - return strings.Compare(a.RealPath, b.RealPath) + if a.RealPath != b.RealPath { + return strings.Compare(a.RealPath, b.RealPath) + } + + // the same path in different archives is a different file + return strings.Compare(a.FullNestedPath(), b.FullNestedPath()) } } diff --git a/syft/file/locations_test.go b/syft/file/locations_test.go index 07e903d90..64b917dcb 100644 --- a/syft/file/locations_test.go +++ b/syft/file/locations_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" "github.com/anchore/syft/internal/evidence" ) @@ -555,3 +556,24 @@ func TestLocationAndCoordinatesSorters(t *testing.T) { }) } } + +func TestSorters_fullPathBreaksTies(t *testing.T) { + // the same path in two archives must sort the same way whatever order the inputs arrive in, or + // output built from map iteration is not reproducible + a := Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"} + b := Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/META-INF/MANIFEST.MF"} + want := []string{a.FullNestedPath(), b.FullNestedPath()} + + for _, in := range [][]Coordinates{{a, b}, {b, a}} { + coords := slices.Clone(in) + slices.SortFunc(coords, CoordinatesSorter(nil)) + assert.Equal(t, want, []string{coords[0].FullNestedPath(), coords[1].FullNestedPath()}, "CoordinatesSorter") + + locs := []Location{NewLocationFromCoordinates(in[0]), NewLocationFromCoordinates(in[1])} + slices.SortFunc(locs, LocationSorter(nil)) + assert.Equal(t, want, []string{locs[0].FullNestedPath(), locs[1].FullNestedPath()}, "LocationSorter") + + got := NewCoordinateSet(in...).ToSlice() + assert.Equal(t, want, []string{got[0].FullNestedPath(), got[1].FullNestedPath()}, "CoordinateSet.ToSlice") + } +} diff --git a/syft/file/mock_resolver.go b/syft/file/mock_resolver.go index 67388796e..c65429f3b 100644 --- a/syft/file/mock_resolver.go +++ b/syft/file/mock_resolver.go @@ -60,9 +60,9 @@ func NewMockResolverForPathsWithMetadata(metadata map[Coordinates]Metadata) *Moc l := NewLocationFromCoordinates(c) locations = append(locations, l) mimeTypeIndex[m.MIMEType] = append(mimeTypeIndex[m.MIMEType], l) - ext := path.Ext(l.RealPath) + ext := path.Ext(l.FullNestedPath()) extension[ext] = append(extension[ext], l) - bn := path.Base(l.RealPath) + bn := path.Base(l.FullNestedPath()) basename[bn] = append(basename[bn], l) } @@ -97,9 +97,9 @@ func NewMockResolverForMediaTypes(mediaTypes map[string][]Location) *MockResolve mediaTypeIndex[mediaType] = append(mediaTypeIndex[mediaType], locs...) for _, l := range locs { locations = append(locations, l) - ext := path.Ext(l.RealPath) + ext := path.Ext(l.FullNestedPath()) extension[ext] = append(extension[ext], l) - bn := path.Base(l.RealPath) + bn := path.Base(l.FullNestedPath()) basename[bn] = append(basename[bn], l) } } @@ -121,7 +121,7 @@ func (r MockResolver) ImageReference() string { // HasPath indicates if the given path exists in the underlying source. func (r MockResolver) HasPath(path string) bool { for _, l := range r.locations { - if l.RealPath == path { + if l.FullNestedPath() == path { return true } } @@ -142,7 +142,7 @@ func (r MockResolver) FileContentsByLocation(location Location) (io.ReadCloser, } } - return nil, fmt.Errorf("no file for location: %v", location) + return nil, fmt.Errorf("no file for location: %q", location.FullNestedPath()) } // FilesByPath returns all Locations that match the given paths. @@ -150,7 +150,7 @@ func (r MockResolver) FilesByPath(paths ...string) ([]Location, error) { var results []Location for _, p := range paths { for _, location := range r.locations { - if p == location.RealPath { + if p == location.LocalPath() { results = append(results, NewLocation(p)) } } @@ -164,7 +164,7 @@ func (r MockResolver) FilesByGlob(patterns ...string) ([]Location, error) { var results []Location for _, pattern := range patterns { for _, location := range r.locations { - matches, err := doublestar.Match(pattern, location.RealPath) + matches, err := doublestar.Match(pattern, location.LocalPath()) if err != nil { return nil, err } diff --git a/syft/format/common/cyclonedxhelpers/to_format_model.go b/syft/format/common/cyclonedxhelpers/to_format_model.go index 7cae828c6..3c381c1e4 100644 --- a/syft/format/common/cyclonedxhelpers/to_format_model.go +++ b/syft/format/common/cyclonedxhelpers/to_format_model.go @@ -14,6 +14,7 @@ import ( "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/file" + formatInternal "github.com/anchore/syft/syft/format/internal" "github.com/anchore/syft/syft/format/internal/cyclonedxutil/helpers" "github.com/anchore/syft/syft/linux" "github.com/anchore/syft/syft/pkg" @@ -42,7 +43,8 @@ func ToFormatModel(s sbom.SBOM) *cyclonedx.BOM { cdxBOM.SerialNumber = uuid.New().URN() cdxBOM.Metadata = toBomDescriptor(s.Descriptor.Name, s.Descriptor.Version, s.Source) - coordinates, locationSorter := getCoordinates(s) + locationSorter, coordinateSorter := formatInternal.GetLocationSorters(s) + top, entries := formatInternal.ArchiveEntries(s.AllCoordinates(), coordinateSorter) // Packages packages := s.Artifacts.Packages.Sorted() @@ -52,40 +54,7 @@ func ToFormatModel(s sbom.SBOM) *cyclonedx.BOM { } components = append(components, toOSComponent(s.Artifacts.LinuxDistribution)...) - artifacts := s.Artifacts - - for _, coordinate := range coordinates { - var metadata *file.Metadata - // File Info - fileMetadata, exists := artifacts.FileMetadata[coordinate] - // no file metadata then don't include in SBOM - // the syft config allows for sometimes only capturing files owned by packages - // so there can be a map miss here where we have less metadata than all coordinates - if !exists { - continue - } - if fileMetadata.Type == stfile.TypeDirectory || - fileMetadata.Type == stfile.TypeSocket || - fileMetadata.Type == stfile.TypeSymLink { - // skip dir, symlinks and sockets for the final bom - continue - } - metadata = &fileMetadata - - // Digests - var digests []file.Digest - if digestsForLocation, exists := artifacts.FileDigests[coordinate]; exists { - digests = digestsForLocation - } - - cdxHashes := digestsToHashes(digests) - components = append(components, cyclonedx.Component{ - BOMRef: string(coordinate.ID()), - Type: cyclonedx.ComponentTypeFile, - Name: metadata.Path, - Hashes: &cdxHashes, - }) - } + components = append(components, toFileComponents(s.Artifacts, entries, top)...) cdxBOM.Components = &components dependencies := toDependencies(s.Relationships) @@ -96,19 +65,41 @@ func ToFormatModel(s sbom.SBOM) *cyclonedx.BOM { return cdxBOM } -func getCoordinates(s sbom.SBOM) ([]file.Coordinates, func(a, b file.Location) int) { - var layers []string - if m, ok := s.Source.Metadata.(source.ImageMetadata); ok { - for _, l := range m.Layers { - layers = append(layers, l.Digest) +// toFileComponents describes each of coordinates that is a regular file with metadata, named by its path +// (within its archive, for a file within one), with the files within it nested in its component. +func toFileComponents(artifacts sbom.Artifacts, entries map[file.Coordinates][]file.Coordinates, coordinates []file.Coordinates) []cyclonedx.Component { + var components []cyclonedx.Component + for _, coordinate := range coordinates { + nested := toFileComponents(artifacts, entries, entries[coordinate]) + // no file metadata then don't include in SBOM + // the syft config allows for sometimes only capturing files owned by packages + // so there can be a map miss here where we have less metadata than all coordinates + metadata, exists := artifacts.FileMetadata[coordinate] + if !isFileComponent(metadata, exists) && len(nested) == 0 { + continue } + + cdxHashes := digestsToHashes(artifacts.FileDigests[coordinate]) + component := cyclonedx.Component{ + BOMRef: string(coordinate.ID()), + Type: cyclonedx.ComponentTypeFile, + Name: coordinate.LocalPath(), + Hashes: &cdxHashes, + } + if len(nested) > 0 { + component.Components = &nested + } + components = append(components, component) } + return components +} - coordSorter := file.CoordinatesSorter(layers) - coordinates := s.AllCoordinates() - - slices.SortFunc(coordinates, coordSorter) - return coordinates, file.LocationSorter(layers) +// isFileComponent skips dirs, symlinks and sockets for the final bom +func isFileComponent(metadata file.Metadata, exists bool) bool { + return exists && + metadata.Type != stfile.TypeDirectory && + metadata.Type != stfile.TypeSocket && + metadata.Type != stfile.TypeSymLink } func digestsToHashes(digests []file.Digest) []cyclonedx.Hash { diff --git a/syft/format/common/cyclonedxhelpers/to_format_model_test.go b/syft/format/common/cyclonedxhelpers/to_format_model_test.go index 5f94b9b57..f0014edee 100644 --- a/syft/format/common/cyclonedxhelpers/to_format_model_test.go +++ b/syft/format/common/cyclonedxhelpers/to_format_model_test.go @@ -608,3 +608,65 @@ func Test_toOSBomRef(t *testing.T) { }) } } + +func Test_nestedArchiveFiles(t *testing.T) { + top := file.NewCoordinates("/etc/os-release", "") + inA := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"} + inB := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt"} + inWar := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/index.html"} + dir := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/c.jar!/META-INF"} + sha := []file.Digest{{Algorithm: "sha256", Value: "abc"}} + s := sbom.SBOM{Artifacts: sbom.Artifacts{ + FileMetadata: map[file.Coordinates]file.Metadata{ + // a directory resolver's metadata path is the path on the host, not the one scanned + top: {Path: "/host/scan/etc/os-release"}, + inA: {}, + inB: {}, + inWar: {}, + dir: {Type: stfile.TypeDirectory}, + }, + FileDigests: map[file.Coordinates][]file.Digest{inA: sha}, + }} + + bom := ToFormatModel(s) + + type node struct { + Name string + Children []node + } + var tree func(cs *[]cyclonedx.Component) []node + tree = func(cs *[]cyclonedx.Component) []node { + if cs == nil { + return nil + } + var out []node + for _, c := range *cs { + require.Equal(t, cyclonedx.ComponentTypeFile, c.Type) + out = append(out, node{Name: c.Name, Children: tree(c.Components)}) + } + return out + } + // each archive holding a file is a component, synthesized when not otherwise described; an archive + // holding only skipped files is left out + assert.Equal(t, []node{ + {Name: "/app.war", Children: []node{ + {Name: "/WEB-INF/lib/a.jar", Children: []node{{Name: "/META-INF/MANIFEST.MF"}}}, + {Name: "/WEB-INF/lib/b.jar", Children: []node{{Name: "/odd:na%me.txt"}}}, + {Name: "/index.html"}, + }}, + {Name: "/etc/os-release"}, + }, tree(bom.Components)) + + decoded, err := helpers.ToSyftModel(bom) + require.NoError(t, err) + + war := file.NewCoordinates("/app.war", "") + a := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar"} + b := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar"} + var got []file.Coordinates + for c := range decoded.Artifacts.FileMetadata { + got = append(got, c) + } + assert.ElementsMatch(t, []file.Coordinates{top, war, a, b, inA, inB, inWar}, got) + assert.Equal(t, map[file.Coordinates][]file.Digest{inA: sha}, decoded.Artifacts.FileDigests) +} diff --git a/syft/format/common/spdxhelpers/to_format_model.go b/syft/format/common/spdxhelpers/to_format_model.go index 2a175ce9d..bd347ffe0 100644 --- a/syft/format/common/spdxhelpers/to_format_model.go +++ b/syft/format/common/spdxhelpers/to_format_model.go @@ -6,6 +6,7 @@ import ( "crypto/sha1" "fmt" "path" + "path/filepath" "slices" "sort" "strings" @@ -53,7 +54,9 @@ func ToFormatModel(s sbom.SBOM) *spdx.Document { rels := relationship.NewIndex(s.Relationships...) packages, otherLicenses := toPackages(rels, s.Artifacts.Packages, s) - allRelationships := toRelationships(rels.All()) + sourceFile := toSourceFile(s) + files, archiveRelationships := toFiles(s, sourceFile) + allRelationships := append(toRelationships(rels.All()), archiveRelationships...) // for valid SPDX we need a document describes relationship describesID := spdx.ElementID("DOCUMENT") @@ -62,8 +65,7 @@ func ToFormatModel(s sbom.SBOM) *spdx.Document { if rootPackage != nil { describesID = rootPackage.PackageSPDXIdentifier - // add all relationships from the document root to all other packages - allRelationships = append(allRelationships, toRootRelationships(rootPackage, packages)...) + allRelationships = append(allRelationships, toRootRelationships(rootPackage, sourceFile, packages, files, allRelationships)...) // append the root package packages = append(packages, rootPackage) @@ -155,25 +157,64 @@ func ToFormatModel(s sbom.SBOM) *spdx.Document { CreatorComment: "", }, Packages: packages, - Files: toFiles(s), + Files: files, Relationships: allRelationships, OtherLicenses: convertOtherLicense(otherLicenses), } } -func toRootRelationships(rootPackage *spdx.Package, packages []*spdx.Package) (out []*spdx.Relationship) { - for _, p := range packages { +// toRootRelationships relates the root package to every package, and to each file nothing else CONTAINS: what is +// within an archive is reached only through the archive's file, and an owned file through its package. When the +// source is itself a file, the root package DESCRIBES it, and a scanned directory CONTAINS those files instead. A +// package containing files must have analyzed them, so the root package then gets a verification code over them. +func toRootRelationships(rootPackage *spdx.Package, sourceFile *file.Coordinates, packages []*spdx.Package, files []*spdx.File, relationships []*spdx.Relationship) (out []*spdx.Relationship) { + contained := map[spdx.ElementID]struct{}{} + for _, r := range relationships { + if r.Relationship == string(helpers.ContainsRelationship) { + contained[r.RefB.ElementRefID] = struct{}{} + } + } + relate := func(a spdx.ElementID, typ helpers.RelationshipType, b spdx.ElementID) { out = append(out, &spdx.Relationship{ - RefA: spdx.DocElementID{ - ElementRefID: rootPackage.PackageSPDXIdentifier, - }, - Relationship: string(helpers.ContainsRelationship), - RefB: spdx.DocElementID{ - ElementRefID: p.PackageSPDXIdentifier, - }, + RefA: spdx.DocElementID{ElementRefID: a}, + Relationship: string(typ), + RefB: spdx.DocElementID{ElementRefID: b}, }) } - return + + fileContainer := rootPackage.PackageSPDXIdentifier + if sourceFile != nil { + id := toSPDXID(*sourceFile) + relate(rootPackage.PackageSPDXIdentifier, helpers.DescribesRelationship, id) + contained[id] = struct{}{} + if isScanRoot(*sourceFile) { + fileContainer = id + } + } + + for _, p := range packages { + relate(rootPackage.PackageSPDXIdentifier, helpers.ContainsRelationship, p.PackageSPDXIdentifier) + } + var sha1s []string + for _, f := range files { + if _, has := contained[f.FileSPDXIdentifier]; has { + continue + } + relate(fileContainer, helpers.ContainsRelationship, f.FileSPDXIdentifier) + if fileContainer != rootPackage.PackageSPDXIdentifier { + continue + } + for _, c := range f.Checksums { + if c.Algorithm == spdx.SHA1 { + sha1s = append(sha1s, c.Value) + break + } + } + } + + rootPackage.PackageVerificationCode = toVerificationCode(sha1s) + rootPackage.FilesAnalyzed = rootPackage.PackageVerificationCode != nil + return out } //nolint:funlen @@ -355,7 +396,7 @@ func toSPDXID(identifiable artifact.Identifiable) spdx.ElementID { return spdx.ElementID(helpers.SanitizeElementID(id)) } p := "" - parts := strings.Split(it.RealPath, "/") + parts := strings.Split(it.FullNestedPath(), "/") for i := len(parts); i > 0; i-- { part := parts[i-1] if len(part) == 0 { @@ -663,8 +704,11 @@ func toRelationships(relationships []artifact.Relationship) (result []*spdx.Rela continue } - // FIXME: we are only currently including Package -> * relationships - if _, ok := r.From.(pkg.Package); !ok { + // FIXME: we are only currently including Package -> * relationships, plus an archive containing a + // package found inside it, which is the only structure nested archive cataloging records for + // non-java packages. Every file-side From is in the document: toFiles covers every coordinate a + // relationship names, and itself relates each archive to the files within it. + if _, ok := r.From.(pkg.Package); !ok && !isFileContainsPackage(r) { log.Debugf("skipping non-package relationship: %+v", r) continue } @@ -683,6 +727,12 @@ func toRelationships(relationships []artifact.Relationship) (result []*spdx.Rela return result } +func isFileContainsPackage(r artifact.Relationship) bool { + _, fromFile := r.From.(file.Coordinates) + _, toPkg := r.To.(pkg.Package) + return fromFile && toPkg && r.Type == artifact.ContainsRelationship +} + func lookupRelationship(ty artifact.RelationshipType) (bool, helpers.RelationshipType, string) { switch ty { case artifact.ContainsRelationship: @@ -697,60 +747,60 @@ func lookupRelationship(ty artifact.RelationshipType) (bool, helpers.Relationshi return false, "", "" } -func toFiles(s sbom.SBOM) (results []*spdx.File) { - artifacts := s.Artifacts +const layerIDCommentPrefix = "layerID: " +// toFiles describes every file, including each archive holding one. A file within an archive is named by +// its path in that archive, and the archive CONTAINS it. +func toFiles(s sbom.SBOM, sourceFile *file.Coordinates) (results []*spdx.File, archiveRelationships []*spdx.Relationship) { _, coordinateSorter := formatInternal.GetLocationSorters(s) + top, entries := formatInternal.ArchiveEntries(s.AllCoordinates(), coordinateSorter) - coordinates := s.AllCoordinates() - slices.SortFunc(coordinates, coordinateSorter) - - for _, c := range coordinates { - var metadata *file.Metadata - if metadataForLocation, exists := artifacts.FileMetadata[c]; exists { - metadata = &metadataForLocation + // a file holding entries is an archive, whatever its content starts with (such as a launch script) + describe := func(c file.Coordinates, name string) *spdx.File { + f := toFile(s.Artifacts, c, name) + if len(entries[c]) > 0 { + f.FileTypes = withArchiveType(f.FileTypes) } + return f + } - var digests []file.Digest - if digestsForLocation, exists := artifacts.FileDigests[c]; exists { - digests = digestsForLocation + var addEntries func(archive file.Coordinates) + addEntries = func(archive file.Coordinates) { + for _, c := range entries[archive] { + archiveRelationships = append(archiveRelationships, &spdx.Relationship{ + RefA: spdx.DocElementID{ElementRefID: toSPDXID(archive)}, + Relationship: string(helpers.ContainsRelationship), + RefB: spdx.DocElementID{ElementRefID: toSPDXID(c)}, + }) + results = append(results, describe(c, strings.TrimPrefix(c.LocalPath(), "/"))) + addEntries(c) } - - // if we don't have any metadata or digests for this location - // then the file is most likely a symlink or non-regular file - // for now we include a 0 sha1 digest as requested by the spdx spec - // TODO: update location code in core SBOM so that we can map complex links - // back to their real file digest location. - if len(digests) == 0 { - digests = append(digests, file.Digest{Algorithm: "sha1", Value: "0000000000000000000000000000000000000000"}) - } - - // TODO: add file classifications (?) and content as a snippet - - var comment string - if c.FileSystemID != "" { - comment = fmt.Sprintf("layerID: %s", c.FileSystemID) - } - - relativePath, err := convertAbsoluteToRelative(c.RealPath) + } + for _, c := range top { + name, err := convertAbsoluteToRelative(c.RealPath) if err != nil { log.Debugf("unable to convert relative path '%s' to absolute path: %s", c.RealPath, err) - relativePath = c.RealPath + name = c.RealPath } - - results = append(results, &spdx.File{ - FileSPDXIdentifier: toSPDXID(c), - FileComment: comment, - // required, no attempt made to determine license information - LicenseConcluded: noAssertion, - FileCopyrightText: noAssertion, - Checksums: toFileChecksums(digests), - FileName: relativePath, - FileTypes: toFileTypes(metadata), - LicenseInfoInFiles: []string{ // required in SPDX 2.2 - helpers.NOASSERTION, - }, - }) + var archive *source.FileMetadata + if isScanRoot(c) { + // the scanned root is the root package, unless it is the file the source is + if sourceFile == nil || c != *sourceFile { + continue + } + name = "." + if m, ok := s.Source.Metadata.(source.FileMetadata); ok { + archive = &m + name = filepath.Base(m.Path) + } + } + f := describe(c, name) + if archive != nil { + // an extracted archive is the archive file, not the directory it was extracted to + f.FileTypes = withArchiveType(toFileTypes(&file.Metadata{MIMEType: archive.MIMEType})) + } + results = append(results, f) + addEntries(c) } // sort by real path then virtual path to ensure the result is stable across multiple runs @@ -760,7 +810,90 @@ func toFiles(s sbom.SBOM) (results []*spdx.File) { } return results[i].FileName < results[j].FileName }) - return results + return results, archiveRelationships +} + +// toSourceFile finds the file the scanned source is: a directory, or an extracted archive, is the scanned root, +// while a single file is indexed beside its parent directory, which was not scanned. +func toSourceFile(s sbom.SBOM) *file.Coordinates { + var top []file.Coordinates + for _, c := range s.AllCoordinates() { + if c.FullNestedPath() == c.RealPath { + top = append(top, c) + } + } + match := isScanRoot + switch m := s.Source.Metadata.(type) { + case source.DirectoryMetadata: + case source.FileMetadata: + name := filepath.Base(m.Path) + single := func(c file.Coordinates) bool { return strings.TrimPrefix(c.RealPath, "/") == name } + if slices.ContainsFunc(top, single) { + match = single + } + default: + return nil + } + i := slices.IndexFunc(top, match) + if i < 0 { + return nil + } + return &top[i] +} + +// withArchiveType adds the archive file type, in place of other +func withArchiveType(types []string) []string { + if slices.Contains(types, string(helpers.ArchiveFileType)) { + return types + } + types = slices.DeleteFunc(types, func(t string) bool { return t == string(helpers.OtherFileType) }) + return append(types, string(helpers.ArchiveFileType)) +} + +func isScanRoot(c file.Coordinates) bool { + return c.RealPath == "" || c.RealPath == "/" +} + +func toFile(artifacts sbom.Artifacts, c file.Coordinates, name string) *spdx.File { + var metadata *file.Metadata + if metadataForLocation, exists := artifacts.FileMetadata[c]; exists { + metadata = &metadataForLocation + } + + var digests []file.Digest + if digestsForLocation, exists := artifacts.FileDigests[c]; exists { + digests = digestsForLocation + } + + // if we don't have any metadata or digests for this location + // then the file is most likely a symlink or non-regular file + // for now we include a 0 sha1 digest as requested by the spdx spec + // TODO: update location code in core SBOM so that we can map complex links + // back to their real file digest location. + if len(digests) == 0 { + digests = append(digests, file.Digest{Algorithm: "sha1", Value: "0000000000000000000000000000000000000000"}) + } + + // TODO: add file classifications (?) and content as a snippet + + var comment string + if c.FileSystemID != "" { + comment = layerIDCommentPrefix + c.FileSystemID + } + + return &spdx.File{ + FileSPDXIdentifier: toSPDXID(c), + FileComment: comment, + // required, no attempt made to determine license information + LicenseConcluded: noAssertion, + FileCopyrightText: noAssertion, + Checksums: toFileChecksums(digests), + FileName: name, + FileTypes: toFileTypes(metadata), + LicenseInfoInFiles: []string{ // required in SPDX 2.2 + helpers.NOASSERTION, + }, + } } func toFileChecksums(digests []file.Digest) (checksums []spdx.Checksum) { @@ -836,7 +969,7 @@ func newPackageVerificationCode(rels *relationship.Index, p pkg.Package, sbom sb // if a sha1 for a file is provided, then the validator will fail if the package does not have // a package verification code coordinates := rels.Coordinates(p, artifact.ContainsRelationship) - var digests []file.Digest + var sha1s []string for _, c := range coordinates { digest := sbom.Artifacts.FileDigests[c] if len(digest) == 0 { @@ -850,27 +983,23 @@ func newPackageVerificationCode(rels *relationship.Index, p pkg.Package, sbom sb break } } - digests = append(digests, d) + sha1s = append(sha1s, d.Value) } + return toVerificationCode(sha1s) +} - if len(digests) == 0 { +func toVerificationCode(sha1s []string) *spdx.PackageVerificationCode { + if len(sha1s) == 0 { return nil } // sort templist in ascending order by SHA1 value - sort.SliceStable(digests, func(i, j int) bool { - return digests[i].Value < digests[j].Value - }) + sort.Strings(sha1s) // filelist = templist with "/n"s removed. /* ordered sequence of SHA1 values with no separators - var b strings.Builder - for _, digest := range digests { - b.WriteString(digest.Value) - } - //nolint:gosec hasher := sha1.New() - _, _ = hasher.Write([]byte(b.String())) + _, _ = hasher.Write([]byte(strings.Join(sha1s, ""))) return &spdx.PackageVerificationCode{ // 7.9.1: Package Verification Code Value // Cardinality: mandatory, one diff --git a/syft/format/common/spdxhelpers/to_format_model_test.go b/syft/format/common/spdxhelpers/to_format_model_test.go index aede4dbee..b1786d42a 100644 --- a/syft/format/common/spdxhelpers/to_format_model_test.go +++ b/syft/format/common/spdxhelpers/to_format_model_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "regexp" + "slices" "strings" "testing" @@ -14,6 +15,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + stfile "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/internal/relationship" "github.com/anchore/syft/internal/sourcemetadata" "github.com/anchore/syft/syft/artifact" @@ -556,7 +558,7 @@ func Test_toFiles(t *testing.T) { tests := []struct { name string in sbom.SBOM - want spdx.File + want []string }{ { name: "File paths are converted to relative in final SPDX collection", @@ -584,16 +586,57 @@ func Test_toFiles(t *testing.T) { }, }, }, - want: spdx.File{ - FileName: "some/path", + want: []string{"some/path"}, + }, + { + name: "the scanned root is the root package, not a file", + in: sbom.SBOM{Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{ + {RealPath: ""}: {Type: stfile.TypeDirectory}, + {RealPath: "/etc"}: {Type: stfile.TypeDirectory}, + }}}, + want: []string{"etc"}, + }, + { + name: "a scanned directory is a file", + in: sbom.SBOM{ + Source: source.Description{Metadata: source.DirectoryMetadata{Path: "/dir"}}, + Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{ + {RealPath: ""}: {Type: stfile.TypeDirectory}, + {RealPath: "/etc"}: {Type: stfile.TypeDirectory}, + }}, }, + want: []string{".", "etc"}, + }, + { + name: "a scanned file is indexed beside its parent directory, which was not scanned", + in: sbom.SBOM{ + Source: source.Description{Metadata: source.FileMetadata{Path: "/dir/go.mod"}}, + Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{ + {RealPath: ""}: {Type: stfile.TypeDirectory}, + {RealPath: "/go.mod"}: {Type: stfile.TypeRegular}, + }}, + }, + want: []string{"go.mod"}, + }, + { + name: "the absolute root is the root package, not a file", + in: sbom.SBOM{Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{ + {RealPath: "/"}: {Type: stfile.TypeDirectory}, + {RealPath: "/etc"}: {Type: stfile.TypeDirectory}, + }}}, + want: []string{"etc"}, }, } for _, test := range tests { - files := toFiles(test.in) - got := files[0] - assert.Equal(t, test.want.FileName, got.FileName) + t.Run(test.name, func(t *testing.T) { + files, _ := toFiles(test.in, toSourceFile(test.in)) + var names []string + for _, f := range files { + names = append(names, f.FileName) + } + assert.Equal(t, test.want, names) + }) } } @@ -762,10 +805,7 @@ func Test_fileIDsForPackage(t *testing.T) { Name: "bogus", } - c := file.Coordinates{ - RealPath: "/path", - FileSystemID: "nowhere", - } + c := file.NewCoordinates("/path", "nowhere") docElementId := func(identifiable artifact.Identifiable) spdx.DocElementID { return spdx.DocElementID{ @@ -824,7 +864,7 @@ func Test_fileIDsForPackage(t *testing.T) { expected: nil, }, { - name: "ignore file-to-package", + name: "include an archive file containing a package", relationships: []artifact.Relationship{ { From: c, @@ -832,6 +872,23 @@ func Test_fileIDsForPackage(t *testing.T) { Type: artifact.ContainsRelationship, }, }, + expected: []*spdx.Relationship{ + { + Relationship: "CONTAINS", + RefA: docElementId(c), + RefB: docElementId(p), + }, + }, + }, + { + name: "ignore other file-to-package", + relationships: []artifact.Relationship{ + { + From: c, + To: p, + Type: artifact.DependencyOfRelationship, + }, + }, expected: nil, }, { @@ -1010,17 +1067,13 @@ func Test_toSPDXID(t *testing.T) { expected string }{ { - name: "short filename", - it: file.Coordinates{ - RealPath: "/short/path/file.txt", - }, + name: "short filename", + it: file.NewCoordinates("/short/path/file.txt", ""), expected: "File-short-path-file.txt", }, { - name: "long filename", - it: file.Coordinates{ - RealPath: "/some/long/path/with/a/lot/of-text/that-contains-a/file.txt", - }, + name: "long filename", + it: file.NewCoordinates("/some/long/path/with/a/lot/of-text/that-contains-a/file.txt", ""), expected: "File-...a-lot-of-text-that-contains-a-file.txt", }, { @@ -1146,6 +1199,287 @@ func Test_otherLicenses(t *testing.T) { } } +func Test_nestedArchiveFiles(t *testing.T) { + top := file.NewCoordinates("/etc/os-release", "sha256:layer") + inA := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", FileSystemID: "sha256:layer"} + inB := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt", FileSystemID: "sha256:layer"} + owned := file.NewCoordinates("/usr/bin/tool", "sha256:layer") + a := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar", FileSystemID: "sha256:layer"} + war := file.NewCoordinates("/app.war", "sha256:layer") + b := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar", FileSystemID: "sha256:layer"} + topPkg := pkg.Package{Name: "top"} + topPkg.SetID() + nestedPkg := pkg.Package{Name: "nested"} + nestedPkg.SetID() + + sources := []source.Description{ + {Name: "img", Metadata: source.ImageMetadata{UserInput: "img:latest", ManifestDigest: "sha256:abc"}}, + {Name: "dir", Metadata: source.DirectoryMetadata{Path: "/dir"}}, + } + for _, src := range sources { + t.Run(src.Name, func(t *testing.T) { + s := sbom.SBOM{ + Source: src, + Artifacts: sbom.Artifacts{ + Packages: pkg.NewCollection(topPkg, nestedPkg), + FileMetadata: map[file.Coordinates]file.Metadata{top: {}, inA: {}, inB: {}, owned: {}}, + }, + Relationships: []artifact.Relationship{ + {From: topPkg, To: owned, Type: artifact.ContainsRelationship}, + {From: a, To: nestedPkg, Type: artifact.ContainsRelationship}, + }, + } + + doc := ToFormatModel(s) + + // each archive holding a described file is described too, named by its path in its own container + names := map[spdx.ElementID]string{} + for _, f := range doc.Files { + names[f.FileSPDXIdentifier] = f.FileName + assert.Equal(t, "layerID: sha256:layer", f.FileComment) + } + assert.Equal(t, map[spdx.ElementID]string{ + toSPDXID(top): "etc/os-release", + toSPDXID(owned): "usr/bin/tool", + toSPDXID(war): "app.war", + toSPDXID(a): "WEB-INF/lib/a.jar", + toSPDXID(b): "WEB-INF/lib/b.jar", + toSPDXID(inA): "META-INF/MANIFEST.MF", + toSPDXID(inB): "odd:na%me.txt", + }, names) + + var described []spdx.ElementID + var contains [][2]spdx.ElementID + for _, r := range doc.Relationships { + switch r.Relationship { + case string(helpers.DescribesRelationship): + described = append(described, r.RefB.ElementRefID) + case string(helpers.ContainsRelationship): + contains = append(contains, [2]spdx.ElementID{r.RefA.ElementRefID, r.RefB.ElementRefID}) + } + } + rootPkg := doc.Packages[len(doc.Packages)-1] + root := rootPkg.PackageSPDXIdentifier + assert.Equal(t, []spdx.ElementID{root}, described) + + // the root contains every package, but only the files nothing else does: not what is within an archive, + // nor an owned file + assert.ElementsMatch(t, [][2]spdx.ElementID{ + {root, toSPDXID(topPkg)}, + {root, toSPDXID(nestedPkg)}, + {root, toSPDXID(top)}, + {root, toSPDXID(war)}, + {toSPDXID(topPkg), toSPDXID(owned)}, + {toSPDXID(a), toSPDXID(nestedPkg)}, + {toSPDXID(war), toSPDXID(a)}, + {toSPDXID(war), toSPDXID(b)}, + {toSPDXID(a), toSPDXID(inA)}, + {toSPDXID(b), toSPDXID(inB)}, + }, contains) + + // a package containing files must have analyzed them + assert.True(t, rootPkg.FilesAnalyzed) + assert.NotNil(t, rootPkg.PackageVerificationCode) + + // other tools may instead have the document describe the top-level files directly + describesFiles := ToFormatModel(s) + for _, r := range describesFiles.Relationships { + if r.RefA.ElementRefID == root && (r.RefB.ElementRefID == toSPDXID(top) || r.RefB.ElementRefID == toSPDXID(war)) { + r.RefA.ElementRefID = "DOCUMENT" + r.Relationship = string(helpers.DescribesRelationship) + } + } + + for _, d := range []struct { + name string + doc *spdx.Document + }{ + {name: "root contains files", doc: doc}, + {name: "document describes files", doc: describesFiles}, + } { + t.Run(d.name, func(t *testing.T) { + decoded, err := ToSyftModel(d.doc) + require.NoError(t, err) + + assert.Equal(t, src.Name, decoded.Source.Name) + assert.IsType(t, src.Metadata, decoded.Source.Metadata) + + // decoded paths are relative, as SPDX names are + relative := func(c file.Coordinates) file.Coordinates { + c.RealPath = strings.TrimPrefix(c.RealPath, "/") + c.FullPath = strings.TrimPrefix(c.FullPath, "/") + return c + } + var got []file.Coordinates + for c := range decoded.Artifacts.FileMetadata { + got = append(got, c) + } + assert.ElementsMatch(t, []file.Coordinates{relative(top), relative(owned), relative(war), relative(a), relative(b), relative(inA), relative(inB)}, got) + + var gotContains []artifact.Relationship + for _, r := range decoded.Relationships { + _, fromFile := r.From.(file.Coordinates) + _, toFile := r.To.(file.Coordinates) + if fromFile && toFile { + gotContains = append(gotContains, r) + } + } + assert.ElementsMatch(t, []artifact.Relationship{ + {From: relative(war), To: relative(a), Type: artifact.ContainsRelationship}, + {From: relative(war), To: relative(b), Type: artifact.ContainsRelationship}, + {From: relative(a), To: relative(inA), Type: artifact.ContainsRelationship}, + {From: relative(b), To: relative(inB), Type: artifact.ContainsRelationship}, + }, gotContains) + }) + } + }) + } +} + +func Test_sourceFile(t *testing.T) { + root := file.NewCoordinates("", "") + war := file.NewCoordinates("/app.war", "") + jar := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/a.jar"} + etc := file.NewCoordinates("/etc/os-release", "") + p := pkg.Package{Name: "p"} + p.SetID() + + tests := []struct { + name string + source source.Description + files []file.Coordinates + // relationships as "from type to", with the root package as "root" + want []string + wantAnalyzed bool + // the file types of the file the root package describes + wantTypes []string + // decoded files, and the file relationships among them, as "from type to" + wantFiles []string + wantContains []string + }{ + { + name: "a scanned directory is described, and contains what is at its root", + source: source.Description{Name: "dir", Metadata: source.DirectoryMetadata{Path: "/dir"}}, + files: []file.Coordinates{root, etc, war, jar}, + want: []string{ + "root DESCRIBES .", + "root CONTAINS p", + ". CONTAINS etc/os-release", + ". CONTAINS app.war", + "app.war CONTAINS a.jar", + }, + // the scanned root decodes back to the scanned root, and holds nothing as an archive would + wantFiles: []string{"", "etc/os-release", "app.war", "app.war!/a.jar"}, + wantContains: []string{"app.war contains app.war!/a.jar"}, + }, + { + name: "an extracted archive is described, and contains what is at its root", + source: source.Description{Name: "t.tar", Metadata: source.FileMetadata{Path: "/dir/t.tar"}}, + files: []file.Coordinates{root, etc}, + want: []string{ + "root DESCRIBES t.tar", + "root CONTAINS p", + "t.tar CONTAINS etc/os-release", + }, + // the scanned root is the directory the archive was extracted to, but describes the archive + wantTypes: []string{"ARCHIVE"}, + // the described archive is the scanned root, holding what is at the root + wantFiles: []string{"", "etc/os-release"}, + }, + { + name: "an extracted archive takes its file types from its MIME type", + source: source.Description{Name: "t.zip", Metadata: source.FileMetadata{Path: "/dir/t.zip", MIMEType: "application/zip"}}, + files: []file.Coordinates{root, etc}, + want: []string{ + "root DESCRIBES t.zip", + "root CONTAINS p", + "t.zip CONTAINS etc/os-release", + }, + wantTypes: []string{"APPLICATION", "ARCHIVE"}, + wantFiles: []string{"", "etc/os-release"}, + }, + { + name: "a scanned file is described, but not its parent directory", + source: source.Description{Name: "app.war", Metadata: source.FileMetadata{Path: "/dir/app.war"}}, + files: []file.Coordinates{root, war, jar}, + want: []string{ + "root DESCRIBES app.war", + "root CONTAINS p", + "app.war CONTAINS a.jar", + }, + // an archive read in place is the scanned root too, so decodes as if it had been extracted + wantFiles: []string{"", "a.jar"}, + }, + { + name: "without the scanned root, the root package contains what is at its root", + source: source.Description{Name: "dir", Metadata: source.DirectoryMetadata{Path: "/dir"}}, + files: []file.Coordinates{etc}, + want: []string{ + "root CONTAINS p", + "root CONTAINS etc/os-release", + }, + wantAnalyzed: true, + wantFiles: []string{"etc/os-release"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + metadata := map[file.Coordinates]file.Metadata{} + for _, c := range tt.files { + metadata[c] = file.Metadata{} + } + s := sbom.SBOM{ + Source: tt.source, + Artifacts: sbom.Artifacts{Packages: pkg.NewCollection(p), FileMetadata: metadata}, + } + + doc := ToFormatModel(s) + + rootPkg := doc.Packages[len(doc.Packages)-1] + names := map[spdx.ElementID]string{rootPkg.PackageSPDXIdentifier: "root", toSPDXID(p): "p"} + for _, f := range doc.Files { + names[f.FileSPDXIdentifier] = f.FileName + } + var got []string + for _, r := range doc.Relationships { + if r.RefA.ElementRefID == "DOCUMENT" { + continue + } + got = append(got, names[r.RefA.ElementRefID]+" "+r.Relationship+" "+names[r.RefB.ElementRefID]) + } + assert.ElementsMatch(t, tt.want, got) + assert.Equal(t, tt.wantAnalyzed, rootPkg.FilesAnalyzed) + if tt.wantTypes != nil { + for _, r := range doc.Relationships { + if r.RefA.ElementRefID == rootPkg.PackageSPDXIdentifier && r.Relationship == string(helpers.DescribesRelationship) { + i := slices.IndexFunc(doc.Files, func(f *spdx.File) bool { return f.FileSPDXIdentifier == r.RefB.ElementRefID }) + require.GreaterOrEqual(t, i, 0) + assert.Equal(t, tt.wantTypes, doc.Files[i].FileTypes) + } + } + } + + decoded, err := ToSyftModel(doc) + require.NoError(t, err) + + var gotFiles []string + for c := range decoded.Artifacts.FileMetadata { + gotFiles = append(gotFiles, c.FullNestedPath()) + } + assert.ElementsMatch(t, tt.wantFiles, gotFiles) + + var gotContains []string + for _, r := range decoded.Relationships { + if from, ok := r.From.(file.Coordinates); ok { + to, _ := r.To.(file.Coordinates) + gotContains = append(gotContains, from.FullNestedPath()+" "+string(r.Type)+" "+to.FullNestedPath()) + } + } + assert.ElementsMatch(t, tt.wantContains, gotContains) + }) + } +} + func Test_toRootPackage_name(t *testing.T) { tests := []struct { name string @@ -1191,3 +1525,27 @@ func Test_toRootPackage_name(t *testing.T) { }) } } + +func Test_toFiles_archiveType(t *testing.T) { + // a file holding entries is an archive, even when its content starts like something else, such as a Spring Boot + // executable jar's launch script + jar := file.NewCoordinates("/app.jar", "") + s := sbom.SBOM{Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{ + jar: {MIMEType: "text/x-shellscript"}, + file.NewCoordinates("/plain.zip", ""): {MIMEType: "application/zip"}, + {RealPath: "/app.jar", FullPath: "/app.jar!/lib/b.jar!/c.txt"}: {MIMEType: "text/plain"}, + }}} + + files, _ := toFiles(s, nil) + got := map[string][]string{} + for _, f := range files { + got[f.FileName] = f.FileTypes + } + assert.Equal(t, map[string][]string{ + "app.jar": {"TEXT", "ARCHIVE"}, + "plain.zip": {"APPLICATION", "ARCHIVE"}, + // added only to hold its entry, with no metadata of its own + "lib/b.jar": {"ARCHIVE"}, + "c.txt": {"TEXT"}, + }, got) +} diff --git a/syft/format/common/spdxhelpers/to_syft_model.go b/syft/format/common/spdxhelpers/to_syft_model.go index eaddc9650..92dd5087d 100644 --- a/syft/format/common/spdxhelpers/to_syft_model.go +++ b/syft/format/common/spdxhelpers/to_syft_model.go @@ -38,6 +38,9 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) { spdxIDMap := make(map[string]any) + // found before the root package and its relationships are removed + sourceFile := findSourceFile(doc) + s := &sbom.SBOM{ Source: extractSource(spdxIDMap, doc), Artifacts: sbom.Artifacts{ @@ -50,7 +53,7 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) { collectSyftPackages(s, spdxIDMap, doc) - collectSyftFiles(s, spdxIDMap, doc) + collectSyftFiles(s, spdxIDMap, doc, sourceFile) s.Relationships = toSyftRelationships(spdxIDMap, doc) @@ -379,26 +382,122 @@ func collectSyftPackages(s *sbom.SBOM, spdxIDMap map[string]any, doc *spdx.Docum } } -func collectSyftFiles(s *sbom.SBOM, spdxIDMap map[string]any, doc *spdx.Document) { +// findSourceFile returns the file the root package DESCRIBES when it CONTAINS files: an archive scanned as the source +// is the scanned root, so what it contains is at the root rather than within an archive. +func findSourceFile(doc *spdx.Document) spdx.ElementID { + roots := findRootPackages(doc) + if len(roots) != 1 { + return "" + } + var described spdx.ElementID + containers := map[spdx.ElementID]bool{} + for _, r := range doc.Relationships { + switch helpers.RelationshipType(r.Relationship) { + case helpers.DescribesRelationship: + if r.RefA.ElementRefID == roots[0].PackageSPDXIdentifier { + described = r.RefB.ElementRefID + } + case helpers.ContainsRelationship: + containers[r.RefA.ElementRefID] = true + } + } + if !containers[described] { + return "" + } + return described +} + +func collectSyftFiles(s *sbom.SBOM, spdxIDMap map[string]any, doc *spdx.Document, sourceFile spdx.ElementID) { + files := map[spdx.ElementID]*spdx.File{} for _, p := range doc.Packages { for _, f := range p.Files { - l := toSyftLocation(f) - spdxIDMap[string(f.FileSPDXIdentifier)] = l + files[f.FileSPDXIdentifier] = f + } + } + for _, f := range doc.Files { + files[f.FileSPDXIdentifier] = f + } - s.Artifacts.FileMetadata[l.Coordinates] = toFileMetadata(f) - s.Artifacts.FileDigests[l.Coordinates] = toFileDigests(f) + nested := newNestedFiles[spdx.ElementID]() + for id, f := range files { + nested.own[id] = toSyftCoordinates(f) + } + if c, ok := nested.own[sourceFile]; ok { + // the scanned root + c.RealPath = "" + nested.own[sourceFile] = c + } + for _, r := range doc.Relationships { + container, entry := r.RefA.ElementRefID, r.RefB.ElementRefID + switch helpers.RelationshipType(r.Relationship) { + case helpers.ContainsRelationship: + case helpers.ContainedByRelationship: + container, entry = entry, container + default: + continue + } + // the scanned root holds what is at the root, not within an archive + if files[container] != nil && files[entry] != nil && !isScanRoot(nested.own[container]) { + nested.container[entry] = container } } - for _, f := range doc.Files { - l := toSyftLocation(f) - spdxIDMap[string(f.FileSPDXIdentifier)] = l + for id, f := range files { + c, ok := nested.coordinates(id) + if !ok { + continue + } + l := file.NewLocationFromCoordinates(c) + spdxIDMap[string(id)] = l s.Artifacts.FileMetadata[l.Coordinates] = toFileMetadata(f) s.Artifacts.FileDigests[l.Coordinates] = toFileDigests(f) } } +// nestedFiles resolves the coordinates of files within archives, where an archive file CONTAINS a file +// named by its path within the archive. +type nestedFiles[K comparable] struct { + own map[K]file.Coordinates + container map[K]K + resolved map[K]file.Coordinates + refused map[K]bool +} + +func newNestedFiles[K comparable]() nestedFiles[K] { + return nestedFiles[K]{own: map[K]file.Coordinates{}, container: map[K]K{}, resolved: map[K]file.Coordinates{}, refused: map[K]bool{}} +} + +// coordinates reports false for a file within a chain a scan would refuse (a bad name, too deep, too long), and +// for everything within it. Such a file is dropped rather than moved, since no scan put it anywhere else. +func (n nestedFiles[K]) coordinates(id K) (file.Coordinates, bool) { + // walk up to a resolved or outermost file, then resolve back down, so a long chain costs no stack + chain := []K{id} + for { + next, ok := n.container[chain[len(chain)-1]] + if _, resolved := n.resolved[chain[len(chain)-1]]; resolved || !ok { + break + } + // a containment cycle ends where it started + delete(n.container, chain[len(chain)-1]) + chain = append(chain, next) + } + for i := len(chain) - 1; i >= 0; i-- { + if _, ok := n.resolved[chain[i]]; ok { + continue + } + c := n.own[chain[i]] + if i+1 < len(chain) { + container := chain[i+1] + nested, ok := internal.NestedCoordinates(n.resolved[container], c.RealPath) + n.refused[chain[i]] = !ok || n.refused[container] + c = nested + } + n.resolved[chain[i]] = c + } + return n.resolved[id], !n.refused[id] +} + func toFileDigests(f *spdx.File) (digests []file.Digest) { for _, digest := range f.Checksums { digests = append(digests, file.Digest{ @@ -452,6 +551,10 @@ func collectDocRelationships(spdxIDMap map[string]any, doc *spdx.Document) (out } a := spdxIDMap[string(r.RefA.ElementRefID)] b := spdxIDMap[string(r.RefB.ElementRefID)] + if rel, ok := fileContainsRelationship(helpers.RelationshipType(r.Relationship), a, b); ok { + out = append(out, rel) + continue + } from, fromOk := a.(pkg.Package) toPackage, toPackageOk := b.(pkg.Package) toLocation, toLocationOk := b.(file.Location) @@ -505,6 +608,31 @@ func collectDocRelationships(spdxIDMap map[string]any, doc *spdx.Document) (out return out } +// fileContainsRelationship decodes an archive file containing a file or package within it. +func fileContainsRelationship(typ helpers.RelationshipType, a, b any) (artifact.Relationship, bool) { + switch typ { + case helpers.ContainsRelationship: + case helpers.ContainedByRelationship: + a, b = b, a + default: + return artifact.Relationship{}, false + } + container, ok := a.(file.Location) + if !ok || isScanRoot(container.Coordinates) { + return artifact.Relationship{}, false + } + var entry artifact.Identifiable + switch to := b.(type) { + case file.Location: + entry = to.Coordinates + case pkg.Package: + entry = to + default: + return artifact.Relationship{}, false + } + return artifact.Relationship{From: container.Coordinates, To: entry, Type: artifact.ContainsRelationship}, true +} + // collectPackageFileRelationships add relationships for direct files func collectPackageFileRelationships(spdxIDMap map[string]any, doc *spdx.Document) (out []artifact.Relationship) { for _, p := range doc.Packages { @@ -530,23 +658,22 @@ func collectPackageFileRelationships(spdxIDMap map[string]any, doc *spdx.Documen } func toSyftCoordinates(f *spdx.File) file.Coordinates { - const layerIDPrefix = "layerID: " - var fileSystemID string - if strings.HasPrefix(f.FileComment, layerIDPrefix) { - fileSystemID = strings.TrimPrefix(f.FileComment, layerIDPrefix) - } - if strings.HasPrefix(string(f.FileSPDXIdentifier), layerIDPrefix) { - fileSystemID = strings.TrimPrefix(string(f.FileSPDXIdentifier), layerIDPrefix) - } - return file.Coordinates{ - RealPath: f.FileName, - FileSystemID: fileSystemID, - } + return coordinatesFromFile(f.FileName, f.FileComment, string(f.FileSPDXIdentifier)) } -func toSyftLocation(f *spdx.File) file.Location { - l := file.NewVirtualLocationFromCoordinates(toSyftCoordinates(f), f.FileName) - return l +func coordinatesFromFile(name, comment, id string) file.Coordinates { + // the scanned root + if name == "." { + name = "" + } + c := file.NewCoordinates(name, "") + if v, ok := strings.CutPrefix(comment, layerIDCommentPrefix); ok { + c.FileSystemID = v + } + if v, ok := strings.CutPrefix(id, layerIDCommentPrefix); ok { + c.FileSystemID = v + } + return c } func requireAndTrimPrefix(val any, prefix string) string { diff --git a/syft/format/common/spdxhelpers/to_syft_model_test.go b/syft/format/common/spdxhelpers/to_syft_model_test.go index 23a8177ca..971e8f4d3 100644 --- a/syft/format/common/spdxhelpers/to_syft_model_test.go +++ b/syft/format/common/spdxhelpers/to_syft_model_test.go @@ -2,6 +2,7 @@ package spdxhelpers import ( "reflect" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -12,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/anchore/packageurl-go" + "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -795,3 +797,24 @@ func Test_skipsPackagesWithGeneratedFromRelationship(t *testing.T) { assert.NotNil(t, s.Artifacts.Packages.Package("1")) assert.Nil(t, s.Artifacts.Packages.Package("1-src")) } + +func Test_nestedFiles_longChainIsCapped(t *testing.T) { + // a document can claim any chain; it decodes no deeper than a scan goes, without recursing per level, and + // what is past that is dropped + n := newNestedFiles[int]() + const length = 100_000 + for i := range length { + n.own[i] = file.NewCoordinates("/a.jar", "") + if i > 0 { + n.container[i] = i - 1 + } + } + + for i := length - 1; i >= 0; i-- { + c, ok := n.coordinates(i) + assert.Equal(t, i <= archive.MaxDepth, ok, "%d", i) + if ok { + assert.Equal(t, "/a.jar"+strings.Repeat("!/a.jar", i), c.FullNestedPath()) + } + } +} diff --git a/syft/format/common/spdxhelpers/to_syft_model_v3.go b/syft/format/common/spdxhelpers/to_syft_model_v3.go index 336c2d30b..19f5f9a3f 100644 --- a/syft/format/common/spdxhelpers/to_syft_model_v3.go +++ b/syft/format/common/spdxhelpers/to_syft_model_v3.go @@ -31,6 +31,9 @@ func ToSyftModelV3(doc *spdx.Document) (*sbom.SBOM, error) { spdxMap := ptrMap[any]{} + // found before the root package and its relationships are removed + sourceFile := v3findSourceFile(doc) + s := &sbom.SBOM{ Source: v3extractSource(spdxMap, doc), Artifacts: sbom.Artifacts{ @@ -45,7 +48,7 @@ func ToSyftModelV3(doc *spdx.Document) (*sbom.SBOM, error) { v3collectSyftPackages(s, spdxMap, relationships, doc) - v3collectSyftFiles(s, spdxMap, doc) + v3collectSyftFiles(s, spdxMap, doc, sourceFile) s.Relationships = v3toSyftRelationships(spdxMap, doc) @@ -282,21 +285,87 @@ func v3collectSyftPackages(s *sbom.SBOM, spdxMap ptrMap[any], relationships ptrM } } -func v3collectSyftFiles(s *sbom.SBOM, spdxMap ptrMap[any], doc *spdx.Document) { - found := ptrMap[struct{}]{} +// v3findSourceFile returns the file the root package DESCRIBES when it CONTAINS files: an archive scanned as the +// source is the scanned root, so what it contains is at the root rather than within an archive. +func v3findSourceFile(doc *spdx.Document) spdx.AnyFile { + roots := v3findRootPackages(doc) + if len(roots) != 1 { + return nil + } + var described spdx.AnyFile + containers := ptrMap[bool]{} + for _, r := range doc.Elements.Relationships() { + from := r.GetFrom() + if from == nil { + continue + } + switch r.GetType() { + case spdx.RelationshipType_Describes: + if from == spdx.AnyElement(roots[0]) { + for _, to := range r.GetTo() { + if f, ok := to.(spdx.AnyFile); ok { + described = f + } + } + } + case spdx.RelationshipType_Contains: + containers.Set(from, true) + } + } + if described == nil || !containers.Get(described) { + return nil + } + return described +} + +func v3collectSyftFiles(s *sbom.SBOM, spdxMap ptrMap[any], doc *spdx.Document, sourceFile spdx.AnyFile) { + nested := newNestedFiles[reflect.Value]() + var files []spdx.AnyFile for _, elementList := range []spdx.ElementList{doc.Elements, doc.RootElements} { for _, f := range elementList.Files() { - if found.Has(f) { + if _, found := nested.own[ptrTo(f)]; found { continue } - found.Set(f, struct{}{}) - l := v3toSyftLocation(f) - spdxMap.Set(f, l) - - s.Artifacts.FileMetadata[l.Coordinates] = v3toFileMetadata(f) - s.Artifacts.FileDigests[l.Coordinates] = v3toFileDigests(f) + nested.own[ptrTo(f)] = v3toSyftCoordinates(f) + files = append(files, f) } } + if sourceFile != nil { + if c, ok := nested.own[ptrTo(sourceFile)]; ok { + // the scanned root + c.RealPath = "" + nested.own[ptrTo(sourceFile)] = c + } + } + for _, r := range doc.Elements.Relationships() { + container, ok := r.GetFrom().(spdx.AnyFile) + if !ok || r.GetType() != spdx.RelationshipType_Contains { + continue + } + for _, to := range r.GetTo() { + entry, ok := to.(spdx.AnyFile) + if !ok { + continue + } + _, knownContainer := nested.own[ptrTo(container)] + _, knownEntry := nested.own[ptrTo(entry)] + if knownContainer && knownEntry && !isScanRoot(nested.own[ptrTo(container)]) { + nested.container[ptrTo(entry)] = ptrTo(container) + } + } + } + + for _, f := range files { + c, ok := nested.coordinates(ptrTo(f)) + if !ok { + continue + } + l := file.NewLocationFromCoordinates(c) + spdxMap.Set(f, l) + + s.Artifacts.FileMetadata[l.Coordinates] = v3toFileMetadata(f) + s.Artifacts.FileDigests[l.Coordinates] = v3toFileDigests(f) + } } func v3toFileDigests(f spdx.AnyFile) (digests []file.Digest) { @@ -341,6 +410,12 @@ func v3collectDocRelationships(spdxMap ptrMap[any], doc *spdx.Document) (out []a for _, to := range r.GetTo() { b := spdxMap.Get(to) + if r.GetType() == spdx.RelationshipType_Contains { + if rel, ok := fileContainsRelationship(helpers.ContainsRelationship, a, b); ok { + out = append(out, rel) + continue + } + } from, fromOk := a.(pkg.Package) toPackage, toPackageOk := b.(pkg.Package) toLocation, toLocationOk := b.(file.Location) @@ -393,23 +468,7 @@ func v3collectDocRelationships(spdxMap ptrMap[any], doc *spdx.Document) (out []a } func v3toSyftCoordinates(f spdx.AnyFile) file.Coordinates { - const layerIDPrefix = "layerID: " - var fileSystemID string - if strings.HasPrefix(f.GetComment(), layerIDPrefix) { - fileSystemID = strings.TrimPrefix(f.GetComment(), layerIDPrefix) - } - if strings.HasPrefix(f.GetID(), layerIDPrefix) { - fileSystemID = strings.TrimPrefix(f.GetID(), layerIDPrefix) - } - return file.Coordinates{ - RealPath: f.GetName(), - FileSystemID: fileSystemID, - } -} - -func v3toSyftLocation(f spdx.AnyFile) file.Location { - l := file.NewVirtualLocationFromCoordinates(v3toSyftCoordinates(f), f.GetName()) - return l + return coordinatesFromFile(f.GetName(), f.GetComment(), f.GetID()) } func v3findQualifierValue(purl packageurl.PackageURL, qualifier string) string { diff --git a/syft/format/common/spdxhelpers/to_syft_model_v3_test.go b/syft/format/common/spdxhelpers/to_syft_model_v3_test.go index f00399b21..90f3ab534 100644 --- a/syft/format/common/spdxhelpers/to_syft_model_v3_test.go +++ b/syft/format/common/spdxhelpers/to_syft_model_v3_test.go @@ -193,14 +193,14 @@ func TestToSyftModelV3_documentConversion(t *testing.T) { assert.Equal(t, "BSD-3-Clause AND GPL-2.0+", pkg2Concluded[0].Value) // Verify files - coords1 := file.Coordinates{RealPath: "/src/main.go"} + coords1 := file.NewCoordinates("/src/main.go", "") digests, ok := result.Artifacts.FileDigests[coords1] require.True(t, ok, "file1 digests not found") require.Len(t, digests, 1) assert.Equal(t, "sha256", digests[0].Algorithm) assert.Equal(t, "abc123", digests[0].Value) - coords2 := file.Coordinates{RealPath: "/src/util.go"} + coords2 := file.NewCoordinates("/src/util.go", "") _, ok = result.Artifacts.FileMetadata[coords2] assert.True(t, ok, "file2 metadata not found") @@ -434,3 +434,44 @@ func Test_v3parseSPDXLicenses(t *testing.T) { require.Len(t, declared, 1) assert.Equal(t, "Apache-2.0 AND BSD-3-Clause", declared[0].Value) } + +func TestToSyftModelV3_nestedArchiveFiles(t *testing.T) { + war := &spdx.File{ID: "war", Name: "/app.war", Comment: "layerID: sha256:layer"} + jar := &spdx.File{ID: "jar", Name: "WEB-INF/lib/a.jar"} + manifest := &spdx.File{ID: "manifest", Name: "META-INF/MANIFEST.MF"} + p := &spdx.Package{ID: "pkg", Name: "a"} + warContainsJar := &spdx.Relationship{From: war, To: spdx.ElementList{jar}, Type: spdx.RelationshipType_Contains} + jarContains := &spdx.Relationship{From: jar, To: spdx.ElementList{manifest, p}, Type: spdx.RelationshipType_Contains} + + doc := &spdx.Document{SpdxDocument: spdx.SpdxDocument{ + Elements: spdx.ElementList{war, jar, manifest, p, warContainsJar, jarContains}, + }} + + s, err := ToSyftModelV3(doc) + require.NoError(t, err) + + wantWar := file.NewCoordinates("/app.war", "sha256:layer") + wantJar := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar", FileSystemID: "sha256:layer"} + wantManifest := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", FileSystemID: "sha256:layer"} + var got []file.Coordinates + for c := range s.Artifacts.FileMetadata { + got = append(got, c) + } + assert.ElementsMatch(t, []file.Coordinates{wantWar, wantJar, wantManifest}, got) + + var fileRels []string + for _, r := range s.Relationships { + if from, ok := r.From.(file.Coordinates); ok { + to := "package" + if c, ok := r.To.(file.Coordinates); ok { + to = c.FullNestedPath() + } + fileRels = append(fileRels, from.FullNestedPath()+" "+string(r.Type)+" "+to) + } + } + assert.ElementsMatch(t, []string{ + "/app.war contains /app.war!/WEB-INF/lib/a.jar", + "/app.war!/WEB-INF/lib/a.jar contains /app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", + "/app.war!/WEB-INF/lib/a.jar contains package", + }, fileRels) +} diff --git a/syft/format/github/internal/model/model.go b/syft/format/github/internal/model/model.go index 874bb88a7..4ce578cb6 100644 --- a/syft/format/github/internal/model/model.go +++ b/syft/format/github/internal/model/model.go @@ -113,11 +113,7 @@ func toPath(s source.Description, p pkg.Package) string { locations := p.Locations.ToSlice() if len(locations) > 0 { location := locations[0] - packagePath := location.RealPath - if location.AccessPath != "" { - packagePath = location.AccessPath - } - packagePath = strings.TrimPrefix(packagePath, "/") + packagePath := strings.TrimPrefix(location.Path(), "/") switch metadata := s.Metadata.(type) { case source.ImageMetadata: image := strings.ReplaceAll(metadata.UserInput, ":/", "//") diff --git a/syft/format/github/internal/model/model_test.go b/syft/format/github/internal/model/model_test.go index 0ea36695c..a2a14df55 100644 --- a/syft/format/github/internal/model/model_test.go +++ b/syft/format/github/internal/model/model_test.go @@ -41,30 +41,21 @@ func sbomFixture() sbom.SBOM { Name: "pkg-1", Version: "1.0.1", Locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/usr/lib", - FileSystemID: "fsid-1", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/usr/lib", "fsid-1")), ), }, { Name: "pkg-2", Version: "2.0.2", Locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/usr/lib", - FileSystemID: "fsid-1", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/usr/lib", "fsid-1")), ), }, { Name: "pkg-3", Version: "3.0.3", Locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/etc", - FileSystemID: "fsid-1", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/etc", "fsid-1")), ), }, } { diff --git a/syft/format/internal/archive_entries.go b/syft/format/internal/archive_entries.go new file mode 100644 index 000000000..30b83b5fc --- /dev/null +++ b/syft/format/internal/archive_entries.go @@ -0,0 +1,41 @@ +package internal + +import ( + "slices" + "strings" + + "github.com/anchore/syft/syft/file" +) + +// ArchiveEntries splits coordinates into those not within an archive and, for each archive, the files +// directly within it, each sorted. An archive not among coordinates is added, so every file within an +// archive has its archive to be described under. +func ArchiveEntries(coordinates []file.Coordinates, sorter func(a, b file.Coordinates) int) (top []file.Coordinates, entries map[file.Coordinates][]file.Coordinates) { + present := file.NewCoordinateSet(coordinates...) + entries = map[file.Coordinates][]file.Coordinates{} + for _, c := range coordinates { + for { + // RealPath is cut off first, so a "!/" in it is not taken for an archive boundary + rest, ok := strings.CutPrefix(c.FullPath, c.RealPath) + if !ok || !strings.HasPrefix(rest, file.ArchivePathSeparator) { + top = append(top, c) + break + } + archive := file.Coordinates{RealPath: c.RealPath, FileSystemID: c.FileSystemID} + if i := strings.LastIndex(rest, file.ArchivePathSeparator); i > 0 { + archive.FullPath = c.FullPath[:len(c.RealPath)+i] + } + entries[archive] = append(entries[archive], c) + if present.Contains(archive) { + break + } + present.Add(archive) + c = archive + } + } + slices.SortFunc(top, sorter) + for _, e := range entries { + slices.SortFunc(e, sorter) + } + return top, entries +} diff --git a/syft/format/internal/archive_entries_test.go b/syft/format/internal/archive_entries_test.go new file mode 100644 index 000000000..20ab42b36 --- /dev/null +++ b/syft/format/internal/archive_entries_test.go @@ -0,0 +1,29 @@ +package internal + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/anchore/syft/syft/file" +) + +func TestArchiveEntries(t *testing.T) { + top := file.NewCoordinates("/etc/passwd", "layer") + odd := file.NewCoordinates("/odd!/name", "layer") // a separator in a real path is not an archive boundary + war := file.NewCoordinates("/app.war", "layer") + jar := file.Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: "/app.war!/WEB-INF/lib/a.jar"} + manifest := file.Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"} + index := file.Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: "/app.war!/index.html"} + + byPath := func(a, b file.Coordinates) int { return strings.Compare(a.FullNestedPath(), b.FullNestedPath()) } + root := file.Coordinates{} // a directory scan's root + gotTop, gotEntries := ArchiveEntries([]file.Coordinates{top, root, odd, manifest, index}, byPath) + + assert.Equal(t, []file.Coordinates{root, war, top, odd}, gotTop) + assert.Equal(t, map[file.Coordinates][]file.Coordinates{ + war: {jar, index}, + jar: {manifest}, + }, gotEntries) +} diff --git a/syft/format/internal/coordinates.go b/syft/format/internal/coordinates.go new file mode 100644 index 000000000..48333d9c6 --- /dev/null +++ b/syft/format/internal/coordinates.go @@ -0,0 +1,74 @@ +package internal + +import ( + "strings" + + "github.com/anchore/syft/internal/archive" + "github.com/anchore/syft/syft/file" +) + +// NestedCoordinates returns the coordinates of the entry named name directly within the archive at container, +// reporting false for a name a scan would refuse as an entry name, for nesting deeper than a scan goes, or for +// a chain longer than archive.MaxPathBytes. A decoded document can then only build chains bounded in depth and +// length. +func NestedCoordinates(container file.Coordinates, name string) (file.Coordinates, bool) { + if nestingDepth(container) >= archive.MaxDepth { + return file.Coordinates{}, false + } + entry, err := archive.EntryName(name) + if err != nil || entry == "" { + return file.Coordinates{}, false + } + chain := container.FullNestedPath() + if len(chain)+len(file.ArchivePathSeparator)+len(entry) > archive.MaxPathBytes { + return file.Coordinates{}, false + } + return file.Coordinates{ + RealPath: container.RealPath, + FileSystemID: container.FileSystemID, + FullPath: chain + file.ArchivePathSeparator + entry, + }, true +} + +// NormalizeCoordinates returns decoded coordinates in the form a resolver builds them, reporting false for a +// FullPath that is not a chain NestedCoordinates could have built within RealPath. Such a file is dropped rather +// than collapsed onto RealPath, where it would merge into the outer archive's own entry. A FullPath merely equal +// to RealPath is cleared. +func NormalizeCoordinates(c file.Coordinates) (file.Coordinates, bool) { + if c.FullPath == "" || c.FullPath == c.RealPath { + c.FullPath = "" + return c, true + } + rest, ok := strings.CutPrefix(c.FullPath, c.RealPath+file.ArchivePathSeparator) + // counted before splitting, so a hostile chain costs nothing to refuse + if !ok || len(c.FullPath) > archive.MaxPathBytes || strings.Count(rest, file.ArchivePathSeparator) >= archive.MaxDepth { + return file.Coordinates{}, false + } + for _, entry := range strings.Split(rest, file.ArchivePathSeparator) { + if cleaned, err := archive.EntryName(entry); err != nil || cleaned == "" || cleaned != entry { + return file.Coordinates{}, false + } + } + return c, true +} + +// NormalizeLocations applies NormalizeCoordinates to each location, dropping those it refuses. +func NormalizeLocations(locations []file.Location) []file.Location { + var out []file.Location + for _, l := range locations { + if c, ok := NormalizeCoordinates(l.Coordinates); ok { + l.Coordinates = c + out = append(out, l) + } + } + return out +} + +// nestingDepth is how many archives deep the file at c is. +func nestingDepth(c file.Coordinates) int { + rest, ok := strings.CutPrefix(c.FullPath, c.RealPath) + if !ok { + return 0 + } + return strings.Count(rest, file.ArchivePathSeparator) +} diff --git a/syft/format/internal/coordinates_test.go b/syft/format/internal/coordinates_test.go new file mode 100644 index 000000000..09a2b1cb2 --- /dev/null +++ b/syft/format/internal/coordinates_test.go @@ -0,0 +1,45 @@ +package internal + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/archive" + "github.com/anchore/syft/syft/file" +) + +func TestNestedCoordinates(t *testing.T) { + war := file.NewCoordinates("/app.war", "layer") + + got, ok := NestedCoordinates(war, "/WEB-INF/lib/a.jar") + require.True(t, ok) + assert.Equal(t, file.Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: "/app.war!/WEB-INF/lib/a.jar"}, got) + + for _, name := range []string{"", "/", "a.jar!/BOOT-INF/lib/b.jar", "a\nb", strings.Repeat("a", 4097)} { + _, ok := NestedCoordinates(war, name) + assert.False(t, ok, "%q", name) + } + + // nesting stops as deep as a scan goes + c := war + for range archive.MaxDepth { + c, ok = NestedCoordinates(c, "a.jar") + require.True(t, ok) + } + _, ok = NestedCoordinates(c, "a.jar") + assert.False(t, ok) + + // so does a chain longer than a path, however it gets there: a long container name shared by many entries + // would otherwise cost memory quadratic in the document + long := file.NewCoordinates("/"+strings.Repeat("a", archive.MaxPathBytes), "layer") + _, ok = NestedCoordinates(long, "a.jar") + assert.False(t, ok) + c = war + for range 3 { + c, ok = NestedCoordinates(c, strings.Repeat("a", 2000)) + } + assert.False(t, ok) +} diff --git a/syft/format/internal/cyclonedxutil/helpers/component_test.go b/syft/format/internal/cyclonedxutil/helpers/component_test.go index 7d7a8d6df..74868732a 100644 --- a/syft/format/internal/cyclonedxutil/helpers/component_test.go +++ b/syft/format/internal/cyclonedxutil/helpers/component_test.go @@ -29,7 +29,7 @@ func Test_encodeComponentProperties(t *testing.T) { input: pkg.Package{ FoundBy: "cataloger", Locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{RealPath: "test"}), + file.NewLocationFromCoordinates(file.NewCoordinates("test", "")), ), Metadata: pkg.ApkDBEntry{ Package: "libc-utils", diff --git a/syft/format/internal/cyclonedxutil/helpers/decoder.go b/syft/format/internal/cyclonedxutil/helpers/decoder.go index aff76f66a..ef7bc1080 100644 --- a/syft/format/internal/cyclonedxutil/helpers/decoder.go +++ b/syft/format/internal/cyclonedxutil/helpers/decoder.go @@ -2,10 +2,13 @@ package helpers import ( "fmt" + "strings" "github.com/CycloneDX/cyclonedx-go" "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/format/internal" "github.com/anchore/syft/syft/linux" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/sbom" @@ -32,11 +35,64 @@ func ToSyftModel(bom *cyclonedx.BOM) (*sbom.SBOM, error) { return nil, err } + if bom.Components != nil { + collectFiles(*bom.Components, nil, s) + } + collectRelationships(bom, s, idMap) return s, nil } +// collectFiles decodes file components, named relative to the scan root; a file component nested in +// another is an entry of that archive. +func collectFiles(components []cyclonedx.Component, container *file.Coordinates, s *sbom.SBOM) { + for i := range components { + component := &components[i] + var coordinates *file.Coordinates + if component.Type == cyclonedx.ComponentTypeFile { + c := file.NewCoordinates("/"+strings.TrimPrefix(component.Name, "/"), "") + if container != nil { + nested, ok := internal.NestedCoordinates(*container, component.Name) + if !ok { + // a chain a scan would refuse is dropped with everything in it, since no scan put it + // anywhere else + continue + } + c = nested + } + coordinates = &c + if s.Artifacts.FileMetadata == nil { + s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{} + } + s.Artifacts.FileMetadata[c] = file.Metadata{} + if digests := toDigests(component.Hashes); len(digests) > 0 { + if s.Artifacts.FileDigests == nil { + s.Artifacts.FileDigests = map[file.Coordinates][]file.Digest{} + } + s.Artifacts.FileDigests[c] = digests + } + } + if component.Components != nil { + collectFiles(*component.Components, coordinates, s) + } + } +} + +func toDigests(hashes *[]cyclonedx.Hash) []file.Digest { + if hashes == nil { + return nil + } + var digests []file.Digest + for _, h := range *hashes { + digests = append(digests, file.Digest{ + Algorithm: strings.ReplaceAll(strings.ToLower(string(h.Algorithm)), "-", ""), + Value: h.Value, + }) + } + return digests +} + func collectBomPackages(bom *cyclonedx.BOM, s *sbom.SBOM, idMap map[string]any) error { componentsPresent := false if bom.Components != nil { diff --git a/syft/format/internal/cyclonedxutil/helpers/decoder_test.go b/syft/format/internal/cyclonedxutil/helpers/decoder_test.go index 5092c3ff0..49bd0f9a5 100644 --- a/syft/format/internal/cyclonedxutil/helpers/decoder_test.go +++ b/syft/format/internal/cyclonedxutil/helpers/decoder_test.go @@ -481,3 +481,27 @@ func Test_useBomRefOverDerivedSyftArtifactID(t *testing.T) { assert.NotEqual(t, "", pkgsWithoutID[0].ID()) } + +func Test_collectFiles_refusedChainIsDropped(t *testing.T) { + // a nested file a scan could not have produced is dropped with everything in it, not moved to the root + fileComponent := func(name string, children ...cyclonedx.Component) cyclonedx.Component { + c := cyclonedx.Component{Type: cyclonedx.ComponentTypeFile, Name: name} + if len(children) > 0 { + c.Components = &children + } + return c + } + s := &sbom.SBOM{} + collectFiles([]cyclonedx.Component{ + fileComponent("/app.war", + fileComponent("WEB-INF/lib/a.jar"), + fileComponent("b.jar!/forged.jar", fileComponent("inner.txt")), + ), + }, nil, s) + + var got []string + for c := range s.Artifacts.FileMetadata { + got = append(got, c.FullNestedPath()) + } + assert.ElementsMatch(t, []string{"/app.war", "/app.war!/WEB-INF/lib/a.jar"}, got) +} diff --git a/syft/format/internal/spdxutil/helpers/source_info.go b/syft/format/internal/spdxutil/helpers/source_info.go index 52bc6479a..7bab77699 100644 --- a/syft/format/internal/spdxutil/helpers/source_info.go +++ b/syft/format/internal/spdxutil/helpers/source_info.go @@ -98,7 +98,7 @@ func SourceInfo(p pkg.Package) string { } var paths []string for _, l := range p.Locations.ToSlice() { - paths = append(paths, l.RealPath) + paths = append(paths, l.FullNestedPath()) } return answer + ": " + strings.Join(paths, ", ") diff --git a/syft/format/internal/testutil/file_relationships.go b/syft/format/internal/testutil/file_relationships.go index 5a522a17a..5c9923329 100644 --- a/syft/format/internal/testutil/file_relationships.go +++ b/syft/format/internal/testutil/file_relationships.go @@ -20,7 +20,7 @@ func AddSampleFileRelationships(s *sbom.SBOM) { for _, f := range files { meta := file.Metadata{} - coords := file.Coordinates{RealPath: f} + coords := file.NewCoordinates(f, "") s.Artifacts.FileMetadata[coords] = meta s.Relationships = append(s.Relationships, artifact.Relationship{ diff --git a/syft/format/spdxjson/encoder_test.go b/syft/format/spdxjson/encoder_test.go index 42912d5d5..29634b81a 100644 --- a/syft/format/spdxjson/encoder_test.go +++ b/syft/format/spdxjson/encoder_test.go @@ -15,6 +15,7 @@ import ( "github.com/anchore/syft/syft/format/internal/testutil" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/sbom" + "github.com/anchore/syft/syft/source" ) var updateSnapshot = flag.Bool("update-spdx-json", false, "update the *.golden files for spdx-json encoders") @@ -123,10 +124,7 @@ func TestSPDX22JSONRequredProperties(t *testing.T) { enc, err := NewFormatEncoderWithConfig(cfg) require.NoError(t, err) - coords := file.Coordinates{ - RealPath: "/some/file", - FileSystemID: "ac897d978b6c38749a1", - } + coords := file.NewCoordinates("/some/file", "ac897d978b6c38749a1") p1 := pkg.Package{ Name: "files-analyzed-true", @@ -284,3 +282,83 @@ func defaultFormatEncoders() []sbom.FormatEncoder { } return encs } + +func TestEncodeDecodeTopLevelFiles(t *testing.T) { + // the root package contains the top-level files, or the scanned root does when it is described + root := file.NewCoordinates("", "") + etc := file.NewCoordinates("/etc/os-release", "") + jar := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar"} + war := file.NewCoordinates("/app.war", "") + tests := []struct { + name string + src source.Description + files []file.Coordinates + wantPaths []string + wantRels []string + }{ + { + name: "img", + src: source.Description{Name: "img", Metadata: source.ImageMetadata{UserInput: "img:latest", ManifestDigest: "sha256:abc"}}, + files: []file.Coordinates{etc, jar}, + wantPaths: []string{"etc/os-release", "app.war", "app.war!/WEB-INF/lib/a.jar"}, + wantRels: []string{"app.war contains app.war!/WEB-INF/lib/a.jar"}, + }, + { + name: "dir", + src: source.Description{Name: "dir", Metadata: source.DirectoryMetadata{Path: "dir"}}, + files: []file.Coordinates{etc, jar}, + wantPaths: []string{"etc/os-release", "app.war", "app.war!/WEB-INF/lib/a.jar"}, + wantRels: []string{"app.war contains app.war!/WEB-INF/lib/a.jar"}, + }, + { + name: "dir-root", + src: source.Description{Name: "dir", Metadata: source.DirectoryMetadata{Path: "dir"}}, + files: []file.Coordinates{root, etc, jar}, + wantPaths: []string{"", "etc/os-release", "app.war", "app.war!/WEB-INF/lib/a.jar"}, + wantRels: []string{"app.war contains app.war!/WEB-INF/lib/a.jar"}, + }, + { + // a scanned archive is the scanned root, holding what is at the root + name: "file", + src: source.Description{Name: "app.war", Metadata: source.FileMetadata{Path: "dir/app.war"}}, + files: []file.Coordinates{war, jar}, + wantPaths: []string{"", "WEB-INF/lib/a.jar"}, + }, + } + for _, tt := range tests { + src := tt.src + for _, enc := range defaultFormatEncoders() { + t.Run(tt.name+"_"+enc.Version(), func(t *testing.T) { + metadata := map[file.Coordinates]file.Metadata{} + for _, c := range tt.files { + metadata[c] = file.Metadata{} + } + var buf bytes.Buffer + require.NoError(t, enc.Encode(&buf, sbom.SBOM{ + Source: src, + Artifacts: sbom.Artifacts{FileMetadata: metadata}, + })) + + s, _, _, err := NewFormatDecoder().Decode(bytes.NewReader(buf.Bytes())) + require.NoError(t, err, "encoded document:\n%s", buf.String()) + + assert.Equal(t, src.Name, s.Source.Name) + assert.IsType(t, src.Metadata, s.Source.Metadata) + + var paths []string + for c := range s.Artifacts.FileMetadata { + paths = append(paths, c.FullNestedPath()) + } + assert.ElementsMatch(t, tt.wantPaths, paths) + + var rels []string + for _, r := range s.Relationships { + from, _ := r.From.(file.Coordinates) + to, _ := r.To.(file.Coordinates) + rels = append(rels, from.FullNestedPath()+" "+string(r.Type)+" "+to.FullNestedPath()) + } + assert.ElementsMatch(t, tt.wantRels, rels) + }) + } + } +} diff --git a/syft/format/spdxtagvalue/encoder_test.go b/syft/format/spdxtagvalue/encoder_test.go index 991d1ad1e..7542c60d8 100644 --- a/syft/format/spdxtagvalue/encoder_test.go +++ b/syft/format/spdxtagvalue/encoder_test.go @@ -9,6 +9,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + stfile "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/format/internal/testutil" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/sbom" @@ -227,3 +229,30 @@ func TestEncodeDecodeSourceWithoutName(t *testing.T) { } } } + +func TestEncodeDecodeRootDirectoryFile(t *testing.T) { + // a directory scan with all file metadata includes the scanned root, which has no relative path of its own + root := file.NewCoordinates("", "") + etc := file.NewCoordinates("/etc", "") + for _, enc := range defaultFormatEncoders() { + t.Run(enc.Version(), func(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, enc.Encode(&buf, sbom.SBOM{ + Source: source.Description{Metadata: source.DirectoryMetadata{Path: "src"}}, + Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{ + root: {Type: stfile.TypeDirectory}, + etc: {Type: stfile.TypeDirectory}, + }}, + })) + + s, _, _, err := NewFormatDecoder().Decode(bytes.NewReader(buf.Bytes())) + require.NoError(t, err, "encoded document:\n%s", buf.String()) + + var names []string + for c := range s.Artifacts.FileMetadata { + names = append(names, c.RealPath) + } + assert.ElementsMatch(t, []string{"", "etc"}, names) + }) + } +} diff --git a/syft/format/syftjson/decoder_test.go b/syft/format/syftjson/decoder_test.go index 9436da0c0..23a6822d2 100644 --- a/syft/format/syftjson/decoder_test.go +++ b/syft/format/syftjson/decoder_test.go @@ -137,10 +137,7 @@ func Test_encodeDecodeFileMetadata(t *testing.T) { FoundBy: "something", Locations: file.NewLocationSet(file.Location{ LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/somewhere", - FileSystemID: "id", - }, + Coordinates: file.NewCoordinates("/somewhere", "id"), }, LocationMetadata: file.LocationMetadata{ Annotations: map[string]string{ @@ -174,10 +171,7 @@ func Test_encodeDecodeFileMetadata(t *testing.T) { } p.SetID() - c := file.Coordinates{ - RealPath: "some-file", - FileSystemID: "some-fs-id", - } + c := file.NewCoordinates("some-file", "some-fs-id") s := sbom.SBOM{ Artifacts: sbom.Artifacts{ diff --git a/syft/format/syftjson/encoder_test.go b/syft/format/syftjson/encoder_test.go index 2415e5640..3b0161b2d 100644 --- a/syft/format/syftjson/encoder_test.go +++ b/syft/format/syftjson/encoder_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "flag" + "fmt" "runtime" "strings" "testing" @@ -90,6 +91,30 @@ func TestEscapeHTML(t *testing.T) { }) } +func TestEncode_sameFileInManyArchivesIsReproducible(t *testing.T) { + // files and package locations that differ only by archive path come out of map iteration, so the + // output is reproducible only if the sort breaks that tie + s := testutil.DirectoryInput(t, t.TempDir()) + s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{} + p := pkg.Package{Name: "shared", Version: "1.0"} + for i := range 20 { + c := file.Coordinates{RealPath: "/app.war", FullPath: fmt.Sprintf("/app.war!/WEB-INF/lib/%02d.jar!/META-INF/MANIFEST.MF", i)} + s.Artifacts.FileMetadata[c] = file.Metadata{} + p.Locations.Add(file.NewLocationFromCoordinates(c)) + } + s.Artifacts.Packages.Add(p) + + encode := func() string { + var buffer bytes.Buffer + require.NoError(t, NewFormatEncoder().Encode(&buffer, s)) + return buffer.String() + } + first := encode() + for range 10 { + require.Equal(t, first, encode()) + } +} + func TestDirectoryEncoder(t *testing.T) { if runtime.GOOS == "windows" { // the golden snapshot embeds the posix source path (and the source ID derived from it) @@ -143,9 +168,7 @@ func TestEncodeFullJSONDocument(t *testing.T) { Name: "package-1", Version: "1.0.1", Locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/a/place/a", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/a/place/a", "")), ), Type: pkg.PythonPkg, FoundBy: "the-cataloger-1", @@ -166,9 +189,7 @@ func TestEncodeFullJSONDocument(t *testing.T) { Name: "package-2", Version: "2.0.1", Locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/b/place/b", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/b/place/b", "")), ), Type: pkg.DebPkg, FoundBy: "the-cataloger-2", diff --git a/syft/format/syftjson/model/package.go b/syft/format/syftjson/model/package.go index e2fb53722..33f2b7099 100644 --- a/syft/format/syftjson/model/package.go +++ b/syft/format/syftjson/model/package.go @@ -172,17 +172,17 @@ func unpackPkgMetadata(p *Package, unpacker packageMetadataUnpacker) error { switch unpacker.MetadataType { case "HackageMetadataType": for _, l := range p.Locations { - if strings.HasSuffix(l.RealPath, ".yaml.lock") { + if strings.HasSuffix(l.FullNestedPath(), ".yaml.lock") { ty = "haskell-hackage-stack-lock-entry" break - } else if strings.HasSuffix(l.RealPath, ".yaml") { + } else if strings.HasSuffix(l.FullNestedPath(), ".yaml") { ty = "haskell-hackage-stack-entry" break } } case "RpmMetadata": for _, l := range p.Locations { - if strings.HasSuffix(l.RealPath, ".rpm") { + if strings.HasSuffix(l.FullNestedPath(), ".rpm") { ty = "rpm-archive" break } @@ -190,7 +190,7 @@ func unpackPkgMetadata(p *Package, unpacker packageMetadataUnpacker) error { case "RustCargoPackageMetadata": var found bool for _, l := range p.Locations { - if strings.HasSuffix(strings.ToLower(l.RealPath), "cargo.lock") { + if strings.HasSuffix(strings.ToLower(l.FullNestedPath()), "cargo.lock") { ty = "rust-cargo-lock-entry" found = true break diff --git a/syft/format/syftjson/to_format_model_test.go b/syft/format/syftjson/to_format_model_test.go index 8ff3fdcc5..dc29c49e1 100644 --- a/syft/format/syftjson/to_format_model_test.go +++ b/syft/format/syftjson/to_format_model_test.go @@ -352,10 +352,7 @@ func Test_toFileType(t *testing.T) { } func Test_toFileMetadataEntry(t *testing.T) { - coords := file.Coordinates{ - RealPath: "/path", - FileSystemID: "x", - } + coords := file.NewCoordinates("/path", "x") tests := []struct { name string metadata *file.Metadata @@ -453,41 +450,17 @@ func Test_toPackageModel_layerOrdering(t *testing.T) { Licenses: pkg.NewLicenseSet(pkg.License{ Value: "MIT", Locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/lic-a", - FileSystemID: "fsid-3", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/lic-a", - FileSystemID: "fsid-1", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/lic-b", - FileSystemID: "fsid-0", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/lic-a", - FileSystemID: "fsid-2", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/lic-a", "fsid-3")), + file.NewLocationFromCoordinates(file.NewCoordinates("/lic-a", "fsid-1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/lic-b", "fsid-0")), + file.NewLocationFromCoordinates(file.NewCoordinates("/lic-a", "fsid-2")), ), }), Locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-3", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-0", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-2", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-3")), + file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-0")), + file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-2")), ), }, layerOrder: []string{ @@ -505,38 +478,26 @@ func Test_toPackageModel_layerOrdering(t *testing.T) { Locations: []file.Location{ { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/lic-b", - FileSystemID: "fsid-0", // important! - }, - AccessPath: "/lic-b", + Coordinates: file.NewCoordinates("/lic-b", "fsid-0"), // important! + AccessPath: "/lic-b", }, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/lic-a", - FileSystemID: "fsid-1", // important! - }, - AccessPath: "/lic-a", + Coordinates: file.NewCoordinates("/lic-a", "fsid-1"), // important! + AccessPath: "/lic-a", }, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/lic-a", - FileSystemID: "fsid-2", // important! - }, - AccessPath: "/lic-a", + Coordinates: file.NewCoordinates("/lic-a", "fsid-2"), // important! + AccessPath: "/lic-a", }, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/lic-a", - FileSystemID: "fsid-3", // important! - }, - AccessPath: "/lic-a", + Coordinates: file.NewCoordinates("/lic-a", "fsid-3"), // important! + AccessPath: "/lic-a", }, }, }, @@ -545,38 +506,26 @@ func Test_toPackageModel_layerOrdering(t *testing.T) { Locations: []file.Location{ { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-0", // important! - }, - AccessPath: "/b", + Coordinates: file.NewCoordinates("/b", "fsid-0"), // important! + AccessPath: "/b", }, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", // important! - }, - AccessPath: "/a", + Coordinates: file.NewCoordinates("/a", "fsid-1"), // important! + AccessPath: "/a", }, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-2", // important! - }, - AccessPath: "/a", + Coordinates: file.NewCoordinates("/a", "fsid-2"), // important! + AccessPath: "/a", }, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-3", // important! - }, - AccessPath: "/a", + Coordinates: file.NewCoordinates("/a", "fsid-3"), // important! + AccessPath: "/a", }, }, }, @@ -609,34 +558,22 @@ func Test_toLocationModel(t *testing.T) { { name: "nil layer order map", locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-2", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-2")), ), layers: nil, // please don't panic! want: []file.Location{ { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, - AccessPath: "/a", + Coordinates: file.NewCoordinates("/a", "fsid-1"), + AccessPath: "/a", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-2", - }, - AccessPath: "/b", + Coordinates: file.NewCoordinates("/b", "fsid-2"), + AccessPath: "/b", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, @@ -645,18 +582,9 @@ func Test_toLocationModel(t *testing.T) { { name: "go case", locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-3", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-2", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-3")), + file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/c", "fsid-2")), ), layers: []string{ "fsid-1", @@ -666,31 +594,22 @@ func Test_toLocationModel(t *testing.T) { want: []file.Location{ { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, - AccessPath: "/b", + Coordinates: file.NewCoordinates("/b", "fsid-1"), + AccessPath: "/b", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-2", - }, - AccessPath: "/c", + Coordinates: file.NewCoordinates("/c", "fsid-2"), + AccessPath: "/c", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-3", - }, - AccessPath: "/a", + Coordinates: file.NewCoordinates("/a", "fsid-3"), + AccessPath: "/a", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, @@ -699,18 +618,9 @@ func Test_toLocationModel(t *testing.T) { { name: "same layer different paths", // prove we can sort by path irrespective of layer locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-1", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/c", "fsid-1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-1")), ), layers: []string{ "fsid-1", @@ -718,31 +628,22 @@ func Test_toLocationModel(t *testing.T) { want: []file.Location{ { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, - AccessPath: "/a", + Coordinates: file.NewCoordinates("/a", "fsid-1"), + AccessPath: "/a", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, - AccessPath: "/b", + Coordinates: file.NewCoordinates("/b", "fsid-1"), + AccessPath: "/b", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-1", - }, - AccessPath: "/c", + Coordinates: file.NewCoordinates("/c", "fsid-1"), + AccessPath: "/c", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, @@ -751,22 +652,10 @@ func Test_toLocationModel(t *testing.T) { { name: "mixed layers and paths", locations: file.NewLocationSet( - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/z", - FileSystemID: "fsid-3", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-2", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }), - file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-2", - }), + file.NewLocationFromCoordinates(file.NewCoordinates("/z", "fsid-3")), + file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-2")), + file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/c", "fsid-2")), ), layers: []string{ "fsid-1", @@ -776,41 +665,29 @@ func Test_toLocationModel(t *testing.T) { want: []file.Location{ { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, - AccessPath: "/b", + Coordinates: file.NewCoordinates("/b", "fsid-1"), + AccessPath: "/b", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-2", - }, - AccessPath: "/a", + Coordinates: file.NewCoordinates("/a", "fsid-2"), + AccessPath: "/a", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-2", - }, - AccessPath: "/c", + Coordinates: file.NewCoordinates("/c", "fsid-2"), + AccessPath: "/c", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, { LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/z", - FileSystemID: "fsid-3", - }, - AccessPath: "/z", + Coordinates: file.NewCoordinates("/z", "fsid-3"), + AccessPath: "/z", }, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, }, @@ -845,35 +722,23 @@ func Test_sortFiles(t *testing.T) { name: "nil layer order map", files: []model.File{ { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-1", + Location: file.NewCoordinates("/a", "fsid-1"), }, { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-2", - }, + ID: "file-2", + Location: file.NewCoordinates("/b", "fsid-2"), }, }, layers: nil, want: []model.File{ { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-1", + Location: file.NewCoordinates("/a", "fsid-1"), }, { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-2", - }, + ID: "file-2", + Location: file.NewCoordinates("/b", "fsid-2"), }, }, }, @@ -881,25 +746,16 @@ func Test_sortFiles(t *testing.T) { name: "layer ordering", files: []model.File{ { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-3", - }, + ID: "file-1", + Location: file.NewCoordinates("/a", "fsid-3"), }, { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, + ID: "file-2", + Location: file.NewCoordinates("/b", "fsid-1"), }, { - ID: "file-3", - Location: file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-2", - }, + ID: "file-3", + Location: file.NewCoordinates("/c", "fsid-2"), }, }, layers: []string{ @@ -909,25 +765,16 @@ func Test_sortFiles(t *testing.T) { }, want: []model.File{ { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, + ID: "file-2", + Location: file.NewCoordinates("/b", "fsid-1"), }, { - ID: "file-3", - Location: file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-2", - }, + ID: "file-3", + Location: file.NewCoordinates("/c", "fsid-2"), }, { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-3", - }, + ID: "file-1", + Location: file.NewCoordinates("/a", "fsid-3"), }, }, }, @@ -935,25 +782,16 @@ func Test_sortFiles(t *testing.T) { name: "same layer different paths", files: []model.File{ { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-1", - }, + ID: "file-1", + Location: file.NewCoordinates("/c", "fsid-1"), }, { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-2", + Location: file.NewCoordinates("/a", "fsid-1"), }, { - ID: "file-3", - Location: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, + ID: "file-3", + Location: file.NewCoordinates("/b", "fsid-1"), }, }, layers: []string{ @@ -961,25 +799,16 @@ func Test_sortFiles(t *testing.T) { }, want: []model.File{ { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-2", + Location: file.NewCoordinates("/a", "fsid-1"), }, { - ID: "file-3", - Location: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, + ID: "file-3", + Location: file.NewCoordinates("/b", "fsid-1"), }, { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/c", - FileSystemID: "fsid-1", - }, + ID: "file-1", + Location: file.NewCoordinates("/c", "fsid-1"), }, }, }, @@ -987,25 +816,16 @@ func Test_sortFiles(t *testing.T) { name: "stability test - preserve original order for equivalent items", files: []model.File{ { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-1", + Location: file.NewCoordinates("/a", "fsid-1"), }, { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-2", + Location: file.NewCoordinates("/a", "fsid-1"), }, { - ID: "file-3", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-3", + Location: file.NewCoordinates("/a", "fsid-1"), }, }, layers: []string{ @@ -1013,25 +833,16 @@ func Test_sortFiles(t *testing.T) { }, want: []model.File{ { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-1", + Location: file.NewCoordinates("/a", "fsid-1"), }, { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-2", + Location: file.NewCoordinates("/a", "fsid-1"), }, { - ID: "file-3", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-1", - }, + ID: "file-3", + Location: file.NewCoordinates("/a", "fsid-1"), }, }, }, @@ -1039,11 +850,8 @@ func Test_sortFiles(t *testing.T) { name: "complex file metadata doesn't affect sorting", files: []model.File{ { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-2", - }, + ID: "file-1", + Location: file.NewCoordinates("/a", "fsid-2"), Metadata: &model.FileMetadataEntry{ Mode: 0644, Type: "file", @@ -1061,11 +869,8 @@ func Test_sortFiles(t *testing.T) { }, }, { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, + ID: "file-2", + Location: file.NewCoordinates("/b", "fsid-1"), Metadata: &model.FileMetadataEntry{ Mode: 0755, Type: "directory", @@ -1082,11 +887,8 @@ func Test_sortFiles(t *testing.T) { }, want: []model.File{ { - ID: "file-2", - Location: file.Coordinates{ - RealPath: "/b", - FileSystemID: "fsid-1", - }, + ID: "file-2", + Location: file.NewCoordinates("/b", "fsid-1"), Metadata: &model.FileMetadataEntry{ Mode: 0755, Type: "directory", @@ -1097,11 +899,8 @@ func Test_sortFiles(t *testing.T) { }, }, { - ID: "file-1", - Location: file.Coordinates{ - RealPath: "/a", - FileSystemID: "fsid-2", - }, + ID: "file-1", + Location: file.NewCoordinates("/a", "fsid-2"), Metadata: &model.FileMetadataEntry{ Mode: 0644, Type: "file", diff --git a/syft/format/syftjson/to_syft_model.go b/syft/format/syftjson/to_syft_model.go index 651f8bba9..a10d5ca76 100644 --- a/syft/format/syftjson/to_syft_model.go +++ b/syft/format/syftjson/to_syft_model.go @@ -80,7 +80,10 @@ func toSyftFiles(files []model.File) sbom.Artifacts { } for _, f := range files { - coord := f.Location + coord, ok := internal.NormalizeCoordinates(f.Location) + if !ok { + continue + } if f.Metadata != nil { fm, err := safeFileModeConvert(f.Metadata.Mode) if err != nil { @@ -90,11 +93,11 @@ func toSyftFiles(files []model.File) sbom.Artifacts { ret.FileMetadata[coord] = file.Metadata{ FileInfo: stereoscopeFile.ManualInfo{ - NameValue: path.Base(coord.RealPath), + NameValue: path.Base(coord.FullNestedPath()), SizeValue: f.Metadata.Size, ModeValue: fm, }, - Path: coord.RealPath, + Path: coord.FullNestedPath(), LinkDestination: f.Metadata.LinkDestination, UserID: f.Metadata.UserID, GroupID: f.Metadata.GroupID, @@ -163,7 +166,7 @@ func toSyftLicenses(m []model.License) (p []pkg.License) { SPDXExpression: l.SPDXExpression, Type: l.Type, URLs: l.URLs, - Locations: file.NewLocationSet(l.Locations...), + Locations: file.NewLocationSet(internal.NormalizeLocations(l.Locations)...), Contents: l.Contents, }) } @@ -239,7 +242,9 @@ func toSyftRelationships(doc *model.Document, catalog *pkg.Collection, relations idMap[doc.Source.ID] = toSyftSource(doc.Source) for _, f := range doc.Files { - idMap[f.ID] = f.Location + if c, ok := internal.NormalizeCoordinates(f.Location); ok { + idMap[f.ID] = c + } } var out []artifact.Relationship @@ -346,7 +351,7 @@ func toSyftPackage(p model.Package, idAliases map[string]string) pkg.Package { Name: p.Name, Version: p.Version, FoundBy: p.FoundBy, - Locations: file.NewLocationSet(p.Locations...), + Locations: file.NewLocationSet(internal.NormalizeLocations(p.Locations)...), Licenses: pkg.NewLicenseSet(toSyftLicenses(p.Licenses)...), Language: p.Language, Type: p.Type, diff --git a/syft/format/syftjson/to_syft_model_test.go b/syft/format/syftjson/to_syft_model_test.go index ed4b7b59b..ccdbaec1d 100644 --- a/syft/format/syftjson/to_syft_model_test.go +++ b/syft/format/syftjson/to_syft_model_test.go @@ -5,6 +5,7 @@ import ( "io/fs" "math" "os" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -295,10 +296,7 @@ func Test_idsHaveChanged(t *testing.T) { } func Test_toSyftFiles(t *testing.T) { - coord := file.Coordinates{ - RealPath: "/somerwhere/place", - FileSystemID: "abc", - } + coord := file.NewCoordinates("/somerwhere/place", "abc") tests := []struct { name string @@ -602,3 +600,39 @@ func Test_safeFileModeConvert(t *testing.T) { }) } } + +func Test_toSyftModel_fullPathIsNormalized(t *testing.T) { + plain := file.NewCoordinates("/app.jar", "") + nested := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/lib/a.jar"} + decoded := []file.Coordinates{ + {RealPath: "/app.jar", FullPath: "/app.jar"}, + {RealPath: "/app.jar", FullPath: "/elsewhere!/app.jar"}, + // deeper than a scan goes + {RealPath: "/app.jar", FullPath: "/app.jar" + strings.Repeat("!/a.jar", 17)}, + // an entry name a scan would refuse + {RealPath: "/app.jar", FullPath: "/app.jar!/a\nb"}, + // longer than a path + {RealPath: "/app.jar", FullPath: "/app.jar!/" + strings.Repeat("a", 4096)}, + nested, + } + + var files []model.File + for _, c := range decoded { + files = append(files, model.File{ID: string(c.ID()), Location: c, Unknowns: []string{"x"}}) + } + got := toSyftFiles(files) + // a refused chain is dropped, not merged into the outer archive's own entry + assert.Equal(t, map[file.Coordinates][]string{plain: {"x"}, nested: {"x"}}, got.Unknowns) + + var locations []file.Location + for _, c := range decoded { + locations = append(locations, file.NewLocationFromCoordinates(c)) + } + p := toSyftPackage(model.Package{PackageBasicData: model.PackageBasicData{Locations: locations}}, nil) + var gotCoordinates []file.Coordinates + for _, l := range p.Locations.ToSlice() { + gotCoordinates = append(gotCoordinates, l.Coordinates) + } + // each location keeps its own access path, so only the coordinates are compared + assert.ElementsMatch(t, []file.Coordinates{plain, nested}, gotCoordinates) +} diff --git a/syft/internal/fileresolver/container_image_all_layers.go b/syft/internal/fileresolver/container_image_all_layers.go index 1a3151e8c..ca2351785 100644 --- a/syft/internal/fileresolver/container_image_all_layers.go +++ b/syft/internal/fileresolver/container_image_all_layers.go @@ -197,7 +197,7 @@ func (r *ContainerImageAllLayers) RelativeFileByPath(location file.Location, pat func (r *ContainerImageAllLayers) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { entry, err := r.img.FileCatalog.Get(location.Reference()) if err != nil { - return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err) + return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.FullNestedPath(), err) } switch entry.Type { diff --git a/syft/internal/fileresolver/container_image_deep_squash_test.go b/syft/internal/fileresolver/container_image_deep_squash_test.go index f88afdf13..6bd9e4d9b 100644 --- a/syft/internal/fileresolver/container_image_deep_squash_test.go +++ b/syft/internal/fileresolver/container_image_deep_squash_test.go @@ -1104,15 +1104,9 @@ func TestContainerImageDeepSquash_MergeLocationStreams_FunCases(t *testing.T) { } func makeLocation(path string, layer int) file.Location { - return file.NewLocationFromCoordinates(file.Coordinates{ - RealPath: path, - FileSystemID: fmt.Sprintf("layer-%d", layer), - }) + return file.NewLocationFromCoordinates(file.NewCoordinates(path, fmt.Sprintf("layer-%d", layer))) } func makeVirtualLocation(path, access string, layer int) file.Location { - return file.NewVirtualLocationFromCoordinates(file.Coordinates{ - RealPath: path, - FileSystemID: fmt.Sprintf("layer-%d", layer), - }, access) + return file.NewVirtualLocationFromCoordinates(file.NewCoordinates(path, fmt.Sprintf("layer-%d", layer)), access) } diff --git a/syft/internal/fileresolver/container_image_squash.go b/syft/internal/fileresolver/container_image_squash.go index efcd2e8e4..b953a3583 100644 --- a/syft/internal/fileresolver/container_image_squash.go +++ b/syft/internal/fileresolver/container_image_squash.go @@ -148,7 +148,7 @@ func (r *ContainerImageSquash) RelativeFileByPath(_ file.Location, path string) func (r *ContainerImageSquash) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { entry, err := r.img.FileCatalog.Get(location.Reference()) if err != nil { - return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err) + return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.FullNestedPath(), err) } switch entry.Type { diff --git a/syft/internal/fileresolver/excluding_file.go b/syft/internal/fileresolver/excluding_file.go index 535f46671..88c706a78 100644 --- a/syft/internal/fileresolver/excluding_file.go +++ b/syft/internal/fileresolver/excluding_file.go @@ -28,14 +28,14 @@ func NewExcludingDecorator(delegate file.Resolver, excludeFn excludeFn) file.Res func (r *excluding) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { if locationMatches(&location, r.excludeFn) { - return nil, fmt.Errorf("no such location: %+v", location.RealPath) + return nil, fmt.Errorf("no such location: %+v", location.FullNestedPath()) } return r.delegate.FileContentsByLocation(location) } func (r *excluding) FileMetadataByLocation(location file.Location) (file.Metadata, error) { if locationMatches(&location, r.excludeFn) { - return file.Metadata{}, fmt.Errorf("no such location: %+v", location.RealPath) + return file.Metadata{}, fmt.Errorf("no such location: %+v", location.FullNestedPath()) } return r.delegate.FileMetadataByLocation(location) } @@ -89,7 +89,7 @@ func (r *excluding) AllLocations(ctx context.Context) <-chan file.Location { } func locationMatches(location *file.Location, exclusionFn excludeFn) bool { - return exclusionFn(location.RealPath) || exclusionFn(location.AccessPath) + return exclusionFn(location.LocalPath()) || exclusionFn(location.AccessPath) } func filterLocations(locations []file.Location, err error, exclusionFn excludeFn) ([]file.Location, error) { diff --git a/syft/internal/fileresolver/unindexed_directory.go b/syft/internal/fileresolver/unindexed_directory.go index f2435ff0d..fbc1c75c6 100644 --- a/syft/internal/fileresolver/unindexed_directory.go +++ b/syft/internal/fileresolver/unindexed_directory.go @@ -89,7 +89,7 @@ func (u UnindexedDirectory) FileContentsByLocation(location file.Location) (io.R return nil, err } if fi.IsDir() { - return nil, fmt.Errorf("unable to get contents of directory: %s", location.RealPath) + return nil, fmt.Errorf("unable to get contents of directory: %s", location.FullNestedPath()) } return f, nil } diff --git a/syft/linux/identify_release.go b/syft/linux/identify_release.go index de45c2db7..077111014 100644 --- a/syft/linux/identify_release.go +++ b/syft/linux/identify_release.go @@ -87,20 +87,20 @@ func IdentifyRelease(resolver file.Resolver) *Release { func tryParseReleaseInfo(resolver file.Resolver, location file.Location, logger logger.Logger, entry parseEntry) *Release { contentReader, err := resolver.FileContentsByLocation(location) if err != nil { - logger.WithFields("error", err, "path", location.RealPath).Trace("unable to get contents") + logger.WithFields("error", err, "path", location.FullNestedPath()).Trace("unable to get contents") return nil } - defer internal.CloseAndLogError(contentReader, location.AccessPath) + defer internal.CloseAndLogError(contentReader, location.FullNestedPath()) content, err := io.ReadAll(io.LimitReader(contentReader, 5*1024*1024)) if err != nil { - logger.WithFields("error", err, "path", location.RealPath).Trace("unable to read contents") + logger.WithFields("error", err, "path", location.FullNestedPath()).Trace("unable to read contents") return nil } release, err := entry.fn(string(content)) if err != nil { - logger.WithFields("error", err, "path", location.RealPath).Trace("unable to parse contents") + logger.WithFields("error", err, "path", location.FullNestedPath()).Trace("unable to parse contents") return nil } diff --git a/syft/linux/supplement_release.go b/syft/linux/supplement_release.go index 33a6815f6..f4366d48a 100644 --- a/syft/linux/supplement_release.go +++ b/syft/linux/supplement_release.go @@ -35,13 +35,13 @@ func supplementDebianVersion(resolver file.Resolver, release *Release) { func readDebianVersionFile(resolver file.Resolver, location file.Location) string { rdr, err := resolver.FileContentsByLocation(location) if err != nil { - log.Debugf("error getting contents for %s: %v", location.RealPath, err) + log.Debugf("error getting contents for %s: %v", location.FullNestedPath(), err) return "" } - defer internal.CloseAndLogError(rdr, location.RealPath) + defer internal.CloseAndLogError(rdr, location.FullNestedPath()) contents, err := io.ReadAll(io.LimitReader(rdr, 5*1024*1024)) if err != nil { - log.Debugf("error reading %s: %v", location.RealPath, err) + log.Debugf("error reading %s: %v", location.FullNestedPath(), err) return "" } version := strings.TrimSpace(string(contents)) diff --git a/syft/pkg/cataloger/ai/cataloger_test.go b/syft/pkg/cataloger/ai/cataloger_test.go index 8a7b3e72c..7b4815cfd 100644 --- a/syft/pkg/cataloger/ai/cataloger_test.go +++ b/syft/pkg/cataloger/ai/cataloger_test.go @@ -10,9 +10,8 @@ import ( "github.com/stretchr/testify/require" "github.com/anchore/syft/internal/tmpdir" - "github.com/anchore/syft/syft/internal/fileresolver" - "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/internal/fileresolver" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" ) diff --git a/syft/pkg/cataloger/ai/identity_dir.go b/syft/pkg/cataloger/ai/identity_dir.go index dd2460092..e7340c193 100644 --- a/syft/pkg/cataloger/ai/identity_dir.go +++ b/syft/pkg/cataloger/ai/identity_dir.go @@ -59,7 +59,7 @@ func readDirHFConfig(resolver file.Resolver, p string) (*file.Location, *hfConfi if err != nil { return nil, nil } - defer internal.CloseAndLogError(rc, p) + defer internal.CloseAndLogError(rc, locations[0].FullNestedPath()) buf, err := readBounded(rc, maxHFConfigSize) if err != nil { @@ -86,7 +86,7 @@ func readDirReadmeFrontmatter(resolver file.Resolver, p string) (*file.Location, if err != nil { return nil, nil } - defer internal.CloseAndLogError(rc, p) + defer internal.CloseAndLogError(rc, locations[0].FullNestedPath()) buf, err := readPrefix(rc, maxReadmePrefixSize) if err != nil { diff --git a/syft/pkg/cataloger/ai/identity_oci.go b/syft/pkg/cataloger/ai/identity_oci.go index 97d58aa6b..d26001f39 100644 --- a/syft/pkg/cataloger/ai/identity_oci.go +++ b/syft/pkg/cataloger/ai/identity_oci.go @@ -32,8 +32,8 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver, // seen wins below, so sort for a stable result across runs. For OCI // artifacts every RealPath is "/", so in practice this is digest order. sort.Slice(modelFileLocs, func(i, j int) bool { - if modelFileLocs[i].RealPath != modelFileLocs[j].RealPath { - return modelFileLocs[i].RealPath < modelFileLocs[j].RealPath + if modelFileLocs[i].FullNestedPath() != modelFileLocs[j].FullNestedPath() { + return modelFileLocs[i].FullNestedPath() < modelFileLocs[j].FullNestedPath() } return modelFileLocs[i].FileSystemID < modelFileLocs[j].FileSystemID }) @@ -128,7 +128,7 @@ func classifyOCIModelFileLayer(resolver file.Resolver, loc file.Location) (*hfCo if err != nil { return nil, nil } - defer internal.CloseAndLogError(rc, loc.RealPath) + defer internal.CloseAndLogError(rc, loc.FullNestedPath()) buf, err := readBounded(rc, maxModelFileLayerSize) if err != nil { @@ -174,7 +174,7 @@ func readLicenseSPDXIDFromFrontmatter(resolver file.Resolver, loc file.Location) if err != nil { return "" } - defer internal.CloseAndLogError(rc, loc.RealPath) + defer internal.CloseAndLogError(rc, loc.FullNestedPath()) buf, err := readPrefix(rc, maxLicensePrefixSize) if err != nil { diff --git a/syft/pkg/cataloger/ai/parse_gguf_model.go b/syft/pkg/cataloger/ai/parse_gguf_model.go index 478cca01d..af3a5b1d4 100644 --- a/syft/pkg/cataloger/ai/parse_gguf_model.go +++ b/syft/pkg/cataloger/ai/parse_gguf_model.go @@ -25,7 +25,7 @@ import ( // parseGGUFModel parses a GGUF model file and returns the discovered package. // This implementation only reads the header portion of the file, not the entire model. func parseGGUFModel(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - defer internal.CloseAndLogError(reader, reader.Path()) + defer internal.CloseAndLogError(reader, reader.FullNestedPath()) td := tmpdir.FromContext(ctx) if td == nil { diff --git a/syft/pkg/cataloger/ai/parse_safetensors_model.go b/syft/pkg/cataloger/ai/parse_safetensors_model.go index da3234240..b90a6cae5 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors_model.go +++ b/syft/pkg/cataloger/ai/parse_safetensors_model.go @@ -18,7 +18,7 @@ import ( // resolution, sibling enrichment, and cross-shard rollup are all handled by // safeTensorsMergeProcessor. func parseSafeTensorsFile(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - defer internal.CloseAndLogError(reader, reader.Path()) + defer internal.CloseAndLogError(reader, reader.FullNestedPath()) header, err := readSafeTensorsHeader(&io.LimitedReader{R: reader, N: maxSafeTensorsHeaderSize + 8}) if err != nil { diff --git a/syft/pkg/cataloger/ai/parse_safetensors_oci.go b/syft/pkg/cataloger/ai/parse_safetensors_oci.go index 8aa04e5b0..519532c0e 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors_oci.go +++ b/syft/pkg/cataloger/ai/parse_safetensors_oci.go @@ -47,7 +47,7 @@ type dockerAIModelConfig struct { // parseSafeTensorsOCIConfig decodes the Docker AI model-config blob func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - defer internal.CloseAndLogError(reader, reader.Path()) + defer internal.CloseAndLogError(reader, reader.FullNestedPath()) body, err := readBounded(reader, maxModelConfigBlobSize) if err != nil { @@ -86,7 +86,7 @@ func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.En // parseSafeTensorsOCILayer decodes the JSON header of a SafeTensors weight // layer fetched from an OCI model artifact func parseSafeTensorsOCILayer(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - defer internal.CloseAndLogError(reader, reader.Path()) + defer internal.CloseAndLogError(reader, reader.FullNestedPath()) header, err := readSafeTensorsHeader(&io.LimitedReader{R: reader, N: maxSafeTensorsHeaderSize + 8}) if err != nil { diff --git a/syft/pkg/cataloger/ai/processor.go b/syft/pkg/cataloger/ai/processor.go index 5cb31c3e7..f15f3c778 100644 --- a/syft/pkg/cataloger/ai/processor.go +++ b/syft/pkg/cataloger/ai/processor.go @@ -56,7 +56,7 @@ func partitionSafeTensorsPackages(pkgs []pkg.Package) (safeTensors, other []pkg. // single scan is one source, so the first package is representative of the rest. func fromOCIArtifact(pkgs []pkg.Package) bool { loc := primaryEvidenceLocation(pkgs[0]) - return loc != nil && loc.RealPath == "/" + return loc != nil && loc.LocalPath() == "/" } // mergeOCIModel treats the whole OCI artifact as a single model: every layer @@ -120,7 +120,7 @@ func groupByParentDir(pkgs []pkg.Package) map[string][]pkg.Package { if loc == nil { continue } - dir := path.Dir(loc.RealPath) + dir := path.Dir(loc.LocalPath()) out[dir] = append(out[dir], p) } return out diff --git a/syft/pkg/cataloger/alpine/parse_apk_db.go b/syft/pkg/cataloger/alpine/parse_apk_db.go index bf721c68d..f5e68f9ba 100644 --- a/syft/pkg/cataloger/alpine/parse_apk_db.go +++ b/syft/pkg/cataloger/alpine/parse_apk_db.go @@ -126,7 +126,7 @@ func parseApkDB(ctx context.Context, resolver file.Resolver, env *generic.Enviro // This should get fixed with https://gitlab.alpinelinux.org/alpine/apk-tools/-/issues/10875 if r == nil { // find the repositories file from the relative directory of the DB file - releases := findReleases(resolver, reader.RealPath) + releases := findReleases(resolver, reader.LocalPath()) if len(releases) > 0 { r = &releases[0] @@ -160,10 +160,10 @@ func findReleases(resolver file.Resolver, dbPath string) []linux.Release { reposReader, err := resolver.FileContentsByLocation(location) if err != nil { - log.Tracef("unable to fetch contents for APK repositories file %q: %+v", reposLocation, err) + log.Tracef("unable to fetch contents for APK repositories file %q: %+v", location.FullNestedPath(), err) return nil } - defer internal.CloseAndLogError(reposReader, location.RealPath) + defer internal.CloseAndLogError(reposReader, location.FullNestedPath()) return parseReleasesFromAPKRepository(file.LocationReadCloser{ Location: location, @@ -176,7 +176,7 @@ func parseReleasesFromAPKRepository(reader file.LocationReadCloser) []linux.Rele reposB, err := io.ReadAll(reader) //nolint:gocritic // regex matching requires full buffer if err != nil { - log.Tracef("unable to read APK repositories file %q: %+v", reader.RealPath, err) + log.Tracef("unable to read APK repositories file %q: %+v", reader.FullNestedPath(), err) return nil } diff --git a/syft/pkg/cataloger/arch/parse_alpm_db.go b/syft/pkg/cataloger/arch/parse_alpm_db.go index ac63a4141..ef1c34316 100644 --- a/syft/pkg/cataloger/arch/parse_alpm_db.go +++ b/syft/pkg/cataloger/arch/parse_alpm_db.go @@ -55,7 +55,7 @@ func parseAlpmDB(ctx context.Context, resolver file.Resolver, env *generic.Envir return nil, nil, nil } - base := path.Dir(reader.RealPath) + base := path.Dir(reader.LocalPath()) var locs []file.Location @@ -102,7 +102,7 @@ func fetchPkgFiles(base string, resolver file.Resolver) ([]pkg.AlpmFileRecord, f if err != nil { return []pkg.AlpmFileRecord{}, loc, unknown.New(loc, fmt.Errorf("failed to get contents: %w", err)) } - defer internal.CloseAndLogError(reader, loc.RealPath) + defer internal.CloseAndLogError(reader, loc.FullNestedPath()) pkgFiles, err := parseMtree(reader) if err != nil { @@ -126,7 +126,7 @@ func fetchBackupFiles(base string, resolver file.Resolver) ([]pkg.AlpmFileRecord if err != nil { return []pkg.AlpmFileRecord{}, loc, unknown.New(loc, fmt.Errorf("failed to get contents: %w", err)) } - defer internal.CloseAndLogError(reader, loc.RealPath) + defer internal.CloseAndLogError(reader, loc.FullNestedPath()) filesMetadata, err := parseAlpmDBEntry(reader) if err != nil { diff --git a/syft/pkg/cataloger/binary/classifier_cataloger_test.go b/syft/pkg/cataloger/binary/classifier_cataloger_test.go index 4138d7e76..b0c56d257 100644 --- a/syft/pkg/cataloger/binary/classifier_cataloger_test.go +++ b/syft/pkg/cataloger/binary/classifier_cataloger_test.go @@ -3661,9 +3661,7 @@ func match(classifier string, paths ...string) pkg.ClassifierMatch { return pkg.ClassifierMatch{ Classifier: classifier, Location: file.NewVirtualLocationFromCoordinates( - file.Coordinates{ - RealPath: realPath, - }, + file.NewCoordinates(realPath, ""), virtualPath, ), } diff --git a/syft/pkg/cataloger/binary/elf_package_cataloger.go b/syft/pkg/cataloger/binary/elf_package_cataloger.go index 943550d2e..41200e54f 100644 --- a/syft/pkg/cataloger/binary/elf_package_cataloger.go +++ b/syft/pkg/cataloger/binary/elf_package_cataloger.go @@ -99,9 +99,9 @@ func (c *elfPackageCataloger) Catalog(ctx context.Context, resolver file.Resolve func parseElfPackageNotes(resolver file.Resolver, location file.Location, c *elfPackageCataloger) (*elfBinaryPackageNotes, elfPackageKey, error) { reader, err := resolver.FileContentsByLocation(location) if err != nil { - return nil, elfPackageKey{}, fmt.Errorf("unable to get binary contents %q: %w", location.Path(), err) + return nil, elfPackageKey{}, fmt.Errorf("unable to get binary contents %q: %w", location.FullNestedPath(), err) } - defer internal.CloseAndLogError(reader, location.AccessPath) + defer internal.CloseAndLogError(reader, location.FullNestedPath()) notes, err := c.parseElfNotes(file.LocationReadCloser{ Location: location, @@ -109,7 +109,7 @@ func parseElfPackageNotes(resolver file.Resolver, location file.Location, c *elf }) if err != nil { - log.WithFields("file", location.Path(), "error", err).Trace("unable to parse ELF notes") + log.WithFields("file", location.FullNestedPath(), "error", err).Trace("unable to parse ELF notes") return nil, elfPackageKey{}, err } @@ -148,7 +148,7 @@ func getELFNotes(r file.LocationReadCloser) (*elfBinaryPackageNotes, error) { f, err := elfutil.NewFile(unionReader) if f == nil || err != nil { - log.WithFields("file", r.Location.Path(), "error", err).Trace("unable to parse binary as ELF") + log.WithFields("file", r.Location.FullNestedPath(), "error", err).Trace("unable to parse binary as ELF") return nil, nil } @@ -182,7 +182,7 @@ func getELFNotes(r file.LocationReadCloser) (*elfBinaryPackageNotes, error) { if metadata, err = unmarshalELFPackageNotesPayload(newPayload); err == nil { return metadata, nil } - log.WithFields("file", r.Location.Path(), "error", err).Trace("unable to unmarshal ELF package notes as JSON") + log.WithFields("file", r.Location.FullNestedPath(), "error", err).Trace("unable to unmarshal ELF package notes as JSON") } } diff --git a/syft/pkg/cataloger/binary/pe_package.go b/syft/pkg/cataloger/binary/pe_package.go index a7cab30e2..9d25b8487 100644 --- a/syft/pkg/cataloger/binary/pe_package.go +++ b/syft/pkg/cataloger/binary/pe_package.go @@ -22,7 +22,7 @@ func newPEPackage(versionResources map[string]string, f file.Location) pkg.Packa if name == "" { // it's possible that the version resources are empty, so we fall back to the file name - name = strings.TrimSuffix(strings.TrimSuffix(path.Base(f.RealPath), ".exe"), ".dll") + name = strings.TrimSuffix(strings.TrimSuffix(path.Base(f.LocalPath()), ".exe"), ".dll") } p := pkg.Package{ diff --git a/syft/pkg/cataloger/binary/pe_package_cataloger.go b/syft/pkg/cataloger/binary/pe_package_cataloger.go index 05410ab97..2d77c24bf 100644 --- a/syft/pkg/cataloger/binary/pe_package_cataloger.go +++ b/syft/pkg/cataloger/binary/pe_package_cataloger.go @@ -22,7 +22,7 @@ func NewPEPackageCataloger() pkg.Cataloger { func parsePE(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { f, err := pe.Read(reader) if err != nil { - return nil, nil, fmt.Errorf("failed to parse PE file %q: %w", reader.RealPath, err) + return nil, nil, fmt.Errorf("failed to parse PE file %q: %w", reader.FullNestedPath(), err) } if f == nil { diff --git a/syft/pkg/cataloger/bitnami/cataloger.go b/syft/pkg/cataloger/bitnami/cataloger.go index 1c215c01c..87fb646cd 100644 --- a/syft/pkg/cataloger/bitnami/cataloger.go +++ b/syft/pkg/cataloger/bitnami/cataloger.go @@ -15,6 +15,7 @@ import ( "github.com/anchore/syft/syft/format" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/pkg/cataloger/internal/embeddedsbom" ) const catalogerName = "bitnami-cataloger" @@ -37,13 +38,13 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment } if s == nil { - log.WithFields("path", reader.RealPath).Trace("file is not an SBOM") + log.WithFields("path", reader.FullNestedPath()).Trace("file is not an SBOM") return nil, nil, nil } // Bitnami exclusively uses SPDX JSON SBOMs if sFormat != "spdx-json" { - log.WithFields("path", reader.RealPath).Trace("file is not an SPDX JSON SBOM") + log.WithFields("path", reader.FullNestedPath()).Trace("file is not an SPDX JSON SBOM") return nil, nil, nil } @@ -75,7 +76,7 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment // Bitnami packages reported in a SPDX file are shipped under the same directory // as the SPDX file itself. - metadata.Path = path.Dir(reader.RealPath) + metadata.Path = path.Dir(reader.LocalPath()) if p.ID() != mainPkgID { metadata.Files = packageFiles(s.Relationships, p, metadata.Path) secondaryPkgsFiles = append(secondaryPkgsFiles, metadata.Files...) @@ -91,12 +92,12 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment } // Resolve all files owned by the main package in the SBOM and update the metadata - if mainPkgFiles, err := mainPkgFiles(resolver, reader.RealPath, secondaryPkgsFiles); err == nil { + if mainPkgFiles, err := mainPkgFiles(resolver, reader.LocalPath(), secondaryPkgsFiles); err == nil { for i, p := range pkgs { if p.ID() == mainPkgID { metadata, ok := p.Metadata.(*pkg.BitnamiSBOMEntry) if !ok { - log.WithFields("spdx-filepath", reader.RealPath).Trace("main package in SBOM does not have Bitnami metadata") + log.WithFields("spdx-filepath", reader.FullNestedPath()).Trace("main package in SBOM does not have Bitnami metadata") continue } @@ -105,30 +106,33 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment } } } else { - log.WithFields("spdx-filepath", reader.RealPath, "error", err).Trace("unable to resolve owned files for main package in SBOM") + log.WithFields("spdx-filepath", reader.FullNestedPath(), "error", err).Trace("unable to resolve owned files for main package in SBOM") } return pkgs, filterRelationships(s.Relationships, pkgs), nil } -// filterRelationships filters out relationships that are not related to Bitnami packages +// filterRelationships filters out relationships that are not from Bitnami packages // and replaces the package information with the one with completed info func filterRelationships(relationships []artifact.Relationship, pkgs []pkg.Package) []artifact.Relationship { var result []artifact.Relationship - for _, r := range relationships { - if value, ok := r.From.(pkg.Package); ok { - found := false - for _, p := range pkgs { - if value.PURL == p.PURL { - r.From = p - found = true - break - } - } - if !found { - continue + for _, r := range embeddedsbom.Relationships(relationships) { + // only what a package says about itself is kept + value, ok := r.From.(pkg.Package) + if !ok { + continue + } + found := false + for _, p := range pkgs { + if value.PURL == p.PURL { + r.From = p + found = true + break } } + if !found { + continue + } if value, ok := r.To.(pkg.Package); ok { found := false for _, p := range pkgs { diff --git a/syft/pkg/cataloger/bitnami/cataloger_test.go b/syft/pkg/cataloger/bitnami/cataloger_test.go index 5721f04a7..2f862f50a 100644 --- a/syft/pkg/cataloger/bitnami/cataloger_test.go +++ b/syft/pkg/cataloger/bitnami/cataloger_test.go @@ -2,6 +2,8 @@ package bitnami import ( "context" + "io" + "strings" "testing" "github.com/stretchr/testify/require" @@ -622,3 +624,67 @@ func TestBitnamiCataloger(t *testing.T) { }) } } + +func Test_filterRelationships_keepsOnlyPackageRelationships(t *testing.T) { + p := pkg.Package{Name: "postgresql", PURL: "pkg:bitnami/postgresql@17.2.0"} + jar := file.NewLocation("app.jar") + inner := file.NewLocation("BOOT-INF/lib/log4j-core-2.14.1.jar") + // as the decoder nests it when the SBOM also says the jar CONTAINS it + nested := file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/app.jar", FullPath: "/app.jar!/BOOT-INF/lib/log4j-core-2.14.1.jar"}) + relationships := []artifact.Relationship{ + {From: p, To: jar, Type: artifact.ContainsRelationship}, + {From: jar, To: inner, Type: artifact.ContainsRelationship}, + {From: jar, To: p, Type: artifact.ContainsRelationship}, + {From: p, To: nested, Type: artifact.ContainsRelationship}, + } + + got := filterRelationships(relationships, []pkg.Package{p}) + + require.Equal(t, relationships[:1], got) +} + +func Test_parseSBOM_nestingInTheSBOMIsNotKept(t *testing.T) { + // a file the SBOM says another CONTAINS decodes as nested within it, and one the SBOM DESCRIBES as the scan + // root; neither is anything the scan found + const head = `{ + "spdxVersion":"SPDX-2.3","dataLicense":"CC0-1.0","SPDXID":"SPDXRef-DOCUMENT","name":"x","documentNamespace":"https://x/y", + "creationInfo":{"created":"2024-01-01T00:00:00Z","creators":["Tool: x"]}, + "packages":[{"name":"postgresql","SPDXID":"SPDXRef-pg","versionInfo":"17.2.0","downloadLocation":"NONE", + "externalRefs":[{"referenceCategory":"PACKAGE-MANAGER","referenceType":"purl","referenceLocator":"pkg:bitnami/postgresql@17.2.0"}]}], + "files":[{"fileName":"app.jar","SPDXID":"SPDXRef-F1","checksums":[{"algorithm":"SHA1","checksumValue":"0000000000000000000000000000000000000000"}]}, + {"fileName":"BOOT-INF/lib/log4j-core-2.14.1.jar","SPDXID":"SPDXRef-F2","checksums":[{"algorithm":"SHA1","checksumValue":"0000000000000000000000000000000000000000"}]}], + "relationships":[` + tests := []struct { + name string + relationships string + }{ + { + name: "nested", + relationships: `{"spdxElementId":"SPDXRef-DOCUMENT","relationshipType":"DESCRIBES","relatedSpdxElement":"SPDXRef-pg"}, + {"spdxElementId":"SPDXRef-pg","relationshipType":"CONTAINS","relatedSpdxElement":"SPDXRef-F2"}, + {"spdxElementId":"SPDXRef-F1","relationshipType":"CONTAINS","relatedSpdxElement":"SPDXRef-F2"}`, + }, + { + name: "scan root", + relationships: `{"spdxElementId":"SPDXRef-DOCUMENT","relationshipType":"DESCRIBES","relatedSpdxElement":"SPDXRef-pg"}, + {"spdxElementId":"SPDXRef-pg","relationshipType":"DESCRIBES","relatedSpdxElement":"SPDXRef-F1"}, + {"spdxElementId":"SPDXRef-pg","relationshipType":"CONTAINS","relatedSpdxElement":"SPDXRef-F1"}, + {"spdxElementId":"SPDXRef-F1","relationshipType":"CONTAINS","relatedSpdxElement":"SPDXRef-F2"}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + loc := file.NewLocation("/opt/bitnami/postgresql/.spdx-postgresql.spdx") + doc := head + tt.relationships + "]}" + _, relationships, err := parseSBOM(context.Background(), file.NewMockResolverForPaths(), nil, file.NewLocationReadCloser(loc, io.NopCloser(strings.NewReader(doc)))) + require.NoError(t, err) + + for _, r := range relationships { + if l, ok := r.To.(file.Location); ok { + require.Empty(t, l.FullPath, "%+v", r) + require.NotContains(t, []string{"", "/"}, l.RealPath, "%+v", r) + } + } + }) + } +} diff --git a/syft/pkg/cataloger/bitnami/package.go b/syft/pkg/cataloger/bitnami/package.go index cb46d5453..f4eb402e1 100644 --- a/syft/pkg/cataloger/bitnami/package.go +++ b/syft/pkg/cataloger/bitnami/package.go @@ -61,7 +61,7 @@ func packageFiles(relationships []artifact.Relationship, p pkg.Package, baseDire // note: the file.Location is from the SBOM, and all files within the Bitnami SBOM by convention // are relative to the /opt/bitnami/PRODUCT directory, so we need to prepend the base directory // so that it's relative to the path found within the container image. - result = append(result, path.Join(baseDirectory, value.RealPath)) + result = append(result, path.Join(baseDirectory, value.LocalPath())) } } } diff --git a/syft/pkg/cataloger/bitnami/parse_components.go b/syft/pkg/cataloger/bitnami/parse_components.go index d2eb6d592..d22424ca4 100644 --- a/syft/pkg/cataloger/bitnami/parse_components.go +++ b/syft/pkg/cataloger/bitnami/parse_components.go @@ -67,7 +67,7 @@ func parseComponentsJSON(_ context.Context, _ file.Resolver, _ *generic.Environm Revision: revision, Architecture: entry.Arch, Distro: entry.Distro, - Path: path.Join(path.Dir(reader.RealPath), name), + Path: path.Join(path.Dir(reader.LocalPath()), name), } p := pkg.Package{ diff --git a/syft/pkg/cataloger/conda/cataloger.go b/syft/pkg/cataloger/conda/cataloger.go index 39667c8c0..61fc009db 100644 --- a/syft/pkg/cataloger/conda/cataloger.go +++ b/syft/pkg/cataloger/conda/cataloger.go @@ -24,7 +24,7 @@ func parseCondaMetaJSON(ctx context.Context, _ file.Resolver, _ *generic.Environ dec := json.NewDecoder(reader) var meta pkg.CondaMetaPackage if err := dec.Decode(&meta); err != nil { - return nil, nil, fmt.Errorf("failed to parse conda-meta package file at %s: %w", reader.RealPath, err) + return nil, nil, fmt.Errorf("failed to parse conda-meta package file at %s: %w", reader.FullNestedPath(), err) } p := pkg.Package{ diff --git a/syft/pkg/cataloger/cpp/parse_conaninfo.go b/syft/pkg/cataloger/cpp/parse_conaninfo.go index 895497174..390b520cb 100644 --- a/syft/pkg/cataloger/cpp/parse_conaninfo.go +++ b/syft/pkg/cataloger/cpp/parse_conaninfo.go @@ -81,9 +81,11 @@ func parseFullRequiresLine(line string, reader file.LocationReadCloser, pkgs *[] // - recipe revision (recipe_hash) func parseConaninfo(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { // First set the base package info by checking the relative path - fullFilePath := string(reader.Location.LocationData.Reference().RealPath) - if len(fullFilePath) == 0 { - fullFilePath = reader.RealPath + // the reference's real path is the absolute host path, which holds the package layout even when the + // scan is rooted below .conan/data + fullFilePath := string(reader.Reference().RealPath) + if fullFilePath == "" { + fullFilePath = reader.LocalPath() } mainMetadata, err := parseConanMetadataFromFilePath(fullFilePath) diff --git a/syft/pkg/cataloger/cpp/parse_conaninfo_test.go b/syft/pkg/cataloger/cpp/parse_conaninfo_test.go index 92fbd1342..450dee9a7 100644 --- a/syft/pkg/cataloger/cpp/parse_conaninfo_test.go +++ b/syft/pkg/cataloger/cpp/parse_conaninfo_test.go @@ -1,8 +1,13 @@ package cpp import ( + "context" + "os" "testing" + "github.com/stretchr/testify/require" + + stereoscopeFile "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -126,3 +131,19 @@ func TestParseConaninfo(t *testing.T) { pkgtest.TestFileParser(t, fixture, parseConaninfo, expected, expectedRelationships) } + +func TestParseConaninfo_pathFromTheFileReference(t *testing.T) { + // a scan rooted at the package directory: only the reference's real path has the package layout + fixture := "testdata/conaninfo/mfast/1.2.2/my_user/my_channel/package/9d1f076b471417647c2022a78d5e2c1f834289ac/conaninfo.txt" + f, err := os.Open(fixture) + require.NoError(t, err) + t.Cleanup(func() { _ = f.Close() }) + + ref := stereoscopeFile.NewFileReference(stereoscopeFile.Path("/home/user/.conan/data/mfast/1.2.2/my_user/my_channel/package/9d1f076b471417647c2022a78d5e2c1f834289ac/conaninfo.txt")) + location := file.NewLocationFromDirectory("/conaninfo.txt", "", *ref) + + pkgs, _, err := parseConaninfo(context.Background(), nil, nil, file.NewLocationReadCloser(location, f)) + require.NoError(t, err) + require.NotEmpty(t, pkgs) + require.Equal(t, "mfast", pkgs[len(pkgs)-1].Name) +} diff --git a/syft/pkg/cataloger/cpp/parse_vcpkgmanifest.go b/syft/pkg/cataloger/cpp/parse_vcpkgmanifest.go index cf11d15bb..fdd86f1d6 100644 --- a/syft/pkg/cataloger/cpp/parse_vcpkgmanifest.go +++ b/syft/pkg/cataloger/cpp/parse_vcpkgmanifest.go @@ -132,7 +132,7 @@ func findManAtLoc(loc file.Location, resolver file.Resolver) (*vcpkg.Vcpkg, erro if err != nil { return nil, err } - defer internal.CloseAndLogError(manCont, loc.RealPath) + defer internal.CloseAndLogError(manCont, loc.FullNestedPath()) manBytes, err := io.ReadAll(io.LimitReader(manCont, maxVcpkgManifestSize)) if err != nil { return nil, err @@ -166,7 +166,7 @@ func findVcpkgConfig(resolver file.Resolver) (*vcpkg.Config, error) { if err != nil { return nil, err } - defer internal.CloseAndLogError(cfgCont, locs[0].RealPath) + defer internal.CloseAndLogError(cfgCont, locs[0].FullNestedPath()) cfgBytes, err := io.ReadAll(io.LimitReader(cfgCont, maxVcpkgManifestSize)) if err != nil { return nil, err diff --git a/syft/pkg/cataloger/debian/package.go b/syft/pkg/cataloger/debian/package.go index c21dc6452..28b09256d 100644 --- a/syft/pkg/cataloger/debian/package.go +++ b/syft/pkg/cataloger/debian/package.go @@ -187,7 +187,7 @@ func addLicenses(ctx context.Context, resolver file.Resolver, dbLocation file.Lo copyrightReader, copyrightLocation := fetchCopyrightContents(resolver, dbLocation, metadata) var licenseStrs []string if copyrightReader != nil && copyrightLocation != nil { - defer internal.CloseAndLogError(copyrightReader, copyrightLocation.AccessPath) + defer internal.CloseAndLogError(copyrightReader, copyrightLocation.FullNestedPath()) // attach the licenses licenseStrs = parseLicensesFromCopyright(copyrightReader) for _, licenseStr := range licenseStrs { @@ -200,7 +200,7 @@ func addLicenses(ctx context.Context, resolver file.Resolver, dbLocation file.Lo if len(licenseStrs) == 0 { sr, sl := fetchCopyrightContents(resolver, dbLocation, metadata) if sr != nil && sl != nil { - defer internal.CloseAndLogError(sr, sl.AccessPath) + defer internal.CloseAndLogError(sr, sl.FullNestedPath()) p.Licenses.Add(pkg.NewLicensesFromReadCloserWithContext(ctx, file.NewLocationReadCloser(*sl, sr))...) } } @@ -246,11 +246,11 @@ func getAdditionalFileListing(resolver file.Resolver, dbLocation file.Location, md5Reader, md5Location := fetchMd5Contents(resolver, dbLocation, m) if md5Reader != nil && md5Location != nil { - defer internal.CloseAndLogError(md5Reader, md5Location.AccessPath) + defer internal.CloseAndLogError(md5Reader, md5Location.FullNestedPath()) // attach the file list records, err := parseDpkgMD5Info(md5Reader) if err != nil { - log.Debugf("dpkg md5sums file %q: %v", md5Location.RealPath, err) + log.Debugf("dpkg md5sums file %q: %v", md5Location.FullNestedPath(), err) } files = append(files, records...) @@ -261,11 +261,11 @@ func getAdditionalFileListing(resolver file.Resolver, dbLocation file.Location, conffilesReader, conffilesLocation := fetchConffileContents(resolver, dbLocation, m) if conffilesReader != nil && conffilesLocation != nil { - defer internal.CloseAndLogError(conffilesReader, conffilesLocation.AccessPath) + defer internal.CloseAndLogError(conffilesReader, conffilesLocation.FullNestedPath()) // attach the file list records, err := parseDpkgConffileInfo(conffilesReader) if err != nil { - log.Debugf("dpkg conffiles file %q: %v", conffilesLocation.RealPath, err) + log.Debugf("dpkg conffiles file %q: %v", conffilesLocation.FullNestedPath(), err) } files = append(files, records...) @@ -288,7 +288,7 @@ func fetchMd5Contents(resolver file.Resolver, dbLocation file.Location, m pkg.Dp // and the md5sum information is under /var/lib/dpkg/info/; however, for distroless the installed // package info is across multiple files under /var/lib/dpkg/status.d/ and the md5sums are contained in // the same directory - searchPath := filepath.Dir(dbLocation.RealPath) + searchPath := filepath.Dir(dbLocation.LocalPath()) if !strings.HasSuffix(searchPath, "status.d") { searchPath = path.Join(searchPath, "info") @@ -327,7 +327,7 @@ func fetchConffileContents(resolver file.Resolver, dbLocation file.Location, m p return nil, nil } - parentPath := filepath.Dir(dbLocation.RealPath) + parentPath := filepath.Dir(dbLocation.LocalPath()) // look for /var/lib/dpkg/info/NAME:ARCH.conffiles name := md5Key(m) diff --git a/syft/pkg/cataloger/debian/parse_dpkg_db.go b/syft/pkg/cataloger/debian/parse_dpkg_db.go index 0168e5fb5..8ab89c5ca 100644 --- a/syft/pkg/cataloger/debian/parse_dpkg_db.go +++ b/syft/pkg/cataloger/debian/parse_dpkg_db.go @@ -37,7 +37,7 @@ var ( func parseDpkgDB(ctx context.Context, resolver file.Resolver, env *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { metadata, err := parseDpkgStatus(reader) if err != nil { - return nil, nil, fmt.Errorf("unable to catalog dpkg DB=%q: %w", reader.RealPath, err) + return nil, nil, fmt.Errorf("unable to catalog dpkg DB=%q: %w", reader.FullNestedPath(), err) } dbLoc := reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation) @@ -61,7 +61,7 @@ func findDpkgInfoFiles(name string, resolver file.Resolver, dbLocation file.Loca // and the md5sum information is under /var/lib/dpkg/info/; however, for distroless the installed // package info is across multiple files under /var/lib/dpkg/status.d/ and the md5sums are contained in // the same directory - searchPath := path.Dir(dbLocation.RealPath) + searchPath := path.Dir(dbLocation.LocalPath()) if !strings.HasSuffix(searchPath, "status.d") { searchPath = path.Join(searchPath, "info") diff --git a/syft/pkg/cataloger/dotnet/deps_binary_cataloger.go b/syft/pkg/cataloger/dotnet/deps_binary_cataloger.go index cde56fd44..d4f4d69ea 100644 --- a/syft/pkg/cataloger/dotnet/deps_binary_cataloger.go +++ b/syft/pkg/cataloger/dotnet/deps_binary_cataloger.go @@ -150,7 +150,7 @@ var runtimeDLLPathPattern = regexp.MustCompile(`/Microsoft\.NETCore\.App/(?P depJsons[j].Location.RealPath + return depJsons[i].Location.FullNestedPath() > depJsons[j].Location.FullNestedPath() }) // we should be processing PE files in a stable order sort.Slice(peFiles, func(i, j int) bool { - return peFiles[i].Location.RealPath > peFiles[j].Location.RealPath + return peFiles[i].Location.FullNestedPath() > peFiles[j].Location.FullNestedPath() }) peFilesByPath := make(map[file.Coordinates][]logicalPE) @@ -191,7 +191,7 @@ func partitionPEs(depJsons []logicalDepsJSON, peFiles []logicalPE) ([]logicalDep var found bool for i := range depJsons { dep := &depJsons[i] - if isParentOf(dep.Location.RealPath, pe.Location.RealPath) && attachAssociatedExecutables(dep, pe) { + if isParentOf(dep.Location.LocalPath(), pe.Location.LocalPath()) && attachAssociatedExecutables(dep, pe) { peFilesByPath[dep.Location.Coordinates] = append(peFilesByPath[dep.Location.Coordinates], pe) found = true // note: we cannot break from the dep JSON search since the same binary could be associated with multiple packages @@ -222,8 +222,8 @@ func partitionPEs(depJsons []logicalDepsJSON, peFiles []logicalPE) ([]logicalDep // attachAssociatedExecutables looks for PE files matching runtime or resource entries // and attaches them to the appropriate package. func attachAssociatedExecutables(dep *logicalDepsJSON, pe logicalPE) bool { - appDir := path.Dir(dep.Location.RealPath) - relativeDllPath := strings.TrimPrefix(strings.TrimPrefix(pe.Location.RealPath, appDir), "/") + appDir := path.Dir(dep.Location.LocalPath()) + relativeDllPath := strings.TrimPrefix(strings.TrimPrefix(pe.Location.LocalPath(), appDir), "/") var found bool for key, p := range dep.PackagesByNameVersion { @@ -471,7 +471,7 @@ func readDepsJSON(resolver file.Resolver, loc file.Location) (*depsJSON, error) if err != nil { return nil, unknown.New(loc, fmt.Errorf("unable to read deps.json file: %w", err)) } - defer internal.CloseAndLogError(reader, loc.RealPath) + defer internal.CloseAndLogError(reader, loc.FullNestedPath()) dj, err := newDepsJSON(file.NewLocationReadCloser(loc, reader)) if err != nil { @@ -479,7 +479,7 @@ func readDepsJSON(resolver file.Resolver, loc file.Location) (*depsJSON, error) } if dj == nil { - return nil, unknown.New(loc, fmt.Errorf("expected to find packages in deps.json but did not: %q", loc.RealPath)) + return nil, unknown.New(loc, fmt.Errorf("expected to find packages in deps.json but did not: %q", loc.FullNestedPath())) } return dj, nil @@ -520,7 +520,7 @@ func readPEFile(resolver file.Resolver, loc file.Location) (*logicalPE, error) { if err != nil { return nil, unknown.New(loc, fmt.Errorf("unable to read PE file: %w", err)) } - defer internal.CloseAndLogError(reader, loc.RealPath) + defer internal.CloseAndLogError(reader, loc.FullNestedPath()) ldpe, err := readLogicalPE(file.NewLocationReadCloser(loc, reader)) if err != nil { @@ -566,7 +566,7 @@ func readELFBundledDepsJSON(resolver file.Resolver, loc file.Location) (*logical if err != nil { return nil, err } - defer internal.CloseAndLogError(reader, loc.RealPath) + defer internal.CloseAndLogError(reader, loc.FullNestedPath()) header := make([]byte, 4) if _, err := io.ReadFull(reader, header); err != nil { diff --git a/syft/pkg/cataloger/dotnet/deps_json.go b/syft/pkg/cataloger/dotnet/deps_json.go index b0a8d2af8..e036d2fe5 100644 --- a/syft/pkg/cataloger/dotnet/deps_json.go +++ b/syft/pkg/cataloger/dotnet/deps_json.go @@ -161,7 +161,7 @@ type logicalDepsJSON struct { } func (l logicalDepsJSON) RootPackage() (logicalDepsJSONPackage, bool) { - rootName := getDepsJSONFilePrefix(l.Location.RealPath) + rootName := getDepsJSONFilePrefix(l.Location.FullNestedPath()) if rootName == "" { return logicalDepsJSONPackage{}, false } diff --git a/syft/pkg/cataloger/dotnet/libman_json.go b/syft/pkg/cataloger/dotnet/libman_json.go index 7f585eb08..f1e8d58a3 100644 --- a/syft/pkg/cataloger/dotnet/libman_json.go +++ b/syft/pkg/cataloger/dotnet/libman_json.go @@ -87,14 +87,14 @@ func newLibmanJSON(reader file.LocationReadCloser) (*libmanJSON, error) { } func findLibmanJSON(resolver file.Resolver, depsJSON file.Location) (*libmanJSON, error) { - parent := path.Dir(depsJSON.RealPath) + parent := path.Dir(depsJSON.LocalPath()) loc := resolver.RelativeFileByPath(depsJSON, path.Join(parent, "libman.json")) if loc == nil { return nil, nil } reader, err := resolver.FileContentsByLocation(*loc) - defer internal.CloseAndLogError(reader, loc.RealPath) + defer internal.CloseAndLogError(reader, loc.FullNestedPath()) if err != nil { return nil, err } diff --git a/syft/pkg/cataloger/dotnet/package.go b/syft/pkg/cataloger/dotnet/package.go index ac2361d60..e8e3f63a3 100644 --- a/syft/pkg/cataloger/dotnet/package.go +++ b/syft/pkg/cataloger/dotnet/package.go @@ -265,7 +265,7 @@ func newDotnetBinaryPackage(versionResources map[string]string, f file.Location) if name == "" { // older .NET runtime dlls may not have any version resources - name = strings.TrimSuffix(strings.TrimSuffix(path.Base(f.RealPath), ".exe"), ".dll") + name = strings.TrimSuffix(strings.TrimSuffix(path.Base(f.FullNestedPath()), ".exe"), ".dll") } ver := findVersionFromVersionResources(versionResources) diff --git a/syft/pkg/cataloger/elixir/parse_mix_lock.go b/syft/pkg/cataloger/elixir/parse_mix_lock.go index e44aaa8c0..ed836293f 100644 --- a/syft/pkg/cataloger/elixir/parse_mix_lock.go +++ b/syft/pkg/cataloger/elixir/parse_mix_lock.go @@ -83,7 +83,7 @@ func parseMixLock(_ context.Context, _ file.Resolver, _ *generic.Environment, re } if name == "" { - log.WithFields("path", reader.RealPath).Debug("skipping empty package name from mix.lock file") + log.WithFields("path", reader.FullNestedPath()).Debug("skipping empty package name from mix.lock file") errs = unknown.Appendf(errs, reader, "skipping empty package name from mix.lock file, for line: %d: %s", lineNum, line) continue } diff --git a/syft/pkg/cataloger/generic/cataloger.go b/syft/pkg/cataloger/generic/cataloger.go index ec4a5384a..e11b1158b 100644 --- a/syft/pkg/cataloger/generic/cataloger.go +++ b/syft/pkg/cataloger/generic/cataloger.go @@ -203,7 +203,7 @@ func (c *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg. errs := sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(c.selectFiles(resolver)), func(req request) (result, error) { location, parser := req.Location, req.Parser - log.WithFields("path", location.RealPath).Trace("parsing file contents") + log.WithFields("path", location.FullNestedPath()).Trace("parsing file contents") discoveredPackages, discoveredRelationships, err := invokeParser(ctx, resolver, location, lgr, parser, &env) if err != nil { @@ -243,15 +243,15 @@ func (c *Cataloger) processSafely(ctx context.Context, p processExecutor, resolv func invokeParser(ctx context.Context, resolver file.Resolver, location file.Location, logger logger.Logger, parser Parser, env *Environment) (pkgs []pkg.Package, rels []artifact.Relationship, err error) { contentReader, err := resolver.FileContentsByLocation(location) if err != nil { - logger.WithFields("location", location.RealPath, "error", err).Debug("unable to fetch contents") + logger.WithFields("location", location.FullNestedPath(), "error", err).Debug("unable to fetch contents") return nil, nil, err } - defer internal.CloseAndLogError(contentReader, location.AccessPath) + defer internal.CloseAndLogError(contentReader, location.FullNestedPath()) defer func() { if r := recover(); r != nil { // a malformed file must not take down the whole SBOM; the caller records this as an unknown - logger.WithFields("location", location.RealPath, "panic", r).Debug("recovered from panic while parsing file") + logger.WithFields("location", location.FullNestedPath(), "panic", r).Debug("recovered from panic while parsing file") logger.Tracef("parser panic stack:\n%s", debug.Stack()) pkgs, rels, err = nil, nil, fmt.Errorf("recovered from panic while parsing file: %v", r) } @@ -260,7 +260,7 @@ func invokeParser(ctx context.Context, resolver file.Resolver, location file.Loc discoveredPackages, discoveredRelationships, err := parser(ctx, resolver, env, file.NewLocationReadCloser(location, contentReader)) if err != nil { // these errors are propagated up, and are likely to be coordinate errors - logger.WithFields("location", location.RealPath, "error", err).Trace("cataloger returned errors") + logger.WithFields("location", location.FullNestedPath(), "error", err).Trace("cataloger returned errors") } return discoveredPackages, discoveredRelationships, err diff --git a/syft/pkg/cataloger/generic/cataloger_test.go b/syft/pkg/cataloger/generic/cataloger_test.go index 93f7813d0..57df02893 100644 --- a/syft/pkg/cataloger/generic/cataloger_test.go +++ b/syft/pkg/cataloger/generic/cataloger_test.go @@ -19,7 +19,7 @@ import ( func Test_Cataloger(t *testing.T) { allParsedPaths := make(map[string]bool) parser := func(_ context.Context, resolver file.Resolver, env *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - allParsedPaths[reader.Path()] = true + allParsedPaths[reader.FullNestedPath()] = true contents, err := io.ReadAll(reader) require.NoError(t, err) @@ -219,7 +219,7 @@ func TestParserPanicBecomesUnknown(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { parser := func(_ context.Context, _ file.Resolver, _ *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - if reader.Path() == "testdata/another-path.txt" { + if reader.FullNestedPath() == "testdata/another-path.txt" { tt.doPanic() } return []pkg.Package{{Name: "good", Locations: file.NewLocationSet(reader.Location)}}, nil, nil @@ -308,7 +308,7 @@ func TestProcessorPanicKeepsResults(t *testing.T) { func Test_CatalogerWithParserByMediaType(t *testing.T) { allParsedPaths := make(map[string]bool) parser := func(_ context.Context, resolver file.Resolver, env *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - allParsedPaths[reader.Path()] = true + allParsedPaths[reader.FullNestedPath()] = true contents, err := io.ReadAll(reader) require.NoError(t, err) @@ -358,7 +358,7 @@ func Test_genericCatalogerReturnsErrors(t *testing.T) { genericErrorReturning := NewCataloger("error returning").WithParserByGlobs(func(ctx context.Context, resolver file.Resolver, environment *Environment, locationReader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { return []pkg.Package{ { - Name: "some-package-" + locationReader.Path(), + Name: "some-package-" + locationReader.FullNestedPath(), }, }, nil, unknown.Newf(locationReader, "unable to read") }, "**/*") diff --git a/syft/pkg/cataloger/gentoo/license.go b/syft/pkg/cataloger/gentoo/license.go index 4fedf5938..cb294fce5 100644 --- a/syft/pkg/cataloger/gentoo/license.go +++ b/syft/pkg/cataloger/gentoo/license.go @@ -71,7 +71,7 @@ func extractLicenses(resolver file.Resolver, closestLocation *file.Location, rea if err := scanner.Err(); err != nil { fields := []any{"error", err} if closestLocation != nil { - fields = append(fields, "path", closestLocation.RealPath) + fields = append(fields, "path", closestLocation.FullNestedPath()) } log.WithFields(fields...).Debug("failed to fully read portage LICENSE") } @@ -115,9 +115,9 @@ func readLicenseGroups(resolver file.Resolver, closestLocation *file.Location) m } groupReader, err := resolver.FileContentsByLocation(*groupLocation) - defer internal.CloseAndLogError(groupReader, groupLocation.RealPath) + defer internal.CloseAndLogError(groupReader, groupLocation.FullNestedPath()) if err != nil { - log.WithFields("path", groupLocation.RealPath, "error", err).Debug("failed to fetch portage LICENSE") + log.WithFields("path", groupLocation.FullNestedPath(), "error", err).Debug("failed to fetch portage LICENSE") return nil } @@ -127,7 +127,7 @@ func readLicenseGroups(resolver file.Resolver, closestLocation *file.Location) m licenseGroups, err = parseLicenseGroups(groupReader) if err != nil { - log.WithFields("path", groupLocation.RealPath, "error", err).Debug("failed to parse portage LICENSE") + log.WithFields("path", groupLocation.FullNestedPath(), "error", err).Debug("failed to parse portage LICENSE") } return licenseGroups diff --git a/syft/pkg/cataloger/gentoo/parse_portage_contents.go b/syft/pkg/cataloger/gentoo/parse_portage_contents.go index db9d68035..f8476a4fb 100644 --- a/syft/pkg/cataloger/gentoo/parse_portage_contents.go +++ b/syft/pkg/cataloger/gentoo/parse_portage_contents.go @@ -24,14 +24,14 @@ var ( // parses individual CONTENTS files from the portage flat-file store (e.g. /var/db/pkg/*/*/CONTENTS). func parsePortageContents(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - cpvMatch := cpvRe.FindStringSubmatch(reader.RealPath) + cpvMatch := cpvRe.FindStringSubmatch(reader.LocalPath()) if cpvMatch == nil { - return nil, nil, fmt.Errorf("failed to match package and version in %s", reader.RealPath) + return nil, nil, fmt.Errorf("failed to match package and version in %s", reader.FullNestedPath()) } name, version := cpvMatch[1], cpvMatch[2] if name == "" || version == "" { - log.WithFields("path", reader.RealPath).Debug("failed to parse portage name and version") + log.WithFields("path", reader.FullNestedPath()).Debug("failed to parse portage name and version") return nil, nil, fmt.Errorf("failed to parse portage name and version") } @@ -64,10 +64,10 @@ func parsePortageContents(ctx context.Context, resolver file.Resolver, _ *generi func addFiles(resolver file.Resolver, dbLocation file.Location, entry *pkg.PortageEntry) { contentsReader, err := resolver.FileContentsByLocation(dbLocation) if err != nil { - log.WithFields("path", dbLocation.RealPath, "error", err).Debug("failed to fetch portage contents") + log.WithFields("path", dbLocation.FullNestedPath(), "error", err).Debug("failed to fetch portage contents") return } - defer internal.CloseAndLogError(contentsReader, dbLocation.RealPath) + defer internal.CloseAndLogError(contentsReader, dbLocation.FullNestedPath()) scanner := internal.NewLineScanner(contentsReader) for scanner.Scan() { @@ -87,12 +87,12 @@ func addFiles(resolver file.Resolver, dbLocation file.Location, entry *pkg.Porta } } if err := scanner.Err(); err != nil { - log.WithFields("path", dbLocation.RealPath, "error", err).Debug("failed to fully read portage contents") + log.WithFields("path", dbLocation.FullNestedPath(), "error", err).Debug("failed to fully read portage contents") } } func addLicenses(ctx context.Context, resolver file.Resolver, dbLocation file.Location, entry *pkg.PortageEntry) (pkg.LicenseSet, []file.Location) { - parentPath := filepath.Dir(dbLocation.RealPath) + parentPath := filepath.Dir(dbLocation.LocalPath()) location := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "LICENSE")) @@ -102,10 +102,10 @@ func addLicenses(ctx context.Context, resolver file.Resolver, dbLocation file.Lo licenseReader, err := resolver.FileContentsByLocation(*location) if err != nil { - log.WithFields("path", dbLocation.RealPath, "error", err).Debug("failed to fetch portage LICENSE") + log.WithFields("path", dbLocation.FullNestedPath(), "error", err).Debug("failed to fetch portage LICENSE") return pkg.NewLicenseSet(), nil } - defer internal.CloseAndLogError(licenseReader, location.RealPath) + defer internal.CloseAndLogError(licenseReader, location.FullNestedPath()) og, spdxExpression := extractLicenses(resolver, location, licenseReader) entry.Licenses = og @@ -115,7 +115,7 @@ func addLicenses(ctx context.Context, resolver file.Resolver, dbLocation file.Lo } func addSize(resolver file.Resolver, dbLocation file.Location, entry *pkg.PortageEntry) []file.Location { - parentPath := filepath.Dir(dbLocation.RealPath) + parentPath := filepath.Dir(dbLocation.LocalPath()) location := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "SIZE")) @@ -125,10 +125,10 @@ func addSize(resolver file.Resolver, dbLocation file.Location, entry *pkg.Portag sizeReader, err := resolver.FileContentsByLocation(*location) if err != nil { - log.WithFields("path", dbLocation.RealPath, "error", err).Debug("failed to fetch portage SIZE") + log.WithFields("path", dbLocation.FullNestedPath(), "error", err).Debug("failed to fetch portage SIZE") return nil } - defer internal.CloseAndLogError(sizeReader, location.RealPath) + defer internal.CloseAndLogError(sizeReader, location.FullNestedPath()) scanner := internal.NewLineScanner(sizeReader) for scanner.Scan() { @@ -139,7 +139,7 @@ func addSize(resolver file.Resolver, dbLocation file.Location, entry *pkg.Portag } } if err := scanner.Err(); err != nil { - log.WithFields("path", location.RealPath, "error", err).Debug("failed to fully read portage SIZE") + log.WithFields("path", location.FullNestedPath(), "error", err).Debug("failed to fully read portage SIZE") } return []file.Location{location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.SupportingEvidenceAnnotation)} diff --git a/syft/pkg/cataloger/githubactions/package.go b/syft/pkg/cataloger/githubactions/package.go index 49c5c83e4..f7f36863b 100644 --- a/syft/pkg/cataloger/githubactions/package.go +++ b/syft/pkg/cataloger/githubactions/package.go @@ -15,7 +15,7 @@ func newPackageFromUsageStatement(use, comment string, location file.Location) ( name, version := parseStepUsageStatement(use, comment) if name == "" { - log.WithFields("file", location.RealPath, "statement", use).Trace("unable to parse github action usage statement") + log.WithFields("file", location.FullNestedPath(), "statement", use).Trace("unable to parse github action usage statement") return nil, fmt.Errorf("unable to parse github action usage statement") } diff --git a/syft/pkg/cataloger/golang/licenses.go b/syft/pkg/cataloger/golang/licenses.go index e3c40af5d..2fdcb8889 100644 --- a/syft/pkg/cataloger/golang/licenses.go +++ b/syft/pkg/cataloger/golang/licenses.go @@ -239,13 +239,13 @@ func (c *goLicenseResolver) findLicensesInSource(ctx context.Context, resolver f func (c *goLicenseResolver) parseLicenseFromLocation(ctx context.Context, l file.Location, resolver file.Resolver) ([]pkg.License, error) { var out []pkg.License - fileName := path.Base(l.RealPath) + fileName := path.Base(l.FullNestedPath()) if licenses.IsLicenseFile(fileName) { contents, err := resolver.FileContentsByLocation(l) if err != nil { return nil, err } - defer internal.CloseAndLogError(contents, l.RealPath) + defer internal.CloseAndLogError(contents, l.FullNestedPath()) out = pkg.NewLicensesFromReadCloserWithContext(ctx, file.NewLocationReadCloser(l, contents)) } return out, nil diff --git a/syft/pkg/cataloger/golang/parse_go_binary.go b/syft/pkg/cataloger/golang/parse_go_binary.go index e33874800..002e44d43 100644 --- a/syft/pkg/cataloger/golang/parse_go_binary.go +++ b/syft/pkg/cataloger/golang/parse_go_binary.go @@ -119,7 +119,7 @@ func (c *goBinaryCataloger) parseGoBinary(ctx context.Context, resolver file.Res if err != nil { return nil, nil, err } - defer internal.CloseAndLogError(reader.ReadCloser, reader.RealPath) + defer internal.CloseAndLogError(reader.ReadCloser, reader.FullNestedPath()) mods, errs := scanFile(ctx, reader.Location, unionReader, c.symbolSelector.enabled()) // scanFile hands back the reconstruction for each binary that turned out to be packed. The readers diff --git a/syft/pkg/cataloger/golang/parse_go_binary_test.go b/syft/pkg/cataloger/golang/parse_go_binary_test.go index 7e33b0155..1b5b58f23 100644 --- a/syft/pkg/cataloger/golang/parse_go_binary_test.go +++ b/syft/pkg/cataloger/golang/parse_go_binary_test.go @@ -163,10 +163,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -209,10 +206,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{}, @@ -254,10 +248,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -292,10 +283,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -366,10 +354,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft@v0.0.0-20221014195457-41bc6bb41035", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -433,10 +418,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft@v0.79.0", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -498,10 +480,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft@v0.79.0", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -555,10 +534,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft@v0.79.0", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -612,10 +588,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft@v0.79.0", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -670,10 +643,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft@v0.0.0-20221014195457-41bc6bb41035", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -742,10 +712,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -763,10 +730,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -820,10 +784,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -840,10 +801,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -895,10 +853,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -916,10 +871,7 @@ func TestBuildGoPkgInfo(t *testing.T) { Type: pkg.GoModulePkg, Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -971,10 +923,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft@v1.0.0-somethingelse%2Bincompatible", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -1030,10 +979,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/anchore/syft", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -1082,10 +1028,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/kuskoman/something-else@v1.2.3", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -1102,10 +1045,7 @@ func TestBuildGoPkgInfo(t *testing.T) { PURL: "pkg:golang/github.com/kuskoman/logstash-exporter@v1.7.0", Locations: file.NewLocationSet( file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ), Metadata: pkg.GolangBinaryBuildinfoEntry{ @@ -1143,10 +1083,7 @@ func TestBuildGoPkgInfo(t *testing.T) { p.SetID() } location := file.NewLocationFromCoordinates( - file.Coordinates{ - RealPath: "/a-path", - FileSystemID: "layer-id", - }, + file.NewCoordinates("/a-path", "layer-id"), ) if test.cfg == nil { @@ -1433,7 +1370,7 @@ func (alwaysErrorReader) Read(_ []byte) (int, error) { } func Test_buildGoPkgInfo_symbolScope(t *testing.T) { - location := file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/a-path", FileSystemID: "layer-id"}) + location := file.NewLocationFromCoordinates(file.NewCoordinates("/a-path", "layer-id")) // the symbols a binary would carry once scanFile has extracted them: one main-package symbol, two // dependency symbols from distinct packages within the same module (so the per-package keying of the @@ -1615,7 +1552,7 @@ var ( // symbols more than once. This happens in production for universal/fat Mach-O binaries: scanFile yields // one build info per architecture and each is recorded under the same location coordinates. func Test_recordStdlibSymbols_merge(t *testing.T) { - coord := file.Coordinates{RealPath: "/a-path", FileSystemID: "layer-id"} + coord := file.NewCoordinates("/a-path", "layer-id") c := newGoBinaryCataloger(CatalogerConfig{}) c.recordStdlibSymbols(coord, map[string][]string{ @@ -1638,7 +1575,7 @@ func Test_recordStdlibSymbols_merge(t *testing.T) { // Test_stdlibSymbolsFor_isolation asserts the returned map is a deep copy: mutating it (or its slices) // must not corrupt the cataloger's guarded internal state. func Test_stdlibSymbolsFor_isolation(t *testing.T) { - coord := file.Coordinates{RealPath: "/a-path", FileSystemID: "layer-id"} + coord := file.NewCoordinates("/a-path", "layer-id") c := newGoBinaryCataloger(CatalogerConfig{}) c.recordStdlibSymbols(coord, map[string][]string{"net/http": {"Get"}}) diff --git a/syft/pkg/cataloger/golang/parse_go_mod.go b/syft/pkg/cataloger/golang/parse_go_mod.go index 7748759aa..8c2e2ecd8 100644 --- a/syft/pkg/cataloger/golang/parse_go_mod.go +++ b/syft/pkg/cataloger/golang/parse_go_mod.go @@ -39,12 +39,13 @@ func newGoModCataloger(opts CatalogerConfig) *goModCataloger { // parseGoModFile takes a go.mod and tries to resolve and lists all packages discovered. func (c *goModCataloger) parseGoModFile(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) (pkgs []pkg.Package, relationships []artifact.Relationship, err error) { + // the go toolchain needs the go.mod on the host: a location without a reference (such as a file within + // an archive) has none, so only the go.mod and go.sum contents are cataloged realPath := string(reader.Location.Reference().RealPath) if windows.HostRunningOnWindows() { // directory resolvers hand back volume-encoded posix paths (/c/...), the go toolchain needs the native path realPath = windows.FromPosix(realPath) } - modDir := filepath.Dir(realPath) digests, err := parseGoSumFile(resolver, reader) if err != nil { log.Debugf("unable to get go.sum: %v", err) @@ -65,12 +66,12 @@ func (c *goModCataloger) parseGoModFile(ctx context.Context, resolver file.Resol var sourceModules map[string]*packages.Module var catalogedModules []pkg.Package - if c.usePackagesLib { + if c.usePackagesLib && realPath != "" { var sourcePackages map[string][]pkgInfo var sourceDependencies map[string][]string var sourceModuleToPkg map[string]artifact.Identifiable - sourcePackages, sourceModules, sourceDependencies, err = c.loadPackages(modDir, reader.Location) + sourcePackages, sourceModules, sourceDependencies, err = c.loadPackages(filepath.Dir(realPath), reader.Location) catalogedModules, sourceModuleToPkg = c.catalogModules(ctx, scanRoot, sourcePackages, sourceModules, reader, digests) relationships = buildModuleRelationships(catalogedModules, sourceDependencies, sourceModuleToPkg) } @@ -334,7 +335,7 @@ func (c *goModCataloger) parseModFileContents(reader file.LocationReadCloser) (* return nil, fmt.Errorf("failed to read go module: %w", err) } - f, err := modfile.Parse(reader.RealPath, contents, nil) + f, err := modfile.Parse(reader.FullNestedPath(), contents, nil) if err != nil { return nil, fmt.Errorf("failed to parse go module: %w", err) } @@ -427,7 +428,7 @@ func parseGoSumFile(resolver file.Resolver, reader file.LocationReadCloser) (map return out, fmt.Errorf("no resolver provided") } - goSumPath := strings.TrimSuffix(reader.RealPath, ".mod") + ".sum" + goSumPath := strings.TrimSuffix(reader.LocalPath(), ".mod") + ".sum" goSumLocation := resolver.RelativeFileByPath(reader.Location, goSumPath) if goSumLocation == nil { return nil, fmt.Errorf("unable to resolve: %s", goSumPath) @@ -436,7 +437,7 @@ func parseGoSumFile(resolver file.Resolver, reader file.LocationReadCloser) (map if err != nil { return nil, err } - defer internal.CloseAndLogError(contents, goSumLocation.AccessPath) + defer internal.CloseAndLogError(contents, goSumLocation.FullNestedPath()) // go.sum has the format like: // github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= diff --git a/syft/pkg/cataloger/golang/scan_binary.go b/syft/pkg/cataloger/golang/scan_binary.go index d017223ba..abb1e1e72 100644 --- a/syft/pkg/cataloger/golang/scan_binary.go +++ b/syft/pkg/cataloger/golang/scan_binary.go @@ -96,7 +96,7 @@ func scanReader(ctx context.Context, location file.Location, r io.ReaderAt, capt if isCancelled(err) { return nil, err } - log.WithFields("file", location.RealPath, "error", err).Trace("unable to fully read golang buildinfo") + log.WithFields("file", location.FullNestedPath(), "error", err).Trace("unable to fully read golang buildinfo") if reportableGap(err) { // the build info is either missing or it is not, and the same err covers both: a partial // reconstruction still hands back everything .go.buildinfo carried, and what it lost is the @@ -128,7 +128,7 @@ func scanReader(ctx context.Context, location file.Location, r io.ReaderAt, capt // bounds only the section elf.NewFile expands as it parses. v, err := getCryptoInformation(contents) if err != nil { - log.WithFields("file", location.RealPath, "error", err).Trace("unable to read golang version info") + log.WithFields("file", location.FullNestedPath(), "error", err).Trace("unable to read golang version info") // don't skip this build info. // we can still catalog packages, even if we can't get the crypto information errs = unknown.Appendf(errs, location, "unable to read golang version info: %w", err) @@ -138,7 +138,7 @@ func scanReader(ctx context.Context, location file.Location, r io.ReaderAt, capt if arch == "" { arch, err = getGOARCHFromBin(contents) if err != nil { - log.WithFields("file", location.RealPath, "error", err).Trace("unable to read golang arch info") + log.WithFields("file", location.FullNestedPath(), "error", err).Trace("unable to read golang arch info") // don't skip this build info. // we can still catalog packages, even if we can't get the arch information errs = unknown.Appendf(errs, location, "unable to read golang arch info: %w", err) @@ -149,7 +149,7 @@ func scanReader(ctx context.Context, location file.Location, r io.ReaderAt, capt if captureSymbols { symbols, err = getSymbols(contents) if err != nil { - log.WithFields("file", location.RealPath, "error", err).Trace("unable to read golang symbol info") + log.WithFields("file", location.FullNestedPath(), "error", err).Trace("unable to read golang symbol info") // don't skip this build info. // we can still catalog packages, even if we can't get the symbol information errs = unknown.Appendf(errs, location, "unable to read golang symbol info: %w", err) diff --git a/syft/pkg/cataloger/haskell/parse_stack_lock.go b/syft/pkg/cataloger/haskell/parse_stack_lock.go index 1f36ea424..66c6c0201 100644 --- a/syft/pkg/cataloger/haskell/parse_stack_lock.go +++ b/syft/pkg/cataloger/haskell/parse_stack_lock.go @@ -44,7 +44,7 @@ func parseStackLock(_ context.Context, _ file.Resolver, _ *generic.Environment, var lockFile stackLock if err := yaml.NewDecoder(reader).Decode(&lockFile); err != nil { - log.WithFields("error", err, "path", reader.RealPath).Trace("failed to parse stack.yaml.lock") + log.WithFields("error", err, "path", reader.FullNestedPath()).Trace("failed to parse stack.yaml.lock") return nil, nil, fmt.Errorf("failed to parse stack.yaml.lock file: %w", err) } diff --git a/syft/pkg/cataloger/haskell/parse_stack_yaml.go b/syft/pkg/cataloger/haskell/parse_stack_yaml.go index cd0fc4167..109625ce7 100644 --- a/syft/pkg/cataloger/haskell/parse_stack_yaml.go +++ b/syft/pkg/cataloger/haskell/parse_stack_yaml.go @@ -25,7 +25,7 @@ func parseStackYaml(_ context.Context, _ file.Resolver, _ *generic.Environment, var stackFile stackYaml if err := yaml.NewDecoder(reader).Decode(&stackFile); err != nil { - log.WithFields("error", err, "path", reader.RealPath).Trace("failed to parse stack.yaml") + log.WithFields("error", err, "path", reader.FullNestedPath()).Trace("failed to parse stack.yaml") return nil, nil, fmt.Errorf("failed to parse stack.yaml file: %w", err) } diff --git a/syft/pkg/cataloger/homebrew/package.go b/syft/pkg/cataloger/homebrew/package.go index 82744d471..df78ebd08 100644 --- a/syft/pkg/cataloger/homebrew/package.go +++ b/syft/pkg/cataloger/homebrew/package.go @@ -16,7 +16,7 @@ func newHomebrewPackage(ctx context.Context, resolver file.Resolver, pd parsedHo lics = append(lics, pkg.NewLicensesFromValues(pd.License)...) } else { // sometimes licenses are included in the parent directory - lics = licenses.FindInDirs(ctx, resolver, path.Dir(formulaLocation.Path())) + lics = licenses.FindInDirs(ctx, resolver, path.Dir(formulaLocation.LocalPath())) } p := pkg.Package{ diff --git a/syft/pkg/cataloger/homebrew/parse_homebrew_formula.go b/syft/pkg/cataloger/homebrew/parse_homebrew_formula.go index 93d866746..8a928cc0e 100644 --- a/syft/pkg/cataloger/homebrew/parse_homebrew_formula.go +++ b/syft/pkg/cataloger/homebrew/parse_homebrew_formula.go @@ -25,7 +25,7 @@ type parsedHomebrewData struct { func parseHomebrewFormula(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { pd, err := parseFormulaFile(reader) if err != nil { - log.WithFields("path", reader.RealPath).Trace("failed to parse formula") + log.WithFields("path", reader.FullNestedPath()).Trace("failed to parse formula") return nil, nil, err } @@ -69,7 +69,7 @@ func parseFormulaFile(reader file.LocationReadCloser) (*parsedHomebrewData, erro } } - pd.Tap = getTapFromPath(reader.RealPath) + pd.Tap = getTapFromPath(reader.LocalPath()) if err := scanner.Err(); err != nil { return nil, err @@ -79,7 +79,7 @@ func parseFormulaFile(reader file.LocationReadCloser) (*parsedHomebrewData, erro return &pd, nil } - pd.Name, pd.Version = getNameAndVersionFromPath(reader.RealPath) + pd.Name, pd.Version = getNameAndVersionFromPath(reader.LocalPath()) return &pd, nil } diff --git a/syft/pkg/cataloger/internal/binutils/branching_matcher.go b/syft/pkg/cataloger/internal/binutils/branching_matcher.go index c18e433e1..d75298b0c 100644 --- a/syft/pkg/cataloger/internal/binutils/branching_matcher.go +++ b/syft/pkg/cataloger/internal/binutils/branching_matcher.go @@ -18,7 +18,7 @@ func BranchingEvidenceMatcher(classifiers ...Classifier) EvidenceMatcher { if err != nil { return nil, err } - defer internal.CloseAndLogError(rdr, context.Location.RealPath) + defer internal.CloseAndLogError(rdr, context.Location.FullNestedPath()) if err != nil { return nil, err } diff --git a/syft/pkg/cataloger/internal/binutils/classifier.go b/syft/pkg/cataloger/internal/binutils/classifier.go index 05b62897e..4dc5f61dd 100644 --- a/syft/pkg/cataloger/internal/binutils/classifier.go +++ b/syft/pkg/cataloger/internal/binutils/classifier.go @@ -162,11 +162,11 @@ func (c ContextualEvidenceMatchers) FileContentsVersionMatcher(patterns ...strin func FileNameTemplateVersionMatcher(fileNamePattern, contentTemplate, catalogerName string) EvidenceMatcher { pat := regexp.MustCompile(fileNamePattern) return func(classifier Classifier, context MatcherContext) ([]pkg.Package, error) { - if !pat.MatchString(context.Location.RealPath) { + if !pat.MatchString(context.Location.LocalPath()) { return nil, nil } - filepathNamedGroupValues := internal.MatchNamedCaptureGroups(pat, context.Location.RealPath) + filepathNamedGroupValues := internal.MatchNamedCaptureGroups(pat, context.Location.LocalPath()) // versions like 3.5 should not match any character, but explicit dot for k, v := range filepathNamedGroupValues { @@ -328,7 +328,7 @@ func MatchPath(path string) EvidenceMatcher { panic("invalid pattern") } return func(_ Classifier, context MatcherContext) ([]pkg.Package, error) { - if doublestar.MatchUnvalidated(path, context.Location.RealPath) { + if doublestar.MatchUnvalidated(path, context.Location.LocalPath()) { return []pkg.Package{}, nil // return non-nil } return nil, nil @@ -339,7 +339,7 @@ func MatchPath(path string) EvidenceMatcher { // relative to a primary file, for example: a VERSION file in the same or a parent directory to another binary func SupportingEvidenceMatcher(relativePathGlob string, evidenceMatcher EvidenceMatcher) EvidenceMatcher { return func(classifier Classifier, context MatcherContext) ([]pkg.Package, error) { - f := path.Dir(context.Location.RealPath) + f := path.Dir(context.Location.LocalPath()) f = path.Join(f, relativePathGlob) f = path.Clean(f) // this would ideally be RelativeFileByPath but with a glob search: @@ -376,7 +376,7 @@ func sharedLibraries(context MatcherContext) ([]string, error) { if err != nil { return nil, err } - defer internal.CloseAndLogError(contents, context.Location.RealPath) + defer internal.CloseAndLogError(contents, context.Location.FullNestedPath()) e, err := elfutil.NewFile(contents) if err != nil { @@ -386,13 +386,13 @@ func sharedLibraries(context MatcherContext) ([]string, error) { // dropped, and nothing downstream would report it. var fmtErr *elf.FormatError if !errors.As(err, &fmtErr) && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { - log.WithFields("file", context.Location.RealPath, "error", err).Debug("unable to parse ELF binary") + log.WithFields("file", context.Location.FullNestedPath(), "error", err).Debug("unable to parse ELF binary") } } if e != nil { symbols, err := e.ImportedLibraries() if err != nil { - log.Debugf("unable to read elf binary at: %s -- %s", context.Location.RealPath, err) + log.Debugf("unable to read elf binary at: %s -- %s", context.Location.FullNestedPath(), err) } return symbols, nil } @@ -404,7 +404,7 @@ func sharedLibraries(context MatcherContext) ([]string, error) { if m != nil { symbols, err := m.ImportedLibraries() if err != nil { - log.Debugf("unable to read macho binary at: %s -- %s", context.Location.RealPath, err) + log.Debugf("unable to read macho binary at: %s -- %s", context.Location.FullNestedPath(), err) } return symbols, nil } @@ -416,7 +416,7 @@ func sharedLibraries(context MatcherContext) ([]string, error) { if p != nil { symbols, err := p.ImportedLibraries() if err != nil { - log.Debugf("unable to read pe binary at: %s -- %s", context.Location.RealPath, err) + log.Debugf("unable to read pe binary at: %s -- %s", context.Location.FullNestedPath(), err) } return symbols, nil } @@ -443,7 +443,7 @@ func collectSupportingEvidence(classifier Classifier, context MatcherContext, re if err != nil { return nil, err } - defer internal.CloseAndLogError(rdr, relativeFile.Path()) + defer internal.CloseAndLogError(rdr, relativeFile.FullNestedPath()) ur, err := unionreader.GetUnionReader(rdr) if err != nil { return nil, err diff --git a/syft/pkg/cataloger/internal/dotnet/pe/pe.go b/syft/pkg/cataloger/internal/dotnet/pe/pe.go index 9939e153e..d527ba6db 100644 --- a/syft/pkg/cataloger/internal/dotnet/pe/pe.go +++ b/syft/pkg/cataloger/internal/dotnet/pe/pe.go @@ -330,7 +330,7 @@ func Read(f file.LocationReadCloser) (*File, error) { parseErr := errors.Join(parseErrs...) if parseErr != nil { - log.Tracef("partially parsed PE file %s: %v", f.RealPath, parseErr) + log.Tracef("partially parsed PE file %s: %v", f.FullNestedPath(), parseErr) } return &File{ diff --git a/syft/pkg/cataloger/internal/embeddedsbom/relationships.go b/syft/pkg/cataloger/internal/embeddedsbom/relationships.go new file mode 100644 index 000000000..1010f592e --- /dev/null +++ b/syft/pkg/cataloger/internal/embeddedsbom/relationships.go @@ -0,0 +1,36 @@ +/* +Package embeddedsbom holds what catalogers that read an SBOM found within the scan share. +*/ +package embeddedsbom + +import ( + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" +) + +// Relationships keeps the relationships decoded from an embedded SBOM that may enter the scan's own results. A +// file the SBOM nests within an archive, or makes the scan root, is nothing this scan found, and encoders would +// rebuild archives and roots from it. +func Relationships(relationships []artifact.Relationship) []artifact.Relationship { + var out []artifact.Relationship + for _, r := range relationships { + if plainFile(r.From) && plainFile(r.To) { + out = append(out, r) + } + } + return out +} + +// plainFile reports false for a file endpoint nested in an archive or at the scan root, and true otherwise. +func plainFile(v artifact.Identifiable) bool { + var c file.Coordinates + switch v := v.(type) { + case file.Location: + c = v.Coordinates + case file.Coordinates: + c = v + default: + return true + } + return c.FullPath == "" && c.RealPath != "" && c.RealPath != "/" +} diff --git a/syft/pkg/cataloger/internal/embeddedsbom/relationships_test.go b/syft/pkg/cataloger/internal/embeddedsbom/relationships_test.go new file mode 100644 index 000000000..0638c7a40 --- /dev/null +++ b/syft/pkg/cataloger/internal/embeddedsbom/relationships_test.go @@ -0,0 +1,33 @@ +package embeddedsbom + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" +) + +func TestRelationships(t *testing.T) { + p := pkg.Package{Name: "postgresql"} + flat := file.NewLocation("/opt/app.jar") + nested := file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/app.jar", FullPath: "/app.jar!/lib/a.jar"}) + root := file.NewLocation("") + + keep := []artifact.Relationship{ + {From: p, To: p, Type: artifact.DependencyOfRelationship}, + {From: p, To: flat, Type: artifact.ContainsRelationship}, + {From: p, To: flat.Coordinates, Type: artifact.ContainsRelationship}, + } + drop := []artifact.Relationship{ + {From: p, To: nested, Type: artifact.ContainsRelationship}, + {From: p, To: nested.Coordinates, Type: artifact.ContainsRelationship}, + {From: p, To: root, Type: artifact.ContainsRelationship}, + {From: p, To: file.NewLocation("/"), Type: artifact.ContainsRelationship}, + {From: root, To: flat, Type: artifact.ContainsRelationship}, + } + + assert.Equal(t, keep, Relationships(append(append([]artifact.Relationship{}, keep...), drop...))) +} diff --git a/syft/pkg/cataloger/internal/licenses/find_licenses.go b/syft/pkg/cataloger/internal/licenses/find_licenses.go index 8a6e4865e..3dcb19064 100644 --- a/syft/pkg/cataloger/internal/licenses/find_licenses.go +++ b/syft/pkg/cataloger/internal/licenses/find_licenses.go @@ -71,7 +71,7 @@ func FindInDirs(ctx context.Context, resolver file.Resolver, dirs ...string) []p func FindRelativeToLocations(ctx context.Context, resolver file.Resolver, locations ...file.Location) []pkg.License { var out []pkg.License for _, location := range locations { - dir := path.Dir(location.AccessPath) + dir := path.Dir(location.LocalPath()) out = append(out, FindInDirs(ctx, resolver, dir)...) } return out @@ -115,9 +115,9 @@ func NewFromValues(ctx context.Context, locations []file.Location, values ...str func readFromResolver(ctx context.Context, resolver file.Resolver, location file.Location) []pkg.License { metadataContents, err := resolver.FileContentsByLocation(location) if err != nil || metadataContents == nil { - log.WithFields("error", err, "path", location.Path()).Trace("unable to license file contents") + log.WithFields("error", err, "path", location.FullNestedPath()).Trace("unable to license file contents") return nil } - defer internal.CloseAndLogError(metadataContents, location.Path()) + defer internal.CloseAndLogError(metadataContents, location.FullNestedPath()) return pkg.NewLicensesFromReadCloserWithContext(ctx, file.NewLocationReadCloser(location, metadataContents)) } diff --git a/syft/pkg/cataloger/internal/pkgtest/observing_resolver.go b/syft/pkg/cataloger/internal/pkgtest/observing_resolver.go index 8fca2ce58..61aaa0b37 100644 --- a/syft/pkg/cataloger/internal/pkgtest/observing_resolver.go +++ b/syft/pkg/cataloger/internal/pkgtest/observing_resolver.go @@ -54,7 +54,7 @@ func (r *ObservingResolver) ObservedPathQuery(input string) bool { // ObservedPathResponses checks if a specific path was returned in any response. func (r *ObservingResolver) ObservedPathResponses(path string) bool { for _, loc := range r.pathResponses { - if loc.RealPath == path { + if loc.FullNestedPath() == path { return true } } @@ -64,7 +64,7 @@ func (r *ObservingResolver) ObservedPathResponses(path string) bool { // ObservedContentQueries checks if a specific file's content was read. func (r *ObservingResolver) ObservedContentQueries(path string) bool { for _, loc := range r.contentQueries { - if loc.RealPath == path { + if loc.FullNestedPath() == path { return true } } @@ -75,7 +75,7 @@ func (r *ObservingResolver) ObservedContentQueries(path string) bool { func (r *ObservingResolver) AllContentQueries() []string { observed := strset.New() for _, loc := range r.contentQueries { - observed.Add(loc.RealPath) + observed.Add(loc.FullNestedPath()) } return observed.List() } diff --git a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go index 22042fcf1..6cc30c14a 100644 --- a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go +++ b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go @@ -495,7 +495,7 @@ func stringPackage(p pkg.Package) string { locs := p.Locations.ToSlice() var loc string if len(locs) > 0 { - loc = p.Locations.ToSlice()[0].RealPath + loc = p.Locations.ToSlice()[0].FullNestedPath() } return fmt.Sprintf("%s @ %s (%s)", p.Name, p.Version, loc) diff --git a/syft/pkg/cataloger/java/archive_entries.go b/syft/pkg/cataloger/java/archive_entries.go new file mode 100644 index 000000000..ed4af5325 --- /dev/null +++ b/syft/pkg/cataloger/java/archive_entries.go @@ -0,0 +1,142 @@ +package java + +import ( + "context" + "fmt" + "strings" + + "github.com/bmatcuk/doublestar/v4" + "github.com/scylladb/go-set/strset" + + "github.com/anchore/syft/internal" + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/file" +) + +// archiveEntries is where the archive parser reads a java archive's entries from, so the same parser +// runs over an archive file and over an archive already extracted to a filesystem. Entry names are +// archive-relative with no leading slash, as zip entry names are. +type archiveEntries interface { + // glob returns the entries matching any pattern. Patterns are written against a leading slash + // ("/META-INF/*"); the names returned have none. + glob(patterns ...string) []string + + // globFold is glob ignoring case, for names java tooling does not agree on the case of + // ("META-INF/LICENSE" vs "meta-inf/license", "X.JAR"). + globFold(patterns ...string) []string + + // contents reads the named entries, keyed by the same names. + contents(ctx context.Context, names ...string) (map[string]string, error) +} + +// zipEntries reads entries out of a java archive file on disk. +type zipEntries struct { + archivePath string + fileManifest intFile.ZipFileManifest +} + +func newZipEntries(ctx context.Context, archivePath string) (*zipEntries, error) { + fileManifest, err := intFile.NewZipFileManifest(ctx, archivePath) + if err != nil { + return nil, fmt.Errorf("unable to read files from java archive: %w", err) + } + return &zipEntries{archivePath: archivePath, fileManifest: fileManifest}, nil +} + +func (z *zipEntries) glob(patterns ...string) []string { + return z.fileManifest.GlobMatch(false, patterns...) +} + +func (z *zipEntries) globFold(patterns ...string) []string { + return z.fileManifest.GlobMatch(true, patterns...) +} + +func (z *zipEntries) contents(ctx context.Context, names ...string) (map[string]string, error) { + return intFile.ContentsFromZip(ctx, z.archivePath, names...) +} + +// resolverEntries reads entries out of an archive the archive cataloger task has extracted. +type resolverEntries struct { + resolver file.Resolver +} + +func (r *resolverEntries) glob(patterns ...string) []string { + seen := strset.New() + var names []string + for _, pattern := range patterns { + locations, err := r.resolver.FilesByGlob(pattern) + if err != nil { + log.WithFields("pattern", pattern, "error", err).Trace("unable to glob archive contents") + continue + } + for _, location := range locations { + name := strings.TrimPrefix(location.LocalPath(), "/") + if seen.Has(name) { + continue + } + seen.Add(name) + names = append(names, name) + } + } + return names +} + +// globFold walks every entry, since a resolver's globs are case-sensitive. The callers ask for a handful +// of names once per archive, so the walk is cheap next to cataloging the archive. +func (r *resolverEntries) globFold(patterns ...string) []string { + var folded []string + for _, pattern := range patterns { + folded = append(folded, strings.ToLower(pattern)) + } + seen := strset.New() + var names []string + for location := range r.resolver.AllLocations(context.Background()) { + name := strings.TrimPrefix(location.LocalPath(), "/") + if seen.Has(name) { + continue + } + for _, pattern := range folded { + if ok, _ := doublestar.Match(pattern, "/"+strings.ToLower(name)); ok { + seen.Add(name) + names = append(names, name) + break + } + } + } + return names +} + +func (r *resolverEntries) contents(_ context.Context, names ...string) (map[string]string, error) { + out := make(map[string]string, len(names)) + for _, name := range names { + locations, err := r.resolver.FilesByPath(name) + if err != nil { + return nil, fmt.Errorf("unable to look up archive entry %q: %w", name, err) + } + if len(locations) == 0 { + return nil, fmt.Errorf("entry %q is not in this archive", name) + } + contents, err := r.read(locations[0]) + if err != nil { + return nil, fmt.Errorf("unable to read archive entry %q: %w", name, err) + } + out[name] = contents + } + return out, nil +} + +// read returns one entry's content, bounded against decompression bombs as zipEntries bounds its own. +func (r *resolverEntries) read(location file.Location) (string, error) { + reader, err := r.resolver.FileContentsByLocation(location) + if err != nil { + return "", err + } + defer internal.CloseAndLogError(reader, location.FullNestedPath()) + + var contents strings.Builder + if err := intFile.SafeCopy(&contents, reader); err != nil { + return "", err + } + return contents.String(), nil +} diff --git a/syft/pkg/cataloger/java/archive_entries_test.go b/syft/pkg/cataloger/java/archive_entries_test.go new file mode 100644 index 000000000..908ced76b --- /dev/null +++ b/syft/pkg/cataloger/java/archive_entries_test.go @@ -0,0 +1,49 @@ +package java + +import ( + "archive/zip" + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/file" +) + +// java tooling doesn't agree on case ("meta-inf/LICENSE", "LIB.JAR"), and the license, version and +// nested-archive lookups have always matched regardless of it +func Test_archiveEntries_globFoldIgnoresCase(t *testing.T) { + names := []string{"meta-inf/LICENSE.txt", "Version.Properties", "lib/DEP.JAR", "META-INF/MANIFEST.MF"} + + archivePath := filepath.Join(t.TempDir(), "app.jar") + f, err := os.Create(archivePath) + require.NoError(t, err) + zw := zip.NewWriter(f) + for _, name := range names { + _, err := zw.Create(name) + require.NoError(t, err) + } + require.NoError(t, zw.Close()) + require.NoError(t, f.Close()) + + fromZip, err := newZipEntries(context.Background(), archivePath) + require.NoError(t, err) + + var rooted []string + for _, name := range names { + rooted = append(rooted, "/"+name) + } + fromResolver := &resolverEntries{resolver: file.NewMockResolverForPaths(rooted...)} + + for name, entries := range map[string]archiveEntries{"zip": fromZip, "resolver": fromResolver} { + t.Run(name, func(t *testing.T) { + assert.ElementsMatch(t, []string{"meta-inf/LICENSE.txt", "META-INF/MANIFEST.MF"}, entries.globFold("/META-INF/*")) + assert.Equal(t, []string{"Version.Properties"}, entries.globFold("/version.properties")) + assert.Equal(t, []string{"lib/DEP.JAR"}, entries.globFold("**/*.jar", "**/*.war")) + assert.Empty(t, entries.glob("/version.properties"), "glob itself stays case-sensitive") + }) + } +} diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index d5608b89a..6827bda66 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -17,6 +17,7 @@ import ( "golang.org/x/exp/maps" "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/archive" intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/tmpdir" @@ -57,12 +58,20 @@ var javaArchiveHashes = []crypto.Hash{ crypto.SHA1, } +// archiveParser reads one java archive, whether this cataloger opened the archive file or the archive +// cataloger task already extracted it. type archiveParser struct { - fileManifest intFile.ZipFileManifest - location file.Location - archivePath string + entries archiveEntries + // location is the archive's location in its parent filesystem; archive.VirtualPath(location) is its + // identity with the chain of containing archives prepended: "app.war:WEB-INF/lib/dep.jar" + location file.Location + fileInfo archiveFilename + // archiveDigests are of the archive file as a whole. When the archive cataloger task extracted the + // archive they come from that extraction, since the archive is not inside itself. + archiveDigests []file.Digest + // contentPath is the directory nested archives are unzipped into. Set only when this cataloger + // opened the archive itself, since only then does it recurse. contentPath string - fileInfo archiveFilename detectNested bool cfg ArchiveCatalogerConfig maven *maven.Resolver @@ -83,7 +92,7 @@ func (gap genericArchiveParserAdapter) parseJavaArchive(ctx context.Context, _ f // processJavaArchive processes an archive for java contents, returning all Java libraries and nested archives func (gap genericArchiveParserAdapter) processJavaArchive(ctx context.Context, reader file.LocationReadCloser, parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) { - parser, cleanupFn, err := newJavaArchiveParser(ctx, reader, true, gap.cfg) + parser, cleanupFn, err := newJavaArchiveParser(ctx, reader, !archive.NestedCatalogingEnabled(ctx), gap.cfg) // note: even on error, we should always run cleanup functions defer cleanupFn() if err != nil { @@ -100,11 +109,12 @@ func uniquePkgKey(groupID string, p *pkg.Package) string { return fmt.Sprintf("%s|%s|%s", groupID, p.Name, p.Version) } -// newJavaArchiveParser returns a new java archive parser object for the given archive. Can be configured to discover -// and parse nested archives or ignore them. +// newJavaArchiveParser returns a parser for an archive file this cataloger opens itself, copied to a +// temp dir. Can be configured to discover and parse nested archives or ignore them. The returned +// cleanup must run even when an error is returned. func newJavaArchiveParser(ctx context.Context, reader file.LocationReadCloser, detectNested bool, cfg ArchiveCatalogerConfig) (*archiveParser, func(), error) { // fetch the last element of the virtual path - virtualElements := strings.Split(reader.Path(), ":") + virtualElements := strings.Split(archive.VirtualPath(reader.Location), ":") currentFilepath := virtualElements[len(virtualElements)-1] td := tmpdir.FromContext(ctx) @@ -116,23 +126,51 @@ func newJavaArchiveParser(ctx context.Context, reader file.LocationReadCloser, d return nil, cleanupFn, fmt.Errorf("unable to process java archive: %w", err) } - fileManifest, err := intFile.NewZipFileManifest(ctx, archivePath) + entries, err := newZipEntries(ctx, archivePath) if err != nil { - return nil, cleanupFn, fmt.Errorf("unable to read files from java archive: %w", err) + return nil, cleanupFn, err + } + + digests, err := getDigestsFromArchive(ctx, archivePath) + if err != nil { + return nil, cleanupFn, err } return &archiveParser{ - fileManifest: fileManifest, - location: reader.Location, - archivePath: archivePath, - contentPath: contentPath, - fileInfo: newJavaArchiveFilename(currentFilepath), - detectNested: detectNested, - cfg: cfg, - maven: maven.NewResolver(nil, cfg.mavenConfig()), + entries: entries, + location: reader.Location, + fileInfo: newJavaArchiveFilename(currentFilepath), + archiveDigests: digests, + contentPath: contentPath, + detectNested: detectNested, + cfg: cfg, + maven: maven.NewResolver(nil, cfg.mavenConfig()), }, cleanupFn, nil } +// newExtractedArchiveParser returns a parser for the archive whose extracted contents the resolver +// holds. Its identity comes from the traversal, since none of it is inside the archive, and it never +// recurses: archives nested inside it are the archive cataloger task's to find. +func newExtractedArchiveParser(trav *archive.Traversal, resolver file.Resolver, cfg ArchiveCatalogerConfig) (*archiveParser, error) { + if resolver == nil { + return nil, fmt.Errorf("no resolver for the contents of %q", trav.Location.FullNestedPath()) + } + return &archiveParser{ + entries: &resolverEntries{resolver: resolver}, + location: trav.Location, + // the name it was reached by, as for isJavaArchiveName + fileInfo: newJavaArchiveFilename(path.Base(trav.Location.AccessPath)), + archiveDigests: trav.Digests, + cfg: cfg, + maven: maven.NewResolver(nil, cfg.mavenConfig()), + }, nil +} + +// virtualPath is the archive's path with the chain of containing archives prepended. +func (j *archiveParser) virtualPath() string { + return archive.VirtualPath(j.location) +} + // parse the loaded archive and return all packages found. func (j *archiveParser) parse(ctx context.Context, parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) { var pkgs []pkg.Package @@ -141,7 +179,7 @@ func (j *archiveParser) parse(ctx context.Context, parentPkg *pkg.Package) ([]pk // find the parent package from the java manifest mainPkg, err := j.discoverMainPackage(ctx) if err != nil { - return nil, nil, fmt.Errorf("could not generate package from %s: %w", j.location, err) + return nil, nil, fmt.Errorf("could not generate package from %s: %w", j.location.FullNestedPath(), err) } // find aux packages from pom.properties/pom.xml and potentially modify the existing parentPkg @@ -189,9 +227,10 @@ func (j *archiveParser) parse(ctx context.Context, parentPkg *pkg.Package) ([]pk } pkgs = append(pkgs, nestedPkgs...) relationships = append(relationships, nestedRelationships...) - } else { + } else if !archive.NestedCatalogingEnabled(ctx) { + // nothing will catalog these, so record them as unknowns // .jar and .war files are present in archives, are others? or generally just consider them top-level? - nestedArchives := j.fileManifest.GlobMatch(true, "**/*.jar", "**/*.war") + nestedArchives := j.entries.globFold("**/*.jar", "**/*.war") if len(nestedArchives) > 0 { slices.Sort(nestedArchives) errs = unknown.Appendf(errs, j.location, "nested archives not cataloged: %v", strings.Join(nestedArchives, ", ")) @@ -224,7 +263,7 @@ func finalizePackage(p *pkg.Package) { // discoverMainPackage parses the root Java manifest used as the parent package to all discovered nested packages. func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package, error) { // search and parse java manifest files - manifestMatches := j.fileManifest.GlobMatch(false, manifestGlob) + manifestMatches := j.entries.glob(manifestGlob) if len(manifestMatches) > 1 { return nil, fmt.Errorf("found multiple manifests in the jar: %+v", manifestMatches) } else if len(manifestMatches) == 0 { @@ -233,32 +272,26 @@ func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package, } // fetch the manifest file - contents, err := intFile.ContentsFromZip(ctx, j.archivePath, manifestMatches...) + contents, err := j.entries.contents(ctx, manifestMatches...) if err != nil { - return nil, fmt.Errorf("unable to extract java manifests (%s): %w", j.location, err) + return nil, fmt.Errorf("unable to extract java manifests (%s): %w", j.location.FullNestedPath(), err) } // parse the manifest file into a rich object manifestContents := contents[manifestMatches[0]] - manifest, err := parseJavaManifest(j.archivePath, strings.NewReader(manifestContents)) + manifest, err := parseJavaManifest(j.virtualPath(), strings.NewReader(manifestContents)) if err != nil { - log.Debugf("failed to parse java manifest (%s): %+v", j.location, err) + log.Debugf("failed to parse java manifest (%s): %+v", j.location.FullNestedPath(), err) return nil, nil } // check for existence of Weave-Classes manifest key in order to exclude jars getting misrepresented as // their targeted counterparts, e.g. newrelic spring and tomcat instrumentation if _, ok := manifest.Main.Get("Weave-Classes"); ok { - log.Debugf("excluding archive due to Weave-Classes manifest entry: %s", j.location) + log.Debugf("excluding archive due to Weave-Classes manifest entry: %s", j.location.FullNestedPath()) return nil, nil } - // grab and assign digest for the entire archive - digests, err := getDigestsFromArchive(ctx, j.archivePath) - if err != nil { - return nil, err - } - name, version, lics, parsedPom := j.discoverNameVersionLicense(ctx, manifest) var pkgPomProject *pkg.JavaPomProject if parsedPom != nil { @@ -276,10 +309,10 @@ func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package, ), Type: j.fileInfo.pkgType(), Metadata: pkg.JavaArchive{ - VirtualPath: j.location.Path(), + VirtualPath: j.virtualPath(), Manifest: manifest, PomProject: pkgPomProject, - ArchiveDigests: digests, + ArchiveDigests: j.archiveDigests, }, }, nil } @@ -392,8 +425,8 @@ type parsedPomProject struct { // discoverMainPackageFromPomInfo attempts to resolve maven groupId, artifactId, version and other info from found pom information func (j *archiveParser) discoverMainPackageFromPomInfo(ctx context.Context) (group, name, version string, parsedPom *parsedPomProject) { // Find the pom.properties/pom.xml if the names seem like a plausible match - properties, _ := pomPropertiesByParentPath(ctx, j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomPropertiesGlob)) - projects, _ := pomProjectByParentPath(ctx, j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomXMLGlob)) + properties, _ := pomPropertiesByParentPath(ctx, j.entries, j.location, j.entries.glob(pomPropertiesGlob)) + projects, _ := pomProjectByParentPath(ctx, j.entries, j.location, j.entries.glob(pomXMLGlob)) artifactsMap := j.buildArtifactsMap(properties) pomProperties, parsedPom := j.findBestPomMatch(properties, projects, artifactsMap) @@ -524,13 +557,13 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(ctx context.Context, paren var pkgs []pkg.Package // pom.properties - properties, err := pomPropertiesByParentPath(ctx, j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomPropertiesGlob)) + properties, err := pomPropertiesByParentPath(ctx, j.entries, j.location, j.entries.glob(pomPropertiesGlob)) if err != nil { return nil, err } // pom.xml - projects, err := pomProjectByParentPath(ctx, j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomXMLGlob)) + projects, err := pomProjectByParentPath(ctx, j.entries, j.location, j.entries.glob(pomXMLGlob)) if err != nil { return nil, err } @@ -573,16 +606,16 @@ func (j *archiveParser) getLicenseFromFileInArchive(ctx context.Context) []pkg.L for _, glob := range []string{"/META-INF/*", "/*"} { var licenseMatches []string - for _, f := range j.fileManifest.GlobMatch(true, glob) { + for _, f := range j.entries.globFold(glob) { if licenses.IsLicenseFile(path.Base(f)) { licenseMatches = append(licenseMatches, f) } } if len(licenseMatches) > 0 { - contents, err := intFile.ContentsFromZip(ctx, j.archivePath, licenseMatches...) + contents, err := j.entries.contents(ctx, licenseMatches...) if err != nil { - log.Debugf("unable to extract java license (%s): %w", j.location, err) + log.Debugf("unable to extract java license (%s): %v", j.location.FullNestedPath(), err) continue } @@ -624,14 +657,14 @@ func (j *archiveParser) versionFromPropertiesFile(ctx context.Context, manifest return "" } - matches := j.fileManifest.GlobMatch(true, "/version.properties") + matches := j.entries.globFold("/version.properties") if len(matches) != 1 { return "" } - contents, err := intFile.ContentsFromZip(ctx, j.archivePath, matches...) + contents, err := j.entries.contents(ctx, matches...) if err != nil { - log.Debugf("unable to extract version.properties (%s): %v", j.location, err) + log.Debugf("unable to extract version.properties (%s): %v", j.location.FullNestedPath(), err) return "" } @@ -651,9 +684,16 @@ func (j *archiveParser) versionFromPropertiesFile(ctx context.Context, manifest return "" } +// discoverPkgsFromNestedArchives recurses into the java archives inside this one, which needs an +// archive file to unzip; an already-extracted archive's nested archives are the archive cataloger +// task's to find. func (j *archiveParser) discoverPkgsFromNestedArchives(ctx context.Context, parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) { + zipped, ok := j.entries.(*zipEntries) + if !ok { + return nil, nil, nil + } // we know that all java archives are zip formatted files, so we can use the shared zip helper - return discoverPkgsFromZip(ctx, j.location, j.archivePath, j.contentPath, j.fileManifest, parentPkg, j.cfg) + return discoverPkgsFromZip(ctx, j.location, zipped.archivePath, j.contentPath, zipped.fileManifest, parentPkg, j.cfg) } // discoverPkgsFromZip finds Java archives within Java archives, returning all listed Java packages found and @@ -676,7 +716,7 @@ func discoverPkgsFromOpeners(ctx context.Context, location file.Location, opener for pathWithinArchive, archiveOpener := range sortedIter(openers) { nestedPkgs, nestedRelationships, err := discoverPkgsFromOpener(ctx, location, pathWithinArchive, archiveOpener, cfg, parentPkg) if err != nil { - log.WithFields("location", location.Path(), "error", err).Debug("unable to discover java packages from opener") + log.WithFields("location", location.FullNestedPath(), "error", err).Debug("unable to discover java packages from opener") continue } @@ -709,7 +749,7 @@ func discoverPkgsFromOpener(ctx context.Context, location file.Location, pathWit } }() - nestedPath := fmt.Sprintf("%s:%s", location.Path(), pathWithinArchive) + nestedPath := fmt.Sprintf("%s:%s", location.AccessPath, pathWithinArchive) nestedLocation := file.NewLocationFromCoordinates(location.Coordinates) nestedLocation.AccessPath = nestedPath gap := newGenericArchiveParserAdapter(cfg) @@ -724,8 +764,8 @@ func discoverPkgsFromOpener(ctx context.Context, location file.Location, pathWit return nestedPkgs, nestedRelationships, nil } -func pomPropertiesByParentPath(ctx context.Context, archivePath string, location file.Location, extractPaths []string) (map[string]pkg.JavaPomProperties, error) { - contentsOfMavenPropertiesFiles, err := intFile.ContentsFromZip(ctx, archivePath, extractPaths...) +func pomPropertiesByParentPath(ctx context.Context, entries archiveEntries, location file.Location, extractPaths []string) (map[string]pkg.JavaPomProperties, error) { + contentsOfMavenPropertiesFiles, err := entries.contents(ctx, extractPaths...) if err != nil { return nil, fmt.Errorf("unable to extract maven files: %w", err) } @@ -734,7 +774,7 @@ func pomPropertiesByParentPath(ctx context.Context, archivePath string, location for filePath, fileContents := range sortedIter(contentsOfMavenPropertiesFiles) { pomProperties, err := parsePomProperties(filePath, strings.NewReader(fileContents)) if err != nil { - log.WithFields("contents-path", filePath, "location", location.Path(), "error", err).Debug("failed to parse pom.properties") + log.WithFields("contents-path", filePath, "location", location.FullNestedPath(), "error", err).Debug("failed to parse pom.properties") continue } @@ -753,8 +793,8 @@ func pomPropertiesByParentPath(ctx context.Context, archivePath string, location return propertiesByParentPath, nil } -func pomProjectByParentPath(ctx context.Context, archivePath string, location file.Location, extractPaths []string) (map[string]*parsedPomProject, error) { - contentsOfMavenProjectFiles, err := intFile.ContentsFromZip(ctx, archivePath, extractPaths...) +func pomProjectByParentPath(ctx context.Context, entries archiveEntries, location file.Location, extractPaths []string) (map[string]*parsedPomProject, error) { + contentsOfMavenProjectFiles, err := entries.contents(ctx, extractPaths...) if err != nil { return nil, fmt.Errorf("unable to extract maven files: %w", err) } @@ -764,7 +804,7 @@ func pomProjectByParentPath(ctx context.Context, archivePath string, location fi // TODO: when we support locations of paths within archives we should start passing the specific pom.xml location object instead of the top jar pom, err := maven.ParsePomXML(strings.NewReader(fileContents)) if err != nil { - log.WithFields("contents-path", filePath, "location", location.Path(), "error", err).Debug("failed to parse pom.xml") + log.WithFields("contents-path", filePath, "location", location.FullNestedPath(), "error", err).Debug("failed to parse pom.xml") continue } if pom == nil { @@ -801,7 +841,7 @@ func newPackageFromMavenData(ctx context.Context, r *maven.Resolver, pomProperti // https://github.com/anchore/syft/issues/1944 vPathSuffix += ":" + pomProperties.GroupID + ":" + pomProperties.ArtifactID } - virtualPath := location.Path() + vPathSuffix + virtualPath := archive.VirtualPath(location) + vPathSuffix var pkgPomProject *pkg.JavaPomProject diff --git a/syft/pkg/cataloger/java/archive_parser_traversal_test.go b/syft/pkg/cataloger/java/archive_parser_traversal_test.go new file mode 100644 index 000000000..4adeb8d3d --- /dev/null +++ b/syft/pkg/cataloger/java/archive_parser_traversal_test.go @@ -0,0 +1,192 @@ +package java + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/archive" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" + "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/syft/source/directorysource" +) + +// Test_virtualPathFromArchiveLocation_colonInEntryPath is the java-cataloger-level counterpart of +// internal/archive's escaping tests: a package reached through an entry whose path contains a colon +// must carry it escaped as %3A end to end in its virtual path, while the maven-derived +// ":groupID:artifactID" suffix that distinguishes a pom.properties package from its containing archive +// is still appended with real, unescaped delimiters. +func Test_virtualPathFromArchiveLocation_colonInEntryPath(t *testing.T) { + jarBytes := makeTestZip(t, map[string][]byte{ + "META-INF/MANIFEST.MF": testManifest("outer-app", "1.0.0"), + "META-INF/maven/com.example/different-lib/pom.properties": []byte( + "groupId=com.example\nartifactId=different-lib\nversion=9.9.9\n", + ), + }) + + // as the archive cataloger task's resolver would present an entry of outer.zip + location := file.NewLocationFromCoordinates(file.Coordinates{RealPath: "outer.zip", FullPath: "outer.zip!/lib/dep:special.jar"}) + + gap := newGenericArchiveParserAdapter(ArchiveCatalogerConfig{}) + pkgs, _, err := gap.processJavaArchive(pkgtest.Context(t), file.LocationReadCloser{ + Location: location, + ReadCloser: io.NopCloser(bytes.NewReader(jarBytes)), + }, nil) + require.NoError(t, err) + require.Len(t, pkgs, 2) + + var mainPkg, auxPkg *pkg.Package + for i := range pkgs { + if pkgs[i].Name == "outer-app" { + mainPkg = &pkgs[i] + } else if pkgs[i].Name == "different-lib" { + auxPkg = &pkgs[i] + } + } + require.NotNil(t, mainPkg, "expected a package from the manifest") + require.NotNil(t, auxPkg, "expected a package from pom.properties") + + mainMetadata, ok := mainPkg.Metadata.(pkg.JavaArchive) + require.True(t, ok) + assert.Equal(t, "outer.zip:lib/dep%3Aspecial.jar", mainMetadata.VirtualPath, + "the entry's own colon must be escaped, and it is the only delimiter in play here") + + auxMetadata, ok := auxPkg.Metadata.(pkg.JavaArchive) + require.True(t, ok) + assert.Equal(t, "outer.zip:lib/dep%3Aspecial.jar:com.example:different-lib", auxMetadata.VirtualPath, + "the maven groupID:artifactID suffix is appended with real, unescaped delimiters") +} + +func Test_nestedArchiveOwnershipSwitch(t *testing.T) { + nestedJar := makeTestZip(t, map[string][]byte{ + "META-INF/MANIFEST.MF": testManifest("nested-lib", "2.0"), + }) + outerJar := makeTestZip(t, map[string][]byte{ + "META-INF/MANIFEST.MF": testManifest("example-app", "1.0.0"), + "lib/nested-lib-2.0.jar": nestedJar, + }) + + parse := func(t *testing.T, ctx context.Context, cfg ArchiveCatalogerConfig) []pkg.Package { + gap := newGenericArchiveParserAdapter(cfg) + pkgs, _, err := gap.processJavaArchive(ctx, file.LocationReadCloser{ + Location: file.NewLocation("example-app-1.0.0.jar"), + ReadCloser: io.NopCloser(bytes.NewReader(outerJar)), + }, nil) + require.NoError(t, err) + return pkgs + } + + t.Run("nested archive cataloging disabled: this cataloger unarchives nested jars itself", func(t *testing.T) { + pkgs := parse(t, pkgtest.Context(t), DefaultArchiveCatalogerConfig()) // MaxDepth defaults to 0 (disabled) + require.Len(t, pkgs, 2) + + var names, virtualPaths []string + for _, p := range pkgs { + names = append(names, p.Name) + metadata, ok := p.Metadata.(pkg.JavaArchive) + require.True(t, ok) + virtualPaths = append(virtualPaths, metadata.VirtualPath) + } + assert.ElementsMatch(t, []string{"example-app", "nested-lib"}, names) + assert.ElementsMatch(t, []string{ + "example-app-1.0.0.jar", + "example-app-1.0.0.jar:lib/nested-lib-2.0.jar", + }, virtualPaths) + }) + + t.Run("nested archive cataloging enabled: this cataloger does not unarchive", func(t *testing.T) { + // only syft.CreateSBOM marks the context in production; it is not reachable from user config + ctx := archive.WithNestedCataloging(pkgtest.Context(t)) + + // no error expected: the nested jar is deliberately left to the archive cataloger task + pkgs := parse(t, ctx, DefaultArchiveCatalogerConfig()) + require.Len(t, pkgs, 1) + assert.Equal(t, "example-app", pkgs[0].Name) + }) +} + +func Test_NewArchiveCataloger_wrappedParserRegistration(t *testing.T) { + // the wrapped-archive parsers exist only because the archive cataloger did not. When it is enabled + // it extracts zip and tar containers itself and this cataloger meets the same jars one nesting level + // down, so registering them would double the work. + // + // Asserted through the globs the cataloger queries, the observable consequence of a registration. + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "placeholder.txt"), []byte("x"), 0o600)) + + queriedGlobs := func(t *testing.T, ctx context.Context, cfg ArchiveCatalogerConfig) *pkgtest.ObservingResolver { + t.Helper() + src, err := directorysource.NewFromPath(dir) + require.NoError(t, err) + resolver, err := src.FileResolver(source.AllLayersScope) + require.NoError(t, err) + + observed := pkgtest.NewObservingResolver(resolver) + _, _, err = NewArchiveCataloger(cfg).Catalog(ctx, observed) + require.NoError(t, err) + return observed + } + + t.Run("wrapped parsers are not registered when the archive cataloger owns recursion", func(t *testing.T) { + cfg := DefaultArchiveCatalogerConfig() + cfg.IncludeIndexedArchives = true + cfg.IncludeUnindexedArchives = true + + observed := queriedGlobs(t, archive.WithNestedCataloging(pkgtest.Context(t)), cfg) + // outside a traversal the cataloger opens no files at all: the task brings every archive back + // as an extracted filesystem, and each is described there + assert.False(t, observed.ObservedPathQuery("**/*.zip"), "no generic zip parser may be registered") + assert.False(t, observed.ObservedPathQuery("**/*.tar"), "no generic tar parser may be registered") + assert.False(t, observed.ObservedPathQuery("**/*.jar"), "archive files are the task's to open") + }) + + t.Run("wrapped parsers stay registered when the feature is off", func(t *testing.T) { + cfg := DefaultArchiveCatalogerConfig() + cfg.IncludeIndexedArchives = true + cfg.IncludeUnindexedArchives = true + + observed := queriedGlobs(t, pkgtest.Context(t), cfg) + assert.True(t, observed.ObservedPathQuery("**/*.zip"), "existing behavior must be unchanged") + assert.True(t, observed.ObservedPathQuery("**/*.tar"), "existing behavior must be unchanged") + }) + + t.Run("the existing include settings still gate them when the feature is off", func(t *testing.T) { + cfg := DefaultArchiveCatalogerConfig() + cfg.IncludeIndexedArchives = false + cfg.IncludeUnindexedArchives = false + + observed := queriedGlobs(t, pkgtest.Context(t), cfg) + assert.False(t, observed.ObservedPathQuery("**/*.zip")) + assert.False(t, observed.ObservedPathQuery("**/*.tar")) + }) +} + +// makeTestZip builds a zip in memory so the parser can be exercised without the java-toolchain-built +// fixtures. +func makeTestZip(t *testing.T, entries map[string][]byte) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for name, content := range entries { + w, err := zw.Create(name) + require.NoError(t, err) + _, err = w.Write(content) + require.NoError(t, err) + } + require.NoError(t, zw.Close()) + return buf.Bytes() +} + +func testManifest(title, version string) []byte { + return fmt.Appendf(nil, "Manifest-Version: 1.0\nImplementation-Title: %s\nImplementation-Version: %s\n", title, version) +} diff --git a/syft/pkg/cataloger/java/archive_source_test.go b/syft/pkg/cataloger/java/archive_source_test.go new file mode 100644 index 000000000..11e206603 --- /dev/null +++ b/syft/pkg/cataloger/java/archive_source_test.go @@ -0,0 +1,229 @@ +package java + +import ( + "archive/zip" + "bytes" + "context" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/archive" + "github.com/anchore/syft/internal/tmpdir" + "github.com/anchore/syft/internal/unknown" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/internal/fileresolver" + "github.com/anchore/syft/syft/pkg" +) + +func Test_archiveContentsCataloger_agreesWithTheArchiveReadingCataloger(t *testing.T) { + // for the same archive, the contents-reading cataloger must report the same packages with the same + // identity as the one reading the archive file; any divergence shows up as an SBOM diff + entries := jarEntries() + cfg := DefaultArchiveCatalogerConfig() + + // the archive-reading cataloger, over a directory holding the jar + archiveDir := t.TempDir() + jarPath := writeJar(t, archiveDir, "example-lib-1.2.3.jar", entries) + archiveResolver, err := fileresolver.NewFromDirectory(archiveDir, "") + require.NoError(t, err) + // the archive-reading cataloger copies the archive to a temp dir first, which the contents-reading + // one does not + archiveCtx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir())) + fromArchive, _, err := NewArchiveCataloger(cfg).Catalog(archiveCtx, archiveResolver) + require.NoError(t, err) + require.NotEmpty(t, fromArchive) + + // the contents-reading cataloger, over the same entries laid out as a filesystem, with the traversal + // the archive cataloger task would have placed on the context + contentsDir := t.TempDir() + explode(t, contentsDir, entries) + contentsResolver, err := fileresolver.NewFromDirectory(contentsDir, "") + require.NoError(t, err) + + // the traversal names the archive exactly as the resolver did for the archive-reading cataloger, so + // both sides derive identity from the same string and the comparison is about where the entries came + // from rather than how the test spelled a path + jarLocations, err := archiveResolver.FilesByGlob("**/*.jar") + require.NoError(t, err) + require.Len(t, jarLocations, 1) + archivePath := jarLocations[0].FullNestedPath() + + digests := digestsOfFile(t, jarPath) + trav := &archive.Traversal{ + Location: file.NewLocation(archivePath), + Digests: digests, + } + ctx := archive.WithTraversal(context.Background(), trav) + fromContents, _, err := NewArchiveCataloger(cfg).Catalog(ctx, contentsResolver) + require.NoError(t, err) + + assert.Equal(t, summarize(fromArchive), summarize(fromContents)) +} + +func Test_archiveContentsCataloger_reportsNothingForAnArchiveThatIsNotJava(t *testing.T) { + // a .zip is an archive and its contents are not a jar's; the archive-reading cataloger draws that + // line with its globs and this one draws it with the same list + dir := t.TempDir() + explode(t, dir, jarEntries()) + resolver, err := fileresolver.NewFromDirectory(dir, "") + require.NoError(t, err) + + ctx := archive.WithTraversal(context.Background(), &archive.Traversal{ + Location: file.NewLocation("/bundle.zip"), + }) + pkgs, _, err := NewArchiveCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver) + require.NoError(t, err) + assert.Empty(t, pkgs) +} + +func Test_archiveContentsCataloger_noManifestIsNoPackage(t *testing.T) { + // unchanged from the archive-reading cataloger, and the whole coverage argument: a jar with no + // META-INF/MANIFEST.MF yields nothing today either, so requiring the contents to be traversed takes + // away less than it appears to. The maven metadata here is never consulted - without a main package + // there is nothing to attach it to. + entries := map[string]string{ + "META-INF/maven/com.other/bundled-dep/pom.properties": "groupId=com.other\nartifactId=bundled-dep\nversion=4.5.6\n", + "com/example/Thing.class": "not really bytecode", + } + + dir := t.TempDir() + explode(t, dir, entries) + resolver, err := fileresolver.NewFromDirectory(dir, "") + require.NoError(t, err) + + ctx := archive.WithTraversal(context.Background(), &archive.Traversal{ + Location: file.NewLocation("no-manifest-1.0.0.jar"), + }) + pkgs, _, err := NewArchiveCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver) + assert.Empty(t, pkgs) + // reported against the archive rather than swallowed, and as an unknown rather than a scan failure + require.Error(t, err) + coordErrs, remaining := unknown.ExtractCoordinateErrors(err) + assert.NoError(t, remaining) + require.Len(t, coordErrs, 1) + assert.Equal(t, "no-manifest-1.0.0.jar", coordErrs[0].Coordinates.RealPath) +} + +func Test_archiveContentsCataloger_readsWhatWasExtractedAndNoMore(t *testing.T) { + // the sharp edge of reading contents rather than the archive: this cataloger sees exactly what the + // extraction produced. An archive truncated by a limit yields a package built from the entries that + // were written, so identity degrades rather than disappearing - the manifest is there, the maven + // metadata is not. + entries := map[string]string{ + "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n" + + "Implementation-Title: example-lib\nImplementation-Version: 1.2.3\n", + // the pom that would have supplied the group id is missing, as truncation would leave it + "com/example/Thing.class": "not really bytecode", + } + + dir := t.TempDir() + explode(t, dir, entries) + resolver, err := fileresolver.NewFromDirectory(dir, "") + require.NoError(t, err) + + ctx := archive.WithTraversal(context.Background(), &archive.Traversal{ + Location: file.NewLocation("example-lib-1.2.3.jar"), + }) + pkgs, _, err := NewArchiveCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver) + require.NoError(t, err) + require.Len(t, pkgs, 1) + assert.Equal(t, "example-lib", pkgs[0].Name) + assert.Equal(t, "1.2.3", pkgs[0].Version) + assert.Equal(t, "pkg:maven/example-lib/example-lib@1.2.3", pkgs[0].PURL, + "with no pom to name the group, the group falls back to the artifact id") +} + +// jarEntries is one java archive's worth of content, used both to build the archive the +// archive-reading cataloger parses and to lay out the filesystem the contents-reading one reads. The +// tests below assert the two agree. +func jarEntries() map[string]string { + return map[string]string{ + "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n" + + "Implementation-Title: example-lib\n" + + "Implementation-Version: 1.2.3\n", + "META-INF/maven/com.example/example-lib/pom.properties": "groupId=com.example\n" + + "artifactId=example-lib\nversion=1.2.3\n", + "META-INF/maven/com.example/example-lib/pom.xml": `4.0.0 +com.exampleexample-lib1.2.3 +Apache License 2.0https://www.apache.org/licenses/LICENSE-2.0 +`, + // a bundled dependency's maven metadata, which becomes a package of its own + "META-INF/maven/com.other/bundled-dep/pom.properties": "groupId=com.other\n" + + "artifactId=bundled-dep\nversion=4.5.6\n", + "com/example/Thing.class": "not really bytecode", + } +} + +func writeJar(t *testing.T, dir, name string, entries map[string]string) string { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + names := make([]string, 0, len(entries)) + for n := range entries { + names = append(names, n) + } + sort.Strings(names) + for _, n := range names { + w, err := zw.Create(n) + require.NoError(t, err) + _, err = w.Write([]byte(entries[n])) + require.NoError(t, err) + } + require.NoError(t, zw.Close()) + + p := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(p, buf.Bytes(), 0o600)) + return p +} + +// explode lays the same entries out as files, what the archive cataloger task hands the sub-pipeline +// once it has extracted an archive. +func explode(t *testing.T, dir string, entries map[string]string) { + t.Helper() + for name, contents := range entries { + p := filepath.Join(dir, filepath.FromSlash(name)) + require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755)) + require.NoError(t, os.WriteFile(p, []byte(contents), 0o600)) + } +} + +func summarize(pkgs []pkg.Package) []string { + var out []string + for _, p := range pkgs { + meta, _ := p.Metadata.(pkg.JavaArchive) + lics := p.Licenses.ToSlice() + names := make([]string, 0, len(lics)) + for _, l := range lics { + names = append(names, l.Value) + } + sort.Strings(names) + out = append(out, p.Name+"@"+p.Version+" purl="+p.PURL+" vpath="+meta.VirtualPath+" lics="+ + joinSorted(names)) + } + sort.Strings(out) + return out +} + +func joinSorted(in []string) string { + out := "" + for i, s := range in { + if i > 0 { + out += "," + } + out += s + } + return out +} + +func digestsOfFile(t *testing.T, path string) []file.Digest { + t.Helper() + digests, err := getDigestsFromArchive(context.Background(), path) + require.NoError(t, err) + require.NotEmpty(t, digests) + return digests +} diff --git a/syft/pkg/cataloger/java/cataloger.go b/syft/pkg/cataloger/java/cataloger.go index 1e48b05e1..3568e3526 100644 --- a/syft/pkg/cataloger/java/cataloger.go +++ b/syft/pkg/cataloger/java/cataloger.go @@ -4,12 +4,24 @@ Package java provides a concrete Cataloger implementation for packages relating package java import ( + "context" + + "github.com/bmatcuk/doublestar/v4" + + "github.com/anchore/syft/internal/archive" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/internal/unknown" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" ) // NewArchiveCataloger returns a new Java archive cataloger object for detecting packages with archives (jar, war, ear, par, sar, jpi, hpi, and native-image formats) func NewArchiveCataloger(cfg ArchiveCatalogerConfig) pkg.Cataloger { + if a := cfg.ArchiveSearchConfig; a.MaxDepth != 0 || a.MaxMemoryBytes != 0 || a.MaxDiskBytes != 0 { + log.Warn("java archive cataloger ignores nested archive depth and limits, set them with syft.CreateSBOMConfig.Archive") + } gap := newGenericArchiveParserAdapter(cfg) c := generic.NewCataloger("java-archive-cataloger"). @@ -26,7 +38,94 @@ func NewArchiveCataloger(cfg ArchiveCatalogerConfig) pkg.Cataloger { gtp := newGenericTarWrappedJavaArchiveParser(cfg) c.WithParserByGlobs(gtp.parseTarWrappedJavaArchive, genericTarGlobs...) } - return c + + return &archiveCataloger{cfg: cfg, extractingCataloger: c} +} + +// archiveCataloger catalogs java archives two ways. Inside an archive the archive cataloger task has +// extracted, the resolver is that archive's contents and the traversal on the context names the +// archive, so the archive to describe is the one the resolver holds. Everywhere else, archives are +// files in the resolver that this cataloger opens itself. +type archiveCataloger struct { + cfg ArchiveCatalogerConfig + extractingCataloger *generic.Cataloger +} + +func (c *archiveCataloger) Name() string { + return c.extractingCataloger.Name() +} + +func (c *archiveCataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { + if trav := archive.TraversalFromContext(ctx); trav != nil { + return c.catalogExtracted(ctx, trav, resolver) + } + + if archive.NestedCatalogingEnabled(ctx) { + // the archive cataloger task extracts every archive and runs this cataloger inside each; opening + // them here too would catalog each twice + return nil, nil, nil + } + + // process archives directly, using the original locally managed java archive extraction + return c.extractingCataloger.Catalog(ctx, resolver) +} + +// catalogExtracted reports the java package of the archive whose contents the resolver holds, if it +// is a java archive. Archives nested inside it are left to the archive cataloger task. +func (c *archiveCataloger) catalogExtracted(ctx context.Context, trav *archive.Traversal, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { + // by the name it was reached by: a content-addressed jar is named like one only by its link + if !isJavaArchiveName(trav.Location.AccessPath) { + return nil, nil, nil + } + + parser, err := newExtractedArchiveParser(trav, resolver, c.cfg) + if err != nil { + return nil, nil, err + } + + // link to the jar this one is nested in, as the legacy recursion does: a DEPENDENCY_OF edge from + // parse, and Parent on whatever does not already have one + parent := enclosingJavaPackage(trav) + pkgs, relationships, err := parser.parse(ctx, parent) + if parent != nil { + for i := range pkgs { + if metadata, ok := pkgs[i].Metadata.(pkg.JavaArchive); ok && metadata.Parent == nil { + metadata.Parent = parent + pkgs[i].Metadata = metadata + } + } + } + if err != nil { + // attributed to the archive, as the generic cataloger attributes a parser failure to its file + return pkgs, relationships, unknown.New(trav.Location, err) + } + return pkgs, relationships, nil +} + +// enclosingJavaPackage returns the main java package of the archive the traversal's archive is nested +// in, if there is one. Of the java packages located at an archive, the main one carries the archive's +// own virtual path; those found from embedded pom files carry it with their group and artifact appended. +func enclosingJavaPackage(trav *archive.Traversal) *pkg.Package { + if trav.Parent == nil { + return nil + } + virtualPath := archive.VirtualPath(trav.Parent.Location) + for _, p := range trav.Parent.Packages { + if metadata, ok := p.Metadata.(pkg.JavaArchive); ok && metadata.VirtualPath == virtualPath { + return &p + } + } + return nil +} + +// isJavaArchiveName reports whether the path matches the globs this cataloger opens files by. +func isJavaArchiveName(path string) bool { + for _, glob := range archiveFormatGlobs { + if ok, _ := doublestar.Match(glob, path); ok { + return true + } + } + return false } // NewPomCataloger returns a cataloger capable of parsing dependencies from a pom.xml file. diff --git a/syft/pkg/cataloger/java/config.go b/syft/pkg/cataloger/java/config.go index 652ca827b..a67033640 100644 --- a/syft/pkg/cataloger/java/config.go +++ b/syft/pkg/cataloger/java/config.go @@ -8,6 +8,10 @@ import ( ) type ArchiveCatalogerConfig struct { + // Deprecated: use syft.CreateSBOMConfig.Archive. Only the two Include* booleans are read here, and + // setting MaxDepth, MaxMemoryBytes or MaxDiskBytes is a configuration error. This will be removed in a + // future release. + // capabilities-fields: IncludeIndexedArchives, IncludeUnindexedArchives cataloging.ArchiveSearchConfig `yaml:",inline" json:"" mapstructure:",squash"` // UseNetwork enables network operations for java package metadata enrichment, such as fetching parent POMs and license information. @@ -37,8 +41,13 @@ type ArchiveCatalogerConfig struct { func DefaultArchiveCatalogerConfig() ArchiveCatalogerConfig { mavenCfg := maven.DefaultConfig() + search := cataloging.DefaultArchiveSearchConfig() return ArchiveCatalogerConfig{ - ArchiveSearchConfig: cataloging.DefaultArchiveSearchConfig(), + // only the booleans: depth and limits are not java settings + ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ + IncludeIndexedArchives: search.IncludeIndexedArchives, + IncludeUnindexedArchives: search.IncludeUnindexedArchives, + }, UseNetwork: mavenCfg.UseNetwork, UseMavenLocalRepository: mavenCfg.UseLocalRepository, MavenLocalRepositoryDir: mavenCfg.LocalRepositoryDir, @@ -75,12 +84,19 @@ func (j ArchiveCatalogerConfig) WithResolveTransitiveDependencies(resolveTransit return j } +// Deprecated: set archive search with syft.CreateSBOMConfig.Archive and the parent depth with +// WithMaxParentRecursiveDepth. This will be removed in a future release. func (j ArchiveCatalogerConfig) WithArchiveTraversal(search cataloging.ArchiveSearchConfig, maxDepth int) ArchiveCatalogerConfig { j.MaxParentRecursiveDepth = maxDepth j.ArchiveSearchConfig = search return j } +func (j ArchiveCatalogerConfig) WithMaxParentRecursiveDepth(depth int) ArchiveCatalogerConfig { + j.MaxParentRecursiveDepth = depth + return j +} + func (j ArchiveCatalogerConfig) mavenConfig() maven.Config { return maven.Config{ UseNetwork: j.UseNetwork, diff --git a/syft/pkg/cataloger/java/graalvm_native_image_cataloger.go b/syft/pkg/cataloger/java/graalvm_native_image_cataloger.go index ab0845360..65a8648bc 100644 --- a/syft/pkg/cataloger/java/graalvm_native_image_cataloger.go +++ b/syft/pkg/cataloger/java/graalvm_native_image_cataloger.go @@ -586,7 +586,7 @@ func fetchPkgs(reader unionreader.UnionReader, location file.Location) ([]pkg.Pa log.Debugf("failed to open the java native-image binary: %v", err) return nil, nil } - filename := location.RealPath + filename := location.FullNestedPath() for _, r := range readers { for _, makeNativeImage := range imageFormats { ni, err := makeNativeImage(filename, r) @@ -645,7 +645,7 @@ func processLocation(location file.Location, resolver file.Resolver) ([]pkg.Pack log.Debugf("error opening file: %v", err) return nil, nil, nil } - defer internal.CloseAndLogError(readerCloser, location.RealPath) + defer internal.CloseAndLogError(readerCloser, location.FullNestedPath()) reader, err := unionreader.GetUnionReader(readerCloser) if err != nil { diff --git a/syft/pkg/cataloger/java/internal/maven/resolver.go b/syft/pkg/cataloger/java/internal/maven/resolver.go index b7d3d8ced..36192fd8c 100644 --- a/syft/pkg/cataloger/java/internal/maven/resolver.go +++ b/syft/pkg/cataloger/java/internal/maven/resolver.go @@ -647,7 +647,8 @@ func (r *Resolver) findParentPomByRelativePath(ctx context.Context, pom *Project if relativePath == "" { return nil } - p := pomLocation.Path() + // relative to the pom within its own archive, so "../" can't clean past the archive boundary + p := pomLocation.LocalPath() p = path.Dir(p) p = path.Join(p, relativePath) p = path.Clean(p) @@ -668,7 +669,7 @@ func (r *Resolver) findParentPomByRelativePath(ctx context.Context, pom *Project Debug("unable to get contents of parent pom by relative path") return nil } - defer internal.CloseAndLogError(parentContents, parentLocation.RealPath) + defer internal.CloseAndLogError(parentContents, parentLocation.FullNestedPath()) parentPom, err := ParsePomXML(parentContents) if err != nil || parentPom == nil { log.WithFields("error", err, "mavenID", r.resolveID(ctx, resolvingProperties, pom), "parentID", parentID, "parentLocation", parentLocation). diff --git a/syft/pkg/cataloger/java/internal/maven/resolver_test.go b/syft/pkg/cataloger/java/internal/maven/resolver_test.go index 259a55c45..3965cb1af 100644 --- a/syft/pkg/cataloger/java/internal/maven/resolver_test.go +++ b/syft/pkg/cataloger/java/internal/maven/resolver_test.go @@ -6,9 +6,11 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/internal/fileresolver" maventest "github.com/anchore/syft/syft/pkg/cataloger/java/internal/maven/test" ) @@ -352,7 +354,7 @@ func Test_relativePathParent(t *testing.T) { loc := locs[0] contents, err := resolver.FileContentsByLocation(loc) require.NoError(t, err) - defer internal.CloseAndLogError(contents, loc.RealPath) + defer internal.CloseAndLogError(contents, loc.FullNestedPath()) pom, err := ParsePomXML(contents) require.NoError(t, err) @@ -368,3 +370,25 @@ func Test_relativePathParent(t *testing.T) { func ptr[T any](value T) *T { return &value } + +// pathRecorder records the paths it is asked for and finds nothing. +type pathRecorder struct { + file.Resolver + paths []string +} + +func (p *pathRecorder) FilesByPath(paths ...string) ([]file.Location, error) { + p.paths = append(p.paths, paths...) + return nil, nil +} + +func Test_relativePathParent_staysInsideTheArchive(t *testing.T) { + recorder := &pathRecorder{} + r := NewResolver(recorder, DefaultConfig()) + pom := &Project{Parent: &Parent{RelativePath: ptr("../../other")}} + nested := file.Coordinates{RealPath: "/data/outer.zip", FullPath: "/data/outer.zip!/a/pom.xml"} + r.pomLocations[pom] = file.NewVirtualLocationFromCoordinates(nested, nested.FullPath) + + assert.Nil(t, r.findParentPomByRelativePath(context.Background(), pom, ID{}, nil)) + assert.Equal(t, []string{"/other/pom.xml"}, recorder.paths) +} diff --git a/syft/pkg/cataloger/java/parse_jvm_release.go b/syft/pkg/cataloger/java/parse_jvm_release.go index c5460ed3b..51617197b 100644 --- a/syft/pkg/cataloger/java/parse_jvm_release.go +++ b/syft/pkg/cataloger/java/parse_jvm_release.go @@ -62,7 +62,7 @@ type jvmCpeInfo struct { func parseJVMRelease(_ context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { ri, err := parseJvmReleaseInfo(reader) if err != nil { - return nil, nil, fmt.Errorf("unable to parse JVM release info %q: %w", reader.Path(), err) + return nil, nil, fmt.Errorf("unable to parse JVM release info %q: %w", reader.FullNestedPath(), err) } if ri == nil { @@ -81,13 +81,17 @@ func parseJVMRelease(_ context.Context, resolver file.Resolver, _ *generic.Envir locations.Add(lic.Locations.ToSlice()...) } - installDir := path.Dir(reader.Path()) + installDir := path.Dir(reader.LocalPath()) files, hasJdk := findJvmFiles(resolver, installDir) // the reason we use the reference to get the real path is in cases where the file cataloger is involved // (thus the real path is not available except for in the original file reference). This is important // since the path is critical for distinguishing between different JVM vendors. - vendor, product := jvmPrimaryVendorProduct(ri, string(reader.Reference().RealPath), hasJdk) + jdkPath := string(reader.Reference().RealPath) + if jdkPath == "" { + jdkPath = reader.LocalPath() + } + vendor, product := jvmPrimaryVendorProduct(ri, jdkPath, hasJdk) p := pkg.Package{ Name: product, @@ -123,7 +127,7 @@ func findJvmFiles(resolver file.Resolver, installDir string) ([]string, bool) { var results []string var hasJdk bool for _, loc := range ownedLocations { - p := loc.Path() + p := loc.LocalPath() results = append(results, p) if !hasJdk && strings.HasSuffix(p, "bin/javac") { hasJdk = true diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index 572dfed8e..72d6ba3f8 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" @@ -40,6 +41,9 @@ func (p pomXMLCataloger) Catalog(ctx context.Context, fileResolver file.Resolver var poms []*maven.Project pomLocations := map[*maven.Project]file.Location{} for _, pomLocation := range locations { + if isArchiveMetaPom(ctx, pomLocation) { + continue + } pom, err := readPomFromLocation(fileResolver, pomLocation) if err != nil || pom == nil { log.WithFields("error", err, "pomLocation", pomLocation).Debug("error while reading pom") @@ -90,7 +94,7 @@ func readPomFromLocation(fileResolver file.Resolver, pomLocation file.Location) if err != nil { return nil, err } - defer internal.CloseAndLogError(contents, pomLocation.RealPath) + defer internal.CloseAndLogError(contents, pomLocation.FullNestedPath()) return maven.ParsePomXML(contents) } @@ -299,6 +303,22 @@ func pomParent(ctx context.Context, r *maven.Resolver, pom *maven.Project) *pkg. } } +// isArchiveMetaPom reports whether a pom.xml sits under META-INF/maven, making it build metadata +// Maven embedded rather than a project pom.xml. Such a pom is a verbatim copy of the build POM, with +// unresolved ${property} references and versions inherited from parent POMs absent from the archive. +// The java-archive-cataloger identifies these archives from pom.properties, whose values are always +// resolved, so skipping them here avoids phantom dependency packages. +// +// This only holds inside an extracted java archive. Anywhere else (an exploded war, test fixtures under +// src/test/resources, a tarball of a source tree) nothing else reads these poms, so they are cataloged. +func isArchiveMetaPom(ctx context.Context, location file.Location) bool { + trav := archive.TraversalFromContext(ctx) + if trav == nil || !isJavaArchiveName(trav.Location.AccessPath) { + return false + } + return strings.Contains(location.LocalPath(), "META-INF/maven/") +} + func cleanDescription(original string) (cleaned string) { descriptionLines := strings.Split(original, "\n") for _, line := range descriptionLines { diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index 44dc5da05..6e40b760b 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -1,12 +1,14 @@ package java import ( + "context" "os" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/file" @@ -690,6 +692,147 @@ func getCommonsTextExpectedPackages(resolved bool) expected { return expected{pkgs, relationships} } +func Test_isArchiveMetaPom(t *testing.T) { + tests := []struct { + name string + path string + expected bool + }{ + { + name: "META-INF maven pom", + path: "META-INF/maven/com.example/my-lib/pom.xml", + expected: true, + }, + { + name: "nested META-INF maven pom", + path: "some/path/META-INF/maven/org.apache/commons/pom.xml", + expected: true, + }, + { + name: "project pom.xml", + path: "pom.xml", + expected: false, + }, + { + name: "module pom.xml", + path: "submodule/pom.xml", + expected: false, + }, + { + name: "META-INF but not maven", + path: "META-INF/pom.xml", + expected: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + loc := file.NewLocation(test.path) + assert.Equal(t, test.expected, isArchiveMetaPom(insideJar(t), loc)) + }) + } + + t.Run("only inside an extracted java archive", func(t *testing.T) { + loc := file.NewLocation("WEB-INF/classes/META-INF/maven/com.example/my-lib/pom.xml") + assert.False(t, isArchiveMetaPom(context.Background(), loc), "outside any archive, e.g. an exploded war") + inTarball := archive.WithTraversal(context.Background(), &archive.Traversal{Location: file.NewLocation("/src.tar.gz")}) + assert.False(t, isArchiveMetaPom(inTarball, loc), "inside an archive java does not describe") + }) +} + +// insideJar is a context as the archive cataloger task sets it up while cataloging a jar's contents. +func insideJar(t *testing.T) context.Context { + return archive.WithTraversal(pkgtest.Context(t), &archive.Traversal{Location: file.NewLocation("/app.jar")}) +} + +func Test_pomCatalogerSkipsMetaInfPoms(t *testing.T) { + // a pom.xml under META-INF/maven/ alone yields no packages: that is archive metadata, not a + // project file + cat := NewPomCataloger(ArchiveCatalogerConfig{ + ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ + IncludeIndexedArchives: true, + IncludeUnindexedArchives: true, + }, + }) + + pkgtest.NewCatalogTester(). + FromDirectory(t, "testdata/pom/meta-inf-archive"). + WithContext(insideJar(t)). + Expects(nil, nil). + TestCataloger(t, cat) + + // the same pom outside a jar is a project file like any other + pkgtest.NewCatalogTester(). + FromDirectory(t, "testdata/pom/meta-inf-archive"). + ExpectsAssertion(func(t *testing.T, pkgs []pkg.Package, _ []artifact.Relationship) { + assert.NotEmpty(t, pkgs) + }). + TestCataloger(t, cat) +} + +func Test_pomCatalogerSkipsMetaInfButKeepsProjectPom(t *testing.T) { + // with both a project pom.xml and a META-INF/maven one, only the project pom.xml is + // cataloged; the META-INF pom and its dependencies are ignored + pomLocation := file.NewLocationSet(file.NewLocation("pom.xml")) + + myApp := pkg.Package{ + Name: "my-app", + Version: "2.0.0", + PURL: "pkg:maven/org.anchore/my-app@2.0.0", + Language: pkg.Java, + Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, + Locations: pomLocation, + Metadata: pkg.JavaArchive{ + PomProject: &pkg.JavaPomProject{ + GroupID: "org.anchore", + ArtifactID: "my-app", + Version: "2.0.0", + }, + }, + } + finalizePackage(&myApp) + + guava := pkg.Package{ + Name: "guava", + Version: "31.1-jre", + PURL: "pkg:maven/com.google.guava/guava@31.1-jre", + Language: pkg.Java, + Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, + Locations: pomLocation, + Metadata: pkg.JavaArchive{ + PomProperties: &pkg.JavaPomProperties{ + GroupID: "com.google.guava", + ArtifactID: "guava", + }, + }, + } + finalizePackage(&guava) + + expectedPkgs := []pkg.Package{myApp, guava} + expectedRelationships := []artifact.Relationship{ + { + From: guava, + To: myApp, + Type: artifact.DependencyOfRelationship, + }, + } + + cat := NewPomCataloger(ArchiveCatalogerConfig{ + ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ + IncludeIndexedArchives: true, + IncludeUnindexedArchives: true, + }, + }) + + pkgtest.NewCatalogTester(). + FromDirectory(t, "testdata/pom/mixed-meta-inf-and-project"). + WithContext(insideJar(t)). + Expects(expectedPkgs, expectedRelationships). + TestCataloger(t, cat) +} + func expectedTransientPackageData() expected { epl2 := pkg.NewLicenseSet(pkg.License{ Value: "Eclipse Public License v2.0", diff --git a/syft/pkg/cataloger/java/testdata/pom/meta-inf-archive/META-INF/maven/com.example/my-lib/pom.xml b/syft/pkg/cataloger/java/testdata/pom/meta-inf-archive/META-INF/maven/com.example/my-lib/pom.xml new file mode 100644 index 000000000..ff3d28844 --- /dev/null +++ b/syft/pkg/cataloger/java/testdata/pom/meta-inf-archive/META-INF/maven/com.example/my-lib/pom.xml @@ -0,0 +1,24 @@ + + + 4.0.0 + + com.example + my-lib + 1.0.0 + + + + org.phantom + phantom-dep + ${phantom.version} + + + org.testonly + test-dep + 2.0.0 + test + + + diff --git a/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/META-INF/maven/com.example/embedded-lib/pom.xml b/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/META-INF/maven/com.example/embedded-lib/pom.xml new file mode 100644 index 000000000..cf2ab1b72 --- /dev/null +++ b/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/META-INF/maven/com.example/embedded-lib/pom.xml @@ -0,0 +1,18 @@ + + + 4.0.0 + + com.example + embedded-lib + 3.0.0 + + + + org.phantom + should-not-appear + ${unresolvable.version} + + + diff --git a/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/pom.xml b/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/pom.xml new file mode 100644 index 000000000..32b8e2426 --- /dev/null +++ b/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/pom.xml @@ -0,0 +1,18 @@ + + + 4.0.0 + + org.anchore + my-app + 2.0.0 + + + + com.google.guava + guava + 31.1-jre + + + diff --git a/syft/pkg/cataloger/java/version_properties_test.go b/syft/pkg/cataloger/java/version_properties_test.go index 2b9680cf1..7f416878a 100644 --- a/syft/pkg/cataloger/java/version_properties_test.go +++ b/syft/pkg/cataloger/java/version_properties_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" @@ -18,13 +17,13 @@ import ( func readManifest(t *testing.T, ctx context.Context, parser *archiveParser) *pkg.JavaManifest { t.Helper() - matches := parser.fileManifest.GlobMatch(false, manifestGlob) + matches := parser.entries.glob(manifestGlob) require.Len(t, matches, 1) - contents, err := intFile.ContentsFromZip(ctx, parser.archivePath, matches...) + contents, err := parser.entries.contents(ctx, matches...) require.NoError(t, err) - manifest, err := parseJavaManifest(parser.archivePath, strings.NewReader(contents[matches[0]])) + manifest, err := parseJavaManifest(parser.virtualPath(), strings.NewReader(contents[matches[0]])) require.NoError(t, err) return manifest diff --git a/syft/pkg/cataloger/java/zip_wrapped_archive_parser.go b/syft/pkg/cataloger/java/zip_wrapped_archive_parser.go index e5a7b6aec..54cc310b5 100644 --- a/syft/pkg/cataloger/java/zip_wrapped_archive_parser.go +++ b/syft/pkg/cataloger/java/zip_wrapped_archive_parser.go @@ -35,7 +35,7 @@ func (gzp genericZipWrappedJavaArchiveParser) parseZipWrappedJavaArchive(ctx con if td == nil { return nil, nil, fmt.Errorf("no temp dir factory in context") } - contentPath, archivePath, cleanupFn, err := saveArchiveToTmp(td, reader.Path(), reader) + contentPath, archivePath, cleanupFn, err := saveArchiveToTmp(td, reader.FullNestedPath(), reader) // note: even on error, we should always run cleanup functions defer cleanupFn() if err != nil { diff --git a/syft/pkg/cataloger/javascript/package.go b/syft/pkg/cataloger/javascript/package.go index 085f186dd..11ca8431f 100644 --- a/syft/pkg/cataloger/javascript/package.go +++ b/syft/pkg/cataloger/javascript/package.go @@ -276,7 +276,7 @@ func addLicenses(name string, resolver file.Resolver, location file.Location) (a return allLicenses } - dir := path.Dir(location.RealPath) + dir := path.Dir(location.LocalPath()) pkgPath := []string{dir, "node_modules"} pkgPath = append(pkgPath, strings.Split(name, "/")...) pkgPath = append(pkgPath, "package.json") @@ -308,7 +308,7 @@ func parseLicensesFromLocation(l file.Location, resolver file.Resolver, pkgFile log.Debugf("error getting file content reader for %s: %v", pkgFile, err) return nil, err } - defer internal.CloseAndLogError(contentReader, l.RealPath) + defer internal.CloseAndLogError(contentReader, l.FullNestedPath()) var pkgJSON packageJSON err = json.NewDecoder(contentReader).Decode(&pkgJSON) diff --git a/syft/pkg/cataloger/javascript/parse_package_lock.go b/syft/pkg/cataloger/javascript/parse_package_lock.go index 66c447d25..a8050c1a0 100644 --- a/syft/pkg/cataloger/javascript/parse_package_lock.go +++ b/syft/pkg/cataloger/javascript/parse_package_lock.go @@ -63,7 +63,7 @@ func newGenericPackageLockAdapter(cfg CatalogerConfig) genericPackageLockAdapter func (a genericPackageLockAdapter) parsePackageLock(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { // in the case we find package-lock.json files in the node_modules directories, skip those // as the whole purpose of the lock file is for the specific dependencies of the root project - if pathContainsNodeModulesDirectory(reader.Path()) { + if pathContainsNodeModulesDirectory(reader.LocalPath()) { return nil, nil, nil } diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock.go b/syft/pkg/cataloger/javascript/parse_yarn_lock.go index 7e65acf68..a10babf2b 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock.go @@ -93,20 +93,20 @@ func readPackageJSONDeps(resolver file.Resolver, lockfileLocation file.Location) pkgJSONLocation := resolver.RelativeFileByPath(lockfileLocation, "package.json") if pkgJSONLocation == nil { - log.WithFields("lockfile", lockfileLocation.RealPath).Debug("could not find package.json for dev dependency detection") + log.WithFields("lockfile", lockfileLocation.FullNestedPath()).Debug("could not find package.json for dev dependency detection") return prod, dev } reader, err := resolver.FileContentsByLocation(*pkgJSONLocation) if err != nil { - log.WithFields("location", pkgJSONLocation.RealPath, "error", err).Debug("could not read package.json for dev dependency detection") + log.WithFields("location", pkgJSONLocation.FullNestedPath(), "error", err).Debug("could not read package.json for dev dependency detection") return prod, dev } - defer internal.CloseAndLogError(reader, pkgJSONLocation.RealPath) + defer internal.CloseAndLogError(reader, pkgJSONLocation.FullNestedPath()) var pkgJSON packageJSON if err := json.NewDecoder(reader).Decode(&pkgJSON); err != nil { - log.WithFields("location", pkgJSONLocation.RealPath, "error", err).Debug("could not parse package.json for dev dependency detection") + log.WithFields("location", pkgJSONLocation.FullNestedPath(), "error", err).Debug("could not parse package.json for dev dependency detection") return prod, dev } @@ -288,7 +288,7 @@ func parseYarnLockYaml(reader io.ReadCloser) ([]yarnPackage, error) { func (a genericYarnLockAdapter) parseYarnLock(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { // in the case we find yarn.lock files in the node_modules directories, skip those // as the whole purpose of the lock file is for the specific dependencies of the project - if pathContainsNodeModulesDirectory(reader.Path()) { + if pathContainsNodeModulesDirectory(reader.LocalPath()) { return nil, nil, nil } diff --git a/syft/pkg/cataloger/kernel/parse_linux_kernel_module_file.go b/syft/pkg/cataloger/kernel/parse_linux_kernel_module_file.go index de92128a2..a323aba6a 100644 --- a/syft/pkg/cataloger/kernel/parse_linux_kernel_module_file.go +++ b/syft/pkg/cataloger/kernel/parse_linux_kernel_module_file.go @@ -34,9 +34,9 @@ func parseLinuxKernelModuleFile(ctx context.Context, _ file.Resolver, _ *generic return nil, nil, fmt.Errorf("unable to get union reader for file: %w", err) } - moduleReader, err := decompressedModuleReader(ctx, reader.RealPath, unionReader) + moduleReader, err := decompressedModuleReader(ctx, reader.FullNestedPath(), unionReader) if err != nil { - return nil, nil, fmt.Errorf("unable to decompress kernel module %q: %w", reader.RealPath, err) + return nil, nil, fmt.Errorf("unable to decompress kernel module %q: %w", reader.FullNestedPath(), err) } defer moduleReader.Close() @@ -48,7 +48,7 @@ func parseLinuxKernelModuleFile(ctx context.Context, _ file.Resolver, _ *generic return nil, nil, nil } - metadata.Path = reader.RealPath + metadata.Path = reader.LocalPath() return []pkg.Package{ newLinuxKernelModulePackage( diff --git a/syft/pkg/cataloger/lua/package.go b/syft/pkg/cataloger/lua/package.go index 78054ebc3..b36a376b7 100644 --- a/syft/pkg/cataloger/lua/package.go +++ b/syft/pkg/cataloger/lua/package.go @@ -13,7 +13,7 @@ import ( func newLuaRocksPackage(ctx context.Context, resolver file.Resolver, u luaRocksPackage, indexLocation file.Location) pkg.Package { license := pkg.NewLicensesFromLocationWithContext(ctx, indexLocation, u.License) if len(license) == 0 { - license = licenses.FindInDirs(ctx, resolver, path.Dir(indexLocation.Path())) + license = licenses.FindInDirs(ctx, resolver, path.Dir(indexLocation.LocalPath())) } p := pkg.Package{ Name: u.Name, diff --git a/syft/pkg/cataloger/lua/parse_rockspec.go b/syft/pkg/cataloger/lua/parse_rockspec.go index 82646aa67..847a476f7 100644 --- a/syft/pkg/cataloger/lua/parse_rockspec.go +++ b/syft/pkg/cataloger/lua/parse_rockspec.go @@ -75,7 +75,7 @@ func parseRockspec(ctx context.Context, resolver file.Resolver, _ *generic.Envir } if name == "" { - log.WithFields("path", reader.Path()).Trace("rockspec has no package name, skipping") + log.WithFields("path", reader.FullNestedPath()).Trace("rockspec has no package name, skipping") return nil, nil, nil } diff --git a/syft/pkg/cataloger/nix/db_cataloger.go b/syft/pkg/cataloger/nix/db_cataloger.go index 825895b69..09f9581c7 100644 --- a/syft/pkg/cataloger/nix/db_cataloger.go +++ b/syft/pkg/cataloger/nix/db_cataloger.go @@ -62,7 +62,7 @@ func (c dbCataloger) catalog(ctx context.Context, resolver file.Resolver) ([]pkg for _, dbLoc := range dbLocs { parser, schema := c.selectDBParser(dbLoc, resolver) if parser == nil { - errs = unknown.Appendf(errs, dbLoc.Coordinates, "unsupported Nix database schema for schema=%d at %q", schema, dbLoc.RealPath) + errs = unknown.Appendf(errs, dbLoc.Coordinates, "unsupported Nix database schema for schema=%d at %q", schema, dbLoc.FullNestedPath()) continue } @@ -80,28 +80,28 @@ func (c dbCataloger) catalog(ctx context.Context, resolver file.Resolver) ([]pkg } func (c dbCataloger) selectDBParser(dbLocation file.Location, resolver file.Resolver) (dbProcessor, int) { - loc := resolver.RelativeFileByPath(dbLocation, path.Join(path.Dir(dbLocation.RealPath), "schema")) + loc := resolver.RelativeFileByPath(dbLocation, path.Join(path.Dir(dbLocation.LocalPath()), "schema")) if loc == nil { - log.WithFields("path", dbLocation.RealPath).Tracef("failed to detect Nix database schema, assuming %d", defaultSchema) + log.WithFields("path", dbLocation.FullNestedPath()).Tracef("failed to detect Nix database schema, assuming %d", defaultSchema) return c.schemaProcessor[defaultSchema], 0 } schemaContents, err := resolver.FileContentsByLocation(*loc) - defer internal.CloseAndLogError(schemaContents, loc.RealPath) + defer internal.CloseAndLogError(schemaContents, loc.FullNestedPath()) if err != nil { - log.WithFields("path", loc.RealPath).Tracef("failed to open Nix database schema file, assuming %d", defaultSchema) + log.WithFields("path", loc.FullNestedPath()).Tracef("failed to open Nix database schema file, assuming %d", defaultSchema) return c.schemaProcessor[defaultSchema], 0 } scanner := bufio.NewScanner(schemaContents) if !scanner.Scan() { - log.WithFields("path", loc.RealPath, "error", scanner.Err()).Tracef("failed to read Nix database schema file, assuming %d", defaultSchema) + log.WithFields("path", loc.FullNestedPath(), "error", scanner.Err()).Tracef("failed to read Nix database schema file, assuming %d", defaultSchema) return c.schemaProcessor[defaultSchema], 0 } schema, err := strconv.Atoi(strings.TrimSpace(scanner.Text())) if err != nil { - log.WithFields("path", loc.RealPath).Tracef("failed to parse Nix database schema file, assuming %d", defaultSchema) + log.WithFields("path", loc.FullNestedPath()).Tracef("failed to parse Nix database schema file, assuming %d", defaultSchema) return c.schemaProcessor[defaultSchema], 0 } @@ -113,7 +113,7 @@ func (c dbCataloger) selectDBParser(dbLocation file.Location, resolver file.Reso schema = defaultSchema } processor = c.schemaProcessor[closestSchema] - log.WithFields("path", loc.RealPath).Tracef("unsupported Nix database schema (%d), treating as closest available schema (%d)", schema, closestSchema) + log.WithFields("path", loc.FullNestedPath()).Tracef("unsupported Nix database schema (%d), treating as closest available schema (%d)", schema, closestSchema) } return processor, schema diff --git a/syft/pkg/cataloger/nix/db_cataloger_v10.go b/syft/pkg/cataloger/nix/db_cataloger_v10.go index 849faffa4..3d8850150 100644 --- a/syft/pkg/cataloger/nix/db_cataloger_v10.go +++ b/syft/pkg/cataloger/nix/db_cataloger_v10.go @@ -19,7 +19,7 @@ var _ dbProcessor = processV10DB func processV10DB(ctx context.Context, config Config, dbLocation file.Location, resolver file.Resolver, catalogerName string) ([]pkg.Package, []artifact.Relationship, error) { dbContents, err := resolver.FileContentsByLocation(dbLocation) - defer internal.CloseAndLogError(dbContents, dbLocation.RealPath) + defer internal.CloseAndLogError(dbContents, dbLocation.FullNestedPath()) if err != nil { return nil, nil, fmt.Errorf("unable to read Nix database: %w", err) } diff --git a/syft/pkg/cataloger/nix/derivation.go b/syft/pkg/cataloger/nix/derivation.go index 97d163c9a..12b740497 100644 --- a/syft/pkg/cataloger/nix/derivation.go +++ b/syft/pkg/cataloger/nix/derivation.go @@ -34,7 +34,7 @@ func newDerivationFromLocation(loc file.Location, resolver file.Resolver) (*deri if err != nil { return nil, fmt.Errorf("failed to read derivation: %w", err) } - defer internal.CloseAndLogError(reader, loc.RealPath) + defer internal.CloseAndLogError(reader, loc.FullNestedPath()) d, err := derivation.ReadDerivation(reader) if err != nil { @@ -60,12 +60,12 @@ func newDerivations() *derivations { } func (c *derivations) add(df derivationFile) { - c.derivationsByDrvPath[df.Location.RealPath] = df + c.derivationsByDrvPath[df.Location.LocalPath()] = df for _, output := range df.Outputs { if output == nil || output.Path == "" { continue } - c.drvPathByOutputPath[output.Path] = df.Location.RealPath + c.drvPathByOutputPath[output.Path] = df.Location.LocalPath() } } diff --git a/syft/pkg/cataloger/nix/package.go b/syft/pkg/cataloger/nix/package.go index 36e8294ed..b214383a5 100644 --- a/syft/pkg/cataloger/nix/package.go +++ b/syft/pkg/cataloger/nix/package.go @@ -21,7 +21,7 @@ func newNixStorePackage(pp nixStorePackage, catalogerName string) pkg.Package { var derivationPath string if pp.derivationFile != nil { - derivationPath = pp.derivationFile.Location.RealPath + derivationPath = pp.derivationFile.Location.LocalPath() locations.Add(pp.derivationFile.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.SupportingEvidenceAnnotation)) } @@ -97,7 +97,7 @@ func newDerivation(df *derivationFile) pkg.NixDerivation { sort.Strings(sources) return pkg.NixDerivation{ - Path: df.Location.RealPath, + Path: df.Location.LocalPath(), System: df.Platform, InputDerivations: inputDerivations, InputSources: sources, diff --git a/syft/pkg/cataloger/nix/store_cataloger.go b/syft/pkg/cataloger/nix/store_cataloger.go index e239330ef..c212b9ffd 100644 --- a/syft/pkg/cataloger/nix/store_cataloger.go +++ b/syft/pkg/cataloger/nix/store_cataloger.go @@ -75,7 +75,7 @@ func (c storeCataloger) finalizeStorePackages(ctx context.Context, resolver file p := newNixStorePackage(pp, c.name) p = licenses.RelativeToPackage(ctx, resolver, p) pkgs = append(pkgs, p) - pkgByStorePath[pp.Location.RealPath] = p + pkgByStorePath[pp.Location.LocalPath()] = p } var relationships []artifact.Relationship @@ -117,7 +117,7 @@ func (c storeCataloger) findDerivationsFromStore(resolver file.Resolver, pkgProt // attach derivations to the packages they belong to for i := range pkgPrototypes { p := &pkgPrototypes[i] - p.derivationFile = dvs.findDerivationForOutputPath(p.Location.RealPath) + p.derivationFile = dvs.findDerivationForOutputPath(p.Location.LocalPath()) } return dvs, errs @@ -130,16 +130,16 @@ func (c storeCataloger) findPackagesFromStore(ctx context.Context, resolver file ctx, cancel := context.WithCancel(ctx) defer cancel() for location := range resolver.AllLocations(ctx) { - matchesStorePath, err := doublestar.Match("**/nix/store/*", location.RealPath) + matchesStorePath, err := doublestar.Match("**/nix/store/*", location.LocalPath()) if err != nil { return nil, fmt.Errorf("failed to match nix store path: %w", err) } - parentStorePath := findParentNixStorePath(location.RealPath) + parentStorePath := findParentNixStorePath(location.LocalPath()) if c.config.CaptureOwnedFiles && parentStorePath != "" { fileInfo, err := resolver.FileMetadataByLocation(location) if err != nil { - log.WithFields("path", location.RealPath).Trace("failed to get file metadata") + log.WithFields("path", location.FullNestedPath()).Trace("failed to get file metadata") continue } @@ -159,7 +159,7 @@ func (c storeCataloger) findPackagesFromStore(ctx context.Context, resolver file continue } - storePath := parseNixStorePath(location.RealPath) + storePath := parseNixStorePath(location.LocalPath()) if storePath == nil || !storePath.isValidPackage() { continue @@ -178,7 +178,7 @@ func (c storeCataloger) findPackagesFromStore(ctx context.Context, resolver file log.WithFields("package", p.nixStorePath.Name).Debug("nix package has no evidence locations associated") continue } - parentStorePath := p.Location.RealPath + parentStorePath := p.Location.LocalPath() files, ok := filesByStorePath[parentStorePath] if !ok { log.WithFields("path", parentStorePath, "nix-store-path", parentStorePath).Debug("found a nix store file for a non-existent package") @@ -193,7 +193,7 @@ func (c storeCataloger) findPackagesFromStore(ctx context.Context, resolver file func filePaths(files []file.Location) []string { var relativePaths []string for _, f := range files { - relativePaths = append(relativePaths, f.RealPath) + relativePaths = append(relativePaths, f.LocalPath()) } return relativePaths } diff --git a/syft/pkg/cataloger/perl/meta.go b/syft/pkg/cataloger/perl/meta.go index a4ca5f5bd..1be6d21fe 100644 --- a/syft/pkg/cataloger/perl/meta.go +++ b/syft/pkg/cataloger/perl/meta.go @@ -107,7 +107,7 @@ func readMyMeta(resolver file.Resolver, installJSONLocation file.Location) (*fil return nil, nil } - p := path.Join(path.Dir(installJSONLocation.Path()), "MYMETA.json") + p := path.Join(path.Dir(installJSONLocation.LocalPath()), "MYMETA.json") locations, err := resolver.FilesByPath(p) if err != nil || len(locations) == 0 { @@ -145,7 +145,7 @@ func decode(resolver file.Resolver, location file.Location, into func(io.Reader, if err != nil { return nil, err } - defer internal.CloseAndLogError(reader, location.Path()) + defer internal.CloseAndLogError(reader, location.FullNestedPath()) var m cpanMeta if err := into(reader, &m); err != nil { diff --git a/syft/pkg/cataloger/perl/parse_install_json.go b/syft/pkg/cataloger/perl/parse_install_json.go index 15e5e2abb..937597f34 100644 --- a/syft/pkg/cataloger/perl/parse_install_json.go +++ b/syft/pkg/cataloger/perl/parse_install_json.go @@ -28,13 +28,13 @@ type installJSON struct { func parseInstallJSON(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { var doc installJSON if err := json.NewDecoder(reader).Decode(&doc); err != nil { - log.WithFields("path", reader.Path(), "error", err).Debug("unable to parse CPAN install.json") + log.WithFields("path", reader.FullNestedPath(), "error", err).Debug("unable to parse CPAN install.json") return nil, nil, nil } name, version := distributionNameVersion(doc) if name == "" || version == "" { - log.WithFields("path", reader.Path()).Debug("CPAN install.json is missing a distribution name or version") + log.WithFields("path", reader.FullNestedPath()).Debug("CPAN install.json is missing a distribution name or version") return nil, nil, nil } diff --git a/syft/pkg/cataloger/perl/parse_packlist.go b/syft/pkg/cataloger/perl/parse_packlist.go index 22cb50240..a84bedfb4 100644 --- a/syft/pkg/cataloger/perl/parse_packlist.go +++ b/syft/pkg/cataloger/perl/parse_packlist.go @@ -40,7 +40,7 @@ var packlistMetadataSuffix = regexp.MustCompile(` \w+=.*$`) // parsePacklist catalogs a distribution installed by any ExtUtils::MakeMaker or Module::Build run, // which covers CPAN.pm (which writes no .meta) and the perl image's own preinstalled distributions. func parsePacklist(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - name := distNameFromPacklistPath(reader.Path()) + name := distNameFromPacklistPath(reader.LocalPath()) if name == "" { // the perl core packlist has no auto/ segment; it lists thousands of files and describes // no single distribution @@ -124,7 +124,7 @@ func readVersionFromPM(resolver file.Resolver, location file.Location) string { if err != nil { return "" } - defer internal.CloseAndLogError(reader, location.Path()) + defer internal.CloseAndLogError(reader, location.FullNestedPath()) return versionFromPM(reader) } diff --git a/syft/pkg/cataloger/perl/parse_release_meta.go b/syft/pkg/cataloger/perl/parse_release_meta.go index 42224a842..643bf67c2 100644 --- a/syft/pkg/cataloger/perl/parse_release_meta.go +++ b/syft/pkg/cataloger/perl/parse_release_meta.go @@ -20,7 +20,7 @@ import ( // 41,008 a MANIFEST (98%). Reading only META.json skips roughly 43% of current releases, skewed toward the // old ones where the advisories concentrate. func parseReleaseMeta(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - if isBuildLeftover(reader.Path()) || !isUnpackedRelease(resolver, reader.Location) { + if isBuildLeftover(reader.LocalPath()) || !isUnpackedRelease(resolver, reader.Location) { return nil, nil, nil } @@ -42,12 +42,12 @@ func parseReleaseMeta(ctx context.Context, resolver file.Resolver, _ *generic.En if err != nil { // a META.yml a strict parser rejects is routine rather than exceptional, so this skips the // directory and never fails the scan - log.WithFields("path", reader.Path(), "error", err).Debug("unable to parse CPAN meta file") + log.WithFields("path", reader.FullNestedPath(), "error", err).Debug("unable to parse CPAN meta file") return nil, nil, nil } if m.Name == "" || m.Version == "" { - log.WithFields("path", reader.Path()).Debug("CPAN meta file is missing a name or version") + log.WithFields("path", reader.FullNestedPath()).Debug("CPAN meta file is missing a name or version") return nil, nil, nil } @@ -64,7 +64,7 @@ func parseReleaseMeta(ctx context.Context, resolver file.Resolver, _ *generic.En // trusted for version. A sibling MANIFEST is what tells the two apart: release tarballs ship one, while a // source checkout does not, since the MANIFEST is generated at build time. func isUnpackedRelease(resolver file.Resolver, metaLocation file.Location) bool { - _, ok := findFile(resolver, path.Join(path.Dir(metaLocation.Path()), "MANIFEST")) + _, ok := findFile(resolver, path.Join(path.Dir(metaLocation.LocalPath()), "MANIFEST")) return ok } @@ -94,7 +94,7 @@ func isBuildLeftover(p string) bool { } func hasSiblingMetaJSON(resolver file.Resolver, metaLocation file.Location) bool { - _, ok := findFile(resolver, path.Join(path.Dir(metaLocation.Path()), "META.json")) + _, ok := findFile(resolver, path.Join(path.Dir(metaLocation.LocalPath()), "META.json")) return ok } diff --git a/syft/pkg/cataloger/perl/perllocal.go b/syft/pkg/cataloger/perl/perllocal.go index 21d892bc4..149785e58 100644 --- a/syft/pkg/cataloger/perl/perllocal.go +++ b/syft/pkg/cataloger/perl/perllocal.go @@ -176,10 +176,10 @@ func perllocalLocations(resolver file.Resolver) []file.Location { func readPerllocal(resolver file.Resolver, location file.Location) []perllocalEntry { reader, err := resolver.FileContentsByLocation(location) if err != nil { - log.WithFields("path", location.Path(), "error", err).Debug("unable to read perllocal.pod") + log.WithFields("path", location.FullNestedPath(), "error", err).Debug("unable to read perllocal.pod") return nil } - defer internal.CloseAndLogError(reader, location.Path()) + defer internal.CloseAndLogError(reader, location.FullNestedPath()) return parsePerllocal(reader) } @@ -217,7 +217,7 @@ func parsePerllocal(r io.Reader) []perllocalEntry { func primaryPath(p pkg.Package) string { for _, location := range p.Locations.ToSlice() { if location.Annotations[pkg.EvidenceAnnotationKey] == pkg.PrimaryEvidenceAnnotation { - return location.Path() + return location.LocalPath() } } return "" diff --git a/syft/pkg/cataloger/perl/perllocal_test.go b/syft/pkg/cataloger/perl/perllocal_test.go index ebf159a34..93f5c399e 100644 --- a/syft/pkg/cataloger/perl/perllocal_test.go +++ b/syft/pkg/cataloger/perl/perllocal_test.go @@ -164,7 +164,7 @@ func Test_perllocalVersion(t *testing.T) { return } require.NotNil(t, evidence) - assert.Equal(t, "/usr/local/lib/perl5/5.40.4/aarch64-linux-gnu/perllocal.pod", evidence.Path()) + assert.Equal(t, "/usr/local/lib/perl5/5.40.4/aarch64-linux-gnu/perllocal.pod", evidence.FullNestedPath()) }) } } diff --git a/syft/pkg/cataloger/perl/processor.go b/syft/pkg/cataloger/perl/processor.go index ee871e516..068ed7c08 100644 --- a/syft/pkg/cataloger/perl/processor.go +++ b/syft/pkg/cataloger/perl/processor.go @@ -212,7 +212,7 @@ func providedModules(p pkg.Package) []string { // belong in relationships and not in the metadata struct. func prereqModules(resolver file.Resolver, p pkg.Package) []string { for _, location := range p.Locations.ToSlice() { - if path.Base(location.Path()) != "MYMETA.json" { + if path.Base(location.FullNestedPath()) != "MYMETA.json" { continue } m, err := decodeMeta(resolver, location) diff --git a/syft/pkg/cataloger/php/interpreter_cataloger.go b/syft/pkg/cataloger/php/interpreter_cataloger.go index f9022d358..016db66b4 100644 --- a/syft/pkg/cataloger/php/interpreter_cataloger.go +++ b/syft/pkg/cataloger/php/interpreter_cataloger.go @@ -182,12 +182,12 @@ func (p interpreterCataloger) catalogExtensions(resolver file.Resolver) ([]pkg.P func (p interpreterCataloger) catalogExtension(resolver file.Resolver, location file.Location) ([]pkg.Package, error) { reader, err := resolver.FileContentsByLocation(location) - defer internal.CloseAndLogError(reader, location.RealPath) if err != nil { - return nil, unknown.ProcessPathErrors(err) + return nil, unknown.New(location, err) } + defer internal.CloseAndLogError(reader, location.FullNestedPath()) - name, cls := p.getClassifier(location.RealPath) + name, cls := p.getClassifier(location.FullNestedPath()) if name == "" || cls == nil { return nil, nil } diff --git a/syft/pkg/cataloger/php/interpreter_cataloger_test.go b/syft/pkg/cataloger/php/interpreter_cataloger_test.go index bff331d5c..5fa5d06e1 100644 --- a/syft/pkg/cataloger/php/interpreter_cataloger_test.go +++ b/syft/pkg/cataloger/php/interpreter_cataloger_test.go @@ -6,7 +6,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/cpe" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" ) @@ -164,3 +166,17 @@ func Test_getClassifier_declaredCPESource(t *testing.T) { require.Len(t, cls.CPEs, 1) assert.Equal(t, cpe.DeclaredSource, cls.CPEs[0].Source) } + +func Test_catalogExtension_unreadableFileKeepsItsCoordinates(t *testing.T) { + nested := file.NewLocationFromCoordinates(file.Coordinates{ + RealPath: "/app.tar", + FullPath: "/app.tar!/usr/lib/php83/modules/bcmath.so", + }) + c := NewInterpreterCataloger().(interpreterCataloger) + + _, err := c.catalogExtension(file.NewMockResolverForPaths(), nested) + + var coordinateErr *unknown.CoordinateError + require.ErrorAs(t, err, &coordinateErr) + assert.Equal(t, nested.Coordinates, coordinateErr.Coordinates) +} diff --git a/syft/pkg/cataloger/python/dependency.go b/syft/pkg/cataloger/python/dependency.go index 29671ed59..336cd5921 100644 --- a/syft/pkg/cataloger/python/dependency.go +++ b/syft/pkg/cataloger/python/dependency.go @@ -287,7 +287,7 @@ func collectPackages(pkgsBySitePackageAndName map[string]map[string]pkg.Package, func deriveSitePackageDir(p pkg.Package) string { for _, l := range packagePrimaryLocations(p) { - sitePackageDir := extractSitePackageDir(l.RealPath) + sitePackageDir := extractSitePackageDir(l.LocalPath()) if sitePackageDir != "" { return sitePackageDir } diff --git a/syft/pkg/cataloger/python/parse_requirements.go b/syft/pkg/cataloger/python/parse_requirements.go index 6622496fd..ed3536d0f 100644 --- a/syft/pkg/cataloger/python/parse_requirements.go +++ b/syft/pkg/cataloger/python/parse_requirements.go @@ -133,7 +133,7 @@ func (rp requirementsParser) parseRequirementsTxt(ctx context.Context, _ file.Re req := newRequirement(line) if req == nil { - log.WithFields("path", reader.RealPath, "line", line).Debug("unable to parse requirements.txt line") + log.WithFields("path", reader.FullNestedPath(), "line", line).Debug("unable to parse requirements.txt line") errs = unknown.Appendf(errs, reader, "unable to parse requirements.txt line: %q", line) continue } @@ -142,7 +142,7 @@ func (rp requirementsParser) parseRequirementsTxt(ctx context.Context, _ file.Re version := parseVersion(req.VersionConstraint, rp.cfg.GuessUnpinnedRequirements) if version == "" { - log.WithFields("path", reader.RealPath, "line", line).Trace("unable to determine package version in requirements.txt line") + log.WithFields("path", reader.FullNestedPath(), "line", line).Trace("unable to determine package version in requirements.txt line") errs = unknown.Appendf(errs, reader, "unable to determine package version in requirements.txt line: %q", line) continue } diff --git a/syft/pkg/cataloger/python/parse_setup.go b/syft/pkg/cataloger/python/parse_setup.go index d8cff06de..b487e7de3 100644 --- a/syft/pkg/cataloger/python/parse_setup.go +++ b/syft/pkg/cataloger/python/parse_setup.go @@ -108,7 +108,7 @@ func (sp setupFileParser) validateAndCreatePackage(ctx context.Context, name, ve if name == "" || version == "" { // the raw line is not logged since this runs once per match, which is quadratic on a long line - log.WithFields("path", reader.RealPath, "name", name, "version", version).Debug("unable to parse package in setup.py") + log.WithFields("path", reader.FullNestedPath(), "name", name, "version", version).Debug("unable to parse package in setup.py") return pkg.Package{}, false } diff --git a/syft/pkg/cataloger/python/parse_wheel_egg.go b/syft/pkg/cataloger/python/parse_wheel_egg.go index 33d2d0161..b33ed693a 100644 --- a/syft/pkg/cataloger/python/parse_wheel_egg.go +++ b/syft/pkg/cataloger/python/parse_wheel_egg.go @@ -52,7 +52,7 @@ func fetchInstalledFiles(resolver file.Resolver, metadataLocation file.Location, // to reconcile the installed-files.txt path to the same layer (or the next adjacent lower layer). // find the installed-files.txt file relative to the directory where the METADATA file resides (in path AND layer structure) - installedFilesPath := path.Join(path.Dir(metadataLocation.RealPath), "installed-files.txt") + installedFilesPath := path.Join(path.Dir(metadataLocation.LocalPath()), "installed-files.txt") installedFilesRef := resolver.RelativeFileByPath(metadataLocation, installedFilesPath) if installedFilesRef != nil { @@ -62,10 +62,10 @@ func fetchInstalledFiles(resolver file.Resolver, metadataLocation file.Location, if err != nil { return nil, nil, err } - defer internal.CloseAndLogError(installedFilesContents, installedFilesPath) + defer internal.CloseAndLogError(installedFilesContents, installedFilesRef.FullNestedPath()) // parse the installed-files contents - installedFiles, err := parseInstalledFiles(installedFilesContents, metadataLocation.RealPath, sitePackagesRootPath) + installedFiles, err := parseInstalledFiles(installedFilesContents, metadataLocation.LocalPath(), sitePackagesRootPath) if err != nil { retErr = unknown.Newf(*installedFilesRef, "unable to parse installed-files.txt for python package: %w", retErr) } @@ -82,7 +82,7 @@ func fetchRecordFiles(resolver file.Resolver, metadataLocation file.Location) (f // to reconcile the RECORD path to the same layer (or the next adjacent lower layer). // find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure) - recordPath := path.Join(path.Dir(metadataLocation.RealPath), "RECORD") + recordPath := path.Join(path.Dir(metadataLocation.LocalPath()), "RECORD") recordRef := resolver.RelativeFileByPath(metadataLocation, recordPath) if recordRef != nil { @@ -92,7 +92,7 @@ func fetchRecordFiles(resolver file.Resolver, metadataLocation file.Location) (f if err != nil { return nil, nil, err } - defer internal.CloseAndLogError(recordContents, recordPath) + defer internal.CloseAndLogError(recordContents, recordRef.FullNestedPath()) // parse the record contents var records []pkg.PythonFileRecord @@ -106,7 +106,7 @@ func fetchRecordFiles(resolver file.Resolver, metadataLocation file.Location) (f // fetchTopLevelPackages finds a corresponding top_level.txt file for the given python package metadata file and returns the set of package names contained. func fetchTopLevelPackages(resolver file.Resolver, metadataLocation file.Location) (pkgs []string, sources []file.Location, err error) { // a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages - parentDir := path.Dir(metadataLocation.RealPath) + parentDir := path.Dir(metadataLocation.LocalPath()) topLevelPath := path.Join(parentDir, "top_level.txt") topLevelLocation := resolver.RelativeFileByPath(metadataLocation, topLevelPath) @@ -120,7 +120,7 @@ func fetchTopLevelPackages(resolver file.Resolver, metadataLocation file.Locatio if err != nil { return nil, nil, err } - defer internal.CloseAndLogError(topLevelContents, topLevelLocation.AccessPath) + defer internal.CloseAndLogError(topLevelContents, topLevelLocation.FullNestedPath()) scanner := bufio.NewScanner(topLevelContents) for scanner.Scan() { @@ -156,7 +156,7 @@ type vcsInfo struct { } func fetchDirectURLData(resolver file.Resolver, metadataLocation file.Location) (d *pkg.PythonDirectURLOriginInfo, sources []file.Location, err error) { - parentDir := path.Dir(metadataLocation.RealPath) + parentDir := path.Dir(metadataLocation.LocalPath()) directURLPath := path.Join(parentDir, "direct_url.json") directURLLocation := resolver.RelativeFileByPath(metadataLocation, directURLPath) @@ -170,7 +170,7 @@ func fetchDirectURLData(resolver file.Resolver, metadataLocation file.Location) if err != nil { return nil, nil, err } - defer internal.CloseAndLogError(directURLContents, directURLLocation.AccessPath) + defer internal.CloseAndLogError(directURLContents, directURLLocation.FullNestedPath()) var directURLJson directURLOrigin if err := json.NewDecoder(directURLContents).Decode(&directURLJson); err != nil { @@ -194,7 +194,7 @@ func assembleEggOrWheelMetadata(resolver file.Resolver, metadataLocation file.Lo if err != nil { return nil, nil, err } - defer internal.CloseAndLogError(metadataContents, metadataLocation.AccessPath) + defer internal.CloseAndLogError(metadataContents, metadataLocation.FullNestedPath()) pd, err := parseWheelOrEggMetadata(file.NewLocationReadCloser(metadataLocation, metadataContents)) if err != nil { @@ -261,7 +261,7 @@ func findLicenses(ctx context.Context, resolver file.Resolver, m parsedData) pkg default: // search for known license paths from RECORDS file - parent := path.Base(path.Dir(m.DistInfoLocation.Path())) + parent := path.Base(path.Dir(m.DistInfoLocation.LocalPath())) candidatePaths := strset.New() for _, f := range m.Files { if !strings.HasPrefix(f.Path, parent) || strings.Count(f.Path, "/") > 1 { diff --git a/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go b/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go index 05c49f5f4..35874ebff 100644 --- a/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go +++ b/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go @@ -48,7 +48,7 @@ func parseWheelOrEggMetadata(locationReader file.LocationReadCloser) (parsedData } // add additional metadata not stored in the egg/wheel metadata file - metadataPath := locationReader.Path() + metadataPath := locationReader.LocalPath() pd.SitePackagesRootPath = determineSitePackagesRootPath(metadataPath) if pd.Licenses != "" || pd.LicenseExpression != "" { @@ -104,13 +104,13 @@ func extractRFC5322Fields(locationReader file.LocationReadCloser) (map[string]an fields[key] = handleSingleOrMultiField(fields[key], val) } else { - log.Debugf("cannot parse field from path: %q from line: %q", locationReader.Path(), line) + log.Debugf("cannot parse field from path: %q from line: %q", locationReader.FullNestedPath(), line) } } } if err := scanner.Err(); err != nil { // the caller discards everything on error, so keep the partial fields and log instead - log.WithFields("path", locationReader.Path(), "error", err).Debug("failed to fully read python wheel/egg metadata") + log.WithFields("path", locationReader.FullNestedPath(), "error", err).Debug("failed to fully read python wheel/egg metadata") } return fields, nil } diff --git a/syft/pkg/cataloger/python/virtual_env.go b/syft/pkg/cataloger/python/virtual_env.go index f6b48b602..d7d5b083c 100644 --- a/syft/pkg/cataloger/python/virtual_env.go +++ b/syft/pkg/cataloger/python/virtual_env.go @@ -97,7 +97,7 @@ func (v virtualEnvInfo) matchVirtualEnvSitePackagesPath(sitePackagePaths []strin sitePackagePathsSet := strset.New(sitePackagePaths...) // the parent directory of the venv config is the top-level directory of the virtualenv // e.g. /app/project1/venv/pyvenv.cfg -> /app/project1/venv - parent := strings.TrimLeft(path.Dir(v.Location.RealPath), "/") + parent := strings.TrimLeft(path.Dir(v.Location.LocalPath()), "/") // we are searchin for the site-packages directory within the virtualenv search := parent + "/lib/python" + v.majorMinorVersion() + "/site-packages" @@ -120,9 +120,9 @@ func (v virtualEnvInfo) matchVirtualEnvSitePackagesPath(sitePackagePaths []strin func parsePyvenvCfg(_ context.Context, resolver file.Resolver, location file.Location) (*virtualEnvInfo, error) { reader, err := resolver.FileContentsByLocation(location) if err != nil { - return nil, fmt.Errorf("unable to read file %q: %w", location.Path(), err) + return nil, fmt.Errorf("unable to read file %q: %w", location.FullNestedPath(), err) } - defer internal.CloseAndLogError(reader, location.Path()) + defer internal.CloseAndLogError(reader, location.FullNestedPath()) cfg, err := parsePyvenvCfgReader(file.NewLocationReadCloser(location, reader)) if err != nil { diff --git a/syft/pkg/cataloger/redhat/parse_rpm_archive.go b/syft/pkg/cataloger/redhat/parse_rpm_archive.go index 5013825b8..9215b305e 100644 --- a/syft/pkg/cataloger/redhat/parse_rpm_archive.go +++ b/syft/pkg/cataloger/redhat/parse_rpm_archive.go @@ -60,7 +60,7 @@ var hashLookup = map[uint8]string{ func parseRpmArchive(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { rpm, err := rpmutils.ReadRpm(reader) if err != nil { - return nil, nil, fmt.Errorf("RPM file found but unable to read: %s (%w)", reader.RealPath, err) + return nil, nil, fmt.Errorf("RPM file found but unable to read: %s (%w)", reader.FullNestedPath(), err) } nevra, err := rpm.Header.GetNEVRA() @@ -269,6 +269,6 @@ func parseEpoch(epoch string) *int { func logRpmArchiveErr(location file.Location, operation string, err error) { if err != nil { - log.WithFields("error", err, "operation", operation, "path", location.RealPath).Trace("unable to parse RPM archive") + log.WithFields("error", err, "operation", operation, "path", location.FullNestedPath()).Trace("unable to parse RPM archive") } } diff --git a/syft/pkg/cataloger/redhat/parse_rpm_db.go b/syft/pkg/cataloger/redhat/parse_rpm_db.go index 22da431b5..971911aeb 100644 --- a/syft/pkg/cataloger/redhat/parse_rpm_db.go +++ b/syft/pkg/cataloger/redhat/parse_rpm_db.go @@ -69,7 +69,7 @@ func parseRpmDB(ctx context.Context, resolver file.Resolver, env *generic.Enviro // This appears to be a holdover, where nowadays only the RSA Header is used. sigs, err := parseSignatures(strings.TrimSpace(entry.PGP), strings.TrimSpace(entry.RSAHeader)) if err != nil { - log.WithFields("error", err, "location", reader.RealPath, "pkg", fmt.Sprintf("%s@%s", entry.Name, entry.Version)).Trace("unable to parse signatures for package %s", entry.Name) + log.WithFields("error", err, "location", reader.FullNestedPath(), "pkg", fmt.Sprintf("%s@%s", entry.Name, entry.Version)).Trace("unable to parse signatures for package %s", entry.Name) sigs = nil } @@ -98,7 +98,7 @@ func parseRpmDB(ctx context.Context, resolver file.Resolver, env *generic.Enviro ) if !pkg.IsValid(&p) { - log.WithFields("location", reader.RealPath, "pkg", fmt.Sprintf("%s@%s", entry.Name, entry.Version)). + log.WithFields("location", reader.FullNestedPath(), "pkg", fmt.Sprintf("%s@%s", entry.Name, entry.Version)). Warn("ignoring invalid package found in RPM DB") errs = unknown.Appendf(errs, reader, "invalild package found; name: %s, version: %s", entry.Name, entry.Version) continue diff --git a/syft/pkg/cataloger/rust/parse_audit_binary.go b/syft/pkg/cataloger/rust/parse_audit_binary.go index e93bae25a..9d1357624 100644 --- a/syft/pkg/cataloger/rust/parse_audit_binary.go +++ b/syft/pkg/cataloger/rust/parse_audit_binary.go @@ -26,7 +26,7 @@ func parseAuditBinary(_ context.Context, _ file.Resolver, _ *generic.Environment return nil, nil, err } - infos, err := parseAuditBinaryEntry(unionReader, reader.RealPath) + infos, err := parseAuditBinaryEntry(unionReader, reader.FullNestedPath()) for _, versionInfo := range infos { auditPkgs, auditRelationships := processAuditVersionInfo(reader.Location, versionInfo) pkgs = append(pkgs, auditPkgs...) diff --git a/syft/pkg/cataloger/sbom/cataloger.go b/syft/pkg/cataloger/sbom/cataloger.go index 0ea8483cd..17a036895 100644 --- a/syft/pkg/cataloger/sbom/cataloger.go +++ b/syft/pkg/cataloger/sbom/cataloger.go @@ -15,6 +15,7 @@ import ( "github.com/anchore/syft/syft/format" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/pkg/cataloger/internal/embeddedsbom" ) const catalogerName = "sbom-cataloger" @@ -40,7 +41,7 @@ func NewCataloger() pkg.Cataloger { func parseSBOM(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { readSeeker, err := adaptToReadSeeker(reader) if err != nil { - return nil, nil, fmt.Errorf("unable to read SBOM file %q: %w", reader.RealPath, err) + return nil, nil, fmt.Errorf("unable to read SBOM file %q: %w", reader.FullNestedPath(), err) } s, _, _, err := format.Decode(readSeeker) if err != nil { @@ -48,12 +49,12 @@ func parseSBOM(_ context.Context, _ file.Resolver, _ *generic.Environment, reade } if s == nil { - log.WithFields("path", reader.RealPath).Trace("file is not an SBOM") + log.WithFields("path", reader.FullNestedPath()).Trace("file is not an SBOM") return nil, nil, nil } var pkgs []pkg.Package - relationships := s.Relationships + relationships := embeddedsbom.Relationships(s.Relationships) for _, p := range s.Artifacts.Packages.Sorted() { // replace all locations on the package with the location of the SBOM file. // Why not keep the original list of locations? Since the "locations" field is meant to capture diff --git a/syft/pkg/cataloger/sbom/cataloger_test.go b/syft/pkg/cataloger/sbom/cataloger_test.go index 7617e9e9b..620d2e711 100644 --- a/syft/pkg/cataloger/sbom/cataloger_test.go +++ b/syft/pkg/cataloger/sbom/cataloger_test.go @@ -259,10 +259,7 @@ func Test_parseSBOM(t *testing.T) { apkgdbLocation := file.NewLocationSet(file.Location{ LocationData: file.LocationData{ - Coordinates: file.Coordinates{ - RealPath: "/lib/apk/db/installed", - FileSystemID: "sha256:e5e13b0c77cbb769548077189c3da2f0a764ceca06af49d8d558e759f5c232bd", - }, + Coordinates: file.NewCoordinates("/lib/apk/db/installed", "sha256:e5e13b0c77cbb769548077189c3da2f0a764ceca06af49d8d558e759f5c232bd"), }, }) @@ -349,10 +346,7 @@ func Test_parseSBOM(t *testing.T) { }, { From: libSSL, - To: file.Coordinates{ - RealPath: "/lib/libssl.so.1.1", - FileSystemID: "sha256:e5e13b0c77cbb769548077189c3da2f0a764ceca06af49d8d558e759f5c232bd", - }, + To: file.NewCoordinates("/lib/libssl.so.1.1", "sha256:e5e13b0c77cbb769548077189c3da2f0a764ceca06af49d8d558e759f5c232bd"), Type: artifact.ContainsRelationship, }, { @@ -362,18 +356,12 @@ func Test_parseSBOM(t *testing.T) { }, { From: baseLayout, - To: file.Coordinates{ - RealPath: "/etc/profile.d/color_prompt.sh.disabled", - FileSystemID: "sha256:e5e13b0c77cbb769548077189c3da2f0a764ceca06af49d8d558e759f5c232bd", - }, + To: file.NewCoordinates("/etc/profile.d/color_prompt.sh.disabled", "sha256:e5e13b0c77cbb769548077189c3da2f0a764ceca06af49d8d558e759f5c232bd"), Type: artifact.ContainsRelationship, }, { From: baseLayout, - To: file.Coordinates{ - RealPath: "/etc/modprobe.d/kms.conf", - FileSystemID: "sha256:e5e13b0c77cbb769548077189c3da2f0a764ceca06af49d8d558e759f5c232bd", - }, + To: file.NewCoordinates("/etc/modprobe.d/kms.conf", "sha256:e5e13b0c77cbb769548077189c3da2f0a764ceca06af49d8d558e759f5c232bd"), Type: artifact.ContainsRelationship, }, { @@ -386,9 +374,7 @@ func Test_parseSBOM(t *testing.T) { for _, p := range expectedPkgs { expectedRelationships = append(expectedRelationships, artifact.Relationship{ From: p, - To: file.Coordinates{ - RealPath: "sbom.syft.json", - }, + To: file.NewCoordinates("sbom.syft.json", ""), Type: artifact.DescribedByRelationship, }) } diff --git a/syft/pkg/cataloger/swipl/parse_pack.go b/syft/pkg/cataloger/swipl/parse_pack.go index f4fbe5d73..e50be797e 100644 --- a/syft/pkg/cataloger/swipl/parse_pack.go +++ b/syft/pkg/cataloger/swipl/parse_pack.go @@ -30,7 +30,7 @@ func parsePackPackage(ctx context.Context, resolver file.Resolver, _ *generic.En version := versionRe.FindSubmatch(data) if name == nil || version == nil { - log.Debugf("encountered pack.pl file without a name and/or version field, ignoring (path=%q)", reader.Path()) + log.WithFields("path", reader.FullNestedPath()).Debug("encountered pack.pl file without a name and/or version field, ignoring") return nil, nil, nil } diff --git a/syft/pkg/cataloger/terraform/parse_tf_lock.go b/syft/pkg/cataloger/terraform/parse_tf_lock.go index 8056c8e2b..064746577 100644 --- a/syft/pkg/cataloger/terraform/parse_tf_lock.go +++ b/syft/pkg/cataloger/terraform/parse_tf_lock.go @@ -25,7 +25,7 @@ func parseTerraformLock(_ context.Context, _ file.Resolver, _ *generic.Environme return nil, nil, fmt.Errorf("failed to read terraform lock file: %w", err) } - err = hclsimple.Decode(reader.RealPath, contents, nil, &lockFile) + err = hclsimple.Decode(reader.FullNestedPath(), contents, nil, &lockFile) if err != nil { return nil, nil, fmt.Errorf("failed to decode terraform lock file: %w", err) } diff --git a/syft/pkg/cataloger/wordpress/parse_plugin.go b/syft/pkg/cataloger/wordpress/parse_plugin.go index a3de5060c..0f776d3ee 100644 --- a/syft/pkg/cataloger/wordpress/parse_plugin.go +++ b/syft/pkg/cataloger/wordpress/parse_plugin.go @@ -47,7 +47,7 @@ func parseWordpressPluginFiles(ctx context.Context, resolver file.Resolver, _ *g // a file shorter than the buffer is expected, so only the header it does have is parsed n, err := io.ReadFull(reader, buffer) if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { - return nil, nil, fmt.Errorf("failed to read %s file: %w", reader.Path(), err) + return nil, nil, fmt.Errorf("failed to read %s file: %w", reader.FullNestedPath(), err) } fields := extractFields(string(buffer[:n])) @@ -56,7 +56,7 @@ func parseWordpressPluginFiles(ctx context.Context, resolver file.Resolver, _ *g version, versionOk := fields["version"] // get a plugin name from a plugin's directory name - pluginInstallDirectory := filepath.Base(filepath.Dir(reader.RealPath)) + pluginInstallDirectory := filepath.Base(filepath.Dir(reader.LocalPath())) if nameOk && name != "" && versionOk && version != "" { var metadata pluginData diff --git a/syft/pkg/collection.go b/syft/pkg/collection.go index 215e7eedf..a566869c8 100644 --- a/syft/pkg/collection.go +++ b/syft/pkg/collection.go @@ -61,7 +61,8 @@ func (c *Collection) Package(id artifact.ID) *Package { return &p } -// PackagesByPath returns all packages that were discovered from the given path. +// PackagesByPath returns all packages that were discovered from the given path: a file's full path or +// access path. A package found within an archive is not returned for the archive's own path. func (c *Collection) PackagesByPath(path string) []Package { c.lock.RLock() defer c.lock.RUnlock() @@ -158,19 +159,22 @@ func (c *Collection) addTypeToIndex(p Package) { func (c *Collection) addPathsToIndex(p Package) { // note: write lock must be held by caller - observedPaths := strset.New() - for _, l := range p.Locations.ToSlice() { - if l.RealPath != "" && !observedPaths.Has(l.RealPath) { - c.addPathToIndex(p.id, l.RealPath) - observedPaths.Add(l.RealPath) - } - if l.AccessPath != "" && l.RealPath != l.AccessPath && !observedPaths.Has(l.AccessPath) { - c.addPathToIndex(p.id, l.AccessPath) - observedPaths.Add(l.AccessPath) - } + for _, path := range indexedPaths(p) { + c.addPathToIndex(p.id, path) } } +// indexedPaths are the paths a package is found by: each location's full path and access path. The real +// path of a nested location is the outermost archive, so it is left out. +func indexedPaths(p Package) []string { + paths := strset.New() + for _, l := range p.Locations.ToSlice() { + paths.Add(l.AccessPath, l.FullNestedPath()) + } + paths.Remove("") + return paths.List() +} + func (c *Collection) addPathToIndex(id artifact.ID, path string) { // note: write lock must be held by caller @@ -223,16 +227,8 @@ func (c *Collection) deleteTypeFromIndex(p Package) { func (c *Collection) deletePathsFromIndex(p Package) { // note: write lock must be held by caller - observedPaths := strset.New() - for _, l := range p.Locations.ToSlice() { - if l.RealPath != "" && !observedPaths.Has(l.RealPath) { - c.deletePathFromIndex(p.id, l.RealPath) - observedPaths.Add(l.RealPath) - } - if l.AccessPath != "" && l.RealPath != l.AccessPath && !observedPaths.Has(l.AccessPath) { - c.deletePathFromIndex(p.id, l.AccessPath) - observedPaths.Add(l.AccessPath) - } + for _, path := range indexedPaths(p) { + c.deletePathFromIndex(p.id, path) } } diff --git a/syft/pkg/collection_test.go b/syft/pkg/collection_test.go index 0d8c510ec..74bedc701 100644 --- a/syft/pkg/collection_test.go +++ b/syft/pkg/collection_test.go @@ -392,10 +392,7 @@ func TestCatalog_MergeRecords(t *testing.T) { CPEs: []cpe.CPE{cpe.Must("cpe:2.3:a:package:1:1:*:*:*:*:*:*:*", cpe.GeneratedSource)}, Locations: file.NewLocationSet( file.NewVirtualLocationFromCoordinates( - file.Coordinates{ - RealPath: "/b/path", - FileSystemID: "a", - }, + file.NewCoordinates("/b/path", "a"), "/another/path", ), ), @@ -405,10 +402,7 @@ func TestCatalog_MergeRecords(t *testing.T) { CPEs: []cpe.CPE{cpe.Must("cpe:2.3:a:package:2:2:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource)}, Locations: file.NewLocationSet( file.NewVirtualLocationFromCoordinates( - file.Coordinates{ - RealPath: "/b/path", - FileSystemID: "b", - }, + file.NewCoordinates("/b/path", "b"), "/another/path", ), ), @@ -417,17 +411,11 @@ func TestCatalog_MergeRecords(t *testing.T) { }, expectedLocations: []file.Location{ file.NewVirtualLocationFromCoordinates( - file.Coordinates{ - RealPath: "/b/path", - FileSystemID: "a", - }, + file.NewCoordinates("/b/path", "a"), "/another/path", ), file.NewVirtualLocationFromCoordinates( - file.Coordinates{ - RealPath: "/b/path", - FileSystemID: "b", - }, + file.NewCoordinates("/b/path", "b"), "/another/path", ), }, @@ -445,6 +433,23 @@ func TestCatalog_MergeRecords(t *testing.T) { } } +func TestCatalog_PackagesByPath_nestedPackagesAreNotFoundByTheOuterArchive(t *testing.T) { + jar := Package{Name: "jar", Type: JavaPkg, Locations: file.NewLocationSet(file.NewLocation("/opt/x.jar"))} + nested := file.Coordinates{RealPath: "/opt/x.jar", FullPath: "/opt/x.jar!/lib/a.so"} + so := Package{Name: "so", Type: BinaryPkg, Locations: file.NewLocationSet(file.NewLocationFromCoordinates(nested))} + c := NewCollection(jar, so) + + names := func(pkgs []Package) []string { + var out []string + for _, p := range pkgs { + out = append(out, p.Name) + } + return out + } + assert.Equal(t, []string{"jar"}, names(c.PackagesByPath("/opt/x.jar"))) + assert.Equal(t, []string{"so"}, names(c.PackagesByPath(nested.FullPath))) +} + func TestCatalog_EnumerateNilCatalog(t *testing.T) { var c *Collection assert.Empty(t, c.Enumerate()) diff --git a/syft/pkg/license.go b/syft/pkg/license.go index df7fe0f78..30389a94f 100644 --- a/syft/pkg/license.go +++ b/syft/pkg/license.go @@ -257,7 +257,7 @@ func (b *licenseBuilder) WithURLs(urls ...string) *licenseBuilder { func (b *licenseBuilder) WithLocations(locations ...file.Location) *licenseBuilder { for _, loc := range locations { - if loc.Path() != "" { + if loc.RealPath != "" { b.locations = append(b.locations, loc) } } diff --git a/syft/pkg/license_set_test.go b/syft/pkg/license_set_test.go index c6e420a3d..fa9692fed 100644 --- a/syft/pkg/license_set_test.go +++ b/syft/pkg/license_set_test.go @@ -64,16 +64,16 @@ func TestLicenseSet_Add(t *testing.T) { { name: "deduplicate licenses with locations", licenses: []License{ - NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "1"})), - NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "1"})), - NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "2"})), + NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.NewCoordinates("/place", "1"))), + NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.NewCoordinates("/place", "1"))), + NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.NewCoordinates("/place", "2"))), }, want: []License{ NewLicenseFromLocationsWithContext( ctx, "MIT", - file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "1"}), - file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "2"}), + file.NewLocationFromCoordinates(file.NewCoordinates("/place", "1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/place", "2")), ), }, }, @@ -81,15 +81,15 @@ func TestLicenseSet_Add(t *testing.T) { name: "same licenses with different locations", licenses: []License{ NewLicenseWithContext(ctx, "MIT"), - NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "2"})), - NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "1"})), + NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.NewCoordinates("/place", "2"))), + NewLicenseFromLocationsWithContext(ctx, "MIT", file.NewLocationFromCoordinates(file.NewCoordinates("/place", "1"))), }, want: []License{ NewLicenseFromLocationsWithContext( ctx, "MIT", - file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "1"}), - file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/place", FileSystemID: "2"}), + file.NewLocationFromCoordinates(file.NewCoordinates("/place", "1")), + file.NewLocationFromCoordinates(file.NewCoordinates("/place", "2")), ), }, }, diff --git a/syft/pkg/package.go b/syft/pkg/package.go index d649db292..79f9089c2 100644 --- a/syft/pkg/package.go +++ b/syft/pkg/package.go @@ -119,10 +119,10 @@ func Less(i, j Package) bool { } return len(iLocations) < len(jLocations) } - if iLocations[l].RealPath == jLocations[l].RealPath { + if iLocations[l].FullNestedPath() == jLocations[l].FullNestedPath() { continue } - return iLocations[l].RealPath < jLocations[l].RealPath + return iLocations[l].FullNestedPath() < jLocations[l].FullNestedPath() } // compare remaining metadata as a final fallback // note: we cannot guarantee that IDs (which digests the metadata) are stable enough to sort on diff --git a/syft/pkg/package_test.go b/syft/pkg/package_test.go index 4c2ad5249..9a4e94a7b 100644 --- a/syft/pkg/package_test.go +++ b/syft/pkg/package_test.go @@ -15,10 +15,7 @@ import ( func TestIDUniqueness(t *testing.T) { ctx := context.TODO() originalLocation := file.NewVirtualLocationFromCoordinates( - file.Coordinates{ - RealPath: "39.0742° N, 21.8243° E", - FileSystemID: "Earth", - }, + file.NewCoordinates("39.0742° N, 21.8243° E", "Earth"), "/Ancient-Greece", ) @@ -230,10 +227,7 @@ func TestIDUniqueness(t *testing.T) { func TestPackage_Merge(t *testing.T) { originalLocation := file.NewVirtualLocationFromCoordinates( - file.Coordinates{ - RealPath: "39.0742° N, 21.8243° E", - FileSystemID: "Earth", - }, + file.NewCoordinates("39.0742° N, 21.8243° E", "Earth"), "/Ancient-Greece", ) @@ -461,3 +455,21 @@ func TestIsValid(t *testing.T) { require.Equal(t, c.want, IsValid(c.given), "when package: %s", c.name) } } + +func TestID_nestedArchiveLocations(t *testing.T) { + id := func(c file.Coordinates) string { + p := Package{Name: "x", Version: "1", Type: NpmPkg, Locations: file.NewLocationSet(file.NewLocationFromCoordinates(c))} + p.SetID() + return string(p.ID()) + } + + assert.NotEqual(t, + id(file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/a.jar!/package.json"}), + id(file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/b.jar!/package.json"}), + "packages in different entries of the same outer archive must not share an ID") + + assert.Equal(t, + id(file.Coordinates{RealPath: "/package.json"}), + id(file.NewCoordinates("/package.json", "")), + "a full path equal to the real path must not change the ID") +} diff --git a/syft/source/directorysource/directory_source.go b/syft/source/directorysource/directory_source.go index 259ab2cac..c6eb43a16 100644 --- a/syft/source/directorysource/directory_source.go +++ b/syft/source/directorysource/directory_source.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" "sync" @@ -18,7 +19,10 @@ import ( "github.com/anchore/syft/syft/source/internal" ) -var _ source.Source = (*directorySource)(nil) +var ( + _ source.Source = (*directorySource)(nil) + _ source.PathExcluder = (*directorySource)(nil) +) type Config struct { Path string @@ -114,6 +118,11 @@ func (s *directorySource) FileResolver(_ source.Scope) (file.Resolver, error) { return s.resolver, nil } +// ExcludedPaths returns the exclusion patterns this source was configured with. +func (s directorySource) ExcludedPaths() []string { + return slices.Clone(s.config.Exclude.Paths) +} + func (s *directorySource) Close() error { s.mutex.Lock() defer s.mutex.Unlock() @@ -127,7 +136,12 @@ func GetDirectoryExclusionFunctions(root string, exclusions []string) ([]fileres return nil, nil } - // this is what directoryResolver.indexTree is doing to get the absolute path: + // the indexer reports symlink-resolved absolute paths to every visitor, so the patterns must be + // anchored to a root resolved the same way, or a root under a symlink (macOS's /var) excludes nothing + if resolved, err := filepath.EvalSymlinks(root); err == nil { + root = resolved + } + root, err := filepath.Abs(root) if err != nil { return nil, err @@ -140,15 +154,17 @@ func GetDirectoryExclusionFunctions(root string, exclusions []string) ([]fileres root += "/" } + // the given patterns are the source's configuration (see source.PathExcluder), so they are not rewritten in place + rooted := make([]string, 0, len(exclusions)) var errors []string - for idx, exclusion := range exclusions { + for _, exclusion := range exclusions { // check exclusions for supported paths, these are all relative to the "scan root" if strings.HasPrefix(exclusion, "./") || strings.HasPrefix(exclusion, "*/") || strings.HasPrefix(exclusion, "**/") { exclusion = strings.TrimPrefix(exclusion, "./") // a trailing slash signals a directory but is otherwise discarded by doublestar.Match, // causing the pattern to silently match nothing (see issue #4839) exclusion = strings.TrimSuffix(exclusion, "/") - exclusions[idx] = root + exclusion + rooted = append(rooted, root+exclusion) } else { errors = append(errors, exclusion) } @@ -160,7 +176,7 @@ func GetDirectoryExclusionFunctions(root string, exclusions []string) ([]fileres return []fileresolver.PathIndexVisitor{ func(_, path string, info os.FileInfo, _ error) error { - for _, exclusion := range exclusions { + for _, exclusion := range rooted { // this is required to handle Windows filepaths path = filepath.ToSlash(path) matches, err := doublestar.Match(exclusion, path) diff --git a/syft/source/exclude.go b/syft/source/exclude.go index f41dc0e31..812cc1575 100644 --- a/syft/source/exclude.go +++ b/syft/source/exclude.go @@ -3,3 +3,9 @@ package source type ExcludeConfig struct { Paths []string } + +// PathExcluder is implemented by a source configured with path exclusion patterns, so the archive +// cataloger can apply the same exclusions inside the archives it extracts. +type PathExcluder interface { + ExcludedPaths() []string +} diff --git a/syft/source/filesource/file_source.go b/syft/source/filesource/file_source.go index a443d2baa..b9e18d58b 100644 --- a/syft/source/filesource/file_source.go +++ b/syft/source/filesource/file_source.go @@ -8,6 +8,7 @@ import ( "os" "path" "path/filepath" + "slices" "sync" "github.com/mholt/archives" @@ -24,7 +25,10 @@ import ( "github.com/anchore/syft/syft/source/internal" ) -var _ source.Source = (*fileSource)(nil) +var ( + _ source.Source = (*fileSource)(nil) + _ source.PathExcluder = (*fileSource)(nil) +) type Config struct { Path string @@ -66,8 +70,14 @@ func New(cfg Config) (source.Source, error) { return nil, fmt.Errorf("given path is a directory: %q", cfg.Path) } + mimeType := stereoFile.MIMEType(f) + var digests []file.Digest if len(cfg.DigestAlgorithms) > 0 { + // reading the MIME type moved past the start of the file + if _, err := f.Seek(0, io.SeekStart); err != nil { + return nil, fmt.Errorf("unable to seek file=%q: %w", cfg.Path, err) + } digests, err = intFile.NewDigestsFromFile(context.TODO(), f, cfg.DigestAlgorithms) if err != nil { return nil, fmt.Errorf("unable to calculate digests for file=%q: %w", cfg.Path, err) @@ -94,7 +104,7 @@ func New(cfg Config) (source.Source, error) { analysisPath: analysisPath, digestForVersion: versionDigest, digests: digests, - mimeType: stereoFile.MIMEType(f), + mimeType: mimeType, }, nil } @@ -153,7 +163,7 @@ func (s fileSource) FileResolver(_ source.Scope) (file.Resolver, error) { if isArchiveAnalysis := fi.IsDir(); isArchiveAnalysis { // this is an analysis of an archive file... we should scan the directory where the archive contents - res, err := fileresolver.NewFromDirectory(s.analysisPath, "", exclusionFunctions...) + res, err := fileresolver.NewFromDirectory(s.analysisPath, s.analysisPath, exclusionFunctions...) if err != nil { return nil, fmt.Errorf("unable to create directory resolver: %w", err) } @@ -172,6 +182,11 @@ func (s fileSource) FileResolver(_ source.Scope) (file.Resolver, error) { return s.resolver, nil } +// ExcludedPaths returns the exclusion patterns this source was configured with. +func (s fileSource) ExcludedPaths() []string { + return slices.Clone(s.config.Exclude.Paths) +} + func (s *fileSource) Close() error { s.mutex.Lock() defer s.mutex.Unlock() diff --git a/syft/source/filesource/file_source_test.go b/syft/source/filesource/file_source_test.go index 1866dd668..900cff8fb 100644 --- a/syft/source/filesource/file_source_test.go +++ b/syft/source/filesource/file_source_test.go @@ -3,6 +3,8 @@ package filesource import ( "archive/zip" "bytes" + "context" + "crypto" "io" "os" "os/exec" @@ -242,6 +244,44 @@ func TestNewFromFile_RarExtension(t *testing.T) { } } +func TestNewFromFile_MIMETypeWithDigests(t *testing.T) { + // the digests read the whole file, which must not leave the MIME type to be read from its end + archivePath := filepath.Join(t.TempDir(), "example.zip") + require.NoError(t, os.WriteFile(archivePath, zipBytes(t, [][2]string{{"a.txt", "a"}}), 0o600)) + + src, err := New(Config{Path: archivePath, DigestAlgorithms: []crypto.Hash{crypto.SHA256}}) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, src.Close()) + }) + + m := src.Describe().Metadata.(source.FileMetadata) + assert.Equal(t, "application/zip", m.MIMEType) + assert.Len(t, m.Digests, 1) +} + +func TestNewFromFile_ArchivePathsAreRootedAtTheArchive(t *testing.T) { + // an extracted archive is scanned like a directory: its root is the empty path, and the temporary directory it + // was extracted to does not appear + archivePath := filepath.Join(t.TempDir(), "example.zip") + require.NoError(t, os.WriteFile(archivePath, zipBytes(t, [][2]string{{"sub/a.txt", "a"}}), 0o600)) + + src, err := New(Config{Path: archivePath}) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, src.Close()) + }) + + res, err := src.FileResolver(source.SquashedScope) + require.NoError(t, err) + + var paths []string + for l := range res.AllLocations(context.Background()) { + paths = append(paths, l.RealPath) + } + assert.ElementsMatch(t, []string{"", "/sub", "/sub/a.txt"}, paths) +} + // createResourceAdapterArchive writes a minimal Java resource adapter archive (a zip file with a .rar extension that // holds a deployment descriptor and a nested jar) into dir and returns its path. func createResourceAdapterArchive(t testing.TB, dir string) string { diff --git a/syft/source/snapsource/manifest.go b/syft/source/snapsource/manifest.go index a2d0b04d2..86f1c79fe 100644 --- a/syft/source/snapsource/manifest.go +++ b/syft/source/snapsource/manifest.go @@ -47,7 +47,7 @@ func parseManifest(resolver file.Resolver) (*snapManifest, error) { if err != nil { return nil, fmt.Errorf("unable to read snap manifest file: %w", err) } - defer internal.CloseAndLogError(reader, manifestFile.RealPath) + defer internal.CloseAndLogError(reader, manifestFile.FullNestedPath()) var manifest snapManifest if err := yaml.NewDecoder(reader).Decode(&manifest); err != nil { diff --git a/syft/source/stereoscopesource/image_source.go b/syft/source/stereoscopesource/image_source.go index c6dc29390..da8c5e18b 100644 --- a/syft/source/stereoscopesource/image_source.go +++ b/syft/source/stereoscopesource/image_source.go @@ -2,6 +2,7 @@ package stereoscopesource import ( "fmt" + "slices" "github.com/bmatcuk/doublestar/v4" "github.com/distribution/reference" @@ -15,7 +16,10 @@ import ( "github.com/anchore/syft/syft/source/internal" ) -var _ source.Source = (*stereoscopeImageSource)(nil) +var ( + _ source.Source = (*stereoscopeImageSource)(nil) + _ source.PathExcluder = (*stereoscopeImageSource)(nil) +) type ImageConfig struct { Reference string @@ -122,6 +126,11 @@ func (s stereoscopeImageSource) FileResolver(scope source.Scope) (file.Resolver, return res, nil } +// ExcludedPaths returns the exclusion patterns this source was configured with. +func (s stereoscopeImageSource) ExcludedPaths() []string { + return slices.Clone(s.config.Exclude.Paths) +} + func (s stereoscopeImageSource) Close() error { if s.image == nil { return nil diff --git a/test/rules/rules.go b/test/rules/rules.go index e001a12bd..cb66f9c9f 100644 --- a/test/rules/rules.go +++ b/test/rules/rules.go @@ -16,7 +16,7 @@ func resourceCleanup(m dsl.Matcher) { m["res"].Type.Implements(`io.Reader`) && m["err"].Type.Implements(`error`) && !m["next"].Text.Matches(`defer internal.CloseAndLogError`)). - Report(`please call "defer internal.CloseAndLogError($res, $loc.RealPath)" right after checking the error returned from $resolver.FileContentsByLocation.`) + Report(`please call "defer internal.CloseAndLogError($res, $loc.FullNestedPath())" right after checking the error returned from $resolver.FileContentsByLocation.`) } // nolint:unused