Merge remote-tracking branch 'upstream/main' into add-nested-archive-cataloging

This commit is contained in:
Keith Zantow
2026-09-10 13:32:07 -04:00
451 changed files with 45797 additions and 2113 deletions
+6
View File
@@ -2,6 +2,8 @@
# This file is partially auto-generated. Run 'go generate ./internal/capabilities' to regenerate.
application: # AUTO-GENERATED - application-level config keys
- key: cpp.vcpkg-allow-git-clone
description: enables Syft to use clone remote repositories for vcpkg custom git registries. (also useful if the builtin vcpkg registry is not cloned locally)
- key: dotnet.dep-packages-must-claim-dll
description: only keep dep.json packages which have a runtime/resource DLL claimed in the deps.json targets section (but not necessarily found on disk). The package is also included if any child package claims a DLL, even if the package itself does not claim a DLL.
- key: dotnet.dep-packages-must-have-dll
@@ -12,6 +14,10 @@ application: # AUTO-GENERATED - application-level config keys
description: treat DLL claims or on-disk evidence for child packages as DLL claims or on-disk evidence for any parent package
- key: dotnet.relax-dll-claims-when-bundling-detected
description: show all packages from the deps.json if bundling tooling is present as a dependency (e.g. ILRepack)
- key: golang.capture-symbols
description: 'capture function symbols from the binary symbol table (pclntab). valid values are: "none" (disabled), "stdlib" (only the synthetic stdlib package), "extended-stdlib" (stdlib plus every module under golang.org/x/), and "all" (all module packages plus stdlib)'
- key: golang.capture-symbols-modules
description: glob patterns matched against go module paths (e.g. github.com/klauspost/**) that should have symbols captured in addition to whatever capture-symbols selects. ** crosses path separators, * does not. a trailing major version suffix is ignored when matching, so github.com/foo/* covers github.com/foo/bar/v2; spelling a suffix out in the pattern selects only that major version. this can only widen the selection, never narrow it, and is inert when capture-symbols is none
- key: golang.local-mod-cache-dir
description: specify an explicit go mod cache directory, if unset this defaults to $GOPATH/pkg/mod or $HOME/go/pkg/mod
- key: golang.local-vendor-dir
+6 -1
View File
@@ -297,6 +297,11 @@ func addCatalogerFieldComment(keyNode, valueNode *yaml.Node, catalogerName strin
if keyNode.LineComment == "" {
keyNode.LineComment = autoGeneratedComment
}
case "purl_types":
// cataloger-level purl_types (for custom catalogers) are AUTO-GENERATED
if keyNode.LineComment == "" {
keyNode.LineComment = autoGeneratedComment
}
case "json_schema_types":
// json_schema_types are AUTO-GENERATED
if keyNode.LineComment == "" {
@@ -367,7 +372,7 @@ func addParserComments(parsersNode *yaml.Node) {
valueNode := parserNode.Content[i+1]
switch keyNode.Value {
case "parser_function", "metadata_types", "package_types", "json_schema_types":
case "parser_function", "metadata_types", "package_types", "purl_types", "json_schema_types":
// add AUTO-GENERATED comment to these fields
if keyNode.LineComment == "" {
keyNode.LineComment = autoGeneratedComment
+38
View File
@@ -3,6 +3,7 @@ package main
import (
"fmt"
"sort"
"strings"
"github.com/scylladb/go-set/strset"
@@ -10,6 +11,7 @@ import (
"github.com/anchore/syft/internal/capabilities"
"github.com/anchore/syft/internal/capabilities/internal"
"github.com/anchore/syft/internal/packagemetadata"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/binary"
)
@@ -192,6 +194,9 @@ func RegenerateCapabilities(catalogerDir string, repoRoot string) (*Statistics,
formatOrphans(orphans))
}
// 5a. Derive PURL types from the discovered package types (purely a function of pkg.Type)
derivePURLTypes(updated)
// 6. Write back to YAML files with comments, preserving existing node trees
fmt.Print(" → Writing updated capabilities files...")
if err := saveCapabilities(catalogerDir, repoRoot, updated, existingNodes); err != nil {
@@ -427,6 +432,39 @@ func (e *EnrichmentData) GetMetadataTypes(catalogerName string) ([]string, bool)
return types, ok
}
// derivePURLTypes populates the PURLTypes field on every cataloger and parser by mapping each
// observed package type to its canonical PURL type via pkg.Type.PackageURLType. this is a pure
// function of the already-discovered package_types, so it needs no additional test observations.
func derivePURLTypes(doc *capabilities.Document) {
for i := range doc.Catalogers {
cat := &doc.Catalogers[i]
cat.PURLTypes = purlTypesFromPackageTypes(cat.PackageTypes)
for j := range cat.Parsers {
cat.Parsers[j].PURLTypes = purlTypesFromPackageTypes(cat.Parsers[j].PackageTypes)
}
}
}
// purlTypesFromPackageTypes maps syft package type strings (e.g. "go-module") to their PURL types
// (e.g. "golang"), returning a deduplicated, sorted slice. package types with no PURL mapping are skipped.
func purlTypesFromPackageTypes(packageTypes []string) []string {
if len(packageTypes) == 0 {
return nil
}
set := strset.New()
for _, pt := range packageTypes {
if purlType := pkg.Type(pt).PackageURLType(); purlType != "" {
set.Add(purlType)
}
}
if set.IsEmpty() {
return nil
}
result := set.List()
sort.Strings(result)
return result
}
// GetPackageTypes returns the package types for the given cataloger name
func (e *EnrichmentData) GetPackageTypes(catalogerName string) ([]string, bool) {
types, ok := e.packageTypes[catalogerName]
@@ -552,3 +552,48 @@ func TestConvertToJSONSchemaTypesFromMetadata(t *testing.T) {
})
}
}
func TestPURLTypesFromPackageTypes(t *testing.T) {
tests := []struct {
name string
packageTypes []string
want []string
}{
{
name: "nil slice returns nil",
packageTypes: nil,
want: nil,
},
{
name: "single package type maps to purl type",
packageTypes: []string{"go-module"},
want: []string{"golang"},
},
{
name: "multiple package types are sorted",
packageTypes: []string{"npm", "go-module"},
want: []string{"golang", "npm"},
},
{
// java and jenkins-plugin both map to maven, so the result is deduplicated
name: "distinct package types collapsing to one purl type are deduplicated",
packageTypes: []string{"java-archive", "jenkins-plugin"},
want: []string{"maven"},
},
{
// unknown package types have no purl mapping and are skipped
name: "package type without a purl mapping is skipped",
packageTypes: []string{"UnknownPackage"},
want: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := purlTypesFromPackageTypes(tt.packageTypes)
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Errorf("purlTypesFromPackageTypes() mismatch (-want +got):\n%s", diff)
}
})
}
}
+2
View File
@@ -82,6 +82,7 @@ type CatalogerEntry struct {
Detectors []Detector `yaml:"detectors,omitempty" json:"detectors,omitempty"` // AUTO-GENERATED - detection methods (only for type=custom)
MetadataTypes []string `yaml:"metadata_types,omitempty" json:"metadata_types,omitempty"` // AUTO-GENERATED - pkg metadata types emitted (only for type=custom)
PackageTypes []string `yaml:"package_types,omitempty" json:"package_types,omitempty"` // AUTO-GENERATED - package types emitted (only for type=custom)
PURLTypes []string `yaml:"purl_types,omitempty" json:"purl_types,omitempty"` // AUTO-GENERATED - PURL types derived from package_types (only for type=custom)
JSONSchemaTypes []string `yaml:"json_schema_types,omitempty" json:"json_schema_types,omitempty"` // AUTO-GENERATED - JSON schema type names (UpperCamelCase)
Capabilities CapabilitySet `yaml:"capabilities,omitempty" json:"capabilities,omitempty"` // MANUAL - config-driven capability definitions (only for type=custom)
}
@@ -92,6 +93,7 @@ type Parser struct {
Detector Detector `yaml:"detector" json:"detector"` // AUTO-GENERATED - how artifacts are detected
MetadataTypes []string `yaml:"metadata_types,omitempty" json:"metadata_types,omitempty"` // AUTO-GENERATED - pkg metadata types emitted by this parser
PackageTypes []string `yaml:"package_types,omitempty" json:"package_types,omitempty"` // AUTO-GENERATED - package types emitted by this parser
PURLTypes []string `yaml:"purl_types,omitempty" json:"purl_types,omitempty"` // AUTO-GENERATED - PURL types derived from package_types
JSONSchemaTypes []string `yaml:"json_schema_types,omitempty" json:"json_schema_types,omitempty"` // AUTO-GENERATED - JSON schema type names (UpperCamelCase)
Capabilities CapabilitySet `yaml:"capabilities,omitempty" json:"capabilities,omitempty"` // MANUAL - config-driven capability definitions
}
+159 -11
View File
@@ -2,6 +2,8 @@ package os
import (
"context"
"encoding/json"
"errors"
"fmt"
"regexp"
@@ -10,6 +12,7 @@ import (
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
)
@@ -19,8 +22,11 @@ func DetectFeatures(_ context.Context, resolver file.Resolver, builder sbomsync.
return
}
err := findRhelFeatures(resolver, s.Artifacts.LinuxDistribution)
if err != nil {
if err := findRhelFeatures(resolver, s.Artifacts.LinuxDistribution); err != nil {
log.WithFields("error", err, "release", s.Artifacts.LinuxDistribution).Trace("error searching for extended support")
}
if err := findUbuntuFeatures(resolver, s.Artifacts.LinuxDistribution, s.Artifacts.Packages); err != nil {
log.WithFields("error", err, "release", s.Artifacts.LinuxDistribution).Trace("error searching for extended support")
}
})
@@ -28,6 +34,9 @@ func DetectFeatures(_ context.Context, resolver file.Resolver, builder sbomsync.
return nil
}
// rhelEUSPatterns match a RHEL content manifest referencing an extended-update-support (EUS) repo.
var rhelEUSPatterns = []*regexp.Regexp{regexp.MustCompile(`baseos-eus`), regexp.MustCompile(`baseos-eus|appstream-eus`)}
func findRhelFeatures(resolver file.Resolver, release *linux.Release) error {
if release == nil || release.ID != "rhel" {
return nil
@@ -37,9 +46,9 @@ func findRhelFeatures(resolver file.Resolver, release *linux.Release) error {
return fmt.Errorf("unable to find content manifests: %w", err)
}
for _, contentManifestFile := range contentManifestFiles {
found, err := hasRhelExtendedSupportInContentManifest(resolver, contentManifestFile)
found, err := fileMatchesAny(resolver, contentManifestFile, rhelEUSPatterns...)
if err != nil {
return fmt.Errorf("unable to read content manifest from %s: %w", contentManifestFile.String(), err)
return fmt.Errorf("unable to read content manifest from %s: %w", contentManifestFile.RealPath, err)
}
if found {
release.ExtendedSupport = true
@@ -49,13 +58,152 @@ func findRhelFeatures(resolver file.Resolver, release *linux.Release) error {
return nil
}
func hasRhelExtendedSupportInContentManifest(resolver file.Resolver, contentManifestFile file.Location) (bool, error) {
contents, err := resolver.FileContentsByLocation(contentManifestFile)
if err != nil {
return false, fmt.Errorf("unable to read content manifest from %s: %w", contentManifestFile.String(), err)
}
defer internal.CloseAndLogError(contents, "content-manifest")
// Ubuntu Pro (formerly Ubuntu Advantage) is Canonical's subscription that unlocks Expanded Security
// Maintenance (ESM): security fixes past a package's standard support window. we detect the two base ESM
// streams, esm-infra (the `main` repo) and esm-apps (`universe`), and collapse them into one ExtendedSupport
// signal; the infra-vs-apps split is disambiguated by the downstream consumer, not by this boolean.
//
// the other Pro streams on the same host (esm.ubuntu.com/fips, /fips-updates, /realtime) are deliberately
// excluded: they are separate compliance products with their own downstream channels, and a host can run
// them with base ESM disabled, so folding them in would be unsound.
//
// detection is deliberately forgiving: apt/status evidence disappears when ESM is disabled, but an installed
// +esmN package is durable proof ESM content is on disk. so ExtendedSupport means "ESM was or is in effect
// for this host or its content", not "currently entitled" (mere eligibility, without evidence, is not enough).
patterns := []*regexp.Regexp{regexp.MustCompile(`baseos-eus`), regexp.MustCompile(`baseos-eus|appstream-eus`)}
// esmAptSourcePattern matches an uncommented apt source (classic or DEB822) or auth entry pointing at the
// plain ESM streams esm.ubuntu.com/infra or /apps. it deliberately scopes to /(infra|apps) so the other
// Pro streams served from the same host (/fips, /fips-updates, /realtime) are never folded into the base
// esm channel. the [^#\n]* before the host ensures a leading `#` (comment) can never satisfy the match.
var esmAptSourcePattern = regexp.MustCompile(`(?m)^[^#\n]*esm\.ubuntu\.com/(infra|apps)`)
// esmVersionPattern matches a dpkg version carrying a numbered ESM pocket suffix, e.g. `...+esm1` or
// `...~esm2`. the trailing \d scopes this to the ESM pocket and excludes other Pro pockets (e.g. +fips).
var esmVersionPattern = regexp.MustCompile(`[~+]esm\d`)
func findUbuntuFeatures(resolver file.Resolver, release *linux.Release, packages *pkg.Collection) error {
if release == nil || release.ID != "ubuntu" {
return nil
}
// an uncommented esm.ubuntu.com apt source or an active pro esm service both prove Pro/ESM was enabled.
// these are the durable, file-based signals present on attached hosts. a read error in one signal must
// not suppress the others, so errors are collected and detection continues to the next signal.
var errs error
for _, check := range []func() (bool, error){
func() (bool, error) { return hasUbuntuESMAptEvidence(resolver) },
func() (bool, error) { return hasActiveUbuntuESMService(resolver) },
} {
found, err := check()
if err != nil {
errs = errors.Join(errs, err)
continue
}
if found {
release.ExtendedSupport = true
return errs
}
}
// fallback for images (typically not pro-attached): an installed package whose version carries an
// +esm/~esm suffix was pulled from an ESM pocket, so ESM was in effect when it was installed.
if hasInstalledESMPackage(packages) {
release.ExtendedSupport = true
}
return errs
}
// hasUbuntuESMAptEvidence reports whether any apt source (classic .list or DEB822 .sources) or auth entry
// references esm.ubuntu.com via an uncommented line.
func hasUbuntuESMAptEvidence(resolver file.Resolver) (bool, error) {
// apt source files live in a directory (classic .list or DEB822 .sources), the credentials file is a fixed path.
sourceLocations, err := resolver.FilesByGlob("/etc/apt/sources.list.d/*")
if err != nil {
return false, fmt.Errorf("unable to find apt esm sources: %w", err)
}
authLocations, err := resolver.FilesByPath("/etc/apt/auth.conf.d/90ubuntu-advantage")
if err != nil {
return false, fmt.Errorf("unable to find apt esm auth: %w", err)
}
// ponytail: a DEB822 source with `Enabled: no` would still match; the pro client removes/comments the
// file on disable rather than toggling that key, so this is an accepted false-positive ceiling. the
// auth.conf.d entry is deleted by the pro client on disable, so it is a durable positive-only signal.
for _, location := range append(sourceLocations, authLocations...) {
match, err := fileMatchesAny(resolver, location, esmAptSourcePattern)
if err != nil {
return false, err
}
if match {
return true, nil
}
}
return false, nil
}
// hasActiveUbuntuESMService reports whether the ubuntu-advantage/pro status cache shows an enabled esm service.
func hasActiveUbuntuESMService(resolver file.Resolver) (bool, error) {
locations, err := resolver.FilesByPath("/var/lib/ubuntu-advantage/status.json")
if err != nil {
return false, fmt.Errorf("unable to find ubuntu-advantage status: %w", err)
}
for _, location := range locations {
enabled, err := hasEnabledESMServiceInStatus(resolver, location)
if err != nil {
return false, err
}
if enabled {
return true, nil
}
}
return false, nil
}
func hasEnabledESMServiceInStatus(resolver file.Resolver, location file.Location) (bool, error) {
contents, err := resolver.FileContentsByLocation(location)
if err != nil {
return false, fmt.Errorf("unable to read ubuntu-advantage status from %s: %w", location.RealPath, err)
}
defer internal.CloseAndLogError(contents, location.RealPath)
var status struct {
Services []struct {
Name string `json:"name"`
Status string `json:"status"`
} `json:"services"`
}
if err := json.NewDecoder(contents).Decode(&status); err != nil {
return false, fmt.Errorf("unable to parse ubuntu-advantage status from %s: %w", location.RealPath, err)
}
// pro status values are enabled/disabled/n-a/warning/—; only "enabled" proves the stream is active.
// warning/expired states are intentionally excluded here and left to the apt-source and installed-package signals.
for _, svc := range status.Services {
if (svc.Name == "esm-infra" || svc.Name == "esm-apps") && svc.Status == "enabled" {
return true, nil
}
}
return false, nil
}
// hasInstalledESMPackage reports whether any installed dpkg package version carries an ESM pocket suffix.
func hasInstalledESMPackage(packages *pkg.Collection) bool {
if packages == nil {
return false
}
for p := range packages.Enumerate(pkg.DebPkg) {
if esmVersionPattern.MatchString(p.Version) {
return true
}
}
return false
}
func fileMatchesAny(resolver file.Resolver, location file.Location, patterns ...*regexp.Regexp) (bool, error) {
contents, err := resolver.FileContentsByLocation(location)
if err != nil {
return false, fmt.Errorf("unable to read %s: %w", location.RealPath, err)
}
defer internal.CloseAndLogError(contents, location.RealPath)
return internal.MatchAnyFromReader(contents, patterns...)
}
+214
View File
@@ -2,6 +2,11 @@ package os_test
import (
"context"
"io"
stdos "os"
"path"
"path/filepath"
"slices"
"testing"
"github.com/stretchr/testify/require"
@@ -11,7 +16,9 @@ import (
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/internal/task"
"github.com/anchore/syft/syft"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
@@ -99,3 +106,210 @@ func Test_EnvironmentTask(t *testing.T) {
})
}
}
func Test_findUbuntuFeatures(t *testing.T) {
tests := []struct {
name string
dir string
releaseID string
packages []pkg.Package
expected bool
}{
{
name: "esm apt source present",
dir: "testdata/ubuntu_esm_apt",
releaseID: "ubuntu",
expected: true,
},
{
name: "esm apt source in DEB822 .sources format",
dir: "testdata/ubuntu_esm_deb822",
releaseID: "ubuntu",
expected: true,
},
{
name: "esm evidence in apt auth.conf.d",
dir: "testdata/ubuntu_esm_authconf",
releaseID: "ubuntu",
expected: true,
},
{
name: "active esm service in ubuntu-advantage status",
dir: "testdata/ubuntu_esm_status",
releaseID: "ubuntu",
expected: true,
},
{
name: "only esm-infra enabled (esm-apps disabled)",
dir: "testdata/ubuntu_esm_infra_only",
releaseID: "ubuntu",
expected: true,
},
{
name: "malformed status.json does not panic or match",
dir: "testdata/ubuntu_esm_malformed",
releaseID: "ubuntu",
expected: false,
},
{
name: "installed +esm package version",
dir: "testdata/ubuntu_plain",
releaseID: "ubuntu",
packages: []pkg.Package{
{Name: "openssl", Version: "1.1.1f-1ubuntu2.19+esm1", Type: pkg.DebPkg},
},
expected: true,
},
{
name: "installed ~esm package version",
dir: "testdata/ubuntu_plain",
releaseID: "ubuntu",
packages: []pkg.Package{
{Name: "libcap2", Version: "1:2.32-1ubuntu0.1~esm1", Type: pkg.DebPkg},
},
expected: true,
},
{
name: "plain ubuntu with no esm evidence",
dir: "testdata/ubuntu_plain",
releaseID: "ubuntu",
packages: []pkg.Package{
{Name: "openssl", Version: "1.1.1f-1ubuntu2.19", Type: pkg.DebPkg},
},
expected: false,
},
{
name: "commented esm source and disabled esm service",
dir: "testdata/ubuntu_esm_disabled",
releaseID: "ubuntu",
expected: false,
},
{
name: "esm disabled but +esm package remains installed",
dir: "testdata/ubuntu_esm_disabled",
releaseID: "ubuntu",
packages: []pkg.Package{
{Name: "openssl", Version: "1.1.1f-1ubuntu2.19+esm1", Type: pkg.DebPkg},
},
expected: true, // durable proof: ESM content is on disk even though the channel is now off
},
{
name: "fips-only pro host is not plain esm",
dir: "testdata/ubuntu_fips_only",
releaseID: "ubuntu",
expected: false, // esm.ubuntu.com/fips must not be folded into the base esm channel
},
{
name: "non-ubuntu is unaffected by esm evidence",
dir: "testdata/ubuntu_esm_apt",
releaseID: "debian",
expected: false,
},
{
name: "non-ubuntu is unaffected by esm package version",
dir: "testdata/ubuntu_plain",
releaseID: "debian",
packages: []pkg.Package{
{Name: "openssl", Version: "1.1.1f-1ubuntu2.19+esm1", Type: pkg.DebPkg},
},
expected: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
resolver := fixtureResolverForDir(t, test.dir)
s := sbom.SBOM{}
s.Artifacts.LinuxDistribution = &linux.Release{ID: test.releaseID}
s.Artifacts.Packages = pkg.NewCollection(test.packages...)
err := os.DetectFeatures(context.Background(), resolver, sbomsync.NewBuilder(&s))
require.NoError(t, err)
require.Equal(t, test.expected, s.Artifacts.LinuxDistribution.ExtendedSupport)
})
}
}
// a read error in the apt-source signal must not suppress the ubuntu-advantage status signal; the three
// signals are meant to be independent fallbacks.
func Test_findUbuntuFeatures_signalErrorDoesNotSuppressOthers(t *testing.T) {
resolver := errOnGlobResolver{
fixtureResolver: fixtureResolverForDir(t, "testdata/ubuntu_esm_status").(fixtureResolver),
errOnGlob: "/etc/apt/sources.list.d/*",
}
s := sbom.SBOM{}
s.Artifacts.LinuxDistribution = &linux.Release{ID: "ubuntu"}
s.Artifacts.Packages = pkg.NewCollection()
require.NoError(t, os.DetectFeatures(context.Background(), resolver, sbomsync.NewBuilder(&s)))
// the apt glob errors, but the status.json signal (esm-infra enabled) should still be consulted
require.True(t, s.Artifacts.LinuxDistribution.ExtendedSupport)
}
// errOnGlobResolver injects a failure for a specific glob to simulate an unreadable apt directory.
type errOnGlobResolver struct {
fixtureResolver
errOnGlob string
}
func (r errOnGlobResolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
if slices.Contains(patterns, r.errOnGlob) {
return nil, stdos.ErrPermission
}
return r.fixtureResolver.FilesByGlob(patterns...)
}
// fixtureResolver maps in-image logical paths to on-disk fixture files so DetectFeatures can be driven
// without building a container image.
type fixtureResolver struct {
file.Resolver // unused methods; nil is fine since only the three below are called
files map[string]string
}
func (r fixtureResolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
var out []file.Location
for logical := range r.files {
for _, p := range patterns {
if ok, _ := path.Match(p, logical); ok {
out = append(out, file.NewLocation(logical))
break
}
}
}
return out, nil
}
func (r fixtureResolver) FilesByPath(paths ...string) ([]file.Location, error) {
var out []file.Location
for _, p := range paths {
if _, ok := r.files[p]; ok {
out = append(out, file.NewLocation(p))
}
}
return out, nil
}
func (r fixtureResolver) FileContentsByLocation(l file.Location) (io.ReadCloser, error) {
return stdos.Open(r.files[l.RealPath])
}
func fixtureResolverForDir(t *testing.T, dir string) file.Resolver {
t.Helper()
files := map[string]string{}
require.NoError(t, filepath.WalkDir(dir, func(p string, d stdos.DirEntry, err error) error {
if err != nil || d.IsDir() {
return err
}
rel, err := filepath.Rel(dir, p)
if err != nil {
return err
}
files["/"+filepath.ToSlash(rel)] = p
return nil
}))
return fixtureResolver{files: files}
}
@@ -0,0 +1,6 @@
# Written by ubuntu-advantage-tools
deb https://esm.ubuntu.com/infra/ubuntu focal-infra-security main
# deb-src https://esm.ubuntu.com/infra/ubuntu focal-infra-security main
deb https://esm.ubuntu.com/infra/ubuntu focal-infra-updates main
# deb-src https://esm.ubuntu.com/infra/ubuntu focal-infra-updates main
@@ -0,0 +1,2 @@
machine esm.ubuntu.com/infra/ubuntu/ login bearer password DUMMYTOKENREDACTED # ubuntu-pro-client
machine esm.ubuntu.com/apps/ubuntu/ login bearer password DUMMYTOKENREDACTED # ubuntu-pro-client
@@ -0,0 +1,5 @@
Types: deb
URIs: https://esm.ubuntu.com/infra/ubuntu
Suites: noble-infra-security noble-infra-updates
Components: main
Signed-By: /usr/share/keyrings/ubuntu-pro-esm-infra-keyring.gpg
@@ -0,0 +1,3 @@
# Written by ubuntu-advantage-tools
# deb https://esm.ubuntu.com/infra/ubuntu focal-infra-security main
# deb-src https://esm.ubuntu.com/infra/ubuntu focal-infra-security main
@@ -0,0 +1,18 @@
{
"_schema_version": "0.1",
"attached": true,
"services": [
{
"name": "esm-apps",
"available": "yes",
"entitled": "yes",
"status": "disabled"
},
{
"name": "esm-infra",
"available": "yes",
"entitled": "yes",
"status": "disabled"
}
]
}
@@ -0,0 +1,21 @@
{
"_schema_version": "0.1",
"attached": true,
"execution_status": "success",
"services": [
{
"name": "esm-apps",
"description": "Expanded Security Maintenance for Applications",
"available": "yes",
"entitled": "yes",
"status": "disabled"
},
{
"name": "esm-infra",
"description": "Expanded Security Maintenance for Infrastructure",
"available": "yes",
"entitled": "yes",
"status": "enabled"
}
]
}
@@ -0,0 +1 @@
{ this is not valid json
@@ -0,0 +1,28 @@
{
"_doc": "Content provided in json response is currently considered Experimental and may change",
"_schema_version": "0.1",
"attached": true,
"services": [
{
"name": "esm-apps",
"description": "Expanded Security Maintenance for Applications",
"available": "yes",
"entitled": "yes",
"status": "enabled"
},
{
"name": "esm-infra",
"description": "Expanded Security Maintenance for Infrastructure",
"available": "yes",
"entitled": "yes",
"status": "enabled"
},
{
"name": "fips",
"description": "NIST-certified FIPS crypto packages",
"available": "no",
"entitled": "no",
"status": "n/a"
}
]
}
@@ -0,0 +1,3 @@
# Written by ubuntu-pro-client
deb https://esm.ubuntu.com/fips/ubuntu focal main
# deb-src https://esm.ubuntu.com/fips/ubuntu focal main
@@ -0,0 +1,28 @@
{
"_schema_version": "0.1",
"attached": true,
"execution_status": "success",
"services": [
{
"name": "esm-apps",
"description": "Expanded Security Maintenance for Applications",
"available": "yes",
"entitled": "yes",
"status": "disabled"
},
{
"name": "esm-infra",
"description": "Expanded Security Maintenance for Infrastructure",
"available": "yes",
"entitled": "yes",
"status": "disabled"
},
{
"name": "fips",
"description": "NIST-certified FIPS crypto packages",
"available": "yes",
"entitled": "yes",
"status": "enabled"
}
]
}
@@ -0,0 +1,5 @@
Types: deb
URIs: http://archive.ubuntu.com/ubuntu/
Suites: noble noble-updates noble-backports
Components: main restricted universe multiverse
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
@@ -27,6 +27,7 @@ var knownNonMetadataTypeNames = strset.New(
var knownMetadataTypeNames = strset.New(
"DotnetPortableExecutableEntry",
"GGUFFileHeader",
"SafeTensorsModelInfo",
)
func DiscoverTypeNames() ([]string, error) {
+5
View File
@@ -9,6 +9,7 @@ func AllTypes() []any {
return []any{
pkg.AlpmDBEntry{},
pkg.ApkDBEntry{},
pkg.AppleAppBundleEntry{},
pkg.BinarySignature{},
pkg.BitnamiSBOMEntry{},
pkg.BunLockEntry{},
@@ -20,6 +21,8 @@ func AllTypes() []any {
pkg.CondaMetaPackage{},
pkg.DartPubspec{},
pkg.DartPubspecLockEntry{},
pkg.DenoLockEntry{},
pkg.DenoRemoteLockEntry{},
pkg.DotnetDepsEntry{},
pkg.DotnetPackagesLockEntry{},
pkg.DotnetPortableExecutableEntry{},
@@ -65,10 +68,12 @@ func AllTypes() []any {
pkg.RubyGemspec{},
pkg.RustBinaryAuditEntry{},
pkg.RustCargoLockEntry{},
pkg.SafeTensorsModelInfo{},
pkg.SnapEntry{},
pkg.SwiftPackageManagerResolvedEntry{},
pkg.SwiplPackEntry{},
pkg.TerraformLockProviderEntry{},
pkg.VcpkgManifest{},
pkg.WordpressPluginEntry{},
pkg.YarnLockEntry{},
}
+5
View File
@@ -73,6 +73,8 @@ var jsonTypes = makeJSONTypes(
jsonNames(pkg.ConaninfoEntry{}, "c-conan-info-entry"),
jsonNames(pkg.DartPubspecLockEntry{}, "dart-pubspec-lock-entry", "DartPubMetadata"),
jsonNames(pkg.DartPubspec{}, "dart-pubspec"),
jsonNames(pkg.DenoLockEntry{}, "deno-lock-entry"),
jsonNames(pkg.DenoRemoteLockEntry{}, "deno-remote-lock-entry"),
jsonNames(pkg.DotnetDepsEntry{}, "dotnet-deps-entry", "DotnetDepsMetadata"),
jsonNames(pkg.DotnetPortableExecutableEntry{}, "dotnet-portable-executable-entry"),
jsonNames(pkg.DpkgArchiveEntry{}, "dpkg-archive-entry"),
@@ -122,11 +124,14 @@ var jsonTypes = makeJSONTypes(
jsonNames(pkg.SnapEntry{}, "snap-entry"),
jsonNames(pkg.WordpressPluginEntry{}, "wordpress-plugin-entry", "WordpressMetadata"),
jsonNames(pkg.HomebrewFormula{}, "homebrew-formula"),
jsonNames(pkg.AppleAppBundleEntry{}, "apple-app-bundle-entry"),
jsonNames(pkg.LuaRocksPackage{}, "luarocks-package"),
jsonNames(pkg.TerraformLockProviderEntry{}, "terraform-lock-provider-entry"),
jsonNames(pkg.DotnetPackagesLockEntry{}, "dotnet-packages-lock-entry"),
jsonNames(pkg.CondaMetaPackage{}, "conda-metadata-entry", "CondaPackageMetadata"),
jsonNames(pkg.GGUFFileHeader{}, "gguf-file-header"),
jsonNames(pkg.SafeTensorsModelInfo{}, "safetensors-model-info"),
jsonNames(pkg.VcpkgManifest{}, "vcpkg-manifest"),
)
func expandLegacyNameVariants(names ...string) []string {
+50 -2
View File
@@ -65,6 +65,30 @@ func (i *Index) Add(relationships ...artifact.Relationship) {
}
func (i *Index) Remove(id artifact.ID) {
// scrub this node's edges from the adjacent nodes' maps before dropping the node itself. without this, the
// edges live on as dangling pointers inside the other endpoints' mappedRelationships, so query methods
// (From/To/References/Coordinates/Contains) would keep returning relationships that have been removed.
if mapped := i.fromID[id]; mapped != nil {
for _, sr := range mapped.allRelated {
if sr.to == id {
continue // self-edge: this node's own maps are deleted below
}
if other := i.toID[sr.to]; other != nil {
other.remove(id, sr)
}
}
}
if mapped := i.toID[id]; mapped != nil {
for _, sr := range mapped.allRelated {
if sr.from == id {
continue // self-edge: this node's own maps are deleted below
}
if other := i.fromID[sr.from]; other != nil {
other.remove(id, sr)
}
}
}
delete(i.fromID, id)
delete(i.toID, id)
@@ -78,6 +102,12 @@ func (i *Index) Remove(id artifact.ID) {
}
func (i *Index) Replace(ogID artifact.ID, replacement artifact.Identifiable) {
if replacement == nil || replacement.ID() == ogID {
// replacing a node with one that has the same ID would re-add the (deduped) edges and then the trailing
// Remove(ogID) would delete them, silently wiping all of the node's relationships. treat it as a no-op.
return
}
for _, mapped := range fromMappedByID(i.fromID, ogID) {
// the stale relationship(i.e. if there's an elder ID in either side) should be discarded
if len(fromMappedByID(i.toID, mapped.relationship.To.ID())) == 0 {
@@ -91,8 +121,10 @@ func (i *Index) Replace(ogID artifact.ID, replacement artifact.Identifiable) {
}
for _, mapped := range fromMappedByID(i.toID, ogID) {
// same as the above
if len(fromMappedByID(i.fromID, mapped.relationship.To.ID())) == 0 {
// same as the above, but check the surviving other endpoint (the From side, since these are edges TO ogID).
// note: this must reference From (not To, which is ogID itself) so that nodes appearing only on the To side
// of relationships (e.g. a go main module that nothing depends on) keep their edges after an ID change.
if len(fromMappedByID(i.fromID, mapped.relationship.From.ID())) == 0 {
continue
}
i.Add(artifact.Relationship{
@@ -206,6 +238,22 @@ func (m *mappedRelationships) add(id artifact.ID, newRelationship *sortableRelat
typeMap[id] = newRelationship
}
// remove deletes a single relationship (the other endpoint keyed by id) from this node's maps. it is the inverse
// of add and is used to keep adjacent nodes consistent when a node is removed from the index.
func (m *mappedRelationships) remove(id artifact.ID, target *sortableRelationship) {
filtered := m.allRelated[:0]
for _, r := range m.allRelated {
if r != target {
filtered = append(filtered, r)
}
}
m.allRelated = filtered
if typeMap := m.typeMap[target.relationship.Type]; typeMap != nil && typeMap[id] == target {
delete(typeMap, id)
}
}
type sortableRelationship struct {
from artifact.ID
to artifact.ID
+126
View File
@@ -380,6 +380,41 @@ func TestReplace(t *testing.T) {
compareRelationships(t, expectedRels, index.All())
}
func TestReplace_NodeOnlyOnToSide(t *testing.T) {
// regression: a node that only appears on the To side of relationships (nothing depends on it, e.g. a go main
// module that other modules are a dependency-of) must keep its incoming edges when its ID changes (such as when a
// compliance rule stubs a missing version and recomputes the package ID). previously the To-side remap guard
// checked ogID itself instead of the surviving From endpoint, so all such edges were dropped.
main := pkg.Package{Name: "main-module"} // empty version, like a go main module
dep1 := pkg.Package{Name: "dep-1", Version: "1.0.0"}
dep2 := pkg.Package{Name: "dep-2", Version: "2.0.0"}
for _, p := range []*pkg.Package{&main, &dep1, &dep2} {
p.SetID()
}
// both deps are a dependency-of main; main has no outgoing edges
r1 := artifact.Relationship{From: dep1, To: main, Type: artifact.DependencyOfRelationship}
r2 := artifact.Relationship{From: dep2, To: main, Type: artifact.DependencyOfRelationship}
index := NewIndex(r1, r2)
// simulate the version stub: main gets a new ID
stubbedMain := main
stubbedMain.Version = "UNKNOWN"
stubbedMain.SetID()
require.NotEqual(t, main.ID(), stubbedMain.ID())
index.Replace(main.ID(), &stubbedMain)
expectedRels := []artifact.Relationship{
{From: dep1, To: stubbedMain, Type: artifact.DependencyOfRelationship},
{From: dep2, To: stubbedMain, Type: artifact.DependencyOfRelationship},
}
compareRelationships(t, expectedRels, index.All())
}
func compareRelationships(t testing.TB, expected, actual []artifact.Relationship) {
assert.Equal(t, len(expected), len(actual), "number of relationships should match")
for _, e := range expected {
@@ -408,3 +443,94 @@ func TestReplace_NoExistingRelations(t *testing.T) {
allRels := index.All()
assert.Len(t, allRels, 0)
}
// Remove must scrub the removed node's edges from adjacent nodes too, otherwise query methods keep returning
// relationships that no longer exist. the original Remove only deleted the removed node's own maps, leaving
// dangling pointers in the other endpoints.
func TestRemove_NoStaleEdgesInAdjacentNodes(t *testing.T) {
p1 := pkg.Package{Name: "pkg-1", Version: "1"}
p2 := pkg.Package{Name: "pkg-2", Version: "2"}
p3 := pkg.Package{Name: "pkg-3", Version: "3"}
for _, p := range []*pkg.Package{&p1, &p2, &p3} {
p.SetID()
}
r12 := artifact.Relationship{From: p1, To: p2, Type: artifact.DependencyOfRelationship}
r32 := artifact.Relationship{From: p3, To: p2, Type: artifact.DependencyOfRelationship}
index := NewIndex(r12, r32)
index.Remove(p1.ID())
// p2 was on the To side of the removed edge; its incoming view must no longer include the stale p1 edge,
// but must still include the live p3 edge.
compareRelationships(t, []artifact.Relationship{r32}, index.To(p2, artifact.DependencyOfRelationship))
assert.False(t, index.Contains(r12), "Contains must not report a removed edge")
assert.Len(t, index.References(p2), 1, "p2 should only reference the surviving p3 edge")
assert.True(t, index.Contains(r32), "surviving edge should remain")
compareRelationships(t, []artifact.Relationship{r32}, index.All())
}
// Replace must leave no dangling references to the old ID in adjacent nodes' query views.
func TestReplace_NoStaleEdgesAfterReplace(t *testing.T) {
main := pkg.Package{Name: "main-module"} // empty version
dep := pkg.Package{Name: "dep", Version: "1.0.0"}
main.SetID()
dep.SetID()
oldEdge := artifact.Relationship{From: dep, To: main, Type: artifact.DependencyOfRelationship}
index := NewIndex(oldEdge)
stubbed := main
stubbed.Version = "UNKNOWN"
stubbed.SetID()
require.NotEqual(t, main.ID(), stubbed.ID())
index.Replace(main.ID(), &stubbed)
newEdge := artifact.Relationship{From: dep, To: stubbed, Type: artifact.DependencyOfRelationship}
// the dep's outgoing view must point at the stubbed node, not the old one
from := index.From(dep, artifact.DependencyOfRelationship)
require.Len(t, from, 1)
assert.Equal(t, stubbed.ID(), from[0].To.ID())
assert.True(t, index.Contains(newEdge))
assert.False(t, index.Contains(oldEdge), "stale edge to old ID must be gone")
assert.Empty(t, index.To(main), "old node should have no incoming edges")
compareRelationships(t, []artifact.Relationship{newEdge}, index.All())
}
// replacing a node with one that has the same ID must be a no-op, not silently wipe its edges.
func TestReplace_SameIDIsNoOp(t *testing.T) {
a := pkg.Package{Name: "a", Version: "1"}
b := pkg.Package{Name: "b", Version: "2"}
a.SetID()
b.SetID()
edge := artifact.Relationship{From: a, To: b, Type: artifact.DependencyOfRelationship}
index := NewIndex(edge)
// same identity object -> same ID
index.Replace(b.ID(), b)
compareRelationships(t, []artifact.Relationship{edge}, index.All())
assert.True(t, index.Contains(edge))
}
// a self-edge on the replaced node should be remapped to replacement->replacement, not dropped.
func TestReplace_SelfEdge(t *testing.T) {
n := pkg.Package{Name: "n"} // empty version
n.SetID()
index := NewIndex(artifact.Relationship{From: n, To: n, Type: artifact.DependencyOfRelationship})
sn := n
sn.Version = "UNKNOWN"
sn.SetID()
require.NotEqual(t, n.ID(), sn.ID())
index.Replace(n.ID(), &sn)
all := index.All()
require.Len(t, all, 1)
assert.Equal(t, sn.ID(), all[0].From.ID())
assert.Equal(t, sn.ID(), all[0].To.ID())
assert.Empty(t, index.From(n), "no edges should reference the old ID")
assert.Empty(t, index.To(n), "no edges should reference the old ID")
}
+13 -1
View File
@@ -5,6 +5,7 @@ import (
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/ai"
"github.com/anchore/syft/syft/pkg/cataloger/alpine"
"github.com/anchore/syft/syft/pkg/cataloger/apple"
"github.com/anchore/syft/syft/pkg/cataloger/arch"
"github.com/anchore/syft/syft/pkg/cataloger/binary"
bitnamiSbomCataloger "github.com/anchore/syft/syft/pkg/cataloger/bitnami"
@@ -56,6 +57,10 @@ const (
// Python ecosystem labels
Python = "python"
// C/C++ ecosystem labels
Cpp = "cpp"
Vcpkg = "vcpkg"
)
//nolint:funlen
@@ -82,6 +87,11 @@ func DefaultPackageTaskFactories() Factories {
// language-specific package declared catalogers ///////////////////////////////////////////////////////////////////////////
newSimplePackageTaskFactory(cpp.NewConanCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "cpp", "conan"),
newPackageTaskFactory(
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
return cpp.NewVcpkgManifestCataloger(cfg.PackagesConfig.Cpp)
},
pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, Cpp, Vcpkg),
newSimplePackageTaskFactory(dart.NewPubspecLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "dart"),
newSimplePackageTaskFactory(dart.NewPubspecCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "dart"),
newSimplePackageTaskFactory(elixir.NewMixLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "elixir"),
@@ -105,7 +115,7 @@ func DefaultPackageTaskFactories() Factories {
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
return javascript.NewLockCataloger(cfg.PackagesConfig.JavaScript)
},
pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, JavaScript, Node, NPM,
pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, JavaScript, Node, NPM, "deno",
),
newSimplePackageTaskFactory(php.NewComposerLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "php", "composer"),
newSimplePackageTaskFactory(php.NewPearCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, pkgcataloging.ImageTag, "php", "pear"),
@@ -184,9 +194,11 @@ func DefaultPackageTaskFactories() Factories {
newSimplePackageTaskFactory(wordpress.NewWordpressPluginCataloger, pkgcataloging.DirectoryTag, pkgcataloging.ImageTag, "wordpress"),
newSimplePackageTaskFactory(terraform.NewLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, "terraform"),
newSimplePackageTaskFactory(homebrew.NewCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, "homebrew"),
newSimplePackageTaskFactory(apple.NewAppBundleCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, "apple"),
newSimplePackageTaskFactory(conda.NewCondaMetaCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.PackageTag, "conda"),
newSimplePackageTaskFactory(snap.NewCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, "snap"),
newSimplePackageTaskFactory(ai.NewGGUFCataloger, pkgcataloging.DirectoryTag, pkgcataloging.ImageTag, "ai", "model", "gguf", "ml"),
newSimplePackageTaskFactory(ai.NewSafeTensorsCataloger, pkgcataloging.DirectoryTag, pkgcataloging.ImageTag, "ai", "model", "safetensors", "ml"),
// deprecated catalogers ////////////////////////////////////////
// these are catalogers that should not be selectable other than specific inclusion via name or "deprecated" tag (to remain backwards compatible)