oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-05 00:46:38 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add abstraction for adding relationships from package cataloger results (#2853)

Browse Source 
* add internal dependency resolver

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor dependency relationship resolution to common object

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* replace cataloger decorator with generic processor

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor resolver to be a single function

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* use common dependency specifier for debian

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* use common dependency specifier for arch

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* use common dependency specifier for alpine

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* allow for generic pkg and rel assertions in testpkg helper

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* do not allow for empty results

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* move stable deduplicate comment

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* remove relationship resolver type

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
2024-05-14 13:27:36 +00:00
committed by GitHub
parent fae6f5d372
commit 4a18895545
32 changed files with 1207 additions and 801 deletions
Download Patch File Download Diff File Expand all files Collapse all files
syft/pkg/cataloger/debian/cataloger.go
+3 -1
View File
@@ -6,6 +6,7 @@ package debian
import (
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/pkg/cataloger/internal/dependency"
)
// NewDBCataloger returns a new Deb package cataloger capable of parsing DPKG status DB flat-file stores.
@@ -13,5 +14,6 @@ func NewDBCataloger() pkg.Cataloger {
return generic.NewCataloger("dpkg-db-cataloger").
// note: these globs have been intentionally split up in order to improve search performance,
// please do NOT combine into: "**/var/lib/dpkg/{status,status.d/*}"
WithParserByGlobs(parseDpkgDB, "**/var/lib/dpkg/status", "**/var/lib/dpkg/status.d/*", "**/lib/opkg/info/*.control", "**/lib/opkg/status")
WithParserByGlobs(parseDpkgDB, "**/var/lib/dpkg/status", "**/var/lib/dpkg/status.d/*", "**/lib/opkg/info/*.control", "**/lib/opkg/status").
WithProcessors(dependency.Processor(dbEntryDependencySpecifier))
}
syft/pkg/cataloger/debian/cataloger_test.go
+62
View File
@@ -1,8 +1,12 @@
package debian
import (
"context"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
@@ -160,6 +164,64 @@ func TestDpkgCataloger(t *testing.T) {
}
}
func Test_CatalogerRelationships(t *testing.T) {
tests := []struct {
name string
fixture string
wantRelationships map[string][]string
}{
{
name: "relationships for coreutils",
fixture: "test-fixtures/var/lib/dpkg/status.d/coreutils-relationships",
wantRelationships: map[string][]string{
"coreutils": {"libacl1", "libattr1", "libc6", "libgmp10", "libselinux1"},
"libacl1": {"libc6"},
"libattr1": {"libc6"},
"libc6": {"libgcc-s1"},
"libgcc-s1": {"gcc-12-base", "libc6"},
"libgmp10": {"libc6"},
"libpcre2-8-0": {"libc6"},
"libselinux1": {"libc6", "libpcre2-8-0"},
},
},
{
name: "relationships from dpkg example docs",
fixture: "test-fixtures/var/lib/dpkg/status.d/doc-examples",
wantRelationships: map[string][]string{
"made-up-package-1": {"gnumach-dev", "hurd-dev", "kernel-headers-2.2.10"},
"made-up-package-2": {"liblua5.1-dev", "libluajit5.1-dev"},
"made-up-package-3": {"bar", "foo"},
// note that the "made-up-package-4" depends on "made-up-package-5" but not via the direct
// package name, but through the "provides" virtual package name "virtual-package-5".
"made-up-package-4": {"made-up-package-5"},
// note that though there is a "default-mta | mail-transport-agent | not-installed"
// dependency choice we raise up the packages that are installed for every choice.
// In this case that means that "default-mta" and "mail-transport-agent".
"mutt": {"default-mta", "libc6", "mail-transport-agent"},
},
},
{
name: "relationships for libpam-runtime",
fixture: "test-fixtures/var/lib/dpkg/status.d/libpam-runtime",
wantRelationships: map[string][]string{
"libpam-runtime": {"cdebconf", "debconf-2.0", "debconf1", "debconf2", "libpam-modules"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pkgs, relationships, err := NewDBCataloger().Catalog(context.Background(), file.NewMockResolverForPaths(tt.fixture))
require.NotEmpty(t, pkgs)
require.NotEmpty(t, relationships)
require.NoError(t, err)
if d := cmp.Diff(tt.wantRelationships, abstractRelationships(t, relationships)); d != "" {
t.Errorf("unexpected relationships (-want +got):\n%s", d)
}
})
}
}
func TestCataloger_Globs(t *testing.T) {
tests := []struct {
name string
syft/pkg/cataloger/debian/dependency.go
+66
View File
@@ -0,0 +1,66 @@
package debian
import (
"strings"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/dependency"
)
var _ dependency.Specifier = dbEntryDependencySpecifier
func dbEntryDependencySpecifier(p pkg.Package) dependency.Specification {
meta, ok := p.Metadata.(pkg.DpkgDBEntry)
if !ok {
log.Tracef("cataloger failed to extract dpkg metadata for package %+v", p.Name)
return dependency.Specification{}
}
provides := []string{p.Name}
for _, key := range meta.Provides {
if key == "" {
continue
}
provides = append(provides, stripVersionSpecifier(key))
}
var allDeps []string
allDeps = append(allDeps, meta.Depends...)
allDeps = append(allDeps, meta.PreDepends...)
var requires []string
for _, depSpecifier := range allDeps {
if depSpecifier == "" {
continue
}
requires = append(requires, splitPackageChoice(depSpecifier)...)
}
return dependency.Specification{
Provides: provides,
Requires: requires,
}
}
func stripVersionSpecifier(s string) string {
// examples:
// libgmp10 (>= 2:6.2.1+dfsg1) --> libgmp10
// libgmp10 --> libgmp10
// foo [i386] --> foo
// default-mta | mail-transport-agent --> default-mta | mail-transport-agent
// kernel-headers-2.2.10 [!hurd-i386] --> kernel-headers-2.2.10
return strings.TrimSpace(internal.SplitAny(s, "[(<>=")[0])
}
func splitPackageChoice(s string) (ret []string) {
fields := strings.Split(s, "|")
for _, field := range fields {
field = strings.TrimSpace(field)
if field != "" {
ret = append(ret, stripVersionSpecifier(field))
}
}
return ret
}
syft/pkg/cataloger/debian/dependency_test.go
+101
View File
@@ -0,0 +1,101 @@
package debian
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/dependency"
)
func Test_dbEntryDependencySpecifier(t *testing.T) {
tests := []struct {
name string
p pkg.Package
want dependency.Specification
}{
{
name: "keeps given values + package name",
p: pkg.Package{
Name: "package-c",
Metadata: pkg.DpkgDBEntry{
Provides: []string{"a-thing"},
Depends: []string{"b-thing"},
},
},
want: dependency.Specification{
Provides: []string{"package-c", "a-thing"},
Requires: []string{"b-thing"},
},
},
{
name: "strip version specifiers + split package deps",
p: pkg.Package{
Name: "package-a",
Metadata: pkg.DpkgDBEntry{
Provides: []string{"foo [i386]"},
Depends: []string{"libgmp10 (>= 2:6.2.1+dfsg1)", "default-mta | mail-transport-agent"},
},
},
want: dependency.Specification{
Provides: []string{"package-a", "foo"},
Requires: []string{"libgmp10", "default-mta", "mail-transport-agent"},
},
},
{
name: "empty dependency data entries",
p: pkg.Package{
Name: "package-a",
Metadata: pkg.DpkgDBEntry{
Provides: []string{""},
Depends: []string{""},
},
},
want: dependency.Specification{
Provides: []string{"package-a"},
Requires: nil,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, dbEntryDependencySpecifier(tt.p))
})
}
}
func Test_stripVersionSpecifier(t *testing.T) {
tests := []struct {
name string
input string
want string
}{
{
name: "package name only",
input: "test",
want: "test",
},
{
name: "with version",
input: "test (1.2.3)",
want: "test",
},
{
name: "multiple packages",
input: "test | other",
want: "test | other",
},
{
name: "with architecture specifiers",
input: "test [amd64 i386]",
want: "test",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, stripVersionSpecifier(tt.input))
})
}
}
syft/pkg/cataloger/debian/parse_dpkg_db.go
+1 -77
View File
@@ -37,7 +37,7 @@ func parseDpkgDB(_ context.Context, resolver file.Resolver, env *generic.Environ
pkgs = append(pkgs, newDpkgPackage(m, reader.Location, resolver, env.LinuxRelease))
}
return pkgs, associateRelationships(pkgs), nil
return pkgs, nil, nil
}
// parseDpkgStatus is a parser function for Debian DB status contents, returning all Debian packages listed.
@@ -238,79 +238,3 @@ func handleNewKeyValue(line string) (key string, val interface{}, err error) {
return "", nil, fmt.Errorf("cannot parse field from line: '%s'", line)
}
// associateRelationships will create relationships between packages based on the "Depends", "Pre-Depends", and "Provides"
// fields for installed packages. if there is an installed package that has a dependency that is (somehow) not installed,
// then that relationship (between the installed and uninstalled package) will NOT be created.
func associateRelationships(pkgs []pkg.Package) (relationships []artifact.Relationship) {
// map["provides" + "package"] -> packages that provide that package
lookup := make(map[string][]pkg.Package)
// read provided and add as keys for lookup keys as well as package names
for _, p := range pkgs {
meta, ok := p.Metadata.(pkg.DpkgDBEntry)
if !ok {
log.Warnf("cataloger failed to extract dpkg 'provides' metadata for package %+v", p.Name)
continue
}
lookup[p.Name] = append(lookup[p.Name], p)
for _, provides := range meta.Provides {
k := stripVersionSpecifier(provides)
lookup[k] = append(lookup[k], p)
}
}
// read "Depends" and "Pre-Depends" and match with keys
for _, p := range pkgs {
meta, ok := p.Metadata.(pkg.DpkgDBEntry)
if !ok {
log.Warnf("cataloger failed to extract dpkg 'dependency' metadata for package %+v", p.Name)
continue
}
var allDeps []string
allDeps = append(allDeps, meta.Depends...)
allDeps = append(allDeps, meta.PreDepends...)
for _, depSpecifier := range allDeps {
deps := splitPackageChoice(depSpecifier)
for _, dep := range deps {
for _, depPkg := range lookup[dep] {
relationships = append(relationships, artifact.Relationship{
From: depPkg,
To: p,
Type: artifact.DependencyOfRelationship,
})
}
}
}
}
return relationships
}
func stripVersionSpecifier(s string) string {
// examples:
// libgmp10 (>= 2:6.2.1+dfsg1) --> libgmp10
// libgmp10 --> libgmp10
// foo [i386] --> foo
// default-mta | mail-transport-agent --> default-mta | mail-transport-agent
// kernel-headers-2.2.10 [!hurd-i386] --> kernel-headers-2.2.10
items := internal.SplitAny(s, "[(<>=")
if len(items) == 0 {
return s
}
return strings.TrimSpace(items[0])
}
func splitPackageChoice(s string) (ret []string) {
fields := strings.Split(s, "|")
for _, field := range fields {
field = strings.TrimSpace(field)
if field != "" {
ret = append(ret, stripVersionSpecifier(field))
}
}
return ret
}
syft/pkg/cataloger/debian/parse_dpkg_db_test.go
+3 -103
View File
@@ -2,7 +2,6 @@ package debian
import (
"bufio"
"context"
"errors"
"fmt"
"os"
@@ -16,7 +15,6 @@ import (
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
)
@@ -28,7 +26,7 @@ func Test_parseDpkgStatus(t *testing.T) {
}{
{
name: "single package",
fixturePath: "test-fixtures/status/single",
fixturePath: "test-fixtures/var/lib/dpkg/status.d/single",
expected: []pkg.DpkgDBEntry{
{
Package: "apt",
@@ -102,7 +100,7 @@ func Test_parseDpkgStatus(t *testing.T) {
},
{
name: "single package with installed size",
fixturePath: "test-fixtures/status/installed-size-4KB",
fixturePath: "test-fixtures/var/lib/dpkg/status.d/installed-size-4KB",
expected: []pkg.DpkgDBEntry{
{
Package: "apt",
@@ -143,7 +141,7 @@ func Test_parseDpkgStatus(t *testing.T) {
},
{
name: "multiple entries",
fixturePath: "test-fixtures/status/multiple",
fixturePath: "test-fixtures/var/lib/dpkg/status.d/multiple",
expected: []pkg.DpkgDBEntry{
{
Package: "no-version",
@@ -434,104 +432,6 @@ func Test_handleNewKeyValue(t *testing.T) {
}
}
func Test_stripVersionSpecifier(t *testing.T) {
tests := []struct {
name string
input string
want string
}{
{
name: "package name only",
input: "test",
want: "test",
},
{
name: "with version",
input: "test (1.2.3)",
want: "test",
},
{
name: "multiple packages",
input: "test | other",
want: "test | other",
},
{
name: "with architecture specifiers",
input: "test [amd64 i386]",
want: "test",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, stripVersionSpecifier(tt.input))
})
}
}
func Test_associateRelationships(t *testing.T) {
tests := []struct {
name string
fixture string
wantRelationships map[string][]string
}{
{
name: "relationships for coreutils",
fixture: "test-fixtures/status/coreutils-relationships",
wantRelationships: map[string][]string{
"coreutils": {"libacl1", "libattr1", "libc6", "libgmp10", "libselinux1"},
"libacl1": {"libc6"},
"libattr1": {"libc6"},
"libc6": {"libgcc-s1"},
"libgcc-s1": {"gcc-12-base", "libc6"},
"libgmp10": {"libc6"},
"libpcre2-8-0": {"libc6"},
"libselinux1": {"libc6", "libpcre2-8-0"},
},
},
{
name: "relationships from dpkg example docs",
fixture: "test-fixtures/status/doc-examples",
wantRelationships: map[string][]string{
"made-up-package-1": {"kernel-headers-2.2.10", "hurd-dev", "gnumach-dev"},
"made-up-package-2": {"libluajit5.1-dev", "liblua5.1-dev"},
"made-up-package-3": {"foo", "bar"},
// note that the "made-up-package-4" depends on "made-up-package-5" but not via the direct
// package name, but through the "provides" virtual package name "virtual-package-5".
"made-up-package-4": {"made-up-package-5"},
// note that though there is a "default-mta | mail-transport-agent | not-installed"
// dependency choice we raise up the packages that are installed for every choice.
// In this case that means that "default-mta" and "mail-transport-agent".
"mutt": {"libc6", "default-mta", "mail-transport-agent"},
},
},
{
name: "relationships for libpam-runtime",
fixture: "test-fixtures/status/libpam-runtime",
wantRelationships: map[string][]string{
"libpam-runtime": {"debconf1", "debconf-2.0", "debconf2", "cdebconf", "libpam-modules"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f, err := os.Open(tt.fixture)
require.NoError(t, err)
reader := file.NewLocationReadCloser(file.NewLocation(tt.fixture), f)
pkgs, relationships, err := parseDpkgDB(context.Background(), nil, &generic.Environment{}, reader)
require.NotEmpty(t, pkgs)
require.NotEmpty(t, relationships)
require.NoError(t, err)
if d := cmp.Diff(tt.wantRelationships, abstractRelationships(t, relationships)); d != "" {
t.Errorf("unexpected relationships (-want +got):\n%s", d)
}
})
}
}
func abstractRelationships(t testing.TB, relationships []artifact.Relationship) map[string][]string {
t.Helper()
syft/pkg/cataloger/debian/test-fixtures/status/coreutils-relationships → syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/coreutils-relationships
View File
syft/pkg/cataloger/debian/test-fixtures/status/doc-examples → syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/doc-examples
View File
syft/pkg/cataloger/debian/test-fixtures/status/empty → syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/empty
View File
syft/pkg/cataloger/debian/test-fixtures/status/installed-size-4KB → syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/installed-size-4KB
View File
syft/pkg/cataloger/debian/test-fixtures/status/libpam-runtime → syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/libpam-runtime
View File
syft/pkg/cataloger/debian/test-fixtures/status/multiple → syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/multiple
View File
syft/pkg/cataloger/debian/test-fixtures/status/single → syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/single
View File