From 4a18895545fbf7559054f445b7ad0aaf6a16776e Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 14 May 2024 09:27:36 -0400 Subject: [PATCH] Add abstraction for adding relationships from package cataloger results (#2853) * add internal dependency resolver Signed-off-by: Alex Goodman * refactor dependency relationship resolution to common object Signed-off-by: Alex Goodman * replace cataloger decorator with generic processor Signed-off-by: Alex Goodman * refactor resolver to be a single function Signed-off-by: Alex Goodman * use common dependency specifier for debian Signed-off-by: Alex Goodman * use common dependency specifier for arch Signed-off-by: Alex Goodman * use common dependency specifier for alpine Signed-off-by: Alex Goodman * allow for generic pkg and rel assertions in testpkg helper Signed-off-by: Alex Goodman * do not allow for empty results Signed-off-by: Alex Goodman * move stable deduplicate comment Signed-off-by: Alex Goodman * remove relationship resolver type Signed-off-by: Alex Goodman --------- Signed-off-by: Alex Goodman --- internal/string_helpers.go | 6 +- internal/string_helpers_test.go | 2 +- syft/pkg/cataloger/alpine/cataloger.go | 4 +- syft/pkg/cataloger/alpine/cataloger_test.go | 230 ++++++++++ syft/pkg/cataloger/alpine/dependency.go | 48 ++ syft/pkg/cataloger/alpine/dependency_test.go | 110 +++++ syft/pkg/cataloger/alpine/parse_apk_db.go | 56 +-- .../pkg/cataloger/alpine/parse_apk_db_test.go | 410 ------------------ .../cataloger/alpine/test-fixtures/multiple | 56 --- .../multiple-1/lib/apk/db/installed | 78 ++++ .../{ => multiple-2/lib/apk/db}/installed | 0 syft/pkg/cataloger/arch/cataloger.go | 90 +--- syft/pkg/cataloger/arch/dependency.go | 48 ++ syft/pkg/cataloger/arch/dependency_test.go | 100 +++++ syft/pkg/cataloger/debian/cataloger.go | 4 +- syft/pkg/cataloger/debian/cataloger_test.go | 62 +++ syft/pkg/cataloger/debian/dependency.go | 66 +++ syft/pkg/cataloger/debian/dependency_test.go | 101 +++++ syft/pkg/cataloger/debian/parse_dpkg_db.go | 78 +--- .../cataloger/debian/parse_dpkg_db_test.go | 106 +---- .../dpkg/status.d}/coreutils-relationships | 0 .../lib/dpkg/status.d}/doc-examples | 0 .../{status => var/lib/dpkg/status.d}/empty | 0 .../lib/dpkg/status.d}/installed-size-4KB | 0 .../lib/dpkg/status.d}/libpam-runtime | 0 .../lib/dpkg/status.d}/multiple | 0 .../{status => var/lib/dpkg/status.d}/single | 0 syft/pkg/cataloger/generic/cataloger.go | 29 +- syft/pkg/cataloger/generic/cataloger_test.go | 4 +- .../cataloger/internal/dependency/resolver.go | 95 ++++ .../internal/dependency/resolver_test.go | 211 +++++++++ .../internal/pkgtest/test_generic_parser.go | 14 +- 32 files changed, 1207 insertions(+), 801 deletions(-) create mode 100644 syft/pkg/cataloger/alpine/dependency.go create mode 100644 syft/pkg/cataloger/alpine/dependency_test.go delete mode 100644 syft/pkg/cataloger/alpine/test-fixtures/multiple create mode 100644 syft/pkg/cataloger/alpine/test-fixtures/multiple-1/lib/apk/db/installed rename syft/pkg/cataloger/alpine/test-fixtures/{ => multiple-2/lib/apk/db}/installed (100%) create mode 100644 syft/pkg/cataloger/arch/dependency.go create mode 100644 syft/pkg/cataloger/arch/dependency_test.go create mode 100644 syft/pkg/cataloger/debian/dependency.go create mode 100644 syft/pkg/cataloger/debian/dependency_test.go rename syft/pkg/cataloger/debian/test-fixtures/{status => var/lib/dpkg/status.d}/coreutils-relationships (100%) rename syft/pkg/cataloger/debian/test-fixtures/{status => var/lib/dpkg/status.d}/doc-examples (100%) rename syft/pkg/cataloger/debian/test-fixtures/{status => var/lib/dpkg/status.d}/empty (100%) rename syft/pkg/cataloger/debian/test-fixtures/{status => var/lib/dpkg/status.d}/installed-size-4KB (100%) rename syft/pkg/cataloger/debian/test-fixtures/{status => var/lib/dpkg/status.d}/libpam-runtime (100%) rename syft/pkg/cataloger/debian/test-fixtures/{status => var/lib/dpkg/status.d}/multiple (100%) rename syft/pkg/cataloger/debian/test-fixtures/{status => var/lib/dpkg/status.d}/single (100%) create mode 100644 syft/pkg/cataloger/internal/dependency/resolver.go create mode 100644 syft/pkg/cataloger/internal/dependency/resolver_test.go diff --git a/internal/string_helpers.go b/internal/string_helpers.go index 4f5c65a4a..f43d5918d 100644 --- a/internal/string_helpers.go +++ b/internal/string_helpers.go @@ -33,5 +33,9 @@ func SplitAny(s string, seps string) []string { splitter := func(r rune) bool { return strings.ContainsRune(seps, r) } - return strings.FieldsFunc(s, splitter) + result := strings.FieldsFunc(s, splitter) + if len(result) == 0 { + return []string{s} + } + return result } diff --git a/internal/string_helpers_test.go b/internal/string_helpers_test.go index 45b90195a..401b28dc3 100644 --- a/internal/string_helpers_test.go +++ b/internal/string_helpers_test.go @@ -123,7 +123,7 @@ func TestSplitAny(t *testing.T) { name: "empty", input: "", fields: ",", - want: []string{}, + want: []string{""}, }, { name: "multiple separators", diff --git a/syft/pkg/cataloger/alpine/cataloger.go b/syft/pkg/cataloger/alpine/cataloger.go index 5df825741..75075108a 100644 --- a/syft/pkg/cataloger/alpine/cataloger.go +++ b/syft/pkg/cataloger/alpine/cataloger.go @@ -6,10 +6,12 @@ package alpine import ( "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" ) // NewDBCataloger returns a new cataloger object initialized for Alpine package DB flat-file stores. func NewDBCataloger() pkg.Cataloger { return generic.NewCataloger("apk-db-cataloger"). - WithParserByGlobs(parseApkDB, pkg.ApkDBGlob) + WithParserByGlobs(parseApkDB, pkg.ApkDBGlob). + WithProcessors(dependency.Processor(dbEntryDependencySpecifier)) } diff --git a/syft/pkg/cataloger/alpine/cataloger_test.go b/syft/pkg/cataloger/alpine/cataloger_test.go index d9ed0e30c..8ac6eaa8e 100644 --- a/syft/pkg/cataloger/alpine/cataloger_test.go +++ b/syft/pkg/cataloger/alpine/cataloger_test.go @@ -3,9 +3,239 @@ package alpine import ( "testing" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" ) +func TestApkDBCataloger(t *testing.T) { + dbLocation := file.NewLocation("lib/apk/db/installed") + + bashPkg := pkg.Package{ + Name: "bash", + Version: "5.2.21-r0", + Type: pkg.ApkPkg, + FoundBy: "apk-db-cataloger", + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("GPL-3.0-or-later", dbLocation), + ), + Locations: file.NewLocationSet(dbLocation), + Metadata: pkg.ApkDBEntry{ + Package: "bash", + OriginPackage: "bash", + Maintainer: "Natanael Copa ", + Version: "5.2.21-r0", + Architecture: "x86_64", + URL: "https://www.gnu.org/software/bash/bash.html", + Description: "The GNU Bourne Again shell", + Size: 448728, + InstalledSize: 1396736, + Dependencies: []string{ + "/bin/sh", "so:libc.musl-x86_64.so.1", "so:libreadline.so.8", + }, + Provides: []string{ + "cmd:bash=5.2.21-r0", + }, + // note: files not provided and not under test + }, + } + + busyboxBinshPkg := pkg.Package{ + Name: "busybox-binsh", + Version: "1.36.1-r15", + Type: pkg.ApkPkg, + FoundBy: "apk-db-cataloger", + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("GPL-2.0-only", dbLocation), + ), + Locations: file.NewLocationSet(dbLocation), + Metadata: pkg.ApkDBEntry{ + Package: "busybox-binsh", + OriginPackage: "busybox", + Maintainer: "Sören Tempel ", + Version: "1.36.1-r15", + Architecture: "x86_64", + URL: "https://busybox.net/", + Description: "busybox ash /bin/sh", + Size: 1543, + InstalledSize: 8192, + Dependencies: []string{ + "busybox=1.36.1-r15", + }, + Provides: []string{ + "/bin/sh", "cmd:sh=1.36.1-r15", + }, + // note: files not provided and not under test + }, + } + + muslPkg := pkg.Package{ + Name: "musl", + Version: "1.2.4_git20230717-r4", + Type: pkg.ApkPkg, + FoundBy: "apk-db-cataloger", + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("MIT", dbLocation), + ), + Locations: file.NewLocationSet(dbLocation), + Metadata: pkg.ApkDBEntry{ + Package: "musl", + OriginPackage: "musl", + Maintainer: "Timo Teräs ", + Version: "1.2.4_git20230717-r4", + Architecture: "x86_64", + URL: "https://musl.libc.org/", + Description: "the musl c library (libc) implementation", + Size: 407278, + InstalledSize: 667648, + Dependencies: []string{}, + Provides: []string{ + "so:libc.musl-x86_64.so.1=1", + }, + // note: files not provided and not under test + }, + } + + readlinePkg := pkg.Package{ + Name: "readline", + Version: "8.2.1-r2", + Type: pkg.ApkPkg, + FoundBy: "apk-db-cataloger", + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("GPL-2.0-or-later", dbLocation), + ), + Locations: file.NewLocationSet(dbLocation), + Metadata: pkg.ApkDBEntry{ + Package: "readline", + OriginPackage: "readline", + Maintainer: "Natanael Copa ", + Version: "8.2.1-r2", + Architecture: "x86_64", + URL: "https://tiswww.cwru.edu/php/chet/readline/rltop.html", + Description: "GNU readline library", + Size: 119878, + InstalledSize: 303104, + Dependencies: []string{ + "so:libc.musl-x86_64.so.1", "so:libncursesw.so.6", + }, + Provides: []string{ + "so:libreadline.so.8=8.2", + }, + // note: files not provided and not under test + }, + } + + expectedPkgs := []pkg.Package{ + bashPkg, + busyboxBinshPkg, + muslPkg, + readlinePkg, + } + + // # apk info --depends bash + // bash-5.2.21-r0 depends on: + // /bin/sh + // so:libc.musl-x86_64.so.1 + // so:libreadline.so.8 + // + // # apk info --who-owns /bin/sh + // /bin/sh is owned by busybox-binsh-1.36.1-r15 + // + // # find / | grep musl + // /lib/ld-musl-x86_64.so.1 + // /lib/libc.musl-x86_64.so.1 + // + // # apk info --who-owns '/lib/libc.musl-x86_64.so.1' + // /lib/libc.musl-x86_64.so.1 is owned by musl-1.2.4_git20230717-r4 + // + // # find / | grep libreadline + // /usr/lib/libreadline.so.8.2 + // /usr/lib/libreadline.so.8 + // + // # apk info --who-owns '/usr/lib/libreadline.so.8' + // /usr/lib/libreadline.so.8 is owned by readline-8.2.1-r2 + + expectedRelationships := []artifact.Relationship{ + { + From: busyboxBinshPkg, + To: bashPkg, + Type: artifact.DependencyOfRelationship, + }, + { + From: readlinePkg, + To: bashPkg, + Type: artifact.DependencyOfRelationship, + }, + { + From: muslPkg, + To: readlinePkg, + Type: artifact.DependencyOfRelationship, + }, + { + From: muslPkg, + To: bashPkg, + Type: artifact.DependencyOfRelationship, + }, + } + + pkgtest.NewCatalogTester(). + FromDirectory(t, "test-fixtures/multiple-1"). + WithCompareOptions(cmpopts.IgnoreFields(pkg.ApkDBEntry{}, "Files", "GitCommit", "Checksum")). + Expects(expectedPkgs, expectedRelationships). + TestCataloger(t, NewDBCataloger()) + +} + +func TestCatalogerDependencyTree(t *testing.T) { + assertion := func(t *testing.T, pkgs []pkg.Package, relationships []artifact.Relationship) { + expected := map[string][]string{ + "alpine-baselayout": {"busybox", "alpine-baselayout-data", "musl"}, + "apk-tools": {"ca-certificates-bundle", "musl", "libcrypto1.1", "libssl1.1", "zlib"}, + "busybox": {"musl"}, + "libc-utils": {"musl-utils"}, + "libcrypto1.1": {"musl"}, + "libssl1.1": {"musl", "libcrypto1.1"}, + "musl-utils": {"scanelf", "musl"}, + "scanelf": {"musl"}, + "ssl_client": {"musl", "libcrypto1.1", "libssl1.1"}, + "zlib": {"musl"}, + } + pkgsByID := make(map[artifact.ID]pkg.Package) + for _, p := range pkgs { + p.SetID() + pkgsByID[p.ID()] = p + } + + actualDependencies := make(map[string][]string) + + for _, r := range relationships { + switch r.Type { + case artifact.DependencyOfRelationship: + to := pkgsByID[r.To.ID()] + from := pkgsByID[r.From.ID()] + actualDependencies[to.Name] = append(actualDependencies[to.Name], from.Name) + default: + t.Fatalf("unexpected relationship type: %+v", r.Type) + } + } + + if d := cmp.Diff(expected, actualDependencies); d != "" { + t.Fail() + t.Log(d) + } + } + + pkgtest.NewCatalogTester(). + FromDirectory(t, "test-fixtures/multiple-2"). + ExpectsAssertion(assertion). + TestCataloger(t, NewDBCataloger()) + +} + func TestCataloger_Globs(t *testing.T) { tests := []struct { name string diff --git a/syft/pkg/cataloger/alpine/dependency.go b/syft/pkg/cataloger/alpine/dependency.go new file mode 100644 index 000000000..ea63f36bb --- /dev/null +++ b/syft/pkg/cataloger/alpine/dependency.go @@ -0,0 +1,48 @@ +package alpine + +import ( + "strings" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" +) + +var _ dependency.Specifier = dbEntryDependencySpecifier + +func dbEntryDependencySpecifier(p pkg.Package) dependency.Specification { + meta, ok := p.Metadata.(pkg.ApkDBEntry) + if !ok { + log.Tracef("cataloger failed to extract apk metadata for package %+v", p.Name) + return dependency.Specification{} + } + + provides := []string{p.Name} + provides = append(provides, stripVersionSpecifiers(meta.Provides)...) + + return dependency.Specification{ + Provides: provides, + Requires: stripVersionSpecifiers(meta.Dependencies), + } +} + +func stripVersionSpecifiers(given []string) []string { + var keys []string + for _, key := range given { + key = stripVersionSpecifier(key) + if key == "" { + continue + } + keys = append(keys, key) + } + return keys +} + +func stripVersionSpecifier(s string) string { + // examples: + // musl>=1 --> musl + // cmd:scanelf=1.3.4-r0 --> cmd:scanelf + + return strings.TrimSpace(internal.SplitAny(s, "<>=")[0]) +} diff --git a/syft/pkg/cataloger/alpine/dependency_test.go b/syft/pkg/cataloger/alpine/dependency_test.go new file mode 100644 index 000000000..a55cba4cb --- /dev/null +++ b/syft/pkg/cataloger/alpine/dependency_test.go @@ -0,0 +1,110 @@ +package alpine + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" +) + +func Test_dbEntryDependencySpecifier(t *testing.T) { + tests := []struct { + name string + p pkg.Package + want dependency.Specification + }{ + { + name: "keeps given values + package name", + p: pkg.Package{ + Name: "package-c", + Metadata: pkg.ApkDBEntry{ + Provides: []string{"a-thing"}, + Dependencies: []string{"b-thing"}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-c", "a-thing"}, + Requires: []string{"b-thing"}, + }, + }, + { + name: "strip version specifiers", + p: pkg.Package{ + Name: "package-a", + Metadata: pkg.ApkDBEntry{ + Provides: []string{"so:libc.musl-x86_64.so.1=1"}, + Dependencies: []string{"so:libc.musl-x86_64.so.2=2"}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-a", "so:libc.musl-x86_64.so.1"}, + Requires: []string{"so:libc.musl-x86_64.so.2"}, + }, + }, + { + name: "empty dependency data entries", + p: pkg.Package{ + Name: "package-a", + Metadata: pkg.ApkDBEntry{ + Provides: []string{""}, + Dependencies: []string{""}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-a"}, + Requires: nil, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, dbEntryDependencySpecifier(tt.p)) + }) + } +} + +func Test_stripVersionSpecifier(t *testing.T) { + tests := []struct { + name string + version string + want string + }{ + { + name: "empty expression", + version: "", + want: "", + }, + { + name: "no expression", + version: "cmd:foo", + want: "cmd:foo", + }, + { + name: "=", + version: "cmd:scanelf=1.3.4-r0", + want: "cmd:scanelf", + }, + { + name: ">=", + version: "cmd:scanelf>=1.3.4-r0", + want: "cmd:scanelf", + }, + { + name: "<", + version: "cmd:scanelf<1.3.4-r0", + want: "cmd:scanelf", + }, + { + name: "ignores file paths", + version: "/bin/sh", + want: "/bin/sh", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, stripVersionSpecifier(tt.version)) + }) + } +} diff --git a/syft/pkg/cataloger/alpine/parse_apk_db.go b/syft/pkg/cataloger/alpine/parse_apk_db.go index 5952ec16c..5948e3035 100644 --- a/syft/pkg/cataloger/alpine/parse_apk_db.go +++ b/syft/pkg/cataloger/alpine/parse_apk_db.go @@ -131,7 +131,7 @@ func parseApkDB(_ context.Context, resolver file.Resolver, env *generic.Environm pkgs = append(pkgs, newPackage(apk, r, reader.Location)) } - return pkgs, discoverPackageDependencies(pkgs), nil + return pkgs, nil, nil } func findReleases(resolver file.Resolver, dbPath string) []linux.Release { @@ -386,57 +386,3 @@ func processChecksum(value string) *file.Digest { Value: value, } } - -func discoverPackageDependencies(pkgs []pkg.Package) (relationships []artifact.Relationship) { - // map["provides" string] -> packages that provide the "p" key - lookup := make(map[string][]pkg.Package) - // read "Provides" (p) and add as keys for lookup keys as well as package names - for _, p := range pkgs { - apkg, ok := p.Metadata.(pkg.ApkDBEntry) - if !ok { - log.Warnf("cataloger failed to extract apk 'provides' metadata for package %+v", p.Name) - continue - } - lookup[p.Name] = append(lookup[p.Name], p) - for _, provides := range apkg.Provides { - k := stripVersionSpecifier(provides) - lookup[k] = append(lookup[k], p) - } - } - - // read "Pull Dependencies" (D) and match with keys - for _, p := range pkgs { - apkg, ok := p.Metadata.(pkg.ApkDBEntry) - if !ok { - log.Warnf("cataloger failed to extract apk dependency metadata for package %+v", p.Name) - continue - } - - for _, depSpecifier := range apkg.Dependencies { - // use the lookup to find what pkg we depend on - dep := stripVersionSpecifier(depSpecifier) - for _, depPkg := range lookup[dep] { - // this is a pkg that package "p" depends on... make a relationship - relationships = append(relationships, artifact.Relationship{ - From: depPkg, - To: p, - Type: artifact.DependencyOfRelationship, - }) - } - } - } - return relationships -} - -func stripVersionSpecifier(s string) string { - // examples: - // musl>=1 --> musl - // cmd:scanelf=1.3.4-r0 --> cmd:scanelf - - items := internal.SplitAny(s, "<>=") - if len(items) == 0 { - return s - } - - return items[0] -} diff --git a/syft/pkg/cataloger/alpine/parse_apk_db_test.go b/syft/pkg/cataloger/alpine/parse_apk_db_test.go index fef3275ab..7918f38f6 100644 --- a/syft/pkg/cataloger/alpine/parse_apk_db_test.go +++ b/syft/pkg/cataloger/alpine/parse_apk_db_test.go @@ -9,11 +9,9 @@ import ( "testing" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/linux" "github.com/anchore/syft/syft/pkg" @@ -689,144 +687,6 @@ func TestSinglePackageDetails(t *testing.T) { } } -func TestMultiplePackages(t *testing.T) { - fixture := "test-fixtures/multiple" - location := file.NewLocation(fixture) - fixtureLocationSet := file.NewLocationSet(location) - expectedPkgs := []pkg.Package{ - { - Name: "libc-utils", - Version: "0.7.2-r0", - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("MPL-2.0 AND MIT", location), - ), - Type: pkg.ApkPkg, - PURL: "pkg:apk/alpine/libc-utils@0.7.2-r0?arch=x86_64&upstream=libc-dev&distro=alpine-3.12", - Locations: fixtureLocationSet, - Metadata: pkg.ApkDBEntry{ - Package: "libc-utils", - OriginPackage: "libc-dev", - Maintainer: "Natanael Copa ", - Version: "0.7.2-r0", - Architecture: "x86_64", - URL: "http://alpinelinux.org", - Description: "Meta package to pull in correct libc", - Size: 1175, - InstalledSize: 4096, - Checksum: "Q1p78yvTLG094tHE1+dToJGbmYzQE=", - GitCommit: "97b1c2842faa3bfa30f5811ffbf16d5ff9f1a479", - Dependencies: []string{"musl-utils"}, - Provides: []string{}, - Files: []pkg.ApkFileRecord{}, - }, - }, - { - Name: "musl-utils", - Version: "1.1.24-r2", - Type: pkg.ApkPkg, - PURL: "pkg:apk/alpine/musl-utils@1.1.24-r2?arch=x86_64&upstream=musl&distro=alpine-3.12", - Locations: fixtureLocationSet, - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("MIT", location), - pkg.NewLicenseFromLocations("BSD", location), - pkg.NewLicenseFromLocations("GPL2+", location), - ), - Metadata: pkg.ApkDBEntry{ - Package: "musl-utils", - OriginPackage: "musl", - Version: "1.1.24-r2", - Description: "the musl c library (libc) implementation", - Maintainer: "Timo Teräs ", - Architecture: "x86_64", - URL: "https://musl.libc.org/", - Size: 37944, - InstalledSize: 151552, - GitCommit: "4024cc3b29ad4c65544ad068b8f59172b5494306", - Dependencies: []string{"scanelf", "so:libc.musl-x86_64.so.1"}, - Provides: []string{"cmd:getconf", "cmd:getent", "cmd:iconv", "cmd:ldconfig", "cmd:ldd"}, - Checksum: "Q1bTtF5526tETKfL+lnigzIDvm+2o=", - Files: []pkg.ApkFileRecord{ - { - Path: "/sbin", - }, - { - Path: "/sbin/ldconfig", - OwnerUID: "0", - OwnerGID: "0", - Permissions: "755", - Digest: &file.Digest{ - Algorithm: "'Q1'+base64(sha1)", - Value: "Q1Kja2+POZKxEkUOZqwSjC6kmaED4=", - }, - }, - { - Path: "/usr", - }, - { - Path: "/usr/bin", - }, - { - Path: "/usr/bin/iconv", - OwnerUID: "0", - OwnerGID: "0", - Permissions: "755", - Digest: &file.Digest{ - Algorithm: "'Q1'+base64(sha1)", - Value: "Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY=", - }, - }, - { - Path: "/usr/bin/ldd", - OwnerUID: "0", - OwnerGID: "0", - Permissions: "755", - Digest: &file.Digest{ - Algorithm: "'Q1'+base64(sha1)", - Value: "Q1yFAhGggmL7ERgbIA7KQxyTzf3ks=", - }, - }, - { - Path: "/usr/bin/getconf", - OwnerUID: "0", - OwnerGID: "0", - Permissions: "755", - Digest: &file.Digest{ - Algorithm: "'Q1'+base64(sha1)", - Value: "Q1dAdYK8M/INibRQF5B3Rw7cmNDDA=", - }, - }, - { - Path: "/usr/bin/getent", - OwnerUID: "0", - OwnerGID: "0", - Permissions: "755", - Digest: &file.Digest{ - Algorithm: "'Q1'+base64(sha1)", - Value: "Q1eR2Dz/WylabgbWMTkd2+hGmEya4=", - }, - }, - }, - }, - }, - } - - expectedRelationships := []artifact.Relationship{ - { - From: expectedPkgs[1], // musl-utils - To: expectedPkgs[0], // libc-utils - Type: artifact.DependencyOfRelationship, - Data: nil, - }, - } - - env := generic.Environment{LinuxRelease: &linux.Release{ - ID: "alpine", - VersionID: "3.12", - }} - - pkgtest.TestFileParserWithEnv(t, fixture, parseApkDB, &env, expectedPkgs, expectedRelationships) -} - func Test_processChecksum(t *testing.T) { tests := []struct { name string @@ -858,237 +718,6 @@ func Test_processChecksum(t *testing.T) { } } -func Test_discoverPackageDependencies(t *testing.T) { - tests := []struct { - name string - genFn func() ([]pkg.Package, []artifact.Relationship) - }{ - { - name: "has no dependency", - genFn: func() ([]pkg.Package, []artifact.Relationship) { - a := pkg.Package{ - Name: "package-a", - Metadata: pkg.ApkDBEntry{ - Provides: []string{"a-thing"}, - }, - } - a.SetID() - b := pkg.Package{ - Name: "package-b", - Metadata: pkg.ApkDBEntry{ - Provides: []string{"b-thing"}, - }, - } - b.SetID() - - return []pkg.Package{a, b}, nil - }, - }, - { - name: "has 1 dependency", - genFn: func() ([]pkg.Package, []artifact.Relationship) { - a := pkg.Package{ - Name: "package-a", - Metadata: pkg.ApkDBEntry{ - Dependencies: []string{"b-thing"}, - }, - } - a.SetID() - b := pkg.Package{ - Name: "package-b", - Metadata: pkg.ApkDBEntry{ - Provides: []string{"b-thing"}, - }, - } - b.SetID() - - return []pkg.Package{a, b}, []artifact.Relationship{ - { - From: b, - To: a, - Type: artifact.DependencyOfRelationship, - }, - } - }, - }, - { - name: "strip version specifiers", - genFn: func() ([]pkg.Package, []artifact.Relationship) { - a := pkg.Package{ - Name: "package-a", - Metadata: pkg.ApkDBEntry{ - Dependencies: []string{"so:libc.musl-x86_64.so.1"}, - }, - } - a.SetID() - b := pkg.Package{ - Name: "package-b", - Metadata: pkg.ApkDBEntry{ - Provides: []string{"so:libc.musl-x86_64.so.1=1"}, - }, - } - b.SetID() - - return []pkg.Package{a, b}, []artifact.Relationship{ - { - From: b, - To: a, - Type: artifact.DependencyOfRelationship, - }, - } - }, - }, - { - name: "strip version specifiers with empty provides value", - genFn: func() ([]pkg.Package, []artifact.Relationship) { - a := pkg.Package{ - Name: "package-a", - Metadata: pkg.ApkDBEntry{ - Dependencies: []string{"so:libc.musl-x86_64.so.1"}, - }, - } - a.SetID() - b := pkg.Package{ - Name: "package-b", - Metadata: pkg.ApkDBEntry{ - Provides: []string{""}, - }, - } - b.SetID() - - return []pkg.Package{a, b}, nil - }, - }, - { - name: "depends on package name", - genFn: func() ([]pkg.Package, []artifact.Relationship) { - a := pkg.Package{ - Name: "package-a", - Metadata: pkg.ApkDBEntry{ - Dependencies: []string{"musl>=1.2"}, - }, - } - a.SetID() - b := pkg.Package{ - Name: "musl", - Metadata: pkg.ApkDBEntry{ - Provides: []string{"so:libc.musl-x86_64.so.1=1"}, - }, - } - b.SetID() - - return []pkg.Package{a, b}, []artifact.Relationship{ - { - From: b, - To: a, - Type: artifact.DependencyOfRelationship, - }, - } - }, - }, - { - name: "depends on package file", - genFn: func() ([]pkg.Package, []artifact.Relationship) { - a := pkg.Package{ - Name: "alpine-baselayout", - Metadata: pkg.ApkDBEntry{ - Dependencies: []string{"/bin/sh"}, - }, - } - a.SetID() - b := pkg.Package{ - Name: "busybox", - Metadata: pkg.ApkDBEntry{ - Provides: []string{"/bin/sh"}, - }, - } - b.SetID() - - return []pkg.Package{a, b}, []artifact.Relationship{ - { - From: b, - To: a, - Type: artifact.DependencyOfRelationship, - }, - } - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - pkgs, wantRelationships := test.genFn() - gotRelationships := discoverPackageDependencies(pkgs) - d := cmp.Diff(wantRelationships, gotRelationships, cmpopts.IgnoreUnexported(pkg.Package{}, file.LocationSet{}, pkg.LicenseSet{})) - if d != "" { - t.Fail() - t.Log(d) - } - }) - } -} - -func TestPackageDbDependenciesByParse(t *testing.T) { - tests := []struct { - fixture string - expected map[string][]string - }{ - { - fixture: "test-fixtures/installed", - expected: map[string][]string{ - "alpine-baselayout": {"alpine-baselayout-data", "busybox", "musl"}, - "apk-tools": {"musl", "ca-certificates-bundle", "musl", "libcrypto1.1", "libssl1.1", "zlib"}, - "busybox": {"musl"}, - "libc-utils": {"musl-utils"}, - "libcrypto1.1": {"musl"}, - "libssl1.1": {"musl", "libcrypto1.1"}, - "musl-utils": {"scanelf", "musl"}, - "scanelf": {"musl"}, - "ssl_client": {"musl", "libcrypto1.1", "libssl1.1"}, - "zlib": {"musl"}, - }, - }, - } - - for _, test := range tests { - t.Run(test.fixture, func(t *testing.T) { - f, err := os.Open(test.fixture) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, f.Close()) }) - - pkgs, relationships, err := parseApkDB(context.Background(), nil, nil, file.LocationReadCloser{ - Location: file.NewLocation(test.fixture), - ReadCloser: f, - }) - require.NoError(t, err) - - pkgsByID := make(map[artifact.ID]pkg.Package) - for _, p := range pkgs { - p.SetID() - pkgsByID[p.ID()] = p - } - - actualDependencies := make(map[string][]string) - - for _, r := range relationships { - switch r.Type { - case artifact.DependencyOfRelationship: - to := pkgsByID[r.To.ID()] - from := pkgsByID[r.From.ID()] - actualDependencies[to.Name] = append(actualDependencies[to.Name], from.Name) - default: - t.Fatalf("unexpected relationship type: %+v", r.Type) - } - } - - if d := cmp.Diff(test.expected, actualDependencies); d != "" { - t.Fail() - t.Log(d) - } - }) - } -} - func Test_parseApkDB_expectedPkgNames(t *testing.T) { tests := []struct { fixture string @@ -1175,45 +804,6 @@ func newLocationReadCloser(t *testing.T, path string) file.LocationReadCloser { return file.NewLocationReadCloser(file.NewLocation(path), f) } -func Test_stripVersionSpecifier(t *testing.T) { - tests := []struct { - name string - version string - want string - }{ - { - name: "empty expression", - version: "", - want: "", - }, - { - name: "no expression", - version: "cmd:foo", - want: "cmd:foo", - }, - { - name: "=", - version: "cmd:scanelf=1.3.4-r0", - want: "cmd:scanelf", - }, - { - name: ">=", - version: "cmd:scanelf>=1.3.4-r0", - want: "cmd:scanelf", - }, - { - name: "<", - version: "cmd:scanelf<1.3.4-r0", - want: "cmd:scanelf", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, stripVersionSpecifier(tt.version)) - }) - } -} - func TestParseReleasesFromAPKRepository(t *testing.T) { tests := []struct { repos string diff --git a/syft/pkg/cataloger/alpine/test-fixtures/multiple b/syft/pkg/cataloger/alpine/test-fixtures/multiple deleted file mode 100644 index 7bf964cf8..000000000 --- a/syft/pkg/cataloger/alpine/test-fixtures/multiple +++ /dev/null @@ -1,56 +0,0 @@ -C:Q1p78yvTLG094tHE1+dToJGbmYzQE= -P:libc-utils -V:0.7.2-r0 -A:x86_64 -S:1175 -I:4096 -T:Meta package to pull in correct libc -U:http://alpinelinux.org -L:MPL-2.0 AND MIT -o:libc-dev -m:Natanael Copa -t:1575749004 -c:97b1c2842faa3bfa30f5811ffbf16d5ff9f1a479 -D:musl-utils - - - - - - - - -C:Q1bTtF5526tETKfL+lnigzIDvm+2o= -P:musl-utils -V:1.1.24-r2 -A:x86_64 -S:37944 -I:151552 -T:the musl c library (libc) implementation -U:https://musl.libc.org/ -L:MIT BSD GPL2+ -o:musl -m:Timo Teräs -t:1584790550 -c:4024cc3b29ad4c65544ad068b8f59172b5494306 -D:scanelf so:libc.musl-x86_64.so.1 -p:cmd:getconf cmd:getent cmd:iconv cmd:ldconfig cmd:ldd -r:libiconv -F:sbin -R:ldconfig -a:0:0:755 -Z:Q1Kja2+POZKxEkUOZqwSjC6kmaED4= -F:usr -F:usr/bin -R:iconv -a:0:0:755 -Z:Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY= -R:ldd -a:0:0:755 -Z:Q1yFAhGggmL7ERgbIA7KQxyTzf3ks= -R:getconf -a:0:0:755 -Z:Q1dAdYK8M/INibRQF5B3Rw7cmNDDA= -R:getent -a:0:0:755 -Z:Q1eR2Dz/WylabgbWMTkd2+hGmEya4= \ No newline at end of file diff --git a/syft/pkg/cataloger/alpine/test-fixtures/multiple-1/lib/apk/db/installed b/syft/pkg/cataloger/alpine/test-fixtures/multiple-1/lib/apk/db/installed new file mode 100644 index 000000000..1a93d9e30 --- /dev/null +++ b/syft/pkg/cataloger/alpine/test-fixtures/multiple-1/lib/apk/db/installed @@ -0,0 +1,78 @@ +P:bash +V:5.2.21-r0 +A:x86_64 +S:448728 +I:1396736 +T:The GNU Bourne Again shell +U:https://www.gnu.org/software/bash/bash.html +L:GPL-3.0-or-later +o:bash +m:Natanael Copa +t:1701073495 +c:6a9559d98850225ba80771901ef1abda91cb29aa +D:/bin/sh so:libc.musl-x86_64.so.1 so:libreadline.so.8 +p:cmd:bash=5.2.21-r0 + +P:busybox-binsh +V:1.36.1-r15 +A:x86_64 +S:1543 +I:8192 +T:busybox ash /bin/sh +U:https://busybox.net/ +L:GPL-2.0-only +o:busybox +m:Sören Tempel +t:1699383189 +c:d1b6f274f29076967826e0ecf6ebcaa5d360272f +k:100 +D:busybox=1.36.1-r15 +p:/bin/sh cmd:sh=1.36.1-r15 +r:busybox-initscripts +F:bin +R:sh +a:0:0:777 +Z:Q1pcfTfDNEbNKQc2s1tia7da05M8Q= + +P:musl +V:1.2.4_git20230717-r4 +A:x86_64 +S:407278 +I:667648 +T:the musl c library (libc) implementation +U:https://musl.libc.org/ +L:MIT +o:musl +m:Timo Teräs +t:1699271358 +c:ca7f2ab5e88794e4e654b40776f8a92256f50639 +p:so:libc.musl-x86_64.so.1=1 +F:lib +R:ld-musl-x86_64.so.1 +a:0:0:755 +Z:Q1+zEJiG53Cxy7DkV5oZQqeWnzybY= +R:libc.musl-x86_64.so.1 +a:0:0:777 +Z:Q17yJ3JFNypA4mxhJJr0ou6CzsJVI= + +P:readline +V:8.2.1-r2 +A:x86_64 +S:119878 +I:303104 +T:GNU readline library +U:https://tiswww.cwru.edu/php/chet/readline/rltop.html +L:GPL-2.0-or-later +o:readline +m:Natanael Copa +t:1684120357 +c:33283848034c9885d984c8e8697c645c57324938 +D:so:libc.musl-x86_64.so.1 so:libncursesw.so.6 +p:so:libreadline.so.8=8.2 +F:etc +R:inputrc +Z:Q1ilcgkuEseXEH6iMo9UNjLn1pPfg= +F:usr +F:usr/lib +R:libreadline.so.8 +a:0:0:777 diff --git a/syft/pkg/cataloger/alpine/test-fixtures/installed b/syft/pkg/cataloger/alpine/test-fixtures/multiple-2/lib/apk/db/installed similarity index 100% rename from syft/pkg/cataloger/alpine/test-fixtures/installed rename to syft/pkg/cataloger/alpine/test-fixtures/multiple-2/lib/apk/db/installed diff --git a/syft/pkg/cataloger/arch/cataloger.go b/syft/pkg/cataloger/arch/cataloger.go index 920dda60a..75153bdd4 100644 --- a/syft/pkg/cataloger/arch/cataloger.go +++ b/syft/pkg/cataloger/arch/cataloger.go @@ -4,96 +4,14 @@ Package arch provides a concrete Cataloger implementations for packages relating package arch import ( - "context" - "strings" - - "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/artifact" - "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" ) -type cataloger struct { - *generic.Cataloger -} - // NewDBCataloger returns a new cataloger object initialized for arch linux pacman database flat-file stores. func NewDBCataloger() pkg.Cataloger { - return cataloger{ - Cataloger: generic.NewCataloger("alpm-db-cataloger"). - WithParserByGlobs(parseAlpmDB, pkg.AlpmDBGlob), - } -} - -func (c cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { - pkgs, rels, err := c.Cataloger.Catalog(ctx, resolver) - if err != nil { - return nil, nil, err - } - - rels = append(rels, associateRelationships(pkgs)...) - - return pkgs, rels, nil -} - -// associateRelationships will create relationships between packages based on the "Depends" and "Provides" -// fields for installed packages. If there is an installed package that has a dependency that is (somehow) not installed, -// then that relationship (between the installed and uninstalled package) will NOT be created. -func associateRelationships(pkgs []pkg.Package) (relationships []artifact.Relationship) { - // map["provides" + "package"] -> packages that provide that package - lookup := make(map[string][]pkg.Package) - - // read providers and add lookup keys as needed - for _, p := range pkgs { - meta, ok := p.Metadata.(pkg.AlpmDBEntry) - if !ok { - log.Warnf("cataloger failed to extract alpm 'provides' metadata for package %+v", p.Name) - continue - } - // allow for lookup by package name - lookup[p.Name] = append(lookup[p.Name], p) - - for _, provides := range meta.Provides { - // allow for lookup by exact specification - lookup[provides] = append(lookup[provides], p) - - // allow for lookup by library name only - k := stripVersionSpecifier(provides) - lookup[k] = append(lookup[k], p) - } - } - - // read "Depends" and match with provider keys - for _, p := range pkgs { - meta, ok := p.Metadata.(pkg.AlpmDBEntry) - if !ok { - log.Warnf("cataloger failed to extract alpm 'dependency' metadata for package %+v", p.Name) - continue - } - - for _, dep := range meta.Depends { - for _, depPkg := range lookup[dep] { - relationships = append(relationships, artifact.Relationship{ - From: depPkg, - To: p, - Type: artifact.DependencyOfRelationship, - }) - } - } - } - return relationships -} - -func stripVersionSpecifier(s string) string { - // examples: - // gcc-libs --> gcc-libs - // libtree-sitter.so=0-64 --> libtree-sitter.so - - items := strings.Split(s, "=") - if len(items) == 0 { - return s - } - - return strings.TrimSpace(items[0]) + return generic.NewCataloger("alpm-db-cataloger"). + WithParserByGlobs(parseAlpmDB, pkg.AlpmDBGlob). + WithProcessors(dependency.Processor(dbEntryDependencySpecifier)) } diff --git a/syft/pkg/cataloger/arch/dependency.go b/syft/pkg/cataloger/arch/dependency.go new file mode 100644 index 000000000..796a61e1b --- /dev/null +++ b/syft/pkg/cataloger/arch/dependency.go @@ -0,0 +1,48 @@ +package arch + +import ( + "strings" + + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" +) + +var _ dependency.Specifier = dbEntryDependencySpecifier + +func dbEntryDependencySpecifier(p pkg.Package) dependency.Specification { + meta, ok := p.Metadata.(pkg.AlpmDBEntry) + if !ok { + log.Tracef("cataloger failed to extract alpm metadata for package %+v", p.Name) + return dependency.Specification{} + } + + provides := []string{p.Name} + for _, key := range meta.Provides { + if key == "" { + continue + } + provides = append(provides, key, stripVersionSpecifier(key)) + } + + var requires []string + for _, depSpecifier := range meta.Depends { + if depSpecifier == "" { + continue + } + requires = append(requires, depSpecifier) + } + + return dependency.Specification{ + Provides: provides, + Requires: requires, + } +} + +func stripVersionSpecifier(s string) string { + // examples: + // gcc-libs --> gcc-libs + // libtree-sitter.so=0-64 --> libtree-sitter.so + + return strings.TrimSpace(strings.Split(s, "=")[0]) +} diff --git a/syft/pkg/cataloger/arch/dependency_test.go b/syft/pkg/cataloger/arch/dependency_test.go new file mode 100644 index 000000000..e4eafae81 --- /dev/null +++ b/syft/pkg/cataloger/arch/dependency_test.go @@ -0,0 +1,100 @@ +package arch + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" +) + +func Test_dbEntryDependencySpecifier(t *testing.T) { + tests := []struct { + name string + p pkg.Package + want dependency.Specification + }{ + { + name: "keeps given values + package name", + p: pkg.Package{ + Name: "package-c", + Metadata: pkg.AlpmDBEntry{ + Provides: []string{"a-thing"}, + Depends: []string{"b-thing"}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-c", "a-thing", "a-thing"}, // note: gets deduplicated downstream + Requires: []string{"b-thing"}, + }, + }, + { + name: "strip version specifiers", + p: pkg.Package{ + Name: "package-a", + Metadata: pkg.AlpmDBEntry{ + Provides: []string{"libtree-sitter.so.me=1-64"}, + Depends: []string{"libtree-sitter.so.thing=2-64"}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-a", "libtree-sitter.so.me=1-64", "libtree-sitter.so.me"}, + Requires: []string{"libtree-sitter.so.thing=2-64"}, + }, + }, + { + name: "empty dependency data entries", + p: pkg.Package{ + Name: "package-a", + Metadata: pkg.AlpmDBEntry{ + Provides: []string{""}, + Depends: []string{""}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-a"}, + Requires: nil, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, dbEntryDependencySpecifier(tt.p)) + }) + } +} + +func Test_stripVersionSpecifier(t *testing.T) { + tests := []struct { + name string + version string + want string + }{ + { + name: "empty expression", + version: "", + want: "", + }, + { + name: "no expression", + version: "gcc-libs", + want: "gcc-libs", + }, + { + name: "=", + version: "libtree-sitter.so=0-64", + want: "libtree-sitter.so", + }, + { + name: "ignores file paths", + version: "/bin/sh", + want: "/bin/sh", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, stripVersionSpecifier(tt.version)) + }) + } +} diff --git a/syft/pkg/cataloger/debian/cataloger.go b/syft/pkg/cataloger/debian/cataloger.go index 713699843..d6876981c 100644 --- a/syft/pkg/cataloger/debian/cataloger.go +++ b/syft/pkg/cataloger/debian/cataloger.go @@ -6,6 +6,7 @@ package debian import ( "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" ) // NewDBCataloger returns a new Deb package cataloger capable of parsing DPKG status DB flat-file stores. @@ -13,5 +14,6 @@ func NewDBCataloger() pkg.Cataloger { return generic.NewCataloger("dpkg-db-cataloger"). // note: these globs have been intentionally split up in order to improve search performance, // please do NOT combine into: "**/var/lib/dpkg/{status,status.d/*}" - WithParserByGlobs(parseDpkgDB, "**/var/lib/dpkg/status", "**/var/lib/dpkg/status.d/*", "**/lib/opkg/info/*.control", "**/lib/opkg/status") + WithParserByGlobs(parseDpkgDB, "**/var/lib/dpkg/status", "**/var/lib/dpkg/status.d/*", "**/lib/opkg/info/*.control", "**/lib/opkg/status"). + WithProcessors(dependency.Processor(dbEntryDependencySpecifier)) } diff --git a/syft/pkg/cataloger/debian/cataloger_test.go b/syft/pkg/cataloger/debian/cataloger_test.go index 5d3addcbf..0f8a70a9a 100644 --- a/syft/pkg/cataloger/debian/cataloger_test.go +++ b/syft/pkg/cataloger/debian/cataloger_test.go @@ -1,8 +1,12 @@ package debian import ( + "context" "testing" + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" @@ -160,6 +164,64 @@ func TestDpkgCataloger(t *testing.T) { } } +func Test_CatalogerRelationships(t *testing.T) { + tests := []struct { + name string + fixture string + wantRelationships map[string][]string + }{ + { + name: "relationships for coreutils", + fixture: "test-fixtures/var/lib/dpkg/status.d/coreutils-relationships", + wantRelationships: map[string][]string{ + "coreutils": {"libacl1", "libattr1", "libc6", "libgmp10", "libselinux1"}, + "libacl1": {"libc6"}, + "libattr1": {"libc6"}, + "libc6": {"libgcc-s1"}, + "libgcc-s1": {"gcc-12-base", "libc6"}, + "libgmp10": {"libc6"}, + "libpcre2-8-0": {"libc6"}, + "libselinux1": {"libc6", "libpcre2-8-0"}, + }, + }, + { + name: "relationships from dpkg example docs", + fixture: "test-fixtures/var/lib/dpkg/status.d/doc-examples", + wantRelationships: map[string][]string{ + "made-up-package-1": {"gnumach-dev", "hurd-dev", "kernel-headers-2.2.10"}, + "made-up-package-2": {"liblua5.1-dev", "libluajit5.1-dev"}, + "made-up-package-3": {"bar", "foo"}, + // note that the "made-up-package-4" depends on "made-up-package-5" but not via the direct + // package name, but through the "provides" virtual package name "virtual-package-5". + "made-up-package-4": {"made-up-package-5"}, + // note that though there is a "default-mta | mail-transport-agent | not-installed" + // dependency choice we raise up the packages that are installed for every choice. + // In this case that means that "default-mta" and "mail-transport-agent". + "mutt": {"default-mta", "libc6", "mail-transport-agent"}, + }, + }, + { + name: "relationships for libpam-runtime", + fixture: "test-fixtures/var/lib/dpkg/status.d/libpam-runtime", + wantRelationships: map[string][]string{ + "libpam-runtime": {"cdebconf", "debconf-2.0", "debconf1", "debconf2", "libpam-modules"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pkgs, relationships, err := NewDBCataloger().Catalog(context.Background(), file.NewMockResolverForPaths(tt.fixture)) + require.NotEmpty(t, pkgs) + require.NotEmpty(t, relationships) + require.NoError(t, err) + + if d := cmp.Diff(tt.wantRelationships, abstractRelationships(t, relationships)); d != "" { + t.Errorf("unexpected relationships (-want +got):\n%s", d) + } + }) + } +} + func TestCataloger_Globs(t *testing.T) { tests := []struct { name string diff --git a/syft/pkg/cataloger/debian/dependency.go b/syft/pkg/cataloger/debian/dependency.go new file mode 100644 index 000000000..9fe39899e --- /dev/null +++ b/syft/pkg/cataloger/debian/dependency.go @@ -0,0 +1,66 @@ +package debian + +import ( + "strings" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" +) + +var _ dependency.Specifier = dbEntryDependencySpecifier + +func dbEntryDependencySpecifier(p pkg.Package) dependency.Specification { + meta, ok := p.Metadata.(pkg.DpkgDBEntry) + if !ok { + log.Tracef("cataloger failed to extract dpkg metadata for package %+v", p.Name) + return dependency.Specification{} + } + provides := []string{p.Name} + for _, key := range meta.Provides { + if key == "" { + continue + } + provides = append(provides, stripVersionSpecifier(key)) + } + + var allDeps []string + allDeps = append(allDeps, meta.Depends...) + allDeps = append(allDeps, meta.PreDepends...) + + var requires []string + for _, depSpecifier := range allDeps { + if depSpecifier == "" { + continue + } + requires = append(requires, splitPackageChoice(depSpecifier)...) + } + + return dependency.Specification{ + Provides: provides, + Requires: requires, + } +} + +func stripVersionSpecifier(s string) string { + // examples: + // libgmp10 (>= 2:6.2.1+dfsg1) --> libgmp10 + // libgmp10 --> libgmp10 + // foo [i386] --> foo + // default-mta | mail-transport-agent --> default-mta | mail-transport-agent + // kernel-headers-2.2.10 [!hurd-i386] --> kernel-headers-2.2.10 + + return strings.TrimSpace(internal.SplitAny(s, "[(<>=")[0]) +} + +func splitPackageChoice(s string) (ret []string) { + fields := strings.Split(s, "|") + for _, field := range fields { + field = strings.TrimSpace(field) + if field != "" { + ret = append(ret, stripVersionSpecifier(field)) + } + } + return ret +} diff --git a/syft/pkg/cataloger/debian/dependency_test.go b/syft/pkg/cataloger/debian/dependency_test.go new file mode 100644 index 000000000..5e2225748 --- /dev/null +++ b/syft/pkg/cataloger/debian/dependency_test.go @@ -0,0 +1,101 @@ +package debian + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" +) + +func Test_dbEntryDependencySpecifier(t *testing.T) { + tests := []struct { + name string + p pkg.Package + want dependency.Specification + }{ + { + name: "keeps given values + package name", + p: pkg.Package{ + Name: "package-c", + Metadata: pkg.DpkgDBEntry{ + Provides: []string{"a-thing"}, + Depends: []string{"b-thing"}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-c", "a-thing"}, + Requires: []string{"b-thing"}, + }, + }, + { + name: "strip version specifiers + split package deps", + p: pkg.Package{ + Name: "package-a", + Metadata: pkg.DpkgDBEntry{ + Provides: []string{"foo [i386]"}, + Depends: []string{"libgmp10 (>= 2:6.2.1+dfsg1)", "default-mta | mail-transport-agent"}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-a", "foo"}, + Requires: []string{"libgmp10", "default-mta", "mail-transport-agent"}, + }, + }, + { + name: "empty dependency data entries", + p: pkg.Package{ + Name: "package-a", + Metadata: pkg.DpkgDBEntry{ + Provides: []string{""}, + Depends: []string{""}, + }, + }, + want: dependency.Specification{ + Provides: []string{"package-a"}, + Requires: nil, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, dbEntryDependencySpecifier(tt.p)) + }) + } +} + +func Test_stripVersionSpecifier(t *testing.T) { + + tests := []struct { + name string + input string + want string + }{ + { + name: "package name only", + input: "test", + want: "test", + }, + { + name: "with version", + input: "test (1.2.3)", + want: "test", + }, + { + name: "multiple packages", + input: "test | other", + want: "test | other", + }, + { + name: "with architecture specifiers", + input: "test [amd64 i386]", + want: "test", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, stripVersionSpecifier(tt.input)) + }) + } +} diff --git a/syft/pkg/cataloger/debian/parse_dpkg_db.go b/syft/pkg/cataloger/debian/parse_dpkg_db.go index 4fb41323c..077e5bef9 100644 --- a/syft/pkg/cataloger/debian/parse_dpkg_db.go +++ b/syft/pkg/cataloger/debian/parse_dpkg_db.go @@ -37,7 +37,7 @@ func parseDpkgDB(_ context.Context, resolver file.Resolver, env *generic.Environ pkgs = append(pkgs, newDpkgPackage(m, reader.Location, resolver, env.LinuxRelease)) } - return pkgs, associateRelationships(pkgs), nil + return pkgs, nil, nil } // parseDpkgStatus is a parser function for Debian DB status contents, returning all Debian packages listed. @@ -238,79 +238,3 @@ func handleNewKeyValue(line string) (key string, val interface{}, err error) { return "", nil, fmt.Errorf("cannot parse field from line: '%s'", line) } - -// associateRelationships will create relationships between packages based on the "Depends", "Pre-Depends", and "Provides" -// fields for installed packages. if there is an installed package that has a dependency that is (somehow) not installed, -// then that relationship (between the installed and uninstalled package) will NOT be created. -func associateRelationships(pkgs []pkg.Package) (relationships []artifact.Relationship) { - // map["provides" + "package"] -> packages that provide that package - lookup := make(map[string][]pkg.Package) - - // read provided and add as keys for lookup keys as well as package names - for _, p := range pkgs { - meta, ok := p.Metadata.(pkg.DpkgDBEntry) - if !ok { - log.Warnf("cataloger failed to extract dpkg 'provides' metadata for package %+v", p.Name) - continue - } - lookup[p.Name] = append(lookup[p.Name], p) - for _, provides := range meta.Provides { - k := stripVersionSpecifier(provides) - lookup[k] = append(lookup[k], p) - } - } - - // read "Depends" and "Pre-Depends" and match with keys - for _, p := range pkgs { - meta, ok := p.Metadata.(pkg.DpkgDBEntry) - if !ok { - log.Warnf("cataloger failed to extract dpkg 'dependency' metadata for package %+v", p.Name) - continue - } - - var allDeps []string - allDeps = append(allDeps, meta.Depends...) - allDeps = append(allDeps, meta.PreDepends...) - - for _, depSpecifier := range allDeps { - deps := splitPackageChoice(depSpecifier) - for _, dep := range deps { - for _, depPkg := range lookup[dep] { - relationships = append(relationships, artifact.Relationship{ - From: depPkg, - To: p, - Type: artifact.DependencyOfRelationship, - }) - } - } - } - } - return relationships -} - -func stripVersionSpecifier(s string) string { - // examples: - // libgmp10 (>= 2:6.2.1+dfsg1) --> libgmp10 - // libgmp10 --> libgmp10 - // foo [i386] --> foo - // default-mta | mail-transport-agent --> default-mta | mail-transport-agent - // kernel-headers-2.2.10 [!hurd-i386] --> kernel-headers-2.2.10 - - items := internal.SplitAny(s, "[(<>=") - if len(items) == 0 { - return s - } - - return strings.TrimSpace(items[0]) -} - -func splitPackageChoice(s string) (ret []string) { - fields := strings.Split(s, "|") - for _, field := range fields { - field = strings.TrimSpace(field) - if field != "" { - ret = append(ret, stripVersionSpecifier(field)) - } - } - return ret -} diff --git a/syft/pkg/cataloger/debian/parse_dpkg_db_test.go b/syft/pkg/cataloger/debian/parse_dpkg_db_test.go index 9a2d4dcf1..603de91e3 100644 --- a/syft/pkg/cataloger/debian/parse_dpkg_db_test.go +++ b/syft/pkg/cataloger/debian/parse_dpkg_db_test.go @@ -2,7 +2,6 @@ package debian import ( "bufio" - "context" "errors" "fmt" "os" @@ -16,7 +15,6 @@ import ( "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/linux" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/pkg/cataloger/generic" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" ) @@ -28,7 +26,7 @@ func Test_parseDpkgStatus(t *testing.T) { }{ { name: "single package", - fixturePath: "test-fixtures/status/single", + fixturePath: "test-fixtures/var/lib/dpkg/status.d/single", expected: []pkg.DpkgDBEntry{ { Package: "apt", @@ -102,7 +100,7 @@ func Test_parseDpkgStatus(t *testing.T) { }, { name: "single package with installed size", - fixturePath: "test-fixtures/status/installed-size-4KB", + fixturePath: "test-fixtures/var/lib/dpkg/status.d/installed-size-4KB", expected: []pkg.DpkgDBEntry{ { Package: "apt", @@ -143,7 +141,7 @@ func Test_parseDpkgStatus(t *testing.T) { }, { name: "multiple entries", - fixturePath: "test-fixtures/status/multiple", + fixturePath: "test-fixtures/var/lib/dpkg/status.d/multiple", expected: []pkg.DpkgDBEntry{ { Package: "no-version", @@ -434,104 +432,6 @@ func Test_handleNewKeyValue(t *testing.T) { } } -func Test_stripVersionSpecifier(t *testing.T) { - - tests := []struct { - name string - input string - want string - }{ - { - name: "package name only", - input: "test", - want: "test", - }, - { - name: "with version", - input: "test (1.2.3)", - want: "test", - }, - { - name: "multiple packages", - input: "test | other", - want: "test | other", - }, - { - name: "with architecture specifiers", - input: "test [amd64 i386]", - want: "test", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, stripVersionSpecifier(tt.input)) - }) - } -} - -func Test_associateRelationships(t *testing.T) { - tests := []struct { - name string - fixture string - wantRelationships map[string][]string - }{ - { - name: "relationships for coreutils", - fixture: "test-fixtures/status/coreutils-relationships", - wantRelationships: map[string][]string{ - "coreutils": {"libacl1", "libattr1", "libc6", "libgmp10", "libselinux1"}, - "libacl1": {"libc6"}, - "libattr1": {"libc6"}, - "libc6": {"libgcc-s1"}, - "libgcc-s1": {"gcc-12-base", "libc6"}, - "libgmp10": {"libc6"}, - "libpcre2-8-0": {"libc6"}, - "libselinux1": {"libc6", "libpcre2-8-0"}, - }, - }, - { - name: "relationships from dpkg example docs", - fixture: "test-fixtures/status/doc-examples", - wantRelationships: map[string][]string{ - "made-up-package-1": {"kernel-headers-2.2.10", "hurd-dev", "gnumach-dev"}, - "made-up-package-2": {"libluajit5.1-dev", "liblua5.1-dev"}, - "made-up-package-3": {"foo", "bar"}, - // note that the "made-up-package-4" depends on "made-up-package-5" but not via the direct - // package name, but through the "provides" virtual package name "virtual-package-5". - "made-up-package-4": {"made-up-package-5"}, - // note that though there is a "default-mta | mail-transport-agent | not-installed" - // dependency choice we raise up the packages that are installed for every choice. - // In this case that means that "default-mta" and "mail-transport-agent". - "mutt": {"libc6", "default-mta", "mail-transport-agent"}, - }, - }, - { - name: "relationships for libpam-runtime", - fixture: "test-fixtures/status/libpam-runtime", - wantRelationships: map[string][]string{ - "libpam-runtime": {"debconf1", "debconf-2.0", "debconf2", "cdebconf", "libpam-modules"}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - f, err := os.Open(tt.fixture) - require.NoError(t, err) - - reader := file.NewLocationReadCloser(file.NewLocation(tt.fixture), f) - - pkgs, relationships, err := parseDpkgDB(context.Background(), nil, &generic.Environment{}, reader) - require.NotEmpty(t, pkgs) - require.NotEmpty(t, relationships) - require.NoError(t, err) - - if d := cmp.Diff(tt.wantRelationships, abstractRelationships(t, relationships)); d != "" { - t.Errorf("unexpected relationships (-want +got):\n%s", d) - } - }) - } -} - func abstractRelationships(t testing.TB, relationships []artifact.Relationship) map[string][]string { t.Helper() diff --git a/syft/pkg/cataloger/debian/test-fixtures/status/coreutils-relationships b/syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/coreutils-relationships similarity index 100% rename from syft/pkg/cataloger/debian/test-fixtures/status/coreutils-relationships rename to syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/coreutils-relationships diff --git a/syft/pkg/cataloger/debian/test-fixtures/status/doc-examples b/syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/doc-examples similarity index 100% rename from syft/pkg/cataloger/debian/test-fixtures/status/doc-examples rename to syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/doc-examples diff --git a/syft/pkg/cataloger/debian/test-fixtures/status/empty b/syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/empty similarity index 100% rename from syft/pkg/cataloger/debian/test-fixtures/status/empty rename to syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/empty diff --git a/syft/pkg/cataloger/debian/test-fixtures/status/installed-size-4KB b/syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/installed-size-4KB similarity index 100% rename from syft/pkg/cataloger/debian/test-fixtures/status/installed-size-4KB rename to syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/installed-size-4KB diff --git a/syft/pkg/cataloger/debian/test-fixtures/status/libpam-runtime b/syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/libpam-runtime similarity index 100% rename from syft/pkg/cataloger/debian/test-fixtures/status/libpam-runtime rename to syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/libpam-runtime diff --git a/syft/pkg/cataloger/debian/test-fixtures/status/multiple b/syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/multiple similarity index 100% rename from syft/pkg/cataloger/debian/test-fixtures/status/multiple rename to syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/multiple diff --git a/syft/pkg/cataloger/debian/test-fixtures/status/single b/syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/single similarity index 100% rename from syft/pkg/cataloger/debian/test-fixtures/status/single rename to syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/single diff --git a/syft/pkg/cataloger/generic/cataloger.go b/syft/pkg/cataloger/generic/cataloger.go index 49052ec88..8009594dd 100644 --- a/syft/pkg/cataloger/generic/cataloger.go +++ b/syft/pkg/cataloger/generic/cataloger.go @@ -12,7 +12,9 @@ import ( "github.com/anchore/syft/syft/pkg" ) -type processor func(resolver file.Resolver, env Environment) []request +type Processor func([]pkg.Package, []artifact.Relationship, error) ([]pkg.Package, []artifact.Relationship, error) + +type requester func(resolver file.Resolver, env Environment) []request type request struct { file.Location @@ -22,12 +24,13 @@ type request struct { // Cataloger implements the Catalog interface and is responsible for dispatching the proper parser function for // a given path or glob pattern. This is intended to be reusable across many package cataloger types. type Cataloger struct { - processor []processor + processors []Processor + requesters []requester upstreamCataloger string } func (c *Cataloger) WithParserByGlobs(parser Parser, globs ...string) *Cataloger { - c.processor = append(c.processor, + c.requesters = append(c.requesters, func(resolver file.Resolver, _ Environment) []request { var requests []request for _, g := range globs { @@ -47,7 +50,7 @@ func (c *Cataloger) WithParserByGlobs(parser Parser, globs ...string) *Cataloger } func (c *Cataloger) WithParserByMimeTypes(parser Parser, types ...string) *Cataloger { - c.processor = append(c.processor, + c.requesters = append(c.requesters, func(resolver file.Resolver, _ Environment) []request { var requests []request log.WithFields("mimetypes", types).Trace("searching for paths matching mimetype") @@ -64,7 +67,7 @@ func (c *Cataloger) WithParserByMimeTypes(parser Parser, types ...string) *Catal } func (c *Cataloger) WithParserByPath(parser Parser, paths ...string) *Cataloger { - c.processor = append(c.processor, + c.requesters = append(c.requesters, func(resolver file.Resolver, _ Environment) []request { var requests []request for _, p := range paths { @@ -83,6 +86,11 @@ func (c *Cataloger) WithParserByPath(parser Parser, paths ...string) *Cataloger return c } +func (c *Cataloger) WithProcessors(processors ...Processor) *Cataloger { + c.processors = append(c.processors, processors...) + return c +} + func makeRequests(parser Parser, locations []file.Location) []request { var requests []request for _, l := range locations { @@ -135,7 +143,14 @@ func (c *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg. relationships = append(relationships, discoveredRelationships...) } - return packages, relationships, nil + return c.process(packages, relationships, nil) +} + +func (c *Cataloger) process(pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) { + for _, proc := range c.processors { + pkgs, rels, err = proc(pkgs, rels, err) + } + return pkgs, rels, err } func invokeParser(ctx context.Context, resolver file.Resolver, location file.Location, logger logger.Logger, parser Parser, env *Environment) ([]pkg.Package, []artifact.Relationship, error) { @@ -158,7 +173,7 @@ func invokeParser(ctx context.Context, resolver file.Resolver, location file.Loc // selectFiles takes a set of file trees and resolves and file references of interest for future cataloging func (c *Cataloger) selectFiles(resolver file.Resolver) []request { var requests []request - for _, proc := range c.processor { + for _, proc := range c.requesters { requests = append(requests, proc(resolver, Environment{})...) } return requests diff --git a/syft/pkg/cataloger/generic/cataloger_test.go b/syft/pkg/cataloger/generic/cataloger_test.go index c22113432..e971b4459 100644 --- a/syft/pkg/cataloger/generic/cataloger_test.go +++ b/syft/pkg/cataloger/generic/cataloger_test.go @@ -159,7 +159,7 @@ func TestClosesFileOnParserPanic(t *testing.T) { resolver := newSpyReturningFileResolver(&spy, "test-fixtures/another-path.txt") ctx := context.TODO() - processors := []processor{ + processors := []requester{ func(resolver file.Resolver, env Environment) []request { return []request{ { @@ -178,7 +178,7 @@ func TestClosesFileOnParserPanic(t *testing.T) { } c := Cataloger{ - processor: processors, + requesters: processors, upstreamCataloger: "unit-test-cataloger", } diff --git a/syft/pkg/cataloger/internal/dependency/resolver.go b/syft/pkg/cataloger/internal/dependency/resolver.go new file mode 100644 index 000000000..d60e3bb3c --- /dev/null +++ b/syft/pkg/cataloger/internal/dependency/resolver.go @@ -0,0 +1,95 @@ +package dependency + +import ( + "sort" + + "github.com/scylladb/go-set/strset" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" +) + +// Specification holds strings that indicate abstract resources that a package provides for other packages and +// requires for itself. These strings can represent anything from file paths, package names, or any other concept +// that is useful for dependency resolution within that packing ecosystem. +type Specification struct { + // Provides holds a list of abstract resources that this package provides for other packages. + Provides []string + + // Requires holds a list of abstract resources that this package requires from other packages. + Requires []string +} + +// Specifier is a function that takes a package and extracts a Specification, describing resources +// the package provides and needs. +type Specifier func(pkg.Package) Specification + +// Processor returns a generic processor that will resolve relationships between packages based on the dependency claims. +func Processor(s Specifier) generic.Processor { + return func(pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) { + // we can't move forward unless all package IDs have been set + for idx, p := range pkgs { + id := p.ID() + if id == "" { + p.SetID() + pkgs[idx] = p + } + } + + rels = append(rels, resolve(s, pkgs)...) + return pkgs, rels, err + } +} + +// resolve will create relationships between packages based on the dependency claims of each package. +func resolve(specifier Specifier, pkgs []pkg.Package) (relationships []artifact.Relationship) { + pkgsProvidingResource := make(map[string][]artifact.ID) + + pkgsByID := make(map[artifact.ID]pkg.Package) + specsByPkg := make(map[artifact.ID]Specification) + + for _, p := range pkgs { + id := p.ID() + pkgsByID[id] = p + specsByPkg[id] = specifier(p) + for _, resource := range deduplicate(specifier(p).Provides) { + pkgsProvidingResource[resource] = append(pkgsProvidingResource[resource], id) + } + } + + seen := strset.New() + for _, dependantPkg := range pkgs { + spec := specsByPkg[dependantPkg.ID()] + for _, resource := range deduplicate(spec.Requires) { + for _, providingPkgID := range pkgsProvidingResource[resource] { + // prevent creating duplicate relationships + pairKey := string(providingPkgID) + "-" + string(dependantPkg.ID()) + if seen.Has(pairKey) { + continue + } + + providingPkg := pkgsByID[providingPkgID] + + relationships = append(relationships, + artifact.Relationship{ + From: providingPkg, + To: dependantPkg, + Type: artifact.DependencyOfRelationship, + }, + ) + + seen.Add(pairKey) + } + } + } + return relationships +} + +func deduplicate(ss []string) []string { + // note: we sort the set such that multiple invocations of this function will be deterministic + set := strset.New(ss...) + list := set.List() + sort.Strings(list) + return list +} diff --git a/syft/pkg/cataloger/internal/dependency/resolver_test.go b/syft/pkg/cataloger/internal/dependency/resolver_test.go new file mode 100644 index 000000000..c6690d958 --- /dev/null +++ b/syft/pkg/cataloger/internal/dependency/resolver_test.go @@ -0,0 +1,211 @@ +package dependency + +import ( + "errors" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/pkg" +) + +func Test_resolve(t *testing.T) { + a := pkg.Package{ + Name: "a", + } + a.SetID() + + b := pkg.Package{ + Name: "b", + } + b.SetID() + + c := pkg.Package{ + Name: "c", + } + c.SetID() + + subjects := []pkg.Package{a, b, c} + + tests := []struct { + name string + s Specifier + want map[string][]string + }{ + { + name: "find relationships between packages", + s: newSpecifierBuilder(). + WithProvides(a /* provides */, "a-resource"). + WithRequires(b /* requires */, "a-resource"). + Specifier(), + want: map[string][]string{ + "b": /* depends on */ {"a"}, + }, + }, + { + name: "deduplicates provider keys", + s: newSpecifierBuilder(). + WithProvides(a /* provides */, "a-resource", "a-resource", "a-resource"). + WithRequires(b /* requires */, "a-resource", "a-resource", "a-resource"). + Specifier(), + want: map[string][]string{ + "b": /* depends on */ {"a"}, + // note: we're NOT seeing: + // "b": /* depends on */ {"a", "a", "a"}, + }, + }, + { + name: "deduplicates crafted relationships", + s: newSpecifierBuilder(). + WithProvides(a /* provides */, "a1-resource", "a2-resource", "a3-resource"). + WithRequires(b /* requires */, "a1-resource", "a2-resource"). + Specifier(), + want: map[string][]string{ + "b": /* depends on */ {"a"}, + // note: we're NOT seeing: + // "b": /* depends on */ {"a", "a"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + relationships := resolve(tt.s, subjects) + if d := cmp.Diff(tt.want, abstractRelationships(t, relationships)); d != "" { + t.Errorf("unexpected relationships (-want +got):\n%s", d) + } + }) + } +} + +type specifierBuilder struct { + provides map[string][]string + requires map[string][]string +} + +func newSpecifierBuilder() *specifierBuilder { + return &specifierBuilder{ + provides: make(map[string][]string), + requires: make(map[string][]string), + } +} + +func (m *specifierBuilder) WithProvides(p pkg.Package, provides ...string) *specifierBuilder { + m.provides[p.Name] = append(m.provides[p.Name], provides...) + return m +} + +func (m *specifierBuilder) WithRequires(p pkg.Package, requires ...string) *specifierBuilder { + m.requires[p.Name] = append(m.requires[p.Name], requires...) + return m +} + +func (m specifierBuilder) Specifier() Specifier { + return func(p pkg.Package) Specification { + return Specification{ + Provides: m.provides[p.Name], + Requires: m.requires[p.Name], + } + } +} + +func abstractRelationships(t testing.TB, relationships []artifact.Relationship) map[string][]string { + t.Helper() + + abstracted := make(map[string][]string) + for _, relationship := range relationships { + fromPkg, ok := relationship.From.(pkg.Package) + if !ok { + continue + } + toPkg, ok := relationship.To.(pkg.Package) + if !ok { + continue + } + + // we build this backwards since we use DependencyOfRelationship instead of DependsOn + abstracted[toPkg.Name] = append(abstracted[toPkg.Name], fromPkg.Name) + } + + return abstracted +} + +func Test_Processor(t *testing.T) { + a := pkg.Package{ + Name: "a", + } + a.SetID() + + b := pkg.Package{ + Name: "b", + } + b.SetID() + + c := pkg.Package{ + Name: "c", + } + c.SetID() + + tests := []struct { + name string + sp Specifier + pkgs []pkg.Package + rels []artifact.Relationship + err error + wantPkgCount int + wantRelCount int + wantErr assert.ErrorAssertionFunc + }{ + { + name: "happy path preserves decorated values", + sp: newSpecifierBuilder(). + WithProvides(b, "b-resource"). + WithRequires(c, "b-resource"). + Specifier(), + pkgs: []pkg.Package{a, b, c}, + rels: []artifact.Relationship{ + { + From: a, + To: b, + Type: artifact.DependencyOfRelationship, + }, + }, + + wantPkgCount: 3, + wantRelCount: 2, // original + new + }, + { + name: "error from cataloger is propagated", + sp: newSpecifierBuilder(). + WithProvides(b, "b-resource"). + WithRequires(c, "b-resource"). + Specifier(), + err: errors.New("surprise!"), + pkgs: []pkg.Package{a, b, c}, + rels: []artifact.Relationship{ + { + From: a, + To: b, + Type: artifact.DependencyOfRelationship, + }, + }, + wantPkgCount: 3, + wantRelCount: 2, // original + new + wantErr: assert.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = assert.NoError + } + + gotPkgs, gotRels, err := Processor(tt.sp)(tt.pkgs, tt.rels, tt.err) + + tt.wantErr(t, err) + assert.Len(t, gotPkgs, tt.wantPkgCount) + assert.Len(t, gotRels, tt.wantRelCount) + }) + } +} diff --git a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go index 11efb263b..227620aa5 100644 --- a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go +++ b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go @@ -44,6 +44,7 @@ type CatalogTester struct { compareOptions []cmp.Option locationComparer locationComparer licenseComparer licenseComparer + customAssertions []func(t *testing.T, pkgs []pkg.Package, relationships []artifact.Relationship) } func NewCatalogTester() *CatalogTester { @@ -164,6 +165,11 @@ func (p *CatalogTester) WithImageResolver(t *testing.T, fixtureName string) *Cat return p } +func (p *CatalogTester) ExpectsAssertion(a func(t *testing.T, pkgs []pkg.Package, relationships []artifact.Relationship)) *CatalogTester { + p.customAssertions = append(p.customAssertions, a) + return p +} + func (p *CatalogTester) IgnoreLocationLayer() *CatalogTester { p.locationComparer = func(x, y file.Location) bool { return cmp.Equal(x.Coordinates.RealPath, y.Coordinates.RealPath) && cmp.Equal(x.AccessPath, y.AccessPath) @@ -250,7 +256,13 @@ func (p *CatalogTester) TestCataloger(t *testing.T, cataloger pkg.Cataloger) { if p.assertResultExpectations { p.wantErr(t, err) p.assertPkgs(t, pkgs, relationships) - } else { + } + + for _, a := range p.customAssertions { + a(t, pkgs, relationships) + } + + if !p.assertResultExpectations && len(p.customAssertions) == 0 { resolver.PruneUnfulfilledPathResponses(p.ignoreUnfulfilledPathResponses, p.ignoreAnyUnfulfilledPaths...) // if we aren't testing the results, we should focus on what was searched for (for glob-centric tests)