From 1414d1fbc3032d817ad9fdf433dc7bdfc9f892f9 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 22 Oct 2020 09:50:31 -0400 Subject: [PATCH] add test coverage for python pacakge cataloger and update catalog interface Signed-off-by: Alex Goodman --- schema/json/schema.json | 34 ++- syft/cataloger/apkdb/parse_apk_db_test.go | 18 +- .../common/generic_cataloger_test.go | 18 +- syft/cataloger/java/archive_parser_test.go | 36 +-- .../javascript/parse_package_json.go | 4 +- .../javascript/parse_package_json_test.go | 13 +- syft/cataloger/python/package_cataloger.go | 96 ++++++-- .../python/package_cataloger_test.go | 207 ++++++++++++++---- .../python/parse_requirements_test.go | 2 - syft/cataloger/python/parse_setup_test.go | 5 - ...eel_egg.go => parse_wheel_egg_metadata.go} | 20 +- ...st.go => parse_wheel_egg_metadata_test.go} | 14 +- .../python/parse_wheel_egg_record.go | 60 +++++ .../python/parse_wheel_egg_record_test.go | 57 +++++ .../python/test-fixtures/dist-info/RECORD | 5 + .../python/test-fixtures/egg-info/RECORD | 6 + .../test-fixtures/partial.dist-info/METADATA | 47 ++++ syft/cataloger/rpmdb/parse_rpmdb_test.go | 7 +- syft/cataloger/ruby/parse_gemspec_test.go | 11 +- syft/pkg/egg_wheel_metadata.go | 11 - syft/pkg/metadata.go | 4 +- syft/pkg/npm_metadata.go | 4 +- syft/pkg/python_package_metadata.go | 23 ++ syft/scope/resolver.go | 5 + syft/scope/resolvers/all_layers_resolver.go | 9 + syft/scope/resolvers/directory_resolver.go | 18 +- syft/scope/resolvers/image_squash_resolver.go | 12 + test/integration/pkg_coverage_test.go | 1 + .../python/dist-info/RECORD | 5 + .../PKG-INFO | 0 30 files changed, 596 insertions(+), 156 deletions(-) rename syft/cataloger/python/{parse_wheel_egg.go => parse_wheel_egg_metadata.go} (60%) rename syft/cataloger/python/{parse_wheel_egg_test.go => parse_wheel_egg_metadata_test.go} (72%) create mode 100644 syft/cataloger/python/parse_wheel_egg_record.go create mode 100644 syft/cataloger/python/parse_wheel_egg_record_test.go create mode 100644 syft/cataloger/python/test-fixtures/dist-info/RECORD create mode 100644 syft/cataloger/python/test-fixtures/egg-info/RECORD create mode 100644 syft/cataloger/python/test-fixtures/partial.dist-info/METADATA delete mode 100644 syft/pkg/egg_wheel_metadata.go create mode 100644 syft/pkg/python_package_metadata.go create mode 100644 test/integration/test-fixtures/image-pkg-coverage/python/dist-info/RECORD rename test/integration/test-fixtures/image-pkg-coverage/python/{someotherpkg-2.19.0-py3.8.egg-info => someotherpkg-3.19.0-py3.8.egg-info}/PKG-INFO (100%) diff --git a/schema/json/schema.json b/schema/json/schema.json index dde279232..83d0a2508 100644 --- a/schema/json/schema.json +++ b/schema/json/schema.json @@ -43,6 +43,9 @@ "author": { "type": "string" }, + "authorEmail": { + "type": "string" + }, "description": { "type": "string" }, @@ -65,6 +68,21 @@ "checksum": { "type": "string" }, + "digest": { + "properties": { + "algorithm": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "algorithm", + "value" + ], + "type": "object" + }, "ownerGid": { "type": "string" }, @@ -76,14 +94,13 @@ }, "permissions": { "type": "string" + }, + "size": { + "type": "string" } }, "required": [ - "checksum", - "ownerGid", - "ownerUid", - "path", - "permissions" + "path" ], "type": "object" } @@ -403,6 +420,9 @@ ], "type": "object" }, + "metadataType": { + "type": "string" + }, "sources": { "type": "null" }, @@ -419,6 +439,7 @@ "licenses", "manifest", "metadata", + "metadataType", "sources", "type", "version" @@ -427,6 +448,9 @@ } ] }, + "platform": { + "type": "string" + }, "pomProperties": { "properties": { "Path": { diff --git a/syft/cataloger/apkdb/parse_apk_db_test.go b/syft/cataloger/apkdb/parse_apk_db_test.go index 3b22cc11d..c8846ac24 100644 --- a/syft/cataloger/apkdb/parse_apk_db_test.go +++ b/syft/cataloger/apkdb/parse_apk_db_test.go @@ -160,10 +160,11 @@ func TestMultiplePackages(t *testing.T) { fixture: "test-fixtures/multiple", expected: []pkg.Package{ { - Name: "libc-utils", - Version: "0.7.2-r0", - Licenses: []string{"BSD"}, - Type: pkg.ApkPkg, + Name: "libc-utils", + Version: "0.7.2-r0", + Licenses: []string{"BSD"}, + Type: pkg.ApkPkg, + MetadataType: pkg.ApkMetadataType, Metadata: pkg.ApkMetadata{ Package: "libc-utils", OriginPackage: "libc-dev", @@ -182,10 +183,11 @@ func TestMultiplePackages(t *testing.T) { }, }, { - Name: "musl-utils", - Version: "1.1.24-r2", - Licenses: []string{"MIT", "BSD", "GPL2+"}, - Type: pkg.ApkPkg, + Name: "musl-utils", + Version: "1.1.24-r2", + Licenses: []string{"MIT", "BSD", "GPL2+"}, + Type: pkg.ApkPkg, + MetadataType: pkg.ApkMetadataType, Metadata: pkg.ApkMetadata{ Package: "musl-utils", OriginPackage: "musl", diff --git a/syft/cataloger/common/generic_cataloger_test.go b/syft/cataloger/common/generic_cataloger_test.go index 39298ec71..6083d390a 100644 --- a/syft/cataloger/common/generic_cataloger_test.go +++ b/syft/cataloger/common/generic_cataloger_test.go @@ -10,25 +10,25 @@ import ( "github.com/anchore/syft/syft/pkg" ) -type testResolver struct { +type testResolverMock struct { contents map[file.Reference]string } -func newTestResolver() *testResolver { - return &testResolver{ +func newTestResolver() *testResolverMock { + return &testResolverMock{ contents: make(map[file.Reference]string), } } -func (r *testResolver) FileContentsByRef(_ file.Reference) (string, error) { +func (r *testResolverMock) FileContentsByRef(_ file.Reference) (string, error) { return "", fmt.Errorf("not implemented") } -func (r *testResolver) MultipleFileContentsByRef(_ ...file.Reference) (map[file.Reference]string, error) { +func (r *testResolverMock) MultipleFileContentsByRef(_ ...file.Reference) (map[file.Reference]string, error) { return r.contents, nil } -func (r *testResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) { +func (r *testResolverMock) FilesByPath(paths ...file.Path) ([]file.Reference, error) { results := make([]file.Reference, len(paths)) for idx, p := range paths { @@ -39,13 +39,17 @@ func (r *testResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) return results, nil } -func (r *testResolver) FilesByGlob(_ ...string) ([]file.Reference, error) { +func (r *testResolverMock) FilesByGlob(_ ...string) ([]file.Reference, error) { path := "/a-path.txt" ref := file.NewFileReference(file.Path(path)) r.contents[ref] = fmt.Sprintf("%s file contents!", path) return []file.Reference{ref}, nil } +func (r *testResolverMock) RelativeFileByPath(_ file.Reference, _ string) (*file.Reference, error) { + return nil, fmt.Errorf("not implemented") +} + func parser(_ string, reader io.Reader) ([]pkg.Package, error) { contents, err := ioutil.ReadAll(reader) if err != nil { diff --git a/syft/cataloger/java/archive_parser_test.go b/syft/cataloger/java/archive_parser_test.go index df290195b..bd402af77 100644 --- a/syft/cataloger/java/archive_parser_test.go +++ b/syft/cataloger/java/archive_parser_test.go @@ -137,10 +137,11 @@ func TestParseJar(t *testing.T) { }, expected: map[string]pkg.Package{ "example-jenkins-plugin": { - Name: "example-jenkins-plugin", - Version: "1.0-SNAPSHOT", - Language: pkg.Java, - Type: pkg.JenkinsPluginPkg, + Name: "example-jenkins-plugin", + Version: "1.0-SNAPSHOT", + Language: pkg.Java, + Type: pkg.JenkinsPluginPkg, + MetadataType: pkg.JavaMetadataType, Metadata: pkg.JavaMetadata{ Manifest: &pkg.JavaManifest{ ManifestVersion: "1.0", @@ -181,10 +182,11 @@ func TestParseJar(t *testing.T) { fixture: "test-fixtures/java-builds/packages/example-java-app-gradle-0.1.0.jar", expected: map[string]pkg.Package{ "example-java-app-gradle": { - Name: "example-java-app-gradle", - Version: "0.1.0", - Language: pkg.Java, - Type: pkg.JavaPkg, + Name: "example-java-app-gradle", + Version: "0.1.0", + Language: pkg.Java, + Type: pkg.JavaPkg, + MetadataType: pkg.JavaMetadataType, Metadata: pkg.JavaMetadata{ Manifest: &pkg.JavaManifest{ ManifestVersion: "1.0", @@ -200,10 +202,11 @@ func TestParseJar(t *testing.T) { }, expected: map[string]pkg.Package{ "example-java-app-maven": { - Name: "example-java-app-maven", - Version: "0.1.0", - Language: pkg.Java, - Type: pkg.JavaPkg, + Name: "example-java-app-maven", + Version: "0.1.0", + Language: pkg.Java, + Type: pkg.JavaPkg, + MetadataType: pkg.JavaMetadataType, Metadata: pkg.JavaMetadata{ Manifest: &pkg.JavaManifest{ ManifestVersion: "1.0", @@ -224,10 +227,11 @@ func TestParseJar(t *testing.T) { }, }, "joda-time": { - Name: "joda-time", - Version: "2.9.2", - Language: pkg.Java, - Type: pkg.JavaPkg, + Name: "joda-time", + Version: "2.9.2", + Language: pkg.Java, + Type: pkg.JavaPkg, + MetadataType: pkg.JavaMetadataType, Metadata: pkg.JavaMetadata{ PomProperties: &pkg.PomProperties{ Path: "META-INF/maven/joda-time/joda-time/pom.properties", diff --git a/syft/cataloger/javascript/parse_package_json.go b/syft/cataloger/javascript/parse_package_json.go index 01d3fe7ba..bcdc68801 100644 --- a/syft/cataloger/javascript/parse_package_json.go +++ b/syft/cataloger/javascript/parse_package_json.go @@ -43,8 +43,8 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { Licenses: []string{p.License}, Language: pkg.JavaScript, Type: pkg.NpmPkg, - MetadataType: pkg.NpmPackageJsonMetadataType, - Metadata: pkg.NpmPackageJsonMetadata{ + MetadataType: pkg.NpmPackageJSONMetadataType, + Metadata: pkg.NpmPackageJSONMetadata{ Author: p.Author, Homepage: p.Homepage, }, diff --git a/syft/cataloger/javascript/parse_package_json_test.go b/syft/cataloger/javascript/parse_package_json_test.go index f5fe044f4..bfe46f4a0 100644 --- a/syft/cataloger/javascript/parse_package_json_test.go +++ b/syft/cataloger/javascript/parse_package_json_test.go @@ -10,12 +10,13 @@ import ( func TestParsePackageJSON(t *testing.T) { expected := pkg.Package{ - Name: "npm", - Version: "6.14.6", - Type: pkg.NpmPkg, - Licenses: []string{"Artistic-2.0"}, - Language: pkg.JavaScript, - Metadata: pkg.NpmPackageJsonMetadata{ + Name: "npm", + Version: "6.14.6", + Type: pkg.NpmPkg, + Licenses: []string{"Artistic-2.0"}, + Language: pkg.JavaScript, + MetadataType: pkg.NpmPackageJSONMetadataType, + Metadata: pkg.NpmPackageJSONMetadata{ Author: "Isaac Z. Schlueter (http://blog.izs.me)", Homepage: "https://docs.npmjs.com/", }, diff --git a/syft/cataloger/python/package_cataloger.go b/syft/cataloger/python/package_cataloger.go index ecf97c214..b23f48c93 100644 --- a/syft/cataloger/python/package_cataloger.go +++ b/syft/cataloger/python/package_cataloger.go @@ -3,6 +3,7 @@ package python import ( "fmt" "path/filepath" + "strings" "github.com/anchore/stereoscope/pkg/file" @@ -11,19 +12,15 @@ import ( "github.com/anchore/syft/syft/scope" ) -const wheelGlob = "**/*dist-info/METADATA" +const ( + eggMetadataGlob = "**/*egg-info/PKG-INFO" + wheelMetadataGlob = "**/*dist-info/METADATA" +) -type PackageCataloger struct { - globs []string -} +type PackageCataloger struct{} // NewPythonPackageCataloger returns a new cataloger for python packages within egg or wheel installation directories. func NewPythonPackageCataloger() *PackageCataloger { - //globParsers := map[string]common.ParserFn{ - // "**/*egg-info/PKG-INFO": parseWheelOrEggMetadata, - // "**/*dist-info/METADATA": parseWheelOrEggMetadata, - //} - return &PackageCataloger{} } @@ -32,33 +29,88 @@ func (c *PackageCataloger) Name() string { } func (c *PackageCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) { - return c.catalogWheels(resolver) -} + // nolint:prealloc + var fileMatches []file.Reference -func (c *PackageCataloger) catalogWheels(resolver scope.Resolver) ([]pkg.Package, error) { - fileMatches, err := resolver.FilesByGlob(wheelGlob) - if err != nil { - return nil, fmt.Errorf("failed to find files by glob: %s", wheelGlob) + for _, glob := range []string{eggMetadataGlob, wheelMetadataGlob} { + matches, err := resolver.FilesByGlob(glob) + if err != nil { + return nil, fmt.Errorf("failed to find files by glob: %s", glob) + } + fileMatches = append(fileMatches, matches...) } var pkgs []pkg.Package for _, ref := range fileMatches { - p, err := c.catalogWheel(resolver, ref) + p, err := c.catalogEggOrWheel(resolver, ref) if err != nil { - return nil, fmt.Errorf("unable to catalog python wheel=%+v: %w", ref.Path, err) + return nil, fmt.Errorf("unable to catalog python package=%+v: %w", ref.Path, err) + } + if p != nil { + pkgs = append(pkgs, *p) } - pkgs = append(pkgs, p) } return pkgs, nil } -func (c *PackageCataloger) catalogWheel(resolver scope.Resolver, wheelRef file.Reference) (pkg.Package, error) { +func (c *PackageCataloger) catalogEggOrWheel(resolver scope.Resolver, metadataRef file.Reference) (*pkg.Package, error) { + var sources = []file.Reference{metadataRef} + + metadataContents, err := resolver.FileContentsByRef(metadataRef) + if err != nil { + return nil, err + } + + metadata, err := parseWheelOrEggMetadata(strings.NewReader(metadataContents)) + if err != nil { + return nil, err + } + // we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory // or for an image... for an image the METADATA file may be present within multiple layers, so it is important // to reconcile the RECORD path to the same layer (or the next adjacent lower layer). - recordPath := filepath.Join(filepath.Dir(string(wheelRef.Path)), "RECORD") - // problem! we don't know which is the right discovered path relative to the given METADATA file! (which layer?) - discoveredPaths, err := resolver.FilesByPath(file.Path(recordPath)) + // lets find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure) + recordPath := filepath.Join(filepath.Dir(string(metadataRef.Path)), "RECORD") + recordRef, err := resolver.RelativeFileByPath(metadataRef, recordPath) + if err != nil { + return nil, err + } + if recordRef != nil { + sources = append(sources, *recordRef) + + recordContents, err := resolver.FileContentsByRef(*recordRef) + if err != nil { + return nil, err + } + + // parse the record contents + records, err := parseWheelOrEggRecord(strings.NewReader(recordContents)) + if err != nil { + return nil, err + } + + // append the record files list to the metadata + metadata.Files = records + } + + // assemble the package + + var licenses []string + if metadata.License != "" { + licenses = []string{metadata.License} + } + + return &pkg.Package{ + Name: metadata.Name, + Version: metadata.Version, + FoundBy: c.Name(), + Source: sources, + Licenses: licenses, + Language: pkg.Python, + Type: pkg.PythonPkg, + MetadataType: pkg.PythonPackageMetadataType, + Metadata: metadata, + }, nil } diff --git a/syft/cataloger/python/package_cataloger_test.go b/syft/cataloger/python/package_cataloger_test.go index 09fc376c4..093a846f1 100644 --- a/syft/cataloger/python/package_cataloger_test.go +++ b/syft/cataloger/python/package_cataloger_test.go @@ -1,70 +1,197 @@ package python import ( + "fmt" + "io" + "io/ioutil" "os" "testing" + "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/pkg" "github.com/go-test/deep" ) -func TestPythonPackageCataloger(t *testing.T) { +type pythonTestResolverMock struct { + metadataReader io.Reader + recordReader io.Reader + metadataRef *file.Reference + recordRef *file.Reference + contents map[file.Reference]string +} + +func newTestResolver(recordReader, metadataReader io.Reader) *pythonTestResolverMock { + var recordRef *file.Reference + if recordReader != nil { + ref := file.NewFileReference("record-path") + recordRef = &ref + } + metadataRef := file.NewFileReference("metadata-path") + return &pythonTestResolverMock{ + recordReader: recordReader, + metadataReader: metadataReader, + metadataRef: &metadataRef, + recordRef: recordRef, + contents: make(map[file.Reference]string), + } +} + +func (r *pythonTestResolverMock) FileContentsByRef(ref file.Reference) (string, error) { + switch ref.Path { + case r.metadataRef.Path: + b, err := ioutil.ReadAll(r.metadataReader) + if err != nil { + return "", err + } + return string(b), nil + case r.recordRef.Path: + b, err := ioutil.ReadAll(r.recordReader) + if err != nil { + return "", err + } + return string(b), nil + } + return "", fmt.Errorf("invalid value given") +} + +func (r *pythonTestResolverMock) MultipleFileContentsByRef(_ ...file.Reference) (map[file.Reference]string, error) { + return nil, fmt.Errorf("not implemented") +} + +func (r *pythonTestResolverMock) FilesByPath(_ ...file.Path) ([]file.Reference, error) { + return nil, fmt.Errorf("not implemented") +} + +func (r *pythonTestResolverMock) FilesByGlob(_ ...string) ([]file.Reference, error) { + return nil, fmt.Errorf("not implemented") +} +func (r *pythonTestResolverMock) RelativeFileByPath(reference file.Reference, _ string) (*file.Reference, error) { + switch reference.Path { + case r.metadataRef.Path: + return r.recordRef, nil + default: + return nil, fmt.Errorf("invalid value given") + } +} + +func TestPythonPackageWheelCataloger(t *testing.T) { tests := []struct { - Fixture string - ExpectedMetadata []pkg.Package + MetadataFixture string + RecordFixture string + ExpectedPackage pkg.Package }{ { - Fixture: "test-fixtures/", - ExpectedMetadata: []pkg.Package{ - { - Name: "requests", - Version: "2.22.0", - Type: pkg.PythonPkg, - Language: pkg.Python, - Licenses: []string{"Apache 2.0"}, - MetadataType: pkg.PythonEggWheelMetadataType, - Metadata: pkg.EggWheelMetadata{ - Name: "requests", - Version: "2.22.0", - License: "Apache 2.0", - Platform: "UNKNOWN", - Author: "Kenneth Reitz", - AuthorEmail: "me@kennethreitz.org", + MetadataFixture: "test-fixtures/egg-info/PKG-INFO", + RecordFixture: "test-fixtures/egg-info/RECORD", + ExpectedPackage: pkg.Package{ + Name: "requests", + Version: "2.22.0", + Type: pkg.PythonPkg, + Language: pkg.Python, + Licenses: []string{"Apache 2.0"}, + FoundBy: "python-package-cataloger", + MetadataType: pkg.PythonPackageMetadataType, + Metadata: pkg.PythonPackageMetadata{ + Name: "requests", + Version: "2.22.0", + License: "Apache 2.0", + Platform: "UNKNOWN", + Author: "Kenneth Reitz", + AuthorEmail: "me@kennethreitz.org", + Files: []pkg.PythonFileRecord{ + {Path: "requests-2.22.0.dist-info/INSTALLER", Digest: pkg.Digest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"}, + {Path: "requests/__init__.py", Digest: pkg.Digest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"}, + {Path: "requests/__pycache__/__version__.cpython-38.pyc"}, + {Path: "requests/__pycache__/utils.cpython-38.pyc"}, + {Path: "requests/__version__.py", Digest: pkg.Digest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"}, + {Path: "requests/utils.py", Digest: pkg.Digest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"}, }, }, - { - Name: "Pygments", - Version: "2.6.1", - Type: pkg.PythonPkg, - Language: pkg.Python, - Licenses: []string{"BSD License"}, - MetadataType: pkg.PythonEggWheelMetadataType, - Metadata: pkg.EggWheelMetadata{ - Name: "Pygments", - Version: "2.6.1", - License: "BSD License", - Platform: "any", - Author: "Georg Brandl", - AuthorEmail: "georg@python.org", + }, + }, + { + MetadataFixture: "test-fixtures/dist-info/METADATA", + RecordFixture: "test-fixtures/dist-info/RECORD", + ExpectedPackage: pkg.Package{ + Name: "Pygments", + Version: "2.6.1", + Type: pkg.PythonPkg, + Language: pkg.Python, + Licenses: []string{"BSD License"}, + FoundBy: "python-package-cataloger", + MetadataType: pkg.PythonPackageMetadataType, + Metadata: pkg.PythonPackageMetadata{ + Name: "Pygments", + Version: "2.6.1", + License: "BSD License", + Platform: "any", + Author: "Georg Brandl", + AuthorEmail: "georg@python.org", + Files: []pkg.PythonFileRecord{ + {Path: "../../../bin/pygmentize", Digest: pkg.Digest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, + {Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: pkg.Digest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, + {Path: "Pygments-2.6.1.dist-info/RECORD"}, + {Path: "pygments/__pycache__/__init__.cpython-38.pyc"}, + {Path: "pygments/util.py", Digest: pkg.Digest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, }, }, }, }, + { + // in casses where the metadata file is available and the record is not we should still record there is a package + MetadataFixture: "test-fixtures/partial.dist-info/METADATA", + ExpectedPackage: pkg.Package{ + Name: "Pygments", + Version: "2.6.1", + Type: pkg.PythonPkg, + Language: pkg.Python, + Licenses: []string{"BSD License"}, + FoundBy: "python-package-cataloger", + MetadataType: pkg.PythonPackageMetadataType, + Metadata: pkg.PythonPackageMetadata{ + Name: "Pygments", + Version: "2.6.1", + License: "BSD License", + Platform: "any", + Author: "Georg Brandl", + AuthorEmail: "georg@python.org", + }, + }, + }, } for _, test := range tests { - t.Run(test.Fixture, func(t *testing.T) { - fixture, err := os.Open(test.Fixture) + t.Run(test.MetadataFixture, func(t *testing.T) { + metadata, err := os.Open(test.MetadataFixture) if err != nil { - t.Fatalf("failed to open fixture: %+v", err) + t.Fatalf("failed to open record: %+v", err) } - actual, err := parseWheelOrEggMetadata(fixture.Name(), fixture) - if err != nil { - t.Fatalf("failed to parse python package: %+v", err) + var record io.Reader + if test.RecordFixture != "" { + record, err = os.Open(test.RecordFixture) + if err != nil { + t.Fatalf("failed to open record: %+v", err) + } } - for _, d := range deep.Equal(actual, &test.ExpectedMetadata) { + resolver := newTestResolver(record, metadata) + + // note that the source is the record ref created by the resolver mock... attach the expected values + test.ExpectedPackage.Source = []file.Reference{*resolver.metadataRef} + if resolver.recordRef != nil { + test.ExpectedPackage.Source = append(test.ExpectedPackage.Source, *resolver.recordRef) + } + + pyPkgCataloger := NewPythonPackageCataloger() + + actual, err := pyPkgCataloger.catalogEggOrWheel(resolver, *resolver.metadataRef) + if err != nil { + t.Fatalf("failed to catalog python package: %+v", err) + } + + for _, d := range deep.Equal(actual, &test.ExpectedPackage) { t.Errorf("diff: %+v", d) } }) diff --git a/syft/cataloger/python/parse_requirements_test.go b/syft/cataloger/python/parse_requirements_test.go index 230bb69ae..8dd66092a 100644 --- a/syft/cataloger/python/parse_requirements_test.go +++ b/syft/cataloger/python/parse_requirements_test.go @@ -37,14 +37,12 @@ func TestParseRequirementsTxt(t *testing.T) { Version: "1.0.0", Language: pkg.Python, Type: pkg.PythonPkg, - Licenses: []string{}, }, "flask": { Name: "flask", Version: "4.0.0", Language: pkg.Python, Type: pkg.PythonPkg, - Licenses: []string{}, }, } fixture, err := os.Open("test-fixtures/requires/requirements.txt") diff --git a/syft/cataloger/python/parse_setup_test.go b/syft/cataloger/python/parse_setup_test.go index 4abf6359c..3b8fa8edc 100644 --- a/syft/cataloger/python/parse_setup_test.go +++ b/syft/cataloger/python/parse_setup_test.go @@ -14,35 +14,30 @@ func TestParseSetup(t *testing.T) { Version: "2.2.0", Language: pkg.Python, Type: pkg.PythonPkg, - Licenses: []string{}, }, "mypy": { Name: "mypy", Version: "v0.770", Language: pkg.Python, Type: pkg.PythonPkg, - Licenses: []string{}, }, "mypy1": { Name: "mypy1", Version: "v0.770", Language: pkg.Python, Type: pkg.PythonPkg, - Licenses: []string{}, }, "mypy2": { Name: "mypy2", Version: "v0.770", Language: pkg.Python, Type: pkg.PythonPkg, - Licenses: []string{}, }, "mypy3": { Name: "mypy3", Version: "v0.770", Language: pkg.Python, Type: pkg.PythonPkg, - Licenses: []string{}, }, } fixture, err := os.Open("test-fixtures/setup/setup.py") diff --git a/syft/cataloger/python/parse_wheel_egg.go b/syft/cataloger/python/parse_wheel_egg_metadata.go similarity index 60% rename from syft/cataloger/python/parse_wheel_egg.go rename to syft/cataloger/python/parse_wheel_egg_metadata.go index 37994ffec..9aff7baa2 100644 --- a/syft/cataloger/python/parse_wheel_egg.go +++ b/syft/cataloger/python/parse_wheel_egg_metadata.go @@ -13,7 +13,7 @@ import ( // parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes), // returning all Python packages listed. -func parseWheelOrEggMetadata(_ string, reader io.Reader) (*pkg.EggWheelMetadata, error) { +func parseWheelOrEggMetadata(reader io.Reader) (pkg.PythonPackageMetadata, error) { fields := make(map[string]string) var key string @@ -35,12 +35,12 @@ func parseWheelOrEggMetadata(_ string, reader io.Reader) (*pkg.EggWheelMetadata, case strings.HasPrefix(line, " "): // a field-body continuation if len(key) == 0 { - return nil, fmt.Errorf("no match for continuation: line: '%s'", line) + return pkg.PythonPackageMetadata{}, fmt.Errorf("no match for continuation: line: '%s'", line) } val, ok := fields[key] if !ok { - return nil, fmt.Errorf("no previous key exists, expecting: %s", key) + return pkg.PythonPackageMetadata{}, fmt.Errorf("no previous key exists, expecting: %s", key) } // concatenate onto previous value val = fmt.Sprintf("%s\n %s", val, strings.TrimSpace(line)) @@ -48,25 +48,25 @@ func parseWheelOrEggMetadata(_ string, reader io.Reader) (*pkg.EggWheelMetadata, default: // parse a new key (note, duplicate keys are overridden) if i := strings.Index(line, ":"); i > 0 { - key = strings.TrimSpace(line[0:i]) + // mapstruct cannot map keys with dashes, and we are expected to persist the "Author-email" field + key = strings.ReplaceAll(strings.TrimSpace(line[0:i]), "-", "") val := strings.TrimSpace(line[i+1:]) fields[key] = val } else { - return nil, fmt.Errorf("cannot parse field from line: '%s'", line) + return pkg.PythonPackageMetadata{}, fmt.Errorf("cannot parse field from line: '%s'", line) } } } if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("failed to parse python wheel/egg: %w", err) + return pkg.PythonPackageMetadata{}, fmt.Errorf("failed to parse python wheel/egg: %w", err) } - var metadata pkg.EggWheelMetadata - + var metadata pkg.PythonPackageMetadata if err := mapstructure.Decode(fields, &metadata); err != nil { - return nil, fmt.Errorf("unable to parse APK metadata: %w", err) + return pkg.PythonPackageMetadata{}, fmt.Errorf("unable to parse APK metadata: %w", err) } - return &metadata, nil + return metadata, nil } diff --git a/syft/cataloger/python/parse_wheel_egg_test.go b/syft/cataloger/python/parse_wheel_egg_metadata_test.go similarity index 72% rename from syft/cataloger/python/parse_wheel_egg_test.go rename to syft/cataloger/python/parse_wheel_egg_metadata_test.go index 3b2eae754..d7cada487 100644 --- a/syft/cataloger/python/parse_wheel_egg_test.go +++ b/syft/cataloger/python/parse_wheel_egg_metadata_test.go @@ -8,14 +8,14 @@ import ( "github.com/go-test/deep" ) -func TestParseEggMetadata(t *testing.T) { +func TestParseWheelEggMetadata(t *testing.T) { tests := []struct { Fixture string - ExpectedMetadata pkg.EggWheelMetadata + ExpectedMetadata pkg.PythonPackageMetadata }{ { Fixture: "test-fixtures/egg-info/PKG-INFO", - ExpectedMetadata: pkg.EggWheelMetadata{ + ExpectedMetadata: pkg.PythonPackageMetadata{ Name: "requests", Version: "2.22.0", License: "Apache 2.0", @@ -26,7 +26,7 @@ func TestParseEggMetadata(t *testing.T) { }, { Fixture: "test-fixtures/dist-info/METADATA", - ExpectedMetadata: pkg.EggWheelMetadata{ + ExpectedMetadata: pkg.PythonPackageMetadata{ Name: "Pygments", Version: "2.6.1", License: "BSD License", @@ -44,12 +44,12 @@ func TestParseEggMetadata(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parseWheelOrEggMetadata(fixture.Name(), fixture) + actual, err := parseWheelOrEggMetadata(fixture) if err != nil { - t.Fatalf("failed to parse egg-info: %+v", err) + t.Fatalf("failed to parse: %+v", err) } - for _, d := range deep.Equal(actual, &test.ExpectedMetadata) { + for _, d := range deep.Equal(actual, test.ExpectedMetadata) { t.Errorf("diff: %+v", d) } }) diff --git a/syft/cataloger/python/parse_wheel_egg_record.go b/syft/cataloger/python/parse_wheel_egg_record.go new file mode 100644 index 000000000..5a25c7e7f --- /dev/null +++ b/syft/cataloger/python/parse_wheel_egg_record.go @@ -0,0 +1,60 @@ +package python + +import ( + "encoding/csv" + "fmt" + "io" + "strings" + + "github.com/anchore/syft/syft/pkg" +) + +// parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes), +// returning all Python packages listed. +func parseWheelOrEggRecord(reader io.Reader) ([]pkg.PythonFileRecord, error) { + var records []pkg.PythonFileRecord + r := csv.NewReader(reader) + + for { + recordList, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("unable to read python record file: %w", err) + } + + if len(recordList) != 3 { + return nil, fmt.Errorf("python record an unexpected length=%d: %q", len(recordList), recordList) + } + + var record pkg.PythonFileRecord + + for idx, item := range recordList { + switch idx { + case 0: + record.Path = item + case 1: + if item == "" { + continue + } + fields := strings.Split(item, "=") + + if len(fields) != 2 { + return nil, fmt.Errorf("unexpected python record digest: %q", item) + } + + record.Digest = pkg.Digest{ + Algorithm: fields[0], + Value: fields[1], + } + case 2: + record.Size = item + } + } + + records = append(records, record) + } + + return records, nil +} diff --git a/syft/cataloger/python/parse_wheel_egg_record_test.go b/syft/cataloger/python/parse_wheel_egg_record_test.go new file mode 100644 index 000000000..515ffdf9b --- /dev/null +++ b/syft/cataloger/python/parse_wheel_egg_record_test.go @@ -0,0 +1,57 @@ +package python + +import ( + "os" + "testing" + + "github.com/anchore/syft/syft/pkg" + "github.com/go-test/deep" +) + +func TestParseWheelEggRecord(t *testing.T) { + tests := []struct { + Fixture string + ExpectedMetadata []pkg.PythonFileRecord + }{ + { + Fixture: "test-fixtures/egg-info/RECORD", + ExpectedMetadata: []pkg.PythonFileRecord{ + {Path: "requests-2.22.0.dist-info/INSTALLER", Digest: pkg.Digest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"}, + {Path: "requests/__init__.py", Digest: pkg.Digest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"}, + {Path: "requests/__pycache__/__version__.cpython-38.pyc"}, + {Path: "requests/__pycache__/utils.cpython-38.pyc"}, + {Path: "requests/__version__.py", Digest: pkg.Digest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"}, + {Path: "requests/utils.py", Digest: pkg.Digest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"}, + }, + }, + { + Fixture: "test-fixtures/dist-info/RECORD", + ExpectedMetadata: []pkg.PythonFileRecord{ + {Path: "../../../bin/pygmentize", Digest: pkg.Digest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, + {Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: pkg.Digest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, + {Path: "Pygments-2.6.1.dist-info/RECORD"}, + {Path: "pygments/__pycache__/__init__.cpython-38.pyc"}, + {Path: "pygments/util.py", Digest: pkg.Digest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, + }, + }, + } + + for _, test := range tests { + t.Run(test.Fixture, func(t *testing.T) { + fixture, err := os.Open(test.Fixture) + if err != nil { + t.Fatalf("failed to open fixture: %+v", err) + } + + actual, err := parseWheelOrEggRecord(fixture) + if err != nil { + t.Fatalf("failed to parse: %+v", err) + } + + for _, d := range deep.Equal(actual, test.ExpectedMetadata) { + t.Errorf("diff: %+v", d) + } + }) + } + +} diff --git a/syft/cataloger/python/test-fixtures/dist-info/RECORD b/syft/cataloger/python/test-fixtures/dist-info/RECORD new file mode 100644 index 000000000..af233f390 --- /dev/null +++ b/syft/cataloger/python/test-fixtures/dist-info/RECORD @@ -0,0 +1,5 @@ +../../../bin/pygmentize,sha256=dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8,220 +Pygments-2.6.1.dist-info/AUTHORS,sha256=PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY,8449 +Pygments-2.6.1.dist-info/RECORD,, +pygments/__pycache__/__init__.cpython-38.pyc,, +pygments/util.py,sha256=586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA,10778 \ No newline at end of file diff --git a/syft/cataloger/python/test-fixtures/egg-info/RECORD b/syft/cataloger/python/test-fixtures/egg-info/RECORD new file mode 100644 index 000000000..a2c2f97c3 --- /dev/null +++ b/syft/cataloger/python/test-fixtures/egg-info/RECORD @@ -0,0 +1,6 @@ +requests-2.22.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +requests/__init__.py,sha256=PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA,3921 +requests/__pycache__/__version__.cpython-38.pyc,, +requests/__pycache__/utils.cpython-38.pyc,, +requests/__version__.py,sha256=Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc,436 +requests/utils.py,sha256=LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A,30049 diff --git a/syft/cataloger/python/test-fixtures/partial.dist-info/METADATA b/syft/cataloger/python/test-fixtures/partial.dist-info/METADATA new file mode 100644 index 000000000..924780dfd --- /dev/null +++ b/syft/cataloger/python/test-fixtures/partial.dist-info/METADATA @@ -0,0 +1,47 @@ +Metadata-Version: 2.1 +Name: Pygments +Version: 2.6.1 +Summary: Pygments is a syntax highlighting package written in Python. +Home-page: https://pygments.org/ +Author: Georg Brandl +Author-email: georg@python.org +License: BSD License +Keywords: syntax highlighting +Platform: any +Classifier: License :: OSI Approved :: BSD License +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: End Users/Desktop +Classifier: Intended Audience :: System Administrators +Classifier: Development Status :: 6 - Mature +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Operating System :: OS Independent +Classifier: Topic :: Text Processing :: Filters +Classifier: Topic :: Utilities +Requires-Python: >=3.5 + + +Pygments +~~~~~~~~ + +Pygments is a syntax highlighting package written in Python. + +It is a generic syntax highlighter suitable for use in code hosting, forums, +wikis or other applications that need to prettify source code. Highlights +are: + +* a wide range of over 500 languages and other text formats is supported +* special attention is paid to details, increasing quality by a fair amount +* support for new languages and formats are added easily +* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image formats that PIL supports and ANSI sequences +* it is usable as a command-line tool and as a library + +:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. +:license: BSD, see LICENSE for details. + diff --git a/syft/cataloger/rpmdb/parse_rpmdb_test.go b/syft/cataloger/rpmdb/parse_rpmdb_test.go index 586961759..bf81f111f 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb_test.go +++ b/syft/cataloger/rpmdb/parse_rpmdb_test.go @@ -11,9 +11,10 @@ import ( func TestParseRpmDB(t *testing.T) { expected := map[string]pkg.Package{ "dive": { - Name: "dive", - Version: "0.9.2-1", - Type: pkg.RpmPkg, + Name: "dive", + Version: "0.9.2-1", + Type: pkg.RpmPkg, + MetadataType: pkg.RpmdbMetadataType, Metadata: pkg.RpmdbMetadata{ Name: "dive", Epoch: 0, diff --git a/syft/cataloger/ruby/parse_gemspec_test.go b/syft/cataloger/ruby/parse_gemspec_test.go index 47bc9fc44..2a32ae0b3 100644 --- a/syft/cataloger/ruby/parse_gemspec_test.go +++ b/syft/cataloger/ruby/parse_gemspec_test.go @@ -10,11 +10,12 @@ import ( func TestParseGemspec(t *testing.T) { var expectedPkg = pkg.Package{ - Name: "bundler", - Version: "2.1.4", - Type: pkg.GemPkg, - Licenses: []string{"MIT"}, - Language: pkg.Ruby, + Name: "bundler", + Version: "2.1.4", + Type: pkg.GemPkg, + Licenses: []string{"MIT"}, + Language: pkg.Ruby, + MetadataType: pkg.GemMetadataType, Metadata: pkg.GemMetadata{ Name: "bundler", Version: "2.1.4", diff --git a/syft/pkg/egg_wheel_metadata.go b/syft/pkg/egg_wheel_metadata.go deleted file mode 100644 index 958257469..000000000 --- a/syft/pkg/egg_wheel_metadata.go +++ /dev/null @@ -1,11 +0,0 @@ -package pkg - -// EggWheelMetadata represents all captured data for a python egg or wheel package. -type EggWheelMetadata struct { - Name string `json:"name" mapstruct:"Name"` - Version string `json:"version" mapstruct:"Version"` - License string `json:"license" mapstruct:"License"` - Author string `json:"author" mapstruct:"Author"` - AuthorEmail string `json:"authorEmail" mapstruct:"Author-email"` - Platform string `json:"platform" mapstruct:"Platform"` -} diff --git a/syft/pkg/metadata.go b/syft/pkg/metadata.go index 280e0951b..c1e402934 100644 --- a/syft/pkg/metadata.go +++ b/syft/pkg/metadata.go @@ -8,7 +8,7 @@ const ( DpkgMetadataType MetadataType = "dpkg-metadata" GemMetadataType MetadataType = "gem-metadata" JavaMetadataType MetadataType = "java-metadata" - NpmPackageJsonMetadataType MetadataType = "npm-package-json-metadata" + NpmPackageJSONMetadataType MetadataType = "npm-package-json-metadata" RpmdbMetadataType MetadataType = "rpmdb-metadata" - PythonEggWheelMetadataType MetadataType = "python-egg-wheel-metadata" + PythonPackageMetadataType MetadataType = "python-package-metadata" ) diff --git a/syft/pkg/npm_metadata.go b/syft/pkg/npm_metadata.go index 705de1486..6df2152d0 100644 --- a/syft/pkg/npm_metadata.go +++ b/syft/pkg/npm_metadata.go @@ -1,7 +1,7 @@ package pkg -// NpmPackageJsonMetadata holds extra information that is used in pkg.Package -type NpmPackageJsonMetadata struct { +// NpmPackageJSONMetadata holds extra information that is used in pkg.Package +type NpmPackageJSONMetadata struct { Name string `mapstructure:"name" json:"name"` Version string `mapstructure:"version" json:"version"` Files []string `mapstructure:"files" json:"files"` diff --git a/syft/pkg/python_package_metadata.go b/syft/pkg/python_package_metadata.go new file mode 100644 index 000000000..cd583c5bc --- /dev/null +++ b/syft/pkg/python_package_metadata.go @@ -0,0 +1,23 @@ +package pkg + +type Digest struct { + Algorithm string `json:"algorithm"` + Value string `json:"value"` +} + +type PythonFileRecord struct { + Path string `json:"path"` + Digest Digest `json:"digest"` + Size string `json:"size"` +} + +// PythonPackageMetadata represents all captured data for a python egg or wheel package. +type PythonPackageMetadata struct { + Name string `json:"name" mapstruct:"Name"` + Version string `json:"version" mapstruct:"Version"` + License string `json:"license" mapstruct:"License"` + Author string `json:"author" mapstruct:"Author"` + AuthorEmail string `json:"authorEmail" mapstruct:"Authoremail"` + Platform string `json:"platform" mapstruct:"Platform"` + Files []PythonFileRecord `json:"files,omitempty"` +} diff --git a/syft/scope/resolver.go b/syft/scope/resolver.go index 74a20783a..d32740f3b 100644 --- a/syft/scope/resolver.go +++ b/syft/scope/resolver.go @@ -23,8 +23,13 @@ type ContentResolver interface { // FileResolver knows how to get file.References for given string paths and globs type FileResolver interface { + // FilesByPath fetches a set of file references which have the given path (for an image, there may be multiple matches) FilesByPath(paths ...file.Path) ([]file.Reference, error) + // FilesByGlob fetches a set of file references which the given glob matches FilesByGlob(patterns ...string) ([]file.Reference, error) + // RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. + // This is helpful when attempting to find a file that is in the same layer or lower as another file. + RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) } // getImageResolver returns the appropriate resolve for a container image given the scope option diff --git a/syft/scope/resolvers/all_layers_resolver.go b/syft/scope/resolvers/all_layers_resolver.go index 757d8129f..049d4a19d 100644 --- a/syft/scope/resolvers/all_layers_resolver.go +++ b/syft/scope/resolvers/all_layers_resolver.go @@ -109,6 +109,15 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, e return uniqueFiles, nil } +func (r *AllLayersResolver) RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) { + entry, err := r.img.FileCatalog.Get(reference) + if err != nil { + return nil, err + } + + return entry.Source.SquashedTree.File(file.Path(path)), nil +} + // MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a // file.Reference is a path relative to a particular layer. func (r *AllLayersResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) { diff --git a/syft/scope/resolvers/directory_resolver.go b/syft/scope/resolvers/directory_resolver.go index 54db895bc..beac7755a 100644 --- a/syft/scope/resolvers/directory_resolver.go +++ b/syft/scope/resolvers/directory_resolver.go @@ -75,6 +75,18 @@ func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]file.Reference, er return result, nil } +func (s *DirectoryResolver) RelativeFileByPath(_ file.Reference, path string) (*file.Reference, error) { + paths, err := s.FilesByPath(file.Path(path)) + if err != nil { + return nil, err + } + if len(paths) == 0 { + return nil, nil + } + + return &paths[0], nil +} + // MultipleFileContentsByRef returns the file contents for all file.References relative a directory. func (s DirectoryResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) { refContents := make(map[file.Reference]string) @@ -91,10 +103,10 @@ func (s DirectoryResolver) MultipleFileContentsByRef(f ...file.Reference) (map[f // FileContentsByRef fetches file contents for a single file reference relative to a directory. // If the path does not exist an error is returned. -func (s DirectoryResolver) FileContentsByRef(ref file.Reference) (string, error) { - contents, err := fileContents(ref.Path) +func (s DirectoryResolver) FileContentsByRef(reference file.Reference) (string, error) { + contents, err := fileContents(reference.Path) if err != nil { - return "", fmt.Errorf("could not read contents of file: %s", ref.Path) + return "", fmt.Errorf("could not read contents of file: %s", reference.Path) } return string(contents), nil diff --git a/syft/scope/resolvers/image_squash_resolver.go b/syft/scope/resolvers/image_squash_resolver.go index 429adff34..1e398f0ac 100644 --- a/syft/scope/resolvers/image_squash_resolver.go +++ b/syft/scope/resolvers/image_squash_resolver.go @@ -73,6 +73,18 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference, return uniqueFiles, nil } +func (r *ImageSquashResolver) RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) { + paths, err := r.FilesByPath(file.Path(path)) + if err != nil { + return nil, err + } + if len(paths) == 0 { + return nil, nil + } + + return &paths[0], nil +} + // MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a // file.Reference is a path relative to a particular layer, in this case only from the squashed representation. func (r *ImageSquashResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) { diff --git a/test/integration/pkg_coverage_test.go b/test/integration/pkg_coverage_test.go index 974421a96..13a5af513 100644 --- a/test/integration/pkg_coverage_test.go +++ b/test/integration/pkg_coverage_test.go @@ -68,6 +68,7 @@ func TestPkgCoverageImage(t *testing.T) { } if pkgCount != len(c.pkgInfo) { + t.Logf("Discovered packages of type %+v", c.pkgType) for a := range catalog.Enumerate(c.pkgType) { t.Log(" ", a) } diff --git a/test/integration/test-fixtures/image-pkg-coverage/python/dist-info/RECORD b/test/integration/test-fixtures/image-pkg-coverage/python/dist-info/RECORD new file mode 100644 index 000000000..af233f390 --- /dev/null +++ b/test/integration/test-fixtures/image-pkg-coverage/python/dist-info/RECORD @@ -0,0 +1,5 @@ +../../../bin/pygmentize,sha256=dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8,220 +Pygments-2.6.1.dist-info/AUTHORS,sha256=PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY,8449 +Pygments-2.6.1.dist-info/RECORD,, +pygments/__pycache__/__init__.cpython-38.pyc,, +pygments/util.py,sha256=586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA,10778 \ No newline at end of file diff --git a/test/integration/test-fixtures/image-pkg-coverage/python/someotherpkg-2.19.0-py3.8.egg-info/PKG-INFO b/test/integration/test-fixtures/image-pkg-coverage/python/someotherpkg-3.19.0-py3.8.egg-info/PKG-INFO similarity index 100% rename from test/integration/test-fixtures/image-pkg-coverage/python/someotherpkg-2.19.0-py3.8.egg-info/PKG-INFO rename to test/integration/test-fixtures/image-pkg-coverage/python/someotherpkg-3.19.0-py3.8.egg-info/PKG-INFO