From 495fb0a45f2f41495e2268787bd1b3ca6fd31f62 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 13 Nov 2020 08:48:27 -0500 Subject: [PATCH 01/13] add sbom document import lib helper function Signed-off-by: Alex Goodman --- syft/distro/distro.go | 2 +- syft/distro/identify_test.go | 4 +- syft/distro/type.go | 55 ++++------ syft/lib.go | 39 +++++++ syft/pkg/catalog.go | 13 ++- syft/pkg/language.go | 32 ++---- syft/pkg/package.go | 10 +- syft/presenter/json/artifact.go | 132 ++++++++++++++++++++--- syft/presenter/json/document.go | 45 +++++--- syft/presenter/json/presenter_test.go | 34 +++++- syft/presenter/json/source.go | 29 +++++ test/integration/document_import_test.go | 92 ++++++++++++++++ 12 files changed, 386 insertions(+), 101 deletions(-) create mode 100644 test/integration/document_import_test.go diff --git a/syft/distro/distro.go b/syft/distro/distro.go index 7e01b2493..446357401 100644 --- a/syft/distro/distro.go +++ b/syft/distro/distro.go @@ -57,5 +57,5 @@ func (d Distro) String() string { // Name provides a string repr of the distro func (d Distro) Name() string { - return d.Type.String() + return string(d.Type) } diff --git a/syft/distro/identify_test.go b/syft/distro/identify_test.go index fc61965c2..4371e20dc 100644 --- a/syft/distro/identify_test.go +++ b/syft/distro/identify_test.go @@ -78,8 +78,8 @@ func TestIdentifyDistro(t *testing.T) { observedDistros := internal.NewStringSet() definedDistros := internal.NewStringSet() - for _, d := range All { - definedDistros.Add(d.String()) + for _, distroType := range All { + definedDistros.Add(string(distroType)) } for _, test := range tests { diff --git a/syft/distro/type.go b/syft/distro/type.go index 0f42fca07..9b6f4c035 100644 --- a/syft/distro/type.go +++ b/syft/distro/type.go @@ -1,37 +1,22 @@ package distro +type Type string + const ( - UnknownDistroType Type = iota - Debian - Ubuntu - RedHat - CentOS - Fedora - Alpine - Busybox - AmazonLinux - OracleLinux - ArchLinux - OpenSuseLeap + UnknownDistroType Type = "UnknownDistroType" + Debian Type = "debian" + Ubuntu Type = "ubuntu" + RedHat Type = "redhat" + CentOS Type = "centos" + Fedora Type = "fedora" + Alpine Type = "alpine" + Busybox Type = "busybox" + AmazonLinux Type = "amazonlinux" + OracleLinux Type = "oraclelinux" + ArchLinux Type = "archlinux" + OpenSuseLeap Type = "opensuseleap" ) -type Type int - -var distroStr = []string{ - "UnknownDistroType", - "debian", - "ubuntu", - "redhat", - "centos", - "fedora", - "alpine", - "busybox", - "amazn", - "oraclelinux", - "archlinux", - "opensuse-leap", -} - var All = []Type{ Debian, Ubuntu, @@ -46,14 +31,6 @@ var All = []Type{ OpenSuseLeap, } -func (t Type) String() string { - if int(t) >= len(distroStr) || t < 0 { - return distroStr[0] - } - - return distroStr[t] -} - // IDMapping connects a distro ID like "ubuntu" to a Distro type var IDMapping = map[string]Type{ "debian": Debian, @@ -68,3 +45,7 @@ var IDMapping = map[string]Type{ "arch": ArchLinux, "opensuse-leap": OpenSuseLeap, } + +func (t Type) String() string { + return string(t) +} diff --git a/syft/lib.go b/syft/lib.go index cb7dc66a1..87f69fe1e 100644 --- a/syft/lib.go +++ b/syft/lib.go @@ -17,7 +17,9 @@ Similar to the cataloging process, Linux distribution identification is also per package syft import ( + "encoding/json" "fmt" + "io" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" @@ -25,6 +27,7 @@ import ( "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/logger" "github.com/anchore/syft/syft/pkg" + jsonPresenter "github.com/anchore/syft/syft/presenter/json" "github.com/anchore/syft/syft/scope" "github.com/wagoodman/go-partybus" ) @@ -79,6 +82,42 @@ func CatalogFromScope(s scope.Scope) (*pkg.Catalog, error) { return cataloger.Catalog(s.Resolver, catalogers...) } +// TODO: we shouldn't return the jsonPresenter.Image object! this is leaky +func CatalogFromJSON(reader io.Reader) (*pkg.Catalog, *distro.Distro, error) { + var doc jsonPresenter.Document + decoder := json.NewDecoder(reader) + if err := decoder.Decode(&doc); err != nil { + return nil, nil, err + } + + var pkgs = make([]pkg.Package, len(doc.Artifacts)) + for i, a := range doc.Artifacts { + pkgs[i] = a.ToPackage() + } + + catalog := pkg.NewCatalog(pkgs...) + + var distroType distro.Type + if doc.Distro.Name == "" { + distroType = distro.UnknownDistroType + } else { + distroType = distro.Type(doc.Distro.Name) + } + + d, err := distro.NewDistro(distroType, doc.Distro.Version, doc.Distro.IDLike) + if err != nil { + return nil, nil, err + } + + //var theImg *jsonPresenter.Image + //if doc.Source.Type == "image" { + // img := doc.Source.Target.(jsonPresenter.Image) + // theImg = &img + //} + + return catalog, &d, nil +} + // SetLogger sets the logger object used for all syft logging calls. func SetLogger(logger logger.Logger) { log.Log = logger diff --git a/syft/pkg/catalog.go b/syft/pkg/catalog.go index 5afde2998..0e7ae49e0 100644 --- a/syft/pkg/catalog.go +++ b/syft/pkg/catalog.go @@ -19,12 +19,18 @@ type Catalog struct { } // NewCatalog returns a new empty Catalog -func NewCatalog() *Catalog { - return &Catalog{ +func NewCatalog(pkgs ...Package) *Catalog { + catalog := Catalog{ byID: make(map[ID]*Package), byType: make(map[Type][]*Package), byFile: make(map[file.Reference][]*Package), } + + for _, p := range pkgs { + catalog.Add(p) + } + + return &catalog } // PackageCount returns the total number of packages that have been added. @@ -111,6 +117,9 @@ func (c *Catalog) Sorted(types ...Type) []*Package { sort.SliceStable(pkgs, func(i, j int) bool { if pkgs[i].Name == pkgs[j].Name { + if pkgs[i].Version == pkgs[j].Version { + return pkgs[i].Type < pkgs[j].Type + } return pkgs[i].Version < pkgs[j].Version } return pkgs[i].Name < pkgs[j].Name diff --git a/syft/pkg/language.go b/syft/pkg/language.go index 75032937d..6fc1b184a 100644 --- a/syft/pkg/language.go +++ b/syft/pkg/language.go @@ -1,25 +1,16 @@ package pkg +type Language string + const ( - UnknownLanguage Language = iota - Java - JavaScript - Python - Ruby - Go + UnknownLanguage Language = "UnknownLanguage" + Java Language = "java" + JavaScript Language = "javascript" + Python Language = "python" + Ruby Language = "ruby" + Go Language = "go" ) -type Language uint - -var languageStr = []string{ - "UnknownLanguage", - "java", - "javascript", - "python", - "ruby", - "go", -} - var AllLanguages = []Language{ Java, JavaScript, @@ -28,9 +19,6 @@ var AllLanguages = []Language{ Go, } -func (t Language) String() string { - if int(t) >= len(languageStr) { - return languageStr[0] - } - return languageStr[t] +func (l Language) String() string { + return string(l) } diff --git a/syft/pkg/package.go b/syft/pkg/package.go index c3224a07f..805c9fb6c 100644 --- a/syft/pkg/package.go +++ b/syft/pkg/package.go @@ -23,11 +23,11 @@ type Package struct { FoundBy string `json:"foundBy"` // the specific cataloger that discovered this package Source []file.Reference `json:"sources"` // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package) // TODO: should we move licenses into metadata? - Licenses []string `json:"licenses"` // licenses discovered with the package metadata - Language Language `json:"language"` // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc) - Type Type `json:"type"` // the package type (e.g. Npm, Yarn, Python, Rpm, Deb, etc) - MetadataType MetadataType `json:"metadataType"` // the shape of the additional data in the "metadata" field - Metadata interface{} `json:"metadata,omitempty"` // additional data found while parsing the package source + Licenses []string `json:"licenses"` // licenses discovered with the package metadata + Language Language `json:"language"` // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc) + Type Type `json:"type"` // the package type (e.g. Npm, Yarn, Python, Rpm, Deb, etc) + MetadataType MetadataType `json:"metadataType,omitempty"` // the shape of the additional data in the "metadata" field + Metadata interface{} `json:"metadata,omitempty"` // additional data found while parsing the package source } // ID returns the package ID, which is unique relative to a package catalog. diff --git a/syft/presenter/json/artifact.go b/syft/presenter/json/artifact.go index 3c7957cef..a031e33d2 100644 --- a/syft/presenter/json/artifact.go +++ b/syft/presenter/json/artifact.go @@ -1,18 +1,36 @@ package json import ( + "encoding/json" + "fmt" + "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/scope" ) type Artifact struct { - Name string `json:"name"` - Version string `json:"version"` - Type string `json:"type"` - FoundBy []string `json:"foundBy"` - Locations Locations `json:"locations,omitempty"` - Licenses []string `json:"licenses"` - Metadata interface{} `json:"metadata,omitempty"` + ArtifactBasicMetadata + ArtifactCustomMetadata +} + +type ArtifactBasicMetadata struct { + Name string `json:"name"` + Version string `json:"version"` + Type string `json:"type"` + FoundBy []string `json:"foundBy"` + Locations Locations `json:"locations,omitempty"` + Licenses []string `json:"licenses"` + Language string `json:"language"` +} + +type ArtifactCustomMetadata struct { + MetadataType pkg.MetadataType `json:"metadataType"` + Metadata interface{} `json:"metadata,omitempty"` +} + +type ArtifactMetadataUnpacker struct { + MetadataType string `json:"metadataType"` + Metadata json.RawMessage `json:"metadata"` } func NewArtifact(p *pkg.Package, s scope.Scope) (Artifact, error) { @@ -22,12 +40,98 @@ func NewArtifact(p *pkg.Package, s scope.Scope) (Artifact, error) { } return Artifact{ - Name: p.Name, - Version: p.Version, - Type: string(p.Type), - FoundBy: []string{p.FoundBy}, - Locations: locations, - Licenses: p.Licenses, - Metadata: p.Metadata, + ArtifactBasicMetadata: ArtifactBasicMetadata{ + Name: p.Name, + Version: p.Version, + Type: string(p.Type), + FoundBy: []string{p.FoundBy}, + Locations: locations, + Licenses: p.Licenses, + Language: string(p.Language), + }, + ArtifactCustomMetadata: ArtifactCustomMetadata{ + MetadataType: p.MetadataType, + Metadata: p.Metadata, + }, }, nil } + +func (a Artifact) ToPackage() pkg.Package { + return pkg.Package{ + // does not include found-by and locations + Name: a.Name, + Version: a.Version, + Licenses: a.Licenses, + Language: pkg.Language(a.Language), + Type: pkg.Type(a.Type), + MetadataType: a.MetadataType, + Metadata: a.Metadata, + } +} + +func (a *Artifact) UnmarshalJSON(b []byte) error { + var basic ArtifactBasicMetadata + if err := json.Unmarshal(b, &basic); err != nil { + return err + } + a.ArtifactBasicMetadata = basic + + var unpacker ArtifactMetadataUnpacker + if err := json.Unmarshal(b, &unpacker); err != nil { + return err + } + + a.MetadataType = pkg.MetadataType(unpacker.MetadataType) + + switch a.MetadataType { + case pkg.RpmdbMetadataType: + var payload pkg.RpmdbMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + a.Metadata = payload + case pkg.PythonPackageMetadataType: + var payload pkg.PythonPackageMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + a.Metadata = payload + case pkg.DpkgMetadataType: + var payload pkg.DpkgMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + a.Metadata = payload + case pkg.ApkMetadataType: + var payload pkg.ApkMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + a.Metadata = payload + case pkg.JavaMetadataType: + var payload pkg.JavaMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + a.Metadata = payload + case pkg.NpmPackageJSONMetadataType: + var payload pkg.NpmPackageJSONMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + a.Metadata = payload + case pkg.GemMetadataType: + var payload pkg.GemMetadata + if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { + return err + } + a.Metadata = payload + case "": + // there may be packages with no metadata, which is OK + default: + return fmt.Errorf("unsupported package metadata type: %+v", a.MetadataType) + + } + + return nil +} diff --git a/syft/presenter/json/document.go b/syft/presenter/json/document.go index 713dc7d7a..e1ca6dc51 100644 --- a/syft/presenter/json/document.go +++ b/syft/presenter/json/document.go @@ -1,18 +1,31 @@ package json import ( + "time" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/version" "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/scope" ) type Document struct { - Artifacts []Artifact `json:"artifacts"` - Source Source `json:"source"` - Distro Distribution `json:"distro"` + Artifacts []Artifact `json:"artifacts"` + Source Source `json:"source"` + Distro Distribution `json:"distro"` + Descriptor Descriptor `json:"descriptor"` } -// Distritbution provides information about a detected Linux Distribution +// Descriptor describes what created the document as well as surrounding metadata +type Descriptor struct { + Name string `json:"name"` + Version string `json:"version"` + ReportTimestamp string `json:"reportTimestamp"` + // TODO: we should include scope option here as well (or in source) +} + +// Distribution provides information about a detected Linux Distribution type Distribution struct { Name string `json:"name"` Version string `json:"version"` @@ -20,23 +33,29 @@ type Distribution struct { } func NewDocument(catalog *pkg.Catalog, s scope.Scope, d distro.Distro) (Document, error) { - doc := Document{ - Artifacts: make([]Artifact, 0), - } - src, err := NewSource(s) if err != nil { return Document{}, nil } - doc.Source = src + distroName := d.Name() if distroName == "UnknownDistroType" { distroName = "" } - doc.Distro = Distribution{ - Name: distroName, - Version: d.FullVersion(), - IDLike: d.IDLike, + + doc := Document{ + Artifacts: make([]Artifact, 0), + Source: src, + Distro: Distribution{ + Name: distroName, + Version: d.FullVersion(), + IDLike: d.IDLike, + }, + Descriptor: Descriptor{ + Name: internal.ApplicationName, + Version: version.FromBuild().Version, + ReportTimestamp: time.Now().Format(time.RFC3339), + }, } for _, p := range catalog.Sorted() { diff --git a/syft/presenter/json/presenter_test.go b/syft/presenter/json/presenter_test.go index b935050a2..f60d53dba 100644 --- a/syft/presenter/json/presenter_test.go +++ b/syft/presenter/json/presenter_test.go @@ -25,11 +25,18 @@ func TestJsonDirsPresenter(t *testing.T) { catalog.Add(pkg.Package{ Name: "package-1", Version: "1.0.1", - Type: pkg.DebPkg, + Type: pkg.PythonPkg, FoundBy: "the-cataloger-1", Source: []file.Reference{ {Path: "/some/path/pkg1"}, }, + Language: pkg.Python, + MetadataType: pkg.PythonPackageMetadataType, + Licenses: []string{"MIT"}, + Metadata: pkg.PythonPackageMetadata{ + Name: "package-1", + Version: "1.0.1", + }, }) catalog.Add(pkg.Package{ Name: "package-2", @@ -39,6 +46,11 @@ func TestJsonDirsPresenter(t *testing.T) { Source: []file.Reference{ {Path: "/some/path/pkg1"}, }, + MetadataType: pkg.DpkgMetadataType, + Metadata: pkg.DpkgMetadata{ + Package: "package-2", + Version: "2.0.1", + }, }) d := distro.NewUnknownDistro() s, err := scope.NewScopeFromDir("/some/path") @@ -87,8 +99,15 @@ func TestJsonImgsPresenter(t *testing.T) { Source: []file.Reference{ *img.SquashedTree().File("/somefile-1.txt"), }, - Type: pkg.DebPkg, - FoundBy: "the-cataloger-1", + Type: pkg.PythonPkg, + FoundBy: "the-cataloger-1", + Language: pkg.Python, + MetadataType: pkg.PythonPackageMetadataType, + Licenses: []string{"MIT"}, + Metadata: pkg.PythonPackageMetadata{ + Name: "package-1", + Version: "1.0.1", + }, }) catalog.Add(pkg.Package{ Name: "package-2", @@ -96,8 +115,13 @@ func TestJsonImgsPresenter(t *testing.T) { Source: []file.Reference{ *img.SquashedTree().File("/somefile-2.txt"), }, - Type: pkg.DebPkg, - FoundBy: "the-cataloger-2", + Type: pkg.DebPkg, + FoundBy: "the-cataloger-2", + MetadataType: pkg.DpkgMetadataType, + Metadata: pkg.DpkgMetadata{ + Package: "package-2", + Version: "2.0.1", + }, }) s, err := scope.NewScopeFromImage(img, scope.AllLayersScope) diff --git a/syft/presenter/json/source.go b/syft/presenter/json/source.go index 471390999..d103bc567 100644 --- a/syft/presenter/json/source.go +++ b/syft/presenter/json/source.go @@ -1,6 +1,7 @@ package json import ( + "encoding/json" "fmt" "github.com/anchore/syft/syft/scope" @@ -11,6 +12,11 @@ type Source struct { Target interface{} `json:"target"` } +type SourceUnpacker struct { + Type string `json:"type"` + Target json.RawMessage `json:"target"` +} + func NewSource(s scope.Scope) (Source, error) { switch src := s.Source.(type) { case scope.ImageSource: @@ -27,3 +33,26 @@ func NewSource(s scope.Scope) (Source, error) { return Source{}, fmt.Errorf("unsupported source: %T", src) } } + +func (s *Source) UnmarshalJSON(b []byte) error { + var unpacker SourceUnpacker + if err := json.Unmarshal(b, &unpacker); err != nil { + return err + } + + s.Type = unpacker.Type + + switch s.Type { + case "image": + var payload Image + if err := json.Unmarshal(unpacker.Target, &payload); err != nil { + return err + } + s.Target = payload + default: + return fmt.Errorf("unsupported package metadata type: %+v", s.Type) + + } + + return nil +} diff --git a/test/integration/document_import_test.go b/test/integration/document_import_test.go new file mode 100644 index 000000000..f42492a21 --- /dev/null +++ b/test/integration/document_import_test.go @@ -0,0 +1,92 @@ +package integration + +import ( + "bytes" + "testing" + + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/anchore/syft/syft" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/presenter/json" + "github.com/anchore/syft/syft/scope" + "github.com/go-test/deep" +) + +func TestCatalogFromJSON(t *testing.T) { + + // ensure each of our fixture images results in roughly the same shape when: + // generate json -> import json -> assert packages and distro are the same (except for select fields) + + tests := []struct { + fixture string + }{ + { + fixture: "image-pkg-coverage", + }, + } + + for _, test := range tests { + t.Run(test.fixture, func(t *testing.T) { + _, cleanup := imagetest.GetFixtureImage(t, "docker-archive", test.fixture) + tarPath := imagetest.GetFixtureImageTarPath(t, test.fixture) + defer cleanup() + + expectedCatalog, s, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, scope.AllLayersScope) + if err != nil { + t.Fatalf("failed to catalog image: %+v", err) + } + + var buf bytes.Buffer + jsonPres := json.NewPresenter(expectedCatalog, *s, *expectedDistro) + if err = jsonPres.Present(&buf); err != nil { + t.Fatalf("failed to write to presenter: %+v", err) + } + + // TODO: test img + + actualCatalog, actualDistro, err := syft.CatalogFromJSON(&buf) + if err != nil { + t.Fatalf("failed to import document: %+v", err) + } + + for _, d := range deep.Equal(actualDistro, expectedDistro) { + t.Errorf(" distro diff: %+v", d) + } + + var actualPackages, expectedPackages []*pkg.Package + + // TODO: take out pkg.RpmdbMetadataType filter + + for _, p := range expectedCatalog.Sorted() { + expectedPackages = append(expectedPackages, p) + } + + for _, p := range actualCatalog.Sorted() { + actualPackages = append(actualPackages, p) + } + + if len(actualPackages) != len(expectedPackages) { + t.Fatalf("mismatched package length: %d != %d", len(actualPackages), len(expectedPackages)) + } + + for i, e := range expectedPackages { + a := actualPackages[i] + + // omit fields that should be missing + e.Source = nil + e.FoundBy = "" + if e.MetadataType == pkg.JavaMetadataType { + metadata := e.Metadata.(pkg.JavaMetadata) + metadata.Parent = nil + e.Metadata = metadata + } + + for _, d := range deep.Equal(a, e) { + t.Errorf(" package %d (name=%s) diff: %+v", i, e.Name, d) + } + } + + }) + } + +} From 9668341a14700822b9eeb70c1540894ec243cbcd Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 13 Nov 2020 11:30:24 -0500 Subject: [PATCH 02/13] rename scope to source Signed-off-by: Alex Goodman --- cmd/cmd.go | 6 +- internal/config/config.go | 10 +- syft/cataloger/catalog.go | 8 +- syft/cataloger/cataloger.go | 4 +- syft/cataloger/common/generic_cataloger.go | 6 +- syft/cataloger/deb/cataloger.go | 8 +- syft/cataloger/deb/cataloger_test.go | 4 +- syft/cataloger/python/package_cataloger.go | 12 +- syft/cataloger/rpmdb/cataloger.go | 4 +- syft/cataloger/rpmdb/parse_rpmdb.go | 3 +- syft/distro/identify.go | 4 +- syft/distro/identify_test.go | 6 +- syft/lib.go | 26 +-- syft/pkg/package.go | 11 +- syft/presenter/cyclonedx/component.go | 2 +- syft/presenter/cyclonedx/presenter.go | 18 +- syft/presenter/cyclonedx/presenter_test.go | 6 +- syft/presenter/json/artifact.go | 4 +- syft/presenter/json/document.go | 6 +- syft/presenter/json/image.go | 4 +- syft/presenter/json/location.go | 10 +- syft/presenter/json/presenter.go | 10 +- syft/presenter/json/presenter_test.go | 6 +- syft/presenter/json/source.go | 10 +- syft/presenter/presenter.go | 4 +- syft/presenter/table/presenter.go | 8 +- syft/presenter/table/presenter_test.go | 7 +- syft/presenter/text/presenter.go | 14 +- syft/presenter/text/presenter_test.go | 8 +- syft/scope/resolvers/docs.go | 4 - syft/scope/scope.go | 166 ------------------ .../all_layers_resolver.go | 6 +- .../all_layers_resolver_test.go | 2 +- .../directory_resolver.go | 2 +- .../directory_resolver_test.go | 2 +- .../image_squash_resolver.go | 4 +- .../image_squash_resolver_test.go | 2 +- syft/{scope => source}/resolver.go | 11 +- syft/source/scheme.go | 55 ++++++ syft/{scope/option.go => source/scope.go} | 12 +- .../option_test.go => source/scope_test.go} | 8 +- syft/source/source.go | 117 ++++++++++++ .../scope_test.go => source/source_test.go} | 24 +-- .../test-fixtures/image-symlinks/Dockerfile | 0 .../test-fixtures/image-symlinks/file-1.txt | 0 .../test-fixtures/image-symlinks/file-2.txt | 0 .../image-symlinks/nested/nested/file-3.txt | 0 .../image-symlinks/new-file-2.txt | 0 .../test-fixtures/path-detected/.vimrc | 0 .../test-fixtures/path-detected/empty | 0 test/integration/distro_test.go | 4 +- test/integration/document_import_test.go | 4 +- test/integration/json_schema_test.go | 10 +- test/integration/pkg_coverage_test.go | 8 +- test/integration/regression_test.go | 4 +- 55 files changed, 340 insertions(+), 334 deletions(-) delete mode 100644 syft/scope/resolvers/docs.go delete mode 100644 syft/scope/scope.go rename syft/{scope/resolvers => source}/all_layers_resolver.go (98%) rename syft/{scope/resolvers => source}/all_layers_resolver_test.go (99%) rename syft/{scope/resolvers => source}/directory_resolver.go (99%) rename syft/{scope/resolvers => source}/directory_resolver_test.go (99%) rename syft/{scope/resolvers => source}/image_squash_resolver.go (98%) rename syft/{scope/resolvers => source}/image_squash_resolver_test.go (99%) rename syft/{scope => source}/resolver.go (86%) create mode 100644 syft/source/scheme.go rename syft/{scope/option.go => source/scope.go} (75%) rename syft/{scope/option_test.go => source/scope_test.go} (55%) create mode 100644 syft/source/source.go rename syft/{scope/scope_test.go => source/source_test.go} (95%) rename syft/{scope/resolvers => source}/test-fixtures/image-symlinks/Dockerfile (100%) rename syft/{scope/resolvers => source}/test-fixtures/image-symlinks/file-1.txt (100%) rename syft/{scope/resolvers => source}/test-fixtures/image-symlinks/file-2.txt (100%) rename syft/{scope/resolvers => source}/test-fixtures/image-symlinks/nested/nested/file-3.txt (100%) rename syft/{scope/resolvers => source}/test-fixtures/image-symlinks/new-file-2.txt (100%) rename syft/{scope => source}/test-fixtures/path-detected/.vimrc (100%) rename syft/{scope => source}/test-fixtures/path-detected/empty (100%) diff --git a/cmd/cmd.go b/cmd/cmd.go index f59b08689..0c4a36327 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -7,7 +7,7 @@ import ( "github.com/spf13/cobra" "github.com/anchore/syft/syft/presenter" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/anchore/stereoscope" "github.com/anchore/syft/internal/config" @@ -49,8 +49,8 @@ func setGlobalCliOptions() { // scan options flag := "scope" rootCmd.Flags().StringP( - "scope", "s", scope.SquashedScope.String(), - fmt.Sprintf("selection of layers to catalog, options=%v", scope.Options)) + "scope", "s", source.SquashedScope.String(), + fmt.Sprintf("selection of layers to catalog, options=%v", source.Options)) if err := viper.BindPFlag(flag, rootCmd.Flags().Lookup(flag)); err != nil { fmt.Printf("unable to bind flag '%s': %+v", flag, err) os.Exit(1) diff --git a/internal/config/config.go b/internal/config/config.go index f0faa32f5..ded48e198 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -8,7 +8,7 @@ import ( "github.com/adrg/xdg" "github.com/anchore/syft/internal" "github.com/anchore/syft/syft/presenter" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/mitchellh/go-homedir" "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -23,7 +23,7 @@ type Application struct { ConfigPath string PresenterOpt presenter.Option Output string `mapstructure:"output"` - ScopeOpt scope.Option + ScopeOpt source.Scope Scope string `mapstructure:"scope"` Quiet bool `mapstructure:"quiet"` Log Logging `mapstructure:"log"` @@ -79,9 +79,9 @@ func (cfg *Application) Build() error { } cfg.PresenterOpt = presenterOption - // set the scope - scopeOption := scope.ParseOption(cfg.Scope) - if scopeOption == scope.UnknownScope { + // set the source + scopeOption := source.ParseOption(cfg.Scope) + if scopeOption == source.UnknownScope { return fmt.Errorf("bad --scope value '%s'", cfg.Scope) } cfg.ScopeOpt = scopeOption diff --git a/syft/cataloger/catalog.go b/syft/cataloger/catalog.go index 022f81855..0724be8b6 100644 --- a/syft/cataloger/catalog.go +++ b/syft/cataloger/catalog.go @@ -5,7 +5,7 @@ import ( "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/hashicorp/go-multierror" "github.com/wagoodman/go-partybus" "github.com/wagoodman/go-progress" @@ -32,11 +32,11 @@ func newMonitor() (*progress.Manual, *progress.Manual) { return &filesProcessed, &packagesDiscovered } -// Catalog a given scope (container image or filesystem) with the given catalogers, returning all discovered packages. +// Catalog a given source (container image or filesystem) with the given catalogers, returning all discovered packages. // In order to efficiently retrieve contents from a underlying container image the content fetch requests are // done in bulk. Specifically, all files of interest are collected from each catalogers and accumulated into a single // request. -func Catalog(resolver scope.Resolver, catalogers ...Cataloger) (*pkg.Catalog, error) { +func Catalog(resolver source.Resolver, catalogers ...Cataloger) (*pkg.Catalog, error) { catalog := pkg.NewCatalog() filesProcessed, packagesDiscovered := newMonitor() @@ -54,6 +54,8 @@ func Catalog(resolver scope.Resolver, catalogers ...Cataloger) (*pkg.Catalog, er log.Debugf("cataloger '%s' discovered '%d' packages", theCataloger.Name(), catalogedPackages) packagesDiscovered.N += int64(catalogedPackages) + // helper function to add synthesized information... + for _, p := range packages { catalog.Add(p) } diff --git a/syft/cataloger/cataloger.go b/syft/cataloger/cataloger.go index 0ee9f5be2..aba71dfb9 100644 --- a/syft/cataloger/cataloger.go +++ b/syft/cataloger/cataloger.go @@ -15,7 +15,7 @@ import ( "github.com/anchore/syft/syft/cataloger/rpmdb" "github.com/anchore/syft/syft/cataloger/ruby" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) // Cataloger describes behavior for an object to participate in parsing container image or file system @@ -25,7 +25,7 @@ type Cataloger interface { // Name returns a string that uniquely describes a cataloger Name() string // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source. - Catalog(resolver scope.Resolver) ([]pkg.Package, error) + Catalog(resolver source.Resolver) ([]pkg.Package, error) } // ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages. diff --git a/syft/cataloger/common/generic_cataloger.go b/syft/cataloger/common/generic_cataloger.go index e21270093..bc17d65ec 100644 --- a/syft/cataloger/common/generic_cataloger.go +++ b/syft/cataloger/common/generic_cataloger.go @@ -9,7 +9,7 @@ import ( "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) // GenericCataloger implements the Catalog interface and is responsible for dispatching the proper parser function for @@ -53,7 +53,7 @@ func (c *GenericCataloger) clear() { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source. -func (c *GenericCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) { +func (c *GenericCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { fileSelection := c.selectFiles(resolver) contents, err := resolver.MultipleFileContentsByRef(fileSelection...) if err != nil { @@ -63,7 +63,7 @@ func (c *GenericCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, erro } // SelectFiles takes a set of file trees and resolves and file references of interest for future cataloging -func (c *GenericCataloger) selectFiles(resolver scope.FileResolver) []file.Reference { +func (c *GenericCataloger) selectFiles(resolver source.FileResolver) []file.Reference { // select by exact path for path, parser := range c.pathParsers { files, err := resolver.FilesByPath(file.Path(path)) diff --git a/syft/cataloger/deb/cataloger.go b/syft/cataloger/deb/cataloger.go index c10f0dd71..e9fdecf54 100644 --- a/syft/cataloger/deb/cataloger.go +++ b/syft/cataloger/deb/cataloger.go @@ -11,7 +11,7 @@ import ( "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) const ( @@ -33,7 +33,7 @@ func (c *Cataloger) Name() string { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files. -func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) { +func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { dbFileMatches, err := resolver.FilesByGlob(dpkgStatusGlob) if err != nil { return nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err) @@ -93,7 +93,7 @@ func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) { return pkgs, nil } -func fetchMd5Contents(resolver scope.Resolver, dbRef file.Reference, pkgs []pkg.Package) (map[string]io.Reader, map[string]file.Reference, error) { +func fetchMd5Contents(resolver source.Resolver, dbRef file.Reference, pkgs []pkg.Package) (map[string]io.Reader, map[string]file.Reference, error) { // fetch all MD5 file contents. This approach is more efficient than fetching each MD5 file one at a time var md5FileMatches []file.Reference @@ -146,7 +146,7 @@ func fetchMd5Contents(resolver scope.Resolver, dbRef file.Reference, pkgs []pkg. return contentsByName, refsByName, nil } -func fetchCopyrightContents(resolver scope.Resolver, dbRef file.Reference, pkgs []pkg.Package) (map[string]io.Reader, map[string]file.Reference, error) { +func fetchCopyrightContents(resolver source.Resolver, dbRef file.Reference, pkgs []pkg.Package) (map[string]io.Reader, map[string]file.Reference, error) { // fetch all copyright file contents. This approach is more efficient than fetching each copyright file one at a time var copyrightFileMatches []file.Reference diff --git a/syft/cataloger/deb/cataloger_test.go b/syft/cataloger/deb/cataloger_test.go index fb36c59e5..34ada8e75 100644 --- a/syft/cataloger/deb/cataloger_test.go +++ b/syft/cataloger/deb/cataloger_test.go @@ -5,7 +5,7 @@ import ( "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/go-test/deep" ) @@ -54,7 +54,7 @@ func TestDpkgCataloger(t *testing.T) { img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-dpkg") defer cleanup() - s, err := scope.NewScopeFromImage(img, scope.AllLayersScope) + s, err := source.NewFromImage(img, source.AllLayersScope) if err != nil { t.Fatal(err) } diff --git a/syft/cataloger/python/package_cataloger.go b/syft/cataloger/python/package_cataloger.go index 051933987..bef3a891f 100644 --- a/syft/cataloger/python/package_cataloger.go +++ b/syft/cataloger/python/package_cataloger.go @@ -12,7 +12,7 @@ import ( "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) const ( @@ -33,7 +33,7 @@ func (c *PackageCataloger) Name() string { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations. -func (c *PackageCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) { +func (c *PackageCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { // nolint:prealloc var fileMatches []file.Reference @@ -59,7 +59,7 @@ func (c *PackageCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, erro } // catalogEggOrWheel takes the primary metadata file reference and returns the python package it represents. -func (c *PackageCataloger) catalogEggOrWheel(resolver scope.Resolver, metadataRef file.Reference) (*pkg.Package, error) { +func (c *PackageCataloger) catalogEggOrWheel(resolver source.Resolver, metadataRef file.Reference) (*pkg.Package, error) { metadata, sources, err := c.assembleEggOrWheelMetadata(resolver, metadataRef) if err != nil { return nil, err @@ -84,7 +84,7 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver scope.Resolver, metadataRe } // fetchRecordFiles finds a corresponding RECORD file for the given python package metadata file and returns the set of file records contained. -func (c *PackageCataloger) fetchRecordFiles(resolver scope.Resolver, metadataRef file.Reference) (files []pkg.PythonFileRecord, sources []file.Reference, err error) { +func (c *PackageCataloger) fetchRecordFiles(resolver source.Resolver, metadataRef file.Reference) (files []pkg.PythonFileRecord, sources []file.Reference, err error) { // we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory // or for an image... for an image the METADATA file may be present within multiple layers, so it is important // to reconcile the RECORD path to the same layer (or the next adjacent lower layer). @@ -116,7 +116,7 @@ func (c *PackageCataloger) fetchRecordFiles(resolver scope.Resolver, metadataRef } // fetchTopLevelPackages finds a corresponding top_level.txt file for the given python package metadata file and returns the set of package names contained. -func (c *PackageCataloger) fetchTopLevelPackages(resolver scope.Resolver, metadataRef file.Reference) (pkgs []string, sources []file.Reference, err error) { +func (c *PackageCataloger) fetchTopLevelPackages(resolver source.Resolver, metadataRef file.Reference) (pkgs []string, sources []file.Reference, err error) { // a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages parentDir := filepath.Dir(string(metadataRef.Path)) topLevelPath := filepath.Join(parentDir, "top_level.txt") @@ -149,7 +149,7 @@ func (c *PackageCataloger) fetchTopLevelPackages(resolver scope.Resolver, metada } // assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from. -func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver scope.Resolver, metadataRef file.Reference) (*pkg.PythonPackageMetadata, []file.Reference, error) { +func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.Resolver, metadataRef file.Reference) (*pkg.PythonPackageMetadata, []file.Reference, error) { var sources = []file.Reference{metadataRef} metadataContents, err := resolver.FileContentsByRef(metadataRef) diff --git a/syft/cataloger/rpmdb/cataloger.go b/syft/cataloger/rpmdb/cataloger.go index 6563f076e..c36ac099b 100644 --- a/syft/cataloger/rpmdb/cataloger.go +++ b/syft/cataloger/rpmdb/cataloger.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) const ( @@ -28,7 +28,7 @@ func (c *Cataloger) Name() string { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation. -func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) { +func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { fileMatches, err := resolver.FilesByGlob(packagesGlob) if err != nil { return nil, fmt.Errorf("failed to find rpmdb's by glob: %w", err) diff --git a/syft/cataloger/rpmdb/parse_rpmdb.go b/syft/cataloger/rpmdb/parse_rpmdb.go index 5a88b45cc..7ec9bc206 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb.go +++ b/syft/cataloger/rpmdb/parse_rpmdb.go @@ -9,6 +9,7 @@ import ( "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" rpmdb "github.com/anchore/go-rpmdb/pkg" "github.com/anchore/syft/internal" @@ -80,7 +81,7 @@ func parseRpmDB(resolver scope.FileResolver, dbRef file.Reference, reader io.Rea return allPkgs, nil } -func extractRpmdbFileRecords(resolver scope.FileResolver, entry *rpmdb.PackageInfo) ([]pkg.RpmdbFileRecord, error) { +func extractRpmdbFileRecords(resolver source.FileResolver, entry *rpmdb.PackageInfo) ([]pkg.RpmdbFileRecord, error) { var records = make([]pkg.RpmdbFileRecord, 0) for _, record := range entry.Files { diff --git a/syft/distro/identify.go b/syft/distro/identify.go index ec7cbcb2a..8868c136b 100644 --- a/syft/distro/identify.go +++ b/syft/distro/identify.go @@ -6,7 +6,7 @@ import ( "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) // returns a distro or nil @@ -18,7 +18,7 @@ type parseEntry struct { } // Identify parses distro-specific files to determine distro metadata like version and release. -func Identify(resolver scope.Resolver) Distro { +func Identify(resolver source.Resolver) Distro { distro := NewUnknownDistro() identityFiles := []parseEntry{ diff --git a/syft/distro/identify_test.go b/syft/distro/identify_test.go index 4371e20dc..2ed9fbc46 100644 --- a/syft/distro/identify_test.go +++ b/syft/distro/identify_test.go @@ -8,7 +8,7 @@ import ( "github.com/anchore/syft/internal" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) func TestIdentifyDistro(t *testing.T) { @@ -84,9 +84,9 @@ func TestIdentifyDistro(t *testing.T) { for _, test := range tests { t.Run(test.fixture, func(t *testing.T) { - s, err := scope.NewScopeFromDir(test.fixture) + s, err := source.NewFromDirectory(test.fixture) if err != nil { - t.Fatalf("unable to produce a new scope for testing: %s", test.fixture) + t.Fatalf("unable to produce a new source for testing: %s", test.fixture) } d := Identify(s.Resolver) diff --git a/syft/lib.go b/syft/lib.go index 87f69fe1e..d2ba2e04e 100644 --- a/syft/lib.go +++ b/syft/lib.go @@ -7,8 +7,8 @@ Here is what the main execution path for syft does: 2. Invoke all catalogers to catalog the image, adding discovered packages to a single catalog object 3. Invoke a single presenter to show the contents of the catalog -A Scope object encapsulates the image object to be cataloged and the user options (catalog all layers vs. squashed layer), -providing a way to inspect paths and file content within the image. The Scope object, not the image object, is used +A Source object encapsulates the image object to be cataloged and the user options (catalog all layers vs. squashed layer), +providing a way to inspect paths and file content within the image. The Source object, not the image object, is used throughout the main execution path. This abstraction allows for decoupling of what is cataloged (a docker image, an OCI image, a filesystem, etc) and how it is cataloged (the individual catalogers). @@ -28,15 +28,15 @@ import ( "github.com/anchore/syft/syft/logger" "github.com/anchore/syft/syft/pkg" jsonPresenter "github.com/anchore/syft/syft/presenter/json" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/wagoodman/go-partybus" ) -// Catalog the given image from a particular perspective (e.g. squashed scope, all-layers scope). Returns the discovered -// set of packages, the identified Linux distribution, and the scope object used to wrap the data source. -func Catalog(userInput string, scoptOpt scope.Option) (*pkg.Catalog, *scope.Scope, *distro.Distro, error) { +// Catalog the given image from a particular perspective (e.g. squashed source, all-layers source). Returns the discovered +// set of packages, the identified Linux distribution, and the source object used to wrap the data source. +func Catalog(userInput string, scoptOpt source.Scope) (*pkg.Catalog, *source.Source, *distro.Distro, error) { log.Info("cataloging image") - s, cleanup, err := scope.NewScope(userInput, scoptOpt) + s, cleanup, err := source.NewSource(userInput, scoptOpt) defer cleanup() if err != nil { return nil, nil, nil, err @@ -53,8 +53,8 @@ func Catalog(userInput string, scoptOpt scope.Option) (*pkg.Catalog, *scope.Scop } // IdentifyDistro attempts to discover what the underlying Linux distribution may be from the available flat files -// provided by the given scope object. If results are inconclusive a "UnknownDistro" Type is returned. -func IdentifyDistro(s scope.Scope) distro.Distro { +// provided by the given source object. If results are inconclusive a "UnknownDistro" Type is returned. +func IdentifyDistro(s source.Source) distro.Distro { d := distro.Identify(s.Resolver) if d.Type != distro.UnknownDistroType { log.Infof("identified distro: %s", d.String()) @@ -64,16 +64,16 @@ func IdentifyDistro(s scope.Scope) distro.Distro { return d } -// Catalog the given scope, which may represent a container image or filesystem. Returns the discovered set of packages. -func CatalogFromScope(s scope.Scope) (*pkg.Catalog, error) { +// Catalog the given source, which may represent a container image or filesystem. Returns the discovered set of packages. +func CatalogFromScope(s source.Source) (*pkg.Catalog, error) { log.Info("building the catalog") // conditionally have two sets of catalogers var catalogers []cataloger.Cataloger switch s.Scheme { - case scope.ImageScheme: + case source.ImageScheme: catalogers = cataloger.ImageCatalogers() - case scope.DirectoryScheme: + case source.DirectoryScheme: catalogers = cataloger.DirectoryCatalogers() default: return nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", s.Scheme) diff --git a/syft/pkg/package.go b/syft/pkg/package.go index 805c9fb6c..3b81f46a2 100644 --- a/syft/pkg/package.go +++ b/syft/pkg/package.go @@ -17,11 +17,12 @@ type ID int64 // Package represents an application or library that has been bundled into a distributable format. type Package struct { - id ID // uniquely identifies a package, set by the cataloger - Name string `json:"manifest"` // the package name - Version string `json:"version"` // the version of the package - FoundBy string `json:"foundBy"` // the specific cataloger that discovered this package - Source []file.Reference `json:"sources"` // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package) + id ID // uniquely identifies a package, set by the cataloger + Name string `json:"manifest"` // the package name + Version string `json:"version"` // the version of the package + FoundBy string `json:"foundBy"` // the specific cataloger that discovered this package + Source []file.Reference `json:"-"` // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package) + Location interface{} `json:"locations"` // TODO: should we move licenses into metadata? Licenses []string `json:"licenses"` // licenses discovered with the package metadata Language Language `json:"language"` // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc) diff --git a/syft/presenter/cyclonedx/component.go b/syft/presenter/cyclonedx/component.go index 2217bb5d1..e540ef9ea 100644 --- a/syft/presenter/cyclonedx/component.go +++ b/syft/presenter/cyclonedx/component.go @@ -15,7 +15,7 @@ type Component struct { Description string `xml:"description,omitempty"` // A description of the component Licenses *[]License `xml:"licenses>license"` // A node describing zero or more license names, SPDX license IDs or expressions PackageURL string `xml:"purl,omitempty"` // Specifies the package-url (PackageURL). The purl, if specified, must be valid and conform to the specification defined at: https://github.com/package-url/purl-spec - // TODO: scope, hashes, copyright, cpe, purl, swid, modified, pedigree, externalReferences + // TODO: source, hashes, copyright, cpe, purl, swid, modified, pedigree, externalReferences // TODO: add user-defined parameters for syft-specific values (image layer index, cataloger, location path, etc.) } diff --git a/syft/presenter/cyclonedx/presenter.go b/syft/presenter/cyclonedx/presenter.go index 1b8accebe..22d9a8238 100644 --- a/syft/presenter/cyclonedx/presenter.go +++ b/syft/presenter/cyclonedx/presenter.go @@ -11,21 +11,21 @@ import ( "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) -// Presenter writes a CycloneDX report from the given Catalog and Scope contents +// Presenter writes a CycloneDX report from the given Catalog and Source contents type Presenter struct { catalog *pkg.Catalog - scope scope.Scope + source source.Source distro distro.Distro } -// NewPresenter creates a CycloneDX presenter from the given Catalog and Scope objects. -func NewPresenter(catalog *pkg.Catalog, s scope.Scope, d distro.Distro) *Presenter { +// NewPresenter creates a CycloneDX presenter from the given Catalog and Source objects. +func NewPresenter(catalog *pkg.Catalog, s source.Source, d distro.Distro) *Presenter { return &Presenter{ catalog: catalog, - scope: s, + source: s, distro: d, } } @@ -34,8 +34,8 @@ func NewPresenter(catalog *pkg.Catalog, s scope.Scope, d distro.Distro) *Present func (pres *Presenter) Present(output io.Writer) error { bom := NewDocumentFromCatalog(pres.catalog, pres.distro) - switch src := pres.scope.Source.(type) { - case scope.DirSource: + switch src := pres.source.Target.(type) { + case source.DirSource: bom.BomDescriptor.Component = &BdComponent{ Component: Component{ Type: "file", @@ -43,7 +43,7 @@ func (pres *Presenter) Present(output io.Writer) error { Version: "", }, } - case scope.ImageSource: + case source.ImageSource: var imageID string var versionStr string if len(src.Img.Metadata.Tags) > 0 { diff --git a/syft/presenter/cyclonedx/presenter_test.go b/syft/presenter/cyclonedx/presenter_test.go index 39778d3c1..df8b3745c 100644 --- a/syft/presenter/cyclonedx/presenter_test.go +++ b/syft/presenter/cyclonedx/presenter_test.go @@ -12,7 +12,7 @@ import ( "github.com/anchore/go-testutils" "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/sergi/go-diff/diffmatchpatch" ) @@ -57,7 +57,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) { }, }) - s, err := scope.NewScopeFromDir("/some/path") + s, err := source.NewFromDirectory("/some/path") if err != nil { t.Fatal(err) } @@ -147,7 +147,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) { }, }) - s, err := scope.NewScopeFromImage(img, scope.AllLayersScope) + s, err := source.NewFromImage(img, source.AllLayersScope) if err != nil { t.Fatal(err) } diff --git a/syft/presenter/json/artifact.go b/syft/presenter/json/artifact.go index a031e33d2..3c15e5353 100644 --- a/syft/presenter/json/artifact.go +++ b/syft/presenter/json/artifact.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) type Artifact struct { @@ -33,7 +33,7 @@ type ArtifactMetadataUnpacker struct { Metadata json.RawMessage `json:"metadata"` } -func NewArtifact(p *pkg.Package, s scope.Scope) (Artifact, error) { +func NewArtifact(p *pkg.Package, s source.Source) (Artifact, error) { locations, err := NewLocations(p, s) if err != nil { return Artifact{}, err diff --git a/syft/presenter/json/document.go b/syft/presenter/json/document.go index e1ca6dc51..78cb55ce2 100644 --- a/syft/presenter/json/document.go +++ b/syft/presenter/json/document.go @@ -7,7 +7,7 @@ import ( "github.com/anchore/syft/internal/version" "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) type Document struct { @@ -22,7 +22,7 @@ type Descriptor struct { Name string `json:"name"` Version string `json:"version"` ReportTimestamp string `json:"reportTimestamp"` - // TODO: we should include scope option here as well (or in source) + // TODO: we should include source option here as well (or in source) } // Distribution provides information about a detected Linux Distribution @@ -32,7 +32,7 @@ type Distribution struct { IDLike string `json:"idLike"` } -func NewDocument(catalog *pkg.Catalog, s scope.Scope, d distro.Distro) (Document, error) { +func NewDocument(catalog *pkg.Catalog, s source.Source, d distro.Distro) (Document, error) { src, err := NewSource(s) if err != nil { return Document{}, nil diff --git a/syft/presenter/json/image.go b/syft/presenter/json/image.go index bffa999bf..ee7796421 100644 --- a/syft/presenter/json/image.go +++ b/syft/presenter/json/image.go @@ -1,7 +1,7 @@ package json import ( - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) type Image struct { @@ -18,7 +18,7 @@ type Layer struct { Size int64 `json:"size"` } -func NewImage(src scope.ImageSource) *Image { +func NewImage(src source.ImageSource) *Image { // populate artifacts... tags := make([]string, len(src.Img.Metadata.Tags)) for idx, tag := range src.Img.Metadata.Tags { diff --git a/syft/presenter/json/location.go b/syft/presenter/json/location.go index 8a0fe1209..2321afa49 100644 --- a/syft/presenter/json/location.go +++ b/syft/presenter/json/location.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) type Locations interface{} @@ -14,9 +14,9 @@ type ImageLocation struct { LayerIndex uint `json:"layerIndex"` } -func NewLocations(p *pkg.Package, s scope.Scope) (Locations, error) { - switch src := s.Source.(type) { - case scope.ImageSource: +func NewLocations(p *pkg.Package, s source.Source) (Locations, error) { + switch src := s.Target.(type) { + case source.ImageSource: locations := make([]ImageLocation, len(p.Source)) for idx := range p.Source { entry, err := src.Img.FileCatalog.Get(p.Source[idx]) @@ -33,7 +33,7 @@ func NewLocations(p *pkg.Package, s scope.Scope) (Locations, error) { } return locations, nil - case scope.DirSource: + case source.DirSource: locations := make([]string, len(p.Source)) for idx := range p.Source { locations[idx] = string(p.Source[idx].Path) diff --git a/syft/presenter/json/presenter.go b/syft/presenter/json/presenter.go index 7c5ee5407..beb2e689b 100644 --- a/syft/presenter/json/presenter.go +++ b/syft/presenter/json/presenter.go @@ -6,25 +6,25 @@ import ( "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) type Presenter struct { catalog *pkg.Catalog - scope scope.Scope + source source.Source distro distro.Distro } -func NewPresenter(catalog *pkg.Catalog, s scope.Scope, d distro.Distro) *Presenter { +func NewPresenter(catalog *pkg.Catalog, s source.Source, d distro.Distro) *Presenter { return &Presenter{ catalog: catalog, - scope: s, + source: s, distro: d, } } func (pres *Presenter) Present(output io.Writer) error { - doc, err := NewDocument(pres.catalog, pres.scope, pres.distro) + doc, err := NewDocument(pres.catalog, pres.source, pres.distro) if err != nil { return err } diff --git a/syft/presenter/json/presenter_test.go b/syft/presenter/json/presenter_test.go index f60d53dba..c7f389df5 100644 --- a/syft/presenter/json/presenter_test.go +++ b/syft/presenter/json/presenter_test.go @@ -10,7 +10,7 @@ import ( "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/sergi/go-diff/diffmatchpatch" ) @@ -53,7 +53,7 @@ func TestJsonDirsPresenter(t *testing.T) { }, }) d := distro.NewUnknownDistro() - s, err := scope.NewScopeFromDir("/some/path") + s, err := source.NewFromDirectory("/some/path") if err != nil { t.Fatal(err) } @@ -124,7 +124,7 @@ func TestJsonImgsPresenter(t *testing.T) { }, }) - s, err := scope.NewScopeFromImage(img, scope.AllLayersScope) + s, err := source.NewFromImage(img, source.AllLayersScope) d := distro.NewUnknownDistro() pres := NewPresenter(catalog, s, d) diff --git a/syft/presenter/json/source.go b/syft/presenter/json/source.go index d103bc567..6172f9bb4 100644 --- a/syft/presenter/json/source.go +++ b/syft/presenter/json/source.go @@ -4,7 +4,7 @@ import ( "encoding/json" "fmt" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) type Source struct { @@ -17,14 +17,14 @@ type SourceUnpacker struct { Target json.RawMessage `json:"target"` } -func NewSource(s scope.Scope) (Source, error) { - switch src := s.Source.(type) { - case scope.ImageSource: +func NewSource(s source.Source) (Source, error) { + switch src := s.Target.(type) { + case source.ImageSource: return Source{ Type: "image", Target: NewImage(src), }, nil - case scope.DirSource: + case source.DirSource: return Source{ Type: "directory", Target: src.Path, diff --git a/syft/presenter/presenter.go b/syft/presenter/presenter.go index 65184a74e..56f00f9e6 100644 --- a/syft/presenter/presenter.go +++ b/syft/presenter/presenter.go @@ -15,7 +15,7 @@ import ( "github.com/anchore/syft/syft/presenter/json" "github.com/anchore/syft/syft/presenter/table" "github.com/anchore/syft/syft/presenter/text" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) // Presenter defines the expected behavior for an object responsible for displaying arbitrary input and processed data @@ -25,7 +25,7 @@ type Presenter interface { } // GetPresenter returns a presenter for images or directories -func GetPresenter(option Option, s scope.Scope, catalog *pkg.Catalog, d *distro.Distro) Presenter { +func GetPresenter(option Option, s source.Source, catalog *pkg.Catalog, d *distro.Distro) Presenter { switch option { case JSONPresenter: return json.NewPresenter(catalog, s, *d) diff --git a/syft/presenter/table/presenter.go b/syft/presenter/table/presenter.go index 29f4631c8..6af222559 100644 --- a/syft/presenter/table/presenter.go +++ b/syft/presenter/table/presenter.go @@ -9,18 +9,18 @@ import ( "github.com/olekukonko/tablewriter" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) type Presenter struct { catalog *pkg.Catalog - scope scope.Scope + source source.Source } -func NewPresenter(catalog *pkg.Catalog, s scope.Scope) *Presenter { +func NewPresenter(catalog *pkg.Catalog, s source.Source) *Presenter { return &Presenter{ catalog: catalog, - scope: s, + source: s, } } diff --git a/syft/presenter/table/presenter_test.go b/syft/presenter/table/presenter_test.go index 4958b1943..afd8ac465 100644 --- a/syft/presenter/table/presenter_test.go +++ b/syft/presenter/table/presenter_test.go @@ -3,14 +3,15 @@ package table import ( "bytes" "flag" - "github.com/go-test/deep" "testing" + "github.com/go-test/deep" + "github.com/anchore/go-testutils" "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/sergi/go-diff/diffmatchpatch" ) @@ -43,7 +44,7 @@ func TestTablePresenter(t *testing.T) { Type: pkg.DebPkg, }) - s, err := scope.NewScopeFromImage(img, scope.AllLayersScope) + s, err := source.NewFromImage(img, source.AllLayersScope) pres := NewPresenter(catalog, s) // run presenter diff --git a/syft/presenter/text/presenter.go b/syft/presenter/text/presenter.go index 3291370d9..aeaf26d67 100644 --- a/syft/presenter/text/presenter.go +++ b/syft/presenter/text/presenter.go @@ -7,18 +7,18 @@ import ( "text/tabwriter" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) type Presenter struct { catalog *pkg.Catalog - scope scope.Scope + source source.Source } -func NewPresenter(catalog *pkg.Catalog, s scope.Scope) *Presenter { +func NewPresenter(catalog *pkg.Catalog, s source.Source) *Presenter { return &Presenter{ catalog: catalog, - scope: s, + source: s, } } @@ -28,10 +28,10 @@ func (pres *Presenter) Present(output io.Writer) error { w := new(tabwriter.Writer) w.Init(output, 0, 8, 0, '\t', tabwriter.AlignRight) - switch src := pres.scope.Source.(type) { - case scope.DirSource: + switch src := pres.source.Target.(type) { + case source.DirSource: fmt.Fprintln(w, fmt.Sprintf("[Path: %s]", src.Path)) - case scope.ImageSource: + case source.ImageSource: fmt.Fprintln(w, "[Image]") for idx, l := range src.Img.Layers { diff --git a/syft/presenter/text/presenter_test.go b/syft/presenter/text/presenter_test.go index 96dbd84c1..9fdcd860a 100644 --- a/syft/presenter/text/presenter_test.go +++ b/syft/presenter/text/presenter_test.go @@ -9,7 +9,7 @@ import ( "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/sergi/go-diff/diffmatchpatch" ) @@ -32,9 +32,9 @@ func TestTextDirPresenter(t *testing.T) { Type: pkg.DebPkg, }) - s, err := scope.NewScopeFromDir("/some/path") + s, err := source.NewFromDirectory("/some/path") if err != nil { - t.Fatalf("unable to create scope: %+v", err) + t.Fatalf("unable to create source: %+v", err) } pres := NewPresenter(catalog, s) @@ -98,7 +98,7 @@ func TestTextImgPresenter(t *testing.T) { l.Metadata.Digest = "sha256:ad8ecdc058976c07e7e347cb89fa9ad86a294b5ceaae6d09713fb035f84115abf3c4a2388a4af3aa60f13b94f4c6846930bdf53" } - s, err := scope.NewScopeFromImage(img, scope.AllLayersScope) + s, err := source.NewFromImage(img, source.AllLayersScope) if err != nil { t.Fatal(err) } diff --git a/syft/scope/resolvers/docs.go b/syft/scope/resolvers/docs.go deleted file mode 100644 index fe8f17e76..000000000 --- a/syft/scope/resolvers/docs.go +++ /dev/null @@ -1,4 +0,0 @@ -/* -Package resolvers provides concrete implementations for the scope.Resolver interface for all supported data sources and scope options. -*/ -package resolvers diff --git a/syft/scope/scope.go b/syft/scope/scope.go deleted file mode 100644 index bf2ed74f3..000000000 --- a/syft/scope/scope.go +++ /dev/null @@ -1,166 +0,0 @@ -/* -Package scope provides an abstraction to allow a user to loosely define a data source to catalog and expose a common interface that -catalogers and use explore and analyze data from the data source. All valid (cataloggable) data sources are defined -within this package. -*/ -package scope - -import ( - "fmt" - "strings" - - "github.com/mitchellh/go-homedir" - - "github.com/spf13/afero" - - "github.com/anchore/stereoscope" - - "github.com/anchore/stereoscope/pkg/image" - "github.com/anchore/syft/syft/scope/resolvers" -) - -const ( - UnknownScheme Scheme = "unknown-scheme" - DirectoryScheme Scheme = "directory-scheme" - ImageScheme Scheme = "image-scheme" -) - -type Scheme string - -// ImageSource represents a data source that is a container image -type ImageSource struct { - Img *image.Image // the image object to be cataloged -} - -// DirSource represents a data source that is a filesystem directory tree -type DirSource struct { - Path string // the root path to be cataloged -} - -// Scope is an object that captures the data source to be cataloged, configuration, and a specific resolver used -// in cataloging (based on the data source and configuration) -type Scope struct { - Option Option // specific perspective to catalog - Resolver Resolver // a Resolver object to use in file path/glob resolution and file contents resolution - Source interface{} // the specific source object to be cataloged - Scheme Scheme // the source data scheme type (directory or image) -} - -// NewScope produces a Scope based on userInput like dir: or image:tag -func NewScope(userInput string, o Option) (Scope, func(), error) { - fs := afero.NewOsFs() - parsedScheme, location, err := detectScheme(fs, image.DetectSource, userInput) - if err != nil { - return Scope{}, func() {}, fmt.Errorf("unable to parse input=%q: %w", userInput, err) - } - - switch parsedScheme { - case DirectoryScheme: - fileMeta, err := fs.Stat(location) - if err != nil { - return Scope{}, func() {}, fmt.Errorf("unable to stat dir=%q: %w", location, err) - } - - if !fileMeta.IsDir() { - return Scope{}, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", location, err) - } - - s, err := NewScopeFromDir(location) - if err != nil { - return Scope{}, func() {}, fmt.Errorf("could not populate scope from path=%q: %w", location, err) - } - return s, func() {}, nil - - case ImageScheme: - img, err := stereoscope.GetImage(location) - cleanup := func() { - stereoscope.Cleanup() - } - - if err != nil || img == nil { - return Scope{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", location, err) - } - - s, err := NewScopeFromImage(img, o) - if err != nil { - return Scope{}, cleanup, fmt.Errorf("could not populate scope with image: %w", err) - } - return s, cleanup, nil - } - - return Scope{}, func() {}, fmt.Errorf("unable to process input for scanning: '%s'", userInput) -} - -// NewScopeFromDir creates a new scope object tailored to catalog a given filesystem directory recursively. -func NewScopeFromDir(path string) (Scope, error) { - return Scope{ - Resolver: &resolvers.DirectoryResolver{ - Path: path, - }, - Source: DirSource{ - Path: path, - }, - Scheme: DirectoryScheme, - }, nil -} - -// NewScopeFromImage creates a new scope object tailored to catalog a given container image, relative to the -// option given (e.g. all-layers, squashed, etc) -func NewScopeFromImage(img *image.Image, option Option) (Scope, error) { - if img == nil { - return Scope{}, fmt.Errorf("no image given") - } - - resolver, err := getImageResolver(img, option) - if err != nil { - return Scope{}, fmt.Errorf("could not determine file resolver: %w", err) - } - - return Scope{ - Option: option, - Resolver: resolver, - Source: ImageSource{ - Img: img, - }, - Scheme: ImageScheme, - }, nil -} - -type sourceDetector func(string) (image.Source, string, error) - -func detectScheme(fs afero.Fs, imageDetector sourceDetector, userInput string) (Scheme, string, error) { - if strings.HasPrefix(userInput, "dir:") { - // blindly trust the user's scheme - dirLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "dir:")) - if err != nil { - return UnknownScheme, "", fmt.Errorf("unable to expand directory path: %w", err) - } - return DirectoryScheme, dirLocation, nil - } - - // we should attempt to let stereoscope determine what the source is first --just because the source is a valid directory - // doesn't mean we yet know if it is an OCI layout directory (to be treated as an image) or if it is a generic filesystem directory. - source, imageSpec, err := imageDetector(userInput) - if err != nil { - return UnknownScheme, "", fmt.Errorf("unable to detect the scheme from %q: %w", userInput, err) - } - - if source == image.UnknownSource { - dirLocation, err := homedir.Expand(userInput) - if err != nil { - return UnknownScheme, "", fmt.Errorf("unable to expand potential directory path: %w", err) - } - - fileMeta, err := fs.Stat(dirLocation) - if err != nil { - return UnknownScheme, "", nil - } - - if fileMeta.IsDir() { - return DirectoryScheme, dirLocation, nil - } - return UnknownScheme, "", nil - } - - return ImageScheme, imageSpec, nil -} diff --git a/syft/scope/resolvers/all_layers_resolver.go b/syft/source/all_layers_resolver.go similarity index 98% rename from syft/scope/resolvers/all_layers_resolver.go rename to syft/source/all_layers_resolver.go index a1c57ccda..a050ad616 100644 --- a/syft/scope/resolvers/all_layers_resolver.go +++ b/syft/source/all_layers_resolver.go @@ -1,4 +1,4 @@ -package resolvers +package source import ( "archive/tar" @@ -8,7 +8,7 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) -// AllLayersResolver implements path and content access for the AllLayers scope option for container image data sources. +// AllLayersResolver implements path and content access for the AllLayers source option for container image data sources. type AllLayersResolver struct { img *image.Image layers []int @@ -41,7 +41,7 @@ func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.Ref if entry.Metadata.TypeFlag == tar.TypeLink || entry.Metadata.TypeFlag == tar.TypeSymlink { // a link may resolve in this layer or higher, assuming a squashed tree is used to search - // we should search all possible resolutions within the valid scope + // we should search all possible resolutions within the valid source for _, subLayerIdx := range r.layers[layerIdx:] { resolvedRef, err := r.img.ResolveLinkByLayerSquash(ref, subLayerIdx) if err != nil { diff --git a/syft/scope/resolvers/all_layers_resolver_test.go b/syft/source/all_layers_resolver_test.go similarity index 99% rename from syft/scope/resolvers/all_layers_resolver_test.go rename to syft/source/all_layers_resolver_test.go index e99ebd8a6..e8076f4cb 100644 --- a/syft/scope/resolvers/all_layers_resolver_test.go +++ b/syft/source/all_layers_resolver_test.go @@ -1,4 +1,4 @@ -package resolvers +package source import ( "testing" diff --git a/syft/scope/resolvers/directory_resolver.go b/syft/source/directory_resolver.go similarity index 99% rename from syft/scope/resolvers/directory_resolver.go rename to syft/source/directory_resolver.go index 5677c6dde..89b717db0 100644 --- a/syft/scope/resolvers/directory_resolver.go +++ b/syft/source/directory_resolver.go @@ -1,4 +1,4 @@ -package resolvers +package source import ( "fmt" diff --git a/syft/scope/resolvers/directory_resolver_test.go b/syft/source/directory_resolver_test.go similarity index 99% rename from syft/scope/resolvers/directory_resolver_test.go rename to syft/source/directory_resolver_test.go index 809148da8..fa234c10e 100644 --- a/syft/scope/resolvers/directory_resolver_test.go +++ b/syft/source/directory_resolver_test.go @@ -1,4 +1,4 @@ -package resolvers +package source import ( "testing" diff --git a/syft/scope/resolvers/image_squash_resolver.go b/syft/source/image_squash_resolver.go similarity index 98% rename from syft/scope/resolvers/image_squash_resolver.go rename to syft/source/image_squash_resolver.go index be1b7fe80..07a9a54d9 100644 --- a/syft/scope/resolvers/image_squash_resolver.go +++ b/syft/source/image_squash_resolver.go @@ -1,4 +1,4 @@ -package resolvers +package source import ( "fmt" @@ -7,7 +7,7 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) -// ImageSquashResolver implements path and content access for the Squashed scope option for container image data sources. +// ImageSquashResolver implements path and content access for the Squashed source option for container image data sources. type ImageSquashResolver struct { img *image.Image } diff --git a/syft/scope/resolvers/image_squash_resolver_test.go b/syft/source/image_squash_resolver_test.go similarity index 99% rename from syft/scope/resolvers/image_squash_resolver_test.go rename to syft/source/image_squash_resolver_test.go index dee432125..88f8074a2 100644 --- a/syft/scope/resolvers/image_squash_resolver_test.go +++ b/syft/source/image_squash_resolver_test.go @@ -1,4 +1,4 @@ -package resolvers +package source import ( "testing" diff --git a/syft/scope/resolver.go b/syft/source/resolver.go similarity index 86% rename from syft/scope/resolver.go rename to syft/source/resolver.go index d32740f3b..6af9feaa2 100644 --- a/syft/scope/resolver.go +++ b/syft/source/resolver.go @@ -1,11 +1,10 @@ -package scope +package source import ( "fmt" "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/image" - "github.com/anchore/syft/syft/scope/resolvers" ) // Resolver is an interface that encompasses how to get specific file references and file contents for a generic data source. @@ -32,13 +31,13 @@ type FileResolver interface { RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) } -// getImageResolver returns the appropriate resolve for a container image given the scope option -func getImageResolver(img *image.Image, option Option) (Resolver, error) { +// getImageResolver returns the appropriate resolve for a container image given the source option +func getImageResolver(img *image.Image, option Scope) (Resolver, error) { switch option { case SquashedScope: - return resolvers.NewImageSquashResolver(img) + return NewImageSquashResolver(img) case AllLayersScope: - return resolvers.NewAllLayersResolver(img) + return NewAllLayersResolver(img) default: return nil, fmt.Errorf("bad option provided: %+v", option) } diff --git a/syft/source/scheme.go b/syft/source/scheme.go new file mode 100644 index 000000000..c12a1f69f --- /dev/null +++ b/syft/source/scheme.go @@ -0,0 +1,55 @@ +package source + +import ( + "fmt" + "strings" + + "github.com/anchore/stereoscope/pkg/image" + "github.com/mitchellh/go-homedir" + "github.com/spf13/afero" +) + +type Scheme string + +const ( + UnknownScheme Scheme = "unknown-scheme" + DirectoryScheme Scheme = "directory-scheme" + ImageScheme Scheme = "image-scheme" +) + +func detectScheme(fs afero.Fs, imageDetector sourceDetector, userInput string) (Scheme, string, error) { + if strings.HasPrefix(userInput, "dir:") { + // blindly trust the user's scheme + dirLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "dir:")) + if err != nil { + return UnknownScheme, "", fmt.Errorf("unable to expand directory path: %w", err) + } + return DirectoryScheme, dirLocation, nil + } + + // we should attempt to let stereoscope determine what the source is first --just because the source is a valid directory + // doesn't mean we yet know if it is an OCI layout directory (to be treated as an image) or if it is a generic filesystem directory. + source, imageSpec, err := imageDetector(userInput) + if err != nil { + return UnknownScheme, "", fmt.Errorf("unable to detect the scheme from %q: %w", userInput, err) + } + + if source == image.UnknownSource { + dirLocation, err := homedir.Expand(userInput) + if err != nil { + return UnknownScheme, "", fmt.Errorf("unable to expand potential directory path: %w", err) + } + + fileMeta, err := fs.Stat(dirLocation) + if err != nil { + return UnknownScheme, "", nil + } + + if fileMeta.IsDir() { + return DirectoryScheme, dirLocation, nil + } + return UnknownScheme, "", nil + } + + return ImageScheme, imageSpec, nil +} diff --git a/syft/scope/option.go b/syft/source/scope.go similarity index 75% rename from syft/scope/option.go rename to syft/source/scope.go index fcfd49302..d870da2da 100644 --- a/syft/scope/option.go +++ b/syft/source/scope.go @@ -1,14 +1,14 @@ -package scope +package source import "strings" const ( - UnknownScope Option = iota + UnknownScope Scope = iota SquashedScope AllLayersScope ) -type Option int +type Scope int var optionStr = []string{ "UnknownScope", @@ -16,12 +16,12 @@ var optionStr = []string{ "AllLayers", } -var Options = []Option{ +var Options = []Scope{ SquashedScope, AllLayersScope, } -func ParseOption(userStr string) Option { +func ParseOption(userStr string) Scope { switch strings.ToLower(userStr) { case strings.ToLower(SquashedScope.String()): return SquashedScope @@ -31,7 +31,7 @@ func ParseOption(userStr string) Option { return UnknownScope } -func (o Option) String() string { +func (o Scope) String() string { if int(o) >= len(optionStr) || o < 0 { return optionStr[0] } diff --git a/syft/scope/option_test.go b/syft/source/scope_test.go similarity index 55% rename from syft/scope/option_test.go rename to syft/source/scope_test.go index a684c2260..0c210e013 100644 --- a/syft/scope/option_test.go +++ b/syft/source/scope_test.go @@ -1,4 +1,4 @@ -package scope +package source import ( "fmt" @@ -6,12 +6,12 @@ import ( ) func TestOptionStringerBoundary(t *testing.T) { - var _ fmt.Stringer = Option(0) + var _ fmt.Stringer = Scope(0) for _, c := range []int{-1, 0, 3} { - option := Option(c) + option := Scope(c) if option.String() != UnknownScope.String() { - t.Errorf("expected Option(%d) to be unknown, found '%+v'", c, option) + t.Errorf("expected Scope(%d) to be unknown, found '%+v'", c, option) } } } diff --git a/syft/source/source.go b/syft/source/source.go new file mode 100644 index 000000000..b3e49d22d --- /dev/null +++ b/syft/source/source.go @@ -0,0 +1,117 @@ +/* +Package source provides an abstraction to allow a user to loosely define a data source to catalog and expose a common interface that +catalogers and use explore and analyze data from the data source. All valid (cataloggable) data sources are defined +within this package. +*/ +package source + +import ( + "fmt" + + "github.com/spf13/afero" + + "github.com/anchore/stereoscope" + + "github.com/anchore/stereoscope/pkg/image" +) + +// ImageSource represents a data source that is a container image +type ImageSource struct { + Img *image.Image // the image object to be cataloged +} + +// DirSource represents a data source that is a filesystem directory tree +type DirSource struct { + Path string // the root path to be cataloged +} + +// Source is an object that captures the data source to be cataloged, configuration, and a specific resolver used +// in cataloging (based on the data source and configuration) +type Source struct { + Scope Scope // specific perspective to catalog + Resolver Resolver // a Resolver object to use in file path/glob resolution and file contents resolution + Target interface{} // the specific source object to be cataloged + Scheme Scheme // the source data scheme type (directory or image) +} + +type sourceDetector func(string) (image.Source, string, error) + +// NewSource produces a Source based on userInput like dir: or image:tag +func NewSource(userInput string, o Scope) (Source, func(), error) { + fs := afero.NewOsFs() + parsedScheme, location, err := detectScheme(fs, image.DetectSource, userInput) + if err != nil { + return Source{}, func() {}, fmt.Errorf("unable to parse input=%q: %w", userInput, err) + } + + switch parsedScheme { + case DirectoryScheme: + fileMeta, err := fs.Stat(location) + if err != nil { + return Source{}, func() {}, fmt.Errorf("unable to stat dir=%q: %w", location, err) + } + + if !fileMeta.IsDir() { + return Source{}, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", location, err) + } + + s, err := NewFromDirectory(location) + if err != nil { + return Source{}, func() {}, fmt.Errorf("could not populate source from path=%q: %w", location, err) + } + return s, func() {}, nil + + case ImageScheme: + img, err := stereoscope.GetImage(location) + cleanup := func() { + stereoscope.Cleanup() + } + + if err != nil || img == nil { + return Source{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", location, err) + } + + s, err := NewFromImage(img, o) + if err != nil { + return Source{}, cleanup, fmt.Errorf("could not populate source with image: %w", err) + } + return s, cleanup, nil + } + + return Source{}, func() {}, fmt.Errorf("unable to process input for scanning: '%s'", userInput) +} + +// NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively. +func NewFromDirectory(path string) (Source, error) { + return Source{ + Resolver: &DirectoryResolver{ + Path: path, + }, + Target: DirSource{ + Path: path, + }, + Scheme: DirectoryScheme, + }, nil +} + +// NewFromImage creates a new source object tailored to catalog a given container image, relative to the +// option given (e.g. all-layers, squashed, etc) +func NewFromImage(img *image.Image, option Scope) (Source, error) { + if img == nil { + return Source{}, fmt.Errorf("no image given") + } + + resolver, err := getImageResolver(img, option) + if err != nil { + return Source{}, fmt.Errorf("could not determine file resolver: %w", err) + } + + return Source{ + Scope: option, + Resolver: resolver, + Target: ImageSource{ + Img: img, + }, + Scheme: ImageScheme, + }, nil +} diff --git a/syft/scope/scope_test.go b/syft/source/source_test.go similarity index 95% rename from syft/scope/scope_test.go rename to syft/source/source_test.go index b6656fc07..27668b1cf 100644 --- a/syft/scope/scope_test.go +++ b/syft/source/source_test.go @@ -1,4 +1,4 @@ -package scope +package source import ( "os" @@ -12,7 +12,7 @@ import ( func TestNewScopeFromImageFails(t *testing.T) { t.Run("no image given", func(t *testing.T) { - _, err := NewScopeFromImage(nil, AllLayersScope) + _, err := NewFromImage(nil, AllLayersScope) if err == nil { t.Errorf("expected an error condition but none was given") } @@ -23,7 +23,7 @@ func TestNewScopeFromImageUnknownOption(t *testing.T) { img := image.Image{} t.Run("unknown option is an error", func(t *testing.T) { - _, err := NewScopeFromImage(&img, UnknownScope) + _, err := NewFromImage(&img, UnknownScope) if err == nil { t.Errorf("expected an error condition but none was given") } @@ -36,10 +36,10 @@ func TestNewScopeFromImage(t *testing.T) { Layers: []*image.Layer{layer}, } - t.Run("create a new Scope object from image", func(t *testing.T) { - _, err := NewScopeFromImage(&img, AllLayersScope) + t.Run("create a new Source object from image", func(t *testing.T) { + _, err := NewFromImage(&img, AllLayersScope) if err != nil { - t.Errorf("unexpected error when creating a new Scope from img: %w", err) + t.Errorf("unexpected error when creating a new Source from img: %w", err) } }) } @@ -79,13 +79,13 @@ func TestDirectoryScope(t *testing.T) { } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { - p, err := NewScopeFromDir(test.input) + p, err := NewFromDirectory(test.input) if err != nil { t.Errorf("could not create NewDirScope: %w", err) } - if p.Source.(DirSource).Path != test.input { - t.Errorf("mismatched stringer: '%s' != '%s'", p.Source.(DirSource).Path, test.input) + if p.Target.(DirSource).Path != test.input { + t.Errorf("mismatched stringer: '%s' != '%s'", p.Target.(DirSource).Path, test.input) } refs, err := p.Resolver.FilesByPath(test.inputPaths...) @@ -123,7 +123,7 @@ func TestMultipleFileContentsByRefContents(t *testing.T) { } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { - p, err := NewScopeFromDir(test.input) + p, err := NewFromDirectory(test.input) if err != nil { t.Errorf("could not create NewDirScope: %w", err) } @@ -163,7 +163,7 @@ func TestMultipleFileContentsByRefNoContents(t *testing.T) { } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { - p, err := NewScopeFromDir(test.input) + p, err := NewFromDirectory(test.input) if err != nil { t.Errorf("could not create NewDirScope: %w", err) } @@ -208,7 +208,7 @@ func TestFilesByGlob(t *testing.T) { } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { - p, err := NewScopeFromDir(test.input) + p, err := NewFromDirectory(test.input) if err != nil { t.Errorf("could not create NewDirScope: %w", err) } diff --git a/syft/scope/resolvers/test-fixtures/image-symlinks/Dockerfile b/syft/source/test-fixtures/image-symlinks/Dockerfile similarity index 100% rename from syft/scope/resolvers/test-fixtures/image-symlinks/Dockerfile rename to syft/source/test-fixtures/image-symlinks/Dockerfile diff --git a/syft/scope/resolvers/test-fixtures/image-symlinks/file-1.txt b/syft/source/test-fixtures/image-symlinks/file-1.txt similarity index 100% rename from syft/scope/resolvers/test-fixtures/image-symlinks/file-1.txt rename to syft/source/test-fixtures/image-symlinks/file-1.txt diff --git a/syft/scope/resolvers/test-fixtures/image-symlinks/file-2.txt b/syft/source/test-fixtures/image-symlinks/file-2.txt similarity index 100% rename from syft/scope/resolvers/test-fixtures/image-symlinks/file-2.txt rename to syft/source/test-fixtures/image-symlinks/file-2.txt diff --git a/syft/scope/resolvers/test-fixtures/image-symlinks/nested/nested/file-3.txt b/syft/source/test-fixtures/image-symlinks/nested/nested/file-3.txt similarity index 100% rename from syft/scope/resolvers/test-fixtures/image-symlinks/nested/nested/file-3.txt rename to syft/source/test-fixtures/image-symlinks/nested/nested/file-3.txt diff --git a/syft/scope/resolvers/test-fixtures/image-symlinks/new-file-2.txt b/syft/source/test-fixtures/image-symlinks/new-file-2.txt similarity index 100% rename from syft/scope/resolvers/test-fixtures/image-symlinks/new-file-2.txt rename to syft/source/test-fixtures/image-symlinks/new-file-2.txt diff --git a/syft/scope/test-fixtures/path-detected/.vimrc b/syft/source/test-fixtures/path-detected/.vimrc similarity index 100% rename from syft/scope/test-fixtures/path-detected/.vimrc rename to syft/source/test-fixtures/path-detected/.vimrc diff --git a/syft/scope/test-fixtures/path-detected/empty b/syft/source/test-fixtures/path-detected/empty similarity index 100% rename from syft/scope/test-fixtures/path-detected/empty rename to syft/source/test-fixtures/path-detected/empty diff --git a/test/integration/distro_test.go b/test/integration/distro_test.go index fdbb5a824..a0ad3fd64 100644 --- a/test/integration/distro_test.go +++ b/test/integration/distro_test.go @@ -6,7 +6,7 @@ import ( "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft" "github.com/anchore/syft/syft/distro" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/go-test/deep" ) @@ -16,7 +16,7 @@ func TestDistroImage(t *testing.T) { tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) defer cleanup() - _, _, actualDistro, err := syft.Catalog("docker-archive:"+tarPath, scope.AllLayersScope) + _, _, actualDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } diff --git a/test/integration/document_import_test.go b/test/integration/document_import_test.go index f42492a21..946421f18 100644 --- a/test/integration/document_import_test.go +++ b/test/integration/document_import_test.go @@ -8,7 +8,7 @@ import ( "github.com/anchore/syft/syft" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/presenter/json" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/go-test/deep" ) @@ -31,7 +31,7 @@ func TestCatalogFromJSON(t *testing.T) { tarPath := imagetest.GetFixtureImageTarPath(t, test.fixture) defer cleanup() - expectedCatalog, s, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, scope.AllLayersScope) + expectedCatalog, s, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } diff --git a/test/integration/json_schema_test.go b/test/integration/json_schema_test.go index 279b74337..15970ee0e 100644 --- a/test/integration/json_schema_test.go +++ b/test/integration/json_schema_test.go @@ -15,7 +15,7 @@ import ( "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/presenter" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" "github.com/xeipuuv/gojsonschema" ) @@ -53,7 +53,7 @@ func validateAgainstV1Schema(t *testing.T, json string) { } } -func testJsonSchema(t *testing.T, catalog *pkg.Catalog, theScope *scope.Scope, prefix string) { +func testJsonSchema(t *testing.T, catalog *pkg.Catalog, theScope *source.Source, prefix string) { // make the json output example dir if it does not exist absJsonSchemaExamplesPath := path.Join(repoRoot(t), jsonSchemaExamplesPath) if _, err := os.Stat(absJsonSchemaExamplesPath); os.IsNotExist(err) { @@ -101,7 +101,7 @@ func TestJsonSchemaImg(t *testing.T) { tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) defer cleanup() - catalog, theScope, _, err := syft.Catalog("docker-archive:"+tarPath, scope.AllLayersScope) + catalog, theScope, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } @@ -118,9 +118,9 @@ func TestJsonSchemaImg(t *testing.T) { } func TestJsonSchemaDirs(t *testing.T) { - catalog, theScope, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", scope.AllLayersScope) + catalog, theScope, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope) if err != nil { - t.Errorf("unable to create scope from dir: %+v", err) + t.Errorf("unable to create source from dir: %+v", err) } var cases []testCase diff --git a/test/integration/pkg_coverage_test.go b/test/integration/pkg_coverage_test.go index 13a5af513..78e821829 100644 --- a/test/integration/pkg_coverage_test.go +++ b/test/integration/pkg_coverage_test.go @@ -9,7 +9,7 @@ import ( "github.com/anchore/syft/internal" "github.com/anchore/syft/syft" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) func TestPkgCoverageImage(t *testing.T) { @@ -18,7 +18,7 @@ func TestPkgCoverageImage(t *testing.T) { tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) defer cleanup() - catalog, _, _, err := syft.Catalog("docker-archive:"+tarPath, scope.AllLayersScope) + catalog, _, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } @@ -100,10 +100,10 @@ func TestPkgCoverageImage(t *testing.T) { } func TestPkgCoverageDirectory(t *testing.T) { - catalog, _, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", scope.AllLayersScope) + catalog, _, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope) if err != nil { - t.Errorf("unable to create scope from dir: %+v", err) + t.Errorf("unable to create source from dir: %+v", err) } observedLanguages := internal.NewStringSet() diff --git a/test/integration/regression_test.go b/test/integration/regression_test.go index 572ddae8d..d018e85b0 100644 --- a/test/integration/regression_test.go +++ b/test/integration/regression_test.go @@ -9,7 +9,7 @@ import ( "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft" - "github.com/anchore/syft/syft/scope" + "github.com/anchore/syft/syft/source" ) func TestRegression212ApkBufferSize(t *testing.T) { @@ -21,7 +21,7 @@ func TestRegression212ApkBufferSize(t *testing.T) { tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) defer cleanup() - catalog, _, _, err := syft.Catalog("docker-archive:"+tarPath, scope.SquashedScope) + catalog, _, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } From b694dacb21299fbd9f78b75a5eefe78b6681efdc Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 13 Nov 2020 12:34:22 -0500 Subject: [PATCH 03/13] add source.Location + reorient Resolvers to use it Signed-off-by: Alex Goodman --- syft/source/all_layers_resolver.go | 71 ++++++++++++++++++++------- syft/source/directory_resolver.go | 72 ++++++++++++++-------------- syft/source/image_squash_resolver.go | 46 +++++++++--------- syft/source/location.go | 46 ++++++++++++++++++ syft/source/resolver.go | 17 ++++--- 5 files changed, 168 insertions(+), 84 deletions(-) create mode 100644 syft/source/location.go diff --git a/syft/source/all_layers_resolver.go b/syft/source/all_layers_resolver.go index a050ad616..1c2be3749 100644 --- a/syft/source/all_layers_resolver.go +++ b/syft/source/all_layers_resolver.go @@ -8,6 +8,8 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) +var _ Resolver = &AllLayersResolver{} + // AllLayersResolver implements path and content access for the AllLayers source option for container image data sources. type AllLayersResolver struct { img *image.Image @@ -61,14 +63,14 @@ func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.Ref } // FilesByPath returns all file.References that match the given paths from any layer in the image. -func (r *AllLayersResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) { +func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) { uniqueFileIDs := file.NewFileReferenceSet() - uniqueFiles := make([]file.Reference, 0) + uniqueLocations := make([]Location, 0) for _, path := range paths { for idx, layerIdx := range r.layers { tree := r.img.Layers[layerIdx].Tree - ref := tree.File(path) + ref := tree.File(file.Path(path)) if ref == nil { // no file found, keep looking through layers continue @@ -91,17 +93,18 @@ func (r *AllLayersResolver) FilesByPath(paths ...file.Path) ([]file.Reference, e if err != nil { return nil, err } - uniqueFiles = append(uniqueFiles, results...) + for _, result := range results { + uniqueLocations = append(uniqueLocations, newLocationFromImage(result, r.img)) + } } } - - return uniqueFiles, nil + return uniqueLocations, nil } // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. -func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) { +func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error) { uniqueFileIDs := file.NewFileReferenceSet() - uniqueFiles := make([]file.Reference, 0) + uniqueLocations := make([]Location, 0) for _, pattern := range patterns { for idx, layerIdx := range r.layers { @@ -128,31 +131,63 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, e if err != nil { return nil, err } - uniqueFiles = append(uniqueFiles, results...) + for _, result := range results { + uniqueLocations = append(uniqueLocations, newLocationFromImage(result, r.img)) + } } } } - return uniqueFiles, nil + return uniqueLocations, nil } -func (r *AllLayersResolver) RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) { - entry, err := r.img.FileCatalog.Get(reference) +func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) *Location { + entry, err := r.img.FileCatalog.Get(location.ref) if err != nil { - return nil, err + return nil } - return entry.Source.SquashedTree.File(file.Path(path)), nil + relativeRef := entry.Source.SquashedTree.File(file.Path(path)) + if relativeRef == nil { + return nil + } + + relativeLocation := newLocationFromImage(*relativeRef, r.img) + + return &relativeLocation } // MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a // file.Reference is a path relative to a particular layer. -func (r *AllLayersResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) { - return r.img.MultipleFileContentsByRef(f...) +func (r *AllLayersResolver) MultipleFileContentsByRef(locations []Location) (map[Location]string, error) { + return mapLocationRefs(r.img.MultipleFileContentsByRef, locations) } // FileContentsByRef fetches file contents for a single file reference, irregardless of the source layer. // If the path does not exist an error is returned. -func (r *AllLayersResolver) FileContentsByRef(ref file.Reference) (string, error) { - return r.img.FileContentsByRef(ref) +func (r *AllLayersResolver) FileContentsByRef(location Location) (string, error) { + return r.img.FileContentsByRef(location.ref) +} + +type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]string, error) + +func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]string, error) { + var fileRefs = make([]file.Reference, len(locations)) + var locationByRefs = make(map[file.Reference]Location) + var results = make(map[Location]string) + + for i, location := range locations { + locationByRefs[location.ref] = location + fileRefs[i] = location.ref + } + + contentsByRef, err := callback(fileRefs...) + if err != nil { + return nil, err + } + + for ref, content := range contentsByRef { + results[locationByRefs[ref]] = content + } + return results, nil } diff --git a/syft/source/directory_resolver.go b/syft/source/directory_resolver.go index 89b717db0..9cbacf6bf 100644 --- a/syft/source/directory_resolver.go +++ b/syft/source/directory_resolver.go @@ -7,11 +7,14 @@ import ( "path" "path/filepath" - "github.com/anchore/stereoscope/pkg/file" + "github.com/docker/distribution/reference" + "github.com/anchore/syft/internal/log" "github.com/bmatcuk/doublestar" ) +var _ Resolver = &DirectoryResolver{} + // DirectoryResolver implements path and content access for the directory data source. type DirectoryResolver struct { Path string @@ -23,11 +26,11 @@ func (s DirectoryResolver) String() string { } // FilesByPath returns all file.References that match the given paths from the directory. -func (s DirectoryResolver) FilesByPath(userPaths ...file.Path) ([]file.Reference, error) { - var references = make([]file.Reference, 0) +func (s DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error) { + var references = make([]Location, 0) for _, userPath := range userPaths { - userStrPath := string(userPath) + userStrPath := userPath if filepath.IsAbs(userStrPath) { // a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is @@ -45,33 +48,24 @@ func (s DirectoryResolver) FilesByPath(userPaths ...file.Path) ([]file.Reference continue } - references = append(references, file.NewFileReference(file.Path(userStrPath))) + references = append(references, newLocation(userStrPath)) } return references, nil } -func fileContents(path file.Path) ([]byte, error) { - contents, err := ioutil.ReadFile(string(path)) - - if err != nil { - return nil, err - } - return contents, nil -} - // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. -func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) { - result := make([]file.Reference, 0) +func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) { + result := make([]Location, 0) for _, pattern := range patterns { pathPattern := path.Join(s.Path, pattern) - matches, err := doublestar.Glob(pathPattern) + pathMatches, err := doublestar.Glob(pathPattern) if err != nil { - return result, err + return nil, err } - for _, match := range matches { - fileMeta, err := os.Stat(match) + for _, matchedPath := range pathMatches { + fileMeta, err := os.Stat(matchedPath) if err != nil { continue } @@ -81,47 +75,55 @@ func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]file.Reference, er continue } - matchedPath := file.Path(match) - result = append(result, file.NewFileReference(matchedPath)) + result = append(result, newLocation(matchedPath)) } } return result, nil } -func (s *DirectoryResolver) RelativeFileByPath(_ file.Reference, path string) (*file.Reference, error) { - paths, err := s.FilesByPath(file.Path(path)) +func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Location { + paths, err := s.FilesByPath(path) if err != nil { - return nil, err + return nil } if len(paths) == 0 { - return nil, nil + return nil } - return &paths[0], nil + return &paths[0] } // MultipleFileContentsByRef returns the file contents for all file.References relative a directory. -func (s DirectoryResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) { - refContents := make(map[file.Reference]string) - for _, fileRef := range f { - contents, err := fileContents(fileRef.Path) +func (s DirectoryResolver) MultipleFileContentsByRef(locations []Location) (map[Location]string, error) { + refContents := make(map[Location]string) + for _, location := range locations { + contents, err := fileContents(location.Path) if err != nil { - return nil, fmt.Errorf("could not read contents of file: %s", fileRef.Path) + return nil, fmt.Errorf("could not read contents of file: %s", location.Path) } - refContents[fileRef] = string(contents) + refContents[location] = string(contents) } return refContents, nil } // FileContentsByRef fetches file contents for a single file reference relative to a directory. // If the path does not exist an error is returned. -func (s DirectoryResolver) FileContentsByRef(reference file.Reference) (string, error) { - contents, err := fileContents(reference.Path) +func (s DirectoryResolver) FileContentsByRef(location Location) (string, error) { + contents, err := fileContents(location.Path) if err != nil { return "", fmt.Errorf("could not read contents of file: %s", reference.Path) } return string(contents), nil } + +func fileContents(path string) ([]byte, error) { + contents, err := ioutil.ReadFile(path) + + if err != nil { + return nil, err + } + return contents, nil +} diff --git a/syft/source/image_squash_resolver.go b/syft/source/image_squash_resolver.go index 07a9a54d9..ea4f3ffd6 100644 --- a/syft/source/image_squash_resolver.go +++ b/syft/source/image_squash_resolver.go @@ -7,6 +7,8 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) +var _ Resolver = &ImageSquashResolver{} + // ImageSquashResolver implements path and content access for the Squashed source option for container image data sources. type ImageSquashResolver struct { img *image.Image @@ -21,13 +23,13 @@ func NewImageSquashResolver(img *image.Image) (*ImageSquashResolver, error) { } // FilesByPath returns all file.References that match the given paths within the squashed representation of the image. -func (r *ImageSquashResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) { +func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) { uniqueFileIDs := file.NewFileReferenceSet() - uniqueFiles := make([]file.Reference, 0) + uniqueLocations := make([]Location, 0) for _, path := range paths { tree := r.img.SquashedTree() - ref := tree.File(path) + ref := tree.File(file.Path(path)) if ref == nil { // no file found, keep looking through layers continue @@ -54,17 +56,17 @@ func (r *ImageSquashResolver) FilesByPath(paths ...file.Path) ([]file.Reference, if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) { uniqueFileIDs.Add(*resolvedRef) - uniqueFiles = append(uniqueFiles, *resolvedRef) + uniqueLocations = append(uniqueLocations, newLocationFromImage(*resolvedRef, r.img)) } } - return uniqueFiles, nil + return uniqueLocations, nil } // FilesByGlob returns all file.References that match the given path glob pattern within the squashed representation of the image. -func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) { +func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error) { uniqueFileIDs := file.NewFileReferenceSet() - uniqueFiles := make([]file.Reference, 0) + uniqueLocations := make([]Location, 0) for _, pattern := range patterns { refs, err := r.img.SquashedTree().FilesByGlob(pattern) @@ -86,42 +88,42 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference, } } - resolvedRefs, err := r.FilesByPath(ref.Path) + resolvedLocations, err := r.FilesByPath(string(ref.Path)) if err != nil { return nil, fmt.Errorf("failed to find files by path (ref=%+v): %w", ref, err) } - for _, resolvedRef := range resolvedRefs { - if !uniqueFileIDs.Contains(resolvedRef) { - uniqueFileIDs.Add(resolvedRef) - uniqueFiles = append(uniqueFiles, resolvedRef) + for _, resolvedLocation := range resolvedLocations { + if !uniqueFileIDs.Contains(resolvedLocation.ref) { + uniqueFileIDs.Add(resolvedLocation.ref) + uniqueLocations = append(uniqueLocations, resolvedLocation) } } } } - return uniqueFiles, nil + return uniqueLocations, nil } -func (r *ImageSquashResolver) RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) { - paths, err := r.FilesByPath(file.Path(path)) +func (r *ImageSquashResolver) RelativeFileByPath(_ Location, path string) *Location { + paths, err := r.FilesByPath(path) if err != nil { - return nil, err + return nil } if len(paths) == 0 { - return nil, nil + return nil } - return &paths[0], nil + return &paths[0] } // MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a // file.Reference is a path relative to a particular layer, in this case only from the squashed representation. -func (r *ImageSquashResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) { - return r.img.MultipleFileContentsByRef(f...) +func (r *ImageSquashResolver) MultipleFileContentsByRef(locations []Location) (map[Location]string, error) { + return mapLocationRefs(r.img.MultipleFileContentsByRef, locations) } // FileContentsByRef fetches file contents for a single file reference, irregardless of the source layer. // If the path does not exist an error is returned. -func (r *ImageSquashResolver) FileContentsByRef(ref file.Reference) (string, error) { - return r.img.FileContentsByRef(ref) +func (r *ImageSquashResolver) FileContentsByRef(location Location) (string, error) { + return r.img.FileContentsByRef(location.ref) } diff --git a/syft/source/location.go b/syft/source/location.go new file mode 100644 index 000000000..1bc5e5ce0 --- /dev/null +++ b/syft/source/location.go @@ -0,0 +1,46 @@ +package source + +import ( + "github.com/anchore/syft/internal/log" + + "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/stereoscope/pkg/image" +) + +type Location struct { + Path string `json:"path"` + LayerIndex uint `json:"layerIndex"` + LayerID string `json:"layerID"` + ref file.Reference +} + +func newLocation(path string) Location { + return Location{ + Path: path, + } +} + +func newLocationFromRef(ref file.Reference) Location { + return Location{ + Path: string(ref.Path), + ref: ref, + } +} + +func newLocationFromImage(ref file.Reference, img *image.Image) Location { + entry, err := img.FileCatalog.Get(ref) + if err != nil { + log.Warnf("unable to find file catalog entry for ref=%+v", ref) + return Location{ + Path: string(ref.Path), + ref: ref, + } + } + + return Location{ + Path: string(ref.Path), + LayerIndex: entry.Source.Metadata.Index, + LayerID: entry.Source.Metadata.Digest, + ref: ref, + } +} diff --git a/syft/source/resolver.go b/syft/source/resolver.go index 6af9feaa2..2bab3bc11 100644 --- a/syft/source/resolver.go +++ b/syft/source/resolver.go @@ -3,7 +3,6 @@ package source import ( "fmt" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/image" ) @@ -15,30 +14,30 @@ type Resolver interface { // ContentResolver knows how to get file content for given file.References type ContentResolver interface { - FileContentsByRef(ref file.Reference) (string, error) - MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) + FileContentsByRef(Location) (string, error) + MultipleFileContentsByRef([]Location) (map[Location]string, error) // TODO: we should consider refactoring to return a set of io.Readers or file.Openers instead of the full contents themselves (allow for optional buffering). } // FileResolver knows how to get file.References for given string paths and globs type FileResolver interface { // FilesByPath fetches a set of file references which have the given path (for an image, there may be multiple matches) - FilesByPath(paths ...file.Path) ([]file.Reference, error) + FilesByPath(paths ...string) ([]Location, error) // FilesByGlob fetches a set of file references which the given glob matches - FilesByGlob(patterns ...string) ([]file.Reference, error) + FilesByGlob(patterns ...string) ([]Location, error) // RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. // This is helpful when attempting to find a file that is in the same layer or lower as another file. - RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) + RelativeFileByPath(_ Location, path string) *Location } // getImageResolver returns the appropriate resolve for a container image given the source option -func getImageResolver(img *image.Image, option Scope) (Resolver, error) { - switch option { +func getImageResolver(img *image.Image, scope Scope) (Resolver, error) { + switch scope { case SquashedScope: return NewImageSquashResolver(img) case AllLayersScope: return NewAllLayersResolver(img) default: - return nil, fmt.Errorf("bad option provided: %+v", option) + return nil, fmt.Errorf("bad scope provided: %+v", scope) } } From aa0d444fd49e19809b92e93fca83cedb2e1473ea Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 13 Nov 2020 13:17:12 -0500 Subject: [PATCH 04/13] fix tests to use location instead of file.Reference Signed-off-by: Alex Goodman --- go.sum | 8 -- internal/config/config.go | 2 +- internal/file/glob_match.go | 2 +- schema/json/schema.json | 72 ++++++++++++--- syft/cataloger/common/generic_cataloger.go | 35 ++++---- .../common/generic_cataloger_test.go | 36 ++++---- syft/cataloger/deb/cataloger.go | 89 ++++++++----------- syft/cataloger/deb/cataloger_test.go | 6 +- syft/cataloger/python/package_cataloger.go | 53 +++++------ .../python/package_cataloger_test.go | 44 ++++----- .../python/parse_wheel_egg_metadata.go | 6 +- .../python/parse_wheel_egg_metadata_test.go | 4 +- syft/cataloger/rpmdb/cataloger.go | 8 +- syft/cataloger/rpmdb/parse_rpmdb.go | 12 +-- syft/cataloger/rpmdb/parse_rpmdb_test.go | 18 ++-- syft/distro/identify.go | 19 ++-- syft/lib.go | 8 +- syft/pkg/catalog.go | 13 +-- syft/pkg/dpkg_metadata.go | 2 +- syft/pkg/package.go | 14 +-- syft/presenter/cyclonedx/bom-extension.go | 2 +- syft/presenter/cyclonedx/document.go | 2 +- syft/presenter/cyclonedx/presenter.go | 4 +- syft/presenter/cyclonedx/presenter_test.go | 13 ++- syft/presenter/json/artifact.go | 20 ++--- syft/presenter/json/location.go | 45 ---------- syft/presenter/json/presenter_test.go | 13 ++- syft/presenter/table/presenter_test.go | 9 +- syft/presenter/text/presenter_test.go | 9 +- syft/source/all_layers_resolver.go | 10 +-- syft/source/all_layers_resolver_test.go | 12 ++- syft/source/directory_resolver.go | 12 ++- syft/source/directory_resolver_test.go | 32 ++++--- syft/source/image_squash_resolver.go | 6 +- syft/source/image_squash_resolver_test.go | 12 ++- syft/source/location.go | 11 +-- syft/source/resolver.go | 4 +- syft/source/scope.go | 2 +- syft/source/source_test.go | 38 ++++---- test/integration/document_import_test.go | 2 +- 40 files changed, 327 insertions(+), 382 deletions(-) delete mode 100644 syft/presenter/json/location.go diff --git a/go.sum b/go.sum index 55021990b..e6ba371c3 100644 --- a/go.sum +++ b/go.sum @@ -124,16 +124,12 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e h1:kty6r0R2JeaNPeWKSYDC+HW3hkqwFh4PP5TQ8pUPYFw= -github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e/go.mod h1:iYuIG0Nai8dR0ri3LhZQKUyO1loxUWAGvoWhXDmjy1A= github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 h1:xbeIbn5F52JVx3RUIajxCj8b0y+9lywspql4sFhcxWQ= github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12/go.mod h1:juoyWXIj7sJ1IDl4E/KIfyLtovbs5XQVSIdaQifFQT8= github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0vW0nnNKJfJieyH/TZ9UYAnTZs5/gHTdAe8= github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E= -github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe h1:m4NSyTo2fVUoUHAV/ZVqE/PFMr/y8oz9HRrhWLk9It0= -github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E= github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409 h1:xKSpDRjmYrEFrdMeDh4AuSUAFc99pdro6YFBKxy2um0= github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= @@ -164,8 +160,6 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4= github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/bmatcuk/doublestar v1.3.1 h1:rT8rxDPsavp9G+4ZULzqhhUSaI/OPsTZNG88Z3i0xvY= -github.com/bmatcuk/doublestar v1.3.1/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/bmatcuk/doublestar v1.3.3 h1:pVP1d49CcQQaNOl+PI6sPybIrIOD/6sux31PFdmhTH0= github.com/bmatcuk/doublestar v1.3.3/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U= @@ -301,8 +295,6 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-test/deep v1.0.6 h1:UHSEyLZUwX9Qoi99vVwvewiMC8mM2bf7XEM2nqvzEn8= -github.com/go-test/deep v1.0.6/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g= diff --git a/internal/config/config.go b/internal/config/config.go index ded48e198..91a4fa770 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -80,7 +80,7 @@ func (cfg *Application) Build() error { cfg.PresenterOpt = presenterOption // set the source - scopeOption := source.ParseOption(cfg.Scope) + scopeOption := source.ParseScope(cfg.Scope) if scopeOption == source.UnknownScope { return fmt.Errorf("bad --scope value '%s'", cfg.Scope) } diff --git a/internal/file/glob_match.go b/internal/file/glob_match.go index 81575de24..07ca126b7 100644 --- a/internal/file/glob_match.go +++ b/internal/file/glob_match.go @@ -1,6 +1,6 @@ package file -// Source: https://research.swtch.com/glob.go +// Locations: https://research.swtch.com/glob.go func GlobMatch(pattern, name string) bool { px := 0 nx := 0 diff --git a/schema/json/schema.json b/schema/json/schema.json index 954c58d92..556fd50e7 100644 --- a/schema/json/schema.json +++ b/schema/json/schema.json @@ -10,6 +10,9 @@ }, "type": "array" }, + "language": { + "type": "string" + }, "licenses": { "anyOf": [ { @@ -24,13 +27,16 @@ ] }, "locations": { - "items": { - "anyOf": [ - { - "type": "string" - }, - { + "anyOf": [ + { + "type": "null" + }, + { + "items": { "properties": { + "layerID": { + "type": "string" + }, "layerIndex": { "type": "integer" }, @@ -39,14 +45,15 @@ } }, "required": [ + "layerID", "layerIndex", "path" ], "type": "object" - } - ] - }, - "type": "array" + }, + "type": "array" + } + ] }, "metadata": { "properties": { @@ -315,6 +322,9 @@ }, "type": "object" }, + "metadataType": { + "type": "string" + }, "name": { "type": "string" }, @@ -327,8 +337,10 @@ }, "required": [ "foundBy", + "language", "licenses", "locations", + "metadataType", "name", "type", "version" @@ -337,6 +349,44 @@ }, "type": "array" }, + "descriptor": { + "properties": { + "name": { + "type": "string" + }, + "reportTimestamp": { + "type": "string" + }, + "version": { + "type": "string" + } + }, + "required": [ + "name", + "reportTimestamp", + "version" + ], + "type": "object" + }, + "distro": { + "properties": { + "idLike": { + "type": "string" + }, + "name": { + "type": "string" + }, + "version": { + "type": "string" + } + }, + "required": [ + "idLike", + "name", + "version" + ], + "type": "object" + }, "source": { "properties": { "target": { @@ -408,6 +458,8 @@ }, "required": [ "artifacts", + "descriptor", + "distro", "source" ], "type": "object" diff --git a/syft/cataloger/common/generic_cataloger.go b/syft/cataloger/common/generic_cataloger.go index bc17d65ec..a35544af9 100644 --- a/syft/cataloger/common/generic_cataloger.go +++ b/syft/cataloger/common/generic_cataloger.go @@ -6,7 +6,6 @@ package common import ( "strings" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" @@ -17,8 +16,8 @@ import ( type GenericCataloger struct { globParsers map[string]ParserFn pathParsers map[string]ParserFn - selectedFiles []file.Reference - parsers map[file.Reference]ParserFn + selectedFiles []source.Location + parsers map[source.Location]ParserFn upstreamCataloger string } @@ -27,8 +26,8 @@ func NewGenericCataloger(pathParsers map[string]ParserFn, globParsers map[string return &GenericCataloger{ globParsers: globParsers, pathParsers: pathParsers, - selectedFiles: make([]file.Reference, 0), - parsers: make(map[file.Reference]ParserFn), + selectedFiles: make([]source.Location, 0), + parsers: make(map[source.Location]ParserFn), upstreamCataloger: upstreamCataloger, } } @@ -39,7 +38,7 @@ func (c *GenericCataloger) Name() string { } // register pairs a set of file references with a parser function for future cataloging (when the file contents are resolved) -func (c *GenericCataloger) register(files []file.Reference, parser ParserFn) { +func (c *GenericCataloger) register(files []source.Location, parser ParserFn) { c.selectedFiles = append(c.selectedFiles, files...) for _, f := range files { c.parsers[f] = parser @@ -48,14 +47,14 @@ func (c *GenericCataloger) register(files []file.Reference, parser ParserFn) { // clear deletes all registered file-reference-to-parser-function pairings from former SelectFiles() and register() calls func (c *GenericCataloger) clear() { - c.selectedFiles = make([]file.Reference, 0) - c.parsers = make(map[file.Reference]ParserFn) + c.selectedFiles = make([]source.Location, 0) + c.parsers = make(map[source.Location]ParserFn) } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source. func (c *GenericCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { fileSelection := c.selectFiles(resolver) - contents, err := resolver.MultipleFileContentsByRef(fileSelection...) + contents, err := resolver.MultipleFileContentsByLocation(fileSelection) if err != nil { return nil, err } @@ -63,10 +62,10 @@ func (c *GenericCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, err } // SelectFiles takes a set of file trees and resolves and file references of interest for future cataloging -func (c *GenericCataloger) selectFiles(resolver source.FileResolver) []file.Reference { +func (c *GenericCataloger) selectFiles(resolver source.FileResolver) []source.Location { // select by exact path for path, parser := range c.pathParsers { - files, err := resolver.FilesByPath(file.Path(path)) + files, err := resolver.FilesByPath(path) if err != nil { log.Warnf("cataloger failed to select files by path: %+v", err) } @@ -90,28 +89,28 @@ func (c *GenericCataloger) selectFiles(resolver source.FileResolver) []file.Refe } // catalog takes a set of file contents and uses any configured parser functions to resolve and return discovered packages -func (c *GenericCataloger) catalog(contents map[file.Reference]string) ([]pkg.Package, error) { +func (c *GenericCataloger) catalog(contents map[source.Location]string) ([]pkg.Package, error) { defer c.clear() packages := make([]pkg.Package, 0) - for reference, parser := range c.parsers { - content, ok := contents[reference] + for location, parser := range c.parsers { + content, ok := contents[location] if !ok { - log.Warnf("cataloger '%s' missing file content: %+v", c.upstreamCataloger, reference) + log.Warnf("cataloger '%s' missing file content: %+v", c.upstreamCataloger, location) continue } - entries, err := parser(string(reference.Path), strings.NewReader(content)) + entries, err := parser(location.Path, strings.NewReader(content)) if err != nil { // TODO: should we fail? or only log? - log.Warnf("cataloger '%s' failed to parse entries (reference=%+v): %+v", c.upstreamCataloger, reference, err) + log.Warnf("cataloger '%s' failed to parse entries (location=%+v): %+v", c.upstreamCataloger, location, err) continue } for _, entry := range entries { entry.FoundBy = c.upstreamCataloger - entry.Source = []file.Reference{reference} + entry.Locations = []source.Location{location} packages = append(packages, entry) } diff --git a/syft/cataloger/common/generic_cataloger_test.go b/syft/cataloger/common/generic_cataloger_test.go index 6083d390a..e82f507a8 100644 --- a/syft/cataloger/common/generic_cataloger_test.go +++ b/syft/cataloger/common/generic_cataloger_test.go @@ -6,48 +6,50 @@ import ( "io/ioutil" "testing" - "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/syft/pkg" ) type testResolverMock struct { - contents map[file.Reference]string + contents map[source.Location]string } func newTestResolver() *testResolverMock { return &testResolverMock{ - contents: make(map[file.Reference]string), + contents: make(map[source.Location]string), } } -func (r *testResolverMock) FileContentsByRef(_ file.Reference) (string, error) { +func (r *testResolverMock) FileContentsByLocation(_ source.Location) (string, error) { return "", fmt.Errorf("not implemented") } -func (r *testResolverMock) MultipleFileContentsByRef(_ ...file.Reference) (map[file.Reference]string, error) { +func (r *testResolverMock) MultipleFileContentsByLocation([]source.Location) (map[source.Location]string, error) { return r.contents, nil } -func (r *testResolverMock) FilesByPath(paths ...file.Path) ([]file.Reference, error) { - results := make([]file.Reference, len(paths)) +func (r *testResolverMock) FilesByPath(paths ...string) ([]source.Location, error) { + results := make([]source.Location, len(paths)) for idx, p := range paths { - results[idx] = file.NewFileReference(p) + results[idx] = source.NewLocation(p) r.contents[results[idx]] = fmt.Sprintf("%s file contents!", p) } return results, nil } -func (r *testResolverMock) FilesByGlob(_ ...string) ([]file.Reference, error) { +func (r *testResolverMock) FilesByGlob(_ ...string) ([]source.Location, error) { path := "/a-path.txt" - ref := file.NewFileReference(file.Path(path)) - r.contents[ref] = fmt.Sprintf("%s file contents!", path) - return []file.Reference{ref}, nil + location := source.NewLocation(path) + r.contents[location] = fmt.Sprintf("%s file contents!", path) + return []source.Location{location}, nil } -func (r *testResolverMock) RelativeFileByPath(_ file.Reference, _ string) (*file.Reference, error) { - return nil, fmt.Errorf("not implemented") +func (r *testResolverMock) RelativeFileByPath(_ source.Location, _ string) *source.Location { + panic(fmt.Errorf("not implemented")) + return nil } func parser(_ string, reader io.Reader) ([]pkg.Package, error) { @@ -94,8 +96,8 @@ func TestGenericCataloger(t *testing.T) { } for _, p := range actualPkgs { - ref := p.Source[0] - exP, ok := expectedPkgs[string(ref.Path)] + ref := p.Locations[0] + exP, ok := expectedPkgs[ref.Path] if !ok { t.Errorf("missing expected pkg: ref=%+v", ref) continue @@ -106,7 +108,7 @@ func TestGenericCataloger(t *testing.T) { } if exP.Name != p.Name { - t.Errorf("bad contents mapping: %+v", p.Source) + t.Errorf("bad contents mapping: %+v", p.Locations) } } } diff --git a/syft/cataloger/deb/cataloger.go b/syft/cataloger/deb/cataloger.go index e9fdecf54..cbd31c03b 100644 --- a/syft/cataloger/deb/cataloger.go +++ b/syft/cataloger/deb/cataloger.go @@ -7,9 +7,9 @@ import ( "fmt" "io" "path" + "path/filepath" "strings" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" ) @@ -40,23 +40,23 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { } var pkgs []pkg.Package - for _, dbRef := range dbFileMatches { - dbContents, err := resolver.FileContentsByRef(dbRef) + for _, dbLocation := range dbFileMatches { + dbContents, err := resolver.FileContentsByLocation(dbLocation) if err != nil { return nil, err } pkgs, err = parseDpkgStatus(strings.NewReader(dbContents)) if err != nil { - return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbRef.Path, err) + return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.Path, err) } - md5ContentsByName, md5RefsByName, err := fetchMd5Contents(resolver, dbRef, pkgs) + md5ContentsByName, md5RefsByName, err := fetchMd5Contents(resolver, dbLocation, pkgs) if err != nil { return nil, fmt.Errorf("unable to find dpkg md5 contents: %w", err) } - copyrightContentsByName, copyrightRefsByName, err := fetchCopyrightContents(resolver, dbRef, pkgs) + copyrightContentsByName, copyrightRefsByName, err := fetchCopyrightContents(resolver, dbLocation, pkgs) if err != nil { return nil, fmt.Errorf("unable to find dpkg copyright contents: %w", err) } @@ -64,7 +64,7 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { for i := range pkgs { p := &pkgs[i] p.FoundBy = c.Name() - p.Source = []file.Reference{dbRef} + p.Locations = []source.Location{dbLocation} if md5Reader, ok := md5ContentsByName[md5Key(*p)]; ok { // attach the file list @@ -74,7 +74,7 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { // keep a record of the file where this was discovered if ref, ok := md5RefsByName[md5Key(*p)]; ok { - p.Source = append(p.Source, ref) + p.Locations = append(p.Locations, ref) } } @@ -85,7 +85,7 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { // keep a record of the file where this was discovered if ref, ok := copyrightRefsByName[p.Name]; ok { - p.Source = append(p.Source, ref) + p.Locations = append(p.Locations, ref) } } } @@ -93,93 +93,82 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { return pkgs, nil } -func fetchMd5Contents(resolver source.Resolver, dbRef file.Reference, pkgs []pkg.Package) (map[string]io.Reader, map[string]file.Reference, error) { +func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) { // fetch all MD5 file contents. This approach is more efficient than fetching each MD5 file one at a time - var md5FileMatches []file.Reference - var nameByRef = make(map[file.Reference]string) - parentPath, err := dbRef.Path.ParentPath() - if err != nil { - return nil, nil, fmt.Errorf("unable to find parent of path=%+v: %w", dbRef.Path, err) - } + var md5FileMatches []source.Location + var nameByRef = make(map[source.Location]string) + parentPath := filepath.Dir(dbLocation.Path) + for _, p := range pkgs { // look for /var/lib/dpkg/info/NAME:ARCH.md5sums name := md5Key(p) - md5sumPath := path.Join(string(parentPath), "info", name+md5sumsExt) - md5SumRef, err := resolver.RelativeFileByPath(dbRef, md5sumPath) - if err != nil { - return nil, nil, fmt.Errorf("unable to find relative md5sum from path=%+v: %w", dbRef.Path, err) - } + md5sumPath := path.Join(parentPath, "info", name+md5sumsExt) + md5SumLocation := resolver.RelativeFileByPath(dbLocation, md5sumPath) - if md5SumRef == nil { + if md5SumLocation == nil { // the most specific key did not work, fallback to just the name // look for /var/lib/dpkg/info/NAME.md5sums name := p.Name - md5sumPath := path.Join(string(parentPath), "info", name+md5sumsExt) - md5SumRef, err = resolver.RelativeFileByPath(dbRef, md5sumPath) - if err != nil { - return nil, nil, fmt.Errorf("unable to find relative md5sum from path=%+v: %w", dbRef.Path, err) - } + md5sumPath := path.Join(parentPath, "info", name+md5sumsExt) + md5SumLocation = resolver.RelativeFileByPath(dbLocation, md5sumPath) } // we should have at least one reference - if md5SumRef != nil { - md5FileMatches = append(md5FileMatches, *md5SumRef) - nameByRef[*md5SumRef] = name + if md5SumLocation != nil { + md5FileMatches = append(md5FileMatches, *md5SumLocation) + nameByRef[*md5SumLocation] = name } } // fetch the md5 contents - md5ContentsByRef, err := resolver.MultipleFileContentsByRef(md5FileMatches...) + md5ContentsByLocation, err := resolver.MultipleFileContentsByLocation(md5FileMatches) if err != nil { return nil, nil, err } // organize content results and refs by a combination of name and architecture var contentsByName = make(map[string]io.Reader) - var refsByName = make(map[string]file.Reference) - for ref, contents := range md5ContentsByRef { - name := nameByRef[ref] + var refsByName = make(map[string]source.Location) + for location, contents := range md5ContentsByLocation { + name := nameByRef[location] contentsByName[name] = strings.NewReader(contents) - refsByName[name] = ref + refsByName[name] = location } return contentsByName, refsByName, nil } -func fetchCopyrightContents(resolver source.Resolver, dbRef file.Reference, pkgs []pkg.Package) (map[string]io.Reader, map[string]file.Reference, error) { +func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) { // fetch all copyright file contents. This approach is more efficient than fetching each copyright file one at a time - var copyrightFileMatches []file.Reference - var nameByRef = make(map[file.Reference]string) + var copyrightFileMatches []source.Location + var nameByLocation = make(map[source.Location]string) for _, p := range pkgs { // look for /usr/share/docs/NAME/copyright files name := p.Name copyrightPath := path.Join(docsPath, name, "copyright") - copyrightRef, err := resolver.RelativeFileByPath(dbRef, copyrightPath) - if err != nil { - return nil, nil, fmt.Errorf("unable to find relative copyright from path=%+v: %w", dbRef.Path, err) - } + copyrightLocation := resolver.RelativeFileByPath(dbLocation, copyrightPath) // we may not have a copyright file for each package, ignore missing files - if copyrightRef != nil { - copyrightFileMatches = append(copyrightFileMatches, *copyrightRef) - nameByRef[*copyrightRef] = name + if copyrightLocation != nil { + copyrightFileMatches = append(copyrightFileMatches, *copyrightLocation) + nameByLocation[*copyrightLocation] = name } } // fetch the copyright contents - copyrightContentsByRef, err := resolver.MultipleFileContentsByRef(copyrightFileMatches...) + copyrightContentsByLocation, err := resolver.MultipleFileContentsByLocation(copyrightFileMatches) if err != nil { return nil, nil, err } // organize content results and refs by package name var contentsByName = make(map[string]io.Reader) - var refsByName = make(map[string]file.Reference) - for ref, contents := range copyrightContentsByRef { - name := nameByRef[ref] + var refsByName = make(map[string]source.Location) + for location, contents := range copyrightContentsByLocation { + name := nameByLocation[location] contentsByName[name] = strings.NewReader(contents) - refsByName[name] = ref + refsByName[name] = location } return contentsByName, refsByName, nil diff --git a/syft/cataloger/deb/cataloger_test.go b/syft/cataloger/deb/cataloger_test.go index 34ada8e75..d340316ed 100644 --- a/syft/cataloger/deb/cataloger_test.go +++ b/syft/cataloger/deb/cataloger_test.go @@ -77,11 +77,11 @@ func TestDpkgCataloger(t *testing.T) { for idx := range actual { a := &actual[idx] // we will test the sources separately - var sourcesList = make([]string, len(a.Source)) - for i, s := range a.Source { + var sourcesList = make([]string, len(a.Locations)) + for i, s := range a.Locations { sourcesList[i] = string(s.Path) } - a.Source = nil + a.Locations = nil for _, d := range deep.Equal(sourcesList, test.sources[a.Name]) { t.Errorf("diff: %+v", d) diff --git a/syft/cataloger/python/package_cataloger.go b/syft/cataloger/python/package_cataloger.go index bef3a891f..0d42a4473 100644 --- a/syft/cataloger/python/package_cataloger.go +++ b/syft/cataloger/python/package_cataloger.go @@ -8,8 +8,6 @@ import ( "github.com/anchore/syft/internal/log" - "github.com/anchore/stereoscope/pkg/file" - "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" @@ -35,7 +33,7 @@ func (c *PackageCataloger) Name() string { // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations. func (c *PackageCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { // nolint:prealloc - var fileMatches []file.Reference + var fileMatches []source.Location for _, glob := range []string{eggMetadataGlob, wheelMetadataGlob} { matches, err := resolver.FilesByGlob(glob) @@ -46,10 +44,10 @@ func (c *PackageCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, err } var pkgs []pkg.Package - for _, ref := range fileMatches { - p, err := c.catalogEggOrWheel(resolver, ref) + for _, location := range fileMatches { + p, err := c.catalogEggOrWheel(resolver, location) if err != nil { - return nil, fmt.Errorf("unable to catalog python package=%+v: %w", ref.Path, err) + return nil, fmt.Errorf("unable to catalog python package=%+v: %w", location.Path, err) } if p != nil { pkgs = append(pkgs, *p) @@ -59,8 +57,8 @@ func (c *PackageCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, err } // catalogEggOrWheel takes the primary metadata file reference and returns the python package it represents. -func (c *PackageCataloger) catalogEggOrWheel(resolver source.Resolver, metadataRef file.Reference) (*pkg.Package, error) { - metadata, sources, err := c.assembleEggOrWheelMetadata(resolver, metadataRef) +func (c *PackageCataloger) catalogEggOrWheel(resolver source.Resolver, metadataLocation source.Location) (*pkg.Package, error) { + metadata, sources, err := c.assembleEggOrWheelMetadata(resolver, metadataLocation) if err != nil { return nil, err } @@ -74,7 +72,7 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver source.Resolver, metadataR Name: metadata.Name, Version: metadata.Version, FoundBy: c.Name(), - Source: sources, + Locations: sources, Licenses: licenses, Language: pkg.Python, Type: pkg.PythonPkg, @@ -84,22 +82,19 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver source.Resolver, metadataR } // fetchRecordFiles finds a corresponding RECORD file for the given python package metadata file and returns the set of file records contained. -func (c *PackageCataloger) fetchRecordFiles(resolver source.Resolver, metadataRef file.Reference) (files []pkg.PythonFileRecord, sources []file.Reference, err error) { +func (c *PackageCataloger) fetchRecordFiles(resolver source.Resolver, metadataLocation source.Location) (files []pkg.PythonFileRecord, sources []source.Location, err error) { // we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory // or for an image... for an image the METADATA file may be present within multiple layers, so it is important // to reconcile the RECORD path to the same layer (or the next adjacent lower layer). // lets find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure) - recordPath := filepath.Join(filepath.Dir(string(metadataRef.Path)), "RECORD") - recordRef, err := resolver.RelativeFileByPath(metadataRef, recordPath) - if err != nil { - return nil, nil, err - } + recordPath := filepath.Join(filepath.Dir(metadataLocation.Path), "RECORD") + recordRef := resolver.RelativeFileByPath(metadataLocation, recordPath) if recordRef != nil { sources = append(sources, *recordRef) - recordContents, err := resolver.FileContentsByRef(*recordRef) + recordContents, err := resolver.FileContentsByLocation(*recordRef) if err != nil { return nil, nil, err } @@ -116,22 +111,20 @@ func (c *PackageCataloger) fetchRecordFiles(resolver source.Resolver, metadataRe } // fetchTopLevelPackages finds a corresponding top_level.txt file for the given python package metadata file and returns the set of package names contained. -func (c *PackageCataloger) fetchTopLevelPackages(resolver source.Resolver, metadataRef file.Reference) (pkgs []string, sources []file.Reference, err error) { +func (c *PackageCataloger) fetchTopLevelPackages(resolver source.Resolver, metadataLocation source.Location) (pkgs []string, sources []source.Location, err error) { // a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages - parentDir := filepath.Dir(string(metadataRef.Path)) + parentDir := filepath.Dir(metadataLocation.Path) topLevelPath := filepath.Join(parentDir, "top_level.txt") - topLevelRef, err := resolver.RelativeFileByPath(metadataRef, topLevelPath) - if err != nil { - return nil, nil, err - } + topLevelRef := resolver.RelativeFileByPath(metadataLocation, topLevelPath) + if topLevelRef == nil { - log.Warnf("missing python package top_level.txt (package=%q)", string(metadataRef.Path)) + log.Warnf("missing python package top_level.txt (package=%q)", metadataLocation.Path) return nil, nil, nil } sources = append(sources, *topLevelRef) - topLevelContents, err := resolver.FileContentsByRef(*topLevelRef) + topLevelContents, err := resolver.FileContentsByLocation(*topLevelRef) if err != nil { return nil, nil, err } @@ -149,21 +142,21 @@ func (c *PackageCataloger) fetchTopLevelPackages(resolver source.Resolver, metad } // assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from. -func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.Resolver, metadataRef file.Reference) (*pkg.PythonPackageMetadata, []file.Reference, error) { - var sources = []file.Reference{metadataRef} +func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.Resolver, metadataLocation source.Location) (*pkg.PythonPackageMetadata, []source.Location, error) { + var sources = []source.Location{metadataLocation} - metadataContents, err := resolver.FileContentsByRef(metadataRef) + metadataContents, err := resolver.FileContentsByLocation(metadataLocation) if err != nil { return nil, nil, err } - metadata, err := parseWheelOrEggMetadata(metadataRef.Path, strings.NewReader(metadataContents)) + metadata, err := parseWheelOrEggMetadata(metadataLocation.Path, strings.NewReader(metadataContents)) if err != nil { return nil, nil, err } // attach any python files found for the given wheel/egg installation - r, s, err := c.fetchRecordFiles(resolver, metadataRef) + r, s, err := c.fetchRecordFiles(resolver, metadataLocation) if err != nil { return nil, nil, err } @@ -171,7 +164,7 @@ func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.Resolver, metadata.Files = r // attach any top-level package names found for the given wheel/egg installation - p, s, err := c.fetchTopLevelPackages(resolver, metadataRef) + p, s, err := c.fetchTopLevelPackages(resolver, metadataLocation) if err != nil { return nil, nil, err } diff --git a/syft/cataloger/python/package_cataloger_test.go b/syft/cataloger/python/package_cataloger_test.go index 91081e8b1..be3033b97 100644 --- a/syft/cataloger/python/package_cataloger_test.go +++ b/syft/cataloger/python/package_cataloger_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/pkg" "github.com/go-test/deep" @@ -18,10 +18,10 @@ type pythonTestResolverMock struct { metadataReader io.Reader recordReader io.Reader topLevelReader io.Reader - metadataRef *file.Reference - recordRef *file.Reference - topLevelRef *file.Reference - contents map[file.Reference]string + metadataRef *source.Location + recordRef *source.Location + topLevelRef *source.Location + contents map[source.Location]string } func newTestResolver(metaPath, recordPath, topPath string) *pythonTestResolverMock { @@ -46,17 +46,17 @@ func newTestResolver(metaPath, recordPath, topPath string) *pythonTestResolverMo } } - var recordRef *file.Reference + var recordRef *source.Location if recordReader != nil { - ref := file.NewFileReference("test-fixtures/dist-info/RECORD") + ref := source.NewLocation("test-fixtures/dist-info/RECORD") recordRef = &ref } - var topLevelRef *file.Reference + var topLevelRef *source.Location if topLevelReader != nil { - ref := file.NewFileReference("test-fixtures/dist-info/top_level.txt") + ref := source.NewLocation("test-fixtures/dist-info/top_level.txt") topLevelRef = &ref } - metadataRef := file.NewFileReference("test-fixtures/dist-info/METADATA") + metadataRef := source.NewLocation("test-fixtures/dist-info/METADATA") return &pythonTestResolverMock{ recordReader: recordReader, metadataReader: metadataReader, @@ -64,11 +64,11 @@ func newTestResolver(metaPath, recordPath, topPath string) *pythonTestResolverMo metadataRef: &metadataRef, recordRef: recordRef, topLevelRef: topLevelRef, - contents: make(map[file.Reference]string), + contents: make(map[source.Location]string), } } -func (r *pythonTestResolverMock) FileContentsByRef(ref file.Reference) (string, error) { +func (r *pythonTestResolverMock) FileContentsByLocation(ref source.Location) (string, error) { switch { case r.topLevelRef != nil && ref.Path == r.topLevelRef.Path: b, err := ioutil.ReadAll(r.topLevelReader) @@ -92,25 +92,25 @@ func (r *pythonTestResolverMock) FileContentsByRef(ref file.Reference) (string, return "", fmt.Errorf("invalid value given") } -func (r *pythonTestResolverMock) MultipleFileContentsByRef(_ ...file.Reference) (map[file.Reference]string, error) { +func (r *pythonTestResolverMock) MultipleFileContentsByLocation(_ []source.Location) (map[source.Location]string, error) { return nil, fmt.Errorf("not implemented") } -func (r *pythonTestResolverMock) FilesByPath(_ ...file.Path) ([]file.Reference, error) { +func (r *pythonTestResolverMock) FilesByPath(_ ...string) ([]source.Location, error) { return nil, fmt.Errorf("not implemented") } -func (r *pythonTestResolverMock) FilesByGlob(_ ...string) ([]file.Reference, error) { +func (r *pythonTestResolverMock) FilesByGlob(_ ...string) ([]source.Location, error) { return nil, fmt.Errorf("not implemented") } -func (r *pythonTestResolverMock) RelativeFileByPath(_ file.Reference, path string) (*file.Reference, error) { +func (r *pythonTestResolverMock) RelativeFileByPath(_ source.Location, path string) *source.Location { switch { case strings.Contains(path, "RECORD"): - return r.recordRef, nil + return r.recordRef case strings.Contains(path, "top_level.txt"): - return r.topLevelRef, nil + return r.topLevelRef default: - return nil, fmt.Errorf("invalid RelativeFileByPath value given: %q", path) + panic(fmt.Errorf("invalid RelativeFileByPath value given: %q", path)) } } @@ -214,13 +214,13 @@ func TestPythonPackageWheelCataloger(t *testing.T) { resolver := newTestResolver(test.MetadataFixture, test.RecordFixture, test.TopLevelFixture) // note that the source is the record ref created by the resolver mock... attach the expected values - test.ExpectedPackage.Source = []file.Reference{*resolver.metadataRef} + test.ExpectedPackage.Locations = []source.Location{*resolver.metadataRef} if resolver.recordRef != nil { - test.ExpectedPackage.Source = append(test.ExpectedPackage.Source, *resolver.recordRef) + test.ExpectedPackage.Locations = append(test.ExpectedPackage.Locations, *resolver.recordRef) } if resolver.topLevelRef != nil { - test.ExpectedPackage.Source = append(test.ExpectedPackage.Source, *resolver.topLevelRef) + test.ExpectedPackage.Locations = append(test.ExpectedPackage.Locations, *resolver.topLevelRef) } // end patching expected values with runtime data... diff --git a/syft/cataloger/python/parse_wheel_egg_metadata.go b/syft/cataloger/python/parse_wheel_egg_metadata.go index dcb90a14c..ca6e40c9f 100644 --- a/syft/cataloger/python/parse_wheel_egg_metadata.go +++ b/syft/cataloger/python/parse_wheel_egg_metadata.go @@ -7,8 +7,6 @@ import ( "path/filepath" "strings" - "github.com/anchore/stereoscope/pkg/file" - "github.com/mitchellh/mapstructure" "github.com/anchore/syft/syft/pkg" @@ -16,7 +14,7 @@ import ( // parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes), // returning all Python packages listed. -func parseWheelOrEggMetadata(path file.Path, reader io.Reader) (pkg.PythonPackageMetadata, error) { +func parseWheelOrEggMetadata(path string, reader io.Reader) (pkg.PythonPackageMetadata, error) { fields := make(map[string]string) var key string @@ -73,7 +71,7 @@ func parseWheelOrEggMetadata(path file.Path, reader io.Reader) (pkg.PythonPackag // add additional metadata not stored in the egg/wheel metadata file - sitePackagesRoot := filepath.Clean(filepath.Join(filepath.Dir(string(path)), "..")) + sitePackagesRoot := filepath.Clean(filepath.Join(filepath.Dir(path), "..")) metadata.SitePackagesRootPath = sitePackagesRoot return metadata, nil diff --git a/syft/cataloger/python/parse_wheel_egg_metadata_test.go b/syft/cataloger/python/parse_wheel_egg_metadata_test.go index 98896eef2..3c04beba2 100644 --- a/syft/cataloger/python/parse_wheel_egg_metadata_test.go +++ b/syft/cataloger/python/parse_wheel_egg_metadata_test.go @@ -4,8 +4,6 @@ import ( "os" "testing" - "github.com/anchore/stereoscope/pkg/file" - "github.com/anchore/syft/syft/pkg" "github.com/go-test/deep" ) @@ -48,7 +46,7 @@ func TestParseWheelEggMetadata(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parseWheelOrEggMetadata(file.Path(test.Fixture), fixture) + actual, err := parseWheelOrEggMetadata(test.Fixture, fixture) if err != nil { t.Fatalf("failed to parse: %+v", err) } diff --git a/syft/cataloger/rpmdb/cataloger.go b/syft/cataloger/rpmdb/cataloger.go index c36ac099b..3ec99e112 100644 --- a/syft/cataloger/rpmdb/cataloger.go +++ b/syft/cataloger/rpmdb/cataloger.go @@ -35,15 +35,15 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { } var pkgs []pkg.Package - for _, dbRef := range fileMatches { - dbContents, err := resolver.FileContentsByRef(dbRef) + for _, location := range fileMatches { + dbContents, err := resolver.FileContentsByLocation(location) if err != nil { return nil, err } - pkgs, err = parseRpmDB(resolver, dbRef, strings.NewReader(dbContents)) + pkgs, err = parseRpmDB(resolver, location, strings.NewReader(dbContents)) if err != nil { - return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", dbRef.Path, err) + return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.Path, err) } } return pkgs, nil diff --git a/syft/cataloger/rpmdb/parse_rpmdb.go b/syft/cataloger/rpmdb/parse_rpmdb.go index 7ec9bc206..05b6ec157 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb.go +++ b/syft/cataloger/rpmdb/parse_rpmdb.go @@ -6,19 +6,15 @@ import ( "io/ioutil" "os" - "github.com/anchore/stereoscope/pkg/file" - - "github.com/anchore/syft/syft/scope" - "github.com/anchore/syft/syft/source" - rpmdb "github.com/anchore/go-rpmdb/pkg" "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" ) // parseApkDb parses an "Packages" RPM DB and returns the Packages listed within it. -func parseRpmDB(resolver scope.FileResolver, dbRef file.Reference, reader io.Reader) ([]pkg.Package, error) { +func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader io.Reader) ([]pkg.Package, error) { f, err := ioutil.TempFile("", internal.ApplicationName+"-rpmdb") if err != nil { return nil, fmt.Errorf("failed to create temp rpmdb file: %w", err) @@ -58,7 +54,7 @@ func parseRpmDB(resolver scope.FileResolver, dbRef file.Reference, reader io.Rea Name: entry.Name, Version: fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does //Version: fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch), - Source: []file.Reference{dbRef}, + Locations: []source.Location{dbLocation}, Type: pkg.RpmPkg, MetadataType: pkg.RpmdbMetadataType, Metadata: pkg.RpmdbMetadata{ @@ -85,7 +81,7 @@ func extractRpmdbFileRecords(resolver source.FileResolver, entry *rpmdb.PackageI var records = make([]pkg.RpmdbFileRecord, 0) for _, record := range entry.Files { - refs, err := resolver.FilesByPath(file.Path(record.Path)) + refs, err := resolver.FilesByPath(record.Path) if err != nil { return nil, fmt.Errorf("failed to resolve path=%+v: %w", record.Path, err) } diff --git a/syft/cataloger/rpmdb/parse_rpmdb_test.go b/syft/cataloger/rpmdb/parse_rpmdb_test.go index ae0f76e2e..2b89e7dff 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb_test.go +++ b/syft/cataloger/rpmdb/parse_rpmdb_test.go @@ -5,7 +5,8 @@ import ( "os" "testing" - "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/syft/pkg" "github.com/go-test/deep" ) @@ -20,24 +21,25 @@ func newTestFileResolver(ignorePaths bool) *rpmdbTestFileResolverMock { } } -func (r *rpmdbTestFileResolverMock) FilesByPath(paths ...file.Path) ([]file.Reference, error) { +func (r *rpmdbTestFileResolverMock) FilesByPath(paths ...string) ([]source.Location, error) { if r.ignorePaths { // act as if no paths exist return nil, nil } // act as if all files exist - var refs = make([]file.Reference, len(paths)) + var locations = make([]source.Location, len(paths)) for i, p := range paths { - refs[i] = file.NewFileReference(p) + locations[i] = source.NewLocation(p) } - return refs, nil + return locations, nil } -func (r *rpmdbTestFileResolverMock) FilesByGlob(...string) ([]file.Reference, error) { +func (r *rpmdbTestFileResolverMock) FilesByGlob(...string) ([]source.Location, error) { return nil, fmt.Errorf("not implemented") } -func (r *rpmdbTestFileResolverMock) RelativeFileByPath(file.Reference, string) (*file.Reference, error) { - return nil, fmt.Errorf("not implemented") +func (r *rpmdbTestFileResolverMock) RelativeFileByPath(source.Location, string) *source.Location { + panic(fmt.Errorf("not implemented")) + return nil } func TestParseRpmDB(t *testing.T) { diff --git a/syft/distro/identify.go b/syft/distro/identify.go index 8868c136b..b029a20a3 100644 --- a/syft/distro/identify.go +++ b/syft/distro/identify.go @@ -4,7 +4,6 @@ import ( "regexp" "strings" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/source" ) @@ -13,7 +12,7 @@ import ( type parseFunc func(string) *Distro type parseEntry struct { - path file.Path + path string fn parseFunc } @@ -41,25 +40,19 @@ func Identify(resolver source.Resolver) Distro { identifyLoop: for _, entry := range identityFiles { - refs, err := resolver.FilesByPath(entry.path) + locations, err := resolver.FilesByPath(entry.path) if err != nil { - log.Errorf("unable to get path refs from %s: %s", entry.path, err) + log.Errorf("unable to get path locations from %s: %s", entry.path, err) break } - if len(refs) == 0 { + if len(locations) == 0 { log.Debugf("No Refs found from path: %s", entry.path) continue } - for _, ref := range refs { - contents, err := resolver.MultipleFileContentsByRef(ref) - content, ok := contents[ref] - - if !ok { - log.Infof("no content present for ref: %s", ref) - continue - } + for _, location := range locations { + content, err := resolver.FileContentsByLocation(location) if err != nil { log.Debugf("unable to get contents from %s: %s", entry.path, err) diff --git a/syft/lib.go b/syft/lib.go index d2ba2e04e..930669371 100644 --- a/syft/lib.go +++ b/syft/lib.go @@ -7,8 +7,8 @@ Here is what the main execution path for syft does: 2. Invoke all catalogers to catalog the image, adding discovered packages to a single catalog object 3. Invoke a single presenter to show the contents of the catalog -A Source object encapsulates the image object to be cataloged and the user options (catalog all layers vs. squashed layer), -providing a way to inspect paths and file content within the image. The Source object, not the image object, is used +A Locations object encapsulates the image object to be cataloged and the user options (catalog all layers vs. squashed layer), +providing a way to inspect paths and file content within the image. The Locations object, not the image object, is used throughout the main execution path. This abstraction allows for decoupling of what is cataloged (a docker image, an OCI image, a filesystem, etc) and how it is cataloged (the individual catalogers). @@ -110,8 +110,8 @@ func CatalogFromJSON(reader io.Reader) (*pkg.Catalog, *distro.Distro, error) { } //var theImg *jsonPresenter.Image - //if doc.Source.Type == "image" { - // img := doc.Source.Target.(jsonPresenter.Image) + //if doc.Locations.Type == "image" { + // img := doc.Locations.Target.(jsonPresenter.Image) // theImg = &img //} diff --git a/syft/pkg/catalog.go b/syft/pkg/catalog.go index 0e7ae49e0..81a1d4652 100644 --- a/syft/pkg/catalog.go +++ b/syft/pkg/catalog.go @@ -4,7 +4,8 @@ import ( "sort" "sync" - "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/internal/log" ) @@ -14,7 +15,7 @@ var nextPackageID int64 type Catalog struct { byID map[ID]*Package byType map[Type][]*Package - byFile map[file.Reference][]*Package + byFile map[source.Location][]*Package lock sync.RWMutex } @@ -23,7 +24,7 @@ func NewCatalog(pkgs ...Package) *Catalog { catalog := Catalog{ byID: make(map[ID]*Package), byType: make(map[Type][]*Package), - byFile: make(map[file.Reference][]*Package), + byFile: make(map[source.Location][]*Package), } for _, p := range pkgs { @@ -44,8 +45,8 @@ func (c *Catalog) Package(id ID) *Package { } // PackagesByFile returns all packages that were discovered from the given source file reference. -func (c *Catalog) PackagesByFile(ref file.Reference) []*Package { - return c.byFile[ref] +func (c *Catalog) PackagesByFile(location source.Location) []*Package { + return c.byFile[location] } // Add a package to the Catalog. @@ -71,7 +72,7 @@ func (c *Catalog) Add(p Package) { c.byType[p.Type] = append(c.byType[p.Type], &p) // store by file references - for _, s := range p.Source { + for _, s := range p.Locations { _, ok := c.byFile[s] if !ok { c.byFile[s] = make([]*Package, 0) diff --git a/syft/pkg/dpkg_metadata.go b/syft/pkg/dpkg_metadata.go index f1e80802e..a29bf8169 100644 --- a/syft/pkg/dpkg_metadata.go +++ b/syft/pkg/dpkg_metadata.go @@ -9,7 +9,7 @@ import ( // at http://manpages.ubuntu.com/manpages/xenial/man1/dpkg-query.1.html in the --showformat section. type DpkgMetadata struct { Package string `mapstructure:"Package" json:"package"` - Source string `mapstructure:"Source" json:"source"` + Source string `mapstructure:"Locations" json:"source"` Version string `mapstructure:"Version" json:"version"` Architecture string `mapstructure:"Architecture" json:"architecture"` Maintainer string `mapstructure:"Maintainer" json:"maintainer"` diff --git a/syft/pkg/package.go b/syft/pkg/package.go index 3b81f46a2..525e5e893 100644 --- a/syft/pkg/package.go +++ b/syft/pkg/package.go @@ -8,7 +8,8 @@ import ( "regexp" "strings" - "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/syft/distro" "github.com/package-url/packageurl-go" ) @@ -17,12 +18,11 @@ type ID int64 // Package represents an application or library that has been bundled into a distributable format. type Package struct { - id ID // uniquely identifies a package, set by the cataloger - Name string `json:"manifest"` // the package name - Version string `json:"version"` // the version of the package - FoundBy string `json:"foundBy"` // the specific cataloger that discovered this package - Source []file.Reference `json:"-"` // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package) - Location interface{} `json:"locations"` + id ID // uniquely identifies a package, set by the cataloger + Name string `json:"manifest"` // the package name + Version string `json:"version"` // the version of the package + FoundBy string `json:"foundBy"` // the specific cataloger that discovered this package + Locations []source.Location `json:"-"` // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package) // TODO: should we move licenses into metadata? Licenses []string `json:"licenses"` // licenses discovered with the package metadata Language Language `json:"language"` // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc) diff --git a/syft/presenter/cyclonedx/bom-extension.go b/syft/presenter/cyclonedx/bom-extension.go index f25713734..8e7ce850d 100644 --- a/syft/presenter/cyclonedx/bom-extension.go +++ b/syft/presenter/cyclonedx/bom-extension.go @@ -8,7 +8,7 @@ import ( "github.com/anchore/syft/internal/version" ) -// Source: https://cyclonedx.org/ext/bom-descriptor/ +// Locations: https://cyclonedx.org/ext/bom-descriptor/ // BomDescriptor represents all metadata surrounding the BOM report (such as when the BOM was made, with which tool, and the item being cataloged). type BomDescriptor struct { diff --git a/syft/presenter/cyclonedx/document.go b/syft/presenter/cyclonedx/document.go index 4539ab73e..f2d2fb866 100644 --- a/syft/presenter/cyclonedx/document.go +++ b/syft/presenter/cyclonedx/document.go @@ -9,7 +9,7 @@ import ( "github.com/google/uuid" ) -// Source: https://github.com/CycloneDX/specification +// Locations: https://github.com/CycloneDX/specification // Document represents a CycloneDX BOM Document. type Document struct { diff --git a/syft/presenter/cyclonedx/presenter.go b/syft/presenter/cyclonedx/presenter.go index 22d9a8238..b73052d51 100644 --- a/syft/presenter/cyclonedx/presenter.go +++ b/syft/presenter/cyclonedx/presenter.go @@ -14,14 +14,14 @@ import ( "github.com/anchore/syft/syft/source" ) -// Presenter writes a CycloneDX report from the given Catalog and Source contents +// Presenter writes a CycloneDX report from the given Catalog and Locations contents type Presenter struct { catalog *pkg.Catalog source source.Source distro distro.Distro } -// NewPresenter creates a CycloneDX presenter from the given Catalog and Source objects. +// NewPresenter creates a CycloneDX presenter from the given Catalog and Locations objects. func NewPresenter(catalog *pkg.Catalog, s source.Source, d distro.Distro) *Presenter { return &Presenter{ catalog: catalog, diff --git a/syft/presenter/cyclonedx/presenter_test.go b/syft/presenter/cyclonedx/presenter_test.go index df8b3745c..90ca5b309 100644 --- a/syft/presenter/cyclonedx/presenter_test.go +++ b/syft/presenter/cyclonedx/presenter_test.go @@ -10,7 +10,6 @@ import ( "github.com/anchore/syft/syft/distro" "github.com/anchore/go-testutils" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" "github.com/sergi/go-diff/diffmatchpatch" @@ -29,7 +28,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) { Version: "1.0.1", Type: pkg.DebPkg, FoundBy: "the-cataloger-1", - Source: []file.Reference{ + Locations: []source.Location{ {Path: "/some/path/pkg1"}, }, Metadata: pkg.DpkgMetadata{ @@ -43,7 +42,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) { Version: "2.0.1", Type: pkg.DebPkg, FoundBy: "the-cataloger-2", - Source: []file.Reference{ + Locations: []source.Location{ {Path: "/some/path/pkg1"}, }, Licenses: []string{ @@ -105,8 +104,8 @@ func TestCycloneDxImgsPresenter(t *testing.T) { catalog.Add(pkg.Package{ Name: "package1", Version: "1.0.1", - Source: []file.Reference{ - *img.SquashedTree().File("/somefile-1.txt"), + Locations: []source.Location{ + source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img), }, Type: pkg.RpmPkg, FoundBy: "the-cataloger-1", @@ -125,8 +124,8 @@ func TestCycloneDxImgsPresenter(t *testing.T) { catalog.Add(pkg.Package{ Name: "package2", Version: "2.0.1", - Source: []file.Reference{ - *img.SquashedTree().File("/somefile-2.txt"), + Locations: []source.Location{ + source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img), }, Type: pkg.RpmPkg, FoundBy: "the-cataloger-2", diff --git a/syft/presenter/json/artifact.go b/syft/presenter/json/artifact.go index 3c15e5353..8a9e586d7 100644 --- a/syft/presenter/json/artifact.go +++ b/syft/presenter/json/artifact.go @@ -14,13 +14,13 @@ type Artifact struct { } type ArtifactBasicMetadata struct { - Name string `json:"name"` - Version string `json:"version"` - Type string `json:"type"` - FoundBy []string `json:"foundBy"` - Locations Locations `json:"locations,omitempty"` - Licenses []string `json:"licenses"` - Language string `json:"language"` + Name string `json:"name"` + Version string `json:"version"` + Type string `json:"type"` + FoundBy []string `json:"foundBy"` + Locations []source.Location `json:"locations"` + Licenses []string `json:"licenses"` + Language string `json:"language"` } type ArtifactCustomMetadata struct { @@ -34,10 +34,6 @@ type ArtifactMetadataUnpacker struct { } func NewArtifact(p *pkg.Package, s source.Source) (Artifact, error) { - locations, err := NewLocations(p, s) - if err != nil { - return Artifact{}, err - } return Artifact{ ArtifactBasicMetadata: ArtifactBasicMetadata{ @@ -45,7 +41,7 @@ func NewArtifact(p *pkg.Package, s source.Source) (Artifact, error) { Version: p.Version, Type: string(p.Type), FoundBy: []string{p.FoundBy}, - Locations: locations, + Locations: p.Locations, Licenses: p.Licenses, Language: string(p.Language), }, diff --git a/syft/presenter/json/location.go b/syft/presenter/json/location.go deleted file mode 100644 index 2321afa49..000000000 --- a/syft/presenter/json/location.go +++ /dev/null @@ -1,45 +0,0 @@ -package json - -import ( - "fmt" - - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/source" -) - -type Locations interface{} - -type ImageLocation struct { - Path string `json:"path"` - LayerIndex uint `json:"layerIndex"` -} - -func NewLocations(p *pkg.Package, s source.Source) (Locations, error) { - switch src := s.Target.(type) { - case source.ImageSource: - locations := make([]ImageLocation, len(p.Source)) - for idx := range p.Source { - entry, err := src.Img.FileCatalog.Get(p.Source[idx]) - if err != nil { - return nil, fmt.Errorf("unable to find layer index for source-idx=%d package=%s", idx, p.Name) - } - - artifactSource := ImageLocation{ - LayerIndex: entry.Source.Metadata.Index, - Path: string(p.Source[idx].Path), - } - - locations[idx] = artifactSource - } - return locations, nil - - case source.DirSource: - locations := make([]string, len(p.Source)) - for idx := range p.Source { - locations[idx] = string(p.Source[idx].Path) - } - return locations, nil - default: - return nil, fmt.Errorf("unable to determine source: %T", src) - } -} diff --git a/syft/presenter/json/presenter_test.go b/syft/presenter/json/presenter_test.go index c7f389df5..94d052dae 100644 --- a/syft/presenter/json/presenter_test.go +++ b/syft/presenter/json/presenter_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/anchore/go-testutils" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/pkg" @@ -27,7 +26,7 @@ func TestJsonDirsPresenter(t *testing.T) { Version: "1.0.1", Type: pkg.PythonPkg, FoundBy: "the-cataloger-1", - Source: []file.Reference{ + Locations: []source.Location{ {Path: "/some/path/pkg1"}, }, Language: pkg.Python, @@ -43,7 +42,7 @@ func TestJsonDirsPresenter(t *testing.T) { Version: "2.0.1", Type: pkg.DebPkg, FoundBy: "the-cataloger-2", - Source: []file.Reference{ + Locations: []source.Location{ {Path: "/some/path/pkg1"}, }, MetadataType: pkg.DpkgMetadataType, @@ -96,8 +95,8 @@ func TestJsonImgsPresenter(t *testing.T) { catalog.Add(pkg.Package{ Name: "package-1", Version: "1.0.1", - Source: []file.Reference{ - *img.SquashedTree().File("/somefile-1.txt"), + Locations: []source.Location{ + source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img), }, Type: pkg.PythonPkg, FoundBy: "the-cataloger-1", @@ -112,8 +111,8 @@ func TestJsonImgsPresenter(t *testing.T) { catalog.Add(pkg.Package{ Name: "package-2", Version: "2.0.1", - Source: []file.Reference{ - *img.SquashedTree().File("/somefile-2.txt"), + Locations: []source.Location{ + source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img), }, Type: pkg.DebPkg, FoundBy: "the-cataloger-2", diff --git a/syft/presenter/table/presenter_test.go b/syft/presenter/table/presenter_test.go index afd8ac465..160e95169 100644 --- a/syft/presenter/table/presenter_test.go +++ b/syft/presenter/table/presenter_test.go @@ -8,7 +8,6 @@ import ( "github.com/go-test/deep" "github.com/anchore/go-testutils" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" @@ -30,16 +29,16 @@ func TestTablePresenter(t *testing.T) { catalog.Add(pkg.Package{ Name: "package-1", Version: "1.0.1", - Source: []file.Reference{ - *img.SquashedTree().File("/somefile-1.txt"), + Locations: []source.Location{ + source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img), }, Type: pkg.DebPkg, }) catalog.Add(pkg.Package{ Name: "package-2", Version: "2.0.1", - Source: []file.Reference{ - *img.SquashedTree().File("/somefile-2.txt"), + Locations: []source.Location{ + source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img), }, Type: pkg.DebPkg, }) diff --git a/syft/presenter/text/presenter_test.go b/syft/presenter/text/presenter_test.go index 9fdcd860a..9144977de 100644 --- a/syft/presenter/text/presenter_test.go +++ b/syft/presenter/text/presenter_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/anchore/go-testutils" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" @@ -75,8 +74,8 @@ func TestTextImgPresenter(t *testing.T) { catalog.Add(pkg.Package{ Name: "package-1", Version: "1.0.1", - Source: []file.Reference{ - *img.SquashedTree().File("/somefile-1.txt"), + Locations: []source.Location{ + source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img), }, FoundBy: "dpkg", Type: pkg.DebPkg, @@ -84,8 +83,8 @@ func TestTextImgPresenter(t *testing.T) { catalog.Add(pkg.Package{ Name: "package-2", Version: "2.0.1", - Source: []file.Reference{ - *img.SquashedTree().File("/somefile-2.txt"), + Locations: []source.Location{ + source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img), }, FoundBy: "dpkg", Metadata: PackageInfo{Name: "package-2", Version: "1.0.2"}, diff --git a/syft/source/all_layers_resolver.go b/syft/source/all_layers_resolver.go index 1c2be3749..1c58e89b1 100644 --- a/syft/source/all_layers_resolver.go +++ b/syft/source/all_layers_resolver.go @@ -94,7 +94,7 @@ func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) { return nil, err } for _, result := range results { - uniqueLocations = append(uniqueLocations, newLocationFromImage(result, r.img)) + uniqueLocations = append(uniqueLocations, NewLocationFromImage(result, r.img)) } } } @@ -132,7 +132,7 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error) return nil, err } for _, result := range results { - uniqueLocations = append(uniqueLocations, newLocationFromImage(result, r.img)) + uniqueLocations = append(uniqueLocations, NewLocationFromImage(result, r.img)) } } } @@ -152,20 +152,20 @@ func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) * return nil } - relativeLocation := newLocationFromImage(*relativeRef, r.img) + relativeLocation := NewLocationFromImage(*relativeRef, r.img) return &relativeLocation } // MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a // file.Reference is a path relative to a particular layer. -func (r *AllLayersResolver) MultipleFileContentsByRef(locations []Location) (map[Location]string, error) { +func (r *AllLayersResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) { return mapLocationRefs(r.img.MultipleFileContentsByRef, locations) } // FileContentsByRef fetches file contents for a single file reference, irregardless of the source layer. // If the path does not exist an error is returned. -func (r *AllLayersResolver) FileContentsByRef(location Location) (string, error) { +func (r *AllLayersResolver) FileContentsByLocation(location Location) (string, error) { return r.img.FileContentsByRef(location.ref) } diff --git a/syft/source/all_layers_resolver_test.go b/syft/source/all_layers_resolver_test.go index e8076f4cb..ac3ea4ebf 100644 --- a/syft/source/all_layers_resolver_test.go +++ b/syft/source/all_layers_resolver_test.go @@ -4,8 +4,6 @@ import ( "testing" "github.com/anchore/stereoscope/pkg/imagetest" - - "github.com/anchore/stereoscope/pkg/file" ) type resolution struct { @@ -97,7 +95,7 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) { t.Fatalf("could not create resolver: %+v", err) } - refs, err := resolver.FilesByPath(file.Path(c.linkPath)) + refs, err := resolver.FilesByPath(c.linkPath) if err != nil { t.Fatalf("could not use resolver: %+v", err) } @@ -109,11 +107,11 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) { for idx, actual := range refs { expected := c.resolutions[idx] - if actual.Path != file.Path(expected.path) { + if actual.Path != expected.path { t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path) } - entry, err := img.FileCatalog.Get(actual) + entry, err := img.FileCatalog.Get(actual.ref) if err != nil { t.Fatalf("failed to get metadata: %+v", err) } @@ -222,11 +220,11 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) { for idx, actual := range refs { expected := c.resolutions[idx] - if actual.Path != file.Path(expected.path) { + if actual.Path != expected.path { t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path) } - entry, err := img.FileCatalog.Get(actual) + entry, err := img.FileCatalog.Get(actual.ref) if err != nil { t.Fatalf("failed to get metadata: %+v", err) } diff --git a/syft/source/directory_resolver.go b/syft/source/directory_resolver.go index 9cbacf6bf..3d6904e74 100644 --- a/syft/source/directory_resolver.go +++ b/syft/source/directory_resolver.go @@ -7,8 +7,6 @@ import ( "path" "path/filepath" - "github.com/docker/distribution/reference" - "github.com/anchore/syft/internal/log" "github.com/bmatcuk/doublestar" ) @@ -48,7 +46,7 @@ func (s DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error) continue } - references = append(references, newLocation(userStrPath)) + references = append(references, NewLocation(userStrPath)) } return references, nil @@ -75,7 +73,7 @@ func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) { continue } - result = append(result, newLocation(matchedPath)) + result = append(result, NewLocation(matchedPath)) } } @@ -95,7 +93,7 @@ func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Locatio } // MultipleFileContentsByRef returns the file contents for all file.References relative a directory. -func (s DirectoryResolver) MultipleFileContentsByRef(locations []Location) (map[Location]string, error) { +func (s DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) { refContents := make(map[Location]string) for _, location := range locations { contents, err := fileContents(location.Path) @@ -110,10 +108,10 @@ func (s DirectoryResolver) MultipleFileContentsByRef(locations []Location) (map[ // FileContentsByRef fetches file contents for a single file reference relative to a directory. // If the path does not exist an error is returned. -func (s DirectoryResolver) FileContentsByRef(location Location) (string, error) { +func (s DirectoryResolver) FileContentsByLocation(location Location) (string, error) { contents, err := fileContents(location.Path) if err != nil { - return "", fmt.Errorf("could not read contents of file: %s", reference.Path) + return "", fmt.Errorf("could not read contents of file: %s", location.Path) } return string(contents), nil diff --git a/syft/source/directory_resolver_test.go b/syft/source/directory_resolver_test.go index fa234c10e..f2bcad89f 100644 --- a/syft/source/directory_resolver_test.go +++ b/syft/source/directory_resolver_test.go @@ -2,8 +2,6 @@ package source import ( "testing" - - "github.com/anchore/stereoscope/pkg/file" ) func TestDirectoryResolver_FilesByPath(t *testing.T) { @@ -58,7 +56,7 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { resolver := DirectoryResolver{c.root} - refs, err := resolver.FilesByPath(file.Path(c.input)) + refs, err := resolver.FilesByPath(c.input) if err != nil { t.Fatalf("could not use resolver: %+v, %+v", err, refs) } @@ -68,7 +66,7 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) { } for _, actual := range refs { - if actual.Path != file.Path(c.expected) { + if actual.Path != c.expected { t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.expected) } } @@ -79,22 +77,22 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) { func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) { cases := []struct { name string - input []file.Path + input []string refCount int }{ { name: "finds multiple files", - input: []file.Path{file.Path("test-fixtures/image-symlinks/file-1.txt"), file.Path("test-fixtures/image-symlinks/file-2.txt")}, + input: []string{"test-fixtures/image-symlinks/file-1.txt", "test-fixtures/image-symlinks/file-2.txt"}, refCount: 2, }, { name: "skips non-existing files", - input: []file.Path{file.Path("test-fixtures/image-symlinks/bogus.txt"), file.Path("test-fixtures/image-symlinks/file-1.txt")}, + input: []string{"test-fixtures/image-symlinks/bogus.txt", "test-fixtures/image-symlinks/file-1.txt"}, refCount: 1, }, { name: "does not return anything for non-existing directories", - input: []file.Path{file.Path("test-fixtures/non-existing/bogus.txt"), file.Path("test-fixtures/non-existing/file-1.txt")}, + input: []string{"test-fixtures/non-existing/bogus.txt", "test-fixtures/non-existing/file-1.txt"}, refCount: 0, }, } @@ -117,47 +115,47 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) { func TestDirectoryResolver_MultipleFileContentsByRef(t *testing.T) { cases := []struct { name string - input []file.Path + input []string refCount int contents []string }{ { name: "gets multiple file contents", - input: []file.Path{file.Path("test-fixtures/image-symlinks/file-1.txt"), file.Path("test-fixtures/image-symlinks/file-2.txt")}, + input: []string{"test-fixtures/image-symlinks/file-1.txt", "test-fixtures/image-symlinks/file-2.txt"}, refCount: 2, }, { name: "skips non-existing files", - input: []file.Path{file.Path("test-fixtures/image-symlinks/bogus.txt"), file.Path("test-fixtures/image-symlinks/file-1.txt")}, + input: []string{"test-fixtures/image-symlinks/bogus.txt", "test-fixtures/image-symlinks/file-1.txt"}, refCount: 1, }, { name: "does not return anything for non-existing directories", - input: []file.Path{file.Path("test-fixtures/non-existing/bogus.txt"), file.Path("test-fixtures/non-existing/file-1.txt")}, + input: []string{"test-fixtures/non-existing/bogus.txt", "test-fixtures/non-existing/file-1.txt"}, refCount: 0, }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - refs := make([]file.Reference, 0) + locations := make([]Location, 0) resolver := DirectoryResolver{"test-fixtures"} for _, p := range c.input { newRefs, err := resolver.FilesByPath(p) if err != nil { - t.Errorf("could not generate refs: %+v", err) + t.Errorf("could not generate locations: %+v", err) } for _, ref := range newRefs { - refs = append(refs, ref) + locations = append(locations, ref) } } - contents, err := resolver.MultipleFileContentsByRef(refs...) + contents, err := resolver.MultipleFileContentsByLocation(locations) if err != nil { t.Fatalf("unable to generate file contents by ref: %+v", err) } if len(contents) != c.refCount { - t.Errorf("unexpected number of refs produced: %d != %d", len(contents), c.refCount) + t.Errorf("unexpected number of locations produced: %d != %d", len(contents), c.refCount) } }) diff --git a/syft/source/image_squash_resolver.go b/syft/source/image_squash_resolver.go index ea4f3ffd6..0e34b3355 100644 --- a/syft/source/image_squash_resolver.go +++ b/syft/source/image_squash_resolver.go @@ -56,7 +56,7 @@ func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) { if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) { uniqueFileIDs.Add(*resolvedRef) - uniqueLocations = append(uniqueLocations, newLocationFromImage(*resolvedRef, r.img)) + uniqueLocations = append(uniqueLocations, NewLocationFromImage(*resolvedRef, r.img)) } } @@ -118,12 +118,12 @@ func (r *ImageSquashResolver) RelativeFileByPath(_ Location, path string) *Locat // MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a // file.Reference is a path relative to a particular layer, in this case only from the squashed representation. -func (r *ImageSquashResolver) MultipleFileContentsByRef(locations []Location) (map[Location]string, error) { +func (r *ImageSquashResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) { return mapLocationRefs(r.img.MultipleFileContentsByRef, locations) } // FileContentsByRef fetches file contents for a single file reference, irregardless of the source layer. // If the path does not exist an error is returned. -func (r *ImageSquashResolver) FileContentsByRef(location Location) (string, error) { +func (r *ImageSquashResolver) FileContentsByLocation(location Location) (string, error) { return r.img.FileContentsByRef(location.ref) } diff --git a/syft/source/image_squash_resolver_test.go b/syft/source/image_squash_resolver_test.go index 88f8074a2..f5d9b2d9f 100644 --- a/syft/source/image_squash_resolver_test.go +++ b/syft/source/image_squash_resolver_test.go @@ -4,8 +4,6 @@ import ( "testing" "github.com/anchore/stereoscope/pkg/imagetest" - - "github.com/anchore/stereoscope/pkg/file" ) func TestImageSquashResolver_FilesByPath(t *testing.T) { @@ -61,7 +59,7 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) { t.Fatalf("could not create resolver: %+v", err) } - refs, err := resolver.FilesByPath(file.Path(c.linkPath)) + refs, err := resolver.FilesByPath(c.linkPath) if err != nil { t.Fatalf("could not use resolver: %+v", err) } @@ -82,11 +80,11 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) { actual := refs[0] - if actual.Path != file.Path(c.resolvePath) { + if actual.Path != c.resolvePath { t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath) } - entry, err := img.FileCatalog.Get(actual) + entry, err := img.FileCatalog.Get(actual.ref) if err != nil { t.Fatalf("failed to get metadata: %+v", err) } @@ -172,11 +170,11 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) { actual := refs[0] - if actual.Path != file.Path(c.resolvePath) { + if actual.Path != c.resolvePath { t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath) } - entry, err := img.FileCatalog.Get(actual) + entry, err := img.FileCatalog.Get(actual.ref) if err != nil { t.Fatalf("failed to get metadata: %+v", err) } diff --git a/syft/source/location.go b/syft/source/location.go index 1bc5e5ce0..f9287d0ab 100644 --- a/syft/source/location.go +++ b/syft/source/location.go @@ -14,20 +14,13 @@ type Location struct { ref file.Reference } -func newLocation(path string) Location { +func NewLocation(path string) Location { return Location{ Path: path, } } -func newLocationFromRef(ref file.Reference) Location { - return Location{ - Path: string(ref.Path), - ref: ref, - } -} - -func newLocationFromImage(ref file.Reference, img *image.Image) Location { +func NewLocationFromImage(ref file.Reference, img *image.Image) Location { entry, err := img.FileCatalog.Get(ref) if err != nil { log.Warnf("unable to find file catalog entry for ref=%+v", ref) diff --git a/syft/source/resolver.go b/syft/source/resolver.go index 2bab3bc11..d6193c4c3 100644 --- a/syft/source/resolver.go +++ b/syft/source/resolver.go @@ -14,8 +14,8 @@ type Resolver interface { // ContentResolver knows how to get file content for given file.References type ContentResolver interface { - FileContentsByRef(Location) (string, error) - MultipleFileContentsByRef([]Location) (map[Location]string, error) + FileContentsByLocation(Location) (string, error) + MultipleFileContentsByLocation([]Location) (map[Location]string, error) // TODO: we should consider refactoring to return a set of io.Readers or file.Openers instead of the full contents themselves (allow for optional buffering). } diff --git a/syft/source/scope.go b/syft/source/scope.go index d870da2da..0df5d7f23 100644 --- a/syft/source/scope.go +++ b/syft/source/scope.go @@ -21,7 +21,7 @@ var Options = []Scope{ AllLayersScope, } -func ParseOption(userStr string) Scope { +func ParseScope(userStr string) Scope { switch strings.ToLower(userStr) { case strings.ToLower(SquashedScope.String()): return SquashedScope diff --git a/syft/source/source_test.go b/syft/source/source_test.go index 27668b1cf..c96aadfaf 100644 --- a/syft/source/source_test.go +++ b/syft/source/source_test.go @@ -4,7 +4,6 @@ import ( "os" "testing" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/image" "github.com/mitchellh/go-homedir" "github.com/spf13/afero" @@ -36,10 +35,10 @@ func TestNewScopeFromImage(t *testing.T) { Layers: []*image.Layer{layer}, } - t.Run("create a new Source object from image", func(t *testing.T) { + t.Run("create a new Locations object from image", func(t *testing.T) { _, err := NewFromImage(&img, AllLayersScope) if err != nil { - t.Errorf("unexpected error when creating a new Source from img: %w", err) + t.Errorf("unexpected error when creating a new Locations from img: %+v", err) } }) } @@ -49,31 +48,31 @@ func TestDirectoryScope(t *testing.T) { desc string input string expString string - inputPaths []file.Path + inputPaths []string expRefs int }{ { desc: "no paths exist", input: "foobar/", - inputPaths: []file.Path{file.Path("/opt/"), file.Path("/other")}, + inputPaths: []string{"/opt/", "/other"}, expRefs: 0, }, { desc: "path detected", input: "test-fixtures", - inputPaths: []file.Path{file.Path("test-fixtures/path-detected/.vimrc")}, + inputPaths: []string{"test-fixtures/path-detected/.vimrc"}, expRefs: 1, }, { desc: "directory ignored", input: "test-fixtures", - inputPaths: []file.Path{file.Path("test-fixtures/path-detected")}, + inputPaths: []string{"test-fixtures/path-detected"}, expRefs: 0, }, { desc: "no files-by-path detected", input: "test-fixtures", - inputPaths: []file.Path{file.Path("test-fixtures/no-path-detected")}, + inputPaths: []string{"test-fixtures/no-path-detected"}, expRefs: 0, }, } @@ -82,7 +81,7 @@ func TestDirectoryScope(t *testing.T) { p, err := NewFromDirectory(test.input) if err != nil { - t.Errorf("could not create NewDirScope: %w", err) + t.Errorf("could not create NewDirScope: %+v", err) } if p.Target.(DirSource).Path != test.input { t.Errorf("mismatched stringer: '%s' != '%s'", p.Target.(DirSource).Path, test.input) @@ -90,7 +89,7 @@ func TestDirectoryScope(t *testing.T) { refs, err := p.Resolver.FilesByPath(test.inputPaths...) if err != nil { - t.Errorf("FilesByPath call produced an error: %w", err) + t.Errorf("FilesByPath call produced an error: %+v", err) } if len(refs) != test.expRefs { t.Errorf("unexpected number of refs returned: %d != %d", len(refs), test.expRefs) @@ -125,20 +124,19 @@ func TestMultipleFileContentsByRefContents(t *testing.T) { t.Run(test.desc, func(t *testing.T) { p, err := NewFromDirectory(test.input) if err != nil { - t.Errorf("could not create NewDirScope: %w", err) + t.Errorf("could not create NewDirScope: %+v", err) } - refs, err := p.Resolver.FilesByPath(file.Path(test.path)) + locations, err := p.Resolver.FilesByPath(test.path) if err != nil { t.Errorf("could not get file references from path: %s, %v", test.path, err) } - if len(refs) != 1 { - t.Fatalf("expected a single ref to be generated but got: %d", len(refs)) + if len(locations) != 1 { + t.Fatalf("expected a single location to be generated but got: %d", len(locations)) } - ref := refs[0] + location := locations[0] - contents, err := p.Resolver.MultipleFileContentsByRef(ref) - content := contents[ref] + content, err := p.Resolver.FileContentsByLocation(location) if content != test.expected { t.Errorf("unexpected contents from file: '%s' != '%s'", content, test.expected) @@ -165,9 +163,9 @@ func TestMultipleFileContentsByRefNoContents(t *testing.T) { t.Run(test.desc, func(t *testing.T) { p, err := NewFromDirectory(test.input) if err != nil { - t.Errorf("could not create NewDirScope: %w", err) + t.Errorf("could not create NewDirScope: %+v", err) } - refs, err := p.Resolver.FilesByPath(file.Path(test.path)) + refs, err := p.Resolver.FilesByPath(test.path) if err != nil { t.Errorf("could not get file references from path: %s, %v", test.path, err) } @@ -210,7 +208,7 @@ func TestFilesByGlob(t *testing.T) { t.Run(test.desc, func(t *testing.T) { p, err := NewFromDirectory(test.input) if err != nil { - t.Errorf("could not create NewDirScope: %w", err) + t.Errorf("could not create NewDirScope: %+v", err) } contents, err := p.Resolver.FilesByGlob(test.glob) diff --git a/test/integration/document_import_test.go b/test/integration/document_import_test.go index 946421f18..438cce148 100644 --- a/test/integration/document_import_test.go +++ b/test/integration/document_import_test.go @@ -73,7 +73,7 @@ func TestCatalogFromJSON(t *testing.T) { a := actualPackages[i] // omit fields that should be missing - e.Source = nil + e.Locations = nil e.FoundBy = "" if e.MetadataType == pkg.JavaMetadataType { metadata := e.Metadata.(pkg.JavaMetadata) From 6f7a4fd3e4e7beeb6f989c4b51e4c6b46d7a1784 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 13 Nov 2020 16:30:31 -0500 Subject: [PATCH 05/13] move source metadata upstream and fix tests Signed-off-by: Alex Goodman --- cmd/cmd.go | 2 +- cmd/root.go | 2 +- schema/json/schema.json | 55 +++++++----------- syft/cataloger/deb/cataloger_test.go | 4 +- syft/cataloger/rpmdb/cataloger.go | 5 +- syft/cataloger/rpmdb/parse_rpmdb.go | 1 + syft/cataloger/rpmdb/parse_rpmdb_test.go | 10 ++-- syft/lib.go | 6 +- syft/pkg/dpkg_metadata.go | 2 +- syft/pkg/package.go | 18 +++--- syft/presenter/cyclonedx/presenter.go | 37 +++++------- syft/presenter/cyclonedx/presenter_test.go | 8 +-- syft/presenter/json/artifact.go | 20 ++++--- syft/presenter/json/document.go | 21 +++---- syft/presenter/json/image.go | 44 -------------- syft/presenter/json/presenter.go | 16 ++--- syft/presenter/json/presenter_test.go | 10 ++-- syft/presenter/json/source.go | 12 ++-- .../snapshot/TestJsonDirsPresenter.golden | 43 ++++++++++++-- .../snapshot/TestJsonImgsPresenter.golden | 47 ++++++++++++--- .../anchore-fixture-image-simple.golden | Bin 23552 -> 0 bytes .../stereoscope-fixture-image-simple.golden | Bin 22016 -> 23552 bytes syft/presenter/presenter.go | 10 ++-- syft/presenter/table/presenter.go | 5 +- syft/presenter/table/presenter_test.go | 5 +- syft/presenter/text/presenter.go | 28 ++++----- syft/presenter/text/presenter_test.go | 6 +- syft/source/image_metadata.go | 44 ++++++++++++++ syft/source/location.go | 14 ++--- syft/source/metadata.go | 8 +++ syft/source/scope.go | 24 +++----- syft/source/scope_test.go | 17 ------ syft/source/source.go | 40 +++++-------- syft/source/source_test.go | 29 ++++----- test/integration/document_import_test.go | 4 +- test/integration/json_schema_test.go | 2 +- 36 files changed, 305 insertions(+), 294 deletions(-) delete mode 100644 syft/presenter/json/image.go delete mode 100644 syft/presenter/json/test-fixtures/snapshot/anchore-fixture-image-simple.golden create mode 100644 syft/source/image_metadata.go create mode 100644 syft/source/metadata.go delete mode 100644 syft/source/scope_test.go diff --git a/cmd/cmd.go b/cmd/cmd.go index 0c4a36327..70197c968 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -50,7 +50,7 @@ func setGlobalCliOptions() { flag := "scope" rootCmd.Flags().StringP( "scope", "s", source.SquashedScope.String(), - fmt.Sprintf("selection of layers to catalog, options=%v", source.Options)) + fmt.Sprintf("selection of layers to catalog, options=%v", source.AllScopes)) if err := viper.BindPFlag(flag, rootCmd.Flags().Lookup(flag)); err != nil { fmt.Printf("unable to bind flag '%s': %+v", flag, err) os.Exit(1) diff --git a/cmd/root.go b/cmd/root.go index ffa936f90..3abccd404 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -100,7 +100,7 @@ func startWorker(userInput string) <-chan error { bus.Publish(partybus.Event{ Type: event.CatalogerFinished, - Value: presenter.GetPresenter(appConfig.PresenterOpt, *scope, catalog, distro), + Value: presenter.GetPresenter(appConfig.PresenterOpt, scope.Metadata, catalog, distro), }) }() return errs diff --git a/schema/json/schema.json b/schema/json/schema.json index 556fd50e7..01f9e3e10 100644 --- a/schema/json/schema.json +++ b/schema/json/schema.json @@ -5,10 +5,7 @@ "items": { "properties": { "foundBy": { - "items": { - "type": "string" - }, - "type": "array" + "type": "string" }, "language": { "type": "string" @@ -27,33 +24,21 @@ ] }, "locations": { - "anyOf": [ - { - "type": "null" - }, - { - "items": { - "properties": { - "layerID": { - "type": "string" - }, - "layerIndex": { - "type": "integer" - }, - "path": { - "type": "string" - } - }, - "required": [ - "layerID", - "layerIndex", - "path" - ], - "type": "object" + "items": { + "properties": { + "layerID": { + "type": "string" }, - "type": "array" - } - ] + "path": { + "type": "string" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "type": "array" }, "metadata": { "properties": { @@ -354,7 +339,7 @@ "name": { "type": "string" }, - "reportTimestamp": { + "scope": { "type": "string" }, "version": { @@ -363,7 +348,7 @@ }, "required": [ "name", - "reportTimestamp", + "scope", "version" ], "type": "object" @@ -432,6 +417,9 @@ "type": "string" }, "type": "array" + }, + "userInput": { + "type": "string" } }, "required": [ @@ -439,7 +427,8 @@ "layers", "mediaType", "size", - "tags" + "tags", + "userInput" ], "type": "object" } diff --git a/syft/cataloger/deb/cataloger_test.go b/syft/cataloger/deb/cataloger_test.go index d340316ed..9a3e390c7 100644 --- a/syft/cataloger/deb/cataloger_test.go +++ b/syft/cataloger/deb/cataloger_test.go @@ -54,7 +54,7 @@ func TestDpkgCataloger(t *testing.T) { img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-dpkg") defer cleanup() - s, err := source.NewFromImage(img, source.AllLayersScope) + s, err := source.NewFromImage(img, source.AllLayersScope, "") if err != nil { t.Fatal(err) } @@ -79,7 +79,7 @@ func TestDpkgCataloger(t *testing.T) { // we will test the sources separately var sourcesList = make([]string, len(a.Locations)) for i, s := range a.Locations { - sourcesList[i] = string(s.Path) + sourcesList[i] = s.Path } a.Locations = nil diff --git a/syft/cataloger/rpmdb/cataloger.go b/syft/cataloger/rpmdb/cataloger.go index 3ec99e112..c9234a1c7 100644 --- a/syft/cataloger/rpmdb/cataloger.go +++ b/syft/cataloger/rpmdb/cataloger.go @@ -12,7 +12,8 @@ import ( ) const ( - packagesGlob = "**/var/lib/rpm/Packages" + packagesGlob = "**/var/lib/rpm/Packages" + catalogerName = "rpmdb-cataloger" ) type Cataloger struct{} @@ -24,7 +25,7 @@ func NewRpmdbCataloger() *Cataloger { // Name returns a string that uniquely describes a cataloger func (c *Cataloger) Name() string { - return "rpmdb-cataloger" + return catalogerName } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation. diff --git a/syft/cataloger/rpmdb/parse_rpmdb.go b/syft/cataloger/rpmdb/parse_rpmdb.go index 05b6ec157..fcf251cd0 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb.go +++ b/syft/cataloger/rpmdb/parse_rpmdb.go @@ -55,6 +55,7 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader Version: fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does //Version: fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch), Locations: []source.Location{dbLocation}, + FoundBy: catalogerName, Type: pkg.RpmPkg, MetadataType: pkg.RpmdbMetadataType, Metadata: pkg.RpmdbMetadata{ diff --git a/syft/cataloger/rpmdb/parse_rpmdb_test.go b/syft/cataloger/rpmdb/parse_rpmdb_test.go index 2b89e7dff..a80fa2523 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb_test.go +++ b/syft/cataloger/rpmdb/parse_rpmdb_test.go @@ -43,7 +43,7 @@ func (r *rpmdbTestFileResolverMock) RelativeFileByPath(source.Location, string) } func TestParseRpmDB(t *testing.T) { - dbRef := file.NewFileReference("test-path") + dbLocation := source.NewLocation("test-path") tests := []struct { fixture string @@ -58,7 +58,8 @@ func TestParseRpmDB(t *testing.T) { "dive": { Name: "dive", Version: "0.9.2-1", - Source: []file.Reference{dbRef}, + Locations: []source.Location{dbLocation}, + FoundBy: catalogerName, Type: pkg.RpmPkg, MetadataType: pkg.RpmdbMetadataType, Metadata: pkg.RpmdbMetadata{ @@ -84,7 +85,8 @@ func TestParseRpmDB(t *testing.T) { "dive": { Name: "dive", Version: "0.9.2-1", - Source: []file.Reference{dbRef}, + Locations: []source.Location{dbLocation}, + FoundBy: catalogerName, Type: pkg.RpmPkg, MetadataType: pkg.RpmdbMetadataType, Metadata: pkg.RpmdbMetadata{ @@ -120,7 +122,7 @@ func TestParseRpmDB(t *testing.T) { fileResolver := newTestFileResolver(test.ignorePaths) - actual, err := parseRpmDB(fileResolver, dbRef, fixture) + actual, err := parseRpmDB(fileResolver, dbLocation, fixture) if err != nil { t.Fatalf("failed to parse rpmdb: %+v", err) } diff --git a/syft/lib.go b/syft/lib.go index 930669371..9a1d81675 100644 --- a/syft/lib.go +++ b/syft/lib.go @@ -36,7 +36,7 @@ import ( // set of packages, the identified Linux distribution, and the source object used to wrap the data source. func Catalog(userInput string, scoptOpt source.Scope) (*pkg.Catalog, *source.Source, *distro.Distro, error) { log.Info("cataloging image") - s, cleanup, err := source.NewSource(userInput, scoptOpt) + s, cleanup, err := source.New(userInput, scoptOpt) defer cleanup() if err != nil { return nil, nil, nil, err @@ -70,13 +70,13 @@ func CatalogFromScope(s source.Source) (*pkg.Catalog, error) { // conditionally have two sets of catalogers var catalogers []cataloger.Cataloger - switch s.Scheme { + switch s.Metadata.Scheme { case source.ImageScheme: catalogers = cataloger.ImageCatalogers() case source.DirectoryScheme: catalogers = cataloger.DirectoryCatalogers() default: - return nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", s.Scheme) + return nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", s.Metadata.Scheme) } return cataloger.Catalog(s.Resolver, catalogers...) diff --git a/syft/pkg/dpkg_metadata.go b/syft/pkg/dpkg_metadata.go index a29bf8169..f1e80802e 100644 --- a/syft/pkg/dpkg_metadata.go +++ b/syft/pkg/dpkg_metadata.go @@ -9,7 +9,7 @@ import ( // at http://manpages.ubuntu.com/manpages/xenial/man1/dpkg-query.1.html in the --showformat section. type DpkgMetadata struct { Package string `mapstructure:"Package" json:"package"` - Source string `mapstructure:"Locations" json:"source"` + Source string `mapstructure:"Source" json:"source"` Version string `mapstructure:"Version" json:"version"` Architecture string `mapstructure:"Architecture" json:"architecture"` Maintainer string `mapstructure:"Maintainer" json:"maintainer"` diff --git a/syft/pkg/package.go b/syft/pkg/package.go index 525e5e893..7e97ee8a8 100644 --- a/syft/pkg/package.go +++ b/syft/pkg/package.go @@ -19,16 +19,16 @@ type ID int64 // Package represents an application or library that has been bundled into a distributable format. type Package struct { id ID // uniquely identifies a package, set by the cataloger - Name string `json:"manifest"` // the package name - Version string `json:"version"` // the version of the package - FoundBy string `json:"foundBy"` // the specific cataloger that discovered this package - Locations []source.Location `json:"-"` // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package) + Name string // the package name + Version string // the version of the package + FoundBy string // the specific cataloger that discovered this package + Locations []source.Location // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package) // TODO: should we move licenses into metadata? - Licenses []string `json:"licenses"` // licenses discovered with the package metadata - Language Language `json:"language"` // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc) - Type Type `json:"type"` // the package type (e.g. Npm, Yarn, Python, Rpm, Deb, etc) - MetadataType MetadataType `json:"metadataType,omitempty"` // the shape of the additional data in the "metadata" field - Metadata interface{} `json:"metadata,omitempty"` // additional data found while parsing the package source + Licenses []string // licenses discovered with the package metadata + Language Language // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc) + Type Type // the package type (e.g. Npm, Yarn, Python, Rpm, Deb, etc) + MetadataType MetadataType // the shape of the additional data in the "metadata" field + Metadata interface{} // additional data found while parsing the package source } // ID returns the package ID, which is unique relative to a package catalog. diff --git a/syft/presenter/cyclonedx/presenter.go b/syft/presenter/cyclonedx/presenter.go index b73052d51..2a28b8db7 100644 --- a/syft/presenter/cyclonedx/presenter.go +++ b/syft/presenter/cyclonedx/presenter.go @@ -16,17 +16,17 @@ import ( // Presenter writes a CycloneDX report from the given Catalog and Locations contents type Presenter struct { - catalog *pkg.Catalog - source source.Source - distro distro.Distro + catalog *pkg.Catalog + srcMetadata source.Metadata + distro distro.Distro } // NewPresenter creates a CycloneDX presenter from the given Catalog and Locations objects. -func NewPresenter(catalog *pkg.Catalog, s source.Source, d distro.Distro) *Presenter { +func NewPresenter(catalog *pkg.Catalog, s source.Metadata, d distro.Distro) *Presenter { return &Presenter{ - catalog: catalog, - source: s, - distro: d, + catalog: catalog, + srcMetadata: s, + distro: d, } } @@ -34,33 +34,26 @@ func NewPresenter(catalog *pkg.Catalog, s source.Source, d distro.Distro) *Prese func (pres *Presenter) Present(output io.Writer) error { bom := NewDocumentFromCatalog(pres.catalog, pres.distro) - switch src := pres.source.Target.(type) { - case source.DirSource: + switch pres.srcMetadata.Scheme { + case source.DirectoryScheme: bom.BomDescriptor.Component = &BdComponent{ Component: Component{ Type: "file", - Name: src.Path, + Name: pres.srcMetadata.Path, Version: "", }, } - case source.ImageSource: - var imageID string - var versionStr string - if len(src.Img.Metadata.Tags) > 0 { - imageID = src.Img.Metadata.Tags[0].Context().Name() - versionStr = src.Img.Metadata.Tags[0].TagStr() - } else { - imageID = src.Img.Metadata.Digest - } + case source.ImageScheme: + // TODO: can we use the tags a bit better? bom.BomDescriptor.Component = &BdComponent{ Component: Component{ Type: "container", - Name: imageID, - Version: versionStr, + Name: pres.srcMetadata.ImageMetadata.UserInput, + Version: pres.srcMetadata.ImageMetadata.Digest, }, } default: - return fmt.Errorf("unsupported source: %T", src) + return fmt.Errorf("unsupported source: %T", pres.srcMetadata.Scheme) } encoder := xml.NewEncoder(output) diff --git a/syft/presenter/cyclonedx/presenter_test.go b/syft/presenter/cyclonedx/presenter_test.go index 90ca5b309..ee29ca62c 100644 --- a/syft/presenter/cyclonedx/presenter_test.go +++ b/syft/presenter/cyclonedx/presenter_test.go @@ -66,7 +66,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) { t.Fatal(err) } - pres := NewPresenter(catalog, s, d) + pres := NewPresenter(catalog, s.Metadata, d) // run presenter err = pres.Present(&buffer) @@ -146,7 +146,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) { }, }) - s, err := source.NewFromImage(img, source.AllLayersScope) + s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input") if err != nil { t.Fatal(err) } @@ -156,7 +156,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) { t.Fatal(err) } - pres := NewPresenter(catalog, s, d) + pres := NewPresenter(catalog, s.Metadata, d) // run presenter err = pres.Present(&buffer) @@ -177,7 +177,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) { if !bytes.Equal(expected, actual) { dmp := diffmatchpatch.New() - diffs := dmp.DiffMain(string(actual), string(expected), true) + diffs := dmp.DiffMain(string(expected), string(actual), true) t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs)) } } diff --git a/syft/presenter/json/artifact.go b/syft/presenter/json/artifact.go index 8a9e586d7..3aeea21ef 100644 --- a/syft/presenter/json/artifact.go +++ b/syft/presenter/json/artifact.go @@ -16,11 +16,11 @@ type Artifact struct { type ArtifactBasicMetadata struct { Name string `json:"name"` Version string `json:"version"` - Type string `json:"type"` - FoundBy []string `json:"foundBy"` + Type pkg.Type `json:"type"` + FoundBy string `json:"foundBy"` Locations []source.Location `json:"locations"` Licenses []string `json:"licenses"` - Language string `json:"language"` + Language pkg.Language `json:"language"` } type ArtifactCustomMetadata struct { @@ -33,17 +33,17 @@ type ArtifactMetadataUnpacker struct { Metadata json.RawMessage `json:"metadata"` } -func NewArtifact(p *pkg.Package, s source.Source) (Artifact, error) { +func NewArtifact(p *pkg.Package) (Artifact, error) { return Artifact{ ArtifactBasicMetadata: ArtifactBasicMetadata{ Name: p.Name, Version: p.Version, - Type: string(p.Type), - FoundBy: []string{p.FoundBy}, + Type: p.Type, + FoundBy: p.FoundBy, Locations: p.Locations, Licenses: p.Licenses, - Language: string(p.Language), + Language: p.Language, }, ArtifactCustomMetadata: ArtifactCustomMetadata{ MetadataType: p.MetadataType, @@ -57,9 +57,11 @@ func (a Artifact) ToPackage() pkg.Package { // does not include found-by and locations Name: a.Name, Version: a.Version, + FoundBy: a.FoundBy, Licenses: a.Licenses, - Language: pkg.Language(a.Language), - Type: pkg.Type(a.Type), + Language: a.Language, + Locations: a.Locations, + Type: a.Type, MetadataType: a.MetadataType, Metadata: a.Metadata, } diff --git a/syft/presenter/json/document.go b/syft/presenter/json/document.go index 78cb55ce2..2c20a1a38 100644 --- a/syft/presenter/json/document.go +++ b/syft/presenter/json/document.go @@ -1,8 +1,6 @@ package json import ( - "time" - "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/version" "github.com/anchore/syft/syft/distro" @@ -19,10 +17,9 @@ type Document struct { // Descriptor describes what created the document as well as surrounding metadata type Descriptor struct { - Name string `json:"name"` - Version string `json:"version"` - ReportTimestamp string `json:"reportTimestamp"` - // TODO: we should include source option here as well (or in source) + Name string `json:"name"` + Version string `json:"version"` + Scope string `json:"scope"` } // Distribution provides information about a detected Linux Distribution @@ -32,8 +29,8 @@ type Distribution struct { IDLike string `json:"idLike"` } -func NewDocument(catalog *pkg.Catalog, s source.Source, d distro.Distro) (Document, error) { - src, err := NewSource(s) +func NewDocument(catalog *pkg.Catalog, srcMetadata source.Metadata, d distro.Distro) (Document, error) { + src, err := NewSource(srcMetadata) if err != nil { return Document{}, nil } @@ -52,14 +49,14 @@ func NewDocument(catalog *pkg.Catalog, s source.Source, d distro.Distro) (Docume IDLike: d.IDLike, }, Descriptor: Descriptor{ - Name: internal.ApplicationName, - Version: version.FromBuild().Version, - ReportTimestamp: time.Now().Format(time.RFC3339), + Name: internal.ApplicationName, + Version: version.FromBuild().Version, + Scope: srcMetadata.Scope.String(), }, } for _, p := range catalog.Sorted() { - art, err := NewArtifact(p, s) + art, err := NewArtifact(p) if err != nil { return Document{}, err } diff --git a/syft/presenter/json/image.go b/syft/presenter/json/image.go deleted file mode 100644 index ee7796421..000000000 --- a/syft/presenter/json/image.go +++ /dev/null @@ -1,44 +0,0 @@ -package json - -import ( - "github.com/anchore/syft/syft/source" -) - -type Image struct { - Layers []Layer `json:"layers"` - Size int64 `json:"size"` - Digest string `json:"digest"` - MediaType string `json:"mediaType"` - Tags []string `json:"tags"` -} - -type Layer struct { - MediaType string `json:"mediaType"` - Digest string `json:"digest"` - Size int64 `json:"size"` -} - -func NewImage(src source.ImageSource) *Image { - // populate artifacts... - tags := make([]string, len(src.Img.Metadata.Tags)) - for idx, tag := range src.Img.Metadata.Tags { - tags[idx] = tag.String() - } - img := Image{ - Digest: src.Img.Metadata.Digest, - Size: src.Img.Metadata.Size, - MediaType: string(src.Img.Metadata.MediaType), - Tags: tags, - Layers: make([]Layer, len(src.Img.Layers)), - } - - // populate image metadata - for idx, l := range src.Img.Layers { - img.Layers[idx] = Layer{ - MediaType: string(l.Metadata.MediaType), - Digest: l.Metadata.Digest, - Size: l.Metadata.Size, - } - } - return &img -} diff --git a/syft/presenter/json/presenter.go b/syft/presenter/json/presenter.go index beb2e689b..00af4ef4e 100644 --- a/syft/presenter/json/presenter.go +++ b/syft/presenter/json/presenter.go @@ -10,21 +10,21 @@ import ( ) type Presenter struct { - catalog *pkg.Catalog - source source.Source - distro distro.Distro + catalog *pkg.Catalog + srcMetadata source.Metadata + distro distro.Distro } -func NewPresenter(catalog *pkg.Catalog, s source.Source, d distro.Distro) *Presenter { +func NewPresenter(catalog *pkg.Catalog, s source.Metadata, d distro.Distro) *Presenter { return &Presenter{ - catalog: catalog, - source: s, - distro: d, + catalog: catalog, + srcMetadata: s, + distro: d, } } func (pres *Presenter) Present(output io.Writer) error { - doc, err := NewDocument(pres.catalog, pres.source, pres.distro) + doc, err := NewDocument(pres.catalog, pres.srcMetadata, pres.distro) if err != nil { return err } diff --git a/syft/presenter/json/presenter_test.go b/syft/presenter/json/presenter_test.go index 94d052dae..57869e08e 100644 --- a/syft/presenter/json/presenter_test.go +++ b/syft/presenter/json/presenter_test.go @@ -56,7 +56,7 @@ func TestJsonDirsPresenter(t *testing.T) { if err != nil { t.Fatal(err) } - pres := NewPresenter(catalog, s, d) + pres := NewPresenter(catalog, s.Metadata, d) // run presenter err = pres.Present(&buffer) @@ -73,7 +73,7 @@ func TestJsonDirsPresenter(t *testing.T) { if !bytes.Equal(expected, actual) { dmp := diffmatchpatch.New() - diffs := dmp.DiffMain(string(actual), string(expected), true) + diffs := dmp.DiffMain(string(expected), string(actual), true) t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs)) } @@ -123,9 +123,9 @@ func TestJsonImgsPresenter(t *testing.T) { }, }) - s, err := source.NewFromImage(img, source.AllLayersScope) + s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input") d := distro.NewUnknownDistro() - pres := NewPresenter(catalog, s, d) + pres := NewPresenter(catalog, s.Metadata, d) // run presenter err = pres.Present(&buffer) @@ -142,7 +142,7 @@ func TestJsonImgsPresenter(t *testing.T) { if !bytes.Equal(expected, actual) { dmp := diffmatchpatch.New() - diffs := dmp.DiffMain(string(actual), string(expected), true) + diffs := dmp.DiffMain(string(expected), string(actual), true) t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs)) } } diff --git a/syft/presenter/json/source.go b/syft/presenter/json/source.go index 6172f9bb4..14561a24f 100644 --- a/syft/presenter/json/source.go +++ b/syft/presenter/json/source.go @@ -17,14 +17,14 @@ type SourceUnpacker struct { Target json.RawMessage `json:"target"` } -func NewSource(s source.Source) (Source, error) { - switch src := s.Target.(type) { - case source.ImageSource: +func NewSource(src source.Metadata) (Source, error) { + switch src.Scheme { + case source.ImageScheme: return Source{ Type: "image", - Target: NewImage(src), + Target: src.ImageMetadata, }, nil - case source.DirSource: + case source.DirectoryScheme: return Source{ Type: "directory", Target: src.Path, @@ -44,7 +44,7 @@ func (s *Source) UnmarshalJSON(b []byte) error { switch s.Type { case "image": - var payload Image + var payload source.ImageMetadata if err := json.Unmarshal(unpacker.Target, &payload); err != nil { return err } diff --git a/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden b/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden index 2193e7875..7324c5349 100644 --- a/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden +++ b/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden @@ -3,14 +3,29 @@ { "name": "package-1", "version": "1.0.1", - "type": "deb", + "type": "python", "foundBy": [ "the-cataloger-1" ], "locations": [ - "/some/path/pkg1" + { + "path": "/some/path/pkg1" + } ], - "licenses": null + "licenses": [ + "MIT" + ], + "language": "python", + "metadataType": "python-package-metadata", + "metadata": { + "name": "package-1", + "version": "1.0.1", + "license": "", + "author": "", + "authorEmail": "", + "platform": "", + "sitePackagesRootPath": "" + } }, { "name": "package-2", @@ -20,9 +35,22 @@ "the-cataloger-2" ], "locations": [ - "/some/path/pkg1" + { + "path": "/some/path/pkg1" + } ], - "licenses": null + "licenses": null, + "language": "", + "metadataType": "dpkg-metadata", + "metadata": { + "package": "package-2", + "source": "", + "version": "2.0.1", + "architecture": "", + "maintainer": "", + "installedSize": 0, + "files": null + } } ], "source": { @@ -33,5 +61,10 @@ "name": "", "version": "", "idLike": "" + }, + "descriptor": { + "name": "syft", + "version": "[not provided]", + "scope": "" } } diff --git a/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden b/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden index f3c7f56a2..91b28d91e 100644 --- a/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden +++ b/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden @@ -3,17 +3,30 @@ { "name": "package-1", "version": "1.0.1", - "type": "deb", + "type": "python", "foundBy": [ "the-cataloger-1" ], "locations": [ { "path": "/somefile-1.txt", - "layerIndex": 0 + "layerID": "sha256:e158b57d6f5a96ef5fd22f2fe76c70b5ba6ff5b2619f9d83125b2aad0492ac7b" } ], - "licenses": null + "licenses": [ + "MIT" + ], + "language": "python", + "metadataType": "python-package-metadata", + "metadata": { + "name": "package-1", + "version": "1.0.1", + "license": "", + "author": "", + "authorEmail": "", + "platform": "", + "sitePackagesRootPath": "" + } }, { "name": "package-2", @@ -25,10 +38,21 @@ "locations": [ { "path": "/somefile-2.txt", - "layerIndex": 1 + "layerID": "sha256:da21056e7bf4308ecea0c0836848a7fe92f38fdcf35bc09ee6d98e7ab7beeebf" } ], - "licenses": null + "licenses": null, + "language": "", + "metadataType": "dpkg-metadata", + "metadata": { + "package": "package-2", + "source": "", + "version": "2.0.1", + "architecture": "", + "maintainer": "", + "installedSize": 0, + "files": null + } } ], "source": { @@ -37,22 +61,22 @@ "layers": [ { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "digest": "sha256:78783bfc74fef84f899b4977561ad1172f87753f82cc2157b06bf097e56dfbce", + "digest": "sha256:e158b57d6f5a96ef5fd22f2fe76c70b5ba6ff5b2619f9d83125b2aad0492ac7b", "size": 22 }, { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "digest": "sha256:54ec7f643dafbf9f27032a5e60afe06248c0e99b50aed54bb0fe28ea4825ccaf", + "digest": "sha256:da21056e7bf4308ecea0c0836848a7fe92f38fdcf35bc09ee6d98e7ab7beeebf", "size": 16 }, { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "digest": "sha256:ec4775a139c45b1ddf9ea8e1cb43385e92e5c0bf6ec2e3f4192372785b18c106", + "digest": "sha256:f0e18aa6032c24659a9c741fc36ca56f589782ea132061ccf6f52b952403da94", "size": 27 } ], "size": 65, - "digest": "sha256:fedd7bcc0b90f071501b662d8e7c9ac7548b88daba6b3deedfdf33f22ed8d95b", + "digest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368", "mediaType": "application/vnd.docker.distribution.manifest.v2+json", "tags": [ "stereoscope-fixture-image-simple:04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7" @@ -63,5 +87,10 @@ "name": "", "version": "", "idLike": "" + }, + "descriptor": { + "name": "syft", + "version": "[not provided]", + "scope": "AllLayers" } } diff --git a/syft/presenter/json/test-fixtures/snapshot/anchore-fixture-image-simple.golden b/syft/presenter/json/test-fixtures/snapshot/anchore-fixture-image-simple.golden deleted file mode 100644 index 3e1d2daaaed20003cb49b3a526de3f0e8cc9713f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23552 zcmeHPTW{M&7WQ+0h01-|1#J1ueMZ1OWSamjP;3^h+X5Q|!CWJ5WztL1Z4ATzeGesB zz9g|^Qxcrb2(rb?%y9UfbI5ZY(AIDxJXo$lO94&>WuwqaCT$4=p;PdWP*+u)Rjj&%R# zhK{$V{xFpOzrOnE=O14G^P%Yi0g#;Er~CCjBFFC1pMv6WFd9sghbDD>?klH18Se$< zYqaz4Gecmo{Qsw|t2OtuPW}}W_vK&W13An88eF$=`m!iPF)4p z04DSAzxxJt)!DL&%{$ym<@;iCS$to&U1d)r_QDn~>r-rSv;RvQo5h(IU1$CD=Ps00 zF-dmT#=~NDtUkwExPLG<*mZgEq`O#czdnoA-Ero7uPcv}Shjyv{ZUM+^V#gM`0f<% zXtDXjseQkk=&$wc{4}=LXMU@j-^=R#%W^r}S5?L2IjY&F!6*2USf z`_*Al<7P1{tMd=V)dmbQl<>zhdl8#)uS0$Ivqt{;NBSQnf`}~tp9OS_C))Znrg9dK z=(zjPZE@gRL#?4d0A1++geobhME##A!884DLAbNc1?@-u^Aw-n?YBF-P#!oVkP*lT zWCWfS0#q?V1qq&^7m#>}0+RO$yqZ0#E>4AH60s4SPnAGFp_89)&LL)WOUJkqgsUT?dn`-8qluB_Hg%Zl9 z=oEuMxJaC83(EIEZrQ&3!YtoyQs_-_j08Jwr-;h%TXwsi8oAn zX{`W_E`~_80OQc{Yza*|NH0}jJQ`^dQjT!+P#6|1Q9(%qNU%{^jgCh>LtLBZ&zB2U z4ibvi(-D{>%1#KG2s+_xOemL9>fTWjYX3WWN&bjUTh{3BK!?g0`WAR@TDD!?pgZHu zlP#{<%_M!qIg&OI-SE5CEt@0uQ?BR{%Di{7S~SkLgyL0J$aY9I+{GL#d_Vqjnl zM8^`O=!r8i@B|K|r#x!yjAzn_IWbn&De0}#QE_#+Zl(rfSVo)2%rqqC6z55BYaEfz zpbrYC1d&CfgYlL+Zj!J{1(Phwea|jIIh%ymn!??g&h6;+GrdEkrMv4POt;A2SIFSE zyhMjZQ`cQWx6x(MU7YprfVoen?YXB+>0KG{+?!%43z4AXPN*Oe5hG&~DNvedib{F~ zPB<%55{^n@k_o!kE*!Ns0ICw!D~F>lM?&r(AY@c7aRxg2Xu*RWrWq zSitya@%CyHU)et1$NRp92Fm-FnQ$av{M8_Sl%e-lVK6vK8F?T&c&(70!6UxWGD3`| z7Mv#5#2v+dB}4yzj{okS8wS&VH}PM5eP#WB_tZZ)$noEOp!|HVFRxcj6Oy$jJR7@&mTX z7k3j3o~;MtGj7LQhUwR8lI>Ml82aGfLjH&PME(coS^wV)AD}mHHoMrwihufaHqHOO z$^QV+=YM4T|LK@-+H*<&@t@Q2{I5Ry@1|WhZr7*Augm}M+blnr|3i7s|JfMcK@FGm zuVRb&mOFY3JYN1OFns=J`5#2ujXK|@|2!LvZ95ZiZT=ry)PLMO|C!`D{=X6ZhBmTQ z-A!XN>M8&2H-DDhbcD&nu^o@cLx;}&GXfcbj6g=<86z;XSZ5QJaZ~+oRsV}lUyk)3 zS^wKT;|!+%PU62`R&*Lx$*xI zB}sn%4+4F|&K(v3%$#a3#;(GvVAp4BJz{k&?-R(@JH@hFua`U2yl zvS0Y%hD>N%f_gKlbYn%`de*QwHy0XcF^{U4(;eFUbxo)!u&dI0`@9x@Er`IjQ+|2-Y>NA$${{|6+ruL&J2q?5t?UtC>nmFXXWPi~qK$OvQv I{$COJAK@Xtvj6}9 diff --git a/syft/presenter/json/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden b/syft/presenter/json/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden index c98a9207cba82e4500322a8abc3c47c933deb265..4e7ce36e08086a679c8a074bf4229f4cafee09bc 100644 GIT binary patch literal 23552 zcmeHPTW{M&7WQ+0h04Bk0ZTq}pAoPR*(N{>6q`lsw!j8KFxQx^Ou8i9#xVTf_fV28 z+lehDlmurpf^3nT3y0r1hdft;wuT$w!Ey~+3UD$g8--RfX-gOgor1?r8KewcYMrpuV-ln@87_}u^Pj~)=F7?q51!VZ8a1@vGf)=xdW$9db?2A4E+ru!E+ zbiO@}hoSWU^~Fy=|M2qPk1ZDnfaLr>-LLl%Id+%+6cmSp(O{Z9wy5j2FP#2lvKN%E z(ayh541vM&|9e|kYp!XX{3|Bz%fG}Ma+d!!xNf8QvM56_Erzl|aaf$$CRQEJavG+G4^Hkuk&(tGu4mQ#o2QB z)nQTNW-%+P^LNG71`INk@JBOy5t~V`Lp}IeBmew8{f`nsM3(;t0o`IlTc5^M&f=I( zx_8|cC%!e+8u|m!h5k>dl7dRq|A`Vj)BhHPJKJ2)e$?Mj@$TJzy|WYLi8BHjfs8;# z;7K7s6(dxT;2C-WiH9g4c`s30=Uybm4W+>bVnr}IF+i0bxJMVD4|s%0JsEEQq0f)J zXa5P6pa1({l*__4{-o@p@6MavBC)4I@`Z)<^zEuw#ebe1fB(;;^R_vf)!xpIT31%n zTl+V=tA_cZ#V&3ZuRmy*wI`MeIdv4pfiLeqrU?kDii2?VB1OAc6Ed9j8~Vp zJTC7~NGBTknov3Sye!WA#^JbYdW^* zB}mZ|XJX(997s=j)Y=)(q!Dvs+*qfiw@ydJ)#1988jN8XZ9ZnDAu*>oPl8+Hh;#;h zP`D(BEE*k*x72ZygjFh-WKr&Wb_vSaB(&BP?yhuhN3UP$9U?8=U5{b9MgG1*2EXMc zIxL#H?h?9f{r_(f<#1&j7g+GX`(4A=@mHP ztV~HbDv3!Z=w7>U)Y<^3N?5NP&bpilxr2a^QMtq!=;)&b4;qyUuB}SZ(8T7!M<$#H z6Qe{OBr5Bai!r*inkhjf+FA*WA`c-0SmPBZ2^q;-fh*NUX^NuGC`jUc!b8*F1 z@z3J*)h52OeLRo%eGUzj_YX7SNWl23LHsB~@2$dMaFjCgKy>h0Aw7dfe4}NA7)>oW zO{|GKivLQ6{{I~R-90x9rvGl@zxepd`v2~!e{hiFzxzP>>7HY-{13PPDP{b={Vyoy z|Li3*;S10ISjK;cvH|L1H}T)0Aw`^P75{zT8o8@Ee_nCmYz^PWUIP6 ziOr~|{MTRoU3MoUOdgKyWHK2#bsnD)$OvQvG6GK+fuY4Zo2ZPN>VG%&zv%SkSpSjr zzuhy=VEXSQ{)=_zEXRL#PyHPK-3P```5c4ge|Y>CB)y;iO$e3*=lJg)7-t#(9m)oG zVz#+vcegdW+hmKOgx^B^=L7Y>n5LQQ|F=oz2j{w*Vf|pYK@DsDe^~R|zyD|D|07D0 zeE%N=`i6r$ECQH2|3)Xay(t6UaB%LH5y%K+1Tq4jEdr;uDia2OH>u<_EHLrsQ3>tqLI(@!bTI!HS65qO`c1*f{W1a> LfsDZaF#`VsZMnln literal 22016 zcmeHPTW{Mo6wY&hg{!^BHt(bi?4fH4w7{?oZPo!Rih|-L)@nbw_Dj_(DDaTeNND759 zO(^Gylx%pk`9g>x$@vkQU+aJCr$?Adj**a*qa%bRf$BZJx$E{dW@V)dm{QxG)<3Ro ze;u_KZR!8^#p^e(PJg>^y0Em6U~HZ4H)TMM-lac56w{+#Ynt3Qshf8{IQ{XU6O?bz z4u2mR0oKRUAU}*zO)sw zWe9o348ax&Bc-5J`xwPiILTv4iM7@sN(GXds91qap{>Si4L<2RK}vuNb?q;$v8@>xEEk6C}{3TV1`cvf95?@upWcDJ9Vbeb8M#G?L->{B$# zW|PTr^n426Hotu})#rLv}OF^L|?jMP|MI?y*BLs@;~xF z!AJ)A@2hz3^95|QlgZ^ar?`iYy8HPLaVEq0@8O7V-*Q3!!RPUy`JZ3>vwh<&Yt~2O zP3Ql*&f+my}_K2xC@y5Xx<01@`LBxw)tdeXvHxL=MUelxpntIcb%FBZ9iuh>x3~WxLGZf$UxFJ* zIRAC9Y!>#vw#%S4#DOYdZnML&M12G{JA8)FsQDKxb@y57zM%oBE#dcI{{d6+y8R!6 zBUs@74%q*w%%`r=`r&BDF1q6UG{2c z3UJ~m4>eloHy{uY2nYlO9t#3(o%If)|HE+n3;zF|ERojqKZyPh{C|S{-wFML{;w05 zAMGHm`F{`kKM49-*Z(mH2@2=G4pz-V|JQaIfc|gk|KC=n6m*Ggy#9C8CAKB}f%JbA zLV^PScfkKk{U7<3`oBj1dsqKQ0{uH6|52m+B}>KM(=NsgV*~^O0s(=5z_&-hJ8NZP z3`%0;5%~T=<_{r=RZf~%8_AhUl(IU}BB9nf>#YaxKth~VHs;Bo{rL}bM%K=Mbpnv^ z``^XtxQf0K)VhkMZsL~r6sLP9rnwHmy%WM*2Q4qXNH#`(gXOZy3y8sfceSAztm+sd z(C9p1d_Ka51X@PVPEH!@XTLFj_l8SxcwYXgm(&5)AJeKnl;+dvJi`6v;lSqRqbq6| zYVZj~*T)+t5_1TJLkc(kF#kJ|Mh?F**)#XJUG2H|Umw=^Bz5kF}c=s+U$C_|4l1eI?cq5s2UNP`8juRG3$pxea zz_25D1qJFAF_2b(%LIwULq-9|h4l%<*&pBZ#F;bF3r4N>iH|*zh!V}6K-xR-He-= len(optionStr) || o < 0 { - return optionStr[0] - } - - return optionStr[o] + return string(o) } diff --git a/syft/source/scope_test.go b/syft/source/scope_test.go deleted file mode 100644 index 0c210e013..000000000 --- a/syft/source/scope_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package source - -import ( - "fmt" - "testing" -) - -func TestOptionStringerBoundary(t *testing.T) { - var _ fmt.Stringer = Scope(0) - - for _, c := range []int{-1, 0, 3} { - option := Scope(c) - if option.String() != UnknownScope.String() { - t.Errorf("expected Scope(%d) to be unknown, found '%+v'", c, option) - } - } -} diff --git a/syft/source/source.go b/syft/source/source.go index b3e49d22d..91ce1009e 100644 --- a/syft/source/source.go +++ b/syft/source/source.go @@ -15,29 +15,18 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) -// ImageSource represents a data source that is a container image -type ImageSource struct { - Img *image.Image // the image object to be cataloged -} - -// DirSource represents a data source that is a filesystem directory tree -type DirSource struct { - Path string // the root path to be cataloged -} - // Source is an object that captures the data source to be cataloged, configuration, and a specific resolver used // in cataloging (based on the data source and configuration) type Source struct { - Scope Scope // specific perspective to catalog - Resolver Resolver // a Resolver object to use in file path/glob resolution and file contents resolution - Target interface{} // the specific source object to be cataloged - Scheme Scheme // the source data scheme type (directory or image) + Resolver Resolver // a Resolver object to use in file path/glob resolution and file contents resolution + Image *image.Image // the image object to be cataloged (image only) + Metadata Metadata } type sourceDetector func(string) (image.Source, string, error) -// NewSource produces a Source based on userInput like dir: or image:tag -func NewSource(userInput string, o Scope) (Source, func(), error) { +// New produces a Source based on userInput like dir: or image:tag +func New(userInput string, o Scope) (Source, func(), error) { fs := afero.NewOsFs() parsedScheme, location, err := detectScheme(fs, image.DetectSource, userInput) if err != nil { @@ -71,7 +60,7 @@ func NewSource(userInput string, o Scope) (Source, func(), error) { return Source{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", location, err) } - s, err := NewFromImage(img, o) + s, err := NewFromImage(img, o, location) if err != nil { return Source{}, cleanup, fmt.Errorf("could not populate source with image: %w", err) } @@ -87,16 +76,16 @@ func NewFromDirectory(path string) (Source, error) { Resolver: &DirectoryResolver{ Path: path, }, - Target: DirSource{ - Path: path, + Metadata: Metadata{ + Scheme: DirectoryScheme, + Path: path, }, - Scheme: DirectoryScheme, }, nil } // NewFromImage creates a new source object tailored to catalog a given container image, relative to the // option given (e.g. all-layers, squashed, etc) -func NewFromImage(img *image.Image, option Scope) (Source, error) { +func NewFromImage(img *image.Image, option Scope, userImageStr string) (Source, error) { if img == nil { return Source{}, fmt.Errorf("no image given") } @@ -107,11 +96,12 @@ func NewFromImage(img *image.Image, option Scope) (Source, error) { } return Source{ - Scope: option, Resolver: resolver, - Target: ImageSource{ - Img: img, + Image: img, + Metadata: Metadata{ + Scope: option, + Scheme: ImageScheme, + ImageMetadata: NewImageMetadata(img, userImageStr), }, - Scheme: ImageScheme, }, nil } diff --git a/syft/source/source_test.go b/syft/source/source_test.go index c96aadfaf..30913c91d 100644 --- a/syft/source/source_test.go +++ b/syft/source/source_test.go @@ -9,41 +9,41 @@ import ( "github.com/spf13/afero" ) -func TestNewScopeFromImageFails(t *testing.T) { +func TestNewFromImageFails(t *testing.T) { t.Run("no image given", func(t *testing.T) { - _, err := NewFromImage(nil, AllLayersScope) + _, err := NewFromImage(nil, AllLayersScope, "") if err == nil { t.Errorf("expected an error condition but none was given") } }) } -func TestNewScopeFromImageUnknownOption(t *testing.T) { +func TestNewFromImageUnknownOption(t *testing.T) { img := image.Image{} t.Run("unknown option is an error", func(t *testing.T) { - _, err := NewFromImage(&img, UnknownScope) + _, err := NewFromImage(&img, UnknownScope, "") if err == nil { t.Errorf("expected an error condition but none was given") } }) } -func TestNewScopeFromImage(t *testing.T) { +func TestNewFromImage(t *testing.T) { layer := image.NewLayer(nil) img := image.Image{ Layers: []*image.Layer{layer}, } t.Run("create a new Locations object from image", func(t *testing.T) { - _, err := NewFromImage(&img, AllLayersScope) + _, err := NewFromImage(&img, AllLayersScope, "") if err != nil { t.Errorf("unexpected error when creating a new Locations from img: %+v", err) } }) } -func TestDirectoryScope(t *testing.T) { +func TestNewFromDirectory(t *testing.T) { testCases := []struct { desc string input string @@ -78,16 +78,16 @@ func TestDirectoryScope(t *testing.T) { } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { - p, err := NewFromDirectory(test.input) + src, err := NewFromDirectory(test.input) if err != nil { t.Errorf("could not create NewDirScope: %+v", err) } - if p.Target.(DirSource).Path != test.input { - t.Errorf("mismatched stringer: '%s' != '%s'", p.Target.(DirSource).Path, test.input) + if src.Metadata.Path != test.input { + t.Errorf("mismatched stringer: '%s' != '%s'", src.Metadata.Path, test.input) } - refs, err := p.Resolver.FilesByPath(test.inputPaths...) + refs, err := src.Resolver.FilesByPath(test.inputPaths...) if err != nil { t.Errorf("FilesByPath call produced an error: %+v", err) } @@ -100,7 +100,7 @@ func TestDirectoryScope(t *testing.T) { } } -func TestMultipleFileContentsByRefContents(t *testing.T) { +func TestMultipleFileContentsByLocation(t *testing.T) { testCases := []struct { desc string input string @@ -136,7 +136,8 @@ func TestMultipleFileContentsByRefContents(t *testing.T) { } location := locations[0] - content, err := p.Resolver.FileContentsByLocation(location) + contents, err := p.Resolver.MultipleFileContentsByLocation([]Location{location}) + content := contents[location] if content != test.expected { t.Errorf("unexpected contents from file: '%s' != '%s'", content, test.expected) @@ -146,7 +147,7 @@ func TestMultipleFileContentsByRefContents(t *testing.T) { } } -func TestMultipleFileContentsByRefNoContents(t *testing.T) { +func TestFilesByPathDoesNotExist(t *testing.T) { testCases := []struct { desc string input string diff --git a/test/integration/document_import_test.go b/test/integration/document_import_test.go index 438cce148..50be36c9c 100644 --- a/test/integration/document_import_test.go +++ b/test/integration/document_import_test.go @@ -37,7 +37,7 @@ func TestCatalogFromJSON(t *testing.T) { } var buf bytes.Buffer - jsonPres := json.NewPresenter(expectedCatalog, *s, *expectedDistro) + jsonPres := json.NewPresenter(expectedCatalog, s.Metadata, *expectedDistro) if err = jsonPres.Present(&buf); err != nil { t.Fatalf("failed to write to presenter: %+v", err) } @@ -73,8 +73,6 @@ func TestCatalogFromJSON(t *testing.T) { a := actualPackages[i] // omit fields that should be missing - e.Locations = nil - e.FoundBy = "" if e.MetadataType == pkg.JavaMetadataType { metadata := e.Metadata.(pkg.JavaMetadata) metadata.Parent = nil diff --git a/test/integration/json_schema_test.go b/test/integration/json_schema_test.go index 15970ee0e..b84bf26f8 100644 --- a/test/integration/json_schema_test.go +++ b/test/integration/json_schema_test.go @@ -67,7 +67,7 @@ func testJsonSchema(t *testing.T, catalog *pkg.Catalog, theScope *source.Source, t.Fatalf("bad distro: %+v", err) } - p := presenter.GetPresenter(presenter.JSONPresenter, *theScope, catalog, &d) + p := presenter.GetPresenter(presenter.JSONPresenter, theScope.Metadata, catalog, &d) if p == nil { t.Fatal("unable to get presenter") } From 91baabe5a10b5a0d29ab3999072e9a6dc0c93a57 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 16 Nov 2020 08:19:57 -0500 Subject: [PATCH 06/13] add image metadata as catalogFromJSON return Signed-off-by: Alex Goodman --- internal/file/glob_match.go | 2 +- syft/lib.go | 24 +++++++++++------------ syft/presenter/cyclonedx/bom-extension.go | 2 +- syft/presenter/cyclonedx/document.go | 2 +- syft/source/source_test.go | 2 +- test/integration/document_import_test.go | 12 +++++++----- 6 files changed, 23 insertions(+), 21 deletions(-) diff --git a/internal/file/glob_match.go b/internal/file/glob_match.go index 07ca126b7..81575de24 100644 --- a/internal/file/glob_match.go +++ b/internal/file/glob_match.go @@ -1,6 +1,6 @@ package file -// Locations: https://research.swtch.com/glob.go +// Source: https://research.swtch.com/glob.go func GlobMatch(pattern, name string) bool { px := 0 nx := 0 diff --git a/syft/lib.go b/syft/lib.go index 9a1d81675..ea06a031f 100644 --- a/syft/lib.go +++ b/syft/lib.go @@ -7,8 +7,8 @@ Here is what the main execution path for syft does: 2. Invoke all catalogers to catalog the image, adding discovered packages to a single catalog object 3. Invoke a single presenter to show the contents of the catalog -A Locations object encapsulates the image object to be cataloged and the user options (catalog all layers vs. squashed layer), -providing a way to inspect paths and file content within the image. The Locations object, not the image object, is used +A Source object encapsulates the image object to be cataloged and the user options (catalog all layers vs. squashed layer), +providing a way to inspect paths and file content within the image. The Source object, not the image object, is used throughout the main execution path. This abstraction allows for decoupling of what is cataloged (a docker image, an OCI image, a filesystem, etc) and how it is cataloged (the individual catalogers). @@ -82,12 +82,12 @@ func CatalogFromScope(s source.Source) (*pkg.Catalog, error) { return cataloger.Catalog(s.Resolver, catalogers...) } -// TODO: we shouldn't return the jsonPresenter.Image object! this is leaky -func CatalogFromJSON(reader io.Reader) (*pkg.Catalog, *distro.Distro, error) { +// CatalogFromJSON takes an existing syft report and generates catalog primitives. +func CatalogFromJSON(reader io.Reader) (*pkg.Catalog, *distro.Distro, *source.ImageMetadata, error) { var doc jsonPresenter.Document decoder := json.NewDecoder(reader) if err := decoder.Decode(&doc); err != nil { - return nil, nil, err + return nil, nil, nil, err } var pkgs = make([]pkg.Package, len(doc.Artifacts)) @@ -106,16 +106,16 @@ func CatalogFromJSON(reader io.Reader) (*pkg.Catalog, *distro.Distro, error) { d, err := distro.NewDistro(distroType, doc.Distro.Version, doc.Distro.IDLike) if err != nil { - return nil, nil, err + return nil, nil, nil, err } - //var theImg *jsonPresenter.Image - //if doc.Locations.Type == "image" { - // img := doc.Locations.Target.(jsonPresenter.Image) - // theImg = &img - //} + var imageMetadata *source.ImageMetadata + if doc.Source.Type == "image" { + payload := doc.Source.Target.(source.ImageMetadata) + imageMetadata = &payload + } - return catalog, &d, nil + return catalog, &d, imageMetadata, nil } // SetLogger sets the logger object used for all syft logging calls. diff --git a/syft/presenter/cyclonedx/bom-extension.go b/syft/presenter/cyclonedx/bom-extension.go index 8e7ce850d..f25713734 100644 --- a/syft/presenter/cyclonedx/bom-extension.go +++ b/syft/presenter/cyclonedx/bom-extension.go @@ -8,7 +8,7 @@ import ( "github.com/anchore/syft/internal/version" ) -// Locations: https://cyclonedx.org/ext/bom-descriptor/ +// Source: https://cyclonedx.org/ext/bom-descriptor/ // BomDescriptor represents all metadata surrounding the BOM report (such as when the BOM was made, with which tool, and the item being cataloged). type BomDescriptor struct { diff --git a/syft/presenter/cyclonedx/document.go b/syft/presenter/cyclonedx/document.go index f2d2fb866..4539ab73e 100644 --- a/syft/presenter/cyclonedx/document.go +++ b/syft/presenter/cyclonedx/document.go @@ -9,7 +9,7 @@ import ( "github.com/google/uuid" ) -// Locations: https://github.com/CycloneDX/specification +// Source: https://github.com/CycloneDX/specification // Document represents a CycloneDX BOM Document. type Document struct { diff --git a/syft/source/source_test.go b/syft/source/source_test.go index 30913c91d..49946cf3b 100644 --- a/syft/source/source_test.go +++ b/syft/source/source_test.go @@ -35,7 +35,7 @@ func TestNewFromImage(t *testing.T) { Layers: []*image.Layer{layer}, } - t.Run("create a new Locations object from image", func(t *testing.T) { + t.Run("create a new source object from image", func(t *testing.T) { _, err := NewFromImage(&img, AllLayersScope, "") if err != nil { t.Errorf("unexpected error when creating a new Locations from img: %+v", err) diff --git a/test/integration/document_import_test.go b/test/integration/document_import_test.go index 50be36c9c..432bd78d3 100644 --- a/test/integration/document_import_test.go +++ b/test/integration/document_import_test.go @@ -31,24 +31,26 @@ func TestCatalogFromJSON(t *testing.T) { tarPath := imagetest.GetFixtureImageTarPath(t, test.fixture) defer cleanup() - expectedCatalog, s, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) + expectedCatalog, expectedSource, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } var buf bytes.Buffer - jsonPres := json.NewPresenter(expectedCatalog, s.Metadata, *expectedDistro) + jsonPres := json.NewPresenter(expectedCatalog, expectedSource.Metadata, *expectedDistro) if err = jsonPres.Present(&buf); err != nil { t.Fatalf("failed to write to presenter: %+v", err) } - // TODO: test img - - actualCatalog, actualDistro, err := syft.CatalogFromJSON(&buf) + actualCatalog, actualDistro, imageMetadata, err := syft.CatalogFromJSON(&buf) if err != nil { t.Fatalf("failed to import document: %+v", err) } + for _, d := range deep.Equal(*imageMetadata, expectedSource.Metadata.ImageMetadata) { + t.Errorf(" image metadata diff: %+v", d) + } + for _, d := range deep.Equal(actualDistro, expectedDistro) { t.Errorf(" distro diff: %+v", d) } From f46de19c6be5ac81fe74a013424c614c513df3ed Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 16 Nov 2020 08:47:24 -0500 Subject: [PATCH 07/13] migrate scope option to image metadata (from source) Signed-off-by: Alex Goodman --- schema/json/schema.json | 8 ++++---- syft/lib.go | 16 +++++----------- .../snapshot/TestCycloneDxDirsPresenter.golden | 4 ++-- .../snapshot/TestCycloneDxImgsPresenter.golden | 8 ++++---- syft/presenter/json/descriptor.go | 7 +++++++ syft/presenter/json/distribution.go | 8 ++++++++ syft/presenter/json/document.go | 15 --------------- syft/presenter/json/source.go | 13 +++++++++++++ .../snapshot/TestJsonDirsPresenter.golden | 11 +++-------- .../snapshot/TestJsonImgsPresenter.golden | 13 +++++-------- syft/source/image_metadata.go | 4 +++- syft/source/metadata.go | 1 - syft/source/source.go | 7 +++---- test/integration/document_import_test.go | 4 ++-- 14 files changed, 59 insertions(+), 60 deletions(-) create mode 100644 syft/presenter/json/descriptor.go create mode 100644 syft/presenter/json/distribution.go diff --git a/schema/json/schema.json b/schema/json/schema.json index 01f9e3e10..edfef558c 100644 --- a/schema/json/schema.json +++ b/schema/json/schema.json @@ -339,16 +339,12 @@ "name": { "type": "string" }, - "scope": { - "type": "string" - }, "version": { "type": "string" } }, "required": [ "name", - "scope", "version" ], "type": "object" @@ -409,6 +405,9 @@ "mediaType": { "type": "string" }, + "scope": { + "type": "string" + }, "size": { "type": "integer" }, @@ -426,6 +425,7 @@ "digest", "layers", "mediaType", + "scope", "size", "tags", "userInput" diff --git a/syft/lib.go b/syft/lib.go index ea06a031f..d57c61f27 100644 --- a/syft/lib.go +++ b/syft/lib.go @@ -83,11 +83,11 @@ func CatalogFromScope(s source.Source) (*pkg.Catalog, error) { } // CatalogFromJSON takes an existing syft report and generates catalog primitives. -func CatalogFromJSON(reader io.Reader) (*pkg.Catalog, *distro.Distro, *source.ImageMetadata, error) { +func CatalogFromJSON(reader io.Reader) (*pkg.Catalog, *distro.Distro, source.Metadata, error) { var doc jsonPresenter.Document decoder := json.NewDecoder(reader) if err := decoder.Decode(&doc); err != nil { - return nil, nil, nil, err + return nil, nil, source.Metadata{}, err } var pkgs = make([]pkg.Package, len(doc.Artifacts)) @@ -104,18 +104,12 @@ func CatalogFromJSON(reader io.Reader) (*pkg.Catalog, *distro.Distro, *source.Im distroType = distro.Type(doc.Distro.Name) } - d, err := distro.NewDistro(distroType, doc.Distro.Version, doc.Distro.IDLike) + theDistro, err := distro.NewDistro(distroType, doc.Distro.Version, doc.Distro.IDLike) if err != nil { - return nil, nil, nil, err + return nil, nil, source.Metadata{}, err } - var imageMetadata *source.ImageMetadata - if doc.Source.Type == "image" { - payload := doc.Source.Target.(source.ImageMetadata) - imageMetadata = &payload - } - - return catalog, &d, imageMetadata, nil + return catalog, &theDistro, doc.Source.ToSourceMetadata(), nil } // SetLogger sets the logger object used for all syft logging calls. diff --git a/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden b/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden index 68ca8ae8e..2e2d0d8ae 100644 --- a/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden +++ b/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden @@ -1,5 +1,5 @@ - + package1 @@ -21,7 +21,7 @@ - 2020-09-23T18:26:58-04:00 + 2020-11-16T08:45:54-05:00 anchore syft diff --git a/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden b/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden index 03f34889a..b4fcc0188 100644 --- a/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden +++ b/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden @@ -1,5 +1,5 @@ - + package1 @@ -21,15 +21,15 @@ - 2020-09-23T18:26:58-04:00 + 2020-11-16T08:45:54-05:00 anchore syft [not provided] - index.docker.io/library/stereoscope-fixture-image-simple - 04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7 + user-image-input + sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368 diff --git a/syft/presenter/json/descriptor.go b/syft/presenter/json/descriptor.go new file mode 100644 index 000000000..de07721fc --- /dev/null +++ b/syft/presenter/json/descriptor.go @@ -0,0 +1,7 @@ +package json + +// Descriptor describes what created the document as well as surrounding metadata +type Descriptor struct { + Name string `json:"name"` + Version string `json:"version"` +} diff --git a/syft/presenter/json/distribution.go b/syft/presenter/json/distribution.go new file mode 100644 index 000000000..567aa50f5 --- /dev/null +++ b/syft/presenter/json/distribution.go @@ -0,0 +1,8 @@ +package json + +// Distribution provides information about a detected Linux Distribution +type Distribution struct { + Name string `json:"name"` + Version string `json:"version"` + IDLike string `json:"idLike"` +} diff --git a/syft/presenter/json/document.go b/syft/presenter/json/document.go index 2c20a1a38..230c9ad8c 100644 --- a/syft/presenter/json/document.go +++ b/syft/presenter/json/document.go @@ -15,20 +15,6 @@ type Document struct { Descriptor Descriptor `json:"descriptor"` } -// Descriptor describes what created the document as well as surrounding metadata -type Descriptor struct { - Name string `json:"name"` - Version string `json:"version"` - Scope string `json:"scope"` -} - -// Distribution provides information about a detected Linux Distribution -type Distribution struct { - Name string `json:"name"` - Version string `json:"version"` - IDLike string `json:"idLike"` -} - func NewDocument(catalog *pkg.Catalog, srcMetadata source.Metadata, d distro.Distro) (Document, error) { src, err := NewSource(srcMetadata) if err != nil { @@ -51,7 +37,6 @@ func NewDocument(catalog *pkg.Catalog, srcMetadata source.Metadata, d distro.Dis Descriptor: Descriptor{ Name: internal.ApplicationName, Version: version.FromBuild().Version, - Scope: srcMetadata.Scope.String(), }, } diff --git a/syft/presenter/json/source.go b/syft/presenter/json/source.go index 14561a24f..a8a1015a4 100644 --- a/syft/presenter/json/source.go +++ b/syft/presenter/json/source.go @@ -56,3 +56,16 @@ func (s *Source) UnmarshalJSON(b []byte) error { return nil } + +func (s *Source) ToSourceMetadata() source.Metadata { + var metadata source.Metadata + switch s.Type { + case "directory": + metadata.Scheme = source.DirectoryScheme + metadata.Path = s.Target.(string) + case "image": + metadata.Scheme = source.ImageScheme + metadata.ImageMetadata = s.Target.(source.ImageMetadata) + } + return metadata +} diff --git a/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden b/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden index 7324c5349..b88f2ffef 100644 --- a/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden +++ b/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden @@ -4,9 +4,7 @@ "name": "package-1", "version": "1.0.1", "type": "python", - "foundBy": [ - "the-cataloger-1" - ], + "foundBy": "the-cataloger-1", "locations": [ { "path": "/some/path/pkg1" @@ -31,9 +29,7 @@ "name": "package-2", "version": "2.0.1", "type": "deb", - "foundBy": [ - "the-cataloger-2" - ], + "foundBy": "the-cataloger-2", "locations": [ { "path": "/some/path/pkg1" @@ -64,7 +60,6 @@ }, "descriptor": { "name": "syft", - "version": "[not provided]", - "scope": "" + "version": "[not provided]" } } diff --git a/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden b/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden index 91b28d91e..854f05ea0 100644 --- a/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden +++ b/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden @@ -4,9 +4,7 @@ "name": "package-1", "version": "1.0.1", "type": "python", - "foundBy": [ - "the-cataloger-1" - ], + "foundBy": "the-cataloger-1", "locations": [ { "path": "/somefile-1.txt", @@ -32,9 +30,7 @@ "name": "package-2", "version": "2.0.1", "type": "deb", - "foundBy": [ - "the-cataloger-2" - ], + "foundBy": "the-cataloger-2", "locations": [ { "path": "/somefile-2.txt", @@ -58,6 +54,8 @@ "source": { "type": "image", "target": { + "userInput": "user-image-input", + "scope": "AllLayers", "layers": [ { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", @@ -90,7 +88,6 @@ }, "descriptor": { "name": "syft", - "version": "[not provided]", - "scope": "AllLayers" + "version": "[not provided]" } } diff --git a/syft/source/image_metadata.go b/syft/source/image_metadata.go index 2a9224d7a..9bd8a30a9 100644 --- a/syft/source/image_metadata.go +++ b/syft/source/image_metadata.go @@ -4,6 +4,7 @@ import "github.com/anchore/stereoscope/pkg/image" type ImageMetadata struct { UserInput string `json:"userInput"` + Scope Scope `json:"scope"` // specific perspective to catalog Layers []LayerMetadata `json:"layers"` Size int64 `json:"size"` Digest string `json:"digest"` @@ -17,7 +18,7 @@ type LayerMetadata struct { Size int64 `json:"size"` } -func NewImageMetadata(img *image.Image, userInput string) ImageMetadata { +func NewImageMetadata(img *image.Image, userInput string, scope Scope) ImageMetadata { // populate artifacts... tags := make([]string, len(img.Metadata.Tags)) for idx, tag := range img.Metadata.Tags { @@ -25,6 +26,7 @@ func NewImageMetadata(img *image.Image, userInput string) ImageMetadata { } theImg := ImageMetadata{ UserInput: userInput, + Scope: scope, Digest: img.Metadata.Digest, Size: img.Metadata.Size, MediaType: string(img.Metadata.MediaType), diff --git a/syft/source/metadata.go b/syft/source/metadata.go index 188118b7b..ef3092564 100644 --- a/syft/source/metadata.go +++ b/syft/source/metadata.go @@ -1,7 +1,6 @@ package source type Metadata struct { - Scope Scope // specific perspective to catalog Scheme Scheme // the source data scheme type (directory or image) ImageMetadata ImageMetadata // all image info (image only) Path string // the root path to be cataloged (directory only) diff --git a/syft/source/source.go b/syft/source/source.go index 91ce1009e..182a87f71 100644 --- a/syft/source/source.go +++ b/syft/source/source.go @@ -85,12 +85,12 @@ func NewFromDirectory(path string) (Source, error) { // NewFromImage creates a new source object tailored to catalog a given container image, relative to the // option given (e.g. all-layers, squashed, etc) -func NewFromImage(img *image.Image, option Scope, userImageStr string) (Source, error) { +func NewFromImage(img *image.Image, scope Scope, userImageStr string) (Source, error) { if img == nil { return Source{}, fmt.Errorf("no image given") } - resolver, err := getImageResolver(img, option) + resolver, err := getImageResolver(img, scope) if err != nil { return Source{}, fmt.Errorf("could not determine file resolver: %w", err) } @@ -99,9 +99,8 @@ func NewFromImage(img *image.Image, option Scope, userImageStr string) (Source, Resolver: resolver, Image: img, Metadata: Metadata{ - Scope: option, Scheme: ImageScheme, - ImageMetadata: NewImageMetadata(img, userImageStr), + ImageMetadata: NewImageMetadata(img, userImageStr, scope), }, }, nil } diff --git a/test/integration/document_import_test.go b/test/integration/document_import_test.go index 432bd78d3..056e92f66 100644 --- a/test/integration/document_import_test.go +++ b/test/integration/document_import_test.go @@ -42,12 +42,12 @@ func TestCatalogFromJSON(t *testing.T) { t.Fatalf("failed to write to presenter: %+v", err) } - actualCatalog, actualDistro, imageMetadata, err := syft.CatalogFromJSON(&buf) + actualCatalog, actualDistro, sourceMetadata, err := syft.CatalogFromJSON(&buf) if err != nil { t.Fatalf("failed to import document: %+v", err) } - for _, d := range deep.Equal(*imageMetadata, expectedSource.Metadata.ImageMetadata) { + for _, d := range deep.Equal(sourceMetadata, expectedSource.Metadata) { t.Errorf(" image metadata diff: %+v", d) } From 4b45c42f5a3650bfd4f32e8315fe3b7ac6f123df Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 16 Nov 2020 13:59:15 -0500 Subject: [PATCH 08/13] make cyclonedx presenter generally reusable (for grype) Signed-off-by: Alex Goodman --- syft/presenter/cyclonedx/bom-extension.go | 32 +++++++++++++++----- syft/presenter/cyclonedx/document.go | 34 +++++++++++----------- syft/presenter/cyclonedx/presenter.go | 29 ++---------------- syft/presenter/cyclonedx/presenter_test.go | 2 +- syft/presenter/json/distribution.go | 15 ++++++++++ syft/presenter/json/document.go | 11 +------ 6 files changed, 62 insertions(+), 61 deletions(-) diff --git a/syft/presenter/cyclonedx/bom-extension.go b/syft/presenter/cyclonedx/bom-extension.go index f25713734..7f99de637 100644 --- a/syft/presenter/cyclonedx/bom-extension.go +++ b/syft/presenter/cyclonedx/bom-extension.go @@ -4,8 +4,7 @@ import ( "encoding/xml" "time" - "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/version" + "github.com/anchore/syft/syft/source" ) // Source: https://cyclonedx.org/ext/bom-descriptor/ @@ -35,15 +34,34 @@ type BdComponent struct { } // NewBomDescriptor returns a new BomDescriptor tailored for the current time and "syft" tool details. -func NewBomDescriptor() *BomDescriptor { - versionInfo := version.FromBuild() - return &BomDescriptor{ +func NewBomDescriptor(name, version string, srcMetadata source.Metadata) *BomDescriptor { + descriptor := BomDescriptor{ XMLName: xml.Name{}, Timestamp: time.Now().Format(time.RFC3339), Tool: &BdTool{ Vendor: "anchore", - Name: internal.ApplicationName, - Version: versionInfo.Version, + Name: name, + Version: version, }, } + + switch srcMetadata.Scheme { + case source.ImageScheme: + descriptor.Component = &BdComponent{ + Component: Component{ + Type: "container", + Name: srcMetadata.ImageMetadata.UserInput, + Version: srcMetadata.ImageMetadata.Digest, + }, + } + case source.DirectoryScheme: + descriptor.Component = &BdComponent{ + Component: Component{ + Type: "file", + Name: srcMetadata.Path, + }, + } + } + + return &descriptor } diff --git a/syft/presenter/cyclonedx/document.go b/syft/presenter/cyclonedx/document.go index 4539ab73e..78752480e 100644 --- a/syft/presenter/cyclonedx/document.go +++ b/syft/presenter/cyclonedx/document.go @@ -3,9 +3,11 @@ package cyclonedx import ( "encoding/xml" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/version" "github.com/anchore/syft/syft/distro" - "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" "github.com/google/uuid" ) @@ -22,19 +24,19 @@ type Document struct { BomDescriptor *BomDescriptor `xml:"bd:metadata"` // The BOM descriptor extension } -// NewDocument returns an empty CycloneDX Document object. -func NewDocument() Document { - return Document{ - XMLNs: "http://cyclonedx.org/schema/bom/1.2", - XMLNsBd: "http://cyclonedx.org/schema/ext/bom-descriptor/1.0", - Version: 1, - SerialNumber: uuid.New().URN(), - } -} - // NewDocumentFromCatalog returns a CycloneDX Document object populated with the catalog contents. -func NewDocumentFromCatalog(catalog *pkg.Catalog, d distro.Distro) Document { - bom := NewDocument() +func NewDocument(catalog *pkg.Catalog, d distro.Distro, srcMetadata source.Metadata) Document { + versionInfo := version.FromBuild() + + doc := Document{ + XMLNs: "http://cyclonedx.org/schema/bom/1.2", + XMLNsBd: "http://cyclonedx.org/schema/ext/bom-descriptor/1.0", + Version: 1, + SerialNumber: uuid.New().URN(), + BomDescriptor: NewBomDescriptor(internal.ApplicationName, versionInfo.Version, srcMetadata), + } + + // attach components for p := range catalog.Enumerate() { component := Component{ Type: "library", // TODO: this is not accurate @@ -51,10 +53,8 @@ func NewDocumentFromCatalog(catalog *pkg.Catalog, d distro.Distro) Document { if len(licenses) > 0 { component.Licenses = &licenses } - bom.Components = append(bom.Components, component) + doc.Components = append(doc.Components, component) } - bom.BomDescriptor = NewBomDescriptor() - - return bom + return doc } diff --git a/syft/presenter/cyclonedx/presenter.go b/syft/presenter/cyclonedx/presenter.go index 2a28b8db7..ad86ac947 100644 --- a/syft/presenter/cyclonedx/presenter.go +++ b/syft/presenter/cyclonedx/presenter.go @@ -5,7 +5,6 @@ package cyclonedx import ( "encoding/xml" - "fmt" "io" "github.com/anchore/syft/syft/distro" @@ -22,39 +21,17 @@ type Presenter struct { } // NewPresenter creates a CycloneDX presenter from the given Catalog and Locations objects. -func NewPresenter(catalog *pkg.Catalog, s source.Metadata, d distro.Distro) *Presenter { +func NewPresenter(catalog *pkg.Catalog, srcMetadata source.Metadata, d distro.Distro) *Presenter { return &Presenter{ catalog: catalog, - srcMetadata: s, + srcMetadata: srcMetadata, distro: d, } } // Present writes the CycloneDX report to the given io.Writer. func (pres *Presenter) Present(output io.Writer) error { - bom := NewDocumentFromCatalog(pres.catalog, pres.distro) - - switch pres.srcMetadata.Scheme { - case source.DirectoryScheme: - bom.BomDescriptor.Component = &BdComponent{ - Component: Component{ - Type: "file", - Name: pres.srcMetadata.Path, - Version: "", - }, - } - case source.ImageScheme: - // TODO: can we use the tags a bit better? - bom.BomDescriptor.Component = &BdComponent{ - Component: Component{ - Type: "container", - Name: pres.srcMetadata.ImageMetadata.UserInput, - Version: pres.srcMetadata.ImageMetadata.Digest, - }, - } - default: - return fmt.Errorf("unsupported source: %T", pres.srcMetadata.Scheme) - } + bom := NewDocument(pres.catalog, pres.distro, pres.srcMetadata) encoder := xml.NewEncoder(output) encoder.Indent("", " ") diff --git a/syft/presenter/cyclonedx/presenter_test.go b/syft/presenter/cyclonedx/presenter_test.go index ee29ca62c..a4ba735fd 100644 --- a/syft/presenter/cyclonedx/presenter_test.go +++ b/syft/presenter/cyclonedx/presenter_test.go @@ -177,7 +177,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) { if !bytes.Equal(expected, actual) { dmp := diffmatchpatch.New() - diffs := dmp.DiffMain(string(expected), string(actual), true) + diffs := dmp.DiffMain(string(actual), string(expected), true) t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs)) } } diff --git a/syft/presenter/json/distribution.go b/syft/presenter/json/distribution.go index 567aa50f5..b19c4ca68 100644 --- a/syft/presenter/json/distribution.go +++ b/syft/presenter/json/distribution.go @@ -1,8 +1,23 @@ package json +import "github.com/anchore/syft/syft/distro" + // Distribution provides information about a detected Linux Distribution type Distribution struct { Name string `json:"name"` Version string `json:"version"` IDLike string `json:"idLike"` } + +func NewDistribution(d distro.Distro) Distribution { + distroName := d.Name() + if distroName == "UnknownDistroType" { + distroName = "" + } + + return Distribution{ + Name: distroName, + Version: d.FullVersion(), + IDLike: d.IDLike, + } +} diff --git a/syft/presenter/json/document.go b/syft/presenter/json/document.go index 230c9ad8c..19b89c9d2 100644 --- a/syft/presenter/json/document.go +++ b/syft/presenter/json/document.go @@ -21,19 +21,10 @@ func NewDocument(catalog *pkg.Catalog, srcMetadata source.Metadata, d distro.Dis return Document{}, nil } - distroName := d.Name() - if distroName == "UnknownDistroType" { - distroName = "" - } - doc := Document{ Artifacts: make([]Artifact, 0), Source: src, - Distro: Distribution{ - Name: distroName, - Version: d.FullVersion(), - IDLike: d.IDLike, - }, + Distro: NewDistribution(d), Descriptor: Descriptor{ Name: internal.ApplicationName, Version: version.FromBuild().Version, From 62b03f3a917ca7f3c681cc8904b98d1e80041ed2 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 16 Nov 2020 15:15:19 -0500 Subject: [PATCH 09/13] dont export structs used for unmarshaling Signed-off-by: Alex Goodman --- syft/presenter/json/artifact.go | 20 ++++++++++---------- syft/presenter/json/distribution.go | 9 +++++---- syft/presenter/json/document.go | 10 ++++++---- syft/presenter/json/presenter.go | 3 +++ syft/presenter/json/source.go | 9 +++++++-- 5 files changed, 31 insertions(+), 20 deletions(-) diff --git a/syft/presenter/json/artifact.go b/syft/presenter/json/artifact.go index 3aeea21ef..e26bdc654 100644 --- a/syft/presenter/json/artifact.go +++ b/syft/presenter/json/artifact.go @@ -9,11 +9,11 @@ import ( ) type Artifact struct { - ArtifactBasicMetadata - ArtifactCustomMetadata + artifactBasicMetadata + artifactCustomMetadata } -type ArtifactBasicMetadata struct { +type artifactBasicMetadata struct { Name string `json:"name"` Version string `json:"version"` Type pkg.Type `json:"type"` @@ -23,12 +23,12 @@ type ArtifactBasicMetadata struct { Language pkg.Language `json:"language"` } -type ArtifactCustomMetadata struct { +type artifactCustomMetadata struct { MetadataType pkg.MetadataType `json:"metadataType"` Metadata interface{} `json:"metadata,omitempty"` } -type ArtifactMetadataUnpacker struct { +type artifactMetadataUnpacker struct { MetadataType string `json:"metadataType"` Metadata json.RawMessage `json:"metadata"` } @@ -36,7 +36,7 @@ type ArtifactMetadataUnpacker struct { func NewArtifact(p *pkg.Package) (Artifact, error) { return Artifact{ - ArtifactBasicMetadata: ArtifactBasicMetadata{ + artifactBasicMetadata: artifactBasicMetadata{ Name: p.Name, Version: p.Version, Type: p.Type, @@ -45,7 +45,7 @@ func NewArtifact(p *pkg.Package) (Artifact, error) { Licenses: p.Licenses, Language: p.Language, }, - ArtifactCustomMetadata: ArtifactCustomMetadata{ + artifactCustomMetadata: artifactCustomMetadata{ MetadataType: p.MetadataType, Metadata: p.Metadata, }, @@ -68,13 +68,13 @@ func (a Artifact) ToPackage() pkg.Package { } func (a *Artifact) UnmarshalJSON(b []byte) error { - var basic ArtifactBasicMetadata + var basic artifactBasicMetadata if err := json.Unmarshal(b, &basic); err != nil { return err } - a.ArtifactBasicMetadata = basic + a.artifactBasicMetadata = basic - var unpacker ArtifactMetadataUnpacker + var unpacker artifactMetadataUnpacker if err := json.Unmarshal(b, &unpacker); err != nil { return err } diff --git a/syft/presenter/json/distribution.go b/syft/presenter/json/distribution.go index b19c4ca68..150334a4c 100644 --- a/syft/presenter/json/distribution.go +++ b/syft/presenter/json/distribution.go @@ -2,13 +2,14 @@ package json import "github.com/anchore/syft/syft/distro" -// Distribution provides information about a detected Linux Distribution +// Distribution provides information about a detected Linux Distribution. type Distribution struct { - Name string `json:"name"` - Version string `json:"version"` - IDLike string `json:"idLike"` + Name string `json:"name"` // Name of the Linux distribution + Version string `json:"version"` // Version of the Linux distribution (major or major.minor version) + IDLike string `json:"idLike"` // the ID_LIKE field found within the /etc/os-release file } +// NewDistribution creates a struct with the Linux distribution to be represented in JSON. func NewDistribution(d distro.Distro) Distribution { distroName := d.Name() if distroName == "UnknownDistroType" { diff --git a/syft/presenter/json/document.go b/syft/presenter/json/document.go index 19b89c9d2..de1ce463b 100644 --- a/syft/presenter/json/document.go +++ b/syft/presenter/json/document.go @@ -8,13 +8,15 @@ import ( "github.com/anchore/syft/syft/source" ) +// Document represents the syft cataloging findings as a JSON document type Document struct { - Artifacts []Artifact `json:"artifacts"` - Source Source `json:"source"` - Distro Distribution `json:"distro"` - Descriptor Descriptor `json:"descriptor"` + Artifacts []Artifact `json:"artifacts"` // Artifacts is the list of packages discovered and placed into the catalog + Source Source `json:"source"` // Source represents the original object that was cataloged + Distro Distribution `json:"distro"` // Distro represents the Linux distribution that was detected from the source + Descriptor Descriptor `json:"descriptor"` // Descriptor is a block containing self-describing information about syft } +// NewDocument creates and populates a new JSON document struct from the given cataloging results. func NewDocument(catalog *pkg.Catalog, srcMetadata source.Metadata, d distro.Distro) (Document, error) { src, err := NewSource(srcMetadata) if err != nil { diff --git a/syft/presenter/json/presenter.go b/syft/presenter/json/presenter.go index 00af4ef4e..dd11381a0 100644 --- a/syft/presenter/json/presenter.go +++ b/syft/presenter/json/presenter.go @@ -9,12 +9,14 @@ import ( "github.com/anchore/syft/syft/source" ) +// Presenter is a JSON presentation object for the syft results type Presenter struct { catalog *pkg.Catalog srcMetadata source.Metadata distro distro.Distro } +// NewPresenter creates a new JSON presenter object for the given cataloging results. func NewPresenter(catalog *pkg.Catalog, s source.Metadata, d distro.Distro) *Presenter { return &Presenter{ catalog: catalog, @@ -23,6 +25,7 @@ func NewPresenter(catalog *pkg.Catalog, s source.Metadata, d distro.Distro) *Pre } } +// Present the catalog results to the given writer. func (pres *Presenter) Present(output io.Writer) error { doc, err := NewDocument(pres.catalog, pres.srcMetadata, pres.distro) if err != nil { diff --git a/syft/presenter/json/source.go b/syft/presenter/json/source.go index a8a1015a4..cd777b93e 100644 --- a/syft/presenter/json/source.go +++ b/syft/presenter/json/source.go @@ -7,16 +7,19 @@ import ( "github.com/anchore/syft/syft/source" ) +// Source object represents the thing that was cataloged type Source struct { Type string `json:"type"` Target interface{} `json:"target"` } -type SourceUnpacker struct { +// sourceUnpacker is used to unmarshal Source objects +type sourceUnpacker struct { Type string `json:"type"` Target json.RawMessage `json:"target"` } +// NewSource creates a new source object to be represented into JSON. func NewSource(src source.Metadata) (Source, error) { switch src.Scheme { case source.ImageScheme: @@ -34,8 +37,9 @@ func NewSource(src source.Metadata) (Source, error) { } } +// UnmarshalJSON populates a source object from JSON bytes. func (s *Source) UnmarshalJSON(b []byte) error { - var unpacker SourceUnpacker + var unpacker sourceUnpacker if err := json.Unmarshal(b, &unpacker); err != nil { return err } @@ -57,6 +61,7 @@ func (s *Source) UnmarshalJSON(b []byte) error { return nil } +// ToSourceMetadata takes a source object represented from JSON and creates a source.Metadata object. func (s *Source) ToSourceMetadata() source.Metadata { var metadata source.Metadata switch s.Type { From c892c3609ee730e2392a7ce931f82eb610617d3a Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 17 Nov 2020 08:25:01 -0500 Subject: [PATCH 10/13] improve doc comments Signed-off-by: Alex Goodman --- cmd/cmd.go | 5 +++-- internal/constants.go | 2 +- internal/docs.go | 4 ++++ internal/file/glob_match.go | 1 + internal/file/opener.go | 2 ++ internal/file/zip_file_manifest.go | 6 ++++++ internal/file/zip_file_traversal.go | 5 +++++ internal/format/color.go | 21 ------------------- internal/log/log.go | 12 +++++++++++ internal/logger/doc.go | 4 ++++ internal/logger/logrus.go | 20 ++++++++++++++++++ internal/stringset.go | 10 +++++++++ internal/version/build.go | 3 +++ internal/version/update.go | 1 + .../python/package_cataloger_test.go | 14 ++++++------- .../python/parse_wheel_egg_record.go | 2 +- .../python/parse_wheel_egg_record_test.go | 14 ++++++------- syft/cataloger/rpmdb/parse_rpmdb.go | 5 ++--- syft/distro/distro.go | 15 ++++++++----- syft/distro/type.go | 4 ++++ syft/event/event.go | 9 ++++++-- syft/logger/logger.go | 3 ++- syft/pkg/apk_metadata.go | 1 + syft/pkg/dpkg_metadata.go | 2 ++ syft/pkg/gem_metadata.go | 1 + syft/pkg/java_metadata.go | 1 + syft/pkg/language.go | 4 ++++ syft/pkg/metadata.go | 16 +++++++------- syft/pkg/package.go | 1 + syft/pkg/python_package_metadata.go | 9 ++++---- syft/pkg/rpmdb_metadata.go | 3 +++ syft/pkg/type.go | 3 +++ syft/presenter/json/artifact.go | 3 +-- syft/presenter/json/source.go | 1 - .../snapshot/TestJsonDirsPresenter.golden | 4 ++-- .../snapshot/TestJsonImgsPresenter.golden | 4 ++-- syft/presenter/text/presenter.go | 2 ++ syft/source/all_layers_resolver.go | 7 +++++-- syft/source/directory_resolver.go | 7 +++++-- syft/source/image_metadata.go | 4 ++++ syft/source/image_squash_resolver.go | 7 +++++-- syft/source/location.go | 9 +++++--- syft/source/metadata.go | 1 + syft/source/resolver.go | 2 +- syft/source/scheme.go | 10 ++++++--- syft/source/scope.go | 10 +++++++-- 46 files changed, 191 insertions(+), 83 deletions(-) create mode 100644 internal/docs.go delete mode 100644 internal/format/color.go create mode 100644 internal/logger/doc.go diff --git a/cmd/cmd.go b/cmd/cmd.go index 70197c968..e39ec4efe 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -4,6 +4,8 @@ import ( "fmt" "os" + "github.com/gookit/color" + "github.com/spf13/cobra" "github.com/anchore/syft/syft/presenter" @@ -11,7 +13,6 @@ import ( "github.com/anchore/stereoscope" "github.com/anchore/syft/internal/config" - "github.com/anchore/syft/internal/format" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/logger" "github.com/anchore/syft/syft" @@ -111,7 +112,7 @@ func logAppConfig() { if err != nil { log.Debugf("Could not display application config: %+v", err) } else { - log.Debugf("Application config:\n%+v", format.Magenta.Format(string(appCfgStr))) + log.Debugf("Application config:\n%+v", color.Magenta.Sprint(appCfgStr)) } } diff --git a/internal/constants.go b/internal/constants.go index 249eaf148..61fc3a2e4 100644 --- a/internal/constants.go +++ b/internal/constants.go @@ -1,4 +1,4 @@ package internal -// note: do not change this +// ApplicationName is the non-capitalized name of the application (do not change this) const ApplicationName = "syft" diff --git a/internal/docs.go b/internal/docs.go new file mode 100644 index 000000000..d06d47f6a --- /dev/null +++ b/internal/docs.go @@ -0,0 +1,4 @@ +/* +Package internal contains miscellaneous functions and objects useful within syft but should not be used externally. +*/ +package internal diff --git a/internal/file/glob_match.go b/internal/file/glob_match.go index 81575de24..6befe32e4 100644 --- a/internal/file/glob_match.go +++ b/internal/file/glob_match.go @@ -1,5 +1,6 @@ package file +// GlobMatch evaluates the given glob pattern against the given "name" string, indicating if there is a match or not. // Source: https://research.swtch.com/glob.go func GlobMatch(pattern, name string) bool { px := 0 diff --git a/internal/file/opener.go b/internal/file/opener.go index c79e2a845..969930f11 100644 --- a/internal/file/opener.go +++ b/internal/file/opener.go @@ -5,10 +5,12 @@ import ( "os" ) +// Opener is an object that stores a path to later be opened as a file. type Opener struct { path string } +// Open the stored path as a io.ReadCloser. func (o Opener) Open() (io.ReadCloser, error) { return os.Open(o.path) } diff --git a/internal/file/zip_file_manifest.go b/internal/file/zip_file_manifest.go index 7b19c2070..d7b7b80be 100644 --- a/internal/file/zip_file_manifest.go +++ b/internal/file/zip_file_manifest.go @@ -12,16 +12,20 @@ import ( "github.com/anchore/syft/internal/log" ) +// ZipFileManifest is a collection of paths and their file metadata. type ZipFileManifest map[string]os.FileInfo +// newZipManifest creates an empty ZipFileManifest. func newZipManifest() ZipFileManifest { return make(ZipFileManifest) } +// Add a new path and it's file metadata to the collection. func (z ZipFileManifest) Add(entry string, info os.FileInfo) { z[entry] = info } +// GlobMatch returns the path keys that match the given value(s). func (z ZipFileManifest) GlobMatch(patterns ...string) []string { uniqueMatches := internal.NewStringSet() @@ -43,6 +47,7 @@ func (z ZipFileManifest) GlobMatch(patterns ...string) []string { return results } +// NewZipFileManifest creates and returns a new ZipFileManifest populated with path and metadata from the given zip archive path. func NewZipFileManifest(archivePath string) (ZipFileManifest, error) { zipReader, err := zip.OpenReader(archivePath) manifest := newZipManifest() @@ -62,6 +67,7 @@ func NewZipFileManifest(archivePath string) (ZipFileManifest, error) { return manifest, nil } +// normalizeZipEntryName takes the given path entry and ensures it is prefixed with "/". func normalizeZipEntryName(entry string) string { if !strings.HasPrefix(entry, "/") { return "/" + entry diff --git a/internal/file/zip_file_traversal.go b/internal/file/zip_file_traversal.go index 0f8fe149e..742a88dbc 100644 --- a/internal/file/zip_file_traversal.go +++ b/internal/file/zip_file_traversal.go @@ -15,6 +15,7 @@ import ( ) const ( + // represents the order of bytes _ = iota KB = 1 << (10 * iota) MB @@ -33,6 +34,7 @@ func newZipTraverseRequest(paths ...string) zipTraversalRequest { return results } +// TraverseFilesInZip enumerates all paths stored within a zip archive using the visitor pattern. func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths ...string) error { request := newZipTraverseRequest(paths...) @@ -63,6 +65,7 @@ func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths return nil } +// ExtractFromZipToUniqueTempFile extracts select paths for the given archive to a temporary directory, returning file openers for each file extracted. func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (map[string]Opener, error) { results := make(map[string]Opener) @@ -121,6 +124,7 @@ func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (m return results, TraverseFilesInZip(archivePath, visitor, paths...) } +// ContentsFromZip extracts select paths for the given archive and returns a set of string contents for each path. func ContentsFromZip(archivePath string, paths ...string) (map[string]string, error) { results := make(map[string]string) @@ -162,6 +166,7 @@ func ContentsFromZip(archivePath string, paths ...string) (map[string]string, er return results, TraverseFilesInZip(archivePath, visitor, paths...) } +// UnzipToDir extracts a zip archive to a target directory. func UnzipToDir(archivePath, targetDir string) error { visitor := func(file *zip.File) error { // the zip-slip attack protection is still being erroneously detected diff --git a/internal/format/color.go b/internal/format/color.go deleted file mode 100644 index fa1757c34..000000000 --- a/internal/format/color.go +++ /dev/null @@ -1,21 +0,0 @@ -package format - -import "fmt" - -const ( - DefaultColor Color = iota + 30 - Red - Green - Yellow - Blue - Magenta - Cyan - White -) - -type Color uint8 - -// TODO: not cross platform (windows...) -func (c Color) Format(s string) string { - return fmt.Sprintf("\x1b[%dm%s\x1b[0m", c, s) -} diff --git a/internal/log/log.go b/internal/log/log.go index 5ea65c839..0dd2199c6 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -1,37 +1,49 @@ +/* +Package log contains the singleton object and helper functions for facilitating logging within the syft library. +*/ package log import "github.com/anchore/syft/syft/logger" +// Log is the singleton used to facilitate logging internally within syft var Log logger.Logger = &nopLogger{} +// Errorf takes a formatted template string and template arguments for the error logging level. func Errorf(format string, args ...interface{}) { Log.Errorf(format, args...) } +// Error logs the given arguments at the error logging level. func Error(args ...interface{}) { Log.Error(args...) } +// Warnf takes a formatted template string and template arguments for the warning logging level. func Warnf(format string, args ...interface{}) { Log.Warnf(format, args...) } +// Warn logs the given arguments at the warning logging level. func Warn(args ...interface{}) { Log.Warn(args...) } +// Infof takes a formatted template string and template arguments for the info logging level. func Infof(format string, args ...interface{}) { Log.Infof(format, args...) } +// Info logs the given arguments at the info logging level. func Info(args ...interface{}) { Log.Info(args...) } +// Debugf takes a formatted template string and template arguments for the debug logging level. func Debugf(format string, args ...interface{}) { Log.Debugf(format, args...) } +// Debug logs the given arguments at the debug logging level. func Debug(args ...interface{}) { Log.Debug(args...) } diff --git a/internal/logger/doc.go b/internal/logger/doc.go new file mode 100644 index 000000000..59647a80c --- /dev/null +++ b/internal/logger/doc.go @@ -0,0 +1,4 @@ +/* +Package logger contains implementations for the syft.logger.Logger interface. +*/ +package logger diff --git a/internal/logger/logrus.go b/internal/logger/logrus.go index 006e5f9c9..96fad888c 100644 --- a/internal/logger/logrus.go +++ b/internal/logger/logrus.go @@ -10,6 +10,7 @@ import ( prefixed "github.com/x-cray/logrus-prefixed-formatter" ) +// LogrusConfig contains all configurable values for the Logrus logger type LogrusConfig struct { EnableConsole bool EnableFile bool @@ -18,16 +19,19 @@ type LogrusConfig struct { FileLocation string } +// LogrusLogger contains all runtime values for using Logrus with the configured output target and input configuration values. type LogrusLogger struct { Config LogrusConfig Logger *logrus.Logger Output io.Writer } +// LogrusNestedLogger is a wrapper for Logrus to enable nested logging configuration (loggers that always attach key-value pairs to all log entries) type LogrusNestedLogger struct { Logger *logrus.Entry } +// NewLogrusLogger creates a new LogrusLogger with the given configuration func NewLogrusLogger(cfg LogrusConfig) *LogrusLogger { appLogger := logrus.New() @@ -76,66 +80,82 @@ func NewLogrusLogger(cfg LogrusConfig) *LogrusLogger { } } +// Debugf takes a formatted template string and template arguments for the debug logging level. func (l *LogrusLogger) Debugf(format string, args ...interface{}) { l.Logger.Debugf(format, args...) } +// Infof takes a formatted template string and template arguments for the info logging level. func (l *LogrusLogger) Infof(format string, args ...interface{}) { l.Logger.Infof(format, args...) } +// Warnf takes a formatted template string and template arguments for the warning logging level. func (l *LogrusLogger) Warnf(format string, args ...interface{}) { l.Logger.Warnf(format, args...) } +// Errorf takes a formatted template string and template arguments for the error logging level. func (l *LogrusLogger) Errorf(format string, args ...interface{}) { l.Logger.Errorf(format, args...) } +// Debug logs the given arguments at the debug logging level. func (l *LogrusLogger) Debug(args ...interface{}) { l.Logger.Debug(args...) } +// Info logs the given arguments at the info logging level. func (l *LogrusLogger) Info(args ...interface{}) { l.Logger.Info(args...) } +// Warn logs the given arguments at the warning logging level. func (l *LogrusLogger) Warn(args ...interface{}) { l.Logger.Warn(args...) } +// Error logs the given arguments at the error logging level. func (l *LogrusLogger) Error(args ...interface{}) { l.Logger.Error(args...) } +// Debugf takes a formatted template string and template arguments for the debug logging level. func (l *LogrusNestedLogger) Debugf(format string, args ...interface{}) { l.Logger.Debugf(format, args...) } +// Infof takes a formatted template string and template arguments for the info logging level. func (l *LogrusNestedLogger) Infof(format string, args ...interface{}) { l.Logger.Infof(format, args...) } +// Warnf takes a formatted template string and template arguments for the warning logging level. func (l *LogrusNestedLogger) Warnf(format string, args ...interface{}) { l.Logger.Warnf(format, args...) } +// Errorf takes a formatted template string and template arguments for the error logging level. func (l *LogrusNestedLogger) Errorf(format string, args ...interface{}) { l.Logger.Errorf(format, args...) } +// Debug logs the given arguments at the debug logging level. func (l *LogrusNestedLogger) Debug(args ...interface{}) { l.Logger.Debug(args...) } +// Info logs the given arguments at the info logging level. func (l *LogrusNestedLogger) Info(args ...interface{}) { l.Logger.Info(args...) } +// Warn logs the given arguments at the warning logging level. func (l *LogrusNestedLogger) Warn(args ...interface{}) { l.Logger.Warn(args...) } +// Error logs the given arguments at the error logging level. func (l *LogrusNestedLogger) Error(args ...interface{}) { l.Logger.Error(args...) } diff --git a/internal/stringset.go b/internal/stringset.go index 41518aaad..2bd68a4b1 100644 --- a/internal/stringset.go +++ b/internal/stringset.go @@ -1,11 +1,16 @@ package internal +import "sort" + +// StringSet represents a set of string types. type StringSet map[string]struct{} +// NewStringSet creates a new empty StringSet. func NewStringSet() StringSet { return make(StringSet) } +// NewStringSetFromSlice creates a StringSet populated with values from the given slice. func NewStringSetFromSlice(start []string) StringSet { ret := make(StringSet) for _, s := range start { @@ -14,19 +19,23 @@ func NewStringSetFromSlice(start []string) StringSet { return ret } +// Add a string to the set. func (s StringSet) Add(i string) { s[i] = struct{}{} } +// Remove a string from the set. func (s StringSet) Remove(i string) { delete(s, i) } +// Contains indicates if the given string is contained within the set. func (s StringSet) Contains(i string) bool { _, ok := s[i] return ok } +// ToSlice returns a sorted slice of strings that are contained within the set. func (s StringSet) ToSlice() []string { ret := make([]string, len(s)) idx := 0 @@ -34,5 +43,6 @@ func (s StringSet) ToSlice() []string { ret[idx] = v idx++ } + sort.Strings(ret) return ret } diff --git a/internal/version/build.go b/internal/version/build.go index e0c960228..3d6702c0c 100644 --- a/internal/version/build.go +++ b/internal/version/build.go @@ -1,3 +1,6 @@ +/* +Package version contains all build time metadata (version, build time, git commit, etc). +*/ package version import ( diff --git a/internal/version/update.go b/internal/version/update.go index bdba75f4d..3aa13ef3a 100644 --- a/internal/version/update.go +++ b/internal/version/update.go @@ -18,6 +18,7 @@ var latestAppVersionURL = struct { path: fmt.Sprintf("/%s/releases/latest/VERSION", internal.ApplicationName), } +// IsUpdateAvailable indicates if there is a newer application version available, and if so, what the new version is. func IsUpdateAvailable() (bool, string, error) { currentVersionStr := FromBuild().Version currentVersion, err := hashiVersion.NewVersion(currentVersionStr) diff --git a/syft/cataloger/python/package_cataloger_test.go b/syft/cataloger/python/package_cataloger_test.go index be3033b97..2ad16da73 100644 --- a/syft/cataloger/python/package_cataloger_test.go +++ b/syft/cataloger/python/package_cataloger_test.go @@ -142,12 +142,12 @@ func TestPythonPackageWheelCataloger(t *testing.T) { AuthorEmail: "me@kennethreitz.org", SitePackagesRootPath: "test-fixtures", Files: []pkg.PythonFileRecord{ - {Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.Digest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"}, - {Path: "requests/__init__.py", Digest: &pkg.Digest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"}, + {Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"}, + {Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"}, {Path: "requests/__pycache__/__version__.cpython-38.pyc"}, {Path: "requests/__pycache__/utils.cpython-38.pyc"}, - {Path: "requests/__version__.py", Digest: &pkg.Digest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"}, - {Path: "requests/utils.py", Digest: &pkg.Digest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"}, + {Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"}, + {Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"}, }, TopLevelPackages: []string{"requests"}, }, @@ -174,11 +174,11 @@ func TestPythonPackageWheelCataloger(t *testing.T) { AuthorEmail: "georg@python.org", SitePackagesRootPath: "test-fixtures", Files: []pkg.PythonFileRecord{ - {Path: "../../../bin/pygmentize", Digest: &pkg.Digest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, - {Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.Digest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, + {Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, + {Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, {Path: "Pygments-2.6.1.dist-info/RECORD"}, {Path: "pygments/__pycache__/__init__.cpython-38.pyc"}, - {Path: "pygments/util.py", Digest: &pkg.Digest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, + {Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, }, TopLevelPackages: []string{"pygments", "something_else"}, }, diff --git a/syft/cataloger/python/parse_wheel_egg_record.go b/syft/cataloger/python/parse_wheel_egg_record.go index 42faafa3f..4f3e828ac 100644 --- a/syft/cataloger/python/parse_wheel_egg_record.go +++ b/syft/cataloger/python/parse_wheel_egg_record.go @@ -44,7 +44,7 @@ func parseWheelOrEggRecord(reader io.Reader) ([]pkg.PythonFileRecord, error) { return nil, fmt.Errorf("unexpected python record digest: %q", item) } - record.Digest = &pkg.Digest{ + record.Digest = &pkg.PythonFileDigest{ Algorithm: fields[0], Value: fields[1], } diff --git a/syft/cataloger/python/parse_wheel_egg_record_test.go b/syft/cataloger/python/parse_wheel_egg_record_test.go index d14868e0f..c0cf578b5 100644 --- a/syft/cataloger/python/parse_wheel_egg_record_test.go +++ b/syft/cataloger/python/parse_wheel_egg_record_test.go @@ -16,22 +16,22 @@ func TestParseWheelEggRecord(t *testing.T) { { Fixture: "test-fixtures/egg-info/RECORD", ExpectedMetadata: []pkg.PythonFileRecord{ - {Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.Digest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"}, - {Path: "requests/__init__.py", Digest: &pkg.Digest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"}, + {Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"}, + {Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"}, {Path: "requests/__pycache__/__version__.cpython-38.pyc"}, {Path: "requests/__pycache__/utils.cpython-38.pyc"}, - {Path: "requests/__version__.py", Digest: &pkg.Digest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"}, - {Path: "requests/utils.py", Digest: &pkg.Digest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"}, + {Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"}, + {Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"}, }, }, { Fixture: "test-fixtures/dist-info/RECORD", ExpectedMetadata: []pkg.PythonFileRecord{ - {Path: "../../../bin/pygmentize", Digest: &pkg.Digest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, - {Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.Digest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, + {Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, + {Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, {Path: "Pygments-2.6.1.dist-info/RECORD"}, {Path: "pygments/__pycache__/__init__.cpython-38.pyc"}, - {Path: "pygments/util.py", Digest: &pkg.Digest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, + {Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, }, }, } diff --git a/syft/cataloger/rpmdb/parse_rpmdb.go b/syft/cataloger/rpmdb/parse_rpmdb.go index fcf251cd0..9f0b3ed80 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb.go +++ b/syft/cataloger/rpmdb/parse_rpmdb.go @@ -51,9 +51,8 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader } p := pkg.Package{ - Name: entry.Name, - Version: fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does - //Version: fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch), + Name: entry.Name, + Version: fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does, instead of fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch) Locations: []source.Location{dbLocation}, FoundBy: catalogerName, Type: pkg.RpmPkg, diff --git a/syft/distro/distro.go b/syft/distro/distro.go index 446357401..a78f5ab33 100644 --- a/syft/distro/distro.go +++ b/syft/distro/distro.go @@ -6,6 +6,7 @@ import ( hashiVer "github.com/hashicorp/go-version" ) +// Distro represents a Linux Distribution. type Distro struct { Type Type Version *hashiVer.Version @@ -20,6 +21,7 @@ func NewUnknownDistro() Distro { } } +// NewDistro creates a new Distro object populated with the given values. func NewDistro(t Type, ver, like string) (Distro, error) { if ver == "" { return Distro{Type: t}, nil @@ -36,6 +38,12 @@ func NewDistro(t Type, ver, like string) (Distro, error) { }, nil } +// Name provides a string repr of the distro +func (d Distro) Name() string { + return string(d.Type) +} + +// MajorVersion returns the major version value from the pseudo-semantically versioned distro version value. func (d Distro) MajorVersion() string { if d.Version == nil { return fmt.Sprint("(version unknown)") @@ -43,10 +51,12 @@ func (d Distro) MajorVersion() string { return fmt.Sprintf("%d", d.Version.Segments()[0]) } +// FullVersion returns the original user version value. func (d Distro) FullVersion() string { return d.RawVersion } +// String returns a human-friendly representation of the Linux distribution. func (d Distro) String() string { versionStr := "(version unknown)" if d.RawVersion != "" { @@ -54,8 +64,3 @@ func (d Distro) String() string { } return fmt.Sprintf("%s %s", d.Type, versionStr) } - -// Name provides a string repr of the distro -func (d Distro) Name() string { - return string(d.Type) -} diff --git a/syft/distro/type.go b/syft/distro/type.go index 9b6f4c035..aac86384a 100644 --- a/syft/distro/type.go +++ b/syft/distro/type.go @@ -1,8 +1,10 @@ package distro +// Type represents the different Linux distribution options type Type string const ( + // represents the set of valid/supported Linux Distributions UnknownDistroType Type = "UnknownDistroType" Debian Type = "debian" Ubuntu Type = "ubuntu" @@ -17,6 +19,7 @@ const ( OpenSuseLeap Type = "opensuseleap" ) +// All contains all Linux distribution options var All = []Type{ Debian, Ubuntu, @@ -46,6 +49,7 @@ var IDMapping = map[string]Type{ "opensuse-leap": OpenSuseLeap, } +// String returns the string representation of the given Linux distribution. func (t Type) String() string { return string(t) } diff --git a/syft/event/event.go b/syft/event/event.go index e76ebd983..caf1e41f5 100644 --- a/syft/event/event.go +++ b/syft/event/event.go @@ -7,7 +7,12 @@ package event import "github.com/wagoodman/go-partybus" const ( + // AppUpdateAvailable is a partybus event that occurs when an application update is available AppUpdateAvailable partybus.EventType = "syft-app-update-available" - CatalogerStarted partybus.EventType = "syft-cataloger-started-event" - CatalogerFinished partybus.EventType = "syft-cataloger-finished-event" + + // CatalogerStarted is a partybus event that occurs when the package cataloging has begun + CatalogerStarted partybus.EventType = "syft-cataloger-started-event" + + // CatalogerFinished is a partybus event that occurs when the package cataloging has completed + CatalogerFinished partybus.EventType = "syft-cataloger-finished-event" ) diff --git a/syft/logger/logger.go b/syft/logger/logger.go index 3816296ff..3ace363f5 100644 --- a/syft/logger/logger.go +++ b/syft/logger/logger.go @@ -1,8 +1,9 @@ /* -Defines the logging interface which is used throughout the syft library. +Package logger defines the logging interface which is used throughout the syft library. */ package logger +// Logger represents the behavior for logging within the syft library. type Logger interface { Errorf(format string, args ...interface{}) Error(args ...interface{}) diff --git a/syft/pkg/apk_metadata.go b/syft/pkg/apk_metadata.go index ef036509a..705528fd0 100644 --- a/syft/pkg/apk_metadata.go +++ b/syft/pkg/apk_metadata.go @@ -35,6 +35,7 @@ type ApkFileRecord struct { Checksum string `json:"checksum,omitempty"` } +// PackageURL returns the PURL for the specific Alpine package (see https://github.com/package-url/purl-spec) func (m ApkMetadata) PackageURL() string { pURL := packageurl.NewPackageURL( // note: this is currently a candidate and not technically within spec diff --git a/syft/pkg/dpkg_metadata.go b/syft/pkg/dpkg_metadata.go index f1e80802e..c810a8198 100644 --- a/syft/pkg/dpkg_metadata.go +++ b/syft/pkg/dpkg_metadata.go @@ -17,11 +17,13 @@ type DpkgMetadata struct { Files []DpkgFileRecord `json:"files"` } +// DpkgFileRecord represents a single file attributed to a debian package. type DpkgFileRecord struct { Path string `json:"path"` MD5 string `json:"md5"` } +// PackageURL returns the PURL for the specific Debian package (see https://github.com/package-url/purl-spec) func (m DpkgMetadata) PackageURL(d distro.Distro) string { pURL := packageurl.NewPackageURL( // TODO: replace with `packageurl.TypeDebian` upon merge of https://github.com/package-url/packageurl-go/pull/21 diff --git a/syft/pkg/gem_metadata.go b/syft/pkg/gem_metadata.go index c06a244b2..51a07a148 100644 --- a/syft/pkg/gem_metadata.go +++ b/syft/pkg/gem_metadata.go @@ -1,5 +1,6 @@ package pkg +// GemMetadata represents all metadata parsed from the gemspec file type GemMetadata struct { Name string `mapstructure:"name" json:"name"` Version string `mapstructure:"version" json:"version"` diff --git a/syft/pkg/java_metadata.go b/syft/pkg/java_metadata.go index d4852eb55..beb03463f 100644 --- a/syft/pkg/java_metadata.go +++ b/syft/pkg/java_metadata.go @@ -26,6 +26,7 @@ type JavaManifest struct { NamedSections map[string]map[string]string `json:"namedSections,omitempty"` } +// PackageURL returns the PURL for the specific Alpine package (see https://github.com/package-url/purl-spec) func (m JavaMetadata) PackageURL() string { if m.PomProperties != nil { pURL := packageurl.NewPackageURL( diff --git a/syft/pkg/language.go b/syft/pkg/language.go index 6fc1b184a..9f2ab7dae 100644 --- a/syft/pkg/language.go +++ b/syft/pkg/language.go @@ -1,8 +1,10 @@ package pkg +// Language represents a single programming language. type Language string const ( + // the full set of supported programming languages UnknownLanguage Language = "UnknownLanguage" Java Language = "java" JavaScript Language = "javascript" @@ -11,6 +13,7 @@ const ( Go Language = "go" ) +// AllLanguages is a set of all programming languages detected by syft. var AllLanguages = []Language{ Java, JavaScript, @@ -19,6 +22,7 @@ var AllLanguages = []Language{ Go, } +// String returns the string representation of the language. func (l Language) String() string { return string(l) } diff --git a/syft/pkg/metadata.go b/syft/pkg/metadata.go index c1e402934..65d58724f 100644 --- a/syft/pkg/metadata.go +++ b/syft/pkg/metadata.go @@ -1,14 +1,16 @@ package pkg +// MetadataType represents the data shape stored within pkg.Package.Metadata. type MetadataType string const ( + // this is the full set of data shapes that can be represented within the pkg.Package.Metadata field UnknownMetadataType MetadataType = "UnknownMetadata" - ApkMetadataType MetadataType = "apk-metadata" - DpkgMetadataType MetadataType = "dpkg-metadata" - GemMetadataType MetadataType = "gem-metadata" - JavaMetadataType MetadataType = "java-metadata" - NpmPackageJSONMetadataType MetadataType = "npm-package-json-metadata" - RpmdbMetadataType MetadataType = "rpmdb-metadata" - PythonPackageMetadataType MetadataType = "python-package-metadata" + ApkMetadataType MetadataType = "ApkMetadata" + DpkgMetadataType MetadataType = "DpkgMetadata" + GemMetadataType MetadataType = "GemMetadata" + JavaMetadataType MetadataType = "JavaMetadata" + NpmPackageJSONMetadataType MetadataType = "NpmPackageJsonMetadata" + RpmdbMetadataType MetadataType = "RpmdbMetadata" + PythonPackageMetadataType MetadataType = "PythonPackageMetadata" ) diff --git a/syft/pkg/package.go b/syft/pkg/package.go index 7e97ee8a8..d2c83a363 100644 --- a/syft/pkg/package.go +++ b/syft/pkg/package.go @@ -14,6 +14,7 @@ import ( "github.com/package-url/packageurl-go" ) +// ID represents a unique value for each package added to a package catalog. type ID int64 // Package represents an application or library that has been bundled into a distributable format. diff --git a/syft/pkg/python_package_metadata.go b/syft/pkg/python_package_metadata.go index 637e6220c..e01771602 100644 --- a/syft/pkg/python_package_metadata.go +++ b/syft/pkg/python_package_metadata.go @@ -1,15 +1,16 @@ package pkg -type Digest struct { +// PythonFileDigest represents the file metadata for a single file attributed to a python package. +type PythonFileDigest struct { Algorithm string `json:"algorithm"` Value string `json:"value"` } // PythonFileRecord represents a single entry within a RECORD file for a python wheel or egg package type PythonFileRecord struct { - Path string `json:"path"` - Digest *Digest `json:"digest,omitempty"` - Size string `json:"size,omitempty"` + Path string `json:"path"` + Digest *PythonFileDigest `json:"digest,omitempty"` + Size string `json:"size,omitempty"` } // PythonPackageMetadata represents all captured data for a python egg or wheel package. diff --git a/syft/pkg/rpmdb_metadata.go b/syft/pkg/rpmdb_metadata.go index c9823c1a8..6b3ea723c 100644 --- a/syft/pkg/rpmdb_metadata.go +++ b/syft/pkg/rpmdb_metadata.go @@ -21,6 +21,7 @@ type RpmdbMetadata struct { Files []RpmdbFileRecord `json:"files"` } +// RpmdbFileRecord represents the file metadata for a single file attributed to a RPM package. type RpmdbFileRecord struct { Path string `json:"path"` Mode RpmdbFileMode `json:"mode"` @@ -28,8 +29,10 @@ type RpmdbFileRecord struct { SHA256 string `json:"sha256"` } +// RpmdbFileMode is the raw file mode for a single file. This can be interpreted as the linux stat.h mode (see https://pubs.opengroup.org/onlinepubs/007908799/xsh/sysstat.h.html) type RpmdbFileMode uint16 +// PackageURL returns the PURL for the specific RHEL package (see https://github.com/package-url/purl-spec) func (m RpmdbMetadata) PackageURL(d distro.Distro) string { pURL := packageurl.NewPackageURL( packageurl.TypeRPM, diff --git a/syft/pkg/type.go b/syft/pkg/type.go index 85dc83159..49d3d3b26 100644 --- a/syft/pkg/type.go +++ b/syft/pkg/type.go @@ -6,6 +6,7 @@ import "github.com/package-url/packageurl-go" type Type string const ( + // the full set of supported packages UnknownPkg Type = "UnknownPackage" ApkPkg Type = "apk" GemPkg Type = "gem" @@ -18,6 +19,7 @@ const ( GoModulePkg Type = "go-module" ) +// AllPkgs represents all supported package types var AllPkgs = []Type{ ApkPkg, GemPkg, @@ -30,6 +32,7 @@ var AllPkgs = []Type{ GoModulePkg, } +// PackageURLType returns the PURL package type for the current package. func (t Type) PackageURLType() string { switch t { case ApkPkg: diff --git a/syft/presenter/json/artifact.go b/syft/presenter/json/artifact.go index e26bdc654..d5bf19872 100644 --- a/syft/presenter/json/artifact.go +++ b/syft/presenter/json/artifact.go @@ -34,7 +34,6 @@ type artifactMetadataUnpacker struct { } func NewArtifact(p *pkg.Package) (Artifact, error) { - return Artifact{ artifactBasicMetadata: artifactBasicMetadata{ Name: p.Name, @@ -67,6 +66,7 @@ func (a Artifact) ToPackage() pkg.Package { } } +// nolint:funlen func (a *Artifact) UnmarshalJSON(b []byte) error { var basic artifactBasicMetadata if err := json.Unmarshal(b, &basic); err != nil { @@ -128,7 +128,6 @@ func (a *Artifact) UnmarshalJSON(b []byte) error { // there may be packages with no metadata, which is OK default: return fmt.Errorf("unsupported package metadata type: %+v", a.MetadataType) - } return nil diff --git a/syft/presenter/json/source.go b/syft/presenter/json/source.go index cd777b93e..7a5ad6ad2 100644 --- a/syft/presenter/json/source.go +++ b/syft/presenter/json/source.go @@ -55,7 +55,6 @@ func (s *Source) UnmarshalJSON(b []byte) error { s.Target = payload default: return fmt.Errorf("unsupported package metadata type: %+v", s.Type) - } return nil diff --git a/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden b/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden index b88f2ffef..4876f5949 100644 --- a/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden +++ b/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden @@ -14,7 +14,7 @@ "MIT" ], "language": "python", - "metadataType": "python-package-metadata", + "metadataType": "PythonPackageMetadata", "metadata": { "name": "package-1", "version": "1.0.1", @@ -37,7 +37,7 @@ ], "licenses": null, "language": "", - "metadataType": "dpkg-metadata", + "metadataType": "DpkgMetadata", "metadata": { "package": "package-2", "source": "", diff --git a/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden b/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden index 854f05ea0..81ba778fd 100644 --- a/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden +++ b/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden @@ -15,7 +15,7 @@ "MIT" ], "language": "python", - "metadataType": "python-package-metadata", + "metadataType": "PythonPackageMetadata", "metadata": { "name": "package-1", "version": "1.0.1", @@ -39,7 +39,7 @@ ], "licenses": null, "language": "", - "metadataType": "dpkg-metadata", + "metadataType": "DpkgMetadata", "metadata": { "package": "package-2", "source": "", diff --git a/syft/presenter/text/presenter.go b/syft/presenter/text/presenter.go index 57764eb8b..8e86fea47 100644 --- a/syft/presenter/text/presenter.go +++ b/syft/presenter/text/presenter.go @@ -10,11 +10,13 @@ import ( "github.com/anchore/syft/syft/source" ) +// Presenter is a human-friendly text presenter to represent package and source data. type Presenter struct { catalog *pkg.Catalog srcMetadata source.Metadata } +// NewPresenter creates a new presenter for the given set of catalog and image data. func NewPresenter(catalog *pkg.Catalog, srcMetadata source.Metadata) *Presenter { return &Presenter{ catalog: catalog, diff --git a/syft/source/all_layers_resolver.go b/syft/source/all_layers_resolver.go index 1c58e89b1..7b94328c1 100644 --- a/syft/source/all_layers_resolver.go +++ b/syft/source/all_layers_resolver.go @@ -102,6 +102,7 @@ func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) { } // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. +// nolint:gocognit func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error) { uniqueFileIDs := file.NewFileReferenceSet() uniqueLocations := make([]Location, 0) @@ -141,6 +142,8 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error) return uniqueLocations, nil } +// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. +// This is helpful when attempting to find a file that is in the same layer or lower as another file. func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) *Location { entry, err := r.img.FileCatalog.Get(location.ref) if err != nil { @@ -157,13 +160,13 @@ func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) * return &relativeLocation } -// MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a +// MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a // file.Reference is a path relative to a particular layer. func (r *AllLayersResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) { return mapLocationRefs(r.img.MultipleFileContentsByRef, locations) } -// FileContentsByRef fetches file contents for a single file reference, irregardless of the source layer. +// FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer. // If the path does not exist an error is returned. func (r *AllLayersResolver) FileContentsByLocation(location Location) (string, error) { return r.img.FileContentsByRef(location.ref) diff --git a/syft/source/directory_resolver.go b/syft/source/directory_resolver.go index 3d6904e74..c05d229c4 100644 --- a/syft/source/directory_resolver.go +++ b/syft/source/directory_resolver.go @@ -80,6 +80,9 @@ func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) { return result, nil } +// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. +// This is helpful when attempting to find a file that is in the same layer or lower as another file. For the +// DirectoryResolver, this is a simple path lookup. func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Location { paths, err := s.FilesByPath(path) if err != nil { @@ -92,7 +95,7 @@ func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Locatio return &paths[0] } -// MultipleFileContentsByRef returns the file contents for all file.References relative a directory. +// MultipleFileContentsByLocation returns the file contents for all file.References relative a directory. func (s DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) { refContents := make(map[Location]string) for _, location := range locations { @@ -106,7 +109,7 @@ func (s DirectoryResolver) MultipleFileContentsByLocation(locations []Location) return refContents, nil } -// FileContentsByRef fetches file contents for a single file reference relative to a directory. +// FileContentsByLocation fetches file contents for a single file reference relative to a directory. // If the path does not exist an error is returned. func (s DirectoryResolver) FileContentsByLocation(location Location) (string, error) { contents, err := fileContents(location.Path) diff --git a/syft/source/image_metadata.go b/syft/source/image_metadata.go index 9bd8a30a9..ce6c59223 100644 --- a/syft/source/image_metadata.go +++ b/syft/source/image_metadata.go @@ -2,6 +2,8 @@ package source import "github.com/anchore/stereoscope/pkg/image" +// ImageMetadata represents all static metadata that defines what a container image is. This is useful to later describe +// "what" was cataloged without needing the more complicated stereoscope Image objects or Resolver objects. type ImageMetadata struct { UserInput string `json:"userInput"` Scope Scope `json:"scope"` // specific perspective to catalog @@ -12,12 +14,14 @@ type ImageMetadata struct { Tags []string `json:"tags"` } +// LayerMetadata represents all static metadata that defines what a container image layer is. type LayerMetadata struct { MediaType string `json:"mediaType"` Digest string `json:"digest"` Size int64 `json:"size"` } +// NewImageMetadata creates a new ImageMetadata object populated from the given stereoscope Image object and user configuration. func NewImageMetadata(img *image.Image, userInput string, scope Scope) ImageMetadata { // populate artifacts... tags := make([]string, len(img.Metadata.Tags)) diff --git a/syft/source/image_squash_resolver.go b/syft/source/image_squash_resolver.go index 0e34b3355..3df142a58 100644 --- a/syft/source/image_squash_resolver.go +++ b/syft/source/image_squash_resolver.go @@ -104,6 +104,9 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error return uniqueLocations, nil } +// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. +// This is helpful when attempting to find a file that is in the same layer or lower as another file. For the +// ImageSquashResolver, this is a simple path lookup. func (r *ImageSquashResolver) RelativeFileByPath(_ Location, path string) *Location { paths, err := r.FilesByPath(path) if err != nil { @@ -116,13 +119,13 @@ func (r *ImageSquashResolver) RelativeFileByPath(_ Location, path string) *Locat return &paths[0] } -// MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a +// MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a // file.Reference is a path relative to a particular layer, in this case only from the squashed representation. func (r *ImageSquashResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) { return mapLocationRefs(r.img.MultipleFileContentsByRef, locations) } -// FileContentsByRef fetches file contents for a single file reference, irregardless of the source layer. +// FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer. // If the path does not exist an error is returned. func (r *ImageSquashResolver) FileContentsByLocation(location Location) (string, error) { return r.img.FileContentsByRef(location.ref) diff --git a/syft/source/location.go b/syft/source/location.go index 1fc1250ae..774fd6413 100644 --- a/syft/source/location.go +++ b/syft/source/location.go @@ -7,18 +7,21 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) +// Location represents a path relative to a particular filesystem. type Location struct { - Path string `json:"path"` - FileSystemID string `json:"layerID,omitempty"` // TODO: comment - ref file.Reference + Path string `json:"path"` // The string path of the location (e.g. /etc/hosts) + FileSystemID string `json:"layerID,omitempty"` // An ID representing the filesystem. For container images this is a layer digest, directories or root filesystem this is blank. + ref file.Reference // The file reference relative to the stereoscope.FileCatalog that has more information about this location. } +// NewLocation creates a new Location representing a path without denoting a filesystem or FileCatalog reference. func NewLocation(path string) Location { return Location{ Path: path, } } +// NewLocationFromImage creates a new Location representing the given path (extracted from the ref) relative to the given image. func NewLocationFromImage(ref file.Reference, img *image.Image) Location { entry, err := img.FileCatalog.Get(ref) if err != nil { diff --git a/syft/source/metadata.go b/syft/source/metadata.go index ef3092564..b9747362e 100644 --- a/syft/source/metadata.go +++ b/syft/source/metadata.go @@ -1,5 +1,6 @@ package source +// Metadata represents any static source data that helps describe "what" was cataloged. type Metadata struct { Scheme Scheme // the source data scheme type (directory or image) ImageMetadata ImageMetadata // all image info (image only) diff --git a/syft/source/resolver.go b/syft/source/resolver.go index d6193c4c3..0f22c2096 100644 --- a/syft/source/resolver.go +++ b/syft/source/resolver.go @@ -19,7 +19,7 @@ type ContentResolver interface { // TODO: we should consider refactoring to return a set of io.Readers or file.Openers instead of the full contents themselves (allow for optional buffering). } -// FileResolver knows how to get file.References for given string paths and globs +// FileResolver knows how to get file.References for given string paths and globs type FileResolver interface { // FilesByPath fetches a set of file references which have the given path (for an image, there may be multiple matches) FilesByPath(paths ...string) ([]Location, error) diff --git a/syft/source/scheme.go b/syft/source/scheme.go index c12a1f69f..d2f1c73eb 100644 --- a/syft/source/scheme.go +++ b/syft/source/scheme.go @@ -9,12 +9,16 @@ import ( "github.com/spf13/afero" ) +// Scheme represents the optional prefixed string at the beginning of a user request (e.g. "docker:"). type Scheme string const ( - UnknownScheme Scheme = "unknown-scheme" - DirectoryScheme Scheme = "directory-scheme" - ImageScheme Scheme = "image-scheme" + // UnknownScheme is the default scheme + UnknownScheme Scheme = "UnknownScheme" + // DirectoryScheme indicates the source being cataloged is a directory on the root filesystem + DirectoryScheme Scheme = "DirectoryScheme" + // ImageScheme indicates the source being cataloged is a container image + ImageScheme Scheme = "ImageScheme" ) func detectScheme(fs afero.Fs, imageDetector sourceDetector, userInput string) (Scheme, string, error) { diff --git a/syft/source/scope.go b/syft/source/scope.go index 92cb0b9b1..e959d1a42 100644 --- a/syft/source/scope.go +++ b/syft/source/scope.go @@ -2,19 +2,25 @@ package source import "strings" +// Scope indicates "how" or from "which perspectives" the source object should be cataloged from. type Scope string const ( - UnknownScope Scope = "UnknownScope" - SquashedScope Scope = "Squashed" + // UnknownScope is the default scope + UnknownScope Scope = "UnknownScope" + // SquashedScope indicates to only catalog content visible from the squashed filesystem representation (what can be seen only within the container at runtime) + SquashedScope Scope = "Squashed" + // AllLayersScope indicates to catalog content on all layers, irregardless if it is visible from the container at runtime. AllLayersScope Scope = "AllLayers" ) +// AllScopes is a slice containing all possible scope options var AllScopes = []Scope{ SquashedScope, AllLayersScope, } +// ParseScope returns a scope as indicated from the given string. func ParseScope(userStr string) Scope { switch strings.ToLower(userStr) { case strings.ToLower(SquashedScope.String()): From 0ad8c53ec2a9ad489633410acb998759fdd4cb3c Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Tue, 17 Nov 2020 10:07:20 -0500 Subject: [PATCH 11/13] Fix cyclonedx test fixture usage of dynamic digest value Signed-off-by: Dan Luhring --- syft/presenter/cyclonedx/presenter_test.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/syft/presenter/cyclonedx/presenter_test.go b/syft/presenter/cyclonedx/presenter_test.go index a4ba735fd..8829b6ec5 100644 --- a/syft/presenter/cyclonedx/presenter_test.go +++ b/syft/presenter/cyclonedx/presenter_test.go @@ -156,6 +156,14 @@ func TestCycloneDxImgsPresenter(t *testing.T) { t.Fatal(err) } + // This accounts for the non-deterministic digest value that we end up with when + // we build a container image dynamically during testing. Ultimately, we should + // use a golden image as a test fixture in place of building this image during + // testing. At that time, this line will no longer be necessary. + // + // This value is sourced from the "version" node in "./test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden" + s.Metadata.ImageMetadata.Digest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368" + pres := NewPresenter(catalog, s.Metadata, d) // run presenter @@ -177,7 +185,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) { if !bytes.Equal(expected, actual) { dmp := diffmatchpatch.New() - diffs := dmp.DiffMain(string(actual), string(expected), true) + diffs := dmp.DiffMain(string(expected), string(actual), true) t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs)) } } From 569a598df703b67a99d12eaff1f6cd620778d9b6 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 17 Nov 2020 12:26:27 -0500 Subject: [PATCH 12/13] minimize pointer usage & order return types consistently Signed-off-by: Alex Goodman --- cmd/cmd.go | 2 +- cmd/root.go | 4 ++-- syft/cataloger/catalog.go | 2 -- syft/lib.go | 20 ++++++++++---------- syft/presenter/presenter.go | 6 +++--- test/integration/distro_test.go | 10 ++-------- test/integration/document_import_test.go | 6 +++--- test/integration/json_schema_test.go | 12 ++++++------ test/integration/pkg_coverage_test.go | 4 ++-- 9 files changed, 29 insertions(+), 37 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index e39ec4efe..4141504a9 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -112,7 +112,7 @@ func logAppConfig() { if err != nil { log.Debugf("Could not display application config: %+v", err) } else { - log.Debugf("Application config:\n%+v", color.Magenta.Sprint(appCfgStr)) + log.Debugf("Application config:\n%+v", color.Magenta.Sprint(string(appCfgStr))) } } diff --git a/cmd/root.go b/cmd/root.go index 3abccd404..7b72fafdd 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -92,7 +92,7 @@ func startWorker(userInput string) <-chan error { } } - catalog, scope, distro, err := syft.Catalog(userInput, appConfig.ScopeOpt) + src, catalog, distro, err := syft.Catalog(userInput, appConfig.ScopeOpt) if err != nil { errs <- fmt.Errorf("failed to catalog input: %+v", err) return @@ -100,7 +100,7 @@ func startWorker(userInput string) <-chan error { bus.Publish(partybus.Event{ Type: event.CatalogerFinished, - Value: presenter.GetPresenter(appConfig.PresenterOpt, scope.Metadata, catalog, distro), + Value: presenter.GetPresenter(appConfig.PresenterOpt, src.Metadata, catalog, distro), }) }() return errs diff --git a/syft/cataloger/catalog.go b/syft/cataloger/catalog.go index 0724be8b6..262fbdbb3 100644 --- a/syft/cataloger/catalog.go +++ b/syft/cataloger/catalog.go @@ -54,8 +54,6 @@ func Catalog(resolver source.Resolver, catalogers ...Cataloger) (*pkg.Catalog, e log.Debugf("cataloger '%s' discovered '%d' packages", theCataloger.Name(), catalogedPackages) packagesDiscovered.N += int64(catalogedPackages) - // helper function to add synthesized information... - for _, p := range packages { catalog.Add(p) } diff --git a/syft/lib.go b/syft/lib.go index d57c61f27..931c3747b 100644 --- a/syft/lib.go +++ b/syft/lib.go @@ -34,22 +34,22 @@ import ( // Catalog the given image from a particular perspective (e.g. squashed source, all-layers source). Returns the discovered // set of packages, the identified Linux distribution, and the source object used to wrap the data source. -func Catalog(userInput string, scoptOpt source.Scope) (*pkg.Catalog, *source.Source, *distro.Distro, error) { +func Catalog(userInput string, scope source.Scope) (source.Source, *pkg.Catalog, distro.Distro, error) { log.Info("cataloging image") - s, cleanup, err := source.New(userInput, scoptOpt) + s, cleanup, err := source.New(userInput, scope) defer cleanup() if err != nil { - return nil, nil, nil, err + return source.Source{}, nil, distro.Distro{}, err } d := IdentifyDistro(s) catalog, err := CatalogFromScope(s) if err != nil { - return nil, nil, nil, err + return source.Source{}, nil, distro.Distro{}, err } - return catalog, &s, &d, nil + return s, catalog, d, nil } // IdentifyDistro attempts to discover what the underlying Linux distribution may be from the available flat files @@ -82,12 +82,12 @@ func CatalogFromScope(s source.Source) (*pkg.Catalog, error) { return cataloger.Catalog(s.Resolver, catalogers...) } -// CatalogFromJSON takes an existing syft report and generates catalog primitives. -func CatalogFromJSON(reader io.Reader) (*pkg.Catalog, *distro.Distro, source.Metadata, error) { +// CatalogFromJSON takes an existing syft report and generates native syft objects. +func CatalogFromJSON(reader io.Reader) (source.Metadata, *pkg.Catalog, distro.Distro, error) { var doc jsonPresenter.Document decoder := json.NewDecoder(reader) if err := decoder.Decode(&doc); err != nil { - return nil, nil, source.Metadata{}, err + return source.Metadata{}, nil, distro.Distro{}, err } var pkgs = make([]pkg.Package, len(doc.Artifacts)) @@ -106,10 +106,10 @@ func CatalogFromJSON(reader io.Reader) (*pkg.Catalog, *distro.Distro, source.Met theDistro, err := distro.NewDistro(distroType, doc.Distro.Version, doc.Distro.IDLike) if err != nil { - return nil, nil, source.Metadata{}, err + return source.Metadata{}, nil, distro.Distro{}, err } - return catalog, &theDistro, doc.Source.ToSourceMetadata(), nil + return doc.Source.ToSourceMetadata(), catalog, theDistro, nil } // SetLogger sets the logger object used for all syft logging calls. diff --git a/syft/presenter/presenter.go b/syft/presenter/presenter.go index 4f1b86224..6c32b60d0 100644 --- a/syft/presenter/presenter.go +++ b/syft/presenter/presenter.go @@ -25,16 +25,16 @@ type Presenter interface { } // GetPresenter returns a presenter for images or directories -func GetPresenter(option Option, srcMetadata source.Metadata, catalog *pkg.Catalog, d *distro.Distro) Presenter { +func GetPresenter(option Option, srcMetadata source.Metadata, catalog *pkg.Catalog, d distro.Distro) Presenter { switch option { case JSONPresenter: - return json.NewPresenter(catalog, srcMetadata, *d) + return json.NewPresenter(catalog, srcMetadata, d) case TextPresenter: return text.NewPresenter(catalog, srcMetadata) case TablePresenter: return table.NewPresenter(catalog) case CycloneDxPresenter: - return cyclonedx.NewPresenter(catalog, srcMetadata, *d) + return cyclonedx.NewPresenter(catalog, srcMetadata, d) default: return nil } diff --git a/test/integration/distro_test.go b/test/integration/distro_test.go index a0ad3fd64..e6a9ce297 100644 --- a/test/integration/distro_test.go +++ b/test/integration/distro_test.go @@ -20,20 +20,14 @@ func TestDistroImage(t *testing.T) { if err != nil { t.Fatalf("failed to catalog image: %+v", err) } - if actualDistro == nil { - t.Fatalf("could not find distro") - } expected, err := distro.NewDistro(distro.Busybox, "1.31.1", "") if err != nil { t.Fatalf("could not create distro: %+v", err) } - diffs := deep.Equal(*actualDistro, expected) - if len(diffs) != 0 { - for _, d := range diffs { - t.Errorf("found distro difference: %+v", d) - } + for _, d := range deep.Equal(actualDistro, expected) { + t.Errorf("found distro difference: %+v", d) } } diff --git a/test/integration/document_import_test.go b/test/integration/document_import_test.go index 056e92f66..86c07a68f 100644 --- a/test/integration/document_import_test.go +++ b/test/integration/document_import_test.go @@ -31,18 +31,18 @@ func TestCatalogFromJSON(t *testing.T) { tarPath := imagetest.GetFixtureImageTarPath(t, test.fixture) defer cleanup() - expectedCatalog, expectedSource, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) + expectedSource, expectedCatalog, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } var buf bytes.Buffer - jsonPres := json.NewPresenter(expectedCatalog, expectedSource.Metadata, *expectedDistro) + jsonPres := json.NewPresenter(expectedCatalog, expectedSource.Metadata, expectedDistro) if err = jsonPres.Present(&buf); err != nil { t.Fatalf("failed to write to presenter: %+v", err) } - actualCatalog, actualDistro, sourceMetadata, err := syft.CatalogFromJSON(&buf) + sourceMetadata, actualCatalog, actualDistro, err := syft.CatalogFromJSON(&buf) if err != nil { t.Fatalf("failed to import document: %+v", err) } diff --git a/test/integration/json_schema_test.go b/test/integration/json_schema_test.go index b84bf26f8..07b1a32cb 100644 --- a/test/integration/json_schema_test.go +++ b/test/integration/json_schema_test.go @@ -53,7 +53,7 @@ func validateAgainstV1Schema(t *testing.T, json string) { } } -func testJsonSchema(t *testing.T, catalog *pkg.Catalog, theScope *source.Source, prefix string) { +func testJsonSchema(t *testing.T, catalog *pkg.Catalog, theScope source.Source, prefix string) { // make the json output example dir if it does not exist absJsonSchemaExamplesPath := path.Join(repoRoot(t), jsonSchemaExamplesPath) if _, err := os.Stat(absJsonSchemaExamplesPath); os.IsNotExist(err) { @@ -67,7 +67,7 @@ func testJsonSchema(t *testing.T, catalog *pkg.Catalog, theScope *source.Source, t.Fatalf("bad distro: %+v", err) } - p := presenter.GetPresenter(presenter.JSONPresenter, theScope.Metadata, catalog, &d) + p := presenter.GetPresenter(presenter.JSONPresenter, theScope.Metadata, catalog, d) if p == nil { t.Fatal("unable to get presenter") } @@ -101,7 +101,7 @@ func TestJsonSchemaImg(t *testing.T) { tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) defer cleanup() - catalog, theScope, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) + src, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } @@ -112,13 +112,13 @@ func TestJsonSchemaImg(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { - testJsonSchema(t, catalog, theScope, "img") + testJsonSchema(t, catalog, src, "img") }) } } func TestJsonSchemaDirs(t *testing.T) { - catalog, theScope, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope) + src, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope) if err != nil { t.Errorf("unable to create source from dir: %+v", err) } @@ -129,7 +129,7 @@ func TestJsonSchemaDirs(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { - testJsonSchema(t, catalog, theScope, "dir") + testJsonSchema(t, catalog, src, "dir") }) } } diff --git a/test/integration/pkg_coverage_test.go b/test/integration/pkg_coverage_test.go index 78e821829..cc25f391a 100644 --- a/test/integration/pkg_coverage_test.go +++ b/test/integration/pkg_coverage_test.go @@ -18,7 +18,7 @@ func TestPkgCoverageImage(t *testing.T) { tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) defer cleanup() - catalog, _, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) + _, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } @@ -100,7 +100,7 @@ func TestPkgCoverageImage(t *testing.T) { } func TestPkgCoverageDirectory(t *testing.T) { - catalog, _, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope) + _, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope) if err != nil { t.Errorf("unable to create source from dir: %+v", err) From 030427bb33767197621d861aba79c45589c50106 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 17 Nov 2020 12:36:07 -0500 Subject: [PATCH 13/13] rename json artifact to package + update resolver integrity checks Signed-off-by: Alex Goodman --- syft/presenter/json/artifact.go | 37 +++++++++++++++++----------- syft/presenter/json/document.go | 6 ++--- syft/source/all_layers_resolver.go | 2 +- syft/source/directory_resolver.go | 2 +- syft/source/image_squash_resolver.go | 2 +- 5 files changed, 28 insertions(+), 21 deletions(-) diff --git a/syft/presenter/json/artifact.go b/syft/presenter/json/artifact.go index d5bf19872..2773ec57f 100644 --- a/syft/presenter/json/artifact.go +++ b/syft/presenter/json/artifact.go @@ -8,12 +8,14 @@ import ( "github.com/anchore/syft/syft/source" ) -type Artifact struct { - artifactBasicMetadata - artifactCustomMetadata +// Package represents a pkg.Package object specialized for JSON marshaling and unmarshaling. +type Package struct { + packageBasicMetadata + packageCustomMetadata } -type artifactBasicMetadata struct { +// packageBasicMetadata contains non-ambiguous values (type-wise) from pkg.Package. +type packageBasicMetadata struct { Name string `json:"name"` Version string `json:"version"` Type pkg.Type `json:"type"` @@ -23,19 +25,22 @@ type artifactBasicMetadata struct { Language pkg.Language `json:"language"` } -type artifactCustomMetadata struct { +// packageCustomMetadata contains ambiguous values (type-wise) from pkg.Package. +type packageCustomMetadata struct { MetadataType pkg.MetadataType `json:"metadataType"` Metadata interface{} `json:"metadata,omitempty"` } -type artifactMetadataUnpacker struct { +// packageMetadataUnpacker is all values needed from Package to disambiguate ambiguous fields during json unmarshaling. +type packageMetadataUnpacker struct { MetadataType string `json:"metadataType"` Metadata json.RawMessage `json:"metadata"` } -func NewArtifact(p *pkg.Package) (Artifact, error) { - return Artifact{ - artifactBasicMetadata: artifactBasicMetadata{ +// NewPackage crates a new Package from the given pkg.Package. +func NewPackage(p *pkg.Package) (Package, error) { + return Package{ + packageBasicMetadata: packageBasicMetadata{ Name: p.Name, Version: p.Version, Type: p.Type, @@ -44,14 +49,15 @@ func NewArtifact(p *pkg.Package) (Artifact, error) { Licenses: p.Licenses, Language: p.Language, }, - artifactCustomMetadata: artifactCustomMetadata{ + packageCustomMetadata: packageCustomMetadata{ MetadataType: p.MetadataType, Metadata: p.Metadata, }, }, nil } -func (a Artifact) ToPackage() pkg.Package { +// ToPackage generates a pkg.Package from the current Package. +func (a Package) ToPackage() pkg.Package { return pkg.Package{ // does not include found-by and locations Name: a.Name, @@ -66,15 +72,16 @@ func (a Artifact) ToPackage() pkg.Package { } } +// UnmarshalJSON is a custom unmarshaller for handling basic values and values with ambiguous types. // nolint:funlen -func (a *Artifact) UnmarshalJSON(b []byte) error { - var basic artifactBasicMetadata +func (a *Package) UnmarshalJSON(b []byte) error { + var basic packageBasicMetadata if err := json.Unmarshal(b, &basic); err != nil { return err } - a.artifactBasicMetadata = basic + a.packageBasicMetadata = basic - var unpacker artifactMetadataUnpacker + var unpacker packageMetadataUnpacker if err := json.Unmarshal(b, &unpacker); err != nil { return err } diff --git a/syft/presenter/json/document.go b/syft/presenter/json/document.go index de1ce463b..27cd6e48e 100644 --- a/syft/presenter/json/document.go +++ b/syft/presenter/json/document.go @@ -10,7 +10,7 @@ import ( // Document represents the syft cataloging findings as a JSON document type Document struct { - Artifacts []Artifact `json:"artifacts"` // Artifacts is the list of packages discovered and placed into the catalog + Artifacts []Package `json:"artifacts"` // Artifacts is the list of packages discovered and placed into the catalog Source Source `json:"source"` // Source represents the original object that was cataloged Distro Distribution `json:"distro"` // Distro represents the Linux distribution that was detected from the source Descriptor Descriptor `json:"descriptor"` // Descriptor is a block containing self-describing information about syft @@ -24,7 +24,7 @@ func NewDocument(catalog *pkg.Catalog, srcMetadata source.Metadata, d distro.Dis } doc := Document{ - Artifacts: make([]Artifact, 0), + Artifacts: make([]Package, 0), Source: src, Distro: NewDistribution(d), Descriptor: Descriptor{ @@ -34,7 +34,7 @@ func NewDocument(catalog *pkg.Catalog, srcMetadata source.Metadata, d distro.Dis } for _, p := range catalog.Sorted() { - art, err := NewArtifact(p) + art, err := NewPackage(p) if err != nil { return Document{}, err } diff --git a/syft/source/all_layers_resolver.go b/syft/source/all_layers_resolver.go index 7b94328c1..4b99bda5b 100644 --- a/syft/source/all_layers_resolver.go +++ b/syft/source/all_layers_resolver.go @@ -8,7 +8,7 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) -var _ Resolver = &AllLayersResolver{} +var _ Resolver = (*AllLayersResolver)(nil) // AllLayersResolver implements path and content access for the AllLayers source option for container image data sources. type AllLayersResolver struct { diff --git a/syft/source/directory_resolver.go b/syft/source/directory_resolver.go index c05d229c4..389914950 100644 --- a/syft/source/directory_resolver.go +++ b/syft/source/directory_resolver.go @@ -11,7 +11,7 @@ import ( "github.com/bmatcuk/doublestar" ) -var _ Resolver = &DirectoryResolver{} +var _ Resolver = (*DirectoryResolver)(nil) // DirectoryResolver implements path and content access for the directory data source. type DirectoryResolver struct { diff --git a/syft/source/image_squash_resolver.go b/syft/source/image_squash_resolver.go index 3df142a58..6be3422e9 100644 --- a/syft/source/image_squash_resolver.go +++ b/syft/source/image_squash_resolver.go @@ -7,7 +7,7 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) -var _ Resolver = &ImageSquashResolver{} +var _ Resolver = (*ImageSquashResolver)(nil) // ImageSquashResolver implements path and content access for the Squashed source option for container image data sources. type ImageSquashResolver struct {