oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-10-31 22:46:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

finalize json output & add schema (#118)

Browse Source
This commit is contained in:
Alex Goodman
2020-08-04 16:05:53 -04:00
committed by GitHub
parent 2560266e38
commit e2a874a277
23 changed files with 1129 additions and 500 deletions
Download Patch File Download Diff File Expand all files Collapse all files
syft/cataloger/apkdb/parse_apk_db.go
+33 -3
View File
@@ -4,9 +4,11 @@ import (
"bufio"
"fmt"
"io"
"path"
"strconv"
"strings"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
"github.com/mitchellh/mapstructure"
)
@@ -52,10 +54,14 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) {
return packages, nil
}
// nolint:funlen
func parseApkDBEntry(reader io.Reader) (*pkg.ApkMetadata, error) {
var entry pkg.ApkMetadata
pkgFields := make(map[string]interface{})
files := make([]string, 0)
files := make([]pkg.ApkFileRecord, 0)
var fileRecord *pkg.ApkFileRecord
lastFile := "/"
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
@@ -70,9 +76,33 @@ func parseApkDBEntry(reader io.Reader) (*pkg.ApkMetadata, error) {
switch key {
case "F":
// extract all file entries, don't store in map
files = append(files, value)
lastFile = "/" + value
continue
case "R":
newFileRecord := pkg.ApkFileRecord{
Path: path.Join(lastFile, value),
}
files = append(files, newFileRecord)
fileRecord = &files[len(files)-1]
case "a":
ownershipFields := strings.Split(value, ":")
if len(ownershipFields) != 3 {
log.Errorf("unexpected APK ownership field: %q", value)
continue
}
if fileRecord == nil {
log.Errorf("ownership field with no parent record: %q", value)
continue
}
fileRecord.OwnerUID = ownershipFields[0]
fileRecord.OwnerGUI = ownershipFields[1]
fileRecord.Permissions = ownershipFields[2]
case "Z":
if fileRecord == nil {
log.Errorf("checksum field with no parent record: %q", value)
continue
}
fileRecord.Checksum = value
case "I", "S":
// coerce to integer
iVal, err := strconv.Atoi(value)
syft/cataloger/apkdb/parse_apk_db_test.go
+75 -3
View File
@@ -31,7 +31,43 @@ func TestSinglePackage(t *testing.T) {
PullDependencies: "scanelf so:libc.musl-x86_64.so.1",
PullChecksum: "Q1bTtF5526tETKfL+lnigzIDvm+2o=",
GitCommitOfAport: "4024cc3b29ad4c65544ad068b8f59172b5494306",
Files: []string{"sbin", "usr", "usr/bin"},
Files: []pkg.ApkFileRecord{
{
Path: "/sbin/ldconfig",
OwnerUID: "0",
OwnerGUI: "0",
Permissions: "755",
Checksum: "Q1Kja2+POZKxEkUOZqwSjC6kmaED4=",
},
{
Path: "/usr/bin/iconv",
OwnerUID: "0",
OwnerGUI: "0",
Permissions: "755",
Checksum: "Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY=",
},
{
Path: "/usr/bin/ldd",
OwnerUID: "0",
OwnerGUI: "0",
Permissions: "755",
Checksum: "Q1yFAhGggmL7ERgbIA7KQxyTzf3ks=",
},
{
Path: "/usr/bin/getconf",
OwnerUID: "0",
OwnerGUI: "0",
Permissions: "755",
Checksum: "Q1dAdYK8M/INibRQF5B3Rw7cmNDDA=",
},
{
Path: "/usr/bin/getent",
OwnerUID: "0",
OwnerGUI: "0",
Permissions: "755",
Checksum: "Q1eR2Dz/WylabgbWMTkd2+hGmEya4=",
},
},
},
},
}
@@ -92,7 +128,7 @@ func TestMultiplePackages(t *testing.T) {
PullChecksum: "Q1p78yvTLG094tHE1+dToJGbmYzQE=",
GitCommitOfAport: "97b1c2842faa3bfa30f5811ffbf16d5ff9f1a479",
PullDependencies: "musl-utils",
Files: []string{},
Files: []pkg.ApkFileRecord{},
},
},
{
@@ -114,7 +150,43 @@ func TestMultiplePackages(t *testing.T) {
PullDependencies: "scanelf so:libc.musl-x86_64.so.1",
PullChecksum: "Q1bTtF5526tETKfL+lnigzIDvm+2o=",
GitCommitOfAport: "4024cc3b29ad4c65544ad068b8f59172b5494306",
Files: []string{"sbin", "usr", "usr/bin"},
Files: []pkg.ApkFileRecord{
{
Path: "/sbin/ldconfig",
OwnerUID: "0",
OwnerGUI: "0",
Permissions: "755",
Checksum: "Q1Kja2+POZKxEkUOZqwSjC6kmaED4=",
},
{
Path: "/usr/bin/iconv",
OwnerUID: "0",
OwnerGUI: "0",
Permissions: "755",
Checksum: "Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY=",
},
{
Path: "/usr/bin/ldd",
OwnerUID: "0",
OwnerGUI: "0",
Permissions: "755",
Checksum: "Q1yFAhGggmL7ERgbIA7KQxyTzf3ks=",
},
{
Path: "/usr/bin/getconf",
OwnerUID: "0",
OwnerGUI: "0",
Permissions: "755",
Checksum: "Q1dAdYK8M/INibRQF5B3Rw7cmNDDA=",
},
{
Path: "/usr/bin/getent",
OwnerUID: "0",
OwnerGUI: "0",
Permissions: "755",
Checksum: "Q1eR2Dz/WylabgbWMTkd2+hGmEya4=",
},
},
},
},
},
syft/pkg/metadata.go
+47 -38
View File
@@ -1,61 +1,70 @@
package pkg
// TODO: consider keeping the remaining values as an embedded map
// Available fields are described at http://manpages.ubuntu.com/manpages/xenial/man1/dpkg-query.1.html
// in the --showformat section
type DpkgMetadata struct {
Package string `mapstructure:"Package"`
Source string `mapstructure:"Source"`
Version string `mapstructure:"Version"`
Package string `mapstructure:"Package" json:"package"`
Source string `mapstructure:"Source" json:"source"`
Version string `mapstructure:"Version" json:"version"`
// TODO: consider keeping the remaining values as an embedded map
}
type RpmMetadata struct {
Epoch int `mapstructure:"Epoch"`
Arch string `mapstructure:"Arch"`
Release string `mapstructure:"Release"`
Epoch int `mapstructure:"Epoch" json:"epoch"`
Arch string `mapstructure:"Arch" json:"architecture"`
Release string `mapstructure:"Release" json:"release"`
// TODO: consider keeping the remaining values as an embedded map
}
type JavaManifest struct {
Name string `mapstructure:"Name"`
ManifestVersion string `mapstructure:"Manifest-Version"`
SpecTitle string `mapstructure:"Specification-Title"`
SpecVersion string `mapstructure:"Specification-Version"`
SpecVendor string `mapstructure:"Specification-Vendor"`
ImplTitle string `mapstructure:"Implementation-Title"`
ImplVersion string `mapstructure:"Implementation-Version"`
ImplVendor string `mapstructure:"Implementation-Vendor"`
Extra map[string]string `mapstructure:",remain"`
Name string `mapstructure:"Name" json:"name"`
ManifestVersion string `mapstructure:"Manifest-Version" json:"manifest-version"`
SpecTitle string `mapstructure:"Specification-Title" json:"specification-title"`
SpecVersion string `mapstructure:"Specification-Version" json:"specification-version"`
SpecVendor string `mapstructure:"Specification-Vendor" json:"specification-vendor"`
ImplTitle string `mapstructure:"Implementation-Title" json:"implementation-title"`
ImplVersion string `mapstructure:"Implementation-Version" json:"implementation-version"`
ImplVendor string `mapstructure:"Implementation-Vendor" json:"implementation-vendor"`
Extra map[string]string `mapstructure:",remain" json:"extra-fields"`
}
type PomProperties struct {
Path string
Name string `mapstructure:"name"`
GroupID string `mapstructure:"groupId"`
ArtifactID string `mapstructure:"artifactId"`
Version string `mapstructure:"version"`
Extra map[string]string `mapstructure:",remain"`
Name string `mapstructure:"name" json:"name"`
GroupID string `mapstructure:"groupId" json:"group-id"`
ArtifactID string `mapstructure:"artifactId" json:"artifact-id"`
Version string `mapstructure:"version" json:"version"`
Extra map[string]string `mapstructure:",remain" json:"extra-fields"`
}
type JavaMetadata struct {
Manifest *JavaManifest `mapstructure:"Manifest"`
PomProperties *PomProperties `mapstructure:"PomProperties"`
Parent *Package
Manifest *JavaManifest `mapstructure:"Manifest" json:"manifest"`
PomProperties *PomProperties `mapstructure:"PomProperties" json:"pom-properties"`
Parent *Package `json:"parent-package"`
}
// source: https://wiki.alpinelinux.org/wiki/Apk_spec
type ApkMetadata struct {
Package string `mapstructure:"P"`
OriginPackage string `mapstructure:"o"`
Maintainer string `mapstructure:"m"`
Version string `mapstructure:"V"`
License string `mapstructure:"L"`
Architecture string `mapstructure:"A"`
URL string `mapstructure:"U"`
Description string `mapstructure:"T"`
Size int `mapstructure:"S"`
InstalledSize int `mapstructure:"I"`
PullDependencies string `mapstructure:"D"`
PullChecksum string `mapstructure:"C"`
GitCommitOfAport string `mapstructure:"c"`
Files []string
Package string `mapstructure:"P" json:"package"`
OriginPackage string `mapstructure:"o" json:"origin-package"`
Maintainer string `mapstructure:"m" json:"maintainer"`
Version string `mapstructure:"V" json:"version"`
License string `mapstructure:"L" json:"license"`
Architecture string `mapstructure:"A" json:"architecture"`
URL string `mapstructure:"U" json:"url"`
Description string `mapstructure:"T" json:"description"`
Size int `mapstructure:"S" json:"size"`
InstalledSize int `mapstructure:"I" json:"installed-size"`
PullDependencies string `mapstructure:"D" json:"pull-dependencies"`
PullChecksum string `mapstructure:"C" json:"pull-checksum"`
GitCommitOfAport string `mapstructure:"c" json:"git-commit-of-apk-port"`
Files []ApkFileRecord `json:"files"`
}
type ApkFileRecord struct {
Path string `json:"path"`
OwnerUID string `json:"owner-uid"`
OwnerGUI string `json:"owner-gid"`
Permissions string `json:"permissions"`
Checksum string `json:"checksum"`
}
syft/pkg/package.go
+9 -9
View File
@@ -12,15 +12,15 @@ type ID int64
// Package represents an application or library that has been bundled into a distributable format
type Package struct {
id ID // this is set when a package is added to the catalog
Name string
Version string
FoundBy string
Source []file.Reference
Licenses []string
Language Language // TODO: should this support multiple languages as a slice?
Type Type
Metadata interface{}
id ID // this is set when a package is added to the catalog
Name string `json:"manifest"`
Version string `json:"version"`
FoundBy string `json:"found-by"`
Source []file.Reference `json:"sources"`
Licenses []string `json:"licenses"` // TODO: should we move this into metadata?
Language Language `json:"language"` // TODO: should this support multiple languages as a slice?
Type Type `json:"type"`
Metadata interface{} `json:"metadata,omitempty"`
}
func (p Package) ID() ID {
syft/presenter/json/presenter.go
+30 -24
View File
@@ -5,7 +5,6 @@ import (
"fmt"
"io"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
)
@@ -24,36 +23,35 @@ func NewPresenter(catalog *pkg.Catalog, s scope.Scope) *Presenter {
type document struct {
Artifacts []artifact `json:"artifacts"`
Image image `json:"image"`
Source string
Image *image `json:"image,omitempty"`
Directory *string `json:"directory,omitempty"`
}
type image struct {
Layers []layer `json:"layers"`
Size int64 `json:"size"`
Digest string `json:"digest"`
MediaType string `json:"mediaType"`
MediaType string `json:"media-type"`
Tags []string `json:"tags"`
}
type layer struct {
MediaType string `json:"mediaType"`
MediaType string `json:"media-type"`
Digest string `json:"digest"`
Size int64 `json:"size"`
}
type source struct {
FoundBy string `json:"foundBy"`
Effects []string `json:"effects"`
FoundBy string `json:"found-by"`
Locations []string `json:"locations"`
}
type artifact struct {
Name string `json:"name"`
Version string `json:"version"`
Type string `json:"type"`
Cataloger string `json:"cataloger"`
Sources []source `json:"sources"`
Metadata interface{} `json:"metadata,omitempty"`
Name string `json:"name"`
Version string `json:"version"`
Type string `json:"type"`
Sources []source `json:"sources"`
Metadata interface{} `json:"metadata,omitempty"`
}
func (pres *Presenter) Present(output io.Writer) error {
@@ -69,15 +67,25 @@ func (pres *Presenter) Present(output io.Writer) error {
for idx, tag := range src.Img.Metadata.Tags {
tags[idx] = tag.String()
}
doc.Image = image{
doc.Image = &image{
Digest: src.Img.Metadata.Digest,
Size: src.Img.Metadata.Size,
MediaType: string(src.Img.Metadata.MediaType),
Tags: tags,
Layers: make([]layer, len(src.Img.Layers)),
}
// populate image metadata
for idx, l := range src.Img.Layers {
doc.Image.Layers[idx] = layer{
MediaType: string(l.Metadata.MediaType),
Digest: l.Metadata.Digest,
Size: l.Metadata.Size,
}
}
case scope.DirSource:
doc.Source = pres.scope.DirSrc.Path
doc.Directory = &pres.scope.DirSrc.Path
default:
return fmt.Errorf("unsupported source: %T", src)
}
@@ -93,8 +101,8 @@ func (pres *Presenter) Present(output io.Writer) error {
for idx := range p.Source {
srcObj := source{
FoundBy: p.FoundBy,
Effects: []string{}, // TODO
FoundBy: p.FoundBy,
Locations: []string{string(p.Source[idx].Path)},
}
art.Sources[idx] = srcObj
}
@@ -102,11 +110,9 @@ func (pres *Presenter) Present(output io.Writer) error {
doc.Artifacts = append(doc.Artifacts, art)
}
bytes, err := json.Marshal(&doc)
if err != nil {
log.Errorf("failed to marshal json (presenter=json): %w", err)
}
_, err = output.Write(bytes)
return err
enc := json.NewEncoder(output)
// prevent > and < from being escaped in the payload
enc.SetEscapeHTML(false)
enc.SetIndent("", " ")
return enc.Encode(&doc)
}
syft/presenter/json/presenter_test.go
-3
View File
@@ -109,7 +109,4 @@ func TestJsonImgsPresenter(t *testing.T) {
diffs := dmp.DiffMain(string(actual), string(expected), true)
t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs))
}
// TODO: add me back in when there is a JSON schema
// validateAgainstV1Schema(t, string(actual))
}
syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden
+17 -1
View File
@@ -1 +1,17 @@
{"artifacts":[{"name":"package-1","version":"1.0.1","type":"deb","cataloger":"","sources":[]},{"name":"package-2","version":"2.0.1","type":"deb","cataloger":"","sources":[]}],"image":{"layers":null,"size":0,"digest":"","mediaType":"","tags":null},"Source":"/some/path"}
{
"artifacts": [
{
"name": "package-1",
"version": "1.0.1",
"type": "deb",
"sources": []
},
{
"name": "package-2",
"version": "2.0.1",
"type": "deb",
"sources": []
}
],
"directory": "/some/path"
}
syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden
+55 -1
View File
@@ -1 +1,55 @@
{"artifacts":[{"name":"package-1","version":"1.0.1","type":"deb","cataloger":"","sources":[{"foundBy":"","effects":[]}]},{"name":"package-2","version":"2.0.1","type":"deb","cataloger":"","sources":[{"foundBy":"","effects":[]}]}],"image":{"layers":[{"mediaType":"","digest":"","size":0},{"mediaType":"","digest":"","size":0},{"mediaType":"","digest":"","size":0}],"size":65,"digest":"sha256:3c53d2d891940f8d8e95acb77b58752f54dc5de9d91d19dd90ced2db76256cea","mediaType":"application/vnd.docker.distribution.manifest.v2+json","tags":["anchore-fixture-image-simple:04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7"]},"Source":""}
{
"artifacts": [
{
"name": "package-1",
"version": "1.0.1",
"type": "deb",
"sources": [
{
"found-by": "",
"locations": [
"/somefile-1.txt"
]
}
]
},
{
"name": "package-2",
"version": "2.0.1",
"type": "deb",
"sources": [
{
"found-by": "",
"locations": [
"/somefile-2.txt"
]
}
]
}
],
"image": {
"layers": [
{
"media-type": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:056c0789fa9ad629ceae6d09713fb035f84115af3c4a88a43aa60f13bc683053",
"size": 22
},
{
"media-type": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:b461c48116592c570a66fed71d5b09662a8172e168b7938cf317af47872cdc9b",
"size": 16
},
{
"media-type": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:00b80053e05c01da485015610d288ce3185fac00d251e2ada02b45a7a7c5f589",
"size": 27
}
],
"size": 65,
"digest": "sha256:3c53d2d891940f8d8e95acb77b58752f54dc5de9d91d19dd90ced2db76256cea",
"media-type": "application/vnd.docker.distribution.manifest.v2+json",
"tags": [
"anchore-fixture-image-simple:04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7"
]
}
}
syft/presenter/table/presenter_test.go
-3
View File
@@ -62,7 +62,4 @@ func TestTablePresenter(t *testing.T) {
diffs := dmp.DiffMain(string(actual), string(expected), true)
t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs))
}
// TODO: add me back in when there is a JSON schema
// validateAgainstV1Schema(t, string(actual))
}
syft/presenter/text/presenter_test.go
+1 -1
View File
@@ -12,7 +12,7 @@ import (
"github.com/sergi/go-diff/diffmatchpatch"
)
var update = flag.Bool("update", false, "update the *.golden files for json presenters")
var update = flag.Bool("update", false, "update the *.golden files for text presenters")
func TestTextDirPresenter(t *testing.T) {
var buffer bytes.Buffer