oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-10-31 22:46:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add OCI support + use URI schemes for user image input (#178)

Browse Source 
* add oci support + update image schemes

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update docs to reflect OCI image sources + URI scheme change

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update to oci-dir

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* bump upstream stereoscope pin

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman
2020-09-25 10:24:24 -04:00
committed by GitHub
parent 45b5cab49a
commit 3d91a66536
23 changed files with 320 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files
syft/presenter/cyclonedx/presenter_test.go
+2 -1
View File
@@ -3,6 +3,7 @@ package cyclonedx
import (
"bytes"
"flag"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/distro"
"regexp"
"testing"
@@ -96,7 +97,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
var buffer bytes.Buffer
catalog := pkg.NewCatalog()
img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-simple")
img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-simple")
defer cleanup()
// populate catalog with test data
syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden
+2 -2
View File
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<bom xmlns="http://cyclonedx.org/schema/bom/1.2" xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0" version="1" serialNumber="urn:uuid:b3b1786a-2de3-4501-b902-58b701b8ad0e">
<bom xmlns="http://cyclonedx.org/schema/bom/1.2" xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0" version="1" serialNumber="urn:uuid:2bbada20-3e87-44ea-9a56-1aa0e4dd01a0">
<components>
<component type="library">
<name>package1</name>
@@ -21,7 +21,7 @@
</component>
</components>
<bd:metadata>
<bd:timestamp>2020-08-30T21:50:50-04:00</bd:timestamp>
<bd:timestamp>2020-09-23T18:26:58-04:00</bd:timestamp>
<bd:tool>
<bd:vendor>anchore</bd:vendor>
<bd:name>syft</bd:name>
syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden
+3 -3
View File
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<bom xmlns="http://cyclonedx.org/schema/bom/1.2" xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0" version="1" serialNumber="urn:uuid:ab970429-e6a2-44a1-810c-f9ed2b7c3147">
<bom xmlns="http://cyclonedx.org/schema/bom/1.2" xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0" version="1" serialNumber="urn:uuid:94dae829-4d5d-482f-afab-27f43f919e2c">
<components>
<component type="library">
<name>package1</name>
@@ -21,14 +21,14 @@
</component>
</components>
<bd:metadata>
<bd:timestamp>2020-08-30T21:50:50-04:00</bd:timestamp>
<bd:timestamp>2020-09-23T18:26:58-04:00</bd:timestamp>
<bd:tool>
<bd:vendor>anchore</bd:vendor>
<bd:name>syft</bd:name>
<bd:version>[not provided]</bd:version>
</bd:tool>
<bd:component type="container">
<name>index.docker.io/library/anchore-fixture-image-simple</name>
<name>index.docker.io/library/stereoscope-fixture-image-simple</name>
<version>04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7</version>
</bd:component>
</bd:metadata>
syft/presenter/json/presenter_test.go
+3 -2
View File
@@ -7,6 +7,7 @@ import (
"github.com/anchore/go-testutils"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/sergi/go-diff/diffmatchpatch"
@@ -72,11 +73,11 @@ func TestJsonImgsPresenter(t *testing.T) {
testImage := "image-simple"
if *update {
testutils.UpdateGoldenFixtureImage(t, testImage)
imagetest.UpdateGoldenFixtureImage(t, testImage)
}
catalog := pkg.NewCatalog()
img := testutils.GetGoldenFixtureImage(t, testImage)
img := imagetest.GetGoldenFixtureImage(t, testImage)
// populate catalog with test data
catalog.Add(pkg.Package{
syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden
+5 -5
View File
@@ -33,25 +33,25 @@
"layers": [
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:056c0789fa9ad629ceae6d09713fb035f84115af3c4a88a43aa60f13bc683053",
"digest": "sha256:e158b57d6f5a96ef5fd22f2fe76c70b5ba6ff5b2619f9d83125b2aad0492ac7b",
"size": 22
},
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:b461c48116592c570a66fed71d5b09662a8172e168b7938cf317af47872cdc9b",
"digest": "sha256:da21056e7bf4308ecea0c0836848a7fe92f38fdcf35bc09ee6d98e7ab7beeebf",
"size": 16
},
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:00b80053e05c01da485015610d288ce3185fac00d251e2ada02b45a7a7c5f589",
"digest": "sha256:f0e18aa6032c24659a9c741fc36ca56f589782ea132061ccf6f52b952403da94",
"size": 27
}
],
"size": 65,
"digest": "sha256:3c53d2d891940f8d8e95acb77b58752f54dc5de9d91d19dd90ced2db76256cea",
"digest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"tags": [
"anchore-fixture-image-simple:04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7"
"stereoscope-fixture-image-simple:04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7"
]
}
}
syft/presenter/json/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden
BIN
View File
Binary file not shown.
syft/presenter/table/presenter_test.go
+2 -1
View File
@@ -7,6 +7,7 @@ import (
"github.com/anchore/go-testutils"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/sergi/go-diff/diffmatchpatch"
@@ -20,7 +21,7 @@ func TestTablePresenter(t *testing.T) {
testImage := "image-simple"
catalog := pkg.NewCatalog()
img, cleanup := testutils.GetFixtureImage(t, "docker-archive", testImage)
img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", testImage)
defer cleanup()
// populate catalog with test data
syft/presenter/text/presenter_test.go
+2 -1
View File
@@ -7,6 +7,7 @@ import (
"github.com/anchore/go-testutils"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/sergi/go-diff/diffmatchpatch"
@@ -67,7 +68,7 @@ func TestTextImgPresenter(t *testing.T) {
var buffer bytes.Buffer
catalog := pkg.NewCatalog()
img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-simple")
img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-simple")
defer cleanup()
// populate catalog with test data
syft/scope/protocol.go
-59
View File
@@ -1,59 +0,0 @@
package scope
import "strings"
// Potentially consider moving this out into a generic package that parses user input.
// Aside from scope, this is the 2nd package that looks at a string to parse the input
// and return an Option type.
const (
// nolint:varcheck,deadcode
unknownProtocol protocolType = iota
imageProtocol
directoryProtocol
)
var protocolStr = []string{
"UnknownProtocol",
"Image",
"Directory",
}
type protocolType int
type protocol struct {
Type protocolType
Value string
}
func newProtocol(userStr string) protocol {
candidates := strings.Split(userStr, "://")
switch len(candidates) {
case 2:
if strings.HasPrefix(userStr, "dir://") {
return protocol{
Type: directoryProtocol,
Value: strings.TrimPrefix(userStr, "dir://"),
}
}
// default to an Image for anything else since stereoscope can handle this
return protocol{
Type: imageProtocol,
Value: userStr,
}
default:
return protocol{
Type: imageProtocol,
Value: userStr,
}
}
}
func (o protocolType) String() string {
if int(o) >= len(protocolStr) || o < 0 {
return protocolStr[0]
}
return protocolStr[o]
}
syft/scope/protocol_test.go
-43
View File
@@ -1,43 +0,0 @@
package scope
import "testing"
func TestNewProtocol(t *testing.T) {
testCases := []struct {
desc string
input string
expType protocolType
expValue string
}{
{
desc: "directory protocol",
input: "dir:///opt/",
expType: directoryProtocol,
expValue: "/opt/",
},
{
desc: "unknown protocol",
input: "s4:///opt/",
expType: imageProtocol,
expValue: "s4:///opt/",
},
{
desc: "docker protocol",
input: "docker://ubuntu:20.04",
expType: imageProtocol,
expValue: "docker://ubuntu:20.04",
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
p := newProtocol(test.input)
if p.Type != test.expType {
t.Errorf("mismatched type in protocol: '%v' != '%v'", p.Type, test.expType)
}
if p.Value != test.expValue {
t.Errorf("mismatched protocol value: '%s' != '%s'", p.Value, test.expValue)
}
})
}
}
syft/scope/resolvers/all_layers_resolver_test.go
+3 -3
View File
@@ -1,9 +1,9 @@
package resolvers
import (
"github.com/anchore/stereoscope/pkg/imagetest"
"testing"
"github.com/anchore/go-testutils"
"github.com/anchore/stereoscope/pkg/file"
)
@@ -83,7 +83,7 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-symlinks")
img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
defer cleanup()
resolver, err := NewAllLayersResolver(img)
@@ -191,7 +191,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-symlinks")
img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
defer cleanup()
resolver, err := NewAllLayersResolver(img)
syft/scope/resolvers/image_squash_resolver_test.go
+3 -3
View File
@@ -1,9 +1,9 @@
package resolvers
import (
"github.com/anchore/stereoscope/pkg/imagetest"
"testing"
"github.com/anchore/go-testutils"
"github.com/anchore/stereoscope/pkg/file"
)
@@ -47,7 +47,7 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-symlinks")
img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
defer cleanup()
resolver, err := NewImageSquashResolver(img)
@@ -122,7 +122,7 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-symlinks")
img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
defer cleanup()
resolver, err := NewImageSquashResolver(img)
syft/scope/scope.go
+55 -22
View File
@@ -7,7 +7,10 @@ package scope
import (
"fmt"
"os"
"strings"
"github.com/anchore/syft/internal/log"
"github.com/spf13/afero"
"github.com/anchore/stereoscope"
@@ -15,6 +18,14 @@ import (
"github.com/anchore/syft/syft/scope/resolvers"
)
const (
unknownScheme scheme = "unknown-scheme"
directoryScheme scheme = "directory-scheme"
imageScheme scheme = "image-scheme"
)
type scheme string
// ImageSource represents a data source that is a container image
type ImageSource struct {
Img *image.Image // the image object to be cataloged
@@ -34,31 +45,36 @@ type Scope struct {
DirSrc DirSource // the specific directory to be cataloged
}
// NewScope produces a Scope based on userInput like dir:// or image:tag
// NewScope produces a Scope based on userInput like dir: or image:tag
func NewScope(userInput string, o Option) (Scope, func(), error) {
protocol := newProtocol(userInput)
fs := afero.NewOsFs()
parsedScheme, location := detectScheme(fs, image.DetectSource, userInput)
switch protocol.Type {
case directoryProtocol:
err := isValidPath(protocol.Value)
switch parsedScheme {
case directoryScheme:
fileMeta, err := fs.Stat(location)
if err != nil {
return Scope{}, func() {}, fmt.Errorf("unable to process path, must exist and be a directory: %w", err)
return Scope{}, nil, fmt.Errorf("unable to stat dir=%q: %w", location, err)
}
s, err := NewScopeFromDir(protocol.Value)
if !fileMeta.IsDir() {
return Scope{}, nil, fmt.Errorf("given path is not a directory (path=%q): %w", location, err)
}
s, err := NewScopeFromDir(location)
if err != nil {
return Scope{}, func() {}, fmt.Errorf("could not populate scope from path (%s): %w", protocol.Value, err)
return Scope{}, func() {}, fmt.Errorf("could not populate scope from path=%q: %w", location, err)
}
return s, func() {}, nil
case imageProtocol:
img, err := stereoscope.GetImage(userInput)
case imageScheme:
img, err := stereoscope.GetImage(location)
cleanup := func() {
stereoscope.Cleanup()
}
if err != nil || img == nil {
return Scope{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", userInput, err)
return Scope{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", location, err)
}
s, err := NewScopeFromImage(img, o)
@@ -66,10 +82,9 @@ func NewScope(userInput string, o Option) (Scope, func(), error) {
return Scope{}, cleanup, fmt.Errorf("could not populate scope with image: %w", err)
}
return s, cleanup, nil
default:
return Scope{}, func() {}, fmt.Errorf("unable to process input for scanning: '%s'", userInput)
}
return Scope{}, func() {}, fmt.Errorf("unable to process input for scanning: '%s'", userInput)
}
// NewScopeFromDir creates a new scope object tailored to catalog a given filesystem directory recursively.
@@ -117,16 +132,34 @@ func (s Scope) Source() interface{} {
return nil
}
// isValidPath ensures that the user-provided input will correspond to a path that exists and is a directory
func isValidPath(userInput string) error {
fileMeta, err := os.Stat(userInput)
type sourceDetector func(string) (image.Source, string, error)
func detectScheme(fs afero.Fs, imageDetector sourceDetector, userInput string) (scheme, string) {
if strings.HasPrefix(userInput, "dir:") {
// blindly trust the user's scheme
return directoryScheme, strings.TrimPrefix(userInput, "dir:")
}
// we should attempt to let stereoscope determine what the source is first --just because the source is a valid directory
// doesn't mean we yet know if it is an OCI layout directory (to be treated as an image) or if it is a generic filesystem directory.
source, imageSpec, err := imageDetector(userInput)
if err != nil {
return err
// this is not necessarily an error we care a
log.Debugf("unable to detect the scheme from %q: %w", userInput, err)
return unknownScheme, ""
}
if fileMeta.IsDir() {
return nil
if source == image.UnknownSource {
fileMeta, err := fs.Stat(userInput)
if err != nil {
return unknownScheme, ""
}
if fileMeta.IsDir() {
return directoryScheme, userInput
}
return unknownScheme, ""
}
return fmt.Errorf("path is not a directory: %s", userInput)
return imageScheme, imageSpec
}
syft/scope/scope_test.go
+165 -19
View File
@@ -1,6 +1,8 @@
package scope
import (
"github.com/spf13/afero"
"os"
"testing"
"github.com/anchore/stereoscope/pkg/file"
@@ -214,39 +216,183 @@ func TestFilesByGlob(t *testing.T) {
}
}
func TestIsValidPath(t *testing.T) {
func TestDetectScheme(t *testing.T) {
type detectorResult struct {
src image.Source
ref string
err error
}
testCases := []struct {
desc string
input string
isError bool
name string
userInput string
dirs []string
detection detectorResult
expectedScheme scheme
expectedLocation string
}{
{
desc: "path is valid",
input: "test-fixtures",
isError: false,
name: "docker-image-ref",
userInput: "wagoodman/dive:latest",
detection: detectorResult{
src: image.DockerDaemonSource,
ref: "wagoodman/dive:latest",
},
expectedScheme: imageScheme,
expectedLocation: "wagoodman/dive:latest",
},
{
desc: "file is invalid",
input: "test-fixtures/.vimrc",
isError: true,
name: "docker-image-ref-no-tag",
userInput: "wagoodman/dive",
detection: detectorResult{
src: image.DockerDaemonSource,
ref: "wagoodman/dive",
},
expectedScheme: imageScheme,
expectedLocation: "wagoodman/dive",
},
{
desc: "path does not exist",
input: "foo/bar/baz",
isError: true,
name: "docker-image-explicit-scheme",
userInput: "docker:wagoodman/dive:latest",
detection: detectorResult{
src: image.DockerDaemonSource,
ref: "wagoodman/dive:latest",
},
expectedScheme: imageScheme,
expectedLocation: "wagoodman/dive:latest",
},
{
name: "docker-image-explicit-scheme-no-tag",
userInput: "docker:wagoodman/dive",
detection: detectorResult{
src: image.DockerDaemonSource,
ref: "wagoodman/dive",
},
expectedScheme: imageScheme,
expectedLocation: "wagoodman/dive",
},
{
name: "docker-image-edge-case",
userInput: "docker:latest",
detection: detectorResult{
src: image.DockerDaemonSource,
ref: "latest",
},
expectedScheme: imageScheme,
// we want to be able to handle this case better, however, I don't see a way to do this
// the user will need to provide more explicit input (docker:docker:latest)
expectedLocation: "latest",
},
{
name: "docker-image-edge-case-explicit",
userInput: "docker:docker:latest",
detection: detectorResult{
src: image.DockerDaemonSource,
ref: "docker:latest",
},
expectedScheme: imageScheme,
// we want to be able to handle this case better, however, I don't see a way to do this
// the user will need to provide more explicit input (docker:docker:latest)
expectedLocation: "docker:latest",
},
{
name: "oci-tar",
userInput: "some/path-to-file",
detection: detectorResult{
src: image.OciTarballSource,
ref: "some/path-to-file",
},
expectedScheme: imageScheme,
expectedLocation: "some/path-to-file",
},
{
name: "oci-dir",
userInput: "some/path-to-dir",
detection: detectorResult{
src: image.OciDirectorySource,
ref: "some/path-to-dir",
},
dirs: []string{"some/path-to-dir"},
expectedScheme: imageScheme,
expectedLocation: "some/path-to-dir",
},
{
name: "guess-dir",
userInput: "some/path-to-dir",
detection: detectorResult{
src: image.UnknownSource,
ref: "",
},
dirs: []string{"some/path-to-dir"},
expectedScheme: directoryScheme,
expectedLocation: "some/path-to-dir",
},
{
name: "generic-dir-does-not-exist",
userInput: "some/path-to-dir",
detection: detectorResult{
src: image.DockerDaemonSource,
ref: "some/path-to-dir",
},
expectedScheme: imageScheme,
expectedLocation: "some/path-to-dir",
},
{
name: "explicit-dir",
userInput: "dir:some/path-to-dir",
detection: detectorResult{
src: image.UnknownSource,
ref: "",
},
dirs: []string{"some/path-to-dir"},
expectedScheme: directoryScheme,
expectedLocation: "some/path-to-dir",
},
{
name: "explicit-current-dir",
userInput: "dir:.",
detection: detectorResult{
src: image.UnknownSource,
ref: "",
},
expectedScheme: directoryScheme,
expectedLocation: ".",
},
{
name: "current-dir",
userInput: ".",
detection: detectorResult{
src: image.UnknownSource,
ref: "",
},
expectedScheme: directoryScheme,
expectedLocation: ".",
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
err := isValidPath(test.input)
if err != nil && !test.isError {
t.Errorf("did not expect and error, got: %w", err)
t.Run(test.name, func(t *testing.T) {
fs := afero.NewMemMapFs()
for _, p := range test.dirs {
err := fs.Mkdir(p, os.ModePerm)
if err != nil {
t.Fatalf("failed to create dummy tar: %+v", err)
}
}
if err == nil && test.isError {
t.Errorf("expected an error but didn't get one")
imageDetector := func(string) (image.Source, string, error) {
return test.detection.src, test.detection.ref, test.detection.err
}
actualScheme, actualLocation := detectScheme(fs, imageDetector, test.userInput)
if actualScheme != test.expectedScheme {
t.Errorf("expected scheme %q , got %q", test.expectedScheme, actualScheme)
}
if actualLocation != test.expectedLocation {
t.Errorf("expected location %q , got %q", test.expectedLocation, actualLocation)
}
})
}
}