Add ability to pull images directly from a registry (#378)

* add registry image source

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* use explicit source for fetching image + add scheme and registry tests

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* adjust test variable name and add credential helper function

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

syft/source/scheme.go

@@ -21,39 +21,39 @@ const (
 	ImageScheme Scheme = "ImageScheme"
 )
 
-func detectScheme(fs afero.Fs, imageDetector sourceDetector, userInput string) (Scheme, string, error) {
+func detectScheme(fs afero.Fs, imageDetector sourceDetector, userInput string) (Scheme, image.Source, string, error) {
 	if strings.HasPrefix(userInput, "dir:") {
 		// blindly trust the user's scheme
 		dirLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "dir:"))
 		if err != nil {
-			return UnknownScheme, "", fmt.Errorf("unable to expand directory path: %w", err)
+			return UnknownScheme, image.UnknownSource, "", fmt.Errorf("unable to expand directory path: %w", err)
 		}
-		return DirectoryScheme, dirLocation, nil
+		return DirectoryScheme, image.UnknownSource, dirLocation, nil
 	}
 
-	// we should attempt to let stereoscope determine what the source is first --just because the source is a valid directory
+	// we should attempt to let stereoscope determine what the source is first --but, just because the source is a valid directory
 	// doesn't mean we yet know if it is an OCI layout directory (to be treated as an image) or if it is a generic filesystem directory.
 	source, imageSpec, err := imageDetector(userInput)
 	if err != nil {
-		return UnknownScheme, "", fmt.Errorf("unable to detect the scheme from %q: %w", userInput, err)
+		return UnknownScheme, image.UnknownSource, "", fmt.Errorf("unable to detect the scheme from %q: %w", userInput, err)
 	}
 
 	if source == image.UnknownSource {
 		dirLocation, err := homedir.Expand(userInput)
 		if err != nil {
-			return UnknownScheme, "", fmt.Errorf("unable to expand potential directory path: %w", err)
+			return UnknownScheme, image.UnknownSource, "", fmt.Errorf("unable to expand potential directory path: %w", err)
 		}
 
 		fileMeta, err := fs.Stat(dirLocation)
 		if err != nil {
-			return UnknownScheme, "", nil
+			return UnknownScheme, source, "", nil
 		}
 
 		if fileMeta.IsDir() {
-			return DirectoryScheme, dirLocation, nil
+			return DirectoryScheme, source, dirLocation, nil
 		}
-		return UnknownScheme, "", nil
+		return UnknownScheme, source, "", nil
 	}
 
-	return ImageScheme, imageSpec, nil
+	return ImageScheme, source, imageSpec, nil
 }

syft/source/scheme_test.go

@@ -0,0 +1,263 @@
+package source
+
+import (
+	"os"
+	"testing"
+
+	"github.com/anchore/stereoscope/pkg/image"
+	"github.com/mitchellh/go-homedir"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDetectScheme(t *testing.T) {
+	type detectorResult struct {
+		src image.Source
+		ref string
+		err error
+	}
+
+	testCases := []struct {
+		name             string
+		userInput        string
+		dirs             []string
+		detection        detectorResult
+		expectedScheme   Scheme
+		expectedLocation string
+	}{
+		{
+			name:      "docker-image-ref",
+			userInput: "wagoodman/dive:latest",
+			detection: detectorResult{
+				src: image.DockerDaemonSource,
+				ref: "wagoodman/dive:latest",
+			},
+			expectedScheme:   ImageScheme,
+			expectedLocation: "wagoodman/dive:latest",
+		},
+		{
+			name:      "docker-image-ref-no-tag",
+			userInput: "wagoodman/dive",
+			detection: detectorResult{
+				src: image.DockerDaemonSource,
+				ref: "wagoodman/dive",
+			},
+			expectedScheme:   ImageScheme,
+			expectedLocation: "wagoodman/dive",
+		},
+		{
+			name:      "registry-image-explicit-scheme",
+			userInput: "registry:wagoodman/dive:latest",
+			detection: detectorResult{
+				src: image.OciRegistrySource,
+				ref: "wagoodman/dive:latest",
+			},
+			expectedScheme:   ImageScheme,
+			expectedLocation: "wagoodman/dive:latest",
+		},
+		{
+			name:      "docker-image-explicit-scheme",
+			userInput: "docker:wagoodman/dive:latest",
+			detection: detectorResult{
+				src: image.DockerDaemonSource,
+				ref: "wagoodman/dive:latest",
+			},
+			expectedScheme:   ImageScheme,
+			expectedLocation: "wagoodman/dive:latest",
+		},
+		{
+			name:      "docker-image-explicit-scheme-no-tag",
+			userInput: "docker:wagoodman/dive",
+			detection: detectorResult{
+				src: image.DockerDaemonSource,
+				ref: "wagoodman/dive",
+			},
+			expectedScheme:   ImageScheme,
+			expectedLocation: "wagoodman/dive",
+		},
+		{
+			name:      "docker-image-edge-case",
+			userInput: "docker:latest",
+			detection: detectorResult{
+				src: image.DockerDaemonSource,
+				ref: "latest",
+			},
+			expectedScheme: ImageScheme,
+			// we want to be able to handle this case better, however, I don't see a way to do this
+			// the user will need to provide more explicit input (docker:docker:latest)
+			expectedLocation: "latest",
+		},
+		{
+			name:      "docker-image-edge-case-explicit",
+			userInput: "docker:docker:latest",
+			detection: detectorResult{
+				src: image.DockerDaemonSource,
+				ref: "docker:latest",
+			},
+			expectedScheme: ImageScheme,
+			// we want to be able to handle this case better, however, I don't see a way to do this
+			// the user will need to provide more explicit input (docker:docker:latest)
+			expectedLocation: "docker:latest",
+		},
+		{
+			name:      "oci-tar",
+			userInput: "some/path-to-file",
+			detection: detectorResult{
+				src: image.OciTarballSource,
+				ref: "some/path-to-file",
+			},
+			expectedScheme:   ImageScheme,
+			expectedLocation: "some/path-to-file",
+		},
+		{
+			name:      "oci-dir",
+			userInput: "some/path-to-dir",
+			detection: detectorResult{
+				src: image.OciDirectorySource,
+				ref: "some/path-to-dir",
+			},
+			dirs:             []string{"some/path-to-dir"},
+			expectedScheme:   ImageScheme,
+			expectedLocation: "some/path-to-dir",
+		},
+		{
+			name:      "guess-dir",
+			userInput: "some/path-to-dir",
+			detection: detectorResult{
+				src: image.UnknownSource,
+				ref: "",
+			},
+			dirs:             []string{"some/path-to-dir"},
+			expectedScheme:   DirectoryScheme,
+			expectedLocation: "some/path-to-dir",
+		},
+		{
+			name:      "generic-dir-does-not-exist",
+			userInput: "some/path-to-dir",
+			detection: detectorResult{
+				src: image.DockerDaemonSource,
+				ref: "some/path-to-dir",
+			},
+			expectedScheme:   ImageScheme,
+			expectedLocation: "some/path-to-dir",
+		},
+		{
+			name:      "explicit-dir",
+			userInput: "dir:some/path-to-dir",
+			detection: detectorResult{
+				src: image.UnknownSource,
+				ref: "",
+			},
+			dirs:             []string{"some/path-to-dir"},
+			expectedScheme:   DirectoryScheme,
+			expectedLocation: "some/path-to-dir",
+		},
+		{
+			name:      "explicit-current-dir",
+			userInput: "dir:.",
+			detection: detectorResult{
+				src: image.UnknownSource,
+				ref: "",
+			},
+			expectedScheme:   DirectoryScheme,
+			expectedLocation: ".",
+		},
+		{
+			name:      "current-dir",
+			userInput: ".",
+			detection: detectorResult{
+				src: image.UnknownSource,
+				ref: "",
+			},
+			expectedScheme:   DirectoryScheme,
+			expectedLocation: ".",
+		},
+		// we should support tilde expansion
+		{
+			name:      "tilde-expansion-image-implicit",
+			userInput: "~/some-path",
+			detection: detectorResult{
+				src: image.OciDirectorySource,
+				ref: "~/some-path",
+			},
+			expectedScheme:   ImageScheme,
+			expectedLocation: "~/some-path",
+		},
+		{
+			name:      "tilde-expansion-dir-implicit",
+			userInput: "~/some-path",
+			detection: detectorResult{
+				src: image.UnknownSource,
+				ref: "",
+			},
+			dirs:             []string{"~/some-path"},
+			expectedScheme:   DirectoryScheme,
+			expectedLocation: "~/some-path",
+		},
+		{
+			name:             "tilde-expansion-dir-explicit-exists",
+			userInput:        "dir:~/some-path",
+			dirs:             []string{"~/some-path"},
+			expectedScheme:   DirectoryScheme,
+			expectedLocation: "~/some-path",
+		},
+		{
+			name:             "tilde-expansion-dir-explicit-dne",
+			userInput:        "dir:~/some-path",
+			expectedScheme:   DirectoryScheme,
+			expectedLocation: "~/some-path",
+		},
+		{
+			name:             "tilde-expansion-dir-implicit-dne",
+			userInput:        "~/some-path",
+			expectedScheme:   UnknownScheme,
+			expectedLocation: "",
+		},
+	}
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+			fs := afero.NewMemMapFs()
+
+			for _, p := range test.dirs {
+				expandedExpectedLocation, err := homedir.Expand(p)
+				if err != nil {
+					t.Fatalf("unable to expand path=%q: %+v", p, err)
+				}
+				err = fs.Mkdir(expandedExpectedLocation, os.ModePerm)
+				if err != nil {
+					t.Fatalf("failed to create dummy tar: %+v", err)
+				}
+			}
+
+			imageDetector := func(string) (image.Source, string, error) {
+				// lean on the users real home directory value
+				switch test.detection.src {
+				case image.OciDirectorySource, image.DockerTarballSource, image.OciTarballSource:
+					expandedExpectedLocation, err := homedir.Expand(test.expectedLocation)
+					if err != nil {
+						t.Fatalf("unable to expand path=%q: %+v", test.expectedLocation, err)
+					}
+					return test.detection.src, expandedExpectedLocation, test.detection.err
+				default:
+					return test.detection.src, test.detection.ref, test.detection.err
+				}
+			}
+
+			actualScheme, actualSource, actualLocation, err := detectScheme(fs, imageDetector, test.userInput)
+			if err != nil {
+				t.Fatalf("unexpected err : %+v", err)
+			}
+
+			assert.Equal(t, test.detection.src, actualSource, "mismatched source")
+			assert.Equal(t, test.expectedScheme, actualScheme, "mismatched scheme")
+
+			// lean on the users real home directory value
+			expandedExpectedLocation, err := homedir.Expand(test.expectedLocation)
+			if err != nil {
+				t.Fatalf("unable to expand path=%q: %+v", test.expectedLocation, err)
+			}
+
+			assert.Equal(t, expandedExpectedLocation, actualLocation, "mismatched location")
+		})
+	}
+}

syft/source/source.go

@@ -8,11 +8,9 @@ package source
 import (
 	"fmt"
 
-	"github.com/spf13/afero"
-
 	"github.com/anchore/stereoscope"
-
 	"github.com/anchore/stereoscope/pkg/image"
+	"github.com/spf13/afero"
 )
 
 // Source is an object that captures the data source to be cataloged, configuration, and a specific resolver used
@@ -25,9 +23,9 @@ type Source struct {
 type sourceDetector func(string) (image.Source, string, error)
 
 // New produces a Source based on userInput like dir: or image:tag
-func New(userInput string) (Source, func(), error) {
+func New(userInput string, registryOptions *image.RegistryOptions) (Source, func(), error) {
 	fs := afero.NewOsFs()
-	parsedScheme, location, err := detectScheme(fs, image.DetectSource, userInput)
+	parsedScheme, imageSource, location, err := detectScheme(fs, image.DetectSource, userInput)
 	if err != nil {
 		return Source{}, func() {}, fmt.Errorf("unable to parse input=%q: %w", userInput, err)
 	}
@@ -50,7 +48,7 @@ func New(userInput string) (Source, func(), error) {
 		return s, func() {}, nil
 
 	case ImageScheme:
-		img, err := stereoscope.GetImage(location)
+		img, err := stereoscope.GetImageFromSource(location, imageSource, registryOptions)
 		cleanup := func() {
 			stereoscope.Cleanup()
 		}

syft/source/source_test.go

@@ -1,12 +1,9 @@
 package source
 
 import (
-	"os"
 	"testing"
 
 	"github.com/anchore/stereoscope/pkg/image"
-	"github.com/mitchellh/go-homedir"
-	"github.com/spf13/afero"
 )
 
 func TestNewFromImageFails(t *testing.T) {
@@ -173,248 +170,3 @@ func TestFilesByGlob(t *testing.T) {
 		})
 	}
 }
-
-func TestDetectScheme(t *testing.T) {
-	type detectorResult struct {
-		src image.Source
-		ref string
-		err error
-	}
-
-	testCases := []struct {
-		name             string
-		userInput        string
-		dirs             []string
-		detection        detectorResult
-		expectedScheme   Scheme
-		expectedLocation string
-	}{
-		{
-			name:      "docker-image-ref",
-			userInput: "wagoodman/dive:latest",
-			detection: detectorResult{
-				src: image.DockerDaemonSource,
-				ref: "wagoodman/dive:latest",
-			},
-			expectedScheme:   ImageScheme,
-			expectedLocation: "wagoodman/dive:latest",
-		},
-		{
-			name:      "docker-image-ref-no-tag",
-			userInput: "wagoodman/dive",
-			detection: detectorResult{
-				src: image.DockerDaemonSource,
-				ref: "wagoodman/dive",
-			},
-			expectedScheme:   ImageScheme,
-			expectedLocation: "wagoodman/dive",
-		},
-		{
-			name:      "docker-image-explicit-scheme",
-			userInput: "docker:wagoodman/dive:latest",
-			detection: detectorResult{
-				src: image.DockerDaemonSource,
-				ref: "wagoodman/dive:latest",
-			},
-			expectedScheme:   ImageScheme,
-			expectedLocation: "wagoodman/dive:latest",
-		},
-		{
-			name:      "docker-image-explicit-scheme-no-tag",
-			userInput: "docker:wagoodman/dive",
-			detection: detectorResult{
-				src: image.DockerDaemonSource,
-				ref: "wagoodman/dive",
-			},
-			expectedScheme:   ImageScheme,
-			expectedLocation: "wagoodman/dive",
-		},
-		{
-			name:      "docker-image-edge-case",
-			userInput: "docker:latest",
-			detection: detectorResult{
-				src: image.DockerDaemonSource,
-				ref: "latest",
-			},
-			expectedScheme: ImageScheme,
-			// we want to be able to handle this case better, however, I don't see a way to do this
-			// the user will need to provide more explicit input (docker:docker:latest)
-			expectedLocation: "latest",
-		},
-		{
-			name:      "docker-image-edge-case-explicit",
-			userInput: "docker:docker:latest",
-			detection: detectorResult{
-				src: image.DockerDaemonSource,
-				ref: "docker:latest",
-			},
-			expectedScheme: ImageScheme,
-			// we want to be able to handle this case better, however, I don't see a way to do this
-			// the user will need to provide more explicit input (docker:docker:latest)
-			expectedLocation: "docker:latest",
-		},
-		{
-			name:      "oci-tar",
-			userInput: "some/path-to-file",
-			detection: detectorResult{
-				src: image.OciTarballSource,
-				ref: "some/path-to-file",
-			},
-			expectedScheme:   ImageScheme,
-			expectedLocation: "some/path-to-file",
-		},
-		{
-			name:      "oci-dir",
-			userInput: "some/path-to-dir",
-			detection: detectorResult{
-				src: image.OciDirectorySource,
-				ref: "some/path-to-dir",
-			},
-			dirs:             []string{"some/path-to-dir"},
-			expectedScheme:   ImageScheme,
-			expectedLocation: "some/path-to-dir",
-		},
-		{
-			name:      "guess-dir",
-			userInput: "some/path-to-dir",
-			detection: detectorResult{
-				src: image.UnknownSource,
-				ref: "",
-			},
-			dirs:             []string{"some/path-to-dir"},
-			expectedScheme:   DirectoryScheme,
-			expectedLocation: "some/path-to-dir",
-		},
-		{
-			name:      "generic-dir-does-not-exist",
-			userInput: "some/path-to-dir",
-			detection: detectorResult{
-				src: image.DockerDaemonSource,
-				ref: "some/path-to-dir",
-			},
-			expectedScheme:   ImageScheme,
-			expectedLocation: "some/path-to-dir",
-		},
-		{
-			name:      "explicit-dir",
-			userInput: "dir:some/path-to-dir",
-			detection: detectorResult{
-				src: image.UnknownSource,
-				ref: "",
-			},
-			dirs:             []string{"some/path-to-dir"},
-			expectedScheme:   DirectoryScheme,
-			expectedLocation: "some/path-to-dir",
-		},
-		{
-			name:      "explicit-current-dir",
-			userInput: "dir:.",
-			detection: detectorResult{
-				src: image.UnknownSource,
-				ref: "",
-			},
-			expectedScheme:   DirectoryScheme,
-			expectedLocation: ".",
-		},
-		{
-			name:      "current-dir",
-			userInput: ".",
-			detection: detectorResult{
-				src: image.UnknownSource,
-				ref: "",
-			},
-			expectedScheme:   DirectoryScheme,
-			expectedLocation: ".",
-		},
-		// we should support tilde expansion
-		{
-			name:      "tilde-expansion-image-implicit",
-			userInput: "~/some-path",
-			detection: detectorResult{
-				src: image.OciDirectorySource,
-				ref: "~/some-path",
-			},
-			expectedScheme:   ImageScheme,
-			expectedLocation: "~/some-path",
-		},
-		{
-			name:      "tilde-expansion-dir-implicit",
-			userInput: "~/some-path",
-			detection: detectorResult{
-				src: image.UnknownSource,
-				ref: "",
-			},
-			dirs:             []string{"~/some-path"},
-			expectedScheme:   DirectoryScheme,
-			expectedLocation: "~/some-path",
-		},
-		{
-			name:             "tilde-expansion-dir-explicit-exists",
-			userInput:        "dir:~/some-path",
-			dirs:             []string{"~/some-path"},
-			expectedScheme:   DirectoryScheme,
-			expectedLocation: "~/some-path",
-		},
-		{
-			name:             "tilde-expansion-dir-explicit-dne",
-			userInput:        "dir:~/some-path",
-			expectedScheme:   DirectoryScheme,
-			expectedLocation: "~/some-path",
-		},
-		{
-			name:             "tilde-expansion-dir-implicit-dne",
-			userInput:        "~/some-path",
-			expectedScheme:   UnknownScheme,
-			expectedLocation: "",
-		},
-	}
-	for _, test := range testCases {
-		t.Run(test.name, func(t *testing.T) {
-			fs := afero.NewMemMapFs()
-
-			for _, p := range test.dirs {
-				expandedExpectedLocation, err := homedir.Expand(p)
-				if err != nil {
-					t.Fatalf("unable to expand path=%q: %+v", p, err)
-				}
-				err = fs.Mkdir(expandedExpectedLocation, os.ModePerm)
-				if err != nil {
-					t.Fatalf("failed to create dummy tar: %+v", err)
-				}
-			}
-
-			imageDetector := func(string) (image.Source, string, error) {
-				// lean on the users real home directory value
-				switch test.detection.src {
-				case image.OciDirectorySource, image.DockerTarballSource, image.OciTarballSource:
-					expandedExpectedLocation, err := homedir.Expand(test.expectedLocation)
-					if err != nil {
-						t.Fatalf("unable to expand path=%q: %+v", test.expectedLocation, err)
-					}
-					return test.detection.src, expandedExpectedLocation, test.detection.err
-				default:
-					return test.detection.src, test.detection.ref, test.detection.err
-				}
-			}
-
-			actualScheme, actualLocation, err := detectScheme(fs, imageDetector, test.userInput)
-			if err != nil {
-				t.Fatalf("unexpected err : %+v", err)
-			}
-
-			if actualScheme != test.expectedScheme {
-				t.Errorf("expected scheme %q , got %q", test.expectedScheme, actualScheme)
-			}
-
-			// lean on the users real home directory value
-			expandedExpectedLocation, err := homedir.Expand(test.expectedLocation)
-			if err != nil {
-				t.Fatalf("unable to expand path=%q: %+v", test.expectedLocation, err)
-			}
-
-			if actualLocation != expandedExpectedLocation {
-				t.Errorf("expected location %q , got %q", expandedExpectedLocation, actualLocation)
-			}
-		})
-	}
-}