Incorporate import changes + add image overwrite option (#294)

* incorporate import changes + add image overwrite option

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update import tests to account for arbitrary json shape

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

cmd/cmd.go

@@ -86,7 +86,7 @@ func setGlobalUploadOptions() {
 	flag := "host"
 	rootCmd.Flags().StringP(
 		flag, "H", "",
-		"the hostname or URL of the Anchore Engine/Enterprise instance to upload to",
+		"the hostname or URL of the Anchore Enterprise instance to upload to",
 	)
 	if err := viper.BindPFlag("anchore.host", rootCmd.Flags().Lookup(flag)); err != nil {
 		fmt.Printf("unable to bind flag '%s': %+v", flag, err)
@@ -96,7 +96,7 @@ func setGlobalUploadOptions() {
 	flag = "username"
 	rootCmd.Flags().StringP(
 		flag, "u", "",
-		"the username to authenticate against Anchore Engine/Enterprise",
+		"the username to authenticate against Anchore Enterprise",
 	)
 	if err := viper.BindPFlag("anchore.username", rootCmd.Flags().Lookup(flag)); err != nil {
 		fmt.Printf("unable to bind flag '%s': %+v", flag, err)
@@ -106,7 +106,7 @@ func setGlobalUploadOptions() {
 	flag = "password"
 	rootCmd.Flags().StringP(
 		flag, "p", "",
-		"the password to authenticate against Anchore Engine/Enterprise",
+		"the password to authenticate against Anchore Enterprise",
 	)
 	if err := viper.BindPFlag("anchore.password", rootCmd.Flags().Lookup(flag)); err != nil {
 		fmt.Printf("unable to bind flag '%s': %+v", flag, err)
@@ -116,12 +116,22 @@ func setGlobalUploadOptions() {
 	flag = "dockerfile"
 	rootCmd.Flags().StringP(
 		flag, "d", "",
-		"include dockerfile for upload to Anchore Engine/Enterprise",
+		"include dockerfile for upload to Anchore Enterprise",
 	)
 	if err := viper.BindPFlag("anchore.dockerfile", rootCmd.Flags().Lookup(flag)); err != nil {
 		fmt.Printf("unable to bind flag '#{flag}': #{err}")
 		os.Exit(1)
 	}
+
+	flag = "overwrite-existing-image"
+	rootCmd.Flags().Bool(
+		flag, false,
+		"overwrite an existing image during the upload to Anchore Enterprise",
+	)
+	if err := viper.BindPFlag("anchore.overwrite-existing-image", rootCmd.Flags().Lookup(flag)); err != nil {
+		fmt.Printf("unable to bind flag '#{flag}': #{err}")
+		os.Exit(1)
+	}
 }
 
 func initAppConfig() {

cmd/root.go

@@ -176,7 +176,16 @@ func doImport(src source.Source, s source.Metadata, catalog *pkg.Catalog, d *dis
 		return fmt.Errorf("failed to create anchore client: %+v", err)
 	}
 
-	if err := c.Import(context.Background(), src.Image.Metadata, s, catalog, d, dockerfileContents); err != nil {
+	importCfg := anchore.ImportConfig{
+		ImageMetadata:           src.Image.Metadata,
+		SourceMetadata:          s,
+		Catalog:                 catalog,
+		Distro:                  d,
+		Dockerfile:              dockerfileContents,
+		OverwriteExistingUpload: appConfig.Anchore.OverwriteExistingImage,
+	}
+
+	if err := c.Import(context.Background(), importCfg); err != nil {
 		return fmt.Errorf("failed to upload results to host=%s: %+v", appConfig.Anchore.Host, err)
 	}
 	return nil

go.mod

@@ -5,11 +5,12 @@ go 1.14
 require (
 	github.com/adrg/xdg v0.2.1
 	github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921
-	github.com/anchore/client-go v0.0.0-20201210022459-59e7a0749c74
+	github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238
 	github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12
 	github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
 	github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
 	github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e
+	github.com/antihax/optional v1.0.0
 	github.com/bmatcuk/doublestar v1.3.3
 	github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible
 	github.com/dustin/go-humanize v1.0.0

go.sum

@@ -128,6 +128,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/anchore/client-go v0.0.0-20201210022459-59e7a0749c74 h1:9kkKTIyXJC+/syUcY6KWxFoJZJ+GWwrIscF+gBY067k=
 github.com/anchore/client-go v0.0.0-20201210022459-59e7a0749c74/go.mod h1:FaODhIA06mxO1E6R32JE0TL1JWZZkmjRIAd4ULvHUKk=
+github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238 h1:/iI+1cj1a27ow0wj378pPJIm8sCSy6I21Tz6oLbLDQY=
+github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238/go.mod h1:FaODhIA06mxO1E6R32JE0TL1JWZZkmjRIAd4ULvHUKk=
 github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 h1:xbeIbn5F52JVx3RUIajxCj8b0y+9lywspql4sFhcxWQ=
 github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12/go.mod h1:juoyWXIj7sJ1IDl4E/KIfyLtovbs5XQVSIdaQifFQT8=
 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0vW0nnNKJfJieyH/TZ9UYAnTZs5/gHTdAe8=

internal/anchore/import.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/antihax/optional"
+
 	"github.com/anchore/client-go/pkg/external"
 	"github.com/anchore/stereoscope/pkg/image"
 	"github.com/anchore/syft/internal/bus"
@@ -17,6 +19,15 @@ import (
 	"github.com/wagoodman/go-progress"
 )
 
+type ImportConfig struct {
+	ImageMetadata           image.Metadata
+	SourceMetadata          source.Metadata
+	Catalog                 *pkg.Catalog
+	Distro                  *distro.Distro
+	Dockerfile              []byte
+	OverwriteExistingUpload bool
+}
+
 func importProgress(source string) (*progress.Stage, *progress.Manual) {
 	stage := &progress.Stage{}
 	prog := &progress.Manual{
@@ -39,8 +50,8 @@ func importProgress(source string) (*progress.Stage, *progress.Manual) {
 }
 
 // nolint:funlen
-func (c *Client) Import(ctx context.Context, imageMetadata image.Metadata, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, dockerfile []byte) error {
+func (c *Client) Import(ctx context.Context, cfg ImportConfig) error {
-	stage, prog := importProgress(imageMetadata.ID)
+	stage, prog := importProgress(c.config.Hostname)
 
 	ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
@@ -60,33 +71,37 @@ func (c *Client) Import(ctx context.Context, imageMetadata image.Metadata, s sou
 	prog.N++
 	sessionID := startOperation.Uuid
 
-	packageDigest, err := importPackageSBOM(authedCtx, c.client.ImportsApi, sessionID, s, catalog, d, stage)
+	packageDigest, err := importPackageSBOM(authedCtx, c.client.ImportsApi, sessionID, cfg.SourceMetadata, cfg.Catalog, cfg.Distro, stage)
 	if err != nil {
 		return fmt.Errorf("failed to import Package SBOM: %w", err)
 	}
 	prog.N++
 
-	manifestDigest, err := importManifest(authedCtx, c.client.ImportsApi, sessionID, imageMetadata.RawManifest, stage)
+	manifestDigest, err := importManifest(authedCtx, c.client.ImportsApi, sessionID, cfg.ImageMetadata.RawManifest, stage)
 	if err != nil {
 		return fmt.Errorf("failed to import Manifest: %w", err)
 	}
 	prog.N++
 
-	configDigest, err := importConfig(authedCtx, c.client.ImportsApi, sessionID, imageMetadata.RawConfig, stage)
+	configDigest, err := importConfig(authedCtx, c.client.ImportsApi, sessionID, cfg.ImageMetadata.RawConfig, stage)
 	if err != nil {
 		return fmt.Errorf("failed to import Config: %w", err)
 	}
 	prog.N++
 
-	dockerfileDigest, err := importDockerfile(authedCtx, c.client.ImportsApi, sessionID, dockerfile, stage)
+	dockerfileDigest, err := importDockerfile(authedCtx, c.client.ImportsApi, sessionID, cfg.Dockerfile, stage)
 	if err != nil {
 		return fmt.Errorf("failed to import Dockerfile: %w", err)
 	}
 	prog.N++
 
 	stage.Current = "finalizing"
-	imageModel := addImageModel(imageMetadata, packageDigest, manifestDigest, dockerfileDigest, configDigest, sessionID)
+	imageModel := addImageModel(cfg.ImageMetadata, packageDigest, manifestDigest, dockerfileDigest, configDigest, sessionID)
-	_, _, err = c.client.ImagesApi.AddImage(authedCtx, imageModel, nil)
+	opts := external.AddImageOpts{
+		Force: optional.NewBool(cfg.OverwriteExistingUpload),
+	}
+
+	_, _, err = c.client.ImagesApi.AddImage(authedCtx, imageModel, &opts)
 	if err != nil {
 		var detail = "no details given"
 		var openAPIErr external.GenericOpenAPIError

internal/anchore/import_config.go

@@ -3,6 +3,7 @@ package anchore
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -14,15 +15,21 @@ import (
 )
 
 type configImportAPI interface {
-	ImportImageConfig(ctx context.Context, sessionID string, contents string) (external.ImageImportContentResponse, *http.Response, error)
+	ImportImageConfig(ctx context.Context, sessionID string, contents interface{}) (external.ImageImportContentResponse, *http.Response, error)
 }
 
-func importConfig(ctx context.Context, api configImportAPI, sessionID string, manifest []byte, stage *progress.Stage) (string, error) {
+func importConfig(ctx context.Context, api configImportAPI, sessionID string, config []byte, stage *progress.Stage) (string, error) {
-	if len(manifest) > 0 {
+	if len(config) > 0 {
 		log.Debug("importing image config")
 		stage.Current = "image config"
 
-		response, httpResponse, err := api.ImportImageConfig(ctx, sessionID, string(manifest))
+		// API requires an object, but we do not verify the shape of this object locally
+		var sender map[string]interface{}
+		if err := json.Unmarshal(config, &sender); err != nil {
+			return "", err
+		}
+
+		response, httpResponse, err := api.ImportImageConfig(ctx, sessionID, sender)
 		if err != nil {
 			var openAPIErr external.GenericOpenAPIError
 			if errors.As(err, &openAPIErr) {

internal/anchore/import_config_test.go

@@ -2,6 +2,7 @@ package anchore
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"strings"
@@ -16,7 +17,7 @@ import (
 
 type mockConfigImportAPI struct {
 	sessionID      string
-	model          string
+	model          interface{}
 	httpResponse   *http.Response
 	err            error
 	ctx            context.Context
@@ -24,7 +25,7 @@ type mockConfigImportAPI struct {
 	wasCalled      bool
 }
 
-func (m *mockConfigImportAPI) ImportImageConfig(ctx context.Context, sessionID string, contents string) (external.ImageImportContentResponse, *http.Response, error) {
+func (m *mockConfigImportAPI) ImportImageConfig(ctx context.Context, sessionID string, contents interface{}) (external.ImageImportContentResponse, *http.Response, error) {
 	m.wasCalled = true
 	m.model = contents
 	m.sessionID = sessionID
@@ -41,16 +42,16 @@ func TestConfigImport(t *testing.T) {
 	sessionID := "my-session"
 
 	tests := []struct {
-		name         string
+		name            string
-		manifest     string
+		manifestJSONStr string
-		api          *mockConfigImportAPI
+		api             *mockConfigImportAPI
-		expectsError bool
+		expectsError    bool
-		expectsCall  bool
+		expectsCall     bool
 	}{
 
 		{
-			name:     "Go case: import works",
+			name:            "Go case: import works",
-			manifest: "the-manifest-contents!",
+			manifestJSONStr: `{ "key": "the-manifest-contents!" }`,
 			api: &mockConfigImportAPI{
 				httpResponse:   &http.Response{StatusCode: 200},
 				responseDigest: "digest!",
@@ -58,14 +59,14 @@ func TestConfigImport(t *testing.T) {
 			expectsCall: true,
 		},
 		{
-			name:        "No manifest provided",
+			name:            "No manifest provided",
-			manifest:    "",
+			manifestJSONStr: "",
-			api:         &mockConfigImportAPI{},
+			api:             &mockConfigImportAPI{},
-			expectsCall: false,
+			expectsCall:     false,
 		},
 		{
-			name:     "API returns an error",
+			name:            "API returns an error",
-			manifest: "the-manifest-contents!",
+			manifestJSONStr: `{ "key": "the-manifest-contents!" }`,
 			api: &mockConfigImportAPI{
 				err: fmt.Errorf("api error, something went wrong"),
 			},
@@ -73,8 +74,8 @@ func TestConfigImport(t *testing.T) {
 			expectsCall:  true,
 		},
 		{
-			name:     "API HTTP-level error",
+			name:            "API HTTP-level error",
-			manifest: "the-manifest-contents!",
+			manifestJSONStr: `{ "key": "the-manifest-contents!" }`,
 			api: &mockConfigImportAPI{
 				httpResponse: &http.Response{StatusCode: 404},
 			},
@@ -86,7 +87,7 @@ func TestConfigImport(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 
-			digest, err := importConfig(context.TODO(), test.api, sessionID, []byte(test.manifest), &progress.Stage{})
+			digest, err := importConfig(context.TODO(), test.api, sessionID, []byte(test.manifestJSONStr), &progress.Stage{})
 
 			// validate error handling
 			if err != nil && !test.expectsError {
@@ -114,7 +115,12 @@ func TestConfigImport(t *testing.T) {
 				t.Errorf("different session ID: %s != %s", test.api.sessionID, sessionID)
 			}
 
-			for _, d := range deep.Equal(test.api.model, test.manifest) {
+			var expected map[string]interface{}
+			if err := json.Unmarshal([]byte(test.manifestJSONStr), &expected); err != nil {
+				t.Fatalf("could not unmarshal expected results")
+			}
+
+			for _, d := range deep.Equal(test.api.model, expected) {
 				t.Errorf("model difference: %s", d)
 			}
 

internal/anchore/import_dockerfile.go

@@ -1,4 +1,3 @@
-// nolint:dupl
 package anchore
 
 import (

internal/anchore/import_manifest.go

@@ -3,6 +3,7 @@ package anchore
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -14,7 +15,7 @@ import (
 )
 
 type manifestImportAPI interface {
-	ImportImageManifest(ctx context.Context, sessionID string, contents string) (external.ImageImportContentResponse, *http.Response, error)
+	ImportImageManifest(ctx context.Context, sessionID string, contents interface{}) (external.ImageImportContentResponse, *http.Response, error)
 }
 
 func importManifest(ctx context.Context, api manifestImportAPI, sessionID string, manifest []byte, stage *progress.Stage) (string, error) {
@@ -22,7 +23,13 @@ func importManifest(ctx context.Context, api manifestImportAPI, sessionID string
 		log.Debug("importing image manifest")
 		stage.Current = "image manifest"
 
-		response, httpResponse, err := api.ImportImageManifest(ctx, sessionID, string(manifest))
+		// API requires an object, but we do not verify the shape of this object locally
+		var sender map[string]interface{}
+		if err := json.Unmarshal(manifest, &sender); err != nil {
+			return "", err
+		}
+
+		response, httpResponse, err := api.ImportImageManifest(ctx, sessionID, sender)
 		if err != nil {
 			var openAPIErr external.GenericOpenAPIError
 			if errors.As(err, &openAPIErr) {

internal/anchore/import_manifest_test.go

@@ -2,6 +2,7 @@ package anchore
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"strings"
@@ -16,7 +17,7 @@ import (
 
 type mockManifestImportAPI struct {
 	sessionID      string
-	model          string
+	model          interface{}
 	httpResponse   *http.Response
 	err            error
 	ctx            context.Context
@@ -24,7 +25,7 @@ type mockManifestImportAPI struct {
 	wasCalled      bool
 }
 
-func (m *mockManifestImportAPI) ImportImageManifest(ctx context.Context, sessionID string, contents string) (external.ImageImportContentResponse, *http.Response, error) {
+func (m *mockManifestImportAPI) ImportImageManifest(ctx context.Context, sessionID string, contents interface{}) (external.ImageImportContentResponse, *http.Response, error) {
 	m.wasCalled = true
 	m.model = contents
 	m.sessionID = sessionID
@@ -50,7 +51,7 @@ func TestManifestImport(t *testing.T) {
 
 		{
 			name:     "Go case: import works",
-			manifest: "the-manifest-contents!",
+			manifest: `{ "key": "the-config-contents!" }`,
 			api: &mockManifestImportAPI{
 				httpResponse:   &http.Response{StatusCode: 200},
 				responseDigest: "digest!",
@@ -65,7 +66,7 @@ func TestManifestImport(t *testing.T) {
 		},
 		{
 			name:     "API returns an error",
-			manifest: "the-manifest-contents!",
+			manifest: `{ "key": "the-config-contents!" }`,
 			api: &mockManifestImportAPI{
 				err: fmt.Errorf("api error, something went wrong"),
 			},
@@ -74,7 +75,7 @@ func TestManifestImport(t *testing.T) {
 		},
 		{
 			name:     "API HTTP-level error",
-			manifest: "the-manifest-contents!",
+			manifest: `{ "key": "the-config-contents!" }`,
 			api: &mockManifestImportAPI{
 				httpResponse: &http.Response{StatusCode: 404},
 			},
@@ -114,7 +115,12 @@ func TestManifestImport(t *testing.T) {
 				t.Errorf("different session ID: %s != %s", test.api.sessionID, sessionID)
 			}
 
-			for _, d := range deep.Equal(test.api.model, test.manifest) {
+			var expected map[string]interface{}
+			if err := json.Unmarshal([]byte(test.manifest), &expected); err != nil {
+				t.Fatalf("could not unmarshal expected results")
+			}
+
+			for _, d := range deep.Equal(test.api.model, expected) {
 				t.Errorf("model difference: %s", d)
 			}
 

internal/config/config.go

@@ -46,12 +46,13 @@ type logging struct {
 
 type anchore struct {
 	// upload options
-	UploadEnabled bool   `yaml:"upload-enabled"  mapstructure:"upload-enabled"` // whether to upload results to Anchore Engine/Enterprise (defaults to "false" unless there is the presence of -h CLI option)
+	UploadEnabled          bool   `yaml:"upload-enabled"  mapstructure:"upload-enabled"`                    // whether to upload results to Anchore Engine/Enterprise (defaults to "false" unless there is the presence of -h CLI option)
-	Host          string `yaml:"host" mapstructure:"host"`                      // -H , hostname of the engine/enterprise instance to upload to
+	Host                   string `yaml:"host" mapstructure:"host"`                                         // -H , hostname of the engine/enterprise instance to upload to
-	Path          string `yaml:"path" mapstructure:"path"`                      // override the engine/enterprise API upload path
+	Path                   string `yaml:"path" mapstructure:"path"`                                         // override the engine/enterprise API upload path
-	Username      string `yaml:"username" mapstructure:"username"`              // -u , username to authenticate upload
+	Username               string `yaml:"username" mapstructure:"username"`                                 // -u , username to authenticate upload
-	Password      string `yaml:"password" mapstructure:"password"`              // -p , password to authenticate upload
+	Password               string `yaml:"password" mapstructure:"password"`                                 // -p , password to authenticate upload
-	Dockerfile    string `yaml:"dockerfile" mapstructure:"dockerfile"`          // -d , dockerfile to attach for upload
+	Dockerfile             string `yaml:"dockerfile" mapstructure:"dockerfile"`                             // -d , dockerfile to attach for upload
+	OverwriteExistingImage bool   `yaml:"overwrite-existing-image" mapstructure:"overwrite-existing-image"` // --overwrite-existing-image , if any of the SBOM components have already been uploaded this flag will ensure they are overwritten with the current upload
 }
 
 type Development struct {

syft/event/parsers/parsers.go

@@ -83,7 +83,7 @@ func ParseImportStarted(e partybus.Event) (string, progress.StagedProgressable,
 		return "", nil, err
 	}
 
-	imgName, ok := e.Source.(string)
+	host, ok := e.Source.(string)
 	if !ok {
 		return "", nil, newPayloadErr(e.Type, "Source", e.Source)
 	}
@@ -93,5 +93,5 @@ func ParseImportStarted(e partybus.Event) (string, progress.StagedProgressable,
 		return "", nil, newPayloadErr(e.Type, "Value", e.Value)
 	}
 
-	return imgName, prog, nil
+	return host, prog, nil
 }

ui/event_handlers.go

@@ -312,7 +312,7 @@ func CatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event partybu
 // ImportStartedHandler shows the intermittent upload progress to Anchore Enterprise.
 // nolint:dupl
 func ImportStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error {
-	_, prog, err := syftEventParsers.ParseImportStarted(event)
+	host, prog, err := syftEventParsers.ParseImportStarted(event)
 	if err != nil {
 		return fmt.Errorf("bad %s event: %w", event.Type, err)
 	}
@@ -348,7 +348,8 @@ func ImportStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.E
 
 		spin := color.Green.Sprint(completedStatus)
 		title = tileFormat.Sprint("Uploaded image")
-		_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate, spin, title))
+		auxInfo := auxInfoFormat.Sprintf("[%s]", host)
+		_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s", spin, title, auxInfo))
 	}()
 	return err
 }