Replace distro type (#742)

* remove strong distro type Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * bump json schema to v3 (breaking distro shape) Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix linting Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * allow for v2 decoding of distro idLikes field in v3 json decoder Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix casing in simple linux release name Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * use discovered name as pretty name in simple linux release Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2026-11-01 23:16:30 +01:00 · 2022-01-12 12:13:42 -05:00
parent dfefd2ea4e
commit 706f291679
74 changed files with 2452 additions and 1136 deletions
@@ -8,31 +8,82 @@ import (
 	"fmt"
 	"net/http"

-	"github.com/anchore/syft/syft/sbom"
-
-	"github.com/anchore/syft/internal/formats/syftjson"
-
-	"github.com/wagoodman/go-progress"
-
 	"github.com/anchore/client-go/pkg/external"
+	"github.com/anchore/syft/internal/formats/syftjson"
+	syftjsonModel "github.com/anchore/syft/internal/formats/syftjson/model"
 	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/sbom"
+	"github.com/wagoodman/go-progress"
 )

 type packageSBOMImportAPI interface {
 	ImportImagePackages(context.Context, string, external.ImagePackageManifest) (external.ImageImportContentResponse, *http.Response, error)
 }

+// importSBOM mirrors all elements found on the syftjson model format object relative to the anchore engine import schema.
+type importSBOM struct {
+	Artifacts             []syftjsonModel.Package      `json:"artifacts"` // Artifacts is the list of packages discovered and placed into the catalog
+	ArtifactRelationships []syftjsonModel.Relationship `json:"artifactRelationships"`
+	Files                 []syftjsonModel.File         `json:"files,omitempty"`   // note: must have omitempty
+	Secrets               []syftjsonModel.Secrets      `json:"secrets,omitempty"` // note: must have omitempty
+	Source                syftjsonModel.Source         `json:"source"`            // Source represents the original object that was cataloged
+	Distro                external.ImportDistribution  `json:"distro"`            // Distro represents the Linux distribution that was detected from the source
+	Descriptor            syftjsonModel.Descriptor     `json:"descriptor"`        // Descriptor is a block containing self-describing information about syft
+	Schema                syftjsonModel.Schema         `json:"schema"`            // Schema is a block reserved for defining the version for the shape of this JSON document and where to find the schema document to validate the shape
+}
+
+// toImportSBOMModel transforms the current sbom shape into what is needed for the current anchore import api shape.
+func toImportSBOMModel(s sbom.SBOM) importSBOM {
+	m := syftjson.ToFormatModel(s)
+
+	var idLike string
+	if len(m.Distro.IDLike) > 0 {
+		idLike = m.Distro.IDLike[0]
+	}
+
+	var version = m.Distro.VersionID // note: version is intentionally not used as the default
+	if version == "" {
+		version = m.Distro.Version
+	}
+
+	var name = m.Distro.ID // note: name is intentionally not used as the default
+	if name == "" {
+		name = m.Distro.Name
+	}
+
+	return importSBOM{
+		Artifacts:             m.Artifacts,
+		ArtifactRelationships: m.ArtifactRelationships,
+		Files:                 m.Files,
+		Secrets:               m.Secrets,
+		Source:                m.Source,
+		Distro: external.ImportDistribution{
+			Name:    name,
+			Version: version,
+			IdLike:  idLike,
+		},
+		Descriptor: m.Descriptor,
+		Schema:     m.Schema,
+	}
+}
+
 func packageSbomModel(s sbom.SBOM) (*external.ImagePackageManifest, error) {
 	var buf bytes.Buffer

-	err := syftjson.Format().Encode(&buf, s)
-	if err != nil {
-		return nil, fmt.Errorf("unable to serialize results: %w", err)
+	doc := toImportSBOMModel(s)
+
+	enc := json.NewEncoder(&buf)
+	// prevent > and < from being escaped in the payload
+	enc.SetEscapeHTML(false)
+	enc.SetIndent("", " ")
+
+	if err := enc.Encode(&doc); err != nil {
+		return nil, fmt.Errorf("unable to encode import JSON model: %w", err)
 	}

 	// the model is 1:1 the JSON output of today. As the schema changes, this will need to be converted into individual mappings.
 	var model external.ImagePackageManifest
-	if err = json.Unmarshal(buf.Bytes(), &model); err != nil {
+	if err := json.Unmarshal(buf.Bytes(), &model); err != nil {
 		return nil, fmt.Errorf("unable to convert JSON output to import model: %w", err)
 	}

@@ -1,7 +1,6 @@
 package anchore

 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -9,16 +8,17 @@ import (
 	"strings"
 	"testing"

-	"github.com/anchore/syft/syft/sbom"
-
 	"github.com/anchore/client-go/pkg/external"
 	"github.com/anchore/syft/internal/formats/syftjson"
-	syftjsonModel "github.com/anchore/syft/internal/formats/syftjson/model"
-	"github.com/anchore/syft/syft/distro"
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/linux"
 	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/sbom"
 	"github.com/anchore/syft/syft/source"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/go-test/deep"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"github.com/wagoodman/go-progress"
 )

@@ -29,107 +29,6 @@ func must(c pkg.CPE, e error) pkg.CPE {
 	return c
 }

-// this test is tailored towards the assumption that the import doc shape and the syft json shape are the same.
-// TODO: replace this as the document shapes diverge.
-func TestPackageSbomToModel(t *testing.T) {
-
-	m := source.Metadata{
-		Scheme: source.ImageScheme,
-		ImageMetadata: source.ImageMetadata{
-			UserInput: "user-in",
-			Layers: []source.LayerMetadata{
-				{
-					MediaType: "layer-metadata-type!",
-					Digest:    "layer-digest",
-					Size:      20,
-				},
-			},
-			Size:           10,
-			ManifestDigest: "sha256:digest!",
-			MediaType:      "mediatype!",
-			Tags:           nil,
-		},
-	}
-
-	d, _ := distro.NewDistro(distro.CentOS, "8.0", "")
-
-	p := pkg.Package{
-		Name:    "name",
-		Version: "version",
-		FoundBy: "foundBy",
-		Locations: []source.Location{
-			{
-				Coordinates: source.Coordinates{
-					RealPath:     "path",
-					FileSystemID: "layerID",
-				},
-			},
-		},
-		Licenses: []string{"license"},
-		Language: pkg.Python,
-		Type:     pkg.PythonPkg,
-		CPEs: []pkg.CPE{
-			must(pkg.NewCPE("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*")),
-		},
-		PURL: "purl",
-	}
-
-	c := pkg.NewCatalog(p)
-
-	sbomResult := sbom.SBOM{
-		Artifacts: sbom.Artifacts{
-			PackageCatalog: c,
-			Distro:         &d,
-		},
-		Source: m,
-	}
-
-	model, err := packageSbomModel(sbomResult)
-	if err != nil {
-		t.Fatalf("unable to generate model from source material: %+v", err)
-	}
-
-	var modelJSON []byte
-
-	modelJSON, err = json.Marshal(&model)
-	if err != nil {
-		t.Fatalf("unable to marshal model: %+v", err)
-	}
-
-	s := sbom.SBOM{
-		Artifacts: sbom.Artifacts{
-			PackageCatalog: c,
-			Distro:         &d,
-		},
-		Source: m,
-	}
-
-	var buf bytes.Buffer
-	if err := syftjson.Format().Encode(&buf, s); err != nil {
-		t.Fatalf("unable to get expected json: %+v", err)
-	}
-
-	// unmarshal expected result
-	var expectedDoc syftjsonModel.Document
-	if err := json.Unmarshal(buf.Bytes(), &expectedDoc); err != nil {
-		t.Fatalf("unable to parse json doc: %+v", err)
-	}
-
-	// unmarshal actual result
-	var actualDoc syftjsonModel.Document
-	if err := json.Unmarshal(modelJSON, &actualDoc); err != nil {
-		t.Fatalf("unable to parse json doc: %+v", err)
-	}
-
-	for _, d := range deep.Equal(actualDoc, expectedDoc) {
-		if strings.HasSuffix(d, "<nil slice> != []") {
-			// do not consider nil vs empty collection semantics as a "difference"
-			continue
-		}
-		t.Errorf("diff: %+v", d)
-	}
-}
-
 type mockPackageSBOMImportAPI struct {
 	sessionID      string
 	model          external.ImagePackageManifest
@@ -150,72 +49,81 @@ func (m *mockPackageSBOMImportAPI) ImportImagePackages(ctx context.Context, sess
 	return external.ImageImportContentResponse{Digest: m.responseDigest}, m.httpResponse, m.err
 }

-func TestPackageSbomImport(t *testing.T) {
-
-	catalog := pkg.NewCatalog(pkg.Package{
-		Name:    "name",
-		Version: "version",
-		FoundBy: "foundBy",
-		Locations: []source.Location{
-			{
-				Coordinates: source.Coordinates{
-					RealPath:     "path",
-					FileSystemID: "layerID",
-				},
-			},
-		},
-		Licenses: []string{"license"},
-		Language: pkg.Python,
-		Type:     pkg.PythonPkg,
-		CPEs: []pkg.CPE{
-			must(pkg.NewCPE("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*")),
-		},
-		PURL:         "purl",
-		MetadataType: pkg.PythonPackageMetadataType,
-		Metadata: pkg.PythonPackageMetadata{
-			Name:        "p-name",
-			Version:     "p-version",
-			License:     "p-license",
-			Author:      "p-author",
-			AuthorEmail: "p-email",
-			Platform:    "p-platform",
-			Files: []pkg.PythonFileRecord{
-				{
-					Path: "p-path",
-					Digest: &pkg.PythonFileDigest{
-						Algorithm: "p-alg",
-						Value:     "p-digest",
-					},
-					Size: "p-size",
-				},
-			},
-			SitePackagesRootPath: "p-site-packages-root",
-			TopLevelPackages:     []string{"top-level"},
-		},
-	})
-
-	m := source.Metadata{
-		Scheme: source.ImageScheme,
-		ImageMetadata: source.ImageMetadata{
-			UserInput:      "user-in",
-			Layers:         nil,
-			Size:           10,
-			ManifestDigest: "sha256:digest!",
-			MediaType:      "mediatype!",
-			Tags:           nil,
-		},
-	}
-
-	d, _ := distro.NewDistro(distro.CentOS, "8.0", "")
-
-	sbomResult := sbom.SBOM{
+func sbomFixture() sbom.SBOM {
+	return sbom.SBOM{
 		Artifacts: sbom.Artifacts{
-			PackageCatalog: catalog,
-			Distro:         &d,
+			PackageCatalog: pkg.NewCatalog(pkg.Package{
+				Name:    "name",
+				Version: "version",
+				FoundBy: "foundBy",
+				Locations: []source.Location{
+					{
+						Coordinates: source.Coordinates{
+							RealPath:     "path",
+							FileSystemID: "layerID",
+						},
+					},
+				},
+				Licenses: []string{"license"},
+				Language: pkg.Python,
+				Type:     pkg.PythonPkg,
+				CPEs: []pkg.CPE{
+					must(pkg.NewCPE("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*")),
+				},
+				PURL:         "purl",
+				MetadataType: pkg.PythonPackageMetadataType,
+				Metadata: pkg.PythonPackageMetadata{
+					Name:        "p-name",
+					Version:     "p-version",
+					License:     "p-license",
+					Author:      "p-author",
+					AuthorEmail: "p-email",
+					Platform:    "p-platform",
+					Files: []pkg.PythonFileRecord{
+						{
+							Path: "p-path",
+							Digest: &pkg.PythonFileDigest{
+								Algorithm: "p-alg",
+								Value:     "p-digest",
+							},
+							Size: "p-size",
+						},
+					},
+					SitePackagesRootPath: "p-site-packages-root",
+					TopLevelPackages:     []string{"top-level"},
+				},
+			}),
+			LinuxDistribution: &linux.Release{
+				ID:        "centos",
+				Version:   "8.0",
+				VersionID: "8.0",
+				IDLike:    []string{"rhel"},
+			},
+		},
+		Relationships: []artifact.Relationship{
+			{
+				From: source.NewLocation("/place1"),
+				To:   source.NewLocation("/place2"),
+				Type: artifact.ContainsRelationship,
+			},
+		},
+		Source: source.Metadata{
+			Scheme: source.ImageScheme,
+			ImageMetadata: source.ImageMetadata{
+				UserInput:      "user-in",
+				Layers:         nil,
+				Size:           10,
+				ManifestDigest: "sha256:digest!",
+				MediaType:      "mediatype!",
+				Tags:           nil,
+			},
 		},
-		Source: m,
 	}

+}
+
+func TestPackageSbomImport(t *testing.T) {
+	sbomResult := sbomFixture()
 	theModel, err := packageSbomModel(sbomResult)
 	if err != nil {
 		t.Fatalf("could not get sbom model: %+v", err)
@@ -280,3 +188,210 @@ func TestPackageSbomImport(t *testing.T) {
 		})
 	}
 }
+
+type modelAssertion func(t *testing.T, model *external.ImagePackageManifest)
+
+func Test_packageSbomModel(t *testing.T) {
+	fix := sbomFixture()
+
+	tests := []struct {
+		name   string
+		sbom   sbom.SBOM
+		traits []modelAssertion
+	}{
+		{
+			name: "distro: has single distro id-like",
+			sbom: sbom.SBOM{
+				Artifacts: sbom.Artifacts{
+					LinuxDistribution: &linux.Release{
+						Name: "centos-name",
+						ID:   "centos-id",
+						IDLike: []string{
+							"centos-id-like-1",
+						},
+						Version:   "version",
+						VersionID: "version-id",
+					},
+				},
+			},
+			traits: []modelAssertion{
+				hasDistroInfo("centos-id", "version-id", "centos-id-like-1"),
+			},
+		},
+		{
+			name: "distro: has multiple distro id-like",
+			sbom: sbom.SBOM{
+				Artifacts: sbom.Artifacts{
+					LinuxDistribution: &linux.Release{
+						Name: "centos-name",
+						ID:   "centos-id",
+						IDLike: []string{
+							"centos-id-like-1",
+							"centos-id-like-2",
+						},
+						Version:   "version",
+						VersionID: "version-id",
+					},
+				},
+			},
+			traits: []modelAssertion{
+				hasDistroInfo("centos-id", "version-id", "centos-id-like-1"),
+			},
+		},
+		{
+			name: "distro: has no distro id-like",
+			sbom: sbom.SBOM{
+				Artifacts: sbom.Artifacts{
+					LinuxDistribution: &linux.Release{
+						Name:      "centos-name",
+						ID:        "centos-id",
+						IDLike:    []string{},
+						Version:   "version",
+						VersionID: "version-id",
+					},
+				},
+			},
+			traits: []modelAssertion{
+				hasDistroInfo("centos-id", "version-id", ""),
+			},
+		},
+		{
+			name: "distro: has no version-id",
+			sbom: sbom.SBOM{
+				Artifacts: sbom.Artifacts{
+					LinuxDistribution: &linux.Release{
+						Name:      "centos-name",
+						ID:        "centos-id",
+						IDLike:    []string{},
+						Version:   "version",
+						VersionID: "",
+					},
+				},
+			},
+			traits: []modelAssertion{
+				hasDistroInfo("centos-id", "version", ""),
+			},
+		},
+		{
+			name: "distro: has no id",
+			sbom: sbom.SBOM{
+				Artifacts: sbom.Artifacts{
+					LinuxDistribution: &linux.Release{
+						Name:      "centos-name",
+						ID:        "",
+						IDLike:    []string{},
+						Version:   "version",
+						VersionID: "version-id",
+					},
+				},
+			},
+			traits: []modelAssertion{
+				hasDistroInfo("centos-name", "version-id", ""),
+			},
+		},
+		{
+			name: "should have expected packages",
+			sbom: fix,
+			traits: []modelAssertion{
+				func(t *testing.T, model *external.ImagePackageManifest) {
+					require.Len(t, model.Artifacts, 1)
+
+					modelPkg := model.Artifacts
+					modelBytes, err := json.Marshal(&modelPkg)
+					require.NoError(t, err)
+
+					fixPkg := syftjson.ToFormatModel(fix).Artifacts
+					fixBytes, err := json.Marshal(&fixPkg)
+					require.NoError(t, err)
+
+					assert.JSONEq(t, string(fixBytes), string(modelBytes))
+				},
+			},
+		},
+		{
+			name: "should have expected relationships",
+			sbom: fix,
+			traits: []modelAssertion{
+				func(t *testing.T, model *external.ImagePackageManifest) {
+					modelPkg := model.ArtifactRelationships
+					modelBytes, err := json.Marshal(&modelPkg)
+					require.NoError(t, err)
+
+					fixPkg := syftjson.ToFormatModel(fix).ArtifactRelationships
+					fixBytes, err := json.Marshal(&fixPkg)
+					require.NoError(t, err)
+
+					assert.JSONEq(t, string(fixBytes), string(modelBytes))
+				},
+			},
+		},
+		{
+			name: "should have expected schema",
+			sbom: fix,
+			traits: []modelAssertion{
+				func(t *testing.T, model *external.ImagePackageManifest) {
+					modelPkg := model.Schema
+					modelBytes, err := json.Marshal(&modelPkg)
+					require.NoError(t, err)
+
+					fixPkg := syftjson.ToFormatModel(fix).Schema
+					fixBytes, err := json.Marshal(&fixPkg)
+					require.NoError(t, err)
+
+					assert.JSONEq(t, string(fixBytes), string(modelBytes))
+				},
+			},
+		},
+		{
+			name: "should have expected descriptor",
+			sbom: fix,
+			traits: []modelAssertion{
+				func(t *testing.T, model *external.ImagePackageManifest) {
+					modelPkg := model.Descriptor
+					modelBytes, err := json.Marshal(&modelPkg)
+					require.NoError(t, err)
+
+					fixPkg := syftjson.ToFormatModel(fix).Descriptor
+					fixBytes, err := json.Marshal(&fixPkg)
+					require.NoError(t, err)
+
+					assert.JSONEq(t, string(fixBytes), string(modelBytes))
+				},
+			},
+		},
+		{
+			name: "should have expected source",
+			sbom: fix,
+			traits: []modelAssertion{
+				func(t *testing.T, model *external.ImagePackageManifest) {
+					modelPkg := model.Source
+					modelBytes, err := json.Marshal(&modelPkg)
+					require.NoError(t, err)
+
+					fixPkg := syftjson.ToFormatModel(fix).Source
+					fixBytes, err := json.Marshal(&fixPkg)
+					require.NoError(t, err)
+
+					assert.JSONEq(t, string(fixBytes), string(modelBytes))
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := packageSbomModel(tt.sbom)
+			require.NoError(t, err)
+			for _, fn := range tt.traits {
+				fn(t, got)
+			}
+		})
+	}
+}
+
+func hasDistroInfo(name, version, idLike string) modelAssertion {
+	return func(t *testing.T, model *external.ImagePackageManifest) {
+		assert.Equal(t, name, model.Distro.Name)
+		assert.Equal(t, version, model.Distro.Version)
+		assert.Equal(t, idLike, model.Distro.IdLike)
+	}
+}