Add dart support (#919)

Co-authored-by: Alex Goodman <alex.goodman@anchore.com>

README.md

@@ -29,6 +29,7 @@ A CLI tool and Go library for generating a Software Bill of Materials (SBOM) fro
 ### Supported Ecosystems
 
 - Alpine (apk)
+- Dart (pubs)
 - Debian (dpkg)
 - Go (go.mod, Go binaries)
 - Java (jar, ear, war, par, sar)

internal/constants.go

@@ -6,5 +6,5 @@ const (
 
 	// JSONSchemaVersion is the current schema version output by the JSON encoder
 	// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
-	JSONSchemaVersion = "3.2.1"
+	JSONSchemaVersion = "3.2.2"
 )

internal/formats/common/spdxhelpers/source_info.go

@@ -13,6 +13,8 @@ func SourceInfo(p pkg.Package) string {
 		answer = "acquired package info from RPM DB"
 	case pkg.ApkPkg:
 		answer = "acquired package info from APK DB"
+	case pkg.DartPubPkg:
+		answer = "acquired package info from pubspec manifest"
 	case pkg.DebPkg:
 		answer = "acquired package info from DPKG DB"
 	case pkg.NpmPkg:

internal/formats/common/spdxhelpers/source_info_test.go

@@ -126,6 +126,14 @@ func Test_SourceInfo(t *testing.T) {
 				"from PHP composer manifest",
 			},
 		},
+		{
+			input: pkg.Package{
+				Type: pkg.DartPubPkg,
+			},
+			expected: []string{
+				"from pubspec manifest",
+			},
+		},
 	}
 	var pkgTypes []pkg.Type
 	for _, test := range tests {

internal/formats/syftjson/model/package.go

@@ -130,6 +130,12 @@ func (p *Package) UnmarshalJSON(b []byte) error {
 			return err
 		}
 		p.Metadata = payload
+	case pkg.DartPubMetadataType:
+		var payload pkg.DartPubMetadata
+		if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
+			return err
+		}
+		p.Metadata = payload
 	default:
 		log.Warnf("unknown package metadata type=%q for packageID=%q", p.MetadataType, p.ID)
 	}

internal/formats/syftjson/test-fixtures/snapshot/TestDirectoryEncoder.golden

@@ -88,7 +88,7 @@
   }
  },
  "schema": {
-  "version": "3.2.1",
+  "version": "3.2.2",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-3.2.1.json"
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-3.2.2.json"
  }
 }

internal/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden

@@ -184,7 +184,7 @@
   }
  },
  "schema": {
-  "version": "3.2.1",
+  "version": "3.2.2",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-3.2.1.json"
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-3.2.2.json"
  }
 }

internal/formats/syftjson/test-fixtures/snapshot/TestImageEncoder.golden

@@ -111,7 +111,7 @@
   }
  },
  "schema": {
-  "version": "3.2.1",
+  "version": "3.2.2",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-3.2.1.json"
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-3.2.2.json"
  }
 }

schema/json/generate.go

@@ -37,6 +37,7 @@ type artifactMetadataContainer struct {
 	Cargo  pkg.CargoPackageMetadata
 	Go     pkg.GolangBinMetadata
 	Php    pkg.PhpComposerJSONMetadata
+	Dart   pkg.DartPubMetadata
 }
 
 func main() {

syft/pkg/cataloger/cataloger.go

@@ -9,6 +9,7 @@ import (
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/apkdb"
+	"github.com/anchore/syft/syft/pkg/cataloger/dart"
 	"github.com/anchore/syft/syft/pkg/cataloger/deb"
 	"github.com/anchore/syft/syft/pkg/cataloger/golang"
 	"github.com/anchore/syft/syft/pkg/cataloger/java"
@@ -61,6 +62,7 @@ func DirectoryCatalogers(cfg Config) []Cataloger {
 		golang.NewGoModuleBinaryCataloger(),
 		golang.NewGoModFileCataloger(),
 		rust.NewCargoLockCataloger(),
+		dart.NewPubspecLockCataloger(),
 	}
 }
 
@@ -80,5 +82,6 @@ func AllCatalogers(cfg Config) []Cataloger {
 		golang.NewGoModuleBinaryCataloger(),
 		golang.NewGoModFileCataloger(),
 		rust.NewCargoLockCataloger(),
+		dart.NewPubspecLockCataloger(),
 	}
 }

syft/pkg/cataloger/dart/cataloger.go

@@ -0,0 +1,14 @@
+package dart
+
+import (
+	"github.com/anchore/syft/syft/pkg/cataloger/common"
+)
+
+// NewPubspecLockCataloger returns a new Dartlang cataloger object base on pubspec lock files.
+func NewPubspecLockCataloger() *common.GenericCataloger {
+	globParsers := map[string]common.ParserFn{
+		"**/pubspec.lock": parsePubspecLock,
+	}
+
+	return common.NewGenericCataloger(nil, globParsers, "dartlang-lock-cataloger")
+}

syft/pkg/cataloger/dart/parse_pubspec_lock.go

@@ -0,0 +1,96 @@
+package dart
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/pkg/cataloger/common"
+	"gopkg.in/yaml.v2"
+)
+
+// integrity check
+var _ common.ParserFn = parsePubspecLock
+
+const defaultPubRegistry string = "https://pub.dartlang.org"
+
+type pubspecLock struct {
+	Packages map[string]pubspecLockPackage `yaml:"packages"`
+	Sdks     map[string]string             `yaml:"sdks"`
+}
+
+type pubspecLockPackage struct {
+	Dependency  string                 `yaml:"dependency" mapstructure:"dependency"`
+	Description pubspecLockDescription `yaml:"description" mapstructure:"description"`
+	Source      string                 `yaml:"source" mapstructure:"source"`
+	Version     string                 `yaml:"version" mapstructure:"version"`
+}
+
+type pubspecLockDescription struct {
+	Name        string `yaml:"name" mapstructure:"name"`
+	URL         string `yaml:"url" mapstructure:"url"`
+	Path        string `yaml:"path" mapstructure:"path"`
+	Ref         string `yaml:"ref" mapstructure:"ref"`
+	ResolvedRef string `yaml:"resolved-ref" mapstructure:"resolved-ref"`
+}
+
+func parsePubspecLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	var packages []*pkg.Package
+
+	dec := yaml.NewDecoder(reader)
+
+	var p pubspecLock
+	if err := dec.Decode(&p); err != nil {
+		return nil, nil, fmt.Errorf("failed to parse pubspec.lock file: %w", err)
+	}
+
+	for name, pubPkg := range p.Packages {
+		packages = append(packages, newPubspecLockPackage(name, pubPkg))
+	}
+
+	return packages, nil, nil
+}
+
+func newPubspecLockPackage(name string, p pubspecLockPackage) *pkg.Package {
+	return &pkg.Package{
+		Name:         name,
+		Version:      p.Version,
+		Language:     pkg.Dart,
+		Type:         pkg.DartPubPkg,
+		MetadataType: pkg.DartPubMetadataType,
+		Metadata: &pkg.DartPubMetadata{
+			Name:      name,
+			Version:   p.Version,
+			HostedURL: p.getHostedURL(),
+			VcsURL:    p.getVcsURL(),
+		},
+	}
+}
+
+func (p *pubspecLockPackage) getVcsURL() string {
+	if p.Source == "git" {
+		if p.Description.Path == "." {
+			return fmt.Sprintf("%s@%s", p.Description.URL, p.Description.ResolvedRef)
+		}
+
+		return fmt.Sprintf("%s@%s#%s", p.Description.URL, p.Description.ResolvedRef, p.Description.Path)
+	}
+
+	return ""
+}
+
+func (p *pubspecLockPackage) getHostedURL() string {
+	if p.Source == "hosted" && p.Description.URL != defaultPubRegistry {
+		u, err := url.Parse(p.Description.URL)
+		if err != nil {
+			log.Debugf("Unable to parse registry url %w", err)
+			return p.Description.URL
+		}
+		return u.Host
+	}
+
+	return ""
+}

syft/pkg/cataloger/dart/parse_pubspec_lock_test.go

@@ -0,0 +1,99 @@
+package dart
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/anchore/syft/syft/pkg"
+)
+
+func assertPackagesEqual(t *testing.T, actual []*pkg.Package, expected map[string]*pkg.Package) {
+	assert.Len(t, actual, len(expected))
+}
+
+func TestParsePubspecLock(t *testing.T) {
+	expected := map[string]*pkg.Package{
+		"ale": {
+			Name:         "ale",
+			Version:      "3.3.0",
+			Language:     pkg.Dart,
+			Type:         pkg.DartPubPkg,
+			MetadataType: pkg.DartPubMetadataType,
+			Metadata: pkg.DartPubMetadata{
+				Name:      "ale",
+				Version:   "3.3.0",
+				HostedURL: "pub.hosted.org",
+			},
+		},
+		"analyzer": {
+			Name:         "analyzer",
+			Version:      "0.40.7",
+			Language:     pkg.Dart,
+			Type:         pkg.DartPubPkg,
+			MetadataType: pkg.DartPubMetadataType,
+			Metadata: pkg.DartPubMetadata{
+				Name:    "analyzer",
+				Version: "0.40.7",
+			},
+		},
+		"ansicolor": {
+			Name:         "ansicolor",
+			Version:      "1.1.1",
+			Language:     pkg.Dart,
+			Type:         pkg.DartPubPkg,
+			MetadataType: pkg.DartPubMetadataType,
+			Metadata: pkg.DartPubMetadata{
+				Name:    "ansicolor",
+				Version: "1.1.1",
+			},
+		},
+		"archive": {
+			Name:         "archive",
+			Version:      "2.0.13",
+			Language:     pkg.Dart,
+			Type:         pkg.DartPubPkg,
+			MetadataType: pkg.DartPubMetadataType,
+			Metadata: pkg.DartPubMetadata{
+				Name:    "archive",
+				Version: "2.0.13",
+			},
+		},
+		"args": {
+			Name:         "args",
+			Version:      "1.6.0",
+			Language:     pkg.Dart,
+			Type:         pkg.DartPubPkg,
+			MetadataType: pkg.DartPubMetadataType,
+			Metadata: pkg.DartPubMetadata{
+				Name:    "args",
+				Version: "1.6.0",
+			},
+		},
+		"key_binder": {
+			Name:         "key_binder",
+			Version:      "1.11.20",
+			Language:     pkg.Dart,
+			Type:         pkg.DartPubPkg,
+			MetadataType: pkg.DartPubMetadataType,
+			Metadata: pkg.DartPubMetadata{
+				Name:    "key_binder",
+				Version: "1.11.20",
+				VcsURL:  "git@github.com:Workiva/key_binder.git#3f7b3a6350e73c7dcac45301c0e18fbd42af02f7",
+			},
+		},
+	}
+
+	fixture, err := os.Open("test-fixtures/pubspec.lock")
+	if err != nil {
+		t.Fatalf("failed to open fixture: %+v", err)
+	}
+
+	actual, _, err := parsePubspecLock(fixture.Name(), fixture)
+	if err != nil {
+		t.Fatalf("failed to parse pubspec.lock: %+v", err)
+	}
+
+	assertPackagesEqual(t, actual, expected)
+}

syft/pkg/cataloger/dart/test-fixtures/pubspec.lock

@@ -0,0 +1,49 @@
+# Generated by pub
+# See https://dart.dev/tools/pub/glossary#lockfile
+packages:
+  ale:
+    dependency: transitive
+    description:
+      name: ale
+      url: "https://pub.hosted.org"
+    source: hosted
+    version: "3.3.0"
+  analyzer:
+    dependency: transitive
+    description:
+      name: analyzer
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "0.40.7"
+  ansicolor:
+    dependency: transitive
+    description:
+      name: ansicolor
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.1.1"
+  archive:
+    dependency: transitive
+    description:
+      name: archive
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "2.0.13"
+  args:
+    dependency: transitive
+    description:
+      name: args
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.6.0"
+  key_binder:
+    dependency: "direct main"
+    description:
+      path: "."
+      ref: HEAD
+      resolved-ref: "3f7b3a6350e73c7dcac45301c0e18fbd42af02f7"
+      url: "git@github.com:Workiva/key_binder.git"
+    source: git
+    version: "1.11.20"
+sdks:
+  dart: ">=2.12.0 <3.0.0"

syft/pkg/dart_pub_metadata.go

@@ -0,0 +1,38 @@
+package pkg
+
+import (
+	"github.com/anchore/packageurl-go"
+	"github.com/anchore/syft/syft/linux"
+)
+
+type DartPubMetadata struct {
+	Name      string `mapstructure:"name" json:"name"`
+	Version   string `mapstructure:"version" json:"version"`
+	HostedURL string `mapstructure:"hosted_url" json:"hosted_url,omitempty"`
+	VcsURL    string `mapstructure:"vcs_url" json:"vcs_url,omitempty"`
+}
+
+func (m DartPubMetadata) PackageURL(_ *linux.Release) string {
+	var qualifiers packageurl.Qualifiers
+
+	if m.HostedURL != "" {
+		qualifiers = append(qualifiers, packageurl.Qualifier{
+			Key:   "hosted_url",
+			Value: m.HostedURL,
+		})
+	} else if m.VcsURL != "" { // Default to using Hosted if somehow both are provided
+		qualifiers = append(qualifiers, packageurl.Qualifier{
+			Key:   "vcs_url",
+			Value: m.VcsURL,
+		})
+	}
+
+	return packageurl.NewPackageURL(
+		packageurl.TypePub,
+		"",
+		m.Name,
+		m.Version,
+		qualifiers,
+		"",
+	).ToString()
+}

syft/pkg/language.go

@@ -19,6 +19,7 @@ const (
 	Ruby            Language = "ruby"
 	Go              Language = "go"
 	Rust            Language = "rust"
+	Dart            Language = "dart"
 )
 
 // AllLanguages is a set of all programming languages detected by syft.
@@ -30,6 +31,7 @@ var AllLanguages = []Language{
 	Ruby,
 	Go,
 	Rust,
+	Dart,
 }
 
 // String returns the string representation of the language.
@@ -62,6 +64,8 @@ func LanguageByName(name string) Language {
 		return Ruby
 	case purlCargoPkgType:
 		return Rust
+	case packageurl.TypePub, string(Dart):
+		return Dart
 	default:
 		return UnknownLanguage
 	}

syft/pkg/language_test.go

@@ -1,9 +1,10 @@
 package pkg
 
 import (
+	"testing"
+
 	"github.com/scylladb/go-set/strset"
 	"github.com/stretchr/testify/assert"
-	"testing"
 )
 
 func TestLanguageFromPURL(t *testing.T) {
@@ -29,6 +30,10 @@ func TestLanguageFromPURL(t *testing.T) {
 			purl: "pkg:golang/github.com/gorilla/context@234fd47e07d1004f0aed9c",
 			want: Go,
 		},
+		{
+			purl: "pkg:pub/util@1.2.34",
+			want: Dart,
+		},
 		{
 			purl: "pkg:cargo/clap@2.33.0",
 			want: Rust,

syft/pkg/metadata.go

@@ -17,6 +17,7 @@ const (
 	JavaMetadataType             MetadataType = "JavaMetadata"
 	NpmPackageJSONMetadataType   MetadataType = "NpmPackageJsonMetadata"
 	RpmdbMetadataType            MetadataType = "RpmdbMetadata"
+	DartPubMetadataType          MetadataType = "DartPubMetadata"
 	PythonPackageMetadataType    MetadataType = "PythonPackageMetadata"
 	RustCargoPackageMetadataType MetadataType = "RustCargoPackageMetadata"
 	KbPackageMetadataType        MetadataType = "KbPackageMetadata"
@@ -31,6 +32,7 @@ var AllMetadataTypes = []MetadataType{
 	JavaMetadataType,
 	NpmPackageJSONMetadataType,
 	RpmdbMetadataType,
+	DartPubMetadataType,
 	PythonPackageMetadataType,
 	RustCargoPackageMetadataType,
 	KbPackageMetadataType,
@@ -45,6 +47,7 @@ var MetadataTypeByName = map[MetadataType]reflect.Type{
 	JavaMetadataType:             reflect.TypeOf(JavaMetadata{}),
 	NpmPackageJSONMetadataType:   reflect.TypeOf(NpmPackageJSONMetadata{}),
 	RpmdbMetadataType:            reflect.TypeOf(RpmdbMetadata{}),
+	DartPubMetadataType:          reflect.TypeOf(DartPubMetadata{}),
 	PythonPackageMetadataType:    reflect.TypeOf(PythonPackageMetadata{}),
 	RustCargoPackageMetadataType: reflect.TypeOf(CargoMetadata{}),
 	KbPackageMetadataType:        reflect.TypeOf(KbPackageMetadata{}),

syft/pkg/type.go

@@ -20,6 +20,7 @@ const (
 	GoModulePkg      Type = "go-module"
 	RustPkg          Type = "rust-crate"
 	KbPkg            Type = "msrc-kb"
+	DartPubPkg       Type = "dart-pub"
 )
 
 // AllPkgs represents all supported package types
@@ -36,6 +37,7 @@ var AllPkgs = []Type{
 	GoModulePkg,
 	RustPkg,
 	KbPkg,
+	DartPubPkg,
 }
 
 // PackageURLType returns the PURL package type for the current package.
@@ -61,6 +63,8 @@ func (t Type) PackageURLType() string {
 		return packageurl.TypeGolang
 	case RustPkg:
 		return "cargo"
+	case DartPubPkg:
+		return packageurl.TypePub
 	default:
 		// TODO: should this be a "generic" purl type instead?
 		return ""
@@ -98,6 +102,8 @@ func TypeByName(name string) Type {
 		return GemPkg
 	case "cargo", "crate":
 		return RustPkg
+	case packageurl.TypePub:
+		return DartPubPkg
 	default:
 		return UnknownPkg
 	}

syft/pkg/type_test.go

@@ -1,9 +1,10 @@
 package pkg
 
 import (
-	"github.com/scylladb/go-set/strset"
 	"testing"
 
+	"github.com/scylladb/go-set/strset"
+
 	"github.com/stretchr/testify/assert"
 )
 
@@ -46,6 +47,10 @@ func TestTypeFromPURL(t *testing.T) {
 			purl:     "pkg:cargo/clap@2.33.0",
 			expected: RustPkg,
 		},
+		{
+			purl:     "pkg:pub/util@1.2.34?hosted_url=pub.hosted.org",
+			expected: DartPubPkg,
+		},
 		{
 			purl:     "pkg:composer/laravel/laravel@5.5.0",
 			expected: PhpComposerPkg,

syft/pkg/url_test.go

@@ -25,6 +25,20 @@ func TestPackageURL(t *testing.T) {
 			},
 			expected: "pkg:golang/github.com/anchore/syft@v0.1.0",
 		},
+		{
+			name: "pub",
+			pkg: Package{
+				Name:    "bad-name",
+				Version: "0.1.0",
+				Type:    DartPubPkg,
+				Metadata: DartPubMetadata{
+					Name:      "name",
+					Version:   "0.2.0",
+					HostedURL: "pub.hosted.org",
+				},
+			},
+			expected: "pkg:pub/name@0.2.0?hosted_url=pub.hosted.org",
+		},
 		{
 			name: "python",
 			pkg: Package{

test/integration/catalog_packages_cases_test.go

@@ -186,6 +186,19 @@ var dirOnlyTestCases = []testCase{
 			"alcaeus/mongo-php-adapter": "1.1.11",
 		},
 	},
+	{
+		name:        "find pubspec lock packages",
+		pkgType:     pkg.DartPubPkg,
+		pkgLanguage: pkg.Dart,
+		pkgInfo: map[string]string{
+			"ansicolor":  "1.1.1",
+			"archive":    "2.0.13",
+			"args":       "1.6.0",
+			"key_binder": "1.11.20",
+			"ale":        "3.3.0",
+			"analyzer":   "0.40.7",
+		},
+	},
 }
 
 var commonTestCases = []testCase{

test/integration/catalog_packages_test.go

@@ -1,9 +1,10 @@
 package integration
 
 import (
-	"github.com/stretchr/testify/require"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/anchore/syft/syft/linux"
 	"github.com/anchore/syft/syft/pkg/cataloger"
 	"github.com/google/go-cmp/cmp"
@@ -64,6 +65,7 @@ func TestPkgCoverageImage(t *testing.T) {
 	// for image scans we should not expect to see any of the following package types
 	definedLanguages.Remove(pkg.Go.String())
 	definedLanguages.Remove(pkg.Rust.String())
+	definedLanguages.Remove(pkg.Dart.String())
 
 	observedPkgs := internal.NewStringSet()
 	definedPkgs := internal.NewStringSet()
@@ -75,6 +77,7 @@ func TestPkgCoverageImage(t *testing.T) {
 	definedPkgs.Remove(string(pkg.KbPkg))
 	definedPkgs.Remove(string(pkg.GoModulePkg))
 	definedPkgs.Remove(string(pkg.RustPkg))
+	definedPkgs.Remove(string(pkg.DartPubPkg))
 
 	var cases []testCase
 	cases = append(cases, commonTestCases...)

test/integration/test-fixtures/image-pkg-coverage/pkgs/dart/pubspec.lock

@@ -0,0 +1,49 @@
+# Generated by pub
+# See https://dart.dev/tools/pub/glossary#lockfile
+packages:
+  ale:
+    dependency: transitive
+    description:
+      name: ale
+      url: "https://pub.hosted.org"
+    source: hosted
+    version: "3.3.0"
+  analyzer:
+    dependency: transitive
+    description:
+      name: analyzer
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "0.40.7"
+  ansicolor:
+    dependency: transitive
+    description:
+      name: ansicolor
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.1.1"
+  archive:
+    dependency: transitive
+    description:
+      name: archive
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "2.0.13"
+  args:
+    dependency: transitive
+    description:
+      name: args
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.6.0"
+  key_binder:
+    dependency: "direct main"
+    description:
+      path: "."
+      ref: HEAD
+      resolved-ref: "3f7b3a6350e73c7dcac45301c0e18fbd42af02f7"
+      url: "git@github.com:Workiva/key_binder.git"
+    source: git
+    version: "1.11.20"
+sdks:
+  dart: ">=2.12.0 <3.0.0"