Adding metadata fields when parsing yarn.lock and poetry.lock (#2350)

* Adding the resolved and integrity fields of yarn.lock to the parsed metadata. This addition is similar to the metadata added when parsing package-lock.json.

Signed-off-by: asi-cider <88270351+asi-cider@users.noreply.github.com>

* fix comment

Signed-off-by: asi-cider <88270351+asi-cider@users.noreply.github.com>

* Adding the Index field to metadeta when parsing poetry.lock similarly to the existing Pipfile metadata

Signed-off-by: asi-cider <88270351+asi-cider@users.noreply.github.com>

* fixing struct accoding to tests

Signed-off-by: asi-cider <88270351+asi-cider@users.noreply.github.com>

* remove old schema change

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* remove empty constants

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* re-generate JSON schema

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update document ref

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix linting

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: asi-cider <88270351+asi-cider@users.noreply.github.com>
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>

syft/pkg/cataloger/python/package.go

@@ -25,7 +25,7 @@ func newPackageForIndex(name, version string, locations ...file.Location) pkg.Pa
 	return p
 }
 
-func newPackageForIndexWithMetadata(name, version string, metadata pkg.PythonPipfileLockEntry, locations ...file.Location) pkg.Package {
+func newPackageForIndexWithMetadata(name, version string, metadata interface{}, locations ...file.Location) pkg.Package {
 	p := pkg.Package{
 		Name:      name,
 		Version:   version,

syft/pkg/cataloger/python/parse_pipfile_lock.go

@@ -62,7 +62,7 @@ func parsePipfileLock(_ context.Context, _ file.Resolver, _ *generic.Environment
 			if pkgMeta.Index != "" {
 				index = sourcesMap[pkgMeta.Index]
 			} else {
-				// https://pipenv.pypa.io/en/latest/advanced/#specifying-package-indexes
+				// https://pipenv.pypa.io/en/latest/indexes.html
 				index = "https://pypi.org/simple"
 			}
 			version := strings.TrimPrefix(pkgMeta.Version, "==")

syft/pkg/cataloger/python/parse_poetry_lock.go

@@ -15,13 +15,18 @@ import (
 // integrity check
 var _ generic.Parser = parsePoetryLock
 
-type poetryMetadata struct {
+type poetryPackageSource struct {
+	URL string `toml:"url"`
+}
+
+type poetryPackages struct {
 	Packages []struct {
-		Name        string `toml:"name"`
-		Version     string `toml:"version"`
-		Category    string `toml:"category"`
-		Description string `toml:"description"`
-		Optional    bool   `toml:"optional"`
+		Name        string              `toml:"name"`
+		Version     string              `toml:"version"`
+		Category    string              `toml:"category"`
+		Description string              `toml:"description"`
+		Optional    bool                `toml:"optional"`
+		Source      poetryPackageSource `toml:"source"`
 	} `toml:"package"`
 }
 
@@ -32,7 +37,7 @@ func parsePoetryLock(_ context.Context, _ file.Resolver, _ *generic.Environment,
 		return nil, nil, fmt.Errorf("unable to load poetry.lock for parsing: %w", err)
 	}
 
-	metadata := poetryMetadata{}
+	metadata := poetryPackages{}
 	err = tree.Unmarshal(&metadata)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to parse poetry.lock: %w", err)
@@ -40,11 +45,19 @@ func parsePoetryLock(_ context.Context, _ file.Resolver, _ *generic.Environment,
 
 	var pkgs []pkg.Package
 	for _, p := range metadata.Packages {
+		var index string
+		if p.Source.URL != "" {
+			index = p.Source.URL
+		} else {
+			// https://python-poetry.org/docs/repositories/
+			index = "https://pypi.org/simple"
+		}
 		pkgs = append(
 			pkgs,
-			newPackageForIndex(
+			newPackageForIndexWithMetadata(
 				p.Name,
 				p.Version,
+				pkg.PythonPoetryLockEntry{Index: index},
 				reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
 			),
 		)

syft/pkg/cataloger/python/parse_poetry_lock_test.go

@@ -20,6 +20,7 @@ func TestParsePoetryLock(t *testing.T) {
 			Locations: locations,
 			Language:  pkg.Python,
 			Type:      pkg.PythonPkg,
+			Metadata:  pkg.PythonPoetryLockEntry{Index: "https://test.pypi.org/simple"},
 		},
 		{
 			Name:      "alabaster",
@@ -28,6 +29,7 @@ func TestParsePoetryLock(t *testing.T) {
 			Locations: locations,
 			Language:  pkg.Python,
 			Type:      pkg.PythonPkg,
+			Metadata:  pkg.PythonPoetryLockEntry{Index: "https://pypi.org/simple"},
 		},
 		{
 			Name:      "appnope",
@@ -36,6 +38,7 @@ func TestParsePoetryLock(t *testing.T) {
 			Locations: locations,
 			Language:  pkg.Python,
 			Type:      pkg.PythonPkg,
+			Metadata:  pkg.PythonPoetryLockEntry{Index: "https://pypi.org/simple"},
 		},
 		{
 			Name:      "asciitree",
@@ -44,6 +47,7 @@ func TestParsePoetryLock(t *testing.T) {
 			Locations: locations,
 			Language:  pkg.Python,
 			Type:      pkg.PythonPkg,
+			Metadata:  pkg.PythonPoetryLockEntry{Index: "https://pypi.org/simple"},
 		},
 	}
 

syft/pkg/cataloger/python/test-fixtures/poetry/poetry.lock

@@ -12,6 +12,11 @@ natsort = "*"
 six = "*"
 sphinx = "*"
 
+[package.source]
+type = "legacy"
+url = "https://test.pypi.org/simple"
+reference = "test"
+
 [package.extras]
 deploy = ["bumpversion", "twine", "wheel"]
 docs = ["sphinx", "sphinx-rtd-theme"]