oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-03 07:56:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add tests for direct-url information and add it to the output purl (#708)

Browse Source 
* add direct_url.json fields to python metadata

Signed-off-by: Christopher Angelo Phillips <christopher.phillips@anchore.com>

* rename DirectURLOrigin struct; add stub for file

Signed-off-by: Christopher Angelo Phillips <christopher.phillips@anchore.com>

* add detection for direct_url.json

Signed-off-by: Christopher Angelo Phillips <christopher.phillips@anchore.com>

* Add tests for direct-url information and add it to the output purl

Signed-off-by: Sambhav Kothari <skothari44@bloomberg.net>

* Update golden snapshot ids after adding new python package metadata field

Signed-off-by: Sambhav Kothari <skothari44@bloomberg.net>

* Add test names for packageurl tests

Signed-off-by: Sambhav Kothari <skothari44@bloomberg.net>

Co-authored-by: Christopher Angelo Phillips <christopher.phillips@anchore.com>
This commit is contained in:
Sambhav Kothari
2021-12-20 15:54:25 -05:00
committed by GitHub
co-authored by Christopher Angelo Phillips
parent 006ba9b557
commit cc20a8f341
12 changed files with 169 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
syft/pkg/cataloger/package_url_test.go
+35 -3
View File
@@ -10,11 +10,13 @@ import (
func TestPackageURL(t *testing.T) {
tests := []struct {
name string
pkg pkg.Package
distro *distro.Distro
expected string
}{
{
name: "golang",
pkg: pkg.Package{
Name: "github.com/anchore/syft",
Version: "v0.1.0",
@@ -23,14 +25,38 @@ func TestPackageURL(t *testing.T) {
expected: "pkg:golang/github.com/anchore/syft@v0.1.0",
},
{
name: "pip with vcs url",
pkg: pkg.Package{
Name: "name",
Version: "v0.1.0",
Name: "bad-name",
Version: "bad-v0.1.0",
Type: pkg.PythonPkg,
Metadata: pkg.PythonPackageMetadata{
Name: "name",
Version: "v0.1.0",
DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{
VCS: "git",
URL: "https://github.com/test/test.git",
CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
},
},
},
expected: "pkg:pypi/name@v0.1.0?vcs_url=git+https:%2F%2Fgithub.com%2Ftest%2Ftest.git@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
},
{
name: "pip without vcs url",
pkg: pkg.Package{
Name: "bad-name",
Version: "bad-v0.1.0",
Type: pkg.PythonPkg,
Metadata: pkg.PythonPackageMetadata{
Name: "name",
Version: "v0.1.0",
},
},
expected: "pkg:pypi/name@v0.1.0",
},
{
name: "gem",
pkg: pkg.Package{
Name: "name",
Version: "v0.1.0",
@@ -39,6 +65,7 @@ func TestPackageURL(t *testing.T) {
expected: "pkg:gem/name@v0.1.0",
},
{
name: "npm",
pkg: pkg.Package{
Name: "name",
Version: "v0.1.0",
@@ -47,6 +74,7 @@ func TestPackageURL(t *testing.T) {
expected: "pkg:npm/name@v0.1.0",
},
{
name: "deb with arch",
distro: &distro.Distro{
Type: distro.Ubuntu,
},
@@ -63,6 +91,7 @@ func TestPackageURL(t *testing.T) {
expected: "pkg:deb/ubuntu/name@v0.1.0?arch=amd64",
},
{
name: "deb with epoch",
distro: &distro.Distro{
Type: distro.CentOS,
},
@@ -81,6 +110,7 @@ func TestPackageURL(t *testing.T) {
expected: "pkg:rpm/centos/name@0.1.0-3?arch=amd64&epoch=2",
},
{
name: "deb with nil epoch",
distro: &distro.Distro{
Type: distro.CentOS,
},
@@ -99,6 +129,7 @@ func TestPackageURL(t *testing.T) {
expected: "pkg:rpm/centos/name@0.1.0-3?arch=amd64",
},
{
name: "deb with unknown distro",
distro: &distro.Distro{
Type: distro.UnknownDistroType,
},
@@ -110,6 +141,7 @@ func TestPackageURL(t *testing.T) {
expected: "pkg:deb/name@v0.1.0",
},
{
name: "cargo",
pkg: pkg.Package{
Name: "name",
Version: "v0.1.0",
@@ -120,7 +152,7 @@ func TestPackageURL(t *testing.T) {
}
for _, test := range tests {
t.Run(string(test.pkg.Type)+"|"+test.expected, func(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
actual := generatePackageURL(test.pkg, test.distro)
if actual != test.expected {
dmp := diffmatchpatch.New()
syft/pkg/cataloger/python/package_cataloger.go
+44
View File
@@ -2,7 +2,9 @@ package python
import (
"bufio"
"encoding/json"
"fmt"
"io/ioutil"
"path/filepath"
"github.com/anchore/syft/internal"
@@ -152,6 +154,40 @@ func (c *PackageCataloger) fetchTopLevelPackages(resolver source.FileResolver, m
return pkgs, sources, nil
}
func (c *PackageCataloger) fetchDirectURLData(resolver source.FileResolver, metadataLocation source.Location) (d *pkg.PythonDirectURLOriginInfo, sources []source.Location, err error) {
parentDir := filepath.Dir(metadataLocation.RealPath)
directURLPath := filepath.Join(parentDir, "direct_url.json")
directURLLocation := resolver.RelativeFileByPath(metadataLocation, directURLPath)
if directURLLocation == nil {
return nil, nil, nil
}
sources = append(sources, *directURLLocation)
directURLContents, err := resolver.FileContentsByLocation(*directURLLocation)
if err != nil {
return nil, nil, err
}
defer internal.CloseAndLogError(directURLContents, directURLLocation.VirtualPath)
buffer, err := ioutil.ReadAll(directURLContents)
if err != nil {
return nil, nil, err
}
var directURLJson pkg.DirectURLOrigin
if err := json.Unmarshal(buffer, &directURLJson); err != nil {
return nil, nil, err
}
return &pkg.PythonDirectURLOriginInfo{
URL: directURLJson.URL,
CommitID: directURLJson.VCSInfo.CommitID,
VCS: directURLJson.VCSInfo.VCS,
}, sources, nil
}
// assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from.
func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.FileResolver, metadataLocation source.Location) (*pkg.PythonPackageMetadata, []source.Location, error) {
var sources = []source.Location{metadataLocation}
@@ -183,5 +219,13 @@ func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.FileResolv
sources = append(sources, s...)
metadata.TopLevelPackages = p
// attach any direct-url package data found for the given wheel/egg installation
d, s, err := c.fetchDirectURLData(resolver, metadataLocation)
if err != nil {
return nil, nil, err
}
sources = append(sources, s...)
metadata.DirectURLOrigin = d
return &metadata, sources, nil
}
syft/pkg/cataloger/python/package_cataloger_test.go
+2
View File
@@ -55,6 +55,7 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
"test-fixtures/dist-info/METADATA",
"test-fixtures/dist-info/RECORD",
"test-fixtures/dist-info/top_level.txt",
"test-fixtures/dist-info/direct_url.json",
},
expectedPackage: pkg.Package{
Name: "Pygments",
@@ -82,6 +83,7 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
{Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"},
},
TopLevelPackages: []string{"pygments", "something_else"},
DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
},
},
},
syft/pkg/cataloger/python/test-fixtures/dist-info/direct_url.json
+1
View File
@@ -0,0 +1 @@
{"url": "https://github.com/python-test/test.git", "vcs_info": {"commit_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "vcs": "git"}}
syft/pkg/python_package_metadata.go
+69 -9
View File
@@ -1,8 +1,10 @@
package pkg
import (
"fmt"
"sort"
"github.com/anchore/packageurl-go"
"github.com/scylladb/go-set/strset"
)
@@ -21,17 +23,45 @@ type PythonFileRecord struct {
Size string `json:"size,omitempty"`
}
type PythonDirectURLOriginInfo struct {
URL string `json:"url"`
CommitID string `json:"commitId,omitempty"`
VCS string `json:"vcs,omitempty"`
}
// PythonPackageMetadata represents all captured data for a python egg or wheel package.
type PythonPackageMetadata struct {
Name string `json:"name" mapstruct:"Name"`
Version string `json:"version" mapstruct:"Version"`
License string `json:"license" mapstruct:"License"`
Author string `json:"author" mapstruct:"Author"`
AuthorEmail string `json:"authorEmail" mapstruct:"Authoremail"`
Platform string `json:"platform" mapstruct:"Platform"`
Files []PythonFileRecord `json:"files,omitempty"`
SitePackagesRootPath string `json:"sitePackagesRootPath"`
TopLevelPackages []string `json:"topLevelPackages,omitempty"`
Name string `json:"name" mapstruct:"Name"`
Version string `json:"version" mapstruct:"Version"`
License string `json:"license" mapstruct:"License"`
Author string `json:"author" mapstruct:"Author"`
AuthorEmail string `json:"authorEmail" mapstruct:"Authoremail"`
Platform string `json:"platform" mapstruct:"Platform"`
Files []PythonFileRecord `json:"files,omitempty"`
SitePackagesRootPath string `json:"sitePackagesRootPath"`
TopLevelPackages []string `json:"topLevelPackages,omitempty"`
DirectURLOrigin *PythonDirectURLOriginInfo `json:"directUrlOrigin,omitempty"`
}
type DirectURLOrigin struct {
URL string `json:"url"`
VCSInfo VCSInfo `json:"vcs_info"`
ArchiveInfo ArchiveInfo `json:"archive_info"`
DirInfo DirInfo `json:"dir_info"`
}
type DirInfo struct {
Editable bool `json:"editable"`
}
type ArchiveInfo struct {
Hash string `json:"hash"`
}
type VCSInfo struct {
CommitID string `json:"commit_id"`
VCS string `json:"vcs"`
RequestedRevision string `json:"requested_revision"`
}
func (m PythonPackageMetadata) OwnedFiles() (result []string) {
@@ -45,3 +75,33 @@ func (m PythonPackageMetadata) OwnedFiles() (result []string) {
sort.Strings(result)
return result
}
func (m PythonPackageMetadata) PackageURL() string {
// generate a purl from the package data
pURL := packageurl.NewPackageURL(
packageurl.TypePyPi,
"",
m.Name,
m.Version,
m.purlQualifiers(),
"")
return pURL.ToString()
}
func (m PythonPackageMetadata) purlQualifiers() packageurl.Qualifiers {
q := packageurl.Qualifiers{}
if m.DirectURLOrigin != nil {
q = append(q, m.DirectURLOrigin.vcsURLQualifier()...)
}
return q
}
func (p PythonDirectURLOriginInfo) vcsURLQualifier() packageurl.Qualifiers {
if p.VCS != "" {
// Taken from https://github.com/package-url/purl-spec/blob/master/PURL-SPECIFICATION.rst#known-qualifiers-keyvalue-pairs
// packageurl-go still doesn't support all qualifier names
return packageurl.Qualifiers{{Key: "vcs_url", Value: fmt.Sprintf("%s+%s@%s", p.VCS, p.URL, p.CommitID)}}
}
return packageurl.Qualifiers{}
}