oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-01 23:16:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add PURL list input/output format (#3853)

Browse Source 
Signed-off-by: Keith Zantow <kzantow@gmail.com>
This commit is contained in:
Keith Zantow
2025-05-12 13:33:24 -04:00
committed by GitHub
parent bea57a4f7d
commit 621d21eb04
28 changed files with 1079 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
syft/format/internal/backfill.go
+137
View File
@@ -0,0 +1,137 @@
package internal
import (
"fmt"
"regexp"
"slices"
"strings"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/cpe"
"github.com/anchore/syft/syft/pkg"
)
// Backfill takes all information present in the package and attempts to fill in any missing information
// from any available sources, such as the Metadata and PURL.
//
// Backfill does not call p.SetID(), but this needs to be called later to ensure it's up to date
func Backfill(p *pkg.Package) {
if p.PURL == "" {
return
}
purl, err := packageurl.FromString(p.PURL)
if err != nil {
log.Debug("unable to parse purl: %s: %w", p.PURL, err)
return
}
var cpes []cpe.CPE
epoch := ""
for _, qualifier := range purl.Qualifiers {
switch qualifier.Key {
case pkg.PURLQualifierCPES:
rawCpes := strings.Split(qualifier.Value, ",")
for _, rawCpe := range rawCpes {
c, err := cpe.New(rawCpe, cpe.DeclaredSource)
if err != nil {
log.Debugf("unable to decode cpe %s in purl %s: %w", rawCpe, p.PURL, err)
continue
}
cpes = append(cpes, c)
}
case pkg.PURLQualifierEpoch:
epoch = qualifier.Value
}
}
if p.Type == "" {
p.Type = pkg.TypeFromPURL(p.PURL)
}
if p.Language == "" {
p.Language = pkg.LanguageFromPURL(p.PURL)
}
if p.Name == "" {
p.Name = nameFromPurl(purl)
}
setVersionFromPurl(p, purl, epoch)
if p.Language == pkg.Java {
setJavaMetadataFromPurl(p, purl)
}
for _, c := range cpes {
if slices.Contains(p.CPEs, c) {
continue
}
p.CPEs = append(p.CPEs, c)
}
}
func setJavaMetadataFromPurl(p *pkg.Package, purl packageurl.PackageURL) {
if p.Type != pkg.JavaPkg {
return
}
if purl.Namespace != "" {
if p.Metadata == nil {
p.Metadata = pkg.JavaArchive{}
}
meta, got := p.Metadata.(pkg.JavaArchive)
if got && meta.PomProperties == nil {
meta.PomProperties = &pkg.JavaPomProperties{}
p.Metadata = meta
}
if meta.PomProperties != nil {
// capture the group id from the purl if it is not already set
if meta.PomProperties.ArtifactID == "" {
meta.PomProperties.ArtifactID = purl.Name
}
if meta.PomProperties.GroupID == "" {
meta.PomProperties.GroupID = purl.Namespace
}
if meta.PomProperties.Version == "" {
meta.PomProperties.Version = purl.Version
}
}
}
}
func setVersionFromPurl(p *pkg.Package, purl packageurl.PackageURL, epoch string) {
if p.Version == "" {
p.Version = purl.Version
}
if epoch != "" && p.Type == pkg.RpmPkg && !epochPrefix.MatchString(p.Version) {
p.Version = fmt.Sprintf("%s:%s", epoch, p.Version)
}
}
var epochPrefix = regexp.MustCompile(`^\d+:`)
// nameFromPurl returns the syft package name of the package from the purl. If the purl includes a namespace,
// the name is prefixed as appropriate based on the PURL type
func nameFromPurl(purl packageurl.PackageURL) string {
if !nameExcludesPurlNamespace(purl.Type) && purl.Namespace != "" {
return fmt.Sprintf("%s/%s", purl.Namespace, purl.Name)
}
return purl.Name
}
func nameExcludesPurlNamespace(purlType string) bool {
switch purlType {
case packageurl.TypeAlpine,
packageurl.TypeAlpm,
packageurl.TypeConan,
packageurl.TypeCpan,
packageurl.TypeDebian,
packageurl.TypeMaven,
packageurl.TypeQpkg,
packageurl.TypeRPM,
packageurl.TypeSWID:
return true
}
return false
}
syft/format/internal/backfill_test.go
+177
View File
@@ -0,0 +1,177 @@
package internal
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/syft/cpe"
"github.com/anchore/syft/syft/pkg"
)
func Test_Backfill(t *testing.T) {
tests := []struct {
name string
in pkg.Package
expected pkg.Package
}{
{
name: "npm type",
in: pkg.Package{
PURL: "pkg:npm/test@3.0.0",
},
expected: pkg.Package{
PURL: "pkg:npm/test@3.0.0",
Type: pkg.NpmPkg,
Language: pkg.JavaScript,
Name: "test",
Version: "3.0.0",
},
},
{
name: "rpm no epoch",
in: pkg.Package{
PURL: "pkg:rpm/redhat/dbus-common@1.12.8-26.el8?arch=noarch&distro=rhel-8.10&upstream=dbus-1.12.8-26.el8.src.rpm",
},
expected: pkg.Package{
PURL: "pkg:rpm/redhat/dbus-common@1.12.8-26.el8?arch=noarch&distro=rhel-8.10&upstream=dbus-1.12.8-26.el8.src.rpm",
Type: pkg.RpmPkg,
Name: "dbus-common",
Version: "1.12.8-26.el8",
},
},
{
name: "rpm epoch",
in: pkg.Package{
PURL: "pkg:rpm/redhat/dbus-common@1.12.8-26.el8?arch=noarch&distro=rhel-8.10&epoch=1&upstream=dbus-1.12.8-26.el8.src.rpm",
},
expected: pkg.Package{
PURL: "pkg:rpm/redhat/dbus-common@1.12.8-26.el8?arch=noarch&distro=rhel-8.10&epoch=1&upstream=dbus-1.12.8-26.el8.src.rpm",
Type: pkg.RpmPkg,
Name: "dbus-common",
Version: "1:1.12.8-26.el8",
},
},
{
name: "bad cpe",
in: pkg.Package{
PURL: "pkg:npm/testp@3.0.0?cpes=cpe:2.3a:testv:testp:3.0.0:*:*:*:*:*:*:*",
},
expected: pkg.Package{
PURL: "pkg:npm/testp@3.0.0?cpes=cpe:2.3a:testv:testp:3.0.0:*:*:*:*:*:*:*",
Type: pkg.NpmPkg,
Language: pkg.JavaScript,
Name: "testp",
Version: "3.0.0",
},
},
{
name: "good cpe",
in: pkg.Package{
PURL: "pkg:npm/testp@3.0.0?cpes=cpe:2.3:a:testv:testp:3.0.0:*:*:*:*:*:*:*",
},
expected: pkg.Package{
PURL: "pkg:npm/testp@3.0.0?cpes=cpe:2.3:a:testv:testp:3.0.0:*:*:*:*:*:*:*",
Type: pkg.NpmPkg,
Language: pkg.JavaScript,
Name: "testp",
Version: "3.0.0",
CPEs: []cpe.CPE{
{
Attributes: cpe.Attributes{
Part: "a",
Vendor: "testv",
Product: "testp",
Version: "3.0.0",
},
Source: cpe.DeclaredSource,
},
},
},
},
{
name: "java type",
in: pkg.Package{
PURL: "pkg:maven/org.apache/some-thing@1.2.3",
},
expected: pkg.Package{
PURL: "pkg:maven/org.apache/some-thing@1.2.3",
Type: pkg.JavaPkg,
Language: pkg.Java,
Name: "some-thing",
Version: "1.2.3",
Metadata: pkg.JavaArchive{
PomProperties: &pkg.JavaPomProperties{
GroupID: "org.apache",
ArtifactID: "some-thing",
Version: "1.2.3",
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Backfill(&tt.in)
tt.in.OverrideID("")
require.Equal(t, tt.expected, tt.in)
})
}
}
func Test_nameFromPurl(t *testing.T) {
tests := []struct {
in string
expected string
}{
{
in: "pkg:npm/some-name@3.0.0",
expected: "some-name",
},
{
in: "pkg:maven/org.apache/some-name@1.2.3",
expected: "some-name",
},
{
in: "pkg:deb/debian/some-name@3.0.0",
expected: "some-name",
},
{
in: "pkg:rpm/redhat/some-name@3.0.0",
expected: "some-name",
},
{
in: "pkg:gem/some-name@3.0.0",
expected: "some-name",
},
{
in: "pkg:apk/alpine/some-name@3.0.0",
expected: "some-name",
},
{
in: "pkg:docker/some-org/some-name@3.0.0",
expected: "some-org/some-name",
},
{
in: "pkg:npm/some-name@3.0.0",
expected: "some-name",
},
{
in: "pkg:npm/some-org/some-name@3.0.0",
expected: "some-org/some-name",
},
{
in: "pkg:oci/library/mysql@8.1.0",
expected: "library/mysql",
},
}
for _, tt := range tests {
t.Run(tt.in, func(t *testing.T) {
p, err := packageurl.FromString(tt.in)
require.NoError(t, err)
got := nameFromPurl(p)
require.Equal(t, tt.expected, got)
})
}
}
syft/format/internal/cyclonedxutil/decoder.go
+1 -13
View File
@@ -1,12 +1,9 @@
package cyclonedxutil
import (
"fmt"
"io"
"github.com/CycloneDX/cyclonedx-go"
"github.com/anchore/syft/syft/format/internal/stream"
)
type Decoder struct {
@@ -20,19 +17,10 @@ func NewDecoder(format cyclonedx.BOMFileFormat) Decoder {
}
func (d Decoder) Decode(r io.Reader) (*cyclonedx.BOM, error) {
reader, err := stream.SeekableReader(r)
if err != nil {
return nil, err
}
doc := &cyclonedx.BOM{
Components: &[]cyclonedx.Component{},
}
if _, err := reader.Seek(0, io.SeekStart); err != nil {
return nil, fmt.Errorf("unable to seek to start of CycloneDX SBOM: %w", err)
}
err = cyclonedx.NewBOMDecoder(reader, d.format).Decode(doc)
err := cyclonedx.NewBOMDecoder(r, d.format).Decode(doc)
if err != nil {
return nil, err
}
syft/format/internal/cyclonedxutil/helpers/component.go
+42 -4
View File
@@ -1,12 +1,14 @@
package helpers
import (
"fmt"
"reflect"
"github.com/CycloneDX/cyclonedx-go"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/format/internal"
"github.com/anchore/syft/syft/internal/packagemetadata"
"github.com/anchore/syft/syft/pkg"
)
@@ -84,7 +86,6 @@ func decodeComponent(c *cyclonedx.Component) *pkg.Package {
}
p := &pkg.Package{
Name: c.Name,
Version: c.Version,
Locations: decodeLocations(values),
Licenses: pkg.NewLicenseSet(decodeLicenses(c)...),
@@ -102,13 +103,50 @@ func decodeComponent(c *cyclonedx.Component) *pkg.Package {
p.Type = pkg.TypeFromPURL(p.PURL)
}
if p.Language == "" {
p.Language = pkg.LanguageFromPURL(p.PURL)
}
setPackageName(p, c)
internal.Backfill(p)
p.SetID()
return p
}
func setPackageName(p *pkg.Package, c *cyclonedx.Component) {
name := c.Name
if c.Group != "" {
switch p.Type {
case pkg.JavaPkg:
if p.Metadata == nil {
p.Metadata = pkg.JavaArchive{}
}
var pomProperties *pkg.JavaPomProperties
javaMetadata, ok := p.Metadata.(pkg.JavaArchive)
if ok {
pomProperties = javaMetadata.PomProperties
if pomProperties == nil {
pomProperties = &pkg.JavaPomProperties{}
javaMetadata.PomProperties = pomProperties
p.Metadata = javaMetadata
}
}
if pomProperties != nil {
if pomProperties.ArtifactID == "" {
pomProperties.ArtifactID = c.Name
}
if pomProperties.GroupID == "" {
pomProperties.GroupID = c.Group
}
if pomProperties.Version == "" {
pomProperties.Version = p.Version
}
}
default:
name = fmt.Sprintf("%s/%s", c.Group, name)
}
}
p.Name = name
}
func decodeLocations(vals map[string]string) file.LocationSet {
v := Decode(reflect.TypeOf([]file.Location{}), vals, "syft:location", CycloneDXFields)
out, ok := v.([]file.Location)
syft/format/internal/stream/seekable_reader.go
+56 -1
View File
@@ -6,13 +6,18 @@ import (
"io"
)
// SeekableReader takes an io.Reader and returns an io.ReadSeeker relative to the current position of the reader.
// Users of this function expect to be able to reset the reader to the current position, not potentially reset the
// reader prior to the location when this reader is provided. An example is a reader with multiple JSON
// documents separated by newlines (JSONL). After reading the first document, if a call is made to decode
// the second and Seek(0, SeekStart) is called it would reset the overall reader back to the first document.
func SeekableReader(reader io.Reader) (io.ReadSeeker, error) {
if reader == nil {
return nil, fmt.Errorf("no bytes provided")
}
if r, ok := reader.(io.ReadSeeker); ok {
return r, nil
return getOffsetReadSeeker(r)
}
content, err := io.ReadAll(reader)
@@ -22,3 +27,53 @@ func SeekableReader(reader io.Reader) (io.ReadSeeker, error) {
return bytes.NewReader(content), nil
}
type offsetReadSeeker struct {
rdr io.ReadSeeker
offset int64
}
// getOffsetReadSeeker returns a new io.ReadSeeker that may wrap another io.ReadSeeker with the current offset, so
// seek calls will be relative to the _current_ position, rather than relative to the reader itself
func getOffsetReadSeeker(r io.ReadSeeker) (io.ReadSeeker, error) {
if r == nil {
return nil, fmt.Errorf("no reader provided")
}
pos, err := r.Seek(0, io.SeekCurrent)
if pos == 0 {
// if the ReadSeeker is currently at 0, we don't need to track an offset
return r, nil
}
return &offsetReadSeeker{
rdr: r,
offset: pos,
}, err
}
func (o *offsetReadSeeker) Read(p []byte) (n int, err error) {
return o.rdr.Read(p)
}
func (o *offsetReadSeeker) Seek(offset int64, whence int) (int64, error) {
switch whence {
case io.SeekStart:
if offset < 0 {
return 0, fmt.Errorf("cannot seek < 0")
}
newOffset, err := o.rdr.Seek(o.offset+offset, io.SeekStart)
return newOffset - o.offset, err
case io.SeekCurrent:
currentOffset, err := o.rdr.Seek(0, io.SeekCurrent)
if err != nil {
return 0, fmt.Errorf("cannot seek current: %w", err)
}
if currentOffset-o.offset+offset < 0 {
return 0, fmt.Errorf("cannot seek < 0")
}
newOffset, err := o.rdr.Seek(offset, io.SeekCurrent)
return newOffset - o.offset, err
}
return 0, fmt.Errorf("only SeekStart and SeekCurrent supported")
}
var _ io.ReadSeeker = (*offsetReadSeeker)(nil)
syft/format/internal/stream/seekable_reader_test.go
+146 -21
View File
@@ -3,10 +3,9 @@ package stream
import (
"bytes"
"io"
"reflect"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -26,7 +25,7 @@ func TestSeekableReader(t *testing.T) {
name: "empty reader",
input: bytes.NewBuffer([]byte{}), // does not implement io.Seeker (but does implement io.Reader)
assert: func(input io.Reader, got io.ReadSeeker) {
impl, ok := got.(*bytes.Reader) // implements bytes.Reader
impl, ok := got.(*bytes.Reader) // contents are copied to a byte slice, accessed via bytes.Reader
require.True(t, ok)
_, err := impl.Seek(0, io.SeekStart)
require.NoError(t, err)
@@ -37,38 +36,28 @@ func TestSeekableReader(t *testing.T) {
},
{
name: "empty read seeker",
input: bytes.NewReader([]byte{}), // implements io.ReadSeeker
input: strings.NewReader(""), // implements io.ReadSeeker, not offset
assert: func(input io.Reader, got io.ReadSeeker) {
impl, ok := got.(*bytes.Reader)
_, ok := got.(*strings.Reader) // same ReadSeeker is returned when not offset
require.True(t, ok)
_, err := impl.Seek(0, io.SeekStart)
_, err := got.Seek(0, io.SeekStart)
require.NoError(t, err)
content, err := io.ReadAll(impl)
content, err := io.ReadAll(got)
require.NoError(t, err)
require.Equal(t, []byte{}, content)
// assert this is the same read seeker (reflect tt.input pointer is the same as the impl pointer
inputImpl, ok := input.(*bytes.Reader)
require.True(t, ok)
assert.Equal(t, reflect.ValueOf(inputImpl).Pointer(), reflect.ValueOf(impl).Pointer())
},
},
{
name: "non-empty read seeker",
input: bytes.NewReader([]byte("hello world!")), // implements io.ReadSeeker
input: strings.NewReader("hello world!"), // implements io.ReadSeeker, not offset
assert: func(input io.Reader, got io.ReadSeeker) {
impl, ok := got.(*bytes.Reader)
_, ok := got.(*strings.Reader) // same ReadSeeker is returned when not offset
require.True(t, ok)
_, err := impl.Seek(0, io.SeekStart)
_, err := got.Seek(0, io.SeekStart)
require.NoError(t, err)
content, err := io.ReadAll(impl)
content, err := io.ReadAll(got)
require.NoError(t, err)
require.Equal(t, []byte("hello world!"), content)
// assert this is the same read seeker (reflect tt.input pointer is the same as the impl pointer
inputImpl, ok := input.(*bytes.Reader)
require.True(t, ok)
assert.Equal(t, reflect.ValueOf(inputImpl).Pointer(), reflect.ValueOf(impl).Pointer())
},
},
{
@@ -84,6 +73,32 @@ func TestSeekableReader(t *testing.T) {
require.Equal(t, []byte("hello world!"), content)
},
},
{
name: "position zero read seeker",
input: strings.NewReader("a string reader"), // implements io.ReadSeeker at position 0
assert: func(input io.Reader, got io.ReadSeeker) {
_, ok := got.(*strings.Reader) // returns the same ReadSeeker
require.True(t, ok)
_, err := got.Seek(0, io.SeekStart)
require.NoError(t, err)
content, err := io.ReadAll(got)
require.NoError(t, err)
require.Equal(t, []byte("a string reader"), content)
},
},
{
name: "offset read seeker",
input: moveOffset(t, bytes.NewReader([]byte{1, 2, 3, 4, 5}), 3), // implements io.ReadSeeker, with an offset
assert: func(input io.Reader, got io.ReadSeeker) {
_, ok := got.(*offsetReadSeeker) // returns an offset-tracking ReadSeeker
require.True(t, ok)
_, err := got.Seek(0, io.SeekStart)
require.NoError(t, err)
content, err := io.ReadAll(got)
require.NoError(t, err)
require.Equal(t, []byte{4, 5}, content)
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -99,3 +114,113 @@ func TestSeekableReader(t *testing.T) {
})
}
}
func Test_offsetReadSeeker(t *testing.T) {
abcd1234 := func() io.ReadSeeker { return strings.NewReader("abcd1234") }
abcd1234offset := func(offset int) func() io.ReadSeeker {
return func() io.ReadSeeker {
r := strings.NewReader("abcd1234")
_, err := r.Seek(int64(offset), io.SeekStart)
require.NoError(t, err)
return r
}
}
tests := []struct {
name string
input func() io.ReadSeeker
seek int64
seek2 int64
whence int
expected string
wantErr require.ErrorAssertionFunc
}{
{
name: "basic reader",
input: abcd1234,
seek: 0,
whence: io.SeekStart,
expected: "abcd1234",
},
{
name: "basic reader offset",
input: abcd1234offset(1),
seek: 0,
whence: io.SeekStart,
expected: "bcd1234",
},
{
name: "basic reader offset both",
input: abcd1234offset(2),
seek: 2,
whence: io.SeekStart,
expected: "1234",
},
{
name: "basic reader offset seek current",
input: abcd1234offset(1),
seek: -1,
whence: io.SeekCurrent,
wantErr: require.Error, // would be < current, which is an error
},
{
name: "valid negative offset from current",
input: abcd1234offset(1),
seek: 2,
seek2: -1,
whence: io.SeekCurrent,
expected: "cd1234",
},
{
name: "basic reader offset multiple",
input: abcd1234offset(2),
seek: 3,
seek2: 2,
whence: io.SeekCurrent,
expected: "4",
},
{
name: "bad whence",
input: abcd1234,
seek: 1,
whence: io.SeekEnd,
wantErr: require.Error,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
rdr := tt.input()
off, err := rdr.Seek(0, io.SeekCurrent)
require.NoError(t, err)
// construct new offsetReadSeeker
sr := offsetReadSeeker{rdr: rdr, offset: off}
_, err = sr.Seek(tt.seek, tt.whence)
if tt.seek2 != 0 {
require.NoError(t, err)
_, err = sr.Seek(tt.seek2, tt.whence)
}
if tt.wantErr != nil {
tt.wantErr(t, err)
return
} else {
require.NoError(t, err)
}
buf := make([]byte, 1024)
n, err := sr.Read(buf)
require.NoError(t, err)
require.Equal(t, tt.expected, string(buf[:n]))
})
}
}
func moveOffset(t *testing.T, reader io.ReadSeeker, offset int64) io.Reader {
pos, err := reader.Seek(offset, io.SeekStart)
require.NoError(t, err)
require.Equal(t, offset, pos)
return reader
}