feat: report unknowns in sbom (#2998)

Signed-off-by: Keith Zantow <kzantow@gmail.com>
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Keith Zantow
2024-10-07 16:11:37 -04:00
committed by GitHub
co-authored by Alex Goodman
parent 4d7ed9f749
commit ccbee94b87
145 changed files with 4420 additions and 233 deletions
@@ -190,6 +190,14 @@ func TestApkDBCataloger(t *testing.T) {
}
func Test_corruptDb(t *testing.T) {
pkgtest.NewCatalogTester().
FromDirectory(t, "test-fixtures/corrupt").
WithCompareOptions(cmpopts.IgnoreFields(pkg.ApkDBEntry{}, "Files", "GitCommit", "Checksum")).
WithError().
TestCataloger(t, NewDBCataloger())
}
func TestCatalogerDependencyTree(t *testing.T) {
assertion := func(t *testing.T, pkgs []pkg.Package, relationships []artifact.Relationship) {
expected := map[string][]string{
+5 -1
View File
@@ -12,6 +12,7 @@ import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
@@ -38,6 +39,7 @@ type parsedData struct {
func parseApkDB(_ context.Context, resolver file.Resolver, env *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
scanner := bufio.NewScanner(reader)
var errs error
var apks []parsedData
var currentEntry parsedData
entryParsingInProgress := false
@@ -81,10 +83,12 @@ func parseApkDB(_ context.Context, resolver file.Resolver, env *generic.Environm
field := parseApkField(line)
if field == nil {
log.Warnf("unable to parse field data from line %q", line)
errs = unknown.Appendf(errs, reader, "unable to parse field data from line %q", line)
continue
}
if len(field.name) == 0 {
log.Warnf("failed to parse field name from line %q", line)
errs = unknown.Appendf(errs, reader, "failed to parse field name from line %q", line)
continue
}
if len(field.value) == 0 {
@@ -131,7 +135,7 @@ func parseApkDB(_ context.Context, resolver file.Resolver, env *generic.Environm
pkgs = append(pkgs, newPackage(apk, r, reader.Location))
}
return pkgs, nil, nil
return pkgs, nil, errs
}
func findReleases(resolver file.Resolver, dbPath string) []linux.Release {
@@ -0,0 +1,43 @@
this is a corrupt db
C:Q1v4QhLje3kWlC8DJj+ZfJTjlJRSU=
P:alpine-baselayout-data
V:3.2.0-r22
A:x86_64
S:11435
I:73728
o:alpine-baselayout
t:1655134784
c:cb70ca5c6d6db0399d2dd09189c5d57827bce5cd
r:alpine-baselayout
F:etc
R:fstab
Z:Q11Q7hNe8QpDS531guqCdrXBzoA/o=
R:group
Z:Q13K+olJg5ayzHSVNUkggZJXuB+9Y=
R:hostname
Z:Q16nVwYVXP/tChvUPdukVD2ifXOmc=
R:hosts
Z:Q1BD6zJKZTRWyqGnPi4tSfd3krsMU=
R:inittab
Z:Q1TsthbhW7QzWRe1E/NKwTOuD4pHc=
R:modules
Z:Q1toogjUipHGcMgECgPJX64SwUT1M=
R:mtab
a:0:0:777
Z:Q1kiljhXXH1LlQroHsEJIkPZg2eiw=
R:passwd
Z:Q1TchuuLUfur0izvfZQZxgN/LJhB8=
R:profile
Z:Q1F3DgXUP+jNZDknmQPPb5t9FSfDg=
R:protocols
Z:Q1omKlp3vgGq2ZqYzyD/KHNdo8rDc=
R:services
Z:Q19WLCv5ItKg4MH7RWfNRh1I7byQc=
R:shadow
a:0:42:640
Z:Q1ltrPIAW2zHeDiajsex2Bdmq3uqA=
R:shells
Z:Q1ojm2YdpCJ6B/apGDaZ/Sdb2xJkA=
R:sysctl.conf
Z:Q14upz3tfnNxZkIEsUhWn7Xoiw96g=
@@ -11,6 +11,14 @@ import (
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
)
func TestAlpmUnknowns(t *testing.T) {
pkgtest.NewCatalogTester().
FromDirectory(t, "test-fixtures/installed").
WithCompareOptions(cmpopts.IgnoreFields(pkg.AlpmFileRecord{}, "Time")).
WithError().
TestCataloger(t, NewDBCataloger())
}
func TestAlpmCataloger(t *testing.T) {
gmpDbLocation := file.NewLocation("var/lib/pacman/local/gmp-6.2.1-2/desc")
treeSitterDbLocation := file.NewLocation("var/lib/pacman/local/tree-sitter-0.22.6-1/desc")
+31 -33
View File
@@ -17,6 +17,7 @@ import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -41,6 +42,8 @@ type parsedData struct {
// parseAlpmDB parses the arch linux pacman database flat-files and returns the packages and relationships found within.
func parseAlpmDB(_ context.Context, resolver file.Resolver, env *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
var errs error
data, err := parseAlpmDBEntry(reader)
if err != nil {
return nil, nil, err
@@ -52,24 +55,25 @@ func parseAlpmDB(_ context.Context, resolver file.Resolver, env *generic.Environ
base := path.Dir(reader.RealPath)
var locs []file.Location
// replace the files found the pacman database with the files from the mtree These contain more metadata and
// thus more useful.
files, fileLoc := fetchPkgFiles(base, resolver)
backups, backupLoc := fetchBackupFiles(base, resolver)
var locs []file.Location
if fileLoc != nil {
files, fileLoc, err := fetchPkgFiles(base, resolver)
errs = unknown.Join(errs, err)
if err == nil {
locs = append(locs, fileLoc.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.SupportingEvidenceAnnotation))
data.Files = files
}
if backupLoc != nil {
backups, backupLoc, err := fetchBackupFiles(base, resolver)
errs = unknown.Join(errs, err)
if err == nil {
locs = append(locs, backupLoc.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.SupportingEvidenceAnnotation))
data.Backup = backups
}
if data.Package == "" {
return nil, nil, nil
return nil, nil, errs
}
return []pkg.Package{
@@ -79,63 +83,56 @@ func parseAlpmDB(_ context.Context, resolver file.Resolver, env *generic.Environ
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
locs...,
),
}, nil, nil
}, nil, errs
}
func fetchPkgFiles(base string, resolver file.Resolver) ([]pkg.AlpmFileRecord, *file.Location) {
func fetchPkgFiles(base string, resolver file.Resolver) ([]pkg.AlpmFileRecord, file.Location, error) {
// TODO: probably want to use MTREE and PKGINFO here
target := path.Join(base, "mtree")
loc, err := getLocation(target, resolver)
if err != nil {
log.WithFields("error", err, "path", target).Trace("failed to find mtree file")
return []pkg.AlpmFileRecord{}, nil
return []pkg.AlpmFileRecord{}, loc, unknown.New(loc, fmt.Errorf("failed to find mtree file: %w", err))
}
if loc == nil {
return []pkg.AlpmFileRecord{}, nil
}
reader, err := resolver.FileContentsByLocation(*loc)
reader, err := resolver.FileContentsByLocation(loc)
if err != nil {
return []pkg.AlpmFileRecord{}, nil
return []pkg.AlpmFileRecord{}, loc, unknown.New(loc, fmt.Errorf("failed to get contents: %w", err))
}
defer internal.CloseAndLogError(reader, loc.RealPath)
pkgFiles, err := parseMtree(reader)
if err != nil {
log.WithFields("error", err, "path", target).Trace("failed to parse mtree file")
return []pkg.AlpmFileRecord{}, nil
return []pkg.AlpmFileRecord{}, loc, unknown.New(loc, fmt.Errorf("failed to parse mtree: %w", err))
}
return pkgFiles, loc
return pkgFiles, loc, nil
}
func fetchBackupFiles(base string, resolver file.Resolver) ([]pkg.AlpmFileRecord, *file.Location) {
func fetchBackupFiles(base string, resolver file.Resolver) ([]pkg.AlpmFileRecord, file.Location, error) {
// We only really do this to get any backup database entries from the files database
target := filepath.Join(base, "files")
loc, err := getLocation(target, resolver)
if err != nil {
log.WithFields("error", err, "path", target).Trace("failed to find alpm files")
return []pkg.AlpmFileRecord{}, nil
}
if loc == nil {
return []pkg.AlpmFileRecord{}, nil
return []pkg.AlpmFileRecord{}, loc, unknown.New(loc, fmt.Errorf("failed to find alpm files: %w", err))
}
reader, err := resolver.FileContentsByLocation(*loc)
reader, err := resolver.FileContentsByLocation(loc)
if err != nil {
return []pkg.AlpmFileRecord{}, nil
return []pkg.AlpmFileRecord{}, loc, unknown.New(loc, fmt.Errorf("failed to get contents: %w", err))
}
defer internal.CloseAndLogError(reader, loc.RealPath)
filesMetadata, err := parseAlpmDBEntry(reader)
if err != nil {
return []pkg.AlpmFileRecord{}, nil
return []pkg.AlpmFileRecord{}, loc, unknown.New(loc, fmt.Errorf("failed to parse alpm db entry: %w", err))
}
if filesMetadata != nil {
return filesMetadata.Backup, loc
return filesMetadata.Backup, loc, nil
}
return []pkg.AlpmFileRecord{}, loc
return []pkg.AlpmFileRecord{}, loc, nil
}
func parseAlpmDBEntry(reader io.Reader) (*parsedData, error) {
@@ -171,20 +168,21 @@ func newScanner(reader io.Reader) *bufio.Scanner {
return scanner
}
func getLocation(path string, resolver file.Resolver) (*file.Location, error) {
func getLocation(path string, resolver file.Resolver) (file.Location, error) {
loc := file.NewLocation(path)
locs, err := resolver.FilesByPath(path)
if err != nil {
return nil, err
return loc, err
}
if len(locs) == 0 {
return nil, fmt.Errorf("could not find file: %s", path)
return loc, fmt.Errorf("could not find file: %s", path)
}
if len(locs) > 1 {
log.WithFields("path", path).Trace("multiple files found for path, using first path")
}
return &locs[0], nil
return locs[0], nil
}
func parseDatabase(b *bufio.Scanner) (*parsedData, error) {
@@ -0,0 +1,34 @@
%NME%
tree-sitter
%VER.6-1
%BASE%
tree-sitter
%DESC%
Incremental parsing library
%BUILDDATE%
1714945746
%INSTALLDATE%
1715026360
%PACKA@archlinux.org>
%SIZE%
223539
%REASON%
1
%LICENSE%
MIT
%VALIDATION%
pgp
%PROVIDE
libtree-sitter.so=0-64
@@ -6,8 +6,10 @@ package binary
import (
"context"
"encoding/json"
"fmt"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -59,12 +61,14 @@ func (c cataloger) Name() string {
func (c cataloger) Catalog(_ context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
var packages []pkg.Package
var relationships []artifact.Relationship
var errs error
for _, cls := range c.classifiers {
log.WithFields("classifier", cls.Class).Trace("cataloging binaries")
newPkgs, err := catalog(resolver, cls)
if err != nil {
log.WithFields("error", err, "classifier", cls.Class).Warn("unable to catalog binary package: %w", err)
log.WithFields("error", err, "classifier", cls.Class).Debugf("unable to catalog binary package: %v", err)
errs = unknown.Join(errs, fmt.Errorf("%s: %w", cls.Class, err))
continue
}
newPackages:
@@ -82,7 +86,7 @@ func (c cataloger) Catalog(_ context.Context, resolver file.Resolver) ([]pkg.Pac
}
}
return packages, relationships, nil
return packages, relationships, errs
}
// mergePackages merges information from the extra package into the target package
@@ -98,6 +102,7 @@ func mergePackages(target *pkg.Package, extra *pkg.Package) {
}
func catalog(resolver file.Resolver, cls Classifier) (packages []pkg.Package, err error) {
var errs error
locations, err := resolver.FilesByGlob(cls.FileGlob)
if err != nil {
return nil, err
@@ -105,11 +110,12 @@ func catalog(resolver file.Resolver, cls Classifier) (packages []pkg.Package, er
for _, location := range locations {
pkgs, err := cls.EvidenceMatcher(cls, matcherContext{resolver: resolver, location: location})
if err != nil {
return nil, err
errs = unknown.Append(errs, location, err)
continue
}
packages = append(packages, pkgs...)
}
return packages, nil
return packages, errs
}
// packagesMatch returns true if the binary packages "match" based on basic criteria
@@ -1668,7 +1668,7 @@ func Test_Cataloger_ResilientToErrors(t *testing.T) {
resolver := &panicyResolver{}
_, _, err := c.Catalog(context.Background(), resolver)
assert.NoError(t, err)
assert.Error(t, err)
assert.True(t, resolver.searchCalled)
}
@@ -11,6 +11,7 @@ import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/mimetype"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/internal/unionreader"
@@ -52,6 +53,7 @@ func (c *elfPackageCataloger) Name() string {
}
func (c *elfPackageCataloger) Catalog(_ context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
var errs error
locations, err := resolver.FilesByMIMEType(mimetype.ExecutableMIMETypeSet.List()...)
if err != nil {
return nil, nil, fmt.Errorf("unable to get binary files by mime type: %w", err)
@@ -62,7 +64,8 @@ func (c *elfPackageCataloger) Catalog(_ context.Context, resolver file.Resolver)
for _, location := range locations {
notes, key, err := parseElfPackageNotes(resolver, location, c)
if err != nil {
return nil, nil, err
errs = unknown.Append(errs, location, err)
continue
}
if notes == nil {
continue
@@ -87,7 +90,7 @@ func (c *elfPackageCataloger) Catalog(_ context.Context, resolver file.Resolver)
// why not return relationships? We have an executable cataloger that will note the dynamic libraries imported by
// each binary. After all files and packages are processed there is a final task that creates package-to-package
// and package-to-file relationships based on the dynamic libraries imported by each binary.
return pkgs, nil, nil
return pkgs, nil, errs
}
func parseElfPackageNotes(resolver file.Resolver, location file.Location, c *elfPackageCataloger) (*elfBinaryPackageNotes, elfPackageKey, error) {
@@ -104,7 +107,7 @@ func parseElfPackageNotes(resolver file.Resolver, location file.Location, c *elf
if err != nil {
log.WithFields("file", location.Path(), "error", err).Trace("unable to parse ELF notes")
return nil, elfPackageKey{}, nil
return nil, elfPackageKey{}, err
}
if notes == nil {
@@ -173,7 +176,7 @@ func getELFNotes(r file.LocationReadCloser) (*elfBinaryPackageNotes, error) {
if len(notes) > headerSize {
var metadata elfBinaryPackageNotes
newPayload := bytes.TrimRight(notes[headerSize:], "\x00")
if err := json.Unmarshal(newPayload, &metadata); err == nil {
if err = json.Unmarshal(newPayload, &metadata); err == nil {
return &metadata, nil
}
log.WithFields("file", r.Location.Path(), "error", err).Trace("unable to unmarshal ELF package notes as JSON")
@@ -3,6 +3,8 @@ package binary
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
@@ -14,6 +16,7 @@ func Test_ELF_Package_Cataloger(t *testing.T) {
name string
fixture string
expected []pkg.Package
wantErr require.ErrorAssertionFunc
}{
{
name: "go case",
@@ -63,6 +66,7 @@ func Test_ELF_Package_Cataloger(t *testing.T) {
},
},
},
wantErr: require.Error,
},
{
name: "fedora 64 bit binaries",
@@ -116,6 +120,7 @@ func Test_ELF_Package_Cataloger(t *testing.T) {
WithImageResolver(t, v.fixture).
IgnoreLocationLayer(). // this fixture can be rebuilt, thus the layer ID will change
Expects(v.expected, nil).
WithErrorAssertion(v.wantErr).
TestCataloger(t, NewELFPackageCataloger())
})
}
@@ -5,9 +5,11 @@ RUN dnf update -y; \
dnf clean all
RUN mkdir -p /usr/local/bin/elftests/elfbinwithnestedlib
RUN mkdir -p /usr/local/bin/elftests/elfbinwithsisterlib
RUN mkdir -p /usr/local/bin/elftests/elfbinwithcorrupt
COPY ./elfbinwithnestedlib /usr/local/bin/elftests/elfbinwithnestedlib
COPY ./elfbinwithsisterlib /usr/local/bin/elftests/elfbinwithsisterlib
COPY ./elfbinwithcorrupt /usr/local/bin/elftests/elfbinwithcorrupt
ENV LD_LIBRARY_PATH=/usr/local/bin/elftests/elfbinwithnestedlib/bin/lib
@@ -16,6 +18,8 @@ RUN make
WORKDIR /usr/local/bin/elftests/elfbinwithsisterlib
RUN make
WORKDIR /usr/local/bin/elftests/elfbinwithcorrupt
RUN make
# let's make the test image smaller, since we only require the built binaries and supporting libraries
FROM busybox:1.36.1-musl
@@ -0,0 +1,6 @@
#include <iostream>
#include "hello_world.h"
void print_hello_world() {
std::cout << "Hello, World!" << std::endl;
}
@@ -0,0 +1,8 @@
#ifndef HELLO_WORLD_H
#define HELLO_WORLD_H
// Function declaration for printing "Hello, World!" to stdout
void print_hello_world();
#endif // HELLO_WORLD_H
@@ -0,0 +1,48 @@
LDFLAGS := -L/lib64 -lstdc++
SRC_DIR := ./
BUILD_DIR := ../build
BIN_DIR := ../bin
LIB_DIR := $(BIN_DIR)/lib
LIB_NAME := hello_world
LIB_SRC := $(SRC_DIR)/hello_world.cpp
LIB_OBJ := $(BUILD_DIR)/$(LIB_NAME).o
LIB_SO := $(LIB_DIR)/lib$(LIB_NAME).so
EXECUTABLE := elfbinwithnestedlib
EXEC_SRC := $(SRC_DIR)/testbin.cpp
EXEC_OBJ := $(BUILD_DIR)/$(EXECUTABLE).o
all: testfixture
$(LIB_SO): $(LIB_OBJ) | $(LIB_DIR)
$(CC) -shared -o $@ $<
echo '{ corrupt json "system": "syftsys","name": "libhello_world.so","version": "0.01","pure:0.01"}' | objcopy --add-section .note.package=/dev/stdin --set-section-flags .note.package=noload,readonly $@
$(LIB_OBJ): $(LIB_SRC) | $(BUILD_DIR)
$(CC) $(CFLAGS) -fPIC -c $< -o $@
$(EXEC_OBJ): $(EXEC_SRC) | $(BUILD_DIR)
$(CC) $(CFLAGS) -c $< -o $@
$(BIN_DIR):
mkdir -p $(BIN_DIR)
$(BUILD_DIR):
mkdir -p $(BUILD_DIR)
$(LIB_DIR):
mkdir -p $(LIB_DIR)
$(BIN_DIR)/$(EXECUTABLE): $(EXEC_OBJ) $(LIB_SO) | $(BIN_DIR)
$(CC) $(CFLAGS) -o $@ $^ -L$(LIB_DIR) -l$(LIB_NAME) $(LDFLAGS)
echo '{corrupt json ..._syfttestfixture:0.01"}' | objcopy --add-section .note.package=/dev/stdin --set-section-flags .note.package=noload,readonly $@
testfixture: $(BIN_DIR)/$(EXECUTABLE)
clean:
rm -rf $(BUILD_DIR) $(LIB_DIR) $(BIN_DIR) $(EXECUTABLE)
.PHONY: all clean
@@ -0,0 +1,8 @@
#include "hello_world.h"
int main() {
// Call the function from the shared library
print_hello_world();
return 0;
}
@@ -0,0 +1,18 @@
CC = g++
CFLAGS = -std=c++17 -Wall -Wextra -pedantic
BUILD_DIR := ./build
BIN_DIR := ./bin
LIB_DIR := $(BIN_DIR)/lib
all: testfixtures
testfixtures:
$(MAKE) -C elfsrc
clean:
rm -rf $(BUILD_DIR) $(BIN_DIR)
.PHONY: all clean testfixtures
+2 -1
View File
@@ -8,6 +8,7 @@ import (
"io"
"strings"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -25,7 +26,7 @@ func parseConanfile(_ context.Context, _ file.Resolver, _ *generic.Environment,
line, err := r.ReadString('\n')
switch {
case errors.Is(err, io.EOF):
return pkgs, nil, nil
return pkgs, nil, unknown.IfEmptyf(pkgs, "unable to determine packages")
case err != nil:
return nil, nil, fmt.Errorf("failed to parse conanfile.txt file: %w", err)
}
+2 -1
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"strings"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -72,7 +73,7 @@ func parseConanLock(_ context.Context, _ file.Resolver, _ *generic.Environment,
}
}
return pkgs, relationships, nil
return pkgs, relationships, unknown.IfEmptyf(pkgs, "unable to determine packages")
}
// handleConanLockV1 handles the parsing of conan lock v1 files (aka v0.4)
@@ -373,3 +373,10 @@ func TestParseConanLockV2(t *testing.T) {
pkgtest.TestFileParser(t, fixture, parseConanLock, expected, expectedRelationships)
}
func Test_corruptConanlock(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/corrupt/conan.lock").
WithError().
TestParser(t, parseConanLock)
}
@@ -0,0 +1,10 @@
{
corrupt json
version": "0.5",
"requires": [
"sound32/1.0#83d4b7bf607b3b60a6546f8b58b5cdd7%1675278904.0791488",
"matrix/1.1#905c3f0babc520684c84127378fefdd0%1675278901.7527816"
],
"build_requires": [],
"python_requires": []
}
@@ -9,6 +9,7 @@ import (
"gopkg.in/yaml.v3"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -85,7 +86,7 @@ func parsePubspecLock(_ context.Context, _ file.Resolver, _ *generic.Environment
)
}
return pkgs, nil, nil
return pkgs, nil, unknown.IfEmptyf(pkgs, "unable to determine packages")
}
func (p *pubspecLockPackage) getVcsURL() string {
@@ -106,3 +106,10 @@ func TestParsePubspecLock(t *testing.T) {
pkgtest.TestFileParser(t, fixture, parsePubspecLock, expected, expectedRelationships)
}
func Test_corruptPubspecLock(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/corrupt/pubspec.lock").
WithError().
TestParser(t, parsePubspecLock)
}
@@ -0,0 +1,7 @@
pa
kages:
ale:
dependency: transitive
descr
s ps:
dart: ">=2.12.0 <3.0.0"
+2 -1
View File
@@ -15,6 +15,7 @@ import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -39,7 +40,7 @@ func parseDpkgDB(_ context.Context, resolver file.Resolver, env *generic.Environ
pkgs = append(pkgs, p)
}
return pkgs, nil, nil
return pkgs, nil, unknown.IfEmptyf(pkgs, "unable to determine packages")
}
func findDpkgInfoFiles(name string, resolver file.Resolver, dbLocation file.Location) []file.Location {
@@ -257,6 +257,17 @@ func Test_parseDpkgStatus(t *testing.T) {
}
}
func Test_corruptEntry(t *testing.T) {
f, err := os.Open("test-fixtures/var/lib/dpkg/status.d/corrupt")
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, f.Close()) })
reader := bufio.NewReader(f)
_, err = parseDpkgStatus(reader)
require.Error(t, err)
}
func TestSourceVersionExtract(t *testing.T) {
tests := []struct {
name string
@@ -312,7 +323,7 @@ func Test_parseDpkgStatus_negativeCases(t *testing.T) {
{
name: "no more packages",
input: `Package: apt`,
wantErr: require.NoError,
wantErr: requireAs(errors.New("unable to determine packages")),
},
{
name: "duplicated key",
@@ -0,0 +1,6 @@
Pakij: apt
Stratus: install ok installed
Prioority: required
Section: admin
Insterface to the configuration settings
* apt-key as an interface to manage authentication keys
@@ -8,6 +8,7 @@ import (
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/relationship"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -126,5 +127,5 @@ func parseDotnetDeps(_ context.Context, _ file.Resolver, _ *generic.Environment,
// this will only consider package-to-package relationships.
relationship.Sort(relationships)
return pkgs, relationships, nil
return pkgs, relationships, unknown.IfEmptyf(pkgs, "unable to determine packages")
}
@@ -9,6 +9,13 @@ import (
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
)
func Test_corruptDotnetDeps(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/glob-paths/src/something.deps.json").
WithError().
TestParser(t, parseDotnetDeps)
}
func TestParseDotnetDeps(t *testing.T) {
fixture := "test-fixtures/TestLibrary.deps.json"
fixtureLocationSet := file.NewLocationSet(file.NewLocation(fixture))
@@ -28,30 +28,26 @@ func parseDotnetPortableExecutable(_ context.Context, _ file.Resolver, _ *generi
peFile, err := pe.NewBytes(by, &pe.Options{})
if err != nil {
// TODO: known-unknown
log.Tracef("unable to create PE instance for file '%s': %v", f.RealPath, err)
return nil, nil, nil
return nil, nil, err
}
err = peFile.Parse()
if err != nil {
// TODO: known-unknown
log.Tracef("unable to parse PE file '%s': %v", f.RealPath, err)
return nil, nil, nil
return nil, nil, err
}
versionResources, err := peFile.ParseVersionResources()
if err != nil {
// TODO: known-unknown
log.Tracef("unable to parse version resources in PE file: %s: %v", f.RealPath, err)
return nil, nil, nil
return nil, nil, fmt.Errorf("unable to parse version resources in PE file: %w", err)
}
dotNetPkg, err := buildDotNetPackage(versionResources, f)
if err != nil {
// TODO: known-unknown
log.Tracef("unable to build dotnet package: %v", err)
return nil, nil, nil
log.Tracef("unable to build dotnet package for: %v %v", f.RealPath, err)
return nil, nil, err
}
return []pkg.Package{dotNetPkg}, nil, nil
@@ -60,12 +56,12 @@ func parseDotnetPortableExecutable(_ context.Context, _ file.Resolver, _ *generi
func buildDotNetPackage(versionResources map[string]string, f file.LocationReadCloser) (dnpkg pkg.Package, err error) {
name := findName(versionResources)
if name == "" {
return dnpkg, fmt.Errorf("unable to find PE name in file: %s", f.RealPath)
return dnpkg, fmt.Errorf("unable to find PE name in file")
}
version := findVersion(versionResources)
if version == "" {
return dnpkg, fmt.Errorf("unable to find PE version in file: %s", f.RealPath)
return dnpkg, fmt.Errorf("unable to find PE version in file")
}
metadata := pkg.DotnetPortableExecutableEntry{
@@ -297,6 +297,13 @@ func TestParseDotnetPortableExecutable(t *testing.T) {
}
}
func Test_corruptDotnetPE(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/glob-paths/src/something.exe").
WithError().
TestParser(t, parseDotnetPortableExecutable)
}
func Test_extractVersion(t *testing.T) {
tests := []struct {
input string
+10 -1
View File
@@ -9,6 +9,7 @@ import (
"regexp"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -22,25 +23,33 @@ var mixLockDelimiter = regexp.MustCompile(`[%{}\n" ,:]+`)
// parseMixLock parses a mix.lock and returns the discovered Elixir packages.
func parseMixLock(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
var errs error
r := bufio.NewReader(reader)
var packages []pkg.Package
lineNum := 0
for {
lineNum++
line, err := r.ReadString('\n')
switch {
case errors.Is(err, io.EOF):
return packages, nil, nil
if errs == nil {
errs = unknown.IfEmptyf(packages, "unable to determine packages")
}
return packages, nil, errs
case err != nil:
return nil, nil, fmt.Errorf("failed to parse mix.lock file: %w", err)
}
tokens := mixLockDelimiter.Split(line, -1)
if len(tokens) < 6 {
errs = unknown.Appendf(errs, reader, "unable to read mix lock line %d: %s", lineNum, line)
continue
}
name, version, hash, hashExt := tokens[1], tokens[4], tokens[5], tokens[len(tokens)-2]
if name == "" {
log.WithFields("path", reader.RealPath).Debug("skipping empty package name from mix.lock file")
errs = unknown.Appendf(errs, reader, "skipping empty package name from mix.lock file, for line: %d: %s", lineNum, line)
continue
}
+2 -1
View File
@@ -2,6 +2,7 @@ package erlang
import (
"context"
"fmt"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
@@ -17,7 +18,7 @@ func parseOTPApp(_ context.Context, _ file.Resolver, _ *generic.Environment, rea
// there are multiple file formats that use the *.app extension, so it's possible that this is not an OTP app file at all
// ... which means we should not return an error here
log.WithFields("error", err).Trace("unable to parse Erlang OTP app")
return nil, nil, nil
return nil, nil, fmt.Errorf("unable to parse Erlang OTP app")
}
var packages []pkg.Package
@@ -41,3 +41,10 @@ func TestParseOTPApplication(t *testing.T) {
})
}
}
func Test_corruptOtpApp(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/corrupt/rabbitmq.app").
WithError().
TestParser(t, parseOTPApp)
}
@@ -4,6 +4,7 @@ import (
"context"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -95,7 +96,7 @@ func parseRebarLock(_ context.Context, _ file.Resolver, _ *generic.Environment,
p.SetID()
packages = append(packages, *p)
}
return packages, nil, nil
return packages, nil, unknown.IfEmptyf(packages, "unable to determine packages")
}
// integrity check
@@ -255,3 +255,10 @@ func TestParseRebarLock(t *testing.T) {
})
}
}
func Test_corruptRebarLock(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/corrupt/rebar.lock").
WithError().
TestParser(t, parseRebarLock)
}
@@ -0,0 +1,9 @@
cation, 'rabbit', [
{description, "RabbitMQ"},
{vsn, "3.12.10"},
{id, "v3.12.9-9-g1f61ca8"},
{modules, ['amqqueue','background_gc']},
{optional_itmq-server#1593
{channel_max, 2047}
]}
]}.
@@ -0,0 +1,11 @@
{"1.2.0",
[{<<"certifi{pkg,<<"certifi">>,<<"2.9.0">>},0},
{<<"idna">>,{pkg,<<"idpkg,<<"parse_trans">>,<<"3.3.1">>},0},
{<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.6">>},0},
{<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.0">>},0}]}.
[
{pkg_hash,[
{<<"certifi">>, <<"6F2A475689DD47F19FB74334859D460A2DC4E3252A3324BD2111B8F0429E7E21">>},
{<<"idna">>, <<"8A63070E9F7D0C62EB9D9FCB360A7DE382448200FBBD1B106CC96D3D8099DF8D">>},
{<<"metrics">>, <<"25F094DEA2CDA98213CECC3AEFF09E940299D950904393B2A29D191C346A8486">>},
{<<"mimerl
+8 -5
View File
@@ -6,6 +6,7 @@ import (
"github.com/anchore/go-logger"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
@@ -151,6 +152,7 @@ func (c *Cataloger) Name() string {
func (c *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
var packages []pkg.Package
var relationships []artifact.Relationship
var errs error
logger := log.Nested("cataloger", c.upstreamCataloger)
@@ -166,7 +168,8 @@ func (c *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.
discoveredPackages, discoveredRelationships, err := invokeParser(ctx, resolver, location, logger, parser, &env)
if err != nil {
continue // logging is handled within invokeParser
// parsers may return errors and valid packages / relationships
errs = unknown.Append(errs, location, err)
}
for _, p := range discoveredPackages {
@@ -176,7 +179,7 @@ func (c *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.
relationships = append(relationships, discoveredRelationships...)
}
return c.process(ctx, resolver, packages, relationships, nil)
return c.process(ctx, resolver, packages, relationships, errs)
}
func (c *Cataloger) process(ctx context.Context, resolver file.Resolver, pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) {
@@ -196,11 +199,11 @@ func invokeParser(ctx context.Context, resolver file.Resolver, location file.Loc
discoveredPackages, discoveredRelationships, err := parser(ctx, resolver, env, file.NewLocationReadCloser(location, contentReader))
if err != nil {
logger.WithFields("location", location.RealPath, "error", err).Warnf("cataloger failed")
return nil, nil, err
// these errors are propagated up, and are likely to be coordinate errors
logger.WithFields("location", location.RealPath, "error", err).Trace("cataloger returned errors")
}
return discoveredPackages, discoveredRelationships, nil
return discoveredPackages, discoveredRelationships, err
}
// selectFiles takes a set of file trees and resolves and file references of interest for future cataloging
@@ -10,6 +10,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -187,3 +188,27 @@ func TestClosesFileOnParserPanic(t *testing.T) {
})
require.True(t, spy.closed)
}
func Test_genericCatalogerReturnsErrors(t *testing.T) {
genericErrorReturning := NewCataloger("error returning").WithParserByGlobs(func(ctx context.Context, resolver file.Resolver, environment *Environment, locationReader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
return []pkg.Package{
{
Name: "some-package-" + locationReader.Path(),
},
}, nil, unknown.Newf(locationReader, "unable to read")
}, "**/*")
m := file.NewMockResolverForPaths(
"test-fixtures/a-path.txt",
"test-fixtures/empty.txt",
)
got, _, errs := genericErrorReturning.Catalog(context.TODO(), m)
// require packages and errors
require.NotEmpty(t, got)
unknowns, others := unknown.ExtractCoordinateErrors(errs)
require.NotEmpty(t, unknowns)
require.Empty(t, others)
}
@@ -35,7 +35,7 @@ func parsePortageContents(_ context.Context, resolver file.Resolver, _ *generic.
name, version := cpvMatch[1], cpvMatch[2]
if name == "" || version == "" {
log.WithFields("path", reader.Location.RealPath).Warnf("failed to parse portage name and version")
return nil, nil, nil
return nil, nil, fmt.Errorf("failed to parse portage name and version")
}
p := pkg.Package{
+5 -4
View File
@@ -1,6 +1,7 @@
package githubactions
import (
"fmt"
"strings"
"github.com/anchore/packageurl-go"
@@ -9,19 +10,19 @@ import (
"github.com/anchore/syft/syft/pkg"
)
func newPackageFromUsageStatement(use string, location file.Location) *pkg.Package {
func newPackageFromUsageStatement(use string, location file.Location) (*pkg.Package, error) {
name, version := parseStepUsageStatement(use)
if name == "" {
log.WithFields("file", location.RealPath, "statement", use).Trace("unable to parse github action usage statement")
return nil
return nil, fmt.Errorf("unable to parse github action usage statement")
}
if strings.Contains(name, ".github/workflows/") {
return newGithubActionWorkflowPackageUsage(name, version, location)
return newGithubActionWorkflowPackageUsage(name, version, location), nil
}
return newGithubActionPackageUsage(name, version, location)
return newGithubActionPackageUsage(name, version, location), nil
}
func newGithubActionWorkflowPackageUsage(name, version string, workflowLocation file.Location) *pkg.Package {
@@ -7,6 +7,7 @@ import (
"gopkg.in/yaml.v3"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -24,14 +25,14 @@ type compositeActionRunsDef struct {
}
func parseCompositeActionForActionUsage(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
contents, err := io.ReadAll(reader)
if err != nil {
return nil, nil, fmt.Errorf("unable to read yaml composite action file: %w", err)
contents, errs := io.ReadAll(reader)
if errs != nil {
return nil, nil, fmt.Errorf("unable to read yaml composite action file: %w", errs)
}
var ca compositeActionDef
if err = yaml.Unmarshal(contents, &ca); err != nil {
return nil, nil, fmt.Errorf("unable to parse yaml composite action file: %w", err)
if errs = yaml.Unmarshal(contents, &ca); errs != nil {
return nil, nil, fmt.Errorf("unable to parse yaml composite action file: %w", errs)
}
// we use a collection to help with deduplication before raising to higher level processing
@@ -42,11 +43,14 @@ func parseCompositeActionForActionUsage(_ context.Context, _ file.Resolver, _ *g
continue
}
p := newPackageFromUsageStatement(step.Uses, reader.Location)
p, err := newPackageFromUsageStatement(step.Uses, reader.Location)
if err != nil {
errs = unknown.Append(errs, reader, err)
}
if p != nil {
pkgs.Add(*p)
}
}
return pkgs.Sorted(), nil, nil
return pkgs.Sorted(), nil, errs
}
@@ -33,3 +33,10 @@ func Test_parseCompositeActionForActionUsage(t *testing.T) {
var expectedRelationships []artifact.Relationship
pkgtest.TestFileParser(t, fixture, parseCompositeActionForActionUsage, expected, expectedRelationships)
}
func Test_corruptCompositeAction(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/corrupt/composite-action.yaml").
WithError().
TestParser(t, parseCompositeActionForActionUsage)
}
@@ -7,6 +7,7 @@ import (
"gopkg.in/yaml.v3"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -37,14 +38,14 @@ type stepDef struct {
}
func parseWorkflowForWorkflowUsage(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
contents, err := io.ReadAll(reader)
if err != nil {
return nil, nil, fmt.Errorf("unable to read yaml workflow file: %w", err)
contents, errs := io.ReadAll(reader)
if errs != nil {
return nil, nil, fmt.Errorf("unable to read yaml workflow file: %w", errs)
}
var wf workflowDef
if err = yaml.Unmarshal(contents, &wf); err != nil {
return nil, nil, fmt.Errorf("unable to parse yaml workflow file: %w", err)
if errs = yaml.Unmarshal(contents, &wf); errs != nil {
return nil, nil, fmt.Errorf("unable to parse yaml workflow file: %w", errs)
}
// we use a collection to help with deduplication before raising to higher level processing
@@ -52,25 +53,28 @@ func parseWorkflowForWorkflowUsage(_ context.Context, _ file.Resolver, _ *generi
for _, job := range wf.Jobs {
if job.Uses != "" {
p := newPackageFromUsageStatement(job.Uses, reader.Location)
p, err := newPackageFromUsageStatement(job.Uses, reader.Location)
if err != nil {
errs = unknown.Append(errs, reader, err)
}
if p != nil {
pkgs.Add(*p)
}
}
}
return pkgs.Sorted(), nil, nil
return pkgs.Sorted(), nil, errs
}
func parseWorkflowForActionUsage(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
contents, err := io.ReadAll(reader)
if err != nil {
return nil, nil, fmt.Errorf("unable to read yaml workflow file: %w", err)
contents, errs := io.ReadAll(reader)
if errs != nil {
return nil, nil, fmt.Errorf("unable to read yaml workflow file: %w", errs)
}
var wf workflowDef
if err = yaml.Unmarshal(contents, &wf); err != nil {
return nil, nil, fmt.Errorf("unable to parse yaml workflow file: %w", err)
if errs = yaml.Unmarshal(contents, &wf); errs != nil {
return nil, nil, fmt.Errorf("unable to parse yaml workflow file: %w", errs)
}
// we use a collection to help with deduplication before raising to higher level processing
@@ -81,12 +85,15 @@ func parseWorkflowForActionUsage(_ context.Context, _ file.Resolver, _ *generic.
if step.Uses == "" {
continue
}
p := newPackageFromUsageStatement(step.Uses, reader.Location)
p, err := newPackageFromUsageStatement(step.Uses, reader.Location)
if err != nil {
errs = unknown.Append(errs, reader, err)
}
if p != nil {
pkgs.Add(*p)
}
}
}
return pkgs.Sorted(), nil, nil
return pkgs.Sorted(), nil, errs
}
@@ -86,3 +86,17 @@ func Test_parseWorkflowForWorkflowUsage(t *testing.T) {
var expectedRelationships []artifact.Relationship
pkgtest.TestFileParser(t, fixture, parseWorkflowForWorkflowUsage, expected, expectedRelationships)
}
func Test_corruptActionWorkflow(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/corrupt/workflow-multi-job.yaml").
WithError().
TestParser(t, parseWorkflowForActionUsage)
}
func Test_corruptWorkflowWorkflow(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/corrupt/workflow-multi-job.yaml").
WithError().
TestParser(t, parseWorkflowForWorkflowUsage)
}
@@ -0,0 +1,13 @@
name: "Bootstrap"
description: "Bootstrap all tools and dependencies"
ints:
go-version:
descrapt-packages:
description: "Space delimited list of tools to install via apt"
default: "libxml2-utils"
rns:
us all cache fingerprints
shell: bash
run: make fingerprints
@@ -0,0 +1,16 @@
name: "Validations"
on:
workflow_dispatch:
pull_request:
push:
branches:
- main
jbs
Statnapshot
key: snapshot-build-${{ github.run_id }}
- name: Run CLI Tests (Linux)
run: make cli
+3 -3
View File
@@ -66,9 +66,9 @@ func (c *goBinaryCataloger) parseGoBinary(_ context.Context, resolver file.Resol
if err != nil {
return nil, nil, err
}
defer internal.CloseAndLogError(reader.ReadCloser, reader.RealPath)
mods := scanFile(unionReader, reader.RealPath)
internal.CloseAndLogError(reader.ReadCloser, reader.RealPath)
mods, errs := scanFile(reader.Location, unionReader)
var rels []artifact.Relationship
for _, mod := range mods {
@@ -81,7 +81,7 @@ func (c *goBinaryCataloger) parseGoBinary(_ context.Context, resolver file.Resol
pkgs = append(pkgs, depPkgs...)
}
return pkgs, rels, nil
return pkgs, rels, errs
}
func createModuleRelationships(main pkg.Package, deps []pkg.Package) []artifact.Relationship {
@@ -157,3 +157,11 @@ func Test_GoSumHashes(t *testing.T) {
})
}
}
func Test_corruptGoMod(t *testing.T) {
c := NewGoModuleFileCataloger(DefaultCatalogerConfig().WithSearchRemoteLicenses(false))
pkgtest.NewCatalogTester().
FromDirectory(t, "test-fixtures/corrupt").
WithError().
TestCataloger(t, c)
}
+14 -9
View File
@@ -9,6 +9,8 @@ import (
"github.com/kastenhq/goversion/version"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/internal/unionreader"
)
@@ -19,20 +21,21 @@ type extendedBuildInfo struct {
}
// scanFile scans file to try to report the Go and module versions.
func scanFile(reader unionreader.UnionReader, filename string) []*extendedBuildInfo {
func scanFile(location file.Location, reader unionreader.UnionReader) ([]*extendedBuildInfo, error) {
// NOTE: multiple readers are returned to cover universal binaries, which are files
// with more than one binary
readers, err := unionreader.GetReaders(reader)
if err != nil {
log.WithFields("error", err).Warnf("failed to open a golang binary")
return nil
readers, errs := unionreader.GetReaders(reader)
if errs != nil {
log.WithFields("error", errs).Warnf("failed to open a golang binary")
return nil, fmt.Errorf("failed to open a golang binary: %w", errs)
}
var builds []*extendedBuildInfo
for _, r := range readers {
bi, err := getBuildInfo(r)
if err != nil {
log.WithFields("file", filename, "error", err).Trace("unable to read golang buildinfo")
log.WithFields("file", location.RealPath, "error", err).Trace("unable to read golang buildinfo")
continue
}
// it's possible the reader just isn't a go binary, in which case just skip it
@@ -42,23 +45,25 @@ func scanFile(reader unionreader.UnionReader, filename string) []*extendedBuildI
v, err := getCryptoInformation(r)
if err != nil {
log.WithFields("file", filename, "error", err).Trace("unable to read golang version info")
log.WithFields("file", location.RealPath, "error", err).Trace("unable to read golang version info")
// don't skip this build info.
// we can still catalog packages, even if we can't get the crypto information
errs = unknown.Appendf(errs, location, "unable to read golang version info: %w", err)
}
arch := getGOARCH(bi.Settings)
if arch == "" {
arch, err = getGOARCHFromBin(r)
if err != nil {
log.WithFields("file", filename, "error", err).Trace("unable to read golang arch info")
log.WithFields("file", location.RealPath, "error", err).Trace("unable to read golang arch info")
// don't skip this build info.
// we can still catalog packages, even if we can't get the arch information
errs = unknown.Appendf(errs, location, "unable to read golang arch info: %w", err)
}
}
builds = append(builds, &extendedBuildInfo{BuildInfo: bi, cryptoSettings: v, arch: arch})
}
return builds
return builds, errs
}
func getCryptoInformation(reader io.ReaderAt) ([]string, error) {
@@ -0,0 +1,11 @@
module github.com/anchore/syft
go 1.18
ruire (
github.com/CycloneDX/cyclonedx-go v0.7.0
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
)
replace github.com/CycloneDX/cyclonedx-go => github.com/CycloneDX/cyclonedx-go v0.6.0
@@ -0,0 +1,4 @@
github.com/CycloneDX/cyclonedx-go v0.6.0/go.mod h1:nQCiF4Tvrg5Ieu8qPhYMvzPGMu5I7fANZkrSsJjl5mg=
github.com/Cycpansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
github6IF
github.com/stretchr/test4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -9,6 +9,7 @@ import (
"gopkg.in/yaml.v3"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -50,7 +51,7 @@ func parseStackLock(_ context.Context, _ file.Resolver, _ *generic.Environment,
if err := yaml.Unmarshal(bytes, &lockFile); err != nil {
log.WithFields("error", err).Tracef("failed to parse stack.yaml.lock file %q", reader.RealPath)
return nil, nil, nil
return nil, nil, fmt.Errorf("failed to parse stack.yaml.lock file")
}
var (
@@ -82,7 +83,7 @@ func parseStackLock(_ context.Context, _ file.Resolver, _ *generic.Environment,
)
}
return pkgs, nil, nil
return pkgs, nil, unknown.IfEmptyf(pkgs, "unable to determine packages")
}
func parseStackPackageEncoding(pkgEncoding string) (name, version, hash string) {
@@ -130,3 +130,10 @@ func TestParseStackLock(t *testing.T) {
pkgtest.TestFileParser(t, fixture, parseStackLock, expectedPkgs, expectedRelationships)
}
func Test_corruptStackLock(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/corrupt/stack.yaml.lock").
WithError().
TestParser(t, parseStackLock)
}
@@ -8,6 +8,7 @@ import (
"gopkg.in/yaml.v3"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -31,7 +32,7 @@ func parseStackYaml(_ context.Context, _ file.Resolver, _ *generic.Environment,
if err := yaml.Unmarshal(bytes, &stackFile); err != nil {
log.WithFields("error", err).Tracef("failed to parse stack.yaml file %q", reader.RealPath)
return nil, nil, nil
return nil, nil, fmt.Errorf("failed to parse stack.yaml file")
}
var pkgs []pkg.Package
@@ -50,5 +51,5 @@ func parseStackYaml(_ context.Context, _ file.Resolver, _ *generic.Environment,
)
}
return pkgs, nil, nil
return pkgs, nil, unknown.IfEmptyf(pkgs, "unable to determine packages")
}
@@ -110,3 +110,10 @@ func TestParseStackYaml(t *testing.T) {
pkgtest.TestFileParser(t, fixture, parseStackYaml, expectedPkgs, expectedRelationships)
}
func Test_corruptStackYaml(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/corrupt/stack.yaml").
WithError().
TestParser(t, parseStackYaml)
}
@@ -0,0 +1,6 @@
flag
extra-package-dbs: []
packa@sha256:fbcf49ecfc3d4da53e797fd0275264cba776ffa324ee223e2a3f4ec2d2c9c4a6,2165
- stm-2.5.0.2@sha256:e4dc6473faaa75fbd7eccab4e3ee1d651d75bb0e49946ef0b8b751ccde771a55,2314
ghc-options:
"$everything": -haddock
@@ -0,0 +1,8 @@
packages
-comp
al:
commit: a5847301404583e16d55cd4d051b8e605d704fbc
git: https://github.com/runtimeverification/haskell-backend.git
subdir: kore
snapshots
- complete//raw.github
@@ -48,7 +48,6 @@ type CatalogTester struct {
func NewCatalogTester() *CatalogTester {
return &CatalogTester{
wantErr: require.NoError,
locationComparer: cmptest.DefaultLocationComparer,
licenseComparer: cmptest.DefaultLicenseComparer,
packageStringer: stringPackage,
@@ -113,7 +112,6 @@ func (p *CatalogTester) WithEnv(env *generic.Environment) *CatalogTester {
}
func (p *CatalogTester) WithError() *CatalogTester {
p.assertResultExpectations = true
p.wantErr = require.Error
return p
}
@@ -226,7 +224,10 @@ func (p *CatalogTester) IgnoreUnfulfilledPathResponses(paths ...string) *Catalog
func (p *CatalogTester) TestParser(t *testing.T, parser generic.Parser) {
t.Helper()
pkgs, relationships, err := parser(context.Background(), p.resolver, p.env, p.reader)
p.wantErr(t, err)
// only test for errors if explicitly requested
if p.wantErr != nil {
p.wantErr(t, err)
}
p.assertPkgs(t, pkgs, relationships)
}
@@ -247,8 +248,12 @@ func (p *CatalogTester) TestCataloger(t *testing.T, cataloger pkg.Cataloger) {
assert.ElementsMatchf(t, p.expectedContentQueries, resolver.AllContentQueries(), "unexpected content queries observed: diff %s", cmp.Diff(p.expectedContentQueries, resolver.AllContentQueries()))
}
if p.assertResultExpectations {
// only test for errors if explicitly requested
if p.wantErr != nil {
p.wantErr(t, err)
}
if p.assertResultExpectations {
p.assertPkgs(t, pkgs, relationships)
}
@@ -256,7 +261,7 @@ func (p *CatalogTester) TestCataloger(t *testing.T, cataloger pkg.Cataloger) {
a(t, pkgs, relationships)
}
if !p.assertResultExpectations && len(p.customAssertions) == 0 {
if !p.assertResultExpectations && len(p.customAssertions) == 0 && p.wantErr == nil {
resolver.PruneUnfulfilledPathResponses(p.ignoreUnfulfilledPathResponses, p.ignoreAnyUnfulfilledPaths...)
// if we aren't testing the results, we should focus on what was searched for (for glob-centric tests)
+19 -6
View File
@@ -16,6 +16,7 @@ import (
intFile "github.com/anchore/syft/internal/file"
"github.com/anchore/syft/internal/licenses"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -133,14 +134,22 @@ func (j *archiveParser) parse(ctx context.Context) ([]pkg.Package, []artifact.Re
}
pkgs = append(pkgs, auxPkgs...)
var errs error
if j.detectNested {
// find nested java archive packages
nestedPkgs, nestedRelationships, err := j.discoverPkgsFromNestedArchives(ctx, parentPkg)
if err != nil {
return nil, nil, err
errs = unknown.Append(errs, j.location, err)
}
pkgs = append(pkgs, nestedPkgs...)
relationships = append(relationships, nestedRelationships...)
} else {
// .jar and .war files are present in archives, are others? or generally just consider them top-level?
nestedArchives := j.fileManifest.GlobMatch(true, "*.jar", "*.war")
if len(nestedArchives) > 0 {
slices.Sort(nestedArchives)
errs = unknown.Appendf(errs, j.location, "nested archives not cataloged: %v", strings.Join(nestedArchives, ", "))
}
}
// lastly, add the parent package to the list (assuming the parent exists)
@@ -166,7 +175,11 @@ func (j *archiveParser) parse(ctx context.Context) ([]pkg.Package, []artifact.Re
p.SetID()
}
return pkgs, relationships, nil
if len(pkgs) == 0 {
errs = unknown.Appendf(errs, j.location, "no package identified in archive")
}
return pkgs, relationships, errs
}
// discoverMainPackage parses the root Java manifest used as the parent package to all discovered nested packages.
@@ -283,14 +296,14 @@ func (j *archiveParser) findLicenseFromJavaMetadata(ctx context.Context, groupID
if parsedPom != nil {
pomLicenses, err = j.maven.resolveLicenses(ctx, parsedPom.project)
if err != nil {
log.WithFields("error", err, "mavenID", j.maven.getMavenID(ctx, parsedPom.project)).Debug("error attempting to resolve pom licenses")
log.WithFields("error", err, "mavenID", j.maven.getMavenID(ctx, parsedPom.project)).Trace("error attempting to resolve pom licenses")
}
}
if err == nil && len(pomLicenses) == 0 {
pomLicenses, err = j.maven.findLicenses(ctx, groupID, artifactID, version)
if err != nil {
log.WithFields("error", err, "mavenID", mavenID{groupID, artifactID, version}).Debug("error attempting to find licenses")
log.WithFields("error", err, "mavenID", mavenID{groupID, artifactID, version}).Trace("error attempting to find licenses")
}
}
@@ -300,7 +313,7 @@ func (j *archiveParser) findLicenseFromJavaMetadata(ctx context.Context, groupID
groupID = strings.Join(packages[:len(packages)-1], ".")
pomLicenses, err = j.maven.findLicenses(ctx, groupID, artifactID, version)
if err != nil {
log.WithFields("error", err, "mavenID", mavenID{groupID, artifactID, version}).Debug("error attempting to find sub-group licenses")
log.WithFields("error", err, "mavenID", mavenID{groupID, artifactID, version}).Trace("error attempting to find sub-group licenses")
}
}
@@ -630,7 +643,7 @@ func newPackageFromMavenData(ctx context.Context, r *mavenResolver, pomPropertie
}
if err != nil {
log.WithFields("error", err, "mavenID", mavenID{pomProperties.GroupID, pomProperties.ArtifactID, pomProperties.Version}).Debug("error attempting to resolve licenses")
log.WithFields("error", err, "mavenID", mavenID{pomProperties.GroupID, pomProperties.ArtifactID, pomProperties.Version}).Trace("error attempting to resolve licenses")
}
licenses := make([]pkg.License, 0)
+16 -2
View File
@@ -91,10 +91,12 @@ func TestParseJar(t *testing.T) {
fixture string
expected map[string]pkg.Package
ignoreExtras []string
wantErr require.ErrorAssertionFunc
}{
{
name: "example-jenkins-plugin",
fixture: "test-fixtures/java-builds/packages/example-jenkins-plugin.hpi",
wantErr: require.Error, // there are nested jars, which are not scanned and result in unknown errors
ignoreExtras: []string{
"Plugin-Version", // has dynamic date
"Built-By", // podman returns the real UID
@@ -153,6 +155,7 @@ func TestParseJar(t *testing.T) {
{
name: "example-java-app-gradle",
fixture: "test-fixtures/java-builds/packages/example-java-app-gradle-0.1.0.jar",
wantErr: require.NoError, // no nested jars
expected: map[string]pkg.Package{
"example-java-app-gradle": {
Name: "example-java-app-gradle",
@@ -226,6 +229,7 @@ func TestParseJar(t *testing.T) {
{
name: "example-java-app-maven",
fixture: "test-fixtures/java-builds/packages/example-java-app-maven-0.1.0.jar",
wantErr: require.NoError, // no nested jars
ignoreExtras: []string{
"Build-Jdk", // can't guarantee the JDK used at build time
"Built-By", // podman returns the real UID
@@ -351,13 +355,15 @@ func TestParseJar(t *testing.T) {
require.NoError(t, err)
actual, _, err := parser.parse(context.Background())
require.NoError(t, err)
if test.wantErr != nil {
test.wantErr(t, err)
}
if len(actual) != len(test.expected) {
for _, a := range actual {
t.Log(" ", a)
}
t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected))
t.Fatalf("unexpected package count; expected: %d got: %d", len(test.expected), len(actual))
}
var parent *pkg.Package
@@ -1488,3 +1494,11 @@ func run(t testing.TB, cmd *exec.Cmd) {
func ptr[T any](value T) *T {
return &value
}
func Test_corruptJarArchive(t *testing.T) {
ap := newGenericArchiveParserAdapter(DefaultArchiveCatalogerConfig())
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/corrupt/example.jar").
WithError().
TestParser(t, ap.parseJavaArchive)
}
+1 -1
View File
@@ -79,7 +79,7 @@ func (r *mavenResolver) resolvePropertyValue(ctx context.Context, propertyValue
}
resolved, err := r.resolveExpression(ctx, resolutionContext, *propertyValue, resolvingProperties)
if err != nil {
log.WithFields("error", err, "propertyValue", *propertyValue).Debug("error resolving maven property")
log.WithFields("error", err, "propertyValue", *propertyValue).Trace("error resolving maven property")
return ""
}
return resolved
+4 -1
View File
@@ -15,6 +15,7 @@ import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -41,11 +42,13 @@ func (p pomXMLCataloger) Catalog(ctx context.Context, fileResolver file.Resolver
r := newMavenResolver(fileResolver, p.cfg)
var errs error
var poms []*gopom.Project
for _, pomLocation := range locations {
pom, err := readPomFromLocation(fileResolver, pomLocation)
if err != nil || pom == nil {
log.WithFields("error", err, "pomLocation", pomLocation).Debug("error while reading pom")
errs = unknown.Appendf(errs, pomLocation, "error reading pom.xml: %w", err)
continue
}
@@ -60,7 +63,7 @@ func (p pomXMLCataloger) Catalog(ctx context.Context, fileResolver file.Resolver
for _, pom := range poms {
pkgs = append(pkgs, processPomXML(ctx, r, pom, r.pomLocations[pom])...)
}
return pkgs, nil, nil
return pkgs, nil, errs
}
func readPomFromLocation(fileResolver file.Resolver, pomLocation file.Location) (*gopom.Project, error) {
@@ -705,3 +705,11 @@ func getCommonsTextExpectedPackages() []pkg.Package {
},
}
}
func Test_corruptPomXml(t *testing.T) {
c := NewPomCataloger(DefaultArchiveCatalogerConfig())
pkgtest.NewCatalogTester().
FromDirectory(t, "test-fixtures/corrupt").
WithError().
TestCataloger(t, c)
}
@@ -10,6 +10,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
)
func Test_parseTarWrappedJavaArchive(t *testing.T) {
@@ -57,3 +58,11 @@ func Test_parseTarWrappedJavaArchive(t *testing.T) {
})
}
}
func Test_corruptTarArchive(t *testing.T) {
ap := newGenericTarWrappedJavaArchiveParser(DefaultArchiveCatalogerConfig())
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/corrupt/example.tar").
WithError().
TestParser(t, ap.parseTarWrappedJavaArchive)
}
@@ -0,0 +1 @@
example archive
@@ -0,0 +1 @@
example archive
@@ -0,0 +1 @@
<this is not a valid pom xml>
@@ -11,7 +11,6 @@ import (
"github.com/mitchellh/mapstructure"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -64,11 +63,8 @@ func parsePackageJSON(_ context.Context, _ file.Resolver, _ *generic.Environment
return nil, nil, fmt.Errorf("failed to parse package.json file: %w", err)
}
if !p.hasNameAndVersionValues() {
log.Debugf("encountered package.json file without a name and/or version field, ignoring (path=%q)", reader.Path())
return nil, nil, nil
}
// always create a package, regardless of having a valid name and/or version,
// a compliance filter later will remove these packages based on compliance rules
pkgs = append(
pkgs,
newPackageJSONPackage(p, reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
@@ -203,10 +199,6 @@ func licensesFromJSON(b []byte) ([]npmPackageLicense, error) {
return nil, errors.New("unmarshal failed")
}
func (p packageJSON) hasNameAndVersionValues() bool {
return p.Name != "" && p.Version != ""
}
// this supports both windows and unix paths
var filepathSeparator = regexp.MustCompile(`[\\/]`)
@@ -210,10 +210,28 @@ func TestParsePackageJSON(t *testing.T) {
}
}
func Test_corruptPackageJSON(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/corrupt/package.json").
WithError().
TestParser(t, parsePackageJSON)
}
func TestParsePackageJSON_Partial(t *testing.T) { // see https://github.com/anchore/syft/issues/311
const fixtureFile = "test-fixtures/pkg-json/package-partial.json"
pkgtest.TestFileParser(t, fixtureFile, parsePackageJSON, nil, nil)
// raise package.json files as packages with any information we find, these will be filtered out
// according to compliance rules later
expectedPkgs := []pkg.Package{
{
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
PURL: packageURL("", ""),
Metadata: pkg.NpmPackage{},
Locations: file.NewLocationSet(file.NewLocation(fixtureFile)),
},
}
pkgtest.TestFileParser(t, fixtureFile, parsePackageJSON, expectedPkgs, nil)
}
func Test_pathContainsNodeModulesDirectory(t *testing.T) {
@@ -9,6 +9,7 @@ import (
"strings"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -100,7 +101,7 @@ func (a genericPackageLockAdapter) parsePackageLock(_ context.Context, resolver
pkg.Sort(pkgs)
return pkgs, nil, nil
return pkgs, nil, unknown.IfEmptyf(pkgs, "unable to determine packages")
}
func (licenses *packageLockLicense) UnmarshalJSON(data []byte) (err error) {
@@ -333,3 +333,11 @@ func TestParsePackageLockLicenseWithArray(t *testing.T) {
adapter := newGenericPackageLockAdapter(CatalogerConfig{})
pkgtest.TestFileParser(t, fixture, adapter.parsePackageLock, expectedPkgs, expectedRelationships)
}
func Test_corruptPackageLock(t *testing.T) {
gap := newGenericPackageLockAdapter(DefaultCatalogerConfig())
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/corrupt/package-lock.json").
WithError().
TestParser(t, gap.parsePackageLock)
}
@@ -11,6 +11,7 @@ import (
"gopkg.in/yaml.v3"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -94,7 +95,7 @@ func parsePnpmLock(_ context.Context, resolver file.Resolver, _ *generic.Environ
pkg.Sort(pkgs)
return pkgs, nil, nil
return pkgs, nil, unknown.IfEmptyf(pkgs, "unable to determine packages")
}
func hasPkg(pkgs []pkg.Package, name, version string) bool {
@@ -144,3 +144,10 @@ func TestParsePnpmV6Lock(t *testing.T) {
pkgtest.TestFileParser(t, fixture, parsePnpmLock, expectedPkgs, expectedRelationships)
}
func Test_corruptPnpmLock(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/corrupt/pnpm-lock.yaml").
WithError().
TestParser(t, parsePnpmLock)
}
@@ -8,6 +8,7 @@ import (
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -112,7 +113,7 @@ func (a genericYarnLockAdapter) parseYarnLock(_ context.Context, resolver file.R
pkg.Sort(pkgs)
return pkgs, nil, nil
return pkgs, nil, unknown.IfEmptyf(pkgs, "unable to determine packages")
}
func findPackageName(line string) string {
@@ -0,0 +1,4 @@
{
"requires": true,
"lockfi
}
@@ -0,0 +1,5 @@
{
"version": "6.14.6",
"name"node": "6 >=6.2.0 || 8 || >=9.3.0"
}
}
@@ -0,0 +1,7 @@
lockfileVersion: 5.4
specifi
lution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==}
dev: true
# removed other packages
+3 -4
View File
@@ -6,9 +6,8 @@ package kernel
import (
"context"
"github.com/hashicorp/go-multierror"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -62,7 +61,7 @@ func (l linuxKernelCataloger) Catalog(ctx context.Context, resolver file.Resolve
kernelPackages, kernelRelationships, err := generic.NewCataloger(l.Name()).WithParserByGlobs(parseLinuxKernelFile, kernelArchiveGlobs...).Catalog(ctx, resolver)
if err != nil {
errs = multierror.Append(errs, err)
errs = unknown.Join(errs, err)
}
allRelationships = append(allRelationships, kernelRelationships...)
@@ -71,7 +70,7 @@ func (l linuxKernelCataloger) Catalog(ctx context.Context, resolver file.Resolve
if l.cfg.CatalogModules {
modulePackages, moduleRelationships, err := generic.NewCataloger(l.Name()).WithParserByGlobs(parseLinuxKernelModuleFile, kernelModuleGlobs...).Catalog(ctx, resolver)
if err != nil {
errs = multierror.Append(errs, err)
errs = unknown.Join(errs, err)
}
allPackages = append(allPackages, modulePackages...)
+2 -1
View File
@@ -2,6 +2,7 @@ package lua
import (
"context"
"fmt"
"strings"
"github.com/anchore/syft/internal/log"
@@ -30,7 +31,7 @@ func parseRockspec(_ context.Context, _ file.Resolver, _ *generic.Environment, r
doc, err := parseRockspecData(reader)
if err != nil {
log.WithFields("error", err).Trace("unable to parse Rockspec app")
return nil, nil, nil
return nil, nil, fmt.Errorf("unable to parse Rockspec app: %w", err)
}
var name, version, license, homepage, description, url string
@@ -106,3 +106,10 @@ func TestParseRockspec(t *testing.T) {
})
}
}
func Test_corruptRockspec(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/corrupt/bad-1.23.0-0.rockspec").
WithError().
TestParser(t, parseRockspec)
}
@@ -0,0 +1,5 @@
package = {"kon
3.7.0-0"
rockspec_fo}
}
@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -48,5 +49,5 @@ func parseComposerLock(_ context.Context, _ file.Resolver, _ *generic.Environmen
}
}
return pkgs, nil, nil
return pkgs, nil, unknown.IfEmptyf(pkgs, "unable to determine packages")
}
@@ -113,3 +113,10 @@ func TestParseComposerFileLock(t *testing.T) {
}
pkgtest.TestFileParser(t, fixture, parseComposerLock, expectedPkgs, expectedRelationships)
}
func Test_corruptComposerLock(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/glob-paths/src/composer.lock").
WithError().
TestParser(t, parseComposerLock)
}
@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -69,5 +70,5 @@ func parseInstalledJSON(_ context.Context, _ file.Resolver, _ *generic.Environme
}
}
return pkgs, nil, nil
return pkgs, nil, unknown.IfEmptyf(pkgs, "unable to determine packages")
}
@@ -142,3 +142,10 @@ func TestParseInstalledJsonComposerV1(t *testing.T) {
})
}
}
func Test_corruptInstalledJSON(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/glob-paths/src/installed.json").
WithError().
TestParser(t, parseInstalledJSON)
}
@@ -33,3 +33,10 @@ func TestParsePeclSerialized(t *testing.T) {
}
pkgtest.TestFileParser(t, fixture, parsePeclSerialized, expectedPkgs, expectedRelationships)
}
func Test_corruptPecl(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/glob-paths/php/.registry/.channel.pecl.php.net/memcached.reg").
WithError().
TestParser(t, parseComposerLock)
}
@@ -80,3 +80,10 @@ func TestParsePipFileLock(t *testing.T) {
pkgtest.TestFileParser(t, fixture, parsePipfileLock, expectedPkgs, expectedRelationships)
}
func Test_corruptPipfileLock(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/glob-paths/src/Pipfile.lock").
WithError().
TestParser(t, parsePipfileLock)
}
@@ -8,6 +8,7 @@ import (
"github.com/BurntSushi/toml"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -57,7 +58,7 @@ func parsePoetryLock(_ context.Context, _ file.Resolver, _ *generic.Environment,
// since we would never expect to create relationships for packages across multiple poetry.lock files
// we should do this on a file parser level (each poetry.lock) instead of a cataloger level (across all
// poetry.lock files)
return pkgs, dependency.Resolve(poetryLockDependencySpecifier, pkgs), nil
return pkgs, dependency.Resolve(poetryLockDependencySpecifier, pkgs), unknown.IfEmptyf(pkgs, "unable to determine packages")
}
func poetryLockPackages(reader file.LocationReadCloser) ([]pkg.Package, error) {
@@ -81,3 +81,10 @@ func TestParsePoetryLock(t *testing.T) {
pkgtest.TestFileParser(t, fixture, parsePoetryLock, expectedPkgs, expectedRelationships)
}
func Test_corruptPoetryLock(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/glob-paths/src/poetry.lock").
WithError().
TestParser(t, parsePoetryLock)
}
@@ -13,6 +13,7 @@ import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -94,6 +95,7 @@ func newRequirementsParser(cfg CatalogerConfig) requirementsParser {
// parseRequirementsTxt takes a Python requirements.txt file, returning all Python packages that are locked to a
// specific version.
func (rp requirementsParser) parseRequirementsTxt(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
var errs error
var packages []pkg.Package
scanner := bufio.NewScanner(reader)
@@ -126,6 +128,7 @@ func (rp requirementsParser) parseRequirementsTxt(_ context.Context, _ file.Reso
req := newRequirement(line)
if req == nil {
log.WithFields("path", reader.RealPath).Warnf("unable to parse requirements.txt line: %q", line)
errs = unknown.Appendf(errs, reader, "unable to parse requirements.txt line: %q", line)
continue
}
@@ -134,6 +137,7 @@ func (rp requirementsParser) parseRequirementsTxt(_ context.Context, _ file.Reso
if version == "" {
log.WithFields("path", reader.RealPath).Tracef("unable to determine package version in requirements.txt line: %q", line)
errs = unknown.Appendf(errs, reader, "unable to determine package version in requirements.txt line: %q", line)
continue
}
@@ -158,7 +162,7 @@ func (rp requirementsParser) parseRequirementsTxt(_ context.Context, _ file.Reso
return nil, nil, fmt.Errorf("failed to parse python requirements file: %w", err)
}
return packages, nil, nil
return packages, nil, unknown.Join(errs, unknown.IfEmptyf(packages, "unable to determine packages"))
}
func parseVersion(version string, guessFromConstraint bool) string {
@@ -353,3 +353,11 @@ func Test_parseVersion(t *testing.T) {
})
}
}
func Test_corruptRequirementsTxt(t *testing.T) {
rp := newRequirementsParser(DefaultCatalogerConfig())
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/glob-paths/src/requirements.txt").
WithError().
TestParser(t, rp.parseRequirementsTxt)
}
@@ -94,3 +94,10 @@ func TestParseRpmFiles(t *testing.T) {
})
}
}
func Test_corruptRpmArchive(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/bad/bad.rpm").
WithError().
TestParser(t, parseRpmArchive)
}
+15 -5
View File
@@ -9,6 +9,7 @@ import (
rpmdb "github.com/knqyf263/go-rpmdb/pkg"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
@@ -59,11 +60,15 @@ func parseRpmDB(_ context.Context, resolver file.Resolver, env *generic.Environm
distro = env.LinuxRelease
}
var errs error
for _, entry := range pkgList {
if entry == nil {
continue
}
files, err := extractRpmFileRecords(resolver, *entry)
errs = unknown.Join(errs, err)
metadata := pkg.RpmDBEntry{
Name: entry.Name,
Version: entry.Version,
@@ -74,7 +79,7 @@ func parseRpmDB(_ context.Context, resolver file.Resolver, env *generic.Environm
Vendor: entry.Vendor,
Size: entry.Size,
ModularityLabel: &entry.Modularitylabel,
Files: extractRpmFileRecords(resolver, *entry),
Files: files,
Provides: entry.Provides,
Requires: entry.Requires,
}
@@ -89,6 +94,7 @@ func parseRpmDB(_ context.Context, resolver file.Resolver, env *generic.Environm
if !pkg.IsValid(&p) {
log.WithFields("location", reader.RealPath, "pkg", fmt.Sprintf("%s@%s", entry.Name, entry.Version)).
Warn("ignoring invalid package found in RPM DB")
errs = unknown.Appendf(errs, reader, "invalild package found; name: %s, version: %s", entry.Name, entry.Version)
continue
}
@@ -96,7 +102,11 @@ func parseRpmDB(_ context.Context, resolver file.Resolver, env *generic.Environm
allPkgs = append(allPkgs, p)
}
return allPkgs, nil, nil
if errs == nil && len(allPkgs) == 0 {
errs = fmt.Errorf("unable to determine packages")
}
return allPkgs, nil, errs
}
// The RPM naming scheme is [name]-[version]-[release]-[arch], where version is implicitly expands to [epoch]:[version].
@@ -112,13 +122,13 @@ func toELVersion(epoch *int, version, release string) string {
return fmt.Sprintf("%s-%s", version, release)
}
func extractRpmFileRecords(resolver file.PathResolver, entry rpmdb.PackageInfo) []pkg.RpmFileRecord {
func extractRpmFileRecords(resolver file.PathResolver, entry rpmdb.PackageInfo) ([]pkg.RpmFileRecord, error) {
var records = make([]pkg.RpmFileRecord, 0)
files, err := entry.InstalledFiles()
if err != nil {
log.Warnf("unable to parse listing of installed files for RPM DB entry: %s", err.Error())
return records
return records, fmt.Errorf("unable to parse listing of installed files for RPM DB entry: %w", err)
}
for _, record := range files {
@@ -138,5 +148,5 @@ func extractRpmFileRecords(resolver file.PathResolver, entry rpmdb.PackageInfo)
})
}
}
return records
return records, nil
}
@@ -214,6 +214,13 @@ func TestToElVersion(t *testing.T) {
}
}
func Test_corruptRpmDbEntry(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/glob-paths/usr/lib/sysimage/rpm/Packages.db").
WithError().
TestParser(t, parseRpmDB)
}
func intRef(i int) *int {
return &i
}
@@ -8,6 +8,7 @@ import (
"strings"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -53,5 +54,5 @@ func parseRpmManifest(_ context.Context, _ file.Resolver, _ *generic.Environment
allPkgs = append(allPkgs, p)
}
return allPkgs, nil, nil
return allPkgs, nil, unknown.IfEmptyf(allPkgs, "unable to determine packages")
}
@@ -7,6 +7,7 @@ import (
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -54,7 +55,7 @@ func parseGemFileLockEntries(_ context.Context, _ file.Resolver, _ *generic.Envi
if err := scanner.Err(); err != nil {
return nil, nil, err
}
return pkgs, nil, nil
return pkgs, nil, unknown.IfEmptyf(pkgs, "unable to determine packages")
}
func isDependencyLine(line string) bool {
@@ -97,3 +97,10 @@ func Test_AuditBinaryCataloger_Globs(t *testing.T) {
})
}
}
func Test_corruptAuditBinary(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/glob-paths/partial-binary").
WithError().
TestParser(t, parseAuditBinary)
}
@@ -3,6 +3,7 @@ package rust
import (
"context"
"errors"
"fmt"
rustaudit "github.com/microsoft/go-rustaudit"
@@ -23,21 +24,22 @@ func parseAuditBinary(_ context.Context, _ file.Resolver, _ *generic.Environment
return nil, nil, err
}
for _, versionInfo := range parseAuditBinaryEntry(unionReader, reader.RealPath) {
infos, err := parseAuditBinaryEntry(unionReader, reader.RealPath)
for _, versionInfo := range infos {
pkgs = append(pkgs, newPackagesFromAudit(reader.Location, versionInfo)...)
}
return pkgs, nil, nil
return pkgs, nil, err
}
// scanFile scans file to try to report the Rust crate dependencies
func parseAuditBinaryEntry(reader unionreader.UnionReader, filename string) []rustaudit.VersionInfo {
func parseAuditBinaryEntry(reader unionreader.UnionReader, filename string) ([]rustaudit.VersionInfo, error) {
// NOTE: multiple readers are returned to cover universal binaries, which are files
// with more than one binary
readers, err := unionreader.GetReaders(reader)
if err != nil {
log.Warnf("rust cataloger: failed to open a binary: %v", err)
return nil
return nil, fmt.Errorf("rust cataloger: failed to open a binary: %w", err)
}
var versionInfos []rustaudit.VersionInfo
@@ -48,14 +50,14 @@ func parseAuditBinaryEntry(reader unionreader.UnionReader, filename string) []ru
if errors.Is(err, rustaudit.ErrNoRustDepInfo) {
// since the cataloger can only select executables and not distinguish if they are a Rust-compiled
// binary, we should not show warnings/logs in this case.
return nil
return nil, nil
}
log.Tracef("rust cataloger: unable to read dependency information (file=%q): %v", filename, err)
return nil
return nil, fmt.Errorf("rust cataloger: unable to read dependency information: %w", err)
}
versionInfos = append(versionInfos, versionInfo)
}
return versionInfos
return versionInfos, nil
}
+2 -1
View File
@@ -6,6 +6,7 @@ import (
"github.com/pelletier/go-toml"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -46,5 +47,5 @@ func parseCargoLock(_ context.Context, _ file.Resolver, _ *generic.Environment,
)
}
return pkgs, nil, nil
return pkgs, nil, unknown.IfEmptyf(pkgs, "unable to determine packages")
}

Some files were not shown because too many files have changed in this diff Show More