Compare commits

...
15 Commits
Author SHA1 Message Date
Christopher Angelo Phillips 23a3173c9f feat: add catalog delete (#1377) 2022-12-12 12:55:12 -05:00
Keith Zantow 17aa8287e6 docs: remove file classifier (#1397) 2022-12-08 16:50:29 +00:00
Christopher Angelo Phillips 730d3e3187 chore: update latest cyclonedx library (#1390) 2022-12-08 11:36:08 -05:00
Keith Zantow 997fbdfcf3 feat: Add Java binary catalogers (#1392) 2022-12-08 10:50:28 -05:00
Marc-Etienne Vargenau 13ceed9336 chore: Update SPDX license list to 3.19 (#1389) 2022-12-08 10:29:27 -05:00
Chapman Pendery 668f102340 fix: add manual vendor/product removal to fix false flags (#1070)
Closes https://github.com/anchore/syft/issues/1066
Closes https://github.com/anchore/grype/issues/800
Closes https://github.com/anchore/grype/issues/491
2022-12-08 09:57:42 -05:00
anchore-actions-token-generator[bot] f1a124209a Update Stereoscope to c5ff155d72f166e2332e160a75c3ff2b8e9c7e2e (#1395)
Signed-off-by: GitHub <noreply@github.com>
2022-12-08 08:32:49 +00:00
Keith Zantow 5dbb3fc41d chore: fix test busybox image sha (#1393) 2022-12-07 20:15:39 -05:00
Keith Zantow 614ea00905 fix: go version not properly identified in binary (#1384) 2022-12-02 13:24:36 -05:00
anchore-actions-token-generator[bot]andwagoodman 247b054ab5 Update Stereoscope to 3b80d983223f6e6fc2d33b0ffa003d30268418e9 (#1376)
Signed-off-by: GitHub <noreply@github.com>

Signed-off-by: GitHub <noreply@github.com>
Co-authored-by: wagoodman <wagoodman@users.noreply.github.com>
2022-11-30 16:11:57 +00:00
Keith Zantow 9e43725951 fix: Update node binary package name (#1375) 2022-11-30 10:30:57 -05:00
Keith Zantow 4f39287216 feat: Generic Binary Cataloger (#1336) 2022-11-29 18:28:10 -05:00
Alex Goodman 7a69e2129b recover from bad parsing of golang binary (#1371)
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2022-11-29 15:56:46 +00:00
Dan Luhring f6996f7b9a Fix parsing of apk databases with large entries (#1365)
Closes https://github.com/anchore/syft/issues/1354
2022-11-29 10:16:36 -05:00
anchore-actions-token-generator[bot]andspiffcs bd523bdb5d Update syft bootstrap tools to latest versions. (#1369)
Co-authored-by: spiffcs <spiffcs@users.noreply.github.com>
2022-11-29 10:13:00 -05:00
65 changed files with 178902 additions and 6542 deletions
+1 -1
View File
@@ -13,7 +13,7 @@ GOLANGCILINT_VERSION = v1.50.1
GOSIMPORTS_VERSION = v0.3.4
BOUNCER_VERSION = v0.4.0
CHRONICLE_VERSION = v0.4.2
GORELEASER_VERSION = v1.13.0
GORELEASER_VERSION = v1.13.1
YAJSV_VERSION = v1.4.1
COSIGN_VERSION = v1.13.1
QUILL_VERSION = v0.2.0
-11
View File
@@ -489,17 +489,6 @@ package:
# same as -s ; SYFT_PACKAGE_CATALOGER_SCOPE env var
scope: "squashed"
# cataloging file classifications is exposed through the power-user subcommand
file-classification:
cataloger:
# enable/disable cataloging of file classifications
# SYFT_FILE_CLASSIFICATION_CATALOGER_ENABLED env var
enabled: true
# the search space to look for file classifications (options: all-layers, squashed)
# SYFT_FILE_CLASSIFICATION_CATALOGER_SCOPE env var
scope: "squashed"
# cataloging file contents is exposed through the power-user subcommand
file-contents:
cataloger:
-29
View File
@@ -22,7 +22,6 @@ func Tasks(app *config.Application) ([]Task, error) {
generateCatalogFileMetadataTask,
generateCatalogFileDigestsTask,
generateCatalogSecretsTask,
generateCatalogFileClassificationsTask,
generateCatalogContentsTask,
}
@@ -162,34 +161,6 @@ func generateCatalogSecretsTask(app *config.Application) (Task, error) {
return task, nil
}
func generateCatalogFileClassificationsTask(app *config.Application) (Task, error) {
if !app.FileClassification.Cataloger.Enabled {
return nil, nil
}
// TODO: in the future we could expose out the classifiers via configuration
classifierCataloger, err := file.NewClassificationCataloger(file.DefaultClassifiers)
if err != nil {
return nil, err
}
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(app.FileClassification.Cataloger.ScopeOpt)
if err != nil {
return nil, err
}
result, err := classifierCataloger.Catalog(resolver)
if err != nil {
return nil, err
}
results.FileClassifications = result
return nil, nil
}
return task, nil
}
func generateCatalogContentsTask(app *config.Application) (Task, error) {
if !app.FileContents.Cataloger.Enabled {
return nil, nil
+4 -4
View File
@@ -3,7 +3,7 @@ module github.com/anchore/syft
go 1.18
require (
github.com/CycloneDX/cyclonedx-go v0.5.2
github.com/CycloneDX/cyclonedx-go v0.7.0
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/acobaugh/osrelease v0.1.0
github.com/adrg/xdg v0.3.3
@@ -52,7 +52,7 @@ require (
require (
github.com/Masterminds/sprig/v3 v3.2.2
github.com/anchore/go-logger v0.0.0-20220728155337-03b66a5207d8
github.com/anchore/stereoscope v0.0.0-20221006201143-d24c9d626b33
github.com/anchore/stereoscope v0.0.0-20221208011002-c5ff155d72f1
github.com/docker/docker v20.10.17+incompatible
github.com/google/go-containerregistry v0.11.0
github.com/in-toto/in-toto-golang v0.4.1-0.20221018183522-731d0640b65f
@@ -124,7 +124,7 @@ require (
github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect
github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490 // indirect
github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect
github.com/containerd/containerd v1.6.8 // indirect
github.com/containerd/containerd v1.6.12 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.12.0 // indirect
github.com/coreos/go-oidc/v3 v3.4.0 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
@@ -208,7 +208,7 @@ require (
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/miekg/pkcs11 v1.1.1 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
+18 -10
View File
@@ -153,8 +153,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw=
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/CycloneDX/cyclonedx-go v0.5.2 h1:CkdGw2R/tZWmEbSypJVZG+3+2SAsDjJirfIrG/RbIVg=
github.com/CycloneDX/cyclonedx-go v0.5.2/go.mod h1:nQCiF4Tvrg5Ieu8qPhYMvzPGMu5I7fANZkrSsJjl5mg=
github.com/CycloneDX/cyclonedx-go v0.7.0 h1:jNxp8hL7UpcvPDFXjY+Y1ibFtsW+e5zyF9QoSmhK/zg=
github.com/CycloneDX/cyclonedx-go v0.7.0/go.mod h1:W5Z9w8pTTL+t+yG3PCiFRGlr8PUlE0pGWzKSJbsyXkg=
github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs=
@@ -196,7 +196,7 @@ github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwT
github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg=
github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
github.com/Microsoft/hcsshim v0.9.4/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
github.com/Microsoft/hcsshim v0.9.5/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU=
github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
@@ -280,8 +280,8 @@ github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZV
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
github.com/anchore/packageurl-go v0.1.1-0.20220428202044-a072fa3cb6d7 h1:kDrYkTSM9uIxaX/P9s0F4nKYNM+hnSgLJdLpqvsaQ/g=
github.com/anchore/packageurl-go v0.1.1-0.20220428202044-a072fa3cb6d7/go.mod h1:Blo6OgJNiYF41ufcgHKkbCKF2MDOMlrqhXv/ij6ocR4=
github.com/anchore/stereoscope v0.0.0-20221006201143-d24c9d626b33 h1:Y+9aMJMTEMH+kJJFEaxqrF6X1t9CkjpWjOzaCo1q5vM=
github.com/anchore/stereoscope v0.0.0-20221006201143-d24c9d626b33/go.mod h1:WOWtswyDxLkBnizq9LihYa9uw88r1FhBvJo7g//Ljcc=
github.com/anchore/stereoscope v0.0.0-20221208011002-c5ff155d72f1 h1:DXUAm/H9chRTEzMfkFyduBIcCiJyFXhCmv3zH3C0HGs=
github.com/anchore/stereoscope v0.0.0-20221208011002-c5ff155d72f1/go.mod h1:/zjVnu2Jdl7xQCUtASegzeEg+IHKrM7SyMqdao3e+Nc=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
@@ -401,8 +401,7 @@ github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTS
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/bombsimon/wsl/v3 v3.3.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
github.com/bradleyjkemp/cupaloy/v2 v2.7.0 h1:AT0vOjO68RcLyenLCHOGZzSNiuto7ziqzq6Q1/3xzMQ=
github.com/bradleyjkemp/cupaloy/v2 v2.7.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=
github.com/breml/bidichk v0.1.1/go.mod h1:zbfeitpevDUGI7V91Uzzuwrn4Vls8MoBMrwtt78jmso=
github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
@@ -516,8 +515,8 @@ github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTV
github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c=
github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s=
github.com/containerd/containerd v1.6.1/go.mod h1:1nJz5xCZPusx6jJU8Frfct988y0NpumIq9ODB0kLtoE=
github.com/containerd/containerd v1.6.8 h1:h4dOFDwzHmqFEP754PgfgTeVXFnLiRc6kiqC7tplDJs=
github.com/containerd/containerd v1.6.8/go.mod h1:By6p5KqPK0/7/CgO/A6t/Gz+CUYUu2zf1hUaaymVXB0=
github.com/containerd/containerd v1.6.12 h1:kJ9b3mOFKf8yqo05Ob+tMoxvt1pbVWhnB0re9Y+k+8c=
github.com/containerd/containerd v1.6.12/go.mod h1:K4Bw7gjgh4TnkmQY+py/PYQGp4e7xgnHAeg87VeWb3A=
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
@@ -526,6 +525,7 @@ github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR
github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk=
github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM=
github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0=
@@ -1376,6 +1376,7 @@ github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
@@ -1408,8 +1409,9 @@ github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4
github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo=
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY=
github.com/mbilski/exhaustivestruct v1.2.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc=
github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg=
@@ -2215,6 +2217,7 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
@@ -2343,6 +2346,7 @@ golang.org/x/net v0.0.0-20220421235706-1d1ef9303861/go.mod h1:CfG3xpIq0wQ8r1q4Su
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458 h1:MgJ6t2zo8v0tbmLCueaCbF1RM+TtB0rs3Lv8DGtOIpY=
golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
@@ -2532,6 +2536,7 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220405210540-1e041c57c461/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -2539,6 +2544,7 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI=
@@ -2878,6 +2884,7 @@ google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX
google.golang.org/genproto v0.0.0-20220422154200-b37d22cd5731/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
@@ -3057,6 +3064,7 @@ k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc=
k8s.io/cri-api v0.23.1/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4=
k8s.io/cri-api v0.25.0/go.mod h1:J1rAyQkSJ2Q6I+aBMOVgg2/cbbebso6FNa0UagiR0kc=
k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
@@ -423,6 +423,7 @@ func Test_processSPDXLicense(t *testing.T) {
"cern-ohl-w-2": "CERN-OHL-W-2.0",
"cern-ohl-w-2.0": "CERN-OHL-W-2.0",
"cern-ohl-w-2.0.0": "CERN-OHL-W-2.0",
"checkmk": "checkmk",
"clartistic": "ClArtistic",
"cnri-jython": "CNRI-Jython",
"cnri-python": "CNRI-Python",
@@ -521,6 +522,7 @@ func Test_processSPDXLicense(t *testing.T) {
"fsfap": "FSFAP",
"fsful": "FSFUL",
"fsfullr": "FSFULLR",
"fsfullrwd": "FSFULLRWD",
"ftl": "FTL",
"gd": "GD",
"gfdl-1": "GFDL-1.1-only",
@@ -668,6 +670,7 @@ func Test_processSPDXLicense(t *testing.T) {
"jasper-2.0.0": "JasPer-2.0",
"jpnic": "JPNIC",
"json": "JSON",
"knuth-ctan": "Knuth-CTAN",
"lal-1": "LAL-1.2",
"lal-1.2": "LAL-1.2",
"lal-1.2.0": "LAL-1.2",
@@ -716,6 +719,7 @@ func Test_processSPDXLicense(t *testing.T) {
"libselinux-1.0": "libselinux-1.0",
"libselinux-1.0.0": "libselinux-1.0",
"libtiff": "libtiff",
"libutil-david-nugent": "libutil-David-Nugent",
"liliq-p-1": "LiLiQ-P-1.1",
"liliq-p-1.1": "LiLiQ-P-1.1",
"liliq-p-1.1.0": "LiLiQ-P-1.1",
File diff suppressed because it is too large Load Diff
+7 -3
View File
@@ -1,9 +1,9 @@
// Code generated by go generate; DO NOT EDIT.
// This file was generated by robots at 2022-10-14 11:46:16.097525 -0400 EDT m=+0.276930979
// This file was generated by robots at 2022-12-02 12:51:33.814385 -0500 EST m=+1.688731869
// using data from https://spdx.org/licenses/licenses.json
package spdxlicense
const Version = "3.18"
const Version = "3.19"
var licenseIDs = map[string]string{
"0bsd": "0BSD",
@@ -372,6 +372,7 @@ var licenseIDs = map[string]string{
"cern-ohl-w-2": "CERN-OHL-W-2.0",
"cern-ohl-w-2.0": "CERN-OHL-W-2.0",
"cern-ohl-w-2.0.0": "CERN-OHL-W-2.0",
"checkmk": "checkmk",
"clartistic": "ClArtistic",
"cnri-jython": "CNRI-Jython",
"cnri-python": "CNRI-Python",
@@ -470,6 +471,7 @@ var licenseIDs = map[string]string{
"fsfap": "FSFAP",
"fsful": "FSFUL",
"fsfullr": "FSFULLR",
"fsfullrwd": "FSFULLRWD",
"ftl": "FTL",
"gd": "GD",
"gfdl-1": "GFDL-1.1-only",
@@ -617,6 +619,7 @@ var licenseIDs = map[string]string{
"jasper-2.0.0": "JasPer-2.0",
"jpnic": "JPNIC",
"json": "JSON",
"knuth-ctan": "Knuth-CTAN",
"lal-1": "LAL-1.2",
"lal-1.2": "LAL-1.2",
"lal-1.2.0": "LAL-1.2",
@@ -665,6 +668,7 @@ var licenseIDs = map[string]string{
"libselinux-1.0": "libselinux-1.0",
"libselinux-1.0.0": "libselinux-1.0",
"libtiff": "libtiff",
"libutil-david-nugent": "libutil-David-Nugent",
"liliq-p-1": "LiLiQ-P-1.1",
"liliq-p-1.1": "LiLiQ-P-1.1",
"liliq-p-1.1.0": "LiLiQ-P-1.1",
@@ -984,7 +988,7 @@ var licenseIDs = map[string]string{
"sspl-1": "SSPL-1.0",
"sspl-1.0": "SSPL-1.0",
"sspl-1.0.0": "SSPL-1.0",
"standardml-nj": "StandardML-NJ",
"standardml-nj": "SMLNJ",
"sugarcrm-1": "SugarCRM-1.1.3",
"sugarcrm-1.1": "SugarCRM-1.1.3",
"sugarcrm-1.1.3": "SugarCRM-1.1.3",
-38
View File
@@ -1,38 +0,0 @@
package file
import (
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/source"
)
type ClassificationCataloger struct {
classifiers []Classifier
}
func NewClassificationCataloger(classifiers []Classifier) (*ClassificationCataloger, error) {
return &ClassificationCataloger{
classifiers: classifiers,
}, nil
}
func (i *ClassificationCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]Classification, error) {
results := make(map[source.Coordinates][]Classification)
numResults := 0
for _, location := range allRegularFiles(resolver) {
for _, classifier := range i.classifiers {
result, err := classifier.Classify(resolver, location)
if err != nil {
log.Warnf("file classification cataloger failed with class=%q at location=%+v: %+v", classifier.Class, location, err)
continue
}
if result != nil {
results[location.Coordinates] = append(results[location.Coordinates], *result)
numResults++
}
}
}
log.Debugf("file classifier discovered %d results", numResults)
return results, nil
}
-210
View File
@@ -1,210 +0,0 @@
package file
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/source"
)
func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
tests := []struct {
name string
fixtureDir string
location string
expected []Classification
expectedErr func(assert.TestingT, error, ...interface{}) bool
}{
{
name: "positive-libpython3.7.so",
fixtureDir: "test-fixtures/classifiers/positive",
location: "libpython3.7.so",
expected: []Classification{
{
Class: "python-binary",
Metadata: map[string]string{
"version": "3.7.4a-vZ9",
},
},
},
expectedErr: assert.NoError,
},
{
name: "positive-python3.6",
fixtureDir: "test-fixtures/classifiers/positive",
location: "python3.6",
expected: []Classification{
{
Class: "python-binary",
Metadata: map[string]string{
"version": "3.6.3a-vZ9",
},
},
},
expectedErr: assert.NoError,
},
{
name: "positive-patchlevel.h",
fixtureDir: "test-fixtures/classifiers/positive",
location: "patchlevel.h",
expected: []Classification{
{
Class: "cpython-source",
Metadata: map[string]string{
"version": "3.9-aZ5",
},
},
},
expectedErr: assert.NoError,
},
{
name: "positive-go",
fixtureDir: "test-fixtures/classifiers/positive",
location: "go",
expected: []Classification{
{
Class: "go-binary",
Metadata: map[string]string{
"version": "1.14",
},
},
},
expectedErr: assert.NoError,
},
{
name: "positive-go-hint",
fixtureDir: "test-fixtures/classifiers/positive",
location: "VERSION",
expected: []Classification{
{
Class: "go-binary-hint",
Metadata: map[string]string{
"version": "1.15",
},
},
},
expectedErr: assert.NoError,
},
{
name: "positive-busybox",
fixtureDir: "test-fixtures/classifiers/positive",
location: "[", // note: busybox is a link to [
expected: []Classification{
{
Class: "busybox-binary",
VirtualPath: "busybox",
Metadata: map[string]string{
"version": "3.33.3",
},
},
},
expectedErr: assert.NoError,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
c, err := NewClassificationCataloger(DefaultClassifiers)
test.expectedErr(t, err)
src, err := source.NewFromDirectory(test.fixtureDir)
test.expectedErr(t, err)
resolver, err := src.FileResolver(source.SquashedScope)
test.expectedErr(t, err)
actualResults, err := c.Catalog(resolver)
test.expectedErr(t, err)
ok := false
for actualLoc, actualClassification := range actualResults {
if test.location == actualLoc.RealPath {
ok = true
assert.Equal(t, test.expected, actualClassification)
}
}
if !ok {
t.Fatalf("could not find test location=%q", test.location)
}
})
}
}
func TestClassifierCataloger_DefaultClassifiers_PositiveCases_Image(t *testing.T) {
tests := []struct {
name string
fixtureImage string
location string
expected []Classification
expectedErr func(assert.TestingT, error, ...interface{}) bool
}{
{
name: "busybox-regression",
fixtureImage: "image-busybox",
location: "/bin/[",
expected: []Classification{
{
Class: "busybox-binary",
VirtualPath: "/bin/busybox",
Metadata: map[string]string{
"version": "1.35.0",
},
},
},
expectedErr: assert.NoError,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
c, err := NewClassificationCataloger(DefaultClassifiers)
test.expectedErr(t, err)
img := imagetest.GetFixtureImage(t, "docker-archive", test.fixtureImage)
src, err := source.NewFromImage(img, "test-img")
test.expectedErr(t, err)
resolver, err := src.FileResolver(source.SquashedScope)
test.expectedErr(t, err)
actualResults, err := c.Catalog(resolver)
test.expectedErr(t, err)
ok := false
for actuaLoc, actualClassification := range actualResults {
if actuaLoc.RealPath == test.location {
ok = true
assert.Equal(t, test.expected, actualClassification)
}
}
if !ok {
t.Fatalf("could not find test location=%q", test.location)
}
})
}
}
func TestClassifierCataloger_DefaultClassifiers_NegativeCases(t *testing.T) {
c, err := NewClassificationCataloger(DefaultClassifiers)
assert.NoError(t, err)
src, err := source.NewFromDirectory("test-fixtures/classifiers/negative")
assert.NoError(t, err)
resolver, err := src.FileResolver(source.SquashedScope)
assert.NoError(t, err)
actualResults, err := c.Catalog(resolver)
assert.NoError(t, err)
assert.Equal(t, 0, len(actualResults))
}
-153
View File
@@ -1,153 +0,0 @@
package file
import (
"bytes"
"fmt"
"io"
"regexp"
"text/template"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/source"
)
var DefaultClassifiers = []Classifier{
{
Class: "python-binary",
FilepathPatterns: []*regexp.Regexp{
regexp.MustCompile(`(.*/|^)python(?P<version>[0-9]+\.[0-9]+)$`),
regexp.MustCompile(`(.*/|^)libpython(?P<version>[0-9]+\.[0-9]+).so.*$`),
},
EvidencePatternTemplates: []string{
`(?m)(?P<version>{{ .version }}\.[0-9]+[-_a-zA-Z0-9]*)`,
},
},
{
Class: "cpython-source",
FilepathPatterns: []*regexp.Regexp{
regexp.MustCompile(`(.*/|^)patchlevel.h$`),
},
EvidencePatternTemplates: []string{
`(?m)#define\s+PY_VERSION\s+"?(?P<version>[0-9\.\-_a-zA-Z]+)"?`,
},
},
{
Class: "go-binary",
FilepathPatterns: []*regexp.Regexp{
regexp.MustCompile(`(.*/|^)go$`),
},
EvidencePatternTemplates: []string{
`(?m)go(?P<version>[0-9]+\.[0-9]+(\.[0-9]+|beta[0-9]+|alpha[0-9]+|rc[0-9]+)?)`,
},
},
{
Class: "nodejs-binary",
FilepathPatterns: []*regexp.Regexp{
regexp.MustCompile(`(.*/|^)node$`),
},
EvidencePatternTemplates: []string{
// regex that matches node.js/vx.y.z
`(?m)node\.js\/v(?P<version>[0-9]+\.[0-9]+\.[0-9]+)`,
},
},
{
Class: "go-binary-hint",
FilepathPatterns: []*regexp.Regexp{
regexp.MustCompile(`(.*/|^)VERSION$`),
},
EvidencePatternTemplates: []string{
`(?m)go(?P<version>[0-9]+\.[0-9]+(\.[0-9]+|beta[0-9]+|alpha[0-9]+|rc[0-9]+)?)`,
},
},
{
Class: "busybox-binary",
FilepathPatterns: []*regexp.Regexp{
regexp.MustCompile(`(.*/|^)busybox$`),
},
EvidencePatternTemplates: []string{
`(?m)BusyBox\s+v(?P<version>[0-9]+\.[0-9]+\.[0-9]+)`,
},
},
}
type Classifier struct {
Class string
FilepathPatterns []*regexp.Regexp
EvidencePatternTemplates []string
}
type Classification struct {
Class string `json:"class"`
VirtualPath string `json:"virtual_path"`
Metadata map[string]string `json:"metadata"`
}
func (c Classifier) Classify(resolver source.FileResolver, location source.Location) (*Classification, error) {
doesFilepathMatch, filepathNamedGroupValues := FilepathMatches(c.FilepathPatterns, location)
if !doesFilepathMatch {
return nil, nil
}
contentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
return nil, err
}
defer internal.CloseAndLogError(contentReader, location.VirtualPath)
// TODO: there is room for improvement here, as this may use an excessive amount of memory. Alternate approach is to leverage a RuneReader.
contents, err := io.ReadAll(contentReader)
if err != nil {
return nil, err
}
var result *Classification
for _, patternTemplate := range c.EvidencePatternTemplates {
tmpl, err := template.New("").Parse(patternTemplate)
if err != nil {
return nil, fmt.Errorf("unable to parse classifier template=%q : %w", patternTemplate, err)
}
patternBuf := &bytes.Buffer{}
err = tmpl.Execute(patternBuf, filepathNamedGroupValues)
if err != nil {
return nil, fmt.Errorf("unable to render template: %w", err)
}
pattern, err := regexp.Compile(patternBuf.String())
if err != nil {
return nil, fmt.Errorf("unable to compile rendered regex=%q: %w", patternBuf.String(), err)
}
if !pattern.Match(contents) {
continue
}
matchMetadata := internal.MatchNamedCaptureGroups(pattern, string(contents))
if result == nil {
result = &Classification{
Class: c.Class,
VirtualPath: location.VirtualPath,
Metadata: matchMetadata,
}
} else {
for key, value := range matchMetadata {
result.Metadata[key] = value
}
}
}
return result, nil
}
func FilepathMatches(patterns []*regexp.Regexp, location source.Location) (bool, map[string]string) {
for _, path := range []string{location.RealPath, location.VirtualPath} {
if path == "" {
continue
}
for _, pattern := range patterns {
if pattern.MatchString(path) {
return true, internal.MatchNamedCaptureGroups(pattern, path)
}
}
}
return false, nil
}
-97
View File
@@ -1,97 +0,0 @@
package file
import (
"regexp"
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/source"
)
func TestFilepathMatches(t *testing.T) {
tests := []struct {
name string
location source.Location
patterns []string
expectedMatches bool
expectedNamedGroups map[string]string
}{
{
name: "simple-filename-match",
location: source.Location{
Coordinates: source.Coordinates{
RealPath: "python2.7",
},
},
patterns: []string{
`python([0-9]+\.[0-9]+)$`,
},
expectedMatches: true,
},
{
name: "filepath-match",
location: source.Location{
Coordinates: source.Coordinates{
RealPath: "/usr/bin/python2.7",
},
},
patterns: []string{
`python([0-9]+\.[0-9]+)$`,
},
expectedMatches: true,
},
{
name: "virtual-filepath-match",
location: source.Location{
VirtualPath: "/usr/bin/python2.7",
},
patterns: []string{
`python([0-9]+\.[0-9]+)$`,
},
expectedMatches: true,
},
{
name: "full-filepath-match",
location: source.Location{
VirtualPath: "/usr/bin/python2.7",
},
patterns: []string{
`.*/bin/python([0-9]+\.[0-9]+)$`,
},
expectedMatches: true,
},
{
name: "anchored-filename-match-FAILS",
location: source.Location{
Coordinates: source.Coordinates{
RealPath: "/usr/bin/python2.7",
},
},
patterns: []string{
`^python([0-9]+\.[0-9]+)$`,
},
expectedMatches: false,
},
{
name: "empty-filename-match-FAILS",
location: source.Location{},
patterns: []string{
`^python([0-9]+\.[0-9]+)$`,
},
expectedMatches: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var patterns []*regexp.Regexp
for _, p := range test.patterns {
patterns = append(patterns, regexp.MustCompile(p))
}
actualMatches, actualNamedGroups := FilepathMatches(patterns, test.location)
assert.Equal(t, test.expectedMatches, actualMatches)
assert.Equal(t, test.expectedNamedGroups, actualNamedGroups)
})
}
}
@@ -1 +0,0 @@
go1.14
@@ -216,7 +216,7 @@ func collectRelationships(bom *cyclonedx.BOM, s *sbom.SBOM, idMap map[string]int
}
for _, t := range *d.Dependencies {
to, toExists := idMap[t.Ref].(artifact.Identifiable)
to, toExists := idMap[t].(artifact.Identifiable)
if !toExists {
continue
}
@@ -173,12 +173,8 @@ func Test_decode(t *testing.T) {
},
Dependencies: &[]cyclonedx.Dependency{
{
Ref: "p1",
Dependencies: &[]cyclonedx.Dependency{
{
Ref: "p2",
},
},
Ref: "p1",
Dependencies: &[]string{"p2"},
},
},
},
@@ -263,8 +259,9 @@ func Test_decode(t *testing.T) {
func Test_missingDataDecode(t *testing.T) {
bom := &cyclonedx.BOM{
Metadata: nil,
Components: &[]cyclonedx.Component{},
Metadata: nil,
Components: &[]cyclonedx.Component{},
SpecVersion: cyclonedx.SpecVersion1_4,
}
_, err := ToSyftModel(bom)
@@ -287,7 +284,9 @@ func Test_missingDataDecode(t *testing.T) {
}
func Test_missingComponentsDecode(t *testing.T) {
bom := &cyclonedx.BOM{}
bom := &cyclonedx.BOM{
SpecVersion: cyclonedx.SpecVersion1_4,
}
bomBytes, _ := json.Marshal(&bom)
decode := GetDecoder(cyclonedx.BOMFileFormatJSON)
@@ -157,8 +157,10 @@ func toDependencies(relationships []artifact.Relationship) []cyclonedx.Dependenc
continue
}
innerDeps := []cyclonedx.Dependency{}
innerDeps = append(innerDeps, cyclonedx.Dependency{Ref: deriveBomRef(*fromPkg)})
// ind dep
innerDeps := []string{}
innerDeps = append(innerDeps, deriveBomRef(*fromPkg))
result = append(result, cyclonedx.Dependency{
Ref: deriveBomRef(*toPkg),
Dependencies: &innerDeps,
+5 -6
View File
@@ -6,12 +6,11 @@ import (
)
type File struct {
ID string `json:"id"`
Location source.Coordinates `json:"location"`
Metadata *FileMetadataEntry `json:"metadata,omitempty"`
Contents string `json:"contents,omitempty"`
Digests []file.Digest `json:"digests,omitempty"`
Classifications []file.Classification `json:"classifications,omitempty"`
ID string `json:"id"`
Location source.Coordinates `json:"location"`
Metadata *FileMetadataEntry `json:"metadata,omitempty"`
Contents string `json:"contents,omitempty"`
Digests []file.Digest `json:"digests,omitempty"`
}
type FileMetadataEntry struct {
+5 -11
View File
@@ -102,23 +102,17 @@ func toFile(s sbom.SBOM) []model.File {
digests = digestsForLocation
}
var classifications []file.Classification
if classificationsForLocation, exists := artifacts.FileClassifications[coordinates]; exists {
classifications = classificationsForLocation
}
var contents string
if contentsForLocation, exists := artifacts.FileContents[coordinates]; exists {
contents = contentsForLocation
}
results = append(results, model.File{
ID: string(coordinates.ID()),
Location: coordinates,
Metadata: toFileMetadataEntry(coordinates, metadata),
Digests: digests,
Classifications: classifications,
Contents: contents,
ID: string(coordinates.ID()),
Location: coordinates,
Metadata: toFileMetadataEntry(coordinates, metadata),
Digests: digests,
Contents: contents,
})
}
+83 -19
View File
@@ -26,6 +26,15 @@ loopNewIDs:
}
}
func (s *orderedIDSet) delete(id artifact.ID) {
for i, existingID := range s.slice {
if existingID == id {
s.slice = append(s.slice[:i], s.slice[i+1:]...)
return
}
}
}
// Catalog represents a collection of Packages.
type Catalog struct {
byID map[artifact.ID]Package
@@ -92,30 +101,32 @@ func (c *Catalog) Packages(ids []artifact.ID) (result []Package) {
return result
}
// Add a package to the Catalog.
func (c *Catalog) Add(p Package) {
// Add n packages to the catalog.
func (c *Catalog) Add(pkgs ...Package) {
c.lock.Lock()
defer c.lock.Unlock()
id := p.ID()
if id == "" {
log.Warnf("found package with empty ID while adding to the catalog: %+v", p)
p.SetID()
id = p.ID()
}
if existing, exists := c.byID[id]; exists {
// there is already a package with this fingerprint merge the existing record with the new one
if err := existing.merge(p); err != nil {
log.Warnf("failed to merge packages: %+v", err)
} else {
c.byID[id] = existing
c.addPathsToIndex(p)
for _, p := range pkgs {
id := p.ID()
if id == "" {
log.Warnf("found package with empty ID while adding to the catalog: %+v", p)
p.SetID()
id = p.ID()
}
return
}
c.addToIndex(p)
if existing, exists := c.byID[id]; exists {
// there is already a package with this fingerprint merge the existing record with the new one
if err := existing.merge(p); err != nil {
log.Warnf("failed to merge packages: %+v", err)
} else {
c.byID[id] = existing
c.addPathsToIndex(p)
}
return
}
c.addToIndex(p)
}
}
func (c *Catalog) addToIndex(p Package) {
@@ -157,6 +168,59 @@ func (c *Catalog) addPathToIndex(id artifact.ID, path string) {
c.idsByPath[path] = pathIndex
}
func (c *Catalog) Delete(ids ...artifact.ID) {
c.lock.Lock()
defer c.lock.Unlock()
for _, id := range ids {
p, exists := c.byID[id]
if !exists {
return
}
delete(c.byID, id)
c.deleteNameFromIndex(p)
c.deleteTypeFromIndex(p)
c.deletePathsFromIndex(p)
}
}
func (c *Catalog) deleteNameFromIndex(p Package) {
nameIndex := c.idsByName[p.Name]
nameIndex.delete(p.id)
c.idsByName[p.Name] = nameIndex
}
func (c *Catalog) deleteTypeFromIndex(p Package) {
typeIndex := c.idsByType[p.Type]
typeIndex.delete(p.id)
c.idsByType[p.Type] = typeIndex
}
func (c *Catalog) deletePathsFromIndex(p Package) {
observedPaths := internal.NewStringSet()
for _, l := range p.Locations.ToSlice() {
if l.RealPath != "" && !observedPaths.Contains(l.RealPath) {
c.deletePathFromIndex(p.id, l.RealPath)
observedPaths.Add(l.RealPath)
}
if l.VirtualPath != "" && l.RealPath != l.VirtualPath && !observedPaths.Contains(l.VirtualPath) {
c.deletePathFromIndex(p.id, l.VirtualPath)
observedPaths.Add(l.VirtualPath)
}
}
}
func (c *Catalog) deletePathFromIndex(id artifact.ID, path string) {
pathIndex := c.idsByPath[path]
pathIndex.delete(id)
if len(pathIndex.slice) == 0 {
delete(c.idsByPath, path)
} else {
c.idsByPath[path] = pathIndex
}
}
// Enumerate all packages for the given type(s), enumerating all packages if no type is specified.
func (c *Catalog) Enumerate(types ...Type) <-chan Package {
channel := make(chan Package)
+142
View File
@@ -16,6 +16,148 @@ type expectedIndexes struct {
byPath map[string]*strset.Set
}
func TestCatalogDeleteRemovesPackages(t *testing.T) {
tests := []struct {
name string
pkgs []Package
deleteIDs []artifact.ID
expectedIndexes expectedIndexes
}{
{
name: "delete one package",
pkgs: []Package{
{
id: "pkg:deb/debian/1",
Name: "debian",
Version: "1",
Type: DebPkg,
Locations: source.NewLocationSet(
source.NewVirtualLocation("/c/path", "/another/path1"),
),
},
{
id: "pkg:deb/debian/2",
Name: "debian",
Version: "2",
Type: DebPkg,
Locations: source.NewLocationSet(
source.NewVirtualLocation("/d/path", "/another/path2"),
),
},
},
deleteIDs: []artifact.ID{
artifact.ID("pkg:deb/debian/1"),
},
expectedIndexes: expectedIndexes{
byType: map[Type]*strset.Set{
DebPkg: strset.New("pkg:deb/debian/2"),
},
byPath: map[string]*strset.Set{
"/d/path": strset.New("pkg:deb/debian/2"),
"/another/path2": strset.New("pkg:deb/debian/2"),
},
},
},
{
name: "delete multiple packages",
pkgs: []Package{
{
id: "pkg:deb/debian/1",
Name: "debian",
Version: "1",
Type: DebPkg,
Locations: source.NewLocationSet(
source.NewVirtualLocation("/c/path", "/another/path1"),
),
},
{
id: "pkg:deb/debian/2",
Name: "debian",
Version: "2",
Type: DebPkg,
Locations: source.NewLocationSet(
source.NewVirtualLocation("/d/path", "/another/path2"),
),
},
{
id: "pkg:deb/debian/3",
Name: "debian",
Version: "3",
Type: DebPkg,
Locations: source.NewLocationSet(
source.NewVirtualLocation("/e/path", "/another/path3"),
),
},
},
deleteIDs: []artifact.ID{
artifact.ID("pkg:deb/debian/1"),
artifact.ID("pkg:deb/debian/3"),
},
expectedIndexes: expectedIndexes{
byType: map[Type]*strset.Set{
DebPkg: strset.New("pkg:deb/debian/2"),
},
byPath: map[string]*strset.Set{
"/d/path": strset.New("pkg:deb/debian/2"),
"/another/path2": strset.New("pkg:deb/debian/2"),
},
},
},
{
name: "delete non-existent package",
pkgs: []Package{
{
id: artifact.ID("pkg:deb/debian/1"),
Name: "debian",
Version: "1",
Type: DebPkg,
Locations: source.NewLocationSet(
source.NewVirtualLocation("/c/path", "/another/path1"),
),
},
{
id: artifact.ID("pkg:deb/debian/2"),
Name: "debian",
Version: "2",
Type: DebPkg,
Locations: source.NewLocationSet(
source.NewVirtualLocation("/d/path", "/another/path2"),
),
},
},
deleteIDs: []artifact.ID{
artifact.ID("pkg:deb/debian/3"),
},
expectedIndexes: expectedIndexes{
byType: map[Type]*strset.Set{
DebPkg: strset.New("pkg:deb/debian/1", "pkg:deb/debian/2"),
},
byPath: map[string]*strset.Set{
"/c/path": strset.New("pkg:deb/debian/1"),
"/another/path1": strset.New("pkg:deb/debian/1"),
"/d/path": strset.New("pkg:deb/debian/2"),
"/another/path2": strset.New("pkg:deb/debian/2"),
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
c := NewCatalog()
for _, p := range test.pkgs {
c.Add(p)
}
for _, id := range test.deleteIDs {
c.Delete(id)
}
assertIndexes(t, c, test.expectedIndexes)
})
}
}
func TestCatalogAddPopulatesIndex(t *testing.T) {
var pkgs = []Package{
+231 -125
View File
@@ -3,13 +3,10 @@ package apkdb
import (
"bufio"
"fmt"
"io"
"path"
"strconv"
"strings"
"github.com/mitchellh/mapstructure"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
@@ -22,28 +19,73 @@ import (
// integrity check
var _ generic.Parser = parseApkDB
// parseApkDb parses individual packages from a given Alpine DB file. For more information on specific fields
// see https://wiki.alpinelinux.org/wiki/Apk_spec .
// parseApkDB parses packages from a given APK installed DB file. For more
// information on specific fields, see https://wiki.alpinelinux.org/wiki/Apk_spec.
//
//nolint:funlen
func parseApkDB(_ source.FileResolver, env *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
// larger capacity for the scanner.
const maxScannerCapacity = 1024 * 1024
// a new larger buffer for the scanner
bufScan := make([]byte, maxScannerCapacity)
pkgs := make([]pkg.Package, 0)
scanner := bufio.NewScanner(reader)
scanner.Buffer(bufScan, maxScannerCapacity)
onDoubleLF := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
for i := 0; i < len(data); i++ {
if i > 0 && data[i-1] == '\n' && data[i] == '\n' {
return i + 1, data[:i-1], nil
var apks []pkg.ApkMetadata
var currentEntry pkg.ApkMetadata
entryParsingInProgress := false
fileParsingCtx := newApkFileParsingContext()
// creating a dedicated append-like function here instead of using `append(...)`
// below since there is nontrivial logic to be performed for each finalized apk
// entry.
appendApk := func(p pkg.ApkMetadata) {
if files := fileParsingCtx.files; len(files) >= 1 {
// attached accumulated files to current package
p.Files = files
// reset file parsing for next use
fileParsingCtx = newApkFileParsingContext()
}
nilFieldsToEmptySlice(&p)
apks = append(apks, p)
}
for scanner.Scan() {
line := scanner.Text()
if line == "" {
// i.e. apk entry separator
if entryParsingInProgress {
// current entry is complete
appendApk(currentEntry)
}
entryParsingInProgress = false
// zero-out currentEntry for use by any future entry
currentEntry = pkg.ApkMetadata{}
continue
}
if !atEOF {
return 0, nil, nil
field := parseApkField(line)
if field == nil {
log.Warnf("unable to parse field data from line %q", line)
continue
}
// deliver the last token (which could be an empty string)
return 0, data, bufio.ErrFinalToken
entryParsingInProgress = true
field.apply(&currentEntry, fileParsingCtx)
}
if entryParsingInProgress {
// There was no final empty line, so currentEntry hasn't been added to the
// collection yet; but we've now reached the end of scanning, so let's be sure to
// add currentEntry to the collection.
appendApk(currentEntry)
}
if err := scanner.Err(); err != nil {
return nil, nil, fmt.Errorf("failed to parse APK installed DB file: %w", err)
}
var r *linux.Release
@@ -51,128 +93,192 @@ func parseApkDB(_ source.FileResolver, env *generic.Environment, reader source.L
r = env.LinuxRelease
}
scanner.Split(onDoubleLF)
for scanner.Scan() {
metadata, err := parseApkDBEntry(strings.NewReader(scanner.Text()))
if err != nil {
return nil, nil, err
}
if metadata != nil {
pkgs = append(pkgs, newPackage(*metadata, r, reader.Location))
}
}
if err := scanner.Err(); err != nil {
return nil, nil, fmt.Errorf("failed to parse APK DB file: %w", err)
pkgs := make([]pkg.Package, 0, len(apks))
for _, apk := range apks {
pkgs = append(pkgs, newPackage(apk, r, reader.Location))
}
return pkgs, discoverPackageDependencies(pkgs), nil
}
// parseApkDBEntry reads and parses a single pkg.ApkMetadata element from the stream, returning nil if their are no more entries.
//
func parseApkField(line string) *apkField {
parts := strings.SplitN(line, ":", 2)
if len(parts) != 2 {
return nil
}
f := apkField{
name: parts[0],
value: parts[1],
}
return &f
}
type apkField struct {
name string
value string
}
//nolint:funlen
func parseApkDBEntry(reader io.Reader) (*pkg.ApkMetadata, error) {
var entry pkg.ApkMetadata
pkgFields := make(map[string]interface{})
func (f apkField) apply(p *pkg.ApkMetadata, ctx *apkFileParsingContext) {
switch f.name {
// APKINDEX field parsing
// We want sane defaults for collections, i.e. an empty array instead of null.
pkgFields["D"] = []string{}
pkgFields["p"] = []string{}
files := make([]pkg.ApkFileRecord, 0)
var fileRecord *pkg.ApkFileRecord
lastFile := "/"
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
line := scanner.Text()
fields := strings.SplitN(line, ":", 2)
if len(fields) != 2 {
continue
case "P":
p.Package = f.value
case "o":
p.OriginPackage = f.value
case "m":
p.Maintainer = f.value
case "V":
p.Version = f.value
case "L":
p.License = f.value
case "A":
p.Architecture = f.value
case "U":
p.URL = f.value
case "T":
p.Description = f.value
case "S":
i, err := strconv.Atoi(f.value)
if err != nil {
log.Warnf("unable to parse value %q for field %q: %w", f.value, f.name, err)
return
}
key := fields[0]
value := strings.TrimSpace(fields[1])
switch key {
case "D", "p":
entries := strings.Split(value, " ")
pkgFields[key] = entries
case "F":
currentFile := "/" + value
newFileRecord := pkg.ApkFileRecord{
Path: currentFile,
}
files = append(files, newFileRecord)
fileRecord = &files[len(files)-1]
// future aux references are relative to previous "F" records
lastFile = currentFile
continue
case "R":
newFileRecord := pkg.ApkFileRecord{
Path: path.Join(lastFile, value),
}
files = append(files, newFileRecord)
fileRecord = &files[len(files)-1]
case "a", "M":
ownershipFields := strings.Split(value, ":")
if len(ownershipFields) < 3 {
log.Warnf("unexpected APK ownership field: %q", value)
continue
}
if fileRecord == nil {
log.Warnf("ownership field with no parent record: %q", value)
continue
}
fileRecord.OwnerUID = ownershipFields[0]
fileRecord.OwnerGID = ownershipFields[1]
fileRecord.Permissions = ownershipFields[2]
// note: there are more optional fields available that we are not capturing, e.g.:
// "0:0:755:Q1JaDEHQHBbizhEzoWK1YxuraNU/4="
case "Z":
if fileRecord == nil {
log.Warnf("checksum field with no parent record: %q", value)
continue
}
fileRecord.Digest = processChecksum(value)
case "I", "S":
// coerce to integer
iVal, err := strconv.Atoi(value)
if err != nil {
return nil, fmt.Errorf("failed to parse APK int: '%+v'", value)
}
pkgFields[key] = iVal
default:
pkgFields[key] = value
p.Size = i
case "I":
i, err := strconv.Atoi(f.value)
if err != nil {
log.Warnf("unable to parse value %q for field %q: %w", f.value, f.name, err)
return
}
p.InstalledSize = i
case "D":
deps := parseListValue(f.value)
p.Dependencies = deps
case "p":
provides := parseListValue(f.value)
p.Provides = provides
case "C":
p.Checksum = f.value
case "c":
p.GitCommit = f.value
// File/directory field parsing:
case "F":
directory := path.Join("/", f.value)
ctx.files = append(ctx.files, pkg.ApkFileRecord{Path: directory})
ctx.indexOfLatestDirectory = len(ctx.files) - 1
case "M":
i := ctx.indexOfLatestDirectory
latest := ctx.files[i]
var ok bool
latest.OwnerUID, latest.OwnerGID, latest.Permissions, ok = processFileInfo(f.value)
if !ok {
log.Warnf("unexpected value for APK ACL field %q: %q", f.name, f.value)
return
}
// save updated directory
ctx.files[i] = latest
case "R":
var regularFile string
dirIndex := ctx.indexOfLatestDirectory
if dirIndex < 0 {
regularFile = path.Join("/", f.value)
} else {
latestDirPath := ctx.files[dirIndex].Path
regularFile = path.Join(latestDirPath, f.value)
}
ctx.files = append(ctx.files, pkg.ApkFileRecord{Path: regularFile})
ctx.indexOfLatestRegularFile = len(ctx.files) - 1
case "a":
i := ctx.indexOfLatestRegularFile
latest := ctx.files[i]
var ok bool
latest.OwnerUID, latest.OwnerGID, latest.Permissions, ok = processFileInfo(f.value)
if !ok {
log.Warnf("unexpected value for APK ACL field %q: %q", f.name, f.value)
return
}
// save updated file
ctx.files[i] = latest
case "Z":
i := ctx.indexOfLatestRegularFile
latest := ctx.files[i]
latest.Digest = processChecksum(f.value)
// save updated file
ctx.files[i] = latest
}
}
func processFileInfo(v string) (uid, gid, perms string, ok bool) {
ok = false
fileInfo := strings.Split(v, ":")
if len(fileInfo) < 3 {
return
}
decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
// By default, mapstructure compares field names in a *case-insensitive* manner.
// That would be the wrong approach here, since these apk files use case
// *sensitive* field names (e.g. 'P' vs. 'p').
MatchName: func(mapKey, fieldName string) bool {
return mapKey == fieldName
},
Result: &entry,
})
if err != nil {
return nil, err
uid = fileInfo[0]
gid = fileInfo[1]
perms = fileInfo[2]
// note: there are more optional fields available that we are not capturing,
// e.g.: "0:0:755:Q1JaDEHQHBbizhEzoWK1YxuraNU/4="
ok = true
return
}
// apkFileParsingContext helps keep track of what file data has been captured so far for the APK currently being parsed.
type apkFileParsingContext struct {
files []pkg.ApkFileRecord
indexOfLatestDirectory int
indexOfLatestRegularFile int
}
func newApkFileParsingContext() *apkFileParsingContext {
return &apkFileParsingContext{
indexOfLatestDirectory: -1, // no directories yet
indexOfLatestRegularFile: -1, // no regular files yet
}
}
// parseListValue parses a space-separated list from an apk entry field value.
func parseListValue(value string) []string {
items := strings.Split(value, " ")
if len(items) >= 1 {
return items
}
if err := decoder.Decode(pkgFields); err != nil {
return nil, fmt.Errorf("unable to parse APK metadata: %w", err)
}
if entry.Package == "" {
return nil, nil
return nil
}
func nilFieldsToEmptySlice(p *pkg.ApkMetadata) {
if p.Dependencies == nil {
p.Dependencies = []string{}
}
entry.Files = files
if p.Provides == nil {
p.Provides = []string{}
}
return &entry, nil
if p.Files == nil {
p.Files = []pkg.ApkFileRecord{}
}
}
func processChecksum(value string) *file.Digest {
+108 -26
View File
@@ -1,8 +1,8 @@
package apkdb
import (
"bufio"
"os"
"path/filepath"
"testing"
"github.com/google/go-cmp/cmp"
@@ -60,16 +60,15 @@ func TestExtraFileAttributes(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
f, err := os.Open("test-fixtures/extra-file-attributes")
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, f.Close()) })
fixturePath := "test-fixtures/extra-file-attributes"
lrc := newLocationReadCloser(t, fixturePath)
reader := bufio.NewReader(f)
pkgs, _, err := parseApkDB(nil, new(generic.Environment), lrc)
assert.NoError(t, err)
require.Len(t, pkgs, 1)
metadata := pkgs[0].Metadata.(pkg.ApkMetadata)
entry, err := parseApkDBEntry(reader)
require.NoError(t, err)
if diff := cmp.Diff(entry.Files, test.expected.Files); diff != "" {
if diff := cmp.Diff(test.expected.Files, metadata.Files); diff != "" {
t.Errorf("Files mismatch (-want +got):\n%s", diff)
}
})
@@ -615,17 +614,14 @@ func TestSinglePackageDetails(t *testing.T) {
for _, test := range tests {
t.Run(test.fixture, func(t *testing.T) {
f, err := os.Open(test.fixture)
lrc := newLocationReadCloser(t, test.fixture)
pkgs, _, err := parseApkDB(nil, new(generic.Environment), lrc)
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, f.Close()) })
require.Len(t, pkgs, 1)
metadata := pkgs[0].Metadata.(pkg.ApkMetadata)
reader := bufio.NewReader(f)
entry, err := parseApkDBEntry(reader)
require.NoError(t, err)
require.NotNil(t, entry)
if diff := cmp.Diff(*entry, test.expected); diff != "" {
if diff := cmp.Diff(test.expected, metadata); diff != "" {
t.Errorf("Entry mismatch (-want +got):\n%s", diff)
}
})
@@ -765,7 +761,6 @@ func TestMultiplePackages(t *testing.T) {
}}
pkgtest.TestFileParserWithEnv(t, fixture, parseApkDB, &env, expectedPkgs, expectedRelationships)
}
func Test_processChecksum(t *testing.T) {
@@ -791,15 +786,15 @@ func Test_processChecksum(t *testing.T) {
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, &tt.want, processChecksum(tt.value))
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, &test.want, processChecksum(test.value))
})
}
}
func Test_discoverPackageDependencies(t *testing.T) {
tests := []struct {
name string
genFn func() ([]pkg.Package, []artifact.Relationship)
@@ -934,9 +929,10 @@ func Test_discoverPackageDependencies(t *testing.T) {
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pkgs, wantRelationships := tt.genFn()
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
pkgs, wantRelationships := test.genFn()
gotRelationships := discoverPackageDependencies(pkgs)
d := cmp.Diff(wantRelationships, gotRelationships, cmpopts.IgnoreUnexported(pkg.Package{}, source.LocationSet{}))
if d != "" {
@@ -1007,3 +1003,89 @@ func TestPackageDbDependenciesByParse(t *testing.T) {
})
}
}
func Test_parseApkDB_expectedPkgNames(t *testing.T) {
tests := []struct {
fixture string
wantPkgNames []string
wantErr assert.ErrorAssertionFunc
}{
{
fixture: "very-large-entries",
wantPkgNames: []string{
"ca-certificates-bundle",
"glibc-locale-posix",
"wolfi-baselayout",
"glibc",
"libcrypto3",
"libssl3",
"zlib",
"apk-tools",
"ncurses-terminfo-base",
"ncurses",
"bash",
"libcap",
"bubblewrap",
"busybox",
"libbrotlicommon1",
"libbrotlidec1",
"libnghttp2-14",
"libcurl4",
"curl",
"expat",
"libpcre2-8-0",
"git",
"binutils",
"libstdc++-dev",
"libgcc",
"libstdc++",
"gmp",
"isl",
"mpfr",
"mpc",
"gcc",
"linux-headers",
"glibc-dev",
"make",
"pkgconf",
"build-base",
"go",
"tree",
"sdk",
},
wantErr: assert.NoError,
},
}
for _, test := range tests {
t.Run(test.fixture, func(t *testing.T) {
fixturePath := filepath.Join("test-fixtures", test.fixture)
lrc := newLocationReadCloser(t, fixturePath)
pkgs, _, err := parseApkDB(nil, new(generic.Environment), lrc)
test.wantErr(t, err)
names := toPackageNames(pkgs)
if diff := cmp.Diff(test.wantPkgNames, names); diff != "" {
t.Errorf("Packages mismatch (-want +got):\n%s", diff)
}
})
}
}
func toPackageNames(pkgs []pkg.Package) []string {
names := make([]string, 0, len(pkgs))
for _, p := range pkgs {
names = append(names, p.Name)
}
return names
}
func newLocationReadCloser(t *testing.T, path string) source.LocationReadCloser {
f, err := os.Open(path)
require.NoError(t, err)
t.Cleanup(func() { f.Close() })
return source.NewLocationReadCloser(source.NewLocation(path), f)
}
File diff suppressed because it is too large Load Diff
+78
View File
@@ -0,0 +1,78 @@
package binary
import (
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
const catalogerName = "binary-cataloger"
func NewCataloger() *Cataloger {
return &Cataloger{}
}
// Cataloger is the cataloger responsible for surfacing evidence of a very limited set of binary files,
// which have been identified by the classifiers. The Cataloger is _NOT_ a place to catalog any and every
// binary, but rather the specific set that has been curated to be important, predominantly related to toolchain-
// related runtimes like Python, Go, Java, or Node. Some exceptions can be made for widely-used binaries such
// as busybox.
type Cataloger struct{}
// Name returns a string that uniquely describes the Cataloger
func (c Cataloger) Name() string {
return catalogerName
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages
// after analyzing the catalog source.
func (c Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
var packages []pkg.Package
var relationships []artifact.Relationship
for _, classifier := range defaultClassifiers {
locations, err := resolver.FilesByGlob(classifier.FileGlob)
if err != nil {
return nil, nil, err
}
for _, location := range locations {
reader, err := resolver.FileContentsByLocation(location)
if err != nil {
return nil, nil, err
}
locationReader := source.NewLocationReadCloser(location, reader)
newPkgs, err := classifier.EvidenceMatcher(classifier, locationReader)
if err != nil {
return nil, nil, err
}
newPackages:
for i := range newPkgs {
newPkg := &newPkgs[i]
for j := range packages {
p := &packages[j]
// consolidate identical packages found in different locations,
// but continue to track each location
if packagesMatch(p, newPkg) {
p.Locations.Add(newPkg.Locations.ToSlice()...)
continue newPackages
}
}
packages = append(packages, *newPkg)
}
}
}
return packages, relationships, nil
}
// packagesMatch returns true if the binary packages "match" based on basic criteria
func packagesMatch(p1 *pkg.Package, p2 *pkg.Package) bool {
if p1.Name != p2.Name ||
p1.Version != p2.Version ||
p1.Language != p2.Language ||
p1.Type != p2.Type {
return false
}
return true
}
+269
View File
@@ -0,0 +1,269 @@
package binary
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
tests := []struct {
name string
fixtureDir string
expected pkg.Package
}{
{
name: "positive-libpython3.7.so",
fixtureDir: "test-fixtures/classifiers/positive",
expected: pkg.Package{
Name: "python",
Version: "3.7.4a-vZ9",
Locations: singleLocation("libpython3.7.so"),
Metadata: pkg.BinaryMetadata{
Classifier: "python-binary-lib",
},
},
},
{
name: "positive-python3.6",
fixtureDir: "test-fixtures/classifiers/positive",
expected: pkg.Package{
Name: "python",
Version: "3.6.3a-vZ9",
Locations: singleLocation("python3.6"),
Metadata: pkg.BinaryMetadata{
Classifier: "python-binary",
},
},
},
{
name: "positive-patchlevel.h",
fixtureDir: "test-fixtures/classifiers/positive",
expected: pkg.Package{
Name: "python",
Version: "3.9-aZ5",
Locations: singleLocation("patchlevel.h"),
Metadata: pkg.BinaryMetadata{
Classifier: "cpython-source",
},
},
},
{
name: "positive-go",
fixtureDir: "test-fixtures/classifiers/positive",
expected: pkg.Package{
Name: "go",
Version: "1.14",
Locations: singleLocation("go"),
Metadata: pkg.BinaryMetadata{
Classifier: "go-binary",
},
},
},
{
name: "positive-node",
fixtureDir: "test-fixtures/classifiers/positive",
expected: pkg.Package{
Name: "node",
Version: "19.2.1",
Locations: singleLocation("node"),
Metadata: pkg.BinaryMetadata{
Classifier: "nodejs-binary",
},
},
},
{
name: "positive-go-hint",
fixtureDir: "test-fixtures/classifiers/positive",
expected: pkg.Package{
Name: "go",
Version: "1.15",
Locations: singleLocation("VERSION"),
Metadata: pkg.BinaryMetadata{
Classifier: "go-binary-hint",
},
},
},
{
name: "positive-busybox",
fixtureDir: "test-fixtures/classifiers/positive",
expected: pkg.Package{
Name: "busybox",
Version: "3.33.3",
Locations: singleLocation("["), // note: busybox is a link to [
Metadata: pkg.BinaryMetadata{
Classifier: "busybox-binary",
VirtualPath: "busybox",
},
},
},
{
name: "positive-java-openjdk",
fixtureDir: "test-fixtures/classifiers/positive/openjdk",
expected: pkg.Package{
Name: "java",
Version: "1.8.0_352-b08",
Type: "binary",
Locations: singleLocation("java"),
Metadata: pkg.BinaryMetadata{
Classifier: "java-binary-openjdk",
VirtualPath: "java",
},
},
},
{
name: "positive-java-oracle",
fixtureDir: "test-fixtures/classifiers/positive/oracle",
expected: pkg.Package{
Name: "java",
Version: "19.0.1+10-21",
Type: "binary",
Locations: singleLocation("java"),
Metadata: pkg.BinaryMetadata{
Classifier: "java-binary-oracle",
VirtualPath: "java",
},
},
},
{
name: "positive-java-oracle-macos",
fixtureDir: "test-fixtures/classifiers/positive/oracle-macos",
expected: pkg.Package{
Name: "java",
Version: "19.0.1+10-21",
Type: "binary",
Locations: singleLocation("java"),
Metadata: pkg.BinaryMetadata{
Classifier: "java-binary-oracle",
VirtualPath: "java",
},
},
},
{
name: "positive-java-ibm",
fixtureDir: "test-fixtures/classifiers/positive/ibm",
expected: pkg.Package{
Name: "java",
Version: "1.8.0-foreman_2022_09_22_15_30-b00",
Type: "binary",
Locations: singleLocation("java"),
Metadata: pkg.BinaryMetadata{
Classifier: "java-binary-ibm",
VirtualPath: "java",
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
c := NewCataloger()
src, err := source.NewFromDirectory(test.fixtureDir)
require.NoError(t, err)
resolver, err := src.FileResolver(source.SquashedScope)
require.NoError(t, err)
packages, _, err := c.Catalog(resolver)
require.NoError(t, err)
ok := false
for _, p := range packages {
if test.expected.Locations.ToSlice()[0].RealPath == p.Locations.ToSlice()[0].RealPath {
ok = true
assertPackagesAreEqual(t, test.expected, p)
}
}
if !ok {
t.Fatalf("could not find test location=%q", test.expected.Locations.ToSlice()[0].RealPath)
}
})
}
}
func TestClassifierCataloger_DefaultClassifiers_PositiveCases_Image(t *testing.T) {
tests := []struct {
name string
fixtureImage string
expected pkg.Package
}{
{
name: "busybox-regression",
fixtureImage: "image-busybox",
expected: pkg.Package{
Name: "busybox",
Version: "1.35.0",
Locations: singleLocation("/bin/["),
Metadata: pkg.BinaryMetadata{
Classifier: "busybox-binary",
VirtualPath: "/bin/busybox",
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
c := NewCataloger()
img := imagetest.GetFixtureImage(t, "docker-archive", test.fixtureImage)
src, err := source.NewFromImage(img, "test-img")
require.NoError(t, err)
resolver, err := src.FileResolver(source.SquashedScope)
require.NoError(t, err)
packages, _, err := c.Catalog(resolver)
require.NoError(t, err)
ok := false
for _, p := range packages {
if test.expected.Locations.ToSlice()[0].RealPath == p.Locations.ToSlice()[0].RealPath {
ok = true
assertPackagesAreEqual(t, test.expected, p)
}
}
if !ok {
t.Fatalf("could not find test location=%q", test.expected.Locations.ToSlice()[0].RealPath)
}
})
}
}
func TestClassifierCataloger_DefaultClassifiers_NegativeCases(t *testing.T) {
c := NewCataloger()
src, err := source.NewFromDirectory("test-fixtures/classifiers/negative")
assert.NoError(t, err)
resolver, err := src.FileResolver(source.SquashedScope)
assert.NoError(t, err)
actualResults, _, err := c.Catalog(resolver)
assert.NoError(t, err)
assert.Equal(t, 0, len(actualResults))
}
func singleLocation(s string) source.LocationSet {
return source.NewLocationSet(source.NewLocation(s))
}
func assertPackagesAreEqual(t *testing.T, expected pkg.Package, p pkg.Package) {
meta1 := expected.Metadata.(pkg.BinaryMetadata)
meta2 := p.Metadata.(pkg.BinaryMetadata)
if expected.Name != p.Name ||
expected.Version != p.Version ||
meta1.Classifier != meta2.Classifier {
assert.Failf(t, "packages not equal", "%v != %v", expected, p)
}
}
+179
View File
@@ -0,0 +1,179 @@
package binary
import (
"bytes"
"fmt"
"io"
"reflect"
"regexp"
"text/template"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/unionreader"
"github.com/anchore/syft/syft/source"
)
var emptyPURL = packageurl.PackageURL{}
// classifier is a generic package classifier that can be used to match a package definition
// to a file that meets the given content criteria of the evidenceMatcher.
type classifier struct {
Class string
// FileGlob is a selector to narrow down file inspection using the **/glob* syntax
FileGlob string
// EvidenceMatcher is what will be used to match against the file in the source
// location. If the matcher returns a package, the file will be considered a candidate.
EvidenceMatcher evidenceMatcher
// Information below is used to specify the Package information when returned
// Package is the name to use for the package
Package string
// Language is the language to classify this package as
Language pkg.Language
// Type is the package type to use for the package
Type pkg.Type
// PURL is the Package URL to use when generating a package
PURL packageurl.PackageURL
// CPEs are the specific CPEs we want to include for this binary with updated version information
CPEs []pkg.CPE
}
// evidenceMatcher is a function called to catalog Packages that match some sort of evidence
type evidenceMatcher func(classifier classifier, reader source.LocationReadCloser) ([]pkg.Package, error)
func fileNameTemplateVersionMatcher(fileNamePattern string, contentTemplate string) evidenceMatcher {
pat := regexp.MustCompile(fileNamePattern)
return func(classifier classifier, reader source.LocationReadCloser) ([]pkg.Package, error) {
if !pat.MatchString(reader.RealPath) {
return nil, nil
}
filepathNamedGroupValues := internal.MatchNamedCaptureGroups(pat, reader.RealPath)
tmpl, err := template.New("").Parse(contentTemplate)
if err != nil {
return nil, fmt.Errorf("unable to parse classifier template=%q : %w", contentTemplate, err)
}
patternBuf := &bytes.Buffer{}
err = tmpl.Execute(patternBuf, filepathNamedGroupValues)
if err != nil {
return nil, fmt.Errorf("unable to render template: %w", err)
}
tmplPattern, err := regexp.Compile(patternBuf.String())
if err != nil {
return nil, fmt.Errorf("unable to compile rendered regex=%q: %w", patternBuf.String(), err)
}
contents, err := getContents(reader)
if err != nil {
return nil, fmt.Errorf("unable to get read contents for file: %w", err)
}
matchMetadata := internal.MatchNamedCaptureGroups(tmplPattern, string(contents))
return singlePackage(classifier, reader, matchMetadata), nil
}
}
func fileContentsVersionMatcher(pattern string) evidenceMatcher {
pat := regexp.MustCompile(pattern)
return func(classifier classifier, reader source.LocationReadCloser) ([]pkg.Package, error) {
contents, err := getContents(reader)
if err != nil {
return nil, fmt.Errorf("unable to get read contents for file: %w", err)
}
matchMetadata := internal.MatchNamedCaptureGroups(pat, string(contents))
return singlePackage(classifier, reader, matchMetadata), nil
}
}
func mustPURL(purl string) packageurl.PackageURL {
p, err := packageurl.FromString(purl)
if err != nil {
panic(fmt.Sprintf("invalid PURL: %s", p))
}
return p
}
func singlePackage(classifier classifier, reader source.LocationReadCloser, matchMetadata map[string]string) []pkg.Package {
version, ok := matchMetadata["version"]
if !ok {
return nil
}
update := matchMetadata["update"]
var cpes []pkg.CPE
for _, cpe := range classifier.CPEs {
cpe.Version = version
cpe.Update = update
cpes = append(cpes, cpe)
}
p := pkg.Package{
Name: classifier.Package,
Version: version,
Language: pkg.Binary,
Locations: source.NewLocationSet(reader.Location),
Type: pkg.BinaryPkg,
CPEs: cpes,
FoundBy: catalogerName,
MetadataType: pkg.BinaryMetadataType,
Metadata: pkg.BinaryMetadata{
Classifier: classifier.Class,
RealPath: reader.RealPath,
VirtualPath: reader.VirtualPath,
},
}
if classifier.Type != "" {
p.Type = classifier.Type
}
if !reflect.DeepEqual(classifier.PURL, emptyPURL) {
purl := classifier.PURL
purl.Version = version
p.PURL = purl.ToString()
}
if classifier.Language != "" {
p.Language = classifier.Language
}
p.SetID()
return []pkg.Package{p}
}
func getContents(reader source.LocationReadCloser) ([]byte, error) {
unionReader, err := unionreader.GetUnionReader(reader.ReadCloser)
if err != nil {
return nil, fmt.Errorf("unable to get union reader for file: %w", err)
}
// TODO: there may be room for improvement here, as this may use an excessive amount of memory. Alternate approach is to leverage a RuneReader.
contents, err := io.ReadAll(unionReader)
if err != nil {
return nil, fmt.Errorf("unable to get contents for file: %w", err)
}
return contents, nil
}
// singleCPE returns a []pkg.CPE based on the cpe string or panics if the CPE is invalid
func singleCPE(cpe string) []pkg.CPE {
return []pkg.CPE{
pkg.MustCPE(cpe),
}
}
@@ -1,7 +1,6 @@
package generic
package binary
import (
"regexp"
"testing"
"github.com/stretchr/testify/require"
@@ -14,35 +13,27 @@ func Test_ClassifierCPEs(t *testing.T) {
tests := []struct {
name string
fixture string
classifier Classifier
classifier classifier
cpes []string
}{
{
name: "no CPEs",
fixture: "test-fixtures/version.txt",
classifier: Classifier{
Package: "some-app",
FilepathPatterns: []*regexp.Regexp{
regexp.MustCompile(".*/version.txt"),
},
EvidencePatterns: []*regexp.Regexp{
regexp.MustCompile(`(?m)my-verison:(?P<version>[0-9.]+)`),
},
CPEs: []pkg.CPE{},
classifier: classifier{
Package: "some-app",
FileGlob: ".*/version.txt",
EvidenceMatcher: fileContentsVersionMatcher(`(?m)my-verison:(?P<version>[0-9.]+)`),
CPEs: []pkg.CPE{},
},
cpes: nil,
},
{
name: "one CPE",
fixture: "test-fixtures/version.txt",
classifier: Classifier{
Package: "some-app",
FilepathPatterns: []*regexp.Regexp{
regexp.MustCompile(".*/version.txt"),
},
EvidencePatterns: []*regexp.Regexp{
regexp.MustCompile(`(?m)my-verison:(?P<version>[0-9.]+)`),
},
classifier: classifier{
Package: "some-app",
FileGlob: ".*/version.txt",
EvidenceMatcher: fileContentsVersionMatcher(`(?m)my-verison:(?P<version>[0-9.]+)`),
CPEs: []pkg.CPE{
pkg.MustCPE("cpe:2.3:a:some:app:*:*:*:*:*:*:*:*"),
},
@@ -54,14 +45,10 @@ func Test_ClassifierCPEs(t *testing.T) {
{
name: "multiple CPEs",
fixture: "test-fixtures/version.txt",
classifier: Classifier{
Package: "some-app",
FilepathPatterns: []*regexp.Regexp{
regexp.MustCompile(".*/version.txt"),
},
EvidencePatterns: []*regexp.Regexp{
regexp.MustCompile(`(?m)my-verison:(?P<version>[0-9.]+)`),
},
classifier: classifier{
Package: "some-app",
FileGlob: ".*/version.txt",
EvidenceMatcher: fileContentsVersionMatcher(`(?m)my-verison:(?P<version>[0-9.]+)`),
CPEs: []pkg.CPE{
pkg.MustCPE("cpe:2.3:a:some:app:*:*:*:*:*:*:*:*"),
pkg.MustCPE("cpe:2.3:a:some:apps:*:*:*:*:*:*:*:*"),
@@ -83,9 +70,13 @@ func Test_ClassifierCPEs(t *testing.T) {
location := locations[0]
readCloser, err := resolver.FileContentsByLocation(location)
require.NoError(t, err)
p, _, err := test.classifier.Examine(source.NewLocationReadCloser(location, readCloser))
pkgs, err := test.classifier.EvidenceMatcher(test.classifier, source.NewLocationReadCloser(location, readCloser))
require.NoError(t, err)
require.Len(t, pkgs, 1)
p := pkgs[0]
var cpes []string
for _, c := range p.CPEs {
cpes = append(cpes, pkg.CPEString(c))
@@ -0,0 +1,89 @@
package binary
import "github.com/anchore/syft/syft/pkg"
var defaultClassifiers = []classifier{
{
Class: "python-binary",
FileGlob: "**/python*",
EvidenceMatcher: fileNameTemplateVersionMatcher(
`(.*/|^)python(?P<version>[0-9]+\.[0-9]+)$`,
`(?m)(?P<version>{{ .version }}\.[0-9]+[-_a-zA-Z0-9]*)`),
Package: "python",
},
{
Class: "python-binary-lib",
FileGlob: "**/libpython*.so*",
EvidenceMatcher: fileNameTemplateVersionMatcher(
`(.*/|^)libpython(?P<version>[0-9]+\.[0-9]+).so.*$`,
`(?m)(?P<version>{{ .version }}\.[0-9]+[-_a-zA-Z0-9]*)`),
Package: "python",
},
{
Class: "cpython-source",
FileGlob: "**/patchlevel.h",
EvidenceMatcher: fileContentsVersionMatcher(
`(?m)#define\s+PY_VERSION\s+"?(?P<version>[0-9\.\-_a-zA-Z]+)"?`),
Package: "python",
},
{
Class: "go-binary",
FileGlob: "**/go",
EvidenceMatcher: fileContentsVersionMatcher(
`(?m)go(?P<version>[0-9]+\.[0-9]+(\.[0-9]+|beta[0-9]+|alpha[0-9]+|rc[0-9]+)?)\x00`),
Package: "go",
CPEs: singleCPE("cpe:2.3:a:golang:go:*:*:*:*:*:*:*:*"),
},
{
Class: "java-binary-openjdk",
FileGlob: "**/java",
EvidenceMatcher: fileContentsVersionMatcher(
// [NUL]openjdk[NUL]java[NUL]1.8[NUL]1.8.0_352-b08[NUL]
`(?m)\x00openjdk\x00java\x00(?P<release>[0-9]+[.0-9]+)\x00(?P<version>[0-9]+[-._a-zA-Z0-9]+)\x00`),
Package: "java",
// TODO the updates might need to be part of the CPE, like: 1.8.0:update152
CPEs: singleCPE("cpe:2.3:a:oracle:openjdk:*:*:*:*:*:*:*:*"),
},
{
Class: "java-binary-ibm",
FileGlob: "**/java",
EvidenceMatcher: fileContentsVersionMatcher(
// [NUL]java[NUL]1.8[NUL][NUL][NUL][NUL]1.8.0-foreman_2022_09_22_15_30-b00[NUL]
`(?m)\x00java\x00(?P<release>[0-9]+[.0-9]+)\x00{4}(?P<version>[0-9]+[-._a-zA-Z0-9]+)\x00`),
Package: "java",
CPEs: singleCPE("cpe:2.3:a:ibm:java:*:*:*:*:*:*:*:*"),
},
{
Class: "java-binary-oracle",
FileGlob: "**/java",
EvidenceMatcher: fileContentsVersionMatcher(
// [NUL]19.0.1+10-21[NUL]
`(?m)\x00(?P<version>[0-9]+[.0-9]+[+][-0-9]+)\x00`),
Package: "java",
CPEs: singleCPE("cpe:2.3:a:oracle:jre:*:*:*:*:*:*:*:*"),
},
{
Class: "nodejs-binary",
FileGlob: "**/node",
EvidenceMatcher: fileContentsVersionMatcher(
`(?m)node\.js\/v(?P<version>[0-9]+\.[0-9]+\.[0-9]+)`),
Package: "node",
Language: pkg.JavaScript,
PURL: mustPURL("pkg:generic/node@version"),
CPEs: singleCPE("cpe:2.3:a:nodejs:node.js:*:*:*:*:*:*:*:*"),
},
{
Class: "go-binary-hint",
FileGlob: "**/VERSION",
EvidenceMatcher: fileContentsVersionMatcher(
`(?m)go(?P<version>[0-9]+\.[0-9]+(\.[0-9]+|beta[0-9]+|alpha[0-9]+|rc[0-9]+)?)`),
Package: "go",
},
{
Class: "busybox-binary",
FileGlob: "**/busybox",
EvidenceMatcher: fileContentsVersionMatcher(
`(?m)BusyBox\s+v(?P<version>[0-9]+\.[0-9]+\.[0-9]+)`),
Package: "busybox",
},
}
@@ -0,0 +1,2 @@
# this should match node 19.2.1
node.js/v19.2.1
Binary file not shown.

After

Width:  |  Height:  |  Size: 39 B

+4 -3
View File
@@ -12,6 +12,7 @@ import (
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/alpm"
"github.com/anchore/syft/syft/pkg/cataloger/apkdb"
"github.com/anchore/syft/syft/pkg/cataloger/binary"
"github.com/anchore/syft/syft/pkg/cataloger/cpp"
"github.com/anchore/syft/syft/pkg/cataloger/dart"
"github.com/anchore/syft/syft/pkg/cataloger/deb"
@@ -40,7 +41,6 @@ func ImageCatalogers(cfg Config) []pkg.Cataloger {
python.NewPythonPackageCataloger(),
php.NewPHPComposerInstalledCataloger(),
javascript.NewJavascriptPackageCataloger(),
javascript.NewNodeBinaryCataloger(),
deb.NewDpkgdbCataloger(),
rpm.NewRpmDBCataloger(),
java.NewJavaCataloger(cfg.Java()),
@@ -49,6 +49,7 @@ func ImageCatalogers(cfg Config) []pkg.Cataloger {
dotnet.NewDotnetDepsCataloger(),
portage.NewPortageCataloger(),
sbom.NewSBOMCataloger(),
binary.NewCataloger(),
}, cfg.Catalogers)
}
@@ -61,7 +62,6 @@ func DirectoryCatalogers(cfg Config) []pkg.Cataloger {
python.NewPythonPackageCataloger(),
php.NewPHPComposerLockCataloger(),
javascript.NewJavascriptLockCataloger(),
javascript.NewNodeBinaryCataloger(),
deb.NewDpkgdbCataloger(),
rpm.NewRpmDBCataloger(),
rpm.NewFileCataloger(),
@@ -78,6 +78,7 @@ func DirectoryCatalogers(cfg Config) []pkg.Cataloger {
portage.NewPortageCataloger(),
haskell.NewHackageCataloger(),
sbom.NewSBOMCataloger(),
binary.NewCataloger(),
}, cfg.Catalogers)
}
@@ -91,7 +92,6 @@ func AllCatalogers(cfg Config) []pkg.Cataloger {
python.NewPythonPackageCataloger(),
javascript.NewJavascriptLockCataloger(),
javascript.NewJavascriptPackageCataloger(),
javascript.NewNodeBinaryCataloger(),
deb.NewDpkgdbCataloger(),
rpm.NewRpmDBCataloger(),
rpm.NewFileCataloger(),
@@ -111,6 +111,7 @@ func AllCatalogers(cfg Config) []pkg.Cataloger {
portage.NewPortageCataloger(),
haskell.NewHackageCataloger(),
sbom.NewSBOMCataloger(),
binary.NewCataloger(),
}, cfg.Catalogers)
}
@@ -11,6 +11,12 @@ type candidateComposite struct {
candidateAddition
}
type candidateRemovalComposite struct {
pkg.Type
candidateKey
candidateRemovals
}
// defaultCandidateAdditions is all of the known cases for product and vendor field values that should be used when
// select package information is discovered
var defaultCandidateAdditions = buildCandidateLookup(
@@ -123,6 +129,16 @@ var defaultCandidateAdditions = buildCandidateLookup(
},
})
var defaultCandidateRemovals = buildCandidateRemovalLookup(
[]candidateRemovalComposite{
// Python packages
{
pkg.PythonPkg,
candidateKey{PkgName: "redis"},
candidateRemovals{VendorsToRemove: []string{"redis"}},
},
})
// buildCandidateLookup is a convenience function for creating the defaultCandidateAdditions set
func buildCandidateLookup(cc []candidateComposite) (ca map[pkg.Type]map[candidateKey]candidateAddition) {
ca = make(map[pkg.Type]map[candidateKey]candidateAddition)
@@ -136,12 +152,30 @@ func buildCandidateLookup(cc []candidateComposite) (ca map[pkg.Type]map[candidat
return ca
}
// buildCandidateRemovalLookup is a convenience function for creating the defaultCandidateRemovals set
func buildCandidateRemovalLookup(cc []candidateRemovalComposite) (ca map[pkg.Type]map[candidateKey]candidateRemovals) {
ca = make(map[pkg.Type]map[candidateKey]candidateRemovals)
for _, c := range cc {
if _, ok := ca[c.Type]; !ok {
ca[c.Type] = make(map[candidateKey]candidateRemovals)
}
ca[c.Type][c.candidateKey] = c.candidateRemovals
}
return ca
}
// candidateKey represents the set of inputs that should be matched on in order to signal more candidate additions to be used.
type candidateKey struct {
Vendor string
PkgName string
}
// candidateRemovals are the specific removals that should be considered during CPE generation (given a specific candidateKey)
type candidateRemovals struct {
ProductsToRemove []string
VendorsToRemove []string
}
// candidateAddition are the specific additions that should be considered during CPE generation (given a specific candidateKey)
type candidateAddition struct {
AdditionalProducts []string
@@ -192,3 +226,35 @@ func findAdditionalProducts(allAdditions map[pkg.Type]map[candidateKey]candidate
return products
}
// findVendorsToRemove searches all possible vendor removals that could be removed during the CPE generation process (given package info + a vendor candidate)
func findVendorsToRemove(allRemovals map[pkg.Type]map[candidateKey]candidateRemovals, ty pkg.Type, pkgName string) (vendors []string) {
removals, ok := allRemovals[ty]
if !ok {
return nil
}
if removal, ok := removals[candidateKey{
PkgName: pkgName,
}]; ok {
vendors = append(vendors, removal.VendorsToRemove...)
}
return vendors
}
// findProductsToRemove searches all possible product removals that could be removed during the CPE generation process (given package info)
func findProductsToRemove(allRemovals map[pkg.Type]map[candidateKey]candidateRemovals, ty pkg.Type, pkgName string) (products []string) {
removals, ok := allRemovals[ty]
if !ok {
return nil
}
if removal, ok := removals[candidateKey{
PkgName: pkgName,
}]; ok {
products = append(products, removal.ProductsToRemove...)
}
return products
}
@@ -154,3 +154,81 @@ func Test_additionalVendors(t *testing.T) {
})
}
}
func Test_findVendorsToRemove(t *testing.T) {
//GIVEN
tests := []struct {
name string
ty pkg.Type
pkgName string
expected []string
}{
{
name: "vendor removal match by input package name",
ty: pkg.JavaPkg,
pkgName: "my-package-name",
expected: []string{"awesome-vendor-addition"},
},
{
name: "vendor removal miss by input package name",
ty: pkg.JavaPkg,
pkgName: "my-package-name-1",
},
}
allRemovals := map[pkg.Type]map[candidateKey]candidateRemovals{
pkg.JavaPkg: {
candidateKey{
PkgName: "my-package-name",
}: {
VendorsToRemove: []string{"awesome-vendor-addition"},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
//WHEN + THEN
assert.Equal(t, test.expected, findVendorsToRemove(allRemovals, test.ty, test.pkgName))
})
}
}
func Test_findProductsToRemove(t *testing.T) {
//GIVEN
tests := []struct {
name string
ty pkg.Type
pkgName string
expected []string
}{
{
name: "vendor removal match by input package name",
ty: pkg.JavaPkg,
pkgName: "my-package-name",
expected: []string{"awesome-vendor-addition"},
},
{
name: "vendor removal miss by input package name",
ty: pkg.JavaPkg,
pkgName: "my-package-name-1",
},
}
allRemovals := map[pkg.Type]map[candidateKey]candidateRemovals{
pkg.JavaPkg: {
candidateKey{
PkgName: "my-package-name",
}: {
ProductsToRemove: []string{"awesome-vendor-addition"},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
//WHEN + THEN
assert.Equal(t, test.expected, findProductsToRemove(allRemovals, test.ty, test.pkgName))
})
}
}
@@ -116,6 +116,9 @@ func candidateVendors(p pkg.Package) []string {
vendors.addValue(findAdditionalVendors(defaultCandidateAdditions, p.Type, p.Name, vendor)...)
}
// remove known mis
vendors.removeByValue(findVendorsToRemove(defaultCandidateRemovals, p.Type, p.Name)...)
return vendors.uniqueValues()
}
@@ -148,6 +151,9 @@ func candidateProducts(p pkg.Package) []string {
// add known candidate additions
products.addValue(findAdditionalProducts(defaultCandidateAdditions, p.Type, p.Name)...)
// remove known candidate removals
products.removeByValue(findProductsToRemove(defaultCandidateRemovals, p.Type, p.Name)...)
return products.uniqueValues()
}
@@ -666,6 +666,27 @@ func TestGeneratePackageCPEs(t *testing.T) {
"cpe:2.3:a:stephanie_morillo:bundler:2.1.4:*:*:*:*:*:*:*",
},
},
{
name: "regression: python redis shadows normal redis",
p: pkg.Package{
Name: "redis",
Version: "2.1.4",
Type: pkg.PythonPkg,
FoundBy: "some-analyzer",
Language: pkg.Python,
},
expected: []string{
"cpe:2.3:a:python-redis:python-redis:2.1.4:*:*:*:*:*:*:*",
"cpe:2.3:a:python-redis:python_redis:2.1.4:*:*:*:*:*:*:*",
"cpe:2.3:a:python-redis:redis:2.1.4:*:*:*:*:*:*:*",
"cpe:2.3:a:python:python-redis:2.1.4:*:*:*:*:*:*:*",
"cpe:2.3:a:python:python_redis:2.1.4:*:*:*:*:*:*:*",
"cpe:2.3:a:python:redis:2.1.4:*:*:*:*:*:*:*",
"cpe:2.3:a:python_redis:python-redis:2.1.4:*:*:*:*:*:*:*",
"cpe:2.3:a:python_redis:python_redis:2.1.4:*:*:*:*:*:*:*",
"cpe:2.3:a:python_redis:redis:2.1.4:*:*:*:*:*:*:*",
},
},
}
for _, test := range tests {
+2
View File
@@ -118,6 +118,8 @@ func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []arti
for _, req := range c.selectFiles(resolver) {
location, parser := req.Location, req.Parser
log.WithFields("path", location.RealPath).Trace("parsing file contents")
contentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
logger.WithFields("location", location.RealPath, "error", err).Warn("unable to fetch contents")
-100
View File
@@ -1,100 +0,0 @@
package generic
import (
"fmt"
"io"
"path"
"regexp"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/unionreader"
"github.com/anchore/syft/syft/source"
)
// Classifier is a generic package classifier that can be used to match a package definition
// to a file that meets the given content criteria of the EvidencePatternTemplates.
type Classifier struct {
Package string
// FilepathPatterns is a list of regular expressions that will be used to match against the file path of a given
// source location. If any of the patterns match, the file will be considered a candidate for parsing.
// If no patterns are provided, the reader is automatically considered a candidate.
FilepathPatterns []*regexp.Regexp
// EvidencePatterns is a list of regular expressions that will be used to match against the file contents of a
// given file in the source location. If any of the patterns match, the file will be considered a candidate for parsing.
EvidencePatterns []*regexp.Regexp
// CPEs are the specific CPEs we want to include for this binary with updated version information
CPEs []pkg.CPE
}
func (c Classifier) Examine(reader source.LocationReadCloser) (p *pkg.Package, r *artifact.Relationship, err error) {
doesFilepathMatch := true
if len(c.FilepathPatterns) > 0 {
doesFilepathMatch, _ = file.FilepathMatches(c.FilepathPatterns, reader.Location)
}
if !doesFilepathMatch {
return nil, nil, fmt.Errorf("location: %s did not match any patterns for package=%q", reader.Location, c.Package)
}
contents, err := getContents(reader)
if err != nil {
return nil, nil, fmt.Errorf("unable to get read contents for file: %w", err)
}
var classifiedPackage *pkg.Package
for _, evidencePattern := range c.EvidencePatterns {
if !evidencePattern.Match(contents) {
continue
}
matchMetadata := internal.MatchNamedCaptureGroups(evidencePattern, string(contents))
version, ok := matchMetadata["version"]
if !ok {
log.Debugf("no version found in binary from pattern %v", evidencePattern)
continue
}
var cpes []pkg.CPE
for _, cpe := range c.CPEs {
cpe.Version = version
if err == nil {
cpes = append(cpes, cpe)
}
}
classifiedPackage = &pkg.Package{
Name: path.Base(reader.VirtualPath),
Version: version,
Language: pkg.Binary,
Locations: source.NewLocationSet(reader.Location),
Type: pkg.BinaryPkg,
CPEs: cpes,
MetadataType: pkg.BinaryMetadataType,
Metadata: pkg.BinaryMetadata{
Classifier: c.Package,
RealPath: reader.RealPath,
VirtualPath: reader.VirtualPath,
},
}
break
}
return classifiedPackage, nil, nil
}
func getContents(reader source.LocationReadCloser) ([]byte, error) {
unionReader, err := unionreader.GetUnionReader(reader.ReadCloser)
if err != nil {
return nil, fmt.Errorf("unable to get union reader for file: %w", err)
}
contents, err := io.ReadAll(unionReader)
if err != nil {
return nil, fmt.Errorf("unable to get contents for file: %w", err)
}
return contents, nil
}
+34 -18
View File
@@ -2,6 +2,8 @@ package golang
import (
"debug/buildinfo"
"fmt"
"io"
"runtime/debug"
"github.com/anchore/syft/internal/log"
@@ -14,31 +16,20 @@ func scanFile(reader unionreader.UnionReader, filename string) ([]*debug.BuildIn
// with more than one binary
readers, err := unionreader.GetReaders(reader)
if err != nil {
log.Warnf("golang cataloger: failed to open a binary: %v", err)
log.WithFields("error", err).Warnf("failed to open a golang binary")
return nil, nil
}
var builds []*debug.BuildInfo
for _, r := range readers {
bi, err := buildinfo.Read(r)
// note: the stdlib does not export the error we need to check for
bi, err := getBuildInfo(r)
if err != nil {
if err.Error() == "not a Go executable" {
// since the cataloger can only select executables and not distinguish if they are a go-compiled
// binary, we should not show warnings/logs in this case.
return nil, nil
}
// in this case we could not read the or parse the file, but not explicitly because it is not a
// go-compiled binary (though it still might be).
// TODO: We should change this back to "warn" eventually.
// But right now it's catching too many cases where the reader IS NOT a Go binary at all.
// It'd be great to see how we can get those cases to be detected and handled above before we get to
// this point in execution.
log.Infof("golang cataloger: unable to read buildinfo (file=%q): %v", filename, err)
return nil, nil
log.WithFields("file", filename, "error", err).Warn("unable to read golang buildinfo")
continue
}
if bi == nil {
continue
}
builds = append(builds, bi)
}
@@ -46,3 +37,28 @@ func scanFile(reader unionreader.UnionReader, filename string) ([]*debug.BuildIn
return builds, archs
}
func getBuildInfo(r io.ReaderAt) (bi *debug.BuildInfo, err error) {
defer func() {
if r := recover(); r != nil {
// this can happen in cases where a malformed binary is passed in can be initially parsed, but not
// used without error later down the line. This is the case with :
// https://github.com/llvm/llvm-project/blob/llvmorg-15.0.6/llvm/test/Object/Inputs/macho-invalid-dysymtab-bad-size
err = fmt.Errorf("recovered from panic: %v", r)
}
}()
bi, err = buildinfo.Read(r)
// note: the stdlib does not export the error we need to check for
if err != nil {
if err.Error() == "not a Go executable" {
// since the cataloger can only select executables and not distinguish if they are a go-compiled
// binary, we should not show warnings/logs in this case.
return
}
// in this case we could not read the or parse the file, but not explicitly because it is not a
// go-compiled binary (though it still might be).
return
}
return
}
@@ -0,0 +1,40 @@
package golang
import (
"fmt"
"io"
"runtime/debug"
"testing"
"github.com/stretchr/testify/assert"
)
func Test_getBuildInfo(t *testing.T) {
type args struct {
r io.ReaderAt
}
tests := []struct {
name string
args args
wantBi *debug.BuildInfo
wantErr assert.ErrorAssertionFunc
}{
{
name: "recover from panic",
args: args{
r: nil, // trying to use a nil reader will cause a panic
},
wantBi: nil, // we should not return anything useful
wantErr: assert.Error,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotBi, err := getBuildInfo(tt.args.r)
if !tt.wantErr(t, err, fmt.Sprintf("getBuildInfo(%v)", tt.args.r)) {
return
}
assert.Equalf(t, tt.wantBi, gotBi, "getBuildInfo(%v)", tt.args.r)
})
}
}
@@ -4,7 +4,6 @@ Package javascript provides a concrete Cataloger implementation for JavaScript e
package javascript
import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
)
@@ -20,8 +19,3 @@ func NewJavascriptLockCataloger() *generic.Cataloger {
WithParserByGlobs(parseYarnLock, "**/yarn.lock").
WithParserByGlobs(parsePnpmLock, "**/pnpm-lock.yaml")
}
func NewNodeBinaryCataloger() *generic.Cataloger {
return generic.NewCataloger("node-binary-cataloger").
WithParserByMimeTypes(parseNodeBinary, internal.ExecutableMIMETypeSet.List()...)
}
@@ -1,43 +0,0 @@
package javascript
import (
"regexp"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/source"
)
var nodeClassifier = generic.Classifier{
Package: "node.js", // Note: this purposely matches the "node.js" string to aid nvd vuln matching
FilepathPatterns: []*regexp.Regexp{
// note: should we just parse all files resolved with executable mimetypes
// regexp that matches node binary
regexp.MustCompile(`(.*/|^)node$`),
},
EvidencePatterns: []*regexp.Regexp{
// regex that matches node.js/vx.y.z
regexp.MustCompile(`(?m)node\.js\/v(?P<version>[0-9]+\.[0-9]+\.[0-9]+)`),
},
CPEs: []pkg.CPE{
pkg.MustCPE("cpe:2.3:a:nodejs:node.js:*:*:*:*:*:*:*:*"),
},
}
func parseNodeBinary(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
p, _, err := nodeClassifier.Examine(reader)
if err != nil {
log.Trace("failed to find node.js package: %+v", err)
return nil, nil, nil // we can silently fail here to reduce warning noise
}
// TODO add node specific metadata to the packages to help with vulnerability matching
if p != nil {
p.Language = pkg.JavaScript
p.SetID()
return []pkg.Package{*p}, nil, nil
}
return nil, nil, nil
}
+7
View File
@@ -0,0 +1,7 @@
package pkg
type FileMetadata struct {
Classifier string `mapstructure:"Classifier" json:"classifier"`
RealPath string `mapstructure:"RealPath" json:"realPath"`
VirtualPath string `mapstructure:"VirtualPath" json:"virtualPath"`
}
+1
View File
@@ -25,6 +25,7 @@ const (
CPP Language = "c++"
Haskell Language = "haskell"
Binary Language = "binary"
File Language = "file"
)
// AllLanguages is a set of all programming languages detected by syft.
+6 -10
View File
@@ -18,13 +18,12 @@ type SBOM struct {
}
type Artifacts struct {
PackageCatalog *pkg.Catalog
FileMetadata map[source.Coordinates]source.FileMetadata
FileDigests map[source.Coordinates][]file.Digest
FileClassifications map[source.Coordinates][]file.Classification
FileContents map[source.Coordinates]string
Secrets map[source.Coordinates][]file.SearchResult
LinuxDistribution *linux.Release
PackageCatalog *pkg.Catalog
FileMetadata map[source.Coordinates]source.FileMetadata
FileDigests map[source.Coordinates][]file.Digest
FileContents map[source.Coordinates]string
Secrets map[source.Coordinates][]file.SearchResult
LinuxDistribution *linux.Release
}
type Descriptor struct {
@@ -55,9 +54,6 @@ func (s SBOM) AllCoordinates() []source.Coordinates {
for coordinates := range s.Artifacts.FileContents {
set.Add(coordinates)
}
for coordinates := range s.Artifacts.FileClassifications {
set.Add(coordinates)
}
for coordinates := range s.Artifacts.FileDigests {
set.Add(coordinates)
}
@@ -1,5 +1,5 @@
# LAYER 0:
FROM busybox:latest
FROM busybox:1.34.0
# LAYER 1:
ADD file-1.txt .
+3 -3
View File
@@ -105,7 +105,7 @@ func TestPackagesCmdFlags(t *testing.T) {
name: "squashed-scope-flag-hidden-packages",
args: []string{"packages", "-o", "json", "-s", "squashed", hiddenPackagesImage},
assertions: []traitAssertion{
assertPackageCount(162),
assertPackageCount(163),
assertNotInOutput("vsftpd"), // hidden package
assertSuccessfulReturnCode,
},
@@ -114,7 +114,7 @@ func TestPackagesCmdFlags(t *testing.T) {
name: "all-layers-scope-flag",
args: []string{"packages", "-o", "json", "-s", "all-layers", hiddenPackagesImage},
assertions: []traitAssertion{
assertPackageCount(163), // packages are now deduplicated for this case
assertPackageCount(164), // packages are now deduplicated for this case
assertInOutput("all-layers"),
assertInOutput("vsftpd"), // hidden package
assertSuccessfulReturnCode,
@@ -127,7 +127,7 @@ func TestPackagesCmdFlags(t *testing.T) {
"SYFT_PACKAGE_CATALOGER_SCOPE": "all-layers",
},
assertions: []traitAssertion{
assertPackageCount(163), // packages are now deduplicated for this case
assertPackageCount(164), // packages are now deduplicated for this case
assertInOutput("all-layers"),
assertInOutput("vsftpd"), // hidden package
assertSuccessfulReturnCode,
@@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
@@ -21,7 +22,7 @@ func TestPackageDeduplication(t *testing.T) {
}{
{
scope: source.AllLayersScope,
packageCount: 172, // without deduplication this would be 618
packageCount: 173, // without deduplication this would be 618
instanceCount: map[string]int{
"basesystem": 1,
"wget": 1,
@@ -40,7 +41,7 @@ func TestPackageDeduplication(t *testing.T) {
},
{
scope: source.SquashedScope,
packageCount: 170,
packageCount: 171,
instanceCount: map[string]int{
"basesystem": 1,
"wget": 1,
@@ -62,6 +63,12 @@ func TestPackageDeduplication(t *testing.T) {
t.Run(string(tt.scope), func(t *testing.T) {
sbom, _ := catalogFixtureImage(t, "image-vertical-package-dups", tt.scope, nil)
for _, p := range sbom.Artifacts.PackageCatalog.Sorted() {
if p.Type == pkg.BinaryPkg {
assert.NotEmpty(t, p.Name)
}
}
assert.Equal(t, tt.packageCount, sbom.Artifacts.PackageCatalog.PackageCount())
for name, expectedInstanceCount := range tt.instanceCount {
pkgs := sbom.Artifacts.PackageCatalog.PackagesByName(name)