mirror of
https://github.com/anchore/syft.git
synced 2025-11-17 08:23:15 +01:00
Compare commits
25 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
365325376a | ||
|
|
153f2321ce | ||
|
|
7bf7bcc461 | ||
|
|
6a21b5e5e2 | ||
|
|
6480c8a425 | ||
|
|
89842bd2f6 | ||
|
|
4a60c41f38 | ||
|
|
2e100f33f3 | ||
|
|
b444f0c2ed | ||
|
|
102d362daf | ||
|
|
66c78d44af | ||
|
|
78a4ab8ced | ||
|
|
25ca33d20e | ||
|
|
60ca241593 | ||
|
|
0f475c8bcd | ||
|
|
199394934d | ||
|
|
8a22d394ed | ||
|
|
bbef262b8f | ||
|
|
4e06a7ab32 | ||
|
|
e5711e9b42 | ||
|
|
f69b1db099 | ||
|
|
fe1ea443c2 | ||
|
|
bfcbf266df | ||
|
|
a400c675fc | ||
|
|
7c154e7c37 |
@ -26,7 +26,7 @@ tools:
|
||||
# used for linting
|
||||
- name: golangci-lint
|
||||
version:
|
||||
want: v2.5.0
|
||||
want: v2.6.2
|
||||
method: github-release
|
||||
with:
|
||||
repo: golangci/golangci-lint
|
||||
@ -90,7 +90,7 @@ tools:
|
||||
# used for running all local and CI tasks
|
||||
- name: task
|
||||
version:
|
||||
want: v3.45.4
|
||||
want: v3.45.5
|
||||
method: github-release
|
||||
with:
|
||||
repo: go-task/task
|
||||
@ -98,7 +98,7 @@ tools:
|
||||
# used for triggering a release
|
||||
- name: gh
|
||||
version:
|
||||
want: v2.82.1
|
||||
want: v2.83.1
|
||||
method: github-release
|
||||
with:
|
||||
repo: cli/cli
|
||||
@ -114,7 +114,7 @@ tools:
|
||||
# used to upload test fixture cache
|
||||
- name: yq
|
||||
version:
|
||||
want: v4.48.1
|
||||
want: v4.48.2
|
||||
method: github-release
|
||||
with:
|
||||
repo: mikefarah/yq
|
||||
|
||||
6
.github/workflows/codeql-analysis.yml
vendored
6
.github/workflows/codeql-analysis.yml
vendored
@ -47,7 +47,7 @@ jobs:
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee #v3.29.5
|
||||
uses: github/codeql-action/init@014f16e7ab1402f30e7c3329d33797e7948572db #v3.29.5
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
@ -58,7 +58,7 @@ jobs:
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@0499de31b99561a6d14a36a5f662c2a54f91beee #v3.29.5
|
||||
uses: github/codeql-action/autobuild@014f16e7ab1402f30e7c3329d33797e7948572db #v3.29.5
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 https://git.io/JvXDl
|
||||
@ -72,4 +72,4 @@ jobs:
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee #v3.29.5
|
||||
uses: github/codeql-action/analyze@014f16e7ab1402f30e7c3329d33797e7948572db #v3.29.5
|
||||
|
||||
@ -31,11 +31,11 @@ jobs:
|
||||
with:
|
||||
repos: ${{ github.event.inputs.repos }}
|
||||
|
||||
- uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a #v2.1.0
|
||||
- uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 #v2.1.4
|
||||
id: generate-token
|
||||
with:
|
||||
app_id: ${{ secrets.TOKEN_APP_ID }}
|
||||
private_key: ${{ secrets.TOKEN_APP_PRIVATE_KEY }}
|
||||
app-id: ${{ secrets.TOKEN_APP_ID }}
|
||||
private-key: ${{ secrets.TOKEN_APP_PRIVATE_KEY }}
|
||||
|
||||
- uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e #v7.0.8
|
||||
with:
|
||||
|
||||
6
.github/workflows/update-bootstrap-tools.yml
vendored
6
.github/workflows/update-bootstrap-tools.yml
vendored
@ -45,11 +45,11 @@ jobs:
|
||||
echo "\`\`\`"
|
||||
} >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
- uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a #v2.1.0
|
||||
- uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 #v2.1.4
|
||||
id: generate-token
|
||||
with:
|
||||
app_id: ${{ secrets.TOKEN_APP_ID }}
|
||||
private_key: ${{ secrets.TOKEN_APP_PRIVATE_KEY }}
|
||||
app-id: ${{ secrets.TOKEN_APP_ID }}
|
||||
private-key: ${{ secrets.TOKEN_APP_PRIVATE_KEY }}
|
||||
|
||||
- uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e #v7.0.8
|
||||
with:
|
||||
|
||||
@ -14,6 +14,9 @@ env:
|
||||
jobs:
|
||||
upgrade-cpe-dictionary-index:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
if: github.repository == 'anchore/syft' # only run for main repo
|
||||
steps:
|
||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0
|
||||
@ -22,18 +25,31 @@ jobs:
|
||||
|
||||
- name: Bootstrap environment
|
||||
uses: ./.github/actions/bootstrap
|
||||
id: bootstrap
|
||||
|
||||
- name: Bootstrap environment
|
||||
uses: ./.github/actions/bootstrap
|
||||
- name: Login to GitHub Container Registry
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | ${{ steps.bootstrap.outputs.oras }} login ghcr.io -u ${{ github.actor }} --password-stdin
|
||||
|
||||
- run: |
|
||||
make generate-cpe-dictionary-index
|
||||
- name: Pull CPE cache from registry
|
||||
run: make generate:cpe-index:cache:pull
|
||||
|
||||
- uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a #v2.1.0
|
||||
- name: Update CPE cache from NVD API
|
||||
run: make generate:cpe-index:cache:update
|
||||
env:
|
||||
NVD_API_KEY: ${{ secrets.NVD_API_KEY }}
|
||||
|
||||
- name: Generate CPE dictionary index
|
||||
run: make generate:cpe-index:build
|
||||
|
||||
- name: Push updated CPE cache to registry
|
||||
run: make generate:cpe-index:cache:push
|
||||
|
||||
- uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 #v2.1.4
|
||||
id: generate-token
|
||||
with:
|
||||
app_id: ${{ secrets.TOKEN_APP_ID }}
|
||||
private_key: ${{ secrets.TOKEN_APP_PRIVATE_KEY }}
|
||||
app-id: ${{ secrets.TOKEN_APP_ID }}
|
||||
private-key: ${{ secrets.TOKEN_APP_PRIVATE_KEY }}
|
||||
|
||||
- uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e #v7.0.8
|
||||
with:
|
||||
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@ -73,3 +73,5 @@ cosign.pub
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
|
||||
|
||||
@ -106,8 +106,8 @@ syft <image> -o <format>
|
||||
Where the `formats` available are:
|
||||
- `syft-json`: Use this to get as much information out of Syft as possible!
|
||||
- `syft-text`: A row-oriented, human-and-machine-friendly output.
|
||||
- `cyclonedx-xml`: A XML report conforming to the [CycloneDX 1.6 specification](https://cyclonedx.org/specification/overview/).
|
||||
- `cyclonedx-xml@1.5`: A XML report conforming to the [CycloneDX 1.5 specification](https://cyclonedx.org/specification/overview/).
|
||||
- `cyclonedx-xml`: An XML report conforming to the [CycloneDX 1.6 specification](https://cyclonedx.org/specification/overview/).
|
||||
- `cyclonedx-xml@1.5`: An XML report conforming to the [CycloneDX 1.5 specification](https://cyclonedx.org/specification/overview/).
|
||||
- `cyclonedx-json`: A JSON report conforming to the [CycloneDX 1.6 specification](https://cyclonedx.org/specification/overview/).
|
||||
- `cyclonedx-json@1.5`: A JSON report conforming to the [CycloneDX 1.5 specification](https://cyclonedx.org/specification/overview/).
|
||||
- `spdx-tag-value`: A tag-value formatted report conforming to the [SPDX 2.3 specification](https://spdx.github.io/spdx-spec/v2.3/).
|
||||
|
||||
@ -1,5 +1,9 @@
|
||||
|
||||
version: "3"
|
||||
|
||||
includes:
|
||||
generate:cpe-index: ./task.d/generate/cpe-index.yaml
|
||||
|
||||
vars:
|
||||
OWNER: anchore
|
||||
PROJECT: syft
|
||||
@ -511,10 +515,11 @@ tasks:
|
||||
- "gofmt -s -w ./internal/spdxlicense"
|
||||
|
||||
generate-cpe-dictionary-index:
|
||||
desc: Generate the CPE index based off of the latest available CPE dictionary
|
||||
dir: "syft/pkg/cataloger/internal/cpegenerate/dictionary"
|
||||
desc: Generate the CPE index from local cache
|
||||
cmds:
|
||||
- "go generate"
|
||||
- task: generate:cpe-index:cache:pull
|
||||
- task: generate:cpe-index:cache:update
|
||||
- task: generate:cpe-index:build
|
||||
|
||||
|
||||
## Build-related targets #################################
|
||||
|
||||
@ -87,8 +87,8 @@ func runCatalogerList(opts *catalogerListOptions) error {
|
||||
}
|
||||
|
||||
func catalogerListReport(opts *catalogerListOptions, allTaskGroups [][]task.Task) (string, error) {
|
||||
defaultCatalogers := options.Flatten(opts.DefaultCatalogers)
|
||||
selectCatalogers := options.Flatten(opts.SelectCatalogers)
|
||||
defaultCatalogers := options.FlattenAndSort(opts.DefaultCatalogers)
|
||||
selectCatalogers := options.FlattenAndSort(opts.SelectCatalogers)
|
||||
selectedTaskGroups, selectionEvidence, err := task.SelectInGroups(
|
||||
allTaskGroups,
|
||||
cataloging.NewSelectionRequest().
|
||||
|
||||
@ -198,9 +198,10 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
|
||||
},
|
||||
Nix: nix.DefaultConfig().
|
||||
WithCaptureOwnedFiles(cfg.Nix.CaptureOwnedFiles),
|
||||
Python: python.CatalogerConfig{
|
||||
GuessUnpinnedRequirements: cfg.Python.GuessUnpinnedRequirements,
|
||||
},
|
||||
Python: python.DefaultCatalogerConfig().
|
||||
WithSearchRemoteLicenses(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.Python), cfg.Python.SearchRemoteLicenses)).
|
||||
WithPypiBaseURL(cfg.Python.PypiBaseURL).
|
||||
WithGuessUnpinnedRequirements(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.Python), cfg.Python.GuessUnpinnedRequirements)),
|
||||
JavaArchive: java.DefaultArchiveCatalogerConfig().
|
||||
WithUseMavenLocalRepository(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.Java, task.Maven), cfg.Java.UseMavenLocalRepository)).
|
||||
WithMavenLocalRepositoryDir(cfg.Java.MavenLocalRepositoryDir).
|
||||
@ -283,10 +284,10 @@ func (cfg *Catalog) PostLoad() error {
|
||||
|
||||
cfg.From = Flatten(cfg.From)
|
||||
|
||||
cfg.Catalogers = Flatten(cfg.Catalogers)
|
||||
cfg.DefaultCatalogers = Flatten(cfg.DefaultCatalogers)
|
||||
cfg.SelectCatalogers = Flatten(cfg.SelectCatalogers)
|
||||
cfg.Enrich = Flatten(cfg.Enrich)
|
||||
cfg.Catalogers = FlattenAndSort(cfg.Catalogers)
|
||||
cfg.DefaultCatalogers = FlattenAndSort(cfg.DefaultCatalogers)
|
||||
cfg.SelectCatalogers = FlattenAndSort(cfg.SelectCatalogers)
|
||||
cfg.Enrich = FlattenAndSort(cfg.Enrich)
|
||||
|
||||
// for backwards compatibility
|
||||
cfg.DefaultCatalogers = append(cfg.DefaultCatalogers, cfg.Catalogers...)
|
||||
@ -311,6 +312,11 @@ func Flatten(commaSeparatedEntries []string) []string {
|
||||
out = append(out, strings.TrimSpace(s))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func FlattenAndSort(commaSeparatedEntries []string) []string {
|
||||
out := Flatten(commaSeparatedEntries)
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@ -320,6 +326,7 @@ var publicisedEnrichmentOptions = []string{
|
||||
task.Golang,
|
||||
task.Java,
|
||||
task.JavaScript,
|
||||
task.Python,
|
||||
}
|
||||
|
||||
func enrichmentEnabled(enrichDirectives []string, features ...string) *bool {
|
||||
|
||||
@ -79,6 +79,98 @@ func TestCatalog_PostLoad(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlatten(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "preserves order of comma-separated values",
|
||||
input: []string{"registry,docker,oci-dir"},
|
||||
expected: []string{"registry", "docker", "oci-dir"},
|
||||
},
|
||||
{
|
||||
name: "preserves order across multiple entries",
|
||||
input: []string{"registry,docker", "oci-dir"},
|
||||
expected: []string{"registry", "docker", "oci-dir"},
|
||||
},
|
||||
{
|
||||
name: "trims whitespace",
|
||||
input: []string{" registry , docker ", " oci-dir "},
|
||||
expected: []string{"registry", "docker", "oci-dir"},
|
||||
},
|
||||
{
|
||||
name: "handles single value",
|
||||
input: []string{"registry"},
|
||||
expected: []string{"registry"},
|
||||
},
|
||||
{
|
||||
name: "handles empty input",
|
||||
input: []string{},
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
name: "preserves reverse alphabetical order",
|
||||
input: []string{"zebra,yankee,xray"},
|
||||
expected: []string{"zebra", "yankee", "xray"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := Flatten(tt.input)
|
||||
assert.Equal(t, tt.expected, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlattenAndSort(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "sorts comma-separated values",
|
||||
input: []string{"registry,docker,oci-dir"},
|
||||
expected: []string{"docker", "oci-dir", "registry"},
|
||||
},
|
||||
{
|
||||
name: "sorts across multiple entries",
|
||||
input: []string{"registry,docker", "oci-dir"},
|
||||
expected: []string{"docker", "oci-dir", "registry"},
|
||||
},
|
||||
{
|
||||
name: "trims whitespace and sorts",
|
||||
input: []string{" registry , docker ", " oci-dir "},
|
||||
expected: []string{"docker", "oci-dir", "registry"},
|
||||
},
|
||||
{
|
||||
name: "handles single value",
|
||||
input: []string{"registry"},
|
||||
expected: []string{"registry"},
|
||||
},
|
||||
{
|
||||
name: "handles empty input",
|
||||
input: []string{},
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
name: "sorts reverse alphabetical order",
|
||||
input: []string{"zebra,yankee,xray"},
|
||||
expected: []string{"xray", "yankee", "zebra"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := FlattenAndSort(tt.input)
|
||||
assert.Equal(t, tt.expected, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_enrichmentEnabled(t *testing.T) {
|
||||
tests := []struct {
|
||||
directives string
|
||||
@ -139,7 +231,7 @@ func Test_enrichmentEnabled(t *testing.T) {
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.directives, func(t *testing.T) {
|
||||
got := enrichmentEnabled(Flatten([]string{test.directives}), test.test)
|
||||
got := enrichmentEnabled(FlattenAndSort([]string{test.directives}), test.test)
|
||||
assert.Equal(t, test.expected, got)
|
||||
})
|
||||
}
|
||||
|
||||
@ -3,7 +3,9 @@ package options
|
||||
import "github.com/anchore/clio"
|
||||
|
||||
type pythonConfig struct {
|
||||
GuessUnpinnedRequirements bool `json:"guess-unpinned-requirements" yaml:"guess-unpinned-requirements" mapstructure:"guess-unpinned-requirements"`
|
||||
SearchRemoteLicenses *bool `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"`
|
||||
PypiBaseURL string `json:"pypi-base-url" yaml:"pypi-base-url" mapstructure:"pypi-base-url"`
|
||||
GuessUnpinnedRequirements *bool `json:"guess-unpinned-requirements" yaml:"guess-unpinned-requirements" mapstructure:"guess-unpinned-requirements"`
|
||||
}
|
||||
|
||||
var _ interface {
|
||||
@ -11,6 +13,8 @@ var _ interface {
|
||||
} = (*pythonConfig)(nil)
|
||||
|
||||
func (o *pythonConfig) DescribeFields(descriptions clio.FieldDescriptionSet) {
|
||||
descriptions.Add(&o.SearchRemoteLicenses, `enables Syft to use the network to fill in more detailed license information`)
|
||||
descriptions.Add(&o.PypiBaseURL, `base Pypi url to use`)
|
||||
descriptions.Add(&o.GuessUnpinnedRequirements, `when running across entries in requirements.txt that do not specify a specific version
|
||||
(e.g. "sqlalchemy >= 1.0.0, <= 2.0.0, != 3.0.0, <= 3.0.0"), attempt to guess what the version could
|
||||
be based on the version requirements specified (e.g. "1.0.0"). When enabled the lowest expressible version
|
||||
|
||||
@ -87,6 +87,7 @@ func TestPkgCoverageImage(t *testing.T) {
|
||||
definedPkgs.Remove(string(pkg.TerraformPkg))
|
||||
definedPkgs.Remove(string(pkg.PhpPeclPkg)) // we have coverage for pear instead
|
||||
definedPkgs.Remove(string(pkg.CondaPkg))
|
||||
definedPkgs.Remove(string(pkg.ModelPkg))
|
||||
|
||||
var cases []testCase
|
||||
cases = append(cases, commonTestCases...)
|
||||
@ -161,6 +162,7 @@ func TestPkgCoverageDirectory(t *testing.T) {
|
||||
definedPkgs.Remove(string(pkg.UnknownPkg))
|
||||
definedPkgs.Remove(string(pkg.CondaPkg))
|
||||
definedPkgs.Remove(string(pkg.PhpPeclPkg)) // this is covered as pear packages
|
||||
definedPkgs.Remove(string(pkg.ModelPkg))
|
||||
|
||||
// for directory scans we should not expect to see any of the following package types
|
||||
definedPkgs.Remove(string(pkg.KbPkg))
|
||||
|
||||
@ -30,10 +30,10 @@ func TestPackageDeduplication(t *testing.T) {
|
||||
locationCount: map[string]int{
|
||||
"basesystem-11-13.el9": 5, // in all layers
|
||||
"curl-minimal-7.76.1-26.el9_3.2.0.1": 2, // base + wget layer
|
||||
"curl-minimal-7.76.1-31.el9": 3, // curl upgrade layer + all above layers
|
||||
"curl-minimal-7.76.1-31.el9_6.1": 3, // curl upgrade layer + all above layers
|
||||
"wget-1.21.1-8.el9_4": 4, // wget + all above layers
|
||||
"vsftpd-3.0.5-6.el9": 2, // vsftpd + all above layers
|
||||
"httpd-2.4.62-4.el9": 1, // last layer
|
||||
"httpd-2.4.62-4.el9_6.4": 1, // last layer
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -47,11 +47,11 @@ func TestPackageDeduplication(t *testing.T) {
|
||||
"httpd": 1, // rpm, binary is now excluded by overlap
|
||||
},
|
||||
locationCount: map[string]int{
|
||||
"basesystem-11-13.el9": 1,
|
||||
"curl-minimal-7.76.1-31.el9": 1, // upgrade
|
||||
"wget-1.21.1-8.el9_4": 1,
|
||||
"vsftpd-3.0.5-6.el9": 1,
|
||||
"httpd-2.4.62-4.el9": 1,
|
||||
"basesystem-11-13.el9": 1,
|
||||
"curl-minimal-7.76.1-31.el9_6.1": 1, // upgrade
|
||||
"wget-1.21.1-8.el9_4": 1,
|
||||
"vsftpd-3.0.5-6.el9": 1,
|
||||
"httpd-2.4.62-4.el9_6.4": 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@ -7,16 +7,16 @@ FROM --platform=linux/amd64 rockylinux:9.3.20231119@sha256:d644d203142cd5b54ad2a
|
||||
# copying the RPM DB from each stage to a final stage in separate layers. This will result in a much smaller image.
|
||||
|
||||
FROM base AS stage1
|
||||
RUN dnf install -y wget
|
||||
RUN dnf install -y wget-1.21.1-8.el9_4
|
||||
|
||||
FROM stage1 AS stage2
|
||||
RUN dnf update -y curl-minimal
|
||||
RUN dnf update -y curl-minimal-7.76.1-31.el9_6.1
|
||||
|
||||
FROM stage2 AS stage3
|
||||
RUN dnf install -y vsftpd
|
||||
RUN dnf install -y vsftpd-3.0.5-6.el9
|
||||
|
||||
FROM stage3 AS stage4
|
||||
RUN dnf install -y httpd
|
||||
RUN dnf install -y httpd-2.4.62-4.el9_6.4
|
||||
|
||||
FROM scratch
|
||||
|
||||
|
||||
41
go.mod
41
go.mod
@ -11,7 +11,6 @@ require (
|
||||
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
|
||||
github.com/acobaugh/osrelease v0.1.0
|
||||
github.com/adrg/xdg v0.5.3
|
||||
github.com/anchore/archiver/v3 v3.5.3-0.20241210171143-5b1d8d1c7c51
|
||||
github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9
|
||||
github.com/anchore/clio v0.0.0-20250319180342-2cfe4b0cb716
|
||||
github.com/anchore/fangs v0.0.0-20250319222917-446a1e748ec2
|
||||
@ -58,14 +57,14 @@ require (
|
||||
github.com/hashicorp/hcl/v2 v2.24.0
|
||||
github.com/iancoleman/strcase v0.3.0
|
||||
github.com/invopop/jsonschema v0.7.0
|
||||
github.com/jedib0t/go-pretty/v6 v6.6.9
|
||||
github.com/jedib0t/go-pretty/v6 v6.7.1
|
||||
github.com/jinzhu/copier v0.4.0
|
||||
github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953
|
||||
github.com/magiconair/properties v1.8.10
|
||||
github.com/mholt/archives v0.1.5
|
||||
github.com/moby/sys/mountinfo v0.7.2
|
||||
github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1
|
||||
github.com/olekukonko/tablewriter v1.0.9
|
||||
github.com/olekukonko/tablewriter v1.1.1
|
||||
github.com/opencontainers/go-digest v1.0.0
|
||||
github.com/pelletier/go-toml v1.9.5
|
||||
github.com/quasilyte/go-ruleguard/dsl v0.3.23
|
||||
@ -90,7 +89,7 @@ require (
|
||||
go.uber.org/goleak v1.3.0
|
||||
go.yaml.in/yaml/v3 v3.0.4
|
||||
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b
|
||||
golang.org/x/mod v0.29.0
|
||||
golang.org/x/mod v0.30.0
|
||||
golang.org/x/net v0.46.0
|
||||
modernc.org/sqlite v1.40.0
|
||||
)
|
||||
@ -131,7 +130,7 @@ require (
|
||||
github.com/charmbracelet/x/term v0.2.1 // indirect
|
||||
github.com/cloudflare/circl v1.6.1 // indirect
|
||||
github.com/containerd/cgroups v1.1.0 // indirect
|
||||
github.com/containerd/containerd v1.7.28 // indirect
|
||||
github.com/containerd/containerd v1.7.29 // indirect
|
||||
github.com/containerd/containerd/api v1.8.0 // indirect
|
||||
github.com/containerd/continuity v0.4.4 // indirect
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
@ -142,7 +141,7 @@ require (
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect
|
||||
github.com/containerd/ttrpc v1.2.7 // indirect
|
||||
github.com/containerd/typeurl/v2 v2.2.0 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.4.1 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.6.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/docker/cli v28.5.1+incompatible // indirect
|
||||
github.com/docker/distribution v2.8.3+incompatible // indirect
|
||||
@ -168,7 +167,6 @@ require (
|
||||
github.com/goccy/go-yaml v1.18.0
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e // indirect
|
||||
github.com/google/s2a-go v0.1.8 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
|
||||
@ -191,7 +189,7 @@ require (
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
||||
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
|
||||
github.com/mikelolasagasti/xz v1.0.1 // indirect
|
||||
github.com/minio/minlz v1.0.1 // indirect
|
||||
@ -209,13 +207,9 @@ require (
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/ncruces/go-strftime v0.1.9 // indirect
|
||||
github.com/nwaples/rardecode v1.1.3 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.2.0 // indirect
|
||||
github.com/olekukonko/errors v1.1.0 // indirect
|
||||
github.com/olekukonko/ll v0.0.9 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/opencontainers/runtime-spec v1.1.0 // indirect
|
||||
github.com/opencontainers/selinux v1.11.0 // indirect
|
||||
github.com/opencontainers/selinux v1.13.0 // indirect
|
||||
github.com/pborman/indent v1.2.1 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
||||
@ -270,7 +264,7 @@ require (
|
||||
golang.org/x/sys v0.37.0 // indirect
|
||||
golang.org/x/term v0.36.0 // indirect
|
||||
golang.org/x/text v0.30.0 // indirect
|
||||
golang.org/x/time v0.12.0 // indirect
|
||||
golang.org/x/time v0.14.0
|
||||
golang.org/x/tools v0.38.0
|
||||
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
|
||||
google.golang.org/api v0.203.0 // indirect
|
||||
@ -287,6 +281,12 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/cespare/xxhash/v2 v2.3.0
|
||||
github.com/gpustack/gguf-parser-go v0.22.1
|
||||
)
|
||||
|
||||
require (
|
||||
cyphar.com/go-pathrs v0.2.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.17 // indirect
|
||||
@ -305,7 +305,20 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 // indirect
|
||||
github.com/aws/smithy-go v1.22.4 // indirect
|
||||
github.com/clipperhouse/displaywidth v0.3.1 // indirect
|
||||
github.com/clipperhouse/stringish v0.1.1 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.2.0 // indirect
|
||||
github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.65 // indirect
|
||||
github.com/henvic/httpretty v0.1.4 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.2.0 // indirect
|
||||
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect
|
||||
github.com/olekukonko/errors v1.1.0 // indirect
|
||||
github.com/olekukonko/ll v0.1.2 // indirect
|
||||
github.com/smallnest/ringbuffer v0.0.0-20241116012123-461381446e3d // indirect
|
||||
gonum.org/v1/gonum v0.15.1 // indirect
|
||||
)
|
||||
|
||||
retract (
|
||||
|
||||
65
go.sum
65
go.sum
@ -59,6 +59,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
|
||||
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||
cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs=
|
||||
cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0=
|
||||
cyphar.com/go-pathrs v0.2.1 h1:9nx1vOgwVvX1mNBWDu93+vaceedpbsDqo+XuBGL40b8=
|
||||
cyphar.com/go-pathrs v0.2.1/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc=
|
||||
dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=
|
||||
dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
@ -108,8 +110,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/anchore/archiver/v3 v3.5.3-0.20241210171143-5b1d8d1c7c51 h1:yhk+P8lF3ZiROjmaVRao9WGTRo4b/wYjoKEiAHWrKwc=
|
||||
github.com/anchore/archiver/v3 v3.5.3-0.20241210171143-5b1d8d1c7c51/go.mod h1:nwuGSd7aZp0rtYt79YggCGafz1RYsclE7pi3fhLwvuw=
|
||||
github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9 h1:p0ZIe0htYOX284Y4axJaGBvXHU0VCCzLN5Wf5XbKStU=
|
||||
github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9/go.mod h1:3ZsFB9tzW3vl4gEiUeuSOMDnwroWxIxJelOOHUp8dSw=
|
||||
github.com/anchore/clio v0.0.0-20250319180342-2cfe4b0cb716 h1:2sIdYJlQESEnyk3Y0WD2vXWW5eD2iMz9Ev8fj1Z8LNA=
|
||||
@ -227,7 +227,6 @@ github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqy
|
||||
github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
@ -263,6 +262,12 @@ github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38
|
||||
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
|
||||
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/clipperhouse/displaywidth v0.3.1 h1:k07iN9gD32177o1y4O1jQMzbLdCrsGJh+blirVYybsk=
|
||||
github.com/clipperhouse/displaywidth v0.3.1/go.mod h1:tgLJKKyaDOCadywag3agw4snxS5kYEuYR6Y9+qWDDYM=
|
||||
github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
|
||||
github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
|
||||
github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY=
|
||||
github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0=
|
||||
github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
@ -277,8 +282,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
|
||||
github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=
|
||||
github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw=
|
||||
github.com/containerd/containerd v1.7.28 h1:Nsgm1AtcmEh4AHAJ4gGlNSaKgXiNccU270Dnf81FQ3c=
|
||||
github.com/containerd/containerd v1.7.28/go.mod h1:azUkWcOvHrWvaiUjSQH0fjzuHIwSPg1WL5PshGP4Szs=
|
||||
github.com/containerd/containerd v1.7.29 h1:90fWABQsaN9mJhGkoVnuzEY+o1XDPbg9BTC9QTAHnuE=
|
||||
github.com/containerd/containerd v1.7.29/go.mod h1:azUkWcOvHrWvaiUjSQH0fjzuHIwSPg1WL5PshGP4Szs=
|
||||
github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0=
|
||||
github.com/containerd/containerd/api v1.8.0/go.mod h1:dFv4lt6S20wTu/hMcP4350RL87qPWLVa/OHOwmmdnYc=
|
||||
github.com/containerd/continuity v0.4.4 h1:/fNVfTJ7wIl/YPMHjf+5H32uFhl63JucB34PlCpMKII=
|
||||
@ -304,8 +309,8 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s=
|
||||
github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI=
|
||||
github.com/cyphar/filepath-securejoin v0.6.0 h1:BtGB77njd6SVO6VztOHfPxKitJvd/VPT+OFBFMOi1Is=
|
||||
github.com/cyphar/filepath-securejoin v0.6.0/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc=
|
||||
github.com/dave/jennifer v1.7.1 h1:B4jJJDHelWcDhlRQxWeo0Npa/pYKBLrirAQoTN45txo=
|
||||
github.com/dave/jennifer v1.7.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc=
|
||||
github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@ -472,8 +477,6 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
|
||||
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
@ -541,6 +544,8 @@ github.com/gookit/assert v0.1.1/go.mod h1:jS5bmIVQZTIwk42uXl4lyj4iaaxx32tqH16CFj
|
||||
github.com/gookit/color v1.2.5/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg=
|
||||
github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA=
|
||||
github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs=
|
||||
github.com/gpustack/gguf-parser-go v0.22.1 h1:FRnEDWqT0Rcplr/R9ctCRSN2+3DhVsf6dnR5/i9JA4E=
|
||||
github.com/gpustack/gguf-parser-go v0.22.1/go.mod h1:y4TwTtDqFWTK+xvprOjRUh+dowgU2TKCX37vRKvGiZ0=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms=
|
||||
@ -590,6 +595,8 @@ github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn
|
||||
github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
|
||||
github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
|
||||
github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
|
||||
github.com/henvic/httpretty v0.1.4 h1:Jo7uwIRWVFxkqOnErcoYfH90o3ddQyVrSANeS4cxYmU=
|
||||
github.com/henvic/httpretty v0.1.4/go.mod h1:Dn60sQTZfbt2dYsdUSNsCljyF4AfdqnuJFDLJA1I4AM=
|
||||
github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI=
|
||||
github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
|
||||
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
|
||||
@ -609,14 +616,15 @@ github.com/invopop/jsonschema v0.7.0 h1:2vgQcBz1n256N+FpX3Jq7Y17AjYt46Ig3zIWyy77
|
||||
github.com/invopop/jsonschema v0.7.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
|
||||
github.com/jedib0t/go-pretty/v6 v6.6.9 h1:PQecJLK3L8ODuVyMe2223b61oRJjrKnmXAncbWTv9MY=
|
||||
github.com/jedib0t/go-pretty/v6 v6.6.9/go.mod h1:YwC5CE4fJ1HFUDeivSV1r//AmANFHyqczZk+U6BDALU=
|
||||
github.com/jedib0t/go-pretty/v6 v6.7.1 h1:bHDSsj93NuJ563hHuM7ohk/wpX7BmRFNIsVv1ssI2/M=
|
||||
github.com/jedib0t/go-pretty/v6 v6.7.1/go.mod h1:YwC5CE4fJ1HFUDeivSV1r//AmANFHyqczZk+U6BDALU=
|
||||
github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
|
||||
github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
@ -676,8 +684,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75 h1:P8UmIzZMYDR+NGImiFvErt6VWfIRPuGM+vyjiEdkmIw=
|
||||
github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
|
||||
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
|
||||
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
|
||||
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||
@ -722,9 +730,11 @@ github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcY
|
||||
github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA=
|
||||
github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
|
||||
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
|
||||
@ -741,16 +751,16 @@ github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1 h1:kpt9ZfKcm+
|
||||
github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1/go.mod h1:qgCw4bBKZX8qMgGeEZzGFVT3notl42dBjNqO2jut0M0=
|
||||
github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 h1:NHrXEjTNQY7P0Zfx1aMrNhpgxHmow66XQtm0aQLY0AE=
|
||||
github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8=
|
||||
github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc=
|
||||
github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
|
||||
github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A=
|
||||
github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
|
||||
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc=
|
||||
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0=
|
||||
github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM=
|
||||
github.com/olekukonko/errors v1.1.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y=
|
||||
github.com/olekukonko/ll v0.0.9 h1:Y+1YqDfVkqMWuEQMclsF9HUR5+a82+dxJuL1HHSRpxI=
|
||||
github.com/olekukonko/ll v0.0.9/go.mod h1:En+sEW0JNETl26+K8eZ6/W4UQ7CYSrrgg/EdIYT2H8g=
|
||||
github.com/olekukonko/tablewriter v1.0.9 h1:XGwRsYLC2bY7bNd93Dk51bcPZksWZmLYuaTHR0FqfL8=
|
||||
github.com/olekukonko/tablewriter v1.0.9/go.mod h1:5c+EBPeSqvXnLLgkm9isDdzR3wjfBkHR9Nhfp3NWrzo=
|
||||
github.com/olekukonko/ll v0.1.2 h1:lkg/k/9mlsy0SxO5aC+WEpbdT5K83ddnNhAepz7TQc0=
|
||||
github.com/olekukonko/ll v0.1.2/go.mod h1:b52bVQRRPObe+yyBl0TxNfhesL0nedD4Cht0/zx55Ew=
|
||||
github.com/olekukonko/tablewriter v1.1.1 h1:b3reP6GCfrHwmKkYwNRFh2rxidGHcT6cgxj/sHiDDx0=
|
||||
github.com/olekukonko/tablewriter v1.1.1/go.mod h1:De/bIcTF+gpBDB3Alv3fEsZA+9unTsSzAg/ZGADCtn4=
|
||||
github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
|
||||
github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
@ -759,8 +769,8 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw
|
||||
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
|
||||
github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg=
|
||||
github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
|
||||
github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU=
|
||||
github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec=
|
||||
github.com/opencontainers/selinux v1.13.0 h1:Zza88GWezyT7RLql12URvoxsbLfjFx988+LGaWfbL84=
|
||||
github.com/opencontainers/selinux v1.13.0/go.mod h1:XxWTed+A/s5NNq4GmYScVy+9jzXhGBVEOAyucdRUY8s=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
@ -809,7 +819,6 @@ github.com/quasilyte/go-ruleguard/dsl v0.3.23 h1:lxjt5B6ZCiBeeNO8/oQsegE6fLeCzuM
|
||||
github.com/quasilyte/go-ruleguard/dsl v0.3.23/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
@ -851,6 +860,8 @@ github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af h1:Sp5TG9f7K39yf
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8=
|
||||
github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY=
|
||||
github.com/smallnest/ringbuffer v0.0.0-20241116012123-461381446e3d h1:3VwvTjiRPA7cqtgOWddEL+JrcijMlXUmj99c/6YyZoY=
|
||||
github.com/smallnest/ringbuffer v0.0.0-20241116012123-461381446e3d/go.mod h1:tAG61zBM1DYRaGIPloumExGvScf08oHuo0kFoOqdbT0=
|
||||
github.com/sorairolake/lzip-go v0.3.8 h1:j5Q2313INdTA80ureWYRhX+1K78mUXfMoPZCw/ivWik=
|
||||
github.com/sorairolake/lzip-go v0.3.8/go.mod h1:JcBqGMV0frlxwrsE9sMWXDjqn3EeVf0/54YPsw66qkU=
|
||||
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
|
||||
@ -1061,8 +1072,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
|
||||
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
|
||||
golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
|
||||
golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@ -1239,8 +1250,8 @@ golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
|
||||
golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
@ -1304,6 +1315,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
|
||||
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
|
||||
gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0=
|
||||
gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
|
||||
@ -3,5 +3,9 @@ package internal
|
||||
const (
|
||||
// JSONSchemaVersion is the current schema version output by the JSON encoder
|
||||
// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
|
||||
JSONSchemaVersion = "16.0.41"
|
||||
JSONSchemaVersion = "16.1.0"
|
||||
|
||||
// Changelog
|
||||
// 16.1.0 - reformulated the python pdm fields (added "URL" and removed the unused "path" field).
|
||||
|
||||
)
|
||||
|
||||
@ -1,17 +1,40 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
"github.com/mholt/archives"
|
||||
|
||||
"github.com/anchore/archiver/v3"
|
||||
"github.com/anchore/syft/internal"
|
||||
)
|
||||
|
||||
// TraverseFilesInTar enumerates all paths stored within a tar archive using the visitor pattern.
|
||||
func TraverseFilesInTar(ctx context.Context, archivePath string, visitor archives.FileHandler) error {
|
||||
tarReader, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to open tar archive (%s): %w", archivePath, err)
|
||||
}
|
||||
defer internal.CloseAndLogError(tarReader, archivePath)
|
||||
|
||||
format, _, err := archives.Identify(ctx, archivePath, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to identify tar compression format: %w", err)
|
||||
}
|
||||
|
||||
extractor, ok := format.(archives.Extractor)
|
||||
if !ok {
|
||||
return fmt.Errorf("file format does not support extraction: %s", archivePath)
|
||||
}
|
||||
|
||||
return extractor.Extract(ctx, tarReader, visitor)
|
||||
}
|
||||
|
||||
// ExtractGlobsFromTarToUniqueTempFile extracts paths matching the given globs within the given archive to a temporary directory, returning file openers for each file extracted.
|
||||
func ExtractGlobsFromTarToUniqueTempFile(archivePath, dir string, globs ...string) (map[string]Opener, error) {
|
||||
func ExtractGlobsFromTarToUniqueTempFile(ctx context.Context, archivePath, dir string, globs ...string) (map[string]Opener, error) {
|
||||
results := make(map[string]Opener)
|
||||
|
||||
// don't allow for full traversal, only select traversal from given paths
|
||||
@ -19,9 +42,7 @@ func ExtractGlobsFromTarToUniqueTempFile(archivePath, dir string, globs ...strin
|
||||
return results, nil
|
||||
}
|
||||
|
||||
visitor := func(file archiver.File) error {
|
||||
defer file.Close()
|
||||
|
||||
visitor := func(_ context.Context, file archives.FileInfo) error {
|
||||
// ignore directories
|
||||
if file.IsDir() {
|
||||
return nil
|
||||
@ -43,7 +64,13 @@ func ExtractGlobsFromTarToUniqueTempFile(archivePath, dir string, globs ...strin
|
||||
// provides a ReadCloser. It is up to the caller to handle closing the file explicitly.
|
||||
defer tempFile.Close()
|
||||
|
||||
if err := safeCopy(tempFile, file.ReadCloser); err != nil {
|
||||
packedFile, err := file.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read file=%q from tar=%q: %w", file.NameInArchive, archivePath, err)
|
||||
}
|
||||
defer internal.CloseAndLogError(packedFile, archivePath)
|
||||
|
||||
if err := safeCopy(tempFile, packedFile); err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q for tar=%q: %w", file.Name(), archivePath, err)
|
||||
}
|
||||
|
||||
@ -52,7 +79,7 @@ func ExtractGlobsFromTarToUniqueTempFile(archivePath, dir string, globs ...strin
|
||||
return nil
|
||||
}
|
||||
|
||||
return results, archiver.Walk(archivePath, visitor)
|
||||
return results, TraverseFilesInTar(ctx, archivePath, visitor)
|
||||
}
|
||||
|
||||
func matchesAnyGlob(name string, globs ...string) bool {
|
||||
|
||||
@ -1,10 +1,12 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
"github.com/scylladb/go-set/strset"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
@ -14,22 +16,25 @@ import (
|
||||
type ZipFileManifest map[string]os.FileInfo
|
||||
|
||||
// NewZipFileManifest creates and returns a new ZipFileManifest populated with path and metadata from the given zip archive path.
|
||||
func NewZipFileManifest(archivePath string) (ZipFileManifest, error) {
|
||||
zipReader, err := OpenZip(archivePath)
|
||||
func NewZipFileManifest(ctx context.Context, archivePath string) (ZipFileManifest, error) {
|
||||
zipReader, err := os.Open(archivePath)
|
||||
manifest := make(ZipFileManifest)
|
||||
if err != nil {
|
||||
log.Debugf("unable to open zip archive (%s): %v", archivePath, err)
|
||||
return manifest, err
|
||||
}
|
||||
defer func() {
|
||||
err = zipReader.Close()
|
||||
if err != nil {
|
||||
if err = zipReader.Close(); err != nil {
|
||||
log.Debugf("unable to close zip archive (%s): %+v", archivePath, err)
|
||||
}
|
||||
}()
|
||||
|
||||
for _, file := range zipReader.File {
|
||||
manifest.Add(file.Name, file.FileInfo())
|
||||
err = archives.Zip{}.Extract(ctx, zipReader, func(_ context.Context, file archives.FileInfo) error {
|
||||
manifest.Add(file.NameInArchive, file.FileInfo)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return manifest, err
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path"
|
||||
@ -24,7 +25,7 @@ func TestNewZipFileManifest(t *testing.T) {
|
||||
|
||||
archiveFilePath := setupZipFileTest(t, sourceDirPath, false)
|
||||
|
||||
actual, err := NewZipFileManifest(archiveFilePath)
|
||||
actual, err := NewZipFileManifest(context.Background(), archiveFilePath)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to extract from unzip archive: %+v", err)
|
||||
}
|
||||
@ -59,7 +60,7 @@ func TestNewZip64FileManifest(t *testing.T) {
|
||||
sourceDirPath := path.Join(cwd, "test-fixtures", "zip-source")
|
||||
archiveFilePath := setupZipFileTest(t, sourceDirPath, true)
|
||||
|
||||
actual, err := NewZipFileManifest(archiveFilePath)
|
||||
actual, err := NewZipFileManifest(context.Background(), archiveFilePath)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to extract from unzip archive: %+v", err)
|
||||
}
|
||||
@ -99,7 +100,7 @@ func TestZipFileManifest_GlobMatch(t *testing.T) {
|
||||
|
||||
archiveFilePath := setupZipFileTest(t, sourceDirPath, false)
|
||||
|
||||
z, err := NewZipFileManifest(archiveFilePath)
|
||||
z, err := NewZipFileManifest(context.Background(), archiveFilePath)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to extract from unzip archive: %+v", err)
|
||||
}
|
||||
|
||||
@ -1,13 +1,15 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
)
|
||||
|
||||
@ -25,7 +27,7 @@ type errZipSlipDetected struct {
|
||||
}
|
||||
|
||||
func (e *errZipSlipDetected) Error() string {
|
||||
return fmt.Sprintf("paths are not allowed to resolve outside of the root prefix (%q). Destination: %q", e.Prefix, e.JoinArgs)
|
||||
return fmt.Sprintf("path traversal detected: paths are not allowed to resolve outside of the root prefix (%q). Destination: %q", e.Prefix, e.JoinArgs)
|
||||
}
|
||||
|
||||
type zipTraversalRequest map[string]struct{}
|
||||
@ -39,38 +41,34 @@ func newZipTraverseRequest(paths ...string) zipTraversalRequest {
|
||||
}
|
||||
|
||||
// TraverseFilesInZip enumerates all paths stored within a zip archive using the visitor pattern.
|
||||
func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths ...string) error {
|
||||
func TraverseFilesInZip(ctx context.Context, archivePath string, visitor archives.FileHandler, paths ...string) error {
|
||||
request := newZipTraverseRequest(paths...)
|
||||
|
||||
zipReader, err := OpenZip(archivePath)
|
||||
zipReader, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to open zip archive (%s): %w", archivePath, err)
|
||||
}
|
||||
defer func() {
|
||||
err = zipReader.Close()
|
||||
if err != nil {
|
||||
if err := zipReader.Close(); err != nil {
|
||||
log.Errorf("unable to close zip archive (%s): %+v", archivePath, err)
|
||||
}
|
||||
}()
|
||||
|
||||
for _, file := range zipReader.File {
|
||||
return archives.Zip{}.Extract(ctx, zipReader, func(ctx context.Context, file archives.FileInfo) error {
|
||||
// if no paths are given then assume that all files should be traversed
|
||||
if len(paths) > 0 {
|
||||
if _, ok := request[file.Name]; !ok {
|
||||
if _, ok := request[file.NameInArchive]; !ok {
|
||||
// this file path is not of interest
|
||||
continue
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if err = visitor(file); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return visitor(ctx, file)
|
||||
})
|
||||
}
|
||||
|
||||
// ExtractFromZipToUniqueTempFile extracts select paths for the given archive to a temporary directory, returning file openers for each file extracted.
|
||||
func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (map[string]Opener, error) {
|
||||
func ExtractFromZipToUniqueTempFile(ctx context.Context, archivePath, dir string, paths ...string) (map[string]Opener, error) {
|
||||
results := make(map[string]Opener)
|
||||
|
||||
// don't allow for full traversal, only select traversal from given paths
|
||||
@ -78,9 +76,8 @@ func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (m
|
||||
return results, nil
|
||||
}
|
||||
|
||||
visitor := func(file *zip.File) error {
|
||||
tempfilePrefix := filepath.Base(filepath.Clean(file.Name)) + "-"
|
||||
|
||||
visitor := func(_ context.Context, file archives.FileInfo) error {
|
||||
tempfilePrefix := filepath.Base(filepath.Clean(file.NameInArchive)) + "-"
|
||||
tempFile, err := os.CreateTemp(dir, tempfilePrefix)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create temp file: %w", err)
|
||||
@ -92,33 +89,32 @@ func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (m
|
||||
|
||||
zippedFile, err := file.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
return fmt.Errorf("unable to read file=%q from zip=%q: %w", file.NameInArchive, archivePath, err)
|
||||
}
|
||||
defer func() {
|
||||
err := zippedFile.Close()
|
||||
if err != nil {
|
||||
log.Errorf("unable to close source file=%q from zip=%q: %+v", file.Name, archivePath, err)
|
||||
if err := zippedFile.Close(); err != nil {
|
||||
log.Errorf("unable to close source file=%q from zip=%q: %+v", file.NameInArchive, archivePath, err)
|
||||
}
|
||||
}()
|
||||
|
||||
if file.FileInfo().IsDir() {
|
||||
return fmt.Errorf("unable to extract directories, only files: %s", file.Name)
|
||||
if file.IsDir() {
|
||||
return fmt.Errorf("unable to extract directories, only files: %s", file.NameInArchive)
|
||||
}
|
||||
|
||||
if err := safeCopy(tempFile, zippedFile); err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.Name, archivePath, err)
|
||||
return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.NameInArchive, archivePath, err)
|
||||
}
|
||||
|
||||
results[file.Name] = Opener{path: tempFile.Name()}
|
||||
results[file.NameInArchive] = Opener{path: tempFile.Name()}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return results, TraverseFilesInZip(archivePath, visitor, paths...)
|
||||
return results, TraverseFilesInZip(ctx, archivePath, visitor, paths...)
|
||||
}
|
||||
|
||||
// ContentsFromZip extracts select paths for the given archive and returns a set of string contents for each path.
|
||||
func ContentsFromZip(archivePath string, paths ...string) (map[string]string, error) {
|
||||
func ContentsFromZip(ctx context.Context, archivePath string, paths ...string) (map[string]string, error) {
|
||||
results := make(map[string]string)
|
||||
|
||||
// don't allow for full traversal, only select traversal from given paths
|
||||
@ -126,37 +122,38 @@ func ContentsFromZip(archivePath string, paths ...string) (map[string]string, er
|
||||
return results, nil
|
||||
}
|
||||
|
||||
visitor := func(file *zip.File) error {
|
||||
visitor := func(_ context.Context, file archives.FileInfo) error {
|
||||
zippedFile, err := file.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
return fmt.Errorf("unable to read file=%q from zip=%q: %w", file.NameInArchive, archivePath, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := zippedFile.Close(); err != nil {
|
||||
log.Errorf("unable to close source file=%q from zip=%q: %+v", file.NameInArchive, archivePath, err)
|
||||
}
|
||||
}()
|
||||
|
||||
if file.FileInfo().IsDir() {
|
||||
return fmt.Errorf("unable to extract directories, only files: %s", file.Name)
|
||||
if file.IsDir() {
|
||||
return fmt.Errorf("unable to extract directories, only files: %s", file.NameInArchive)
|
||||
}
|
||||
|
||||
var buffer bytes.Buffer
|
||||
if err := safeCopy(&buffer, zippedFile); err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.Name, archivePath, err)
|
||||
return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.NameInArchive, archivePath, err)
|
||||
}
|
||||
|
||||
results[file.Name] = buffer.String()
|
||||
results[file.NameInArchive] = buffer.String()
|
||||
|
||||
err = zippedFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to close source file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return results, TraverseFilesInZip(archivePath, visitor, paths...)
|
||||
return results, TraverseFilesInZip(ctx, archivePath, visitor, paths...)
|
||||
}
|
||||
|
||||
// UnzipToDir extracts a zip archive to a target directory.
|
||||
func UnzipToDir(archivePath, targetDir string) error {
|
||||
visitor := func(file *zip.File) error {
|
||||
joinedPath, err := safeJoin(targetDir, file.Name)
|
||||
func UnzipToDir(ctx context.Context, archivePath, targetDir string) error {
|
||||
visitor := func(_ context.Context, file archives.FileInfo) error {
|
||||
joinedPath, err := SafeJoin(targetDir, file.NameInArchive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -164,11 +161,11 @@ func UnzipToDir(archivePath, targetDir string) error {
|
||||
return extractSingleFile(file, joinedPath, archivePath)
|
||||
}
|
||||
|
||||
return TraverseFilesInZip(archivePath, visitor)
|
||||
return TraverseFilesInZip(ctx, archivePath, visitor)
|
||||
}
|
||||
|
||||
// safeJoin ensures that any destinations do not resolve to a path above the prefix path.
|
||||
func safeJoin(prefix string, dest ...string) (string, error) {
|
||||
// SafeJoin ensures that any destinations do not resolve to a path above the prefix path.
|
||||
func SafeJoin(prefix string, dest ...string) (string, error) {
|
||||
joinResult := filepath.Join(append([]string{prefix}, dest...)...)
|
||||
cleanJoinResult := filepath.Clean(joinResult)
|
||||
if !strings.HasPrefix(cleanJoinResult, filepath.Clean(prefix)) {
|
||||
@ -181,13 +178,18 @@ func safeJoin(prefix string, dest ...string) (string, error) {
|
||||
return joinResult, nil
|
||||
}
|
||||
|
||||
func extractSingleFile(file *zip.File, expandedFilePath, archivePath string) error {
|
||||
func extractSingleFile(file archives.FileInfo, expandedFilePath, archivePath string) error {
|
||||
zippedFile, err := file.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
return fmt.Errorf("unable to read file=%q from zip=%q: %w", file.NameInArchive, archivePath, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := zippedFile.Close(); err != nil {
|
||||
log.Errorf("unable to close source file=%q from zip=%q: %+v", file.NameInArchive, archivePath, err)
|
||||
}
|
||||
}()
|
||||
|
||||
if file.FileInfo().IsDir() {
|
||||
if file.IsDir() {
|
||||
err = os.MkdirAll(expandedFilePath, file.Mode())
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create dir=%q from zip=%q: %w", expandedFilePath, archivePath, err)
|
||||
@ -202,20 +204,16 @@ func extractSingleFile(file *zip.File, expandedFilePath, archivePath string) err
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create dest file=%q from zip=%q: %w", expandedFilePath, archivePath, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := outputFile.Close(); err != nil {
|
||||
log.Errorf("unable to close dest file=%q from zip=%q: %+v", outputFile.Name(), archivePath, err)
|
||||
}
|
||||
}()
|
||||
|
||||
if err := safeCopy(outputFile, zippedFile); err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q to dest=%q for zip=%q: %w", file.Name, outputFile.Name(), archivePath, err)
|
||||
}
|
||||
|
||||
err = outputFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to close dest file=%q from zip=%q: %w", outputFile.Name(), archivePath, err)
|
||||
return fmt.Errorf("unable to copy source=%q to dest=%q for zip=%q: %w", file.NameInArchive, outputFile.Name(), archivePath, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = zippedFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to close source file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -4,6 +4,8 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@ -17,6 +19,7 @@ import (
|
||||
|
||||
"github.com/go-test/deep"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func equal(r1, r2 io.Reader) (bool, error) {
|
||||
@ -55,7 +58,7 @@ func TestUnzipToDir(t *testing.T) {
|
||||
expectedPaths := len(expectedZipArchiveEntries)
|
||||
observedPaths := 0
|
||||
|
||||
err = UnzipToDir(archiveFilePath, unzipDestinationDir)
|
||||
err = UnzipToDir(context.Background(), archiveFilePath, unzipDestinationDir)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to unzip archive: %+v", err)
|
||||
}
|
||||
@ -145,7 +148,7 @@ func TestContentsFromZip(t *testing.T) {
|
||||
paths = append(paths, p)
|
||||
}
|
||||
|
||||
actual, err := ContentsFromZip(archivePath, paths...)
|
||||
actual, err := ContentsFromZip(context.Background(), archivePath, paths...)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to extract from unzip archive: %+v", err)
|
||||
}
|
||||
@ -307,9 +310,528 @@ func TestSafeJoin(t *testing.T) {
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(fmt.Sprintf("%+v:%+v", test.prefix, test.args), func(t *testing.T) {
|
||||
actual, err := safeJoin(test.prefix, test.args...)
|
||||
actual, err := SafeJoin(test.prefix, test.args...)
|
||||
test.errAssertion(t, err)
|
||||
assert.Equal(t, test.expected, actual)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSymlinkProtection demonstrates that SafeJoin protects against symlink-based
|
||||
// directory traversal attacks by validating that archive entry paths cannot escape
|
||||
// the extraction directory.
|
||||
func TestSafeJoin_SymlinkProtection(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
archivePath string // Path as it would appear in the archive
|
||||
expectError bool
|
||||
description string
|
||||
}{
|
||||
{
|
||||
name: "path traversal via ../",
|
||||
archivePath: "../../../outside/file.txt",
|
||||
expectError: true,
|
||||
description: "Archive entry with ../ trying to escape extraction dir",
|
||||
},
|
||||
{
|
||||
name: "absolute path symlink target",
|
||||
archivePath: "../../../sensitive.txt",
|
||||
expectError: true,
|
||||
description: "Simulates symlink pointing outside via relative path",
|
||||
},
|
||||
{
|
||||
name: "safe relative path within extraction dir",
|
||||
archivePath: "subdir/safe.txt",
|
||||
expectError: false,
|
||||
description: "Normal file path that stays within extraction directory",
|
||||
},
|
||||
{
|
||||
name: "safe path with internal ../",
|
||||
archivePath: "dir1/../dir2/file.txt",
|
||||
expectError: false,
|
||||
description: "Path with ../ that still resolves within extraction dir",
|
||||
},
|
||||
{
|
||||
name: "deeply nested traversal",
|
||||
archivePath: "../../../../../../tmp/evil.txt",
|
||||
expectError: true,
|
||||
description: "Multiple levels of ../ trying to escape",
|
||||
},
|
||||
{
|
||||
name: "single parent directory escape",
|
||||
archivePath: "../",
|
||||
expectError: true,
|
||||
description: "Simple one-level escape attempt",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Create temp directories to simulate extraction scenario
|
||||
tmpDir := t.TempDir()
|
||||
extractDir := filepath.Join(tmpDir, "extract")
|
||||
outsideDir := filepath.Join(tmpDir, "outside")
|
||||
|
||||
require.NoError(t, os.MkdirAll(extractDir, 0755))
|
||||
require.NoError(t, os.MkdirAll(outsideDir, 0755))
|
||||
|
||||
// Create a file outside extraction dir that an attacker might target
|
||||
outsideFile := filepath.Join(outsideDir, "sensitive.txt")
|
||||
require.NoError(t, os.WriteFile(outsideFile, []byte("sensitive data"), 0644))
|
||||
|
||||
// Test SafeJoin - this is what happens when processing archive entries
|
||||
result, err := SafeJoin(extractDir, tt.archivePath)
|
||||
|
||||
if tt.expectError {
|
||||
// Should block malicious paths
|
||||
require.Error(t, err, "Expected SafeJoin to reject malicious path")
|
||||
var zipSlipErr *errZipSlipDetected
|
||||
assert.ErrorAs(t, err, &zipSlipErr, "Error should be errZipSlipDetected type")
|
||||
assert.Empty(t, result, "Result should be empty for blocked paths")
|
||||
} else {
|
||||
// Should allow safe paths
|
||||
require.NoError(t, err, "Expected SafeJoin to allow safe path")
|
||||
assert.NotEmpty(t, result, "Result should not be empty for safe paths")
|
||||
assert.True(t, strings.HasPrefix(filepath.Clean(result), filepath.Clean(extractDir)),
|
||||
"Safe path should resolve within extraction directory")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnzipToDir_SymlinkAttacks tests UnzipToDir function with malicious ZIP archives
|
||||
// containing symlink entries that attempt path traversal attacks.
|
||||
//
|
||||
// EXPECTED BEHAVIOR: UnzipToDir should either:
|
||||
// 1. Detect and reject symlinks explicitly with a security error, OR
|
||||
// 2. Extract them safely (library converts symlinks to regular files)
|
||||
func TestUnzipToDir_SymlinkAttacks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
symlinkName string
|
||||
fileName string
|
||||
errContains string
|
||||
}{
|
||||
{
|
||||
name: "direct symlink to outside directory",
|
||||
symlinkName: "evil_link",
|
||||
fileName: "evil_link/payload.txt",
|
||||
errContains: "not a directory", // attempt to write through symlink leaf (which is not a directory)
|
||||
},
|
||||
{
|
||||
name: "directory symlink attack",
|
||||
symlinkName: "safe_dir/link",
|
||||
fileName: "safe_dir/link/payload.txt",
|
||||
errContains: "not a directory", // attempt to write through symlink (which is not a directory)
|
||||
},
|
||||
{
|
||||
name: "symlink without payload file",
|
||||
symlinkName: "standalone_link",
|
||||
fileName: "", // no payload file
|
||||
errContains: "", // no error expected, symlink without payload is safe
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
// create outside target directory
|
||||
outsideDir := filepath.Join(tempDir, "outside_target")
|
||||
require.NoError(t, os.MkdirAll(outsideDir, 0755))
|
||||
|
||||
// create extraction directory
|
||||
extractDir := filepath.Join(tempDir, "extract")
|
||||
require.NoError(t, os.MkdirAll(extractDir, 0755))
|
||||
|
||||
maliciousZip := createMaliciousZipWithSymlink(t, tempDir, tt.symlinkName, outsideDir, tt.fileName)
|
||||
|
||||
err := UnzipToDir(context.Background(), maliciousZip, extractDir)
|
||||
|
||||
// check error expectations
|
||||
if tt.errContains != "" {
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), tt.errContains)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
analyzeExtractionDirectory(t, extractDir)
|
||||
|
||||
// check if payload file escaped extraction directory
|
||||
if tt.fileName != "" {
|
||||
maliciousFile := filepath.Join(outsideDir, filepath.Base(tt.fileName))
|
||||
checkFileOutsideExtraction(t, maliciousFile)
|
||||
}
|
||||
|
||||
// check if symlink was created pointing outside
|
||||
symlinkPath := filepath.Join(extractDir, tt.symlinkName)
|
||||
checkSymlinkCreation(t, symlinkPath, extractDir, outsideDir)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestContentsFromZip_SymlinkAttacks tests the ContentsFromZip function with malicious
|
||||
// ZIP archives containing symlink entries.
|
||||
//
|
||||
// EXPECTED BEHAVIOR: ContentsFromZip should either:
|
||||
// 1. Reject symlinks explicitly, OR
|
||||
// 2. Return empty content for symlinks (library behavior)
|
||||
//
|
||||
// Though ContentsFromZip doesn't write to disk, but if symlinks are followed, it could read sensitive
|
||||
// files from outside the archive.
|
||||
func TestContentsFromZip_SymlinkAttacks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
symlinkName string
|
||||
symlinkTarget string
|
||||
requestPath string
|
||||
errContains string
|
||||
}{
|
||||
{
|
||||
name: "request symlink entry directly",
|
||||
symlinkName: "evil_link",
|
||||
symlinkTarget: "/etc/hosts", // attempt to read sensitive file
|
||||
requestPath: "evil_link",
|
||||
errContains: "", // no error expected - library returns symlink metadata
|
||||
},
|
||||
{
|
||||
name: "symlink in nested directory",
|
||||
symlinkName: "nested/link",
|
||||
symlinkTarget: "/etc/hosts",
|
||||
requestPath: "nested/link",
|
||||
errContains: "", // no error expected - library returns symlink metadata
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
// create malicious ZIP with symlink entry (no payload file needed)
|
||||
maliciousZip := createMaliciousZipWithSymlink(t, tempDir, tt.symlinkName, tt.symlinkTarget, "")
|
||||
|
||||
contents, err := ContentsFromZip(context.Background(), maliciousZip, tt.requestPath)
|
||||
|
||||
// check error expectations
|
||||
if tt.errContains != "" {
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), tt.errContains)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
|
||||
// verify symlink handling - library should return symlink target as content (metadata)
|
||||
content, found := contents[tt.requestPath]
|
||||
require.True(t, found, "symlink entry should be found in results")
|
||||
|
||||
// verify symlink was NOT followed (content should be target path or empty)
|
||||
if content != "" && content != tt.symlinkTarget {
|
||||
// content is not empty and not the symlink target - check if actual file was read
|
||||
if _, statErr := os.Stat(tt.symlinkTarget); statErr == nil {
|
||||
targetContent, readErr := os.ReadFile(tt.symlinkTarget)
|
||||
if readErr == nil && string(targetContent) == content {
|
||||
t.Errorf("critical issue!... symlink was FOLLOWED and external file content was read!")
|
||||
t.Logf(" symlink: %s → %s", tt.requestPath, tt.symlinkTarget)
|
||||
t.Logf(" content length: %d bytes", len(content))
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestExtractFromZipToUniqueTempFile_SymlinkAttacks tests the ExtractFromZipToUniqueTempFile
|
||||
// function with malicious ZIP archives containing symlink entries.
|
||||
//
|
||||
// EXPECTED BEHAVIOR: ExtractFromZipToUniqueTempFile should either:
|
||||
// 1. Reject symlinks explicitly, OR
|
||||
// 2. Extract them safely (library converts to empty files, filepath.Base sanitizes names)
|
||||
//
|
||||
// This function uses filepath.Base() on the archive entry name for temp file prefix and
|
||||
// os.CreateTemp() which creates files in the specified directory, so it should be protected.
|
||||
func TestExtractFromZipToUniqueTempFile_SymlinkAttacks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
symlinkName string
|
||||
symlinkTarget string
|
||||
requestPath string
|
||||
errContains string
|
||||
}{
|
||||
{
|
||||
name: "extract symlink entry to temp file",
|
||||
symlinkName: "evil_link",
|
||||
symlinkTarget: "/etc/passwd",
|
||||
requestPath: "evil_link",
|
||||
errContains: "", // no error expected - library extracts symlink metadata
|
||||
},
|
||||
{
|
||||
name: "extract nested symlink",
|
||||
symlinkName: "nested/dir/link",
|
||||
symlinkTarget: "/tmp/outside",
|
||||
requestPath: "nested/dir/link",
|
||||
errContains: "", // no error expected
|
||||
},
|
||||
{
|
||||
name: "extract path traversal symlink name",
|
||||
symlinkName: "../../escape",
|
||||
symlinkTarget: "/tmp/outside",
|
||||
requestPath: "../../escape",
|
||||
errContains: "", // no error expected - filepath.Base sanitizes name
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
maliciousZip := createMaliciousZipWithSymlink(t, tempDir, tt.symlinkName, tt.symlinkTarget, "")
|
||||
|
||||
// create temp directory for extraction
|
||||
extractTempDir := filepath.Join(tempDir, "temp_extract")
|
||||
require.NoError(t, os.MkdirAll(extractTempDir, 0755))
|
||||
|
||||
openers, err := ExtractFromZipToUniqueTempFile(context.Background(), maliciousZip, extractTempDir, tt.requestPath)
|
||||
|
||||
// check error expectations
|
||||
if tt.errContains != "" {
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), tt.errContains)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
|
||||
// verify symlink was extracted
|
||||
opener, found := openers[tt.requestPath]
|
||||
require.True(t, found, "symlink entry should be extracted")
|
||||
|
||||
// verify temp file is within temp directory
|
||||
tempFilePath := opener.path
|
||||
cleanTempDir := filepath.Clean(extractTempDir)
|
||||
cleanTempFile := filepath.Clean(tempFilePath)
|
||||
require.True(t, strings.HasPrefix(cleanTempFile, cleanTempDir),
|
||||
"temp file must be within temp directory: %s not in %s", cleanTempFile, cleanTempDir)
|
||||
|
||||
// verify symlink was NOT followed (content should be target path or empty)
|
||||
f, openErr := opener.Open()
|
||||
require.NoError(t, openErr)
|
||||
defer f.Close()
|
||||
|
||||
content, readErr := io.ReadAll(f)
|
||||
require.NoError(t, readErr)
|
||||
|
||||
// check if symlink was followed (content matches actual file)
|
||||
if len(content) > 0 && string(content) != tt.symlinkTarget {
|
||||
if _, statErr := os.Stat(tt.symlinkTarget); statErr == nil {
|
||||
targetContent, readErr := os.ReadFile(tt.symlinkTarget)
|
||||
if readErr == nil && string(targetContent) == string(content) {
|
||||
t.Errorf("critical issue!... symlink was FOLLOWED and external file content was copied!")
|
||||
t.Logf(" symlink: %s → %s", tt.requestPath, tt.symlinkTarget)
|
||||
t.Logf(" content length: %d bytes", len(content))
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// forensicFindings contains the results of analyzing an extraction directory
|
||||
type forensicFindings struct {
|
||||
symlinksFound []forensicSymlink
|
||||
regularFiles []string
|
||||
directories []string
|
||||
symlinkVulnerabilities []string
|
||||
}
|
||||
|
||||
type forensicSymlink struct {
|
||||
path string
|
||||
target string
|
||||
escapesExtraction bool
|
||||
resolvedPath string
|
||||
}
|
||||
|
||||
// analyzeExtractionDirectory walks the extraction directory and detects symlinks that point
|
||||
// outside the extraction directory. It is silent unless vulnerabilities are found.
|
||||
func analyzeExtractionDirectory(t *testing.T, extractDir string) forensicFindings {
|
||||
t.Helper()
|
||||
|
||||
findings := forensicFindings{}
|
||||
|
||||
filepath.Walk(extractDir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
// only log if there's an error walking the directory
|
||||
t.Logf("Error walking %s: %v", path, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
relPath := strings.TrimPrefix(path, extractDir+"/")
|
||||
if relPath == "" {
|
||||
relPath = "."
|
||||
}
|
||||
|
||||
// use Lstat to detect symlinks without following them
|
||||
linfo, lerr := os.Lstat(path)
|
||||
if lerr == nil && linfo.Mode()&os.ModeSymlink != 0 {
|
||||
target, _ := os.Readlink(path)
|
||||
|
||||
// resolve to see where it actually points
|
||||
var resolvedPath string
|
||||
var escapesExtraction bool
|
||||
|
||||
if filepath.IsAbs(target) {
|
||||
// absolute symlink
|
||||
resolvedPath = target
|
||||
cleanExtractDir := filepath.Clean(extractDir)
|
||||
escapesExtraction = !strings.HasPrefix(filepath.Clean(target), cleanExtractDir)
|
||||
|
||||
if escapesExtraction {
|
||||
t.Errorf("critical issue!... absolute symlink created: %s → %s", relPath, target)
|
||||
t.Logf(" this symlink points outside the extraction directory")
|
||||
findings.symlinkVulnerabilities = append(findings.symlinkVulnerabilities,
|
||||
fmt.Sprintf("absolute symlink: %s → %s", relPath, target))
|
||||
}
|
||||
} else {
|
||||
// relative symlink - resolve it
|
||||
resolvedPath = filepath.Join(filepath.Dir(path), target)
|
||||
cleanResolved := filepath.Clean(resolvedPath)
|
||||
cleanExtractDir := filepath.Clean(extractDir)
|
||||
|
||||
escapesExtraction = !strings.HasPrefix(cleanResolved, cleanExtractDir)
|
||||
|
||||
if escapesExtraction {
|
||||
t.Errorf("critical issue!... symlink escapes extraction dir: %s → %s", relPath, target)
|
||||
t.Logf(" symlink resolves to: %s (outside extraction directory)", cleanResolved)
|
||||
findings.symlinkVulnerabilities = append(findings.symlinkVulnerabilities,
|
||||
fmt.Sprintf("relative symlink escape: %s → %s (resolves to %s)", relPath, target, cleanResolved))
|
||||
}
|
||||
}
|
||||
|
||||
findings.symlinksFound = append(findings.symlinksFound, forensicSymlink{
|
||||
path: relPath,
|
||||
target: target,
|
||||
escapesExtraction: escapesExtraction,
|
||||
resolvedPath: resolvedPath,
|
||||
})
|
||||
} else {
|
||||
// regular file or directory - collect silently
|
||||
if info.IsDir() {
|
||||
findings.directories = append(findings.directories, relPath)
|
||||
} else {
|
||||
findings.regularFiles = append(findings.regularFiles, relPath)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
return findings
|
||||
}
|
||||
|
||||
// checkFileOutsideExtraction checks if a file was written outside the extraction directory.
|
||||
// Returns true if the file exists (vulnerability), false otherwise. Silent on success.
|
||||
func checkFileOutsideExtraction(t *testing.T, filePath string) bool {
|
||||
t.Helper()
|
||||
|
||||
if stat, err := os.Stat(filePath); err == nil {
|
||||
content, _ := os.ReadFile(filePath)
|
||||
t.Errorf("critical issue!... file written OUTSIDE extraction directory!")
|
||||
t.Logf(" location: %s", filePath)
|
||||
t.Logf(" size: %d bytes", stat.Size())
|
||||
t.Logf(" content: %s", string(content))
|
||||
t.Logf(" ...this means an attacker can write files to arbitrary locations on the filesystem")
|
||||
return true
|
||||
}
|
||||
// no file found outside extraction directory...
|
||||
return false
|
||||
}
|
||||
|
||||
// checkSymlinkCreation verifies if a symlink was created at the expected path and reports
|
||||
// whether it points outside the extraction directory. Silent unless a symlink is found.
|
||||
func checkSymlinkCreation(t *testing.T, symlinkPath, extractDir, expectedTarget string) bool {
|
||||
t.Helper()
|
||||
|
||||
if linfo, err := os.Lstat(symlinkPath); err == nil {
|
||||
if linfo.Mode()&os.ModeSymlink != 0 {
|
||||
target, _ := os.Readlink(symlinkPath)
|
||||
|
||||
if expectedTarget != "" && target == expectedTarget {
|
||||
t.Errorf("critical issue!... symlink pointing outside extraction dir was created!")
|
||||
t.Logf(" Symlink: %s → %s", symlinkPath, target)
|
||||
return true
|
||||
}
|
||||
|
||||
// Check if it escapes even if target doesn't match expected
|
||||
if filepath.IsAbs(target) {
|
||||
cleanExtractDir := filepath.Clean(extractDir)
|
||||
if !strings.HasPrefix(filepath.Clean(target), cleanExtractDir) {
|
||||
t.Errorf("critical issue!... absolute symlink escapes extraction dir!")
|
||||
t.Logf(" symlink: %s → %s", symlinkPath, target)
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
// if it exists but is not a symlink, that's good (attack was thwarted)...
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// createMaliciousZipWithSymlink creates a ZIP archive containing a symlink entry pointing to an arbitrary target,
|
||||
// followed by a file entry that attempts to write through that symlink.
|
||||
// returns the path to the created ZIP archive.
|
||||
func createMaliciousZipWithSymlink(t *testing.T, tempDir, symlinkName, symlinkTarget, fileName string) string {
|
||||
t.Helper()
|
||||
|
||||
maliciousZip := filepath.Join(tempDir, "malicious.zip")
|
||||
zipFile, err := os.Create(maliciousZip)
|
||||
require.NoError(t, err)
|
||||
defer zipFile.Close()
|
||||
|
||||
zw := zip.NewWriter(zipFile)
|
||||
|
||||
// create parent directories if the symlink is nested
|
||||
if dir := filepath.Dir(symlinkName); dir != "." {
|
||||
dirHeader := &zip.FileHeader{
|
||||
Name: dir + "/",
|
||||
Method: zip.Store,
|
||||
}
|
||||
dirHeader.SetMode(os.ModeDir | 0755)
|
||||
_, err = zw.CreateHeader(dirHeader)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// create symlink entry pointing outside extraction directory
|
||||
// note: ZIP format stores symlinks as regular files with the target path as content
|
||||
symlinkHeader := &zip.FileHeader{
|
||||
Name: symlinkName,
|
||||
Method: zip.Store,
|
||||
}
|
||||
symlinkHeader.SetMode(os.ModeSymlink | 0755)
|
||||
|
||||
symlinkWriter, err := zw.CreateHeader(symlinkHeader)
|
||||
require.NoError(t, err)
|
||||
|
||||
// write the symlink target as the file content (this is how ZIP stores symlinks)
|
||||
_, err = symlinkWriter.Write([]byte(symlinkTarget))
|
||||
require.NoError(t, err)
|
||||
|
||||
// create file entry that will be written through the symlink
|
||||
if fileName != "" {
|
||||
payloadContent := []byte("MALICIOUS PAYLOAD - This should NOT be written outside extraction dir!")
|
||||
payloadHeader := &zip.FileHeader{
|
||||
Name: fileName,
|
||||
Method: zip.Deflate,
|
||||
}
|
||||
payloadHeader.SetMode(0644)
|
||||
|
||||
payloadWriter, err := zw.CreateHeader(payloadHeader)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = payloadWriter.Write(payloadContent)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
require.NoError(t, zw.Close())
|
||||
require.NoError(t, zipFile.Close())
|
||||
|
||||
return maliciousZip
|
||||
}
|
||||
|
||||
@ -1,229 +0,0 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
)
|
||||
|
||||
// directoryEndLen, readByf, directoryEnd, and findSignatureInBlock were copied from the golang stdlib, specifically:
|
||||
// - https://github.com/golang/go/blob/go1.16.4/src/archive/zip/struct.go
|
||||
// - https://github.com/golang/go/blob/go1.16.4/src/archive/zip/reader.go
|
||||
// findArchiveStartOffset is derived from the same stdlib utils, specifically the readDirectoryEnd function.
|
||||
|
||||
const (
|
||||
directoryEndLen = 22
|
||||
directory64LocLen = 20
|
||||
directory64EndLen = 56
|
||||
directory64LocSignature = 0x07064b50
|
||||
directory64EndSignature = 0x06064b50
|
||||
)
|
||||
|
||||
// ZipReadCloser is a drop-in replacement for zip.ReadCloser (from zip.OpenReader) that additionally considers zips
|
||||
// that have bytes prefixed to the front of the archive (common with self-extracting jars).
|
||||
type ZipReadCloser struct {
|
||||
*zip.Reader
|
||||
io.Closer
|
||||
}
|
||||
|
||||
// OpenZip provides a ZipReadCloser for the given filepath.
|
||||
func OpenZip(filepath string) (*ZipReadCloser, error) {
|
||||
f, err := os.Open(filepath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// some archives may have bytes prepended to the front of the archive, such as with self executing JARs. We first
|
||||
// need to find the start of the archive and keep track of this offset.
|
||||
offset, err := findArchiveStartOffset(f, fi.Size())
|
||||
if err != nil {
|
||||
log.Debugf("cannot find beginning of zip archive=%q : %v", filepath, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if _, err := f.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, fmt.Errorf("unable to seek to beginning of archive: %w", err)
|
||||
}
|
||||
|
||||
if offset > math.MaxInt64 {
|
||||
return nil, fmt.Errorf("archive start offset too large: %v", offset)
|
||||
}
|
||||
offset64 := int64(offset)
|
||||
|
||||
size := fi.Size() - offset64
|
||||
|
||||
r, err := zip.NewReader(io.NewSectionReader(f, offset64, size), size)
|
||||
if err != nil {
|
||||
log.Debugf("unable to open ZipReadCloser @ %q: %v", filepath, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &ZipReadCloser{
|
||||
Reader: r,
|
||||
Closer: f,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type readBuf []byte
|
||||
|
||||
func (b *readBuf) uint16() uint16 {
|
||||
v := binary.LittleEndian.Uint16(*b)
|
||||
*b = (*b)[2:]
|
||||
return v
|
||||
}
|
||||
|
||||
func (b *readBuf) uint32() uint32 {
|
||||
v := binary.LittleEndian.Uint32(*b)
|
||||
*b = (*b)[4:]
|
||||
return v
|
||||
}
|
||||
|
||||
func (b *readBuf) uint64() uint64 {
|
||||
v := binary.LittleEndian.Uint64(*b)
|
||||
*b = (*b)[8:]
|
||||
return v
|
||||
}
|
||||
|
||||
type directoryEnd struct {
|
||||
diskNbr uint32 // unused
|
||||
dirDiskNbr uint32 // unused
|
||||
dirRecordsThisDisk uint64 // unused
|
||||
directoryRecords uint64
|
||||
directorySize uint64
|
||||
directoryOffset uint64 // relative to file
|
||||
}
|
||||
|
||||
// note: this is derived from readDirectoryEnd within the archive/zip package
|
||||
func findArchiveStartOffset(r io.ReaderAt, size int64) (startOfArchive uint64, err error) {
|
||||
// look for directoryEndSignature in the last 1k, then in the last 65k
|
||||
var buf []byte
|
||||
var directoryEndOffset int64
|
||||
for i, bLen := range []int64{1024, 65 * 1024} {
|
||||
if bLen > size {
|
||||
bLen = size
|
||||
}
|
||||
buf = make([]byte, int(bLen))
|
||||
if _, err := r.ReadAt(buf, size-bLen); err != nil && !errors.Is(err, io.EOF) {
|
||||
return 0, err
|
||||
}
|
||||
if p := findSignatureInBlock(buf); p >= 0 {
|
||||
buf = buf[p:]
|
||||
directoryEndOffset = size - bLen + int64(p)
|
||||
break
|
||||
}
|
||||
if i == 1 || bLen == size {
|
||||
return 0, zip.ErrFormat
|
||||
}
|
||||
}
|
||||
|
||||
if buf == nil {
|
||||
// we were unable to find the directoryEndSignature block
|
||||
return 0, zip.ErrFormat
|
||||
}
|
||||
|
||||
// read header into struct
|
||||
b := readBuf(buf[4:]) // skip signature
|
||||
d := &directoryEnd{
|
||||
diskNbr: uint32(b.uint16()),
|
||||
dirDiskNbr: uint32(b.uint16()),
|
||||
dirRecordsThisDisk: uint64(b.uint16()),
|
||||
directoryRecords: uint64(b.uint16()),
|
||||
directorySize: uint64(b.uint32()),
|
||||
directoryOffset: uint64(b.uint32()),
|
||||
}
|
||||
// Calculate where the zip data actually begins
|
||||
|
||||
// These values mean that the file can be a zip64 file
|
||||
if d.directoryRecords == 0xffff || d.directorySize == 0xffff || d.directoryOffset == 0xffffffff {
|
||||
p, err := findDirectory64End(r, directoryEndOffset)
|
||||
if err == nil && p >= 0 {
|
||||
directoryEndOffset = p
|
||||
err = readDirectory64End(r, p, d)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
startOfArchive = uint64(directoryEndOffset) - d.directorySize - d.directoryOffset
|
||||
|
||||
// Make sure directoryOffset points to somewhere in our file.
|
||||
if d.directoryOffset >= uint64(size) {
|
||||
return 0, zip.ErrFormat
|
||||
}
|
||||
return startOfArchive, nil
|
||||
}
|
||||
|
||||
// findDirectory64End tries to read the zip64 locator just before the
|
||||
// directory end and returns the offset of the zip64 directory end if
|
||||
// found.
|
||||
func findDirectory64End(r io.ReaderAt, directoryEndOffset int64) (int64, error) {
|
||||
locOffset := directoryEndOffset - directory64LocLen
|
||||
if locOffset < 0 {
|
||||
return -1, nil // no need to look for a header outside the file
|
||||
}
|
||||
buf := make([]byte, directory64LocLen)
|
||||
if _, err := r.ReadAt(buf, locOffset); err != nil {
|
||||
return -1, err
|
||||
}
|
||||
b := readBuf(buf)
|
||||
if sig := b.uint32(); sig != directory64LocSignature {
|
||||
return -1, nil
|
||||
}
|
||||
if b.uint32() != 0 { // number of the disk with the start of the zip64 end of central directory
|
||||
return -1, nil // the file is not a valid zip64-file
|
||||
}
|
||||
p := b.uint64() // relative offset of the zip64 end of central directory record
|
||||
if b.uint32() != 1 { // total number of disks
|
||||
return -1, nil // the file is not a valid zip64-file
|
||||
}
|
||||
return int64(p), nil
|
||||
}
|
||||
|
||||
// readDirectory64End reads the zip64 directory end and updates the
|
||||
// directory end with the zip64 directory end values.
|
||||
func readDirectory64End(r io.ReaderAt, offset int64, d *directoryEnd) (err error) {
|
||||
buf := make([]byte, directory64EndLen)
|
||||
if _, err := r.ReadAt(buf, offset); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b := readBuf(buf)
|
||||
if sig := b.uint32(); sig != directory64EndSignature {
|
||||
return errors.New("could not read directory64End")
|
||||
}
|
||||
|
||||
b = b[12:] // skip dir size, version and version needed (uint64 + 2x uint16)
|
||||
d.diskNbr = b.uint32() // number of this disk
|
||||
d.dirDiskNbr = b.uint32() // number of the disk with the start of the central directory
|
||||
d.dirRecordsThisDisk = b.uint64() // total number of entries in the central directory on this disk
|
||||
d.directoryRecords = b.uint64() // total number of entries in the central directory
|
||||
d.directorySize = b.uint64() // size of the central directory
|
||||
d.directoryOffset = b.uint64() // offset of start of central directory with respect to the starting disk number
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func findSignatureInBlock(b []byte) int {
|
||||
for i := len(b) - directoryEndLen; i >= 0; i-- {
|
||||
// defined from directoryEndSignature
|
||||
if b[i] == 'P' && b[i+1] == 'K' && b[i+2] == 0x05 && b[i+3] == 0x06 {
|
||||
// n is length of comment
|
||||
n := int(b[i+directoryEndLen-2]) | int(b[i+directoryEndLen-1])<<8
|
||||
if n+directoryEndLen+i <= len(b) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
@ -1,50 +0,0 @@
|
||||
//go:build !windows
|
||||
// +build !windows
|
||||
|
||||
package file
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestFindArchiveStartOffset(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
archivePrep func(tb testing.TB) string
|
||||
expected uint64
|
||||
}{
|
||||
{
|
||||
name: "standard, non-nested zip",
|
||||
archivePrep: prepZipSourceFixture,
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "zip with prepended bytes",
|
||||
archivePrep: prependZipSourceFixtureWithString(t, "junk at the beginning of the file..."),
|
||||
expected: 36,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
archivePath := test.archivePrep(t)
|
||||
f, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
t.Fatalf("could not open archive %q: %+v", archivePath, err)
|
||||
}
|
||||
fi, err := os.Stat(f.Name())
|
||||
if err != nil {
|
||||
t.Fatalf("unable to stat archive: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := findArchiveStartOffset(f, fi.Size())
|
||||
if err != nil {
|
||||
t.Fatalf("unable to find offset: %+v", err)
|
||||
}
|
||||
assert.Equal(t, test.expected, actual)
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -27,6 +27,7 @@ func AllTypes() []any {
|
||||
pkg.ELFBinaryPackageNoteJSONPayload{},
|
||||
pkg.ElixirMixLockEntry{},
|
||||
pkg.ErlangRebarLockEntry{},
|
||||
pkg.GGUFFileHeader{},
|
||||
pkg.GitHubActionsUseStatement{},
|
||||
pkg.GolangBinaryBuildinfoEntry{},
|
||||
pkg.GolangModuleEntry{},
|
||||
@ -49,6 +50,7 @@ func AllTypes() []any {
|
||||
pkg.PhpComposerLockEntry{},
|
||||
pkg.PhpPearEntry{},
|
||||
pkg.PhpPeclEntry{},
|
||||
pkg.PnpmLockEntry{},
|
||||
pkg.PortageEntry{},
|
||||
pkg.PythonPackage{},
|
||||
pkg.PythonPdmLockEntry{},
|
||||
|
||||
@ -95,10 +95,11 @@ var jsonTypes = makeJSONTypes(
|
||||
jsonNames(pkg.NpmPackage{}, "javascript-npm-package", "NpmPackageJsonMetadata"),
|
||||
jsonNames(pkg.NpmPackageLockEntry{}, "javascript-npm-package-lock-entry", "NpmPackageLockJsonMetadata"),
|
||||
jsonNames(pkg.YarnLockEntry{}, "javascript-yarn-lock-entry", "YarnLockJsonMetadata"),
|
||||
jsonNames(pkg.PnpmLockEntry{}, "javascript-pnpm-lock-entry"),
|
||||
jsonNames(pkg.PEBinary{}, "pe-binary"),
|
||||
jsonNames(pkg.PhpComposerLockEntry{}, "php-composer-lock-entry", "PhpComposerJsonMetadata"),
|
||||
jsonNamesWithoutLookup(pkg.PhpComposerInstalledEntry{}, "php-composer-installed-entry", "PhpComposerJsonMetadata"), // the legacy value is split into two types, where the other is preferred
|
||||
jsonNames(pkg.PhpPeclEntry{}, "php-pecl-entry", "PhpPeclMetadata"),
|
||||
jsonNames(pkg.PhpPeclEntry{}, "php-pecl-entry", "PhpPeclMetadata"), //nolint:staticcheck
|
||||
jsonNames(pkg.PhpPearEntry{}, "php-pear-entry"),
|
||||
jsonNames(pkg.PortageEntry{}, "portage-db-entry", "PortageMetadata"),
|
||||
jsonNames(pkg.PythonPackage{}, "python-package", "PythonPackageMetadata"),
|
||||
@ -123,6 +124,7 @@ var jsonTypes = makeJSONTypes(
|
||||
jsonNames(pkg.TerraformLockProviderEntry{}, "terraform-lock-provider-entry"),
|
||||
jsonNames(pkg.DotnetPackagesLockEntry{}, "dotnet-packages-lock-entry"),
|
||||
jsonNames(pkg.CondaMetaPackage{}, "conda-metadata-entry", "CondaPackageMetadata"),
|
||||
jsonNames(pkg.GGUFFileHeader{}, "gguf-file-header"),
|
||||
)
|
||||
|
||||
func expandLegacyNameVariants(names ...string) []string {
|
||||
|
||||
@ -3,6 +3,7 @@ package task
|
||||
import (
|
||||
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/ai"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/alpine"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/arch"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/binary"
|
||||
@ -52,6 +53,9 @@ const (
|
||||
JavaScript = "javascript"
|
||||
Node = "node"
|
||||
NPM = "npm"
|
||||
|
||||
// Python ecosystem labels
|
||||
Python = "python"
|
||||
)
|
||||
|
||||
//nolint:funlen
|
||||
@ -109,7 +113,7 @@ func DefaultPackageTaskFactories() Factories {
|
||||
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
|
||||
return python.NewPackageCataloger(cfg.PackagesConfig.Python)
|
||||
},
|
||||
pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "python",
|
||||
pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, Python,
|
||||
),
|
||||
newSimplePackageTaskFactory(ruby.NewGemFileLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "ruby", "gem"),
|
||||
newSimplePackageTaskFactory(ruby.NewGemSpecCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "ruby", "gem", "gemspec"),
|
||||
@ -127,7 +131,7 @@ func DefaultPackageTaskFactories() Factories {
|
||||
pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "dotnet", "c#",
|
||||
),
|
||||
newSimplePackageTaskFactory(dotnet.NewDotnetPackagesLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.ImageTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "dotnet", "c#"),
|
||||
newSimplePackageTaskFactory(python.NewInstalledPackageCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "python"),
|
||||
newSimplePackageTaskFactory(python.NewInstalledPackageCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, Python),
|
||||
newPackageTaskFactory(
|
||||
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
|
||||
return golang.NewGoModuleBinaryCataloger(cfg.PackagesConfig.Golang)
|
||||
@ -175,12 +179,13 @@ func DefaultPackageTaskFactories() Factories {
|
||||
newSimplePackageTaskFactory(homebrew.NewCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, "homebrew"),
|
||||
newSimplePackageTaskFactory(conda.NewCondaMetaCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.PackageTag, "conda"),
|
||||
newSimplePackageTaskFactory(snap.NewCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, "snap"),
|
||||
newSimplePackageTaskFactory(ai.NewGGUFCataloger, pkgcataloging.DirectoryTag, pkgcataloging.ImageTag, "ai", "model", "gguf", "ml"),
|
||||
|
||||
// deprecated catalogers ////////////////////////////////////////
|
||||
// these are catalogers that should not be selectable other than specific inclusion via name or "deprecated" tag (to remain backwards compatible)
|
||||
newSimplePackageTaskFactory(dotnet.NewDotnetDepsCataloger, pkgcataloging.DeprecatedTag), // TODO: remove in syft v2.0
|
||||
newSimplePackageTaskFactory(dotnet.NewDotnetPortableExecutableCataloger, pkgcataloging.DeprecatedTag), // TODO: remove in syft v2.0
|
||||
newSimplePackageTaskFactory(php.NewPeclCataloger, pkgcataloging.DeprecatedTag), // TODO: remove in syft v2.0
|
||||
newSimplePackageTaskFactory(nix.NewStoreCataloger, pkgcataloging.DeprecatedTag), // TODO: remove in syft v2.0
|
||||
newSimplePackageTaskFactory(dotnet.NewDotnetDepsCataloger, pkgcataloging.DeprecatedTag), //nolint:staticcheck // TODO: remove in syft v2.0
|
||||
newSimplePackageTaskFactory(dotnet.NewDotnetPortableExecutableCataloger, pkgcataloging.DeprecatedTag), //nolint:staticcheck // TODO: remove in syft v2.0
|
||||
newSimplePackageTaskFactory(php.NewPeclCataloger, pkgcataloging.DeprecatedTag), //nolint:staticcheck // TODO: remove in syft v2.0
|
||||
newSimplePackageTaskFactory(nix.NewStoreCataloger, pkgcataloging.DeprecatedTag), //nolint:staticcheck // TODO: remove in syft v2.0
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,7 +4,8 @@ import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/archiver/v3"
|
||||
"github.com/mholt/archives"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
@ -57,9 +58,10 @@ func (c unknownsLabelerTask) finalize(resolver file.Resolver, s *sbom.SBOM) {
|
||||
}
|
||||
|
||||
if c.IncludeUnexpandedArchives {
|
||||
ctx := context.Background()
|
||||
for coords := range s.Artifacts.FileMetadata {
|
||||
unarchiver, notArchiveErr := archiver.ByExtension(coords.RealPath)
|
||||
if unarchiver != nil && notArchiveErr == nil && !hasPackageReference(coords) {
|
||||
format, _, notArchiveErr := archives.Identify(ctx, coords.RealPath, nil)
|
||||
if format != nil && notArchiveErr == nil && !hasPackageReference(coords) {
|
||||
s.Artifacts.Unknowns[coords] = append(s.Artifacts.Unknowns[coords], "archive not cataloged")
|
||||
}
|
||||
}
|
||||
|
||||
4148
schema/json/schema-16.0.42.json
Normal file
4148
schema/json/schema-16.0.42.json
Normal file
File diff suppressed because it is too large
Load Diff
4193
schema/json/schema-16.0.43.json
Normal file
4193
schema/json/schema-16.0.43.json
Normal file
File diff suppressed because it is too large
Load Diff
4258
schema/json/schema-16.1.0.json
Normal file
4258
schema/json/schema-16.1.0.json
Normal file
File diff suppressed because it is too large
Load Diff
@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "anchore.io/schema/syft/json/16.0.41/document",
|
||||
"$id": "anchore.io/schema/syft/json/16.1.0/document",
|
||||
"$ref": "#/$defs/Document",
|
||||
"$defs": {
|
||||
"AlpmDbEntry": {
|
||||
@ -130,7 +130,8 @@
|
||||
"description": "Digests contains file content hashes for integrity verification"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
"type": "object",
|
||||
"description": "AlpmFileRecord represents a single file entry within an Arch Linux package with its associated metadata tracked by pacman."
|
||||
},
|
||||
"ApkDbEntry": {
|
||||
"properties": {
|
||||
@ -433,16 +434,19 @@
|
||||
"CPE": {
|
||||
"properties": {
|
||||
"cpe": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Value is the CPE string identifier."
|
||||
},
|
||||
"source": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Source is the source where this CPE was obtained or generated from."
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": [
|
||||
"cpe"
|
||||
]
|
||||
],
|
||||
"description": "CPE represents a Common Platform Enumeration identifier used for matching packages to known vulnerabilities in security databases."
|
||||
},
|
||||
"ClassifierMatch": {
|
||||
"properties": {
|
||||
@ -747,19 +751,23 @@
|
||||
"Descriptor": {
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Name is the name of the tool that generated this SBOM (e.g., \"syft\")."
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Version is the version of the tool that generated this SBOM."
|
||||
},
|
||||
"configuration": true
|
||||
"configuration": {
|
||||
"description": "Configuration contains the tool configuration used during SBOM generation."
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": [
|
||||
"name",
|
||||
"version"
|
||||
],
|
||||
"description": "Descriptor describes what created the document as well as surrounding metadata"
|
||||
"description": "Descriptor identifies the tool that generated this SBOM document, including its name, version, and configuration used during catalog generation."
|
||||
},
|
||||
"Digest": {
|
||||
"properties": {
|
||||
@ -1285,58 +1293,71 @@
|
||||
"File": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "ID is a unique identifier for this file within the SBOM."
|
||||
},
|
||||
"location": {
|
||||
"$ref": "#/$defs/Coordinates"
|
||||
"$ref": "#/$defs/Coordinates",
|
||||
"description": "Location is the file path and layer information where this file was found."
|
||||
},
|
||||
"metadata": {
|
||||
"$ref": "#/$defs/FileMetadataEntry"
|
||||
"$ref": "#/$defs/FileMetadataEntry",
|
||||
"description": "Metadata contains filesystem metadata such as permissions, ownership, and file type."
|
||||
},
|
||||
"contents": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Contents is the file contents for small files."
|
||||
},
|
||||
"digests": {
|
||||
"items": {
|
||||
"$ref": "#/$defs/Digest"
|
||||
},
|
||||
"type": "array"
|
||||
"type": "array",
|
||||
"description": "Digests contains cryptographic hashes of the file contents."
|
||||
},
|
||||
"licenses": {
|
||||
"items": {
|
||||
"$ref": "#/$defs/FileLicense"
|
||||
},
|
||||
"type": "array"
|
||||
"type": "array",
|
||||
"description": "Licenses contains license information discovered within this file."
|
||||
},
|
||||
"executable": {
|
||||
"$ref": "#/$defs/Executable"
|
||||
"$ref": "#/$defs/Executable",
|
||||
"description": "Executable contains executable metadata if this file is a binary."
|
||||
},
|
||||
"unknowns": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
"type": "array",
|
||||
"description": "Unknowns contains unknown fields for forward compatibility."
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"location"
|
||||
]
|
||||
],
|
||||
"description": "File represents a file discovered during cataloging with its metadata, content digests, licenses, and relationships to packages."
|
||||
},
|
||||
"FileLicense": {
|
||||
"properties": {
|
||||
"value": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Value is the raw license identifier or text as found in the file."
|
||||
},
|
||||
"spdxExpression": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "SPDXExpression is the parsed SPDX license expression."
|
||||
},
|
||||
"type": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Type is the license type classification (e.g., declared, concluded, discovered)."
|
||||
},
|
||||
"evidence": {
|
||||
"$ref": "#/$defs/FileLicenseEvidence"
|
||||
"$ref": "#/$defs/FileLicenseEvidence",
|
||||
"description": "Evidence contains supporting evidence for this license detection."
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
@ -1344,18 +1365,22 @@
|
||||
"value",
|
||||
"spdxExpression",
|
||||
"type"
|
||||
]
|
||||
],
|
||||
"description": "FileLicense represents license information discovered within a file's contents or metadata, including the matched license text and SPDX expression."
|
||||
},
|
||||
"FileLicenseEvidence": {
|
||||
"properties": {
|
||||
"confidence": {
|
||||
"type": "integer"
|
||||
"type": "integer",
|
||||
"description": "Confidence is the confidence score for this license detection (0-100)."
|
||||
},
|
||||
"offset": {
|
||||
"type": "integer"
|
||||
"type": "integer",
|
||||
"description": "Offset is the byte offset where the license text starts in the file."
|
||||
},
|
||||
"extent": {
|
||||
"type": "integer"
|
||||
"type": "integer",
|
||||
"description": "Extent is the length of the license text in bytes."
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
@ -1363,30 +1388,38 @@
|
||||
"confidence",
|
||||
"offset",
|
||||
"extent"
|
||||
]
|
||||
],
|
||||
"description": "FileLicenseEvidence contains supporting evidence for a license detection in a file, including the byte offset, extent, and confidence level."
|
||||
},
|
||||
"FileMetadataEntry": {
|
||||
"properties": {
|
||||
"mode": {
|
||||
"type": "integer"
|
||||
"type": "integer",
|
||||
"description": "Mode is the Unix file permission mode in octal format."
|
||||
},
|
||||
"type": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Type is the file type (e.g., \"RegularFile\", \"Directory\", \"SymbolicLink\")."
|
||||
},
|
||||
"linkDestination": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "LinkDestination is the target path for symbolic links."
|
||||
},
|
||||
"userID": {
|
||||
"type": "integer"
|
||||
"type": "integer",
|
||||
"description": "UserID is the file owner user ID."
|
||||
},
|
||||
"groupID": {
|
||||
"type": "integer"
|
||||
"type": "integer",
|
||||
"description": "GroupID is the file owner group ID."
|
||||
},
|
||||
"mimeType": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "MIMEType is the MIME type of the file contents."
|
||||
},
|
||||
"size": {
|
||||
"type": "integer"
|
||||
"type": "integer",
|
||||
"description": "Size is the file size in bytes."
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
@ -1397,7 +1430,50 @@
|
||||
"groupID",
|
||||
"mimeType",
|
||||
"size"
|
||||
]
|
||||
],
|
||||
"description": "FileMetadataEntry contains filesystem-level metadata attributes such as permissions, ownership, type, and size for a cataloged file."
|
||||
},
|
||||
"GgufFileHeader": {
|
||||
"properties": {
|
||||
"ggufVersion": {
|
||||
"type": "integer",
|
||||
"description": "GGUFVersion is the GGUF format version (e.g., 3)"
|
||||
},
|
||||
"fileSize": {
|
||||
"type": "integer",
|
||||
"description": "FileSize is the size of the GGUF file in bytes (best-effort if available from resolver)"
|
||||
},
|
||||
"architecture": {
|
||||
"type": "string",
|
||||
"description": "Architecture is the model architecture (from general.architecture, e.g., \"qwen3moe\", \"llama\")"
|
||||
},
|
||||
"quantization": {
|
||||
"type": "string",
|
||||
"description": "Quantization is the quantization type (e.g., \"IQ4_NL\", \"Q4_K_M\")"
|
||||
},
|
||||
"parameters": {
|
||||
"type": "integer",
|
||||
"description": "Parameters is the number of model parameters (if present in header)"
|
||||
},
|
||||
"tensorCount": {
|
||||
"type": "integer",
|
||||
"description": "TensorCount is the number of tensors in the model"
|
||||
},
|
||||
"header": {
|
||||
"type": "object",
|
||||
"description": "RemainingKeyValues contains the remaining key-value pairs from the GGUF header that are not already\nrepresented as typed fields above. This preserves additional metadata fields for reference\n(namespaced with general.*, llama.*, etc.) while avoiding duplication."
|
||||
},
|
||||
"metadataHash": {
|
||||
"type": "string",
|
||||
"description": "MetadataKeyValuesHash is a xx64 hash of all key-value pairs from the GGUF header metadata.\nThis hash is computed over the complete header metadata (including the fields extracted\ninto typed fields above) and provides a stable identifier for the model configuration\nacross different file locations or remotes. It allows matching identical models even\nwhen stored in different repositories or with different filenames."
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": [
|
||||
"ggufVersion",
|
||||
"tensorCount"
|
||||
],
|
||||
"description": "GGUFFileHeader represents metadata extracted from a GGUF (GPT-Generated Unified Format) model file."
|
||||
},
|
||||
"GithubActionsUseStatement": {
|
||||
"properties": {
|
||||
@ -1545,7 +1621,8 @@
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
"type": "array",
|
||||
"description": "IDLikes represents a list of distribution IDs that this Linux distribution is similar to or derived from, as defined in os-release ID_LIKE field."
|
||||
},
|
||||
"JavaArchive": {
|
||||
"properties": {
|
||||
@ -1876,15 +1953,48 @@
|
||||
"integrity": {
|
||||
"type": "string",
|
||||
"description": "Integrity is Subresource Integrity hash for verification using standard SRI format (sha512-... or sha1-...). npm changed from SHA-1 to SHA-512 in newer versions. For registry sources this is the integrity from registry, for remote tarballs it's SHA-512 of the file. npm verifies tarball matches this hash before unpacking, throwing EINTEGRITY error if mismatch detected."
|
||||
},
|
||||
"dependencies": {
|
||||
"patternProperties": {
|
||||
".*": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"description": "Dependencies is a map of dependencies and their version markers, i.e. \"lodash\": \"^1.0.0\""
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": [
|
||||
"resolved",
|
||||
"integrity"
|
||||
"integrity",
|
||||
"dependencies"
|
||||
],
|
||||
"description": "NpmPackageLockEntry represents a single entry within the \"packages\" section of a package-lock.json file."
|
||||
},
|
||||
"JavascriptPnpmLockEntry": {
|
||||
"properties": {
|
||||
"resolution": {
|
||||
"$ref": "#/$defs/PnpmLockResolution",
|
||||
"description": "Resolution is the resolution information for the package"
|
||||
},
|
||||
"dependencies": {
|
||||
"patternProperties": {
|
||||
".*": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"description": "Dependencies is a map of dependencies and their versions"
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": [
|
||||
"resolution",
|
||||
"dependencies"
|
||||
],
|
||||
"description": "PnpmLockEntry represents a single entry in the \"packages\" section of a pnpm-lock.yaml file."
|
||||
},
|
||||
"JavascriptYarnLockEntry": {
|
||||
"properties": {
|
||||
"resolved": {
|
||||
@ -1894,12 +2004,22 @@
|
||||
"integrity": {
|
||||
"type": "string",
|
||||
"description": "Integrity is Subresource Integrity hash for verification (SRI format)"
|
||||
},
|
||||
"dependencies": {
|
||||
"patternProperties": {
|
||||
".*": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"description": "Dependencies is a map of dependencies and their versions"
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": [
|
||||
"resolved",
|
||||
"integrity"
|
||||
"integrity",
|
||||
"dependencies"
|
||||
],
|
||||
"description": "YarnLockEntry represents a single entry section of a yarn.lock file."
|
||||
},
|
||||
@ -1931,28 +2051,34 @@
|
||||
"License": {
|
||||
"properties": {
|
||||
"value": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Value is the raw license identifier or expression as found."
|
||||
},
|
||||
"spdxExpression": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "SPDXExpression is the parsed SPDX license expression."
|
||||
},
|
||||
"type": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Type is the license type classification (e.g., declared, concluded, discovered)."
|
||||
},
|
||||
"urls": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
"type": "array",
|
||||
"description": "URLs are URLs where license text or information can be found."
|
||||
},
|
||||
"locations": {
|
||||
"items": {
|
||||
"$ref": "#/$defs/Location"
|
||||
},
|
||||
"type": "array"
|
||||
"type": "array",
|
||||
"description": "Locations are file locations where this license was discovered."
|
||||
},
|
||||
"contents": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Contents is the full license text content."
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
@ -1962,7 +2088,8 @@
|
||||
"type",
|
||||
"urls",
|
||||
"locations"
|
||||
]
|
||||
],
|
||||
"description": "License represents software license information discovered for a package, including SPDX expressions and supporting evidence locations."
|
||||
},
|
||||
"LinuxKernelArchive": {
|
||||
"properties": {
|
||||
@ -2087,64 +2214,84 @@
|
||||
"LinuxRelease": {
|
||||
"properties": {
|
||||
"prettyName": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "PrettyName is a human-readable operating system name with version."
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Name is the operating system name without version information."
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "ID is the lower-case operating system identifier (e.g., \"ubuntu\", \"rhel\")."
|
||||
},
|
||||
"idLike": {
|
||||
"$ref": "#/$defs/IDLikes"
|
||||
"$ref": "#/$defs/IDLikes",
|
||||
"description": "IDLike is a list of operating system IDs this distribution is similar to or derived from."
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Version is the operating system version including codename if available."
|
||||
},
|
||||
"versionID": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "VersionID is the operating system version number or identifier."
|
||||
},
|
||||
"versionCodename": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "VersionCodename is the operating system release codename (e.g., \"jammy\", \"bullseye\")."
|
||||
},
|
||||
"buildID": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "BuildID is a build identifier for the operating system."
|
||||
},
|
||||
"imageID": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "ImageID is an identifier for container or cloud images."
|
||||
},
|
||||
"imageVersion": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "ImageVersion is the version for container or cloud images."
|
||||
},
|
||||
"variant": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Variant is the operating system variant name (e.g., \"Server\", \"Workstation\")."
|
||||
},
|
||||
"variantID": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "VariantID is the lower-case operating system variant identifier."
|
||||
},
|
||||
"homeURL": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "HomeURL is the homepage URL for the operating system."
|
||||
},
|
||||
"supportURL": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "SupportURL is the support or help URL for the operating system."
|
||||
},
|
||||
"bugReportURL": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "BugReportURL is the bug reporting URL for the operating system."
|
||||
},
|
||||
"privacyPolicyURL": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "PrivacyPolicyURL is the privacy policy URL for the operating system."
|
||||
},
|
||||
"cpeName": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "CPEName is the Common Platform Enumeration name for the operating system."
|
||||
},
|
||||
"supportEnd": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "SupportEnd is the end of support date or version identifier."
|
||||
},
|
||||
"extendedSupport": {
|
||||
"type": "boolean"
|
||||
"type": "boolean",
|
||||
"description": "ExtendedSupport indicates whether extended security or support is available."
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
"type": "object",
|
||||
"description": "LinuxRelease contains Linux distribution identification and version information extracted from /etc/os-release or similar system files."
|
||||
},
|
||||
"Location": {
|
||||
"properties": {
|
||||
@ -2240,7 +2387,7 @@
|
||||
"product_id",
|
||||
"kb"
|
||||
],
|
||||
"description": "MicrosoftKbPatch is slightly odd in how it is expected to map onto data."
|
||||
"description": "MicrosoftKbPatch represents a Windows Knowledge Base patch identifier associated with a specific Microsoft product from the MSRC (Microsoft Security Response Center)."
|
||||
},
|
||||
"NixDerivation": {
|
||||
"properties": {
|
||||
@ -2474,6 +2621,9 @@
|
||||
{
|
||||
"$ref": "#/$defs/ErlangRebarLockEntry"
|
||||
},
|
||||
{
|
||||
"$ref": "#/$defs/GgufFileHeader"
|
||||
},
|
||||
{
|
||||
"$ref": "#/$defs/GithubActionsUseStatement"
|
||||
},
|
||||
@ -2507,6 +2657,9 @@
|
||||
{
|
||||
"$ref": "#/$defs/JavascriptNpmPackageLockEntry"
|
||||
},
|
||||
{
|
||||
"$ref": "#/$defs/JavascriptPnpmLockEntry"
|
||||
},
|
||||
{
|
||||
"$ref": "#/$defs/JavascriptYarnLockEntry"
|
||||
},
|
||||
@ -2958,6 +3111,19 @@
|
||||
],
|
||||
"description": "PhpPeclEntry represents a single package entry found within php pecl metadata files."
|
||||
},
|
||||
"PnpmLockResolution": {
|
||||
"properties": {
|
||||
"integrity": {
|
||||
"type": "string",
|
||||
"description": "Integrity is Subresource Integrity hash for verification (SRI format)"
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": [
|
||||
"integrity"
|
||||
],
|
||||
"description": "PnpmLockResolution contains package resolution metadata from pnpm lockfiles, including the integrity hash used for verification."
|
||||
},
|
||||
"PortageDbEntry": {
|
||||
"properties": {
|
||||
"installedSize": {
|
||||
@ -3134,6 +3300,23 @@
|
||||
],
|
||||
"description": "PythonPackage represents all captured data for a python egg or wheel package (specifically as outlined in the PyPA core metadata specification https://packaging.python.org/en/latest/specifications/core-metadata/)."
|
||||
},
|
||||
"PythonPdmFileEntry": {
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string",
|
||||
"description": "URL is the file download URL"
|
||||
},
|
||||
"digest": {
|
||||
"$ref": "#/$defs/PythonFileDigest",
|
||||
"description": "Digest is the hash digest of the file hosted at the URL"
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": [
|
||||
"url",
|
||||
"digest"
|
||||
]
|
||||
},
|
||||
"PythonPdmLockEntry": {
|
||||
"properties": {
|
||||
"summary": {
|
||||
@ -3142,27 +3325,75 @@
|
||||
},
|
||||
"files": {
|
||||
"items": {
|
||||
"$ref": "#/$defs/PythonFileRecord"
|
||||
"$ref": "#/$defs/PythonPdmFileEntry"
|
||||
},
|
||||
"type": "array",
|
||||
"description": "Files are the package files with their paths and hash digests"
|
||||
"description": "Files are the package files with their paths and hash digests (for the base package without extras)"
|
||||
},
|
||||
"marker": {
|
||||
"type": "string",
|
||||
"description": "Marker is the \"environment\" --conditional expressions that determine whether a package should be installed based on the runtime environment"
|
||||
},
|
||||
"requiresPython": {
|
||||
"type": "string",
|
||||
"description": "RequiresPython specifies the Python version requirement (e.g., \"\u003e=3.6\")."
|
||||
},
|
||||
"dependencies": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array",
|
||||
"description": "Dependencies are the dependency specifications, without environment qualifiers"
|
||||
"description": "Dependencies are the dependency specifications for the base package (without extras)"
|
||||
},
|
||||
"extras": {
|
||||
"items": {
|
||||
"$ref": "#/$defs/PythonPdmLockExtraVariant"
|
||||
},
|
||||
"type": "array",
|
||||
"description": "Extras contains variants for different extras combinations (PDM may have multiple entries per package)"
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": [
|
||||
"summary",
|
||||
"files",
|
||||
"dependencies"
|
||||
"files"
|
||||
],
|
||||
"description": "PythonPdmLockEntry represents a single package entry within a pdm.lock file."
|
||||
},
|
||||
"PythonPdmLockExtraVariant": {
|
||||
"properties": {
|
||||
"extras": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array",
|
||||
"description": "Extras are the optional extras enabled for this variant (e.g., [\"toml\"], [\"dev\"], or [\"toml\", \"dev\"])"
|
||||
},
|
||||
"dependencies": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array",
|
||||
"description": "Dependencies are the dependencies specific to this extras variant"
|
||||
},
|
||||
"files": {
|
||||
"items": {
|
||||
"$ref": "#/$defs/PythonPdmFileEntry"
|
||||
},
|
||||
"type": "array",
|
||||
"description": "Files are the package files specific to this variant (only populated if different from base)"
|
||||
},
|
||||
"marker": {
|
||||
"type": "string",
|
||||
"description": "Marker is the environment conditional expression for this variant (e.g., \"python_version \u003c \\\"3.11\\\"\")"
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": [
|
||||
"extras"
|
||||
],
|
||||
"description": "PythonPdmLockExtraVariant represents a specific extras combination variant within a PDM lock file."
|
||||
},
|
||||
"PythonPipRequirementsEntry": {
|
||||
"properties": {
|
||||
"name": {
|
||||
@ -3443,22 +3674,28 @@
|
||||
"Relationship": {
|
||||
"properties": {
|
||||
"parent": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Parent is the ID of the parent artifact in this relationship."
|
||||
},
|
||||
"child": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Child is the ID of the child artifact in this relationship."
|
||||
},
|
||||
"type": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Type is the relationship type (e.g., \"contains\", \"dependency-of\", \"ancestor-of\")."
|
||||
},
|
||||
"metadata": true
|
||||
"metadata": {
|
||||
"description": "Metadata contains additional relationship-specific metadata."
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": [
|
||||
"parent",
|
||||
"child",
|
||||
"type"
|
||||
]
|
||||
],
|
||||
"description": "Relationship represents a directed relationship between two artifacts in the SBOM, such as package-contains-file or package-depends-on-package."
|
||||
},
|
||||
"RpmArchive": {
|
||||
"properties": {
|
||||
@ -3805,17 +4042,20 @@
|
||||
"Schema": {
|
||||
"properties": {
|
||||
"version": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Version is the JSON schema version for this document format."
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "URL is the URL to the JSON schema definition document."
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": [
|
||||
"version",
|
||||
"url"
|
||||
]
|
||||
],
|
||||
"description": "Schema specifies the JSON schema version and URL reference that defines the structure and validation rules for this document format."
|
||||
},
|
||||
"SnapEntry": {
|
||||
"properties": {
|
||||
@ -3853,21 +4093,28 @@
|
||||
"Source": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "ID is a unique identifier for the analyzed source artifact."
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Name is the name of the analyzed artifact (e.g., image name, directory path)."
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Version is the version of the analyzed artifact (e.g., image tag)."
|
||||
},
|
||||
"supplier": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Supplier is supplier information, which can be user-provided for NTIA minimum elements compliance."
|
||||
},
|
||||
"type": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Type is the source type (e.g., \"image\", \"directory\", \"file\")."
|
||||
},
|
||||
"metadata": true
|
||||
"metadata": {
|
||||
"description": "Metadata contains additional source-specific metadata."
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": [
|
||||
@ -3877,7 +4124,7 @@
|
||||
"type",
|
||||
"metadata"
|
||||
],
|
||||
"description": "Instead, the Supplier can be determined by the user of syft and passed as a config or flag to help fulfill the NTIA minimum elements."
|
||||
"description": "Source represents the artifact that was analyzed to generate this SBOM, such as a container image, directory, or file archive."
|
||||
},
|
||||
"SwiftPackageManagerLockEntry": {
|
||||
"properties": {
|
||||
|
||||
95
syft/format/cpes/decoder.go
Normal file
95
syft/format/cpes/decoder.go
Normal file
@ -0,0 +1,95 @@
|
||||
package cpes
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/cpe"
|
||||
"github.com/anchore/syft/syft/format/internal"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
)
|
||||
|
||||
const ID sbom.FormatID = "cpes"
|
||||
const version = "1"
|
||||
|
||||
var _ sbom.FormatDecoder = (*decoder)(nil)
|
||||
|
||||
type decoder struct{}
|
||||
|
||||
func NewFormatDecoder() sbom.FormatDecoder {
|
||||
return decoder{}
|
||||
}
|
||||
|
||||
func (d decoder) Decode(r io.Reader) (*sbom.SBOM, sbom.FormatID, string, error) {
|
||||
if r == nil {
|
||||
return nil, "", "", fmt.Errorf("no reader provided")
|
||||
}
|
||||
s, err := toSyftModel(r)
|
||||
return s, ID, version, err
|
||||
}
|
||||
|
||||
func (d decoder) Identify(r io.Reader) (sbom.FormatID, string) {
|
||||
if r == nil {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(r)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
// skip whitespace only lines
|
||||
continue
|
||||
}
|
||||
|
||||
err := cpe.ValidateString(line)
|
||||
if err != nil {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
return ID, version
|
||||
}
|
||||
|
||||
return "", ""
|
||||
}
|
||||
|
||||
func toSyftModel(r io.Reader) (*sbom.SBOM, error) {
|
||||
var errs []error
|
||||
pkgs := pkg.NewCollection()
|
||||
|
||||
scanner := bufio.NewScanner(r)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// skip invalid CPEs
|
||||
c, err := cpe.New(line, "")
|
||||
if err != nil {
|
||||
log.WithFields("error", err, "line", line).Debug("unable to parse cpe")
|
||||
continue
|
||||
}
|
||||
|
||||
p := pkg.Package{
|
||||
Name: c.Attributes.Product,
|
||||
Version: c.Attributes.Version,
|
||||
CPEs: []cpe.CPE{c},
|
||||
}
|
||||
|
||||
internal.Backfill(&p)
|
||||
p.SetID()
|
||||
pkgs.Add(p)
|
||||
}
|
||||
|
||||
return &sbom.SBOM{
|
||||
Artifacts: sbom.Artifacts{
|
||||
Packages: pkgs,
|
||||
},
|
||||
}, errors.Join(errs...)
|
||||
}
|
||||
171
syft/format/cpes/decoder_test.go
Normal file
171
syft/format/cpes/decoder_test.go
Normal file
@ -0,0 +1,171 @@
|
||||
package cpes
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/cpe"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
)
|
||||
|
||||
func Test_CPEProvider(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
userInput string
|
||||
sbom *sbom.SBOM
|
||||
}{
|
||||
{
|
||||
name: "takes a single cpe",
|
||||
userInput: "cpe:/a:apache:log4j:2.14.1",
|
||||
sbom: &sbom.SBOM{
|
||||
Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(pkg.Package{
|
||||
Name: "log4j",
|
||||
Version: "2.14.1",
|
||||
CPEs: []cpe.CPE{
|
||||
cpe.Must("cpe:/a:apache:log4j:2.14.1", ""),
|
||||
},
|
||||
}),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "takes multiple cpes",
|
||||
userInput: `cpe:/a:apache:log4j:2.14.1
|
||||
cpe:2.3:a:f5:nginx:*:*:*:*:*:*:*:*;
|
||||
cpe:2.3:a:f5:nginx:0.5.2:*:*:*:*:*:*:*;
|
||||
cpe:2.3:a:f5:nginx:0.5.3:*:*:*:*:*:*:*;`,
|
||||
sbom: &sbom.SBOM{
|
||||
Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(
|
||||
pkg.Package{
|
||||
Name: "log4j",
|
||||
Version: "2.14.1",
|
||||
CPEs: []cpe.CPE{
|
||||
cpe.Must("cpe:/a:apache:log4j:2.14.1", ""),
|
||||
},
|
||||
},
|
||||
pkg.Package{
|
||||
Name: "nginx",
|
||||
Version: "",
|
||||
CPEs: []cpe.CPE{
|
||||
cpe.Must("cpe:2.3:a:f5:nginx:*:*:*:*:*:*:*:*;", ""),
|
||||
},
|
||||
},
|
||||
pkg.Package{
|
||||
Name: "nginx",
|
||||
Version: "0.5.2",
|
||||
CPEs: []cpe.CPE{
|
||||
cpe.Must("cpe:2.3:a:f5:nginx:0.5.2:*:*:*:*:*:*:*;", ""),
|
||||
},
|
||||
},
|
||||
pkg.Package{
|
||||
Name: "nginx",
|
||||
Version: "0.5.3",
|
||||
CPEs: []cpe.CPE{
|
||||
cpe.Must("cpe:2.3:a:f5:nginx:0.5.3:*:*:*:*:*:*:*;", ""),
|
||||
},
|
||||
},
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "takes cpe with no version",
|
||||
userInput: "cpe:/a:apache:log4j",
|
||||
sbom: &sbom.SBOM{
|
||||
Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(pkg.Package{
|
||||
Name: "log4j",
|
||||
CPEs: []cpe.CPE{
|
||||
cpe.Must("cpe:/a:apache:log4j", ""),
|
||||
},
|
||||
}),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "takes CPE 2.3 format",
|
||||
userInput: "cpe:2.3:a:apache:log4j:2.14.1:*:*:*:*:*:*:*",
|
||||
sbom: &sbom.SBOM{
|
||||
Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(pkg.Package{
|
||||
Name: "log4j",
|
||||
Version: "2.14.1",
|
||||
CPEs: []cpe.CPE{
|
||||
cpe.Must("cpe:2.3:a:apache:log4j:2.14.1:*:*:*:*:*:*:*", ""),
|
||||
},
|
||||
}),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "deduces target SW from CPE - known target_sw",
|
||||
userInput: "cpe:2.3:a:amazon:opensearch:*:*:*:*:*:ruby:*:*",
|
||||
sbom: &sbom.SBOM{
|
||||
Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(pkg.Package{
|
||||
Name: "opensearch",
|
||||
Type: pkg.GemPkg,
|
||||
CPEs: []cpe.CPE{
|
||||
cpe.Must("cpe:2.3:a:amazon:opensearch:*:*:*:*:*:ruby:*:*", ""),
|
||||
},
|
||||
}),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "handles unknown target_sw CPE field",
|
||||
userInput: "cpe:2.3:a:amazon:opensearch:*:*:*:*:*:loremipsum:*:*",
|
||||
sbom: &sbom.SBOM{
|
||||
Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(pkg.Package{
|
||||
Name: "opensearch",
|
||||
Type: "",
|
||||
CPEs: []cpe.CPE{
|
||||
cpe.Must("cpe:2.3:a:amazon:opensearch:*:*:*:*:*:loremipsum:*:*", ""),
|
||||
},
|
||||
}),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid prefix",
|
||||
userInput: "dir:test-fixtures/cpe",
|
||||
sbom: &sbom.SBOM{
|
||||
Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
syftPkgOpts := []cmp.Option{
|
||||
cmpopts.IgnoreFields(pkg.Package{}, "id", "Language"),
|
||||
cmpopts.IgnoreUnexported(pkg.Package{}, file.LocationSet{}, pkg.LicenseSet{}),
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dec := NewFormatDecoder()
|
||||
|
||||
decodedSBOM, _, _, err := dec.Decode(strings.NewReader(tc.userInput))
|
||||
require.NoError(t, err)
|
||||
|
||||
gotSyftPkgs := decodedSBOM.Artifacts.Packages.Sorted()
|
||||
wantSyftPkgs := tc.sbom.Artifacts.Packages.Sorted()
|
||||
require.Equal(t, len(gotSyftPkgs), len(wantSyftPkgs))
|
||||
for idx, wantPkg := range wantSyftPkgs {
|
||||
if d := cmp.Diff(wantPkg, gotSyftPkgs[idx], syftPkgOpts...); d != "" {
|
||||
t.Errorf("unexpected Syft Package (-want +got):\n%s", d)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -3,6 +3,7 @@ package format
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/anchore/syft/syft/format/cpes"
|
||||
"github.com/anchore/syft/syft/format/cyclonedxjson"
|
||||
"github.com/anchore/syft/syft/format/cyclonedxxml"
|
||||
"github.com/anchore/syft/syft/format/purls"
|
||||
@ -26,6 +27,7 @@ func Decoders() []sbom.FormatDecoder {
|
||||
spdxtagvalue.NewFormatDecoder(),
|
||||
spdxjson.NewFormatDecoder(),
|
||||
purls.NewFormatDecoder(),
|
||||
cpes.NewFormatDecoder(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,11 +1,13 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/anchore/archiver/v3"
|
||||
"github.com/mholt/archives"
|
||||
|
||||
"github.com/anchore/packageurl-go"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
@ -153,8 +155,8 @@ func trimRelative(s string) string {
|
||||
|
||||
// isArchive returns true if the path appears to be an archive
|
||||
func isArchive(path string) bool {
|
||||
_, err := archiver.ByExtension(path)
|
||||
return err == nil
|
||||
format, _, err := archives.Identify(context.Background(), path, nil)
|
||||
return err == nil && format != nil
|
||||
}
|
||||
|
||||
func toDependencies(s *sbom.SBOM, p pkg.Package) (out []string) {
|
||||
|
||||
@ -10,13 +10,31 @@ import (
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/cpe"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
cataloger "github.com/anchore/syft/syft/pkg/cataloger/common/cpe"
|
||||
)
|
||||
|
||||
// Backfill takes all information present in the package and attempts to fill in any missing information
|
||||
// from any available sources, such as the Metadata and PURL.
|
||||
// from any available sources, such as the Metadata, PURL, or CPEs.
|
||||
//
|
||||
// Backfill does not call p.SetID(), but this needs to be called later to ensure it's up to date
|
||||
func Backfill(p *pkg.Package) {
|
||||
backfillFromPurl(p)
|
||||
backfillFromCPE(p)
|
||||
}
|
||||
|
||||
func backfillFromCPE(p *pkg.Package) {
|
||||
if len(p.CPEs) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
c := p.CPEs[0]
|
||||
|
||||
if p.Type == "" {
|
||||
p.Type = cataloger.TargetSoftwareToPackageType(c.Attributes.TargetSW)
|
||||
}
|
||||
}
|
||||
|
||||
func backfillFromPurl(p *pkg.Package) {
|
||||
if p.PURL == "" {
|
||||
return
|
||||
}
|
||||
|
||||
@ -121,6 +121,20 @@ func Test_Backfill(t *testing.T) {
|
||||
Metadata: pkg.JavaArchive{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "target-sw from CPE",
|
||||
in: pkg.Package{
|
||||
CPEs: []cpe.CPE{
|
||||
cpe.Must("cpe:2.3:a:amazon:opensearch:*:*:*:*:*:ruby:*:*", ""),
|
||||
},
|
||||
},
|
||||
expected: pkg.Package{
|
||||
CPEs: []cpe.CPE{
|
||||
cpe.Must("cpe:2.3:a:amazon:opensearch:*:*:*:*:*:ruby:*:*", ""),
|
||||
},
|
||||
Type: pkg.GemPkg,
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@ -40,8 +40,11 @@ func EncodeComponent(p pkg.Package, supplier string, locationSorter func(a, b fi
|
||||
}
|
||||
|
||||
componentType := cyclonedx.ComponentTypeLibrary
|
||||
if p.Type == pkg.BinaryPkg {
|
||||
switch p.Type {
|
||||
case pkg.BinaryPkg:
|
||||
componentType = cyclonedx.ComponentTypeApplication
|
||||
case pkg.ModelPkg:
|
||||
componentType = cyclonedx.ComponentTypeMachineLearningModel
|
||||
}
|
||||
|
||||
return cyclonedx.Component{
|
||||
|
||||
@ -62,7 +62,7 @@ func collectPackages(component *cyclonedx.Component, s *sbom.SBOM, idMap map[str
|
||||
switch component.Type {
|
||||
case cyclonedx.ComponentTypeOS:
|
||||
case cyclonedx.ComponentTypeContainer:
|
||||
case cyclonedx.ComponentTypeApplication, cyclonedx.ComponentTypeFramework, cyclonedx.ComponentTypeLibrary:
|
||||
case cyclonedx.ComponentTypeApplication, cyclonedx.ComponentTypeFramework, cyclonedx.ComponentTypeLibrary, cyclonedx.ComponentTypeMachineLearningModel:
|
||||
p := decodeComponent(component)
|
||||
idMap[component.BOMRef] = p
|
||||
if component.BOMRef != "" {
|
||||
|
||||
@ -40,6 +40,7 @@ func Test_OriginatorSupplier(t *testing.T) {
|
||||
pkg.PhpComposerInstalledEntry{},
|
||||
pkg.PhpPearEntry{},
|
||||
pkg.PhpPeclEntry{},
|
||||
pkg.PnpmLockEntry{},
|
||||
pkg.PortageEntry{},
|
||||
pkg.PythonPipfileLockEntry{},
|
||||
pkg.PythonPdmLockEntry{},
|
||||
@ -54,6 +55,7 @@ func Test_OriginatorSupplier(t *testing.T) {
|
||||
pkg.OpamPackage{},
|
||||
pkg.YarnLockEntry{},
|
||||
pkg.TerraformLockProviderEntry{},
|
||||
pkg.GGUFFileHeader{},
|
||||
)
|
||||
tests := []struct {
|
||||
name string
|
||||
@ -347,10 +349,10 @@ func Test_OriginatorSupplier(t *testing.T) {
|
||||
name: "from python PDM lock",
|
||||
input: pkg.Package{
|
||||
Metadata: pkg.PythonPdmLockEntry{
|
||||
Files: []pkg.PythonFileRecord{
|
||||
Files: []pkg.PythonPdmFileEntry{
|
||||
{
|
||||
Path: "",
|
||||
Digest: &pkg.PythonFileDigest{
|
||||
URL: "https://pypi.org/project/testpkg/1.2.3/file1.tar.gz",
|
||||
Digest: pkg.PythonFileDigest{
|
||||
Algorithm: "sha256",
|
||||
Value: "3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651",
|
||||
},
|
||||
|
||||
@ -82,6 +82,8 @@ func SourceInfo(p pkg.Package) string {
|
||||
answer = "acquired package info from Homebrew formula"
|
||||
case pkg.TerraformPkg:
|
||||
answer = "acquired package info from Terraform dependency lock file"
|
||||
case pkg.ModelPkg:
|
||||
answer = "acquired package info from AI artifact (e.g. GGUF File"
|
||||
default:
|
||||
answer = "acquired package info from the following paths"
|
||||
}
|
||||
|
||||
@ -351,6 +351,14 @@ func Test_SourceInfo(t *testing.T) {
|
||||
"acquired package info from Terraform dependency lock file",
|
||||
},
|
||||
},
|
||||
{
|
||||
input: pkg.Package{
|
||||
Type: pkg.ModelPkg,
|
||||
},
|
||||
expected: []string{
|
||||
"",
|
||||
},
|
||||
},
|
||||
}
|
||||
var pkgTypes []pkg.Type
|
||||
for _, test := range tests {
|
||||
|
||||
@ -35,14 +35,23 @@ func (d *Document) UnmarshalJSON(data []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Descriptor describes what created the document as well as surrounding metadata
|
||||
// Descriptor identifies the tool that generated this SBOM document, including its name, version, and configuration used during catalog generation.
|
||||
type Descriptor struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
// Name is the name of the tool that generated this SBOM (e.g., "syft").
|
||||
Name string `json:"name"`
|
||||
|
||||
// Version is the version of the tool that generated this SBOM.
|
||||
Version string `json:"version"`
|
||||
|
||||
// Configuration contains the tool configuration used during SBOM generation.
|
||||
Configuration interface{} `json:"configuration,omitempty"`
|
||||
}
|
||||
|
||||
// Schema specifies the JSON schema version and URL reference that defines the structure and validation rules for this document format.
|
||||
type Schema struct {
|
||||
// Version is the JSON schema version for this document format.
|
||||
Version string `json:"version"`
|
||||
URL string `json:"url"`
|
||||
|
||||
// URL is the URL to the JSON schema definition document.
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
@ -10,25 +10,55 @@ import (
|
||||
"github.com/anchore/syft/syft/license"
|
||||
)
|
||||
|
||||
// File represents a file discovered during cataloging with its metadata, content digests, licenses, and relationships to packages.
|
||||
type File struct {
|
||||
ID string `json:"id"`
|
||||
Location file.Coordinates `json:"location"`
|
||||
Metadata *FileMetadataEntry `json:"metadata,omitempty"`
|
||||
Contents string `json:"contents,omitempty"`
|
||||
Digests []file.Digest `json:"digests,omitempty"`
|
||||
Licenses []FileLicense `json:"licenses,omitempty"`
|
||||
Executable *file.Executable `json:"executable,omitempty"`
|
||||
Unknowns []string `json:"unknowns,omitempty"`
|
||||
// ID is a unique identifier for this file within the SBOM.
|
||||
ID string `json:"id"`
|
||||
|
||||
// Location is the file path and layer information where this file was found.
|
||||
Location file.Coordinates `json:"location"`
|
||||
|
||||
// Metadata contains filesystem metadata such as permissions, ownership, and file type.
|
||||
Metadata *FileMetadataEntry `json:"metadata,omitempty"`
|
||||
|
||||
// Contents is the file contents for small files.
|
||||
Contents string `json:"contents,omitempty"`
|
||||
|
||||
// Digests contains cryptographic hashes of the file contents.
|
||||
Digests []file.Digest `json:"digests,omitempty"`
|
||||
|
||||
// Licenses contains license information discovered within this file.
|
||||
Licenses []FileLicense `json:"licenses,omitempty"`
|
||||
|
||||
// Executable contains executable metadata if this file is a binary.
|
||||
Executable *file.Executable `json:"executable,omitempty"`
|
||||
|
||||
// Unknowns contains unknown fields for forward compatibility.
|
||||
Unknowns []string `json:"unknowns,omitempty"`
|
||||
}
|
||||
|
||||
// FileMetadataEntry contains filesystem-level metadata attributes such as permissions, ownership, type, and size for a cataloged file.
|
||||
type FileMetadataEntry struct {
|
||||
Mode int `json:"mode"`
|
||||
Type string `json:"type"`
|
||||
// Mode is the Unix file permission mode in octal format.
|
||||
Mode int `json:"mode"`
|
||||
|
||||
// Type is the file type (e.g., "RegularFile", "Directory", "SymbolicLink").
|
||||
Type string `json:"type"`
|
||||
|
||||
// LinkDestination is the target path for symbolic links.
|
||||
LinkDestination string `json:"linkDestination,omitempty"`
|
||||
UserID int `json:"userID"`
|
||||
GroupID int `json:"groupID"`
|
||||
MIMEType string `json:"mimeType"`
|
||||
Size int64 `json:"size"`
|
||||
|
||||
// UserID is the file owner user ID.
|
||||
UserID int `json:"userID"`
|
||||
|
||||
// GroupID is the file owner group ID.
|
||||
GroupID int `json:"groupID"`
|
||||
|
||||
// MIMEType is the MIME type of the file contents.
|
||||
MIMEType string `json:"mimeType"`
|
||||
|
||||
// Size is the file size in bytes.
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type auxFileMetadataEntry FileMetadataEntry
|
||||
@ -82,17 +112,31 @@ type sbomImportLegacyFileMetadataEntry struct {
|
||||
Size int64 `json:"Size"`
|
||||
}
|
||||
|
||||
// FileLicense represents license information discovered within a file's contents or metadata, including the matched license text and SPDX expression.
|
||||
type FileLicense struct {
|
||||
Value string `json:"value"`
|
||||
SPDXExpression string `json:"spdxExpression"`
|
||||
Type license.Type `json:"type"`
|
||||
Evidence *FileLicenseEvidence `json:"evidence,omitempty"`
|
||||
// Value is the raw license identifier or text as found in the file.
|
||||
Value string `json:"value"`
|
||||
|
||||
// SPDXExpression is the parsed SPDX license expression.
|
||||
SPDXExpression string `json:"spdxExpression"`
|
||||
|
||||
// Type is the license type classification (e.g., declared, concluded, discovered).
|
||||
Type license.Type `json:"type"`
|
||||
|
||||
// Evidence contains supporting evidence for this license detection.
|
||||
Evidence *FileLicenseEvidence `json:"evidence,omitempty"`
|
||||
}
|
||||
|
||||
// FileLicenseEvidence contains supporting evidence for a license detection in a file, including the byte offset, extent, and confidence level.
|
||||
type FileLicenseEvidence struct {
|
||||
// Confidence is the confidence score for this license detection (0-100).
|
||||
Confidence int `json:"confidence"`
|
||||
Offset int `json:"offset"`
|
||||
Extent int `json:"extent"`
|
||||
|
||||
// Offset is the byte offset where the license text starts in the file.
|
||||
Offset int `json:"offset"`
|
||||
|
||||
// Extent is the length of the license text in bytes.
|
||||
Extent int `json:"extent"`
|
||||
}
|
||||
|
||||
type intOrStringFileType struct {
|
||||
|
||||
@ -4,28 +4,67 @@ import (
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
// IDLikes represents a list of distribution IDs that this Linux distribution is similar to or derived from, as defined in os-release ID_LIKE field.
|
||||
type IDLikes []string
|
||||
|
||||
// LinuxRelease contains Linux distribution identification and version information extracted from /etc/os-release or similar system files.
|
||||
type LinuxRelease struct {
|
||||
PrettyName string `json:"prettyName,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
ID string `json:"id,omitempty"`
|
||||
IDLike IDLikes `json:"idLike,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
VersionID string `json:"versionID,omitempty"`
|
||||
VersionCodename string `json:"versionCodename,omitempty"`
|
||||
BuildID string `json:"buildID,omitempty"`
|
||||
ImageID string `json:"imageID,omitempty"`
|
||||
ImageVersion string `json:"imageVersion,omitempty"`
|
||||
Variant string `json:"variant,omitempty"`
|
||||
VariantID string `json:"variantID,omitempty"`
|
||||
HomeURL string `json:"homeURL,omitempty"`
|
||||
SupportURL string `json:"supportURL,omitempty"`
|
||||
BugReportURL string `json:"bugReportURL,omitempty"`
|
||||
PrivacyPolicyURL string `json:"privacyPolicyURL,omitempty"`
|
||||
CPEName string `json:"cpeName,omitempty"`
|
||||
SupportEnd string `json:"supportEnd,omitempty"`
|
||||
ExtendedSupport bool `json:"extendedSupport,omitempty"`
|
||||
// PrettyName is a human-readable operating system name with version.
|
||||
PrettyName string `json:"prettyName,omitempty"`
|
||||
|
||||
// Name is the operating system name without version information.
|
||||
Name string `json:"name,omitempty"`
|
||||
|
||||
// ID is the lower-case operating system identifier (e.g., "ubuntu", "rhel").
|
||||
ID string `json:"id,omitempty"`
|
||||
|
||||
// IDLike is a list of operating system IDs this distribution is similar to or derived from.
|
||||
IDLike IDLikes `json:"idLike,omitempty"`
|
||||
|
||||
// Version is the operating system version including codename if available.
|
||||
Version string `json:"version,omitempty"`
|
||||
|
||||
// VersionID is the operating system version number or identifier.
|
||||
VersionID string `json:"versionID,omitempty"`
|
||||
|
||||
// VersionCodename is the operating system release codename (e.g., "jammy", "bullseye").
|
||||
VersionCodename string `json:"versionCodename,omitempty"`
|
||||
|
||||
// BuildID is a build identifier for the operating system.
|
||||
BuildID string `json:"buildID,omitempty"`
|
||||
|
||||
// ImageID is an identifier for container or cloud images.
|
||||
ImageID string `json:"imageID,omitempty"`
|
||||
|
||||
// ImageVersion is the version for container or cloud images.
|
||||
ImageVersion string `json:"imageVersion,omitempty"`
|
||||
|
||||
// Variant is the operating system variant name (e.g., "Server", "Workstation").
|
||||
Variant string `json:"variant,omitempty"`
|
||||
|
||||
// VariantID is the lower-case operating system variant identifier.
|
||||
VariantID string `json:"variantID,omitempty"`
|
||||
|
||||
// HomeURL is the homepage URL for the operating system.
|
||||
HomeURL string `json:"homeURL,omitempty"`
|
||||
|
||||
// SupportURL is the support or help URL for the operating system.
|
||||
SupportURL string `json:"supportURL,omitempty"`
|
||||
|
||||
// BugReportURL is the bug reporting URL for the operating system.
|
||||
BugReportURL string `json:"bugReportURL,omitempty"`
|
||||
|
||||
// PrivacyPolicyURL is the privacy policy URL for the operating system.
|
||||
PrivacyPolicyURL string `json:"privacyPolicyURL,omitempty"`
|
||||
|
||||
// CPEName is the Common Platform Enumeration name for the operating system.
|
||||
CPEName string `json:"cpeName,omitempty"`
|
||||
|
||||
// SupportEnd is the end of support date or version identifier.
|
||||
SupportEnd string `json:"supportEnd,omitempty"`
|
||||
|
||||
// ExtendedSupport indicates whether extended security or support is available.
|
||||
ExtendedSupport bool `json:"extendedSupport,omitempty"`
|
||||
}
|
||||
|
||||
func (s *IDLikes) UnmarshalJSON(data []byte) error {
|
||||
|
||||
@ -36,22 +36,40 @@ type PackageBasicData struct {
|
||||
PURL string `json:"purl"`
|
||||
}
|
||||
|
||||
// cpes is a collection of Common Platform Enumeration identifiers for a package.
|
||||
type cpes []CPE
|
||||
|
||||
// CPE represents a Common Platform Enumeration identifier used for matching packages to known vulnerabilities in security databases.
|
||||
type CPE struct {
|
||||
Value string `json:"cpe"`
|
||||
// Value is the CPE string identifier.
|
||||
Value string `json:"cpe"`
|
||||
|
||||
// Source is the source where this CPE was obtained or generated from.
|
||||
Source string `json:"source,omitempty"`
|
||||
}
|
||||
|
||||
// licenses is a collection of license findings associated with a package.
|
||||
type licenses []License
|
||||
|
||||
// License represents software license information discovered for a package, including SPDX expressions and supporting evidence locations.
|
||||
type License struct {
|
||||
Value string `json:"value"`
|
||||
SPDXExpression string `json:"spdxExpression"`
|
||||
Type license.Type `json:"type"`
|
||||
URLs []string `json:"urls"`
|
||||
Locations []file.Location `json:"locations"`
|
||||
Contents string `json:"contents,omitempty"`
|
||||
// Value is the raw license identifier or expression as found.
|
||||
Value string `json:"value"`
|
||||
|
||||
// SPDXExpression is the parsed SPDX license expression.
|
||||
SPDXExpression string `json:"spdxExpression"`
|
||||
|
||||
// Type is the license type classification (e.g., declared, concluded, discovered).
|
||||
Type license.Type `json:"type"`
|
||||
|
||||
// URLs are URLs where license text or information can be found.
|
||||
URLs []string `json:"urls"`
|
||||
|
||||
// Locations are file locations where this license was discovered.
|
||||
Locations []file.Location `json:"locations"`
|
||||
|
||||
// Contents is the full license text content.
|
||||
Contents string `json:"contents,omitempty"`
|
||||
}
|
||||
|
||||
func newModelLicensesFromValues(licenses []string) (ml []License) {
|
||||
|
||||
@ -1,8 +1,16 @@
|
||||
package model
|
||||
|
||||
// Relationship represents a directed relationship between two artifacts in the SBOM, such as package-contains-file or package-depends-on-package.
|
||||
type Relationship struct {
|
||||
Parent string `json:"parent"`
|
||||
Child string `json:"child"`
|
||||
Type string `json:"type"`
|
||||
// Parent is the ID of the parent artifact in this relationship.
|
||||
Parent string `json:"parent"`
|
||||
|
||||
// Child is the ID of the child artifact in this relationship.
|
||||
Child string `json:"child"`
|
||||
|
||||
// Type is the relationship type (e.g., "contains", "dependency-of", "ancestor-of").
|
||||
Type string `json:"type"`
|
||||
|
||||
// Metadata contains additional relationship-specific metadata.
|
||||
Metadata interface{} `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
@ -11,18 +11,25 @@ import (
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
// Source object represents the thing that was cataloged
|
||||
// Note: syft currently makes no claims or runs any logic to determine the Supplier field below
|
||||
|
||||
// Instead, the Supplier can be determined by the user of syft and passed as a config or flag to help fulfill
|
||||
// the NTIA minimum elements. For mor information see the NTIA framing document below
|
||||
// https://www.ntia.gov/files/ntia/publications/framingsbom_20191112.pdf
|
||||
// Source represents the artifact that was analyzed to generate this SBOM, such as a container image, directory, or file archive.
|
||||
// The Supplier field can be provided by users to fulfill NTIA minimum elements requirements.
|
||||
type Source struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Supplier string `json:"supplier,omitempty"`
|
||||
Type string `json:"type"`
|
||||
// ID is a unique identifier for the analyzed source artifact.
|
||||
ID string `json:"id"`
|
||||
|
||||
// Name is the name of the analyzed artifact (e.g., image name, directory path).
|
||||
Name string `json:"name"`
|
||||
|
||||
// Version is the version of the analyzed artifact (e.g., image tag).
|
||||
Version string `json:"version"`
|
||||
|
||||
// Supplier is supplier information, which can be user-provided for NTIA minimum elements compliance.
|
||||
Supplier string `json:"supplier,omitempty"`
|
||||
|
||||
// Type is the source type (e.g., "image", "directory", "file").
|
||||
Type string `json:"type"`
|
||||
|
||||
// Metadata contains additional source-specific metadata.
|
||||
Metadata interface{} `json:"metadata"`
|
||||
}
|
||||
|
||||
|
||||
@ -58,6 +58,7 @@ type AlpmDBEntry struct {
|
||||
Depends []string `mapstructure:"depends" json:"depends,omitempty"`
|
||||
}
|
||||
|
||||
// AlpmFileRecord represents a single file entry within an Arch Linux package with its associated metadata tracked by pacman.
|
||||
type AlpmFileRecord struct {
|
||||
// Path is the file path relative to the filesystem root
|
||||
Path string `mapstruture:"path" json:"path,omitempty"`
|
||||
|
||||
16
syft/pkg/cataloger/ai/cataloger.go
Normal file
16
syft/pkg/cataloger/ai/cataloger.go
Normal file
@ -0,0 +1,16 @@
|
||||
/*
|
||||
Package ai provides concrete Cataloger implementations for AI artifacts and machine learning models,
|
||||
including support for GGUF (GPT-Generated Unified Format) model files.
|
||||
*/
|
||||
package ai
|
||||
|
||||
import (
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/generic"
|
||||
)
|
||||
|
||||
// NewGGUFCataloger returns a new cataloger instance for GGUF model files.
|
||||
func NewGGUFCataloger() pkg.Cataloger {
|
||||
return generic.NewCataloger("gguf-cataloger").
|
||||
WithParserByGlobs(parseGGUFModel, "**/*.gguf")
|
||||
}
|
||||
140
syft/pkg/cataloger/ai/cataloger_test.go
Normal file
140
syft/pkg/cataloger/ai/cataloger_test.go
Normal file
@ -0,0 +1,140 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
|
||||
)
|
||||
|
||||
func TestGGUFCataloger_Globs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fixture string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "obtain gguf files",
|
||||
fixture: "test-fixtures/glob-paths",
|
||||
expected: []string{
|
||||
"models/model.gguf",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
pkgtest.NewCatalogTester().
|
||||
FromDirectory(t, test.fixture).
|
||||
ExpectsResolverContentQueries(test.expected).
|
||||
TestCataloger(t, NewGGUFCataloger())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGGUFCataloger(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(t *testing.T) string
|
||||
expectedPackages []pkg.Package
|
||||
expectedRelationships []artifact.Relationship
|
||||
}{
|
||||
{
|
||||
name: "catalog single GGUF file",
|
||||
setup: func(t *testing.T) string {
|
||||
dir := t.TempDir()
|
||||
data := newTestGGUFBuilder().
|
||||
withVersion(3).
|
||||
withStringKV("general.architecture", "llama").
|
||||
withStringKV("general.name", "llama3-8b").
|
||||
withStringKV("general.version", "3.0").
|
||||
withStringKV("general.license", "Apache-2.0").
|
||||
withStringKV("general.quantization", "Q4_K_M").
|
||||
withUint64KV("general.parameter_count", 8030000000).
|
||||
withStringKV("general.some_random_kv", "foobar").
|
||||
build()
|
||||
|
||||
path := filepath.Join(dir, "llama3-8b.gguf")
|
||||
os.WriteFile(path, data, 0644)
|
||||
return dir
|
||||
},
|
||||
expectedPackages: []pkg.Package{
|
||||
{
|
||||
Name: "llama3-8b",
|
||||
Version: "3.0",
|
||||
Type: pkg.ModelPkg,
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
pkg.NewLicenseFromFields("Apache-2.0", "", nil),
|
||||
),
|
||||
Metadata: pkg.GGUFFileHeader{
|
||||
Architecture: "llama",
|
||||
Quantization: "Unknown",
|
||||
Parameters: 0,
|
||||
GGUFVersion: 3,
|
||||
TensorCount: 0,
|
||||
MetadataKeyValuesHash: "6e3d368066455ce4",
|
||||
RemainingKeyValues: map[string]interface{}{
|
||||
"general.some_random_kv": "foobar",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedRelationships: nil,
|
||||
},
|
||||
{
|
||||
name: "catalog GGUF file with minimal metadata",
|
||||
setup: func(t *testing.T) string {
|
||||
dir := t.TempDir()
|
||||
data := newTestGGUFBuilder().
|
||||
withVersion(3).
|
||||
withStringKV("general.architecture", "gpt2").
|
||||
withStringKV("general.name", "gpt2-small").
|
||||
withStringKV("gpt2.context_length", "1024").
|
||||
withUint32KV("gpt2.embedding_length", 768).
|
||||
build()
|
||||
|
||||
path := filepath.Join(dir, "gpt2-small.gguf")
|
||||
os.WriteFile(path, data, 0644)
|
||||
return dir
|
||||
},
|
||||
expectedPackages: []pkg.Package{
|
||||
{
|
||||
Name: "gpt2-small",
|
||||
Version: "",
|
||||
Type: pkg.ModelPkg,
|
||||
Licenses: pkg.NewLicenseSet(),
|
||||
Metadata: pkg.GGUFFileHeader{
|
||||
Architecture: "gpt2",
|
||||
Quantization: "Unknown",
|
||||
Parameters: 0,
|
||||
GGUFVersion: 3,
|
||||
TensorCount: 0,
|
||||
MetadataKeyValuesHash: "9dc6f23591062a27",
|
||||
RemainingKeyValues: map[string]interface{}{
|
||||
"gpt2.context_length": "1024",
|
||||
"gpt2.embedding_length": uint32(768),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedRelationships: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
fixtureDir := tt.setup(t)
|
||||
|
||||
// Use pkgtest to catalog and compare
|
||||
pkgtest.NewCatalogTester().
|
||||
FromDirectory(t, fixtureDir).
|
||||
Expects(tt.expectedPackages, tt.expectedRelationships).
|
||||
IgnoreLocationLayer().
|
||||
IgnorePackageFields("FoundBy", "Locations").
|
||||
TestCataloger(t, NewGGUFCataloger())
|
||||
})
|
||||
}
|
||||
}
|
||||
22
syft/pkg/cataloger/ai/package.go
Normal file
22
syft/pkg/cataloger/ai/package.go
Normal file
@ -0,0 +1,22 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
func newGGUFPackage(metadata *pkg.GGUFFileHeader, modelName, version, license string, locations ...file.Location) pkg.Package {
|
||||
p := pkg.Package{
|
||||
Name: modelName,
|
||||
Version: version,
|
||||
Locations: file.NewLocationSet(locations...),
|
||||
Type: pkg.ModelPkg,
|
||||
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValues(license)...),
|
||||
Metadata: *metadata,
|
||||
// NOTE: PURL is intentionally not set as the package-url spec
|
||||
// has not yet finalized support for ML model packages
|
||||
}
|
||||
p.SetID()
|
||||
|
||||
return p
|
||||
}
|
||||
121
syft/pkg/cataloger/ai/package_test.go
Normal file
121
syft/pkg/cataloger/ai/package_test.go
Normal file
@ -0,0 +1,121 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
|
||||
)
|
||||
|
||||
func TestNewGGUFPackage(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
metadata *pkg.GGUFFileHeader
|
||||
input struct {
|
||||
modelName string
|
||||
version string
|
||||
license string
|
||||
locations []file.Location
|
||||
}
|
||||
expected pkg.Package
|
||||
}{
|
||||
{
|
||||
name: "complete GGUF package with all fields",
|
||||
input: struct {
|
||||
modelName string
|
||||
version string
|
||||
license string
|
||||
locations []file.Location
|
||||
}{
|
||||
modelName: "llama3-8b",
|
||||
version: "3.0",
|
||||
license: "Apache-2.0",
|
||||
locations: []file.Location{file.NewLocation("/models/llama3-8b.gguf")},
|
||||
},
|
||||
metadata: &pkg.GGUFFileHeader{
|
||||
Architecture: "llama",
|
||||
Quantization: "Q4_K_M",
|
||||
Parameters: 8030000000,
|
||||
GGUFVersion: 3,
|
||||
TensorCount: 291,
|
||||
RemainingKeyValues: map[string]any{
|
||||
"general.random_kv": "foobar",
|
||||
},
|
||||
},
|
||||
expected: pkg.Package{
|
||||
Name: "llama3-8b",
|
||||
Version: "3.0",
|
||||
Type: pkg.ModelPkg,
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
pkg.NewLicenseFromFields("Apache-2.0", "", nil),
|
||||
),
|
||||
Metadata: pkg.GGUFFileHeader{
|
||||
Architecture: "llama",
|
||||
Quantization: "Q4_K_M",
|
||||
Parameters: 8030000000,
|
||||
GGUFVersion: 3,
|
||||
TensorCount: 291,
|
||||
RemainingKeyValues: map[string]any{
|
||||
"general.random_kv": "foobar",
|
||||
},
|
||||
},
|
||||
Locations: file.NewLocationSet(file.NewLocation("/models/llama3-8b.gguf")),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "minimal GGUF package",
|
||||
input: struct {
|
||||
modelName string
|
||||
version string
|
||||
license string
|
||||
locations []file.Location
|
||||
}{
|
||||
modelName: "gpt2-small",
|
||||
version: "1.0",
|
||||
license: "MIT",
|
||||
locations: []file.Location{file.NewLocation("/models/simple.gguf")},
|
||||
},
|
||||
metadata: &pkg.GGUFFileHeader{
|
||||
Architecture: "gpt2",
|
||||
GGUFVersion: 3,
|
||||
TensorCount: 50,
|
||||
},
|
||||
expected: pkg.Package{
|
||||
Name: "gpt2-small",
|
||||
Version: "1.0",
|
||||
Type: pkg.ModelPkg,
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
pkg.NewLicenseFromFields("MIT", "", nil),
|
||||
),
|
||||
Metadata: pkg.GGUFFileHeader{
|
||||
Architecture: "gpt2",
|
||||
GGUFVersion: 3,
|
||||
TensorCount: 50,
|
||||
},
|
||||
Locations: file.NewLocationSet(file.NewLocation("/models/simple.gguf")),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
actual := newGGUFPackage(
|
||||
tt.metadata,
|
||||
tt.input.modelName,
|
||||
tt.input.version,
|
||||
tt.input.license,
|
||||
tt.input.locations...,
|
||||
)
|
||||
|
||||
// Verify metadata type
|
||||
_, ok := actual.Metadata.(pkg.GGUFFileHeader)
|
||||
require.True(t, ok, "metadata should be GGUFFileHeader")
|
||||
|
||||
// Use AssertPackagesEqual for comprehensive comparison
|
||||
pkgtest.AssertPackagesEqual(t, tt.expected, actual)
|
||||
})
|
||||
}
|
||||
}
|
||||
63
syft/pkg/cataloger/ai/parse_gguf.go
Normal file
63
syft/pkg/cataloger/ai/parse_gguf.go
Normal file
@ -0,0 +1,63 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
gguf_parser "github.com/gpustack/gguf-parser-go"
|
||||
)
|
||||
|
||||
// GGUF file format constants
|
||||
const (
|
||||
ggufMagicNumber = 0x46554747 // "GGUF" in little-endian
|
||||
maxHeaderSize = 50 * 1024 * 1024 // 50MB for large tokenizer vocabularies
|
||||
)
|
||||
|
||||
// copyHeader copies the GGUF header from the reader to the writer.
|
||||
// It validates the magic number first, then copies the rest of the data.
|
||||
// The reader should be wrapped with io.LimitedReader to prevent OOM issues.
|
||||
func copyHeader(w io.Writer, r io.Reader) error {
|
||||
// Read initial chunk to validate magic number
|
||||
// GGUF format: magic(4) + version(4) + tensor_count(8) + metadata_kv_count(8) + metadata_kvs + tensors_info
|
||||
initialBuf := make([]byte, 24) // Enough for magic, version, tensor count, and kv count
|
||||
if _, err := io.ReadFull(r, initialBuf); err != nil {
|
||||
return fmt.Errorf("failed to read GGUF header prefix: %w", err)
|
||||
}
|
||||
|
||||
// Verify magic number
|
||||
magic := binary.LittleEndian.Uint32(initialBuf[0:4])
|
||||
if magic != ggufMagicNumber {
|
||||
return fmt.Errorf("invalid GGUF magic number: 0x%08X", magic)
|
||||
}
|
||||
|
||||
// Write the initial buffer to the writer
|
||||
if _, err := w.Write(initialBuf); err != nil {
|
||||
return fmt.Errorf("failed to write GGUF header prefix: %w", err)
|
||||
}
|
||||
|
||||
// Copy the rest of the header from reader to writer
|
||||
// The LimitedReader will return EOF once maxHeaderSize is reached
|
||||
if _, err := io.Copy(w, r); err != nil {
|
||||
return fmt.Errorf("failed to copy GGUF header: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Helper to convert gguf_parser metadata to simpler types
|
||||
func convertGGUFMetadataKVs(kvs gguf_parser.GGUFMetadataKVs) map[string]interface{} {
|
||||
result := make(map[string]interface{})
|
||||
|
||||
for _, kv := range kvs {
|
||||
// Skip standard fields that are extracted separately
|
||||
switch kv.Key {
|
||||
case "general.architecture", "general.name", "general.license",
|
||||
"general.version", "general.parameter_count", "general.quantization":
|
||||
continue
|
||||
}
|
||||
result[kv.Key] = kv.Value
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
135
syft/pkg/cataloger/ai/parse_gguf_model.go
Normal file
135
syft/pkg/cataloger/ai/parse_gguf_model.go
Normal file
@ -0,0 +1,135 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/cespare/xxhash/v2"
|
||||
gguf_parser "github.com/gpustack/gguf-parser-go"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/unknown"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/generic"
|
||||
)
|
||||
|
||||
// parseGGUFModel parses a GGUF model file and returns the discovered package.
|
||||
// This implementation only reads the header portion of the file, not the entire model.
|
||||
func parseGGUFModel(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
defer internal.CloseAndLogError(reader, reader.Path())
|
||||
|
||||
// Create a temporary file for the library to parse
|
||||
// The library requires a file path, so we create a temp file
|
||||
tempFile, err := os.CreateTemp("", "syft-gguf-*.gguf")
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to create temp file: %w", err)
|
||||
}
|
||||
tempPath := tempFile.Name()
|
||||
defer os.Remove(tempPath)
|
||||
|
||||
// Copy and validate the GGUF file header using LimitedReader to prevent OOM
|
||||
// We use LimitedReader to cap reads at maxHeaderSize (50MB)
|
||||
limitedReader := &io.LimitedReader{R: reader, N: maxHeaderSize}
|
||||
if err := copyHeader(tempFile, limitedReader); err != nil {
|
||||
tempFile.Close()
|
||||
return nil, nil, fmt.Errorf("failed to copy GGUF header: %w", err)
|
||||
}
|
||||
tempFile.Close()
|
||||
|
||||
// Parse using gguf-parser-go with options to skip unnecessary data
|
||||
ggufFile, err := gguf_parser.ParseGGUFFile(tempPath,
|
||||
gguf_parser.SkipLargeMetadata(),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to parse GGUF file: %w", err)
|
||||
}
|
||||
|
||||
// Extract metadata
|
||||
metadata := ggufFile.Metadata()
|
||||
|
||||
// Extract version separately (will be set on Package.Version)
|
||||
modelVersion := extractVersion(ggufFile.Header.MetadataKV)
|
||||
|
||||
// Convert to syft metadata structure
|
||||
syftMetadata := &pkg.GGUFFileHeader{
|
||||
Architecture: metadata.Architecture,
|
||||
Quantization: metadata.FileTypeDescriptor,
|
||||
Parameters: uint64(metadata.Parameters),
|
||||
GGUFVersion: uint32(ggufFile.Header.Version),
|
||||
TensorCount: ggufFile.Header.TensorCount,
|
||||
RemainingKeyValues: convertGGUFMetadataKVs(ggufFile.Header.MetadataKV),
|
||||
MetadataKeyValuesHash: computeKVMetadataHash(ggufFile.Header.MetadataKV),
|
||||
}
|
||||
|
||||
// If model name is not in metadata, use filename
|
||||
if metadata.Name == "" {
|
||||
metadata.Name = extractModelNameFromPath(reader.Path())
|
||||
}
|
||||
|
||||
// Create package from metadata
|
||||
p := newGGUFPackage(
|
||||
syftMetadata,
|
||||
metadata.Name,
|
||||
modelVersion,
|
||||
metadata.License,
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
)
|
||||
|
||||
return []pkg.Package{p}, nil, unknown.IfEmptyf([]pkg.Package{p}, "unable to parse GGUF file")
|
||||
}
|
||||
|
||||
// computeKVMetadataHash computes a stable hash of the KV metadata for use as a global identifier
|
||||
func computeKVMetadataHash(metadata gguf_parser.GGUFMetadataKVs) string {
|
||||
// Sort the KV pairs by key for stable hashing
|
||||
sortedKVs := make([]gguf_parser.GGUFMetadataKV, len(metadata))
|
||||
copy(sortedKVs, metadata)
|
||||
sort.Slice(sortedKVs, func(i, j int) bool {
|
||||
return sortedKVs[i].Key < sortedKVs[j].Key
|
||||
})
|
||||
|
||||
// Marshal sorted KVs to JSON for stable hashing
|
||||
jsonBytes, err := json.Marshal(sortedKVs)
|
||||
if err != nil {
|
||||
log.Debugf("failed to marshal metadata for hashing: %v", err)
|
||||
return ""
|
||||
}
|
||||
|
||||
// Compute xxhash
|
||||
hash := xxhash.Sum64(jsonBytes)
|
||||
return fmt.Sprintf("%016x", hash) // 16 hex chars (64 bits)
|
||||
}
|
||||
|
||||
// extractVersion attempts to extract version from metadata KV pairs
|
||||
func extractVersion(kvs gguf_parser.GGUFMetadataKVs) string {
|
||||
for _, kv := range kvs {
|
||||
if kv.Key == "general.version" {
|
||||
if v, ok := kv.Value.(string); ok && v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// extractModelNameFromPath extracts the model name from the file path
|
||||
func extractModelNameFromPath(path string) string {
|
||||
// Get the base filename
|
||||
base := filepath.Base(path)
|
||||
|
||||
// Remove .gguf extension
|
||||
name := strings.TrimSuffix(base, ".gguf")
|
||||
|
||||
return name
|
||||
}
|
||||
|
||||
// integrity check
|
||||
var _ generic.Parser = parseGGUFModel
|
||||
128
syft/pkg/cataloger/ai/test_helpers_test.go
Normal file
128
syft/pkg/cataloger/ai/test_helpers_test.go
Normal file
@ -0,0 +1,128 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
)
|
||||
|
||||
// GGUF type constants for test builder
|
||||
// https://github.com/ggml-org/ggml/blob/master/docs/gguf.md
|
||||
const (
|
||||
ggufMagic = 0x46554747 // "GGUF" in little-endian
|
||||
ggufTypeUint8 = 0
|
||||
ggufTypeInt8 = 1
|
||||
ggufTypeUint16 = 2
|
||||
ggufTypeInt16 = 3
|
||||
ggufTypeUint32 = 4
|
||||
ggufTypeInt32 = 5
|
||||
ggufTypeFloat32 = 6
|
||||
ggufTypeBool = 7
|
||||
ggufTypeString = 8
|
||||
ggufTypeArray = 9
|
||||
ggufTypeUint64 = 10
|
||||
ggufTypeInt64 = 11
|
||||
ggufTypeFloat64 = 12
|
||||
)
|
||||
|
||||
// testGGUFBuilder helps build GGUF files for testing
|
||||
type testGGUFBuilder struct {
|
||||
buf *bytes.Buffer
|
||||
version uint32
|
||||
tensorCount uint64
|
||||
kvPairs []testKVPair
|
||||
}
|
||||
|
||||
type testKVPair struct {
|
||||
key string
|
||||
valueType uint32
|
||||
value interface{}
|
||||
}
|
||||
|
||||
func newTestGGUFBuilder() *testGGUFBuilder {
|
||||
return &testGGUFBuilder{
|
||||
buf: new(bytes.Buffer),
|
||||
version: 3,
|
||||
tensorCount: 0,
|
||||
kvPairs: []testKVPair{},
|
||||
}
|
||||
}
|
||||
|
||||
func (b *testGGUFBuilder) withVersion(v uint32) *testGGUFBuilder {
|
||||
b.version = v
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *testGGUFBuilder) withTensorCount(count uint64) *testGGUFBuilder {
|
||||
b.tensorCount = count
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *testGGUFBuilder) withStringKV(key, value string) *testGGUFBuilder {
|
||||
b.kvPairs = append(b.kvPairs, testKVPair{key: key, valueType: ggufTypeString, value: value})
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *testGGUFBuilder) withUint64KV(key string, value uint64) *testGGUFBuilder {
|
||||
b.kvPairs = append(b.kvPairs, testKVPair{key: key, valueType: ggufTypeUint64, value: value})
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *testGGUFBuilder) withUint32KV(key string, value uint32) *testGGUFBuilder {
|
||||
b.kvPairs = append(b.kvPairs, testKVPair{key: key, valueType: ggufTypeUint32, value: value})
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *testGGUFBuilder) writeString(s string) {
|
||||
binary.Write(b.buf, binary.LittleEndian, uint64(len(s)))
|
||||
b.buf.WriteString(s)
|
||||
}
|
||||
|
||||
func (b *testGGUFBuilder) build() []byte {
|
||||
// Write magic number "GGUF"
|
||||
binary.Write(b.buf, binary.LittleEndian, uint32(ggufMagic))
|
||||
|
||||
// Write version
|
||||
binary.Write(b.buf, binary.LittleEndian, b.version)
|
||||
|
||||
// Write tensor count
|
||||
binary.Write(b.buf, binary.LittleEndian, b.tensorCount)
|
||||
|
||||
// Write KV count
|
||||
binary.Write(b.buf, binary.LittleEndian, uint64(len(b.kvPairs)))
|
||||
|
||||
// Write KV pairs
|
||||
for _, kv := range b.kvPairs {
|
||||
// Write key
|
||||
b.writeString(kv.key)
|
||||
// Write value type
|
||||
binary.Write(b.buf, binary.LittleEndian, kv.valueType)
|
||||
// Write value based on type
|
||||
switch kv.valueType {
|
||||
case ggufTypeString:
|
||||
b.writeString(kv.value.(string))
|
||||
case ggufTypeUint32:
|
||||
binary.Write(b.buf, binary.LittleEndian, kv.value.(uint32))
|
||||
case ggufTypeUint64:
|
||||
binary.Write(b.buf, binary.LittleEndian, kv.value.(uint64))
|
||||
case ggufTypeUint8:
|
||||
binary.Write(b.buf, binary.LittleEndian, kv.value.(uint8))
|
||||
case ggufTypeInt32:
|
||||
binary.Write(b.buf, binary.LittleEndian, kv.value.(int32))
|
||||
case ggufTypeBool:
|
||||
var v uint8
|
||||
if kv.value.(bool) {
|
||||
v = 1
|
||||
}
|
||||
binary.Write(b.buf, binary.LittleEndian, v)
|
||||
}
|
||||
}
|
||||
|
||||
return b.buf.Bytes()
|
||||
}
|
||||
|
||||
// buildInvalidMagic creates a file with invalid magic number
|
||||
func (b *testGGUFBuilder) buildInvalidMagic() []byte {
|
||||
buf := new(bytes.Buffer)
|
||||
binary.Write(buf, binary.LittleEndian, uint32(0x12345678))
|
||||
return buf.Bytes()
|
||||
}
|
||||
@ -1403,6 +1403,22 @@ func Test_Cataloger_PositiveCases(t *testing.T) {
|
||||
Metadata: metadata("ffmpeg-library"),
|
||||
},
|
||||
},
|
||||
{
|
||||
logicalFixture: "elixir/1.19.1/linux-amd64",
|
||||
expected: pkg.Package{
|
||||
Name: "elixir",
|
||||
Version: "1.19.1",
|
||||
Type: "binary",
|
||||
PURL: "pkg:generic/elixir@1.19.1",
|
||||
Locations: locations("elixir", "lib/elixir/ebin/elixir.app"),
|
||||
Metadata: pkg.BinarySignature{
|
||||
Matches: []pkg.ClassifierMatch{
|
||||
match("elixir-binary", "elixir"),
|
||||
match("elixir-library", "lib/elixir/ebin/elixir.app"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
|
||||
@ -663,6 +663,26 @@ func DefaultClassifiers() []binutils.Classifier {
|
||||
PURL: mustPURL("pkg:generic/ffmpeg@version"),
|
||||
CPEs: singleCPE("cpe:2.3:a:ffmpeg:ffmpeg:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource),
|
||||
},
|
||||
{
|
||||
Class: "elixir-binary",
|
||||
FileGlob: "**/elixir",
|
||||
EvidenceMatcher: m.FileContentsVersionMatcher(
|
||||
`(?m)ELIXIR_VERSION=(?P<version>[0-9]+\.[0-9]+\.[0-9]+)`),
|
||||
Package: "elixir",
|
||||
PURL: mustPURL("pkg:generic/elixir@version"),
|
||||
CPEs: []cpe.CPE{
|
||||
cpe.Must("cpe:2.3:a:elixir-lang:elixir:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource),
|
||||
},
|
||||
},
|
||||
{
|
||||
Class: "elixir-library",
|
||||
FileGlob: "**/elixir/ebin/elixir.app",
|
||||
EvidenceMatcher: m.FileContentsVersionMatcher(
|
||||
`(?m)\{vsn,"(?P<version>[0-9]+\.[0-9]+\.[0-9]+(-[a-z0-9]+)?)"\}`),
|
||||
Package: "elixir",
|
||||
PURL: mustPURL("pkg:generic/elixir@version"),
|
||||
CPEs: singleCPE("cpe:2.3:a:elixir-lang:elixir:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource),
|
||||
},
|
||||
}
|
||||
|
||||
return append(classifiers, defaultJavaClassifiers()...)
|
||||
|
||||
@ -0,0 +1,20 @@
|
||||
#!/bin/sh
|
||||
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: 2021 The Elixir Team
|
||||
# SPDX-FileCopyrightText: 2012 Plataformatec
|
||||
|
||||
set -e
|
||||
|
||||
ELIXIR_VERSION=1.19.1
|
||||
|
||||
if [ $# -eq 0 ] || { [ $# -eq 1 ] && { [ "$1" = "--help" ] || [ "$1" = "-h" ]; }; }; then
|
||||
cat <<USAGE >&2
|
||||
Usage: $(basename "$0") [options] [.exs file] [data]
|
||||
|
||||
## General options
|
||||
|
||||
-e "COMMAND" Evaluates the given command (*)
|
||||
-h, --help Prints this message (standalone)
|
||||
-r "FILE" Requires the given files/patterns (*)
|
||||
-S SCRIPT Finds and executes the given script in \$PATH
|
||||
@ -0,0 +1,19 @@
|
||||
{application,elixir,
|
||||
[{description,"elixir"},
|
||||
{vsn,"1.19.1"},
|
||||
{modules,
|
||||
['Elixir.Access','Elixir.Agent.Server','Elixir.Agent',
|
||||
'Elixir.Application','Elixir.ArgumentError',
|
||||
elixir_overridable,elixir_parser,elixir_quote,elixir_rewrite,
|
||||
elixir_sup,elixir_tokenizer,elixir_utils,iex]},
|
||||
{registered,[elixir_sup,elixir_config,elixir_code_server]},
|
||||
{applications,[kernel,stdlib,compiler]},
|
||||
{mod,{elixir,[]}},
|
||||
{env,
|
||||
[{ansi_syntax_colors,
|
||||
[{atom,cyan},
|
||||
{binary,default_color},
|
||||
{operator,default_color}]},
|
||||
{check_endianness,true},
|
||||
{dbg_callback,{'Elixir.Macro',dbg,[]}},
|
||||
{time_zone_database,'Elixir.Calendar.UTCOnlyTimeZoneDatabase'}]}]}.
|
||||
58
syft/pkg/cataloger/common/cpe/target_software_to_pkg_type.go
Normal file
58
syft/pkg/cataloger/common/cpe/target_software_to_pkg_type.go
Normal file
@ -0,0 +1,58 @@
|
||||
package cpe
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
// TargetSoftwareToPackageType is derived from looking at target_software attributes in the NVD dataset
|
||||
// TODO: ideally this would be driven from the store, where we can resolve ecosystem aliases directly
|
||||
func TargetSoftwareToPackageType(tsw string) pkg.Type {
|
||||
tsw = strings.NewReplacer("-", "_", " ", "_").Replace(strings.ToLower(tsw))
|
||||
switch tsw {
|
||||
case "alpine", "apk":
|
||||
return pkg.ApkPkg
|
||||
case "debian", "dpkg":
|
||||
return pkg.DebPkg
|
||||
case "java", "maven", "ant", "gradle", "jenkins", "jenkins_ci", "kafka", "logstash", "mule", "nifi", "solr", "spark", "storm", "struts", "tomcat", "zookeeper", "log4j":
|
||||
return pkg.JavaPkg
|
||||
case "javascript", "node", "nodejs", "node.js", "npm", "yarn", "apache", "jquery", "next.js", "prismjs":
|
||||
return pkg.NpmPkg
|
||||
case "c", "c++", "c/c++", "conan", "gnu_c++", "qt":
|
||||
return pkg.ConanPkg
|
||||
case "dart":
|
||||
return pkg.DartPubPkg
|
||||
case "redhat", "rpm", "redhat_enterprise_linux", "rhel", "suse", "suse_linux", "opensuse", "opensuse_linux", "fedora", "centos", "oracle_linux", "ol":
|
||||
return pkg.RpmPkg
|
||||
case "elixir", "hex":
|
||||
return pkg.HexPkg
|
||||
case "erlang":
|
||||
return pkg.ErlangOTPPkg
|
||||
case ".net", ".net_framework", "asp", "asp.net", "dotnet", "dotnet_framework", "c#", "csharp", "nuget":
|
||||
return pkg.DotnetPkg
|
||||
case "ruby", "gem", "nokogiri", "ruby_on_rails":
|
||||
return pkg.GemPkg
|
||||
case "rust", "cargo", "crates":
|
||||
return pkg.RustPkg
|
||||
case "python", "pip", "pypi", "flask":
|
||||
return pkg.PythonPkg
|
||||
case "kb", "knowledgebase", "msrc", "mskb", "microsoft":
|
||||
return pkg.KbPkg
|
||||
case "portage", "gentoo":
|
||||
return pkg.PortagePkg
|
||||
case "go", "golang", "gomodule":
|
||||
return pkg.GoModulePkg
|
||||
case "linux_kernel", "linux", "z/linux":
|
||||
return pkg.LinuxKernelPkg
|
||||
case "php":
|
||||
return pkg.PhpComposerPkg
|
||||
case "swift":
|
||||
return pkg.SwiftPkg
|
||||
case "wordpress", "wordpress_plugin", "wordpress_":
|
||||
return pkg.WordpressPluginPkg
|
||||
case "lua", "luarocks":
|
||||
return pkg.LuaRocksPkg
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@ -1,9 +1,12 @@
|
||||
package dart
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/anchore/packageurl-go"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/internal/licenses"
|
||||
)
|
||||
|
||||
func newPubspecLockPackage(name string, raw pubspecLockPackage, locations ...file.Location) pkg.Package {
|
||||
@ -29,7 +32,7 @@ func newPubspecLockPackage(name string, raw pubspecLockPackage, locations ...fil
|
||||
return p
|
||||
}
|
||||
|
||||
func newPubspecPackage(raw pubspecPackage, locations ...file.Location) pkg.Package {
|
||||
func newPubspecPackage(ctx context.Context, resolver file.Resolver, raw pubspecPackage, locations ...file.Location) pkg.Package {
|
||||
var env *pkg.DartPubspecEnvironment
|
||||
if raw.Environment.SDK != "" || raw.Environment.Flutter != "" {
|
||||
// this is required only after pubspec v2, but might have been optional before this
|
||||
@ -58,6 +61,8 @@ func newPubspecPackage(raw pubspecPackage, locations ...file.Location) pkg.Packa
|
||||
|
||||
p.SetID()
|
||||
|
||||
p = licenses.RelativeToPackage(ctx, resolver, p)
|
||||
|
||||
return p
|
||||
}
|
||||
|
||||
|
||||
@ -29,7 +29,7 @@ type dartPubspecEnvironment struct {
|
||||
Flutter string `mapstructure:"flutter" yaml:"flutter"`
|
||||
}
|
||||
|
||||
func parsePubspec(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
func parsePubspec(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
var pkgs []pkg.Package
|
||||
|
||||
dec := yaml.NewDecoder(reader)
|
||||
@ -41,6 +41,8 @@ func parsePubspec(_ context.Context, _ file.Resolver, _ *generic.Environment, re
|
||||
|
||||
pkgs = append(pkgs,
|
||||
newPubspecPackage(
|
||||
ctx,
|
||||
resolver,
|
||||
p,
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
),
|
||||
|
||||
@ -12,6 +12,7 @@ import (
|
||||
// binary cataloger will search for .dll and .exe files and create packages based off of the version resources embedded
|
||||
// as a resource directory within the executable. If there is no evidence of a .NET runtime (a CLR header) then no
|
||||
// package will be created.
|
||||
//
|
||||
// Deprecated: use depsBinaryCataloger instead which combines the PE and deps.json data which yields more accurate results (will be removed in syft v2.0).
|
||||
type binaryCataloger struct {
|
||||
}
|
||||
|
||||
@ -13,12 +13,14 @@ func NewDotnetDepsBinaryCataloger(config CatalogerConfig) pkg.Cataloger {
|
||||
}
|
||||
|
||||
// NewDotnetDepsCataloger returns a cataloger based on deps.json file contents.
|
||||
//
|
||||
// Deprecated: use NewDotnetDepsBinaryCataloger instead which combines the PE and deps.json data which yields more accurate results (will be removed in syft v2.0).
|
||||
func NewDotnetDepsCataloger() pkg.Cataloger {
|
||||
return &depsCataloger{}
|
||||
}
|
||||
|
||||
// NewDotnetPortableExecutableCataloger returns a cataloger based on PE file contents.
|
||||
//
|
||||
// Deprecated: use NewDotnetDepsBinaryCataloger instead which combines the PE and deps.json data which yields more accurate results (will be removed in syft v2.0).
|
||||
func NewDotnetPortableExecutableCataloger() pkg.Cataloger {
|
||||
return &binaryCataloger{}
|
||||
|
||||
@ -9,6 +9,7 @@ import (
|
||||
)
|
||||
|
||||
// depsCataloger will search for deps.json file contents.
|
||||
//
|
||||
// Deprecated: use depsBinaryCataloger instead which combines the PE and deps.json data which yields more accurate results (will be removed in syft v2.0).
|
||||
type depsCataloger struct {
|
||||
}
|
||||
|
||||
@ -4,18 +4,20 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/internal/licenses"
|
||||
)
|
||||
|
||||
// resolveModuleLicenses finds and parses license files for Go modules
|
||||
func resolveModuleLicenses(ctx context.Context, pkgInfos []pkgInfo, fs afero.Fs) pkg.LicenseSet {
|
||||
licenses := pkg.NewLicenseSet()
|
||||
func resolveModuleLicenses(ctx context.Context, scanRoot string, pkgInfos []pkgInfo, fs afero.Fs) pkg.LicenseSet {
|
||||
out := pkg.NewLicenseSet()
|
||||
|
||||
for _, info := range pkgInfos {
|
||||
modDir, pkgDir, err := getAbsolutePkgPaths(info)
|
||||
@ -23,22 +25,32 @@ func resolveModuleLicenses(ctx context.Context, pkgInfos []pkgInfo, fs afero.Fs)
|
||||
continue
|
||||
}
|
||||
|
||||
licenseFiles, err := findAllLicenseCandidatesUpwards(pkgDir, licenseRegexp, modDir, fs)
|
||||
licenseFiles, err := findAllLicenseCandidatesUpwards(pkgDir, modDir, fs)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, f := range licenseFiles {
|
||||
contents, err := fs.Open(f)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
licenses.Add(pkg.NewLicensesFromReadCloserWithContext(ctx, file.NewLocationReadCloser(file.Location{}, contents))...)
|
||||
_ = contents.Close()
|
||||
out.Add(readLicenses(ctx, scanRoot, fs, f)...)
|
||||
}
|
||||
}
|
||||
|
||||
return licenses
|
||||
return out
|
||||
}
|
||||
|
||||
func readLicenses(ctx context.Context, scanRoot string, fs afero.Fs, f string) []pkg.License {
|
||||
contents, err := fs.Open(f)
|
||||
if err != nil {
|
||||
log.WithFields("file", f, "error", err).Debug("unable to read license file")
|
||||
return nil
|
||||
}
|
||||
defer internal.CloseAndLogError(contents, f)
|
||||
location := file.Location{}
|
||||
if scanRoot != "" && strings.HasPrefix(f, scanRoot) {
|
||||
// include location when licenses are found within the scan target
|
||||
location = file.NewLocation(strings.TrimPrefix(f, scanRoot))
|
||||
}
|
||||
return pkg.NewLicensesFromReadCloserWithContext(ctx, file.NewLocationReadCloser(location, contents))
|
||||
}
|
||||
|
||||
/*
|
||||
@ -60,7 +72,7 @@ When we should consider redesign tip to stem:
|
||||
- We need to consider the case here where nested modules are visited by accident and licenses
|
||||
are erroneously associated to a 'parent module'; bubble up currently prevents this
|
||||
*/
|
||||
func findAllLicenseCandidatesUpwards(dir string, r *regexp.Regexp, stopAt string, fs afero.Fs) ([]string, error) {
|
||||
func findAllLicenseCandidatesUpwards(dir string, stopAt string, fs afero.Fs) ([]string, error) {
|
||||
// Validate that both paths are absolute
|
||||
if !filepath.IsAbs(dir) {
|
||||
return nil, fmt.Errorf("dir must be an absolute path, got: %s", dir)
|
||||
@ -69,25 +81,16 @@ func findAllLicenseCandidatesUpwards(dir string, r *regexp.Regexp, stopAt string
|
||||
return nil, fmt.Errorf("stopAt must be an absolute path, got: %s", stopAt)
|
||||
}
|
||||
|
||||
licenses, err := findLicenseCandidates(dir, r, stopAt, fs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Ensure we return an empty slice rather than nil for consistency
|
||||
if licenses == nil {
|
||||
return []string{}, nil
|
||||
}
|
||||
return licenses, nil
|
||||
return findLicenseCandidates(dir, stopAt, fs)
|
||||
}
|
||||
|
||||
func findLicenseCandidates(dir string, r *regexp.Regexp, stopAt string, fs afero.Fs) ([]string, error) {
|
||||
func findLicenseCandidates(dir string, stopAt string, fs afero.Fs) ([]string, error) {
|
||||
// stop if we've gone outside the stopAt directory
|
||||
if !strings.HasPrefix(dir, stopAt) {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
licenses, err := findLicensesInDir(dir, r, fs)
|
||||
out, err := findLicensesInDir(dir, fs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -95,17 +98,17 @@ func findLicenseCandidates(dir string, r *regexp.Regexp, stopAt string, fs afero
|
||||
parent := filepath.Dir(dir)
|
||||
// can't go any higher up the directory tree: "/" case
|
||||
if parent == dir {
|
||||
return licenses, nil
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// search parent directory and combine results
|
||||
parentLicenses, err := findLicenseCandidates(parent, r, stopAt, fs)
|
||||
parentLicenses, err := findLicenseCandidates(parent, stopAt, fs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Combine current directory licenses with parent directory licenses
|
||||
return append(licenses, parentLicenses...), nil
|
||||
return append(out, parentLicenses...), nil
|
||||
}
|
||||
|
||||
func getAbsolutePkgPaths(info pkgInfo) (modDir string, pkgDir string, err error) {
|
||||
@ -126,8 +129,8 @@ func getAbsolutePkgPaths(info pkgInfo) (modDir string, pkgDir string, err error)
|
||||
return modDir, pkgDir, nil
|
||||
}
|
||||
|
||||
func findLicensesInDir(dir string, r *regexp.Regexp, fs afero.Fs) ([]string, error) {
|
||||
var licenses []string
|
||||
func findLicensesInDir(dir string, fs afero.Fs) ([]string, error) {
|
||||
var out []string
|
||||
|
||||
dirContents, err := afero.ReadDir(fs, dir)
|
||||
if err != nil {
|
||||
@ -139,11 +142,11 @@ func findLicensesInDir(dir string, r *regexp.Regexp, fs afero.Fs) ([]string, err
|
||||
continue
|
||||
}
|
||||
|
||||
if r.MatchString(f.Name()) {
|
||||
if licenses.IsLicenseFile(f.Name()) {
|
||||
path := filepath.Join(dir, f.Name())
|
||||
licenses = append(licenses, path)
|
||||
out = append(out, path)
|
||||
}
|
||||
}
|
||||
|
||||
return licenses, nil
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@ -70,8 +70,8 @@ func TestFindAllLicenseCandidatesUpwards(t *testing.T) {
|
||||
fs.MkdirAll("/empty/dir/tree", 0755)
|
||||
// No license files
|
||||
},
|
||||
expectedFiles: []string{},
|
||||
description: "Should return empty slice when no license files found",
|
||||
expectedFiles: nil,
|
||||
description: "Should return nil when no license files found",
|
||||
},
|
||||
{
|
||||
name: "handles directory at filesystem root",
|
||||
@ -205,7 +205,7 @@ func TestFindAllLicenseCandidatesUpwards(t *testing.T) {
|
||||
tt.setupFS(fs)
|
||||
|
||||
// Run the function
|
||||
result, err := findAllLicenseCandidatesUpwards(tt.startDir, licenseRegexp, tt.stopAt, fs)
|
||||
result, err := findAllLicenseCandidatesUpwards(tt.startDir, tt.stopAt, fs)
|
||||
|
||||
// Check error expectation
|
||||
if tt.expectedError {
|
||||
|
||||
@ -19,23 +19,21 @@ import (
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/storage/memory"
|
||||
"github.com/scylladb/go-set/strset"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/cache"
|
||||
"github.com/anchore/syft/internal/licenses"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/internal/licenses"
|
||||
)
|
||||
|
||||
type goLicenseResolver struct {
|
||||
catalogerName string
|
||||
opts CatalogerConfig
|
||||
localModCacheDir fs.FS
|
||||
localVendorDir fs.FS
|
||||
licenseCache cache.Resolver[[]pkg.License]
|
||||
lowerLicenseFileNames *strset.Set
|
||||
catalogerName string
|
||||
opts CatalogerConfig
|
||||
localModCacheDir fs.FS
|
||||
localVendorDir fs.FS
|
||||
licenseCache cache.Resolver[[]pkg.License]
|
||||
}
|
||||
|
||||
func newGoLicenseResolver(catalogerName string, opts CatalogerConfig) goLicenseResolver {
|
||||
@ -59,23 +57,14 @@ func newGoLicenseResolver(catalogerName string, opts CatalogerConfig) goLicenseR
|
||||
}
|
||||
|
||||
return goLicenseResolver{
|
||||
catalogerName: catalogerName,
|
||||
opts: opts,
|
||||
localModCacheDir: localModCacheDir,
|
||||
localVendorDir: localVendorDir,
|
||||
licenseCache: cache.GetResolverCachingErrors[[]pkg.License]("golang", "v2"),
|
||||
lowerLicenseFileNames: strset.New(lowercaseLicenseFiles()...),
|
||||
catalogerName: catalogerName,
|
||||
opts: opts,
|
||||
localModCacheDir: localModCacheDir,
|
||||
localVendorDir: localVendorDir,
|
||||
licenseCache: cache.GetResolverCachingErrors[[]pkg.License]("golang", "v2"),
|
||||
}
|
||||
}
|
||||
|
||||
func lowercaseLicenseFiles() []string {
|
||||
fileNames := licenses.FileNames()
|
||||
for i := range fileNames {
|
||||
fileNames[i] = strings.ToLower(fileNames[i])
|
||||
}
|
||||
return fileNames
|
||||
}
|
||||
|
||||
func remotesForModule(proxies []string, noProxy []string, module string) []string {
|
||||
for _, pattern := range noProxy {
|
||||
if matched, err := path.Match(pattern, module); err == nil && matched {
|
||||
@ -194,7 +183,7 @@ func (c *goLicenseResolver) findLicensesInFS(ctx context.Context, urlPrefix stri
|
||||
log.Debugf("nil entry for %s#%s", urlPrefix, filePath)
|
||||
return nil
|
||||
}
|
||||
if !c.lowerLicenseFileNames.Has(strings.ToLower(d.Name())) {
|
||||
if !licenses.IsLicenseFile(d.Name()) {
|
||||
return nil
|
||||
}
|
||||
rdr, err := fsys.Open(filePath)
|
||||
@ -203,11 +192,11 @@ func (c *goLicenseResolver) findLicensesInFS(ctx context.Context, urlPrefix stri
|
||||
return nil
|
||||
}
|
||||
defer internal.CloseAndLogError(rdr, filePath)
|
||||
licenses := pkg.NewLicensesFromReadCloserWithContext(ctx, file.NewLocationReadCloser(file.NewLocation(filePath), rdr))
|
||||
foundLicenses := pkg.NewLicensesFromReadCloserWithContext(ctx, file.NewLocationReadCloser(file.NewLocation(filePath), rdr))
|
||||
// since these licenses are found in an external fs.FS, not in the scanned source,
|
||||
// get rid of the locations but keep information about the where the license was found
|
||||
// by prepending the urlPrefix to the internal path for an accurate representation
|
||||
for _, l := range licenses {
|
||||
for _, l := range foundLicenses {
|
||||
l.URLs = []string{urlPrefix + filePath}
|
||||
l.Locations = file.NewLocationSet()
|
||||
out = append(out, l)
|
||||
@ -246,7 +235,7 @@ func (c *goLicenseResolver) findLicensesInSource(ctx context.Context, resolver f
|
||||
func (c *goLicenseResolver) parseLicenseFromLocation(ctx context.Context, l file.Location, resolver file.Resolver) ([]pkg.License, error) {
|
||||
var out []pkg.License
|
||||
fileName := path.Base(l.RealPath)
|
||||
if c.lowerLicenseFileNames.Has(strings.ToLower(fileName)) {
|
||||
if licenses.IsLicenseFile(fileName) {
|
||||
contents, err := resolver.FileContentsByLocation(l)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@ -7,7 +7,7 @@ import (
|
||||
"go/build"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@ -20,14 +20,11 @@ import (
|
||||
"github.com/anchore/syft/internal/unknown"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/internal/fileresolver"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/generic"
|
||||
)
|
||||
|
||||
var (
|
||||
licenseRegexp = regexp.MustCompile(`^(?i)((UN)?LICEN(S|C)E|COPYING|NOTICE).*$`)
|
||||
)
|
||||
|
||||
type goModCataloger struct {
|
||||
licenseResolver goLicenseResolver
|
||||
}
|
||||
@ -46,9 +43,14 @@ func (c *goModCataloger) parseGoModFile(ctx context.Context, resolver file.Resol
|
||||
log.Debugf("unable to get go.sum: %v", err)
|
||||
}
|
||||
|
||||
scanRoot := ""
|
||||
if dir, ok := resolver.(*fileresolver.Directory); ok && dir != nil {
|
||||
scanRoot = dir.Chroot.Base()
|
||||
}
|
||||
|
||||
// source analysis using go toolchain if available
|
||||
syftSourcePackages, sourceModules, sourceDependencies, unknownErr := c.loadPackages(modDir, reader.Location)
|
||||
catalogedModules, sourceModuleToPkg := c.catalogModules(ctx, syftSourcePackages, sourceModules, reader, digests)
|
||||
catalogedModules, sourceModuleToPkg := c.catalogModules(ctx, scanRoot, syftSourcePackages, sourceModules, reader, digests)
|
||||
relationships := buildModuleRelationships(catalogedModules, sourceDependencies, sourceModuleToPkg)
|
||||
|
||||
// base case go.mod file parsing
|
||||
@ -208,12 +210,16 @@ func (c *goModCataloger) visitPackages(
|
||||
}
|
||||
}
|
||||
}
|
||||
pkgs[module.Path] = append(pkgs[module.Path], pkgInfo{
|
||||
|
||||
info := pkgInfo{
|
||||
pkgPath: p.PkgPath,
|
||||
modulePath: module.Path,
|
||||
pkgDir: pkgDir,
|
||||
moduleDir: module.Dir,
|
||||
})
|
||||
}
|
||||
if !slices.Contains(pkgs[module.Path], info) { // avoid duplicates
|
||||
pkgs[module.Path] = append(pkgs[module.Path], info)
|
||||
}
|
||||
modules[p.Module.Path] = module
|
||||
|
||||
return true
|
||||
@ -224,6 +230,7 @@ func (c *goModCataloger) visitPackages(
|
||||
// create syft packages from Go modules found by the go toolchain
|
||||
func (c *goModCataloger) catalogModules(
|
||||
ctx context.Context,
|
||||
scanRoot string,
|
||||
pkgs map[string][]pkgInfo,
|
||||
modules map[string]*packages.Module,
|
||||
reader file.LocationReadCloser,
|
||||
@ -243,7 +250,7 @@ func (c *goModCataloger) catalogModules(
|
||||
}
|
||||
|
||||
pkgInfos := pkgs[m.Path]
|
||||
moduleLicenses := resolveModuleLicenses(ctx, pkgInfos, afero.NewOsFs())
|
||||
moduleLicenses := resolveModuleLicenses(ctx, scanRoot, pkgInfos, afero.NewOsFs())
|
||||
// we do out of source lookups for module parsing
|
||||
// locations are NOT included in the SBOM because of this
|
||||
goModulePkg := pkg.Package{
|
||||
|
||||
@ -1,15 +1,22 @@
|
||||
package homebrew
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
|
||||
"github.com/anchore/packageurl-go"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/internal/licenses"
|
||||
)
|
||||
|
||||
func newHomebrewPackage(pd parsedHomebrewData, formulaLocation file.Location) pkg.Package {
|
||||
var licenses []string
|
||||
func newHomebrewPackage(ctx context.Context, resolver file.Resolver, pd parsedHomebrewData, formulaLocation file.Location) pkg.Package {
|
||||
var lics []pkg.License
|
||||
if pd.License != "" {
|
||||
licenses = append(licenses, pd.License)
|
||||
lics = append(lics, pkg.NewLicensesFromValues(pd.License)...)
|
||||
} else {
|
||||
// sometimes licenses are included in the parent directory
|
||||
lics = licenses.FindInDirs(ctx, resolver, path.Dir(formulaLocation.Path()))
|
||||
}
|
||||
|
||||
p := pkg.Package{
|
||||
@ -17,7 +24,7 @@ func newHomebrewPackage(pd parsedHomebrewData, formulaLocation file.Location) pk
|
||||
Version: pd.Version,
|
||||
Type: pkg.HomebrewPkg,
|
||||
Locations: file.NewLocationSet(formulaLocation.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
|
||||
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValues(licenses...)...),
|
||||
Licenses: pkg.NewLicenseSet(lics...),
|
||||
FoundBy: "homebrew-cataloger",
|
||||
PURL: packageURL(pd.Name, pd.Version),
|
||||
Metadata: pkg.HomebrewFormula{
|
||||
|
||||
@ -22,7 +22,7 @@ type parsedHomebrewData struct {
|
||||
License string
|
||||
}
|
||||
|
||||
func parseHomebrewFormula(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
func parseHomebrewFormula(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
pd, err := parseFormulaFile(reader)
|
||||
if err != nil {
|
||||
log.WithFields("path", reader.RealPath).Trace("failed to parse formula")
|
||||
@ -35,6 +35,8 @@ func parseHomebrewFormula(_ context.Context, _ file.Resolver, _ *generic.Environ
|
||||
|
||||
return []pkg.Package{
|
||||
newHomebrewPackage(
|
||||
ctx,
|
||||
resolver,
|
||||
*pd,
|
||||
reader.Location,
|
||||
),
|
||||
|
||||
167
syft/pkg/cataloger/internal/cpegenerate/README.md
Normal file
167
syft/pkg/cataloger/internal/cpegenerate/README.md
Normal file
@ -0,0 +1,167 @@
|
||||
# CPE Generation
|
||||
|
||||
This package generates Common Platform Enumeration (CPE) identifiers for software packages discovered by Syft.
|
||||
CPEs are standardized identifiers that enable vulnerability matching by linking packages to known vulnerabilities in databases like the National Vulnerability Database (NVD).
|
||||
|
||||
## Overview
|
||||
|
||||
CPE generation in Syft uses a **two-tier approach** to balance accuracy and coverage:
|
||||
|
||||
1. **Dictionary Lookups** (Authoritative): Pre-validated CPEs from the official NIST CPE dictionary
|
||||
2. **Heuristic Generation** (Fallback): Intelligent generation based on package metadata and ecosystem-specific patterns
|
||||
|
||||
This dual approach ensures:
|
||||
- **High accuracy** for packages in the NIST dictionary (no false positives)
|
||||
- **Broad coverage** for packages not yet in the dictionary (maximizes vulnerability detection)
|
||||
- **Fast performance** with an embedded, indexed CPE dictionary (~814KB)
|
||||
|
||||
## Why It Matters
|
||||
|
||||
CPEs link discovered packages to security vulnerabilities (CVEs) in tools like Grype. Without accurate CPE generation, vulnerability scanning misses security issues.
|
||||
|
||||
## How It Works
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ Syft Package Discovery │
|
||||
└──────────────────┬──────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────┐
|
||||
│ CPE Generation │
|
||||
│ (this package) │
|
||||
└──────────┬──────────┘
|
||||
│
|
||||
┌───────────┴────────────┐
|
||||
│ │
|
||||
▼ ▼
|
||||
┌──────────────────┐ ┌─────────────────────┐
|
||||
│ Dictionary │ │ Heuristic │
|
||||
│ Lookup │ │ Generation │
|
||||
│ │ │ │
|
||||
│ • Embedded index │ │ • Ecosystem rules │
|
||||
│ • ~22K entries │ │ • Vendor/product │
|
||||
│ • 11 ecosystems │ │ candidates │
|
||||
└──────────────────┘ │ • Curated mappings │
|
||||
│ • Smart filters │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
### Dictionary Generation Process
|
||||
|
||||
The dictionary is generated offline and embedded into the Syft binary for fast, offline lookups.
|
||||
|
||||
**Location**: `dictionary/index-generator/`
|
||||
|
||||
**Process**:
|
||||
1. **Fetch**: Retrieves CPE data from NVD Products API using incremental updates
|
||||
2. **Cache**: Stores raw API responses in ORAS registry for reuse (`.cpe-cache/`)
|
||||
3. **Filter**:
|
||||
- Removes CPEs without reference URLs
|
||||
- Excludes hardware (`h`) and OS (`o`) CPEs (keeps only applications `a`)
|
||||
4. **Index by Ecosystem**:
|
||||
- Extracts package names from reference URLs (npm, pypi, rubygems, etc.)
|
||||
- Creates index: `ecosystem → package_name → [CPE strings]`
|
||||
5. **Embed**: Generates `data/cpe-index.json` embedded via `go:embed` directive
|
||||
|
||||
### Runtime CPE Lookup/Generation
|
||||
|
||||
**Entry Point**: `generate.go`
|
||||
|
||||
When Syft discovers a package:
|
||||
|
||||
1. **Check for Declared CPEs**: If package metadata already contains CPEs (from SBOM imports), skip generation
|
||||
2. **Try Dictionary Lookup** (`FromDictionaryFind`):
|
||||
- Loads embedded CPE index (singleton, loaded once)
|
||||
- Looks up by ecosystem + package name
|
||||
- Returns pre-validated CPEs if found
|
||||
- Marks source as `NVDDictionaryLookupSource`
|
||||
3. **Fallback to Heuristic Generation** (`FromPackageAttributes`):
|
||||
- Generates vendor/product/targetSW candidates using ecosystem-specific logic
|
||||
- Creates CPE permutations from candidates
|
||||
- Applies filters to remove known false positives
|
||||
- Marks source as `GeneratedSource`
|
||||
|
||||
### Supported Ecosystems
|
||||
|
||||
**Dictionary Lookups** (11 ecosystems):
|
||||
npm, RubyGems, PyPI, Jenkins Plugins, crates.io, PHP, Go Modules, WordPress Plugins/Themes
|
||||
|
||||
**Heuristic Generation** (all package types):
|
||||
All dictionary ecosystems plus Java, .NET/NuGet, Alpine APK, Debian/RPM, and any other package type Syft discovers
|
||||
|
||||
### Ecosystem-Specific Intelligence
|
||||
|
||||
The heuristic generator uses per-ecosystem strategies:
|
||||
|
||||
- **Java**: Extracts vendor from groupId, product from artifactId
|
||||
- **Python**: Parses author fields, adds `_project` suffix variants
|
||||
- **Go**: Extracts org/repo from module paths (`github.com/org/repo`)
|
||||
- **JavaScript**: Handles npm scope patterns (`@scope/package`)
|
||||
|
||||
### Curated Mappings & Filters
|
||||
|
||||
- **500+ curated mappings**: `curl` → `haxx`, `spring-boot` → `pivotal`, etc.
|
||||
- **Filters**: Prevent false positives (Jenkins plugins vs. core, Jira client vs. server)
|
||||
- **Validation**: Ensures CPE syntax correctness before returning
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### Embedded Index Format
|
||||
|
||||
```json
|
||||
{
|
||||
"ecosystems": {
|
||||
"npm": {
|
||||
"lodash": ["cpe:2.3:a:lodash:lodash:*:*:*:*:*:node.js:*:*"]
|
||||
},
|
||||
"pypi": {
|
||||
"Django": ["cpe:2.3:a:djangoproject:django:*:*:*:*:*:python:*:*"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The dictionary generator maps packages to ecosystems using reference URL patterns (npmjs.com, pypi.org, rubygems.org, etc.).
|
||||
|
||||
## Maintenance
|
||||
|
||||
### Updating the CPE Dictionary
|
||||
|
||||
The CPE dictionary should be updated periodically to include new packages:
|
||||
|
||||
```bash
|
||||
# Full workflow: pull cache → update from NVD → build index
|
||||
make generate:cpe-index
|
||||
|
||||
# Or run individual steps:
|
||||
make generate:cpe-index:cache:pull # Pull cached CPE data from ORAS
|
||||
make generate:cpe-index:cache:update # Fetch updates from NVD Products API
|
||||
make generate:cpe-index:build # Generate cpe-index.json from cache
|
||||
```
|
||||
|
||||
**Optional**: Set `NVD_API_KEY` for faster updates (50 req/30s vs 5 req/30s)
|
||||
|
||||
This workflow:
|
||||
1. Pulls existing cache from ORAS registry (avoids re-fetching all ~1.5M CPEs)
|
||||
2. Fetches only products modified since last update from NVD Products API
|
||||
3. Builds indexed dictionary (~814KB, ~22K entries)
|
||||
4. Pushes updated cache for team reuse
|
||||
|
||||
### Extending CPE Generation
|
||||
|
||||
**Add dictionary support for a new ecosystem:**
|
||||
1. Add URL pattern in `index-generator/generate.go`
|
||||
2. Regenerate index with `make generate:cpe-index`
|
||||
|
||||
**Improve heuristic generation:**
|
||||
1. Modify ecosystem-specific file (e.g., `java.go`, `python.go`)
|
||||
2. Add curated mappings to `candidate_by_package_type.go`
|
||||
|
||||
**Key files:**
|
||||
- `generate.go` - Main generation logic
|
||||
- `dictionary/` - Dictionary generator and embedded index
|
||||
- `candidate_by_package_type.go` - Ecosystem-specific candidates
|
||||
- `filter.go` - Filtering rules
|
||||
@ -653,6 +653,9 @@
|
||||
"dbCharts": [
|
||||
"cpe:2.3:a:jenkins:dbcharts:*:*:*:*:*:jenkins:*:*"
|
||||
],
|
||||
"deadmanssnitch": [
|
||||
"cpe:2.3:a:jenkins:dead_man\\'s_snitch:*:*:*:*:*:jenkins:*:*"
|
||||
],
|
||||
"debian-package-builder": [
|
||||
"cpe:2.3:a:jenkins:debian_package_builder:*:*:*:*:*:jenkins:*:*"
|
||||
],
|
||||
@ -1360,6 +1363,9 @@
|
||||
"oic-auth": [
|
||||
"cpe:2.3:a:jenkins:openid_connect_authentication:*:*:*:*:*:jenkins:*:*"
|
||||
],
|
||||
"oidc-provider": [
|
||||
"cpe:2.3:a:jenkins:openid_connect_provider:*:*:*:*:*:jenkins:*:*"
|
||||
],
|
||||
"ontrack": [
|
||||
"cpe:2.3:a:jenkins:ontrack:*:*:*:*:*:jenkins:*:*"
|
||||
],
|
||||
@ -1531,6 +1537,9 @@
|
||||
"qualys-pc": [
|
||||
"cpe:2.3:a:qualys:policy_compliance:*:*:*:*:*:jenkins:*:*"
|
||||
],
|
||||
"qualys-was": [
|
||||
"cpe:2.3:a:qualys:web_application_screening:*:*:*:*:*:jenkins:*:*"
|
||||
],
|
||||
"quayio-trigger": [
|
||||
"cpe:2.3:a:jenkins:quay.io_trigger:*:*:*:*:*:jenkins:*:*"
|
||||
],
|
||||
@ -2164,6 +2173,9 @@
|
||||
"@azure/ms-rest-nodeauth": [
|
||||
"cpe:2.3:a:microsoft:ms-rest-nodeauth:*:*:*:*:*:node.js:*:*"
|
||||
],
|
||||
"@backstage/backend-common": [
|
||||
"cpe:2.3:a:linuxfoundation:backstage_backend-common:*:*:*:*:*:node.js:*:*"
|
||||
],
|
||||
"@backstage/plugin-auth-backend": [
|
||||
"cpe:2.3:a:linuxfoundation:auth_backend:*:*:*:*:*:node.js:*:*"
|
||||
],
|
||||
@ -3035,6 +3047,9 @@
|
||||
"electron-packager": [
|
||||
"cpe:2.3:a:electron-packager_project:electron-packager:*:*:*:*:*:node.js:*:*"
|
||||
],
|
||||
"electron-pdf": [
|
||||
"cpe:2.3:a:fraserxu:electron-pdf:*:*:*:*:*:node.js:*:*"
|
||||
],
|
||||
"elliptic": [
|
||||
"cpe:2.3:a:indutny:elliptic:*:*:*:*:*:node.js:*:*"
|
||||
],
|
||||
@ -5284,6 +5299,9 @@
|
||||
"ts-process-promises": [
|
||||
"cpe:2.3:a:ts-process-promises_project:ts-process-promises:*:*:*:*:*:node.js:*:*"
|
||||
],
|
||||
"tsup": [
|
||||
"cpe:2.3:a:egoist:tsup:*:*:*:*:*:node.js:*:*"
|
||||
],
|
||||
"ua-parser": [
|
||||
"cpe:2.3:a:ua-parser_project:ua-parser:*:*:*:*:*:node.js:*:*"
|
||||
],
|
||||
@ -5552,6 +5570,9 @@
|
||||
"alfnru/password_recovery": [
|
||||
"cpe:2.3:a:password_recovery_project:password_recovery:*:*:*:*:*:roundcube:*:*"
|
||||
],
|
||||
"couleurcitron/tarteaucitron-wp": [
|
||||
"cpe:2.3:a:couleurcitron:tarteaucitron-wp:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"dev-lancer/minecraft-motd-parser": [
|
||||
"cpe:2.3:a:jgniecki:minecraft_motd_parser:*:*:*:*:*:*:*:*"
|
||||
],
|
||||
@ -7259,6 +7280,9 @@
|
||||
"ab-press-optimizer-lite": [
|
||||
"cpe:2.3:a:abpressoptimizer:ab_press_optimizer:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"abitgone-commentsafe": [
|
||||
"cpe:2.3:a:abitgone:abitgone_commentsafe:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"about-me": [
|
||||
"cpe:2.3:a:about-me_project:about-me:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -7605,6 +7629,9 @@
|
||||
"advanced-backgrounds": [
|
||||
"cpe:2.3:a:wpbackgrounds:advanced_wordpress_backgrounds:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"advanced-blocks-pro": [
|
||||
"cpe:2.3:a:essamamdani:advanced_blocks_pro:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"advanced-booking-calendar": [
|
||||
"cpe:2.3:a:elbtide:advanced_booking_calendar:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -7702,6 +7729,9 @@
|
||||
"affiliatebooster-blocks": [
|
||||
"cpe:2.3:a:affiliatebooster:affiliate_booster:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"affiliateimportereb": [
|
||||
"cpe:2.3:a:cr1000:affiliateimportereb:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"affiliates-manager": [
|
||||
"cpe:2.3:a:wpaffiliatemanager:affiliates_manager:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -8408,6 +8438,9 @@
|
||||
"cpe:2.3:a:dotstore:woocommerce_category_banner_management:*:*:*:*:*:wordpress:*:*",
|
||||
"cpe:2.3:a:multidots:banner_management_for_woocommerce:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"bannerlid": [
|
||||
"cpe:2.3:a:web_lid:bannerlid:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"barcode-scanner-lite-pos-to-manage-products-inventory-and-orders": [
|
||||
"cpe:2.3:a:ukrsolution:barcode_scanner_and_inventory_manager:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -8516,6 +8549,9 @@
|
||||
"better-elementor-addons": [
|
||||
"cpe:2.3:a:kitforest:better_elementor_addons:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"better-follow-button-for-jetpack": [
|
||||
"cpe:2.3:a:antonpug:better_flow_button_for_jetpack:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"better-font-awesome": [
|
||||
"cpe:2.3:a:better_font_awesome_project:better_font_awesome:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -8770,6 +8806,9 @@
|
||||
"bp-cover": [
|
||||
"cpe:2.3:a:buddypress_cover_project:buddypress_cover:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"bp-email-assign-templates": [
|
||||
"cpe:2.3:a:shanebp:bp_email_assign_templates:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"bp-profile-search": [
|
||||
"cpe:2.3:a:dontdream:bp_profile_search:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -9240,6 +9279,9 @@
|
||||
"chained-quiz": [
|
||||
"cpe:2.3:a:kibokolabs:chained_quiz:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"chalet-montagne-com-tools": [
|
||||
"cpe:2.3:a:alpium:chalet-montagne.com_tools:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"chamber-dashboard-business-directory": [
|
||||
"cpe:2.3:a:chamber_dashboard_business_directory_project:chamber_dashboard_business_directory:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -9252,6 +9294,9 @@
|
||||
"change-memory-limit": [
|
||||
"cpe:2.3:a:simon99:change_memory_limit:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"change-table-prefix": [
|
||||
"cpe:2.3:a:youngtechleads:change_table_prefix:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"change-uploaded-file-permissions": [
|
||||
"cpe:2.3:a:change_uploaded_file_permissions_project:change_uploaded_file_permissions:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -9550,6 +9595,9 @@
|
||||
"commenttweets": [
|
||||
"cpe:2.3:a:theresehansen:commenttweets:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"common-tools-for-site": [
|
||||
"cpe:2.3:a:chetanvaghela:common_tools_for_site:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"commonsbooking": [
|
||||
"cpe:2.3:a:wielebenwir:commonsbooking:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -10041,6 +10089,9 @@
|
||||
"csv-importer": [
|
||||
"cpe:2.3:a:deniskobozev:csv_importer:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"csv-mass-importer": [
|
||||
"cpe:2.3:a:aleapp:csv_mass_importer:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"ct-commerce": [
|
||||
"cpe:2.3:a:ujwolbastakoti:ct_commerce:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -10798,6 +10849,9 @@
|
||||
"easy-svg": [
|
||||
"cpe:2.3:a:benjaminzekavica:easy_svg_support:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"easy-svg-upload": [
|
||||
"cpe:2.3:a:delowerhossain:easy_svg_upload:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"easy-table": [
|
||||
"cpe:2.3:a:easy_table_project:easy_table:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -11286,6 +11340,9 @@
|
||||
"exit-intent-popups-by-optimonk": [
|
||||
"cpe:2.3:a:optimonk:optimonk\\:popups\\,_personalization_\\\u0026_a\\/b_testing:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"exit-notifier": [
|
||||
"cpe:2.3:a:cvstech:exit_notifier:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"exmage-wp-image-links": [
|
||||
"cpe:2.3:a:villatheme:exmage:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -11325,6 +11382,9 @@
|
||||
"exquisite-paypal-donation": [
|
||||
"cpe:2.3:a:exquisite_paypal_donation_project:exquisite_paypal_donation:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"extended-search-plugin": [
|
||||
"cpe:2.3:a:jakesnyder:enhanced_search_box:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"extensions-for-cf7": [
|
||||
"cpe:2.3:a:hasthemes:extensions_for_cf7:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -11571,6 +11631,7 @@
|
||||
"cpe:2.3:a:five_minute_webshop_project:five_minute_webshop:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"fl3r-feelbox": [
|
||||
"cpe:2.3:a:armandofiore:fl3r_feelbox:*:*:*:*:*:wordpress:*:*",
|
||||
"cpe:2.3:a:fl3r-feelbox_project:fl3r-feelbox:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"flash-album-gallery": [
|
||||
@ -12235,6 +12296,9 @@
|
||||
"google-sitemap-plugin": [
|
||||
"cpe:2.3:a:bestwebsoft:google_sitemap:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"google-website-translator": [
|
||||
"cpe:2.3:a:prisna:google_website_translator:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"googleanalytics": [
|
||||
"cpe:2.3:a:sharethis:dashboard_for_google_analytics:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -12634,6 +12698,9 @@
|
||||
"hunk-companion": [
|
||||
"cpe:2.3:a:themehunk:hunk_companion:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"hurrytimer": [
|
||||
"cpe:2.3:a:nabillemsieh:hurrytimer:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"hyphenator": [
|
||||
"cpe:2.3:a:benedictb\\/maciejgryniuk:hyphenator:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -12907,6 +12974,9 @@
|
||||
"cpe:2.3:a:cm-wp:woody_code_snippets:*:*:*:*:*:wordpress:*:*",
|
||||
"cpe:2.3:a:webcraftic:woody_ad_snippets:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"insert-php-code-snippet": [
|
||||
"cpe:2.3:a:f1logic:insert_php_code_snippet:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"insight-core": [
|
||||
"cpe:2.3:a:thememove:insight_core:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -13011,6 +13081,9 @@
|
||||
"ip-blacklist-cloud": [
|
||||
"cpe:2.3:a:ip_blacklist_cloud_project:ip_blacklist_cloud:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"ip-vault-wp-firewall": [
|
||||
"cpe:2.3:a:youtag:two-factor_authentication:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"ip2location-country-blocker": [
|
||||
"cpe:2.3:a:ip2location:country_blocker:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -13557,6 +13630,9 @@
|
||||
"list-category-posts": [
|
||||
"cpe:2.3:a:fernandobriano:list_category_posts:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"list-children": [
|
||||
"cpe:2.3:a:sizeable:list_children:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"list-last-changes": [
|
||||
"cpe:2.3:a:rolandbaer:list_last_changes:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -13854,6 +13930,9 @@
|
||||
"manual-image-crop": [
|
||||
"cpe:2.3:a:manual_image_crop_project:manual_image_crop:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"mapfig-studio": [
|
||||
"cpe:2.3:a:acugis:mapfig_studio:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"mapping-multiple-urls-redirect-same-page": [
|
||||
"cpe:2.3:a:mapping_multiple_urls_redirect_same_page_project:mapping_multiple_urls_redirect_same_page:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -14237,6 +14316,9 @@
|
||||
"monetize": [
|
||||
"cpe:2.3:a:monetize_project:monetize:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"monitor-chat": [
|
||||
"cpe:2.3:a:edwardstoever:monitor.chat:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"month-name-translation-benaceur": [
|
||||
"cpe:2.3:a:benaceur-php:month_name_translation_benaceur:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -14306,6 +14388,9 @@
|
||||
"mq-woocommerce-products-price-bulk-edit": [
|
||||
"cpe:2.3:a:mq-woocommerce-products-price-bulk-edit_project:mq-woocommerce-products-price-bulk-edit:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"ms-registration": [
|
||||
"cpe:2.3:a:alphaefficiencyteam:custom_login_and_registration:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"ms-reviews": [
|
||||
"cpe:2.3:a:ms-reviews_project:ms-reviews:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -14438,7 +14523,7 @@
|
||||
"cpe:2.3:a:stormhillmedia:mybook_table_bookstore:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"mycred": [
|
||||
"cpe:2.3:a:mycred:mycred:*:*:*:*:*:wordpress:*:*"
|
||||
"cpe:2.3:a:wpexperts:mycred:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"mycryptocheckout": [
|
||||
"cpe:2.3:a:plainviewplugins:mycryptocheckout:*:*:*:*:*:wordpress:*:*"
|
||||
@ -14625,12 +14710,18 @@
|
||||
"ninjafirewall": [
|
||||
"cpe:2.3:a:nintechnet:ninjafirewall:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"ninjateam-telegram": [
|
||||
"cpe:2.3:a:ninjateam:chat_for_telegram:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"nirweb-support": [
|
||||
"cpe:2.3:a:nirweb:nirweb_support:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"nitropack": [
|
||||
"cpe:2.3:a:nitropack:nitropack:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"nix-anti-spam-light": [
|
||||
"cpe:2.3:a:nixsolutions:nix_anti-spam_light:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"nktagcloud": [
|
||||
"cpe:2.3:a:better_tag_cloud_project:better_tag_cloud:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -15186,6 +15277,9 @@
|
||||
"cpe:2.3:a:greentreelabs:gallery_photoblocks:*:*:*:*:*:wordpress:*:*",
|
||||
"cpe:2.3:a:wpchill:gallery_photoblocks:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"photokit": [
|
||||
"cpe:2.3:a:jackzhu:photokit:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"photoshow": [
|
||||
"cpe:2.3:a:codepeople:smart_image_gallery:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -15511,6 +15605,9 @@
|
||||
"postman-smtp": [
|
||||
"cpe:2.3:a:postman-smtp_project:postman-smtp:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"postmash": [
|
||||
"cpe:2.3:a:jmash:postmash:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"postmatic": [
|
||||
"cpe:2.3:a:gopostmatic:replyable:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -15761,6 +15858,9 @@
|
||||
"pure-chat": [
|
||||
"cpe:2.3:a:purechat:pure_chat:*:*:*:*:*:*:*:*"
|
||||
],
|
||||
"pure-css-circle-progress-bar": [
|
||||
"cpe:2.3:a:shafayat:pure_css_circle_progress_bar:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"purple-xmls-google-product-feed-for-woocommerce": [
|
||||
"cpe:2.3:a:dpl:product_feed_on_woocommerce_for_google\\,_awin\\,_shareasale\\,_bing\\,_and_more:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -15964,6 +16064,9 @@
|
||||
"react-webcam": [
|
||||
"cpe:2.3:a:react_webcam_project:react_webcam:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"reaction-buttons": [
|
||||
"cpe:2.3:a:jakob42:reaction_buttons:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"read-and-understood": [
|
||||
"cpe:2.3:a:read_and_understood_project:read_and_understood:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -16124,6 +16227,9 @@
|
||||
"reservation-studio-widget": [
|
||||
"cpe:2.3:a:pvmg:reservation.studio:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"reset": [
|
||||
"cpe:2.3:a:smartzminds:reset:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"resize-at-upload-plus": [
|
||||
"cpe:2.3:a:resize_at_upload_plus_project:resize_at_upload_plus:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -16527,6 +16633,9 @@
|
||||
"sellkit": [
|
||||
"cpe:2.3:a:artbees:sellkit:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"send-email-only-on-reply-to-my-comment": [
|
||||
"cpe:2.3:a:yasirwazir:send_email_only_on_reply_to_my_comment:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"send-emails-with-mandrill": [
|
||||
"cpe:2.3:a:millermedia:mandrill:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -17092,6 +17201,9 @@
|
||||
"site-editor": [
|
||||
"cpe:2.3:a:siteeditor:site_editor:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"site-mailer": [
|
||||
"cpe:2.3:a:elementor:site_mailer:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"site-offline": [
|
||||
"cpe:2.3:a:freehtmldesigns:site_offline:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -17780,6 +17892,9 @@
|
||||
"svg-support": [
|
||||
"cpe:2.3:a:benbodhi:svg_support:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"svg-uploads-support": [
|
||||
"cpe:2.3:a:ablyperu:svg_uploads_support:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"svg-vector-icon-plugin": [
|
||||
"cpe:2.3:a:wp_svg_icons_project:wp_svg_icons:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -17859,6 +17974,7 @@
|
||||
"cpe:2.3:a:tainacan:tainacan:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"tarteaucitronjs": [
|
||||
"cpe:2.3:a:amauri:tarteaucitron.io:*:*:*:*:*:wordpress:*:*",
|
||||
"cpe:2.3:a:tarteaucitron.js_-_cookies_legislation_\\\u0026_gdpr_project:tarteaucitron.js_-_cookies_legislation_\\\u0026_gdpr:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"taskbuilder": [
|
||||
@ -18106,6 +18222,9 @@
|
||||
"timeline-widget-addon-for-elementor": [
|
||||
"cpe:2.3:a:coolplugins:timeline_widget_for_elementor:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"timer-countdown": [
|
||||
"cpe:2.3:a:yaidier:countdown_timer:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"timesheet": [
|
||||
"cpe:2.3:a:bestwebsoft:timesheet:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -18249,9 +18368,15 @@
|
||||
"tripetto": [
|
||||
"cpe:2.3:a:tripetto:tripetto:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"tripplan": [
|
||||
"cpe:2.3:a:checklist:trip_plan:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"truebooker-appointment-booking": [
|
||||
"cpe:2.3:a:themetechmount:truebooker:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"trx_addons": [
|
||||
"cpe:2.3:a:themerex:addons:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"ts-webfonts-for-conoha": [
|
||||
"cpe:2.3:a:gmo:typesquare_webfonts_for_conoha:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -18457,9 +18582,15 @@
|
||||
"ultimate-weather-plugin": [
|
||||
"cpe:2.3:a:ultimate-weather_project:ultimate-weather:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"ultimate-woocommerce-auction-pro": [
|
||||
"cpe:2.3:a:auctionplugin:ultimate_wordpress_auction_plugin:*:*:*:*:pro:wordpress:*:*"
|
||||
],
|
||||
"ultimate-wp-query-search-filter": [
|
||||
"cpe:2.3:a:ultimate_wp_query_search_filter_project:ultimate_wp_query_search_filter:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"ultimate-youtube-video-player": [
|
||||
"cpe:2.3:a:codelizar:ultimate_youtube_video_\\\u0026_shorts_player_with_vimeo:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"ultra-companion": [
|
||||
"cpe:2.3:a:wpoperation:ultra_companion:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -19198,6 +19329,9 @@
|
||||
"woo-esto": [
|
||||
"cpe:2.3:a:rebing:woocommerce_esto:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"woo-exfood": [
|
||||
"cpe:2.3:a:exthemes:woocommerce_food:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"woo-floating-cart-lite": [
|
||||
"cpe:2.3:a:xplodedthemes:xt_floating_cart_for_woocommerce:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -19267,6 +19401,9 @@
|
||||
"woo-shipping-dpd-baltic": [
|
||||
"cpe:2.3:a:dpdgroup:woocommerce_shipping:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"woo-slider-pro-drag-drop-slider-builder-for-woocommerce": [
|
||||
"cpe:2.3:a:binarycarpenter:woo_slider_pro:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"woo-smart-compare": [
|
||||
"cpe:2.3:a:wpclever:wpc_smart_compare_for_woocommerce:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -19820,6 +19957,9 @@
|
||||
"cpe:2.3:a:kigurumi:csv_exporter:*:*:*:*:*:wordpress:*:*",
|
||||
"cpe:2.3:a:wp_csv_exporter_project:wp_csv_exporter:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"wp-curriculo-vitae": [
|
||||
"cpe:2.3:a:williamluis:wp-curriculo_vitae_free:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"wp-custom-admin-interface": [
|
||||
"cpe:2.3:a:wp_custom_admin_interface_project:wp_custom_admin_interface:*:*:*:*:*:*:*:*"
|
||||
],
|
||||
@ -19891,7 +20031,8 @@
|
||||
"cpe:2.3:a:display_users_project:display_users:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"wp-docs": [
|
||||
"cpe:2.3:a:androidbubble:wp_docs:*:*:*:*:*:wordpress:*:*"
|
||||
"cpe:2.3:a:androidbubble:wp_docs:*:*:*:*:*:wordpress:*:*",
|
||||
"cpe:2.3:a:fahadmahmood:wp_docs:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"wp-domain-redirect": [
|
||||
"cpe:2.3:a:wp_domain_redirect_project:wp_domain_redirect:*:*:*:*:*:wordpress:*:*"
|
||||
@ -20795,6 +20936,9 @@
|
||||
"wp-table-builder": [
|
||||
"cpe:2.3:a:dotcamp:wp_table_builder:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"wp-table-manager": [
|
||||
"cpe:2.3:a:joomunited:wp_table_manager:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"wp-table-reloaded": [
|
||||
"cpe:2.3:a:wp-table_reloaded_project:wp-table_reloaded:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -21139,6 +21283,9 @@
|
||||
"wppizza": [
|
||||
"cpe:2.3:a:wp-pizza:wppizza:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"wpquiz": [
|
||||
"cpe:2.3:a:bauc:wpquiz:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"wprequal": [
|
||||
"cpe:2.3:a:kevinbrent:wprequal:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -21169,6 +21316,9 @@
|
||||
"wpsolr-search-engine": [
|
||||
"cpe:2.3:a:wpsolr:wpsolr-search-engine:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"wpstickybar-sticky-bar-sticky-header": [
|
||||
"cpe:2.3:a:a17lab:wpstickybar:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"wpstream": [
|
||||
"cpe:2.3:a:wpstream:wpstream:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -21276,6 +21426,9 @@
|
||||
"xtremelocator": [
|
||||
"cpe:2.3:a:xtremelocator:xtremelocator:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"xv-random-quotes": [
|
||||
"cpe:2.3:a:xavivars:xv_random_quotes:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"yabp": [
|
||||
"cpe:2.3:a:tromit:yabp:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -21362,6 +21515,9 @@
|
||||
"yotpo-social-reviews-for-woocommerce": [
|
||||
"cpe:2.3:a:yotpo:yotpo:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"yotuwp-easy-youtube-embed": [
|
||||
"cpe:2.3:a:yotuwp:video_gallery:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"yourchannel": [
|
||||
"cpe:2.3:a:plugin:yourchannel:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@ -21782,6 +21938,9 @@
|
||||
"pressmart": [
|
||||
"cpe:2.3:a:presslayouts:pressmart:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"puzzles": [
|
||||
"cpe:2.3:a:themerex:puzzles:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"regina-lite": [
|
||||
"cpe:2.3:a:machothemes:regina_lite:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
|
||||
@ -1,3 +0,0 @@
|
||||
package dictionary
|
||||
|
||||
//go:generate go run ./index-generator/ -o data/cpe-index.json
|
||||
6
syft/pkg/cataloger/internal/cpegenerate/dictionary/index-generator/.gitignore
vendored
Normal file
6
syft/pkg/cataloger/internal/cpegenerate/dictionary/index-generator/.gitignore
vendored
Normal file
@ -0,0 +1,6 @@
|
||||
# ORAS cache directory - raw CPE data from NVD API
|
||||
.cpe-cache/
|
||||
|
||||
# Build artifacts
|
||||
index-generator
|
||||
.tmp-*
|
||||
@ -0,0 +1,370 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
const cacheDir = ".cpe-cache"
|
||||
|
||||
// IncrementMetadata tracks a single fetch increment for a monthly batch
|
||||
type IncrementMetadata struct {
|
||||
FetchedAt time.Time `json:"fetchedAt"`
|
||||
LastModStartDate time.Time `json:"lastModStartDate"`
|
||||
LastModEndDate time.Time `json:"lastModEndDate"`
|
||||
Products int `json:"products"`
|
||||
StartIndex int `json:"startIndex"` // API pagination start index
|
||||
EndIndex int `json:"endIndex"` // API pagination end index (last fetched)
|
||||
}
|
||||
|
||||
// MonthlyBatchMetadata tracks all increments for a specific month
|
||||
type MonthlyBatchMetadata struct {
|
||||
Complete bool `json:"complete"`
|
||||
TotalProducts int `json:"totalProducts"`
|
||||
Increments []IncrementMetadata `json:"increments"`
|
||||
}
|
||||
|
||||
// CacheMetadata tracks the state of the CPE cache using monthly time-based organization
|
||||
type CacheMetadata struct {
|
||||
LastFullRefresh time.Time `json:"lastFullRefresh"`
|
||||
LastStartIndex int `json:"lastStartIndex"` // last successfully processed startIndex for resume
|
||||
TotalProducts int `json:"totalProducts"`
|
||||
MonthlyBatches map[string]*MonthlyBatchMetadata `json:"monthlyBatches"` // key is "YYYY-MM"
|
||||
}
|
||||
|
||||
// CacheManager handles local caching of CPE data
|
||||
type CacheManager struct {
|
||||
cacheDir string
|
||||
}
|
||||
|
||||
// NewCacheManager creates a new cache manager
|
||||
func NewCacheManager() *CacheManager {
|
||||
return &CacheManager{
|
||||
cacheDir: cacheDir,
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureCacheDir ensures the cache directory exists
|
||||
func (m *CacheManager) EnsureCacheDir() error {
|
||||
if err := os.MkdirAll(m.cacheDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create cache directory: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadMetadata loads the cache metadata
|
||||
func (m *CacheManager) LoadMetadata() (*CacheMetadata, error) {
|
||||
metadataPath := filepath.Join(m.cacheDir, "metadata.json")
|
||||
|
||||
// check if metadata file exists
|
||||
if _, err := os.Stat(metadataPath); os.IsNotExist(err) {
|
||||
// return empty metadata for first run
|
||||
return &CacheMetadata{
|
||||
LastFullRefresh: time.Time{},
|
||||
TotalProducts: 0,
|
||||
MonthlyBatches: make(map[string]*MonthlyBatchMetadata),
|
||||
}, nil
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(metadataPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read metadata: %w", err)
|
||||
}
|
||||
|
||||
var metadata CacheMetadata
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal metadata: %w", err)
|
||||
}
|
||||
|
||||
// ensure MonthlyBatches map is initialized
|
||||
if metadata.MonthlyBatches == nil {
|
||||
metadata.MonthlyBatches = make(map[string]*MonthlyBatchMetadata)
|
||||
}
|
||||
|
||||
return &metadata, nil
|
||||
}
|
||||
|
||||
// SaveMetadata saves the cache metadata
|
||||
func (m *CacheManager) SaveMetadata(metadata *CacheMetadata) error {
|
||||
if err := m.EnsureCacheDir(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
metadataPath := filepath.Join(m.cacheDir, "metadata.json")
|
||||
|
||||
data, err := json.MarshalIndent(metadata, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal metadata: %w", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(metadataPath, data, 0600); err != nil {
|
||||
return fmt.Errorf("failed to write metadata: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveProductsToMonthlyFile saves products to a monthly file (initial.json or YYYY-MM.json)
|
||||
// uses atomic write pattern with temp file + rename for safety
|
||||
func (m *CacheManager) SaveProductsToMonthlyFile(filename string, products []NVDProduct) error {
|
||||
if err := m.EnsureCacheDir(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
filePath := filepath.Join(m.cacheDir, filename)
|
||||
tempPath := filePath + ".tmp"
|
||||
|
||||
// marshal products to JSON
|
||||
data, err := json.MarshalIndent(products, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal products: %w", err)
|
||||
}
|
||||
|
||||
// write to temp file first
|
||||
if err := os.WriteFile(tempPath, data, 0600); err != nil {
|
||||
return fmt.Errorf("failed to write temp file: %w", err)
|
||||
}
|
||||
|
||||
// atomic rename
|
||||
if err := os.Rename(tempPath, filePath); err != nil {
|
||||
// cleanup temp file on error
|
||||
_ = os.Remove(tempPath)
|
||||
return fmt.Errorf("failed to rename temp file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadMonthlyFile loads products from a monthly file
|
||||
func (m *CacheManager) LoadMonthlyFile(filename string) ([]NVDProduct, error) {
|
||||
filePath := filepath.Join(m.cacheDir, filename)
|
||||
|
||||
data, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return []NVDProduct{}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("failed to read %s: %w", filename, err)
|
||||
}
|
||||
|
||||
var products []NVDProduct
|
||||
if err := json.Unmarshal(data, &products); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal %s: %w", filename, err)
|
||||
}
|
||||
|
||||
return products, nil
|
||||
}
|
||||
|
||||
// GetMonthKey returns the "YYYY-MM" key for a given time
|
||||
func GetMonthKey(t time.Time) string {
|
||||
return t.Format("2006-01")
|
||||
}
|
||||
|
||||
// SaveProducts saves products grouped by modification month
|
||||
// this is called after fetching from the API to organize products into monthly files
|
||||
func (m *CacheManager) SaveProducts(products []NVDProduct, isFullRefresh bool, metadata *CacheMetadata, increment IncrementMetadata) error {
|
||||
if len(products) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if isFullRefresh {
|
||||
return m.saveFullRefresh(products, metadata)
|
||||
}
|
||||
|
||||
return m.saveIncrementalUpdate(products, metadata, increment)
|
||||
}
|
||||
|
||||
// saveFullRefresh saves all products to initial.json
|
||||
func (m *CacheManager) saveFullRefresh(products []NVDProduct, metadata *CacheMetadata) error {
|
||||
if err := m.SaveProductsToMonthlyFile("initial.json", products); err != nil {
|
||||
return fmt.Errorf("failed to save initial.json: %w", err)
|
||||
}
|
||||
|
||||
metadata.LastFullRefresh = time.Now()
|
||||
metadata.TotalProducts = len(products)
|
||||
metadata.LastStartIndex = 0 // reset on full refresh
|
||||
metadata.MonthlyBatches = make(map[string]*MonthlyBatchMetadata)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// saveIncrementalUpdate saves products grouped by modification month to monthly files
|
||||
func (m *CacheManager) saveIncrementalUpdate(products []NVDProduct, metadata *CacheMetadata, increment IncrementMetadata) error {
|
||||
productsByMonth, err := groupProductsByMonth(products)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for monthKey, monthProducts := range productsByMonth {
|
||||
if err := m.saveMonthlyBatch(monthKey, monthProducts, metadata, increment); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// update last processed index for resume capability
|
||||
metadata.LastStartIndex = increment.EndIndex
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// groupProductsByMonth groups products by their lastModified month
|
||||
func groupProductsByMonth(products []NVDProduct) (map[string][]NVDProduct, error) {
|
||||
productsByMonth := make(map[string][]NVDProduct)
|
||||
|
||||
for _, product := range products {
|
||||
lastMod, err := time.Parse(time.RFC3339, product.CPE.LastModified)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse lastModified for %s: %w", product.CPE.CPENameID, err)
|
||||
}
|
||||
|
||||
monthKey := GetMonthKey(lastMod)
|
||||
productsByMonth[monthKey] = append(productsByMonth[monthKey], product)
|
||||
}
|
||||
|
||||
return productsByMonth, nil
|
||||
}
|
||||
|
||||
// saveMonthlyBatch saves products for a specific month, merging with existing data
|
||||
func (m *CacheManager) saveMonthlyBatch(monthKey string, monthProducts []NVDProduct, metadata *CacheMetadata, increment IncrementMetadata) error {
|
||||
filename := monthKey + ".json"
|
||||
|
||||
// load existing products for this month
|
||||
existing, err := m.LoadMonthlyFile(filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load existing %s: %w", filename, err)
|
||||
}
|
||||
|
||||
// merge products (newer wins)
|
||||
merged := mergeProducts(existing, monthProducts)
|
||||
|
||||
// atomically save merged products
|
||||
if err := m.SaveProductsToMonthlyFile(filename, merged); err != nil {
|
||||
return fmt.Errorf("failed to save %s: %w", filename, err)
|
||||
}
|
||||
|
||||
// update metadata
|
||||
updateMonthlyBatchMetadata(metadata, monthKey, monthProducts, merged, increment)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// mergeProducts deduplicates products by CPENameID, with newer products overwriting older ones
|
||||
func mergeProducts(existing, updated []NVDProduct) []NVDProduct {
|
||||
productMap := make(map[string]NVDProduct)
|
||||
|
||||
for _, p := range existing {
|
||||
productMap[p.CPE.CPENameID] = p
|
||||
}
|
||||
for _, p := range updated {
|
||||
productMap[p.CPE.CPENameID] = p
|
||||
}
|
||||
|
||||
merged := make([]NVDProduct, 0, len(productMap))
|
||||
for _, p := range productMap {
|
||||
merged = append(merged, p)
|
||||
}
|
||||
|
||||
return merged
|
||||
}
|
||||
|
||||
// updateMonthlyBatchMetadata updates the metadata for a monthly batch
|
||||
func updateMonthlyBatchMetadata(metadata *CacheMetadata, monthKey string, newProducts, allProducts []NVDProduct, increment IncrementMetadata) {
|
||||
if metadata.MonthlyBatches[monthKey] == nil {
|
||||
metadata.MonthlyBatches[monthKey] = &MonthlyBatchMetadata{
|
||||
Complete: false,
|
||||
Increments: []IncrementMetadata{},
|
||||
}
|
||||
}
|
||||
|
||||
batchMeta := metadata.MonthlyBatches[monthKey]
|
||||
batchMeta.Increments = append(batchMeta.Increments, IncrementMetadata{
|
||||
FetchedAt: increment.FetchedAt,
|
||||
LastModStartDate: increment.LastModStartDate,
|
||||
LastModEndDate: increment.LastModEndDate,
|
||||
Products: len(newProducts),
|
||||
StartIndex: increment.StartIndex,
|
||||
EndIndex: increment.EndIndex,
|
||||
})
|
||||
batchMeta.TotalProducts = len(allProducts)
|
||||
}
|
||||
|
||||
// LoadAllProducts loads and merges all cached products from monthly files
|
||||
// returns a deduplicated slice of products (newer products override older ones by CPENameID)
|
||||
func (m *CacheManager) LoadAllProducts() ([]NVDProduct, error) {
|
||||
// check if cache directory exists
|
||||
if _, err := os.Stat(m.cacheDir); os.IsNotExist(err) {
|
||||
return []NVDProduct{}, nil
|
||||
}
|
||||
|
||||
productMap := make(map[string]NVDProduct)
|
||||
|
||||
// load initial.json first (if it exists)
|
||||
initial, err := m.LoadMonthlyFile("initial.json")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load initial.json: %w", err)
|
||||
}
|
||||
|
||||
for _, p := range initial {
|
||||
productMap[p.CPE.CPENameID] = p
|
||||
}
|
||||
|
||||
// load all monthly files (YYYY-MM.json)
|
||||
entries, err := os.ReadDir(m.cacheDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read cache directory: %w", err)
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" {
|
||||
continue
|
||||
}
|
||||
|
||||
// skip metadata.json and initial.json
|
||||
if entry.Name() == "metadata.json" || entry.Name() == "initial.json" {
|
||||
continue
|
||||
}
|
||||
|
||||
// load monthly file
|
||||
products, err := m.LoadMonthlyFile(entry.Name())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load %s: %w", entry.Name(), err)
|
||||
}
|
||||
|
||||
// merge products (newer wins based on lastModified)
|
||||
for _, p := range products {
|
||||
existing, exists := productMap[p.CPE.CPENameID]
|
||||
if !exists {
|
||||
productMap[p.CPE.CPENameID] = p
|
||||
continue
|
||||
}
|
||||
|
||||
// compare lastModified timestamps to keep the newer one
|
||||
newMod, _ := time.Parse(time.RFC3339, p.CPE.LastModified)
|
||||
existingMod, _ := time.Parse(time.RFC3339, existing.CPE.LastModified)
|
||||
|
||||
if newMod.After(existingMod) {
|
||||
productMap[p.CPE.CPENameID] = p
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// convert map to slice
|
||||
allProducts := make([]NVDProduct, 0, len(productMap))
|
||||
for _, p := range productMap {
|
||||
allProducts = append(allProducts, p)
|
||||
}
|
||||
|
||||
return allProducts, nil
|
||||
}
|
||||
|
||||
// CleanCache removes the local cache directory
|
||||
func (m *CacheManager) CleanCache() error {
|
||||
if err := os.RemoveAll(m.cacheDir); err != nil {
|
||||
return fmt.Errorf("failed to clean cache: %w", err)
|
||||
}
|
||||
fmt.Println("Cache cleaned successfully")
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,319 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCacheManager_MonthlyFileOperations(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
cacheManager := &CacheManager{cacheDir: tmpDir}
|
||||
|
||||
testProducts := []NVDProduct{
|
||||
{
|
||||
CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:product1:1.0:*:*:*:*:*:*:*",
|
||||
CPENameID: "product1-id",
|
||||
LastModified: "2024-11-15T10:00:00.000Z",
|
||||
Titles: []NVDTitle{{Title: "Test Product 1", Lang: "en"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:product2:2.0:*:*:*:*:*:*:*",
|
||||
CPENameID: "product2-id",
|
||||
LastModified: "2024-11-20T10:00:00.000Z",
|
||||
Titles: []NVDTitle{{Title: "Test Product 2", Lang: "en"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("save and load monthly file", func(t *testing.T) {
|
||||
err := cacheManager.SaveProductsToMonthlyFile("2024-11.json", testProducts)
|
||||
require.NoError(t, err)
|
||||
|
||||
expectedPath := filepath.Join(tmpDir, "2024-11.json")
|
||||
require.FileExists(t, expectedPath)
|
||||
|
||||
loaded, err := cacheManager.LoadMonthlyFile("2024-11.json")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, loaded, 2)
|
||||
assert.Equal(t, testProducts[0].CPE.CPEName, loaded[0].CPE.CPEName)
|
||||
assert.Equal(t, testProducts[1].CPE.CPEName, loaded[1].CPE.CPEName)
|
||||
})
|
||||
|
||||
t.Run("atomic save with temp file", func(t *testing.T) {
|
||||
err := cacheManager.SaveProductsToMonthlyFile("2024-12.json", testProducts)
|
||||
require.NoError(t, err)
|
||||
|
||||
// temp file should not exist after successful save
|
||||
tempPath := filepath.Join(tmpDir, "2024-12.json.tmp")
|
||||
require.NoFileExists(t, tempPath)
|
||||
|
||||
// actual file should exist
|
||||
finalPath := filepath.Join(tmpDir, "2024-12.json")
|
||||
require.FileExists(t, finalPath)
|
||||
})
|
||||
|
||||
t.Run("load non-existent file returns empty", func(t *testing.T) {
|
||||
loaded, err := cacheManager.LoadMonthlyFile("2025-01.json")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, loaded)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCacheManager_Metadata(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
cacheManager := &CacheManager{cacheDir: tmpDir}
|
||||
|
||||
t.Run("load metadata on first run", func(t *testing.T) {
|
||||
metadata, err := cacheManager.LoadMetadata()
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, metadata)
|
||||
|
||||
assert.NotNil(t, metadata.MonthlyBatches)
|
||||
assert.True(t, metadata.LastFullRefresh.IsZero())
|
||||
assert.Equal(t, 0, metadata.LastStartIndex)
|
||||
assert.Equal(t, 0, metadata.TotalProducts)
|
||||
})
|
||||
|
||||
t.Run("save and load metadata with monthly batches", func(t *testing.T) {
|
||||
now := time.Now()
|
||||
metadata := &CacheMetadata{
|
||||
LastFullRefresh: now,
|
||||
LastStartIndex: 4000,
|
||||
TotalProducts: 1500,
|
||||
MonthlyBatches: map[string]*MonthlyBatchMetadata{
|
||||
"2024-11": {
|
||||
Complete: true,
|
||||
TotalProducts: 1000,
|
||||
Increments: []IncrementMetadata{
|
||||
{
|
||||
FetchedAt: now,
|
||||
LastModStartDate: now.Add(-24 * time.Hour),
|
||||
LastModEndDate: now,
|
||||
Products: 1000,
|
||||
StartIndex: 0,
|
||||
EndIndex: 2000,
|
||||
},
|
||||
},
|
||||
},
|
||||
"2024-12": {
|
||||
Complete: false,
|
||||
TotalProducts: 500,
|
||||
Increments: []IncrementMetadata{
|
||||
{
|
||||
FetchedAt: now,
|
||||
LastModStartDate: now.Add(-12 * time.Hour),
|
||||
LastModEndDate: now,
|
||||
Products: 500,
|
||||
StartIndex: 0,
|
||||
EndIndex: 1000,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
err := cacheManager.SaveMetadata(metadata)
|
||||
require.NoError(t, err)
|
||||
|
||||
loadedMetadata, err := cacheManager.LoadMetadata()
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, metadata.TotalProducts, loadedMetadata.TotalProducts)
|
||||
assert.Equal(t, metadata.LastStartIndex, loadedMetadata.LastStartIndex)
|
||||
assert.Equal(t, 2, len(loadedMetadata.MonthlyBatches))
|
||||
assert.True(t, loadedMetadata.MonthlyBatches["2024-11"].Complete)
|
||||
assert.False(t, loadedMetadata.MonthlyBatches["2024-12"].Complete)
|
||||
assert.Equal(t, 1000, loadedMetadata.MonthlyBatches["2024-11"].TotalProducts)
|
||||
assert.Len(t, loadedMetadata.MonthlyBatches["2024-11"].Increments, 1)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCacheManager_LoadAllProducts(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
cacheManager := &CacheManager{cacheDir: tmpDir}
|
||||
|
||||
t.Run("load and merge monthly files", func(t *testing.T) {
|
||||
// save initial.json with base products
|
||||
initialProducts := []NVDProduct{
|
||||
{CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:product1:*:*:*:*:*:*:*:*",
|
||||
CPENameID: "product1-id",
|
||||
LastModified: "2024-10-01T10:00:00.000Z",
|
||||
}},
|
||||
{CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:product2:*:*:*:*:*:*:*:*",
|
||||
CPENameID: "product2-id",
|
||||
LastModified: "2024-10-15T10:00:00.000Z",
|
||||
}},
|
||||
}
|
||||
err := cacheManager.SaveProductsToMonthlyFile("initial.json", initialProducts)
|
||||
require.NoError(t, err)
|
||||
|
||||
// save 2024-11.json with updated product2 and new product3
|
||||
novemberProducts := []NVDProduct{
|
||||
{CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:product2:*:*:*:*:*:*:*:*",
|
||||
CPENameID: "product2-id",
|
||||
LastModified: "2024-11-05T10:00:00.000Z", // newer version
|
||||
}},
|
||||
{CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:product3:*:*:*:*:*:*:*:*",
|
||||
CPENameID: "product3-id",
|
||||
LastModified: "2024-11-10T10:00:00.000Z",
|
||||
}},
|
||||
}
|
||||
err = cacheManager.SaveProductsToMonthlyFile("2024-11.json", novemberProducts)
|
||||
require.NoError(t, err)
|
||||
|
||||
// load all products
|
||||
allProducts, err := cacheManager.LoadAllProducts()
|
||||
require.NoError(t, err)
|
||||
|
||||
// should have 3 unique products (product2 from Nov overwrites Oct version)
|
||||
require.Len(t, allProducts, 3)
|
||||
|
||||
// verify we got all products
|
||||
cpeNames := make(map[string]string) // CPENameID -> LastModified
|
||||
for _, product := range allProducts {
|
||||
cpeNames[product.CPE.CPENameID] = product.CPE.LastModified
|
||||
}
|
||||
|
||||
assert.Contains(t, cpeNames, "product1-id")
|
||||
assert.Contains(t, cpeNames, "product2-id")
|
||||
assert.Contains(t, cpeNames, "product3-id")
|
||||
|
||||
// product2 should be the newer version from November
|
||||
assert.Equal(t, "2024-11-05T10:00:00.000Z", cpeNames["product2-id"])
|
||||
})
|
||||
|
||||
t.Run("empty directory", func(t *testing.T) {
|
||||
emptyDir := t.TempDir()
|
||||
emptyCache := &CacheManager{cacheDir: emptyDir}
|
||||
|
||||
allProducts, err := emptyCache.LoadAllProducts()
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, allProducts)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCacheManager_CleanCache(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
cacheManager := &CacheManager{cacheDir: tmpDir}
|
||||
|
||||
// create some cache files
|
||||
testProducts := []NVDProduct{
|
||||
{CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*",
|
||||
CPENameID: "test-id",
|
||||
LastModified: "2024-11-01T10:00:00.000Z",
|
||||
}},
|
||||
}
|
||||
err := cacheManager.SaveProductsToMonthlyFile("initial.json", testProducts)
|
||||
require.NoError(t, err)
|
||||
|
||||
// verify cache exists
|
||||
require.DirExists(t, tmpDir)
|
||||
|
||||
// clean cache
|
||||
err = cacheManager.CleanCache()
|
||||
require.NoError(t, err)
|
||||
|
||||
// verify cache is removed
|
||||
_, err = os.Stat(tmpDir)
|
||||
assert.True(t, os.IsNotExist(err))
|
||||
}
|
||||
|
||||
func TestCacheManager_SaveProducts(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
cacheManager := &CacheManager{cacheDir: tmpDir}
|
||||
|
||||
t.Run("full refresh saves to initial.json", func(t *testing.T) {
|
||||
metadata := &CacheMetadata{
|
||||
MonthlyBatches: make(map[string]*MonthlyBatchMetadata),
|
||||
}
|
||||
|
||||
products := []NVDProduct{
|
||||
{CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:product1:*:*:*:*:*:*:*:*",
|
||||
CPENameID: "p1",
|
||||
LastModified: "2024-10-01T10:00:00.000Z",
|
||||
}},
|
||||
}
|
||||
|
||||
increment := IncrementMetadata{
|
||||
FetchedAt: time.Now(),
|
||||
Products: 1,
|
||||
}
|
||||
|
||||
err := cacheManager.SaveProducts(products, true, metadata, increment)
|
||||
require.NoError(t, err)
|
||||
|
||||
// verify initial.json exists
|
||||
initialPath := filepath.Join(tmpDir, "initial.json")
|
||||
require.FileExists(t, initialPath)
|
||||
|
||||
// verify metadata updated
|
||||
assert.NotZero(t, metadata.LastFullRefresh)
|
||||
assert.Equal(t, 1, metadata.TotalProducts)
|
||||
assert.Empty(t, metadata.MonthlyBatches)
|
||||
})
|
||||
|
||||
t.Run("incremental update groups by month", func(t *testing.T) {
|
||||
metadata := &CacheMetadata{
|
||||
LastFullRefresh: time.Now().Add(-30 * 24 * time.Hour),
|
||||
MonthlyBatches: make(map[string]*MonthlyBatchMetadata),
|
||||
}
|
||||
|
||||
products := []NVDProduct{
|
||||
{CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:product1:*:*:*:*:*:*:*:*",
|
||||
CPENameID: "p1",
|
||||
LastModified: "2024-11-05T10:00:00.000Z",
|
||||
}},
|
||||
{CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:product2:*:*:*:*:*:*:*:*",
|
||||
CPENameID: "p2",
|
||||
LastModified: "2024-11-15T10:00:00.000Z",
|
||||
}},
|
||||
{CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:product3:*:*:*:*:*:*:*:*",
|
||||
CPENameID: "p3",
|
||||
LastModified: "2024-12-01T10:00:00.000Z",
|
||||
}},
|
||||
}
|
||||
|
||||
increment := IncrementMetadata{
|
||||
FetchedAt: time.Now(),
|
||||
Products: 3,
|
||||
}
|
||||
|
||||
err := cacheManager.SaveProducts(products, false, metadata, increment)
|
||||
require.NoError(t, err)
|
||||
|
||||
// verify monthly files exist
|
||||
nov2024Path := filepath.Join(tmpDir, "2024-11.json")
|
||||
dec2024Path := filepath.Join(tmpDir, "2024-12.json")
|
||||
require.FileExists(t, nov2024Path)
|
||||
require.FileExists(t, dec2024Path)
|
||||
|
||||
// verify metadata has monthly batches
|
||||
assert.Len(t, metadata.MonthlyBatches, 2)
|
||||
assert.Contains(t, metadata.MonthlyBatches, "2024-11")
|
||||
assert.Contains(t, metadata.MonthlyBatches, "2024-12")
|
||||
|
||||
// verify 2024-11 has 2 products
|
||||
assert.Equal(t, 2, metadata.MonthlyBatches["2024-11"].TotalProducts)
|
||||
assert.Len(t, metadata.MonthlyBatches["2024-11"].Increments, 1)
|
||||
|
||||
// verify 2024-12 has 1 product
|
||||
assert.Equal(t, 1, metadata.MonthlyBatches["2024-12"].TotalProducts)
|
||||
})
|
||||
}
|
||||
@ -1,11 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"slices"
|
||||
"strings"
|
||||
@ -15,39 +10,6 @@ import (
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/internal/cpegenerate/dictionary"
|
||||
)
|
||||
|
||||
func generateIndexedDictionaryJSON(rawGzipData io.Reader) ([]byte, error) {
|
||||
gzipReader, err := gzip.NewReader(rawGzipData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to decompress CPE dictionary: %w", err)
|
||||
}
|
||||
defer gzipReader.Close()
|
||||
|
||||
// Read XML data
|
||||
data, err := io.ReadAll(gzipReader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to read CPE dictionary: %w", err)
|
||||
}
|
||||
|
||||
// Unmarshal XML
|
||||
var cpeList CpeList
|
||||
if err := xml.Unmarshal(data, &cpeList); err != nil {
|
||||
return nil, fmt.Errorf("unable to unmarshal CPE dictionary XML: %w", err)
|
||||
}
|
||||
|
||||
// Filter out data that's not applicable here
|
||||
cpeList = filterCpeList(cpeList)
|
||||
|
||||
// Create indexed dictionary to help with looking up CPEs
|
||||
indexedDictionary := indexCPEList(cpeList)
|
||||
|
||||
// Convert to JSON
|
||||
jsonData, err := json.MarshalIndent(indexedDictionary, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to marshal CPE dictionary to JSON: %w", err)
|
||||
}
|
||||
return jsonData, nil
|
||||
}
|
||||
|
||||
// filterCpeList removes CPE items that are not applicable to software packages.
|
||||
func filterCpeList(cpeList CpeList) CpeList {
|
||||
var processedCpeList CpeList
|
||||
|
||||
@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
@ -15,22 +16,37 @@ import (
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/internal/cpegenerate/dictionary"
|
||||
)
|
||||
|
||||
func Test_generateIndexedDictionaryJSON(t *testing.T) {
|
||||
func Test_processCPEList(t *testing.T) {
|
||||
// load test data from XML file (legacy format for testing backward compatibility)
|
||||
f, err := os.Open("testdata/official-cpe-dictionary_v2.3.xml")
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
|
||||
// Create a buffer to store the gzipped data in memory
|
||||
// create a buffer to store the gzipped data in memory
|
||||
buf := new(bytes.Buffer)
|
||||
|
||||
w := gzip.NewWriter(buf)
|
||||
_, err = io.Copy(w, f)
|
||||
require.NoError(t, err)
|
||||
|
||||
// (finalize the gzip stream)
|
||||
// finalize the gzip stream
|
||||
err = w.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
dictionaryJSON, err := generateIndexedDictionaryJSON(buf)
|
||||
// decompress and parse XML to get CpeList
|
||||
gzipReader, err := gzip.NewReader(buf)
|
||||
require.NoError(t, err)
|
||||
defer gzipReader.Close()
|
||||
|
||||
data, err := io.ReadAll(gzipReader)
|
||||
require.NoError(t, err)
|
||||
|
||||
var cpeList CpeList
|
||||
err = xml.Unmarshal(data, &cpeList)
|
||||
require.NoError(t, err)
|
||||
|
||||
// process the CPE list
|
||||
dictionaryJSON, err := processCPEList(cpeList)
|
||||
assert.NoError(t, err)
|
||||
|
||||
expected, err := os.ReadFile("./testdata/expected-cpe-index.json")
|
||||
@ -40,7 +56,7 @@ func Test_generateIndexedDictionaryJSON(t *testing.T) {
|
||||
dictionaryJSONString := string(dictionaryJSON)
|
||||
|
||||
if diff := cmp.Diff(expectedDictionaryJSONString, dictionaryJSONString); diff != "" {
|
||||
t.Errorf("generateIndexedDictionaryJSON() mismatch (-want +got):\n%s", diff)
|
||||
t.Errorf("processCPEList() mismatch (-want +got):\n%s", diff)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,49 +1,217 @@
|
||||
// This program downloads the latest CPE dictionary from NIST and processes it into a JSON file that can be embedded into Syft for more accurate CPE results.
|
||||
// This program fetches CPE data from the NVD Products API and processes it into a JSON file that can be embedded into Syft for more accurate CPE results.
|
||||
// ORAS caching is managed by Taskfile tasks - this program only works with local cache.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
func mainE() error {
|
||||
var outputFilename string
|
||||
flag.StringVar(&outputFilename, "o", "", "file location to save CPE index")
|
||||
var forceFullRefresh bool
|
||||
var cacheOnly bool
|
||||
flag.StringVar(&outputFilename, "o", "", "file location to save CPE index (required for build mode)")
|
||||
flag.BoolVar(&forceFullRefresh, "full", false, "force full refresh instead of incremental update")
|
||||
flag.BoolVar(&cacheOnly, "cache-only", false, "only update cache from NVD API, don't generate index")
|
||||
flag.Parse()
|
||||
|
||||
if outputFilename == "" {
|
||||
return errors.New("-o is required")
|
||||
// validate flags
|
||||
if !cacheOnly && outputFilename == "" {
|
||||
return errors.New("-o is required (unless using -cache-only)")
|
||||
}
|
||||
|
||||
// Download and decompress file
|
||||
fmt.Println("Fetching CPE dictionary...")
|
||||
resp, err := http.Get(cpeDictionaryURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to get CPE dictionary: %w", err)
|
||||
if cacheOnly && outputFilename != "" {
|
||||
return errors.New("-cache-only and -o cannot be used together")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
cacheManager := NewCacheManager()
|
||||
|
||||
// MODE 1: Update cache only (called by task generate:cpe-index:update-cache)
|
||||
if cacheOnly {
|
||||
return updateCache(ctx, cacheManager, forceFullRefresh)
|
||||
}
|
||||
|
||||
// MODE 2: Generate index from existing cache (called by task generate:cpe-index:build)
|
||||
return generateIndexFromCache(cacheManager, outputFilename)
|
||||
}
|
||||
|
||||
// updateCache fetches new/updated CPE data from NVD API and saves to local cache
|
||||
func updateCache(ctx context.Context, cacheManager *CacheManager, forceFullRefresh bool) error {
|
||||
metadata, err := cacheManager.LoadMetadata()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load metadata: %w", err)
|
||||
}
|
||||
|
||||
lastModStartDate, isFullRefresh := determineUpdateMode(metadata, forceFullRefresh)
|
||||
|
||||
// use resume index if available
|
||||
resumeFromIndex := 0
|
||||
if !isFullRefresh && metadata.LastStartIndex > 0 {
|
||||
resumeFromIndex = metadata.LastStartIndex
|
||||
fmt.Printf("Resuming from index %d...\n", resumeFromIndex)
|
||||
}
|
||||
|
||||
allProducts, increment, err := fetchProducts(ctx, lastModStartDate, resumeFromIndex)
|
||||
if err != nil {
|
||||
// if we have partial products, save them before returning error
|
||||
if len(allProducts) > 0 {
|
||||
fmt.Printf("\nError occurred but saving %d products fetched so far...\n", len(allProducts))
|
||||
if saveErr := saveAndReportResults(cacheManager, allProducts, isFullRefresh, metadata, increment); saveErr != nil {
|
||||
fmt.Printf("WARNING: Failed to save partial progress: %v\n", saveErr)
|
||||
} else {
|
||||
fmt.Println("Partial progress saved successfully. Run again to resume from this point.")
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if len(allProducts) == 0 {
|
||||
fmt.Println("No products fetched (already up to date)")
|
||||
return nil
|
||||
}
|
||||
|
||||
return saveAndReportResults(cacheManager, allProducts, isFullRefresh, metadata, increment)
|
||||
}
|
||||
|
||||
// determineUpdateMode decides whether to do a full refresh or incremental update
|
||||
func determineUpdateMode(metadata *CacheMetadata, forceFullRefresh bool) (time.Time, bool) {
|
||||
if forceFullRefresh || metadata.LastFullRefresh.IsZero() {
|
||||
fmt.Println("Performing full refresh of CPE data")
|
||||
return time.Time{}, true
|
||||
}
|
||||
|
||||
fmt.Printf("Performing incremental update since %s\n", metadata.LastFullRefresh.Format("2006-01-02"))
|
||||
return metadata.LastFullRefresh, false
|
||||
}
|
||||
|
||||
// fetchProducts fetches products from the NVD API
|
||||
func fetchProducts(ctx context.Context, lastModStartDate time.Time, resumeFromIndex int) ([]NVDProduct, IncrementMetadata, error) {
|
||||
apiClient := NewNVDAPIClient()
|
||||
fmt.Println("Fetching CPE data from NVD Products API...")
|
||||
|
||||
var allProducts []NVDProduct
|
||||
var totalResults int
|
||||
var firstStartIndex, lastEndIndex int
|
||||
|
||||
onPageFetched := func(startIndex int, response NVDProductsResponse) error {
|
||||
if totalResults == 0 {
|
||||
totalResults = response.TotalResults
|
||||
firstStartIndex = startIndex
|
||||
}
|
||||
lastEndIndex = startIndex + response.ResultsPerPage
|
||||
allProducts = append(allProducts, response.Products...)
|
||||
fmt.Printf("Fetched %d/%d products...\n", len(allProducts), totalResults)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := apiClient.FetchProductsSince(ctx, lastModStartDate, resumeFromIndex, onPageFetched); err != nil {
|
||||
// return partial products with increment metadata so they can be saved
|
||||
increment := IncrementMetadata{
|
||||
FetchedAt: time.Now(),
|
||||
LastModStartDate: lastModStartDate,
|
||||
LastModEndDate: time.Now(),
|
||||
Products: len(allProducts),
|
||||
StartIndex: firstStartIndex,
|
||||
EndIndex: lastEndIndex,
|
||||
}
|
||||
return allProducts, increment, fmt.Errorf("failed to fetch products from NVD API: %w", err)
|
||||
}
|
||||
|
||||
increment := IncrementMetadata{
|
||||
FetchedAt: time.Now(),
|
||||
LastModStartDate: lastModStartDate,
|
||||
LastModEndDate: time.Now(),
|
||||
Products: len(allProducts),
|
||||
StartIndex: firstStartIndex,
|
||||
EndIndex: lastEndIndex,
|
||||
}
|
||||
|
||||
return allProducts, increment, nil
|
||||
}
|
||||
|
||||
// saveAndReportResults saves products and metadata, then reports success
|
||||
func saveAndReportResults(cacheManager *CacheManager, allProducts []NVDProduct, isFullRefresh bool, metadata *CacheMetadata, increment IncrementMetadata) error {
|
||||
fmt.Println("Saving products to cache...")
|
||||
if err := cacheManager.SaveProducts(allProducts, isFullRefresh, metadata, increment); err != nil {
|
||||
return fmt.Errorf("failed to save products: %w", err)
|
||||
}
|
||||
|
||||
if err := cacheManager.SaveMetadata(metadata); err != nil {
|
||||
return fmt.Errorf("failed to save metadata: %w", err)
|
||||
}
|
||||
|
||||
fmt.Println("Cache updated successfully!")
|
||||
if isFullRefresh {
|
||||
fmt.Printf("Total products in cache: %d\n", len(allProducts))
|
||||
} else {
|
||||
fmt.Printf("Added/updated %d products\n", len(allProducts))
|
||||
fmt.Printf("Grouped into %d monthly files\n", len(metadata.MonthlyBatches))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// generateIndexFromCache generates the CPE index from cached data only
|
||||
func generateIndexFromCache(cacheManager *CacheManager, outputFilename string) error {
|
||||
fmt.Println("Loading cached products...")
|
||||
allProducts, err := cacheManager.LoadAllProducts()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load cached products: %w", err)
|
||||
}
|
||||
|
||||
if len(allProducts) == 0 {
|
||||
return fmt.Errorf("no cached data available - run 'task generate:cpe-index:cache:pull' and 'task generate:cpe-index:cache:update' first")
|
||||
}
|
||||
|
||||
fmt.Printf("Loaded %d products from cache\n", len(allProducts))
|
||||
fmt.Println("Converting products to CPE list...")
|
||||
cpeList := ProductsToCpeList(allProducts)
|
||||
|
||||
fmt.Println("Generating index...")
|
||||
dictionaryJSON, err := generateIndexedDictionaryJSON(resp.Body)
|
||||
dictionaryJSON, err := processCPEList(cpeList)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write CPE index (JSON data) to disk
|
||||
err = os.WriteFile(outputFilename, dictionaryJSON, 0600)
|
||||
if err != nil {
|
||||
// ensure parent directory exists
|
||||
outputDir := filepath.Dir(outputFilename)
|
||||
if err := os.MkdirAll(outputDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create output directory: %w", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(outputFilename, dictionaryJSON, 0600); err != nil {
|
||||
return fmt.Errorf("unable to write processed CPE dictionary to file: %w", err)
|
||||
}
|
||||
|
||||
fmt.Println("Done!")
|
||||
|
||||
fmt.Println("CPE index generated successfully!")
|
||||
return nil
|
||||
}
|
||||
|
||||
// processCPEList filters and indexes a CPE list, returning JSON bytes
|
||||
func processCPEList(cpeList CpeList) ([]byte, error) {
|
||||
// filter out data that's not applicable
|
||||
cpeList = filterCpeList(cpeList)
|
||||
|
||||
// create indexed dictionary to help with looking up CPEs
|
||||
indexedDictionary := indexCPEList(cpeList)
|
||||
|
||||
// convert to JSON
|
||||
jsonData, err := json.MarshalIndent(indexedDictionary, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to marshal CPE dictionary to JSON: %w", err)
|
||||
}
|
||||
return jsonData, nil
|
||||
}
|
||||
|
||||
// errExit prints an error and exits with a non-zero exit code.
|
||||
func errExit(err error) {
|
||||
log.Printf("command failed: %s", err)
|
||||
|
||||
@ -23,4 +23,5 @@ type CpeList struct {
|
||||
CpeItems []CpeItem `xml:"cpe-item"`
|
||||
}
|
||||
|
||||
const cpeDictionaryURL = "https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz"
|
||||
// cpeDictionaryURL is deprecated - we now use the NVD Products API
|
||||
// const cpeDictionaryURL = "https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz"
|
||||
|
||||
@ -0,0 +1,66 @@
|
||||
package main
|
||||
|
||||
// nvd_adapter.go converts NVD API responses to the existing CpeList/CpeItem structures
|
||||
// this allows the existing filtering and indexing logic to work without modification
|
||||
|
||||
// ProductsToCpeList converts NVD API products to the legacy CpeList format
|
||||
func ProductsToCpeList(products []NVDProduct) CpeList {
|
||||
var cpeItems []CpeItem
|
||||
|
||||
for _, product := range products {
|
||||
item := productToCpeItem(product)
|
||||
cpeItems = append(cpeItems, item)
|
||||
}
|
||||
|
||||
return CpeList{
|
||||
CpeItems: cpeItems,
|
||||
}
|
||||
}
|
||||
|
||||
// productToCpeItem converts a single NVD API product to a CpeItem
|
||||
func productToCpeItem(product NVDProduct) CpeItem {
|
||||
details := product.CPE
|
||||
|
||||
item := CpeItem{
|
||||
// use CPE 2.2 format for the Name field (legacy compatibility)
|
||||
// note: the old XML feed had both 2.2 and 2.3 formats
|
||||
// for now, we'll use 2.3 format in both places since that's what the API provides
|
||||
Name: details.CPEName,
|
||||
}
|
||||
|
||||
// extract title (prefer English)
|
||||
for _, title := range details.Titles {
|
||||
if title.Lang == "en" {
|
||||
item.Title = title.Title
|
||||
break
|
||||
}
|
||||
}
|
||||
// fallback to first title if no English title found
|
||||
if item.Title == "" && len(details.Titles) > 0 {
|
||||
item.Title = details.Titles[0].Title
|
||||
}
|
||||
|
||||
// convert references
|
||||
if len(details.Refs) > 0 {
|
||||
item.References.Reference = make([]struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Body string `xml:",chardata"`
|
||||
}, len(details.Refs))
|
||||
|
||||
for i, ref := range details.Refs {
|
||||
item.References.Reference[i].Href = ref.Ref
|
||||
item.References.Reference[i].Body = ref.Type
|
||||
}
|
||||
}
|
||||
|
||||
// set CPE 2.3 information
|
||||
item.Cpe23Item.Name = details.CPEName
|
||||
|
||||
// handle deprecation
|
||||
if details.Deprecated && len(details.DeprecatedBy) > 0 {
|
||||
// use the first deprecated-by CPE (the old format only supported one)
|
||||
item.Cpe23Item.Deprecation.DeprecatedBy.Name = details.DeprecatedBy[0].CPEName
|
||||
}
|
||||
|
||||
return item
|
||||
}
|
||||
@ -0,0 +1,235 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestProductToCpeItem(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
product NVDProduct
|
||||
expected CpeItem
|
||||
}{
|
||||
{
|
||||
name: "basic product conversion",
|
||||
product: NVDProduct{
|
||||
CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*",
|
||||
Deprecated: false,
|
||||
Titles: []NVDTitle{
|
||||
{Title: "Test Product", Lang: "en"},
|
||||
},
|
||||
Refs: []NVDRef{
|
||||
{Ref: "https://example.com/product", Type: "Vendor"},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: CpeItem{
|
||||
Name: "cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*",
|
||||
Title: "Test Product",
|
||||
References: struct {
|
||||
Reference []struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Body string `xml:",chardata"`
|
||||
} `xml:"reference"`
|
||||
}{
|
||||
Reference: []struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Body string `xml:",chardata"`
|
||||
}{
|
||||
{Href: "https://example.com/product", Body: "Vendor"},
|
||||
},
|
||||
},
|
||||
Cpe23Item: struct {
|
||||
Name string `xml:"name,attr"`
|
||||
Deprecation struct {
|
||||
DeprecatedBy struct {
|
||||
Name string `xml:"name,attr"`
|
||||
} `xml:"deprecated-by"`
|
||||
} `xml:"deprecation"`
|
||||
}{
|
||||
Name: "cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "deprecated product",
|
||||
product: NVDProduct{
|
||||
CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:old:1.0:*:*:*:*:*:*:*",
|
||||
Deprecated: true,
|
||||
DeprecatedBy: []NVDDeprecatedBy{
|
||||
{CPEName: "cpe:2.3:a:vendor:new:1.0:*:*:*:*:*:*:*", CPENameID: "test-uuid-123"},
|
||||
},
|
||||
Titles: []NVDTitle{
|
||||
{Title: "Old Product", Lang: "en"},
|
||||
},
|
||||
Refs: []NVDRef{
|
||||
{Ref: "https://example.com/old", Type: "Vendor"},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: CpeItem{
|
||||
Name: "cpe:2.3:a:vendor:old:1.0:*:*:*:*:*:*:*",
|
||||
Title: "Old Product",
|
||||
References: struct {
|
||||
Reference []struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Body string `xml:",chardata"`
|
||||
} `xml:"reference"`
|
||||
}{
|
||||
Reference: []struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Body string `xml:",chardata"`
|
||||
}{
|
||||
{Href: "https://example.com/old", Body: "Vendor"},
|
||||
},
|
||||
},
|
||||
Cpe23Item: struct {
|
||||
Name string `xml:"name,attr"`
|
||||
Deprecation struct {
|
||||
DeprecatedBy struct {
|
||||
Name string `xml:"name,attr"`
|
||||
} `xml:"deprecated-by"`
|
||||
} `xml:"deprecation"`
|
||||
}{
|
||||
Name: "cpe:2.3:a:vendor:old:1.0:*:*:*:*:*:*:*",
|
||||
Deprecation: struct {
|
||||
DeprecatedBy struct {
|
||||
Name string `xml:"name,attr"`
|
||||
} `xml:"deprecated-by"`
|
||||
}{
|
||||
DeprecatedBy: struct {
|
||||
Name string `xml:"name,attr"`
|
||||
}{
|
||||
Name: "cpe:2.3:a:vendor:new:1.0:*:*:*:*:*:*:*",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "product with multiple titles prefers English",
|
||||
product: NVDProduct{
|
||||
CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*",
|
||||
Titles: []NVDTitle{
|
||||
{Title: "Produit", Lang: "fr"},
|
||||
{Title: "Product", Lang: "en"},
|
||||
{Title: "Producto", Lang: "es"},
|
||||
},
|
||||
Refs: []NVDRef{
|
||||
{Ref: "https://example.com", Type: "Vendor"},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: CpeItem{
|
||||
Name: "cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*",
|
||||
Title: "Product",
|
||||
References: struct {
|
||||
Reference []struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Body string `xml:",chardata"`
|
||||
} `xml:"reference"`
|
||||
}{
|
||||
Reference: []struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Body string `xml:",chardata"`
|
||||
}{
|
||||
{Href: "https://example.com", Body: "Vendor"},
|
||||
},
|
||||
},
|
||||
Cpe23Item: struct {
|
||||
Name string `xml:"name,attr"`
|
||||
Deprecation struct {
|
||||
DeprecatedBy struct {
|
||||
Name string `xml:"name,attr"`
|
||||
} `xml:"deprecated-by"`
|
||||
} `xml:"deprecation"`
|
||||
}{
|
||||
Name: "cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := productToCpeItem(tt.product)
|
||||
|
||||
assert.Equal(t, tt.expected.Name, result.Name)
|
||||
assert.Equal(t, tt.expected.Title, result.Title)
|
||||
assert.Equal(t, tt.expected.Cpe23Item.Name, result.Cpe23Item.Name)
|
||||
assert.Equal(t, tt.expected.Cpe23Item.Deprecation.DeprecatedBy.Name, result.Cpe23Item.Deprecation.DeprecatedBy.Name)
|
||||
|
||||
require.Equal(t, len(tt.expected.References.Reference), len(result.References.Reference))
|
||||
for i := range tt.expected.References.Reference {
|
||||
assert.Equal(t, tt.expected.References.Reference[i].Href, result.References.Reference[i].Href)
|
||||
assert.Equal(t, tt.expected.References.Reference[i].Body, result.References.Reference[i].Body)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductsToCpeList(t *testing.T) {
|
||||
products := []NVDProduct{
|
||||
{
|
||||
CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:product1:1.0:*:*:*:*:*:*:*",
|
||||
Titles: []NVDTitle{
|
||||
{Title: "Product 1", Lang: "en"},
|
||||
},
|
||||
Refs: []NVDRef{
|
||||
{Ref: "https://npmjs.com/package/product1", Type: "Vendor"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:product2:2.0:*:*:*:*:*:*:*",
|
||||
Titles: []NVDTitle{
|
||||
{Title: "Product 2", Lang: "en"},
|
||||
},
|
||||
Refs: []NVDRef{
|
||||
{Ref: "https://pypi.org/project/product2", Type: "Vendor"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := ProductsToCpeList(products)
|
||||
|
||||
require.Len(t, result.CpeItems, 2)
|
||||
assert.Equal(t, "cpe:2.3:a:vendor:product1:1.0:*:*:*:*:*:*:*", result.CpeItems[0].Name)
|
||||
assert.Equal(t, "Product 1", result.CpeItems[0].Title)
|
||||
assert.Equal(t, "cpe:2.3:a:vendor:product2:2.0:*:*:*:*:*:*:*", result.CpeItems[1].Name)
|
||||
assert.Equal(t, "Product 2", result.CpeItems[1].Title)
|
||||
}
|
||||
|
||||
func TestProductsToCpeList_MultipleProducts(t *testing.T) {
|
||||
products := []NVDProduct{
|
||||
{
|
||||
CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:product1:*:*:*:*:*:*:*:*",
|
||||
Titles: []NVDTitle{{Title: "Product 1", Lang: "en"}},
|
||||
Refs: []NVDRef{{Ref: "https://example.com/1", Type: "Vendor"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
CPE: NVDProductDetails{
|
||||
CPEName: "cpe:2.3:a:vendor:product2:*:*:*:*:*:*:*:*",
|
||||
Titles: []NVDTitle{{Title: "Product 2", Lang: "en"}},
|
||||
Refs: []NVDRef{{Ref: "https://example.com/2", Type: "Vendor"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := ProductsToCpeList(products)
|
||||
|
||||
require.Len(t, result.CpeItems, 2)
|
||||
assert.Equal(t, "cpe:2.3:a:vendor:product1:*:*:*:*:*:*:*:*", result.CpeItems[0].Cpe23Item.Name)
|
||||
assert.Equal(t, "cpe:2.3:a:vendor:product2:*:*:*:*:*:*:*:*", result.CpeItems[1].Cpe23Item.Name)
|
||||
}
|
||||
@ -0,0 +1,286 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
const (
|
||||
nvdProductsAPIURL = "https://services.nvd.nist.gov/rest/json/cpes/2.0"
|
||||
resultsPerPage = 2000 // maximum allowed by NVD API
|
||||
|
||||
// rate limits per NVD API documentation
|
||||
unauthenticatedRequestsPer30Seconds = 5
|
||||
authenticatedRequestsPer30Seconds = 50
|
||||
|
||||
// retry configuration for rate limiting
|
||||
maxRetries = 5
|
||||
baseRetryDelay = 30 * time.Second // NVD uses 30-second rolling windows
|
||||
)
|
||||
|
||||
// NVDAPIClient handles communication with the NVD Products API
|
||||
type NVDAPIClient struct {
|
||||
httpClient *http.Client
|
||||
rateLimiter *rate.Limiter
|
||||
apiKey string
|
||||
}
|
||||
|
||||
// NVDProductsResponse represents the JSON response from the NVD Products API
|
||||
type NVDProductsResponse struct {
|
||||
ResultsPerPage int `json:"resultsPerPage"`
|
||||
StartIndex int `json:"startIndex"`
|
||||
TotalResults int `json:"totalResults"`
|
||||
Format string `json:"format"`
|
||||
Version string `json:"version"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
Products []NVDProduct `json:"products"`
|
||||
}
|
||||
|
||||
// NVDProduct represents a single product entry from the API
|
||||
type NVDProduct struct {
|
||||
CPE NVDProductDetails `json:"cpe"`
|
||||
}
|
||||
|
||||
// NVDProductDetails contains the CPE and reference information
|
||||
type NVDProductDetails struct {
|
||||
CPEName string `json:"cpeName"`
|
||||
Deprecated bool `json:"deprecated"`
|
||||
DeprecatedBy []NVDDeprecatedBy `json:"deprecatedBy,omitempty"`
|
||||
CPENameID string `json:"cpeNameId"`
|
||||
Created string `json:"created"`
|
||||
LastModified string `json:"lastModified"`
|
||||
Titles []NVDTitle `json:"titles"`
|
||||
Refs []NVDRef `json:"refs"`
|
||||
}
|
||||
|
||||
// NVDTitle represents a title in a specific language
|
||||
type NVDTitle struct {
|
||||
Title string `json:"title"`
|
||||
Lang string `json:"lang"`
|
||||
}
|
||||
|
||||
// NVDRef represents a reference URL
|
||||
type NVDRef struct {
|
||||
Ref string `json:"ref"`
|
||||
Type string `json:"type,omitempty"`
|
||||
}
|
||||
|
||||
// NVDDeprecatedBy represents a CPE that replaces a deprecated one
|
||||
type NVDDeprecatedBy struct {
|
||||
CPEName string `json:"cpeName"`
|
||||
CPENameID string `json:"cpeNameId"`
|
||||
}
|
||||
|
||||
// NewNVDAPIClient creates a new NVD API client
|
||||
// it reads the NVD_API_KEY environment variable for authenticated requests
|
||||
func NewNVDAPIClient() *NVDAPIClient {
|
||||
apiKey := os.Getenv("NVD_API_KEY")
|
||||
|
||||
// determine rate limit based on authentication
|
||||
requestsPer30Seconds := unauthenticatedRequestsPer30Seconds
|
||||
if apiKey != "" {
|
||||
requestsPer30Seconds = authenticatedRequestsPer30Seconds
|
||||
fmt.Printf("Using authenticated NVD API access (%d requests per 30 seconds)\n", requestsPer30Seconds)
|
||||
} else {
|
||||
fmt.Printf("Using unauthenticated NVD API access (%d requests per 30 seconds)\n", requestsPer30Seconds)
|
||||
fmt.Println("Set NVD_API_KEY environment variable for higher rate limits")
|
||||
}
|
||||
|
||||
// create rate limiter with 10% safety margin to avoid hitting limits
|
||||
// X requests per 30 seconds * 0.9 = (X * 0.9) / 30 requests per second
|
||||
effectiveRate := float64(requestsPer30Seconds) * 0.9 / 30.0
|
||||
limiter := rate.NewLimiter(rate.Limit(effectiveRate), 1)
|
||||
fmt.Printf("Rate limiter configured: %.2f requests/second (with 10%% safety margin)\n", effectiveRate)
|
||||
|
||||
return &NVDAPIClient{
|
||||
httpClient: &http.Client{
|
||||
Timeout: 60 * time.Second,
|
||||
},
|
||||
rateLimiter: limiter,
|
||||
apiKey: apiKey,
|
||||
}
|
||||
}
|
||||
|
||||
// PageCallback is called after each page is successfully fetched
|
||||
// it receives the startIndex and the response for that page
|
||||
type PageCallback func(startIndex int, response NVDProductsResponse) error
|
||||
|
||||
// FetchProductsSince fetches all products modified since the given date
|
||||
// if lastModStartDate is zero, fetches all products
|
||||
// calls onPageFetched callback after each successful page fetch for incremental saving
|
||||
// if resumeFromIndex > 0, starts fetching from that index
|
||||
func (c *NVDAPIClient) FetchProductsSince(ctx context.Context, lastModStartDate time.Time, resumeFromIndex int, onPageFetched PageCallback) error {
|
||||
startIndex := resumeFromIndex
|
||||
|
||||
for {
|
||||
resp, err := c.fetchPage(ctx, startIndex, lastModStartDate)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to fetch page at index %d: %w", startIndex, err)
|
||||
}
|
||||
|
||||
// call callback to save progress immediately
|
||||
if onPageFetched != nil {
|
||||
if err := onPageFetched(startIndex, resp); err != nil {
|
||||
return fmt.Errorf("callback failed at index %d: %w", startIndex, err)
|
||||
}
|
||||
}
|
||||
|
||||
// check if we've fetched all results
|
||||
if startIndex+resp.ResultsPerPage >= resp.TotalResults {
|
||||
fmt.Printf("Fetched %d/%d products (complete)\n", resp.TotalResults, resp.TotalResults)
|
||||
break
|
||||
}
|
||||
|
||||
startIndex += resp.ResultsPerPage
|
||||
fmt.Printf("Fetched %d/%d products...\n", startIndex, resp.TotalResults)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// fetchPage fetches a single page of results from the NVD API with retry logic for rate limiting
|
||||
func (c *NVDAPIClient) fetchPage(ctx context.Context, startIndex int, lastModStartDate time.Time) (NVDProductsResponse, error) {
|
||||
var lastErr error
|
||||
|
||||
for attempt := 0; attempt < maxRetries; attempt++ {
|
||||
// wait for rate limiter
|
||||
if err := c.rateLimiter.Wait(ctx); err != nil {
|
||||
return NVDProductsResponse{}, fmt.Errorf("rate limiter error: %w", err)
|
||||
}
|
||||
|
||||
// build request URL
|
||||
url := fmt.Sprintf("%s?resultsPerPage=%d&startIndex=%d", nvdProductsAPIURL, resultsPerPage, startIndex)
|
||||
|
||||
// add date range if specified (incremental update)
|
||||
if !lastModStartDate.IsZero() {
|
||||
// NVD API requires RFC3339 format: 2024-01-01T00:00:00.000
|
||||
lastModStartStr := lastModStartDate.Format("2006-01-02T15:04:05.000")
|
||||
url += fmt.Sprintf("&lastModStartDate=%s", lastModStartStr)
|
||||
}
|
||||
|
||||
// create request
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return NVDProductsResponse{}, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
// add API key header if available
|
||||
if c.apiKey != "" {
|
||||
req.Header.Set("apiKey", c.apiKey)
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "syft-cpe-dictionary-generator")
|
||||
|
||||
// execute request
|
||||
httpResp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return NVDProductsResponse{}, fmt.Errorf("failed to execute request: %w", err)
|
||||
}
|
||||
|
||||
// handle rate limiting
|
||||
if httpResp.StatusCode == http.StatusTooManyRequests {
|
||||
lastErr = c.handleRateLimit(ctx, httpResp, attempt)
|
||||
continue // retry
|
||||
}
|
||||
|
||||
// handle HTTP status codes
|
||||
statusResponse, handled, err := c.handleHTTPStatus(httpResp, startIndex)
|
||||
if handled {
|
||||
// either error or special case (404 with empty results)
|
||||
return statusResponse, err
|
||||
}
|
||||
|
||||
// success - parse response
|
||||
var response NVDProductsResponse
|
||||
if err := json.NewDecoder(httpResp.Body).Decode(&response); err != nil {
|
||||
httpResp.Body.Close()
|
||||
return NVDProductsResponse{}, fmt.Errorf("failed to decode response: %w", err)
|
||||
}
|
||||
|
||||
httpResp.Body.Close()
|
||||
return response, nil
|
||||
}
|
||||
|
||||
return NVDProductsResponse{}, fmt.Errorf("max retries (%d) exceeded: %w", maxRetries, lastErr)
|
||||
}
|
||||
|
||||
// handleRateLimit handles HTTP 429 responses by parsing Retry-After and waiting
|
||||
func (c *NVDAPIClient) handleRateLimit(ctx context.Context, httpResp *http.Response, attempt int) error {
|
||||
body, _ := io.ReadAll(httpResp.Body)
|
||||
httpResp.Body.Close()
|
||||
|
||||
// parse Retry-After header
|
||||
retryAfter := parseRetryAfter(httpResp.Header.Get("Retry-After"))
|
||||
if retryAfter == 0 {
|
||||
// use exponential backoff if no Retry-After header
|
||||
retryAfter = baseRetryDelay * time.Duration(1<<uint(attempt))
|
||||
}
|
||||
|
||||
err := fmt.Errorf("rate limited (429): %s", string(body))
|
||||
fmt.Printf("Rate limited (429), retrying in %v (attempt %d/%d)...\n", retryAfter, attempt+1, maxRetries)
|
||||
|
||||
select {
|
||||
case <-time.After(retryAfter):
|
||||
return err // return to retry
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// handleHTTPStatus handles non-429 HTTP status codes
|
||||
// returns (response, handled, error) where:
|
||||
// - handled=true means the status was processed (either success case like 404 or error)
|
||||
// - handled=false means continue to normal response parsing
|
||||
func (c *NVDAPIClient) handleHTTPStatus(httpResp *http.Response, startIndex int) (NVDProductsResponse, bool, error) {
|
||||
// handle 404 as "no results found" (common when querying recent dates with no updates)
|
||||
if httpResp.StatusCode == http.StatusNotFound {
|
||||
httpResp.Body.Close()
|
||||
return NVDProductsResponse{
|
||||
ResultsPerPage: 0,
|
||||
StartIndex: startIndex,
|
||||
TotalResults: 0,
|
||||
Products: []NVDProduct{},
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
// check for other non-200 status codes
|
||||
if httpResp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(httpResp.Body)
|
||||
httpResp.Body.Close()
|
||||
return NVDProductsResponse{}, true, fmt.Errorf("unexpected status code %d: %s", httpResp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
// status OK - let caller parse response
|
||||
return NVDProductsResponse{}, false, nil
|
||||
}
|
||||
|
||||
// parseRetryAfter parses the Retry-After header from HTTP 429 responses
|
||||
// returns 0 if the header is missing or invalid
|
||||
func parseRetryAfter(header string) time.Duration {
|
||||
if header == "" {
|
||||
return 0
|
||||
}
|
||||
|
||||
// try parsing as seconds (most common format)
|
||||
if seconds, err := strconv.Atoi(header); err == nil {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
|
||||
// try parsing as HTTP date (less common)
|
||||
if t, err := time.Parse(time.RFC1123, header); err == nil {
|
||||
duration := time.Until(t)
|
||||
if duration > 0 {
|
||||
return duration
|
||||
}
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
123
syft/pkg/cataloger/internal/licenses/find_licenses.go
Normal file
123
syft/pkg/cataloger/internal/licenses/find_licenses.go
Normal file
@ -0,0 +1,123 @@
|
||||
package licenses
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
// RelativeToPackage searches for licenses in the same directory as primary evidence locations
|
||||
// on the package and returns the package with licenses set and ID reset if the package has no licenses already
|
||||
func RelativeToPackage(ctx context.Context, resolver file.Resolver, p pkg.Package) pkg.Package {
|
||||
// if licenses were already found, don't search for more
|
||||
if !p.Licenses.Empty() {
|
||||
return p
|
||||
}
|
||||
var out []pkg.License
|
||||
for _, l := range p.Locations.ToUnorderedSlice() {
|
||||
if evidenceType, ok := l.Annotations[pkg.EvidenceAnnotationKey]; ok && evidenceType != pkg.PrimaryEvidenceAnnotation {
|
||||
continue
|
||||
}
|
||||
// search for license files relative to any primary evidence on the package
|
||||
out = append(out, FindRelativeToLocations(ctx, resolver, l)...)
|
||||
}
|
||||
if len(out) > 0 {
|
||||
p.Licenses = pkg.NewLicenseSet(out...)
|
||||
p.SetID()
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// FindAtLocations creates License objects by reading license files directly the provided locations
|
||||
func FindAtLocations(ctx context.Context, resolver file.Resolver, locations ...file.Location) []pkg.License {
|
||||
var out []pkg.License
|
||||
for _, loc := range locations {
|
||||
out = append(out, readFromResolver(ctx, resolver, loc)...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FindAtPaths creates License objects by reading license files directly at the provided paths
|
||||
func FindAtPaths(ctx context.Context, resolver file.Resolver, paths ...string) []pkg.License {
|
||||
var out []pkg.License
|
||||
for _, p := range paths {
|
||||
locs, err := resolver.FilesByPath(p)
|
||||
if err != nil {
|
||||
log.WithFields("error", err, "path", p).Trace("unable to resolve license path")
|
||||
continue
|
||||
}
|
||||
for _, loc := range locs {
|
||||
out = append(out, readFromResolver(ctx, resolver, loc)...)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FindInDirs creates License objects by searching for known license files in the provided directories
|
||||
func FindInDirs(ctx context.Context, resolver file.Resolver, dirs ...string) []pkg.License {
|
||||
var out []pkg.License
|
||||
for _, dir := range dirs {
|
||||
glob := path.Join(dir, "*") // only search in the directory
|
||||
out = append(out, FindByGlob(ctx, resolver, glob)...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FindRelativeToLocations creates License objects by searching for known license files relative to the provided locations, in the same directory path
|
||||
func FindRelativeToLocations(ctx context.Context, resolver file.Resolver, locations ...file.Location) []pkg.License {
|
||||
var out []pkg.License
|
||||
for _, location := range locations {
|
||||
dir := path.Dir(location.AccessPath)
|
||||
out = append(out, FindInDirs(ctx, resolver, dir)...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FindByGlob creates License objects by searching for license files with the provided glob.
|
||||
// only file names which match licenses.LowerFileNames() case-insensitive will be included,
|
||||
// so a recursive glob search such as: `<path>/**/*` will only attempt to read LICENSE files it finds, for example
|
||||
func FindByGlob(ctx context.Context, resolver file.Resolver, glob string) []pkg.License {
|
||||
locs, err := resolver.FilesByGlob(glob)
|
||||
if err != nil {
|
||||
log.WithFields("glob", glob, "error", err).Debug("error searching for license files")
|
||||
return nil
|
||||
}
|
||||
var out []pkg.License
|
||||
for _, l := range locs {
|
||||
fileName := path.Base(l.Path())
|
||||
if IsLicenseFile(fileName) {
|
||||
out = append(out, readFromResolver(ctx, resolver, l)...)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func NewFromValues(ctx context.Context, locations []file.Location, values ...string) []pkg.License {
|
||||
if len(locations) == 0 {
|
||||
return pkg.NewLicensesFromValuesWithContext(ctx, values...)
|
||||
}
|
||||
|
||||
var out []pkg.License
|
||||
for _, value := range values {
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, pkg.NewLicenseFromLocationsWithContext(ctx, value, locations...))
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func readFromResolver(ctx context.Context, resolver file.Resolver, location file.Location) []pkg.License {
|
||||
metadataContents, err := resolver.FileContentsByLocation(location)
|
||||
if err != nil || metadataContents == nil {
|
||||
log.WithFields("error", err, "path", location.Path()).Trace("unable to license file contents")
|
||||
return nil
|
||||
}
|
||||
defer internal.CloseAndLogError(metadataContents, location.Path())
|
||||
return pkg.NewLicensesFromReadCloserWithContext(ctx, file.NewLocationReadCloser(location, metadataContents))
|
||||
}
|
||||
171
syft/pkg/cataloger/internal/licenses/find_licenses_test.go
Normal file
171
syft/pkg/cataloger/internal/licenses/find_licenses_test.go
Normal file
@ -0,0 +1,171 @@
|
||||
package licenses
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/internal/licenses"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/internal/fileresolver"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
// scanner is used by all tests
|
||||
var scanner = getScanner()
|
||||
|
||||
func Test_FindRelativeLicenses(t *testing.T) {
|
||||
resolver := fileresolver.NewFromUnindexedDirectory("testdata")
|
||||
sourceTxtResolved, err := resolver.FilesByPath("source.txt")
|
||||
require.NoError(t, err)
|
||||
|
||||
sourceTxt := file.NewLocationSet(sourceTxtResolved[0].WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation))
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
resolver file.Resolver
|
||||
p pkg.Package
|
||||
expected pkg.LicenseSet
|
||||
}{
|
||||
{
|
||||
name: "existing license",
|
||||
resolver: resolver,
|
||||
p: pkg.Package{
|
||||
Locations: sourceTxt,
|
||||
Licenses: pkg.NewLicenseSet(pkg.NewLicense("GPL-2.0")),
|
||||
},
|
||||
expected: pkg.NewLicenseSet(pkg.NewLicense("GPL-2.0")),
|
||||
},
|
||||
{
|
||||
name: "no licenses",
|
||||
resolver: fileresolver.Empty{},
|
||||
p: pkg.Package{
|
||||
Locations: sourceTxt,
|
||||
},
|
||||
expected: pkg.NewLicenseSet(),
|
||||
},
|
||||
{
|
||||
name: "found relative license",
|
||||
resolver: resolver,
|
||||
p: pkg.Package{
|
||||
Locations: sourceTxt,
|
||||
},
|
||||
expected: pkg.NewLicenseSet(pkg.NewLicense("MIT")),
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.TODO()
|
||||
ctx = licenses.SetContextLicenseScanner(ctx, scanner)
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := RelativeToPackage(ctx, tt.resolver, tt.p)
|
||||
require.Equal(t, licenseNames(tt.expected.ToSlice()), licenseNames(got.Licenses.ToSlice()))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_Finders(t *testing.T) {
|
||||
resolver := fileresolver.NewFromUnindexedDirectory("testdata")
|
||||
|
||||
// prepare context with license scanner
|
||||
ctx := context.TODO()
|
||||
ctx = licenses.SetContextLicenseScanner(ctx, scanner)
|
||||
|
||||
// resolve known files
|
||||
licenseLocs, err := resolver.FilesByPath("LICENSE")
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, licenseLocs)
|
||||
licenseLoc := licenseLocs[0]
|
||||
|
||||
sourceLocs, err := resolver.FilesByPath("source.txt")
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, sourceLocs)
|
||||
sourceLoc := sourceLocs[0]
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
finder func(t *testing.T) []pkg.License
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "FindAtLocations finds LICENSE content",
|
||||
finder: func(t *testing.T) []pkg.License {
|
||||
return FindAtLocations(ctx, resolver, licenseLoc)
|
||||
},
|
||||
expected: []string{"MIT"},
|
||||
},
|
||||
{
|
||||
name: "FindAtLocations with empty resolver returns none",
|
||||
finder: func(t *testing.T) []pkg.License {
|
||||
return FindAtLocations(ctx, fileresolver.Empty{}, licenseLoc)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "FindAtPaths finds LICENSE by path",
|
||||
finder: func(t *testing.T) []pkg.License {
|
||||
return FindAtPaths(ctx, resolver, "LICENSE")
|
||||
},
|
||||
expected: []string{"MIT"},
|
||||
},
|
||||
{
|
||||
name: "FindInDirs finds LICENSE in directory",
|
||||
finder: func(t *testing.T) []pkg.License {
|
||||
return FindInDirs(ctx, resolver, ".")
|
||||
},
|
||||
expected: []string{"MIT"},
|
||||
},
|
||||
{
|
||||
name: "FindRelativeToLocations finds LICENSE relative to source.txt",
|
||||
finder: func(t *testing.T) []pkg.License {
|
||||
return FindRelativeToLocations(ctx, resolver, sourceLoc)
|
||||
},
|
||||
expected: []string{"MIT"},
|
||||
},
|
||||
{
|
||||
name: "FindByGlob finds LICENSE with glob",
|
||||
finder: func(t *testing.T) []pkg.License {
|
||||
return FindByGlob(ctx, resolver, "*")
|
||||
},
|
||||
expected: []string{"MIT"},
|
||||
},
|
||||
{
|
||||
name: "FindByGlob finds LICENSE with recursive glob",
|
||||
finder: func(t *testing.T) []pkg.License {
|
||||
return FindByGlob(ctx, resolver, "**/*")
|
||||
},
|
||||
expected: []string{"MIT"},
|
||||
},
|
||||
{
|
||||
name: "NewFromValues with locations returns license values",
|
||||
finder: func(t *testing.T) []pkg.License {
|
||||
return NewFromValues(ctx, []file.Location{licenseLoc}, "MIT")
|
||||
},
|
||||
expected: []string{"MIT"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tt.finder(t)
|
||||
require.Equal(t, tt.expected, licenseNames(got))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func licenseNames(slice []pkg.License) []string {
|
||||
var out []string
|
||||
for _, l := range slice {
|
||||
out = append(out, l.SPDXExpression)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func getScanner() licenses.Scanner {
|
||||
s, err := licenses.NewDefaultScanner()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
62
syft/pkg/cataloger/internal/licenses/names.go
Normal file
62
syft/pkg/cataloger/internal/licenses/names.go
Normal file
@ -0,0 +1,62 @@
|
||||
package licenses
|
||||
|
||||
import (
|
||||
"math"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/scylladb/go-set/strset"
|
||||
|
||||
"github.com/anchore/syft/internal/licenses"
|
||||
)
|
||||
|
||||
var licenseRegexp = regexp.MustCompile(`^(?i)(?:(?:UN|MIT-)?LICEN[S|C]E|COPYING|NOTICE).*$`)
|
||||
|
||||
// lowerFileNames is a strset.Set of lowercased filenames
|
||||
var lowerFileNames = func() *strset.Set {
|
||||
lowerNames := strset.New()
|
||||
for _, fileName := range licenses.FileNames() {
|
||||
lowerNames.Add(strings.ToLower(fileName))
|
||||
}
|
||||
return lowerNames
|
||||
}()
|
||||
|
||||
// lowerFileNamesSorted is a sorted slice of lowercased filenames
|
||||
var lowerFileNamesSorted = func() []string {
|
||||
out := lowerFileNames.List()
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}()
|
||||
|
||||
// remove duplicate names that match the regex, keep any extras to test after regex check
|
||||
var minLength, extraFileNames = func() (int, []string) {
|
||||
minSize := math.MaxInt
|
||||
var extras []string
|
||||
for _, name := range lowerFileNamesSorted {
|
||||
if len(name) < minSize {
|
||||
minSize = len(name)
|
||||
}
|
||||
if licenseRegexp.MatchString(name) {
|
||||
continue
|
||||
}
|
||||
extras = append(extras, name)
|
||||
}
|
||||
return minSize, extras
|
||||
}()
|
||||
|
||||
// IsLicenseFile returns true if the name matches known license file name patterns
|
||||
func IsLicenseFile(name string) bool {
|
||||
if len(name) < minLength {
|
||||
return false
|
||||
}
|
||||
if licenseRegexp.MatchString(name) {
|
||||
return true
|
||||
}
|
||||
for _, licenseFile := range extraFileNames {
|
||||
if strings.EqualFold(licenseFile, name) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
45
syft/pkg/cataloger/internal/licenses/names_test.go
Normal file
45
syft/pkg/cataloger/internal/licenses/names_test.go
Normal file
@ -0,0 +1,45 @@
|
||||
package licenses
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func Test_IsLicenseFile(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want bool
|
||||
}{
|
||||
// positive cases (should be detected as license files)
|
||||
{"plain LICENSE", "LICENSE", true},
|
||||
{"lowercase license", "license", true},
|
||||
{"license with extension", "LICENSE.txt", true},
|
||||
{"mixed case", "LiCeNsE", true},
|
||||
{"copying", "COPYING", true},
|
||||
{"AL2.0", "AL2.0", true},
|
||||
{"notice", "NOTICE", true},
|
||||
{"mit-license", "MIT-License", true},
|
||||
{"unlicense", "UNLICENSE", true},
|
||||
{"licence variant", "LICENCE", true},
|
||||
{"license markdown", "license.md", true},
|
||||
|
||||
// negative cases (should NOT be detected)
|
||||
{"AL1.0", "AL1.0", false},
|
||||
{"readme", "README", false},
|
||||
{"readme with ext", "README.md", false},
|
||||
{"not a license", "not_a_license", false},
|
||||
{"licensor (prefix-like but not)", "LICENSOR", false},
|
||||
{"too short (below minLength)", "a", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := IsLicenseFile(tt.input)
|
||||
if got != tt.want {
|
||||
t.Fatalf("IsLicenseFile(%q) = %v, want %v", tt.input, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
7
syft/pkg/cataloger/internal/licenses/testdata/LICENSE
vendored
Normal file
7
syft/pkg/cataloger/internal/licenses/testdata/LICENSE
vendored
Normal file
@ -0,0 +1,7 @@
|
||||
Copyright 2025 Some Place, Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
1
syft/pkg/cataloger/internal/licenses/testdata/source.txt
vendored
Normal file
1
syft/pkg/cataloger/internal/licenses/testdata/source.txt
vendored
Normal file
@ -0,0 +1 @@
|
||||
a source file
|
||||
@ -26,6 +26,7 @@ import (
|
||||
"github.com/anchore/syft/internal/relationship"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/internal/fileresolver"
|
||||
"github.com/anchore/syft/syft/linux"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/generic"
|
||||
@ -81,6 +82,7 @@ func NewCatalogTester() *CatalogTester {
|
||||
locationComparer: cmptest.DefaultLocationComparer,
|
||||
licenseComparer: cmptest.DefaultLicenseComparer,
|
||||
packageStringer: stringPackage,
|
||||
resolver: fileresolver.Empty{},
|
||||
ignoreUnfulfilledPathResponses: map[string][]string{
|
||||
"FilesByPath": {
|
||||
// most catalogers search for a linux release, which will not be fulfilled in testing
|
||||
|
||||
@ -17,13 +17,13 @@ import (
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
intFile "github.com/anchore/syft/internal/file"
|
||||
"github.com/anchore/syft/internal/licenses"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/unknown"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/generic"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/internal/licenses"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/java/internal/maven"
|
||||
)
|
||||
|
||||
@ -55,15 +55,14 @@ var javaArchiveHashes = []crypto.Hash{
|
||||
}
|
||||
|
||||
type archiveParser struct {
|
||||
fileManifest intFile.ZipFileManifest
|
||||
location file.Location
|
||||
archivePath string
|
||||
contentPath string
|
||||
fileInfo archiveFilename
|
||||
detectNested bool
|
||||
cfg ArchiveCatalogerConfig
|
||||
maven *maven.Resolver
|
||||
licenseScanner licenses.Scanner
|
||||
fileManifest intFile.ZipFileManifest
|
||||
location file.Location
|
||||
archivePath string
|
||||
contentPath string
|
||||
fileInfo archiveFilename
|
||||
detectNested bool
|
||||
cfg ArchiveCatalogerConfig
|
||||
maven *maven.Resolver
|
||||
}
|
||||
|
||||
type genericArchiveParserAdapter struct {
|
||||
@ -101,11 +100,6 @@ func uniquePkgKey(groupID string, p *pkg.Package) string {
|
||||
// newJavaArchiveParser returns a new java archive parser object for the given archive. Can be configured to discover
|
||||
// and parse nested archives or ignore them.
|
||||
func newJavaArchiveParser(ctx context.Context, reader file.LocationReadCloser, detectNested bool, cfg ArchiveCatalogerConfig) (*archiveParser, func(), error) {
|
||||
licenseScanner, err := licenses.ContextLicenseScanner(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("could not build license scanner for java archive parser: %w", err)
|
||||
}
|
||||
|
||||
// fetch the last element of the virtual path
|
||||
virtualElements := strings.Split(reader.Path(), ":")
|
||||
currentFilepath := virtualElements[len(virtualElements)-1]
|
||||
@ -115,21 +109,20 @@ func newJavaArchiveParser(ctx context.Context, reader file.LocationReadCloser, d
|
||||
return nil, cleanupFn, fmt.Errorf("unable to process java archive: %w", err)
|
||||
}
|
||||
|
||||
fileManifest, err := intFile.NewZipFileManifest(archivePath)
|
||||
fileManifest, err := intFile.NewZipFileManifest(ctx, archivePath)
|
||||
if err != nil {
|
||||
return nil, cleanupFn, fmt.Errorf("unable to read files from java archive: %w", err)
|
||||
}
|
||||
|
||||
return &archiveParser{
|
||||
fileManifest: fileManifest,
|
||||
location: reader.Location,
|
||||
archivePath: archivePath,
|
||||
contentPath: contentPath,
|
||||
fileInfo: newJavaArchiveFilename(currentFilepath),
|
||||
detectNested: detectNested,
|
||||
cfg: cfg,
|
||||
maven: maven.NewResolver(nil, cfg.mavenConfig()),
|
||||
licenseScanner: licenseScanner,
|
||||
fileManifest: fileManifest,
|
||||
location: reader.Location,
|
||||
archivePath: archivePath,
|
||||
contentPath: contentPath,
|
||||
fileInfo: newJavaArchiveFilename(currentFilepath),
|
||||
detectNested: detectNested,
|
||||
cfg: cfg,
|
||||
maven: maven.NewResolver(nil, cfg.mavenConfig()),
|
||||
}, cleanupFn, nil
|
||||
}
|
||||
|
||||
@ -233,7 +226,7 @@ func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package,
|
||||
}
|
||||
|
||||
// fetch the manifest file
|
||||
contents, err := intFile.ContentsFromZip(j.archivePath, manifestMatches...)
|
||||
contents, err := intFile.ContentsFromZip(ctx, j.archivePath, manifestMatches...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to extract java manifests (%s): %w", j.location, err)
|
||||
}
|
||||
@ -394,8 +387,9 @@ type parsedPomProject struct {
|
||||
|
||||
// discoverMainPackageFromPomInfo attempts to resolve maven groupId, artifactId, version and other info from found pom information
|
||||
func (j *archiveParser) discoverMainPackageFromPomInfo(ctx context.Context) (group, name, version string, parsedPom *parsedPomProject) {
|
||||
properties, _ := pomPropertiesByParentPath(j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomPropertiesGlob))
|
||||
projects, _ := pomProjectByParentPath(j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomXMLGlob))
|
||||
// Find the pom.properties/pom.xml if the names seem like a plausible match
|
||||
properties, _ := pomPropertiesByParentPath(ctx, j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomPropertiesGlob))
|
||||
projects, _ := pomProjectByParentPath(ctx, j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomXMLGlob))
|
||||
|
||||
artifactsMap := j.buildArtifactsMap(properties)
|
||||
pomProperties, parsedPom := j.findBestPomMatch(properties, projects, artifactsMap)
|
||||
@ -526,13 +520,13 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(ctx context.Context, paren
|
||||
var pkgs []pkg.Package
|
||||
|
||||
// pom.properties
|
||||
properties, err := pomPropertiesByParentPath(j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomPropertiesGlob))
|
||||
properties, err := pomPropertiesByParentPath(ctx, j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomPropertiesGlob))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// pom.xml
|
||||
projects, err := pomProjectByParentPath(j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomXMLGlob))
|
||||
projects, err := pomProjectByParentPath(ctx, j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomXMLGlob))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -569,16 +563,20 @@ func getDigestsFromArchive(ctx context.Context, archivePath string) ([]file.Dige
|
||||
}
|
||||
|
||||
func (j *archiveParser) getLicenseFromFileInArchive(ctx context.Context) ([]pkg.License, error) {
|
||||
var out []pkg.License
|
||||
for _, filename := range licenses.FileNames() {
|
||||
licenseMatches := j.fileManifest.GlobMatch(true, "/META-INF/"+filename)
|
||||
if len(licenseMatches) == 0 {
|
||||
// Try the root directory if it's not in META-INF
|
||||
licenseMatches = j.fileManifest.GlobMatch(true, "/"+filename)
|
||||
// prefer identified licenses, fall back to unknown
|
||||
var identified []pkg.License
|
||||
var unidentified []pkg.License
|
||||
|
||||
for _, glob := range []string{"/META-INF/*", "/*"} {
|
||||
var licenseMatches []string
|
||||
for _, f := range j.fileManifest.GlobMatch(true, glob) {
|
||||
if licenses.IsLicenseFile(path.Base(f)) {
|
||||
licenseMatches = append(licenseMatches, f)
|
||||
}
|
||||
}
|
||||
|
||||
if len(licenseMatches) > 0 {
|
||||
contents, err := intFile.ContentsFromZip(j.archivePath, licenseMatches...)
|
||||
contents, err := intFile.ContentsFromZip(ctx, j.archivePath, licenseMatches...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to extract java license (%s): %w", j.location, err)
|
||||
}
|
||||
@ -586,15 +584,28 @@ func (j *archiveParser) getLicenseFromFileInArchive(ctx context.Context) ([]pkg.
|
||||
for _, licenseMatch := range licenseMatches {
|
||||
licenseContents := contents[licenseMatch]
|
||||
r := strings.NewReader(licenseContents)
|
||||
lics := pkg.NewLicensesFromReadCloserWithContext(ctx, file.NewLocationReadCloser(j.location, io.NopCloser(r)))
|
||||
if len(lics) > 0 {
|
||||
out = append(out, lics...)
|
||||
foundLicenses := pkg.NewLicensesFromReadCloserWithContext(ctx, file.NewLocationReadCloser(j.location, io.NopCloser(r)))
|
||||
for _, l := range foundLicenses {
|
||||
if l.SPDXExpression != "" {
|
||||
identified = append(identified, l)
|
||||
} else {
|
||||
unidentified = append(unidentified, l)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// prefer licenses found in /META-INF
|
||||
if len(identified) > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
if len(identified) == 0 {
|
||||
return unidentified, nil
|
||||
}
|
||||
|
||||
return identified, nil
|
||||
}
|
||||
|
||||
func (j *archiveParser) discoverPkgsFromNestedArchives(ctx context.Context, parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
@ -606,7 +617,7 @@ func (j *archiveParser) discoverPkgsFromNestedArchives(ctx context.Context, pare
|
||||
// associating each discovered package to the given parent package.
|
||||
func discoverPkgsFromZip(ctx context.Context, location file.Location, archivePath, contentPath string, fileManifest intFile.ZipFileManifest, parentPkg *pkg.Package, cfg ArchiveCatalogerConfig) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
// search and parse pom.properties files & fetch the contents
|
||||
openers, err := intFile.ExtractFromZipToUniqueTempFile(archivePath, contentPath, fileManifest.GlobMatch(false, archiveFormatGlobs...)...)
|
||||
openers, err := intFile.ExtractFromZipToUniqueTempFile(ctx, archivePath, contentPath, fileManifest.GlobMatch(false, archiveFormatGlobs...)...)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("unable to extract files from zip: %w", err)
|
||||
}
|
||||
@ -670,8 +681,8 @@ func discoverPkgsFromOpener(ctx context.Context, location file.Location, pathWit
|
||||
return nestedPkgs, nestedRelationships, nil
|
||||
}
|
||||
|
||||
func pomPropertiesByParentPath(archivePath string, location file.Location, extractPaths []string) (map[string]pkg.JavaPomProperties, error) {
|
||||
contentsOfMavenPropertiesFiles, err := intFile.ContentsFromZip(archivePath, extractPaths...)
|
||||
func pomPropertiesByParentPath(ctx context.Context, archivePath string, location file.Location, extractPaths []string) (map[string]pkg.JavaPomProperties, error) {
|
||||
contentsOfMavenPropertiesFiles, err := intFile.ContentsFromZip(ctx, archivePath, extractPaths...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to extract maven files: %w", err)
|
||||
}
|
||||
@ -699,8 +710,8 @@ func pomPropertiesByParentPath(archivePath string, location file.Location, extra
|
||||
return propertiesByParentPath, nil
|
||||
}
|
||||
|
||||
func pomProjectByParentPath(archivePath string, location file.Location, extractPaths []string) (map[string]*parsedPomProject, error) {
|
||||
contentsOfMavenProjectFiles, err := intFile.ContentsFromZip(archivePath, extractPaths...)
|
||||
func pomProjectByParentPath(ctx context.Context, archivePath string, location file.Location, extractPaths []string) (map[string]*parsedPomProject, error) {
|
||||
contentsOfMavenProjectFiles, err := intFile.ContentsFromZip(ctx, archivePath, extractPaths...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to extract maven files: %w", err)
|
||||
}
|
||||
@ -799,7 +810,7 @@ func packageIdentitiesMatch(p pkg.Package, parentPkg *pkg.Package) bool {
|
||||
switch {
|
||||
case !ok:
|
||||
log.WithFields("package", p.String()).Trace("unable to extract java metadata to check for matching package identity for package: %s", p.Name)
|
||||
case !parentOk:
|
||||
default: // !parentOk
|
||||
log.WithFields("package", parentPkg.String()).Trace("unable to extract java metadata to check for matching package identity for package: %s", parentPkg.Name)
|
||||
}
|
||||
// if we can't extract metadata, we can check for matching identities via the package name
|
||||
|
||||
@ -72,8 +72,7 @@ func TestSearchMavenForLicenses(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// setup parser
|
||||
ap, cleanupFn, err := newJavaArchiveParser(
|
||||
ctx,
|
||||
ap, cleanupFn, err := newJavaArchiveParser(context.Background(),
|
||||
file.LocationReadCloser{
|
||||
Location: file.NewLocation(fixture.Name()),
|
||||
ReadCloser: fixture,
|
||||
@ -373,8 +372,7 @@ func TestParseJar(t *testing.T) {
|
||||
UseNetwork: false,
|
||||
UseMavenLocalRepository: false,
|
||||
}
|
||||
parser, cleanupFn, err := newJavaArchiveParser(
|
||||
ctx,
|
||||
parser, cleanupFn, err := newJavaArchiveParser(context.Background(),
|
||||
file.LocationReadCloser{
|
||||
Location: file.NewLocation(fixture.Name()),
|
||||
ReadCloser: fixture,
|
||||
@ -1478,7 +1476,6 @@ func Test_parseJavaArchive_regressions(t *testing.T) {
|
||||
}
|
||||
|
||||
func Test_deterministicMatchingPomProperties(t *testing.T) {
|
||||
ctx := pkgtest.Context()
|
||||
tests := []struct {
|
||||
fixture string
|
||||
expected maven.ID
|
||||
@ -1502,8 +1499,7 @@ func Test_deterministicMatchingPomProperties(t *testing.T) {
|
||||
fixture, err := os.Open(fixturePath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parser, cleanupFn, err := newJavaArchiveParser(
|
||||
ctx,
|
||||
parser, cleanupFn, err := newJavaArchiveParser(context.Background(),
|
||||
file.LocationReadCloser{
|
||||
Location: file.NewLocation(fixture.Name()),
|
||||
ReadCloser: fixture,
|
||||
@ -1640,8 +1636,7 @@ func Test_jarPomPropertyResolutionDoesNotPanic(t *testing.T) {
|
||||
|
||||
ctx := context.TODO()
|
||||
// setup parser
|
||||
ap, cleanupFn, err := newJavaArchiveParser(
|
||||
ctx,
|
||||
ap, cleanupFn, err := newJavaArchiveParser(context.Background(),
|
||||
file.LocationReadCloser{
|
||||
Location: file.NewLocation(fixture.Name()),
|
||||
ReadCloser: fixture,
|
||||
|
||||
@ -11,6 +11,7 @@ import (
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/internal/licenses"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/java/internal/maven"
|
||||
)
|
||||
|
||||
@ -65,7 +66,7 @@ func (p pomXMLCataloger) Catalog(ctx context.Context, fileResolver file.Resolver
|
||||
continue
|
||||
}
|
||||
resolved[id] = mainPkg
|
||||
pkgs = append(pkgs, *mainPkg)
|
||||
pkgs = append(pkgs, licenses.RelativeToPackage(ctx, fileResolver, *mainPkg))
|
||||
}
|
||||
|
||||
// catalog all dependencies
|
||||
@ -117,7 +118,7 @@ func newPackageFromMavenPom(ctx context.Context, r *maven.Resolver, pom *maven.P
|
||||
if err != nil {
|
||||
log.Tracef("error resolving licenses: %v", err)
|
||||
}
|
||||
licenses := toPkgLicenses(ctx, &location, pomLicenses)
|
||||
pkgLicenses := toPkgLicenses(ctx, &location, pomLicenses)
|
||||
|
||||
m := pkg.JavaArchive{
|
||||
PomProject: &pkg.JavaPomProject{
|
||||
@ -137,7 +138,7 @@ func newPackageFromMavenPom(ctx context.Context, r *maven.Resolver, pom *maven.P
|
||||
Locations: file.NewLocationSet(
|
||||
location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
),
|
||||
Licenses: pkg.NewLicenseSet(licenses...),
|
||||
Licenses: pkg.NewLicenseSet(pkgLicenses...),
|
||||
Language: pkg.Java,
|
||||
Type: pkg.JavaPkg,
|
||||
FoundBy: pomCatalogerName,
|
||||
@ -231,7 +232,7 @@ func newPackageFromDependency(ctx context.Context, r *maven.Resolver, pom *maven
|
||||
id := r.ResolveDependencyID(ctx, pom, dep)
|
||||
|
||||
var err error
|
||||
var licenses []pkg.License
|
||||
var pkgLicenses []pkg.License
|
||||
dependencyPom, depErr := r.FindPom(ctx, id.GroupID, id.ArtifactID, id.Version)
|
||||
if depErr != nil {
|
||||
err = errors.Join(err, depErr)
|
||||
@ -240,7 +241,7 @@ func newPackageFromDependency(ctx context.Context, r *maven.Resolver, pom *maven
|
||||
var pomProject *pkg.JavaPomProject
|
||||
if dependencyPom != nil {
|
||||
depLicenses, _ := r.ResolveLicenses(ctx, dependencyPom)
|
||||
licenses = append(licenses, toPkgLicenses(ctx, nil, depLicenses)...)
|
||||
pkgLicenses = append(pkgLicenses, toPkgLicenses(ctx, nil, depLicenses)...)
|
||||
pomProject = &pkg.JavaPomProject{
|
||||
Parent: pomParent(ctx, r, dependencyPom),
|
||||
GroupID: id.GroupID,
|
||||
@ -265,7 +266,7 @@ func newPackageFromDependency(ctx context.Context, r *maven.Resolver, pom *maven
|
||||
Name: id.ArtifactID,
|
||||
Version: id.Version,
|
||||
Locations: file.NewLocationSet(locations...),
|
||||
Licenses: pkg.NewLicenseSet(licenses...),
|
||||
Licenses: pkg.NewLicenseSet(pkgLicenses...),
|
||||
PURL: packageURL(id.ArtifactID, id.Version, m),
|
||||
Language: pkg.Java,
|
||||
Type: pkg.JavaPkg, // TODO: should we differentiate between packages from jar/war/zip versus packages from a pom.xml that were not installed yet?
|
||||
|
||||
@ -70,7 +70,7 @@ func (gtp genericTarWrappedJavaArchiveParser) parseTarWrappedJavaArchive(ctx con
|
||||
}
|
||||
|
||||
func discoverPkgsFromTar(ctx context.Context, location file.Location, archivePath, contentPath string, cfg ArchiveCatalogerConfig) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
openers, err := intFile.ExtractGlobsFromTarToUniqueTempFile(archivePath, contentPath, archiveFormatGlobs...)
|
||||
openers, err := intFile.ExtractGlobsFromTarToUniqueTempFile(ctx, archivePath, contentPath, archiveFormatGlobs...)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("unable to extract files from tar: %w", err)
|
||||
}
|
||||
|
||||
@ -41,7 +41,7 @@ func (gzp genericZipWrappedJavaArchiveParser) parseZipWrappedJavaArchive(ctx con
|
||||
// functions support zips with shell scripts prepended to the file. Specifically, the helpers use the central
|
||||
// header at the end of the file to determine where the beginning of the zip payload is (unlike the standard lib
|
||||
// or archiver).
|
||||
fileManifest, err := intFile.NewZipFileManifest(archivePath)
|
||||
fileManifest, err := intFile.NewZipFileManifest(ctx, archivePath)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("unable to read files from java archive: %w", err)
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user