mirror of
https://github.com/anchore/syft.git
synced 2026-10-12 06:07:20 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9872ff36ba | ||
|
|
31c503124f | ||
|
|
2c5e193f7a | ||
|
|
3a23cfff1d | ||
|
|
443de210ca | ||
|
|
1af8b1acaa | ||
|
|
c185657d71 | ||
|
|
48ee12be0c | ||
|
|
0b05f0ed69 | ||
|
|
138cb1be0e | ||
|
|
6755377554 | ||
|
|
540c08a41b |
+1
-1
@@ -2,7 +2,7 @@ tools:
|
||||
# we want to use a pinned version of binny to manage the toolchain (so binny manages itself!)
|
||||
- name: binny
|
||||
version:
|
||||
want: v0.11.1
|
||||
want: v0.11.2
|
||||
method: github-release
|
||||
with:
|
||||
repo: anchore/binny
|
||||
|
||||
@@ -37,7 +37,7 @@ runs:
|
||||
- name: Restore tool cache
|
||||
if: inputs.tools == 'true'
|
||||
id: tool-cache
|
||||
uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1
|
||||
uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
|
||||
with:
|
||||
path: ${{ github.workspace }}/.tool
|
||||
key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-tool-${{ hashFiles('.binny.yaml') }}
|
||||
@@ -66,7 +66,7 @@ runs:
|
||||
|
||||
- name: Restore ORAS cache from github actions
|
||||
if: inputs.download-test-fixture-cache == 'true'
|
||||
uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1
|
||||
uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
|
||||
with:
|
||||
path: ${{ github.workspace }}/.tmp/oras-cache
|
||||
key: ${{ inputs.cache-key-prefix }}-oras-cache
|
||||
|
||||
@@ -150,13 +150,13 @@ jobs:
|
||||
uses: ./.github/actions/bootstrap
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef #v3.6.0
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 #v3.7.0
|
||||
with:
|
||||
username: ${{ secrets.ANCHOREOSSWRITE_DH_USERNAME }}
|
||||
password: ${{ secrets.ANCHOREOSSWRITE_DH_PAT }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef #v3.6.0
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 #v3.7.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
|
||||
+2
-1
@@ -545,7 +545,8 @@ tasks:
|
||||
- "go generate ./internal/capabilities/..."
|
||||
- "gofmt -s -w ./internal/capabilities"
|
||||
# now that we have the latest capabilities, run completeness tests to ensure this is self-consistent
|
||||
- "SYFT_ENABLE_COMPLETENESS_TESTS=true go test ./internal/capabilities/... -count=1"
|
||||
# note: -p 1 forces sequential package execution to avoid race between generate (writes) and internal (reads)
|
||||
- "SYFT_ENABLE_COMPLETENESS_TESTS=true go test -p 1 ./internal/capabilities/... -count=1"
|
||||
|
||||
|
||||
## Build-related targets #################################
|
||||
|
||||
@@ -59,7 +59,9 @@ const (
|
||||
nonImageSchemeHelp = ` {{.appName}} {{.command}} dir:path/to/yourproject read directly from a path on disk (any directory)
|
||||
{{.appName}} {{.command}} file:path/to/yourproject/file read directly from a path on disk (any single file)
|
||||
`
|
||||
scanSchemeHelp = "\n " + schemeHelpHeader + "\n" + imageSchemeHelp + nonImageSchemeHelp
|
||||
modelSchemeHelp = ` {{.appName}} {{.command}} oci-model-registry:ai/llama3.2 scan an OCI model artifact from a registry (e.g. Docker Hub AI models)
|
||||
`
|
||||
scanSchemeHelp = "\n " + schemeHelpHeader + "\n" + imageSchemeHelp + modelSchemeHelp + nonImageSchemeHelp
|
||||
|
||||
scanHelp = scanExample + scanSchemeHelp
|
||||
)
|
||||
|
||||
@@ -35,7 +35,8 @@ func TestYarnPackageLockDirectory(t *testing.T) {
|
||||
sbom, _ := catalogDirectory(t, "test-fixtures/yarn-lock")
|
||||
|
||||
foundPackages := strset.New()
|
||||
expectedPackages := strset.New("async@0.9.2", "async@3.2.3", "merge-objects@1.0.5", "should-type@1.3.0", "@4lolo/resize-observer-polyfill@1.5.2")
|
||||
// merge-objects and should-type are devDependencies in package.json and are excluded by default
|
||||
expectedPackages := strset.New("async@0.9.2", "async@3.2.3", "@4lolo/resize-observer-polyfill@1.5.2")
|
||||
|
||||
for actualPkg := range sbom.Artifacts.Packages.Enumerate(pkg.NpmPkg) {
|
||||
for _, actualLocation := range actualPkg.Locations.ToSlice() {
|
||||
|
||||
@@ -23,12 +23,12 @@ require (
|
||||
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
|
||||
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
|
||||
github.com/anchore/packageurl-go v0.1.1-0.20250220190351-d62adb6e1115
|
||||
github.com/anchore/stereoscope v0.1.19
|
||||
github.com/anchore/stereoscope v0.1.20
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
|
||||
github.com/aquasecurity/go-pep440-version v0.0.1
|
||||
github.com/bitnami/go-version v0.0.0-20250131085805-b1f57a8634ef
|
||||
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb
|
||||
github.com/bmatcuk/doublestar/v4 v4.9.2
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.0
|
||||
github.com/charmbracelet/bubbles v0.21.0
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
github.com/charmbracelet/lipgloss v1.1.0
|
||||
@@ -42,7 +42,7 @@ require (
|
||||
github.com/github/go-spdx/v2 v2.3.6
|
||||
github.com/gkampitakis/go-snaps v0.5.19
|
||||
github.com/go-git/go-billy/v5 v5.7.0
|
||||
github.com/go-git/go-git/v5 v5.16.4
|
||||
github.com/go-git/go-git/v5 v5.16.5
|
||||
github.com/go-test/deep v1.1.1
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0
|
||||
github.com/gohugoio/hashstructure v0.6.0
|
||||
@@ -139,7 +139,7 @@ require (
|
||||
github.com/containerd/typeurl/v2 v2.2.3 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.6.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/docker/cli v29.1.5+incompatible // indirect
|
||||
github.com/docker/cli v29.2.0+incompatible // indirect
|
||||
github.com/docker/distribution v2.8.3+incompatible // indirect
|
||||
github.com/docker/docker v28.5.2+incompatible // indirect
|
||||
github.com/docker/docker-credential-helpers v0.9.4 // indirect
|
||||
|
||||
@@ -152,8 +152,8 @@ github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZV
|
||||
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
|
||||
github.com/anchore/packageurl-go v0.1.1-0.20250220190351-d62adb6e1115 h1:ZyRCmiEjnoGJZ1+Ah0ZZ/mKKqNhGcUZBl0s7PTTDzvY=
|
||||
github.com/anchore/packageurl-go v0.1.1-0.20250220190351-d62adb6e1115/go.mod h1:KoYIv7tdP5+CC9VGkeZV4/vGCKsY55VvoG+5dadg4YI=
|
||||
github.com/anchore/stereoscope v0.1.19 h1:1G5LVmRN1Sz6qNezpVAEeN7QfWwCE9zw9TJK1ZGnkvw=
|
||||
github.com/anchore/stereoscope v0.1.19/go.mod h1:+laNHlk05xA2YqgEzq8mxkFzclL3NRdeNIsiQQVeZZ4=
|
||||
github.com/anchore/stereoscope v0.1.20 h1:32720yZ/YtvzF5tvsoRL/ibdAJzOdIaR444fDXW4arQ=
|
||||
github.com/anchore/stereoscope v0.1.20/go.mod h1:6Ef0xQAuN2Ito7eV9A9pYjD1x/0cX5fy56MwgEGyrB4=
|
||||
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
|
||||
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
||||
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
@@ -231,8 +231,8 @@ github.com/bitnami/go-version v0.0.0-20250131085805-b1f57a8634ef h1:TSFnfbbu2oAO
|
||||
github.com/bitnami/go-version v0.0.0-20250131085805-b1f57a8634ef/go.mod h1:9iglf1GG4oNRJ39bZ5AZrjgAFD2RwQbXw6Qf7Cs47wo=
|
||||
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4=
|
||||
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
|
||||
github.com/bmatcuk/doublestar/v4 v4.9.2 h1:b0mc6WyRSYLjzofB2v/0cuDUZ+MqoGyH3r0dVij35GI=
|
||||
github.com/bmatcuk/doublestar/v4 v4.9.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs=
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
|
||||
github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU=
|
||||
github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs=
|
||||
github.com/bodgit/sevenzip v1.6.1 h1:kikg2pUMYC9ljU7W9SaqHXhym5HyKm8/M/jd31fYan4=
|
||||
@@ -351,8 +351,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
|
||||
github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
|
||||
github.com/docker/cli v29.1.5+incompatible h1:GckbANUt3j+lsnQ6eCcQd70mNSOismSHWt8vk2AX8ao=
|
||||
github.com/docker/cli v29.1.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
github.com/docker/cli v29.2.0+incompatible h1:9oBd9+YM7rxjZLfyMGxjraKBKE4/nVyvVfN4qNl9XRM=
|
||||
github.com/docker/cli v29.2.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
|
||||
github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
|
||||
github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
|
||||
@@ -436,8 +436,8 @@ github.com/go-git/go-billy/v5 v5.7.0 h1:83lBUJhGWhYp0ngzCMSgllhUSuoHP1iEWYjsPl9n
|
||||
github.com/go-git/go-billy/v5 v5.7.0/go.mod h1:/1IUejTKH8xipsAcdfcSAlUlo2J7lkYV8GTKxAT/L3E=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
|
||||
github.com/go-git/go-git/v5 v5.16.4 h1:7ajIEZHZJULcyJebDLo99bGgS0jRrOxzZG4uCk2Yb2Y=
|
||||
github.com/go-git/go-git/v5 v5.16.4/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8=
|
||||
github.com/go-git/go-git/v5 v5.16.5 h1:mdkuqblwr57kVfXri5TTH+nMFLNUxIj9Z7F5ykFbw5s=
|
||||
github.com/go-git/go-git/v5 v5.16.5/go.mod h1:QOMLpNf1qxuSY4StA/ArOdfFR2TrKEjJiye2kel2m+M=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
|
||||
@@ -3,10 +3,12 @@ package internal
|
||||
const (
|
||||
// JSONSchemaVersion is the current schema version output by the JSON encoder
|
||||
// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
|
||||
JSONSchemaVersion = "16.1.2"
|
||||
JSONSchemaVersion = "16.1.3"
|
||||
|
||||
// Changelog
|
||||
// 16.1.0 - reformulated the python pdm fields (added "URL" and removed the unused "path" field).
|
||||
// 16.1.1 - correct elf package osCpe field according to the document of systemd (also add appCpe field)
|
||||
// 16.1.2 - placeholder for 16.1.2 changelog
|
||||
// 16.1.3 - add GGUFFileParts to GGUFFileHeader metadata
|
||||
|
||||
)
|
||||
|
||||
@@ -82,12 +82,29 @@ func assembleTypeContainer(items []any) (any, map[string]string) {
|
||||
return reflect.New(structType).Elem().Interface(), mapping
|
||||
}
|
||||
|
||||
//nolint:funlen
|
||||
func build() *jsonschema.Schema {
|
||||
// create metadata mapping first so we can use it in the Namer function for self-referential types
|
||||
pkgMetadataContainer, pkgMetadataMapping := assembleTypeContainer(packagemetadata.AllTypes())
|
||||
pkgMetadataContainerType := reflect.TypeOf(pkgMetadataContainer)
|
||||
|
||||
// create a set of valid metadata display names for lookup
|
||||
// (since Namer now returns display names, the schema definitions use display names as keys)
|
||||
pkgMetadataDisplayNames := make(map[string]struct{}, len(pkgMetadataMapping))
|
||||
for _, displayName := range pkgMetadataMapping {
|
||||
pkgMetadataDisplayNames[displayName] = struct{}{}
|
||||
}
|
||||
|
||||
reflector := &jsonschema.Reflector{
|
||||
BaseSchemaID: schemaID(),
|
||||
AllowAdditionalProperties: true,
|
||||
Namer: func(r reflect.Type) string {
|
||||
return strings.TrimPrefix(r.Name(), "JSON")
|
||||
name := strings.TrimPrefix(r.Name(), "JSON")
|
||||
// if this is a metadata type, use the mapped name for consistent references
|
||||
if mappedName, ok := pkgMetadataMapping[name]; ok {
|
||||
return mappedName
|
||||
}
|
||||
return name
|
||||
},
|
||||
CommentMap: make(map[string]string),
|
||||
}
|
||||
@@ -123,9 +140,6 @@ func build() *jsonschema.Schema {
|
||||
copyAliasFieldComments(reflector.CommentMap, repoRoot)
|
||||
}
|
||||
|
||||
pkgMetadataContainer, pkgMetadataMapping := assembleTypeContainer(packagemetadata.AllTypes())
|
||||
pkgMetadataContainerType := reflect.TypeOf(pkgMetadataContainer)
|
||||
|
||||
// srcMetadataContainer := assembleTypeContainer(sourcemetadata.AllTypes())
|
||||
// srcMetadataContainerType := reflect.TypeOf(srcMetadataContainer)
|
||||
|
||||
@@ -144,11 +158,10 @@ func build() *jsonschema.Schema {
|
||||
continue
|
||||
}
|
||||
|
||||
displayName, ok := pkgMetadataMapping[typeName]
|
||||
if ok {
|
||||
// this is a package metadata type...
|
||||
documentSchema.Definitions[displayName] = definition
|
||||
metadataNames = append(metadataNames, displayName)
|
||||
if _, ok := pkgMetadataDisplayNames[typeName]; ok {
|
||||
// this is a package metadata type (typeName is already the display name from Namer)
|
||||
documentSchema.Definitions[typeName] = definition
|
||||
metadataNames = append(metadataNames, typeName)
|
||||
} else {
|
||||
// this is a type that the metadata type uses (e.g. DpkgFileRecord)
|
||||
documentSchema.Definitions[typeName] = definition
|
||||
|
||||
@@ -26,6 +26,7 @@ var knownNonMetadataTypeNames = strset.New(
|
||||
// known to be metadata types themselves. Adding to this list will prevent the removal of the type from the schema.
|
||||
var knownMetadataTypeNames = strset.New(
|
||||
"DotnetPortableExecutableEntry",
|
||||
"GGUFFileHeader",
|
||||
)
|
||||
|
||||
func DiscoverTypeNames() ([]string, error) {
|
||||
|
||||
@@ -94,25 +94,33 @@ func findMetadataDefinitionNamesInFile(path string) ([]string, []string, error)
|
||||
|
||||
// loop over all types declared in the type declaration
|
||||
for _, typ := range spec.Specs {
|
||||
// check if the type is a struct type
|
||||
spec, ok := typ.(*ast.TypeSpec)
|
||||
if !ok || spec.Type == nil {
|
||||
typeSpec, ok := typ.(*ast.TypeSpec)
|
||||
if !ok || typeSpec.Type == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
structType, ok := spec.Type.(*ast.StructType)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
// check if the struct type ends with "Metadata"
|
||||
name := spec.Name.String()
|
||||
name := typeSpec.Name.String()
|
||||
|
||||
// only look for exported types that end with "Metadata"
|
||||
if isMetadataTypeCandidate(name) {
|
||||
// print the full declaration of the struct type
|
||||
metadataDefinitions = append(metadataDefinitions, name)
|
||||
if !isMetadataTypeCandidate(name) {
|
||||
continue
|
||||
}
|
||||
|
||||
metadataDefinitions = append(metadataDefinitions, name)
|
||||
|
||||
// handle struct types (e.g., "type FooMetadata struct {...}")
|
||||
if structType, ok := typeSpec.Type.(*ast.StructType); ok {
|
||||
usedTypeNames = append(usedTypeNames, typeNamesUsedInStruct(structType)...)
|
||||
continue
|
||||
}
|
||||
|
||||
// handle type definitions from another type (e.g., "type FooMetadata BarMetadata")
|
||||
// if the base type is NOT a metadata candidate, track it as used
|
||||
// (e.g., we want both ImageMetadata and OCIModelMetadata which is an alias to it)
|
||||
if ident, ok := typeSpec.Type.(*ast.Ident); ok {
|
||||
if !isMetadataTypeCandidate(ident.Name) {
|
||||
usedTypeNames = append(usedTypeNames, ident.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,5 +6,5 @@ import "github.com/anchore/syft/syft/source"
|
||||
|
||||
// AllTypes returns a list of all source metadata types that syft supports (that are represented in the source.Description.Metadata field).
|
||||
func AllTypes() []any {
|
||||
return []any{source.DirectoryMetadata{}, source.FileMetadata{}, source.ImageMetadata{}, source.SnapMetadata{}}
|
||||
return []any{source.DirectoryMetadata{}, source.FileMetadata{}, source.ImageMetadata{}, source.OCIModelMetadata{}, source.SnapMetadata{}}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ var jsonNameFromType = map[reflect.Type][]string{
|
||||
reflect.TypeOf(source.FileMetadata{}): {"file"},
|
||||
reflect.TypeOf(source.ImageMetadata{}): {"image"},
|
||||
reflect.TypeOf(source.SnapMetadata{}): {"snap"},
|
||||
reflect.TypeOf(source.OCIModelMetadata{}): {"oci-model"},
|
||||
}
|
||||
|
||||
func AllTypeNames() []string {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "anchore.io/schema/syft/json/16.1.2/document",
|
||||
"$id": "anchore.io/schema/syft/json/16.1.3/document",
|
||||
"$ref": "#/$defs/Document",
|
||||
"$defs": {
|
||||
"AlpmDbEntry": {
|
||||
@@ -1478,6 +1478,13 @@
|
||||
"metadataHash": {
|
||||
"type": "string",
|
||||
"description": "MetadataKeyValuesHash is a xx64 hash of all key-value pairs from the GGUF header metadata.\nThis hash is computed over the complete header metadata (including the fields extracted\ninto typed fields above) and provides a stable identifier for the model configuration\nacross different file locations or remotes. It allows matching identical models even\nwhen stored in different repositories or with different filenames."
|
||||
},
|
||||
"parts": {
|
||||
"items": {
|
||||
"$ref": "#/$defs/GgufFileHeader"
|
||||
},
|
||||
"type": "array",
|
||||
"description": "Parts contains headers from additional GGUF files that were merged\ninto this package during post-processing (e.g., from OCI layers without model names)."
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
|
||||
@@ -477,7 +477,7 @@ func (c *CreateSBOMConfig) Create(ctx context.Context, src source.Source) (*sbom
|
||||
|
||||
func findDefaultTags(src source.Description) ([]string, error) {
|
||||
switch m := src.Metadata.(type) {
|
||||
case source.ImageMetadata:
|
||||
case source.ImageMetadata, source.OCIModelMetadata:
|
||||
return []string{pkgcataloging.ImageTag, filecataloging.FileTag}, nil
|
||||
case source.FileMetadata, source.DirectoryMetadata:
|
||||
return []string{pkgcataloging.DirectoryTag, filecataloging.FileTag}, nil
|
||||
|
||||
@@ -13,16 +13,18 @@ import (
|
||||
)
|
||||
|
||||
var _ Resolver = (*MockResolver)(nil)
|
||||
var _ OCIMediaTypeResolver = (*MockResolver)(nil)
|
||||
|
||||
// MockResolver implements the FileResolver interface and is intended for use *only in test code*.
|
||||
// It provides an implementation that can resolve local filesystem paths using only a provided discrete list of file
|
||||
// paths, which are typically paths to test fixtures.
|
||||
type MockResolver struct {
|
||||
locations []Location
|
||||
metadata map[Coordinates]Metadata
|
||||
mimeTypeIndex map[string][]Location
|
||||
extension map[string][]Location
|
||||
basename map[string][]Location
|
||||
locations []Location
|
||||
metadata map[Coordinates]Metadata
|
||||
mimeTypeIndex map[string][]Location
|
||||
mediaTypeIndex map[string][]Location
|
||||
extension map[string][]Location
|
||||
basename map[string][]Location
|
||||
}
|
||||
|
||||
// NewMockResolverForPaths creates a new MockResolver, where the only resolvable
|
||||
@@ -72,6 +74,34 @@ func NewMockResolverForPathsWithMetadata(metadata map[Coordinates]Metadata) *Moc
|
||||
}
|
||||
}
|
||||
|
||||
// NewMockResolverForMediaTypes creates a MockResolver that can resolve files by media type.
|
||||
// The mediaTypes map specifies which locations should be returned for each media type.
|
||||
func NewMockResolverForMediaTypes(mediaTypes map[string][]Location) *MockResolver {
|
||||
var locations []Location
|
||||
mediaTypeIndex := make(map[string][]Location)
|
||||
extension := make(map[string][]Location)
|
||||
basename := make(map[string][]Location)
|
||||
|
||||
for mediaType, locs := range mediaTypes {
|
||||
mediaTypeIndex[mediaType] = append(mediaTypeIndex[mediaType], locs...)
|
||||
for _, l := range locs {
|
||||
locations = append(locations, l)
|
||||
ext := path.Ext(l.RealPath)
|
||||
extension[ext] = append(extension[ext], l)
|
||||
bn := path.Base(l.RealPath)
|
||||
basename[bn] = append(basename[bn], l)
|
||||
}
|
||||
}
|
||||
|
||||
return &MockResolver{
|
||||
locations: locations,
|
||||
metadata: make(map[Coordinates]Metadata),
|
||||
mediaTypeIndex: mediaTypeIndex,
|
||||
extension: extension,
|
||||
basename: basename,
|
||||
}
|
||||
}
|
||||
|
||||
// HasPath indicates if the given path exists in the underlying source.
|
||||
func (r MockResolver) HasPath(path string) bool {
|
||||
for _, l := range r.locations {
|
||||
@@ -189,6 +219,14 @@ func (r MockResolver) FilesByMIMEType(types ...string) ([]Location, error) {
|
||||
return locations, nil
|
||||
}
|
||||
|
||||
func (r MockResolver) FilesByMediaType(types ...string) ([]Location, error) {
|
||||
var locations []Location
|
||||
for _, ty := range types {
|
||||
locations = append(locations, r.mediaTypeIndex[ty]...)
|
||||
}
|
||||
return locations, nil
|
||||
}
|
||||
|
||||
func (r MockResolver) FilesByExtension(extensions ...string) ([]Location, error) {
|
||||
var results []Location
|
||||
for _, ext := range extensions {
|
||||
|
||||
@@ -52,6 +52,17 @@ type PathResolver interface {
|
||||
RelativeFileByPath(_ Location, path string) *Location
|
||||
}
|
||||
|
||||
// OCIMediaTypeResolver resolves single files as a layer in an OCI artifact for a given media type.
|
||||
type OCIMediaTypeResolver interface {
|
||||
// FilesByMediaType fetches a set of file references which the contents have been classified as one of the given Media Types.
|
||||
// The implementation for this may vary, however, this was first implemented to classify ai globs stored in OCI images.
|
||||
// The following considerations should be made when implementing:
|
||||
// - only return locations to files (NOT directories)
|
||||
// - locations for the implementer should be "/" and the fsid should be the layer digest the glob was found
|
||||
// - locations should be used with the FileContents API to return readers to the temporary data
|
||||
FilesByMediaType(types ...string) ([]Location, error)
|
||||
}
|
||||
|
||||
// LocationResolver provides iteration over all file locations in a source.
|
||||
type LocationResolver interface {
|
||||
// AllLocations returns a channel of all file references from the underlying source.
|
||||
|
||||
@@ -35,6 +35,7 @@ const (
|
||||
spdxPrimaryPurposeOther = "OTHER"
|
||||
|
||||
prefixImage = "Image"
|
||||
prefixOCIModel = "OCIModel"
|
||||
prefixDirectory = "Directory"
|
||||
prefixFile = "File"
|
||||
prefixSnap = "Snap"
|
||||
@@ -215,6 +216,36 @@ func toRootPackage(s source.Description) *spdx.Package {
|
||||
}
|
||||
}
|
||||
|
||||
case source.OCIModelMetadata:
|
||||
prefix = prefixOCIModel
|
||||
purpose = spdxPrimaryPurposeContainer
|
||||
|
||||
qualifiers := packageurl.Qualifiers{
|
||||
{
|
||||
Key: "arch",
|
||||
Value: m.Architecture,
|
||||
},
|
||||
}
|
||||
|
||||
ref, _ := reference.Parse(m.UserInput)
|
||||
if ref, ok := ref.(reference.NamedTagged); ok {
|
||||
qualifiers = append(qualifiers, packageurl.Qualifier{
|
||||
Key: "tag",
|
||||
Value: ref.Tag(),
|
||||
})
|
||||
}
|
||||
|
||||
c := toChecksum(m.ManifestDigest)
|
||||
if c != nil {
|
||||
checksums = append(checksums, *c)
|
||||
purl = &packageurl.PackageURL{
|
||||
Type: "oci",
|
||||
Name: s.Name,
|
||||
Version: m.ManifestDigest,
|
||||
Qualifiers: qualifiers,
|
||||
}
|
||||
}
|
||||
|
||||
case source.DirectoryMetadata:
|
||||
prefix = prefixDirectory
|
||||
purpose = spdxPrimaryPurposeFile
|
||||
|
||||
@@ -316,6 +316,81 @@ func Test_toFormatModel(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "oci-model",
|
||||
in: sbom.SBOM{
|
||||
Source: source.Description{
|
||||
Name: "llama",
|
||||
Version: "sha256:d34db33f",
|
||||
Supplier: "Model Provider",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "model-repo/llama:latest",
|
||||
ManifestDigest: "sha256:d34db33f",
|
||||
},
|
||||
},
|
||||
Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(pkg.Package{
|
||||
Name: "pkg-1",
|
||||
Version: "version-1",
|
||||
}),
|
||||
},
|
||||
},
|
||||
expected: &spdx.Document{
|
||||
SPDXIdentifier: "DOCUMENT",
|
||||
SPDXVersion: spdx.Version,
|
||||
DataLicense: spdx.DataLicense,
|
||||
DocumentName: "llama",
|
||||
Packages: []*spdx.Package{
|
||||
{
|
||||
PackageSPDXIdentifier: "Package-pkg-1-pkg-1",
|
||||
PackageName: "pkg-1",
|
||||
PackageVersion: "version-1",
|
||||
PackageSupplier: &spdx.Supplier{
|
||||
Supplier: "Model Provider",
|
||||
SupplierType: "Organization",
|
||||
},
|
||||
},
|
||||
{
|
||||
PackageSPDXIdentifier: "DocumentRoot-OCIModel-llama",
|
||||
PackageName: "llama",
|
||||
PackageVersion: "sha256:d34db33f",
|
||||
PrimaryPackagePurpose: "CONTAINER",
|
||||
PackageChecksums: []spdx.Checksum{{Algorithm: "SHA256", Value: "d34db33f"}},
|
||||
PackageExternalReferences: []*v2_3.PackageExternalReference{
|
||||
{
|
||||
Category: "PACKAGE-MANAGER",
|
||||
RefType: "purl",
|
||||
Locator: "pkg:oci/llama@sha256%3Ad34db33f?arch=&tag=latest",
|
||||
},
|
||||
},
|
||||
PackageSupplier: &spdx.Supplier{
|
||||
Supplier: "Model Provider",
|
||||
SupplierType: "Organization",
|
||||
},
|
||||
},
|
||||
},
|
||||
Relationships: []*spdx.Relationship{
|
||||
{
|
||||
RefA: spdx.DocElementID{
|
||||
ElementRefID: "DocumentRoot-OCIModel-llama",
|
||||
},
|
||||
RefB: spdx.DocElementID{
|
||||
ElementRefID: "Package-pkg-1-pkg-1",
|
||||
},
|
||||
Relationship: spdx.RelationshipContains,
|
||||
},
|
||||
{
|
||||
RefA: spdx.DocElementID{
|
||||
ElementRefID: "DOCUMENT",
|
||||
},
|
||||
RefB: spdx.DocElementID{
|
||||
ElementRefID: "DocumentRoot-OCIModel-llama",
|
||||
},
|
||||
Relationship: spdx.RelationshipDescribes,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
|
||||
@@ -122,6 +122,9 @@ func toPath(s source.Description, p pkg.Package) string {
|
||||
case source.ImageMetadata:
|
||||
image := strings.ReplaceAll(metadata.UserInput, ":/", "//")
|
||||
return fmt.Sprintf("%s:/%s", image, packagePath)
|
||||
case source.OCIModelMetadata:
|
||||
image := strings.ReplaceAll(metadata.UserInput, ":/", "//")
|
||||
return fmt.Sprintf("%s:/%s", image, packagePath)
|
||||
case source.FileMetadata:
|
||||
path := trimRelative(metadata.Path)
|
||||
if isArchive(metadata.Path) {
|
||||
|
||||
@@ -178,6 +178,11 @@ func Test_toGithubModel(t *testing.T) {
|
||||
metadata: source.SnapMetadata{},
|
||||
testPath: "name:/etc",
|
||||
},
|
||||
{
|
||||
name: "oci-model",
|
||||
metadata: source.OCIModelMetadata{UserInput: "model-repo/llama:latest"},
|
||||
testPath: "model-repo/llama:latest:/etc",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
|
||||
@@ -12,6 +12,8 @@ func DocumentName(src source.Description) string {
|
||||
switch metadata := src.Metadata.(type) {
|
||||
case source.ImageMetadata:
|
||||
return metadata.UserInput
|
||||
case source.OCIModelMetadata:
|
||||
return metadata.UserInput
|
||||
case source.DirectoryMetadata:
|
||||
return metadata.Path
|
||||
case source.FileMetadata:
|
||||
|
||||
@@ -54,6 +54,17 @@ func Test_DocumentName(t *testing.T) {
|
||||
},
|
||||
expected: "some/name",
|
||||
},
|
||||
{
|
||||
name: "oci-model",
|
||||
srcMetadata: source.Description{
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "model-repo/name:tag",
|
||||
ID: "id",
|
||||
ManifestDigest: "digest",
|
||||
},
|
||||
},
|
||||
expected: "model-repo/name:tag",
|
||||
},
|
||||
{
|
||||
name: "named",
|
||||
srcMetadata: source.Description{
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
const (
|
||||
InputImage = "image"
|
||||
InputOCIModel = "oci-model"
|
||||
InputDirectory = "dir"
|
||||
InputFile = "file"
|
||||
InputSnap = "snap"
|
||||
@@ -30,6 +31,8 @@ func DocumentNamespace(name string, src source.Description, desc sbom.Descriptor
|
||||
switch src.Metadata.(type) {
|
||||
case source.ImageMetadata:
|
||||
input = InputImage
|
||||
case source.OCIModelMetadata:
|
||||
input = InputOCIModel
|
||||
case source.DirectoryMetadata:
|
||||
input = InputDirectory
|
||||
case source.FileMetadata:
|
||||
|
||||
@@ -61,6 +61,18 @@ func Test_DocumentNamespace(t *testing.T) {
|
||||
},
|
||||
expected: "https://anchore.com/syft/snap/my-name-",
|
||||
},
|
||||
{
|
||||
name: "oci-model",
|
||||
inputName: "my-name",
|
||||
src: source.Description{
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "model-repo/name:tag",
|
||||
ID: "id",
|
||||
ManifestDigest: "digest",
|
||||
},
|
||||
},
|
||||
expected: "https://anchore.com/syft/oci-model/my-name-",
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
|
||||
@@ -83,7 +83,7 @@ func SourceInfo(p pkg.Package) string {
|
||||
case pkg.TerraformPkg:
|
||||
answer = "acquired package info from Terraform dependency lock file"
|
||||
case pkg.ModelPkg:
|
||||
answer = "acquired package info from AI artifact (e.g. GGUF File"
|
||||
answer = "acquired package info from AI artifact (e.g. GGUF File)"
|
||||
default:
|
||||
answer = "acquired package info from the following paths"
|
||||
}
|
||||
|
||||
@@ -190,6 +190,37 @@ func TestSource_UnmarshalJSON(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "oci-model",
|
||||
input: []byte(`{
|
||||
"id": "foobar",
|
||||
"type": "oci-model",
|
||||
"metadata": {
|
||||
"userInput": "model-repo/llama:latest",
|
||||
"imageID": "sha256:e7b300aee9f9bf3433d32bc9305bfdd22183beb59d933b48d77ab56ba53a197a",
|
||||
"manifestDigest": "sha256:e515aad2ed234a5072c4d2ef86a1cb77d5bfe4b11aa865d9214875734c4eeb3c",
|
||||
"mediaType": "application/vnd.oci.image.manifest.v1+json",
|
||||
"tags": [],
|
||||
"imageSize": 5576169,
|
||||
"layers": [],
|
||||
"repoDigests": []
|
||||
}
|
||||
}`),
|
||||
expected: &Source{
|
||||
ID: "foobar",
|
||||
Type: "oci-model",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "model-repo/llama:latest",
|
||||
ID: "sha256:e7b300aee9f9bf3433d32bc9305bfdd22183beb59d933b48d77ab56ba53a197a",
|
||||
ManifestDigest: "sha256:e515aad2ed234a5072c4d2ef86a1cb77d5bfe4b11aa865d9214875734c4eeb3c",
|
||||
MediaType: "application/vnd.oci.image.manifest.v1+json",
|
||||
Tags: []string{},
|
||||
Size: 5576169,
|
||||
Layers: []source.LayerMetadata{},
|
||||
RepoDigests: []string{},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "unknown source type",
|
||||
input: []byte(`{
|
||||
|
||||
@@ -325,7 +325,17 @@ func toSourceModel(src source.Description) model.Source {
|
||||
Metadata: src.Metadata,
|
||||
}
|
||||
|
||||
if metadata, ok := src.Metadata.(source.ImageMetadata); ok {
|
||||
switch metadata := src.Metadata.(type) {
|
||||
case source.ImageMetadata:
|
||||
// ensure that empty collections are not shown as null
|
||||
if metadata.RepoDigests == nil {
|
||||
metadata.RepoDigests = []string{}
|
||||
}
|
||||
if metadata.Tags == nil {
|
||||
metadata.Tags = []string{}
|
||||
}
|
||||
m.Metadata = metadata
|
||||
case source.OCIModelMetadata:
|
||||
// ensure that empty collections are not shown as null
|
||||
if metadata.RepoDigests == nil {
|
||||
metadata.RepoDigests = []string{}
|
||||
|
||||
@@ -161,6 +161,34 @@ func Test_toSourceModel(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "oci-model",
|
||||
src: source.Description{
|
||||
ID: "test-id",
|
||||
Name: "some-name",
|
||||
Version: "some-version",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "user-input",
|
||||
ID: "id...",
|
||||
ManifestDigest: "digest...",
|
||||
MediaType: "type...",
|
||||
},
|
||||
},
|
||||
expected: model.Source{
|
||||
ID: "test-id",
|
||||
Name: "some-name",
|
||||
Version: "some-version",
|
||||
Type: "oci-model",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "user-input",
|
||||
ID: "id...",
|
||||
ManifestDigest: "digest...",
|
||||
MediaType: "type...",
|
||||
RepoDigests: []string{},
|
||||
Tags: []string{},
|
||||
},
|
||||
},
|
||||
},
|
||||
// below are regression tests for when the name/version are not provided
|
||||
// historically we've hoisted up the name/version from the metadata, now it is a simple pass-through
|
||||
{
|
||||
@@ -225,6 +253,30 @@ func Test_toSourceModel(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "oci-model - no name/version",
|
||||
src: source.Description{
|
||||
ID: "test-id",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "user-input",
|
||||
ID: "id...",
|
||||
ManifestDigest: "digest...",
|
||||
MediaType: "type...",
|
||||
},
|
||||
},
|
||||
expected: model.Source{
|
||||
ID: "test-id",
|
||||
Type: "oci-model",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "user-input",
|
||||
ID: "id...",
|
||||
ManifestDigest: "digest...",
|
||||
MediaType: "type...",
|
||||
RepoDigests: []string{},
|
||||
Tags: []string{},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
|
||||
@@ -130,6 +130,32 @@ func Test_toSyftSourceData(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "oci-model",
|
||||
src: model.Source{
|
||||
ID: "the-id",
|
||||
Name: "some-name",
|
||||
Version: "some-version",
|
||||
Type: "oci-model",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "user-input",
|
||||
ID: "id...",
|
||||
ManifestDigest: "digest...",
|
||||
MediaType: "type...",
|
||||
},
|
||||
},
|
||||
expected: &source.Description{
|
||||
ID: "the-id",
|
||||
Name: "some-name",
|
||||
Version: "some-version",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "user-input",
|
||||
ID: "id...",
|
||||
ManifestDigest: "digest...",
|
||||
MediaType: "type...",
|
||||
},
|
||||
},
|
||||
},
|
||||
// below are regression tests for when the name/version are not provided
|
||||
// historically we've hoisted up the name/version from the metadata, now it is a simple pass-through
|
||||
{
|
||||
@@ -192,6 +218,28 @@ func Test_toSyftSourceData(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "oci-model - no name/version",
|
||||
src: model.Source{
|
||||
ID: "the-id",
|
||||
Type: "oci-model",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "user-input",
|
||||
ID: "id...",
|
||||
ManifestDigest: "digest...",
|
||||
MediaType: "type...",
|
||||
},
|
||||
},
|
||||
expected: &source.Description{
|
||||
ID: "the-id",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "user-input",
|
||||
ID: "id...",
|
||||
ManifestDigest: "digest...",
|
||||
MediaType: "type...",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
|
||||
@@ -65,6 +65,8 @@ func validateSourcePlatform(src source.Source, cfg *GetSourceConfig) error {
|
||||
switch meta.(type) {
|
||||
case *source.ImageMetadata, source.ImageMetadata:
|
||||
return nil
|
||||
case *source.OCIModelMetadata, source.OCIModelMetadata:
|
||||
return nil
|
||||
case *source.SnapMetadata, source.SnapMetadata:
|
||||
return nil
|
||||
default:
|
||||
|
||||
@@ -31,7 +31,8 @@ func TestGetProviders_Sources(t *testing.T) {
|
||||
t.Errorf("Expected no error for Sources parameter, got: %v", err)
|
||||
}
|
||||
|
||||
if len(providers) != 1 {
|
||||
t.Errorf("Expected 1 providers, got %d", len(providers))
|
||||
// Registry tag has two providers: OCIModel and Image
|
||||
if len(providers) != 2 {
|
||||
t.Errorf("Expected 2 providers, got %d", len(providers))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,6 +111,10 @@ func TestValidateSourcePlatform_SupportedMetadataTypes(t *testing.T) {
|
||||
metadata: source.FileMetadata{},
|
||||
wantErr: require.Error,
|
||||
},
|
||||
{
|
||||
name: "oci-model",
|
||||
metadata: source.OCIModelMetadata{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
package fileresolver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
var _ file.Resolver = (*ContainerImageModel)(nil)
|
||||
var _ file.OCIMediaTypeResolver = (*ContainerImageModel)(nil)
|
||||
|
||||
// LayerInfo holds information about an OCI model layer file stored on disk.
|
||||
type LayerInfo struct {
|
||||
TempPath string // Path to the temp file on disk
|
||||
MediaType string // OCI media type of the layer
|
||||
}
|
||||
|
||||
// ContainerImageModel is a file.Resolver implementation that provides access to
|
||||
// GGUF header data fetched from OCI model artifacts via range-GET requests.
|
||||
// This does not fetch the entire model from the registry, only a sliver of it.
|
||||
type ContainerImageModel struct {
|
||||
tempDir string // temp directory containing all layer files
|
||||
layerFiles map[string]LayerInfo // digest -> layer info (temp path + media type)
|
||||
locations map[string]file.Location // digest -> location
|
||||
}
|
||||
|
||||
// NewContainerImageModel creates a new resolver with the given temp directory and layer files.
|
||||
func NewContainerImageModel(tempDir string, layerFiles map[string]LayerInfo) *ContainerImageModel {
|
||||
// Create locations for all layer files
|
||||
// Each location has RealPath="/", FileSystemID=digest, AccessPath="/"
|
||||
locations := make(map[string]file.Location, len(layerFiles))
|
||||
for digest := range layerFiles {
|
||||
// Use NewVirtualLocationFromCoordinates with digest as FileSystemID
|
||||
coords := file.NewCoordinates("/", digest)
|
||||
locations[digest] = file.NewVirtualLocationFromCoordinates(coords, "/")
|
||||
}
|
||||
|
||||
return &ContainerImageModel{
|
||||
tempDir: tempDir,
|
||||
layerFiles: layerFiles,
|
||||
locations: locations,
|
||||
}
|
||||
}
|
||||
|
||||
// FilesByMediaType returns locations for layers matching the given media type patterns.
|
||||
// Patterns support glob-style matching (e.g., "application/vnd.docker.ai*").
|
||||
func (r *ContainerImageModel) FilesByMediaType(types ...string) ([]file.Location, error) {
|
||||
var matches []file.Location
|
||||
|
||||
for digest, info := range r.layerFiles {
|
||||
for _, pattern := range types {
|
||||
matched, err := filepath.Match(pattern, info.MediaType)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid media type pattern %q: %w", pattern, err)
|
||||
}
|
||||
if matched {
|
||||
if loc, ok := r.locations[digest]; ok {
|
||||
matches = append(matches, loc)
|
||||
}
|
||||
break // Don't add the same location twice
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return matches, nil
|
||||
}
|
||||
|
||||
// FileContentsByLocation returns the contents of the file at the given location.
|
||||
// The location's FileSystemID contains the layer digest, which is used to look up the temp file.
|
||||
// This method is used as part of the content selection in the generic cataloger when locations
|
||||
// are returned by searching for contents by media type.
|
||||
func (r *ContainerImageModel) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
|
||||
// Look up the temp file path using the digest stored in FileSystemID
|
||||
digest := location.FileSystemID
|
||||
info, ok := r.layerFiles[digest]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no file found for digest %q", digest)
|
||||
}
|
||||
return os.Open(info.TempPath)
|
||||
}
|
||||
|
||||
// FileMetadataByLocation returns metadata for the file at the given location.
|
||||
func (r *ContainerImageModel) FileMetadataByLocation(_ file.Location) (m file.Metadata, err error) {
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// HasPath checks if the given path exists in the resolver.
|
||||
func (r *ContainerImageModel) HasPath(path string) bool {
|
||||
// The virtual path is "/" for all files
|
||||
if path == "/" && len(r.layerFiles) > 0 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FilesByPath returns locations for files matching the given paths.
|
||||
func (r *ContainerImageModel) FilesByPath(_ ...string) ([]file.Location, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// FilesByGlob returns locations for files matching the given glob patterns.
|
||||
func (r *ContainerImageModel) FilesByGlob(_ ...string) ([]file.Location, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// FilesByMIMEType returns locations for files with the given MIME types.
|
||||
// This is not implemented for OCI model artifacts as we don't have MIME type detection.
|
||||
func (r *ContainerImageModel) FilesByMIMEType(_ ...string) ([]file.Location, error) {
|
||||
// Not implemented - OCI model artifacts don't have MIME type detection
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// RelativeFileByPath returns a file at the given path relative to the reference location.
|
||||
// This is not applicable for OCI model artifacts.
|
||||
func (r *ContainerImageModel) RelativeFileByPath(_ file.Location, _ string) *file.Location {
|
||||
// Not implemented - no layer hierarchy in OCI model artifacts
|
||||
return nil
|
||||
}
|
||||
|
||||
// AllLocations returns all file locations in the resolver.
|
||||
func (r *ContainerImageModel) AllLocations(ctx context.Context) <-chan file.Location {
|
||||
ch := make(chan file.Location)
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
for _, loc := range r.locations {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case ch <- loc:
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package fileresolver
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
const ggufLayerMediaType = "application/vnd.docker.ai.gguf.v3"
|
||||
|
||||
func TestOCIModelResolver_FilesByMediaType(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
layerFiles map[string]LayerInfo
|
||||
patterns []string
|
||||
expected int
|
||||
}{
|
||||
{
|
||||
name: "exact match GGUF",
|
||||
layerFiles: map[string]LayerInfo{
|
||||
"sha256:abc123": {TempPath: filepath.Join(tempDir, "f1"), MediaType: ggufLayerMediaType},
|
||||
},
|
||||
patterns: []string{ggufLayerMediaType},
|
||||
expected: 1,
|
||||
},
|
||||
{
|
||||
name: "glob match docker ai",
|
||||
layerFiles: map[string]LayerInfo{
|
||||
"sha256:abc123": {TempPath: filepath.Join(tempDir, "f1"), MediaType: ggufLayerMediaType},
|
||||
},
|
||||
patterns: []string{"application/vnd.docker.ai*"},
|
||||
expected: 1,
|
||||
},
|
||||
{
|
||||
name: "no match",
|
||||
layerFiles: map[string]LayerInfo{
|
||||
"sha256:abc123": {TempPath: filepath.Join(tempDir, "f1"), MediaType: ggufLayerMediaType},
|
||||
},
|
||||
patterns: []string{"application/json"},
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "multiple patterns match multiple files",
|
||||
layerFiles: map[string]LayerInfo{
|
||||
"sha256:abc123": {TempPath: filepath.Join(tempDir, "f1"), MediaType: ggufLayerMediaType},
|
||||
"sha256:def456": {TempPath: filepath.Join(tempDir, "f2"), MediaType: "application/octet-stream"},
|
||||
},
|
||||
patterns: []string{ggufLayerMediaType, "application/octet-stream"},
|
||||
expected: 2,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
resolver := NewContainerImageModel(tempDir, test.layerFiles)
|
||||
|
||||
locations, err := resolver.FilesByMediaType(test.patterns...)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, locations, test.expected)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOCIModelResolver_FileContentsByLocation(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
content := []byte("test gguf content")
|
||||
|
||||
tempFile := filepath.Join(tempDir, "test.gguf")
|
||||
require.NoError(t, os.WriteFile(tempFile, content, 0600))
|
||||
|
||||
digest := "sha256:abc123"
|
||||
layerFiles := map[string]LayerInfo{
|
||||
digest: {TempPath: tempFile, MediaType: ggufLayerMediaType},
|
||||
}
|
||||
|
||||
resolver := NewContainerImageModel(tempDir, layerFiles)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
digest string
|
||||
wantErr bool
|
||||
wantData []byte
|
||||
errSubstr string
|
||||
}{
|
||||
{
|
||||
name: "valid location returns content",
|
||||
digest: digest,
|
||||
wantErr: false,
|
||||
wantData: content,
|
||||
},
|
||||
{
|
||||
name: "invalid digest returns error",
|
||||
digest: "sha256:invalid",
|
||||
wantErr: true,
|
||||
errSubstr: "no file found for digest",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
loc := file.NewVirtualLocationFromCoordinates(
|
||||
file.NewCoordinates("/", test.digest),
|
||||
"/",
|
||||
)
|
||||
|
||||
reader, err := resolver.FileContentsByLocation(loc)
|
||||
|
||||
if test.wantErr {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), test.errSubstr)
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
|
||||
data, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, test.wantData, data)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -9,8 +9,17 @@ import (
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/generic"
|
||||
)
|
||||
|
||||
const (
|
||||
catalogerName = "gguf-cataloger"
|
||||
ggufLayerMediaType = "application/vnd.docker.ai*"
|
||||
)
|
||||
|
||||
// NewGGUFCataloger returns a new cataloger instance for GGUF model files.
|
||||
// It supports both traditional file-based discovery and OCI layer-aware discovery
|
||||
// when the source for the SBOM is the oci model source
|
||||
func NewGGUFCataloger() pkg.Cataloger {
|
||||
return generic.NewCataloger("gguf-cataloger").
|
||||
WithParserByGlobs(parseGGUFModel, "**/*.gguf")
|
||||
return generic.NewCataloger(catalogerName).
|
||||
WithParserByGlobs(parseGGUFModel, "**/*.gguf").
|
||||
WithParserByMediaType(parseGGUFModel, ggufLayerMediaType).
|
||||
WithProcessors(ggufMergeProcessor)
|
||||
}
|
||||
|
||||
@@ -122,6 +122,10 @@ func extractVersion(kvs gguf_parser.GGUFMetadataKVs) string {
|
||||
|
||||
// extractModelNameFromPath extracts the model name from the file path
|
||||
func extractModelNameFromPath(path string) string {
|
||||
// we do not want to return a name from filepath if it's not a distinct gguf file
|
||||
if !strings.Contains(path, ".gguf") {
|
||||
return ""
|
||||
}
|
||||
// Get the base filename
|
||||
base := filepath.Base(path)
|
||||
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
// ggufMergeProcessor consolidates multiple GGUF packages into a single package
|
||||
// representing the AI model. When scanning OCI images with multiple layers,
|
||||
// each layer may produce a separate package. This processor finds the package
|
||||
// with a name and merges metadata from nameless packages into its GGUFFileParts field.
|
||||
// Only packages with a non-empty name are returned in the final result.
|
||||
func ggufMergeProcessor(pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
if err != nil {
|
||||
return pkgs, rels, err
|
||||
}
|
||||
|
||||
if len(pkgs) == 0 {
|
||||
return pkgs, rels, err
|
||||
}
|
||||
|
||||
// Separate packages with names from those without
|
||||
var namedPkgs []pkg.Package
|
||||
var namelessHeaders []pkg.GGUFFileHeader
|
||||
|
||||
for _, p := range pkgs {
|
||||
if p.Name != "" {
|
||||
namedPkgs = append(namedPkgs, p)
|
||||
} else {
|
||||
if header, ok := p.Metadata.(pkg.GGUFFileHeader); ok {
|
||||
// We do not want a kv hash for nameless headers
|
||||
header.MetadataKeyValuesHash = ""
|
||||
namelessHeaders = append(namelessHeaders, header)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If there are no named packages, return nothing
|
||||
if len(namedPkgs) == 0 {
|
||||
return nil, rels, err
|
||||
}
|
||||
|
||||
// merge nameless headers into a single named package;
|
||||
// if there are multiple named packages, return them without trying to merge headers.
|
||||
// we cannot determine which nameless headers belong to which package
|
||||
// this is because the order we receive the gguf headers in is not guaranteed
|
||||
// to match the layer order in the original oci image
|
||||
if len(namedPkgs) == 1 && len(namelessHeaders) > 0 {
|
||||
winner := &namedPkgs[0]
|
||||
if header, ok := winner.Metadata.(pkg.GGUFFileHeader); ok {
|
||||
header.Parts = namelessHeaders
|
||||
winner.Metadata = header
|
||||
}
|
||||
}
|
||||
|
||||
// Largest number of key value
|
||||
|
||||
return namedPkgs, rels, err
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
func Test_ggufMergeProcessor(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
pkgs []pkg.Package
|
||||
wantPkgCount int
|
||||
wantFilePartCount int
|
||||
}{
|
||||
{
|
||||
name: "single named package merges nameless headers",
|
||||
pkgs: []pkg.Package{
|
||||
{Name: "model", Metadata: pkg.GGUFFileHeader{MetadataKeyValuesHash: "abc"}},
|
||||
{Name: "", Metadata: pkg.GGUFFileHeader{MetadataKeyValuesHash: "part1"}},
|
||||
{Name: "", Metadata: pkg.GGUFFileHeader{MetadataKeyValuesHash: "part2"}},
|
||||
},
|
||||
wantPkgCount: 1,
|
||||
wantFilePartCount: 2,
|
||||
},
|
||||
{
|
||||
name: "multiple named packages returns all without merging",
|
||||
pkgs: []pkg.Package{
|
||||
{Name: "model1", Metadata: pkg.GGUFFileHeader{}},
|
||||
{Name: "model2", Metadata: pkg.GGUFFileHeader{}},
|
||||
{Name: "", Metadata: pkg.GGUFFileHeader{}},
|
||||
},
|
||||
wantPkgCount: 2,
|
||||
wantFilePartCount: 0,
|
||||
},
|
||||
{
|
||||
name: "no named packages returns empty result",
|
||||
pkgs: []pkg.Package{
|
||||
{Name: "", Metadata: pkg.GGUFFileHeader{}},
|
||||
{Name: "", Metadata: pkg.GGUFFileHeader{}},
|
||||
},
|
||||
wantPkgCount: 0,
|
||||
wantFilePartCount: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, _, err := ggufMergeProcessor(test.pkgs, nil, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, got, test.wantPkgCount)
|
||||
|
||||
if test.wantPkgCount == 1 && test.wantFilePartCount > 0 {
|
||||
header, ok := got[0].Metadata.(pkg.GGUFFileHeader)
|
||||
require.True(t, ok)
|
||||
assert.Len(t, header.Parts, test.wantFilePartCount)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2164,6 +2164,11 @@ func (p *panicyResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (p *panicyResolver) FilesByMediaType(_ ...string) ([]file.Location, error) {
|
||||
p.searchCalled = true
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (p *panicyResolver) RelativeFileByPath(_ file.Location, _ string) *file.Location {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -114,6 +114,26 @@ func (c *Cataloger) WithParserByPath(parser Parser, paths ...string) *Cataloger
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *Cataloger) WithParserByMediaType(parser Parser, types ...string) *Cataloger {
|
||||
c.requesters = append(c.requesters,
|
||||
func(resolver file.Resolver, _ Environment) []request {
|
||||
var requests []request
|
||||
log.WithFields("mediatypes", types).Trace("searching content matching mediatypes")
|
||||
ociResolver, ok := resolver.(file.OCIMediaTypeResolver)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
matches, err := ociResolver.FilesByMediaType(types...)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
requests = append(requests, makeRequests(parser, matches)...)
|
||||
return requests
|
||||
},
|
||||
)
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *Cataloger) WithProcessors(processors ...Processor) *Cataloger {
|
||||
for _, p := range processors {
|
||||
c.processors = append(c.processors, processorWrapper{Processor: p})
|
||||
|
||||
@@ -138,6 +138,10 @@ func (m spyReturningFileResolver) FilesByMIMEType(types ...string) ([]file.Locat
|
||||
return m.m.FilesByMIMEType(types...)
|
||||
}
|
||||
|
||||
func (m spyReturningFileResolver) FilesByMediaType(types ...string) ([]file.Location, error) {
|
||||
return m.m.FilesByMediaType(types...)
|
||||
}
|
||||
|
||||
func (m spyReturningFileResolver) RelativeFileByPath(f file.Location, path string) *file.Location {
|
||||
return m.m.RelativeFileByPath(f, path)
|
||||
}
|
||||
@@ -189,6 +193,55 @@ func TestClosesFileOnParserPanic(t *testing.T) {
|
||||
require.True(t, spy.closed)
|
||||
}
|
||||
|
||||
func Test_CatalogerWithParserByMediaType(t *testing.T) {
|
||||
allParsedPaths := make(map[string]bool)
|
||||
parser := func(_ context.Context, resolver file.Resolver, env *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
allParsedPaths[reader.Path()] = true
|
||||
contents, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
|
||||
if len(contents) == 0 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
p := pkg.Package{
|
||||
Name: string(contents),
|
||||
Locations: file.NewLocationSet(reader.Location),
|
||||
}
|
||||
|
||||
return []pkg.Package{p}, nil, nil
|
||||
}
|
||||
|
||||
upstream := "media-type-cataloger"
|
||||
|
||||
// Create locations with test fixtures that exist on disk
|
||||
loc1 := file.NewLocation("test-fixtures/a-path.txt")
|
||||
loc2 := file.NewLocation("test-fixtures/another-path.txt")
|
||||
|
||||
// Create a mock resolver that maps media types to locations
|
||||
resolver := file.NewMockResolverForMediaTypes(map[string][]file.Location{
|
||||
"application/vnd.test.model": {loc1, loc2},
|
||||
})
|
||||
|
||||
cataloger := NewCataloger(upstream).
|
||||
WithParserByMediaType(parser, "application/vnd.test.model")
|
||||
|
||||
actualPkgs, _, err := cataloger.Catalog(context.Background(), resolver)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify both files were parsed
|
||||
assert.True(t, allParsedPaths["test-fixtures/a-path.txt"], "expected a-path.txt to be parsed")
|
||||
assert.True(t, allParsedPaths["test-fixtures/another-path.txt"], "expected another-path.txt to be parsed")
|
||||
|
||||
// Verify packages were created
|
||||
assert.Len(t, actualPkgs, 2)
|
||||
|
||||
// Verify FoundBy is set correctly
|
||||
for _, p := range actualPkgs {
|
||||
assert.Equal(t, upstream, p.FoundBy)
|
||||
}
|
||||
}
|
||||
|
||||
func Test_genericCatalogerReturnsErrors(t *testing.T) {
|
||||
genericErrorReturning := NewCataloger("error returning").WithParserByGlobs(func(ctx context.Context, resolver file.Resolver, environment *Environment, locationReader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
return []pkg.Package{
|
||||
|
||||
@@ -97,6 +97,15 @@ func Test_candidateVendorsForAPK(t *testing.T) {
|
||||
},
|
||||
expected: []string{"rake", "ruby-lang"},
|
||||
},
|
||||
{
|
||||
name: "libavif",
|
||||
pkg: pkg.Package{
|
||||
Metadata: pkg.ApkDBEntry{
|
||||
Package: "libavif",
|
||||
},
|
||||
},
|
||||
expected: []string{"aomedia", "libavif"},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
|
||||
@@ -410,6 +410,11 @@ var defaultCandidateAdditions = buildCandidateLookup(
|
||||
candidateKey{PkgName: "git"},
|
||||
candidateAddition{AdditionalVendors: []string{"git-scm"}},
|
||||
},
|
||||
{
|
||||
pkg.ApkPkg,
|
||||
candidateKey{PkgName: "libavif"},
|
||||
candidateAddition{AdditionalVendors: []string{"aomedia"}},
|
||||
},
|
||||
{
|
||||
pkg.ApkPkg,
|
||||
candidateKey{PkgName: "bind"},
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+20
-3
@@ -10,6 +10,23 @@ import (
|
||||
|
||||
const cacheDir = ".cpe-cache"
|
||||
|
||||
// parseNVDTimestamp parses timestamps from NVD API which may or may not include timezone
|
||||
func parseNVDTimestamp(ts string) (time.Time, error) {
|
||||
// try RFC3339 first (with timezone)
|
||||
if t, err := time.Parse(time.RFC3339, ts); err == nil {
|
||||
return t, nil
|
||||
}
|
||||
// try without timezone (NVD sometimes returns this format)
|
||||
if t, err := time.Parse("2006-01-02T15:04:05.000", ts); err == nil {
|
||||
return t, nil
|
||||
}
|
||||
// try without milliseconds
|
||||
if t, err := time.Parse("2006-01-02T15:04:05", ts); err == nil {
|
||||
return t, nil
|
||||
}
|
||||
return time.Time{}, fmt.Errorf("unable to parse timestamp: %s", ts)
|
||||
}
|
||||
|
||||
// IncrementMetadata tracks a single fetch increment for a monthly batch
|
||||
type IncrementMetadata struct {
|
||||
FetchedAt time.Time `json:"fetchedAt"`
|
||||
@@ -215,7 +232,7 @@ func groupProductsByMonth(products []NVDProduct) (map[string][]NVDProduct, error
|
||||
productsByMonth := make(map[string][]NVDProduct)
|
||||
|
||||
for _, product := range products {
|
||||
lastMod, err := time.Parse(time.RFC3339, product.CPE.LastModified)
|
||||
lastMod, err := parseNVDTimestamp(product.CPE.LastModified)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse lastModified for %s: %w", product.CPE.CPENameID, err)
|
||||
}
|
||||
@@ -342,8 +359,8 @@ func (m *CacheManager) LoadAllProducts() ([]NVDProduct, error) {
|
||||
}
|
||||
|
||||
// compare lastModified timestamps to keep the newer one
|
||||
newMod, _ := time.Parse(time.RFC3339, p.CPE.LastModified)
|
||||
existingMod, _ := time.Parse(time.RFC3339, existing.CPE.LastModified)
|
||||
newMod, _ := parseNVDTimestamp(p.CPE.LastModified)
|
||||
existingMod, _ := parseNVDTimestamp(existing.CPE.LastModified)
|
||||
|
||||
if newMod.After(existingMod) {
|
||||
productMap[p.CPE.CPENameID] = p
|
||||
|
||||
+56
@@ -317,3 +317,59 @@ func TestCacheManager_SaveProducts(t *testing.T) {
|
||||
assert.Equal(t, 1, metadata.MonthlyBatches["2024-12"].TotalProducts)
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseNVDTimestamp(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expectError bool
|
||||
expectedStr string // expected time formatted as 2006-01-02T15:04:05
|
||||
}{
|
||||
{
|
||||
name: "RFC3339 with Z timezone",
|
||||
input: "2024-11-15T10:00:00.000Z",
|
||||
expectError: false,
|
||||
expectedStr: "2024-11-15T10:00:00",
|
||||
},
|
||||
{
|
||||
name: "RFC3339 with offset timezone",
|
||||
input: "2024-11-15T10:00:00+00:00",
|
||||
expectError: false,
|
||||
expectedStr: "2024-11-15T10:00:00",
|
||||
},
|
||||
{
|
||||
name: "without timezone (NVD format)",
|
||||
input: "2026-01-27T15:53:34.823",
|
||||
expectError: false,
|
||||
expectedStr: "2026-01-27T15:53:34",
|
||||
},
|
||||
{
|
||||
name: "without timezone or milliseconds",
|
||||
input: "2024-11-15T10:00:00",
|
||||
expectError: false,
|
||||
expectedStr: "2024-11-15T10:00:00",
|
||||
},
|
||||
{
|
||||
name: "invalid format",
|
||||
input: "not-a-timestamp",
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
name: "empty string",
|
||||
input: "",
|
||||
expectError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, err := parseNVDTimestamp(tt.input)
|
||||
if tt.expectError {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.expectedStr, result.Format("2006-01-02T15:04:05"))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,33 +53,26 @@ func updateCache(ctx context.Context, cacheManager *CacheManager, forceFullRefre
|
||||
|
||||
lastModStartDate, isFullRefresh := determineUpdateMode(metadata, forceFullRefresh)
|
||||
|
||||
// use resume index if available
|
||||
resumeFromIndex := 0
|
||||
if !isFullRefresh && metadata.LastStartIndex > 0 {
|
||||
resumeFromIndex = metadata.LastStartIndex
|
||||
fmt.Printf("Resuming from index %d...\n", resumeFromIndex)
|
||||
}
|
||||
|
||||
allProducts, increment, err := fetchProducts(ctx, lastModStartDate, resumeFromIndex)
|
||||
products, increment, err := fetchProducts(ctx, lastModStartDate)
|
||||
if err != nil {
|
||||
// if we have partial products, save them before returning error
|
||||
if len(allProducts) > 0 {
|
||||
fmt.Printf("\nError occurred but saving %d products fetched so far...\n", len(allProducts))
|
||||
if saveErr := saveAndReportResults(cacheManager, allProducts, isFullRefresh, metadata, increment); saveErr != nil {
|
||||
if len(products) > 0 {
|
||||
fmt.Printf("\nError occurred but saving %d products fetched so far...\n", len(products))
|
||||
if saveErr := saveAndReportResults(cacheManager, products, isFullRefresh, metadata, increment); saveErr != nil {
|
||||
fmt.Printf("WARNING: Failed to save partial progress: %v\n", saveErr)
|
||||
} else {
|
||||
fmt.Println("Partial progress saved successfully. Run again to resume from this point.")
|
||||
fmt.Println("Partial progress saved successfully.")
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if len(allProducts) == 0 {
|
||||
if len(products) == 0 {
|
||||
fmt.Println("No products fetched (already up to date)")
|
||||
return nil
|
||||
}
|
||||
|
||||
return saveAndReportResults(cacheManager, allProducts, isFullRefresh, metadata, increment)
|
||||
return saveAndReportResults(cacheManager, products, isFullRefresh, metadata, increment)
|
||||
}
|
||||
|
||||
// determineUpdateMode decides whether to do a full refresh or incremental update
|
||||
@@ -94,48 +87,25 @@ func determineUpdateMode(metadata *CacheMetadata, forceFullRefresh bool) (time.T
|
||||
}
|
||||
|
||||
// fetchProducts fetches products from the NVD API
|
||||
func fetchProducts(ctx context.Context, lastModStartDate time.Time, resumeFromIndex int) ([]NVDProduct, IncrementMetadata, error) {
|
||||
func fetchProducts(ctx context.Context, lastModStartDate time.Time) ([]NVDProduct, IncrementMetadata, error) {
|
||||
apiClient := NewNVDAPIClient()
|
||||
fmt.Println("Fetching CPE data from NVD Products API...")
|
||||
|
||||
var allProducts []NVDProduct
|
||||
var totalResults int
|
||||
var firstStartIndex, lastEndIndex int
|
||||
|
||||
onPageFetched := func(startIndex int, response NVDProductsResponse) error {
|
||||
if totalResults == 0 {
|
||||
totalResults = response.TotalResults
|
||||
firstStartIndex = startIndex
|
||||
}
|
||||
lastEndIndex = startIndex + response.ResultsPerPage
|
||||
allProducts = append(allProducts, response.Products...)
|
||||
fmt.Printf("Fetched %d/%d products...\n", len(allProducts), totalResults)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := apiClient.FetchProductsSince(ctx, lastModStartDate, resumeFromIndex, onPageFetched); err != nil {
|
||||
// return partial products with increment metadata so they can be saved
|
||||
increment := IncrementMetadata{
|
||||
FetchedAt: time.Now(),
|
||||
LastModStartDate: lastModStartDate,
|
||||
LastModEndDate: time.Now(),
|
||||
Products: len(allProducts),
|
||||
StartIndex: firstStartIndex,
|
||||
EndIndex: lastEndIndex,
|
||||
}
|
||||
return allProducts, increment, fmt.Errorf("failed to fetch products from NVD API: %w", err)
|
||||
}
|
||||
products, err := apiClient.FetchProductsSince(ctx, lastModStartDate)
|
||||
|
||||
increment := IncrementMetadata{
|
||||
FetchedAt: time.Now(),
|
||||
LastModStartDate: lastModStartDate,
|
||||
LastModEndDate: time.Now(),
|
||||
Products: len(allProducts),
|
||||
StartIndex: firstStartIndex,
|
||||
EndIndex: lastEndIndex,
|
||||
Products: len(products),
|
||||
}
|
||||
|
||||
return allProducts, increment, nil
|
||||
if err != nil {
|
||||
// return partial products so they can be saved
|
||||
return products, increment, fmt.Errorf("failed to fetch products from NVD API: %w", err)
|
||||
}
|
||||
|
||||
return products, increment, nil
|
||||
}
|
||||
|
||||
// saveAndReportResults saves products and metadata, then reports success
|
||||
|
||||
+96
-55
@@ -24,6 +24,9 @@ const (
|
||||
// retry configuration for rate limiting
|
||||
maxRetries = 5
|
||||
baseRetryDelay = 30 * time.Second // NVD uses 30-second rolling windows
|
||||
|
||||
// NVD API has a maximum date range of 120 days for queries with date filters
|
||||
maxDateRangeDays = 120
|
||||
)
|
||||
|
||||
// NVDAPIClient handles communication with the NVD Products API
|
||||
@@ -109,45 +112,98 @@ func NewNVDAPIClient() *NVDAPIClient {
|
||||
}
|
||||
}
|
||||
|
||||
// PageCallback is called after each page is successfully fetched
|
||||
// it receives the startIndex and the response for that page
|
||||
type PageCallback func(startIndex int, response NVDProductsResponse) error
|
||||
// FetchProductsSince fetches all products modified since the given date.
|
||||
// If lastModStartDate is zero, fetches all products (no date filter).
|
||||
// If lastModStartDate is set, fetches in 120-day chunks (NVD API limit) from that date to now.
|
||||
// Returns partial results on error so progress can be saved.
|
||||
func (c *NVDAPIClient) FetchProductsSince(ctx context.Context, lastModStartDate time.Time) ([]NVDProduct, error) {
|
||||
// if no date filter, fetch all products in a single pass
|
||||
if lastModStartDate.IsZero() {
|
||||
return c.fetchDateRange(ctx, time.Time{}, time.Time{})
|
||||
}
|
||||
|
||||
// FetchProductsSince fetches all products modified since the given date
|
||||
// if lastModStartDate is zero, fetches all products
|
||||
// calls onPageFetched callback after each successful page fetch for incremental saving
|
||||
// if resumeFromIndex > 0, starts fetching from that index
|
||||
func (c *NVDAPIClient) FetchProductsSince(ctx context.Context, lastModStartDate time.Time, resumeFromIndex int, onPageFetched PageCallback) error {
|
||||
startIndex := resumeFromIndex
|
||||
// fetch in 120-day chunks from lastModStartDate to now
|
||||
chunks := buildDateChunks(lastModStartDate, time.Now().UTC())
|
||||
if len(chunks) > 1 {
|
||||
fmt.Printf("Date range spans %d chunks of up to %d days each\n", len(chunks), maxDateRangeDays)
|
||||
}
|
||||
|
||||
var allProducts []NVDProduct
|
||||
for i, chunk := range chunks {
|
||||
if len(chunks) > 1 {
|
||||
fmt.Printf("Fetching chunk %d/%d: %s to %s\n", i+1, len(chunks),
|
||||
chunk.start.Format("2006-01-02"), chunk.end.Format("2006-01-02"))
|
||||
}
|
||||
|
||||
products, err := c.fetchDateRange(ctx, chunk.start, chunk.end)
|
||||
if err != nil {
|
||||
// return partial results so caller can save progress
|
||||
return allProducts, err
|
||||
}
|
||||
|
||||
allProducts = append(allProducts, products...)
|
||||
if len(chunks) > 1 {
|
||||
fmt.Printf("Chunk %d complete: %d products (total so far: %d)\n", i+1, len(products), len(allProducts))
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("Fetched %d products total\n", len(allProducts))
|
||||
return allProducts, nil
|
||||
}
|
||||
|
||||
// dateChunk represents a date range for fetching
|
||||
type dateChunk struct {
|
||||
start time.Time
|
||||
end time.Time
|
||||
}
|
||||
|
||||
// buildDateChunks splits a date range into chunks of maxDateRangeDays
|
||||
func buildDateChunks(start, end time.Time) []dateChunk {
|
||||
var chunks []dateChunk
|
||||
chunkStart := start
|
||||
|
||||
for chunkStart.Before(end) {
|
||||
chunkEnd := chunkStart.AddDate(0, 0, maxDateRangeDays)
|
||||
if chunkEnd.After(end) {
|
||||
chunkEnd = end
|
||||
}
|
||||
chunks = append(chunks, dateChunk{start: chunkStart, end: chunkEnd})
|
||||
chunkStart = chunkEnd
|
||||
}
|
||||
|
||||
return chunks
|
||||
}
|
||||
|
||||
// fetchDateRange fetches all products within a single date range (must be <= 120 days).
|
||||
// If start and end are both zero, fetches all products without date filtering.
|
||||
func (c *NVDAPIClient) fetchDateRange(ctx context.Context, start, end time.Time) ([]NVDProduct, error) {
|
||||
var products []NVDProduct
|
||||
startIndex := 0
|
||||
|
||||
for {
|
||||
resp, err := c.fetchPage(ctx, startIndex, lastModStartDate)
|
||||
resp, err := c.fetchPage(ctx, startIndex, start, end)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to fetch page at index %d: %w", startIndex, err)
|
||||
return products, fmt.Errorf("failed to fetch page at index %d: %w", startIndex, err)
|
||||
}
|
||||
|
||||
// call callback to save progress immediately
|
||||
if onPageFetched != nil {
|
||||
if err := onPageFetched(startIndex, resp); err != nil {
|
||||
return fmt.Errorf("callback failed at index %d: %w", startIndex, err)
|
||||
}
|
||||
}
|
||||
products = append(products, resp.Products...)
|
||||
fmt.Printf(" Fetched %d/%d products...\n", len(products), resp.TotalResults)
|
||||
|
||||
// check if we've fetched all results
|
||||
if startIndex+resp.ResultsPerPage >= resp.TotalResults {
|
||||
fmt.Printf("Fetched %d/%d products (complete)\n", resp.TotalResults, resp.TotalResults)
|
||||
break
|
||||
}
|
||||
|
||||
startIndex += resp.ResultsPerPage
|
||||
fmt.Printf("Fetched %d/%d products...\n", startIndex, resp.TotalResults)
|
||||
}
|
||||
|
||||
return nil
|
||||
return products, nil
|
||||
}
|
||||
|
||||
// fetchPage fetches a single page of results from the NVD API with retry logic for rate limiting
|
||||
func (c *NVDAPIClient) fetchPage(ctx context.Context, startIndex int, lastModStartDate time.Time) (NVDProductsResponse, error) {
|
||||
// if both start and end are zero, fetches without date filtering
|
||||
// if start and end are set, they must form a range <= 120 days (enforced by caller)
|
||||
func (c *NVDAPIClient) fetchPage(ctx context.Context, startIndex int, start, end time.Time) (NVDProductsResponse, error) {
|
||||
var lastErr error
|
||||
|
||||
for attempt := 0; attempt < maxRetries; attempt++ {
|
||||
@@ -160,10 +216,12 @@ func (c *NVDAPIClient) fetchPage(ctx context.Context, startIndex int, lastModSta
|
||||
url := fmt.Sprintf("%s?resultsPerPage=%d&startIndex=%d", nvdProductsAPIURL, resultsPerPage, startIndex)
|
||||
|
||||
// add date range if specified (incremental update)
|
||||
if !lastModStartDate.IsZero() {
|
||||
// NVD API requires RFC3339 format: 2024-01-01T00:00:00.000
|
||||
lastModStartStr := lastModStartDate.Format("2006-01-02T15:04:05.000")
|
||||
url += fmt.Sprintf("&lastModStartDate=%s", lastModStartStr)
|
||||
// NVD API requires both lastModStartDate and lastModEndDate when either is present
|
||||
if !start.IsZero() && !end.IsZero() {
|
||||
// NVD API requires this format: 2024-01-01T00:00:00.000
|
||||
startStr := start.Format("2006-01-02T15:04:05.000")
|
||||
endStr := end.Format("2006-01-02T15:04:05.000")
|
||||
url += fmt.Sprintf("&lastModStartDate=%s&lastModEndDate=%s", startStr, endStr)
|
||||
}
|
||||
|
||||
// create request
|
||||
@@ -191,14 +249,12 @@ func (c *NVDAPIClient) fetchPage(ctx context.Context, startIndex int, lastModSta
|
||||
continue // retry
|
||||
}
|
||||
|
||||
// handle HTTP status codes
|
||||
statusResponse, handled, err := c.handleHTTPStatus(httpResp, startIndex)
|
||||
if handled {
|
||||
// either error or special case (404 with empty results)
|
||||
return statusResponse, err
|
||||
// check for error status codes
|
||||
if err := checkHTTPStatus(httpResp); err != nil {
|
||||
return NVDProductsResponse{}, err
|
||||
}
|
||||
|
||||
// success - parse response
|
||||
// parse response
|
||||
var response NVDProductsResponse
|
||||
if err := json.NewDecoder(httpResp.Body).Decode(&response); err != nil {
|
||||
httpResp.Body.Close()
|
||||
@@ -235,31 +291,16 @@ func (c *NVDAPIClient) handleRateLimit(ctx context.Context, httpResp *http.Respo
|
||||
}
|
||||
}
|
||||
|
||||
// handleHTTPStatus handles non-429 HTTP status codes
|
||||
// returns (response, handled, error) where:
|
||||
// - handled=true means the status was processed (either success case like 404 or error)
|
||||
// - handled=false means continue to normal response parsing
|
||||
func (c *NVDAPIClient) handleHTTPStatus(httpResp *http.Response, startIndex int) (NVDProductsResponse, bool, error) {
|
||||
// handle 404 as "no results found" (common when querying recent dates with no updates)
|
||||
if httpResp.StatusCode == http.StatusNotFound {
|
||||
httpResp.Body.Close()
|
||||
return NVDProductsResponse{
|
||||
ResultsPerPage: 0,
|
||||
StartIndex: startIndex,
|
||||
TotalResults: 0,
|
||||
Products: []NVDProduct{},
|
||||
}, true, nil
|
||||
// checkHTTPStatus returns an error for non-200 status codes.
|
||||
// NVD API returns 200 with TotalResults=0 when there are no results,
|
||||
// so any non-200 status (including 404) indicates an actual error.
|
||||
func checkHTTPStatus(httpResp *http.Response) error {
|
||||
if httpResp.StatusCode == http.StatusOK {
|
||||
return nil
|
||||
}
|
||||
|
||||
// check for other non-200 status codes
|
||||
if httpResp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(httpResp.Body)
|
||||
httpResp.Body.Close()
|
||||
return NVDProductsResponse{}, true, fmt.Errorf("unexpected status code %d: %s", httpResp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
// status OK - let caller parse response
|
||||
return NVDProductsResponse{}, false, nil
|
||||
body, _ := io.ReadAll(httpResp.Body)
|
||||
httpResp.Body.Close()
|
||||
return fmt.Errorf("NVD API error (status %d): %s", httpResp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
// parseRetryAfter parses the Retry-After header from HTTP 429 responses
|
||||
|
||||
+233
@@ -0,0 +1,233 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBuildDateChunks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
start time.Time
|
||||
end time.Time
|
||||
expectedChunks int
|
||||
validateChunks func(t *testing.T, chunks []dateChunk)
|
||||
}{
|
||||
{
|
||||
name: "single chunk when range is under 120 days",
|
||||
start: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
end: time.Date(2025, 2, 1, 0, 0, 0, 0, time.UTC), // 31 days
|
||||
expectedChunks: 1,
|
||||
validateChunks: func(t *testing.T, chunks []dateChunk) {
|
||||
assert.Equal(t, time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), chunks[0].start)
|
||||
assert.Equal(t, time.Date(2025, 2, 1, 0, 0, 0, 0, time.UTC), chunks[0].end)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "single chunk when range is exactly 120 days",
|
||||
start: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
end: time.Date(2025, 5, 1, 0, 0, 0, 0, time.UTC), // 120 days
|
||||
expectedChunks: 1,
|
||||
validateChunks: func(t *testing.T, chunks []dateChunk) {
|
||||
assert.Equal(t, time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), chunks[0].start)
|
||||
assert.Equal(t, time.Date(2025, 5, 1, 0, 0, 0, 0, time.UTC), chunks[0].end)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "two chunks when range is 121 days",
|
||||
start: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
end: time.Date(2025, 5, 2, 0, 0, 0, 0, time.UTC), // 121 days
|
||||
expectedChunks: 2,
|
||||
validateChunks: func(t *testing.T, chunks []dateChunk) {
|
||||
// first chunk: Jan 1 to May 1 (120 days)
|
||||
assert.Equal(t, time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), chunks[0].start)
|
||||
assert.Equal(t, time.Date(2025, 5, 1, 0, 0, 0, 0, time.UTC), chunks[0].end)
|
||||
// second chunk: May 1 to May 2 (1 day)
|
||||
assert.Equal(t, time.Date(2025, 5, 1, 0, 0, 0, 0, time.UTC), chunks[1].start)
|
||||
assert.Equal(t, time.Date(2025, 5, 2, 0, 0, 0, 0, time.UTC), chunks[1].end)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multiple chunks for a full year",
|
||||
start: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
end: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), // 366 days (leap year)
|
||||
expectedChunks: 4,
|
||||
validateChunks: func(t *testing.T, chunks []dateChunk) {
|
||||
// verify chunks are contiguous (each chunk starts where previous ended)
|
||||
for i := 1; i < len(chunks); i++ {
|
||||
assert.Equal(t, chunks[i-1].end, chunks[i].start, "chunks should be contiguous")
|
||||
}
|
||||
// verify first and last
|
||||
assert.Equal(t, time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), chunks[0].start)
|
||||
assert.Equal(t, time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), chunks[len(chunks)-1].end)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty result when start equals end",
|
||||
start: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
end: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
expectedChunks: 0,
|
||||
},
|
||||
{
|
||||
name: "empty result when start is after end",
|
||||
start: time.Date(2025, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
end: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
expectedChunks: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
chunks := buildDateChunks(tt.start, tt.end)
|
||||
assert.Len(t, chunks, tt.expectedChunks)
|
||||
if tt.validateChunks != nil && len(chunks) > 0 {
|
||||
tt.validateChunks(t, chunks)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDateChunks_ChunkSizeNeverExceeds120Days(t *testing.T) {
|
||||
// test with various date ranges to ensure no chunk exceeds 120 days
|
||||
testCases := []struct {
|
||||
start time.Time
|
||||
end time.Time
|
||||
}{
|
||||
{time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)},
|
||||
{time.Date(2023, 6, 15, 0, 0, 0, 0, time.UTC), time.Date(2025, 2, 1, 0, 0, 0, 0, time.UTC)},
|
||||
{time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2024, 12, 31, 0, 0, 0, 0, time.UTC)},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
chunks := buildDateChunks(tc.start, tc.end)
|
||||
for i, chunk := range chunks {
|
||||
days := chunk.end.Sub(chunk.start).Hours() / 24
|
||||
assert.LessOrEqual(t, days, float64(maxDateRangeDays),
|
||||
"chunk %d exceeds max days: start=%s, end=%s, days=%.0f",
|
||||
i, chunk.start.Format("2006-01-02"), chunk.end.Format("2006-01-02"), days)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mockResponseBody creates an io.ReadCloser from a string for testing
|
||||
type mockReadCloser struct {
|
||||
io.Reader
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (m *mockReadCloser) Close() error {
|
||||
m.closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func newMockResponse(statusCode int, body string) *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: statusCode,
|
||||
Body: &mockReadCloser{Reader: strings.NewReader(body)},
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckHTTPStatus(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
statusCode int
|
||||
body string
|
||||
expectError bool
|
||||
errorContains string
|
||||
}{
|
||||
{
|
||||
name: "200 OK returns nil",
|
||||
statusCode: http.StatusOK,
|
||||
body: `{"totalResults": 0}`,
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "404 returns error",
|
||||
statusCode: http.StatusNotFound,
|
||||
body: "",
|
||||
expectError: true,
|
||||
errorContains: "status 404",
|
||||
},
|
||||
{
|
||||
name: "400 Bad Request returns error with body",
|
||||
statusCode: http.StatusBadRequest,
|
||||
body: "Both lastModStartDate and lastModEndDate are required",
|
||||
expectError: true,
|
||||
errorContains: "lastModStartDate and lastModEndDate are required",
|
||||
},
|
||||
{
|
||||
name: "500 Internal Server Error returns error",
|
||||
statusCode: http.StatusInternalServerError,
|
||||
body: "Internal server error",
|
||||
expectError: true,
|
||||
errorContains: "status 500",
|
||||
},
|
||||
{
|
||||
name: "503 Service Unavailable returns error",
|
||||
statusCode: http.StatusServiceUnavailable,
|
||||
body: "Service temporarily unavailable",
|
||||
expectError: true,
|
||||
errorContains: "status 503",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
resp := newMockResponse(tt.statusCode, tt.body)
|
||||
err := checkHTTPStatus(resp)
|
||||
|
||||
if tt.expectError {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tt.errorContains)
|
||||
// verify body was closed on error
|
||||
assert.True(t, resp.Body.(*mockReadCloser).closed, "response body should be closed on error")
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
// verify body was NOT closed on success (caller needs to read it)
|
||||
assert.False(t, resp.Body.(*mockReadCloser).closed, "response body should not be closed on success")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRetryAfter(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
header string
|
||||
expected time.Duration
|
||||
}{
|
||||
{
|
||||
name: "empty header returns 0",
|
||||
header: "",
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "numeric seconds",
|
||||
header: "30",
|
||||
expected: 30 * time.Second,
|
||||
},
|
||||
{
|
||||
name: "numeric seconds - larger value",
|
||||
header: "120",
|
||||
expected: 120 * time.Second,
|
||||
},
|
||||
{
|
||||
name: "invalid value returns 0",
|
||||
header: "not-a-number",
|
||||
expected: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := parseRetryAfter(tt.header)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -208,6 +208,11 @@ func (r *ObservingResolver) FilesByMIMEType(types ...string) ([]file.Location, e
|
||||
return locs, err
|
||||
}
|
||||
|
||||
// FilesByMediaType returns files matching the given media types.
|
||||
func (r *ObservingResolver) FilesByMediaType(_ ...string) ([]file.Location, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// RelativeFileByPath returns a file at a path relative to the given location.
|
||||
func (r *ObservingResolver) RelativeFileByPath(location file.Location, path string) *file.Location {
|
||||
const methodName = "RelativeFileByPath"
|
||||
|
||||
@@ -23,20 +23,21 @@ var _ generic.Parser = parsePackageJSON
|
||||
|
||||
// packageJSON represents a JavaScript package.json file
|
||||
type packageJSON struct {
|
||||
Version string `json:"version"`
|
||||
Latest []string `json:"latest"`
|
||||
Author person `json:"author"`
|
||||
Authors people `json:"authors"`
|
||||
Contributors people `json:"contributors"`
|
||||
Maintainers people `json:"maintainers"`
|
||||
License json.RawMessage `json:"license"`
|
||||
Licenses json.RawMessage `json:"licenses"`
|
||||
Name string `json:"name"`
|
||||
Homepage string `json:"homepage"`
|
||||
Description string `json:"description"`
|
||||
Dependencies map[string]string `json:"dependencies"`
|
||||
Repository repository `json:"repository"`
|
||||
Private bool `json:"private"`
|
||||
Version string `json:"version"`
|
||||
Latest []string `json:"latest"`
|
||||
Author person `json:"author"`
|
||||
Authors people `json:"authors"`
|
||||
Contributors people `json:"contributors"`
|
||||
Maintainers people `json:"maintainers"`
|
||||
License json.RawMessage `json:"license"`
|
||||
Licenses json.RawMessage `json:"licenses"`
|
||||
Name string `json:"name"`
|
||||
Homepage string `json:"homepage"`
|
||||
Description string `json:"description"`
|
||||
Dependencies map[string]string `json:"dependencies"`
|
||||
DevDependencies map[string]string `json:"devDependencies"`
|
||||
Repository repository `json:"repository"`
|
||||
Private bool `json:"private"`
|
||||
}
|
||||
|
||||
type person struct {
|
||||
|
||||
@@ -3,6 +3,7 @@ package javascript
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
@@ -13,6 +14,7 @@ import (
|
||||
"github.com/goccy/go-yaml"
|
||||
"github.com/scylladb/go-set/strset"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/unknown"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
@@ -51,7 +53,7 @@ type yarnPackage struct {
|
||||
Version string
|
||||
Resolved string
|
||||
Integrity string
|
||||
Dependencies map[string]string // We don't currently support dependencies for yarn v1 lock files
|
||||
Dependencies map[string]string
|
||||
}
|
||||
|
||||
type yarnV2PackageEntry struct {
|
||||
@@ -71,6 +73,98 @@ func newGenericYarnLockAdapter(cfg CatalogerConfig) genericYarnLockAdapter {
|
||||
}
|
||||
}
|
||||
|
||||
// readPackageJSONDeps reads the package.json adjacent to the given lockfile location.
|
||||
// NOTE: in yarn workspaces, only the root package.json is consulted. Packages declared
|
||||
// as devDependencies only in a workspace package.json (not the root) will not be detected
|
||||
// as dev-only. This is a safe degradation — those packages will be included in the SBOM
|
||||
// rather than incorrectly filtered out.
|
||||
func readPackageJSONDeps(resolver file.Resolver, lockfileLocation file.Location) (prod, dev map[string]string) {
|
||||
prod = make(map[string]string)
|
||||
dev = make(map[string]string)
|
||||
|
||||
if resolver == nil {
|
||||
return prod, dev
|
||||
}
|
||||
|
||||
pkgJSONLocation := resolver.RelativeFileByPath(lockfileLocation, "package.json")
|
||||
if pkgJSONLocation == nil {
|
||||
log.WithFields("lockfile", lockfileLocation.RealPath).Debug("could not find package.json for dev dependency detection")
|
||||
return prod, dev
|
||||
}
|
||||
|
||||
reader, err := resolver.FileContentsByLocation(*pkgJSONLocation)
|
||||
if err != nil {
|
||||
log.WithFields("location", pkgJSONLocation.RealPath, "error", err).Debug("could not read package.json for dev dependency detection")
|
||||
return prod, dev
|
||||
}
|
||||
defer internal.CloseAndLogError(reader, pkgJSONLocation.RealPath)
|
||||
|
||||
var pkgJSON packageJSON
|
||||
if err := json.NewDecoder(reader).Decode(&pkgJSON); err != nil {
|
||||
log.WithFields("location", pkgJSONLocation.RealPath, "error", err).Debug("could not parse package.json for dev dependency detection")
|
||||
return prod, dev
|
||||
}
|
||||
|
||||
if pkgJSON.Dependencies != nil {
|
||||
prod = pkgJSON.Dependencies
|
||||
}
|
||||
if pkgJSON.DevDependencies != nil {
|
||||
dev = pkgJSON.DevDependencies
|
||||
}
|
||||
|
||||
return prod, dev
|
||||
}
|
||||
|
||||
// findReachable returns all package names reachable from the given roots via BFS
|
||||
// through the dependency graph.
|
||||
func findReachable(roots map[string]string, pkgByName map[string]yarnPackage) map[string]bool {
|
||||
visited := make(map[string]bool)
|
||||
queue := make([]string, 0, len(roots))
|
||||
|
||||
for name := range roots {
|
||||
queue = append(queue, name)
|
||||
}
|
||||
|
||||
for len(queue) > 0 {
|
||||
name := queue[0]
|
||||
queue = queue[1:]
|
||||
|
||||
if visited[name] {
|
||||
continue
|
||||
}
|
||||
visited[name] = true
|
||||
|
||||
if pkg, exists := pkgByName[name]; exists {
|
||||
for depName := range pkg.Dependencies {
|
||||
if !visited[depName] {
|
||||
queue = append(queue, depName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return visited
|
||||
}
|
||||
|
||||
func findDevOnlyPkgs(yarnPkgs []yarnPackage, prodDeps, devDeps map[string]string) map[string]bool {
|
||||
pkgByName := make(map[string]yarnPackage)
|
||||
for _, p := range yarnPkgs {
|
||||
pkgByName[p.Name] = p
|
||||
}
|
||||
|
||||
prodTransitive := findReachable(prodDeps, pkgByName)
|
||||
devTransitive := findReachable(devDeps, pkgByName)
|
||||
|
||||
devOnly := make(map[string]bool)
|
||||
for name := range devTransitive {
|
||||
if !prodTransitive[name] {
|
||||
devOnly[name] = true
|
||||
}
|
||||
}
|
||||
|
||||
return devOnly
|
||||
}
|
||||
|
||||
func parseYarnV1LockFile(reader io.ReadCloser) ([]yarnPackage, error) {
|
||||
content, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
@@ -190,9 +284,17 @@ func (a genericYarnLockAdapter) parseYarnLock(ctx context.Context, resolver file
|
||||
return nil, nil, fmt.Errorf("failed to parse yarn.lock file: %w", err)
|
||||
}
|
||||
|
||||
packages := make([]pkg.Package, len(yarnPkgs))
|
||||
for i, p := range yarnPkgs {
|
||||
packages[i] = newYarnLockPackage(ctx, a.cfg, resolver, reader.Location, p.Name, p.Version, p.Resolved, p.Integrity, p.Dependencies)
|
||||
// get dependencies from sibling package.json for dev dependency detection
|
||||
prodDeps, devDeps := readPackageJSONDeps(resolver, reader.Location)
|
||||
|
||||
devOnlyPkgs := findDevOnlyPkgs(yarnPkgs, prodDeps, devDeps)
|
||||
|
||||
packages := make([]pkg.Package, 0, len(yarnPkgs))
|
||||
for _, p := range yarnPkgs {
|
||||
if devOnlyPkgs[p.Name] && !a.cfg.IncludeDevDependencies {
|
||||
continue
|
||||
}
|
||||
packages = append(packages, newYarnLockPackage(ctx, a.cfg, resolver, reader.Location, p.Name, p.Version, p.Resolved, p.Integrity, p.Dependencies))
|
||||
}
|
||||
|
||||
pkg.Sort(packages)
|
||||
|
||||
@@ -739,6 +739,274 @@ func TestParseYarnFindPackageNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseYarnLock_DevDependencies(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fixtureDir string
|
||||
includeDev bool
|
||||
expected func(file.LocationSet) ([]pkg.Package, []artifact.Relationship)
|
||||
}{
|
||||
{
|
||||
name: "v1 include dev dependencies",
|
||||
fixtureDir: "test-fixtures/yarn-dev-deps",
|
||||
includeDev: true,
|
||||
expected: func(locations file.LocationSet) ([]pkg.Package, []artifact.Relationship) {
|
||||
pkgs := []pkg.Package{
|
||||
{
|
||||
Name: "dev-only-transitive",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/dev-only-transitive@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "https://registry.yarnpkg.com/dev-only-transitive/-/dev-only-transitive-1.0.0.tgz#abc123",
|
||||
Integrity: "sha512-devonlytransitive==",
|
||||
Dependencies: map[string]string{},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "dev-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/dev-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "https://registry.yarnpkg.com/dev-pkg/-/dev-pkg-1.0.0.tgz#def456",
|
||||
Integrity: "sha512-devpkg==",
|
||||
Dependencies: map[string]string{
|
||||
"dev-only-transitive": "^1.0.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "prod-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/prod-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "https://registry.yarnpkg.com/prod-pkg/-/prod-pkg-1.0.0.tgz#ghi789",
|
||||
Integrity: "sha512-prodpkg==",
|
||||
Dependencies: map[string]string{
|
||||
"shared-pkg": "^1.0.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "shared-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/shared-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "https://registry.yarnpkg.com/shared-pkg/-/shared-pkg-1.0.0.tgz#jkl012",
|
||||
Integrity: "sha512-sharedpkg==",
|
||||
Dependencies: map[string]string{},
|
||||
},
|
||||
},
|
||||
}
|
||||
rels := []artifact.Relationship{
|
||||
{
|
||||
From: pkgs[0], // dev-only-transitive
|
||||
To: pkgs[1], // dev-pkg
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
},
|
||||
{
|
||||
From: pkgs[3], // shared-pkg
|
||||
To: pkgs[2], // prod-pkg
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
},
|
||||
}
|
||||
return pkgs, rels
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "v1 exclude dev dependencies",
|
||||
fixtureDir: "test-fixtures/yarn-dev-deps",
|
||||
includeDev: false,
|
||||
expected: func(locations file.LocationSet) ([]pkg.Package, []artifact.Relationship) {
|
||||
pkgs := []pkg.Package{
|
||||
{
|
||||
Name: "prod-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/prod-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "https://registry.yarnpkg.com/prod-pkg/-/prod-pkg-1.0.0.tgz#ghi789",
|
||||
Integrity: "sha512-prodpkg==",
|
||||
Dependencies: map[string]string{
|
||||
"shared-pkg": "^1.0.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "shared-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/shared-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "https://registry.yarnpkg.com/shared-pkg/-/shared-pkg-1.0.0.tgz#jkl012",
|
||||
Integrity: "sha512-sharedpkg==",
|
||||
Dependencies: map[string]string{},
|
||||
},
|
||||
},
|
||||
}
|
||||
rels := []artifact.Relationship{
|
||||
{
|
||||
From: pkgs[1], // shared-pkg
|
||||
To: pkgs[0], // prod-pkg
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
},
|
||||
}
|
||||
return pkgs, rels
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "v2 (berry) include dev dependencies",
|
||||
fixtureDir: "test-fixtures/yarn-berry-dev-deps",
|
||||
includeDev: true,
|
||||
expected: func(locations file.LocationSet) ([]pkg.Package, []artifact.Relationship) {
|
||||
pkgs := []pkg.Package{
|
||||
{
|
||||
Name: "dev-only-transitive",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/dev-only-transitive@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "dev-only-transitive@npm:1.0.0",
|
||||
Integrity: "abc123",
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "dev-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/dev-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "dev-pkg@npm:1.0.0",
|
||||
Integrity: "def456",
|
||||
Dependencies: map[string]string{
|
||||
"dev-only-transitive": "^1.0.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "prod-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/prod-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "prod-pkg@npm:1.0.0",
|
||||
Integrity: "ghi789",
|
||||
Dependencies: map[string]string{
|
||||
"shared-pkg": "^1.0.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "shared-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/shared-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "shared-pkg@npm:1.0.0",
|
||||
Integrity: "jkl012",
|
||||
},
|
||||
},
|
||||
}
|
||||
rels := []artifact.Relationship{
|
||||
{
|
||||
From: pkgs[0], // dev-only-transitive
|
||||
To: pkgs[1], // dev-pkg
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
},
|
||||
{
|
||||
From: pkgs[3], // shared-pkg
|
||||
To: pkgs[2], // prod-pkg
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
},
|
||||
}
|
||||
return pkgs, rels
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "v2 (berry) exclude dev dependencies",
|
||||
fixtureDir: "test-fixtures/yarn-berry-dev-deps",
|
||||
includeDev: false,
|
||||
expected: func(locations file.LocationSet) ([]pkg.Package, []artifact.Relationship) {
|
||||
pkgs := []pkg.Package{
|
||||
{
|
||||
Name: "prod-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/prod-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "prod-pkg@npm:1.0.0",
|
||||
Integrity: "ghi789",
|
||||
Dependencies: map[string]string{
|
||||
"shared-pkg": "^1.0.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "shared-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/shared-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "shared-pkg@npm:1.0.0",
|
||||
Integrity: "jkl012",
|
||||
},
|
||||
},
|
||||
}
|
||||
rels := []artifact.Relationship{
|
||||
{
|
||||
From: pkgs[1], // shared-pkg
|
||||
To: pkgs[0], // prod-pkg
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
},
|
||||
}
|
||||
return pkgs, rels
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
fixture := tt.fixtureDir + "/yarn.lock"
|
||||
locations := file.NewLocationSet(file.NewLocation(fixture))
|
||||
expectedPkgs, expectedRels := tt.expected(locations)
|
||||
|
||||
adapter := newGenericYarnLockAdapter(CatalogerConfig{IncludeDevDependencies: tt.includeDev})
|
||||
pkgtest.NewCatalogTester().
|
||||
FromDirectory(t, tt.fixtureDir).
|
||||
FromFile(t, fixture).
|
||||
Expects(expectedPkgs, expectedRels).
|
||||
TestParser(t, adapter.parseYarnLock)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func generateMockYarnRegistryHandler(responseFixture string) func(w http.ResponseWriter, r *http.Request) {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"name": "test-project",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"prod-pkg": "^1.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"dev-pkg": "^1.0.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
# This file is generated by running "yarn install" inside your project.
|
||||
# Manual changes might be lost - proceed with caution!
|
||||
|
||||
__metadata:
|
||||
version: 6
|
||||
cacheKey: 8
|
||||
|
||||
"dev-only-transitive@npm:^1.0.0":
|
||||
version: 1.0.0
|
||||
resolution: "dev-only-transitive@npm:1.0.0"
|
||||
checksum: abc123
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"dev-pkg@npm:^1.0.0":
|
||||
version: 1.0.0
|
||||
resolution: "dev-pkg@npm:1.0.0"
|
||||
dependencies:
|
||||
dev-only-transitive: ^1.0.0
|
||||
checksum: def456
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"prod-pkg@npm:^1.0.0":
|
||||
version: 1.0.0
|
||||
resolution: "prod-pkg@npm:1.0.0"
|
||||
dependencies:
|
||||
shared-pkg: ^1.0.0
|
||||
checksum: ghi789
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"shared-pkg@npm:^1.0.0":
|
||||
version: 1.0.0
|
||||
resolution: "shared-pkg@npm:1.0.0"
|
||||
checksum: jkl012
|
||||
languageName: node
|
||||
linkType: hard
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"name": "test-project",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"prod-pkg": "^1.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"dev-pkg": "^1.0.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
|
||||
# yarn lockfile v1
|
||||
|
||||
|
||||
dev-only-transitive@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/dev-only-transitive/-/dev-only-transitive-1.0.0.tgz#abc123"
|
||||
integrity sha512-devonlytransitive==
|
||||
|
||||
dev-pkg@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/dev-pkg/-/dev-pkg-1.0.0.tgz#def456"
|
||||
integrity sha512-devpkg==
|
||||
dependencies:
|
||||
dev-only-transitive "^1.0.0"
|
||||
|
||||
prod-pkg@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/prod-pkg/-/prod-pkg-1.0.0.tgz#ghi789"
|
||||
integrity sha512-prodpkg==
|
||||
dependencies:
|
||||
shared-pkg "^1.0.0"
|
||||
|
||||
shared-pkg@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/shared-pkg/-/shared-pkg-1.0.0.tgz#jkl012"
|
||||
integrity sha512-sharedpkg==
|
||||
@@ -80,6 +80,10 @@ func (r *rpmdbTestFileResolverMock) FilesByMIMEType(...string) ([]file.Location,
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func (r *rpmdbTestFileResolverMock) FilesByMediaType(...string) ([]file.Location, error) {
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func TestParseRpmDB(t *testing.T) {
|
||||
ctx := context.TODO()
|
||||
packagesLocation := file.NewLocation("test-fixtures/Packages")
|
||||
|
||||
@@ -34,4 +34,8 @@ type GGUFFileHeader struct {
|
||||
// across different file locations or remotes. It allows matching identical models even
|
||||
// when stored in different repositories or with different filenames.
|
||||
MetadataKeyValuesHash string `json:"metadataHash,omitempty" cyclonedx:"metadataHash"`
|
||||
|
||||
// Parts contains headers from additional GGUF files that were merged
|
||||
// into this package during post-processing (e.g., from OCI layers without model names).
|
||||
Parts []GGUFFileHeader `json:"parts,omitempty" cyclonedx:"parts"`
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ type ImageMetadata struct {
|
||||
Variant string `json:"architectureVariant,omitempty"`
|
||||
OS string `json:"os"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
Annotations map[string]string `json:"annotations,omitempty" id:"-"` // critical: do not consider annotations as an identifiable part of the source image
|
||||
}
|
||||
|
||||
// LayerMetadata represents all static metadata that defines what a container image layer is.
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
// DeriveImageID derives an artifact ID from the given image metadata. The order of data precedence is:
|
||||
// 1. prefer a digest of the raw container image manifest
|
||||
// 2. if no manifest digest is available, calculate a chain ID from the image layer metadata
|
||||
// 3. if no layer metadata is available, use the user input string
|
||||
//
|
||||
// in all cases, if an alias is provided, it is additionally considered in the ID calculation. This allows for the
|
||||
// same image to be scanned multiple times with different aliases and be considered logically different.
|
||||
func DeriveImageID(alias source.Alias, metadata source.ImageMetadata) artifact.ID {
|
||||
var input string
|
||||
|
||||
if len(metadata.RawManifest) > 0 {
|
||||
input = digest.Canonical.FromBytes(metadata.RawManifest).String()
|
||||
} else {
|
||||
// calculate chain ID for image sources where manifestDigest is not available
|
||||
// https://github.com/opencontainers/image-spec/blob/main/config.md#layer-chainid
|
||||
input = calculateChainID(metadata.Layers)
|
||||
if input == "" {
|
||||
// TODO what happens here if image has no layers?
|
||||
// is this case possible?
|
||||
input = digest.Canonical.FromString(metadata.UserInput).String()
|
||||
}
|
||||
}
|
||||
|
||||
if !alias.IsEmpty() {
|
||||
// if the user provided an alias, we want to consider that in the artifact ID. This way if the user
|
||||
// scans the same item but is considered to be logically different, then ID will express that.
|
||||
aliasStr := fmt.Sprintf(":%s@%s", alias.Name, alias.Version)
|
||||
input = digest.Canonical.FromString(input + aliasStr).String()
|
||||
}
|
||||
|
||||
return ArtifactIDFromDigest(input)
|
||||
}
|
||||
|
||||
// https://github.com/opencontainers/image-spec/blob/main/config.md#layer-chainid
|
||||
func calculateChainID(lm []source.LayerMetadata) string {
|
||||
if len(lm) < 1 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// DiffID(L0) = digest of layer 0
|
||||
// https://github.com/anchore/stereoscope/blob/1b1b744a919964f38d14e1416fb3f25221b761ce/pkg/image/layer_metadata.go#L19-L32
|
||||
chainID := lm[0].Digest
|
||||
id := chain(chainID, lm[1:])
|
||||
|
||||
return id
|
||||
}
|
||||
|
||||
func chain(chainID string, layers []source.LayerMetadata) string {
|
||||
if len(layers) < 1 {
|
||||
return chainID
|
||||
}
|
||||
|
||||
chainID = digest.Canonical.FromString(layers[0].Digest + " " + chainID).String()
|
||||
return chain(chainID, layers[1:])
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
func TestDeriveImageID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
alias source.Alias
|
||||
metadata source.ImageMetadata
|
||||
want artifact.ID
|
||||
}{
|
||||
{
|
||||
name: "use raw manifest over chain ID or user input",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
Layers: []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
RawManifest: []byte("raw-manifest"),
|
||||
},
|
||||
want: func() artifact.ID {
|
||||
hasher := sha256.New()
|
||||
hasher.Write([]byte("raw-manifest"))
|
||||
return artifact.ID(fmt.Sprintf("%x", hasher.Sum(nil)))
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "use chain ID over user input",
|
||||
metadata: source.ImageMetadata{
|
||||
Layers: []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
},
|
||||
want: func() artifact.ID {
|
||||
metadata := []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
}
|
||||
return artifact.ID(strings.TrimPrefix(calculateChainID(metadata), "sha256:"))
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "use user input last",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
},
|
||||
want: func() artifact.ID {
|
||||
hasher := sha256.New()
|
||||
hasher.Write([]byte("user-input"))
|
||||
return artifact.ID(fmt.Sprintf("%x", hasher.Sum(nil)))
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "without alias (first)",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
Layers: []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
RawManifest: []byte("raw-manifest"),
|
||||
},
|
||||
want: "85298926ecd92ed57688f13039017160cd728f04dd0d2d10a10629007106f107",
|
||||
},
|
||||
{
|
||||
name: "always consider alias (first)",
|
||||
alias: source.Alias{
|
||||
Name: "alias",
|
||||
Version: "version",
|
||||
},
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
Layers: []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
RawManifest: []byte("raw-manifest"),
|
||||
},
|
||||
want: "a8717e42449960c1dd4963f2f22bd69c7c105e7e82445be0a65aa1825d62ff0d",
|
||||
},
|
||||
{
|
||||
name: "without alias (last)",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
},
|
||||
want: "ab0dff627d80b9753193d7280bec8f45e8ec6b4cb0912c6fffcf7cd782d9739e",
|
||||
},
|
||||
{
|
||||
name: "always consider alias (last)",
|
||||
alias: source.Alias{
|
||||
Name: "alias",
|
||||
Version: "version",
|
||||
},
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
},
|
||||
want: "fe86c0eecd5654d3c0c0b2176aa394aef6440347c241aa8d9b628dfdde4287cf",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, DeriveImageID(tt.alias, tt.metadata))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ensures same metadata produces identical IDs
|
||||
// regardless of whether the source is stereoscope-based or OCI model-based. Both source types
|
||||
// use DeriveImageID with ImageMetadata
|
||||
// this test captures known-good IDs that must remain
|
||||
// stable across refactors to maintain consistency.
|
||||
//
|
||||
// IMPORTANT: If any of these tests fail after a refactor, it means the artifact ID generation
|
||||
// has changed and will break consistency between stereoscope images and OCI model sources.
|
||||
func TestDeriveImageID_CrossSourceConsistency(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
alias source.Alias
|
||||
metadata source.ImageMetadata
|
||||
wantID artifact.ID
|
||||
}{
|
||||
{
|
||||
name: "raw manifest with layers - typical container image",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "docker.io/library/alpine:latest",
|
||||
ManifestDigest: "sha256:abc123",
|
||||
Layers: []source.LayerMetadata{
|
||||
{Digest: "sha256:layer1", MediaType: "application/vnd.oci.image.layer.v1.tar+gzip", Size: 1000},
|
||||
{Digest: "sha256:layer2", MediaType: "application/vnd.oci.image.layer.v1.tar+gzip", Size: 2000},
|
||||
},
|
||||
RawManifest: []byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}`),
|
||||
},
|
||||
// snapshot: this ID must remain stable for stereoscope/oci-model consistency
|
||||
wantID: "b22c7289dd3b4785a3795c90e15d16bd66bd29b444b8974fe29ed0443ce50405",
|
||||
},
|
||||
{
|
||||
name: "raw manifest only - minimal image",
|
||||
metadata: source.ImageMetadata{
|
||||
RawManifest: []byte(`{"schemaVersion":2}`),
|
||||
},
|
||||
// snapshot: this ID must remain stable
|
||||
wantID: "bafebd36189ad3688b7b3915ea55d461e0bfcfbdde11e54b0a123999fb6be50f",
|
||||
},
|
||||
{
|
||||
name: "chain ID fallback - no raw manifest",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "some-image",
|
||||
Layers: []source.LayerMetadata{
|
||||
{Digest: "sha256:aaa111"},
|
||||
{Digest: "sha256:bbb222"},
|
||||
},
|
||||
},
|
||||
// snapshot: chain ID calculation must remain stable
|
||||
wantID: "0ba9c8d271e6708871505d362e37267c5fb7910066c04d3115b89ba4d34aa180",
|
||||
},
|
||||
{
|
||||
name: "user input fallback - no manifest or layers",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "registry.example.com/org/model:v1.0",
|
||||
},
|
||||
// snapshot: user input hash must remain stable
|
||||
wantID: "a5a8733a3ba3eb99a8ebebcd40c4053f9b896ea6e2217ebc6e885573f20baccf",
|
||||
},
|
||||
{
|
||||
name: "with alias - same image different logical identity",
|
||||
alias: source.Alias{
|
||||
Name: "my-custom-name",
|
||||
Version: "1.0.0",
|
||||
},
|
||||
metadata: source.ImageMetadata{
|
||||
RawManifest: []byte(`{"schemaVersion":2}`),
|
||||
},
|
||||
// snapshot: alias must affect ID deterministically
|
||||
wantID: "9eae41c0efc30023368c29089bac007f2c9d0b40a0ee034081a17c4c22f55ac6",
|
||||
},
|
||||
{
|
||||
name: "annotations has no effect on ID",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "registry.example.com/org/model:v1.0",
|
||||
Annotations: map[string]string{
|
||||
"annotation1": "value1",
|
||||
},
|
||||
},
|
||||
// snapshot: user input hash must remain stable
|
||||
wantID: "a5a8733a3ba3eb99a8ebebcd40c4053f9b896ea6e2217ebc6e885573f20baccf",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := DeriveImageID(tt.alias, tt.metadata)
|
||||
assert.Equal(t, tt.wantID, got, "ID must remain stable for cross-source consistency")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalculateChainID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
layers []source.LayerMetadata
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "empty layers returns empty string",
|
||||
layers: []source.LayerMetadata{},
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "single layer returns digest",
|
||||
layers: []source.LayerMetadata{
|
||||
{Digest: "sha256:abc123"},
|
||||
},
|
||||
want: "sha256:abc123",
|
||||
},
|
||||
{
|
||||
name: "multiple layers calculates chain ID",
|
||||
layers: []source.LayerMetadata{
|
||||
{Digest: "a"},
|
||||
{Digest: "b"},
|
||||
{Digest: "c"},
|
||||
},
|
||||
// snapshot - this value should not change
|
||||
want: "sha256:1dfe230e220ef0e6bc0a8978d23d72b95769e76a62879a5f49267d8c007ab43d",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, calculateChainID(tt.layers))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
package source
|
||||
|
||||
// OCIModelMetadata is an AI model from an OCI registry, which is a specialized form of ImageMetadata.
|
||||
type OCIModelMetadata ImageMetadata
|
||||
@@ -0,0 +1,245 @@
|
||||
package ocimodelsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/google/go-containerregistry/pkg/name"
|
||||
v1 "github.com/google/go-containerregistry/pkg/v1"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/internal/fileresolver"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
"github.com/anchore/syft/syft/source/internal"
|
||||
)
|
||||
|
||||
var _ source.Source = (*ociModelSource)(nil)
|
||||
|
||||
// Config holds the input configuration for an OCI model artifact source.
|
||||
type Config struct {
|
||||
Reference string
|
||||
RegistryOptions *image.RegistryOptions
|
||||
Alias source.Alias
|
||||
}
|
||||
|
||||
// ociModelSource implements the source.Source interface for OCI model artifacts.
|
||||
type ociModelSource struct {
|
||||
id artifact.ID
|
||||
reference string
|
||||
alias source.Alias
|
||||
metadata source.OCIModelMetadata
|
||||
tempDir string
|
||||
resolver interface {
|
||||
file.Resolver
|
||||
file.OCIMediaTypeResolver
|
||||
}
|
||||
mutex *sync.Mutex
|
||||
}
|
||||
|
||||
// NewFromRegistry creates a new OCI model source by fetching the model artifact from a registry.
|
||||
func NewFromRegistry(ctx context.Context, cfg Config) (source.Source, error) {
|
||||
client := newRegistryClient(cfg.RegistryOptions)
|
||||
art, err := validateAndFetchArtifact(ctx, client, cfg.Reference)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
metadata := buildMetadata(art)
|
||||
tempDir, resolver, err := fetchAndStoreGGUFHeaders(ctx, client, art)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
id := internal.DeriveImageID(cfg.Alias, source.ImageMetadata(metadata))
|
||||
return &ociModelSource{
|
||||
id: id,
|
||||
reference: cfg.Reference,
|
||||
alias: cfg.Alias,
|
||||
metadata: metadata,
|
||||
tempDir: tempDir,
|
||||
resolver: resolver,
|
||||
mutex: &sync.Mutex{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// validateAndFetchArtifact fetches and validates a model artifact in a single registry call.
|
||||
func validateAndFetchArtifact(ctx context.Context, client *registryClient, reference string) (*modelArtifact, error) {
|
||||
art, err := client.fetchModelArtifact(ctx, reference)
|
||||
if err != nil {
|
||||
// errNotModelArtifact is wrapped, so callers can use errors.Is() to check
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(art.GGUFLayers) == 0 {
|
||||
return nil, fmt.Errorf("model artifact has no GGUF layers")
|
||||
}
|
||||
|
||||
return art, nil
|
||||
}
|
||||
|
||||
// fetchAndStoreGGUFHeaders fetches GGUF layer headers and stores them in temp files.
|
||||
func fetchAndStoreGGUFHeaders(ctx context.Context, client *registryClient, artifact *modelArtifact) (string, *fileresolver.ContainerImageModel, error) {
|
||||
tempDir, err := os.MkdirTemp("", "syft-oci-gguf")
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("failed to create temp directory: %w", err)
|
||||
}
|
||||
|
||||
layerFiles := make(map[string]fileresolver.LayerInfo)
|
||||
for _, layer := range artifact.GGUFLayers {
|
||||
li, err := fetchSingleGGUFHeader(ctx, client, artifact.Reference, layer, tempDir)
|
||||
if err != nil {
|
||||
osErr := os.RemoveAll(tempDir)
|
||||
if osErr != nil {
|
||||
log.Errorf("unable to remove temp directory (%s): %v", tempDir, err)
|
||||
}
|
||||
return "", nil, err
|
||||
}
|
||||
layerFiles[layer.Digest.String()] = li
|
||||
}
|
||||
|
||||
resolver := fileresolver.NewContainerImageModel(tempDir, layerFiles)
|
||||
|
||||
return tempDir, resolver, nil
|
||||
}
|
||||
|
||||
// fetchSingleGGUFHeader fetches a single GGUF layer header and writes it to a temp file.
|
||||
func fetchSingleGGUFHeader(ctx context.Context, client *registryClient, ref name.Reference, layer v1.Descriptor, tempDir string) (fileresolver.LayerInfo, error) {
|
||||
headerData, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxHeaderBytes)
|
||||
if err != nil {
|
||||
return fileresolver.LayerInfo{}, fmt.Errorf("failed to fetch GGUF layer header: %w", err)
|
||||
}
|
||||
|
||||
digestStr := layer.Digest.String()
|
||||
safeDigest := strings.ReplaceAll(digestStr, ":", "-")
|
||||
tempPath := filepath.Join(tempDir, safeDigest+".gguf")
|
||||
if err := os.WriteFile(tempPath, headerData, 0600); err != nil {
|
||||
return fileresolver.LayerInfo{}, fmt.Errorf("failed to write temp file: %w", err)
|
||||
}
|
||||
|
||||
return fileresolver.LayerInfo{
|
||||
TempPath: tempPath,
|
||||
MediaType: string(layer.MediaType),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// buildMetadata constructs OCIModelMetadata from a modelArtifact.
|
||||
func buildMetadata(artifact *modelArtifact) source.OCIModelMetadata {
|
||||
// layers
|
||||
layers := make([]source.LayerMetadata, len(artifact.Manifest.Layers))
|
||||
for i, layer := range artifact.Manifest.Layers {
|
||||
layers[i] = source.LayerMetadata{
|
||||
MediaType: string(layer.MediaType),
|
||||
Digest: layer.Digest.String(),
|
||||
Size: layer.Size,
|
||||
}
|
||||
}
|
||||
|
||||
// tags
|
||||
var tags []string
|
||||
if tagged, ok := artifact.Reference.(interface{ TagStr() string }); ok {
|
||||
if tag := tagged.TagStr(); tag != "" {
|
||||
tags = []string{tag}
|
||||
}
|
||||
}
|
||||
|
||||
// digests
|
||||
var repoDigests []string
|
||||
if artifact.ManifestDigest != "" {
|
||||
repoDigests = []string{artifact.Reference.Context().String() + "@" + artifact.ManifestDigest}
|
||||
}
|
||||
|
||||
// metadata
|
||||
return source.OCIModelMetadata{
|
||||
UserInput: artifact.Reference.String(),
|
||||
ID: artifact.ManifestDigest,
|
||||
ManifestDigest: artifact.ManifestDigest,
|
||||
MediaType: string(artifact.Manifest.MediaType),
|
||||
Tags: tags,
|
||||
Size: calculateTotalSize(layers),
|
||||
Layers: layers,
|
||||
RawManifest: artifact.RawManifest,
|
||||
RawConfig: artifact.RawConfig,
|
||||
RepoDigests: repoDigests,
|
||||
Architecture: artifact.Config.Architecture,
|
||||
Variant: artifact.Config.Variant,
|
||||
OS: artifact.Config.OS,
|
||||
Labels: artifact.Config.Config.Labels,
|
||||
Annotations: extractManifestAnnotations(artifact.Manifest),
|
||||
}
|
||||
}
|
||||
|
||||
// extractManifestAnnotations extracts annotations from the manifest.
|
||||
func extractManifestAnnotations(manifest *v1.Manifest) map[string]string {
|
||||
if manifest == nil || manifest.Annotations == nil {
|
||||
return make(map[string]string)
|
||||
}
|
||||
return manifest.Annotations
|
||||
}
|
||||
|
||||
// calculateTotalSize sums up the size of all layers.
|
||||
func calculateTotalSize(layers []source.LayerMetadata) int64 {
|
||||
var total int64
|
||||
for _, layer := range layers {
|
||||
total += layer.Size
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// ID returns the artifact ID.
|
||||
func (s *ociModelSource) ID() artifact.ID {
|
||||
return s.id
|
||||
}
|
||||
|
||||
// Describe returns a description of the source.
|
||||
func (s *ociModelSource) Describe() source.Description {
|
||||
name := s.reference
|
||||
version := ""
|
||||
supplier := ""
|
||||
|
||||
if !s.alias.IsEmpty() {
|
||||
if s.alias.Name != "" {
|
||||
name = s.alias.Name
|
||||
}
|
||||
if s.alias.Version != "" {
|
||||
version = s.alias.Version
|
||||
}
|
||||
if s.alias.Supplier != "" {
|
||||
supplier = s.alias.Supplier
|
||||
}
|
||||
}
|
||||
|
||||
return source.Description{
|
||||
ID: string(s.id),
|
||||
Name: name,
|
||||
Version: version,
|
||||
Supplier: supplier,
|
||||
Metadata: s.metadata,
|
||||
}
|
||||
}
|
||||
|
||||
// FileResolver returns a file resolver for accessing header of GGUF files.
|
||||
func (s *ociModelSource) FileResolver(_ source.Scope) (file.Resolver, error) {
|
||||
return s.resolver, nil
|
||||
}
|
||||
|
||||
// Close cleans up temporary files. Safe to call multiple times.
|
||||
func (s *ociModelSource) Close() error {
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
|
||||
if s.tempDir == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := os.RemoveAll(s.tempDir)
|
||||
s.tempDir = ""
|
||||
s.resolver = nil
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package ocimodelsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
type ociModelSourceProvider struct {
|
||||
reference string
|
||||
registryOpts *image.RegistryOptions
|
||||
alias source.Alias
|
||||
}
|
||||
|
||||
// NewSourceProvider creates a new OCI model artifact source provider.
|
||||
func NewSourceProvider(reference string, registryOpts *image.RegistryOptions, alias source.Alias) source.Provider {
|
||||
return &ociModelSourceProvider{
|
||||
reference: reference,
|
||||
registryOpts: registryOpts,
|
||||
alias: alias,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ociModelSourceProvider) Name() string {
|
||||
return "oci-model"
|
||||
}
|
||||
|
||||
func (p *ociModelSourceProvider) Provide(ctx context.Context) (source.Source, error) {
|
||||
cfg := Config{
|
||||
Reference: p.reference,
|
||||
RegistryOptions: p.registryOpts,
|
||||
Alias: p.alias,
|
||||
}
|
||||
return NewFromRegistry(ctx, cfg)
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
package ocimodelsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/google/go-containerregistry/pkg/authn"
|
||||
"github.com/google/go-containerregistry/pkg/name"
|
||||
v1 "github.com/google/go-containerregistry/pkg/v1"
|
||||
"github.com/google/go-containerregistry/pkg/v1/remote"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
)
|
||||
|
||||
// errNotModelArtifact is returned when a reference does not point to a model artifact.
|
||||
var errNotModelArtifact = errors.New("not an OCI model artifact")
|
||||
|
||||
const (
|
||||
// Model artifact media types as per Docker's OCI artifacts for AI model packaging
|
||||
// Reference: https://www.docker.com/blog/oci-artifacts-for-ai-model-packaging/
|
||||
modelConfigMediaTypePrefix = "application/vnd.docker.ai.model.config."
|
||||
ggufLayerMediaType = "application/vnd.docker.ai.gguf.v3"
|
||||
|
||||
// Maximum bytes to read/return for GGUF headers
|
||||
maxHeaderBytes = 8 * 1024 * 1024 // 8 MB
|
||||
)
|
||||
|
||||
// registryClient handles OCI registry interactions for model artifacts.
|
||||
type registryClient struct {
|
||||
options []remote.Option
|
||||
}
|
||||
|
||||
// newRegistryClient creates a new registry client with authentication from RegistryOptions.
|
||||
func newRegistryClient(registryOpts *image.RegistryOptions) *registryClient {
|
||||
opts := buildRemoteOptions(registryOpts)
|
||||
|
||||
return ®istryClient{
|
||||
options: opts,
|
||||
}
|
||||
}
|
||||
|
||||
// buildRemoteOptions converts stereoscope RegistryOptions to go-containerregistry remote.Options.
|
||||
func buildRemoteOptions(registryOpts *image.RegistryOptions) []remote.Option {
|
||||
var opts []remote.Option
|
||||
|
||||
if registryOpts == nil {
|
||||
return opts
|
||||
}
|
||||
|
||||
// Build authenticator
|
||||
authenticator := buildAuthenticator(registryOpts)
|
||||
opts = append(opts, remote.WithAuth(authenticator))
|
||||
|
||||
// Handle TLS settings
|
||||
if registryOpts.InsecureSkipTLSVerify {
|
||||
if transport, ok := remote.DefaultTransport.(*http.Transport); ok {
|
||||
transport = transport.Clone()
|
||||
if transport.TLSClientConfig == nil {
|
||||
transport.TLSClientConfig = &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
}
|
||||
}
|
||||
transport.TLSClientConfig.InsecureSkipVerify = true //#nosec G402 -- user explicitly requested insecure TLS
|
||||
opts = append(opts, remote.WithTransport(transport))
|
||||
}
|
||||
}
|
||||
|
||||
// Handle insecure HTTP
|
||||
if registryOpts.InsecureUseHTTP {
|
||||
opts = append(opts, remote.WithTransport(http.DefaultTransport))
|
||||
}
|
||||
|
||||
return opts
|
||||
}
|
||||
|
||||
// buildAuthenticator creates an authn.Authenticator from RegistryOptions.
|
||||
func buildAuthenticator(registryOpts *image.RegistryOptions) authn.Authenticator {
|
||||
// If credentials are provided, use them
|
||||
if len(registryOpts.Credentials) > 0 {
|
||||
// Use the first credential set (we could enhance this to match by authority)
|
||||
cred := registryOpts.Credentials[0]
|
||||
|
||||
if cred.Token != "" {
|
||||
return &authn.Bearer{Token: cred.Token}
|
||||
}
|
||||
|
||||
if cred.Username != "" || cred.Password != "" {
|
||||
return &authn.Basic{
|
||||
Username: cred.Username,
|
||||
Password: cred.Password,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to anonymous authenticator
|
||||
return authn.Anonymous
|
||||
}
|
||||
|
||||
// modelArtifact represents a parsed OCI model artifact.
|
||||
type modelArtifact struct {
|
||||
Reference name.Reference
|
||||
Manifest *v1.Manifest
|
||||
Config *v1.ConfigFile
|
||||
RawManifest []byte
|
||||
RawConfig []byte
|
||||
ManifestDigest string
|
||||
GGUFLayers []v1.Descriptor
|
||||
}
|
||||
|
||||
func (c *registryClient) fetchModelArtifact(ctx context.Context, refStr string) (*modelArtifact, error) {
|
||||
ref, err := name.ParseReference(refStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse reference %q: %w", refStr, err)
|
||||
}
|
||||
|
||||
opts := c.options
|
||||
opts = append(opts, remote.WithContext(ctx))
|
||||
desc, err := remote.Get(ref, opts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch descriptor: %w", err)
|
||||
}
|
||||
|
||||
manifest := &v1.Manifest{}
|
||||
if err := json.Unmarshal(desc.Manifest, manifest); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal manifest: %w", err)
|
||||
}
|
||||
|
||||
if !isModelArtifact(manifest) {
|
||||
return nil, fmt.Errorf("%w (config media type: %s)", errNotModelArtifact, manifest.Config.MediaType)
|
||||
}
|
||||
|
||||
img, err := desc.Image()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get image: %w", err)
|
||||
}
|
||||
|
||||
configFile, err := img.ConfigFile()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get config file: %w", err)
|
||||
}
|
||||
|
||||
rawConfig, err := img.RawConfigFile()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get raw config: %w", err)
|
||||
}
|
||||
|
||||
ggufLayers := extractGGUFLayers(manifest)
|
||||
|
||||
return &modelArtifact{
|
||||
Reference: ref,
|
||||
Manifest: manifest,
|
||||
Config: configFile,
|
||||
RawManifest: desc.Manifest,
|
||||
RawConfig: rawConfig,
|
||||
ManifestDigest: desc.Digest.String(),
|
||||
GGUFLayers: ggufLayers,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// isModelArtifact checks if the manifest represents a model artifact.
|
||||
func isModelArtifact(manifest *v1.Manifest) bool {
|
||||
return strings.HasPrefix(string(manifest.Config.MediaType), modelConfigMediaTypePrefix)
|
||||
}
|
||||
|
||||
// extractGGUFLayers extracts GGUF layer descriptors from the manifest.
|
||||
func extractGGUFLayers(manifest *v1.Manifest) []v1.Descriptor {
|
||||
var ggufLayers []v1.Descriptor
|
||||
for _, layer := range manifest.Layers {
|
||||
if string(layer.MediaType) == ggufLayerMediaType {
|
||||
ggufLayers = append(ggufLayers, layer)
|
||||
}
|
||||
}
|
||||
return ggufLayers
|
||||
}
|
||||
|
||||
func (c *registryClient) fetchBlobRange(ctx context.Context, ref name.Reference, digest v1.Hash, maxBytes int64) ([]byte, error) {
|
||||
repo := ref.Context()
|
||||
|
||||
opts := c.options
|
||||
opts = append(opts, remote.WithContext(ctx))
|
||||
layer, err := remote.Layer(repo.Digest(digest.String()), opts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch layer: %w", err)
|
||||
}
|
||||
|
||||
reader, err := layer.Compressed()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get layer reader: %w", err)
|
||||
}
|
||||
// this defer is what causes the download to stop
|
||||
// 1. io.ReadFull(reader, data) reads exactly 8MB into the buffer
|
||||
// 2. The function returns with data[:n]
|
||||
// 3. defer reader.Close() executes, closing the HTTP response body
|
||||
// 4. Closing the response body closes the underlying TCP connection
|
||||
// 5. The server receives TCP FIN/RST and stops sending
|
||||
// note: some data is already in flight when we close so we will see > 8mb over the wire
|
||||
// the full image will not download given we terminate the reader early here
|
||||
defer reader.Close()
|
||||
|
||||
// Note: this is not some arbitrary number picked out of the blue.
|
||||
// This is based on the specification of header data found here:
|
||||
// https://github.com/ggml-org/ggml/blob/master/docs/gguf.md#file-structure
|
||||
data := make([]byte, maxBytes)
|
||||
n, err := io.ReadFull(reader, data)
|
||||
if err != nil && err != io.ErrUnexpectedEOF {
|
||||
// ErrUnexpectedEOF is okay - it means the file is smaller than maxBytes
|
||||
return nil, fmt.Errorf("failed to read layer data: %w", err)
|
||||
}
|
||||
|
||||
return data[:n], nil
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package ocimodelsource
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
v1 "github.com/google/go-containerregistry/pkg/v1"
|
||||
"github.com/google/go-containerregistry/pkg/v1/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestIsModelArtifact(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
manifest *v1.Manifest
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "valid model artifact",
|
||||
manifest: &v1.Manifest{
|
||||
Config: v1.Descriptor{
|
||||
MediaType: modelConfigMediaTypePrefix + "v1+json",
|
||||
},
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "container image",
|
||||
manifest: &v1.Manifest{
|
||||
Config: v1.Descriptor{
|
||||
MediaType: types.DockerConfigJSON,
|
||||
},
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "empty media type",
|
||||
manifest: &v1.Manifest{
|
||||
Config: v1.Descriptor{
|
||||
MediaType: "",
|
||||
},
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
result := isModelArtifact(test.manifest)
|
||||
assert.Equal(t, test.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractGGUFLayers(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
manifest *v1.Manifest
|
||||
expected int
|
||||
}{
|
||||
{
|
||||
name: "single GGUF layer",
|
||||
manifest: &v1.Manifest{
|
||||
Layers: []v1.Descriptor{
|
||||
{MediaType: types.MediaType(ggufLayerMediaType), Digest: v1.Hash{Algorithm: "sha256", Hex: "abc"}},
|
||||
},
|
||||
},
|
||||
expected: 1,
|
||||
},
|
||||
{
|
||||
name: "multiple GGUF layers",
|
||||
manifest: &v1.Manifest{
|
||||
Layers: []v1.Descriptor{
|
||||
{MediaType: types.MediaType(ggufLayerMediaType), Digest: v1.Hash{Algorithm: "sha256", Hex: "abc"}},
|
||||
{MediaType: types.MediaType(ggufLayerMediaType), Digest: v1.Hash{Algorithm: "sha256", Hex: "def"}},
|
||||
},
|
||||
},
|
||||
expected: 2,
|
||||
},
|
||||
{
|
||||
name: "mixed layers",
|
||||
manifest: &v1.Manifest{
|
||||
Layers: []v1.Descriptor{
|
||||
{MediaType: types.MediaType(ggufLayerMediaType), Digest: v1.Hash{Algorithm: "sha256", Hex: "abc"}},
|
||||
{MediaType: types.DockerLayer, Digest: v1.Hash{Algorithm: "sha256", Hex: "def"}},
|
||||
{MediaType: types.MediaType(ggufLayerMediaType), Digest: v1.Hash{Algorithm: "sha256", Hex: "ghi"}},
|
||||
},
|
||||
},
|
||||
expected: 2,
|
||||
},
|
||||
{
|
||||
name: "no GGUF layers",
|
||||
manifest: &v1.Manifest{
|
||||
Layers: []v1.Descriptor{
|
||||
{MediaType: types.DockerLayer},
|
||||
},
|
||||
},
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "empty layers",
|
||||
manifest: &v1.Manifest{
|
||||
Layers: []v1.Descriptor{},
|
||||
},
|
||||
expected: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
result := extractGGUFLayers(test.manifest)
|
||||
assert.Len(t, result, test.expected)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"github.com/anchore/syft/syft/source"
|
||||
"github.com/anchore/syft/syft/source/directorysource"
|
||||
"github.com/anchore/syft/syft/source/filesource"
|
||||
"github.com/anchore/syft/syft/source/ocimodelsource"
|
||||
"github.com/anchore/syft/syft/source/snapsource"
|
||||
"github.com/anchore/syft/syft/source/stereoscopesource"
|
||||
)
|
||||
@@ -43,6 +44,14 @@ func All(userInput string, cfg *Config) []collections.TaggedValue[source.Provide
|
||||
// --from docker, registry, etc.
|
||||
Join(stereoscopeProviders.Select(PullTag)...).
|
||||
|
||||
// --from oci-model, registry (for select cases only)
|
||||
// OCI model artifacts with header-only fetching
|
||||
// note: we don't want to use the "pull" tag since it's not actually pulling the full image,
|
||||
// instead we want to match on registry since these models are stored in OCI registries.
|
||||
// This does mean that this must be placed after the pull provider, which is ideal since we don't want to
|
||||
// unnecessarily pull registry headers first if the more common case is the pull providers.
|
||||
Join(tagProvider(ocimodelsource.NewSourceProvider(userInput, cfg.RegistryOptions, cfg.Alias), "registry")).
|
||||
|
||||
// --from snap (remote only)
|
||||
Join(tagProvider(snapsource.NewRemoteSourceProvider(userInput, cfg.Exclude, cfg.DigestAlgorithms, cfg.Alias), SnapTag))
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
"github.com/distribution/reference"
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
@@ -36,7 +35,7 @@ type stereoscopeImageSource struct {
|
||||
func New(img *image.Image, cfg ImageConfig) source.Source {
|
||||
metadata := imageMetadataFromStereoscopeImage(img, cfg.Reference)
|
||||
return &stereoscopeImageSource{
|
||||
id: deriveIDFromStereoscopeImage(cfg.Alias, metadata),
|
||||
id: internal.DeriveImageID(cfg.Alias, metadata),
|
||||
config: cfg,
|
||||
image: img,
|
||||
metadata: metadata,
|
||||
@@ -163,61 +162,6 @@ func imageMetadataFromStereoscopeImage(img *image.Image, reference string) sourc
|
||||
}
|
||||
}
|
||||
|
||||
// deriveIDFromStereoscopeImage derives an artifact ID from the given image metadata. The order of data precedence is:
|
||||
// 1. prefer a digest of the raw container image manifest
|
||||
// 2. if no manifest digest is available, calculate a chain ID from the image layer metadata
|
||||
// 3. if no layer metadata is available, use the user input string
|
||||
//
|
||||
// in all cases, if an alias is provided, it is additionally considered in the ID calculation. This allows for the
|
||||
// same image to be scanned multiple times with different aliases and be considered logically different.
|
||||
func deriveIDFromStereoscopeImage(alias source.Alias, metadata source.ImageMetadata) artifact.ID {
|
||||
var input string
|
||||
|
||||
if len(metadata.RawManifest) > 0 {
|
||||
input = digest.Canonical.FromBytes(metadata.RawManifest).String()
|
||||
} else {
|
||||
// calculate chain ID for image sources where manifestDigest is not available
|
||||
// https://github.com/opencontainers/image-spec/blob/main/config.md#layer-chainid
|
||||
input = calculateChainID(metadata.Layers)
|
||||
if input == "" {
|
||||
// TODO what happens here if image has no layers?
|
||||
// is this case possible?
|
||||
input = digest.Canonical.FromString(metadata.UserInput).String()
|
||||
}
|
||||
}
|
||||
|
||||
if !alias.IsEmpty() {
|
||||
// if the user provided an alias, we want to consider that in the artifact ID. This way if the user
|
||||
// scans the same item but is considered to be logically different, then ID will express that.
|
||||
aliasStr := fmt.Sprintf(":%s@%s", alias.Name, alias.Version)
|
||||
input = digest.Canonical.FromString(input + aliasStr).String()
|
||||
}
|
||||
|
||||
return internal.ArtifactIDFromDigest(input)
|
||||
}
|
||||
|
||||
func calculateChainID(lm []source.LayerMetadata) string {
|
||||
if len(lm) < 1 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// DiffID(L0) = digest of layer 0
|
||||
// https://github.com/anchore/stereoscope/blob/1b1b744a919964f38d14e1416fb3f25221b761ce/pkg/image/layer_metadata.go#L19-L32
|
||||
chainID := lm[0].Digest
|
||||
id := chain(chainID, lm[1:])
|
||||
|
||||
return id
|
||||
}
|
||||
|
||||
func chain(chainID string, layers []source.LayerMetadata) string {
|
||||
if len(layers) < 1 {
|
||||
return chainID
|
||||
}
|
||||
|
||||
chainID = digest.Canonical.FromString(layers[0].Digest + " " + chainID).String()
|
||||
return chain(chainID, layers[1:])
|
||||
}
|
||||
|
||||
func getImageExclusionFunction(exclusions []string) func(string) bool {
|
||||
if len(exclusions) == 0 {
|
||||
return nil
|
||||
|
||||
@@ -2,8 +2,6 @@ package stereoscopesource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -12,7 +10,6 @@ import (
|
||||
|
||||
"github.com/anchore/stereoscope"
|
||||
"github.com/anchore/stereoscope/pkg/imagetest"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/internal/testutil"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
@@ -112,146 +109,6 @@ func Test_StereoscopeImage_Exclusions(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func Test_StereoscopeImageSource_ID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
alias source.Alias
|
||||
metadata source.ImageMetadata
|
||||
want artifact.ID
|
||||
}{
|
||||
{
|
||||
name: "use raw manifest over chain ID or user input",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
Layers: []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
RawManifest: []byte("raw-manifest"),
|
||||
},
|
||||
want: func() artifact.ID {
|
||||
hasher := sha256.New()
|
||||
hasher.Write([]byte("raw-manifest"))
|
||||
return artifact.ID(fmt.Sprintf("%x", hasher.Sum(nil)))
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "use chain ID over user input",
|
||||
metadata: source.ImageMetadata{
|
||||
//UserInput: "user-input",
|
||||
Layers: []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
},
|
||||
want: func() artifact.ID {
|
||||
metadata := []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
}
|
||||
return artifact.ID(strings.TrimPrefix(calculateChainID(metadata), "sha256:"))
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "use user input last",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
},
|
||||
want: func() artifact.ID {
|
||||
hasher := sha256.New()
|
||||
hasher.Write([]byte("user-input"))
|
||||
return artifact.ID(fmt.Sprintf("%x", hasher.Sum(nil)))
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "without alias (first)",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
Layers: []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
RawManifest: []byte("raw-manifest"),
|
||||
},
|
||||
want: "85298926ecd92ed57688f13039017160cd728f04dd0d2d10a10629007106f107",
|
||||
},
|
||||
{
|
||||
name: "always consider alias (first)",
|
||||
alias: source.Alias{
|
||||
Name: "alias",
|
||||
Version: "version",
|
||||
},
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
Layers: []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
RawManifest: []byte("raw-manifest"),
|
||||
},
|
||||
want: "a8717e42449960c1dd4963f2f22bd69c7c105e7e82445be0a65aa1825d62ff0d",
|
||||
},
|
||||
{
|
||||
name: "without alias (last)",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
},
|
||||
want: "ab0dff627d80b9753193d7280bec8f45e8ec6b4cb0912c6fffcf7cd782d9739e",
|
||||
},
|
||||
{
|
||||
name: "always consider alias (last)",
|
||||
alias: source.Alias{
|
||||
Name: "alias",
|
||||
Version: "version",
|
||||
},
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
},
|
||||
want: "fe86c0eecd5654d3c0c0b2176aa394aef6440347c241aa8d9b628dfdde4287cf",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, deriveIDFromStereoscopeImage(tt.alias, tt.metadata))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_Describe(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
Reference in New Issue
Block a user