mirror of
https://github.com/anchore/syft.git
synced 2025-11-17 16:33:21 +01:00
fix: first pass pr fixes
Signed-off-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com>
This commit is contained in:
parent
08c0572fb7
commit
f664f9eaf2
@ -124,7 +124,7 @@ var jsonTypes = makeJSONTypes(
|
|||||||
jsonNames(pkg.TerraformLockProviderEntry{}, "terraform-lock-provider-entry"),
|
jsonNames(pkg.TerraformLockProviderEntry{}, "terraform-lock-provider-entry"),
|
||||||
jsonNames(pkg.DotnetPackagesLockEntry{}, "dotnet-packages-lock-entry"),
|
jsonNames(pkg.DotnetPackagesLockEntry{}, "dotnet-packages-lock-entry"),
|
||||||
jsonNames(pkg.CondaMetaPackage{}, "conda-metadata-entry", "CondaPackageMetadata"),
|
jsonNames(pkg.CondaMetaPackage{}, "conda-metadata-entry", "CondaPackageMetadata"),
|
||||||
jsonNames(pkg.GGUFFileMetadata{}, "gguf-file-metadata", "GGUFFileMetadata"),
|
jsonNames(pkg.GGUFFileMetadata{}, "gguf-file-metadata"),
|
||||||
)
|
)
|
||||||
|
|
||||||
func expandLegacyNameVariants(names ...string) []string {
|
func expandLegacyNameVariants(names ...string) []string {
|
||||||
|
|||||||
@ -179,7 +179,7 @@ func DefaultPackageTaskFactories() Factories {
|
|||||||
newSimplePackageTaskFactory(homebrew.NewCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, "homebrew"),
|
newSimplePackageTaskFactory(homebrew.NewCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, "homebrew"),
|
||||||
newSimplePackageTaskFactory(conda.NewCondaMetaCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.PackageTag, "conda"),
|
newSimplePackageTaskFactory(conda.NewCondaMetaCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.PackageTag, "conda"),
|
||||||
newSimplePackageTaskFactory(snap.NewCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, "snap"),
|
newSimplePackageTaskFactory(snap.NewCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, "snap"),
|
||||||
newSimplePackageTaskFactory(aiartifact.NewGGUFCataloger, pkgcataloging.DirectoryTag, pkgcataloging.ImageTag, "ai-artifact", "model", "gguf", "ml"),
|
newSimplePackageTaskFactory(aiartifact.NewGGUFCataloger, pkgcataloging.DirectoryTag, pkgcataloging.ImageTag, "ai", "model", "gguf", "ml"),
|
||||||
|
|
||||||
// deprecated catalogers ////////////////////////////////////////
|
// deprecated catalogers ////////////////////////////////////////
|
||||||
// these are catalogers that should not be selectable other than specific inclusion via name or "deprecated" tag (to remain backwards compatible)
|
// these are catalogers that should not be selectable other than specific inclusion via name or "deprecated" tag (to remain backwards compatible)
|
||||||
|
|||||||
@ -83,7 +83,7 @@ func SourceInfo(p pkg.Package) string {
|
|||||||
case pkg.TerraformPkg:
|
case pkg.TerraformPkg:
|
||||||
answer = "acquired package info from Terraform dependency lock file"
|
answer = "acquired package info from Terraform dependency lock file"
|
||||||
case pkg.ModelPkg:
|
case pkg.ModelPkg:
|
||||||
answer = "acquired package info from AI artifact"
|
answer = "acquired package info from AI artifact (e.g. GGUF File"
|
||||||
default:
|
default:
|
||||||
answer = "acquired package info from the following paths"
|
answer = "acquired package info from the following paths"
|
||||||
}
|
}
|
||||||
|
|||||||
@ -11,6 +11,6 @@ import (
|
|||||||
|
|
||||||
// NewGGUFCataloger returns a new cataloger instance for GGUF model files.
|
// NewGGUFCataloger returns a new cataloger instance for GGUF model files.
|
||||||
func NewGGUFCataloger() pkg.Cataloger {
|
func NewGGUFCataloger() pkg.Cataloger {
|
||||||
return generic.NewCataloger("gguf-cataloger").
|
return generic.NewCataloger("model-gguf-cataloger").
|
||||||
WithParserByGlobs(parseGGUFModel, "**/*.gguf")
|
WithParserByGlobs(parseGGUFModel, "**/*.gguf")
|
||||||
}
|
}
|
||||||
|
|||||||
@ -75,14 +75,10 @@ func TestGGUFCataloger_Globs(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
fixtureDir := tt.setup(t)
|
fixtureDir := tt.setup(t)
|
||||||
|
|
||||||
tester := pkgtest.NewCatalogTester().
|
pkgtest.NewCatalogTester().
|
||||||
FromDirectory(t, fixtureDir)
|
FromDirectory(t, fixtureDir).
|
||||||
|
ExpectsResolverContentQueries(tt.expected).
|
||||||
if len(tt.expected) > 0 {
|
TestCataloger(t, NewGGUFCataloger())
|
||||||
tester.ExpectsResolverContentQueries(tt.expected)
|
|
||||||
}
|
|
||||||
|
|
||||||
tester.TestCataloger(t, NewGGUFCataloger())
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -29,7 +29,7 @@ func TestNewGGUFPackage(t *testing.T) {
|
|||||||
Parameters: 8030000000,
|
Parameters: 8030000000,
|
||||||
GGUFVersion: 3,
|
GGUFVersion: 3,
|
||||||
TensorCount: 291,
|
TensorCount: 291,
|
||||||
Header: map[string]interface{}{},
|
Header: map[string]any{},
|
||||||
TruncatedHeader: false,
|
TruncatedHeader: false,
|
||||||
},
|
},
|
||||||
locations: []file.Location{file.NewLocation("/models/llama3-8b.gguf")},
|
locations: []file.Location{file.NewLocation("/models/llama3-8b.gguf")},
|
||||||
@ -101,85 +101,3 @@ func TestNewGGUFPackage(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewGGUFPackage_IDUniqueness(t *testing.T) {
|
|
||||||
// Test that different packages get different IDs
|
|
||||||
metadata1 := &pkg.GGUFFileMetadata{
|
|
||||||
ModelFormat: "gguf",
|
|
||||||
ModelName: "model-1",
|
|
||||||
ModelVersion: "1.0",
|
|
||||||
Architecture: "llama",
|
|
||||||
GGUFVersion: 3,
|
|
||||||
TensorCount: 100,
|
|
||||||
}
|
|
||||||
|
|
||||||
metadata2 := &pkg.GGUFFileMetadata{
|
|
||||||
ModelFormat: "gguf",
|
|
||||||
ModelName: "model-2",
|
|
||||||
ModelVersion: "1.0",
|
|
||||||
Architecture: "llama",
|
|
||||||
GGUFVersion: 3,
|
|
||||||
TensorCount: 100,
|
|
||||||
}
|
|
||||||
|
|
||||||
loc := file.NewLocation("/models/test.gguf")
|
|
||||||
p1 := newGGUFPackage(metadata1, loc)
|
|
||||||
p2 := newGGUFPackage(metadata2, loc)
|
|
||||||
|
|
||||||
assert.NotEqual(t, p1.ID(), p2.ID(), "different packages should have different IDs")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewGGUFPackage_IDConsistency(t *testing.T) {
|
|
||||||
// Test that same metadata produces same ID
|
|
||||||
metadata := &pkg.GGUFFileMetadata{
|
|
||||||
ModelFormat: "gguf",
|
|
||||||
ModelName: "test-model",
|
|
||||||
ModelVersion: "1.0",
|
|
||||||
Architecture: "llama",
|
|
||||||
GGUFVersion: 3,
|
|
||||||
TensorCount: 100,
|
|
||||||
}
|
|
||||||
|
|
||||||
loc := file.NewLocation("/models/test.gguf")
|
|
||||||
p1 := newGGUFPackage(metadata, loc)
|
|
||||||
p2 := newGGUFPackage(metadata, loc)
|
|
||||||
|
|
||||||
assert.Equal(t, p1.ID(), p2.ID(), "identical packages should have identical IDs")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewGGUFPackage_MetadataPreservation(t *testing.T) {
|
|
||||||
// Ensure all metadata fields are preserved in the package
|
|
||||||
metadata := &pkg.GGUFFileMetadata{
|
|
||||||
ModelFormat: "gguf",
|
|
||||||
ModelName: "preservation-test",
|
|
||||||
ModelVersion: "2.0",
|
|
||||||
License: "MIT",
|
|
||||||
Architecture: "llama",
|
|
||||||
Quantization: "Q4_K_M",
|
|
||||||
Parameters: 7000000000,
|
|
||||||
GGUFVersion: 3,
|
|
||||||
TensorCount: 219,
|
|
||||||
Hash: "abc123",
|
|
||||||
Header: map[string]interface{}{"custom.field": "value"},
|
|
||||||
TruncatedHeader: false,
|
|
||||||
}
|
|
||||||
|
|
||||||
loc := file.NewLocation("/models/test.gguf")
|
|
||||||
p := newGGUFPackage(metadata, loc)
|
|
||||||
|
|
||||||
extractedMetadata, ok := p.Metadata.(pkg.GGUFFileMetadata)
|
|
||||||
require.True(t, ok)
|
|
||||||
|
|
||||||
assert.Equal(t, metadata.ModelFormat, extractedMetadata.ModelFormat)
|
|
||||||
assert.Equal(t, metadata.ModelName, extractedMetadata.ModelName)
|
|
||||||
assert.Equal(t, metadata.ModelVersion, extractedMetadata.ModelVersion)
|
|
||||||
assert.Equal(t, metadata.License, extractedMetadata.License)
|
|
||||||
assert.Equal(t, metadata.Architecture, extractedMetadata.Architecture)
|
|
||||||
assert.Equal(t, metadata.Quantization, extractedMetadata.Quantization)
|
|
||||||
assert.Equal(t, metadata.Parameters, extractedMetadata.Parameters)
|
|
||||||
assert.Equal(t, metadata.GGUFVersion, extractedMetadata.GGUFVersion)
|
|
||||||
assert.Equal(t, metadata.TensorCount, extractedMetadata.TensorCount)
|
|
||||||
assert.Equal(t, metadata.Hash, extractedMetadata.Hash)
|
|
||||||
assert.Equal(t, metadata.TruncatedHeader, extractedMetadata.TruncatedHeader)
|
|
||||||
assert.Equal(t, metadata.Header, extractedMetadata.Header)
|
|
||||||
}
|
|
||||||
|
|||||||
@ -41,7 +41,7 @@ const (
|
|||||||
ggufTypeArray = 12
|
ggufTypeArray = 12
|
||||||
)
|
)
|
||||||
|
|
||||||
const unkownGGUFData = "unknown"
|
const unknownGGUFData = "unknown"
|
||||||
|
|
||||||
//nolint:funlen
|
//nolint:funlen
|
||||||
func parseGGUFHeader(data []byte, location string) (*pkg.GGUFFileMetadata, error) {
|
func parseGGUFHeader(data []byte, location string) (*pkg.GGUFFileMetadata, error) {
|
||||||
@ -135,7 +135,7 @@ func parseGGUFHeader(data []byte, location string) (*pkg.GGUFFileMetadata, error
|
|||||||
metadata.ModelVersion = version
|
metadata.ModelVersion = version
|
||||||
delete(kvMap, "general.version")
|
delete(kvMap, "general.version")
|
||||||
} else {
|
} else {
|
||||||
metadata.ModelVersion = unkownGGUFData
|
metadata.ModelVersion = unknownGGUFData
|
||||||
}
|
}
|
||||||
|
|
||||||
// Extract parameters count if present
|
// Extract parameters count if present
|
||||||
@ -153,7 +153,7 @@ func parseGGUFHeader(data []byte, location string) (*pkg.GGUFFileMetadata, error
|
|||||||
metadata.Quantization = inferQuantizationFromFilename(location)
|
metadata.Quantization = inferQuantizationFromFilename(location)
|
||||||
// Note: we keep general.quantized_by in Header since it's not directly mapped to a field
|
// Note: we keep general.quantized_by in Header since it's not directly mapped to a field
|
||||||
} else {
|
} else {
|
||||||
metadata.Quantization = unkownGGUFData
|
metadata.Quantization = unknownGGUFData
|
||||||
}
|
}
|
||||||
|
|
||||||
// Compute hash of metadata for stable identifier
|
// Compute hash of metadata for stable identifier
|
||||||
@ -310,7 +310,7 @@ func inferQuantizationFromFilename(filename string) string {
|
|||||||
if match := quantPattern.FindString(filename); match != "" {
|
if match := quantPattern.FindString(filename); match != "" {
|
||||||
return match
|
return match
|
||||||
}
|
}
|
||||||
return unkownGGUFData
|
return unknownGGUFData
|
||||||
}
|
}
|
||||||
|
|
||||||
// computeMetadataHash computes a stable hash of the metadata for use as a global identifier
|
// computeMetadataHash computes a stable hash of the metadata for use as a global identifier
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user