chore: pr comments

Signed-off-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com>
This commit is contained in:
Christopher Phillips 2025-11-13 00:32:08 -05:00
parent cdb41b0c76
commit b80592f735
No known key found for this signature in database
6 changed files with 55 additions and 74 deletions

View File

@ -14,6 +14,28 @@ import (
) )
func TestGGUFCataloger_Globs(t *testing.T) { func TestGGUFCataloger_Globs(t *testing.T) {
tests := []struct {
name string
fixture string
expected []string
}{
{
name: "obtain gguf files",
fixture: "test-fixtures/glob-paths",
expected: []string{
"models/model.gguf",
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
pkgtest.NewCatalogTester().
FromDirectory(t, test.fixture).
ExpectsResolverContentQueries(test.expected).
TestCataloger(t, NewGGUFCataloger())
})
}
} }
func TestGGUFCataloger_Integration(t *testing.T) { func TestGGUFCataloger_Integration(t *testing.T) {
@ -50,15 +72,15 @@ func TestGGUFCataloger_Integration(t *testing.T) {
pkg.NewLicenseFromFields("Apache-2.0", "", nil), pkg.NewLicenseFromFields("Apache-2.0", "", nil),
), ),
Metadata: pkg.GGUFFileHeader{ Metadata: pkg.GGUFFileHeader{
ModelName: "llama3-8b", ModelName: "llama3-8b",
ModelVersion: "3.0", ModelVersion: "3.0",
License: "Apache-2.0", License: "Apache-2.0",
Architecture: "llama", Architecture: "llama",
Quantization: "Unknown", Quantization: "Unknown",
Parameters: 0, Parameters: 0,
GGUFVersion: 3, GGUFVersion: 3,
TensorCount: 0, TensorCount: 0,
Header: map[string]interface{}{}, Header: map[string]interface{}{},
}, },
}, },
}, },
@ -77,8 +99,8 @@ func TestGGUFCataloger_Integration(t *testing.T) {
IgnoreLocationLayer(). IgnoreLocationLayer().
IgnorePackageFields("FoundBy", "Locations"). // These are set by the cataloger IgnorePackageFields("FoundBy", "Locations"). // These are set by the cataloger
WithCompareOptions( WithCompareOptions(
// Ignore Hash as it's computed dynamically // Ignore MetadataHash as it's computed dynamically
cmpopts.IgnoreFields(pkg.GGUFFileHeader{}, "Hash"), cmpopts.IgnoreFields(pkg.GGUFFileHeader{}, "MetadataHash"),
) )
tester.TestCataloger(t, NewGGUFCataloger()) tester.TestCataloger(t, NewGGUFCataloger())

View File

@ -1,22 +1,11 @@
package ai package ai
import ( import (
"encoding/json"
"fmt"
"github.com/cespare/xxhash/v2"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
) )
func newGGUFPackage(metadata *pkg.GGUFFileHeader, locations ...file.Location) pkg.Package { func newGGUFPackage(metadata *pkg.GGUFFileHeader, locations ...file.Location) pkg.Package {
// Compute hash if not already set
if metadata.Hash == "" {
metadata.Hash = computeMetadataHash(metadata)
}
p := pkg.Package{ p := pkg.Package{
Name: metadata.ModelName, Name: metadata.ModelName,
Version: metadata.ModelVersion, Version: metadata.ModelVersion,
@ -37,33 +26,3 @@ func newGGUFPackage(metadata *pkg.GGUFFileHeader, locations ...file.Location) pk
return p return p
} }
// computeMetadataHash computes a stable hash of the metadata for use as a global identifier
func computeMetadataHash(metadata *pkg.GGUFFileHeader) string {
// Create a stable representation of the metadata
hashData := struct {
Format string
Name string
Version string
Architecture string
GGUFVersion uint32
TensorCount uint64
}{
Name: metadata.ModelName,
Version: metadata.ModelVersion,
Architecture: metadata.Architecture,
GGUFVersion: metadata.GGUFVersion,
TensorCount: metadata.TensorCount,
}
// Marshal to JSON for stable hashing
jsonBytes, err := json.Marshal(hashData)
if err != nil {
log.Debugf("failed to marshal metadata for hashing: %v", err)
return ""
}
// Compute xxhash
hash := xxhash.Sum64(jsonBytes)
return fmt.Sprintf("%016x", hash) // 16 hex chars (64 bits)
}

View File

@ -21,15 +21,15 @@ func TestNewGGUFPackage(t *testing.T) {
{ {
name: "complete GGUF package with all fields", name: "complete GGUF package with all fields",
metadata: &pkg.GGUFFileHeader{ metadata: &pkg.GGUFFileHeader{
ModelName: "llama3-8b-instruct", ModelName: "llama3-8b-instruct",
ModelVersion: "3.0", ModelVersion: "3.0",
License: "Apache-2.0", License: "Apache-2.0",
Architecture: "llama", Architecture: "llama",
Quantization: "Q4_K_M", Quantization: "Q4_K_M",
Parameters: 8030000000, Parameters: 8030000000,
GGUFVersion: 3, GGUFVersion: 3,
TensorCount: 291, TensorCount: 291,
Header: map[string]any{}, Header: map[string]any{},
}, },
locations: []file.Location{file.NewLocation("/models/llama3-8b.gguf")}, locations: []file.Location{file.NewLocation("/models/llama3-8b.gguf")},
checkFunc: func(t *testing.T, p pkg.Package) { checkFunc: func(t *testing.T, p pkg.Package) {

View File

@ -64,13 +64,13 @@ func parseGGUFModel(_ context.Context, _ file.Resolver, _ *generic.Environment,
// Convert to syft metadata structure // Convert to syft metadata structure
syftMetadata := &pkg.GGUFFileHeader{ syftMetadata := &pkg.GGUFFileHeader{
ModelName: metadata.Name, ModelName: metadata.Name,
ModelVersion: extractVersion(ggufFile.Header.MetadataKV), ModelVersion: extractVersion(ggufFile.Header.MetadataKV),
License: metadata.License, License: metadata.License,
Architecture: metadata.Architecture, Architecture: metadata.Architecture,
Quantization: metadata.FileTypeDescriptor, Quantization: metadata.FileTypeDescriptor,
Parameters: uint64(metadata.Parameters), Parameters: uint64(metadata.Parameters),
GGUFVersion: uint32(ggufFile.Header.Version), GGUFVersion: uint32(ggufFile.Header.Version),
TensorCount: ggufFile.Header.TensorCount, TensorCount: ggufFile.Header.TensorCount,
Header: convertGGUFMetadataKVs(ggufFile.Header.MetadataKV), Header: convertGGUFMetadataKVs(ggufFile.Header.MetadataKV),
MetadataHash: computeKVMetadataHash(ggufFile.Header.MetadataKV), MetadataHash: computeKVMetadataHash(ggufFile.Header.MetadataKV),

View File

@ -3,7 +3,7 @@ package ai
import ( import (
"fmt" "fmt"
"os" "os"
gguf_parser "github.com/gpustack/gguf-parser-go" gguf_parser "github.com/gpustack/gguf-parser-go"
) )
@ -14,21 +14,21 @@ func main() {
withStringKV("general.architecture", "llama"). withStringKV("general.architecture", "llama").
withStringKV("general.name", "test-model"). withStringKV("general.name", "test-model").
build() build()
// Write to temp file // Write to temp file
tempFile, err := os.CreateTemp("", "test-*.gguf") tempFile, err := os.CreateTemp("", "test-*.gguf")
if err != nil { if err != nil {
panic(err) panic(err)
} }
defer os.Remove(tempFile.Name()) defer os.Remove(tempFile.Name())
if _, err := tempFile.Write(data); err != nil { if _, err := tempFile.Write(data); err != nil {
panic(err) panic(err)
} }
tempFile.Close() tempFile.Close()
fmt.Printf("Wrote %d bytes to %s\n", len(data), tempFile.Name()) fmt.Printf("Wrote %d bytes to %s\n", len(data), tempFile.Name())
// Try to parse it // Try to parse it
fmt.Println("Attempting to parse...") fmt.Println("Attempting to parse...")
gf, err := gguf_parser.ParseGGUFFile(tempFile.Name(), gguf_parser.SkipLargeMetadata()) gf, err := gguf_parser.ParseGGUFFile(tempFile.Name(), gguf_parser.SkipLargeMetadata())
@ -36,6 +36,6 @@ func main() {
fmt.Printf("Parse error: %v\n", err) fmt.Printf("Parse error: %v\n", err)
return return
} }
fmt.Printf("Success! Model: %s\n", gf.Metadata().Name) fmt.Printf("Success! Model: %s\n", gf.Metadata().Name)
} }