mirror of
https://github.com/anchore/syft.git
synced 2025-11-17 08:23:15 +01:00
chore: pr comments
Signed-off-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com>
This commit is contained in:
parent
cdb41b0c76
commit
b80592f735
@ -14,6 +14,28 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestGGUFCataloger_Globs(t *testing.T) {
|
func TestGGUFCataloger_Globs(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
fixture string
|
||||||
|
expected []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "obtain gguf files",
|
||||||
|
fixture: "test-fixtures/glob-paths",
|
||||||
|
expected: []string{
|
||||||
|
"models/model.gguf",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
pkgtest.NewCatalogTester().
|
||||||
|
FromDirectory(t, test.fixture).
|
||||||
|
ExpectsResolverContentQueries(test.expected).
|
||||||
|
TestCataloger(t, NewGGUFCataloger())
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGGUFCataloger_Integration(t *testing.T) {
|
func TestGGUFCataloger_Integration(t *testing.T) {
|
||||||
@ -50,15 +72,15 @@ func TestGGUFCataloger_Integration(t *testing.T) {
|
|||||||
pkg.NewLicenseFromFields("Apache-2.0", "", nil),
|
pkg.NewLicenseFromFields("Apache-2.0", "", nil),
|
||||||
),
|
),
|
||||||
Metadata: pkg.GGUFFileHeader{
|
Metadata: pkg.GGUFFileHeader{
|
||||||
ModelName: "llama3-8b",
|
ModelName: "llama3-8b",
|
||||||
ModelVersion: "3.0",
|
ModelVersion: "3.0",
|
||||||
License: "Apache-2.0",
|
License: "Apache-2.0",
|
||||||
Architecture: "llama",
|
Architecture: "llama",
|
||||||
Quantization: "Unknown",
|
Quantization: "Unknown",
|
||||||
Parameters: 0,
|
Parameters: 0,
|
||||||
GGUFVersion: 3,
|
GGUFVersion: 3,
|
||||||
TensorCount: 0,
|
TensorCount: 0,
|
||||||
Header: map[string]interface{}{},
|
Header: map[string]interface{}{},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@ -77,8 +99,8 @@ func TestGGUFCataloger_Integration(t *testing.T) {
|
|||||||
IgnoreLocationLayer().
|
IgnoreLocationLayer().
|
||||||
IgnorePackageFields("FoundBy", "Locations"). // These are set by the cataloger
|
IgnorePackageFields("FoundBy", "Locations"). // These are set by the cataloger
|
||||||
WithCompareOptions(
|
WithCompareOptions(
|
||||||
// Ignore Hash as it's computed dynamically
|
// Ignore MetadataHash as it's computed dynamically
|
||||||
cmpopts.IgnoreFields(pkg.GGUFFileHeader{}, "Hash"),
|
cmpopts.IgnoreFields(pkg.GGUFFileHeader{}, "MetadataHash"),
|
||||||
)
|
)
|
||||||
|
|
||||||
tester.TestCataloger(t, NewGGUFCataloger())
|
tester.TestCataloger(t, NewGGUFCataloger())
|
||||||
|
|||||||
@ -1,22 +1,11 @@
|
|||||||
package ai
|
package ai
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/cespare/xxhash/v2"
|
|
||||||
|
|
||||||
"github.com/anchore/syft/internal/log"
|
|
||||||
"github.com/anchore/syft/syft/file"
|
"github.com/anchore/syft/syft/file"
|
||||||
"github.com/anchore/syft/syft/pkg"
|
"github.com/anchore/syft/syft/pkg"
|
||||||
)
|
)
|
||||||
|
|
||||||
func newGGUFPackage(metadata *pkg.GGUFFileHeader, locations ...file.Location) pkg.Package {
|
func newGGUFPackage(metadata *pkg.GGUFFileHeader, locations ...file.Location) pkg.Package {
|
||||||
// Compute hash if not already set
|
|
||||||
if metadata.Hash == "" {
|
|
||||||
metadata.Hash = computeMetadataHash(metadata)
|
|
||||||
}
|
|
||||||
|
|
||||||
p := pkg.Package{
|
p := pkg.Package{
|
||||||
Name: metadata.ModelName,
|
Name: metadata.ModelName,
|
||||||
Version: metadata.ModelVersion,
|
Version: metadata.ModelVersion,
|
||||||
@ -37,33 +26,3 @@ func newGGUFPackage(metadata *pkg.GGUFFileHeader, locations ...file.Location) pk
|
|||||||
|
|
||||||
return p
|
return p
|
||||||
}
|
}
|
||||||
|
|
||||||
// computeMetadataHash computes a stable hash of the metadata for use as a global identifier
|
|
||||||
func computeMetadataHash(metadata *pkg.GGUFFileHeader) string {
|
|
||||||
// Create a stable representation of the metadata
|
|
||||||
hashData := struct {
|
|
||||||
Format string
|
|
||||||
Name string
|
|
||||||
Version string
|
|
||||||
Architecture string
|
|
||||||
GGUFVersion uint32
|
|
||||||
TensorCount uint64
|
|
||||||
}{
|
|
||||||
Name: metadata.ModelName,
|
|
||||||
Version: metadata.ModelVersion,
|
|
||||||
Architecture: metadata.Architecture,
|
|
||||||
GGUFVersion: metadata.GGUFVersion,
|
|
||||||
TensorCount: metadata.TensorCount,
|
|
||||||
}
|
|
||||||
|
|
||||||
// Marshal to JSON for stable hashing
|
|
||||||
jsonBytes, err := json.Marshal(hashData)
|
|
||||||
if err != nil {
|
|
||||||
log.Debugf("failed to marshal metadata for hashing: %v", err)
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compute xxhash
|
|
||||||
hash := xxhash.Sum64(jsonBytes)
|
|
||||||
return fmt.Sprintf("%016x", hash) // 16 hex chars (64 bits)
|
|
||||||
}
|
|
||||||
|
|||||||
@ -21,15 +21,15 @@ func TestNewGGUFPackage(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "complete GGUF package with all fields",
|
name: "complete GGUF package with all fields",
|
||||||
metadata: &pkg.GGUFFileHeader{
|
metadata: &pkg.GGUFFileHeader{
|
||||||
ModelName: "llama3-8b-instruct",
|
ModelName: "llama3-8b-instruct",
|
||||||
ModelVersion: "3.0",
|
ModelVersion: "3.0",
|
||||||
License: "Apache-2.0",
|
License: "Apache-2.0",
|
||||||
Architecture: "llama",
|
Architecture: "llama",
|
||||||
Quantization: "Q4_K_M",
|
Quantization: "Q4_K_M",
|
||||||
Parameters: 8030000000,
|
Parameters: 8030000000,
|
||||||
GGUFVersion: 3,
|
GGUFVersion: 3,
|
||||||
TensorCount: 291,
|
TensorCount: 291,
|
||||||
Header: map[string]any{},
|
Header: map[string]any{},
|
||||||
},
|
},
|
||||||
locations: []file.Location{file.NewLocation("/models/llama3-8b.gguf")},
|
locations: []file.Location{file.NewLocation("/models/llama3-8b.gguf")},
|
||||||
checkFunc: func(t *testing.T, p pkg.Package) {
|
checkFunc: func(t *testing.T, p pkg.Package) {
|
||||||
|
|||||||
@ -64,13 +64,13 @@ func parseGGUFModel(_ context.Context, _ file.Resolver, _ *generic.Environment,
|
|||||||
|
|
||||||
// Convert to syft metadata structure
|
// Convert to syft metadata structure
|
||||||
syftMetadata := &pkg.GGUFFileHeader{
|
syftMetadata := &pkg.GGUFFileHeader{
|
||||||
ModelName: metadata.Name,
|
ModelName: metadata.Name,
|
||||||
ModelVersion: extractVersion(ggufFile.Header.MetadataKV),
|
ModelVersion: extractVersion(ggufFile.Header.MetadataKV),
|
||||||
License: metadata.License,
|
License: metadata.License,
|
||||||
Architecture: metadata.Architecture,
|
Architecture: metadata.Architecture,
|
||||||
Quantization: metadata.FileTypeDescriptor,
|
Quantization: metadata.FileTypeDescriptor,
|
||||||
Parameters: uint64(metadata.Parameters),
|
Parameters: uint64(metadata.Parameters),
|
||||||
GGUFVersion: uint32(ggufFile.Header.Version),
|
GGUFVersion: uint32(ggufFile.Header.Version),
|
||||||
TensorCount: ggufFile.Header.TensorCount,
|
TensorCount: ggufFile.Header.TensorCount,
|
||||||
Header: convertGGUFMetadataKVs(ggufFile.Header.MetadataKV),
|
Header: convertGGUFMetadataKVs(ggufFile.Header.MetadataKV),
|
||||||
MetadataHash: computeKVMetadataHash(ggufFile.Header.MetadataKV),
|
MetadataHash: computeKVMetadataHash(ggufFile.Header.MetadataKV),
|
||||||
|
|||||||
@ -3,7 +3,7 @@ package ai
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
gguf_parser "github.com/gpustack/gguf-parser-go"
|
gguf_parser "github.com/gpustack/gguf-parser-go"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -14,21 +14,21 @@ func main() {
|
|||||||
withStringKV("general.architecture", "llama").
|
withStringKV("general.architecture", "llama").
|
||||||
withStringKV("general.name", "test-model").
|
withStringKV("general.name", "test-model").
|
||||||
build()
|
build()
|
||||||
|
|
||||||
// Write to temp file
|
// Write to temp file
|
||||||
tempFile, err := os.CreateTemp("", "test-*.gguf")
|
tempFile, err := os.CreateTemp("", "test-*.gguf")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
defer os.Remove(tempFile.Name())
|
defer os.Remove(tempFile.Name())
|
||||||
|
|
||||||
if _, err := tempFile.Write(data); err != nil {
|
if _, err := tempFile.Write(data); err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
tempFile.Close()
|
tempFile.Close()
|
||||||
|
|
||||||
fmt.Printf("Wrote %d bytes to %s\n", len(data), tempFile.Name())
|
fmt.Printf("Wrote %d bytes to %s\n", len(data), tempFile.Name())
|
||||||
|
|
||||||
// Try to parse it
|
// Try to parse it
|
||||||
fmt.Println("Attempting to parse...")
|
fmt.Println("Attempting to parse...")
|
||||||
gf, err := gguf_parser.ParseGGUFFile(tempFile.Name(), gguf_parser.SkipLargeMetadata())
|
gf, err := gguf_parser.ParseGGUFFile(tempFile.Name(), gguf_parser.SkipLargeMetadata())
|
||||||
@ -36,6 +36,6 @@ func main() {
|
|||||||
fmt.Printf("Parse error: %v\n", err)
|
fmt.Printf("Parse error: %v\n", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("Success! Model: %s\n", gf.Metadata().Name)
|
fmt.Printf("Success! Model: %s\n", gf.Metadata().Name)
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user