Merge remote-tracking branch 'upstream/main' into add-nested-archive-cataloging

test/cli/model_cataloging_test.go

@@ -0,0 +1,62 @@
+package cli
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// modelSize is larger than the 50 MB syft copies from a model to parse its header, and than the tensor data
+// the header fixture declares.
+const modelSize = 512 << 20
+
+// TestGGUFModelIsCataloged scans a model larger than the header copy syft parses: a real GGUF header followed
+// by a sparse range standing in for the tensor data. Models under the copy size are parsed whole, so only a
+// model this large catches a parser that checks the declared tensor data against the bytes it was given.
+func TestGGUFModelIsCataloged(t *testing.T) {
+	root := t.TempDir()
+	header, err := os.ReadFile("testdata/models/gguf-header-only/header.gguf")
+	require.NoError(t, err)
+	modelPath := filepath.Join(root, "qwen2.5-0.5b-instruct-q2_k.gguf")
+	require.NoError(t, os.WriteFile(modelPath, header, 0o600))
+	require.NoError(t, os.Truncate(modelPath, modelSize))
+
+	out := filepath.Join(t.TempDir(), "sbom.json")
+	cmd, stdout, stderr := runSyft(t, nil, "scan", "dir:"+root, "-o", "syft-json="+out)
+	defer logOutputOnFailure(t, cmd, stdout, stderr)
+	assertSuccessfulReturnCode(t, stdout, stderr, cmd.ProcessState.ExitCode())
+
+	b, err := os.ReadFile(out)
+	require.NoError(t, err, "no SBOM written")
+	var doc struct {
+		Artifacts []struct {
+			Name     string `json:"name"`
+			Type     string `json:"type"`
+			Licenses []struct {
+				Value string `json:"value"`
+			} `json:"licenses"`
+			Metadata struct {
+				Architecture string `json:"architecture"`
+				Quantization string `json:"quantization"`
+				Parameters   uint64 `json:"parameters"`
+				TensorCount  uint64 `json:"tensorCount"`
+			} `json:"metadata"`
+		} `json:"artifacts"`
+	}
+	require.NoError(t, json.Unmarshal(b, &doc), "SBOM is not valid JSON")
+	require.Len(t, doc.Artifacts, 1, "expected the model package")
+
+	model := doc.Artifacts[0]
+	assert.Equal(t, "qwen2.5-0.5b-instruct", model.Name)
+	assert.Equal(t, "model", model.Type)
+	require.Len(t, model.Licenses, 1)
+	assert.Equal(t, "apache-2.0", model.Licenses[0].Value)
+	assert.Equal(t, "qwen2", model.Metadata.Architecture)
+	assert.Equal(t, "Q2_K_L", model.Metadata.Quantization)
+	assert.Equal(t, uint64(290), model.Metadata.TensorCount)
+	assert.Equal(t, uint64(494032768), model.Metadata.Parameters)
+}

test/cli/testdata/models/gguf-header-only/generate.go

@@ -0,0 +1,126 @@
+// generate writes header.gguf: the header of a GGUF model shaped like Qwen2.5-0.5B-Instruct at Q2_K (290
+// tensors declaring about 175 MB of data), padded to the tensor data alignment. Tests extend a copy to a
+// model's size as a sparse file, which stands in for the full model without storing its data.
+//
+// This file lives under testdata/ so the Go build system ignores it. Run it when changing the fixture:
+//
+//	go run ./testdata/models/gguf-header-only/generate.go ./testdata/models/gguf-header-only/header.gguf
+package main
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"os"
+)
+
+const (
+	typeUint32 = 4
+	typeString = 8
+	typeArray  = 9
+
+	ggmlF32 = 0
+	ggmlQ2K = 10
+	ggmlQ3K = 11
+	ggmlQ4K = 12
+	fileQ2K = 10
+)
+
+type tensor struct {
+	name string
+	typ  uint32
+	dims []uint64
+}
+
+type writer struct{ bytes.Buffer }
+
+func (w *writer) le(v any) { _ = binary.Write(w, binary.LittleEndian, v) }
+
+func (w *writer) str(s string) {
+	w.le(uint64(len(s)))
+	w.WriteString(s)
+}
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Fprintln(os.Stderr, "usage: generate <output path>")
+		os.Exit(1)
+	}
+
+	const (
+		hidden = 896
+		kvDim  = 128
+		ffn    = 4864
+		vocab  = 151936
+		layers = 24
+	)
+	tensors := []tensor{{"token_embd.weight", ggmlQ4K, []uint64{hidden, vocab}}}
+	for i := range layers {
+		blk := func(n string) string { return fmt.Sprintf("blk.%d.%s", i, n) }
+		tensors = append(tensors,
+			tensor{blk("attn_norm.weight"), ggmlF32, []uint64{hidden}},
+			tensor{blk("attn_q.weight"), ggmlQ2K, []uint64{hidden, hidden}},
+			tensor{blk("attn_q.bias"), ggmlF32, []uint64{hidden}},
+			tensor{blk("attn_k.weight"), ggmlQ2K, []uint64{hidden, kvDim}},
+			tensor{blk("attn_k.bias"), ggmlF32, []uint64{kvDim}},
+			tensor{blk("attn_v.weight"), ggmlQ3K, []uint64{hidden, kvDim}},
+			tensor{blk("attn_v.bias"), ggmlF32, []uint64{kvDim}},
+			tensor{blk("attn_output.weight"), ggmlQ3K, []uint64{hidden, hidden}},
+			tensor{blk("ffn_norm.weight"), ggmlF32, []uint64{hidden}},
+			tensor{blk("ffn_gate.weight"), ggmlQ2K, []uint64{hidden, ffn}},
+			tensor{blk("ffn_up.weight"), ggmlQ2K, []uint64{hidden, ffn}},
+			tensor{blk("ffn_down.weight"), ggmlQ3K, []uint64{ffn, hidden}},
+		)
+	}
+	tensors = append(tensors, tensor{"output_norm.weight", ggmlF32, []uint64{hidden}})
+
+	var w writer
+	w.le(uint32(0x46554747)) // "GGUF"
+	w.le(uint32(3))
+	w.le(uint64(len(tensors)))
+	w.le(uint64(6)) // metadata KV count
+
+	w.str("general.architecture")
+	w.le(uint32(typeString))
+	w.str("qwen2")
+	w.str("general.name")
+	w.le(uint32(typeString))
+	w.str("qwen2.5-0.5b-instruct")
+	w.str("general.license")
+	w.le(uint32(typeString))
+	w.str("apache-2.0")
+	w.str("general.file_type")
+	w.le(uint32(typeUint32))
+	w.le(uint32(fileQ2K))
+	w.str("qwen2.block_count")
+	w.le(uint32(typeUint32))
+	w.le(uint32(layers))
+	// a vocabulary the parser skips over
+	w.str("tokenizer.ggml.tokens")
+	w.le(uint32(typeArray))
+	w.le(uint32(typeString))
+	w.le(uint64(3))
+	for _, tok := range []string{"!", "\"", "#"} {
+		w.str(tok)
+	}
+
+	var offset uint64
+	for _, t := range tensors {
+		w.str(t.name)
+		w.le(uint32(len(t.dims)))
+		for _, d := range t.dims {
+			w.le(d)
+		}
+		w.le(t.typ)
+		w.le(offset)
+		offset += 1 << 20 // not used by syft; real offsets depend on each type's block size
+	}
+
+	// pad to the default 32-byte alignment, where the tensor data starts
+	w.Write(make([]byte, (32-w.Len()%32)%32))
+
+	if err := os.WriteFile(os.Args[1], w.Bytes(), 0o644); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}