diff --git a/internal/safetensors.go b/internal/safetensors.go new file mode 100644 index 000000000..deae046f5 --- /dev/null +++ b/internal/safetensors.go @@ -0,0 +1,7 @@ +package internal + +// MaxSafeTensorsHeaderSize caps a safetensors header's JSON (excluding its +// 8-byte length prefix). Shared by the ai cataloger and the OCI model source so +// dir and OCI scans accept the same headers. Real-world shard headers are well +// under 1 MB. +const MaxSafeTensorsHeaderSize = 8 * 1024 * 1024 diff --git a/syft/pkg/cataloger/ai/huggingface.go b/syft/pkg/cataloger/ai/huggingface.go index dc8333e70..9b9d644b1 100644 --- a/syft/pkg/cataloger/ai/huggingface.go +++ b/syft/pkg/cataloger/ai/huggingface.go @@ -15,6 +15,12 @@ type hfConfig struct { NameOrPath string `json:"_name_or_path"` } +// looksLikeHF reports whether the config carries any field we use. Other JSON +// (generation_config.json, unrelated app configs) is ignored. +func (c hfConfig) looksLikeHF() bool { + return len(c.Architectures) > 0 || c.NameOrPath != "" +} + func applyHFConfig(md *pkg.SafeTensorsModelInfo, cfg *hfConfig) { if md.Architecture == "" && len(cfg.Architectures) > 0 { md.Architecture = cfg.Architectures[0] @@ -23,8 +29,8 @@ func applyHFConfig(md *pkg.SafeTensorsModelInfo, cfg *hfConfig) { // readmeFrontmatter holds the subset of YAML frontmatter fields we extract. type readmeFrontmatter struct { - License string `yaml:"license"` - BaseModel []string `yaml:"base_model"` + Licenses []string + BaseModel []string } type licenseFrontmatter struct { @@ -32,21 +38,33 @@ type licenseFrontmatter struct { } // extractFrontmatterBlock returns the YAML bytes between the first and second -// "---" delimiters of a file +// "---" delimiter lines of a file func extractFrontmatterBlock(buf []byte) []byte { trimmed := bytes.TrimLeft(buf, "\xef\xbb\xbf \t\r\n") - if !bytes.HasPrefix(trimmed, []byte("---")) { + first, rest, _ := bytes.Cut(trimmed, []byte("\n")) + if !isFrontmatterDelimiter(first) { return nil } - rest := trimmed[3:] - if i := bytes.IndexByte(rest, '\n'); i >= 0 { - rest = rest[i+1:] + for off := 0; off < len(rest); { + line, _, found := bytes.Cut(rest[off:], []byte("\n")) + if isFrontmatterDelimiter(line) { + if off == 0 || off > maxFrontmatterSize { + return nil + } + return rest[:off] + } + if !found { + break + } + off += len(line) + 1 } - block, _, found := bytes.Cut(rest, []byte("\n---")) - if !found { - return nil - } - return block + return nil +} + +// isFrontmatterDelimiter reports whether a line is exactly "---", ignoring +// trailing whitespace and \r. +func isFrontmatterDelimiter(line []byte) bool { + return bytes.Equal(bytes.TrimRight(line, " \t\r"), []byte("---")) } // parseFrontmatter decodes a Hugging Face model card YAML frontmatter block @@ -57,8 +75,9 @@ func parseFrontmatter(buf []byte) *readmeFrontmatter { return nil } + // both fields may be a scalar or a list, so decode them as nodes var raw struct { - License string `yaml:"license"` + License yaml.Node `yaml:"license"` BaseModel yaml.Node `yaml:"base_model"` } if err := yaml.Unmarshal(block, &raw); err != nil { @@ -66,16 +85,37 @@ func parseFrontmatter(buf []byte) *readmeFrontmatter { return nil } - fm := readmeFrontmatter{License: raw.License} - switch raw.BaseModel.Kind { + return &readmeFrontmatter{ + Licenses: yamlStrings(raw.License), + BaseModel: yamlStrings(raw.BaseModel), + } +} + +// yamlStrings returns the non-empty, non-null values of a scalar or a sequence +// of scalars, keeping at most maxFrontmatterValues. Anything else yields nil. +func yamlStrings(n yaml.Node) []string { + switch n.Kind { case yaml.ScalarNode: - if raw.BaseModel.Value != "" { - fm.BaseModel = []string{raw.BaseModel.Value} + if isYAMLString(&n) { + return []string{n.Value} } case yaml.SequenceNode: - _ = raw.BaseModel.Decode(&fm.BaseModel) + var out []string + for _, c := range n.Content { + if len(out) == maxFrontmatterValues { + break + } + if c.Kind == yaml.ScalarNode && isYAMLString(c) { + out = append(out, c.Value) + } + } + return out } - return &fm + return nil +} + +func isYAMLString(n *yaml.Node) bool { + return n.Value != "" && n.ShortTag() != "!!null" } // parseLicenseFrontmatter returns the producer-declared SPDX identifier diff --git a/syft/pkg/cataloger/ai/identity_dir.go b/syft/pkg/cataloger/ai/identity_dir.go index eaa0d1efa..dd2460092 100644 --- a/syft/pkg/cataloger/ai/identity_dir.go +++ b/syft/pkg/cataloger/ai/identity_dir.go @@ -3,7 +3,6 @@ package ai import ( "context" "encoding/json" - "io" "path" "github.com/anchore/syft/internal" @@ -13,42 +12,42 @@ import ( ) // resolveSafeTensorsDirIdentity handles the directory-scan case for safe tensors -// find config.json beside the model files (walking up parent directories to the -// scanned source root if no sibling exists) and a sibling README.md. It returns +// find config.json beside the model files (or one directory up, for the shard +// subdirectory layout) and a sibling README.md. It returns // the group's name candidates, resolved licenses, and supporting evidence. func resolveSafeTensorsDirIdentity(ctx context.Context, resolver file.Resolver, dir string, md *pkg.SafeTensorsModelInfo) safeTensorsIdentity { id := safeTensorsIdentity{fallbackName: safeTensorsDirName(dir)} if loc, cfg := findDirHFConfig(resolver, dir); cfg != nil { applyHFConfig(md, cfg) - id.nameOrPath = cfg.NameOrPath + id.nameOrPath = sanitizeModelName(cfg.NameOrPath) id.supporting = append(id.supporting, *loc) } if loc, fm := readDirReadmeFrontmatter(resolver, path.Join(dir, "README.md")); fm != nil { - if fm.License != "" { - id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, fm.License) + if len(fm.Licenses) > 0 { + id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, fm.Licenses...) } - if id.nameOrPath == "" && len(fm.BaseModel) > 0 { - id.nameOrPath = fm.BaseModel[0] + if id.nameOrPath == "" { + id.nameOrPath = firstModelName(fm.BaseModel) } id.supporting = append(id.supporting, *loc) } return id } -// findDirHFConfig looks for a config.json beside the model files +// findDirHFConfig looks for an HF config.json beside the model files, then in +// the parent directory (covers weights kept in a subdirectory of the model). It +// does not walk further up, so an unrelated config.json higher in the tree can't +// name every model below it. func findDirHFConfig(resolver file.Resolver, dir string) (*file.Location, *hfConfig) { - for { - if loc, cfg := readDirHFConfig(resolver, path.Join(dir, "config.json")); cfg != nil { - return loc, cfg - } - parent := path.Dir(dir) - if parent == dir { - return nil, nil // reached the source root - } - dir = parent + if loc, cfg := readDirHFConfig(resolver, path.Join(dir, "config.json")); cfg != nil { + return loc, cfg } + if parent := path.Dir(dir); parent != dir { + return readDirHFConfig(resolver, path.Join(parent, "config.json")) + } + return nil, nil } func readDirHFConfig(resolver file.Resolver, p string) (*file.Location, *hfConfig) { @@ -62,11 +61,19 @@ func readDirHFConfig(resolver file.Resolver, p string) (*file.Location, *hfConfi } defer internal.CloseAndLogError(rc, p) + buf, err := readBounded(rc, maxHFConfigSize) + if err != nil { + log.Debugf("failed to read %s: %v", p, err) + return nil, nil + } var cfg hfConfig - if err := json.NewDecoder(rc).Decode(&cfg); err != nil { + if err := json.Unmarshal(buf, &cfg); err != nil { log.Debugf("failed to decode %s: %v", p, err) return nil, nil } + if !cfg.looksLikeHF() { + return nil, nil + } return &locations[0], &cfg } @@ -81,8 +88,9 @@ func readDirReadmeFrontmatter(resolver file.Resolver, p string) (*file.Location, } defer internal.CloseAndLogError(rc, p) - buf, err := io.ReadAll(io.LimitReader(rc, 1024*1024)) + buf, err := readPrefix(rc, maxReadmePrefixSize) if err != nil { + log.Debugf("failed to read %s: %v", p, err) return nil, nil } fm := parseFrontmatter(buf) diff --git a/syft/pkg/cataloger/ai/identity_oci.go b/syft/pkg/cataloger/ai/identity_oci.go index 6209a3f8b..97d58aa6b 100644 --- a/syft/pkg/cataloger/ai/identity_oci.go +++ b/syft/pkg/cataloger/ai/identity_oci.go @@ -4,8 +4,8 @@ import ( "bytes" "context" "encoding/json" - "io" "path" + "sort" gcrname "github.com/google/go-containerregistry/pkg/name" @@ -28,9 +28,18 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver, log.Debugf("failed to list docker AI model-file layers: %v", err) } - // Collect config / readme candidates separately so the layer-iteration order - // returned by the resolver doesn't decide the precedence. - var configName, readmeName, readmeLicense string + // the resolver returns layers in map order, and the first config / README + // seen wins below, so sort for a stable result across runs. For OCI + // artifacts every RealPath is "/", so in practice this is digest order. + sort.Slice(modelFileLocs, func(i, j int) bool { + if modelFileLocs[i].RealPath != modelFileLocs[j].RealPath { + return modelFileLocs[i].RealPath < modelFileLocs[j].RealPath + } + return modelFileLocs[i].FileSystemID < modelFileLocs[j].FileSystemID + }) + + var configName, readmeName string + var readmeLicenses []string var supporting []file.Location for _, loc := range modelFileLocs { cfg, fm := classifyOCIModelFileLayer(resolver, loc) @@ -38,15 +47,15 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver, case cfg != nil: applyHFConfig(md, cfg) if configName == "" { - configName = cfg.NameOrPath + configName = sanitizeModelName(cfg.NameOrPath) } supporting = append(supporting, loc) case fm != nil: - if readmeLicense == "" { - readmeLicense = fm.License + if len(readmeLicenses) == 0 { + readmeLicenses = fm.Licenses } - if readmeName == "" && len(fm.BaseModel) > 0 { - readmeName = fm.BaseModel[0] + if readmeName == "" { + readmeName = firstModelName(fm.BaseModel) } supporting = append(supporting, loc) } @@ -64,7 +73,7 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver, supporting: supporting, } - // License precedence: a dedicated vnd.docker.ai.license layer is a + // License precedence: a dedicated vnd.docker.ai.license layer // outranks the free-text license field in a model card's README frontmatter. licLocs, err := ociResolver.FilesByMediaType(dockerAILicenseMediaType) if err != nil { @@ -74,8 +83,8 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver, case len(licLocs) > 0: id.licenses = identifyLicenseLayers(ctx, resolver, licLocs) id.supporting = append(id.supporting, licLocs...) - case readmeLicense != "": - id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, readmeLicense) + case len(readmeLicenses) > 0: + id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, readmeLicenses...) } return id @@ -108,10 +117,11 @@ func ociImageRefBasename(resolver file.Resolver) string { return path.Base(parsed.Context().RepositoryStr()) } -// classifyOCIModelFileLayer reads up to 4 MiB of a model.file layer and decodes -// it as either an HF config.json or a README model card's YAML frontmatter, -// based on its leading bytes. It returns whichever it recognized; both are nil -// when the layer is neither (or fails to decode). The caller owns precedence and +// classifyOCIModelFileLayer reads a model.file layer (up to maxModelFileLayerSize) +// and decodes it as either an HF config.json or a README model card's YAML +// frontmatter, based on its leading bytes. It returns whichever it recognized; +// both are nil when the layer is neither, fails to decode, or is JSON that +// doesn't look like an HF config (e.g. generation_config.json). The caller owns precedence and // metadata enrichment. func classifyOCIModelFileLayer(resolver file.Resolver, loc file.Location) (*hfConfig, *readmeFrontmatter) { rc, err := resolver.FileContentsByLocation(loc) @@ -120,17 +130,18 @@ func classifyOCIModelFileLayer(resolver file.Resolver, loc file.Location) (*hfCo } defer internal.CloseAndLogError(rc, loc.RealPath) - buf, err := io.ReadAll(io.LimitReader(rc, 4*1024*1024)) + buf, err := readBounded(rc, maxModelFileLayerSize) if err != nil { + log.Debugf("failed to read docker AI model-file layer: %v", err) return nil, nil } trimmed := bytes.TrimLeft(buf, "\xef\xbb\xbf \t\r\n") switch { case bytes.HasPrefix(trimmed, []byte("---")): - return nil, parseFrontmatter(buf) + return nil, parseFrontmatter(trimmed) case bytes.HasPrefix(trimmed, []byte("{")): var cfg hfConfig - if err := json.Unmarshal(buf, &cfg); err != nil { + if err := json.Unmarshal(trimmed, &cfg); err != nil || !cfg.looksLikeHF() { return nil, nil } return &cfg, nil @@ -165,8 +176,9 @@ func readLicenseSPDXIDFromFrontmatter(resolver file.Resolver, loc file.Location) } defer internal.CloseAndLogError(rc, loc.RealPath) - buf, err := io.ReadAll(io.LimitReader(rc, 64*1024)) + buf, err := readPrefix(rc, maxLicensePrefixSize) if err != nil { + log.Debugf("failed to read docker AI license layer: %v", err) return "" } return parseLicenseFrontmatter(buf) diff --git a/syft/pkg/cataloger/ai/limits.go b/syft/pkg/cataloger/ai/limits.go new file mode 100644 index 000000000..9de7bfd58 --- /dev/null +++ b/syft/pkg/cataloger/ai/limits.go @@ -0,0 +1,46 @@ +package ai + +import ( + "fmt" + "io" + + "github.com/anchore/syft/internal" +) + +// size limits for model companion files and headers. JSON reads that hit a +// limit fail (see readBounded) instead of parsing a truncated document. +// Frontmatter-only reads (README, license) take a prefix instead, since the +// frontmatter sits at the top and is itself capped by maxFrontmatterSize. +const ( + maxHFConfigSize = 4 * 1024 * 1024 + maxReadmePrefixSize = 1 * 1024 * 1024 // dir scans; OCI README layers use maxModelFileLayerSize + maxModelFileLayerSize = 4 * 1024 * 1024 + maxLicensePrefixSize = 64 * 1024 + maxModelConfigBlobSize = 1 * 1024 * 1024 + maxFrontmatterSize = 256 * 1024 + maxFrontmatterValues = 32 + maxModelNameLength = 256 + + // maxSafeTensorsHeaderSize caps the header JSON (excluding its 8-byte length + // prefix). It is shared with the OCI source so dir and OCI scans accept the + // same headers. + maxSafeTensorsHeaderSize = internal.MaxSafeTensorsHeaderSize +) + +// readPrefix reads at most limit bytes. It is for frontmatter-only reads, where +// the rest of the file is never needed. +func readPrefix(r io.Reader, limit int64) ([]byte, error) { + return io.ReadAll(io.LimitReader(r, limit)) +} + +// readBounded reads at most limit bytes and reports overflow instead of silently truncating. +func readBounded(r io.Reader, limit int64) ([]byte, error) { + b, err := io.ReadAll(io.LimitReader(r, limit+1)) + if err != nil { + return nil, err + } + if int64(len(b)) > limit { + return nil, fmt.Errorf("content exceeds %d bytes", limit) + } + return b, nil +} diff --git a/syft/pkg/cataloger/ai/naming.go b/syft/pkg/cataloger/ai/naming.go index ed2571dcb..65de1d367 100644 --- a/syft/pkg/cataloger/ai/naming.go +++ b/syft/pkg/cataloger/ai/naming.go @@ -1,15 +1,44 @@ package ai -import "path" +import ( + "path" + "strings" + "unicode/utf8" +) // pickSafeTensorsName implements the documented naming precedence chain: func pickSafeTensorsName(nameOrPath, fallbackName string) string { - if nameOrPath != "" { - return path.Base(nameOrPath) + if name := sanitizeModelName(nameOrPath); name != "" { + return name } return fallbackName } +// firstModelName returns the first value that sanitizes to a usable name. +func firstModelName(values []string) string { + for _, v := range values { + if name := sanitizeModelName(v); name != "" { + return name + } + } + return "" +} + +// sanitizeModelName reduces a producer-supplied name or path (_name_or_path, +// base_model) to its last path element, capped at maxModelNameLength runes. It +// returns "" when nothing usable is left. +func sanitizeModelName(nameOrPath string) string { + base := path.Base(strings.ReplaceAll(nameOrPath, `\`, "/")) + switch base { + case "/", ".", "..", "": + return "" + } + if utf8.RuneCountInString(base) > maxModelNameLength { + base = string([]rune(base)[:maxModelNameLength]) + } + return base +} + // safeTensorsDirName returns the directory-scan naming fallback: the base name // of the group's parent directory (the group key is already that directory). func safeTensorsDirName(directory string) string { diff --git a/syft/pkg/cataloger/ai/parse_safetensors.go b/syft/pkg/cataloger/ai/parse_safetensors.go index bc1ff1402..119a182ff 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors.go +++ b/syft/pkg/cataloger/ai/parse_safetensors.go @@ -15,9 +15,7 @@ import ( // SafeTensors file format: [8 bytes u64 LE header size] [N bytes JSON header] [tensor data]. // Reference: https://github.com/huggingface/safetensors#format -const ( - maxSafeTensorsHeaderSize = 100 * 1024 * 1024 // 100MB ceiling on header JSON to prevent OOM -) +// The header JSON is capped at maxSafeTensorsHeaderSize (see limits.go). // safeTensorsHeader is the decoded JSON header. Tensor entries live alongside a // reserved "__metadata__" key holding a string-to-string producer map. We decode diff --git a/syft/pkg/cataloger/ai/parse_safetensors_oci.go b/syft/pkg/cataloger/ai/parse_safetensors_oci.go index 7fd9f3b3e..5b517b846 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors_oci.go +++ b/syft/pkg/cataloger/ai/parse_safetensors_oci.go @@ -50,7 +50,7 @@ type dockerAIModelConfig struct { func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { defer internal.CloseAndLogError(reader, reader.Path()) - body, err := io.ReadAll(io.LimitReader(reader, 1024*1024)) + body, err := readBounded(reader, maxModelConfigBlobSize) if err != nil { return nil, nil, fmt.Errorf("failed to read docker AI model config: %w", err) } diff --git a/syft/pkg/cataloger/ai/parse_safetensors_test.go b/syft/pkg/cataloger/ai/parse_safetensors_test.go index 943892c40..7b98cb915 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors_test.go +++ b/syft/pkg/cataloger/ai/parse_safetensors_test.go @@ -8,6 +8,8 @@ import ( "io" "os" "path/filepath" + "slices" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -48,7 +50,7 @@ func buildSafeTensorsFile(t *testing.T, metadata map[string]string, tensors map[ // // The naming precedence (owned by the merge processor's pickSafeTensorsName) is: // 1. config.json _name_or_path (path.Base applied), found beside the model or -// by walking up parent directories to the scan root +// in its parent directory // 2. otherwise the model's immediate parent directory base name // → drop (no package) when neither yields a usable name // @@ -231,9 +233,9 @@ func TestSafeTensorsCataloger(t *testing.T) { expectedPackages: nil, }, { - // rung 1 via parent-walk: findDirHFConfig walks up from the model - // directory, so a config.json in an ancestor names a nested model. - name: "config.json in an ancestor directory names a nested model", + // rung 1 via the parent: findDirHFConfig also checks the model + // directory's parent, so a config.json there names a nested model. + name: "config.json in the parent directory names a nested model", setup: func(t *testing.T) string { dir := t.TempDir() model(t, filepath.Join(dir, "dir", "someothertensor")) @@ -248,6 +250,87 @@ func TestSafeTensorsCataloger(t *testing.T) { }, }, }, + { + // the walk stops at the parent: a config.json two levels up is not used. + name: "config.json two levels up is not used", + setup: func(t *testing.T) string { + dir := t.TempDir() + model(t, filepath.Join(dir, "a", "b", "someothertensor")) + writeFile(t, filepath.Join(dir, "a", "config.json"), `{"_name_or_path":"org/TooFar"}`) + return dir + }, + expectedPackages: []pkg.Package{ + { + Name: "someothertensor", + Type: pkg.ModelPkg, + Metadata: wantMetadata(""), + }, + }, + }, + { + // JSON without any HF field is not treated as a model config, so the + // lookup keeps going to the parent. + name: "non-HF config.json is skipped", + setup: func(t *testing.T) string { + dir := t.TempDir() + modelDir := filepath.Join(dir, "app", "sometensor") + model(t, modelDir) + writeFile(t, filepath.Join(modelDir, "config.json"), `{"port":8080}`) + writeFile(t, filepath.Join(dir, "app", "config.json"), `{"_name_or_path":"org/Parent"}`) + return dir + }, + expectedPackages: []pkg.Package{ + { + Name: "Parent", + Type: pkg.ModelPkg, + Metadata: wantMetadata(""), + }, + }, + }, + { + // an oversized config.json is skipped entirely, while a large README + // still yields its frontmatter (only a prefix is read). + name: "oversized config.json is ignored and a large README still names the model", + setup: func(t *testing.T) string { + dir := t.TempDir() + modelDir := filepath.Join(dir, "sometensor") + model(t, modelDir) + writeFile(t, filepath.Join(modelDir, "config.json"), + `{"architectures":["X"],"_name_or_path":"`+strings.Repeat("A", maxHFConfigSize)+`"}`) + writeFile(t, filepath.Join(modelDir, "README.md"), + "---\nlicense: MIT\nbase_model: org/Readme\n---\n"+strings.Repeat("A", maxReadmePrefixSize)) + return dir + }, + expectedPackages: []pkg.Package{ + { + Name: "Readme", + Type: pkg.ModelPkg, + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromFields("MIT", "", nil), + ), + Metadata: wantMetadata(""), + }, + }, + }, + { + // a _name_or_path that sanitizes to nothing doesn't block README base_model. + name: "degenerate _name_or_path falls through to README base_model", + setup: func(t *testing.T) string { + dir := t.TempDir() + modelDir := filepath.Join(dir, "sometensor") + model(t, modelDir) + writeFile(t, filepath.Join(modelDir, "config.json"), `{"_name_or_path":"."}`) + writeFile(t, filepath.Join(modelDir, "README.md"), "---\nbase_model: org/Readme\n---\n") + return dir + }, + expectedPackages: []pkg.Package{ + { + Name: "Readme", + Type: pkg.ModelPkg, + Metadata: wantMetadata(""), + }, + }, + }, { // grouping: independent models in one scan are grouped by their own // parent directory and each named from it. @@ -601,6 +684,60 @@ spdx-id: Apache-2.0 assertHasLicense(t, out[0], "Apache-2.0") }) + t.Run("OCI: config precedence is stable across layer order", func(t *testing.T) { + dir := t.TempDir() + a := filepath.Join(dir, "a.json") + b := filepath.Join(dir, "b.json") + gen := filepath.Join(dir, "generation_config.json") + require.NoError(t, os.WriteFile(a, []byte(`{"_name_or_path":"org/first"}`), 0o644)) + require.NoError(t, os.WriteFile(b, []byte(`{"_name_or_path":"org/second"}`), 0o644)) + require.NoError(t, os.WriteFile(gen, []byte(`{"max_new_tokens":10}`), 0o644)) + + configMd := pkg.SafeTensorsModelInfo{Format: "safetensors", TensorCount: 1} + for _, order := range [][]string{{a, b, gen}, {gen, b, a}} { + var locs []file.Location + for _, p := range order { + locs = append(locs, file.NewLocation(p)) + } + resolver := file.NewMockResolverForMediaTypes(map[string][]file.Location{ + dockerAIModelFileMediaType: locs, + }) + out, _, err := safeTensorsMergeProcessor(context.Background(), resolver, []pkg.Package{ociPkg(configMd)}, nil, nil) + require.NoError(t, err) + require.Len(t, out, 1) + assert.Equal(t, "first", out[0].Name) + // the non-HF generation config is not recorded as supporting evidence + assert.Len(t, out[0].Locations.ToSlice(), 3) + } + }) + + t.Run("dir scan: non-HF config.json in the parent is not supporting evidence", func(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.json") + require.NoError(t, os.WriteFile(cfgPath, []byte(`{"port":8080}`), 0o644)) + loc, cfg := findDirHFConfig(file.NewMockResolverForPaths(cfgPath), filepath.Join(dir, "model")) + assert.Nil(t, loc) + assert.Nil(t, cfg) + }) + + t.Run("OCI: a large license layer still yields its frontmatter spdx-id", func(t *testing.T) { + dir := t.TempDir() + licensePath := filepath.Join(dir, "LICENSE") + require.NoError(t, os.WriteFile(licensePath, + []byte("---\nspdx-id: Apache-2.0\n---\n"+strings.Repeat("A", maxLicensePrefixSize)), 0o644)) + assert.Equal(t, "Apache-2.0", readLicenseSPDXIDFromFrontmatter(file.NewMockResolverForPaths(licensePath), file.NewLocation(licensePath))) + }) + + t.Run("OCI: oversized model.file layer is rejected", func(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.json") + require.NoError(t, os.WriteFile(cfgPath, + []byte(`{"_name_or_path":"org/x","pad":"`+strings.Repeat("A", maxModelFileLayerSize)+`"}`), 0o644)) + cfg, fm := classifyOCIModelFileLayer(file.NewMockResolverForPaths(cfgPath), file.NewLocation(cfgPath)) + assert.Nil(t, cfg) + assert.Nil(t, fm) + }) + t.Run("names packages and still propagates an upstream error", func(t *testing.T) { // A non-nil upstream error (e.g. another file in the run failed to parse) // must NOT short-circuit naming. The package is still nameless at this @@ -622,7 +759,7 @@ spdx-id: Apache-2.0 // (or neither, asserting the drop path). // // Precedence (highest → lowest): -// 1. config.json _name_or_path (path.Base applied; both dir-scan and OCI) +// 1. config.json _name_or_path (sanitized to its last path element; both dir-scan and OCI) // 2. fallback name — OCI image-ref last segment, or dir-scan parent directory // base name (the merge processor computes the right one per group) // → drop (empty name) when nothing matches @@ -663,6 +800,31 @@ func TestSafeTensorsNamingPrecedence(t *testing.T) { want: "tiny-llama", }, + // sanitizing + { + name: "windows-style path uses the last element", + nameOrPath: `C:\models\win-model`, + fallbackName: "fallback-name", + want: "win-model", + }, + { + name: "degenerate path falls back", + nameOrPath: "..", + fallbackName: "fallback-name", + want: "fallback-name", + }, + { + name: "root path falls back", + nameOrPath: "/", + fallbackName: "fallback-name", + want: "fallback-name", + }, + { + name: "long names are truncated", + nameOrPath: "org/" + strings.Repeat("x", maxModelNameLength+10), + want: strings.Repeat("x", maxModelNameLength), + }, + // drops { name: "drop: both rungs empty", @@ -1036,25 +1198,78 @@ func TestParseFrontmatter(t *testing.T) { name string input string wantNil bool - wantLicense string + wantLicenses []string wantBaseModel []string }{ { name: "list base_model", input: "---\nlicense: mit\nbase_model:\n - org/Model\n---\nbody", - wantLicense: "mit", + wantLicenses: []string{"mit"}, wantBaseModel: []string{"org/Model"}, }, { name: "scalar base_model", input: "---\nlicense: apache-2.0\nbase_model: org/Model\n---\n", - wantLicense: "apache-2.0", + wantLicenses: []string{"apache-2.0"}, wantBaseModel: []string{"org/Model"}, }, { - name: "leading BOM", - input: "\xef\xbb\xbf---\nlicense: mit\n---\n", - wantLicense: "mit", + name: "list license keeps base_model", + input: "---\nlicense: [mit, apache-2.0]\nbase_model: org/Model\n---\n", + wantLicenses: []string{"mit", "apache-2.0"}, + wantBaseModel: []string{"org/Model"}, + }, + { + name: "mapping-valued fields are ignored, not fatal", + input: "---\nlicense: {name: mit}\nbase_model: org/Model\n---\n", + wantBaseModel: []string{"org/Model"}, + }, + { + name: "leading BOM", + input: "\xef\xbb\xbf---\nlicense: mit\n---\n", + wantLicenses: []string{"mit"}, + }, + { + name: "null license is ignored", + input: "---\nlicense: null\nbase_model: ~\n---\n", + wantLicenses: nil, + wantBaseModel: nil, + }, + { + name: "empty block", + input: "---\n---\n", + wantNil: true, + }, + { + name: "oversized block is rejected", + input: "---\nlicense: mit\n" + strings.Repeat("# pad\n", maxFrontmatterSize/6+1) + "---\n", + wantNil: true, + }, + { + name: "closing delimiter at EOF with trailing whitespace", + input: "---\nlicense: mit\n--- ", + wantLicenses: []string{"mit"}, + }, + { + name: "list values are capped", + input: "---\nlicense: [" + strings.Repeat("mit,", maxFrontmatterValues+5) + "mit]\n---\n", + wantLicenses: slices.Repeat([]string{"mit"}, maxFrontmatterValues), + }, + { + name: "CRLF delimiters", + input: "---\r\nlicense: mit\r\n---\r\nbody", + wantLicenses: []string{"mit"}, + }, + { + name: "opening delimiter must be exactly ---", + input: "----\nlicense: mit\n---\n", + wantNil: true, + }, + { + name: "a ---x line inside the block does not close it", + input: "---\nbase_model: org/Model\n---x: 1\nlicense: mit\n---\n", + wantLicenses: []string{"mit"}, + wantBaseModel: []string{"org/Model"}, }, { name: "no frontmatter", @@ -1082,7 +1297,7 @@ func TestParseFrontmatter(t *testing.T) { return } require.NotNil(t, fm) - assert.Equal(t, tt.wantLicense, fm.License) + assert.Equal(t, tt.wantLicenses, fm.Licenses) if tt.wantBaseModel != nil { assert.Equal(t, tt.wantBaseModel, fm.BaseModel) } diff --git a/syft/pkg/cataloger/ai/processor.go b/syft/pkg/cataloger/ai/processor.go index fd0e29292..5cb31c3e7 100644 --- a/syft/pkg/cataloger/ai/processor.go +++ b/syft/pkg/cataloger/ai/processor.go @@ -104,10 +104,10 @@ func mergeDirModels(ctx context.Context, resolver file.Resolver, pkgs []pkg.Pack // groupByParentDir buckets filesystem-scanned models by the directory their // primary-evidence file lives in. - +// // This encodes a deliberate assumption: a directory holds one logical // model, so every .safetensors file in a directory is treated as a shard of the -// same modeland merged into one package. +// same model and merged into one package. // The trade-off is that if a directory happens to contain several unrelated models, // they are merged into one package rather than reported separately. // We accept that because the conventional on-disk layout gives each model diff --git a/syft/pkg/cataloger/ai/testdata/safetensors/extract_header.go b/syft/pkg/cataloger/ai/testdata/safetensors/extract_header.go index 4f7d72a60..8445adcb7 100644 --- a/syft/pkg/cataloger/ai/testdata/safetensors/extract_header.go +++ b/syft/pkg/cataloger/ai/testdata/safetensors/extract_header.go @@ -28,9 +28,9 @@ import ( const ( safetensorsLayerMediaType = "application/vnd.docker.ai.safetensors" - // 8 MB matches maxHeaderBytes in the OCI model source. Real model headers - // are well under 1 MB; the extra slack covers outliers. - fetchBytes = 8 * 1024 * 1024 + // matches maxSafeTensorsHeaderBytes in the OCI model source (8 MB header + // cap plus the 8-byte length prefix). Real model headers are well under 1 MB. + fetchBytes = 8*1024*1024 + 8 ) func main() { diff --git a/syft/source/ocimodelsource/oci_model_source.go b/syft/source/ocimodelsource/oci_model_source.go index c93e9c3fa..77ab1b187 100644 --- a/syft/source/ocimodelsource/oci_model_source.go +++ b/syft/source/ocimodelsource/oci_model_source.go @@ -135,8 +135,8 @@ func fetchAndStoreModelHeaders(ctx context.Context, client *registryClient, arti } } - // SafeTensors weight-layer headers. We only pull the leading prefix (same - // budget as a GGUF header) + // SafeTensors weight-layer headers. We only pull the leading + // maxSafeTensorsHeaderBytes if artifact.Format == modelFormatSafeTensors { for _, layer := range artifact.SafeTensorsLayers { li, err := fetchSafeTensorsLayerHeader(ctx, client, artifact.Reference, layer, tempDir) @@ -174,7 +174,7 @@ func storeConfigBlobAsLayer(artifact *modelArtifact, tempDir string) (fileresolv // fetchCompanionLayer downloads a companion (non-weight) layer to a temp file func fetchCompanionLayer(ctx context.Context, client *registryClient, ref name.Reference, layer v1.Descriptor, tempDir string) (fileresolver.LayerInfo, error) { - data, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxCompanionBytes) + data, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxCompanionBytes+1) if err != nil { return fileresolver.LayerInfo{}, fmt.Errorf("failed to fetch companion layer: %w", err) } @@ -191,7 +191,7 @@ func fetchCompanionLayer(ctx context.Context, client *registryClient, ref name.R // fetchSingleGGUFHeader fetches a single GGUF layer header and writes it to a temp file func fetchSingleGGUFHeader(ctx context.Context, client *registryClient, ref name.Reference, layer v1.Descriptor, tempDir string) (fileresolver.LayerInfo, error) { - headerData, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxWeightHeaderBytes) + headerData, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxGGUFHeaderBytes) if err != nil { return fileresolver.LayerInfo{}, fmt.Errorf("failed to fetch GGUF layer header: %w", err) } @@ -212,7 +212,7 @@ func fetchSingleGGUFHeader(ctx context.Context, client *registryClient, ref name // fetchSafeTensorsLayerHeader fetches the leading bytes of a SafeTensors weight // layer (enough to cover the JSON header) and writes them to a temp file func fetchSafeTensorsLayerHeader(ctx context.Context, client *registryClient, ref name.Reference, layer v1.Descriptor, tempDir string) (fileresolver.LayerInfo, error) { - headerData, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxWeightHeaderBytes) + headerData, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxSafeTensorsHeaderBytes) if err != nil { return fileresolver.LayerInfo{}, fmt.Errorf("failed to fetch safetensors layer header: %w", err) } diff --git a/syft/source/ocimodelsource/registry_client.go b/syft/source/ocimodelsource/registry_client.go index 8e6c6b9c7..0d4be3a22 100644 --- a/syft/source/ocimodelsource/registry_client.go +++ b/syft/source/ocimodelsource/registry_client.go @@ -16,6 +16,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/anchore/stereoscope/pkg/image" + "github.com/anchore/syft/internal" ) // errNotModelArtifact is returned when a reference does not point to a model artifact. @@ -37,17 +38,20 @@ const ( modelFormatGGUF = "gguf" modelFormatSafeTensors = "safetensors" - // maxWeightHeaderBytes is the leading slice we range-GET from a (multi-GB) - // weight layer — enough to cover the GGUF/safetensors header. Note this is - // smaller than the ai cataloger's own maxSafeTensorsHeaderSize (100 MB) parse - // ceiling: a safetensors header between the two is parseable from a directory - // scan but truncated here, so its shard would go uncounted on an OCI scan. - // Keep this comfortably above real-world header sizes. - maxWeightHeaderBytes = 8 * 1024 * 1024 // 8 MB + // maxGGUFHeaderBytes is the leading slice we range-GET from a (multi-GB) + // GGUF weight layer, enough to cover its header. See + // https://github.com/ggml-org/ggml/blob/master/docs/gguf.md#file-structure + maxGGUFHeaderBytes = 8 * 1024 * 1024 // 8 MB + + // maxSafeTensorsHeaderBytes is the leading slice we range-GET from a + // safetensors weight layer: the 8-byte length prefix plus the same header + // ceiling the ai cataloger applies to directory scans. + maxSafeTensorsHeaderBytes = internal.MaxSafeTensorsHeaderSize + 8 // maxCompanionBytes caps a whole companion blob (README, config.json, - // license); these are small by convention. Matches the 4 MB read cap in - // classifyOCIModelFileLayer. + // license); these are small by convention. It is the upper bound of the ai + // cataloger's per-layer caps. One extra byte is fetched so the cataloger can + // tell an oversized blob from one that is exactly at the cap. maxCompanionBytes = 4 * 1024 * 1024 // 4 MB ) @@ -279,9 +283,6 @@ func (c *registryClient) fetchBlobRange(ctx context.Context, ref name.Reference, } defer reader.Close() - // Note: this is not some arbitrary number picked out of the blue. - // This is based on the specification of header data found here: - // https://github.com/ggml-org/ggml/blob/master/docs/gguf.md#file-structure data := make([]byte, maxBytes) n, err := io.ReadFull(reader, data)