oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-04 08:26:38 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix(ai): bound model companion file reads and validate their contents (#5344)

Browse Source 
Companion files next to AI models (`config.json`, `README.md`, and the OCI model.file, license and model config layers) are now read with named size limits, and are only used when they look like what they claim to be.

- JSON companions are read whole up to a limit and skipped when larger, rather than being fully buffered or parsed truncated. OCI companion layers are fetched one byte past the cap so oversize is detectable.
- README and license layers read a prefix. The frontmatter block is capped at 256 KB and list values at 32.
- dir and OCI scans share one 8 MB safetensors header cap. Directory scans used to allow 100 MB.
- `config.json` is only used when it has `architectures` or `_name_or_path`, and is looked up beside the model or in its parent directory, no longer all the way up to the scan root
- OCI model.file layers are processed in a stable order, so the chosen name no longer changes between runs
- README frontmatter:
  - a list-form `license:` is accepted and no longer drops `base_model`
  - `null` values are ignored
  - delimiters must be exact `---` lines
- model names from `_name_or_path` / `base_model`:
  - Windows paths are handled
  - `.`, `..` and `/` fall back to the next name source
  - names are capped at 256 characters

Behavior changes worth noting:
- a `config.json` two or more directories above the model no longer names it
- JSON `config.json` files without `architectures` or `_name_or_path` (e.g. `generation_config.json`, app configs) are ignored
- safetensors headers over 8 MB in directory scans are now rejected, matching OCI scans (real shard headers are well under 1 MB)

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
2026-09-29 09:33:57 -04:00
committed by GitHub
parent 10af8342c0
commit 50dbb33e66
13 changed files with 456 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
syft/pkg/cataloger/ai/identity_dir.go
+28 -20
View File
@@ -3,7 +3,6 @@ package ai
import (
"context"
"encoding/json"
"io"
"path"
"github.com/anchore/syft/internal"
@@ -13,42 +12,42 @@ import (
)
// resolveSafeTensorsDirIdentity handles the directory-scan case for safe tensors
// find config.json beside the model files (walking up parent directories to the
// scanned source root if no sibling exists) and a sibling README.md. It returns
// find config.json beside the model files (or one directory up, for the shard
// subdirectory layout) and a sibling README.md. It returns
// the group's name candidates, resolved licenses, and supporting evidence.
func resolveSafeTensorsDirIdentity(ctx context.Context, resolver file.Resolver, dir string, md *pkg.SafeTensorsModelInfo) safeTensorsIdentity {
id := safeTensorsIdentity{fallbackName: safeTensorsDirName(dir)}
if loc, cfg := findDirHFConfig(resolver, dir); cfg != nil {
applyHFConfig(md, cfg)
id.nameOrPath = cfg.NameOrPath
id.nameOrPath = sanitizeModelName(cfg.NameOrPath)
id.supporting = append(id.supporting, *loc)
}
if loc, fm := readDirReadmeFrontmatter(resolver, path.Join(dir, "README.md")); fm != nil {
if fm.License != "" {
id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, fm.License)
if len(fm.Licenses) > 0 {
id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, fm.Licenses...)
}
if id.nameOrPath == "" && len(fm.BaseModel) > 0 {
id.nameOrPath = fm.BaseModel[0]
if id.nameOrPath == "" {
id.nameOrPath = firstModelName(fm.BaseModel)
}
id.supporting = append(id.supporting, *loc)
}
return id
}
// findDirHFConfig looks for a config.json beside the model files
// findDirHFConfig looks for an HF config.json beside the model files, then in
// the parent directory (covers weights kept in a subdirectory of the model). It
// does not walk further up, so an unrelated config.json higher in the tree can't
// name every model below it.
func findDirHFConfig(resolver file.Resolver, dir string) (*file.Location, *hfConfig) {
for {
if loc, cfg := readDirHFConfig(resolver, path.Join(dir, "config.json")); cfg != nil {
return loc, cfg
}
parent := path.Dir(dir)
if parent == dir {
return nil, nil // reached the source root
}
dir = parent
if loc, cfg := readDirHFConfig(resolver, path.Join(dir, "config.json")); cfg != nil {
return loc, cfg
}
if parent := path.Dir(dir); parent != dir {
return readDirHFConfig(resolver, path.Join(parent, "config.json"))
}
return nil, nil
}
func readDirHFConfig(resolver file.Resolver, p string) (*file.Location, *hfConfig) {
@@ -62,11 +61,19 @@ func readDirHFConfig(resolver file.Resolver, p string) (*file.Location, *hfConfi
}
defer internal.CloseAndLogError(rc, p)
buf, err := readBounded(rc, maxHFConfigSize)
if err != nil {
log.Debugf("failed to read %s: %v", p, err)
return nil, nil
}
var cfg hfConfig
if err := json.NewDecoder(rc).Decode(&cfg); err != nil {
if err := json.Unmarshal(buf, &cfg); err != nil {
log.Debugf("failed to decode %s: %v", p, err)
return nil, nil
}
if !cfg.looksLikeHF() {
return nil, nil
}
return &locations[0], &cfg
}
@@ -81,8 +88,9 @@ func readDirReadmeFrontmatter(resolver file.Resolver, p string) (*file.Location,
}
defer internal.CloseAndLogError(rc, p)
buf, err := io.ReadAll(io.LimitReader(rc, 1024*1024))
buf, err := readPrefix(rc, maxReadmePrefixSize)
if err != nil {
log.Debugf("failed to read %s: %v", p, err)
return nil, nil
}
fm := parseFrontmatter(buf)