mirror of
https://github.com/anchore/syft.git
synced 2026-11-03 07:56:36 +01:00
Alex Goodman
50dbb33e66
Companion files next to AI models (`config.json`, `README.md`, and the OCI model.file, license and model config layers) are now read with named size limits, and are only used when they look like what they claim to be. - JSON companions are read whole up to a limit and skipped when larger, rather than being fully buffered or parsed truncated. OCI companion layers are fetched one byte past the cap so oversize is detectable. - README and license layers read a prefix. The frontmatter block is capped at 256 KB and list values at 32. - dir and OCI scans share one 8 MB safetensors header cap. Directory scans used to allow 100 MB. - `config.json` is only used when it has `architectures` or `_name_or_path`, and is looked up beside the model or in its parent directory, no longer all the way up to the scan root - OCI model.file layers are processed in a stable order, so the chosen name no longer changes between runs - README frontmatter: - a list-form `license:` is accepted and no longer drops `base_model` - `null` values are ignored - delimiters must be exact `---` lines - model names from `_name_or_path` / `base_model`: - Windows paths are handled - `.`, `..` and `/` fall back to the next name source - names are capped at 256 characters Behavior changes worth noting: - a `config.json` two or more directories above the model no longer names it - JSON `config.json` files without `architectures` or `_name_or_path` (e.g. `generation_config.json`, app configs) are ignored - safetensors headers over 8 MB in directory scans are now rejected, matching OCI scans (real shard headers are well under 1 MB) Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
102 lines
3.0 KiB
Go
102 lines
3.0 KiB
Go
package ai
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"path"
|
|
|
|
"github.com/anchore/syft/internal"
|
|
"github.com/anchore/syft/internal/log"
|
|
"github.com/anchore/syft/syft/file"
|
|
"github.com/anchore/syft/syft/pkg"
|
|
)
|
|
|
|
// resolveSafeTensorsDirIdentity handles the directory-scan case for safe tensors
|
|
// find config.json beside the model files (or one directory up, for the shard
|
|
// subdirectory layout) and a sibling README.md. It returns
|
|
// the group's name candidates, resolved licenses, and supporting evidence.
|
|
func resolveSafeTensorsDirIdentity(ctx context.Context, resolver file.Resolver, dir string, md *pkg.SafeTensorsModelInfo) safeTensorsIdentity {
|
|
id := safeTensorsIdentity{fallbackName: safeTensorsDirName(dir)}
|
|
|
|
if loc, cfg := findDirHFConfig(resolver, dir); cfg != nil {
|
|
applyHFConfig(md, cfg)
|
|
id.nameOrPath = sanitizeModelName(cfg.NameOrPath)
|
|
id.supporting = append(id.supporting, *loc)
|
|
}
|
|
|
|
if loc, fm := readDirReadmeFrontmatter(resolver, path.Join(dir, "README.md")); fm != nil {
|
|
if len(fm.Licenses) > 0 {
|
|
id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, fm.Licenses...)
|
|
}
|
|
if id.nameOrPath == "" {
|
|
id.nameOrPath = firstModelName(fm.BaseModel)
|
|
}
|
|
id.supporting = append(id.supporting, *loc)
|
|
}
|
|
return id
|
|
}
|
|
|
|
// findDirHFConfig looks for an HF config.json beside the model files, then in
|
|
// the parent directory (covers weights kept in a subdirectory of the model). It
|
|
// does not walk further up, so an unrelated config.json higher in the tree can't
|
|
// name every model below it.
|
|
func findDirHFConfig(resolver file.Resolver, dir string) (*file.Location, *hfConfig) {
|
|
if loc, cfg := readDirHFConfig(resolver, path.Join(dir, "config.json")); cfg != nil {
|
|
return loc, cfg
|
|
}
|
|
if parent := path.Dir(dir); parent != dir {
|
|
return readDirHFConfig(resolver, path.Join(parent, "config.json"))
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func readDirHFConfig(resolver file.Resolver, p string) (*file.Location, *hfConfig) {
|
|
locations, err := resolver.FilesByPath(p)
|
|
if err != nil || len(locations) == 0 {
|
|
return nil, nil
|
|
}
|
|
rc, err := resolver.FileContentsByLocation(locations[0])
|
|
if err != nil {
|
|
return nil, nil
|
|
}
|
|
defer internal.CloseAndLogError(rc, p)
|
|
|
|
buf, err := readBounded(rc, maxHFConfigSize)
|
|
if err != nil {
|
|
log.Debugf("failed to read %s: %v", p, err)
|
|
return nil, nil
|
|
}
|
|
var cfg hfConfig
|
|
if err := json.Unmarshal(buf, &cfg); err != nil {
|
|
log.Debugf("failed to decode %s: %v", p, err)
|
|
return nil, nil
|
|
}
|
|
if !cfg.looksLikeHF() {
|
|
return nil, nil
|
|
}
|
|
return &locations[0], &cfg
|
|
}
|
|
|
|
func readDirReadmeFrontmatter(resolver file.Resolver, p string) (*file.Location, *readmeFrontmatter) {
|
|
locations, err := resolver.FilesByPath(p)
|
|
if err != nil || len(locations) == 0 {
|
|
return nil, nil
|
|
}
|
|
rc, err := resolver.FileContentsByLocation(locations[0])
|
|
if err != nil {
|
|
return nil, nil
|
|
}
|
|
defer internal.CloseAndLogError(rc, p)
|
|
|
|
buf, err := readPrefix(rc, maxReadmePrefixSize)
|
|
if err != nil {
|
|
log.Debugf("failed to read %s: %v", p, err)
|
|
return nil, nil
|
|
}
|
|
fm := parseFrontmatter(buf)
|
|
if fm == nil {
|
|
return nil, nil
|
|
}
|
|
return &locations[0], fm
|
|
}
|