mirror of
https://github.com/anchore/syft.git
synced 2026-11-03 07:56:36 +01:00
Alex Goodman
50dbb33e66
Companion files next to AI models (`config.json`, `README.md`, and the OCI model.file, license and model config layers) are now read with named size limits, and are only used when they look like what they claim to be. - JSON companions are read whole up to a limit and skipped when larger, rather than being fully buffered or parsed truncated. OCI companion layers are fetched one byte past the cap so oversize is detectable. - README and license layers read a prefix. The frontmatter block is capped at 256 KB and list values at 32. - dir and OCI scans share one 8 MB safetensors header cap. Directory scans used to allow 100 MB. - `config.json` is only used when it has `architectures` or `_name_or_path`, and is looked up beside the model or in its parent directory, no longer all the way up to the scan root - OCI model.file layers are processed in a stable order, so the chosen name no longer changes between runs - README frontmatter: - a list-form `license:` is accepted and no longer drops `base_model` - `null` values are ignored - delimiters must be exact `---` lines - model names from `_name_or_path` / `base_model`: - Windows paths are handled - `.`, `..` and `/` fall back to the next name source - names are capped at 256 characters Behavior changes worth noting: - a `config.json` two or more directories above the model no longer names it - JSON `config.json` files without `architectures` or `_name_or_path` (e.g. `generation_config.json`, app configs) are ignored - safetensors headers over 8 MB in directory scans are now rejected, matching OCI scans (real shard headers are well under 1 MB) Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
52 lines
1.3 KiB
Go
52 lines
1.3 KiB
Go
package ai
|
|
|
|
import (
|
|
"path"
|
|
"strings"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
// pickSafeTensorsName implements the documented naming precedence chain:
|
|
func pickSafeTensorsName(nameOrPath, fallbackName string) string {
|
|
if name := sanitizeModelName(nameOrPath); name != "" {
|
|
return name
|
|
}
|
|
return fallbackName
|
|
}
|
|
|
|
// firstModelName returns the first value that sanitizes to a usable name.
|
|
func firstModelName(values []string) string {
|
|
for _, v := range values {
|
|
if name := sanitizeModelName(v); name != "" {
|
|
return name
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// sanitizeModelName reduces a producer-supplied name or path (_name_or_path,
|
|
// base_model) to its last path element, capped at maxModelNameLength runes. It
|
|
// returns "" when nothing usable is left.
|
|
func sanitizeModelName(nameOrPath string) string {
|
|
base := path.Base(strings.ReplaceAll(nameOrPath, `\`, "/"))
|
|
switch base {
|
|
case "/", ".", "..", "":
|
|
return ""
|
|
}
|
|
if utf8.RuneCountInString(base) > maxModelNameLength {
|
|
base = string([]rune(base)[:maxModelNameLength])
|
|
}
|
|
return base
|
|
}
|
|
|
|
// safeTensorsDirName returns the directory-scan naming fallback: the base name
|
|
// of the group's parent directory (the group key is already that directory).
|
|
func safeTensorsDirName(directory string) string {
|
|
base := path.Base(directory)
|
|
switch base {
|
|
case "/", ".", "":
|
|
return ""
|
|
}
|
|
return base
|
|
}
|