diff --git a/syft/format/internal/cyclonedxutil/helpers/component_test.go b/syft/format/internal/cyclonedxutil/helpers/component_test.go index 2b1f44421..7d7a8d6df 100644 --- a/syft/format/internal/cyclonedxutil/helpers/component_test.go +++ b/syft/format/internal/cyclonedxutil/helpers/component_test.go @@ -345,6 +345,73 @@ func Test_decodeComponent(t *testing.T) { }, wantPURL: "pkg:rpm/centos/acl@2.2.53-1.el8?arch=x86_64&upstream=acl-2.2.53-1.el8.src.rpm&distro=centos-8", }, + { + // regression: the self-referencing Parts field made decoding recurse without bound + name: "decode safetensors model metadata", + component: cyclonedx.Component{ + Name: "lib", + Version: "UNKNOWN", + Type: cyclonedx.ComponentTypeMachineLearningModel, + Properties: &[]cyclonedx.Property{ + {Name: "syft:package:foundBy", Value: "safetensors-cataloger"}, + {Name: "syft:package:type", Value: "model"}, + {Name: "syft:package:metadataType", Value: "safetensors-model-info"}, + {Name: "syft:metadata:format", Value: "safetensors"}, + {Name: "syft:metadata:metadataHash", Value: "afcae99661c33f88"}, + {Name: "syft:metadata:parameters", Value: "16"}, + {Name: "syft:metadata:quantization", Value: "I8"}, + {Name: "syft:metadata:shardCount", Value: "1"}, + {Name: "syft:metadata:tensorCount", Value: "1"}, + }, + }, + wantMetadata: pkg.SafeTensorsModelInfo{ + Format: "safetensors", + MetadataHash: "afcae99661c33f88", + Parameters: 16, + Quantization: "I8", + ShardCount: 1, + TensorCount: 1, + }, + }, + { + name: "decode safetensors model metadata with parts", + component: cyclonedx.Component{ + Name: "model", + Type: cyclonedx.ComponentTypeMachineLearningModel, + Properties: &[]cyclonedx.Property{ + {Name: "syft:package:type", Value: "model"}, + {Name: "syft:package:metadataType", Value: "safetensors-model-info"}, + {Name: "syft:metadata:format", Value: "safetensors"}, + {Name: "syft:metadata:parts:0:format", Value: "safetensors"}, + {Name: "syft:metadata:parts:0:tensorCount", Value: "2"}, + {Name: "syft:metadata:parts:1:tensorCount", Value: "3"}, + }, + }, + wantMetadata: pkg.SafeTensorsModelInfo{ + Format: "safetensors", + Parts: []pkg.SafeTensorsModelInfo{ + {Format: "safetensors", TensorCount: 2}, + {TensorCount: 3}, + }, + }, + }, + { + name: "decode gguf model metadata", + component: cyclonedx.Component{ + Name: "model", + Type: cyclonedx.ComponentTypeMachineLearningModel, + Properties: &[]cyclonedx.Property{ + {Name: "syft:package:type", Value: "model"}, + {Name: "syft:package:metadataType", Value: "gguf-file-header"}, + {Name: "syft:metadata:ggufVersion", Value: "3"}, + {Name: "syft:metadata:tensorCount", Value: "291"}, + }, + }, + wantMetadata: pkg.GGUFFileHeader{ + GGUFVersion: 3, + TensorCount: 291, + }, + }, { name: "generate a purl from package type", component: cyclonedx.Component{ diff --git a/syft/format/internal/cyclonedxutil/helpers/property_encoder.go b/syft/format/internal/cyclonedxutil/helpers/property_encoder.go index 2aeb8b6d0..bdfb000fa 100644 --- a/syft/format/internal/cyclonedxutil/helpers/property_encoder.go +++ b/syft/format/internal/cyclonedxutil/helpers/property_encoder.go @@ -171,7 +171,7 @@ func Decode(typ reflect.Type, values map[string]string, prefix string, fn FieldN v := reflect.New(typ) - decode(values, v, prefix, fn) + decode(values, sortedPropertyNames(values), v, prefix, fn) switch { case isSlice && isPtr: @@ -192,11 +192,11 @@ func DecodeInto(obj any, values map[string]string, prefix string, fn FieldName) value = value.Elem() } - decode(values, value, prefix, fn) + decode(values, sortedPropertyNames(values), value, prefix, fn) } //nolint:funlen,gocognit,gocyclo -func decode(vals map[string]string, value reflect.Value, prefix string, fn FieldName) bool { +func decode(vals map[string]string, names propertyNames, value reflect.Value, prefix string, fn FieldName) bool { if !value.IsValid() { return false } @@ -211,7 +211,7 @@ func decode(vals map[string]string, value reflect.Value, prefix string, fn Field if v.IsNil() { v = reflect.New(t) } - if decode(vals, v.Elem(), prefix, fn) && value.CanSet() { + if decode(vals, names, v.Elem(), prefix, fn) && value.CanSet() { o := v.Interface() log.Tracef("%v", o) value.Set(v) @@ -273,7 +273,7 @@ func decode(vals map[string]string, value reflect.Value, prefix string, fn Field newType = t.Elem() } v := reflect.New(newType) - if decode(vals, v.Elem(), str, fn) { + if decode(vals, names, v.Elem(), str, fn) { // append to slice if t.Kind() != reflect.Pointer { v = v.Elem() @@ -308,7 +308,8 @@ func decode(vals map[string]string, value reflect.Value, prefix string, fn Field newKeyType = keyType.Elem() } k := reflect.New(newKeyType) - if !decode(keyVals, k.Elem(), key, fn) { + // a key is decoded from the single property holding it, so it has no nested property names + if !decode(keyVals, nil, k.Elem(), key, fn) { log.Debugf("unable to decode key for: %s", key) continue } @@ -322,7 +323,7 @@ func decode(vals map[string]string, value reflect.Value, prefix string, fn Field newValueType = valueType.Elem() } v := reflect.New(newValueType) - if decode(vals, v.Elem(), key, fn) { + if decode(vals, names, v.Elem(), key, fn) { if valueType.Kind() != reflect.Pointer { v = v.Elem() } @@ -339,6 +340,13 @@ func decode(vals map[string]string, value reflect.Value, prefix string, fn Field return false } case reflect.Struct: + // every field is decoded from the property named prefix or from one nested below it, so there is + // nothing to decode when there is none. Stopping here also bounds the recursion for a type that refers + // to itself (e.g. a Parts []T field): otherwise decoding the first element of an absent []T decodes + // the same []T one level deeper, without end. + if !names.hasPrefix(prefix) { + return false + } values := false for i := 0; i < typ.NumField(); i++ { f := typ.Field(i) @@ -349,7 +357,7 @@ func decode(vals map[string]string, value reflect.Value, prefix string, fn Field continue } - if decode(vals, v, name, fn) { + if decode(vals, names, v, name, fn) { values = true } } @@ -361,6 +369,34 @@ func decode(vals map[string]string, value reflect.Value, prefix string, fn Field return true } +// propertyNames holds the names of the properties being decoded, sorted, so that decode can find whether a property +// is nested below a prefix without scanning every property for every struct it decodes +type propertyNames []string + +func sortedPropertyNames(vals map[string]string) propertyNames { + names := make(propertyNames, 0, len(vals)) + for name := range vals { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// hasPrefix reports whether there is a property named prefix or any property nested below it +func (names propertyNames) hasPrefix(prefix string) bool { + if prefix == "" { + return len(names) > 0 + } + i := sort.SearchStrings(names, prefix) + if i < len(names) && names[i] == prefix { + return true + } + // names sorting between prefix and prefix+":" (e.g. "parts:10" between "parts:1" and "parts:1:") are skipped + nested := prefix + ":" + i += sort.SearchStrings(names[i:], nested) + return i < len(names) && strings.HasPrefix(names[i], nested) +} + func PtrToStruct(ptr any) any { v := reflect.ValueOf(ptr) if v.IsZero() && v.Type().Kind() != reflect.Struct { diff --git a/syft/format/internal/cyclonedxutil/helpers/property_encoder_test.go b/syft/format/internal/cyclonedxutil/helpers/property_encoder_test.go index 6b16c937c..b326c44c7 100644 --- a/syft/format/internal/cyclonedxutil/helpers/property_encoder_test.go +++ b/syft/format/internal/cyclonedxutil/helpers/property_encoder_test.go @@ -41,6 +41,13 @@ type T5 struct { Map map[string]string } +// TRecursive refers to itself, like pkg.SafeTensorsModelInfo and pkg.GGUFFileHeader do through their Parts field +type TRecursive struct { + Name string + Children []TRecursive + Next *TRecursive +} + func Test_EncodeDecodeCycle(t *testing.T) { val := 99 @@ -122,6 +129,33 @@ func Test_EncodeDecodeCycle(t *testing.T) { {"t2 elem 1"}, }, }, + { + name: "recursive type without children", + value: &TRecursive{ + Name: "root", + }, + }, + { + name: "recursive type with nested children", + value: &TRecursive{ + Name: "root", + Children: []TRecursive{ + { + Name: "child 0", + Children: []TRecursive{ + {Name: "grandchild 0"}, + }, + }, + { + Name: "child 1", + Next: &TRecursive{Name: "next of child 1"}, + }, + }, + Next: &TRecursive{ + Name: "next", + }, + }, + }, { name: "map of strings", value: &T5{ @@ -154,3 +188,56 @@ func Test_EncodeDecodeCycle(t *testing.T) { }) } } + +func Test_propertyNames_hasPrefix(t *testing.T) { + names := sortedPropertyNames(map[string]string{ + "props:name": "root", + "props:children:10:x": "a", + "props:children:2:x": "b", + "props:children:3": "c", + "props:children:40": "d", + "props:children:4:x": "e", + }) + + tests := []struct { + prefix string + want bool + }{ + {prefix: "", want: true}, + {prefix: "props", want: true}, + {prefix: "props:name", want: true}, + {prefix: "props:nam", want: false}, + {prefix: "props:children", want: true}, + {prefix: "props:children:0", want: false}, + // only "props:children:10:x" shares this prefix as a string + {prefix: "props:children:1", want: false}, + {prefix: "props:children:10", want: true}, + {prefix: "props:children:2", want: true}, + {prefix: "props:children:3", want: true}, + {prefix: "props:children:3:x", want: false}, + // "props:children:40" sorts between this prefix and the nested "props:children:4:x" + {prefix: "props:children:4", want: true}, + {prefix: "other", want: false}, + } + for _, test := range tests { + t.Run(test.prefix, func(t *testing.T) { + assert.Equal(t, test.want, names.hasPrefix(test.prefix)) + }) + } + + assert.False(t, sortedPropertyNames(nil).hasPrefix("")) +} + +func Test_DecodeRecursiveTypeWithManyChildren(t *testing.T) { + // more than ten children, so that "children:1" sorts next to "children:10" + value := &TRecursive{Name: "root"} + for i := 0; i < 12; i++ { + value.Children = append(value.Children, TRecursive{ + Children: []TRecursive{{Name: "grandchild"}}, + }) + } + + vals := Encode(value, "props", OptionalJSONTag) + + assert.EqualValues(t, value, Decode(reflect.TypeOf(value), vals, "props", OptionalJSONTag)) +}