mirror of
https://github.com/anchore/syft.git
synced 2026-11-02 07:26:34 +01:00
feat: initial recursive archive cataloging impl
Signed-off-by: Keith Zantow <keith.zantow@anchore.com>
This commit is contained in:
Keith Zantow
@@ -0,0 +1,328 @@
|
||||
// Package index provides a concurrent key-split (radix) index and a forward/reverse pair over it.
|
||||
//
|
||||
// Ported from a prototype dir scan package whose search strategy this
|
||||
// exists to serve: a path index that answers "which entries end in .jar" and "which entries live
|
||||
// under this directory" without walking a tree, so a cataloger's `**/*.ext` glob is a lookup keyed
|
||||
// on the extension rather than a scan of every path.
|
||||
//
|
||||
// Kept close to the original so the two can be diffed. The lock type is local (see lock.go) rather
|
||||
// than pulled in from the prototype's sync/atomic package, which carries unrelated helpers.
|
||||
package index
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ----------------------- KeySplitIndex -----------------------
|
||||
|
||||
type KeySplitIndex[T any] struct {
|
||||
Node[T]
|
||||
}
|
||||
|
||||
var _ interface {
|
||||
json.Marshaler
|
||||
json.Unmarshaler
|
||||
} = (*KeySplitIndex[int])(nil)
|
||||
|
||||
func (n *KeySplitIndex[T]) MarshalJSON() ([]byte, error) {
|
||||
values := map[string]T{}
|
||||
collectValues(values, "", &n.Node)
|
||||
buf := &bytes.Buffer{}
|
||||
enc := json.NewEncoder(buf)
|
||||
enc.SetEscapeHTML(false)
|
||||
err := enc.Encode(values)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func (n *KeySplitIndex[T]) UnmarshalJSON(bytes []byte) error {
|
||||
values := map[string]T{}
|
||||
err := json.Unmarshal(bytes, &values)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for k, v := range values {
|
||||
n.Set(k, v)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func collectValues[T any](values map[string]T, key string, node *Node[T]) {
|
||||
if node.set {
|
||||
values[key] = node.value
|
||||
}
|
||||
for segment, child := range node.keyMap {
|
||||
collectValues(values, key+segment, child)
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------- Node -----------------------
|
||||
|
||||
type NodeMap[T any] map[string]*Node[T]
|
||||
|
||||
type Node[T any] struct {
|
||||
lockable
|
||||
set bool
|
||||
value T
|
||||
keyMap NodeMap[T]
|
||||
keyByChar map[rune]string
|
||||
}
|
||||
|
||||
type NodeUpdateFunc[T any] func(current *Node[T]) (newValue T)
|
||||
|
||||
func (n *Node[T]) Value() (v T) {
|
||||
unlock := n.RLock()
|
||||
v = n.value
|
||||
unlock()
|
||||
return
|
||||
}
|
||||
|
||||
func (n *Node[T]) Contains(s string) bool {
|
||||
_, equal := n._find(s)
|
||||
return equal
|
||||
}
|
||||
|
||||
func (n *Node[T]) Get(s string) (out T) {
|
||||
v, equal := n._find(s)
|
||||
if !equal || v == nil {
|
||||
return
|
||||
}
|
||||
out = v.value
|
||||
return
|
||||
}
|
||||
|
||||
func (n *Node[T]) Set(name string, value T) {
|
||||
node := n._makeNodeP(nil, name, nil)
|
||||
node.SetValue(value)
|
||||
}
|
||||
|
||||
func (n *Node[T]) ByPrefix(s string) []T {
|
||||
v, _ := n._find(s)
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
return v.Collect()
|
||||
}
|
||||
|
||||
func (n *Node[T]) Update(name string, f NodeUpdateFunc[T]) {
|
||||
unlock := n.Lock()
|
||||
node := n._makeNodeP(&unlock, name, nil)
|
||||
v := f(node)
|
||||
node.SetValue(v)
|
||||
unlock()
|
||||
}
|
||||
|
||||
func (n *Node[T]) SetValue(value T) {
|
||||
unlock := n.Lock()
|
||||
n.value = value
|
||||
n.set = true
|
||||
unlock()
|
||||
}
|
||||
|
||||
func (n *Node[T]) UnsetValue() {
|
||||
unlock := n.Lock()
|
||||
var zero T
|
||||
n.value = zero
|
||||
n.set = false
|
||||
unlock()
|
||||
}
|
||||
|
||||
func (n *Node[T]) Collect() (values []T) {
|
||||
n._collect(&values)
|
||||
return values
|
||||
}
|
||||
|
||||
func (n *Node[T]) _collect(values *[]T) {
|
||||
unlock := n.RLock()
|
||||
if n.set {
|
||||
*values = append(*values, n.value)
|
||||
}
|
||||
|
||||
for _, v := range n.keyMap {
|
||||
v._collect(values)
|
||||
}
|
||||
|
||||
unlock()
|
||||
}
|
||||
|
||||
// _startsWith returns the node starting with the given string
|
||||
func (n *Node[T]) _find(s string) (node *Node[T], equal bool) {
|
||||
defer n.RLock()()
|
||||
|
||||
if s == "" {
|
||||
return n, true
|
||||
}
|
||||
|
||||
ch := rune(s[0])
|
||||
key := n.keyByChar[ch]
|
||||
if key == "" {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
offset := len(key)
|
||||
|
||||
// check our part of the key matches
|
||||
if offset > 1 {
|
||||
for i := 1; i < len(s) && i < offset; i++ {
|
||||
if s[i] != key[i] {
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
next := n.keyMap[key]
|
||||
|
||||
// our key matched, it's equal -- just return the node
|
||||
if offset == len(s) {
|
||||
return next, true
|
||||
}
|
||||
|
||||
// our key matched, it's longer -- just return the node as a non-equality match
|
||||
if offset >= len(s) {
|
||||
return next, false
|
||||
}
|
||||
|
||||
// our key matched, but it's shorter -- return what we find for the next portion
|
||||
return next._find(s[offset:])
|
||||
}
|
||||
|
||||
//nolint:funlen,gocognit
|
||||
func (n *Node[T]) _makeNodeP(unlock *func(), name string, nodeIfEmpty *Node[T]) *Node[T] {
|
||||
if name == "" {
|
||||
return n
|
||||
}
|
||||
|
||||
if unlock == nil {
|
||||
rUnlock := n.RLock()
|
||||
n = n._makeNodeP(&rUnlock, name, nodeIfEmpty)
|
||||
rUnlock()
|
||||
return n
|
||||
}
|
||||
|
||||
ch := rune(name[0])
|
||||
|
||||
if n.keyByChar == nil || n.keyMap == nil {
|
||||
n._exclusiveLock(unlock)
|
||||
|
||||
if n.keyByChar == nil {
|
||||
n.keyByChar = map[rune]string{}
|
||||
}
|
||||
if n.keyMap == nil {
|
||||
n.keyMap = NodeMap[T]{}
|
||||
}
|
||||
}
|
||||
|
||||
key, ok := n.keyByChar[ch]
|
||||
|
||||
// no entry for the given character
|
||||
if !ok {
|
||||
// check again with an exclusive lock before creating a new entry
|
||||
n._exclusiveLock(unlock)
|
||||
|
||||
key, ok = n.keyByChar[ch]
|
||||
if !ok {
|
||||
// no entry for the given character, create one; this is all we have to do
|
||||
newNode := nodeIfEmpty
|
||||
if newNode == nil {
|
||||
newNode = &Node[T]{}
|
||||
}
|
||||
n.keyMap[name] = newNode
|
||||
n.keyByChar[ch] = name
|
||||
|
||||
return newNode
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
case key == name:
|
||||
existingNode := n.keyMap[key]
|
||||
if existingNode == nil {
|
||||
// try this again, something changed between read checks and exclusive lock
|
||||
return n._makeNodeP(unlock, name, nodeIfEmpty)
|
||||
}
|
||||
|
||||
return existingNode
|
||||
case strings.HasPrefix(key, name):
|
||||
// existing key is longer than my key, we can just use the existing and make a new sub-entry for the longer key
|
||||
n._exclusiveLock(unlock)
|
||||
|
||||
existingNode := n.keyMap[key]
|
||||
if existingNode == nil {
|
||||
// try this again, something changed between read checks and exclusive lock
|
||||
return n._makeNodeP(unlock, name, nodeIfEmpty)
|
||||
}
|
||||
delete(n.keyMap, key)
|
||||
|
||||
newNode := nodeIfEmpty
|
||||
if newNode == nil {
|
||||
newNode = &Node[T]{}
|
||||
}
|
||||
n.keyMap[name] = newNode
|
||||
n.keyByChar[ch] = name
|
||||
|
||||
next := key[len(name):]
|
||||
_ = newNode._makeNodeP(nil, next, existingNode)
|
||||
|
||||
return newNode
|
||||
case strings.HasPrefix(name, key):
|
||||
// existing key is shorter than my key, we can just take the substring to remove the
|
||||
// existing string prefix and set the new node as a child of the existing node
|
||||
n._exclusiveLock(unlock)
|
||||
|
||||
existingNode := n.keyMap[key]
|
||||
if existingNode == nil {
|
||||
// try this again, something changed between read checks and exclusive lock
|
||||
return n._makeNodeP(unlock, name, nodeIfEmpty)
|
||||
}
|
||||
|
||||
next := name[len(key):]
|
||||
return existingNode._makeNodeP(nil, next, nodeIfEmpty)
|
||||
default:
|
||||
// neither the existing key nor my key contains a prefix of the other, so we find
|
||||
// the longest common prefix and split BOTH entries as children of a new entry with this common prefix
|
||||
n._exclusiveLock(unlock)
|
||||
|
||||
commonLength := 1 // the first character already matches
|
||||
minLength := len(key)
|
||||
if len(name) < minLength {
|
||||
minLength = len(name)
|
||||
}
|
||||
for ; commonLength < minLength; commonLength++ {
|
||||
if key[commonLength] != name[commonLength] {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
existingNode := n.keyMap[key]
|
||||
if existingNode == nil {
|
||||
// try this again, something changed between read checks and exclusive lock
|
||||
return n._makeNodeP(unlock, name, nodeIfEmpty)
|
||||
}
|
||||
delete(n.keyMap, key)
|
||||
|
||||
newNode := nodeIfEmpty
|
||||
if newNode == nil {
|
||||
newNode = &Node[T]{}
|
||||
}
|
||||
parentNode := &Node[T]{}
|
||||
parentNode._makeNodeP(nil, key[commonLength:], existingNode)
|
||||
parentNode._makeNodeP(nil, name[commonLength:], newNode)
|
||||
|
||||
common := key[:commonLength]
|
||||
n.keyMap[common] = parentNode
|
||||
n.keyByChar[ch] = common
|
||||
|
||||
return newNode
|
||||
}
|
||||
}
|
||||
|
||||
func (n *Node[T]) _exclusiveLock(unlocker *func()) {
|
||||
if !n.isExclusiveLock(*unlocker) {
|
||||
(*unlocker)()
|
||||
*unlocker = n.Lock()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Test_KeySplitIndex_Create(t *testing.T) {
|
||||
fi := KeySplitIndex[int]{}
|
||||
|
||||
fi.Set("one", 1)
|
||||
|
||||
requireEqualValues(t, &fi, m{
|
||||
"one": 1,
|
||||
})
|
||||
|
||||
fi.Set("onesie", 101)
|
||||
|
||||
requireEqualValues(t, &fi, m{
|
||||
"one": m{ // there is a value here
|
||||
"sie": 101,
|
||||
},
|
||||
})
|
||||
|
||||
fi.Set("apple", 99)
|
||||
|
||||
requireEqualValues(t, &fi, m{
|
||||
"one": m{
|
||||
"sie": 101,
|
||||
},
|
||||
"apple": 99,
|
||||
})
|
||||
|
||||
fi.Set("alice", 999)
|
||||
|
||||
requireEqualValues(t, &fi, m{
|
||||
"one": m{
|
||||
"sie": 101,
|
||||
},
|
||||
"a": m{
|
||||
"pple": 99,
|
||||
"lice": 999,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
type m = map[string]any
|
||||
|
||||
func requireEqualValues[T comparable](t *testing.T, value *KeySplitIndex[T], expected m) {
|
||||
_requireNodeValues(t, &value.Node, "", expected)
|
||||
}
|
||||
|
||||
func _requireNodeValues[T comparable](t *testing.T, value *Node[T], path string, expected any) {
|
||||
switch expected := expected.(type) {
|
||||
case T:
|
||||
if value.value != expected {
|
||||
t.Fatalf("values not equal: %v != %v", value.value, value)
|
||||
}
|
||||
case m:
|
||||
if len(value.keyMap) != len(expected) {
|
||||
t.Fatalf("number of children not equal: %v != %v", len(value.keyMap), len(expected))
|
||||
}
|
||||
for k, v := range expected {
|
||||
v2, ok := value.keyMap[k]
|
||||
if !ok {
|
||||
t.Fatalf("missing key: %s", k)
|
||||
} else {
|
||||
_requireNodeValues(t, v2, path, v)
|
||||
}
|
||||
}
|
||||
default:
|
||||
panic(fmt.Errorf("invalid type: %#v", expected))
|
||||
}
|
||||
}
|
||||
|
||||
func Test_KeySplitIndex_Get(t *testing.T) {
|
||||
fi := KeySplitIndex[int]{}
|
||||
|
||||
const (
|
||||
one = 1
|
||||
two = 2
|
||||
once = 11
|
||||
onesie = 11
|
||||
)
|
||||
|
||||
fi.Set("one", one)
|
||||
require.Equal(t, one, fi.Get("one"))
|
||||
|
||||
fi.Set("two", two)
|
||||
require.Equal(t, one, fi.Get("one"))
|
||||
require.Equal(t, two, fi.Get("two"))
|
||||
|
||||
fi.Set("once", once)
|
||||
require.Equal(t, one, fi.Get("one"))
|
||||
require.Equal(t, two, fi.Get("two"))
|
||||
require.Equal(t, once, fi.Get("once"))
|
||||
|
||||
fi.Set("onesie", onesie)
|
||||
require.Equal(t, one, fi.Get("one"))
|
||||
require.Equal(t, two, fi.Get("two"))
|
||||
require.Equal(t, once, fi.Get("once"))
|
||||
require.Equal(t, onesie, fi.Get("onesie"))
|
||||
|
||||
fi.Set("apple", 1000)
|
||||
require.Equal(t, one, fi.Get("one"))
|
||||
require.Equal(t, two, fi.Get("two"))
|
||||
require.Equal(t, once, fi.Get("once"))
|
||||
require.Equal(t, onesie, fi.Get("onesie"))
|
||||
|
||||
fi.Set("alice", 1001)
|
||||
require.Equal(t, one, fi.Get("one"))
|
||||
require.Equal(t, two, fi.Get("two"))
|
||||
require.Equal(t, once, fi.Get("once"))
|
||||
require.Equal(t, onesie, fi.Get("onesie"))
|
||||
|
||||
requireHasAll(t, []int{one, once, onesie}, fi.ByPrefix("o")...)
|
||||
requireHasAll(t, []int{one, once, onesie}, fi.ByPrefix("on")...)
|
||||
requireHasAll(t, []int{one, onesie}, fi.ByPrefix("one")...)
|
||||
requireHasAll(t, []int{onesie}, fi.ByPrefix("ones")...)
|
||||
requireHasAll(t, []int{onesie}, fi.ByPrefix("onesi")...)
|
||||
requireHasAll(t, []int{onesie}, fi.ByPrefix("onesie")...)
|
||||
requireHasAll(t, nil, fi.ByPrefix("onesies")...)
|
||||
requireHasAll(t, []int{two}, fi.ByPrefix("t")...)
|
||||
requireHasAll(t, []int{two}, fi.ByPrefix("tw")...)
|
||||
requireHasAll(t, []int{two}, fi.ByPrefix("two")...)
|
||||
requireHasAll(t, []int{}, fi.ByPrefix("twos")...)
|
||||
|
||||
require.Equal(t, 0, fi.Get("invalid")) // returns the zero value
|
||||
}
|
||||
|
||||
func Test_KeySplitIndex_serialization(t *testing.T) {
|
||||
index1 := KeySplitIndex[int]{}
|
||||
|
||||
const (
|
||||
one = 1
|
||||
two = 2
|
||||
once = 11
|
||||
onesie = 111
|
||||
)
|
||||
|
||||
index1.Set("one", one)
|
||||
index1.Set("two", two)
|
||||
index1.Set("once", once)
|
||||
index1.Set("onesie", onesie)
|
||||
|
||||
serialized, err := json.Marshal(&index1)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.JSONEq(t, `{"one":1,"two":2,"once":11,"onesie":111}`, string(serialized))
|
||||
|
||||
var index2 KeySplitIndex[int]
|
||||
err = json.Unmarshal(serialized, &index2)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, &index1, &index2)
|
||||
}
|
||||
|
||||
// requireHasAll asserts the values are exactly the required ones, in any order. Carried over with the
|
||||
// index's own tests so they run here unchanged.
|
||||
func requireHasAll[T comparable](t *testing.T, values []T, required ...T) {
|
||||
t.Helper()
|
||||
require.Len(t, values, len(required))
|
||||
nextFound:
|
||||
for _, ex := range required {
|
||||
for _, existing := range values {
|
||||
if existing == ex {
|
||||
continue nextFound
|
||||
}
|
||||
}
|
||||
require.Fail(t, fmt.Sprintf("value %v isn't in slice %v", ex, values))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// lockable is a read-write lock that hands back its own unlock function.
|
||||
//
|
||||
// Returning the unlocker rather than exposing Unlock is what makes the index's lock upgrades safe to
|
||||
// write: a caller holds an opaque `unlock`, and the node it is walking can tell whether that unlock
|
||||
// belongs to the write lock or the read lock and upgrade only when it has to. Carried over from the
|
||||
// prototype this index came from, where the pattern is used throughout.
|
||||
type lockable struct {
|
||||
lock sync.RWMutex
|
||||
}
|
||||
|
||||
func (l *lockable) Lock() (unlock func()) {
|
||||
l.lock.Lock()
|
||||
return l.lock.Unlock
|
||||
}
|
||||
|
||||
func (l *lockable) RLock() (unlock func()) {
|
||||
l.lock.RLock()
|
||||
return l.lock.RUnlock
|
||||
}
|
||||
|
||||
// isExclusiveLock reports whether the given unlocker is the write lock's rather than the read
|
||||
// lock's. Compared by function pointer because that is the only handle the caller was given.
|
||||
func (l *lockable) isExclusiveLock(fn func()) bool {
|
||||
return reflect.ValueOf(l.lock.Unlock).Pointer() == reflect.ValueOf(fn).Pointer()
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package index
|
||||
|
||||
// PrefixSuffix index combines two KeySplitIndexes to simplify indexing and searching by prefix and suffix
|
||||
type PrefixSuffix[T any] struct {
|
||||
forwardIndex KeySplitIndex[T]
|
||||
reverseIndex KeySplitIndex[T]
|
||||
}
|
||||
|
||||
func (f *PrefixSuffix[T]) Set(key string, value T) {
|
||||
f.forwardIndex.Set(key, value)
|
||||
f.reverseIndex.Set(reverse(key), value)
|
||||
}
|
||||
|
||||
func (f *PrefixSuffix[T]) Update(key string, fn NodeUpdateFunc[T]) {
|
||||
f.forwardIndex.Update(key, fn)
|
||||
f.reverseIndex.Update(reverse(key), fn)
|
||||
}
|
||||
|
||||
func (f *PrefixSuffix[T]) Get(key string) T {
|
||||
return f.forwardIndex.Get(key)
|
||||
}
|
||||
|
||||
func (f *PrefixSuffix[T]) ByPrefix(prefix string) []T {
|
||||
return f.forwardIndex.ByPrefix(prefix)
|
||||
}
|
||||
|
||||
func (f *PrefixSuffix[T]) BySuffix(suffix string) []T {
|
||||
return f.reverseIndex.ByPrefix(reverse(suffix))
|
||||
}
|
||||
|
||||
func reverse(s string) string {
|
||||
runes := []rune(s)
|
||||
for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
|
||||
runes[i], runes[j] = runes[j], runes[i]
|
||||
}
|
||||
return string(runes)
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Test_PrefixSuffix(t *testing.T) {
|
||||
i := PrefixSuffix[int]{}
|
||||
i.Set("one", 1)
|
||||
i.Set("once", 11)
|
||||
i.Set("onesie", 111)
|
||||
i.Set("two", 2)
|
||||
i.Set("done", 99)
|
||||
require.Equal(t, 1, i.Get("one"))
|
||||
require.Equal(t, 11, i.Get("once"))
|
||||
require.Equal(t, 111, i.Get("onesie"))
|
||||
require.Equal(t, 2, i.Get("two"))
|
||||
requireHasAll(t, i.ByPrefix("on"), 1, 11, 111)
|
||||
requireHasAll(t, i.BySuffix("e"), 1, 11, 111, 99)
|
||||
requireHasAll(t, i.BySuffix(""), 1, 2, 11, 111, 99)
|
||||
requireHasAll(t, i.BySuffix("one"), 1, 99)
|
||||
requireHasAll(t, i.BySuffix("sie"), 111)
|
||||
}
|
||||
|
||||
func Test_reverse(t *testing.T) {
|
||||
require.Equal(t, "case", reverse("esac"))
|
||||
}
|
||||
Reference in New Issue
Block a user