mirror of
https://github.com/anchore/syft.git
synced 2026-10-31 14:36:33 +01:00
fix: separate golang license caches from mod dir (#2852)
Signed-off-by: Keith Zantow <kzantow@gmail.com>
This commit is contained in:
Vendored
+51
@@ -0,0 +1,51 @@
|
||||
# Caching
|
||||
|
||||
All caches are created from a global `manager`. By defaut this is a `bypassedCache`, which performs no caching.
|
||||
One benefit of this is that tests don't need to worry about caching causing issues unless they explicitly need
|
||||
to test the cache and can opt-in using the `cache.TestCache(t)` helper.
|
||||
|
||||
Syft sets a `filesystemCache` when the [cache options](../../cmd/syft/internal/options/cache.go) are loaded.
|
||||
|
||||
When using the `filesystemCache` all items are stored on disk under a root directory, generally in the form of:
|
||||
```
|
||||
<rootDir>/<named-cache>/<data-version>/path/to/data
|
||||
```
|
||||
|
||||
# Using the cache
|
||||
|
||||
The easiest and preferred method to use the cache is a `cache.Resolver`, which automatically creates a `<data-version>`
|
||||
based on the _structure_ of the provided type.
|
||||
If the structure changes in any way it will end up with a new version key and all will re populate this new key,
|
||||
ignoring cached values from older, different versions.
|
||||
The resolver will store items using the `json` package to serialize/deserialize values, so to save space
|
||||
it is encouraged to use `omitempty`. For example:
|
||||
|
||||
```go
|
||||
type myCacheItem struct {
|
||||
Name string `json:"name",omitempty`
|
||||
}
|
||||
```
|
||||
|
||||
It is possible to use core types such as `pkg.Package` as long as they support the standard `json` serialization,
|
||||
but this is discouraged in order to decouple changes to them from affecting the information stored in the cache.
|
||||
|
||||
To get a cache for this type:
|
||||
```go
|
||||
resolver := cache.GetResolver[myCacheItem]("myCacheName", "v1")
|
||||
```
|
||||
|
||||
Using the `resolver` is a single call, which manages checking for items in the cache, expiry times,
|
||||
and if not found invoking the callback to populate the cache and return a value:
|
||||
```go
|
||||
data := resolver.Resolve("some/cache/key", func() (myCacheItem, error) {
|
||||
// do things to return a myCacheItem or error
|
||||
})
|
||||
```
|
||||
|
||||
If it is common that checking for an item will result in errors, and you do not want to re-run the resolve function
|
||||
when errors are encountered, instead of using `GetResolver`, you can use `GetResolverCachingErrors`, which is useful
|
||||
for things such as resolving artifacts over a network, where a number of them will not be resolved, and you do not want
|
||||
to continue to have the expense of running the network resolution. This should be used when it is acceptable a network
|
||||
outage and cached errors is an acceptable risk.
|
||||
|
||||
An example can be seen in the [golang cataloger](../../syft/pkg/cataloger/golang/licenses.go) fetching remote licenses.
|
||||
Vendored
+24
@@ -0,0 +1,24 @@
|
||||
package cache
|
||||
|
||||
import "io"
|
||||
|
||||
type bypassedCache struct{}
|
||||
|
||||
func (b *bypassedCache) Read(_ string) (ReaderAtCloser, error) {
|
||||
return nil, errNotFound
|
||||
}
|
||||
|
||||
func (b *bypassedCache) Write(_ string, contents io.Reader) error {
|
||||
if closer, ok := contents.(io.Closer); ok {
|
||||
_ = closer.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *bypassedCache) GetCache(_, _ string) Cache {
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *bypassedCache) RootDirs() []string {
|
||||
return nil
|
||||
}
|
||||
Vendored
+18
@@ -0,0 +1,18 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Test_bypassedCache(t *testing.T) {
|
||||
m := bypassedCache{}
|
||||
cache := m.GetCache("name", "version")
|
||||
err := cache.Write("test", strings.NewReader("value"))
|
||||
require.NoError(t, err)
|
||||
rdr, err := cache.Read("test")
|
||||
require.Nil(t, rdr)
|
||||
require.ErrorIs(t, err, errNotFound)
|
||||
}
|
||||
Vendored
+48
@@ -0,0 +1,48 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"io"
|
||||
)
|
||||
|
||||
// Manager is responsible for managing cache data and instantiating all caches
|
||||
type Manager interface {
|
||||
// GetCache returns a cache scoped to the given named, versioned data
|
||||
GetCache(name, version string) Cache
|
||||
|
||||
// RootDirs returns any root directories this cache manager uses
|
||||
RootDirs() []string
|
||||
}
|
||||
|
||||
// ReaderAtCloser is an amalgamation of: io.Reader, io.ReaderAt, and io.Closer
|
||||
type ReaderAtCloser interface {
|
||||
io.Reader
|
||||
io.ReaderAt
|
||||
io.Closer
|
||||
}
|
||||
|
||||
// Cache is what the application interacts with to get and set cached data
|
||||
type Cache interface {
|
||||
// Read returns a reader for the cache value, if found and not expired
|
||||
// or errors when unable to find / expired
|
||||
Read(key string) (ReaderAtCloser, error)
|
||||
|
||||
// Write writes the contents of the reader to the cache
|
||||
// and closes it, if the reader implements io.Closer
|
||||
Write(key string, contents io.Reader) error
|
||||
}
|
||||
|
||||
// GetManager returns the global cache manager, which is used to instantiate all caches
|
||||
func GetManager() Manager {
|
||||
return manager
|
||||
}
|
||||
|
||||
// SetManager sets the global cache manager, which is used to instantiate all caches
|
||||
func SetManager(m Manager) {
|
||||
if m == nil {
|
||||
manager = &bypassedCache{}
|
||||
} else {
|
||||
manager = m
|
||||
}
|
||||
}
|
||||
|
||||
var manager Manager = &bypassedCache{}
|
||||
Vendored
+21
@@ -0,0 +1,21 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Test_SetManager(t *testing.T) {
|
||||
original := GetManager()
|
||||
defer SetManager(original)
|
||||
|
||||
SetManager(NewInMemory(0))
|
||||
|
||||
require.NotNil(t, GetManager())
|
||||
require.IsType(t, &filesystemCache{}, GetManager())
|
||||
|
||||
SetManager(nil)
|
||||
require.NotNil(t, GetManager())
|
||||
require.IsType(t, &bypassedCache{}, GetManager())
|
||||
}
|
||||
Vendored
+40
@@ -0,0 +1,40 @@
|
||||
package cache
|
||||
|
||||
import "fmt"
|
||||
|
||||
// GetResolverCachingErrors returns a Resolver that caches errors and will return them
|
||||
// instead of continuing to call the provided resolve functions
|
||||
func GetResolverCachingErrors[T any](name, version string) Resolver[T] {
|
||||
return &errorResolver[T]{
|
||||
resolver: GetResolver[errResponse[T]](name, version),
|
||||
}
|
||||
}
|
||||
|
||||
type errResponse[T any] struct {
|
||||
Error string `json:"err,omitempty"`
|
||||
Value T `json:"val,omitempty"`
|
||||
}
|
||||
|
||||
type errorResolver[T any] struct {
|
||||
resolver Resolver[errResponse[T]]
|
||||
}
|
||||
|
||||
func (r *errorResolver[T]) Resolve(key string, resolver resolverFunc[T]) (T, error) {
|
||||
v, err := r.resolver.Resolve(key, func() (errResponse[T], error) {
|
||||
v, err := resolver()
|
||||
out := errResponse[T]{
|
||||
Value: v,
|
||||
}
|
||||
if err != nil {
|
||||
out.Error = err.Error()
|
||||
}
|
||||
return out, nil
|
||||
})
|
||||
if err != nil {
|
||||
return v.Value, err
|
||||
}
|
||||
if v.Error != "" {
|
||||
return v.Value, fmt.Errorf(v.Error)
|
||||
}
|
||||
return v.Value, nil
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Test_errorResolver(t *testing.T) {
|
||||
original := GetManager()
|
||||
defer SetManager(original)
|
||||
SetManager(NewInMemory(time.Hour))
|
||||
|
||||
resolver := GetResolverCachingErrors[string]("theCache", "theVersion")
|
||||
|
||||
resolveCount := 0
|
||||
resolveFunc := func() (string, error) {
|
||||
resolveCount++
|
||||
return "theValue", nil
|
||||
}
|
||||
|
||||
val, err := resolver.Resolve("theKey", resolveFunc)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, resolveCount)
|
||||
require.Equal(t, "theValue", val)
|
||||
|
||||
val, err = resolver.Resolve("theKey", resolveFunc)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, resolveCount)
|
||||
require.Equal(t, "theValue", val)
|
||||
|
||||
errorCount := 0
|
||||
errorFunc := func() (string, error) {
|
||||
errorCount++
|
||||
return "", fmt.Errorf("an error")
|
||||
}
|
||||
|
||||
_, err = resolver.Resolve("errorValue", errorFunc)
|
||||
require.ErrorContains(t, err, "an error")
|
||||
require.Equal(t, 1, errorCount)
|
||||
|
||||
_, err = resolver.Resolve("errorValue", errorFunc)
|
||||
require.ErrorContains(t, err, "an error")
|
||||
require.Equal(t, 1, errorCount)
|
||||
}
|
||||
Vendored
+120
@@ -0,0 +1,120 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
)
|
||||
|
||||
// NewFromDir creates a new cache manager which returns caches stored on disk, rooted at the given directory
|
||||
func NewFromDir(dir string, ttl time.Duration) (Manager, error) {
|
||||
dir = filepath.Clean(dir)
|
||||
fsys, err := subFs(afero.NewOsFs(), dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &filesystemCache{
|
||||
dir: dir,
|
||||
fs: fsys,
|
||||
ttl: ttl,
|
||||
}, nil
|
||||
}
|
||||
|
||||
const filePermissions = 0700
|
||||
const dirPermissions = os.ModeDir | filePermissions
|
||||
|
||||
type filesystemCache struct {
|
||||
dir string
|
||||
fs afero.Fs
|
||||
ttl time.Duration
|
||||
}
|
||||
|
||||
func (d *filesystemCache) GetCache(name, version string) Cache {
|
||||
fsys, err := subFs(d.fs, name, version)
|
||||
if err != nil {
|
||||
log.Warnf("error getting cache for: %s/%s: %v", name, version, err)
|
||||
return &bypassedCache{}
|
||||
}
|
||||
return &filesystemCache{
|
||||
dir: filepath.Join(d.dir, name, version),
|
||||
fs: fsys,
|
||||
ttl: d.ttl,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *filesystemCache) RootDirs() []string {
|
||||
if d.dir == "" {
|
||||
return nil
|
||||
}
|
||||
return []string{d.dir}
|
||||
}
|
||||
|
||||
func (d *filesystemCache) Read(key string) (ReaderAtCloser, error) {
|
||||
path := makeDiskKey(key)
|
||||
f, err := d.fs.Open(path)
|
||||
if err != nil {
|
||||
log.Tracef("no cache entry for %s %s: %v", d.dir, key, err)
|
||||
return nil, errNotFound
|
||||
} else if stat, err := f.Stat(); err != nil || stat == nil || time.Since(stat.ModTime()) > d.ttl {
|
||||
log.Tracef("cache entry is too old for %s %s", d.dir, key)
|
||||
return nil, errExpired
|
||||
}
|
||||
log.Tracef("using cache for %s %s", d.dir, key)
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (d *filesystemCache) Write(key string, contents io.Reader) error {
|
||||
path := makeDiskKey(key)
|
||||
return afero.WriteReader(d.fs, path, contents)
|
||||
}
|
||||
|
||||
// subFs returns a writable directory with the given name under the root cache directory returned from findRoot,
|
||||
// the directory will be created if it does not exist
|
||||
func subFs(fsys afero.Fs, subDirs ...string) (afero.Fs, error) {
|
||||
dir := filepath.Join(subDirs...)
|
||||
dir = filepath.Clean(dir)
|
||||
stat, err := fsys.Stat(dir)
|
||||
if errors.Is(err, afero.ErrFileNotFound) {
|
||||
err = fsys.MkdirAll(dir, dirPermissions)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create directory at '%s/%s': %v", dir, strings.Join(subDirs, "/"), err)
|
||||
}
|
||||
stat, err = fsys.Stat(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err != nil || stat == nil || !stat.IsDir() {
|
||||
return nil, fmt.Errorf("unable to verify directory '%s': %v", dir, err)
|
||||
}
|
||||
fsys = afero.NewBasePathFs(fsys, dir)
|
||||
return fsys, err
|
||||
}
|
||||
|
||||
var keyReplacer = regexp.MustCompile("[^-._/a-zA-Z0-9]")
|
||||
|
||||
// makeDiskKey makes a safe sub-path but not escape forward slashes, this allows for logical partitioning on disk
|
||||
func makeDiskKey(key string) string {
|
||||
// encode single dot directory
|
||||
if key == "." {
|
||||
return "%2E"
|
||||
}
|
||||
// replace any disallowed chars with encoded form
|
||||
key = keyReplacer.ReplaceAllStringFunc(key, url.QueryEscape)
|
||||
// allow . in names but not ..
|
||||
key = strings.ReplaceAll(key, "..", "%2E%2E")
|
||||
return key
|
||||
}
|
||||
|
||||
var errNotFound = fmt.Errorf("not found")
|
||||
var errExpired = fmt.Errorf("expired")
|
||||
Vendored
+94
@@ -0,0 +1,94 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
)
|
||||
|
||||
func Test_filesystemCache(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, err := NewFromDir(dir, 1*time.Minute)
|
||||
require.NoError(t, err)
|
||||
|
||||
cacheName := "test"
|
||||
cacheVersion := "v1"
|
||||
cache := man.GetCache(cacheName, cacheVersion)
|
||||
|
||||
cacheKey := "test-key"
|
||||
contentsValue := "some contents to cache"
|
||||
|
||||
err = cache.Write(cacheKey, strings.NewReader(contentsValue))
|
||||
require.NoError(t, err)
|
||||
|
||||
rdr, err := cache.Read(cacheKey)
|
||||
require.NoError(t, err)
|
||||
defer internal.CloseAndLogError(rdr, cacheKey)
|
||||
|
||||
contents, err := io.ReadAll(rdr)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, contentsValue, string(contents))
|
||||
|
||||
// check the contents were actually written to disk as expected
|
||||
contents, err = os.ReadFile(filepath.Join(dir, cacheName, cacheVersion, cacheKey))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, contentsValue, string(contents))
|
||||
|
||||
_, err = cache.Read("otherKey")
|
||||
require.ErrorIs(t, err, errNotFound)
|
||||
}
|
||||
|
||||
func Test_makeDiskKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
in string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
in: "",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
in: ".",
|
||||
expected: "%2E",
|
||||
},
|
||||
{
|
||||
in: "..",
|
||||
expected: "%2E%2E",
|
||||
},
|
||||
{
|
||||
in: "github.com",
|
||||
expected: "github.com",
|
||||
},
|
||||
{
|
||||
in: "../github.com",
|
||||
expected: "%2E%2E/github.com",
|
||||
},
|
||||
{
|
||||
in: "github.com/../..",
|
||||
expected: "github.com/%2E%2E/%2E%2E",
|
||||
},
|
||||
{
|
||||
in: "github.com/%2E../..",
|
||||
expected: "github.com/%252E%2E%2E/%2E%2E",
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.in, func(t *testing.T) {
|
||||
got := makeDiskKey(test.in)
|
||||
// validate appropriate escaping
|
||||
require.Equal(t, test.expected, got)
|
||||
// also validate that unescaped string matches original
|
||||
unescaped, err := url.QueryUnescape(got)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, test.in, unescaped)
|
||||
})
|
||||
}
|
||||
}
|
||||
Vendored
+71
@@ -0,0 +1,71 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"github.com/mitchellh/hashstructure/v2"
|
||||
)
|
||||
|
||||
// hashType returns a stable hash based on the structure of the type
|
||||
func hashType[T any]() string {
|
||||
// get the base type and hash an empty instance
|
||||
var t T
|
||||
empty := emptyValue(reflect.TypeOf(t)).Interface()
|
||||
hash, err := hashstructure.Hash(empty, hashstructure.FormatV2, &hashstructure.HashOptions{
|
||||
ZeroNil: false,
|
||||
IgnoreZeroValue: false,
|
||||
SlicesAsSets: false,
|
||||
UseStringer: false,
|
||||
})
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("unable to use type as cache key: %w", err))
|
||||
}
|
||||
return fmt.Sprintf("%x", hash)
|
||||
}
|
||||
|
||||
func emptyValue(t reflect.Type) reflect.Value {
|
||||
switch t.Kind() {
|
||||
case reflect.Pointer:
|
||||
e := t.Elem()
|
||||
v := emptyValue(e)
|
||||
if v.CanAddr() {
|
||||
return v.Addr()
|
||||
}
|
||||
ptrv := reflect.New(e)
|
||||
ptrv.Elem().Set(v)
|
||||
return ptrv
|
||||
case reflect.Slice:
|
||||
v := emptyValue(t.Elem())
|
||||
s := reflect.MakeSlice(t, 1, 1)
|
||||
s.Index(0).Set(v)
|
||||
return s
|
||||
case reflect.Struct:
|
||||
v := reflect.New(t).Elem()
|
||||
// get all empty field values, too
|
||||
for i := 0; i < v.NumField(); i++ {
|
||||
f := t.Field(i)
|
||||
if isIgnored(f) {
|
||||
continue
|
||||
}
|
||||
fv := v.Field(i)
|
||||
if fv.CanSet() {
|
||||
fv.Set(emptyValue(f.Type))
|
||||
}
|
||||
}
|
||||
return v
|
||||
default:
|
||||
return reflect.New(t).Elem()
|
||||
}
|
||||
}
|
||||
|
||||
func isIgnored(f reflect.StructField) bool {
|
||||
if !f.IsExported() {
|
||||
return true
|
||||
}
|
||||
tag := f.Tag.Get("hash")
|
||||
if tag == "-" || tag == "ignore" {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
Vendored
+123
@@ -0,0 +1,123 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/mitchellh/hashstructure/v2"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Test_hashType(t *testing.T) {
|
||||
type t1 struct {
|
||||
Name string
|
||||
}
|
||||
type t2 struct {
|
||||
Name string
|
||||
}
|
||||
type generic[T any] struct {
|
||||
Val T
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
hash func() string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "struct 1",
|
||||
hash: func() string { return hashType[t1]() },
|
||||
expected: "d106c3ffbf98a0b1",
|
||||
},
|
||||
{
|
||||
name: "slice of struct 1",
|
||||
hash: func() string { return hashType[[]t1]() },
|
||||
expected: "8122ace4ee1af0b4",
|
||||
},
|
||||
{
|
||||
name: "slice of struct 2",
|
||||
hash: func() string { return hashType[[]t2]() },
|
||||
expected: "8cc04b5808be5bf9",
|
||||
},
|
||||
{
|
||||
name: "ptr 1",
|
||||
hash: func() string { return hashType[*t1]() },
|
||||
expected: "d106c3ffbf98a0b1", // same hash as t1, which is ok since the structs are the same
|
||||
},
|
||||
{
|
||||
name: "slice of ptr 1",
|
||||
hash: func() string { return hashType[[]*t1]() },
|
||||
expected: "8122ace4ee1af0b4", // same hash as []t1, again underlying serialization is the same
|
||||
},
|
||||
{
|
||||
name: "slice of ptr 2",
|
||||
hash: func() string { return hashType[[]*t2]() },
|
||||
expected: "8cc04b5808be5bf9", // same hash as []t2, underlying serialization is the same
|
||||
},
|
||||
{
|
||||
name: "slice of ptr of slice of ptr",
|
||||
hash: func() string { return hashType[[]*[]*t1]() },
|
||||
expected: "500d9f5b3a5977ce",
|
||||
},
|
||||
{
|
||||
name: "generic 1",
|
||||
hash: func() string { return hashType[generic[t1]]() },
|
||||
expected: "b5fbb30e24400e81",
|
||||
},
|
||||
{
|
||||
name: "generic 2",
|
||||
hash: func() string { return hashType[generic[t2]]() },
|
||||
expected: "becdb767c6b22bfa",
|
||||
},
|
||||
{
|
||||
name: "generic with ptr 1",
|
||||
hash: func() string { return hashType[generic[*t1]]() },
|
||||
expected: "30c8855bf290fd83",
|
||||
},
|
||||
{
|
||||
name: "generic with ptr 2",
|
||||
hash: func() string { return hashType[generic[*t2]]() },
|
||||
expected: "b66366b6ce9e6361",
|
||||
},
|
||||
{
|
||||
name: "generic with slice 1",
|
||||
hash: func() string { return hashType[generic[[]t1]]() },
|
||||
expected: "d2ed158942fa6c29",
|
||||
},
|
||||
{
|
||||
name: "generic with slice 2",
|
||||
hash: func() string { return hashType[generic[[]t2]]() },
|
||||
expected: "7a7bec575871c179",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
require.Equal(t, test.expected, test.hash())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_hashIgnores(t *testing.T) {
|
||||
hash := func(v any) string {
|
||||
v, err := hashstructure.Hash(v, hashstructure.FormatV2, &hashstructure.HashOptions{})
|
||||
require.NoError(t, err)
|
||||
return fmt.Sprintf("%x", v)
|
||||
}
|
||||
type t1 struct {
|
||||
Name string
|
||||
notExported string
|
||||
}
|
||||
require.Equal(t, hash(t1{notExported: "a value"}), hashType[t1]())
|
||||
|
||||
type t2 struct {
|
||||
Name string
|
||||
Exported string `hash:"ignore"`
|
||||
}
|
||||
require.Equal(t, hash(t2{Exported: "another value"}), hashType[t2]())
|
||||
|
||||
type t3 struct {
|
||||
Name string
|
||||
Exported string `hash:"-"`
|
||||
}
|
||||
require.Equal(t, hash(t3{Exported: "still valued"}), hashType[t3]())
|
||||
}
|
||||
Vendored
+16
@@ -0,0 +1,16 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
// NewInMemory returns an in-memory only cache manager
|
||||
func NewInMemory(ttl time.Duration) Manager {
|
||||
return &filesystemCache{
|
||||
dir: "",
|
||||
fs: afero.NewMemMapFs(),
|
||||
ttl: ttl,
|
||||
}
|
||||
}
|
||||
Vendored
+37
@@ -0,0 +1,37 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
)
|
||||
|
||||
func Test_NewInMemory(t *testing.T) {
|
||||
man := NewInMemory(time.Hour)
|
||||
|
||||
cacheName := "test"
|
||||
cacheVersion := "v1"
|
||||
cache := man.GetCache(cacheName, cacheVersion)
|
||||
|
||||
cacheKey := "test-key"
|
||||
contentsValue := "some contents to cache"
|
||||
|
||||
err := cache.Write(cacheKey, strings.NewReader(contentsValue))
|
||||
require.NoError(t, err)
|
||||
|
||||
rdr, err := cache.Read(cacheKey)
|
||||
require.NoError(t, err)
|
||||
defer internal.CloseAndLogError(rdr, cacheKey)
|
||||
|
||||
contents, err := io.ReadAll(rdr)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, contentsValue, string(contents))
|
||||
|
||||
_, err = cache.Read("otherKey")
|
||||
require.ErrorIs(t, err, errNotFound)
|
||||
}
|
||||
Vendored
+83
@@ -0,0 +1,83 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
)
|
||||
|
||||
// Resolver interface provides a single Resolve method, which will return from cache
|
||||
// or call the provided resolve function to get the value if not available in cache
|
||||
type Resolver[T any] interface {
|
||||
// Resolve attempts to resolve the given key from cache and convert it to the type of the cache,
|
||||
// or calls the resolver function if unable to resolve a cached value
|
||||
Resolve(key string, resolver resolverFunc[T]) (T, error)
|
||||
}
|
||||
|
||||
// GetResolver returns a cache resolver for persistent cached data across Syft runs, stored in a unique
|
||||
// location based on the provided name and versioned by the type
|
||||
func GetResolver[T any](name, version string) Resolver[T] {
|
||||
typeHash := hashType[T]()
|
||||
versionKey := path.Join(version, typeHash)
|
||||
return &cacheResolver[T]{
|
||||
name: fmt.Sprintf("%s/%s", name, versionKey),
|
||||
cache: manager.GetCache(name, versionKey),
|
||||
}
|
||||
}
|
||||
|
||||
const resolverKeySuffix = ".json"
|
||||
|
||||
type resolverFunc[T any] func() (T, error)
|
||||
|
||||
type cacheResolver[T any] struct {
|
||||
name string
|
||||
cache Cache
|
||||
}
|
||||
|
||||
var _ interface {
|
||||
Resolver[int]
|
||||
} = (*cacheResolver[int])(nil)
|
||||
|
||||
func (r *cacheResolver[T]) Resolve(key string, resolver resolverFunc[T]) (T, error) {
|
||||
key += resolverKeySuffix
|
||||
|
||||
rdr, err := r.cache.Read(key)
|
||||
if rdr == nil || err != nil {
|
||||
return r.resolveAndCache(key, resolver)
|
||||
}
|
||||
defer internal.CloseAndLogError(rdr, key)
|
||||
|
||||
dec := json.NewDecoder(rdr)
|
||||
if dec == nil {
|
||||
log.Tracef("error getting cache json decoder for %s %v: %v", r.name, key, err)
|
||||
return r.resolveAndCache(key, resolver)
|
||||
}
|
||||
var t T
|
||||
err = dec.Decode(&t)
|
||||
if err != nil {
|
||||
log.Tracef("error decoding cached entry for %s %v: %v", r.name, key, err)
|
||||
return r.resolveAndCache(key, resolver)
|
||||
}
|
||||
// no error, able to resolve from cache
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (r *cacheResolver[T]) resolveAndCache(key string, resolver func() (T, error)) (T, error) {
|
||||
t, err := resolver()
|
||||
if err != nil {
|
||||
return t, err
|
||||
}
|
||||
var data bytes.Buffer
|
||||
enc := json.NewEncoder(&data)
|
||||
enc.SetEscapeHTML(false)
|
||||
err = enc.Encode(t)
|
||||
if err != nil {
|
||||
return t, err
|
||||
}
|
||||
err = r.cache.Write(key, &data)
|
||||
return t, err
|
||||
}
|
||||
Vendored
+92
@@ -0,0 +1,92 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Test_Resolver(t *testing.T) {
|
||||
original := GetManager()
|
||||
defer SetManager(original)
|
||||
SetManager(NewInMemory(time.Hour))
|
||||
|
||||
type sub struct {
|
||||
Name string
|
||||
Value bool
|
||||
}
|
||||
|
||||
type thing struct {
|
||||
Value string
|
||||
Values []int
|
||||
Subs []*sub
|
||||
}
|
||||
|
||||
versionHash := hashType[thing]()
|
||||
cache := GetManager().GetCache("test", "v7/"+versionHash)
|
||||
|
||||
resolver := GetResolver[thing]("test", "v7")
|
||||
require.NotNil(t, resolver)
|
||||
|
||||
require.IsType(t, &cacheResolver[thing]{}, resolver)
|
||||
cr := resolver.(*cacheResolver[thing])
|
||||
|
||||
require.IsType(t, cache, cr.cache)
|
||||
|
||||
resolveErrCount := 0
|
||||
resolveThingErr := func() (thing, error) {
|
||||
resolveErrCount++
|
||||
return thing{}, fmt.Errorf("an error")
|
||||
}
|
||||
|
||||
_, err := resolver.Resolve("err", resolveThingErr)
|
||||
require.ErrorContains(t, err, "an error")
|
||||
require.Equal(t, 1, resolveErrCount)
|
||||
|
||||
_, err = resolver.Resolve("err", resolveThingErr)
|
||||
require.ErrorContains(t, err, "an error")
|
||||
require.Equal(t, 2, resolveErrCount)
|
||||
|
||||
aThing := thing{
|
||||
Value: "a value",
|
||||
Values: []int{7, 8, 9},
|
||||
Subs: []*sub{
|
||||
{
|
||||
Name: "sub1",
|
||||
Value: true,
|
||||
},
|
||||
{
|
||||
Name: "sub2",
|
||||
Value: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
resolveThingCount := 0
|
||||
resolveThing := func() (thing, error) {
|
||||
resolveThingCount++
|
||||
return aThing, nil
|
||||
}
|
||||
|
||||
val, err := resolver.Resolve("thing", resolveThing)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, resolveThingCount)
|
||||
require.Equal(t, aThing, val)
|
||||
|
||||
val, err = resolver.Resolve("thing", resolveThing)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, resolveThingCount)
|
||||
require.Equal(t, aThing, val)
|
||||
|
||||
rdr, err := cache.Read("thing" + resolverKeySuffix)
|
||||
require.NoError(t, err)
|
||||
decoder := json.NewDecoder(rdr)
|
||||
|
||||
var val2 thing
|
||||
err = decoder.Decode(&val2)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, aThing, val2)
|
||||
}
|
||||
Reference in New Issue
Block a user