fix: separate golang license caches from mod dir (#2852)

Signed-off-by: Keith Zantow <kzantow@gmail.com>
This commit is contained in:
Keith Zantow
2024-06-12 19:12:35 -04:00
committed by GitHub
parent dd723bb3c5
commit ca0cc52d47
31 changed files with 1496 additions and 149 deletions
+51
View File
@@ -0,0 +1,51 @@
# Caching
All caches are created from a global `manager`. By defaut this is a `bypassedCache`, which performs no caching.
One benefit of this is that tests don't need to worry about caching causing issues unless they explicitly need
to test the cache and can opt-in using the `cache.TestCache(t)` helper.
Syft sets a `filesystemCache` when the [cache options](../../cmd/syft/internal/options/cache.go) are loaded.
When using the `filesystemCache` all items are stored on disk under a root directory, generally in the form of:
```
<rootDir>/<named-cache>/<data-version>/path/to/data
```
# Using the cache
The easiest and preferred method to use the cache is a `cache.Resolver`, which automatically creates a `<data-version>`
based on the _structure_ of the provided type.
If the structure changes in any way it will end up with a new version key and all will re populate this new key,
ignoring cached values from older, different versions.
The resolver will store items using the `json` package to serialize/deserialize values, so to save space
it is encouraged to use `omitempty`. For example:
```go
type myCacheItem struct {
Name string `json:"name",omitempty`
}
```
It is possible to use core types such as `pkg.Package` as long as they support the standard `json` serialization,
but this is discouraged in order to decouple changes to them from affecting the information stored in the cache.
To get a cache for this type:
```go
resolver := cache.GetResolver[myCacheItem]("myCacheName", "v1")
```
Using the `resolver` is a single call, which manages checking for items in the cache, expiry times,
and if not found invoking the callback to populate the cache and return a value:
```go
data := resolver.Resolve("some/cache/key", func() (myCacheItem, error) {
// do things to return a myCacheItem or error
})
```
If it is common that checking for an item will result in errors, and you do not want to re-run the resolve function
when errors are encountered, instead of using `GetResolver`, you can use `GetResolverCachingErrors`, which is useful
for things such as resolving artifacts over a network, where a number of them will not be resolved, and you do not want
to continue to have the expense of running the network resolution. This should be used when it is acceptable a network
outage and cached errors is an acceptable risk.
An example can be seen in the [golang cataloger](../../syft/pkg/cataloger/golang/licenses.go) fetching remote licenses.
+24
View File
@@ -0,0 +1,24 @@
package cache
import "io"
type bypassedCache struct{}
func (b *bypassedCache) Read(_ string) (ReaderAtCloser, error) {
return nil, errNotFound
}
func (b *bypassedCache) Write(_ string, contents io.Reader) error {
if closer, ok := contents.(io.Closer); ok {
_ = closer.Close()
}
return nil
}
func (b *bypassedCache) GetCache(_, _ string) Cache {
return b
}
func (b *bypassedCache) RootDirs() []string {
return nil
}
+18
View File
@@ -0,0 +1,18 @@
package cache
import (
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func Test_bypassedCache(t *testing.T) {
m := bypassedCache{}
cache := m.GetCache("name", "version")
err := cache.Write("test", strings.NewReader("value"))
require.NoError(t, err)
rdr, err := cache.Read("test")
require.Nil(t, rdr)
require.ErrorIs(t, err, errNotFound)
}
+48
View File
@@ -0,0 +1,48 @@
package cache
import (
"io"
)
// Manager is responsible for managing cache data and instantiating all caches
type Manager interface {
// GetCache returns a cache scoped to the given named, versioned data
GetCache(name, version string) Cache
// RootDirs returns any root directories this cache manager uses
RootDirs() []string
}
// ReaderAtCloser is an amalgamation of: io.Reader, io.ReaderAt, and io.Closer
type ReaderAtCloser interface {
io.Reader
io.ReaderAt
io.Closer
}
// Cache is what the application interacts with to get and set cached data
type Cache interface {
// Read returns a reader for the cache value, if found and not expired
// or errors when unable to find / expired
Read(key string) (ReaderAtCloser, error)
// Write writes the contents of the reader to the cache
// and closes it, if the reader implements io.Closer
Write(key string, contents io.Reader) error
}
// GetManager returns the global cache manager, which is used to instantiate all caches
func GetManager() Manager {
return manager
}
// SetManager sets the global cache manager, which is used to instantiate all caches
func SetManager(m Manager) {
if m == nil {
manager = &bypassedCache{}
} else {
manager = m
}
}
var manager Manager = &bypassedCache{}
+21
View File
@@ -0,0 +1,21 @@
package cache
import (
"testing"
"github.com/stretchr/testify/require"
)
func Test_SetManager(t *testing.T) {
original := GetManager()
defer SetManager(original)
SetManager(NewInMemory(0))
require.NotNil(t, GetManager())
require.IsType(t, &filesystemCache{}, GetManager())
SetManager(nil)
require.NotNil(t, GetManager())
require.IsType(t, &bypassedCache{}, GetManager())
}
+40
View File
@@ -0,0 +1,40 @@
package cache
import "fmt"
// GetResolverCachingErrors returns a Resolver that caches errors and will return them
// instead of continuing to call the provided resolve functions
func GetResolverCachingErrors[T any](name, version string) Resolver[T] {
return &errorResolver[T]{
resolver: GetResolver[errResponse[T]](name, version),
}
}
type errResponse[T any] struct {
Error string `json:"err,omitempty"`
Value T `json:"val,omitempty"`
}
type errorResolver[T any] struct {
resolver Resolver[errResponse[T]]
}
func (r *errorResolver[T]) Resolve(key string, resolver resolverFunc[T]) (T, error) {
v, err := r.resolver.Resolve(key, func() (errResponse[T], error) {
v, err := resolver()
out := errResponse[T]{
Value: v,
}
if err != nil {
out.Error = err.Error()
}
return out, nil
})
if err != nil {
return v.Value, err
}
if v.Error != "" {
return v.Value, fmt.Errorf(v.Error)
}
return v.Value, nil
}
+47
View File
@@ -0,0 +1,47 @@
package cache
import (
"fmt"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func Test_errorResolver(t *testing.T) {
original := GetManager()
defer SetManager(original)
SetManager(NewInMemory(time.Hour))
resolver := GetResolverCachingErrors[string]("theCache", "theVersion")
resolveCount := 0
resolveFunc := func() (string, error) {
resolveCount++
return "theValue", nil
}
val, err := resolver.Resolve("theKey", resolveFunc)
require.NoError(t, err)
require.Equal(t, 1, resolveCount)
require.Equal(t, "theValue", val)
val, err = resolver.Resolve("theKey", resolveFunc)
require.NoError(t, err)
require.Equal(t, 1, resolveCount)
require.Equal(t, "theValue", val)
errorCount := 0
errorFunc := func() (string, error) {
errorCount++
return "", fmt.Errorf("an error")
}
_, err = resolver.Resolve("errorValue", errorFunc)
require.ErrorContains(t, err, "an error")
require.Equal(t, 1, errorCount)
_, err = resolver.Resolve("errorValue", errorFunc)
require.ErrorContains(t, err, "an error")
require.Equal(t, 1, errorCount)
}
+120
View File
@@ -0,0 +1,120 @@
package cache
import (
"errors"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/spf13/afero"
"github.com/anchore/syft/internal/log"
)
// NewFromDir creates a new cache manager which returns caches stored on disk, rooted at the given directory
func NewFromDir(dir string, ttl time.Duration) (Manager, error) {
dir = filepath.Clean(dir)
fsys, err := subFs(afero.NewOsFs(), dir)
if err != nil {
return nil, err
}
return &filesystemCache{
dir: dir,
fs: fsys,
ttl: ttl,
}, nil
}
const filePermissions = 0700
const dirPermissions = os.ModeDir | filePermissions
type filesystemCache struct {
dir string
fs afero.Fs
ttl time.Duration
}
func (d *filesystemCache) GetCache(name, version string) Cache {
fsys, err := subFs(d.fs, name, version)
if err != nil {
log.Warnf("error getting cache for: %s/%s: %v", name, version, err)
return &bypassedCache{}
}
return &filesystemCache{
dir: filepath.Join(d.dir, name, version),
fs: fsys,
ttl: d.ttl,
}
}
func (d *filesystemCache) RootDirs() []string {
if d.dir == "" {
return nil
}
return []string{d.dir}
}
func (d *filesystemCache) Read(key string) (ReaderAtCloser, error) {
path := makeDiskKey(key)
f, err := d.fs.Open(path)
if err != nil {
log.Tracef("no cache entry for %s %s: %v", d.dir, key, err)
return nil, errNotFound
} else if stat, err := f.Stat(); err != nil || stat == nil || time.Since(stat.ModTime()) > d.ttl {
log.Tracef("cache entry is too old for %s %s", d.dir, key)
return nil, errExpired
}
log.Tracef("using cache for %s %s", d.dir, key)
return f, nil
}
func (d *filesystemCache) Write(key string, contents io.Reader) error {
path := makeDiskKey(key)
return afero.WriteReader(d.fs, path, contents)
}
// subFs returns a writable directory with the given name under the root cache directory returned from findRoot,
// the directory will be created if it does not exist
func subFs(fsys afero.Fs, subDirs ...string) (afero.Fs, error) {
dir := filepath.Join(subDirs...)
dir = filepath.Clean(dir)
stat, err := fsys.Stat(dir)
if errors.Is(err, afero.ErrFileNotFound) {
err = fsys.MkdirAll(dir, dirPermissions)
if err != nil {
return nil, fmt.Errorf("unable to create directory at '%s/%s': %v", dir, strings.Join(subDirs, "/"), err)
}
stat, err = fsys.Stat(dir)
if err != nil {
return nil, err
}
}
if err != nil || stat == nil || !stat.IsDir() {
return nil, fmt.Errorf("unable to verify directory '%s': %v", dir, err)
}
fsys = afero.NewBasePathFs(fsys, dir)
return fsys, err
}
var keyReplacer = regexp.MustCompile("[^-._/a-zA-Z0-9]")
// makeDiskKey makes a safe sub-path but not escape forward slashes, this allows for logical partitioning on disk
func makeDiskKey(key string) string {
// encode single dot directory
if key == "." {
return "%2E"
}
// replace any disallowed chars with encoded form
key = keyReplacer.ReplaceAllStringFunc(key, url.QueryEscape)
// allow . in names but not ..
key = strings.ReplaceAll(key, "..", "%2E%2E")
return key
}
var errNotFound = fmt.Errorf("not found")
var errExpired = fmt.Errorf("expired")
+94
View File
@@ -0,0 +1,94 @@
package cache
import (
"io"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/internal"
)
func Test_filesystemCache(t *testing.T) {
dir := t.TempDir()
man, err := NewFromDir(dir, 1*time.Minute)
require.NoError(t, err)
cacheName := "test"
cacheVersion := "v1"
cache := man.GetCache(cacheName, cacheVersion)
cacheKey := "test-key"
contentsValue := "some contents to cache"
err = cache.Write(cacheKey, strings.NewReader(contentsValue))
require.NoError(t, err)
rdr, err := cache.Read(cacheKey)
require.NoError(t, err)
defer internal.CloseAndLogError(rdr, cacheKey)
contents, err := io.ReadAll(rdr)
require.NoError(t, err)
require.Equal(t, contentsValue, string(contents))
// check the contents were actually written to disk as expected
contents, err = os.ReadFile(filepath.Join(dir, cacheName, cacheVersion, cacheKey))
require.NoError(t, err)
require.Equal(t, contentsValue, string(contents))
_, err = cache.Read("otherKey")
require.ErrorIs(t, err, errNotFound)
}
func Test_makeDiskKey(t *testing.T) {
tests := []struct {
in string
expected string
}{
{
in: "",
expected: "",
},
{
in: ".",
expected: "%2E",
},
{
in: "..",
expected: "%2E%2E",
},
{
in: "github.com",
expected: "github.com",
},
{
in: "../github.com",
expected: "%2E%2E/github.com",
},
{
in: "github.com/../..",
expected: "github.com/%2E%2E/%2E%2E",
},
{
in: "github.com/%2E../..",
expected: "github.com/%252E%2E%2E/%2E%2E",
},
}
for _, test := range tests {
t.Run(test.in, func(t *testing.T) {
got := makeDiskKey(test.in)
// validate appropriate escaping
require.Equal(t, test.expected, got)
// also validate that unescaped string matches original
unescaped, err := url.QueryUnescape(got)
require.NoError(t, err)
require.Equal(t, test.in, unescaped)
})
}
}
+71
View File
@@ -0,0 +1,71 @@
package cache
import (
"fmt"
"reflect"
"github.com/mitchellh/hashstructure/v2"
)
// hashType returns a stable hash based on the structure of the type
func hashType[T any]() string {
// get the base type and hash an empty instance
var t T
empty := emptyValue(reflect.TypeOf(t)).Interface()
hash, err := hashstructure.Hash(empty, hashstructure.FormatV2, &hashstructure.HashOptions{
ZeroNil: false,
IgnoreZeroValue: false,
SlicesAsSets: false,
UseStringer: false,
})
if err != nil {
panic(fmt.Errorf("unable to use type as cache key: %w", err))
}
return fmt.Sprintf("%x", hash)
}
func emptyValue(t reflect.Type) reflect.Value {
switch t.Kind() {
case reflect.Pointer:
e := t.Elem()
v := emptyValue(e)
if v.CanAddr() {
return v.Addr()
}
ptrv := reflect.New(e)
ptrv.Elem().Set(v)
return ptrv
case reflect.Slice:
v := emptyValue(t.Elem())
s := reflect.MakeSlice(t, 1, 1)
s.Index(0).Set(v)
return s
case reflect.Struct:
v := reflect.New(t).Elem()
// get all empty field values, too
for i := 0; i < v.NumField(); i++ {
f := t.Field(i)
if isIgnored(f) {
continue
}
fv := v.Field(i)
if fv.CanSet() {
fv.Set(emptyValue(f.Type))
}
}
return v
default:
return reflect.New(t).Elem()
}
}
func isIgnored(f reflect.StructField) bool {
if !f.IsExported() {
return true
}
tag := f.Tag.Get("hash")
if tag == "-" || tag == "ignore" {
return true
}
return false
}
+123
View File
@@ -0,0 +1,123 @@
package cache
import (
"fmt"
"testing"
"github.com/mitchellh/hashstructure/v2"
"github.com/stretchr/testify/require"
)
func Test_hashType(t *testing.T) {
type t1 struct {
Name string
}
type t2 struct {
Name string
}
type generic[T any] struct {
Val T
}
tests := []struct {
name string
hash func() string
expected string
}{
{
name: "struct 1",
hash: func() string { return hashType[t1]() },
expected: "d106c3ffbf98a0b1",
},
{
name: "slice of struct 1",
hash: func() string { return hashType[[]t1]() },
expected: "8122ace4ee1af0b4",
},
{
name: "slice of struct 2",
hash: func() string { return hashType[[]t2]() },
expected: "8cc04b5808be5bf9",
},
{
name: "ptr 1",
hash: func() string { return hashType[*t1]() },
expected: "d106c3ffbf98a0b1", // same hash as t1, which is ok since the structs are the same
},
{
name: "slice of ptr 1",
hash: func() string { return hashType[[]*t1]() },
expected: "8122ace4ee1af0b4", // same hash as []t1, again underlying serialization is the same
},
{
name: "slice of ptr 2",
hash: func() string { return hashType[[]*t2]() },
expected: "8cc04b5808be5bf9", // same hash as []t2, underlying serialization is the same
},
{
name: "slice of ptr of slice of ptr",
hash: func() string { return hashType[[]*[]*t1]() },
expected: "500d9f5b3a5977ce",
},
{
name: "generic 1",
hash: func() string { return hashType[generic[t1]]() },
expected: "b5fbb30e24400e81",
},
{
name: "generic 2",
hash: func() string { return hashType[generic[t2]]() },
expected: "becdb767c6b22bfa",
},
{
name: "generic with ptr 1",
hash: func() string { return hashType[generic[*t1]]() },
expected: "30c8855bf290fd83",
},
{
name: "generic with ptr 2",
hash: func() string { return hashType[generic[*t2]]() },
expected: "b66366b6ce9e6361",
},
{
name: "generic with slice 1",
hash: func() string { return hashType[generic[[]t1]]() },
expected: "d2ed158942fa6c29",
},
{
name: "generic with slice 2",
hash: func() string { return hashType[generic[[]t2]]() },
expected: "7a7bec575871c179",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
require.Equal(t, test.expected, test.hash())
})
}
}
func Test_hashIgnores(t *testing.T) {
hash := func(v any) string {
v, err := hashstructure.Hash(v, hashstructure.FormatV2, &hashstructure.HashOptions{})
require.NoError(t, err)
return fmt.Sprintf("%x", v)
}
type t1 struct {
Name string
notExported string
}
require.Equal(t, hash(t1{notExported: "a value"}), hashType[t1]())
type t2 struct {
Name string
Exported string `hash:"ignore"`
}
require.Equal(t, hash(t2{Exported: "another value"}), hashType[t2]())
type t3 struct {
Name string
Exported string `hash:"-"`
}
require.Equal(t, hash(t3{Exported: "still valued"}), hashType[t3]())
}
+16
View File
@@ -0,0 +1,16 @@
package cache
import (
"time"
"github.com/spf13/afero"
)
// NewInMemory returns an in-memory only cache manager
func NewInMemory(ttl time.Duration) Manager {
return &filesystemCache{
dir: "",
fs: afero.NewMemMapFs(),
ttl: ttl,
}
}
+37
View File
@@ -0,0 +1,37 @@
package cache
import (
"io"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/internal"
)
func Test_NewInMemory(t *testing.T) {
man := NewInMemory(time.Hour)
cacheName := "test"
cacheVersion := "v1"
cache := man.GetCache(cacheName, cacheVersion)
cacheKey := "test-key"
contentsValue := "some contents to cache"
err := cache.Write(cacheKey, strings.NewReader(contentsValue))
require.NoError(t, err)
rdr, err := cache.Read(cacheKey)
require.NoError(t, err)
defer internal.CloseAndLogError(rdr, cacheKey)
contents, err := io.ReadAll(rdr)
require.NoError(t, err)
require.Equal(t, contentsValue, string(contents))
_, err = cache.Read("otherKey")
require.ErrorIs(t, err, errNotFound)
}
+83
View File
@@ -0,0 +1,83 @@
package cache
import (
"bytes"
"encoding/json"
"fmt"
"path"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
)
// Resolver interface provides a single Resolve method, which will return from cache
// or call the provided resolve function to get the value if not available in cache
type Resolver[T any] interface {
// Resolve attempts to resolve the given key from cache and convert it to the type of the cache,
// or calls the resolver function if unable to resolve a cached value
Resolve(key string, resolver resolverFunc[T]) (T, error)
}
// GetResolver returns a cache resolver for persistent cached data across Syft runs, stored in a unique
// location based on the provided name and versioned by the type
func GetResolver[T any](name, version string) Resolver[T] {
typeHash := hashType[T]()
versionKey := path.Join(version, typeHash)
return &cacheResolver[T]{
name: fmt.Sprintf("%s/%s", name, versionKey),
cache: manager.GetCache(name, versionKey),
}
}
const resolverKeySuffix = ".json"
type resolverFunc[T any] func() (T, error)
type cacheResolver[T any] struct {
name string
cache Cache
}
var _ interface {
Resolver[int]
} = (*cacheResolver[int])(nil)
func (r *cacheResolver[T]) Resolve(key string, resolver resolverFunc[T]) (T, error) {
key += resolverKeySuffix
rdr, err := r.cache.Read(key)
if rdr == nil || err != nil {
return r.resolveAndCache(key, resolver)
}
defer internal.CloseAndLogError(rdr, key)
dec := json.NewDecoder(rdr)
if dec == nil {
log.Tracef("error getting cache json decoder for %s %v: %v", r.name, key, err)
return r.resolveAndCache(key, resolver)
}
var t T
err = dec.Decode(&t)
if err != nil {
log.Tracef("error decoding cached entry for %s %v: %v", r.name, key, err)
return r.resolveAndCache(key, resolver)
}
// no error, able to resolve from cache
return t, nil
}
func (r *cacheResolver[T]) resolveAndCache(key string, resolver func() (T, error)) (T, error) {
t, err := resolver()
if err != nil {
return t, err
}
var data bytes.Buffer
enc := json.NewEncoder(&data)
enc.SetEscapeHTML(false)
err = enc.Encode(t)
if err != nil {
return t, err
}
err = r.cache.Write(key, &data)
return t, err
}
+92
View File
@@ -0,0 +1,92 @@
package cache
import (
"encoding/json"
"fmt"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func Test_Resolver(t *testing.T) {
original := GetManager()
defer SetManager(original)
SetManager(NewInMemory(time.Hour))
type sub struct {
Name string
Value bool
}
type thing struct {
Value string
Values []int
Subs []*sub
}
versionHash := hashType[thing]()
cache := GetManager().GetCache("test", "v7/"+versionHash)
resolver := GetResolver[thing]("test", "v7")
require.NotNil(t, resolver)
require.IsType(t, &cacheResolver[thing]{}, resolver)
cr := resolver.(*cacheResolver[thing])
require.IsType(t, cache, cr.cache)
resolveErrCount := 0
resolveThingErr := func() (thing, error) {
resolveErrCount++
return thing{}, fmt.Errorf("an error")
}
_, err := resolver.Resolve("err", resolveThingErr)
require.ErrorContains(t, err, "an error")
require.Equal(t, 1, resolveErrCount)
_, err = resolver.Resolve("err", resolveThingErr)
require.ErrorContains(t, err, "an error")
require.Equal(t, 2, resolveErrCount)
aThing := thing{
Value: "a value",
Values: []int{7, 8, 9},
Subs: []*sub{
{
Name: "sub1",
Value: true,
},
{
Name: "sub2",
Value: false,
},
},
}
resolveThingCount := 0
resolveThing := func() (thing, error) {
resolveThingCount++
return aThing, nil
}
val, err := resolver.Resolve("thing", resolveThing)
require.NoError(t, err)
require.Equal(t, 1, resolveThingCount)
require.Equal(t, aThing, val)
val, err = resolver.Resolve("thing", resolveThing)
require.NoError(t, err)
require.Equal(t, 1, resolveThingCount)
require.Equal(t, aThing, val)
rdr, err := cache.Read("thing" + resolverKeySuffix)
require.NoError(t, err)
decoder := json.NewDecoder(rdr)
var val2 thing
err = decoder.Decode(&val2)
require.NoError(t, err)
require.Equal(t, aThing, val2)
}