fix: fix parsing for complex toml types (#2965)

* fix: fix parsing for complex toml types
---------
Signed-off-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com>
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>

.binny.yaml

@@ -58,7 +58,7 @@ tools:
   # used to release all artifacts
   - name: goreleaser
     version:
-      want: v2.0.0
+      want: v2.0.1
     method: github-release
     with:
       repo: goreleaser/goreleaser
@@ -111,7 +111,7 @@ tools:
   # used for triggering a release
   - name: gh
     version:
-      want: v2.50.0
+      want: v2.51.0
     method: github-release
     with:
       repo: cli/cli

.github/workflows/benchmark-testing.yaml

@@ -17,7 +17,7 @@ jobs:
     # the job by event.
     steps:
       - name: Checkout code
-        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Bootstrap environment
         uses: ./.github/actions/bootstrap

.github/workflows/codeql-analysis.yml

@@ -36,7 +36,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 #v4.1.6
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
 
     - name: Install Go
       uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 #v5.0.1
@@ -45,7 +45,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff #v3.25.8
+      uses: github/codeql-action/init@23acc5c183826b7a8a97bce3cecc52db901f8251 #v3.25.10
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -56,7 +56,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff #v3.25.8
+      uses: github/codeql-action/autobuild@23acc5c183826b7a8a97bce3cecc52db901f8251 #v3.25.10
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -70,4 +70,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff #v3.25.8
+      uses: github/codeql-action/analyze@23acc5c183826b7a8a97bce3cecc52db901f8251 #v3.25.10

.github/workflows/detect-schema-changes.yaml

@@ -30,7 +30,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
 
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 #v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
 
       - run: python .github/scripts/labeler.py
         env:

.github/workflows/release-version-file.yaml

@@ -20,7 +20,7 @@ jobs:
   release:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 #v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
 
       - name: Update version file
         run: make ci-release-version-file

.github/workflows/release.yaml

@@ -15,7 +15,7 @@ jobs:
     environment: release
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 #v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
 
       - name: Check if running on main
         if: github.ref != 'refs/heads/main'
@@ -105,7 +105,7 @@ jobs:
       # required for goreleaser signs section with cosign
       id-token: write
     steps:
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 #v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
         with:
           fetch-depth: 0
 

.github/workflows/update-bootstrap-tools.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.repository == 'anchore/syft' # only run for main repo
     steps:
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 #v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
 
       - name: Bootstrap environment
         uses: ./.github/actions/bootstrap

.github/workflows/update-cpe-dictionary-index.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.repository == 'anchore/syft' # only run for main repo
     steps:
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 #v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
 
       - name: Bootstrap environment
         uses: ./.github/actions/bootstrap

.github/workflows/update-stereoscope-release.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.repository == 'anchore/syft' # only run for main repo
     steps:
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 #v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
 
       - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 #v5.0.1
         with:

.github/workflows/validations.yaml

@@ -17,7 +17,7 @@ jobs:
     name: "Static analysis"
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 #v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
 
       - name: Bootstrap environment
         uses: ./.github/actions/bootstrap
@@ -31,7 +31,7 @@ jobs:
     name: "Unit tests"
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 #v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
 
       - name: Bootstrap environment
         uses: ./.github/actions/bootstrap
@@ -87,7 +87,7 @@ jobs:
     name: "Integration tests"
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 #v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
 
       - name: Bootstrap environment
         uses: ./.github/actions/bootstrap
@@ -109,7 +109,7 @@ jobs:
     name: "Build snapshot artifacts"
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 #v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
 
       - name: Bootstrap environment
         uses: ./.github/actions/bootstrap
@@ -139,7 +139,7 @@ jobs:
     needs: [Build-Snapshot-Artifacts]
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 #v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
 
       - name: Bootstrap environment
         uses: ./.github/actions/bootstrap
@@ -190,7 +190,7 @@ jobs:
       - name: Install Cosign
         uses: sigstore/cosign-installer@v3.5.0
 
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 #v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
 
       - name: Bootstrap environment
         uses: ./.github/actions/bootstrap
@@ -234,7 +234,7 @@ jobs:
     needs: [Build-Snapshot-Artifacts]
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 #v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
 
       - name: Bootstrap environment
         uses: ./.github/actions/bootstrap

cmd/syft/internal/commands/attest.go

@@ -43,6 +43,7 @@ type attestOptions struct {
 	options.UpdateCheck `yaml:",inline" mapstructure:",squash"`
 	options.Catalog     `yaml:",inline" mapstructure:",squash"`
 	Attest              options.Attest `yaml:"attest" mapstructure:"attest"`
+	Cache               options.Cache  `json:"-" yaml:"cache" mapstructure:"cache"`
 }
 
 func Attest(app clio.Application) *cobra.Command {
@@ -77,6 +78,7 @@ func defaultAttestOptions() attestOptions {
 		Output:      defaultAttestOutputOptions(),
 		UpdateCheck: options.DefaultUpdateCheck(),
 		Catalog:     options.DefaultCatalog(),
+		Cache:       options.DefaultCache(),
 	}
 }
 

cmd/syft/internal/commands/scan.go

@@ -68,6 +68,7 @@ type scanOptions struct {
 	options.Output      `yaml:",inline" mapstructure:",squash"`
 	options.UpdateCheck `yaml:",inline" mapstructure:",squash"`
 	options.Catalog     `yaml:",inline" mapstructure:",squash"`
+	Cache               options.Cache `json:"-" yaml:"cache" mapstructure:"cache"`
 }
 
 func defaultScanOptions() *scanOptions {
@@ -75,6 +76,7 @@ func defaultScanOptions() *scanOptions {
 		Output:      options.DefaultOutput(),
 		UpdateCheck: options.DefaultUpdateCheck(),
 		Catalog:     options.DefaultCatalog(),
+		Cache:       options.DefaultCache(),
 	}
 }
 

cmd/syft/internal/options/cache.go

@@ -0,0 +1,131 @@
+package options
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/adrg/xdg"
+	"github.com/mitchellh/go-homedir"
+
+	"github.com/anchore/clio"
+	"github.com/anchore/syft/internal/cache"
+	"github.com/anchore/syft/internal/log"
+)
+
+// Cache provides configuration for the Syft caching behavior
+type Cache struct {
+	Dir string `yaml:"dir" mapstructure:"dir"`
+	TTL string `yaml:"ttl" mapstructure:"ttl"`
+}
+
+func (c *Cache) DescribeFields(descriptions clio.FieldDescriptionSet) {
+	descriptions.Add(&c.Dir, "root directory to cache any downloaded content; empty string will use an in-memory cache")
+	descriptions.Add(&c.TTL, "time to live for cached data; setting this to 0 will disable caching entirely")
+}
+
+func (c *Cache) PostLoad() error {
+	ttl, err := parseDuration(c.TTL)
+	if err != nil {
+		log.Warnf("unable to parse duration '%v', using default (%s) due to: %v", c.TTL, durationToString(defaultTTL()), err)
+		ttl = defaultTTL()
+	}
+	// if TTL is <= 0, disable caching entirely
+	if ttl <= 0 {
+		cache.SetManager(nil)
+		return nil
+	}
+	// if dir == "" but we have a TTL, use an in-memory cache
+	if c.Dir == "" {
+		cache.SetManager(cache.NewInMemory(ttl))
+		return nil
+	}
+	dir, err := homedir.Expand(c.Dir)
+	if err != nil {
+		log.Warnf("unable to expand cache directory %s: %v", c.Dir, err)
+		cache.SetManager(cache.NewInMemory(ttl))
+	} else {
+		m, err := cache.NewFromDir(dir, ttl)
+		if err != nil {
+			log.Warnf("unable to get filesystem cache at %s: %v", c.Dir, err)
+			cache.SetManager(cache.NewInMemory(ttl))
+		} else {
+			cache.SetManager(m)
+		}
+	}
+	return nil
+}
+
+var _ interface {
+	clio.PostLoader
+	clio.FieldDescriber
+} = (*Cache)(nil)
+
+func DefaultCache() Cache {
+	return Cache{
+		Dir: defaultDir(),
+		TTL: durationToString(defaultTTL()),
+	}
+}
+
+func defaultTTL() time.Duration {
+	return 7 * 24 * time.Hour
+}
+
+func defaultDir() string {
+	var err error
+	cacheRoot := xdg.CacheHome
+	if cacheRoot == "" {
+		cacheRoot, err = homedir.Dir()
+		if err != nil {
+			cacheRoot = os.TempDir()
+			log.Debugf("unable to get stable cache directory due to: %v, defaulting cache to temp dir: %s", err, cacheRoot)
+		} else {
+			cacheRoot = filepath.Join(cacheRoot, ".cache")
+		}
+	}
+
+	return filepath.Join(cacheRoot, "syft")
+}
+
+func durationToString(duration time.Duration) string {
+	days := int64(duration / (24 * time.Hour))
+	remain := duration % (24 * time.Hour)
+	out := ""
+	if days > 0 {
+		out = fmt.Sprintf("%vd", days)
+	}
+	if remain != 0 {
+		out += remain.String()
+	}
+	if out == "" {
+		return "0"
+	}
+	return out
+}
+
+func parseDuration(duration string) (time.Duration, error) {
+	whitespace := regexp.MustCompile(`\s+`)
+	duration = strings.ToLower(whitespace.ReplaceAllString(duration, ""))
+	parts := strings.SplitN(duration, "d", 2)
+	var days time.Duration
+	var remain time.Duration
+	var err error
+	if len(parts) > 1 {
+		numDays, daysErr := strconv.Atoi(parts[0])
+		if daysErr != nil {
+			return 0, daysErr
+		}
+		days = time.Duration(numDays) * 24 * time.Hour
+		if len(parts[1]) > 0 {
+			remain, err = time.ParseDuration(parts[1])
+		}
+	} else {
+		remain, err = time.ParseDuration(duration)
+	}
+	return days + remain, err
+}

cmd/syft/internal/options/cache_test.go

@@ -0,0 +1,313 @@
+package options
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/adrg/xdg"
+	"github.com/mitchellh/go-homedir"
+	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/internal/cache"
+)
+
+func Test_defaultDir(t *testing.T) {
+	tmpDir := filepath.Join(t.TempDir(), "cache-temp")
+	xdgCacheDir := filepath.Join(tmpDir, "fake-xdg-cache")
+	homeDir := filepath.Join(tmpDir, "fake-home")
+
+	tests := []struct {
+		name     string
+		env      map[string]string
+		expected string
+	}{
+		{
+			name: "no-xdg",
+			env: map[string]string{
+				"HOME": homeDir,
+			},
+			expected: homeDir,
+		},
+		{
+			name: "xdg-cache",
+			env: map[string]string{
+				"XDG_CACHE_HOME": xdgCacheDir,
+			},
+			expected: xdgCacheDir,
+		},
+	}
+
+	// capture all the initial environment variables to reset them before we reset library caches
+	env := map[string]string{
+		"HOME":            "",
+		"XDG_DATA_HOME":   "",
+		"XDG_DATA_DIRS":   "",
+		"XDG_CONFIG_HOME": "",
+		"XDG_CONFIG_DIRS": "",
+		"XDG_STATE_HOME":  "",
+		"XDG_CACHE_HOME":  "",
+		"XDG_RUNTIME_DIR": "",
+	}
+	for k := range env {
+		env[k] = os.Getenv(k)
+	}
+
+	unsetEnv := func(t *testing.T) {
+		for k := range env {
+			t.Setenv(k, "")
+		}
+	}
+
+	resetEnv := func() {
+		for k, v := range env {
+			if v == "" {
+				_ = os.Unsetenv(k)
+			} else {
+				_ = os.Setenv(k, v)
+			}
+		}
+		homedir.Reset()
+		xdg.Reload()
+	}
+
+	t.Cleanup(resetEnv)
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			defer resetEnv()
+
+			unsetEnv(t)
+			for k, v := range test.env {
+				t.Setenv(k, v)
+			}
+			homedir.Reset()
+			xdg.Reload()
+
+			got := defaultDir()
+
+			require.True(t, strings.HasPrefix(got, test.expected))
+		})
+	}
+}
+
+func Test_cacheOptions(t *testing.T) {
+	tmp := t.TempDir()
+	tests := []struct {
+		name string
+		opts Cache
+		test func(t *testing.T)
+	}{
+		{
+			name: "disable-1",
+			opts: Cache{
+				Dir: "some-dir",
+				TTL: "0",
+			},
+			test: func(t *testing.T) {
+				c := cache.GetManager().GetCache("test-disable-1", "v-disable-1")
+				err := c.Write("key-disable-1", strings.NewReader("some-value-disable-1"))
+				require.NoError(t, err)
+				rdr, err := c.Read("key-disable-1")
+				require.Nil(t, rdr)
+				require.Error(t, err)
+			},
+		},
+		{
+			name: "disable-2",
+			opts: Cache{
+				Dir: "some-dir",
+				TTL: "0s",
+			},
+			test: func(t *testing.T) {
+				c := cache.GetManager().GetCache("test-disable-2", "v-disable-2")
+				err := c.Write("key-disable-2", strings.NewReader("some-value-disable-2"))
+				require.NoError(t, err)
+				rdr, err := c.Read("key-disable-2")
+				require.Nil(t, rdr)
+				require.Error(t, err)
+			},
+		},
+
+		{
+			name: "disable-3",
+			opts: Cache{
+				Dir: "some-dir",
+				TTL: "0d",
+			},
+			test: func(t *testing.T) {
+				c := cache.GetManager().GetCache("test-disable-3", "v-disable-3")
+				err := c.Write("key-disable-3", strings.NewReader("some-value-disable-3"))
+				require.NoError(t, err)
+				rdr, err := c.Read("key-disable-3")
+				require.Nil(t, rdr)
+				require.Error(t, err)
+			},
+		},
+		{
+			name: "in-memory",
+			opts: Cache{
+				Dir: "",
+				TTL: "10m",
+			},
+			test: func(t *testing.T) {
+				c := cache.GetManager().GetCache("test-mem", "v-mem")
+				err := c.Write("key-mem", strings.NewReader("some-value-mem"))
+				require.NoError(t, err)
+				rdr, err := c.Read("key-mem")
+				require.NotNil(t, rdr)
+				defer internal.CloseAndLogError(rdr, "")
+				require.NoError(t, err)
+				data, err := io.ReadAll(rdr)
+				require.NoError(t, err)
+				require.Equal(t, "some-value-mem", string(data))
+				require.NoDirExists(t, filepath.Join("test-mem", "v-mem"))
+			},
+		},
+		{
+			name: "on disk",
+			opts: Cache{
+				Dir: tmp,
+				TTL: "10m",
+			},
+			test: func(t *testing.T) {
+				c := cache.GetManager().GetCache("test-disk", "v-disk")
+				err := c.Write("key-disk", strings.NewReader("some-value-disk"))
+				require.NoError(t, err)
+				rdr, err := c.Read("key-disk")
+				require.NotNil(t, rdr)
+				defer internal.CloseAndLogError(rdr, "")
+				require.NoError(t, err)
+				data, err := io.ReadAll(rdr)
+				require.NoError(t, err)
+				require.Equal(t, "some-value-disk", string(data))
+				require.DirExists(t, filepath.Join(tmp, "test-disk", "v-disk"))
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			original := cache.GetManager()
+			defer cache.SetManager(original)
+
+			err := test.opts.PostLoad()
+			require.NoError(t, err)
+
+			test.test(t)
+		})
+	}
+}
+
+func Test_parseDuration(t *testing.T) {
+	tests := []struct {
+		duration string
+		expect   time.Duration
+		err      require.ErrorAssertionFunc
+	}{
+		{
+			duration: "0d",
+			expect:   0,
+		},
+		{
+			duration: "0m",
+			expect:   0,
+		},
+		{
+			duration: "0s",
+			expect:   0,
+		},
+		{
+			duration: "0",
+			expect:   0,
+		},
+		{
+			duration: "1d",
+			expect:   24 * time.Hour,
+		},
+		{
+			duration: "7d",
+			expect:   7 * 24 * time.Hour,
+		},
+		{
+			duration: "365D",
+			expect:   365 * 24 * time.Hour,
+		},
+		{
+			duration: "7d1h1m1s",
+			expect:   7*24*time.Hour + time.Hour + time.Minute + time.Second,
+		},
+		{
+			duration: "7d  1h 1m 1s",
+			expect:   7*24*time.Hour + time.Hour + time.Minute + time.Second,
+		},
+		{
+			duration: "2h",
+			expect:   2 * time.Hour,
+		},
+		{
+			duration: "2h5m",
+			expect:   2*time.Hour + 5*time.Minute,
+		},
+		{
+			duration: "2h 5m",
+			expect:   2*time.Hour + 5*time.Minute,
+		},
+		{
+			duration: "d24h",
+			err:      require.Error,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.duration, func(t *testing.T) {
+			got, err := parseDuration(test.duration)
+			if test.err != nil {
+				test.err(t, err)
+				return
+			} else {
+				require.NoError(t, err)
+			}
+			require.Equal(t, test.expect, got)
+		})
+	}
+}
+
+func Test_durationToString(t *testing.T) {
+	tests := []struct {
+		duration time.Duration
+		expect   string
+		err      require.ErrorAssertionFunc
+	}{
+		{
+			expect:   "1d",
+			duration: 24 * time.Hour,
+		},
+		{
+			expect:   "7d",
+			duration: 7 * 24 * time.Hour,
+		},
+		{
+			expect:   "7d1h1m1s",
+			duration: 7*24*time.Hour + time.Hour + time.Minute + time.Second,
+		},
+		{
+			expect:   "2h0m0s",
+			duration: 2 * time.Hour,
+		},
+		{
+			expect:   "2h5m0s",
+			duration: 2*time.Hour + 5*time.Minute,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.expect, func(t *testing.T) {
+			got := durationToString(test.duration)
+			require.Equal(t, test.expect, got)
+		})
+	}
+}

go.mod

@@ -71,7 +71,7 @@ require (
 	github.com/spf13/afero v1.11.0
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.9.0
-	github.com/vbatts/go-mtree v0.5.3
+	github.com/vbatts/go-mtree v0.5.4
 	github.com/vifraa/gopom v1.0.0
 	github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651
 	github.com/wagoodman/go-progress v0.0.0-20230925121702-07e42b3cdba0
@@ -81,12 +81,16 @@ require (
 	golang.org/x/mod v0.18.0
 	golang.org/x/net v0.26.0
 	gopkg.in/yaml.v3 v3.0.1
-	modernc.org/sqlite v1.30.0
+	modernc.org/sqlite v1.30.1
 )
 
 require google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect
 
-require github.com/magiconair/properties v1.8.7
+require (
+	github.com/BurntSushi/toml v1.4.0
+	github.com/adrg/xdg v0.4.0
+	github.com/magiconair/properties v1.8.7
+)
 
 require (
 	dario.cat/mergo v1.0.0 // indirect
@@ -98,7 +102,6 @@ require (
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/Microsoft/hcsshim v0.11.4 // indirect
 	github.com/ProtonMail/go-crypto v1.0.0 // indirect
-	github.com/adrg/xdg v0.4.0 // indirect
 	github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect
 	github.com/andybalholm/brotli v1.0.4 // indirect
 	github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492 // indirect
@@ -240,7 +243,7 @@ require (
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
-	modernc.org/libc v1.50.9 // indirect
+	modernc.org/libc v1.52.1 // indirect
 	modernc.org/mathutil v1.6.0 // indirect
 	modernc.org/memory v1.8.0 // indirect
 	modernc.org/strutil v1.2.0 // indirect

go.sum

@@ -57,6 +57,8 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg6
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
+github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/CycloneDX/cyclonedx-go v0.9.0 h1:inaif7qD8bivyxp7XLgxUYtOXWtDez7+j72qKTMQTb8=
 github.com/CycloneDX/cyclonedx-go v0.9.0/go.mod h1:NE/EWvzELOFlG6+ljX/QeMlVt9VKcTwu8u0ccsACEsw=
@@ -791,8 +793,8 @@ github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
 github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
-github.com/vbatts/go-mtree v0.5.3 h1:S/jYlfG8rZ+a0bhZd+RANXejy7M4Js8fq9U+XoWTd5w=
-github.com/vbatts/go-mtree v0.5.3/go.mod h1:eXsdoPMdL2jcJx6HweWi9lYQxBsTp4lNhqqAjgkZUg8=
+github.com/vbatts/go-mtree v0.5.4 h1:OMAb8jaCyiFA7zXj0Zc/oARcxBDBoeu2LizjB8BVJl0=
+github.com/vbatts/go-mtree v0.5.4/go.mod h1:5GqJbVhm9BBiCc4K5uc/c42FPgXulHaQs4sFUEfIWMo=
 github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
 github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY=
 github.com/vifraa/gopom v1.0.0 h1:L9XlKbyvid8PAIK8nr0lihMApJQg/12OBvMA28BcWh0=
@@ -1359,16 +1361,16 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 modernc.org/cc/v4 v4.21.2 h1:dycHFB/jDc3IyacKipCNSDrjIC0Lm1hyoWOZTRR20Lk=
 modernc.org/cc/v4 v4.21.2/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
-modernc.org/ccgo/v4 v4.17.8 h1:yyWBf2ipA0Y9GGz/MmCmi3EFpKgeS7ICrAFes+suEbs=
-modernc.org/ccgo/v4 v4.17.8/go.mod h1:buJnJ6Fn0tyAdP/dqePbrrvLyr6qslFfTbFrCuaYvtA=
+modernc.org/ccgo/v4 v4.17.10 h1:6wrtRozgrhCxieCeJh85QsxkX/2FFrT9hdaWPlbn4Zo=
+modernc.org/ccgo/v4 v4.17.10/go.mod h1:0NBHgsqTTpm9cA5z2ccErvGZmtntSM9qD2kFAs6pjXM=
 modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
 modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
 modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
 modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
 modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
 modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
-modernc.org/libc v1.50.9 h1:hIWf1uz55lorXQhfoEoezdUHjxzuO6ceshET/yWjSjk=
-modernc.org/libc v1.50.9/go.mod h1:15P6ublJ9FJR8YQCGy8DeQ2Uwur7iW9Hserr/T3OFZE=
+modernc.org/libc v1.52.1 h1:uau0VoiT5hnR+SpoWekCKbLqm7v6dhRL3hI+NQhgN3M=
+modernc.org/libc v1.52.1/go.mod h1:HR4nVzFDSDizP620zcMCgjb1/8xk2lg5p/8yjfGv1IQ=
 modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
 modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
 modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
@@ -1377,8 +1379,8 @@ modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
 modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
 modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
 modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
-modernc.org/sqlite v1.30.0 h1:8YhPUs/HTnlEgErn/jSYQTwHN/ex8CjHHjg+K9iG7LM=
-modernc.org/sqlite v1.30.0/go.mod h1:cgkTARJ9ugeXSNaLBPK3CqbOe7Ec7ZhWPoMFGldEYEw=
+modernc.org/sqlite v1.30.1 h1:YFhPVfu2iIgUf9kuA1CR7iiHdcEEsI2i+yjRYHscyxk=
+modernc.org/sqlite v1.30.1/go.mod h1:DUmsiWQDaAvU4abhc/N+djlom/L2o8f7gZ95RCvyoLU=
 modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
 modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
 modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=

internal/cache/README.md

@@ -0,0 +1,51 @@
+# Caching
+
+All caches are created from a global `manager`. By defaut this is a `bypassedCache`, which performs no caching.
+One benefit of this is that tests don't need to worry about caching causing issues unless they explicitly need
+to test the cache and can opt-in using the `cache.TestCache(t)` helper.
+
+Syft sets a `filesystemCache` when the [cache options](../../cmd/syft/internal/options/cache.go) are loaded.
+
+When using the `filesystemCache` all items are stored on disk under a root directory, generally in the form of:
+```
+<rootDir>/<named-cache>/<data-version>/path/to/data
+```
+
+# Using the cache
+
+The easiest and preferred method to use the cache is a `cache.Resolver`, which automatically creates a `<data-version>`
+based on the _structure_ of the provided type.
+If the structure changes in any way it will end up with a new version key and all will re populate this new key,
+ignoring cached values from older, different versions.
+The resolver will store items using the `json` package to serialize/deserialize values, so to save space
+it is encouraged  to use `omitempty`. For example:
+
+```go
+type myCacheItem struct {
+	Name string `json:"name",omitempty`
+}
+```
+
+It is possible to use core types such as `pkg.Package` as long as they support the standard `json` serialization,
+but this is discouraged in order to decouple changes to them from affecting the information stored in the cache.
+
+To get a cache for this type:
+```go
+resolver := cache.GetResolver[myCacheItem]("myCacheName", "v1")
+```
+
+Using the `resolver` is a single call, which manages checking for items in the cache, expiry times,
+and if not found invoking the callback to populate the cache and return a value:
+```go
+data := resolver.Resolve("some/cache/key", func() (myCacheItem, error) {
+	// do things to return a myCacheItem or error
+})
+```
+
+If it is common that checking for an item will result in errors, and you do not want to re-run the resolve function
+when errors are encountered, instead of using `GetResolver`, you can use `GetResolverCachingErrors`, which is useful
+for things such as resolving artifacts over a network, where a number of them will not be resolved, and you do not want
+to continue to have the expense of running the network resolution. This should be used when it is acceptable a network
+outage and cached errors is an acceptable risk.
+
+An example can be seen in the [golang cataloger](../../syft/pkg/cataloger/golang/licenses.go) fetching remote licenses.

internal/cache/bypass.go

@@ -0,0 +1,24 @@
+package cache
+
+import "io"
+
+type bypassedCache struct{}
+
+func (b *bypassedCache) Read(_ string) (ReaderAtCloser, error) {
+	return nil, errNotFound
+}
+
+func (b *bypassedCache) Write(_ string, contents io.Reader) error {
+	if closer, ok := contents.(io.Closer); ok {
+		_ = closer.Close()
+	}
+	return nil
+}
+
+func (b *bypassedCache) GetCache(_, _ string) Cache {
+	return b
+}
+
+func (b *bypassedCache) RootDirs() []string {
+	return nil
+}

internal/cache/bypass_test.go

@@ -0,0 +1,18 @@
+package cache
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_bypassedCache(t *testing.T) {
+	m := bypassedCache{}
+	cache := m.GetCache("name", "version")
+	err := cache.Write("test", strings.NewReader("value"))
+	require.NoError(t, err)
+	rdr, err := cache.Read("test")
+	require.Nil(t, rdr)
+	require.ErrorIs(t, err, errNotFound)
+}

internal/cache/cache.go

@@ -0,0 +1,49 @@
+package cache
+
+import (
+	"io"
+)
+
+// Manager is responsible for managing cache data and instantiating all caches
+type Manager interface {
+	// GetCache returns a cache scoped to the given named, versioned data
+	GetCache(name, version string) Cache
+
+	// RootDirs returns any root directories this cache manager uses
+	RootDirs() []string
+}
+
+// ReaderAtCloser is an amalgamation of: io.Reader, io.ReaderAt, and io.Closer
+type ReaderAtCloser interface {
+	io.Reader
+	io.ReaderAt
+	io.Closer
+}
+
+// Cache is what the application interacts with to get and set cached data
+type Cache interface {
+	// Read returns a reader for the cache value, if found and not expired
+	// or errors when unable to find / expired
+	Read(key string) (ReaderAtCloser, error)
+
+	// Write writes the contents of the reader to the cache
+	// and closes it, if the reader implements io.Closer
+	Write(key string, contents io.Reader) error
+}
+
+// GetManager returns the global cache manager, which is used to instantiate all caches
+func GetManager() Manager {
+	return manager
+}
+
+// SetManager sets the global cache manager, which is used to instantiate all caches.
+// Setting this to nil disables caching.
+func SetManager(m Manager) {
+	if m == nil {
+		manager = &bypassedCache{}
+	} else {
+		manager = m
+	}
+}
+
+var manager Manager = &bypassedCache{}

internal/cache/cache_test.go

@@ -0,0 +1,32 @@
+package cache
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_SetManager(t *testing.T) {
+	original := GetManager()
+	defer SetManager(original)
+
+	SetManager(nil)
+
+	require.NotNil(t, GetManager())
+	require.IsType(t, &bypassedCache{}, GetManager())
+
+	SetManager(NewInMemory(0))
+
+	require.NotNil(t, GetManager())
+	require.IsType(t, &bypassedCache{}, GetManager())
+
+	SetManager(NewInMemory(1 * time.Hour))
+
+	require.NotNil(t, GetManager())
+	require.IsType(t, &filesystemCache{}, GetManager())
+
+	SetManager(nil)
+	require.NotNil(t, GetManager())
+	require.IsType(t, &bypassedCache{}, GetManager())
+}

internal/cache/error_resolver.go

@@ -0,0 +1,40 @@
+package cache
+
+import "fmt"
+
+// GetResolverCachingErrors returns a Resolver that caches errors and will return them
+// instead of continuing to call the provided resolve functions
+func GetResolverCachingErrors[T any](name, version string) Resolver[T] {
+	return &errorResolver[T]{
+		resolver: GetResolver[errResponse[T]](name, version),
+	}
+}
+
+type errResponse[T any] struct {
+	Error string `json:"err,omitempty"`
+	Value T      `json:"val,omitempty"`
+}
+
+type errorResolver[T any] struct {
+	resolver Resolver[errResponse[T]]
+}
+
+func (r *errorResolver[T]) Resolve(key string, resolver resolverFunc[T]) (T, error) {
+	v, err := r.resolver.Resolve(key, func() (errResponse[T], error) {
+		v, err := resolver()
+		out := errResponse[T]{
+			Value: v,
+		}
+		if err != nil {
+			out.Error = err.Error()
+		}
+		return out, nil
+	})
+	if err != nil {
+		return v.Value, err
+	}
+	if v.Error != "" {
+		return v.Value, fmt.Errorf(v.Error)
+	}
+	return v.Value, nil
+}

internal/cache/error_resolver_test.go

@@ -0,0 +1,47 @@
+package cache
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_errorResolver(t *testing.T) {
+	original := GetManager()
+	defer SetManager(original)
+	SetManager(NewInMemory(time.Hour))
+
+	resolver := GetResolverCachingErrors[string]("theCache", "theVersion")
+
+	resolveCount := 0
+	resolveFunc := func() (string, error) {
+		resolveCount++
+		return "theValue", nil
+	}
+
+	val, err := resolver.Resolve("theKey", resolveFunc)
+	require.NoError(t, err)
+	require.Equal(t, 1, resolveCount)
+	require.Equal(t, "theValue", val)
+
+	val, err = resolver.Resolve("theKey", resolveFunc)
+	require.NoError(t, err)
+	require.Equal(t, 1, resolveCount)
+	require.Equal(t, "theValue", val)
+
+	errorCount := 0
+	errorFunc := func() (string, error) {
+		errorCount++
+		return "", fmt.Errorf("an error")
+	}
+
+	_, err = resolver.Resolve("errorValue", errorFunc)
+	require.ErrorContains(t, err, "an error")
+	require.Equal(t, 1, errorCount)
+
+	_, err = resolver.Resolve("errorValue", errorFunc)
+	require.ErrorContains(t, err, "an error")
+	require.Equal(t, 1, errorCount)
+}

internal/cache/filesystem.go

@@ -0,0 +1,120 @@
+package cache
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/spf13/afero"
+
+	"github.com/anchore/syft/internal/log"
+)
+
+// NewFromDir creates a new cache manager which returns caches stored on disk, rooted at the given directory
+func NewFromDir(dir string, ttl time.Duration) (Manager, error) {
+	dir = filepath.Clean(dir)
+	fsys, err := subFs(afero.NewOsFs(), dir)
+	if err != nil {
+		return nil, err
+	}
+	return &filesystemCache{
+		dir: dir,
+		fs:  fsys,
+		ttl: ttl,
+	}, nil
+}
+
+const filePermissions = 0700
+const dirPermissions = os.ModeDir | filePermissions
+
+type filesystemCache struct {
+	dir string
+	fs  afero.Fs
+	ttl time.Duration
+}
+
+func (d *filesystemCache) GetCache(name, version string) Cache {
+	fsys, err := subFs(d.fs, name, version)
+	if err != nil {
+		log.Warnf("error getting cache for: %s/%s: %v", name, version, err)
+		return &bypassedCache{}
+	}
+	return &filesystemCache{
+		dir: filepath.Join(d.dir, name, version),
+		fs:  fsys,
+		ttl: d.ttl,
+	}
+}
+
+func (d *filesystemCache) RootDirs() []string {
+	if d.dir == "" {
+		return nil
+	}
+	return []string{d.dir}
+}
+
+func (d *filesystemCache) Read(key string) (ReaderAtCloser, error) {
+	path := makeDiskKey(key)
+	f, err := d.fs.Open(path)
+	if err != nil {
+		log.Tracef("no cache entry for %s %s: %v", d.dir, key, err)
+		return nil, errNotFound
+	} else if stat, err := f.Stat(); err != nil || stat == nil || time.Since(stat.ModTime()) > d.ttl {
+		log.Tracef("cache entry is too old for %s %s", d.dir, key)
+		return nil, errExpired
+	}
+	log.Tracef("using cache for %s %s", d.dir, key)
+	return f, nil
+}
+
+func (d *filesystemCache) Write(key string, contents io.Reader) error {
+	path := makeDiskKey(key)
+	return afero.WriteReader(d.fs, path, contents)
+}
+
+// subFs returns a writable directory with the given name under the root cache directory returned from findRoot,
+// the directory will be created if it does not exist
+func subFs(fsys afero.Fs, subDirs ...string) (afero.Fs, error) {
+	dir := filepath.Join(subDirs...)
+	dir = filepath.Clean(dir)
+	stat, err := fsys.Stat(dir)
+	if errors.Is(err, afero.ErrFileNotFound) {
+		err = fsys.MkdirAll(dir, dirPermissions)
+		if err != nil {
+			return nil, fmt.Errorf("unable to create directory at '%s/%s': %v", dir, strings.Join(subDirs, "/"), err)
+		}
+		stat, err = fsys.Stat(dir)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if err != nil || stat == nil || !stat.IsDir() {
+		return nil, fmt.Errorf("unable to verify directory '%s': %v", dir, err)
+	}
+	fsys = afero.NewBasePathFs(fsys, dir)
+	return fsys, err
+}
+
+var keyReplacer = regexp.MustCompile("[^-._/a-zA-Z0-9]")
+
+// makeDiskKey makes a safe sub-path but not escape forward slashes, this allows for logical partitioning on disk
+func makeDiskKey(key string) string {
+	// encode single dot directory
+	if key == "." {
+		return "%2E"
+	}
+	// replace any disallowed chars with encoded form
+	key = keyReplacer.ReplaceAllStringFunc(key, url.QueryEscape)
+	// allow . in names but not ..
+	key = strings.ReplaceAll(key, "..", "%2E%2E")
+	return key
+}
+
+var errNotFound = fmt.Errorf("not found")
+var errExpired = fmt.Errorf("expired")

internal/cache/filesystem_test.go

@@ -0,0 +1,94 @@
+package cache
+
+import (
+	"io"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/syft/internal"
+)
+
+func Test_filesystemCache(t *testing.T) {
+	dir := t.TempDir()
+	man, err := NewFromDir(dir, 1*time.Minute)
+	require.NoError(t, err)
+
+	cacheName := "test"
+	cacheVersion := "v1"
+	cache := man.GetCache(cacheName, cacheVersion)
+
+	cacheKey := "test-key"
+	contentsValue := "some contents to cache"
+
+	err = cache.Write(cacheKey, strings.NewReader(contentsValue))
+	require.NoError(t, err)
+
+	rdr, err := cache.Read(cacheKey)
+	require.NoError(t, err)
+	defer internal.CloseAndLogError(rdr, cacheKey)
+
+	contents, err := io.ReadAll(rdr)
+	require.NoError(t, err)
+	require.Equal(t, contentsValue, string(contents))
+
+	// check the contents were actually written to disk as expected
+	contents, err = os.ReadFile(filepath.Join(dir, cacheName, cacheVersion, cacheKey))
+	require.NoError(t, err)
+	require.Equal(t, contentsValue, string(contents))
+
+	_, err = cache.Read("otherKey")
+	require.ErrorIs(t, err, errNotFound)
+}
+
+func Test_makeDiskKey(t *testing.T) {
+	tests := []struct {
+		in       string
+		expected string
+	}{
+		{
+			in:       "",
+			expected: "",
+		},
+		{
+			in:       ".",
+			expected: "%2E",
+		},
+		{
+			in:       "..",
+			expected: "%2E%2E",
+		},
+		{
+			in:       "github.com",
+			expected: "github.com",
+		},
+		{
+			in:       "../github.com",
+			expected: "%2E%2E/github.com",
+		},
+		{
+			in:       "github.com/../..",
+			expected: "github.com/%2E%2E/%2E%2E",
+		},
+		{
+			in:       "github.com/%2E../..",
+			expected: "github.com/%252E%2E%2E/%2E%2E",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.in, func(t *testing.T) {
+			got := makeDiskKey(test.in)
+			// validate appropriate escaping
+			require.Equal(t, test.expected, got)
+			// also validate that unescaped string matches original
+			unescaped, err := url.QueryUnescape(got)
+			require.NoError(t, err)
+			require.Equal(t, test.in, unescaped)
+		})
+	}
+}

internal/cache/hash_type.go

@@ -0,0 +1,71 @@
+package cache
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/mitchellh/hashstructure/v2"
+)
+
+// hashType returns a stable hash based on the structure of the type
+func hashType[T any]() string {
+	// get the base type and hash an empty instance
+	var t T
+	empty := emptyValue(reflect.TypeOf(t)).Interface()
+	hash, err := hashstructure.Hash(empty, hashstructure.FormatV2, &hashstructure.HashOptions{
+		ZeroNil:         false,
+		IgnoreZeroValue: false,
+		SlicesAsSets:    false,
+		UseStringer:     false,
+	})
+	if err != nil {
+		panic(fmt.Errorf("unable to use type as cache key: %w", err))
+	}
+	return fmt.Sprintf("%x", hash)
+}
+
+func emptyValue(t reflect.Type) reflect.Value {
+	switch t.Kind() {
+	case reflect.Pointer:
+		e := t.Elem()
+		v := emptyValue(e)
+		if v.CanAddr() {
+			return v.Addr()
+		}
+		ptrv := reflect.New(e)
+		ptrv.Elem().Set(v)
+		return ptrv
+	case reflect.Slice:
+		v := emptyValue(t.Elem())
+		s := reflect.MakeSlice(t, 1, 1)
+		s.Index(0).Set(v)
+		return s
+	case reflect.Struct:
+		v := reflect.New(t).Elem()
+		// get all empty field values, too
+		for i := 0; i < v.NumField(); i++ {
+			f := t.Field(i)
+			if isIgnored(f) {
+				continue
+			}
+			fv := v.Field(i)
+			if fv.CanSet() {
+				fv.Set(emptyValue(f.Type))
+			}
+		}
+		return v
+	default:
+		return reflect.New(t).Elem()
+	}
+}
+
+func isIgnored(f reflect.StructField) bool {
+	if !f.IsExported() {
+		return true
+	}
+	tag := f.Tag.Get("hash")
+	if tag == "-" || tag == "ignore" {
+		return true
+	}
+	return false
+}

internal/cache/hash_type_test.go

@@ -0,0 +1,123 @@
+package cache
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/mitchellh/hashstructure/v2"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_hashType(t *testing.T) {
+	type t1 struct {
+		Name string
+	}
+	type t2 struct {
+		Name string
+	}
+	type generic[T any] struct {
+		Val T
+	}
+	tests := []struct {
+		name     string
+		hash     func() string
+		expected string
+	}{
+		{
+			name:     "struct 1",
+			hash:     func() string { return hashType[t1]() },
+			expected: "d106c3ffbf98a0b1",
+		},
+		{
+			name:     "slice of struct 1",
+			hash:     func() string { return hashType[[]t1]() },
+			expected: "8122ace4ee1af0b4",
+		},
+		{
+			name:     "slice of struct 2",
+			hash:     func() string { return hashType[[]t2]() },
+			expected: "8cc04b5808be5bf9",
+		},
+		{
+			name:     "ptr 1",
+			hash:     func() string { return hashType[*t1]() },
+			expected: "d106c3ffbf98a0b1", // same hash as t1, which is ok since the structs are the same
+		},
+		{
+			name:     "slice of ptr 1",
+			hash:     func() string { return hashType[[]*t1]() },
+			expected: "8122ace4ee1af0b4", // same hash as []t1, again underlying serialization is the same
+		},
+		{
+			name:     "slice of ptr 2",
+			hash:     func() string { return hashType[[]*t2]() },
+			expected: "8cc04b5808be5bf9", // same hash as []t2, underlying serialization is the same
+		},
+		{
+			name:     "slice of ptr of slice of ptr",
+			hash:     func() string { return hashType[[]*[]*t1]() },
+			expected: "500d9f5b3a5977ce",
+		},
+		{
+			name:     "generic 1",
+			hash:     func() string { return hashType[generic[t1]]() },
+			expected: "b5fbb30e24400e81",
+		},
+		{
+			name:     "generic 2",
+			hash:     func() string { return hashType[generic[t2]]() },
+			expected: "becdb767c6b22bfa",
+		},
+		{
+			name:     "generic with ptr 1",
+			hash:     func() string { return hashType[generic[*t1]]() },
+			expected: "30c8855bf290fd83",
+		},
+		{
+			name:     "generic with ptr 2",
+			hash:     func() string { return hashType[generic[*t2]]() },
+			expected: "b66366b6ce9e6361",
+		},
+		{
+			name:     "generic with slice 1",
+			hash:     func() string { return hashType[generic[[]t1]]() },
+			expected: "d2ed158942fa6c29",
+		},
+		{
+			name:     "generic with slice 2",
+			hash:     func() string { return hashType[generic[[]t2]]() },
+			expected: "7a7bec575871c179",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			require.Equal(t, test.expected, test.hash())
+		})
+	}
+}
+
+func Test_hashIgnores(t *testing.T) {
+	hash := func(v any) string {
+		v, err := hashstructure.Hash(v, hashstructure.FormatV2, &hashstructure.HashOptions{})
+		require.NoError(t, err)
+		return fmt.Sprintf("%x", v)
+	}
+	type t1 struct {
+		Name        string
+		notExported string
+	}
+	require.Equal(t, hash(t1{notExported: "a value"}), hashType[t1]())
+
+	type t2 struct {
+		Name     string
+		Exported string `hash:"ignore"`
+	}
+	require.Equal(t, hash(t2{Exported: "another value"}), hashType[t2]())
+
+	type t3 struct {
+		Name     string
+		Exported string `hash:"-"`
+	}
+	require.Equal(t, hash(t3{Exported: "still valued"}), hashType[t3]())
+}

internal/cache/memory.go

@@ -0,0 +1,19 @@
+package cache
+
+import (
+	"time"
+
+	"github.com/spf13/afero"
+)
+
+// NewInMemory returns an in-memory only cache manager
+func NewInMemory(ttl time.Duration) Manager {
+	if ttl <= 0 {
+		return &bypassedCache{}
+	}
+	return &filesystemCache{
+		dir: "",
+		fs:  afero.NewMemMapFs(),
+		ttl: ttl,
+	}
+}

internal/cache/memory_test.go

@@ -0,0 +1,37 @@
+package cache
+
+import (
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/syft/internal"
+)
+
+func Test_NewInMemory(t *testing.T) {
+	man := NewInMemory(time.Hour)
+
+	cacheName := "test"
+	cacheVersion := "v1"
+	cache := man.GetCache(cacheName, cacheVersion)
+
+	cacheKey := "test-key"
+	contentsValue := "some contents to cache"
+
+	err := cache.Write(cacheKey, strings.NewReader(contentsValue))
+	require.NoError(t, err)
+
+	rdr, err := cache.Read(cacheKey)
+	require.NoError(t, err)
+	defer internal.CloseAndLogError(rdr, cacheKey)
+
+	contents, err := io.ReadAll(rdr)
+	require.NoError(t, err)
+	require.Equal(t, contentsValue, string(contents))
+
+	_, err = cache.Read("otherKey")
+	require.ErrorIs(t, err, errNotFound)
+}

internal/cache/resolver.go

@@ -0,0 +1,83 @@
+package cache
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"path"
+
+	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/internal/log"
+)
+
+// Resolver interface provides a single Resolve method, which will return from cache
+// or call the provided resolve function to get the value if not available in cache
+type Resolver[T any] interface {
+	// Resolve attempts to resolve the given key from cache and convert it to the type of the cache,
+	// or calls the resolver function if unable to resolve a cached value
+	Resolve(key string, resolver resolverFunc[T]) (T, error)
+}
+
+// GetResolver returns a cache resolver for persistent cached data across Syft runs, stored in a unique
+// location based on the provided name and versioned by the type
+func GetResolver[T any](name, version string) Resolver[T] {
+	typeHash := hashType[T]()
+	versionKey := path.Join(version, typeHash)
+	return &cacheResolver[T]{
+		name:  fmt.Sprintf("%s/%s", name, versionKey),
+		cache: manager.GetCache(name, versionKey),
+	}
+}
+
+const resolverKeySuffix = ".json"
+
+type resolverFunc[T any] func() (T, error)
+
+type cacheResolver[T any] struct {
+	name  string
+	cache Cache
+}
+
+var _ interface {
+	Resolver[int]
+} = (*cacheResolver[int])(nil)
+
+func (r *cacheResolver[T]) Resolve(key string, resolver resolverFunc[T]) (T, error) {
+	key += resolverKeySuffix
+
+	rdr, err := r.cache.Read(key)
+	if rdr == nil || err != nil {
+		return r.resolveAndCache(key, resolver)
+	}
+	defer internal.CloseAndLogError(rdr, key)
+
+	dec := json.NewDecoder(rdr)
+	if dec == nil {
+		log.Tracef("error getting cache json decoder for %s %v: %v", r.name, key, err)
+		return r.resolveAndCache(key, resolver)
+	}
+	var t T
+	err = dec.Decode(&t)
+	if err != nil {
+		log.Tracef("error decoding cached entry for %s %v: %v", r.name, key, err)
+		return r.resolveAndCache(key, resolver)
+	}
+	// no error, able to resolve from cache
+	return t, nil
+}
+
+func (r *cacheResolver[T]) resolveAndCache(key string, resolver func() (T, error)) (T, error) {
+	t, err := resolver()
+	if err != nil {
+		return t, err
+	}
+	var data bytes.Buffer
+	enc := json.NewEncoder(&data)
+	enc.SetEscapeHTML(false)
+	err = enc.Encode(t)
+	if err != nil {
+		return t, err
+	}
+	err = r.cache.Write(key, &data)
+	return t, err
+}

internal/cache/resolver_test.go

@@ -0,0 +1,92 @@
+package cache
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_Resolver(t *testing.T) {
+	original := GetManager()
+	defer SetManager(original)
+	SetManager(NewInMemory(time.Hour))
+
+	type sub struct {
+		Name  string
+		Value bool
+	}
+
+	type thing struct {
+		Value  string
+		Values []int
+		Subs   []*sub
+	}
+
+	versionHash := hashType[thing]()
+	cache := GetManager().GetCache("test", "v7/"+versionHash)
+
+	resolver := GetResolver[thing]("test", "v7")
+	require.NotNil(t, resolver)
+
+	require.IsType(t, &cacheResolver[thing]{}, resolver)
+	cr := resolver.(*cacheResolver[thing])
+
+	require.IsType(t, cache, cr.cache)
+
+	resolveErrCount := 0
+	resolveThingErr := func() (thing, error) {
+		resolveErrCount++
+		return thing{}, fmt.Errorf("an error")
+	}
+
+	_, err := resolver.Resolve("err", resolveThingErr)
+	require.ErrorContains(t, err, "an error")
+	require.Equal(t, 1, resolveErrCount)
+
+	_, err = resolver.Resolve("err", resolveThingErr)
+	require.ErrorContains(t, err, "an error")
+	require.Equal(t, 2, resolveErrCount)
+
+	aThing := thing{
+		Value:  "a value",
+		Values: []int{7, 8, 9},
+		Subs: []*sub{
+			{
+				Name:  "sub1",
+				Value: true,
+			},
+			{
+				Name:  "sub2",
+				Value: false,
+			},
+		},
+	}
+
+	resolveThingCount := 0
+	resolveThing := func() (thing, error) {
+		resolveThingCount++
+		return aThing, nil
+	}
+
+	val, err := resolver.Resolve("thing", resolveThing)
+	require.NoError(t, err)
+	require.Equal(t, 1, resolveThingCount)
+	require.Equal(t, aThing, val)
+
+	val, err = resolver.Resolve("thing", resolveThing)
+	require.NoError(t, err)
+	require.Equal(t, 1, resolveThingCount)
+	require.Equal(t, aThing, val)
+
+	rdr, err := cache.Read("thing" + resolverKeySuffix)
+	require.NoError(t, err)
+	decoder := json.NewDecoder(rdr)
+
+	var val2 thing
+	err = decoder.Decode(&val2)
+	require.NoError(t, err)
+	require.Equal(t, aThing, val2)
+}

syft/pkg/cataloger/golang/cataloger.go

@@ -20,12 +20,8 @@ const (
 
 // NewGoModuleFileCataloger returns a new cataloger object that searches within go.mod files.
 func NewGoModuleFileCataloger(opts CatalogerConfig) pkg.Cataloger {
-	c := goModCataloger{
-		licenses: newGoLicenses(modFileCatalogerName, opts),
-	}
-
 	return generic.NewCataloger(modFileCatalogerName).
-		WithParserByGlobs(c.parseGoModFile, "**/go.mod")
+		WithParserByGlobs(newGoModCataloger(opts).parseGoModFile, "**/go.mod")
 }
 
 // NewGoModuleBinaryCataloger returns a new cataloger object that searches within binaries built by the go compiler.

syft/pkg/cataloger/golang/config.go

@@ -41,6 +41,7 @@ type MainModuleVersionConfig struct {
 func DefaultCatalogerConfig() CatalogerConfig {
 	g := CatalogerConfig{
 		MainModuleVersion: DefaultMainModuleVersionConfig(),
+		LocalModCacheDir:  defaultGoModDir(),
 	}
 
 	// first process the proxy settings
@@ -67,24 +68,25 @@ func DefaultCatalogerConfig() CatalogerConfig {
 		}
 	}
 
-	if g.LocalModCacheDir == "" {
-		goPath := os.Getenv("GOPATH")
-
-		if goPath == "" {
-			homeDir, err := homedir.Dir()
-			if err != nil {
-				log.Debug("unable to determine user home dir: %v", err)
-			} else {
-				goPath = filepath.Join(homeDir, "go")
-			}
-		}
-		if goPath != "" {
-			g.LocalModCacheDir = filepath.Join(goPath, "pkg", "mod")
-		}
-	}
 	return g
 }
 
+// defaultGoModDir returns $GOPATH/pkg/mod or $HOME/go/pkg/mod based on environment variables available
+func defaultGoModDir() string {
+	goPath := os.Getenv("GOPATH")
+
+	if goPath == "" {
+		homeDir, err := homedir.Dir()
+		if err != nil {
+			log.Warnf("unable to determine GOPATH or user home dir: %w", err)
+			return ""
+		}
+		goPath = filepath.Join(homeDir, "go")
+	}
+
+	return filepath.Join(goPath, "pkg", "mod")
+}
+
 func DefaultMainModuleVersionConfig() MainModuleVersionConfig {
 	return MainModuleVersionConfig{
 		FromLDFlags:       true,

syft/pkg/cataloger/golang/config_test.go

@@ -1,6 +1,7 @@
 package golang
 
 import (
+	"path/filepath"
 	"testing"
 
 	"github.com/mitchellh/go-homedir"
@@ -47,7 +48,7 @@ func Test_Config(t *testing.T) {
 			opts: opts{},
 			expected: CatalogerConfig{
 				SearchLocalModCacheLicenses: false,
-				LocalModCacheDir:            "/go/pkg/mod",
+				LocalModCacheDir:            filepath.Join("/go", "pkg", "mod"),
 				SearchRemoteLicenses:        false,
 				Proxies:                     []string{"https://my.proxy"},
 				NoProxy:                     []string{"my.private", "no.proxy"},

syft/pkg/cataloger/golang/licenses.go

@@ -21,25 +21,41 @@ import (
 	"github.com/scylladb/go-set/strset"
 
 	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/internal/cache"
 	"github.com/anchore/syft/internal/licenses"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/file"
-	"github.com/anchore/syft/syft/internal/fileresolver"
+	"github.com/anchore/syft/syft/license"
 	"github.com/anchore/syft/syft/pkg"
 )
 
-type goLicenses struct {
+type goLicense struct {
+	Value          string       `json:"val,omitempty"`
+	SPDXExpression string       `json:"spdx,omitempty"`
+	Type           license.Type `json:"type,omitempty"`
+	URLs           []string     `json:"urls,omitempty"`
+	Locations      []string     `json:"locations,omitempty"`
+}
+
+type goLicenseResolver struct {
 	catalogerName         string
 	opts                  CatalogerConfig
-	localModCacheResolver file.WritableResolver
+	localModCacheDir      fs.FS
+	licenseCache          cache.Resolver[[]goLicense]
 	lowerLicenseFileNames *strset.Set
 }
 
-func newGoLicenses(catalogerName string, opts CatalogerConfig) goLicenses {
-	return goLicenses{
+func newGoLicenseResolver(catalogerName string, opts CatalogerConfig) goLicenseResolver {
+	var localModCacheDir fs.FS
+	if opts.SearchLocalModCacheLicenses {
+		localModCacheDir = os.DirFS(opts.LocalModCacheDir)
+	}
+
+	return goLicenseResolver{
 		catalogerName:         catalogerName,
 		opts:                  opts,
-		localModCacheResolver: modCacheResolver(opts.LocalModCacheDir),
+		localModCacheDir:      localModCacheDir,
+		licenseCache:          cache.GetResolverCachingErrors[[]goLicense]("golang", "v1"),
 		lowerLicenseFileNames: strset.New(lowercaseLicenseFiles()...),
 	}
 }
@@ -63,98 +79,95 @@ func remotesForModule(proxies []string, noProxy []string, module string) []strin
 	return proxies
 }
 
-func modCacheResolver(modCacheDir string) file.WritableResolver {
-	var r file.WritableResolver
+func (c *goLicenseResolver) getLicenses(resolver file.Resolver, moduleName, moduleVersion string) ([]pkg.License, error) {
+	// search the scan target first, ignoring local and remote sources
+	goLicenses, err := c.findLicensesInSource(resolver,
+		fmt.Sprintf(`**/go/pkg/mod/%s@%s/*`, processCaps(moduleName), moduleVersion),
+	)
+	if err != nil || len(goLicenses) > 0 {
+		return toPkgLicenses(goLicenses), err
+	}
 
-	if modCacheDir == "" {
-		log.Trace("unable to determine mod cache directory, skipping mod cache resolver")
-		r = fileresolver.Empty{}
-	} else {
-		stat, err := os.Stat(modCacheDir)
-
-		if os.IsNotExist(err) || stat == nil || !stat.IsDir() {
-			log.Tracef("unable to open mod cache directory: %s, skipping mod cache resolver", modCacheDir)
-			r = fileresolver.Empty{}
-		} else {
-			r = fileresolver.NewFromUnindexedDirectory(modCacheDir)
+	// look in the local host mod directory...
+	if c.opts.SearchLocalModCacheLicenses {
+		goLicenses, err = c.getLicensesFromLocal(moduleName, moduleVersion)
+		if err != nil || len(goLicenses) > 0 {
+			return toPkgLicenses(goLicenses), err
 		}
 	}
 
-	return r
-}
-
-func (c *goLicenses) getLicenses(resolver file.Resolver, moduleName, moduleVersion string) (licenses []pkg.License, err error) {
-	licenses, err = c.findLicenses(resolver,
-		fmt.Sprintf(`**/go/pkg/mod/%s@%s/*`, processCaps(moduleName), moduleVersion),
-	)
-	if err != nil || len(licenses) > 0 {
-		return requireCollection(licenses), err
+	// download from remote sources
+	if c.opts.SearchRemoteLicenses {
+		goLicenses, err = c.getLicensesFromRemote(moduleName, moduleVersion)
 	}
 
-	// look in the local host mod cache...
-	licenses, err = c.getLicensesFromLocal(moduleName, moduleVersion)
-	if err != nil || len(licenses) > 0 {
-		return requireCollection(licenses), err
-	}
-
-	// we did not find it yet and remote searching was enabled
-	licenses, err = c.getLicensesFromRemote(moduleName, moduleVersion)
-	return requireCollection(licenses), err
+	return toPkgLicenses(goLicenses), err
 }
 
-func (c *goLicenses) getLicensesFromLocal(moduleName, moduleVersion string) ([]pkg.License, error) {
-	if !c.opts.SearchLocalModCacheLicenses {
+func (c *goLicenseResolver) getLicensesFromLocal(moduleName, moduleVersion string) ([]goLicense, error) {
+	if c.localModCacheDir == nil {
 		return nil, nil
 	}
 
-	// if we're running against a directory on the filesystem, it may not include the
-	// user's homedir / GOPATH, so we defer to using the localModCacheResolver
-	return c.findLicenses(c.localModCacheResolver, moduleSearchGlob(moduleName, moduleVersion))
-}
+	subdir := moduleDir(moduleName, moduleVersion)
 
-func (c *goLicenses) getLicensesFromRemote(moduleName, moduleVersion string) ([]pkg.License, error) {
-	if !c.opts.SearchRemoteLicenses {
-		return nil, nil
-	}
-
-	proxies := remotesForModule(c.opts.Proxies, c.opts.NoProxy, moduleName)
-
-	fsys, err := getModule(proxies, moduleName, moduleVersion)
+	// get the local subdirectory containing the specific go module
+	dir, err := fs.Sub(c.localModCacheDir, subdir)
 	if err != nil {
 		return nil, err
 	}
 
-	dir := moduleDir(moduleName, moduleVersion)
-
-	// populate the mod cache with the results
-	err = fs.WalkDir(fsys, ".", func(filePath string, d fs.DirEntry, err error) error {
-		if err != nil {
-			log.Debug(err)
-			return nil
-		}
-		if d.IsDir() {
-			return nil
-		}
-		f, err := fsys.Open(filePath)
-		if err != nil {
-			return err
-		}
-		return c.localModCacheResolver.Write(file.NewLocation(path.Join(dir, filePath)), f)
-	})
-
-	if err != nil {
-		log.Tracef("remote proxy walk failed for: %s", moduleName)
-	}
-
-	return c.findLicenses(c.localModCacheResolver, moduleSearchGlob(moduleName, moduleVersion))
+	// if we're running against a directory on the filesystem, it may not include the
+	// user's homedir / GOPATH, so we defer to using the localModCacheResolver
+	// we use $GOPATH/pkg/mod to avoid leaking information about the user's system
+	return c.findLicensesInFS("file://$GOPATH/pkg/mod/"+subdir+"/", dir)
 }
 
-func (c *goLicenses) findLicenses(resolver file.Resolver, globMatch string) (out []pkg.License, err error) {
-	out = make([]pkg.License, 0)
-	if resolver == nil {
-		return
-	}
+func (c *goLicenseResolver) getLicensesFromRemote(moduleName, moduleVersion string) ([]goLicense, error) {
+	return c.licenseCache.Resolve(fmt.Sprintf("%s/%s", moduleName, moduleVersion), func() ([]goLicense, error) {
+		proxies := remotesForModule(c.opts.Proxies, c.opts.NoProxy, moduleName)
 
+		urlPrefix, fsys, err := getModule(proxies, moduleName, moduleVersion)
+		if err != nil {
+			return nil, err
+		}
+
+		return c.findLicensesInFS(urlPrefix, fsys)
+	})
+}
+
+func (c *goLicenseResolver) findLicensesInFS(urlPrefix string, fsys fs.FS) ([]goLicense, error) {
+	var out []goLicense
+	err := fs.WalkDir(fsys, ".", func(filePath string, d fs.DirEntry, _ error) error {
+		if !c.lowerLicenseFileNames.Has(strings.ToLower(d.Name())) {
+			return nil
+		}
+		rdr, err := fsys.Open(filePath)
+		if err != nil {
+			log.Debugf("error opening license file %s: %v", filePath, err)
+			return nil
+		}
+		defer internal.CloseAndLogError(rdr, filePath)
+		parsed, err := licenses.Parse(rdr, file.NewLocation(filePath))
+		if err != nil {
+			log.Debugf("error parsing license file %s: %v", filePath, err)
+			return nil
+		}
+		// since these licenses are found in an external fs.FS, not in the scanned source,
+		// get rid of the locations but keep information about the where the license was found
+		// by prepending the urlPrefix to the internal path for an accurate representation
+		for _, l := range toGoLicenses(parsed) {
+			l.URLs = []string{urlPrefix + filePath}
+			l.Locations = nil
+			out = append(out, l)
+		}
+		return nil
+	})
+	return out, err
+}
+
+func (c *goLicenseResolver) findLicensesInSource(resolver file.Resolver, globMatch string) ([]goLicense, error) {
+	var out []goLicense
 	locations, err := resolver.FilesByGlob(globMatch)
 	if err != nil {
 		return nil, err
@@ -168,11 +181,19 @@ func (c *goLicenses) findLicenses(resolver file.Resolver, globMatch string) (out
 		out = append(out, parsed...)
 	}
 
-	return
+	// if we have a directory but simply don't have any found license files, indicate this so we
+	// don't re-download modules continually
+	if len(locations) > 0 && len(out) == 0 {
+		return nil, noLicensesFound{
+			glob: globMatch,
+		}
+	}
+
+	return out, nil
 }
 
-func (c *goLicenses) parseLicenseFromLocation(l file.Location, resolver file.Resolver) ([]pkg.License, error) {
-	var out []pkg.License
+func (c *goLicenseResolver) parseLicenseFromLocation(l file.Location, resolver file.Resolver) ([]goLicense, error) {
+	var out []goLicense
 	fileName := path.Base(l.RealPath)
 	if c.lowerLicenseFileNames.Has(strings.ToLower(fileName)) {
 		contents, err := resolver.FileContentsByLocation(l)
@@ -185,7 +206,7 @@ func (c *goLicenses) parseLicenseFromLocation(l file.Location, resolver file.Res
 			return nil, err
 		}
 
-		out = append(out, parsed...)
+		out = append(out, toGoLicenses(parsed)...)
 	}
 	return out, nil
 }
@@ -194,13 +215,9 @@ func moduleDir(moduleName, moduleVersion string) string {
 	return fmt.Sprintf("%s@%s", processCaps(moduleName), moduleVersion)
 }
 
-func moduleSearchGlob(moduleName, moduleVersion string) string {
-	return fmt.Sprintf("%s/*", moduleDir(moduleName, moduleVersion))
-}
-
-func requireCollection(licenses []pkg.License) []pkg.License {
+func requireCollection[T any](licenses []T) []T {
 	if licenses == nil {
-		return make([]pkg.License, 0)
+		return make([]T, 0)
 	}
 	return licenses
 }
@@ -213,18 +230,19 @@ func processCaps(s string) string {
 	})
 }
 
-func getModule(proxies []string, moduleName, moduleVersion string) (fsys fs.FS, err error) {
+func getModule(proxies []string, moduleName, moduleVersion string) (urlPrefix string, fsys fs.FS, err error) {
 	for _, proxy := range proxies {
 		u, _ := url.Parse(proxy)
 		if proxy == "direct" {
-			fsys, err = getModuleRepository(moduleName, moduleVersion)
+			urlPrefix, fsys, err = getModuleRepository(moduleName, moduleVersion)
 			continue
 		}
 		switch u.Scheme {
 		case "https", "http":
-			fsys, err = getModuleProxy(proxy, moduleName, moduleVersion)
+			urlPrefix, fsys, err = getModuleProxy(proxy, moduleName, moduleVersion)
 		case "file":
 			p := filepath.Join(u.Path, moduleName, "@v", moduleVersion)
+			urlPrefix = path.Join("file://", p) + "/"
 			fsys = os.DirFS(p)
 		}
 		if fsys != nil {
@@ -234,13 +252,13 @@ func getModule(proxies []string, moduleName, moduleVersion string) (fsys fs.FS,
 	return
 }
 
-func getModuleProxy(proxy string, moduleName string, moduleVersion string) (out fs.FS, _ error) {
+func getModuleProxy(proxy string, moduleName string, moduleVersion string) (moduleURL string, out fs.FS, _ error) {
 	u := fmt.Sprintf("%s/%s/@v/%s.zip", proxy, moduleName, moduleVersion)
 
 	// get the module zip
 	resp, err := http.Get(u) //nolint:gosec
 	if err != nil {
-		return nil, err
+		return "", nil, err
 	}
 	defer func() { _ = resp.Body.Close() }()
 
@@ -250,25 +268,25 @@ func getModuleProxy(proxy string, moduleName string, moduleVersion string) (out
 		// try lowercasing it; some packages have mixed casing that really messes up the proxy
 		resp, err = http.Get(u) //nolint:gosec
 		if err != nil {
-			return nil, err
+			return "", nil, err
 		}
 		defer func() { _ = resp.Body.Close() }()
 		if resp.StatusCode != http.StatusOK {
-			return nil, fmt.Errorf("failed to get module zip: %s", resp.Status)
+			return "", nil, fmt.Errorf("failed to get module zip: %s", resp.Status)
 		}
 	}
 
 	// read the zip
 	b, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return nil, err
+		return "", nil, err
 	}
 
 	out, err = zip.NewReader(bytes.NewReader(b), resp.ContentLength)
 	versionPath := findVersionPath(out, ".")
 	out = getSubFS(out, versionPath)
 
-	return out, err
+	return u + "#" + versionPath + "/", out, err
 }
 
 func findVersionPath(f fs.FS, dir string) string {
@@ -288,26 +306,111 @@ func findVersionPath(f fs.FS, dir string) string {
 	return ""
 }
 
-func getModuleRepository(moduleName string, moduleVersion string) (fs.FS, error) {
+func getModuleRepository(moduleName string, moduleVersion string) (string, fs.FS, error) {
 	repoName := moduleName
 	parts := strings.Split(moduleName, "/")
 	if len(parts) > 2 {
 		repoName = fmt.Sprintf("%s/%s/%s", parts[0], parts[1], parts[2])
 	}
 
+	// see if there's a hash and use that if so, otherwise use a tag
+	splitVersion := strings.Split(moduleVersion, "-")
+	var cloneRefName plumbing.ReferenceName
+	refPath := ""
+	if len(splitVersion) < 3 {
+		tagName := splitVersion[0]
+		cloneRefName = plumbing.NewTagReferenceName(tagName)
+		refPath = "/tags/" + tagName
+	}
+
 	f := memfs.New()
 	buf := &bytes.Buffer{}
-	_, err := git.Clone(memory.NewStorage(), f, &git.CloneOptions{
-		URL:           fmt.Sprintf("https://%s", repoName),
-		ReferenceName: plumbing.NewTagReferenceName(moduleVersion), // FIXME version might be a SHA
+	repoURL := fmt.Sprintf("https://%s", repoName)
+	r, err := git.Clone(memory.NewStorage(), f, &git.CloneOptions{
+		URL:           repoURL,
+		ReferenceName: cloneRefName,
 		SingleBranch:  true,
 		Depth:         1,
 		Progress:      buf,
 	})
-
 	if err != nil {
-		return nil, fmt.Errorf("%w -- %s", err, buf.String())
+		return "", nil, fmt.Errorf("%w -- %s", err, buf.String())
 	}
 
-	return billyFSAdapter{fs: f}, nil
+	if len(splitVersion) > 2 {
+		sha := splitVersion[len(splitVersion)-1]
+		hash, err := r.ResolveRevision(plumbing.Revision(sha))
+		if err != nil || hash == nil {
+			log.Tracef("unable to resolve hash %s: %v", sha, err)
+		} else {
+			w, err := r.Worktree()
+			if err != nil {
+				log.Tracef("unable to get worktree, using default: %v", err)
+			}
+			err = w.Checkout(&git.CheckoutOptions{
+				Hash: *hash,
+			})
+			if err != nil {
+				log.Tracef("unable to checkout commit, using default: %v", err)
+			} else {
+				refPath = "/refs/" + hash.String()
+			}
+		}
+	}
+
+	return repoURL + refPath + "/", billyFSAdapter{fs: f}, err
+}
+
+type noLicensesFound struct {
+	glob string
+}
+
+func (l noLicensesFound) Error() string {
+	return fmt.Sprintf("unable to find license information matching: %s", l.glob)
+}
+
+var _ error = (*noLicensesFound)(nil)
+
+func toPkgLicenses(goLicenses []goLicense) []pkg.License {
+	var out []pkg.License
+	for _, l := range goLicenses {
+		out = append(out, pkg.License{
+			Value:          l.Value,
+			SPDXExpression: l.SPDXExpression,
+			Type:           l.Type,
+			URLs:           l.URLs,
+			Locations:      toPkgLocations(l.Locations),
+		})
+	}
+	return requireCollection(out)
+}
+
+func toPkgLocations(goLocations []string) file.LocationSet {
+	out := file.NewLocationSet()
+	for _, l := range goLocations {
+		out.Add(file.NewLocation(l))
+	}
+	return out
+}
+
+func toGoLicenses(pkgLicenses []pkg.License) []goLicense {
+	var out []goLicense
+	for _, l := range pkgLicenses {
+		out = append(out, goLicense{
+			Value:          l.Value,
+			SPDXExpression: l.SPDXExpression,
+			Type:           l.Type,
+			URLs:           l.URLs,
+			Locations:      toGoLocations(l.Locations),
+		})
+	}
+	return out
+}
+
+func toGoLocations(locations file.LocationSet) []string {
+	var out []string
+	for _, l := range locations.ToSlice() {
+		out = append(out, l.RealPath)
+	}
+	return out
 }

syft/pkg/cataloger/golang/licenses_test.go

@@ -8,6 +8,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"path"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -36,7 +37,8 @@ func Test_LocalLicenseSearch(t *testing.T) {
 				Value:          "Apache-2.0",
 				SPDXExpression: "Apache-2.0",
 				Type:           license.Concluded,
-				Locations:      file.NewLocationSet(loc1),
+				URLs:           []string{"file://$GOPATH/pkg/mod/" + loc1.RealPath},
+				Locations:      file.NewLocationSet(),
 			},
 		},
 		{
@@ -46,7 +48,8 @@ func Test_LocalLicenseSearch(t *testing.T) {
 				Value:          "MIT",
 				SPDXExpression: "MIT",
 				Type:           license.Concluded,
-				Locations:      file.NewLocationSet(loc2),
+				URLs:           []string{"file://$GOPATH/pkg/mod/" + loc2.RealPath},
+				Locations:      file.NewLocationSet(),
 			},
 		},
 		{
@@ -56,7 +59,8 @@ func Test_LocalLicenseSearch(t *testing.T) {
 				Value:          "Apache-2.0",
 				SPDXExpression: "Apache-2.0",
 				Type:           license.Concluded,
-				Locations:      file.NewLocationSet(loc3),
+				URLs:           []string{"file://$GOPATH/pkg/mod/" + loc3.RealPath},
+				Locations:      file.NewLocationSet(),
 			},
 		},
 	}
@@ -66,11 +70,11 @@ func Test_LocalLicenseSearch(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			l := newGoLicenses(
+			l := newGoLicenseResolver(
 				"",
 				CatalogerConfig{
 					SearchLocalModCacheLicenses: true,
-					LocalModCacheDir:            path.Join(wd, "test-fixtures", "licenses", "pkg", "mod"),
+					LocalModCacheDir:            filepath.Join(wd, "test-fixtures", "licenses", "pkg", "mod"),
 				},
 			)
 			licenses, err := l.getLicenses(fileresolver.Empty{}, test.name, test.version)
@@ -97,7 +101,7 @@ func Test_RemoteProxyLicenseSearch(t *testing.T) {
 
 		wd, err := os.Getwd()
 		require.NoError(t, err)
-		testDir := path.Join(wd, "test-fixtures", "licenses", "pkg", "mod", processCaps(modPath)+"@"+modVersion)
+		testDir := filepath.Join(wd, "test-fixtures", "licenses", "pkg", "mod", processCaps(modPath)+"@"+modVersion)
 
 		archive := zip.NewWriter(buf)
 
@@ -106,9 +110,9 @@ func Test_RemoteProxyLicenseSearch(t *testing.T) {
 		for _, f := range entries {
 			// the zip files downloaded contain a path to the repo that somewhat matches where it ends up on disk,
 			// so prefix entries with something similar
-			writer, err := archive.Create(path.Join("github.com/something/some@version", f.Name()))
+			writer, err := archive.Create(path.Join(moduleDir(modPath, modVersion), f.Name()))
 			require.NoError(t, err)
-			contents, err := os.ReadFile(path.Join(testDir, f.Name()))
+			contents, err := os.ReadFile(filepath.Join(testDir, f.Name()))
 			require.NoError(t, err)
 			_, err = writer.Write(contents)
 			require.NoError(t, err)
@@ -136,7 +140,8 @@ func Test_RemoteProxyLicenseSearch(t *testing.T) {
 				Value:          "Apache-2.0",
 				SPDXExpression: "Apache-2.0",
 				Type:           license.Concluded,
-				Locations:      file.NewLocationSet(loc1),
+				URLs:           []string{server.URL + "/github.com/someorg/somename/@v/v0.3.2.zip#" + loc1.RealPath},
+				Locations:      file.NewLocationSet(),
 			},
 		},
 		{
@@ -146,21 +151,20 @@ func Test_RemoteProxyLicenseSearch(t *testing.T) {
 				Value:          "MIT",
 				SPDXExpression: "MIT",
 				Type:           license.Concluded,
-				Locations:      file.NewLocationSet(loc2),
+				URLs:           []string{server.URL + "/github.com/CapORG/CapProject/@v/v4.111.5.zip#" + loc2.RealPath},
+				Locations:      file.NewLocationSet(),
 			},
 		},
 	}
 
-	modDir := path.Join(t.TempDir())
-
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			l := newGoLicenses(
+
+			l := newGoLicenseResolver(
 				"",
 				CatalogerConfig{
 					SearchRemoteLicenses: true,
 					Proxies:              []string{server.URL},
-					LocalModCacheDir:     modDir,
 				},
 			)
 

syft/pkg/cataloger/golang/package.go

@@ -15,7 +15,7 @@ func (c *goBinaryCataloger) newGoBinaryPackage(resolver file.Resolver, dep *debu
 		dep = dep.Replace
 	}
 
-	licenses, err := c.licenses.getLicenses(resolver, dep.Path, dep.Version)
+	licenses, err := c.licenseResolver.getLicenses(resolver, dep.Path, dep.Version)
 	if err != nil {
 		log.Tracef("error getting licenses for golang package: %s %v", dep.Path, err)
 	}

syft/pkg/cataloger/golang/parse_go_binary.go

@@ -38,21 +38,21 @@ var (
 	// inject the correct version into the main module of the build process
 
 	knownBuildFlagPatterns = []*regexp.Regexp{
-		regexp.MustCompile(`(?m)\.([gG]it)?([bB]uild)?[vV]er(sion)?=(\S+/)*(?P<version>v?\d+.\d+.\d+[-\w]*)`),
-		regexp.MustCompile(`(?m)\.([tT]ag)=(\S+/)*(?P<version>v?\d+.\d+.\d+[-\w]*)`),
+		regexp.MustCompile(`(?m)\.[a-zA-Z0-9]*([rR]elease)?([gG]it)?([bB]uild)?[vV]er(sion)?=(\S+/)*(?P<version>v?\d+.\d+.\d+[-\w]*)`),
+		regexp.MustCompile(`(?m)\.[a-zA-Z0-9]*([tT]ag)=(\S+/)*(?P<version>v?\d+.\d+.\d+[-\w]*)`),
 	}
 )
 
 const devel = "(devel)"
 
 type goBinaryCataloger struct {
-	licenses          goLicenses
+	licenseResolver   goLicenseResolver
 	mainModuleVersion MainModuleVersionConfig
 }
 
 func newGoBinaryCataloger(opts CatalogerConfig) *goBinaryCataloger {
 	return &goBinaryCataloger{
-		licenses:          newGoLicenses(binaryCatalogerName, opts),
+		licenseResolver:   newGoLicenseResolver(binaryCatalogerName, opts),
 		mainModuleVersion: opts.MainModuleVersion,
 	}
 }

syft/pkg/cataloger/golang/parse_go_binary_test.go

@@ -1080,6 +1080,18 @@ func Test_extractVersionFromLDFlags(t *testing.T) {
 			wantMajorVersion: "4",
 			wantFullVersion:  "v4.5.7",
 		},
+		{
+			name:             "TiDB 7.5.0 ldflags",
+			ldflags:          `build	-ldflags="-X \"github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.0\" -X \"github.com/pingcap/tidb/pkg/util/versioninfo.TiDBBuildTS=2023-11-24 08:51:04\" -X \"github.com/pingcap/tidb/pkg/util/versioninfo.TiDBGitHash=069631e2ecfedc000ffb92c67207bea81380f020\" -X \"github.com/pingcap/tidb/pkg/util/versioninfo.TiDBGitBranch=heads/refs/tags/v7.5.0\" -X \"github.com/pingcap/tidb/pkg/util/versioninfo.TiDBEdition=Community\" "`,
+			wantMajorVersion: "7",
+			wantFullVersion:  "v7.5.0",
+		},
+		{
+			name:             "TiDB 6.1.7 ldflags",
+			ldflags:          `build	-ldflags="-X \"github.com/pingcap/tidb/parser/mysql.TiDBReleaseVersion=v6.1.7\" -X \"github.com/pingcap/tidb/util/versioninfo.TiDBBuildTS=2023-07-04 12:06:03\" -X \"github.com/pingcap/tidb/util/versioninfo.TiDBGitHash=613ecc5f731b2843e1d53a43915e2cd8da795936\" -X \"github.com/pingcap/tidb/util/versioninfo.TiDBGitBranch=heads/refs/tags/v6.1.7\" -X \"github.com/pingcap/tidb/util/versioninfo.TiDBEdition=Community\" "`,
+			wantMajorVersion: "6",
+			wantFullVersion:  "v6.1.7",
+		},
 		//////////////////////////////////////////////////////////////////
 		// negative cases
 		{

syft/pkg/cataloger/golang/parse_go_mod.go

@@ -19,7 +19,13 @@ import (
 )
 
 type goModCataloger struct {
-	licenses goLicenses
+	licenseResolver goLicenseResolver
+}
+
+func newGoModCataloger(opts CatalogerConfig) *goModCataloger {
+	return &goModCataloger{
+		licenseResolver: newGoLicenseResolver(modFileCatalogerName, opts),
+	}
 }
 
 // parseGoModFile takes a go.mod and lists all packages discovered.
@@ -44,7 +50,7 @@ func (c *goModCataloger) parseGoModFile(_ context.Context, resolver file.Resolve
 	}
 
 	for _, m := range f.Require {
-		licenses, err := c.licenses.getLicenses(resolver, m.Mod.Path, m.Mod.Version)
+		licenses, err := c.licenseResolver.getLicenses(resolver, m.Mod.Path, m.Mod.Version)
 		if err != nil {
 			log.Tracef("error getting licenses for package: %s %v", m.Mod.Path, err)
 		}
@@ -65,7 +71,7 @@ func (c *goModCataloger) parseGoModFile(_ context.Context, resolver file.Resolve
 
 	// remove any old packages and replace with new ones...
 	for _, m := range f.Replace {
-		licenses, err := c.licenses.getLicenses(resolver, m.New.Path, m.New.Version)
+		licenses, err := c.licenseResolver.getLicenses(resolver, m.New.Path, m.New.Version)
 		if err != nil {
 			log.Tracef("error getting licenses for package: %s %v", m.New.Path, err)
 		}

syft/pkg/cataloger/golang/parse_go_mod_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/anchore/syft/syft/file"
+	"github.com/anchore/syft/syft/internal/fileresolver"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
 )
@@ -91,10 +92,11 @@ func TestParseGoMod(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.fixture, func(t *testing.T) {
-			c := goModCataloger{}
+			c := newGoModCataloger(DefaultCatalogerConfig())
 			pkgtest.NewCatalogTester().
 				FromFile(t, test.fixture).
 				Expects(test.expected, nil).
+				WithResolver(fileresolver.Empty{}).
 				TestParser(t, c.parseGoModFile)
 		})
 	}

syft/pkg/cataloger/internal/cpegenerate/dictionary/index-generator/generate.go

@@ -102,20 +102,29 @@ func normalizeCPE(cpe *wfn.Attributes) *wfn.Attributes {
 }
 
 const (
-	prefixForNPMPackages          = "https://www.npmjs.com/package/"
-	prefixForRubyGems             = "https://rubygems.org/gems/"
-	prefixForRubyGemsHTTP         = "http://rubygems.org/gems/"
-	prefixForNativeRubyGems       = "https://github.com/ruby/"
-	prefixForPyPIPackages         = "https://pypi.org/project/"
-	prefixForJenkinsPlugins       = "https://plugins.jenkins.io/"
-	prefixForJenkinsPluginsGitHub = "https://github.com/jenkinsci/"
-	prefixForRustCrates           = "https://crates.io/crates/"
-	prefixForPHPPear              = "https://pear.php.net/"
-	prefixForPHPPearHTTP          = "http://pear.php.net/"
-	prefixForPHPPecl              = "https://pecl.php.net/"
-	prefixForPHPPeclHTTP          = "http://pecl.php.net/"
-	prefixForPHPComposer          = "https://packagist.org/packages/"
-	prefixForGoModules            = "https://pkg.go.dev/"
+	prefixForNPMPackages                   = "https://www.npmjs.com/package/"
+	prefixForRubyGems                      = "https://rubygems.org/gems/"
+	prefixForRubyGemsHTTP                  = "http://rubygems.org/gems/"
+	prefixForNativeRubyGems                = "https://github.com/ruby/"
+	prefixForPyPIPackages                  = "https://pypi.org/project/"
+	prefixForJenkinsPlugins                = "https://plugins.jenkins.io/"
+	prefixForJenkinsPluginsGitHub          = "https://github.com/jenkinsci/"
+	prefixForRustCrates                    = "https://crates.io/crates/"
+	prefixForPHPPear                       = "https://pear.php.net/"
+	prefixForPHPPearHTTP                   = "http://pear.php.net/"
+	prefixForPHPPecl                       = "https://pecl.php.net/"
+	prefixForPHPPeclHTTP                   = "http://pecl.php.net/"
+	prefixForPHPComposer                   = "https://packagist.org/packages/"
+	prefixForGoModules                     = "https://pkg.go.dev/"
+	prefixForWordpressPlugins              = "https://wordpress.org/plugins/"
+	prefixForWordpressPluginsTracBrowser   = "https://plugins.trac.wordpress.org/browser/"
+	prefixForWordpressPluginsTracLog       = "https://plugins.trac.wordpress.org/log/"
+	prefixForWordpressPluginsGitHubArchive = "https://github.com/wp-plugins/"
+	prefixForWordpressPluginsWordfence     = "https://www.wordfence.com/threat-intel/vulnerabilities/wordpress-plugins/"
+	prefixForWordpressThemes               = "https://wordpress.org/themes/"
+	prefixForWordpressThemesTracBrowser    = "https://themes.trac.wordpress.org/browser/"
+	prefixForWordpressThemesTracLog        = "https://themes.trac.wordpress.org/log/"
+	prefixForWordpressThemesWordfence      = "https://www.wordfence.com/threat-intel/vulnerabilities/wordpress-themes/"
 )
 
 // indexCPEList creates an index of CPEs by ecosystem.
@@ -164,6 +173,12 @@ func indexCPEList(list CpeList) *dictionary.Indexed {
 
 			case strings.HasPrefix(ref, prefixForGoModules):
 				addEntryForGoModulePackage(indexed, ref, cpeItemName)
+
+			case strings.HasPrefix(ref, prefixForWordpressPlugins), strings.HasPrefix(ref, prefixForWordpressPluginsTracBrowser), strings.HasPrefix(ref, prefixForWordpressPluginsTracLog), strings.HasPrefix(ref, prefixForWordpressPluginsGitHubArchive), strings.HasPrefix(ref, prefixForWordpressPluginsWordfence):
+				addEntryForWordpressPlugin(indexed, ref, cpeItemName)
+
+			case strings.HasPrefix(ref, prefixForWordpressThemes), strings.HasPrefix(ref, prefixForWordpressThemesTracBrowser), strings.HasPrefix(ref, prefixForWordpressThemesTracLog), strings.HasPrefix(ref, prefixForWordpressThemesWordfence):
+				addEntryForWordpressTheme(indexed, ref, cpeItemName)
 			}
 		}
 	}
@@ -183,6 +198,37 @@ func updateIndex(indexed *dictionary.Indexed, ecosystem string, pkgName string,
 	indexed.EcosystemPackages[ecosystem][pkgName].Add(cpe)
 }
 
+func addEntryForWordpressPlugin(indexed *dictionary.Indexed, ref string, cpeItemName string) {
+	// Prune off the non-package-name parts of the URL
+	ref = strings.TrimPrefix(ref, prefixForWordpressPlugins)
+	ref = strings.TrimPrefix(ref, prefixForWordpressPluginsTracBrowser)
+	ref = strings.TrimPrefix(ref, prefixForWordpressPluginsTracLog)
+	ref = strings.TrimPrefix(ref, prefixForWordpressPluginsGitHubArchive)
+	ref = strings.TrimPrefix(ref, prefixForWordpressPluginsWordfence)
+	ref = strings.Split(ref, "?")[0]
+	ref = strings.Split(ref, "/")[0]
+	if ref == "" {
+		return
+	}
+
+	updateIndex(indexed, dictionary.EcosystemWordpressPlugins, ref, cpeItemName)
+}
+
+func addEntryForWordpressTheme(indexed *dictionary.Indexed, ref string, cpeItemName string) {
+	// Prune off the non-package-name parts of the URL
+	ref = strings.TrimPrefix(ref, prefixForWordpressThemes)
+	ref = strings.TrimPrefix(ref, prefixForWordpressThemesTracBrowser)
+	ref = strings.TrimPrefix(ref, prefixForWordpressThemesTracLog)
+	ref = strings.TrimPrefix(ref, prefixForWordpressThemesWordfence)
+	ref = strings.Split(ref, "?")[0]
+	ref = strings.Split(ref, "/")[0]
+	if ref == "" {
+		return
+	}
+
+	updateIndex(indexed, dictionary.EcosystemWordpressThemes, ref, cpeItemName)
+}
+
 func addEntryForRustCrate(indexed *dictionary.Indexed, ref string, cpeItemName string) {
 	// Prune off the non-package-name parts of the URL
 	ref = strings.TrimPrefix(ref, prefixForRustCrates)

syft/pkg/cataloger/internal/cpegenerate/dictionary/index-generator/generate_test.go

@@ -243,6 +243,123 @@ func Test_addEntryFuncs(t *testing.T) {
 				},
 			},
 		},
+		{
+			name:             "addEntryForWordpressPlugin",
+			addEntryFunc:     addEntryForWordpressPlugin,
+			inputRef:         "https://wordpress.org/plugins/armadillo/releases",
+			inputCpeItemName: "cpe:2.3:a:armadillo:armadillo:1.23:*:*:*:*:wordpress:*:*",
+			expectedIndexed: dictionary.Indexed{
+				EcosystemPackages: map[string]dictionary.Packages{
+					dictionary.EcosystemWordpressPlugins: {
+						"armadillo": dictionary.NewSet("cpe:2.3:a:armadillo:armadillo:1.23:*:*:*:*:wordpress:*:*"),
+					},
+				},
+			},
+		},
+		{
+			name:             "addEntryForWordpressPlugin from Trac Browser",
+			addEntryFunc:     addEntryForWordpressPlugin,
+			inputRef:         "https://plugins.trac.wordpress.org/browser/armadillo/something",
+			inputCpeItemName: "cpe:2.3:a:armadillo:armadillo:1.23:*:*:*:*:wordpress:*:*",
+			expectedIndexed: dictionary.Indexed{
+				EcosystemPackages: map[string]dictionary.Packages{
+					dictionary.EcosystemWordpressPlugins: {
+						"armadillo": dictionary.NewSet("cpe:2.3:a:armadillo:armadillo:1.23:*:*:*:*:wordpress:*:*"),
+					},
+				},
+			},
+		},
+		{
+			name:             "addEntryForWordpressPlugin from Trac Log",
+			addEntryFunc:     addEntryForWordpressPlugin,
+			inputRef:         "https://plugins.trac.wordpress.org/log/armadillo/log",
+			inputCpeItemName: "cpe:2.3:a:armadillo:armadillo:1.23:*:*:*:*:wordpress:*:*",
+			expectedIndexed: dictionary.Indexed{
+				EcosystemPackages: map[string]dictionary.Packages{
+					dictionary.EcosystemWordpressPlugins: {
+						"armadillo": dictionary.NewSet("cpe:2.3:a:armadillo:armadillo:1.23:*:*:*:*:wordpress:*:*"),
+					},
+				},
+			},
+		},
+		{
+			name:             "addEntryForWordpressPlugin from GitHub wp-plugins archive",
+			addEntryFunc:     addEntryForWordpressPlugin,
+			inputRef:         "https://github.com/wp-plugins/armadillo/something",
+			inputCpeItemName: "cpe:2.3:a:armadillo:armadillo:1.23:*:*:*:*:wordpress:*:*",
+			expectedIndexed: dictionary.Indexed{
+				EcosystemPackages: map[string]dictionary.Packages{
+					dictionary.EcosystemWordpressPlugins: {
+						"armadillo": dictionary.NewSet("cpe:2.3:a:armadillo:armadillo:1.23:*:*:*:*:wordpress:*:*"),
+					},
+				},
+			},
+		},
+		{
+			name:             "addEntryForWordpressPlugin wordfence",
+			addEntryFunc:     addEntryForWordpressPlugin,
+			inputRef:         "https://www.wordfence.com/threat-intel/vulnerabilities/wordpress-plugins/armadillo/skjfhskdjhf/12344",
+			inputCpeItemName: "cpe:2.3:a:armadillo:armadillo:1.23:*:*:*:*:wordpress:*:*",
+			expectedIndexed: dictionary.Indexed{
+				EcosystemPackages: map[string]dictionary.Packages{
+					dictionary.EcosystemWordpressPlugins: {
+						"armadillo": dictionary.NewSet("cpe:2.3:a:armadillo:armadillo:1.23:*:*:*:*:wordpress:*:*"),
+					},
+				},
+			},
+		},
+		{
+			name:             "addEntryForWordpressTheme",
+			addEntryFunc:     addEntryForWordpressTheme,
+			inputRef:         "https://wordpress.org/themes/basic/releases",
+			inputCpeItemName: "cpe:2.3:a:basic:basic:1.23:*:*:*:*:wordpress:*:*",
+			expectedIndexed: dictionary.Indexed{
+				EcosystemPackages: map[string]dictionary.Packages{
+					dictionary.EcosystemWordpressThemes: {
+						"basic": dictionary.NewSet("cpe:2.3:a:basic:basic:1.23:*:*:*:*:wordpress:*:*"),
+					},
+				},
+			},
+		},
+		{
+			name:             "addEntryForWordpressTheme from Trac Browser",
+			addEntryFunc:     addEntryForWordpressTheme,
+			inputRef:         "https://themes.trac.wordpress.org/browser/basic/something",
+			inputCpeItemName: "cpe:2.3:a:basic:basic:1.23:*:*:*:*:wordpress:*:*",
+			expectedIndexed: dictionary.Indexed{
+				EcosystemPackages: map[string]dictionary.Packages{
+					dictionary.EcosystemWordpressThemes: {
+						"basic": dictionary.NewSet("cpe:2.3:a:basic:basic:1.23:*:*:*:*:wordpress:*:*"),
+					},
+				},
+			},
+		},
+		{
+			name:             "addEntryForWordpressTheme from Trac Log",
+			addEntryFunc:     addEntryForWordpressTheme,
+			inputRef:         "https://themes.trac.wordpress.org/log/basic/log",
+			inputCpeItemName: "cpe:2.3:a:basic:basic:1.23:*:*:*:*:wordpress:*:*",
+			expectedIndexed: dictionary.Indexed{
+				EcosystemPackages: map[string]dictionary.Packages{
+					dictionary.EcosystemWordpressThemes: {
+						"basic": dictionary.NewSet("cpe:2.3:a:basic:basic:1.23:*:*:*:*:wordpress:*:*"),
+					},
+				},
+			},
+		},
+		{
+			name:             "addEntryForWordpressTheme wordfence",
+			addEntryFunc:     addEntryForWordpressTheme,
+			inputRef:         "https://www.wordfence.com/threat-intel/vulnerabilities/wordpress-themes/basic/skjfhskdjhf/12344",
+			inputCpeItemName: "cpe:2.3:a:basic:basic:1.23:*:*:*:*:wordpress:*:*",
+			expectedIndexed: dictionary.Indexed{
+				EcosystemPackages: map[string]dictionary.Packages{
+					dictionary.EcosystemWordpressThemes: {
+						"basic": dictionary.NewSet("cpe:2.3:a:basic:basic:1.23:*:*:*:*:wordpress:*:*"),
+					},
+				},
+			},
+		},
 	}
 
 	for _, tt := range tests {

syft/pkg/cataloger/internal/cpegenerate/dictionary/index-generator/testdata/expected-cpe-index.json

@@ -106,6 +106,58 @@
       "unicycle": [
         "cpe:2.3:a:unicycle_project:unicycle:*:*:*:*:*:rust:*:*"
       ]
+    },
+    "wordpress_plugins": {
+      "0mk-shortener": [
+        "cpe:2.3:a:0mk_shortener_project:0mk_shortener:*:*:*:*:*:wordpress:*:*"
+      ],
+      "jobboardwp": [
+        "cpe:2.3:a:ultimatemember:jobboardwp:*:*:*:*:*:wordpress:*:*"
+      ],
+      "nofollow": [
+        "cpe:2.3:a:ultimate_nofollow_project:ultimate_nofollow:*:*:*:*:*:wordpress:*:*"
+      ],
+      "petfinder-listings": [
+        "cpe:2.3:a:unboxinteractive:petfinder-listings:*:*:*:*:*:wordpress:*:*"
+      ],
+      "ultimate-instagram-feed": [
+        "cpe:2.3:a:ultimate_instagram_feed_project:ultimate_instagram_feed:*:*:*:*:*:wordpress:*:*"
+      ],
+      "ultimate-member": [
+        "cpe:2.3:a:ultimatemember:ultimate_member:*:*:*:*:*:wordpress:*:*",
+        "cpe:2.3:a:ultimatemember:user_profile_\\\u0026_membership:*:*:*:*:*:wordpress:*:*"
+      ],
+      "ultimate-noindex-nofollow-tool-ii": [
+        "cpe:2.3:a:ultimate_noindex_nofollow_tool_ii_project:ultimate_noindex_nofollow_tool_ii:*:*:*:*:*:wordpress:*:*"
+      ],
+      "ultimate-sms-notifications": [
+        "cpe:2.3:a:ultimatesmsnotifications:ultimate_sms_notifications_for_woocommerce:*:*:*:*:*:wordpress:*:*"
+      ],
+      "ultimate-weather-plugin": [
+        "cpe:2.3:a:ultimate-weather_project:ultimate-weather:*:*:*:*:*:wordpress:*:*"
+      ],
+      "ultimate-wp-query-search-filter": [
+        "cpe:2.3:a:ultimate_wp_query_search_filter_project:ultimate_wp_query_search_filter:*:*:*:*:*:wordpress:*:*"
+      ],
+      "uncanny-automator": [
+        "cpe:2.3:a:uncannyowl:uncanny_automator:*:*:*:*:*:wordpress:*:*"
+      ],
+      "uncanny-learndash-toolkit": [
+        "cpe:2.3:a:uncannyowl:uncanny_toolkit_for_learndash:*:*:*:*:*:*:*:*",
+        "cpe:2.3:a:uncannyowl:uncanny_toolkit_for_learndash:*:*:*:*:*:wordpress:*:*"
+      ],
+      "unconfirmed": [
+        "cpe:2.3:a:unconfirmed_project:unconfirmed:*:*:*:*:*:wordpress:*:*"
+      ],
+      "underconstruction": [
+        "cpe:2.3:a:underconstruction_project:underconstruction:*:*:*:*:*:wordpress:*:*"
+      ],
+      "wp-cleanfix": [
+        "cpe:2.3:a:undolog:wp_cleanfix:*:*:*:*:*:wordpress:*:*"
+      ],
+      "wp-ultra-simple-paypal-shopping-cart": [
+        "cpe:2.3:a:ultra-prod:wordpress_ultra_simple_paypal_shopping_cart:*:*:*:*:*:wordpress:*:*"
+      ]
     }
   }
 }

syft/pkg/cataloger/internal/cpegenerate/dictionary/index-generator/testdata/official-cpe-dictionary_v2.3.xml

@@ -25093,6 +25093,14 @@
     </references>
     <cpe-23:cpe23-item name="cpe:2.3:a:yaml_project:yaml:2.3.0:*:*:*:*:go:*:*"/>
   </cpe-item>
+  <cpe-item name="cpe:/a:0mk_shortener_project:0mk_shortener:0.2::~~~wordpress~~">
+    <title xml:lang="en-US">0mk Shortener Project 0mk Shortener 0.2 for WordPress</title>
+    <references>
+      <reference href="https://www.wordfence.com/threat-intel/vulnerabilities/wordpress-plugins/0mk-shortener/0mk-shortener-02-cross-site-request-forgery-to-stored-cross-site-scripting">Advisory</reference>
+      <reference href="https://wordpress.org/plugins/0mk-shortener/">Project</reference>
+    </references>
+    <cpe-23:cpe23-item name="cpe:2.3:a:0mk_shortener_project:0mk_shortener:0.2:*:*:*:*:wordpress:*:*"/>
+  </cpe-item>
 </cpe-list>
 </cpe-list>
 </cpe-list>

syft/pkg/cataloger/internal/cpegenerate/dictionary/types.go

@@ -8,15 +8,17 @@ import (
 )
 
 const (
-	EcosystemNPM            = "npm"
-	EcosystemRubyGems       = "rubygems"
-	EcosystemPyPI           = "pypi"
-	EcosystemPHPPear        = "php_pear"
-	EcosystemPHPPecl        = "php_pecl"
-	EcosystemPHPComposer    = "php_composer"
-	EcosystemJenkinsPlugins = "jenkins_plugins"
-	EcosystemRustCrates     = "rust_crates"
-	EcosystemGoModules      = "go_modules"
+	EcosystemNPM              = "npm"
+	EcosystemRubyGems         = "rubygems"
+	EcosystemPyPI             = "pypi"
+	EcosystemPHPPear          = "php_pear"
+	EcosystemPHPPecl          = "php_pecl"
+	EcosystemPHPComposer      = "php_composer"
+	EcosystemJenkinsPlugins   = "jenkins_plugins"
+	EcosystemRustCrates       = "rust_crates"
+	EcosystemGoModules        = "go_modules"
+	EcosystemWordpressPlugins = "wordpress_plugins"
+	EcosystemWordpressThemes  = "wordpress_themes"
 )
 
 type Indexed struct {

syft/pkg/cataloger/internal/cpegenerate/generate.go

@@ -96,6 +96,13 @@ func FromDictionaryFind(p pkg.Package) ([]cpe.CPE, bool) {
 	case pkg.GoModulePkg:
 		cpes, ok = dict.EcosystemPackages[dictionary.EcosystemGoModules][p.Name]
 
+	case pkg.WordpressPluginPkg:
+		metadata, valid := p.Metadata.(pkg.WordpressPluginEntry)
+		if !valid {
+			return parsedCPEs, false
+		}
+		cpes, ok = dict.EcosystemPackages[dictionary.EcosystemWordpressPlugins][metadata.PluginInstallDirectory]
+
 	default:
 		// The dictionary doesn't support this package type yet.
 		return parsedCPEs, false
@@ -129,23 +136,26 @@ func FromDictionaryFind(p pkg.Package) ([]cpe.CPE, bool) {
 func FromPackageAttributes(p pkg.Package) []cpe.CPE {
 	vendors := candidateVendors(p)
 	products := candidateProducts(p)
+	targetSWs := candidateTargetSw(p)
 	if len(products) == 0 {
 		return nil
 	}
 
 	keys := strset.New()
 	cpes := make([]cpe.Attributes, 0)
-	for _, product := range products {
-		for _, vendor := range vendors {
-			// prevent duplicate entries...
-			key := fmt.Sprintf("%s|%s|%s", product, vendor, p.Version)
-			if keys.Has(key) {
-				continue
-			}
-			keys.Add(key)
-			// add a new entry...
-			if c := newCPE(product, vendor, p.Version, cpe.Any); c != nil {
-				cpes = append(cpes, *c)
+	for _, ts := range targetSWs {
+		for _, product := range products {
+			for _, vendor := range vendors {
+				// prevent duplicate entries...
+				key := fmt.Sprintf("%s|%s|%s|%s", product, vendor, p.Version, ts)
+				if keys.Has(key) {
+					continue
+				}
+				keys.Add(key)
+				// add a new entry...
+				if c := newCPE(product, vendor, p.Version, ts); c != nil {
+					cpes = append(cpes, *c)
+				}
 			}
 		}
 	}
@@ -162,6 +172,13 @@ func FromPackageAttributes(p pkg.Package) []cpe.CPE {
 	return result
 }
 
+func candidateTargetSw(p pkg.Package) []string {
+	if p.Type == pkg.WordpressPluginPkg {
+		return []string{"wordpress"}
+	}
+	return []string{cpe.Any}
+}
+
 //nolint:funlen
 func candidateVendors(p pkg.Package) []string {
 	// in ecosystems where the packaging metadata does not have a clear field to indicate a vendor (or a field that

syft/pkg/cataloger/internal/cpegenerate/generate_test.go

@@ -719,6 +719,31 @@ func TestGeneratePackageCPEs(t *testing.T) {
 				"cpe:2.3:a:ruby_rake:ruby_rake:2.7.6-r0:*:*:*:*:*:*:*",
 			},
 		},
+		{
+			name: "wordpress plugin",
+			p: pkg.Package{
+				Name:    "WP Coder",
+				Version: "2.5.1",
+				Type:    pkg.WordpressPluginPkg,
+				Metadata: pkg.WordpressPluginEntry{
+					PluginInstallDirectory: "wp-coder",
+					Author:                 "Wow-Company",
+					AuthorURI:              "https://wow-estore.com",
+				},
+			},
+			expected: []string{
+				"cpe:2.3:a:wow-company:wp-coder:2.5.1:*:*:*:*:wordpress:*:*",
+				"cpe:2.3:a:wow-company:wp_coder:2.5.1:*:*:*:*:wordpress:*:*", // this is the correct CPE relative to CVE-2021-25053
+				"cpe:2.3:a:wow-estore:wp-coder:2.5.1:*:*:*:*:wordpress:*:*",
+				"cpe:2.3:a:wow-estore:wp_coder:2.5.1:*:*:*:*:wordpress:*:*",
+				"cpe:2.3:a:wow:wp-coder:2.5.1:*:*:*:*:wordpress:*:*",
+				"cpe:2.3:a:wow:wp_coder:2.5.1:*:*:*:*:wordpress:*:*",
+				"cpe:2.3:a:wow_company:wp-coder:2.5.1:*:*:*:*:wordpress:*:*",
+				"cpe:2.3:a:wow_company:wp_coder:2.5.1:*:*:*:*:wordpress:*:*",
+				"cpe:2.3:a:wow_estore:wp-coder:2.5.1:*:*:*:*:wordpress:*:*",
+				"cpe:2.3:a:wow_estore:wp_coder:2.5.1:*:*:*:*:wordpress:*:*",
+			},
+		},
 	}
 
 	for _, test := range tests {

syft/pkg/cataloger/internal/cpegenerate/wordpress.go

@@ -21,12 +21,18 @@ func candidateVendorsForWordpressPlugin(p pkg.Package) fieldCandidateSet {
 
 	vendors := newFieldCandidateSet()
 
+	if metadata.Author != "" {
+		vendors.addValue(strings.ToLower(metadata.Author))
+	}
+
 	if metadata.AuthorURI != "" {
 		matchMap := internal.MatchNamedCaptureGroups(vendorFromURLRegexp, metadata.AuthorURI)
 		if vendor, ok := matchMap["vendor"]; ok && vendor != "" {
-			vendors.addValue(vendor)
+			vendors.addValue(strings.ToLower(vendor))
 		}
-	} else {
+	}
+
+	if len(vendors) == 0 {
 		// add plugin_name + _project as a vendor if no Author URI found
 		vendors.addValue(fmt.Sprintf("%s_project", normalizeWordpressPluginName(p.Name)))
 	}

syft/pkg/cataloger/internal/cpegenerate/wordpress_test.go

@@ -24,7 +24,7 @@ func Test_candidateVendorsForWordpressPlugin(t *testing.T) {
 					AuthorURI:              "https://automattic.com/wordpress-plugins/",
 				},
 			},
-			expected: []string{"automattic"},
+			expected: []string{"automattic - anti-spam team", "automattic"},
 		},
 		{
 			name: "All-in-One WP Migration",
@@ -47,7 +47,7 @@ func Test_candidateVendorsForWordpressPlugin(t *testing.T) {
 					AuthorURI:              "https://bookingultrapro.com/",
 				},
 			},
-			expected: []string{"bookingultrapro"},
+			expected: []string{"booking ultra pro", "bookingultrapro"},
 		},
 		{
 			name: "Coming Soon Chop Chop",
@@ -58,7 +58,7 @@ func Test_candidateVendorsForWordpressPlugin(t *testing.T) {
 					AuthorURI:              "https://www.chop-chop.org",
 				},
 			},
-			expected: []string{"chop-chop"},
+			expected: []string{"chop-chop.org", "chop-chop"},
 		},
 		{
 			name: "Access Code Feeder",
@@ -71,6 +71,22 @@ func Test_candidateVendorsForWordpressPlugin(t *testing.T) {
 			// When a plugin as no `Author URI` use plugin_name + _project as a vendor
 			expected: []string{"access_code_feeder_project"},
 		},
+		{
+			name: "WP Coder",
+			pkg: pkg.Package{
+				Name: "WP Coder",
+				Metadata: pkg.WordpressPluginEntry{
+					PluginInstallDirectory: "wp-coder",
+					Author:                 "Wow-Company",
+					AuthorURI:              "https://wow-estore.com/",
+				},
+			},
+			// found in the wild https://plugins.trac.wordpress.org/browser/wp-coder/tags/2.5.1/wp-coder.php
+			expected: []string{
+				"wow-company", // this is the correct answer relative to CPEs registered on CVE-2021-25053
+				"wow-estore",
+			},
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {

syft/pkg/cataloger/python/parse_poetry_lock.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"fmt"
 	"sort"
-	"strings"
 
-	"github.com/pelletier/go-toml"
+	"github.com/BurntSushi/toml"
 
+	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
@@ -19,7 +19,9 @@ import (
 var _ generic.Parser = parsePoetryLock
 
 type poetryPackageSource struct {
-	URL string `toml:"url"`
+	URL       string `toml:"url"`
+	Type      string `toml:"type"`
+	Reference string `toml:"reference"`
 }
 
 type poetryPackages struct {
@@ -27,14 +29,15 @@ type poetryPackages struct {
 }
 
 type poetryPackage struct {
-	Name         string                             `toml:"name"`
-	Version      string                             `toml:"version"`
-	Category     string                             `toml:"category"`
-	Description  string                             `toml:"description"`
-	Optional     bool                               `toml:"optional"`
-	Source       poetryPackageSource                `toml:"source"`
-	Dependencies map[string]poetryPackageDependency `toml:"dependencies"`
-	Extras       map[string][]string                `toml:"extras"`
+	Name                  string                    `toml:"name"`
+	Version               string                    `toml:"version"`
+	Category              string                    `toml:"category"`
+	Description           string                    `toml:"description"`
+	Optional              bool                      `toml:"optional"`
+	Source                poetryPackageSource       `toml:"source"`
+	DependenciesUnmarshal map[string]toml.Primitive `toml:"dependencies"`
+	Extras                map[string][]string       `toml:"extras"`
+	Dependencies          map[string][]poetryPackageDependency
 }
 
 type poetryPackageDependency struct {
@@ -44,41 +47,6 @@ type poetryPackageDependency struct {
 	Extras   []string `toml:"extras"`
 }
 
-func (d *poetryPackageDependency) UnmarshalText(data []byte) error {
-	// attempt to parse as a map first
-	var dep map[string]interface{}
-	if err := toml.Unmarshal(data, &dep); err == nil {
-		if extras, ok := dep["extras"]; ok {
-			if extrasList, ok := extras.([]string); ok {
-				d.Extras = extrasList
-			}
-		}
-
-		if markers, ok := dep["markers"]; ok {
-			if markersString, ok := markers.(string); ok {
-				d.Markers = markersString
-			}
-		}
-
-		if version, ok := dep["version"]; ok {
-			if versionString, ok := version.(string); ok {
-				d.Version = versionString
-			}
-		}
-		return nil
-	}
-
-	if strings.ContainsAny(string(data), "[]{}") {
-		// odds are this is really a malformed toml array or object
-		return fmt.Errorf("unable to parse poetry dependency: version is malformed array/object: %q", string(data))
-	}
-
-	// assume this is a simple version string
-	d.Version = string(data)
-
-	return nil
-}
-
 // parsePoetryLock is a parser function for poetry.lock contents, returning all python packages discovered.
 func parsePoetryLock(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
 	pkgs, err := poetryLockPackages(reader)
@@ -93,15 +61,33 @@ func parsePoetryLock(_ context.Context, _ file.Resolver, _ *generic.Environment,
 }
 
 func poetryLockPackages(reader file.LocationReadCloser) ([]pkg.Package, error) {
-	tree, err := toml.LoadReader(reader)
+	metadata := poetryPackages{}
+	md, err := toml.NewDecoder(reader).Decode(&metadata)
 	if err != nil {
-		return nil, fmt.Errorf("unable to load poetry.lock for parsing: %w", err)
+		return nil, fmt.Errorf("failed to read poetry lock package: %w", err)
 	}
 
-	metadata := poetryPackages{}
-	err = tree.Unmarshal(&metadata)
-	if err != nil {
-		return nil, fmt.Errorf("unable to parse poetry.lock: %w", err)
+	for i, p := range metadata.Packages {
+		dependencies := make(map[string][]poetryPackageDependency)
+		for pkgName, du := range p.DependenciesUnmarshal {
+			var (
+				single    string
+				singleObj poetryPackageDependency
+				multiObj  []poetryPackageDependency
+			)
+
+			switch {
+			case md.PrimitiveDecode(du, &single) == nil:
+				dependencies[pkgName] = append(dependencies[pkgName], poetryPackageDependency{Version: single})
+			case md.PrimitiveDecode(du, &singleObj) == nil:
+				dependencies[pkgName] = append(dependencies[pkgName], singleObj)
+			case md.PrimitiveDecode(du, &multiObj) == nil:
+				dependencies[pkgName] = append(dependencies[pkgName], multiObj...)
+			default:
+				log.Trace("failed to decode poetry lock package dependencies for %s; skipping", pkgName)
+			}
+		}
+		metadata.Packages[i].Dependencies = dependencies
 	}
 
 	var pkgs []pkg.Package
@@ -137,13 +123,15 @@ func extractIndex(p poetryPackage) string {
 
 func extractPoetryDependencies(p poetryPackage) []pkg.PythonPoetryLockDependencyEntry {
 	var deps []pkg.PythonPoetryLockDependencyEntry
-	for name, dep := range p.Dependencies {
-		deps = append(deps, pkg.PythonPoetryLockDependencyEntry{
-			Name:    name,
-			Version: dep.Version,
-			Extras:  dep.Extras,
-			Markers: dep.Markers,
-		})
+	for name, dependencies := range p.Dependencies {
+		for _, d := range dependencies {
+			deps = append(deps, pkg.PythonPoetryLockDependencyEntry{
+				Name:    name,
+				Version: d.Version,
+				Extras:  d.Extras,
+				Markers: d.Markers,
+			})
+		}
 	}
 	sort.Slice(deps, func(i, j int) bool {
 		return deps[i].Name < deps[j].Name

syft/pkg/cataloger/python/parse_poetry_lock_test.go

@@ -24,7 +24,11 @@ func TestParsePoetryLock(t *testing.T) {
 				Index: "https://test.pypi.org/simple",
 				Dependencies: []pkg.PythonPoetryLockDependencyEntry{
 					{Name: "docutils", Version: "*"},
+					{Name: "msal", Version: ">=0.4.1,<2.0.0"},
 					{Name: "natsort", Version: "*"},
+					{Name: "packaging", Version: "*"},
+					{Name: "portalocker", Version: ">=1.0,<3", Markers: `platform_system != "Windows"`},
+					{Name: "portalocker", Version: ">=1.6,<3", Markers: `platform_system == "Windows"`},
 					{Name: "six", Version: "*"},
 					{Name: "sphinx", Version: "*"},
 				},

syft/pkg/cataloger/python/test-fixtures/poetry/dev-deps/poetry.lock

@@ -11,6 +11,13 @@ docutils = "*"
 natsort = "*"
 six = "*"
 sphinx = "*"
+packaging = "*"
+msal = {version = ">=0.4.1,<2.0.0"}
+malformed = [ [ { version = "1.2" } ] ]
+portalocker = [
+    {version = ">=1.0,<3", markers = "platform_system != \"Windows\""},
+    {version = ">=1.6,<3", markers = "platform_system == \"Windows\""},
+]
 
 [package.source]
 type = "legacy"

syft/pkg/cataloger/wordpress/parse_plugin.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"path/filepath"
 	"regexp"
+	"strings"
 
 	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/syft/artifact"
@@ -39,7 +40,6 @@ type pluginData struct {
 
 func parseWordpressPluginFiles(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
 	var pkgs []pkg.Package
-	var fields = make(map[string]interface{})
 	buffer := make([]byte, contentBufferSize)
 
 	_, err := reader.Read(buffer)
@@ -47,13 +47,7 @@ func parseWordpressPluginFiles(_ context.Context, _ file.Resolver, _ *generic.En
 		return nil, nil, fmt.Errorf("failed to read %s file: %w", reader.Location.Path(), err)
 	}
 
-	fileContent := string(buffer)
-	for field, pattern := range patterns {
-		matchMap := internal.MatchNamedCaptureGroups(pattern, fileContent)
-		if value := matchMap[field]; value != "" {
-			fields[field] = value
-		}
-	}
+	fields := extractFields(string(buffer))
 
 	name, nameOk := fields["name"]
 	version, versionOk := fields["version"]
@@ -96,3 +90,15 @@ func parseWordpressPluginFiles(_ context.Context, _ file.Resolver, _ *generic.En
 
 	return pkgs, nil, nil
 }
+
+func extractFields(in string) map[string]any {
+	var fields = make(map[string]interface{})
+
+	for field, pattern := range patterns {
+		matchMap := internal.MatchNamedCaptureGroups(pattern, in)
+		if value := matchMap[field]; value != "" {
+			fields[field] = strings.TrimSpace(value)
+		}
+	}
+	return fields
+}

syft/pkg/cataloger/wordpress/parse_plugin_test.go

@@ -3,6 +3,8 @@ package wordpress
 import (
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
@@ -30,3 +32,28 @@ func TestParseWordpressPluginFiles(t *testing.T) {
 
 	pkgtest.TestFileParser(t, fixture, parseWordpressPluginFiles, []pkg.Package{expectedPkg}, nil)
 }
+
+func Test_extractFields(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want map[string]any
+	}{
+		{
+			name: "carriage returns are stripped",
+			in:   "Plugin Name: WP Migration\r\nVersion: 5.3\r\nLicense: GPLv3\r\nAuthor: MonsterInsights\r\nAuthor URI: https://servmask.com/\r\n",
+			want: map[string]any{
+				"name":       "WP Migration",
+				"version":    "5.3",
+				"license":    "GPLv3",
+				"author":     "MonsterInsights",
+				"author_uri": "https://servmask.com/",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, extractFields(tt.in))
+		})
+	}
+}

syft/source/directorysource/cache_excludes.go

@@ -0,0 +1,33 @@
+package directorysource
+
+import (
+	"os"
+	"strings"
+
+	"github.com/anchore/syft/internal/cache"
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/internal/fileresolver"
+)
+
+// we do not want to cache things and then subsequently scan them, if, for example a user runs `syft /` twice
+func excludeCachePathVisitors() []fileresolver.PathIndexVisitor {
+	var out []fileresolver.PathIndexVisitor
+	for _, dir := range cache.GetManager().RootDirs() {
+		out = append(out, excludeCacheDirPathVisitor{
+			dir: dir,
+		}.excludeCacheDir)
+	}
+	return out
+}
+
+type excludeCacheDirPathVisitor struct {
+	dir string
+}
+
+func (d excludeCacheDirPathVisitor) excludeCacheDir(_, path string, _ os.FileInfo, _ error) error {
+	if strings.HasPrefix(path, d.dir) {
+		log.Tracef("skipping cache path: %s", path)
+		return fileresolver.ErrSkipPath
+	}
+	return nil
+}

syft/source/directorysource/directory_source.go

@@ -142,6 +142,9 @@ func (s *directorySource) FileResolver(_ source.Scope) (file.Resolver, error) {
 			return nil, err
 		}
 
+		// this should be the only file resolver that might have overlap with where files are cached
+		exclusionFunctions = append(exclusionFunctions, excludeCachePathVisitors()...)
+
 		res, err := fileresolver.NewFromDirectory(s.config.Path, s.config.Base, exclusionFunctions...)
 		if err != nil {
 			return nil, fmt.Errorf("unable to create directory resolver: %w", err)