Migrate location-related structs to the file package (#1751)

* migrate location structs to file package

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* replace source.Location refs with file package call

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix linting

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* remove hardlink test for file based catalogers

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* remove hardlink test for all-regular-files testing

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* migrate file resolver implementations to separate package

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix linting

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* [wip] migrate resolvers to internal

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* migrate resolvers to syft/internal

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

---------

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
Signed-off-by: <>

syft/pkg/cataloger/python/cataloger_test.go

@@ -5,9 +5,9 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
-	"github.com/anchore/syft/syft/source"
 )
 
 func Test_PackageCataloger(t *testing.T) {
@@ -46,7 +46,7 @@ func Test_PackageCataloger(t *testing.T) {
 				Type:     pkg.PythonPkg,
 				Language: pkg.Python,
 				Licenses: pkg.NewLicenseSet(
-					pkg.NewLicenseFromLocations("Apache 2.0", source.NewLocation("test-fixtures/egg-info/PKG-INFO")),
+					pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("test-fixtures/egg-info/PKG-INFO")),
 				),
 				FoundBy:      "python-package-cataloger",
 				MetadataType: pkg.PythonPackageMetadataType,
@@ -84,7 +84,7 @@ func Test_PackageCataloger(t *testing.T) {
 				Type:     pkg.PythonPkg,
 				Language: pkg.Python,
 				Licenses: pkg.NewLicenseSet(
-					pkg.NewLicenseFromLocations("BSD License", source.NewLocation("test-fixtures/dist-info/METADATA")),
+					pkg.NewLicenseFromLocations("BSD License", file.NewLocation("test-fixtures/dist-info/METADATA")),
 				),
 				FoundBy:      "python-package-cataloger",
 				MetadataType: pkg.PythonPackageMetadataType,
@@ -122,7 +122,7 @@ func Test_PackageCataloger(t *testing.T) {
 				Type:     pkg.PythonPkg,
 				Language: pkg.Python,
 				Licenses: pkg.NewLicenseSet(
-					pkg.NewLicenseFromLocations("BSD License", source.NewLocation("test-fixtures/malformed-record/dist-info/METADATA")),
+					pkg.NewLicenseFromLocations("BSD License", file.NewLocation("test-fixtures/malformed-record/dist-info/METADATA")),
 				),
 				FoundBy:      "python-package-cataloger",
 				MetadataType: pkg.PythonPackageMetadataType,
@@ -154,7 +154,7 @@ func Test_PackageCataloger(t *testing.T) {
 				Type:     pkg.PythonPkg,
 				Language: pkg.Python,
 				Licenses: pkg.NewLicenseSet(
-					pkg.NewLicenseFromLocations("BSD License", source.NewLocation("test-fixtures/partial.dist-info/METADATA")),
+					pkg.NewLicenseFromLocations("BSD License", file.NewLocation("test-fixtures/partial.dist-info/METADATA")),
 				),
 				FoundBy:      "python-package-cataloger",
 				MetadataType: pkg.PythonPackageMetadataType,
@@ -178,7 +178,7 @@ func Test_PackageCataloger(t *testing.T) {
 				Type:     pkg.PythonPkg,
 				Language: pkg.Python,
 				Licenses: pkg.NewLicenseSet(
-					pkg.NewLicenseFromLocations("Apache 2.0", source.NewLocation("test-fixtures/test.egg-info")),
+					pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("test-fixtures/test.egg-info")),
 				),
 				FoundBy:      "python-package-cataloger",
 				MetadataType: pkg.PythonPackageMetadataType,
@@ -196,12 +196,12 @@ func Test_PackageCataloger(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			resolver := source.NewMockResolverForPaths(test.fixtures...)
+			resolver := file.NewMockResolverForPaths(test.fixtures...)
 
 			locations, err := resolver.FilesByPath(test.fixtures...)
 			require.NoError(t, err)
 
-			test.expectedPackage.Locations = source.NewLocationSet(locations...)
+			test.expectedPackage.Locations = file.NewLocationSet(locations...)
 
 			pkgtest.NewCatalogTester().
 				WithResolver(resolver).
@@ -225,7 +225,7 @@ func Test_PackageCataloger_IgnorePackage(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.MetadataFixture, func(t *testing.T) {
-			resolver := source.NewMockResolverForPaths(test.MetadataFixture)
+			resolver := file.NewMockResolverForPaths(test.MetadataFixture)
 
 			actual, _, err := NewPythonPackageCataloger().Catalog(resolver)
 			require.NoError(t, err)

syft/pkg/cataloger/python/package.go

@@ -4,15 +4,15 @@ import (
 	"fmt"
 
 	"github.com/anchore/packageurl-go"
+	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
-	"github.com/anchore/syft/syft/source"
 )
 
-func newPackageForIndex(name, version string, locations ...source.Location) pkg.Package {
+func newPackageForIndex(name, version string, locations ...file.Location) pkg.Package {
 	p := pkg.Package{
 		Name:      name,
 		Version:   version,
-		Locations: source.NewLocationSet(locations...),
+		Locations: file.NewLocationSet(locations...),
 		PURL:      packageURL(name, version, nil),
 		Language:  pkg.Python,
 		Type:      pkg.PythonPkg,
@@ -23,11 +23,11 @@ func newPackageForIndex(name, version string, locations ...source.Location) pkg.
 	return p
 }
 
-func newPackageForIndexWithMetadata(name, version string, metadata pkg.PythonPipfileLockMetadata, locations ...source.Location) pkg.Package {
+func newPackageForIndexWithMetadata(name, version string, metadata pkg.PythonPipfileLockMetadata, locations ...file.Location) pkg.Package {
 	p := pkg.Package{
 		Name:         name,
 		Version:      version,
-		Locations:    source.NewLocationSet(locations...),
+		Locations:    file.NewLocationSet(locations...),
 		PURL:         packageURL(name, version, nil),
 		Language:     pkg.Python,
 		Type:         pkg.PythonPkg,
@@ -40,11 +40,11 @@ func newPackageForIndexWithMetadata(name, version string, metadata pkg.PythonPip
 	return p
 }
 
-func newPackageForRequirementsWithMetadata(name, version string, metadata pkg.PythonRequirementsMetadata, locations ...source.Location) pkg.Package {
+func newPackageForRequirementsWithMetadata(name, version string, metadata pkg.PythonRequirementsMetadata, locations ...file.Location) pkg.Package {
 	p := pkg.Package{
 		Name:         name,
 		Version:      version,
-		Locations:    source.NewLocationSet(locations...),
+		Locations:    file.NewLocationSet(locations...),
 		PURL:         packageURL(name, version, nil),
 		Language:     pkg.Python,
 		Type:         pkg.PythonPkg,
@@ -57,12 +57,12 @@ func newPackageForRequirementsWithMetadata(name, version string, metadata pkg.Py
 	return p
 }
 
-func newPackageForPackage(m parsedData, sources ...source.Location) pkg.Package {
+func newPackageForPackage(m parsedData, sources ...file.Location) pkg.Package {
 	p := pkg.Package{
 		Name:         m.Name,
 		Version:      m.Version,
 		PURL:         packageURL(m.Name, m.Version, &m.PythonPackageMetadata),
-		Locations:    source.NewLocationSet(sources...),
+		Locations:    file.NewLocationSet(sources...),
 		Licenses:     pkg.NewLicenseSet(pkg.NewLicensesFromLocation(m.LicenseLocation, m.Licenses)...),
 		Language:     pkg.Python,
 		Type:         pkg.PythonPkg,

syft/pkg/cataloger/python/parse_pipfile_lock.go

@@ -8,9 +8,9 @@ import (
 	"strings"
 
 	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
-	"github.com/anchore/syft/syft/source"
 )
 
 type pipfileLock struct {
@@ -41,7 +41,7 @@ type Dependency struct {
 var _ generic.Parser = parsePipfileLock
 
 // parsePipfileLock is a parser function for Pipfile.lock contents, returning "Default" python packages discovered.
-func parsePipfileLock(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
+func parsePipfileLock(_ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
 	pkgs := make([]pkg.Package, 0)
 	dec := json.NewDecoder(reader)
 

syft/pkg/cataloger/python/parse_pipfile_lock_test.go

@@ -4,15 +4,15 @@ import (
 	"testing"
 
 	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
-	"github.com/anchore/syft/syft/source"
 )
 
 func TestParsePipFileLock(t *testing.T) {
 
 	fixture := "test-fixtures/pipfile-lock/Pipfile.lock"
-	locations := source.NewLocationSet(source.NewLocation(fixture))
+	locations := file.NewLocationSet(file.NewLocation(fixture))
 	expectedPkgs := []pkg.Package{
 		{
 			Name:         "aio-pika",

syft/pkg/cataloger/python/parse_poetry_lock.go

@@ -6,9 +6,9 @@ import (
 	"github.com/pelletier/go-toml"
 
 	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
-	"github.com/anchore/syft/syft/source"
 )
 
 // integrity check
@@ -25,7 +25,7 @@ type poetryMetadata struct {
 }
 
 // parsePoetryLock is a parser function for poetry.lock contents, returning all python packages discovered.
-func parsePoetryLock(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
+func parsePoetryLock(_ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
 	tree, err := toml.LoadReader(reader)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to load poetry.lock for parsing: %w", err)

syft/pkg/cataloger/python/parse_poetry_lock_test.go

@@ -4,14 +4,14 @@ import (
 	"testing"
 
 	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
-	"github.com/anchore/syft/syft/source"
 )
 
 func TestParsePoetryLock(t *testing.T) {
 	fixture := "test-fixtures/poetry/poetry.lock"
-	locations := source.NewLocationSet(source.NewLocation(fixture))
+	locations := file.NewLocationSet(file.NewLocation(fixture))
 	expectedPkgs := []pkg.Package{
 		{
 			Name:      "added-value",

syft/pkg/cataloger/python/parse_requirements.go

@@ -9,9 +9,9 @@ import (
 
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
-	"github.com/anchore/syft/syft/source"
 )
 
 var _ generic.Parser = parseRequirementsTxt
@@ -23,7 +23,7 @@ var (
 
 // parseRequirementsTxt takes a Python requirements.txt file, returning all Python packages that are locked to a
 // specific version.
-func parseRequirementsTxt(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
+func parseRequirementsTxt(_ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
 	var packages []pkg.Package
 
 	scanner := bufio.NewScanner(reader)

syft/pkg/cataloger/python/parse_requirements_test.go

@@ -4,14 +4,14 @@ import (
 	"testing"
 
 	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
-	"github.com/anchore/syft/syft/source"
 )
 
 func TestParseRequirementsTxt(t *testing.T) {
 	fixture := "test-fixtures/requires/requirements.txt"
-	locations := source.NewLocationSet(source.NewLocation(fixture))
+	locations := file.NewLocationSet(file.NewLocation(fixture))
 	expectedPkgs := []pkg.Package{
 		{
 			Name:         "flask",

syft/pkg/cataloger/python/parse_setup.go

@@ -7,9 +7,9 @@ import (
 
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
-	"github.com/anchore/syft/syft/source"
 )
 
 // integrity check
@@ -22,7 +22,7 @@ var _ generic.Parser = parseSetup
 //	" mypy2 == v0.770", ' mypy3== v0.770',  --> match(name=mypy2 version=v0.770), match(name=mypy3, version=v0.770)
 var pinnedDependency = regexp.MustCompile(`['"]\W?(\w+\W?==\W?[\w.]*)`)
 
-func parseSetup(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
+func parseSetup(_ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
 	var packages []pkg.Package
 
 	scanner := bufio.NewScanner(reader)

syft/pkg/cataloger/python/parse_setup_test.go

@@ -6,9 +6,9 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
-	"github.com/anchore/syft/syft/source"
 )
 
 func TestParseSetup(t *testing.T) {
@@ -65,7 +65,7 @@ func TestParseSetup(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.fixture, func(t *testing.T) {
-			locations := source.NewLocationSet(source.NewLocation(tt.fixture))
+			locations := file.NewLocationSet(file.NewLocation(tt.fixture))
 			for i := range tt.expected {
 				tt.expected[i].Locations = locations
 			}

syft/pkg/cataloger/python/parse_wheel_egg.go

@@ -10,13 +10,13 @@ import (
 	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
-	"github.com/anchore/syft/syft/source"
 )
 
 // parseWheelOrEgg takes the primary metadata file reference and returns the python package it represents.
-func parseWheelOrEgg(resolver source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
+func parseWheelOrEgg(resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
 	pd, sources, err := assembleEggOrWheelMetadata(resolver, reader.Location)
 	if err != nil {
 		return nil, nil, err
@@ -37,7 +37,7 @@ func parseWheelOrEgg(resolver source.FileResolver, _ *generic.Environment, reade
 }
 
 // fetchRecordFiles finds a corresponding installed-files.txt file for the given python package metadata file and returns the set of file records contained.
-func fetchInstalledFiles(resolver source.FileResolver, metadataLocation source.Location, sitePackagesRootPath string) (files []pkg.PythonFileRecord, sources []source.Location, err error) {
+func fetchInstalledFiles(resolver file.Resolver, metadataLocation file.Location, sitePackagesRootPath string) (files []pkg.PythonFileRecord, sources []file.Location, err error) {
 	// we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory
 	// or for an image... for an image the METADATA file may be present within multiple layers, so it is important
 	// to reconcile the installed-files.txt path to the same layer (or the next adjacent lower layer).
@@ -68,7 +68,7 @@ func fetchInstalledFiles(resolver source.FileResolver, metadataLocation source.L
 }
 
 // fetchRecordFiles finds a corresponding RECORD file for the given python package metadata file and returns the set of file records contained.
-func fetchRecordFiles(resolver source.FileResolver, metadataLocation source.Location) (files []pkg.PythonFileRecord, sources []source.Location, err error) {
+func fetchRecordFiles(resolver file.Resolver, metadataLocation file.Location) (files []pkg.PythonFileRecord, sources []file.Location, err error) {
 	// we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory
 	// or for an image... for an image the METADATA file may be present within multiple layers, so it is important
 	// to reconcile the RECORD path to the same layer (or the next adjacent lower layer).
@@ -95,7 +95,7 @@ func fetchRecordFiles(resolver source.FileResolver, metadataLocation source.Loca
 }
 
 // fetchTopLevelPackages finds a corresponding top_level.txt file for the given python package metadata file and returns the set of package names contained.
-func fetchTopLevelPackages(resolver source.FileResolver, metadataLocation source.Location) (pkgs []string, sources []source.Location, err error) {
+func fetchTopLevelPackages(resolver file.Resolver, metadataLocation file.Location) (pkgs []string, sources []file.Location, err error) {
 	// a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages
 	parentDir := filepath.Dir(metadataLocation.RealPath)
 	topLevelPath := filepath.Join(parentDir, "top_level.txt")
@@ -125,7 +125,7 @@ func fetchTopLevelPackages(resolver source.FileResolver, metadataLocation source
 	return pkgs, sources, nil
 }
 
-func fetchDirectURLData(resolver source.FileResolver, metadataLocation source.Location) (d *pkg.PythonDirectURLOriginInfo, sources []source.Location, err error) {
+func fetchDirectURLData(resolver file.Resolver, metadataLocation file.Location) (d *pkg.PythonDirectURLOriginInfo, sources []file.Location, err error) {
 	parentDir := filepath.Dir(metadataLocation.RealPath)
 	directURLPath := filepath.Join(parentDir, "direct_url.json")
 	directURLLocation := resolver.RelativeFileByPath(metadataLocation, directURLPath)
@@ -160,8 +160,8 @@ func fetchDirectURLData(resolver source.FileResolver, metadataLocation source.Lo
 }
 
 // assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from.
-func assembleEggOrWheelMetadata(resolver source.FileResolver, metadataLocation source.Location) (*parsedData, []source.Location, error) {
-	var sources = []source.Location{
+func assembleEggOrWheelMetadata(resolver file.Resolver, metadataLocation file.Location) (*parsedData, []file.Location, error) {
+	var sources = []file.Location{
 		metadataLocation.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
 	}
 

syft/pkg/cataloger/python/parse_wheel_egg_metadata.go

@@ -9,15 +9,15 @@ import (
 
 	"github.com/mitchellh/mapstructure"
 
-	"github.com/anchore/syft/internal/file"
+	intFile "github.com/anchore/syft/internal/file"
 	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
-	"github.com/anchore/syft/syft/source"
 )
 
 type parsedData struct {
 	Licenses                  string `mapstructure:"License"`
-	LicenseLocation           source.Location
+	LicenseLocation           file.Location
 	pkg.PythonPackageMetadata `mapstructure:",squash"`
 }
 
@@ -81,7 +81,7 @@ func parseWheelOrEggMetadata(path string, reader io.Reader) (parsedData, error)
 
 	pd.SitePackagesRootPath = determineSitePackagesRootPath(path)
 	if pd.Licenses != "" {
-		pd.LicenseLocation = source.NewLocation(path)
+		pd.LicenseLocation = file.NewLocation(path)
 	}
 
 	return pd, nil
@@ -91,7 +91,7 @@ func parseWheelOrEggMetadata(path string, reader io.Reader) (parsedData, error)
 // of egg metadata (as opposed to a directory that contains more metadata
 // files).
 func isEggRegularFile(path string) bool {
-	return file.GlobMatch(eggInfoGlob, path)
+	return intFile.GlobMatch(eggInfoGlob, path)
 }
 
 // determineSitePackagesRootPath returns the path of the site packages root,

syft/pkg/cataloger/python/parse_wheel_egg_metadata_test.go

@@ -6,8 +6,8 @@ import (
 
 	"github.com/go-test/deep"
 
+	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
-	"github.com/anchore/syft/syft/source"
 )
 
 func TestParseWheelEggMetadata(t *testing.T) {
@@ -19,7 +19,7 @@ func TestParseWheelEggMetadata(t *testing.T) {
 			Fixture: "test-fixtures/egg-info/PKG-INFO",
 			ExpectedMetadata: parsedData{
 				"Apache 2.0",
-				source.NewLocation("test-fixtures/egg-info/PKG-INFO"),
+				file.NewLocation("test-fixtures/egg-info/PKG-INFO"),
 				pkg.PythonPackageMetadata{
 					Name:                 "requests",
 					Version:              "2.22.0",
@@ -34,7 +34,7 @@ func TestParseWheelEggMetadata(t *testing.T) {
 			Fixture: "test-fixtures/dist-info/METADATA",
 			ExpectedMetadata: parsedData{
 				"BSD License",
-				source.NewLocation("test-fixtures/dist-info/METADATA"),
+				file.NewLocation("test-fixtures/dist-info/METADATA"),
 				pkg.PythonPackageMetadata{
 					Name:                 "Pygments",
 					Version:              "2.6.1",
@@ -135,7 +135,7 @@ func TestParseWheelEggMetadataInvalid(t *testing.T) {
 			Fixture: "test-fixtures/egg-info/PKG-INFO-INVALID",
 			ExpectedMetadata: parsedData{
 				"",
-				source.Location{},
+				file.Location{},
 				pkg.PythonPackageMetadata{
 					Name:                 "mxnet",
 					Version:              "1.8.0",