diff --git a/syft/pkg/cataloger/python/dependency.go b/syft/pkg/cataloger/python/dependency.go index 50b7d6c4a..29671ed59 100644 --- a/syft/pkg/cataloger/python/dependency.go +++ b/syft/pkg/cataloger/python/dependency.go @@ -15,7 +15,7 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" ) -func poetryLockDependencySpecifier(p pkg.Package) dependency.Specification { //nolint:dupl // this is very similar to the uv lock dependency specifier, but should remain separate +func poetryLockDependencySpecifier(p pkg.Package) dependency.Specification { meta, ok := p.Metadata.(pkg.PythonPoetryLockEntry) if !ok { log.Tracef("cataloger failed to extract poetry lock metadata for package %+v", p.Name) diff --git a/syft/pkg/cataloger/python/parse_uv_lock.go b/syft/pkg/cataloger/python/parse_uv_lock.go index 5ac0b3a1c..26c2492e6 100644 --- a/syft/pkg/cataloger/python/parse_uv_lock.go +++ b/syft/pkg/cataloger/python/parse_uv_lock.go @@ -5,12 +5,13 @@ import ( "context" "fmt" "io" + "maps" + "slices" "sort" "strings" "github.com/BurntSushi/toml" - "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -45,8 +46,12 @@ type uvPackage struct { Metadata uvMetadata `toml:"metadata"` } -type uvDependencies []struct { - Name string `toml:"name"` +type uvDependencies []uvDependency + +type uvDependency struct { + Name string `toml:"name"` + // Version is only written by uv when more than one version of this name is locked + Version string `toml:"version"` Extras []string `toml:"extra"` Markers string `toml:"marker"` } @@ -83,12 +88,16 @@ func newUvLockParser(cfg CatalogerConfig) uvLockParser { // parseUvLock is a parser function for uv.lock contents, returning all the pakcages discovered func (ulp uvLockParser) parseUvLock(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - pkgs, err := ulp.uvLockPackages(ctx, reader) + pkgs, specs, err := ulp.uvLockPackages(ctx, reader) if err != nil { return nil, nil, err } - return pkgs, dependency.Resolve(uvLockDependencySpecifier, pkgs), err + specifier := func(p pkg.Package) dependency.Specification { + return specs[p.ID()] + } + + return pkgs, dependency.Resolve(specifier, pkgs), err } func extractUvIndex(p uvPackage) string { @@ -139,105 +148,114 @@ func newPythonUvLockEntry(p uvPackage) pkg.PythonUvLockEntry { } } -func (ulp uvLockParser) uvLockPackages(ctx context.Context, reader file.LocationReadCloser) ([]pkg.Package, error) { +// uvLockPackages returns the packages in the lock along with the dependency specification for each (by package ID). +func (ulp uvLockParser) uvLockPackages(ctx context.Context, reader file.LocationReadCloser) ([]pkg.Package, map[artifact.ID]dependency.Specification, error) { var parsedLockFileVersion uvLockFileVersion // we cannot use the reader twice, so we read the contents first --uv.lock files tend to be small enough contents, err := io.ReadAll(reader) //nolint:gocritic // multi-pass parse requires []byte if err != nil { - return nil, unknown.New(reader.Location, fmt.Errorf("failed to read uv lock file: %w", err)) + return nil, nil, unknown.New(reader.Location, fmt.Errorf("failed to read uv lock file: %w", err)) } _, err = toml.NewDecoder(bytes.NewReader(contents)).Decode(&parsedLockFileVersion) if err != nil { - return nil, fmt.Errorf("failed to read uv lock version: %w", err) + return nil, nil, fmt.Errorf("failed to read uv lock version: %w", err) } // We will need to add some logic to parse and branch on different // lock file versions should they arise, but this gets us // started down this road for now. if parsedLockFileVersion.Version > 1 { - return nil, fmt.Errorf("could not parse uv lock file version %d", parsedLockFileVersion.Version) + return nil, nil, fmt.Errorf("could not parse uv lock file version %d", parsedLockFileVersion.Version) } var parsedLockFile uvLockFile _, err = toml.NewDecoder(bytes.NewReader(contents)).Decode(&parsedLockFile) if err != nil { - return nil, fmt.Errorf("failed to parse uv lock packages: %w", err) - } - - // The uv lock file doesn't store the dependency version in the dependency structure. - // Thus, we need a name -> version map for invoking extractUvDependencies. - // We then, of course, have to pass it down the call stack. - var pkgVerMap = make(map[string]string) - for _, p := range parsedLockFile.Packages { - pkgVerMap[p.Name] = p.Version + return nil, nil, fmt.Errorf("failed to parse uv lock packages: %w", err) } var pkgs []pkg.Package + specs := make(map[artifact.ID]dependency.Specification) for _, p := range parsedLockFile.Packages { - pkgs = append(pkgs, - newPackageForIndexWithMetadata( - ctx, - ulp.licenseResolver, - p.Name, - p.Version, - newPythonUvLockEntry(p), - reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), - ), + np := newPackageForIndexWithMetadata( + ctx, + ulp.licenseResolver, + p.Name, + p.Version, + newPythonUvLockEntry(p), + reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ) + pkgs = append(pkgs, np) + specs[np.ID()] = uvLockDependencySpecification(p) } - return pkgs, unknown.IfEmptyf(pkgs, "unable to determine packages") + return pkgs, specs, unknown.IfEmptyf(pkgs, "unable to determine packages") } -func isDependencyForUvExtra(dep pkg.PythonUvLockDependencyEntry) bool { +func isDependencyForUvExtra(dep uvDependency) bool { return strings.Contains(dep.Markers, "extra ==") } -// This is identical to poetryLockDependencySpecifier since it operates on identical -// data structures. Keeping it separate for now since it's always possible for data -// structures to change down the line. -// It *is* possible we may be able to merge the Uv and Poetry data structures -func uvLockDependencySpecifier(p pkg.Package) dependency.Specification { //nolint:dupl // this is very similar to the poetry lock dependency specifier, but should remain separate - meta, ok := p.Metadata.(pkg.PythonUvLockEntry) - if !ok { - log.Tracef("cataloger failed to extract UV lock metadata for package %+v", p.Name) - return dependency.Specification{} - } - - provides := []string{packageRef(p.Name, "")} - +// uvLockDependencySpecification is built from the raw lock entry rather than the package metadata. When a lock +// holds several versions of one name (a forked resolution), uv records which version each dependency entry means, +// and that is the only way to pair a dependent with the right one. The pairing is carried by the resulting +// relationships, so the version does not need to live on the metadata. +func uvLockDependencySpecification(p uvPackage) dependency.Specification { var requires []string - - for _, dep := range meta.Dependencies { + for _, dep := range p.Dependencies { if isDependencyForUvExtra(dep) { continue } - - requires = append(requires, packageRef(dep.Name, "")) - - for _, extra := range dep.Extras { - requires = append(requires, packageRef(dep.Name, extra)) - } + requires = append(requires, uvLockRequires(dep)...) } var variants []dependency.ProvidesRequires - for _, extra := range meta.Extras { + for _, extra := range slices.Sorted(maps.Keys(p.OptionalDependencies)) { + var extraRequires []string + for _, dep := range p.OptionalDependencies[extra] { + extraRequires = append(extraRequires, uvLockRequires(dep)...) + } variants = append(variants, dependency.ProvidesRequires{ - Provides: []string{packageRef(p.Name, extra.Name)}, - Requires: extractPackageNames(extra.Dependencies), + Provides: uvLockProvides(p, extra), + Requires: extraRequires, }, ) } return dependency.Specification{ ProvidesRequires: dependency.ProvidesRequires{ - Provides: provides, + Provides: uvLockProvides(p, ""), Requires: requires, }, Variants: variants, } } + +// uvLockProvides offers both the bare ref (for dependency entries without a version) and the versioned ref (for +// dependency entries uv pinned to this version). +func uvLockProvides(p uvPackage, extra string) []string { + return []string{packageRef(p.Name, extra), uvLockPackageRef(p.Name, p.Version, extra)} +} + +// uvLockRequires always requires the base package, plus each extra individually (name[extra1] and name[extra2], +// never name[extra1,extra2]). +func uvLockRequires(dep uvDependency) []string { + refs := []string{uvLockPackageRef(dep.Name, dep.Version, "")} + for _, extra := range dep.Extras { + refs = append(refs, uvLockPackageRef(dep.Name, dep.Version, extra)) + } + return refs +} + +func uvLockPackageRef(name, version, extra string) string { + ref := packageRef(name, extra) + version = strings.TrimSpace(version) + if version == "" { + return ref + } + return ref + "@" + version +} diff --git a/syft/pkg/cataloger/python/parse_uv_lock_test.go b/syft/pkg/cataloger/python/parse_uv_lock_test.go index 582ac9eb6..9aafc2471 100644 --- a/syft/pkg/cataloger/python/parse_uv_lock_test.go +++ b/syft/pkg/cataloger/python/parse_uv_lock_test.go @@ -129,6 +129,79 @@ func TestParseUvLock(t *testing.T) { pkgtest.TestFileParser(t, fixture, uvLockParser.parseUvLock, expectedPkgs, expectedRelationships) } +func TestParseUvLockForkedVersions(t *testing.T) { + fixture := "testdata/uv/forked-versions/uv.lock" + locations := file.NewLocationSet(file.NewLocation(fixture)) + index := "https://pypi.org/simple" + + newPkg := func(name, version string, meta pkg.PythonUvLockEntry) pkg.Package { + return pkg.Package{ + Name: name, + Version: version, + Locations: locations, + PURL: "pkg:pypi/" + name + "@" + version, + Language: pkg.Python, + Type: pkg.PythonPkg, + Metadata: meta, + } + } + + forkdemo := newPkg("forkdemo", "0.1.0", pkg.PythonUvLockEntry{ + Index: ".", + Dependencies: []pkg.PythonUvLockDependencyEntry{ + {Name: "pandas", Markers: "python_full_version < '3.11'"}, + {Name: "pandas", Markers: "python_full_version >= '3.11'"}, + }, + Extras: []pkg.PythonUvLockExtraEntry{ + {Name: "legacy", Dependencies: []string{"numpy"}}, + }, + }) + numpy1264 := newPkg("numpy", "1.26.4", pkg.PythonUvLockEntry{Index: index}) + numpy226 := newPkg("numpy", "2.2.6", pkg.PythonUvLockEntry{Index: index}) + numpy246 := newPkg("numpy", "2.4.6", pkg.PythonUvLockEntry{Index: index}) + numpy253 := newPkg("numpy", "2.5.3", pkg.PythonUvLockEntry{Index: index}) + pandas233 := newPkg("pandas", "2.3.3", pkg.PythonUvLockEntry{ + Index: index, + Dependencies: []pkg.PythonUvLockDependencyEntry{ + {Name: "numpy", Markers: "python_full_version < '3.10'"}, + {Name: "numpy", Markers: "python_full_version == '3.10.*'"}, + }, + }) + pandas306 := newPkg("pandas", "3.0.6", pkg.PythonUvLockEntry{ + Index: index, + Dependencies: []pkg.PythonUvLockDependencyEntry{ + {Name: "numpy", Markers: "python_full_version == '3.11.*'"}, + {Name: "numpy", Markers: "python_full_version >= '3.12'"}, + }, + }) + + dependencyOf := func(from, to pkg.Package) artifact.Relationship { + return artifact.Relationship{From: from, To: to, Type: artifact.DependencyOfRelationship} + } + + // each dependent links only to the versions its entries name, including through an optional dependency + expectedRelationships := []artifact.Relationship{ + dependencyOf(pandas233, forkdemo), + dependencyOf(pandas306, forkdemo), + dependencyOf(numpy1264, forkdemo), + dependencyOf(numpy1264, pandas233), + dependencyOf(numpy226, pandas233), + dependencyOf(numpy246, pandas306), + dependencyOf(numpy253, pandas306), + } + + uvLockParser := newUvLockParser(DefaultCatalogerConfig()) + pkgtest.TestFileParser(t, fixture, uvLockParser.parseUvLock, []pkg.Package{ + forkdemo, + numpy1264, + numpy226, + numpy246, + numpy253, + pandas233, + pandas306, + }, expectedRelationships) +} + func TestParseUvLockWithLicenseEnrichment(t *testing.T) { ctx := context.TODO() fixture := "testdata/pypi-remote/uv.lock" diff --git a/syft/pkg/cataloger/python/testdata/uv/forked-versions/uv.lock b/syft/pkg/cataloger/python/testdata/uv/forked-versions/uv.lock new file mode 100644 index 000000000..bcc73c082 --- /dev/null +++ b/syft/pkg/cataloger/python/testdata/uv/forked-versions/uv.lock @@ -0,0 +1,54 @@ +version = 1 +requires-python = ">=3.9" + +[[package]] +name = "forkdemo" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "pandas", version = "2.3.3", marker = "python_full_version < '3.11'" }, + { name = "pandas", version = "3.0.6", marker = "python_full_version >= '3.11'" }, +] + +[package.optional-dependencies] +legacy = [ + { name = "numpy", version = "1.26.4" }, +] + +[[package]] +name = "numpy" +version = "1.26.4" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "numpy" +version = "2.2.6" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "numpy" +version = "2.4.6" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "numpy" +version = "2.5.3" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "pandas" +version = "2.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "1.26.4", marker = "python_full_version < '3.10'" }, + { name = "numpy", version = "2.2.6", marker = "python_full_version == '3.10.*'" }, +] + +[[package]] +name = "pandas" +version = "3.0.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.4.6", marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.5.3", marker = "python_full_version >= '3.12'" }, +]