fix: fix parsing for complex toml types (#2965)

* fix: fix parsing for complex toml types
---------
Signed-off-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com>
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>

go.mod

@@ -87,6 +87,7 @@ require (
 require google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect
 
 require (
+	github.com/BurntSushi/toml v1.4.0
 	github.com/adrg/xdg v0.4.0
 	github.com/magiconair/properties v1.8.7
 )

go.sum

@@ -57,6 +57,8 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg6
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
+github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/CycloneDX/cyclonedx-go v0.9.0 h1:inaif7qD8bivyxp7XLgxUYtOXWtDez7+j72qKTMQTb8=
 github.com/CycloneDX/cyclonedx-go v0.9.0/go.mod h1:NE/EWvzELOFlG6+ljX/QeMlVt9VKcTwu8u0ccsACEsw=

syft/pkg/cataloger/python/parse_poetry_lock.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"fmt"
 	"sort"
-	"strings"
 
-	"github.com/pelletier/go-toml"
+	"github.com/BurntSushi/toml"
 
+	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
@@ -19,7 +19,9 @@ import (
 var _ generic.Parser = parsePoetryLock
 
 type poetryPackageSource struct {
-	URL string `toml:"url"`
+	URL       string `toml:"url"`
+	Type      string `toml:"type"`
+	Reference string `toml:"reference"`
 }
 
 type poetryPackages struct {
@@ -27,14 +29,15 @@ type poetryPackages struct {
 }
 
 type poetryPackage struct {
-	Name         string                             `toml:"name"`
+	Name                  string                    `toml:"name"`
-	Version      string                             `toml:"version"`
+	Version               string                    `toml:"version"`
-	Category     string                             `toml:"category"`
+	Category              string                    `toml:"category"`
-	Description  string                             `toml:"description"`
+	Description           string                    `toml:"description"`
-	Optional     bool                               `toml:"optional"`
+	Optional              bool                      `toml:"optional"`
-	Source       poetryPackageSource                `toml:"source"`
+	Source                poetryPackageSource       `toml:"source"`
-	Dependencies map[string]poetryPackageDependency `toml:"dependencies"`
+	DependenciesUnmarshal map[string]toml.Primitive `toml:"dependencies"`
-	Extras       map[string][]string                `toml:"extras"`
+	Extras                map[string][]string       `toml:"extras"`
+	Dependencies          map[string][]poetryPackageDependency
 }
 
 type poetryPackageDependency struct {
@@ -44,41 +47,6 @@ type poetryPackageDependency struct {
 	Extras   []string `toml:"extras"`
 }
 
-func (d *poetryPackageDependency) UnmarshalText(data []byte) error {
-	// attempt to parse as a map first
-	var dep map[string]interface{}
-	if err := toml.Unmarshal(data, &dep); err == nil {
-		if extras, ok := dep["extras"]; ok {
-			if extrasList, ok := extras.([]string); ok {
-				d.Extras = extrasList
-			}
-		}
-
-		if markers, ok := dep["markers"]; ok {
-			if markersString, ok := markers.(string); ok {
-				d.Markers = markersString
-			}
-		}
-
-		if version, ok := dep["version"]; ok {
-			if versionString, ok := version.(string); ok {
-				d.Version = versionString
-			}
-		}
-		return nil
-	}
-
-	if strings.ContainsAny(string(data), "[]{}") {
-		// odds are this is really a malformed toml array or object
-		return fmt.Errorf("unable to parse poetry dependency: version is malformed array/object: %q", string(data))
-	}
-
-	// assume this is a simple version string
-	d.Version = string(data)
-
-	return nil
-}
-
 // parsePoetryLock is a parser function for poetry.lock contents, returning all python packages discovered.
 func parsePoetryLock(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
 	pkgs, err := poetryLockPackages(reader)
@@ -93,15 +61,33 @@ func parsePoetryLock(_ context.Context, _ file.Resolver, _ *generic.Environment,
 }
 
 func poetryLockPackages(reader file.LocationReadCloser) ([]pkg.Package, error) {
-	tree, err := toml.LoadReader(reader)
+	metadata := poetryPackages{}
+	md, err := toml.NewDecoder(reader).Decode(&metadata)
 	if err != nil {
-		return nil, fmt.Errorf("unable to load poetry.lock for parsing: %w", err)
+		return nil, fmt.Errorf("failed to read poetry lock package: %w", err)
 	}
 
-	metadata := poetryPackages{}
+	for i, p := range metadata.Packages {
-	err = tree.Unmarshal(&metadata)
+		dependencies := make(map[string][]poetryPackageDependency)
-	if err != nil {
+		for pkgName, du := range p.DependenciesUnmarshal {
-		return nil, fmt.Errorf("unable to parse poetry.lock: %w", err)
+			var (
+				single    string
+				singleObj poetryPackageDependency
+				multiObj  []poetryPackageDependency
+			)
+
+			switch {
+			case md.PrimitiveDecode(du, &single) == nil:
+				dependencies[pkgName] = append(dependencies[pkgName], poetryPackageDependency{Version: single})
+			case md.PrimitiveDecode(du, &singleObj) == nil:
+				dependencies[pkgName] = append(dependencies[pkgName], singleObj)
+			case md.PrimitiveDecode(du, &multiObj) == nil:
+				dependencies[pkgName] = append(dependencies[pkgName], multiObj...)
+			default:
+				log.Trace("failed to decode poetry lock package dependencies for %s; skipping", pkgName)
+			}
+		}
+		metadata.Packages[i].Dependencies = dependencies
 	}
 
 	var pkgs []pkg.Package
@@ -137,13 +123,15 @@ func extractIndex(p poetryPackage) string {
 
 func extractPoetryDependencies(p poetryPackage) []pkg.PythonPoetryLockDependencyEntry {
 	var deps []pkg.PythonPoetryLockDependencyEntry
-	for name, dep := range p.Dependencies {
+	for name, dependencies := range p.Dependencies {
-		deps = append(deps, pkg.PythonPoetryLockDependencyEntry{
+		for _, d := range dependencies {
-			Name:    name,
+			deps = append(deps, pkg.PythonPoetryLockDependencyEntry{
-			Version: dep.Version,
+				Name:    name,
-			Extras:  dep.Extras,
+				Version: d.Version,
-			Markers: dep.Markers,
+				Extras:  d.Extras,
-		})
+				Markers: d.Markers,
+			})
+		}
 	}
 	sort.Slice(deps, func(i, j int) bool {
 		return deps[i].Name < deps[j].Name

syft/pkg/cataloger/python/parse_poetry_lock_test.go

@@ -24,7 +24,11 @@ func TestParsePoetryLock(t *testing.T) {
 				Index: "https://test.pypi.org/simple",
 				Dependencies: []pkg.PythonPoetryLockDependencyEntry{
 					{Name: "docutils", Version: "*"},
+					{Name: "msal", Version: ">=0.4.1,<2.0.0"},
 					{Name: "natsort", Version: "*"},
+					{Name: "packaging", Version: "*"},
+					{Name: "portalocker", Version: ">=1.0,<3", Markers: `platform_system != "Windows"`},
+					{Name: "portalocker", Version: ">=1.6,<3", Markers: `platform_system == "Windows"`},
 					{Name: "six", Version: "*"},
 					{Name: "sphinx", Version: "*"},
 				},

syft/pkg/cataloger/python/test-fixtures/poetry/dev-deps/poetry.lock

@@ -11,6 +11,13 @@ docutils = "*"
 natsort = "*"
 six = "*"
 sphinx = "*"
+packaging = "*"
+msal = {version = ">=0.4.1,<2.0.0"}
+malformed = [ [ { version = "1.2" } ] ]
+portalocker = [
+    {version = ">=1.0,<3", markers = "platform_system != \"Windows\""},
+    {version = ">=1.6,<3", markers = "platform_system == \"Windows\""},
+]
 
 [package.source]
 type = "legacy"