syft/syft/pkg/cataloger/debian/parse_dpkg_db.go

package debian

import (
	"bufio"
	"context"
	"errors"
	"fmt"
	"io"
	"path"
	"regexp"
	"strings"

	"github.com/dustin/go-humanize"
	"github.com/go-viper/mapstructure/v2"

	"github.com/anchore/go-sync"
	"github.com/anchore/syft/internal"
	"github.com/anchore/syft/internal/log"
	"github.com/anchore/syft/internal/unknown"
	"github.com/anchore/syft/syft/artifact"
	"github.com/anchore/syft/syft/cataloging"
	"github.com/anchore/syft/syft/file"
	"github.com/anchore/syft/syft/pkg"
	"github.com/anchore/syft/syft/pkg/cataloger/generic"
)

const (
	deinstallStatus string = "deinstall"
)

var (
	errEndOfPackages = fmt.Errorf("no more packages to read")
	sourceRegexp     = regexp.MustCompile(`(?P<name>\S+)( \((?P<version>.*)\))?`)
)

// parseDpkgDB reads a dpkg database "status" file (and surrounding data files) and returns the packages and relationships found.
func parseDpkgDB(ctx context.Context, resolver file.Resolver, env *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
	metadata, err := parseDpkgStatus(reader)
	if err != nil {
		return nil, nil, fmt.Errorf("unable to catalog dpkg DB=%q: %w", reader.RealPath, err)
	}

	dbLoc := reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)
	var pkgs []pkg.Package
	_ = sync.CollectSlice(&ctx, cataloging.ExecutorFile, sync.ToSeq(metadata), func(m dpkgExtractedMetadata) (pkg.Package, error) {
		return newDpkgPackage(ctx, m, dbLoc, resolver, env.LinuxRelease, findDpkgInfoFiles(m.Package, resolver, reader.Location)...), nil
	}, &pkgs)

	return pkgs, nil, unknown.IfEmptyf(pkgs, "unable to determine packages")
}

func findDpkgInfoFiles(name string, resolver file.Resolver, dbLocation file.Location) []file.Location {
	if resolver == nil {
		return nil
	}
	if strings.TrimSpace(name) == "" {
		return nil
	}

	// for typical debian-base distributions, the installed package info is at /var/lib/dpkg/status
	// and the md5sum information is under /var/lib/dpkg/info/; however, for distroless the installed
	// package info is across multiple files under /var/lib/dpkg/status.d/ and the md5sums are contained in
	// the same directory
	searchPath := path.Dir(dbLocation.RealPath)

	if !strings.HasSuffix(searchPath, "status.d") {
		searchPath = path.Join(searchPath, "info")
	}

	// look for /var/lib/dpkg/info/NAME.*
	locations, err := resolver.FilesByGlob(path.Join(searchPath, name+".*"))
	if err != nil {
		log.WithFields("error", err, "pkg", name).Trace("failed to fetch related dpkg info files")
		return nil
	}

	return locations
}

// parseDpkgStatus is a parser function for Debian DB status contents, returning the raw metadata for all Debian
// packages listed. Conversion to pkg.DpkgDBEntry is deferred to the package-building stage so that fields which
// are not part of the final entry (e.g. License) remain available to the caller.
func parseDpkgStatus(reader io.Reader) ([]dpkgExtractedMetadata, error) {
	buffedReader := bufio.NewReader(reader)
	var metadata []dpkgExtractedMetadata

	continueProcessing := true
	for continueProcessing {
		entry, err := parseDpkgStatusEntry(buffedReader)
		if err != nil {
			if errors.Is(err, errEndOfPackages) {
				continueProcessing = false
			} else {
				return nil, err
			}
		}
		if entry == nil {
			continue
		}

		metadata = append(metadata, *entry)
	}

	return metadata, nil
}

// dpkgExtractedMetadata is an adapter struct to capture the fields from the dpkg status file, however, the final
// pkg.DpkgMetadata struct has different types for some fields (e.g. Provides, Depends, and PreDepends is []string, not a string).
type dpkgExtractedMetadata struct {
	Package       string `mapstructure:"Package"`
	Source        string `mapstructure:"Source"`
	Version       string `mapstructure:"Version"`
	SourceVersion string `mapstructure:"SourceVersion"`
	Architecture  string `mapstructure:"Architecture"`
	Maintainer    string `mapstructure:"Maintainer"`
	InstalledSize int    `mapstructure:"InstalledSize"`
	Description   string `mapstructure:"Description"`
	Provides      string `mapstructure:"Provides"`
	Depends       string `mapstructure:"Depends"`
	PreDepends    string `mapstructure:"PreDepends"` // note: original doc is Pre-Depends
	Status        string `mapstructure:"Status"`
	License       string `mapstructure:"License"`
	Conffiles     string `mapstructure:"Conffiles"`
}

func (d *dpkgExtractedMetadata) toDpkgEntry() pkg.DpkgDBEntry {
	entry := pkg.DpkgDBEntry{
		Package:       d.Package,
		Source:        d.Source,
		Version:       d.Version,
		SourceVersion: d.SourceVersion,
		Architecture:  d.Architecture,
		Maintainer:    d.Maintainer,
		InstalledSize: d.InstalledSize,
		Description:   d.Description,
		Provides:      splitPkgList(d.Provides),
		Depends:       splitPkgList(d.Depends),
		PreDepends:    splitPkgList(d.PreDepends),
		// note: licenses and conffiles are handled separately
	}

	// there may be an optional conffiles section that we should persist as files
	if d.Conffiles != "" {
		var err error
		entry.Files, err = parseDpkgConffileInfo(strings.NewReader(d.Conffiles))
		if err != nil {
			// toDpkgEntry has no error return, a log line is all that's available here
			log.Debugf("dpkg status conffiles field for package %q: %v", d.Package, err)
		}
	}

	if entry.Files == nil {
		// ensure the default value for a collection is never nil since this may be shown as JSON
		entry.Files = make([]pkg.DpkgFileRecord, 0)
	}

	return entry
}

// parseDpkgStatusEntry returns an individual Dpkg entry, or returns errEndOfPackages if there are no more packages to parse from the reader.
func parseDpkgStatusEntry(reader *bufio.Reader) (*dpkgExtractedMetadata, error) {
	var retErr error
	dpkgFields, err := extractAllFields(reader)
	if err != nil {
		if !errors.Is(err, errEndOfPackages) {
			return nil, err
		}
		if len(dpkgFields) == 0 {
			return nil, err
		}
		retErr = err
	}

	raw := dpkgExtractedMetadata{}
	err = mapstructure.Decode(dpkgFields, &raw)
	if err != nil {
		return nil, err
	}

	// Skip entries which have been removed but not purged, e.g. "rc" status in dpkg -l
	if strings.Contains(raw.Status, deinstallStatus) {
		return nil, nil
	}

	sourceName, sourceVersion := extractSourceVersion(raw.Source)
	if sourceName != "" {
		raw.Source = sourceName
	}
	if sourceVersion != "" {
		raw.SourceVersion = sourceVersion
	}

	if raw.Package == "" {
		return nil, retErr
	}

	return &raw, retErr
}

func splitPkgList(pkgList string) (ret []string) {
	fields := strings.Split(pkgList, ",")
	for _, field := range fields {
		field = strings.TrimSpace(field)
		if field != "" {
			ret = append(ret, field)
		}
	}
	return ret
}

func extractAllFields(reader *bufio.Reader) (map[string]any, error) {
	dpkgFields := make(map[string]any)
	var key string

	for {
		line, err := reader.ReadString('\n')
		if err != nil {
			if errors.Is(err, io.EOF) {
				return dpkgFields, errEndOfPackages
			}
			return nil, err
		}

		line = strings.TrimRight(line, "\n")

		// empty line indicates end of entry
		if len(line) == 0 {
			// if the entry has not started, keep parsing lines
			if len(dpkgFields) == 0 {
				continue
			}
			break
		}

		switch {
		case strings.HasPrefix(line, " "):
			// a field-body continuation
			if len(key) == 0 {
				return nil, fmt.Errorf("no match for continuation: line: '%s'", line)
			}

			val, ok := dpkgFields[key]
			if !ok {
				return nil, fmt.Errorf("no previous key exists, expecting: %s", key)
			}
			// concatenate onto previous value
			val = fmt.Sprintf("%s\n %s", val, strings.TrimSpace(line))
			dpkgFields[key] = val
		default:
			// parse a new key
			var val any
			key, val, err = handleNewKeyValue(line)
			if err != nil {
				log.Tracef("parsing dpkg status: extracting key-value from line: %s err: %v", line, err)
				continue
			}

			if _, ok := dpkgFields[key]; ok {
				return nil, fmt.Errorf("duplicate key discovered: %s", key)
			}
			dpkgFields[key] = val
		}
	}
	return dpkgFields, nil
}

// If the source entry string is of the form "<name> (<version>)" then parse and return the components, if
// of the "<name>" form, then return name and nil. Yocto (opkg/ipk) instead writes the bitbake recipe filename
// ("<name>_<version>.bb"), which is split the same way.
func extractSourceVersion(source string) (string, string) {
	if recipe, ok := strings.CutSuffix(source, ".bb"); ok && !strings.ContainsAny(recipe, " (") {
		// bitbake reserves "_" as the recipe name / version separator
		name, version, _ := strings.Cut(recipe, "_")
		return name, version
	}
	// special handling for the Source field since it has formatted data
	match := internal.MatchNamedCaptureGroups(sourceRegexp, source)
	return match["name"], match["version"]
}

// handleNewKeyValue parse a new key-value pair from the given unprocessed line
func handleNewKeyValue(line string) (key string, val any, err error) {
	if i := strings.Index(line, ":"); i > 0 {
		key = strings.TrimSpace(line[0:i])
		// mapstruct cant handle "-"
		key = strings.ReplaceAll(key, "-", "")
		val := strings.TrimSpace(line[i+1:])

		// further processing of values based on the key that was discovered
		switch key {
		case "InstalledSize":
			s, err := humanize.ParseBytes(val)
			if err != nil {
				return "", nil, fmt.Errorf("bad installed-size value=%q: %w", val, err)
			}
			return key, int(s), nil
		default:
			return key, val, nil
		}
	}

	return "", nil, fmt.Errorf("cannot parse field from line: '%s'", line)
}