feat: add Debian archive (.deb) file cataloger (#3704)

* feat: add Debian archive (.deb) file cataloger

Add a cataloger that parses Debian package (.deb) archive files directly,
allowing Syft to discover packages from .deb files without requiring
them to be installed on the system. This implements issue #3315.

Key features:
- Parse .deb AR archives to extract package metadata
- Support for gzip, xz, and zstd compressed control files
- Extract package metadata from control files
- Process file information from md5sums files
- Mark configuration files from conffiles entries
- Handle trailing slashes in archive member names

Signed-off-by: Alan Pope <alan.pope@anchore.com>

* chore: run go mod tidy to fix failing workflow

Signed-off-by: Alan Pope <alan.pope@anchore.com>

* add license processing to dpkg archive cataloger + add tests

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update json schema with dpkg archive type

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alan Pope <alan.pope@anchore.com>
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alan Pope
2025-03-19 20:03:21 +00:00
committed by GitHub
co-authored by Alex Goodman
parent be0959cabf
commit 5fa8e9c6e9
19 changed files with 3506 additions and 12 deletions
@@ -54,6 +54,9 @@ func Originator(p pkg.Package) (typ string, author string) { //nolint: funlen
case pkg.DpkgDBEntry:
author = metadata.Maintainer
case pkg.DpkgArchiveEntry:
author = metadata.Maintainer
case pkg.JavaArchive:
if metadata.Manifest != nil {
author = metadata.Manifest.Main.MustGet("Specification-Vendor")
@@ -109,7 +109,7 @@ func Test_OriginatorSupplier(t *testing.T) {
supplier: "Organization: Microsoft Corporation",
},
{
name: "from dpkg",
name: "from dpkg DB",
input: pkg.Package{
Metadata: pkg.DpkgDBEntry{
Maintainer: "auth",
@@ -118,6 +118,16 @@ func Test_OriginatorSupplier(t *testing.T) {
originator: "Person: auth",
supplier: "Person: auth",
},
{
name: "from dpkg archive",
input: pkg.Package{
Metadata: pkg.DpkgArchiveEntry{
Maintainer: "auth",
},
},
originator: "Person: auth",
supplier: "Person: auth",
},
{
name: "from gem",
input: pkg.Package{
@@ -20,6 +20,7 @@ func AllTypes() []any {
pkg.DotnetDepsEntry{},
pkg.DotnetPackagesLockEntry{},
pkg.DotnetPortableExecutableEntry{},
pkg.DpkgArchiveEntry{},
pkg.DpkgDBEntry{},
pkg.ELFBinaryPackageNoteJSONPayload{},
pkg.ElixirMixLockEntry{},
+1
View File
@@ -74,6 +74,7 @@ var jsonTypes = makeJSONTypes(
jsonNames(pkg.DartPubspecLockEntry{}, "dart-pubspec-lock-entry", "DartPubMetadata"),
jsonNames(pkg.DotnetDepsEntry{}, "dotnet-deps-entry", "DotnetDepsMetadata"),
jsonNames(pkg.DotnetPortableExecutableEntry{}, "dotnet-portable-executable-entry"),
jsonNames(pkg.DpkgArchiveEntry{}, "dpkg-archive-entry"),
jsonNames(pkg.DpkgDBEntry{}, "dpkg-db-entry", "DpkgMetadata"),
jsonNames(pkg.ELFBinaryPackageNoteJSONPayload{}, "elf-binary-package-note-json-payload"),
jsonNames(pkg.RubyGemspec{}, "ruby-gemspec", "GemMetadata"),
+6
View File
@@ -17,3 +17,9 @@ func NewDBCataloger() pkg.Cataloger {
WithParserByGlobs(parseDpkgDB, "**/lib/dpkg/status", "**/lib/dpkg/status.d/*", "**/lib/opkg/info/*.control", "**/lib/opkg/status").
WithProcessors(dependency.Processor(dbEntryDependencySpecifier))
}
// NewArchiveCataloger returns a new Debian package cataloger object capable of parsing .deb archive files
func NewArchiveCataloger() pkg.Cataloger {
return generic.NewCataloger("deb-archive-cataloger").
WithParserByGlobs(parseDebArchive, "**/*.deb")
}
@@ -36,6 +36,7 @@ func TestDpkgCataloger(t *testing.T) {
file.NewLocation("/var/lib/dpkg/info/libpam-runtime.conffiles").WithAnnotation(pkg.EvidenceAnnotationKey, pkg.SupportingEvidenceAnnotation),
file.NewLocation("/usr/share/doc/libpam-runtime/copyright").WithAnnotation(pkg.EvidenceAnnotationKey, pkg.SupportingEvidenceAnnotation),
),
PURL: "pkg:deb/debian/libpam-runtime@1.1.8-3.6?arch=all&distro=debian-12&upstream=pam",
Type: pkg.DebPkg,
Metadata: pkg.DpkgDBEntry{
Package: "libpam-runtime",
@@ -224,6 +225,70 @@ func Test_CatalogerRelationships(t *testing.T) {
}
}
func TestDpkgArchiveCataloger(t *testing.T) {
tests := []struct {
name string
expected []pkg.Package
}{
{
name: "image-single-dpkg",
expected: []pkg.Package{
{
Name: "zlib1g",
Version: "1:1.3.dfsg-3.1ubuntu2.1",
FoundBy: "deb-archive-cataloger",
Locations: file.NewLocationSet(
file.NewLocation("/zlib1g.deb"),
),
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("Zlib"),
),
PURL: "pkg:deb/zlib1g@1%3A1.3.dfsg-3.1ubuntu2.1?arch=amd64&upstream=zlib",
Type: pkg.DebPkg,
Metadata: pkg.DpkgArchiveEntry{
Package: "zlib1g",
Source: "zlib",
Version: "1:1.3.dfsg-3.1ubuntu2.1",
Architecture: "amd64",
Maintainer: "Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>",
InstalledSize: 163,
Description: `compression library - runtime
zlib is a library implementing the deflate compression method found
in gzip and PKZIP. This package includes the shared library.`,
Provides: []string{"libz1"},
Depends: []string{"libc6 (>= 2.14)"},
Files: []pkg.DpkgFileRecord{
{
Path: "/usr/lib/x86_64-linux-gnu/libz.so.1.3",
Digest: &file.Digest{Algorithm: "md5", Value: "4447b36fc5cd1b044f089553b4166f09"},
},
{
Path: "/usr/share/doc/zlib1g/changelog.Debian.gz",
Digest: &file.Digest{Algorithm: "md5", Value: "8b870c2e94c0cf780e2a65329cf11fdc"},
},
{
Path: "/usr/share/doc/zlib1g/copyright",
Digest: &file.Digest{Algorithm: "md5", Value: "d348307d5bf18267bcbada155a715a3e"},
},
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := NewArchiveCataloger()
pkgtest.NewCatalogTester().
WithImageResolver(t, tt.name).
IgnoreLocationLayer(). // this fixture can be rebuilt, thus the layer ID will change
Expects(tt.expected, nil).
TestCataloger(t, c)
})
}
}
func TestCataloger_Globs(t *testing.T) {
tests := []struct {
name string
+20 -1
View File
@@ -24,7 +24,7 @@ const (
func newDpkgPackage(d pkg.DpkgDBEntry, dbLocation file.Location, resolver file.Resolver, release *linux.Release, evidence ...file.Location) pkg.Package {
// TODO: separate pr to license refactor, but explore extracting dpkg-specific license parsing into a separate function
licenses := make([]pkg.License, 0)
var licenses []pkg.License
locations := file.NewLocationSet(dbLocation.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation))
locations.Add(evidence...)
@@ -54,6 +54,25 @@ func newDpkgPackage(d pkg.DpkgDBEntry, dbLocation file.Location, resolver file.R
return p
}
func newDebArchivePackage(location file.Location, metadata pkg.DpkgArchiveEntry, licenseStrings []string) pkg.Package {
p := pkg.Package{
Name: metadata.Package,
Version: metadata.Version,
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValues(licenseStrings...)...),
Type: pkg.DebPkg,
PURL: packageURL(
pkg.DpkgDBEntry(metadata),
// we don't know the distro information, but since this is a deb file then we can reasonably assume it is a debian-based distro
&linux.Release{IDLike: []string{"debian"}},
),
Metadata: metadata,
Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
}
p.SetID()
return p
}
// PackageURL returns the PURL for the specific Debian package (see https://github.com/package-url/purl-spec)
func packageURL(m pkg.DpkgDBEntry, distro *linux.Release) string {
if distro == nil {
@@ -0,0 +1,238 @@
package debian
import (
"archive/tar"
"bytes"
"context"
"fmt"
"io"
"path/filepath"
"regexp"
"strings"
"github.com/blakesmith/ar"
"github.com/mholt/archives"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
)
// parseDebArchive parses a Debian package archive (.deb) file and returns the packages it contains.
// A .deb file is an ar archive containing three main files:
// - debian-binary: Version of the .deb format (usually "2.0")
// - control.tar.gz/xz/zst: Contains package metadata (control file, md5sums, conffiles)
// - data.tar.gz/xz/zst: Contains the actual files to be installed (not processed by this cataloger)
//
// This function extracts and processes the control information to create package metadata.
func parseDebArchive(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
arReader := ar.NewReader(reader)
var metadata *pkg.DpkgArchiveEntry
var licenses []string
var unknownErr error
for {
header, err := arReader.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, nil, fmt.Errorf("failed to read ar header: %w", err)
}
switch {
case strings.HasPrefix(header.Name, "control.tar"):
// Decompress the control.tar.* file
dcReader, err := decompressionStream(ctx, arReader, header.Name)
if err != nil {
return nil, nil, unknown.New(reader.Location, fmt.Errorf("failed to decompress control.tar.* file: %w", err))
}
metadata, err = processControlTar(dcReader)
if err != nil {
return nil, nil, unknown.New(reader.Location, fmt.Errorf("failed to process control.tar.* file: %w", err))
}
case strings.HasPrefix(header.Name, "data.tar"):
// Decompress the data.tar.* file
dcReader, err := decompressionStream(ctx, arReader, header.Name)
if err != nil {
return nil, nil, unknown.New(reader.Location, fmt.Errorf("failed to decompress data.tar.* file: %w", err))
}
licenses, err = processDataTar(dcReader)
if err != nil {
unknownErr = unknown.Append(unknownErr, reader.Location, fmt.Errorf("failed to process data.tar.* file: %w", err))
}
}
}
if metadata == nil {
return nil, nil, unknown.New(reader.Location, fmt.Errorf("no application found described in .dpkg archive"))
}
return []pkg.Package{
newDebArchivePackage(reader.Location, *metadata, licenses),
}, nil, nil
}
// this is the pattern you'd expect to see in a tar header for a debian package license file ()
var archiveHeaderLicensePathPattern = regexp.MustCompile(`^\.?/usr/share/doc/[^/]+/copyright$`)
func processDataTar(dcReader io.ReadCloser) ([]string, error) {
defer internal.CloseAndLogError(dcReader, "")
var licenses []string
tarReader := tar.NewReader(dcReader)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return licenses, err
}
// look for /usr/share/docs/*/copyright files, parse each one for license claims
// TODO: in the future we can add archive sub indexes to the locations to see where within
// the dpkg archive the license was found
if archiveHeaderLicensePathPattern.MatchString(header.Name) {
licenses = append(licenses, parseLicensesFromCopyright(tarReader)...)
}
}
return licenses, nil
}
func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) {
defer internal.CloseAndLogError(dcReader, "")
// Extract control, md5sums, and conffiles files from control.tar
tarReader := tar.NewReader(dcReader)
controlFileContent, md5Content, confContent, err := readControlFiles(tarReader)
if err != nil {
return nil, fmt.Errorf("failed to read control files: %w", err)
}
if controlFileContent == nil {
return nil, fmt.Errorf("control file not found in archive")
}
metadata, err := newDpkgArchiveMetadata(controlFileContent, md5Content, confContent)
if err != nil {
return nil, fmt.Errorf("failed to create package metadata: %w", err)
}
return &metadata, nil
}
func newDpkgArchiveMetadata(controlFile, md5sums, confFiles []byte) (pkg.DpkgArchiveEntry, error) {
// parse the control file to get package metadata
metadata, err := parseControlFile(string(controlFile))
if err != nil {
return pkg.DpkgArchiveEntry{}, fmt.Errorf("failed to parse control file: %w", err)
}
// parse MD5 sums to get file records
var files []pkg.DpkgFileRecord
if len(md5sums) > 0 {
files = parseDpkgMD5Info(bytes.NewReader(md5sums))
}
// mark config files
if len(confFiles) > 0 {
markConfigFiles(confFiles, files)
}
metadata.Files = files
return metadata, nil
}
func decompressionStream(ctx context.Context, r io.Reader, filePath string) (io.ReadCloser, error) {
format, stream, err := archives.Identify(ctx, filePath, r)
if err != nil {
return nil, fmt.Errorf("failed to identify compression format: %w", err)
}
decompressor, ok := format.(archives.Decompressor)
if !ok {
return nil, fmt.Errorf("file format does not support decompression: %s", filePath)
}
rc, err := decompressor.OpenReader(stream)
if err != nil {
return nil, fmt.Errorf("failed to create decompression reader: %w", err)
}
return rc, nil
}
// readControlFiles extracts important files from the control.tar archive
func readControlFiles(tarReader *tar.Reader) (controlFile, md5sums, conffiles []byte, err error) {
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, nil, nil, err
}
switch filepath.Base(header.Name) {
case "control":
controlFile, err = io.ReadAll(tarReader)
if err != nil {
return nil, nil, nil, err
}
case "md5sums":
md5sums, err = io.ReadAll(tarReader)
if err != nil {
return nil, nil, nil, err
}
case "conffiles":
conffiles, err = io.ReadAll(tarReader)
if err != nil {
return nil, nil, nil, err
}
}
}
return controlFile, md5sums, conffiles, nil
}
// parseControlFile parses the content of a debian control file into package metadata
func parseControlFile(controlFileContent string) (pkg.DpkgArchiveEntry, error) {
// Reuse the existing dpkg status file parsing logic
reader := strings.NewReader(controlFileContent)
entries, err := parseDpkgStatus(reader)
if err != nil {
return pkg.DpkgArchiveEntry{}, fmt.Errorf("failed to parse control file: %w", err)
}
if len(entries) == 0 {
return pkg.DpkgArchiveEntry{}, fmt.Errorf("no package entries found in control file")
}
// We expect only one entry from a .deb control file
return pkg.DpkgArchiveEntry(entries[0]), nil
}
// markConfigFiles marks files that are listed in conffiles as configuration files
func markConfigFiles(conffilesContent []byte, files []pkg.DpkgFileRecord) {
// Parse the conffiles content into DpkgFileRecord entries
confFiles := parseDpkgConffileInfo(bytes.NewReader(conffilesContent))
// Create a map for quick lookup of config files by path
configPathMap := make(map[string]struct{})
for _, confFile := range confFiles {
configPathMap[confFile.Path] = struct{}{}
}
// Mark files as config files if they're in the conffiles list
for i := range files {
if _, exists := configPathMap[files[i].Path]; exists {
files[i].IsConfigFile = true
}
}
}
@@ -0,0 +1,142 @@
package debian
import (
"archive/tar"
"bytes"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
)
func TestReadControlFiles(t *testing.T) {
tarBytes := createTestTarWithControlFiles(t)
tarReader := bytes.NewReader(tarBytes)
reader := tar.NewReader(tarReader)
controlFile, md5sums, conffiles, err := readControlFiles(reader)
require.NoError(t, err)
assert.NotNil(t, controlFile, "expected control file to be found")
assert.NotNil(t, md5sums, "expected md5sums file to be found")
assert.NotNil(t, conffiles, "expected conffiles file to be found")
assert.Contains(t, string(controlFile), "Package: test-package")
assert.Contains(t, string(md5sums), "d41d8cd98f00b204e9800998ecf8427e")
assert.Contains(t, string(conffiles), "/etc/test")
}
// createTestTarWithControlFiles creates a simple in-memory tar file with test control files
func createTestTarWithControlFiles(t *testing.T) []byte {
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
// Add control file
controlContent := `Package: test-package
Version: 1.0.0
Architecture: all
Maintainer: Test <test@example.com>
Description: Test package
`
err := tw.WriteHeader(&tar.Header{
Name: "control",
Mode: 0644,
Size: int64(len(controlContent)),
})
require.NoError(t, err)
_, err = tw.Write([]byte(controlContent))
require.NoError(t, err)
// Add md5sums file
md5Content := "d41d8cd98f00b204e9800998ecf8427e usr/bin/test-command\n"
err = tw.WriteHeader(&tar.Header{
Name: "md5sums",
Mode: 0644,
Size: int64(len(md5Content)),
})
require.NoError(t, err)
_, err = tw.Write([]byte(md5Content))
require.NoError(t, err)
// Add conffiles file
conffilesContent := "/etc/test/config.conf\n"
err = tw.WriteHeader(&tar.Header{
Name: "conffiles",
Mode: 0644,
Size: int64(len(conffilesContent)),
})
require.NoError(t, err)
_, err = tw.Write([]byte(conffilesContent))
require.NoError(t, err)
// Close the tar writer
err = tw.Close()
require.NoError(t, err)
return buf.Bytes()
}
func TestMarkConfigFiles(t *testing.T) {
// Create test data
conffilesContent := []byte("/usr/bin/test-command\n/etc/test/config.conf\n")
files := []pkg.DpkgFileRecord{
{
Path: "/usr/bin/test-command",
Digest: &file.Digest{
Algorithm: "md5",
Value: "d41d8cd98f00b204e9800998ecf8427e",
},
},
{
Path: "/etc/test/config.conf",
Digest: &file.Digest{
Algorithm: "md5",
Value: "d41d8cd98f00b204e9800998ecf8427e",
},
},
{
Path: "/usr/bin/other-command",
Digest: &file.Digest{
Algorithm: "md5",
Value: "d41d8cd98f00b204e9800998ecf8427e",
},
},
}
markConfigFiles(conffilesContent, files)
assert.True(t, files[0].IsConfigFile, "first file should be marked as config file")
assert.True(t, files[1].IsConfigFile, "second file should be marked as config file")
assert.False(t, files[2].IsConfigFile, "third file should not be marked as config file")
}
func TestParseControlFile(t *testing.T) {
controlContent := `Package: test-package
Version: 1.2.3-4
Architecture: amd64
Maintainer: Test User <test@example.com>
Installed-Size: 1234
Depends: libc6, libtest
Description: This is a test package
More description text
And even more details
`
metadata, err := parseControlFile(controlContent)
require.NoError(t, err)
assert.Equal(t, "test-package", metadata.Package)
assert.Equal(t, "1.2.3-4", metadata.Version)
assert.Equal(t, "amd64", metadata.Architecture)
assert.Equal(t, "Test User <test@example.com>", metadata.Maintainer)
assert.Equal(t, 1234, metadata.InstalledSize)
assert.Contains(t, metadata.Description, "This is a test package")
assert.Len(t, metadata.Depends, 2)
assert.Contains(t, metadata.Depends, "libc6")
assert.Contains(t, metadata.Depends, "libtest")
}
@@ -0,0 +1,9 @@
PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"
NAME="Debian GNU/Linux"
VERSION_ID="12"
VERSION="12 (bookworm)"
VERSION_CODENAME=bookworm
ID=debian
HOME_URL="https://www.debian.org/"
SUPPORT_URL="https://www.debian.org/support"
BUG_REPORT_URL="https://bugs.debian.org/"
@@ -0,0 +1,9 @@
FROM ubuntu:22.04 AS downloader
RUN apt-get update && apt-get install -y curl
RUN curl -o zlib1g.deb http://archive.ubuntu.com/ubuntu/pool/main/z/zlib/zlib1g_1.3.dfsg-3.1ubuntu2.1_amd64.deb
FROM scratch
COPY --from=downloader /zlib1g.deb /zlib1g.deb
+2
View File
@@ -12,6 +12,8 @@ const DpkgDBGlob = "**/var/lib/dpkg/{status,status.d/**}"
var _ FileOwner = (*DpkgDBEntry)(nil)
type DpkgArchiveEntry DpkgDBEntry
// DpkgDBEntry represents all captured data for a Debian package DB entry; available fields are described
// at http://manpages.ubuntu.com/manpages/xenial/man1/dpkg-query.1.html in the --showformat section.
// Additional information about how these fields are used can be found at
+7 -5
View File
@@ -39,7 +39,7 @@ func PURLQualifiers(vars map[string]string, release *linux.Release) (q packageur
})
}
distroQualifiers := []string{}
var distroQualifiers []string
if release == nil {
return q
@@ -55,10 +55,12 @@ func PURLQualifiers(vars map[string]string, release *linux.Release) (q packageur
distroQualifiers = append(distroQualifiers, release.BuildID)
}
q = append(q, packageurl.Qualifier{
Key: PURLQualifierDistro,
Value: strings.Join(distroQualifiers, "-"),
})
if len(distroQualifiers) > 0 {
q = append(q, packageurl.Qualifier{
Key: PURLQualifierDistro,
Value: strings.Join(distroQualifiers, "-"),
})
}
return q
}