Add support for scanning IPK packages using the deb-archive-cataloger (#5111)

* Add support for scanning IPK packages using the deb-archive-cataloger

Signed-off-by: Silas Parker <silas@srp.me.uk>

* fix(debian): share License field parsing between opkg status and archive control files

- the control-file `License:` fallback now goes through the same helper as the opkg status DB path, so a package reports the same licenses whether found installed or as an archive
- bitbake `&` / `|` are rewritten to SPDX `AND` / `OR` and the whole value is validated as one expression instead of split by hand, which kept grouping like `(A & B) | C` intact
- non-SPDX values fall back to a deduped list of names with operators and parens dropped (previously the status DB path emitted `&`, `(BSD`, etc as licenses)
- fields over 4KB are ignored, bounding work per archive and keeping deep paren nesting away from the recursive expression parser

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix(debian): split Yocto recipe filenames in the Source field

Yocto writes the bitbake recipe filename (`Source: zlib_1.3.2.bb`) where dpkg expects a source package name. This is now split into `Source: zlib` / `SourceVersion: 1.3.2` for both ipk archives and opkg status DB entries, instead of surfacing the `.bb` filename as the upstream package.

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix(debian): do not label ipk archives as Debian packages

- `.ipk` archives no longer get a `pkg:deb` PURL from the assumed debian distro; opkg-based distros (Yocto, OpenWrt) have no purl type, so these now match packages found in an opkg status DB, which already get no PURL
- docs now cover `.ipk` and note that only ar-framed ipks are supported (OpenWrt's older gzipped-tar framing is rejected as an invalid archive)

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* test(debian): assemble the ipk fixture at build time instead of committing it

The fixture Dockerfile now builds a Yocto-shaped `.ipk` from a checked-in `control/` directory (copied from the original poky-tiny build) using `tar`, `zstd` and binutils `ar`, the same way bitbake's `package_ipk` packs one. This drops the prebuilt binary and the 20-30 minute Yocto `Dockerfile.build` while keeping the same archive layout and control metadata.

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Silas Parker <silas@srp.me.uk>
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Silas Parker
2026-10-07 20:41:13 +00:00
committed by GitHub
co-authored by Alex Goodman
parent b0a8de7bf6
commit 546c872c9d
12 changed files with 262 additions and 39 deletions
@@ -75,6 +75,7 @@ catalogers:
method: glob # AUTO-GENERATED method: glob # AUTO-GENERATED
criteria: # AUTO-GENERATED criteria: # AUTO-GENERATED
- '**/*.deb' - '**/*.deb'
- '**/*.ipk'
metadata_types: # AUTO-GENERATED metadata_types: # AUTO-GENERATED
- pkg.DpkgArchiveEntry - pkg.DpkgArchiveEntry
package_types: # AUTO-GENERATED package_types: # AUTO-GENERATED
+2 -2
View File
@@ -18,8 +18,8 @@ func NewDBCataloger() pkg.Cataloger {
WithProcessors(dependency.Processor(dbEntryDependencySpecifier)) WithProcessors(dependency.Processor(dbEntryDependencySpecifier))
} }
// NewArchiveCataloger returns a new Debian package cataloger object capable of parsing .deb archive files // NewArchiveCataloger returns a new Debian package cataloger object capable of parsing .deb and .ipk (opkg) archive files
func NewArchiveCataloger() pkg.Cataloger { func NewArchiveCataloger() pkg.Cataloger {
return generic.NewCataloger("deb-archive-cataloger"). return generic.NewCataloger("deb-archive-cataloger").
WithParserByGlobs(parseDebArchive, "**/*.deb") WithParserByGlobs(parseDebArchive, "**/*.deb", "**/*.ipk")
} }
@@ -276,6 +276,36 @@ func TestDpkgArchiveCataloger(t *testing.T) {
}, },
}, },
}, },
}, {
name: "image-single-ipk",
expected: []pkg.Package{
{
Name: "libz1",
Version: "1.3.2-r0",
FoundBy: "deb-archive-cataloger",
Locations: file.NewLocationSet(
file.NewLocation("/libz1_1.3.2-r0_x86-64-v3.ipk"),
),
Type: pkg.DebPkg,
Metadata: pkg.DpkgArchiveEntry{
Package: "libz1",
Source: "zlib",
SourceVersion: "1.3.2",
Version: "1.3.2-r0",
Architecture: "x86-64-v3",
Maintainer: "Poky Maintainers <poky@lists.yoctoproject.org>",
InstalledSize: 0,
Description: `Zlib Compression Library
Zlib is a general-purpose, patent-free, lossless data compression library
which is used by many different programs.`,
Provides: []string{"zlib"},
Depends: []string{"musl (>= 1.2.6+git0+9fa28ece75)"},
},
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocationsWithContext(ctx, "Zlib"),
),
},
},
}, },
} }
+61 -15
View File
@@ -9,6 +9,8 @@ import (
"sort" "sort"
"strings" "strings"
"github.com/scylladb/go-set/strset"
"github.com/anchore/packageurl-go" "github.com/anchore/packageurl-go"
"github.com/anchore/syft/internal" "github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
@@ -61,32 +63,65 @@ func newDpkgPackage(ctx context.Context, d dpkgExtractedMetadata, dbLocation fil
return p return p
} }
// a declared License field holds a single expression, a few hundred bytes at most in practice. Anything
// much larger only feeds the expression parser (which recurses on parentheses) attacker-sized input.
const maxDeclaredLicenseLen = 4096
var (
// opkg/ipkg (Yocto via bitbake) declare licenses with "&" for AND and "|" for OR
bitbakeLicenseOperators = strings.NewReplacer("&", " AND ", "|", " OR ")
tightenParens = strings.NewReplacer("( ", "(", " )", ")")
licenseListSeparators = strings.NewReplacer("(", " ", ")", " ")
)
// extractDeclaredLicenses converts a License field from the status DB into a license set. Returns nil // extractDeclaredLicenses converts a License field from the status DB into a license set. Returns nil
// for empty input so standard dpkg entries (which never declare License inline) incur no allocation. // for empty input so standard dpkg entries (which never declare License inline) incur no allocation.
// Mirrors the alpine cataloger's approach: keep the value whole if it parses as a valid SPDX expression,
// otherwise split on whitespace to handle space-separated lists.
func extractDeclaredLicenses(ctx context.Context, raw string, dbLocation file.Location) []pkg.License { func extractDeclaredLicenses(ctx context.Context, raw string, dbLocation file.Location) []pkg.License {
values := declaredLicenseValues(raw)
if len(values) == 0 {
return nil
}
return pkg.NewLicensesFromLocationWithContext(ctx, dbLocation, values...)
}
// declaredLicenseValues converts an inline License field (opkg status DB or an archive control file) into
// license values. Bitbake operators are rewritten to SPDX ones and the value is kept whole if it then parses
// as a valid SPDX expression (mirroring the alpine cataloger), otherwise it is treated as a list of license
// names with operators and grouping dropped.
func declaredLicenseValues(raw string) []string {
raw = strings.TrimSpace(raw)
if raw == "" { if raw == "" {
return nil return nil
} }
licenseStrings := []string{raw} if len(raw) > maxDeclaredLicenseLen {
if _, err := license.ParseExpression(raw); err != nil { log.WithFields("length", len(raw)).Debug("ignoring oversized declared license field")
licenseStrings = strings.Fields(raw) return nil
} }
return pkg.NewLicensesFromLocationWithContext(ctx, dbLocation, licenseStrings...)
expr := tightenParens.Replace(strings.Join(strings.Fields(bitbakeLicenseOperators.Replace(raw)), " "))
if _, err := license.ParseExpression(expr); err == nil {
return []string{expr}
}
var values []string
seen := strset.New()
for _, field := range strings.Fields(licenseListSeparators.Replace(expr)) {
if field == "AND" || field == "OR" || seen.Has(field) {
continue
}
seen.Add(field)
values = append(values, field)
}
return values
} }
func newDebArchivePackage(ctx context.Context, location file.Location, metadata pkg.DpkgArchiveEntry, licenseStrings []string) pkg.Package { func newDebArchivePackage(ctx context.Context, location file.Location, metadata pkg.DpkgArchiveEntry, licenseStrings []string) pkg.Package {
p := pkg.Package{ p := pkg.Package{
Name: metadata.Package, Name: metadata.Package,
Version: metadata.Version, Version: metadata.Version,
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, licenseStrings...)...), Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, licenseStrings...)...),
Type: pkg.DebPkg, Type: pkg.DebPkg,
PURL: packageURL( PURL: packageURL(pkg.DpkgDBEntry(metadata), archiveDistro(location)),
pkg.DpkgDBEntry(metadata),
// we don't know the distro information, but since this is a deb file then we can reasonably assume it is a debian-based distro
&linux.Release{IDLike: []string{"debian"}},
),
Metadata: metadata, Metadata: metadata,
Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)), Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
} }
@@ -95,6 +130,17 @@ func newDebArchivePackage(ctx context.Context, location file.Location, metadata
return p return p
} }
// archiveDistro guesses the distro an archive was built for. We don't know the distro information, but a .deb
// can reasonably be assumed to target a debian-based distro. An .ipk targets opkg-based distros (Yocto,
// OpenWrt) which have no purl type, so it gets no distro and therefore no PURL, the same as packages found in
// an opkg status DB.
func archiveDistro(location file.Location) *linux.Release {
if strings.EqualFold(path.Ext(location.Path()), ".ipk") {
return nil
}
return &linux.Release{IDLike: []string{"debian"}}
}
// PackageURL returns the PURL for the specific Debian package (see https://github.com/package-url/purl-spec) // PackageURL returns the PURL for the specific Debian package (see https://github.com/package-url/purl-spec)
func packageURL(m pkg.DpkgDBEntry, distro *linux.Release) string { func packageURL(m pkg.DpkgDBEntry, distro *linux.Release) string {
if distro == nil { if distro == nil {
+31
View File
@@ -3,6 +3,7 @@ package debian
import ( import (
"context" "context"
"sort" "sort"
"strings"
"testing" "testing"
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
@@ -145,6 +146,36 @@ func Test_extractDeclaredLicenses(t *testing.T) {
raw: "GPL-2.0 BSD-3-Clause", raw: "GPL-2.0 BSD-3-Clause",
expected: []string{"GPL-2.0", "BSD-3-Clause"}, expected: []string{"GPL-2.0", "BSD-3-Clause"},
}, },
{
name: "bitbake operators become an SPDX expression",
raw: "MIT & (BSD-3-Clause | Apache-2.0) & ( GPL-2.0-or-later )",
expected: []string{"MIT AND (BSD-3-Clause OR Apache-2.0) AND (GPL-2.0-or-later)"},
},
{
name: "bitbake grouping is preserved rather than split",
raw: "(MIT & BSD-3-Clause) | Apache-2.0",
expected: []string{"(MIT AND BSD-3-Clause) OR Apache-2.0"},
},
{
name: "parenthesized alternatives are preserved",
raw: "(MIT) | (Apache-2.0)",
expected: []string{"(MIT) OR (Apache-2.0)"},
},
{
name: "non-SPDX names drop operators and grouping",
raw: "GPLv2+ & (LGPLv2.1 | BSD) & GPLv2+",
expected: []string{"GPLv2+", "LGPLv2.1", "BSD"},
},
{
name: "oversized field is ignored",
raw: strings.Repeat("MIT & ", maxDeclaredLicenseLen),
expected: nil,
},
{
name: "deeply nested parens are ignored",
raw: strings.Repeat("(", maxDeclaredLicenseLen) + "MIT" + strings.Repeat(")", maxDeclaredLicenseLen),
expected: nil,
},
} }
for _, test := range tests { for _, test := range tests {
+46 -15
View File
@@ -2,6 +2,7 @@ package debian
import ( import (
"archive/tar" "archive/tar"
"bytes"
"context" "context"
"errors" "errors"
"fmt" "fmt"
@@ -23,18 +24,27 @@ import (
"github.com/anchore/syft/syft/pkg/cataloger/generic" "github.com/anchore/syft/syft/pkg/cataloger/generic"
) )
// parseDebArchive parses a Debian package archive (.deb) file and returns the packages it contains. // parseDebArchive parses a Debian package archive (.deb) or opkg package (.ipk) file and returns the packages it
// A .deb file is an ar archive containing three main files: // contains. Both are ar archives containing three main files:
// - debian-binary: Version of the .deb format (usually "2.0") // - debian-binary: Version of the archive format (usually "2.0")
// - control.tar.gz/xz/zst: Contains package metadata (control file, md5sums, conffiles) // - control.tar.gz/xz/zst: Contains package metadata (control file, md5sums, conffiles)
// - data.tar.gz/xz/zst: Contains the actual files to be installed (not processed by this cataloger) // - data.tar.gz/xz/zst: Contains the actual files to be installed (not processed by this cataloger)
// //
// This function extracts and processes the control information to create package metadata. // This function extracts and processes the control information to create package metadata. Licenses come
// from copyright files in the data tar, falling back to the control file's License field (common for .ipk).
// Only ar-framed .ipk files are supported; the older gzipped-tar framing (e.g. OpenWrt's ipkg-build) is
// rejected as an invalid archive.
func parseDebArchive(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { func parseDebArchive(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
arReader := ar.NewReader(reader) validatedReader, err := newValidatedArReader(reader)
if err != nil {
return nil, nil, fmt.Errorf("invalid ar archive: %w", err)
}
arReader := ar.NewReader(validatedReader)
var metadata *pkg.DpkgArchiveEntry var metadata *pkg.DpkgArchiveEntry
var licenses []string var licenses []string
var ctrlLicenses []string
var unknownErr error var unknownErr error
var sawControl, sawData bool var sawControl, sawData bool
for { for {
@@ -56,7 +66,7 @@ func parseDebArchive(ctx context.Context, _ file.Resolver, _ *generic.Environmen
if err != nil { if err != nil {
return nil, nil, unknown.New(reader.Location, fmt.Errorf("failed to decompress control.tar.* file: %w", err)) return nil, nil, unknown.New(reader.Location, fmt.Errorf("failed to decompress control.tar.* file: %w", err))
} }
metadata, err = processControlTar(dcReader) metadata, ctrlLicenses, err = processControlTar(dcReader)
switch { switch {
case err != nil && metadata == nil: case err != nil && metadata == nil:
return nil, nil, unknown.New(reader.Location, fmt.Errorf("failed to process control.tar.* file: %w", err)) return nil, nil, unknown.New(reader.Location, fmt.Errorf("failed to process control.tar.* file: %w", err))
@@ -87,6 +97,10 @@ func parseDebArchive(ctx context.Context, _ file.Resolver, _ *generic.Environmen
return nil, nil, unknown.New(reader.Location, fmt.Errorf("no application found described in .dpkg archive")) return nil, nil, unknown.New(reader.Location, fmt.Errorf("no application found described in .dpkg archive"))
} }
if len(licenses) == 0 && len(ctrlLicenses) > 0 {
licenses = ctrlLicenses
}
// a partial parse still yields a usable package, so report what went wrong alongside it rather than // a partial parse still yields a usable package, so report what went wrong alongside it rather than
// dropping either one // dropping either one
return []pkg.Package{ return []pkg.Package{
@@ -128,10 +142,10 @@ func processDataTar(dcReader io.ReadCloser) ([]string, error) {
} }
// processControlTar always returns whatever metadata it managed to parse, even alongside a non-nil // processControlTar always returns whatever metadata it managed to parse, even alongside a non-nil
// error. A non-nil error with non-nil metadata means the package is usable but incomplete (e.g. a // error, along with any licenses declared in the control file's License field. A non-nil error with
// clipped file listing); nil metadata means nothing usable was found and the caller should treat it // non-nil metadata means the package is usable but incomplete (e.g. a clipped file listing); nil
// as fatal. // metadata means nothing usable was found and the caller should treat it as fatal.
func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) { func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, []string, error) {
defer internal.CloseAndLogError(dcReader, "") defer internal.CloseAndLogError(dcReader, "")
tarReader := tar.NewReader(dcReader) tarReader := tar.NewReader(dcReader)
@@ -140,6 +154,7 @@ func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) {
var files []pkg.DpkgFileRecord var files []pkg.DpkgFileRecord
var confFileRecords []pkg.DpkgFileRecord var confFileRecords []pkg.DpkgFileRecord
var listingErr error var listingErr error
var licenses []string
for { for {
header, err := tarReader.Next() header, err := tarReader.Next()
@@ -147,7 +162,7 @@ func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) {
break break
} }
if err != nil { if err != nil {
return metadata, fmt.Errorf("failed to read control tar: %w", err) return metadata, nil, fmt.Errorf("failed to read control tar: %w", err)
} }
switch filepath.Base(header.Name) { switch filepath.Base(header.Name) {
@@ -155,12 +170,13 @@ func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) {
// parseDpkgStatus already streams via bufio.Reader // parseDpkgStatus already streams via bufio.Reader
entries, err := parseDpkgStatus(tarReader) entries, err := parseDpkgStatus(tarReader)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to parse control file: %w", err) return nil, nil, fmt.Errorf("failed to parse control file: %w", err)
} }
if len(entries) == 0 { if len(entries) == 0 {
return nil, fmt.Errorf("no package entries found in control file") return nil, nil, fmt.Errorf("no package entries found in control file")
} }
entry := pkg.DpkgArchiveEntry(entries[0].toDpkgEntry()) entry := pkg.DpkgArchiveEntry(entries[0].toDpkgEntry())
licenses = declaredLicenseValues(entries[0].License)
metadata = &entry metadata = &entry
case "md5sums": case "md5sums":
// parseDpkgMD5Info streams via bufio.Scanner and reports its own clipping/scan errors // parseDpkgMD5Info streams via bufio.Scanner and reports its own clipping/scan errors
@@ -176,7 +192,7 @@ func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) {
} }
if metadata == nil { if metadata == nil {
return nil, fmt.Errorf("control file not found in archive") return nil, nil, fmt.Errorf("control file not found in archive")
} }
if len(confFileRecords) > 0 && len(files) > 0 { if len(confFileRecords) > 0 && len(files) > 0 {
@@ -195,7 +211,22 @@ func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) {
// a clipped or unreadable file listing leaves the package usable, so hand both back and let the // a clipped or unreadable file listing leaves the package usable, so hand both back and let the
// caller decide // caller decide
return metadata, listingErr return metadata, licenses, listingErr
}
// newValidatedArReader checks for the ar global header before handing the stream to the ar reader, which
// otherwise skips those bytes unchecked and reads garbage headers from non-ar input.
func newValidatedArReader(reader io.ReadCloser) (io.ReadCloser, error) {
prefix := make([]byte, len(ar.GLOBAL_HEADER))
if _, err := io.ReadFull(reader, prefix); err != nil {
return nil, fmt.Errorf("failed to read ar header: %w", err)
}
if !bytes.Equal(prefix, []byte(ar.GLOBAL_HEADER)) {
return nil, fmt.Errorf("expected ar header %q, got %q", string(ar.GLOBAL_HEADER), string(prefix))
}
return io.NopCloser(io.MultiReader(bytes.NewReader(prefix), reader)), nil
} }
// a .deb's member sizes are bounded by the archive, but what they decompress to is not: xz and zstd // a .deb's member sizes are bounded by the archive, but what they decompress to is not: xz and zstd
@@ -26,12 +26,13 @@ import (
func TestProcessControlTar(t *testing.T) { func TestProcessControlTar(t *testing.T) {
tarBytes := createTestTarWithControlFiles(t) tarBytes := createTestTarWithControlFiles(t)
metadata, err := processControlTar(io.NopCloser(bytes.NewReader(tarBytes))) metadata, licenses, err := processControlTar(io.NopCloser(bytes.NewReader(tarBytes)))
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, metadata) require.NotNil(t, metadata)
assert.Equal(t, "test-package", metadata.Package) assert.Equal(t, "test-package", metadata.Package)
assert.Equal(t, []string{"MIT"}, licenses)
assert.Equal(t, "1.0.0", metadata.Version) assert.Equal(t, "1.0.0", metadata.Version)
// md5sums should have been parsed into file records // md5sums should have been parsed into file records
@@ -43,6 +44,14 @@ func TestProcessControlTar(t *testing.T) {
assert.True(t, metadata.Files[0].IsConfigFile, "file listed in conffiles should be marked as config") assert.True(t, metadata.Files[0].IsConfigFile, "file listed in conffiles should be marked as config")
} }
func TestParseDebArchive_RejectsInvalidArchiveHeader(t *testing.T) {
reader := file.NewLocationReadCloser(file.NewLocation("test.deb"), io.NopCloser(bytes.NewReader([]byte("not-an-ar-archive"))))
_, _, err := parseDebArchive(context.Background(), nil, nil, reader)
require.Error(t, err)
assert.Contains(t, err.Error(), "ar archive")
}
func TestProcessControlTar_ConfigFileMarking(t *testing.T) { func TestProcessControlTar_ConfigFileMarking(t *testing.T) {
// Create a tar where conffiles lists paths that overlap with md5sums entries // Create a tar where conffiles lists paths that overlap with md5sums entries
var buf bytes.Buffer var buf bytes.Buffer
@@ -61,9 +70,10 @@ func TestProcessControlTar_ConfigFileMarking(t *testing.T) {
require.NoError(t, tw.Close()) require.NoError(t, tw.Close())
metadata, err := processControlTar(io.NopCloser(bytes.NewReader(buf.Bytes()))) metadata, licenses, err := processControlTar(io.NopCloser(bytes.NewReader(buf.Bytes())))
require.NoError(t, err) require.NoError(t, err)
require.Len(t, metadata.Files, 3) require.Len(t, metadata.Files, 3)
assert.Empty(t, licenses)
assert.True(t, metadata.Files[0].IsConfigFile, "first file should be marked as config file") assert.True(t, metadata.Files[0].IsConfigFile, "first file should be marked as config file")
assert.True(t, metadata.Files[1].IsConfigFile, "second file should be marked as config file") assert.True(t, metadata.Files[1].IsConfigFile, "second file should be marked as config file")
@@ -75,7 +85,7 @@ func createTestTarWithControlFiles(t *testing.T) []byte {
var buf bytes.Buffer var buf bytes.Buffer
tw := tar.NewWriter(&buf) tw := tar.NewWriter(&buf)
controlContent := "Package: test-package\nVersion: 1.0.0\nArchitecture: all\nMaintainer: Test <test@example.com>\nDescription: Test package\n" controlContent := "Package: test-package\nVersion: 1.0.0\nArchitecture: all\nMaintainer: Test <test@example.com>\nDescription: Test package\nLicense: MIT\n"
writeTarEntry(t, tw, "control", controlContent) writeTarEntry(t, tw, "control", controlContent)
md5Content := "d41d8cd98f00b204e9800998ecf8427e usr/bin/test-command\n" md5Content := "d41d8cd98f00b204e9800998ecf8427e usr/bin/test-command\n"
@@ -165,7 +175,7 @@ func Test_processControlTar_rejectsABombedControlMember(t *testing.T) {
writeTarEntry(t, tw, "md5sums", strings.Repeat("d41d8cd98f00b204e9800998ecf8427e usr/bin/x\n", 20000)) writeTarEntry(t, tw, "md5sums", strings.Repeat("d41d8cd98f00b204e9800998ecf8427e usr/bin/x\n", 20000))
require.NoError(t, tw.Close()) require.NoError(t, tw.Close())
metadata, err := processControlTar(newBoundedReadCloser(io.NopCloser(bytes.NewReader(tarBuf.Bytes())), 4096)) metadata, _, err := processControlTar(newBoundedReadCloser(io.NopCloser(bytes.NewReader(tarBuf.Bytes())), 4096))
require.ErrorIs(t, err, errDecompressedTooLarge) require.ErrorIs(t, err, errDecompressedTooLarge)
require.NotNil(t, metadata, "a control tar read failure after \"control\" was parsed must not drop the package") require.NotNil(t, metadata, "a control tar read failure after \"control\" was parsed must not drop the package")
@@ -181,7 +191,7 @@ func Test_processControlTar_clippedListingIsUsableButReported(t *testing.T) {
writeTarEntry(t, tw, "md5sums", strings.Repeat("d41d8cd98f00b204e9800998ecf8427e usr/bin/x\n", maxDpkgFileRecords+10)) writeTarEntry(t, tw, "md5sums", strings.Repeat("d41d8cd98f00b204e9800998ecf8427e usr/bin/x\n", maxDpkgFileRecords+10))
require.NoError(t, tw.Close()) require.NoError(t, tw.Close())
metadata, err := processControlTar(io.NopCloser(bytes.NewReader(tarBuf.Bytes()))) metadata, _, err := processControlTar(io.NopCloser(bytes.NewReader(tarBuf.Bytes())))
require.Error(t, err, "clipping must be reported") require.Error(t, err, "clipping must be reported")
require.ErrorIs(t, err, errClippedFileListing) require.ErrorIs(t, err, errClippedFileListing)
@@ -418,3 +428,18 @@ func Test_decompressionStream_failsEarlyInsteadOfBuffering(t *testing.T) {
assert.Less(t, bounded, uint64(8*intFile.MB), assert.Less(t, bounded, uint64(8*intFile.MB),
"a bounded stream must stop at the cap, not buffer the whole member and check afterwards") "a bounded stream must stop at the cap, not buffer the whole member and check afterwards")
} }
func TestProcessControlTar_DeclaredLicense(t *testing.T) {
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
controlContent := "Package: test-package\nVersion: 1.0.0\nArchitecture: all\nLicense: MIT & (BSD-3-Clause | Apache-2.0) & ( GPL-2.0-or-later )\n"
writeTarEntry(t, tw, "control", controlContent)
require.NoError(t, tw.Close())
metadata, licenses, err := processControlTar(io.NopCloser(bytes.NewReader(buf.Bytes())))
require.NoError(t, err)
require.NotNil(t, metadata)
assert.Equal(t, []string{"MIT AND (BSD-3-Clause OR Apache-2.0) AND (GPL-2.0-or-later)"}, licenses)
}
+10 -2
View File
@@ -183,9 +183,11 @@ func parseDpkgStatusEntry(reader *bufio.Reader) (*dpkgExtractedMetadata, error)
} }
sourceName, sourceVersion := extractSourceVersion(raw.Source) sourceName, sourceVersion := extractSourceVersion(raw.Source)
if sourceName != "" {
raw.Source = sourceName
}
if sourceVersion != "" { if sourceVersion != "" {
raw.SourceVersion = sourceVersion raw.SourceVersion = sourceVersion
raw.Source = sourceName
} }
if raw.Package == "" { if raw.Package == "" {
@@ -263,8 +265,14 @@ func extractAllFields(reader *bufio.Reader) (map[string]any, error) {
} }
// If the source entry string is of the form "<name> (<version>)" then parse and return the components, if // If the source entry string is of the form "<name> (<version>)" then parse and return the components, if
// of the "<name>" form, then return name and nil // of the "<name>" form, then return name and nil. Yocto (opkg/ipk) instead writes the bitbake recipe filename
// ("<name>_<version>.bb"), which is split the same way.
func extractSourceVersion(source string) (string, string) { func extractSourceVersion(source string) (string, string) {
if recipe, ok := strings.CutSuffix(source, ".bb"); ok && !strings.ContainsAny(recipe, " (") {
// bitbake reserves "_" as the recipe name / version separator
name, version, _ := strings.Cut(recipe, "_")
return name, version
}
// special handling for the Source field since it has formatted data // special handling for the Source field since it has formatted data
match := internal.MatchNamedCaptureGroups(sourceRegexp, source) match := internal.MatchNamedCaptureGroups(sourceRegexp, source)
return match["name"], match["version"] return match["name"], match["version"]
@@ -366,6 +366,16 @@ func TestSourceVersionExtract(t *testing.T) {
input: "", input: "",
expected: []string{"", ""}, expected: []string{"", ""},
}, },
{
name: "bitbake recipe with version",
input: "zlib_1.3.2.bb",
expected: []string{"zlib", "1.3.2"},
},
{
name: "bitbake recipe without version",
input: "packagegroup-core-boot.bb",
expected: []string{"packagegroup-core-boot", ""},
},
} }
for _, test := range tests { for _, test := range tests {
@@ -0,0 +1,21 @@
# assembles a Yocto-style .ipk the same way bitbake's package_ipk does (opkg-build: a GNU ar archive of
# debian-binary, control.tar.gz and data.tar.zst) without paying for a full Yocto build. control/ is copied
# verbatim from libz1_1.3.2-r0_x86-64-v3.ipk built by poky-tiny (wrynose); the library itself is a stub
# since only the archive shape and control metadata matter here.
FROM alpine:3.18.4@sha256:48d9183eb12a05c99bcc0bf44a003607b8e941e1d4f41f9ad12bdcc4b5672f86 AS build
RUN apk add --no-cache binutils zstd
COPY control /pkg/control
RUN mkdir -p /pkg/data/usr/lib \
&& echo stub > /pkg/data/usr/lib/libz.so.1.3.2 \
&& ln -s libz.so.1.3.2 /pkg/data/usr/lib/libz.so.1 \
&& cd /pkg \
&& echo 2.0 > debian-binary \
&& tar -C control -czf control.tar.gz . \
&& tar -C data -cf - . | zstd -q -o data.tar.zst \
&& ar rcD /libz1_1.3.2-r0_x86-64-v3.ipk debian-binary control.tar.gz data.tar.zst
FROM scratch
COPY --from=build /libz1_1.3.2-r0_x86-64-v3.ipk /
@@ -0,0 +1,15 @@
Package: libz1
Version: 1.3.2-r0
Description: Zlib Compression Library
Zlib is a general-purpose, patent-free, lossless data compression library
which is used by many different programs.
Section: libs
Priority: optional
Maintainer: Poky Maintainers <poky@lists.yoctoproject.org>
License: Zlib
Architecture: x86-64-v3
OE: zlib
Homepage: http://zlib.net/
Depends: musl (>= 1.2.6+git0+9fa28ece75)
Provides: zlib
Source: zlib_1.3.2.bb
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -e
if [ x"$D" = "x" ]; then
if [ -x /sbin/ldconfig ]; then /sbin/ldconfig ; fi
fi