mirror of
https://github.com/anchore/syft.git
synced 2026-11-03 16:06:36 +01:00
Add support for scanning IPK packages using the deb-archive-cataloger (#5111)
* Add support for scanning IPK packages using the deb-archive-cataloger Signed-off-by: Silas Parker <silas@srp.me.uk> * fix(debian): share License field parsing between opkg status and archive control files - the control-file `License:` fallback now goes through the same helper as the opkg status DB path, so a package reports the same licenses whether found installed or as an archive - bitbake `&` / `|` are rewritten to SPDX `AND` / `OR` and the whole value is validated as one expression instead of split by hand, which kept grouping like `(A & B) | C` intact - non-SPDX values fall back to a deduped list of names with operators and parens dropped (previously the status DB path emitted `&`, `(BSD`, etc as licenses) - fields over 4KB are ignored, bounding work per archive and keeping deep paren nesting away from the recursive expression parser Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix(debian): split Yocto recipe filenames in the Source field Yocto writes the bitbake recipe filename (`Source: zlib_1.3.2.bb`) where dpkg expects a source package name. This is now split into `Source: zlib` / `SourceVersion: 1.3.2` for both ipk archives and opkg status DB entries, instead of surfacing the `.bb` filename as the upstream package. Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix(debian): do not label ipk archives as Debian packages - `.ipk` archives no longer get a `pkg:deb` PURL from the assumed debian distro; opkg-based distros (Yocto, OpenWrt) have no purl type, so these now match packages found in an opkg status DB, which already get no PURL - docs now cover `.ipk` and note that only ar-framed ipks are supported (OpenWrt's older gzipped-tar framing is rejected as an invalid archive) Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * test(debian): assemble the ipk fixture at build time instead of committing it The fixture Dockerfile now builds a Yocto-shaped `.ipk` from a checked-in `control/` directory (copied from the original poky-tiny build) using `tar`, `zstd` and binutils `ar`, the same way bitbake's `package_ipk` packs one. This drops the prebuilt binary and the 20-30 minute Yocto `Dockerfile.build` while keeping the same archive layout and control metadata. Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Silas Parker <silas@srp.me.uk> Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
co-authored by
Alex Goodman
parent
b0a8de7bf6
commit
546c872c9d
@@ -75,6 +75,7 @@ catalogers:
|
|||||||
method: glob # AUTO-GENERATED
|
method: glob # AUTO-GENERATED
|
||||||
criteria: # AUTO-GENERATED
|
criteria: # AUTO-GENERATED
|
||||||
- '**/*.deb'
|
- '**/*.deb'
|
||||||
|
- '**/*.ipk'
|
||||||
metadata_types: # AUTO-GENERATED
|
metadata_types: # AUTO-GENERATED
|
||||||
- pkg.DpkgArchiveEntry
|
- pkg.DpkgArchiveEntry
|
||||||
package_types: # AUTO-GENERATED
|
package_types: # AUTO-GENERATED
|
||||||
|
|||||||
@@ -18,8 +18,8 @@ func NewDBCataloger() pkg.Cataloger {
|
|||||||
WithProcessors(dependency.Processor(dbEntryDependencySpecifier))
|
WithProcessors(dependency.Processor(dbEntryDependencySpecifier))
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewArchiveCataloger returns a new Debian package cataloger object capable of parsing .deb archive files
|
// NewArchiveCataloger returns a new Debian package cataloger object capable of parsing .deb and .ipk (opkg) archive files
|
||||||
func NewArchiveCataloger() pkg.Cataloger {
|
func NewArchiveCataloger() pkg.Cataloger {
|
||||||
return generic.NewCataloger("deb-archive-cataloger").
|
return generic.NewCataloger("deb-archive-cataloger").
|
||||||
WithParserByGlobs(parseDebArchive, "**/*.deb")
|
WithParserByGlobs(parseDebArchive, "**/*.deb", "**/*.ipk")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -276,6 +276,36 @@ func TestDpkgArchiveCataloger(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
}, {
|
||||||
|
name: "image-single-ipk",
|
||||||
|
expected: []pkg.Package{
|
||||||
|
{
|
||||||
|
Name: "libz1",
|
||||||
|
Version: "1.3.2-r0",
|
||||||
|
FoundBy: "deb-archive-cataloger",
|
||||||
|
Locations: file.NewLocationSet(
|
||||||
|
file.NewLocation("/libz1_1.3.2-r0_x86-64-v3.ipk"),
|
||||||
|
),
|
||||||
|
Type: pkg.DebPkg,
|
||||||
|
Metadata: pkg.DpkgArchiveEntry{
|
||||||
|
Package: "libz1",
|
||||||
|
Source: "zlib",
|
||||||
|
SourceVersion: "1.3.2",
|
||||||
|
Version: "1.3.2-r0",
|
||||||
|
Architecture: "x86-64-v3",
|
||||||
|
Maintainer: "Poky Maintainers <poky@lists.yoctoproject.org>",
|
||||||
|
InstalledSize: 0,
|
||||||
|
Description: `Zlib Compression Library
|
||||||
|
Zlib is a general-purpose, patent-free, lossless data compression library
|
||||||
|
which is used by many different programs.`,
|
||||||
|
Provides: []string{"zlib"},
|
||||||
|
Depends: []string{"musl (>= 1.2.6+git0+9fa28ece75)"},
|
||||||
|
},
|
||||||
|
Licenses: pkg.NewLicenseSet(
|
||||||
|
pkg.NewLicenseFromLocationsWithContext(ctx, "Zlib"),
|
||||||
|
),
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/scylladb/go-set/strset"
|
||||||
|
|
||||||
"github.com/anchore/packageurl-go"
|
"github.com/anchore/packageurl-go"
|
||||||
"github.com/anchore/syft/internal"
|
"github.com/anchore/syft/internal"
|
||||||
"github.com/anchore/syft/internal/log"
|
"github.com/anchore/syft/internal/log"
|
||||||
@@ -61,32 +63,65 @@ func newDpkgPackage(ctx context.Context, d dpkgExtractedMetadata, dbLocation fil
|
|||||||
return p
|
return p
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// a declared License field holds a single expression, a few hundred bytes at most in practice. Anything
|
||||||
|
// much larger only feeds the expression parser (which recurses on parentheses) attacker-sized input.
|
||||||
|
const maxDeclaredLicenseLen = 4096
|
||||||
|
|
||||||
|
var (
|
||||||
|
// opkg/ipkg (Yocto via bitbake) declare licenses with "&" for AND and "|" for OR
|
||||||
|
bitbakeLicenseOperators = strings.NewReplacer("&", " AND ", "|", " OR ")
|
||||||
|
tightenParens = strings.NewReplacer("( ", "(", " )", ")")
|
||||||
|
licenseListSeparators = strings.NewReplacer("(", " ", ")", " ")
|
||||||
|
)
|
||||||
|
|
||||||
// extractDeclaredLicenses converts a License field from the status DB into a license set. Returns nil
|
// extractDeclaredLicenses converts a License field from the status DB into a license set. Returns nil
|
||||||
// for empty input so standard dpkg entries (which never declare License inline) incur no allocation.
|
// for empty input so standard dpkg entries (which never declare License inline) incur no allocation.
|
||||||
// Mirrors the alpine cataloger's approach: keep the value whole if it parses as a valid SPDX expression,
|
|
||||||
// otherwise split on whitespace to handle space-separated lists.
|
|
||||||
func extractDeclaredLicenses(ctx context.Context, raw string, dbLocation file.Location) []pkg.License {
|
func extractDeclaredLicenses(ctx context.Context, raw string, dbLocation file.Location) []pkg.License {
|
||||||
|
values := declaredLicenseValues(raw)
|
||||||
|
if len(values) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return pkg.NewLicensesFromLocationWithContext(ctx, dbLocation, values...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// declaredLicenseValues converts an inline License field (opkg status DB or an archive control file) into
|
||||||
|
// license values. Bitbake operators are rewritten to SPDX ones and the value is kept whole if it then parses
|
||||||
|
// as a valid SPDX expression (mirroring the alpine cataloger), otherwise it is treated as a list of license
|
||||||
|
// names with operators and grouping dropped.
|
||||||
|
func declaredLicenseValues(raw string) []string {
|
||||||
|
raw = strings.TrimSpace(raw)
|
||||||
if raw == "" {
|
if raw == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
licenseStrings := []string{raw}
|
if len(raw) > maxDeclaredLicenseLen {
|
||||||
if _, err := license.ParseExpression(raw); err != nil {
|
log.WithFields("length", len(raw)).Debug("ignoring oversized declared license field")
|
||||||
licenseStrings = strings.Fields(raw)
|
return nil
|
||||||
}
|
}
|
||||||
return pkg.NewLicensesFromLocationWithContext(ctx, dbLocation, licenseStrings...)
|
|
||||||
|
expr := tightenParens.Replace(strings.Join(strings.Fields(bitbakeLicenseOperators.Replace(raw)), " "))
|
||||||
|
if _, err := license.ParseExpression(expr); err == nil {
|
||||||
|
return []string{expr}
|
||||||
|
}
|
||||||
|
|
||||||
|
var values []string
|
||||||
|
seen := strset.New()
|
||||||
|
for _, field := range strings.Fields(licenseListSeparators.Replace(expr)) {
|
||||||
|
if field == "AND" || field == "OR" || seen.Has(field) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen.Add(field)
|
||||||
|
values = append(values, field)
|
||||||
|
}
|
||||||
|
return values
|
||||||
}
|
}
|
||||||
|
|
||||||
func newDebArchivePackage(ctx context.Context, location file.Location, metadata pkg.DpkgArchiveEntry, licenseStrings []string) pkg.Package {
|
func newDebArchivePackage(ctx context.Context, location file.Location, metadata pkg.DpkgArchiveEntry, licenseStrings []string) pkg.Package {
|
||||||
p := pkg.Package{
|
p := pkg.Package{
|
||||||
Name: metadata.Package,
|
Name: metadata.Package,
|
||||||
Version: metadata.Version,
|
Version: metadata.Version,
|
||||||
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, licenseStrings...)...),
|
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, licenseStrings...)...),
|
||||||
Type: pkg.DebPkg,
|
Type: pkg.DebPkg,
|
||||||
PURL: packageURL(
|
PURL: packageURL(pkg.DpkgDBEntry(metadata), archiveDistro(location)),
|
||||||
pkg.DpkgDBEntry(metadata),
|
|
||||||
// we don't know the distro information, but since this is a deb file then we can reasonably assume it is a debian-based distro
|
|
||||||
&linux.Release{IDLike: []string{"debian"}},
|
|
||||||
),
|
|
||||||
Metadata: metadata,
|
Metadata: metadata,
|
||||||
Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
|
Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
|
||||||
}
|
}
|
||||||
@@ -95,6 +130,17 @@ func newDebArchivePackage(ctx context.Context, location file.Location, metadata
|
|||||||
return p
|
return p
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// archiveDistro guesses the distro an archive was built for. We don't know the distro information, but a .deb
|
||||||
|
// can reasonably be assumed to target a debian-based distro. An .ipk targets opkg-based distros (Yocto,
|
||||||
|
// OpenWrt) which have no purl type, so it gets no distro and therefore no PURL, the same as packages found in
|
||||||
|
// an opkg status DB.
|
||||||
|
func archiveDistro(location file.Location) *linux.Release {
|
||||||
|
if strings.EqualFold(path.Ext(location.Path()), ".ipk") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &linux.Release{IDLike: []string{"debian"}}
|
||||||
|
}
|
||||||
|
|
||||||
// PackageURL returns the PURL for the specific Debian package (see https://github.com/package-url/purl-spec)
|
// PackageURL returns the PURL for the specific Debian package (see https://github.com/package-url/purl-spec)
|
||||||
func packageURL(m pkg.DpkgDBEntry, distro *linux.Release) string {
|
func packageURL(m pkg.DpkgDBEntry, distro *linux.Release) string {
|
||||||
if distro == nil {
|
if distro == nil {
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package debian
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"sort"
|
"sort"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/google/go-cmp/cmp"
|
"github.com/google/go-cmp/cmp"
|
||||||
@@ -145,6 +146,36 @@ func Test_extractDeclaredLicenses(t *testing.T) {
|
|||||||
raw: "GPL-2.0 BSD-3-Clause",
|
raw: "GPL-2.0 BSD-3-Clause",
|
||||||
expected: []string{"GPL-2.0", "BSD-3-Clause"},
|
expected: []string{"GPL-2.0", "BSD-3-Clause"},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "bitbake operators become an SPDX expression",
|
||||||
|
raw: "MIT & (BSD-3-Clause | Apache-2.0) & ( GPL-2.0-or-later )",
|
||||||
|
expected: []string{"MIT AND (BSD-3-Clause OR Apache-2.0) AND (GPL-2.0-or-later)"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "bitbake grouping is preserved rather than split",
|
||||||
|
raw: "(MIT & BSD-3-Clause) | Apache-2.0",
|
||||||
|
expected: []string{"(MIT AND BSD-3-Clause) OR Apache-2.0"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "parenthesized alternatives are preserved",
|
||||||
|
raw: "(MIT) | (Apache-2.0)",
|
||||||
|
expected: []string{"(MIT) OR (Apache-2.0)"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non-SPDX names drop operators and grouping",
|
||||||
|
raw: "GPLv2+ & (LGPLv2.1 | BSD) & GPLv2+",
|
||||||
|
expected: []string{"GPLv2+", "LGPLv2.1", "BSD"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "oversized field is ignored",
|
||||||
|
raw: strings.Repeat("MIT & ", maxDeclaredLicenseLen),
|
||||||
|
expected: nil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "deeply nested parens are ignored",
|
||||||
|
raw: strings.Repeat("(", maxDeclaredLicenseLen) + "MIT" + strings.Repeat(")", maxDeclaredLicenseLen),
|
||||||
|
expected: nil,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package debian
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"archive/tar"
|
"archive/tar"
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -23,18 +24,27 @@ import (
|
|||||||
"github.com/anchore/syft/syft/pkg/cataloger/generic"
|
"github.com/anchore/syft/syft/pkg/cataloger/generic"
|
||||||
)
|
)
|
||||||
|
|
||||||
// parseDebArchive parses a Debian package archive (.deb) file and returns the packages it contains.
|
// parseDebArchive parses a Debian package archive (.deb) or opkg package (.ipk) file and returns the packages it
|
||||||
// A .deb file is an ar archive containing three main files:
|
// contains. Both are ar archives containing three main files:
|
||||||
// - debian-binary: Version of the .deb format (usually "2.0")
|
// - debian-binary: Version of the archive format (usually "2.0")
|
||||||
// - control.tar.gz/xz/zst: Contains package metadata (control file, md5sums, conffiles)
|
// - control.tar.gz/xz/zst: Contains package metadata (control file, md5sums, conffiles)
|
||||||
// - data.tar.gz/xz/zst: Contains the actual files to be installed (not processed by this cataloger)
|
// - data.tar.gz/xz/zst: Contains the actual files to be installed (not processed by this cataloger)
|
||||||
//
|
//
|
||||||
// This function extracts and processes the control information to create package metadata.
|
// This function extracts and processes the control information to create package metadata. Licenses come
|
||||||
|
// from copyright files in the data tar, falling back to the control file's License field (common for .ipk).
|
||||||
|
// Only ar-framed .ipk files are supported; the older gzipped-tar framing (e.g. OpenWrt's ipkg-build) is
|
||||||
|
// rejected as an invalid archive.
|
||||||
func parseDebArchive(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
|
func parseDebArchive(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
|
||||||
arReader := ar.NewReader(reader)
|
validatedReader, err := newValidatedArReader(reader)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("invalid ar archive: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
arReader := ar.NewReader(validatedReader)
|
||||||
|
|
||||||
var metadata *pkg.DpkgArchiveEntry
|
var metadata *pkg.DpkgArchiveEntry
|
||||||
var licenses []string
|
var licenses []string
|
||||||
|
var ctrlLicenses []string
|
||||||
var unknownErr error
|
var unknownErr error
|
||||||
var sawControl, sawData bool
|
var sawControl, sawData bool
|
||||||
for {
|
for {
|
||||||
@@ -56,7 +66,7 @@ func parseDebArchive(ctx context.Context, _ file.Resolver, _ *generic.Environmen
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, unknown.New(reader.Location, fmt.Errorf("failed to decompress control.tar.* file: %w", err))
|
return nil, nil, unknown.New(reader.Location, fmt.Errorf("failed to decompress control.tar.* file: %w", err))
|
||||||
}
|
}
|
||||||
metadata, err = processControlTar(dcReader)
|
metadata, ctrlLicenses, err = processControlTar(dcReader)
|
||||||
switch {
|
switch {
|
||||||
case err != nil && metadata == nil:
|
case err != nil && metadata == nil:
|
||||||
return nil, nil, unknown.New(reader.Location, fmt.Errorf("failed to process control.tar.* file: %w", err))
|
return nil, nil, unknown.New(reader.Location, fmt.Errorf("failed to process control.tar.* file: %w", err))
|
||||||
@@ -87,6 +97,10 @@ func parseDebArchive(ctx context.Context, _ file.Resolver, _ *generic.Environmen
|
|||||||
return nil, nil, unknown.New(reader.Location, fmt.Errorf("no application found described in .dpkg archive"))
|
return nil, nil, unknown.New(reader.Location, fmt.Errorf("no application found described in .dpkg archive"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(licenses) == 0 && len(ctrlLicenses) > 0 {
|
||||||
|
licenses = ctrlLicenses
|
||||||
|
}
|
||||||
|
|
||||||
// a partial parse still yields a usable package, so report what went wrong alongside it rather than
|
// a partial parse still yields a usable package, so report what went wrong alongside it rather than
|
||||||
// dropping either one
|
// dropping either one
|
||||||
return []pkg.Package{
|
return []pkg.Package{
|
||||||
@@ -128,10 +142,10 @@ func processDataTar(dcReader io.ReadCloser) ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// processControlTar always returns whatever metadata it managed to parse, even alongside a non-nil
|
// processControlTar always returns whatever metadata it managed to parse, even alongside a non-nil
|
||||||
// error. A non-nil error with non-nil metadata means the package is usable but incomplete (e.g. a
|
// error, along with any licenses declared in the control file's License field. A non-nil error with
|
||||||
// clipped file listing); nil metadata means nothing usable was found and the caller should treat it
|
// non-nil metadata means the package is usable but incomplete (e.g. a clipped file listing); nil
|
||||||
// as fatal.
|
// metadata means nothing usable was found and the caller should treat it as fatal.
|
||||||
func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) {
|
func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, []string, error) {
|
||||||
defer internal.CloseAndLogError(dcReader, "")
|
defer internal.CloseAndLogError(dcReader, "")
|
||||||
|
|
||||||
tarReader := tar.NewReader(dcReader)
|
tarReader := tar.NewReader(dcReader)
|
||||||
@@ -140,6 +154,7 @@ func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) {
|
|||||||
var files []pkg.DpkgFileRecord
|
var files []pkg.DpkgFileRecord
|
||||||
var confFileRecords []pkg.DpkgFileRecord
|
var confFileRecords []pkg.DpkgFileRecord
|
||||||
var listingErr error
|
var listingErr error
|
||||||
|
var licenses []string
|
||||||
|
|
||||||
for {
|
for {
|
||||||
header, err := tarReader.Next()
|
header, err := tarReader.Next()
|
||||||
@@ -147,7 +162,7 @@ func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) {
|
|||||||
break
|
break
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return metadata, fmt.Errorf("failed to read control tar: %w", err)
|
return metadata, nil, fmt.Errorf("failed to read control tar: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
switch filepath.Base(header.Name) {
|
switch filepath.Base(header.Name) {
|
||||||
@@ -155,12 +170,13 @@ func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) {
|
|||||||
// parseDpkgStatus already streams via bufio.Reader
|
// parseDpkgStatus already streams via bufio.Reader
|
||||||
entries, err := parseDpkgStatus(tarReader)
|
entries, err := parseDpkgStatus(tarReader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to parse control file: %w", err)
|
return nil, nil, fmt.Errorf("failed to parse control file: %w", err)
|
||||||
}
|
}
|
||||||
if len(entries) == 0 {
|
if len(entries) == 0 {
|
||||||
return nil, fmt.Errorf("no package entries found in control file")
|
return nil, nil, fmt.Errorf("no package entries found in control file")
|
||||||
}
|
}
|
||||||
entry := pkg.DpkgArchiveEntry(entries[0].toDpkgEntry())
|
entry := pkg.DpkgArchiveEntry(entries[0].toDpkgEntry())
|
||||||
|
licenses = declaredLicenseValues(entries[0].License)
|
||||||
metadata = &entry
|
metadata = &entry
|
||||||
case "md5sums":
|
case "md5sums":
|
||||||
// parseDpkgMD5Info streams via bufio.Scanner and reports its own clipping/scan errors
|
// parseDpkgMD5Info streams via bufio.Scanner and reports its own clipping/scan errors
|
||||||
@@ -176,7 +192,7 @@ func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if metadata == nil {
|
if metadata == nil {
|
||||||
return nil, fmt.Errorf("control file not found in archive")
|
return nil, nil, fmt.Errorf("control file not found in archive")
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(confFileRecords) > 0 && len(files) > 0 {
|
if len(confFileRecords) > 0 && len(files) > 0 {
|
||||||
@@ -195,7 +211,22 @@ func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) {
|
|||||||
|
|
||||||
// a clipped or unreadable file listing leaves the package usable, so hand both back and let the
|
// a clipped or unreadable file listing leaves the package usable, so hand both back and let the
|
||||||
// caller decide
|
// caller decide
|
||||||
return metadata, listingErr
|
return metadata, licenses, listingErr
|
||||||
|
}
|
||||||
|
|
||||||
|
// newValidatedArReader checks for the ar global header before handing the stream to the ar reader, which
|
||||||
|
// otherwise skips those bytes unchecked and reads garbage headers from non-ar input.
|
||||||
|
func newValidatedArReader(reader io.ReadCloser) (io.ReadCloser, error) {
|
||||||
|
prefix := make([]byte, len(ar.GLOBAL_HEADER))
|
||||||
|
if _, err := io.ReadFull(reader, prefix); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read ar header: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !bytes.Equal(prefix, []byte(ar.GLOBAL_HEADER)) {
|
||||||
|
return nil, fmt.Errorf("expected ar header %q, got %q", string(ar.GLOBAL_HEADER), string(prefix))
|
||||||
|
}
|
||||||
|
|
||||||
|
return io.NopCloser(io.MultiReader(bytes.NewReader(prefix), reader)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// a .deb's member sizes are bounded by the archive, but what they decompress to is not: xz and zstd
|
// a .deb's member sizes are bounded by the archive, but what they decompress to is not: xz and zstd
|
||||||
|
|||||||
@@ -26,12 +26,13 @@ import (
|
|||||||
func TestProcessControlTar(t *testing.T) {
|
func TestProcessControlTar(t *testing.T) {
|
||||||
tarBytes := createTestTarWithControlFiles(t)
|
tarBytes := createTestTarWithControlFiles(t)
|
||||||
|
|
||||||
metadata, err := processControlTar(io.NopCloser(bytes.NewReader(tarBytes)))
|
metadata, licenses, err := processControlTar(io.NopCloser(bytes.NewReader(tarBytes)))
|
||||||
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NotNil(t, metadata)
|
require.NotNil(t, metadata)
|
||||||
|
|
||||||
assert.Equal(t, "test-package", metadata.Package)
|
assert.Equal(t, "test-package", metadata.Package)
|
||||||
|
assert.Equal(t, []string{"MIT"}, licenses)
|
||||||
assert.Equal(t, "1.0.0", metadata.Version)
|
assert.Equal(t, "1.0.0", metadata.Version)
|
||||||
|
|
||||||
// md5sums should have been parsed into file records
|
// md5sums should have been parsed into file records
|
||||||
@@ -43,6 +44,14 @@ func TestProcessControlTar(t *testing.T) {
|
|||||||
assert.True(t, metadata.Files[0].IsConfigFile, "file listed in conffiles should be marked as config")
|
assert.True(t, metadata.Files[0].IsConfigFile, "file listed in conffiles should be marked as config")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseDebArchive_RejectsInvalidArchiveHeader(t *testing.T) {
|
||||||
|
reader := file.NewLocationReadCloser(file.NewLocation("test.deb"), io.NopCloser(bytes.NewReader([]byte("not-an-ar-archive"))))
|
||||||
|
|
||||||
|
_, _, err := parseDebArchive(context.Background(), nil, nil, reader)
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "ar archive")
|
||||||
|
}
|
||||||
|
|
||||||
func TestProcessControlTar_ConfigFileMarking(t *testing.T) {
|
func TestProcessControlTar_ConfigFileMarking(t *testing.T) {
|
||||||
// Create a tar where conffiles lists paths that overlap with md5sums entries
|
// Create a tar where conffiles lists paths that overlap with md5sums entries
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
@@ -61,9 +70,10 @@ func TestProcessControlTar_ConfigFileMarking(t *testing.T) {
|
|||||||
|
|
||||||
require.NoError(t, tw.Close())
|
require.NoError(t, tw.Close())
|
||||||
|
|
||||||
metadata, err := processControlTar(io.NopCloser(bytes.NewReader(buf.Bytes())))
|
metadata, licenses, err := processControlTar(io.NopCloser(bytes.NewReader(buf.Bytes())))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Len(t, metadata.Files, 3)
|
require.Len(t, metadata.Files, 3)
|
||||||
|
assert.Empty(t, licenses)
|
||||||
|
|
||||||
assert.True(t, metadata.Files[0].IsConfigFile, "first file should be marked as config file")
|
assert.True(t, metadata.Files[0].IsConfigFile, "first file should be marked as config file")
|
||||||
assert.True(t, metadata.Files[1].IsConfigFile, "second file should be marked as config file")
|
assert.True(t, metadata.Files[1].IsConfigFile, "second file should be marked as config file")
|
||||||
@@ -75,7 +85,7 @@ func createTestTarWithControlFiles(t *testing.T) []byte {
|
|||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
tw := tar.NewWriter(&buf)
|
tw := tar.NewWriter(&buf)
|
||||||
|
|
||||||
controlContent := "Package: test-package\nVersion: 1.0.0\nArchitecture: all\nMaintainer: Test <test@example.com>\nDescription: Test package\n"
|
controlContent := "Package: test-package\nVersion: 1.0.0\nArchitecture: all\nMaintainer: Test <test@example.com>\nDescription: Test package\nLicense: MIT\n"
|
||||||
writeTarEntry(t, tw, "control", controlContent)
|
writeTarEntry(t, tw, "control", controlContent)
|
||||||
|
|
||||||
md5Content := "d41d8cd98f00b204e9800998ecf8427e usr/bin/test-command\n"
|
md5Content := "d41d8cd98f00b204e9800998ecf8427e usr/bin/test-command\n"
|
||||||
@@ -165,7 +175,7 @@ func Test_processControlTar_rejectsABombedControlMember(t *testing.T) {
|
|||||||
writeTarEntry(t, tw, "md5sums", strings.Repeat("d41d8cd98f00b204e9800998ecf8427e usr/bin/x\n", 20000))
|
writeTarEntry(t, tw, "md5sums", strings.Repeat("d41d8cd98f00b204e9800998ecf8427e usr/bin/x\n", 20000))
|
||||||
require.NoError(t, tw.Close())
|
require.NoError(t, tw.Close())
|
||||||
|
|
||||||
metadata, err := processControlTar(newBoundedReadCloser(io.NopCloser(bytes.NewReader(tarBuf.Bytes())), 4096))
|
metadata, _, err := processControlTar(newBoundedReadCloser(io.NopCloser(bytes.NewReader(tarBuf.Bytes())), 4096))
|
||||||
|
|
||||||
require.ErrorIs(t, err, errDecompressedTooLarge)
|
require.ErrorIs(t, err, errDecompressedTooLarge)
|
||||||
require.NotNil(t, metadata, "a control tar read failure after \"control\" was parsed must not drop the package")
|
require.NotNil(t, metadata, "a control tar read failure after \"control\" was parsed must not drop the package")
|
||||||
@@ -181,7 +191,7 @@ func Test_processControlTar_clippedListingIsUsableButReported(t *testing.T) {
|
|||||||
writeTarEntry(t, tw, "md5sums", strings.Repeat("d41d8cd98f00b204e9800998ecf8427e usr/bin/x\n", maxDpkgFileRecords+10))
|
writeTarEntry(t, tw, "md5sums", strings.Repeat("d41d8cd98f00b204e9800998ecf8427e usr/bin/x\n", maxDpkgFileRecords+10))
|
||||||
require.NoError(t, tw.Close())
|
require.NoError(t, tw.Close())
|
||||||
|
|
||||||
metadata, err := processControlTar(io.NopCloser(bytes.NewReader(tarBuf.Bytes())))
|
metadata, _, err := processControlTar(io.NopCloser(bytes.NewReader(tarBuf.Bytes())))
|
||||||
|
|
||||||
require.Error(t, err, "clipping must be reported")
|
require.Error(t, err, "clipping must be reported")
|
||||||
require.ErrorIs(t, err, errClippedFileListing)
|
require.ErrorIs(t, err, errClippedFileListing)
|
||||||
@@ -418,3 +428,18 @@ func Test_decompressionStream_failsEarlyInsteadOfBuffering(t *testing.T) {
|
|||||||
assert.Less(t, bounded, uint64(8*intFile.MB),
|
assert.Less(t, bounded, uint64(8*intFile.MB),
|
||||||
"a bounded stream must stop at the cap, not buffer the whole member and check afterwards")
|
"a bounded stream must stop at the cap, not buffer the whole member and check afterwards")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProcessControlTar_DeclaredLicense(t *testing.T) {
|
||||||
|
var buf bytes.Buffer
|
||||||
|
tw := tar.NewWriter(&buf)
|
||||||
|
|
||||||
|
controlContent := "Package: test-package\nVersion: 1.0.0\nArchitecture: all\nLicense: MIT & (BSD-3-Clause | Apache-2.0) & ( GPL-2.0-or-later )\n"
|
||||||
|
writeTarEntry(t, tw, "control", controlContent)
|
||||||
|
|
||||||
|
require.NoError(t, tw.Close())
|
||||||
|
|
||||||
|
metadata, licenses, err := processControlTar(io.NopCloser(bytes.NewReader(buf.Bytes())))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, metadata)
|
||||||
|
assert.Equal(t, []string{"MIT AND (BSD-3-Clause OR Apache-2.0) AND (GPL-2.0-or-later)"}, licenses)
|
||||||
|
}
|
||||||
|
|||||||
@@ -183,9 +183,11 @@ func parseDpkgStatusEntry(reader *bufio.Reader) (*dpkgExtractedMetadata, error)
|
|||||||
}
|
}
|
||||||
|
|
||||||
sourceName, sourceVersion := extractSourceVersion(raw.Source)
|
sourceName, sourceVersion := extractSourceVersion(raw.Source)
|
||||||
|
if sourceName != "" {
|
||||||
|
raw.Source = sourceName
|
||||||
|
}
|
||||||
if sourceVersion != "" {
|
if sourceVersion != "" {
|
||||||
raw.SourceVersion = sourceVersion
|
raw.SourceVersion = sourceVersion
|
||||||
raw.Source = sourceName
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if raw.Package == "" {
|
if raw.Package == "" {
|
||||||
@@ -263,8 +265,14 @@ func extractAllFields(reader *bufio.Reader) (map[string]any, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// If the source entry string is of the form "<name> (<version>)" then parse and return the components, if
|
// If the source entry string is of the form "<name> (<version>)" then parse and return the components, if
|
||||||
// of the "<name>" form, then return name and nil
|
// of the "<name>" form, then return name and nil. Yocto (opkg/ipk) instead writes the bitbake recipe filename
|
||||||
|
// ("<name>_<version>.bb"), which is split the same way.
|
||||||
func extractSourceVersion(source string) (string, string) {
|
func extractSourceVersion(source string) (string, string) {
|
||||||
|
if recipe, ok := strings.CutSuffix(source, ".bb"); ok && !strings.ContainsAny(recipe, " (") {
|
||||||
|
// bitbake reserves "_" as the recipe name / version separator
|
||||||
|
name, version, _ := strings.Cut(recipe, "_")
|
||||||
|
return name, version
|
||||||
|
}
|
||||||
// special handling for the Source field since it has formatted data
|
// special handling for the Source field since it has formatted data
|
||||||
match := internal.MatchNamedCaptureGroups(sourceRegexp, source)
|
match := internal.MatchNamedCaptureGroups(sourceRegexp, source)
|
||||||
return match["name"], match["version"]
|
return match["name"], match["version"]
|
||||||
|
|||||||
@@ -366,6 +366,16 @@ func TestSourceVersionExtract(t *testing.T) {
|
|||||||
input: "",
|
input: "",
|
||||||
expected: []string{"", ""},
|
expected: []string{"", ""},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "bitbake recipe with version",
|
||||||
|
input: "zlib_1.3.2.bb",
|
||||||
|
expected: []string{"zlib", "1.3.2"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "bitbake recipe without version",
|
||||||
|
input: "packagegroup-core-boot.bb",
|
||||||
|
expected: []string{"packagegroup-core-boot", ""},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# assembles a Yocto-style .ipk the same way bitbake's package_ipk does (opkg-build: a GNU ar archive of
|
||||||
|
# debian-binary, control.tar.gz and data.tar.zst) without paying for a full Yocto build. control/ is copied
|
||||||
|
# verbatim from libz1_1.3.2-r0_x86-64-v3.ipk built by poky-tiny (wrynose); the library itself is a stub
|
||||||
|
# since only the archive shape and control metadata matter here.
|
||||||
|
FROM alpine:3.18.4@sha256:48d9183eb12a05c99bcc0bf44a003607b8e941e1d4f41f9ad12bdcc4b5672f86 AS build
|
||||||
|
|
||||||
|
RUN apk add --no-cache binutils zstd
|
||||||
|
|
||||||
|
COPY control /pkg/control
|
||||||
|
|
||||||
|
RUN mkdir -p /pkg/data/usr/lib \
|
||||||
|
&& echo stub > /pkg/data/usr/lib/libz.so.1.3.2 \
|
||||||
|
&& ln -s libz.so.1.3.2 /pkg/data/usr/lib/libz.so.1 \
|
||||||
|
&& cd /pkg \
|
||||||
|
&& echo 2.0 > debian-binary \
|
||||||
|
&& tar -C control -czf control.tar.gz . \
|
||||||
|
&& tar -C data -cf - . | zstd -q -o data.tar.zst \
|
||||||
|
&& ar rcD /libz1_1.3.2-r0_x86-64-v3.ipk debian-binary control.tar.gz data.tar.zst
|
||||||
|
|
||||||
|
FROM scratch
|
||||||
|
COPY --from=build /libz1_1.3.2-r0_x86-64-v3.ipk /
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
Package: libz1
|
||||||
|
Version: 1.3.2-r0
|
||||||
|
Description: Zlib Compression Library
|
||||||
|
Zlib is a general-purpose, patent-free, lossless data compression library
|
||||||
|
which is used by many different programs.
|
||||||
|
Section: libs
|
||||||
|
Priority: optional
|
||||||
|
Maintainer: Poky Maintainers <poky@lists.yoctoproject.org>
|
||||||
|
License: Zlib
|
||||||
|
Architecture: x86-64-v3
|
||||||
|
OE: zlib
|
||||||
|
Homepage: http://zlib.net/
|
||||||
|
Depends: musl (>= 1.2.6+git0+9fa28ece75)
|
||||||
|
Provides: zlib
|
||||||
|
Source: zlib_1.3.2.bb
|
||||||
+5
@@ -0,0 +1,5 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -e
|
||||||
|
if [ x"$D" = "x" ]; then
|
||||||
|
if [ -x /sbin/ldconfig ]; then /sbin/ldconfig ; fi
|
||||||
|
fi
|
||||||
Reference in New Issue
Block a user