Add relationships for ALPM packages (arch linux) (#2851)

* add alpm relationships

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* tweak reader linter rule to check for reader impl

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update JSON schema with alpm dependency information

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman 2024-05-07 13:29:46 -04:00 committed by GitHub
parent e7b6284039
commit ada8f009d2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 3078 additions and 266 deletions

View File

@ -3,5 +3,5 @@ package internal
const ( const (
// JSONSchemaVersion is the current schema version output by the JSON encoder // JSONSchemaVersion is the current schema version output by the JSON encoder
// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment. // This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
JSONSchemaVersion = "16.0.7" JSONSchemaVersion = "16.0.8"
) )

File diff suppressed because it is too large Load Diff

View File

@ -1,6 +1,6 @@
{ {
"$schema": "https://json-schema.org/draft/2020-12/schema", "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "anchore.io/schema/syft/json/16.0.7/document", "$id": "anchore.io/schema/syft/json/16.0.8/document",
"$ref": "#/$defs/Document", "$ref": "#/$defs/Document",
"$defs": { "$defs": {
"AlpmDbEntry": { "AlpmDbEntry": {
@ -46,6 +46,18 @@
"$ref": "#/$defs/AlpmFileRecord" "$ref": "#/$defs/AlpmFileRecord"
}, },
"type": "array" "type": "array"
},
"provides": {
"items": {
"type": "string"
},
"type": "array"
},
"depends": {
"items": {
"type": "string"
},
"type": "array"
} }
}, },
"type": "object", "type": "object",

View File

@ -27,6 +27,8 @@ type AlpmDBEntry struct {
Reason int `mapstructure:"reason" json:"reason"` Reason int `mapstructure:"reason" json:"reason"`
Files []AlpmFileRecord `mapstructure:"files" json:"files"` Files []AlpmFileRecord `mapstructure:"files" json:"files"`
Backup []AlpmFileRecord `mapstructure:"backup" json:"backup"` Backup []AlpmFileRecord `mapstructure:"backup" json:"backup"`
Provides []string `mapstructure:"provides" json:"provides,omitempty"`
Depends []string `mapstructure:"depends" json:"depends,omitempty"`
} }
type AlpmFileRecord struct { type AlpmFileRecord struct {

View File

@ -4,12 +4,96 @@ Package arch provides a concrete Cataloger implementations for packages relating
package arch package arch
import ( import (
"context"
"strings"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic" "github.com/anchore/syft/syft/pkg/cataloger/generic"
) )
type cataloger struct {
*generic.Cataloger
}
// NewDBCataloger returns a new cataloger object initialized for arch linux pacman database flat-file stores. // NewDBCataloger returns a new cataloger object initialized for arch linux pacman database flat-file stores.
func NewDBCataloger() pkg.Cataloger { func NewDBCataloger() pkg.Cataloger {
return generic.NewCataloger("alpm-db-cataloger"). return cataloger{
WithParserByGlobs(parseAlpmDB, pkg.AlpmDBGlob) Cataloger: generic.NewCataloger("alpm-db-cataloger").
WithParserByGlobs(parseAlpmDB, pkg.AlpmDBGlob),
}
}
func (c cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
pkgs, rels, err := c.Cataloger.Catalog(ctx, resolver)
if err != nil {
return nil, nil, err
}
rels = append(rels, associateRelationships(pkgs)...)
return pkgs, rels, nil
}
// associateRelationships will create relationships between packages based on the "Depends" and "Provides"
// fields for installed packages. If there is an installed package that has a dependency that is (somehow) not installed,
// then that relationship (between the installed and uninstalled package) will NOT be created.
func associateRelationships(pkgs []pkg.Package) (relationships []artifact.Relationship) {
// map["provides" + "package"] -> packages that provide that package
lookup := make(map[string][]pkg.Package)
// read providers and add lookup keys as needed
for _, p := range pkgs {
meta, ok := p.Metadata.(pkg.AlpmDBEntry)
if !ok {
log.Warnf("cataloger failed to extract alpm 'provides' metadata for package %+v", p.Name)
continue
}
// allow for lookup by package name
lookup[p.Name] = append(lookup[p.Name], p)
for _, provides := range meta.Provides {
// allow for lookup by exact specification
lookup[provides] = append(lookup[provides], p)
// allow for lookup by library name only
k := stripVersionSpecifier(provides)
lookup[k] = append(lookup[k], p)
}
}
// read "Depends" and match with provider keys
for _, p := range pkgs {
meta, ok := p.Metadata.(pkg.AlpmDBEntry)
if !ok {
log.Warnf("cataloger failed to extract alpm 'dependency' metadata for package %+v", p.Name)
continue
}
for _, dep := range meta.Depends {
for _, depPkg := range lookup[dep] {
relationships = append(relationships, artifact.Relationship{
From: depPkg,
To: p,
Type: artifact.DependencyOfRelationship,
})
}
}
}
return relationships
}
func stripVersionSpecifier(s string) string {
// examples:
// gcc-libs --> gcc-libs
// libtree-sitter.so=0-64 --> libtree-sitter.so
items := strings.Split(s, "=")
if len(items) == 0 {
return s
}
return strings.TrimSpace(items[0])
} }

View File

@ -12,169 +12,289 @@ import (
) )
func TestAlpmCataloger(t *testing.T) { func TestAlpmCataloger(t *testing.T) {
dbLocation := file.NewLocation("var/lib/pacman/local/gmp-6.2.1-2/desc") gmpDbLocation := file.NewLocation("var/lib/pacman/local/gmp-6.2.1-2/desc")
expectedPkgs := []pkg.Package{ treeSitterDbLocation := file.NewLocation("var/lib/pacman/local/tree-sitter-0.22.6-1/desc")
{ emacsDbLocation := file.NewLocation("var/lib/pacman/local/emacs-29.3-3/desc")
Name: "gmp", fuzzyDbLocation := file.NewLocation("var/lib/pacman/local/fuzzy-1.2-3/desc")
Version: "6.2.1-2", madeupDbLocation := file.NewLocation("var/lib/pacman/local/madeup-20.30-4/desc")
Type: pkg.AlpmPkg,
FoundBy: "alpm-db-cataloger", treeSitterPkg := pkg.Package{
Licenses: pkg.NewLicenseSet( Name: "tree-sitter",
pkg.NewLicenseFromLocations("LGPL3", dbLocation), Version: "0.22.6-1",
pkg.NewLicenseFromLocations("GPL", dbLocation), Type: pkg.AlpmPkg,
), FoundBy: "alpm-db-cataloger",
Locations: file.NewLocationSet(dbLocation), Licenses: pkg.NewLicenseSet(
CPEs: nil, pkg.NewLicenseFromLocations("MIT", treeSitterDbLocation),
PURL: "", ),
Metadata: pkg.AlpmDBEntry{ Locations: file.NewLocationSet(treeSitterDbLocation),
BasePackage: "gmp", Metadata: pkg.AlpmDBEntry{
Package: "gmp", BasePackage: "tree-sitter",
Version: "6.2.1-2", Package: "tree-sitter",
Description: "A free library for arbitrary precision arithmetic", Version: "0.22.6-1",
Architecture: "x86_64", Description: "Incremental parsing library",
Size: 1044438, Architecture: "x86_64",
Packager: "Antonio Rojas <arojas@archlinux.org>", Size: 223539,
URL: "https://gmplib.org/", Packager: "Daniel M. Capella <polyzen@archlinux.org>",
Validation: "pgp", URL: "https://github.com/tree-sitter/tree-sitter",
Reason: 1, Validation: "pgp",
Files: []pkg.AlpmFileRecord{ Reason: 1,
{ Files: []pkg.AlpmFileRecord{},
Path: "/usr", Backup: []pkg.AlpmFileRecord{},
Type: "dir", Provides: []string{"libtree-sitter.so=0-64"},
Digests: []file.Digest{},
},
{
Path: "/usr/include",
Type: "dir",
Digests: []file.Digest{},
},
{
Path: "/usr/include/gmp.h",
Size: "84140",
Digests: []file.Digest{
{Algorithm: "md5", Value: "76595f70565c72550eb520809bf86856"},
{Algorithm: "sha256", Value: "91a614b9202453153fe3b7512d15e89659108b93ce8841c8e13789eb85da9e3a"},
},
},
{
Path: "/usr/include/gmpxx.h",
Size: "129113",
Digests: []file.Digest{
{Algorithm: "md5", Value: "ea3d21de4bcf7c696799c5c55dd3655b"},
{Algorithm: "sha256", Value: "0011ae411a0bc1030e07d968b32fdc1343f5ac2a17b7d28f493e7976dde2ac82"},
},
},
{
Path: "/usr/lib",
Type: "dir",
Digests: []file.Digest{},
},
{
Path: "/usr/lib/libgmp.so",
Type: "link",
Link: "libgmp.so.10.4.1",
Digests: []file.Digest{},
},
{
Path: "/usr/lib/libgmp.so.10",
Type: "link",
Link: "libgmp.so.10.4.1",
Digests: []file.Digest{},
},
{
Path: "/usr/lib/libgmp.so.10.4.1",
Size: "663224",
Digests: []file.Digest{
{Algorithm: "md5", Value: "d6d03eadacdd9048d5b2adf577e9d722"},
{Algorithm: "sha256", Value: "39898bd3d8d6785222432fa8b8aef7ce3b7e5bbfc66a52b7c0da09bed4adbe6a"},
},
},
{
Path: "/usr/lib/libgmpxx.so",
Type: "link",
Link: "libgmpxx.so.4.6.1",
Digests: []file.Digest{},
},
{
Path: "/usr/lib/libgmpxx.so.4",
Type: "link",
Link: "libgmpxx.so.4.6.1",
Digests: []file.Digest{},
},
{
Path: "/usr/lib/libgmpxx.so.4.6.1",
Size: "30680",
Digests: []file.Digest{
{Algorithm: "md5", Value: "dd5f0c4d635fa599fa7f4339c0e8814d"},
{Algorithm: "sha256", Value: "0ef67cbde4841f58d2e4b41f59425eb87c9eeaf4e649c060b326342c53bedbec"},
},
},
{
Path: "/usr/lib/pkgconfig",
Type: "dir",
Digests: []file.Digest{},
},
{
Path: "/usr/lib/pkgconfig/gmp.pc",
Size: "245",
Digests: []file.Digest{
{Algorithm: "md5", Value: "a91a9f1b66218cb77b9cd2cdf341756d"},
{Algorithm: "sha256", Value: "4e9de547a48c4e443781e9fa702a1ec5a23ee28b4bc520306cff2541a855be37"},
},
},
{
Path: "/usr/lib/pkgconfig/gmpxx.pc",
Size: "280",
Digests: []file.Digest{
{Algorithm: "md5", Value: "8c0f54e987934352177a6a30a811b001"},
{Algorithm: "sha256", Value: "fc5dbfbe75977057ba50953d94b9daecf696c9fdfe5b94692b832b44ecca871b"},
},
},
{
Path: "/usr/share",
Type: "dir",
Digests: []file.Digest{},
},
{
Path: "/usr/share/info",
Type: "dir",
Digests: []file.Digest{},
},
{
Path: "/usr/share/info/gmp.info-1.gz",
Size: "85892",
Digests: []file.Digest{
{Algorithm: "md5", Value: "63304d4d2f0247fb8a999fae66a81c19"},
{Algorithm: "sha256", Value: "86288c1531a2789db5da8b9838b5cde4db07bda230ae11eba23a1f33698bd14e"},
},
},
{
Path: "/usr/share/info/gmp.info-2.gz",
Size: "48484",
Digests: []file.Digest{
{Algorithm: "md5", Value: "4bb0dadec416d305232cac6eae712ff7"},
{Algorithm: "sha256", Value: "b7443c1b529588d98a074266087f79b595657ac7274191c34b10a9ceedfa950e"},
},
},
{
Path: "/usr/share/info/gmp.info.gz",
Size: "2380",
Digests: []file.Digest{
{Algorithm: "md5", Value: "cf6880fb0d862ee1da0d13c3831b5720"},
{Algorithm: "sha256", Value: "a13c8eecda3f3e5ad1e09773e47a9686f07d9d494eaddf326f3696bbef1548fd"},
},
},
},
Backup: []pkg.AlpmFileRecord{},
},
}, },
} }
// TODO: relationships are not under test yet emacsPkg := pkg.Package{
var expectedRelationships []artifact.Relationship Name: "emacs",
Version: "29.3-3",
Type: pkg.AlpmPkg,
FoundBy: "alpm-db-cataloger",
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("GPL3", emacsDbLocation),
),
Locations: file.NewLocationSet(emacsDbLocation),
Metadata: pkg.AlpmDBEntry{
BasePackage: "emacs",
Package: "emacs",
Version: "29.3-3",
Description: "The extensible, customizable, self-documenting real-time display editor",
Architecture: "x86_64",
Size: 126427862,
Packager: "Frederik Schwan <freswa@archlinux.org>",
URL: "https://www.gnu.org/software/emacs/emacs.html",
Validation: "pgp",
Files: []pkg.AlpmFileRecord{},
Backup: []pkg.AlpmFileRecord{},
Depends: []string{"libtree-sitter.so=0-64"},
},
}
fuzzyPkg := pkg.Package{
Name: "fuzzy",
Version: "1.2-3",
Type: pkg.AlpmPkg,
FoundBy: "alpm-db-cataloger",
Locations: file.NewLocationSet(
fuzzyDbLocation,
file.NewLocation("var/lib/pacman/local/fuzzy-1.2-3/files"),
),
Metadata: pkg.AlpmDBEntry{
Package: "fuzzy",
Version: "1.2-3",
Files: []pkg.AlpmFileRecord{},
Backup: []pkg.AlpmFileRecord{
{
Path: "/etc/fuzzy.conf",
Digests: []file.Digest{
{Algorithm: "md5", Value: "79fce043df7dfc676ae5ecb903762d8b"},
},
},
},
Depends: []string{"tree-sitter"},
},
}
madeupPkg := pkg.Package{
Name: "madeup",
Version: "20.30-4",
Type: pkg.AlpmPkg,
FoundBy: "alpm-db-cataloger",
Locations: file.NewLocationSet(madeupDbLocation),
Metadata: pkg.AlpmDBEntry{
Package: "madeup",
Version: "20.30-4",
Files: []pkg.AlpmFileRecord{},
Backup: []pkg.AlpmFileRecord{},
Depends: []string{"libtree-sitter.so"},
},
}
gmpPkg := pkg.Package{
Name: "gmp",
Version: "6.2.1-2",
Type: pkg.AlpmPkg,
FoundBy: "alpm-db-cataloger",
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("LGPL3", gmpDbLocation),
pkg.NewLicenseFromLocations("GPL", gmpDbLocation),
),
Locations: file.NewLocationSet(
gmpDbLocation,
file.NewLocation("var/lib/pacman/local/gmp-6.2.1-2/files"),
file.NewLocation("var/lib/pacman/local/gmp-6.2.1-2/mtree"),
),
Metadata: pkg.AlpmDBEntry{
BasePackage: "gmp",
Package: "gmp",
Version: "6.2.1-2",
Description: "A free library for arbitrary precision arithmetic",
Architecture: "x86_64",
Size: 1044438,
Packager: "Antonio Rojas <arojas@archlinux.org>",
URL: "https://gmplib.org/",
Validation: "pgp",
Reason: 1,
Depends: []string{"gcc-libs", "sh", "libtree-sitter.so=1-64"},
Files: []pkg.AlpmFileRecord{
{
Path: "/usr",
Type: "dir",
Digests: []file.Digest{},
},
{
Path: "/usr/include",
Type: "dir",
Digests: []file.Digest{},
},
{
Path: "/usr/include/gmp.h",
Size: "84140",
Digests: []file.Digest{
{Algorithm: "md5", Value: "76595f70565c72550eb520809bf86856"},
{Algorithm: "sha256", Value: "91a614b9202453153fe3b7512d15e89659108b93ce8841c8e13789eb85da9e3a"},
},
},
{
Path: "/usr/include/gmpxx.h",
Size: "129113",
Digests: []file.Digest{
{Algorithm: "md5", Value: "ea3d21de4bcf7c696799c5c55dd3655b"},
{Algorithm: "sha256", Value: "0011ae411a0bc1030e07d968b32fdc1343f5ac2a17b7d28f493e7976dde2ac82"},
},
},
{
Path: "/usr/lib",
Type: "dir",
Digests: []file.Digest{},
},
{
Path: "/usr/lib/libgmp.so",
Type: "link",
Link: "libgmp.so.10.4.1",
Digests: []file.Digest{},
},
{
Path: "/usr/lib/libgmp.so.10",
Type: "link",
Link: "libgmp.so.10.4.1",
Digests: []file.Digest{},
},
{
Path: "/usr/lib/libgmp.so.10.4.1",
Size: "663224",
Digests: []file.Digest{
{Algorithm: "md5", Value: "d6d03eadacdd9048d5b2adf577e9d722"},
{Algorithm: "sha256", Value: "39898bd3d8d6785222432fa8b8aef7ce3b7e5bbfc66a52b7c0da09bed4adbe6a"},
},
},
{
Path: "/usr/lib/libgmpxx.so",
Type: "link",
Link: "libgmpxx.so.4.6.1",
Digests: []file.Digest{},
},
{
Path: "/usr/lib/libgmpxx.so.4",
Type: "link",
Link: "libgmpxx.so.4.6.1",
Digests: []file.Digest{},
},
{
Path: "/usr/lib/libgmpxx.so.4.6.1",
Size: "30680",
Digests: []file.Digest{
{Algorithm: "md5", Value: "dd5f0c4d635fa599fa7f4339c0e8814d"},
{Algorithm: "sha256", Value: "0ef67cbde4841f58d2e4b41f59425eb87c9eeaf4e649c060b326342c53bedbec"},
},
},
{
Path: "/usr/lib/pkgconfig",
Type: "dir",
Digests: []file.Digest{},
},
{
Path: "/usr/lib/pkgconfig/gmp.pc",
Size: "245",
Digests: []file.Digest{
{Algorithm: "md5", Value: "a91a9f1b66218cb77b9cd2cdf341756d"},
{Algorithm: "sha256", Value: "4e9de547a48c4e443781e9fa702a1ec5a23ee28b4bc520306cff2541a855be37"},
},
},
{
Path: "/usr/lib/pkgconfig/gmpxx.pc",
Size: "280",
Digests: []file.Digest{
{Algorithm: "md5", Value: "8c0f54e987934352177a6a30a811b001"},
{Algorithm: "sha256", Value: "fc5dbfbe75977057ba50953d94b9daecf696c9fdfe5b94692b832b44ecca871b"},
},
},
{
Path: "/usr/share",
Type: "dir",
Digests: []file.Digest{},
},
{
Path: "/usr/share/info",
Type: "dir",
Digests: []file.Digest{},
},
{
Path: "/usr/share/info/gmp.info-1.gz",
Size: "85892",
Digests: []file.Digest{
{Algorithm: "md5", Value: "63304d4d2f0247fb8a999fae66a81c19"},
{Algorithm: "sha256", Value: "86288c1531a2789db5da8b9838b5cde4db07bda230ae11eba23a1f33698bd14e"},
},
},
{
Path: "/usr/share/info/gmp.info-2.gz",
Size: "48484",
Digests: []file.Digest{
{Algorithm: "md5", Value: "4bb0dadec416d305232cac6eae712ff7"},
{Algorithm: "sha256", Value: "b7443c1b529588d98a074266087f79b595657ac7274191c34b10a9ceedfa950e"},
},
},
{
Path: "/usr/share/info/gmp.info.gz",
Size: "2380",
Digests: []file.Digest{
{Algorithm: "md5", Value: "cf6880fb0d862ee1da0d13c3831b5720"},
{Algorithm: "sha256", Value: "a13c8eecda3f3e5ad1e09773e47a9686f07d9d494eaddf326f3696bbef1548fd"},
},
},
},
Backup: []pkg.AlpmFileRecord{},
},
}
expectedPkgs := []pkg.Package{
treeSitterPkg,
emacsPkg,
fuzzyPkg,
madeupPkg,
gmpPkg,
}
expectedRelationships := []artifact.Relationship{
{ // exact spec lookup
From: treeSitterPkg,
To: emacsPkg,
Type: artifact.DependencyOfRelationship,
},
{ // package name lookup
From: treeSitterPkg,
To: fuzzyPkg,
Type: artifact.DependencyOfRelationship,
},
{ // library name lookup
From: treeSitterPkg,
To: madeupPkg,
Type: artifact.DependencyOfRelationship,
},
}
pkgtest.NewCatalogTester(). pkgtest.NewCatalogTester().
FromDirectory(t, "test-fixtures/gmp-fixture"). FromDirectory(t, "test-fixtures/installed").
WithCompareOptions(cmpopts.IgnoreFields(pkg.AlpmFileRecord{}, "Time")). WithCompareOptions(cmpopts.IgnoreFields(pkg.AlpmFileRecord{}, "Time")).
Expects(expectedPkgs, expectedRelationships). Expects(expectedPkgs, expectedRelationships).
TestCataloger(t, NewDBCataloger()) TestCataloger(t, NewDBCataloger())

View File

@ -9,13 +9,16 @@ import (
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
) )
func newPackage(m *parsedData, release *linux.Release, dbLocation file.Location) pkg.Package { func newPackage(m *parsedData, release *linux.Release, dbLocation file.Location, otherLocations ...file.Location) pkg.Package {
licenseCandidates := strings.Split(m.Licenses, "\n") licenseCandidates := strings.Split(m.Licenses, "\n")
locs := file.NewLocationSet(dbLocation)
locs.Add(otherLocations...)
p := pkg.Package{ p := pkg.Package{
Name: m.Package, Name: m.Package,
Version: m.Version, Version: m.Version,
Locations: file.NewLocationSet(dbLocation), Locations: locs,
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromLocation(dbLocation.WithoutAnnotations(), licenseCandidates...)...), Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromLocation(dbLocation.WithoutAnnotations(), licenseCandidates...)...),
Type: pkg.AlpmPkg, Type: pkg.AlpmPkg,
PURL: packageURL(m, release), PURL: packageURL(m, release),

View File

@ -6,6 +6,7 @@ import (
"context" "context"
"fmt" "fmt"
"io" "io"
"path"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings" "strings"
@ -15,6 +16,7 @@ import (
"github.com/vbatts/go-mtree" "github.com/vbatts/go-mtree"
"github.com/anchore/syft/internal" "github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
@ -44,33 +46,26 @@ func parseAlpmDB(_ context.Context, resolver file.Resolver, env *generic.Environ
return nil, nil, err return nil, nil, err
} }
base := filepath.Dir(reader.RealPath) if data == nil {
r, err := getFileReader(filepath.Join(base, "mtree"), resolver) return nil, nil, nil
if err != nil {
return nil, nil, err
} }
pkgFiles, err := parseMtree(r) base := path.Dir(reader.RealPath)
if err != nil {
return nil, nil, err
}
// replace the files found the pacman database with the files from the mtree These contain more metadata and // replace the files found the pacman database with the files from the mtree These contain more metadata and
// thus more useful. // thus more useful.
// TODO: probably want to use MTREE and PKGINFO here files, fileLoc := fetchPkgFiles(base, resolver)
data.Files = pkgFiles backups, backupLoc := fetchBackupFiles(base, resolver)
// We only really do this to get any backup database entries from the files database var locs []file.Location
files := filepath.Join(base, "files") if fileLoc != nil {
_, err = getFileReader(files, resolver) locs = append(locs, fileLoc.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.SupportingEvidenceAnnotation))
if err != nil { data.Files = files
return nil, nil, err
} }
filesMetadata, err := parseAlpmDBEntry(reader)
if err != nil { if backupLoc != nil {
return nil, nil, err locs = append(locs, backupLoc.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.SupportingEvidenceAnnotation))
} else if filesMetadata != nil { data.Backup = backups
data.Backup = filesMetadata.Backup
} }
if data.Package == "" { if data.Package == "" {
@ -82,10 +77,67 @@ func parseAlpmDB(_ context.Context, resolver file.Resolver, env *generic.Environ
data, data,
env.LinuxRelease, env.LinuxRelease,
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
locs...,
), ),
}, nil, nil }, nil, nil
} }
func fetchPkgFiles(base string, resolver file.Resolver) ([]pkg.AlpmFileRecord, *file.Location) {
// TODO: probably want to use MTREE and PKGINFO here
target := path.Join(base, "mtree")
loc, err := getLocation(target, resolver)
if err != nil {
log.WithFields("error", err, "path", target).Trace("failed to find mtree file")
return []pkg.AlpmFileRecord{}, nil
}
if loc == nil {
return []pkg.AlpmFileRecord{}, nil
}
reader, err := resolver.FileContentsByLocation(*loc)
if err != nil {
return []pkg.AlpmFileRecord{}, nil
}
defer internal.CloseAndLogError(reader, loc.RealPath)
pkgFiles, err := parseMtree(reader)
if err != nil {
log.WithFields("error", err, "path", target).Trace("failed to parse mtree file")
return []pkg.AlpmFileRecord{}, nil
}
return pkgFiles, loc
}
func fetchBackupFiles(base string, resolver file.Resolver) ([]pkg.AlpmFileRecord, *file.Location) {
// We only really do this to get any backup database entries from the files database
target := filepath.Join(base, "files")
loc, err := getLocation(target, resolver)
if err != nil {
log.WithFields("error", err, "path", target).Trace("failed to find alpm files")
return []pkg.AlpmFileRecord{}, nil
}
if loc == nil {
return []pkg.AlpmFileRecord{}, nil
}
reader, err := resolver.FileContentsByLocation(*loc)
if err != nil {
return []pkg.AlpmFileRecord{}, nil
}
defer internal.CloseAndLogError(reader, loc.RealPath)
filesMetadata, err := parseAlpmDBEntry(reader)
if err != nil {
return []pkg.AlpmFileRecord{}, nil
}
if filesMetadata != nil {
return filesMetadata.Backup, loc
}
return []pkg.AlpmFileRecord{}, loc
}
func parseAlpmDBEntry(reader io.Reader) (*parsedData, error) { func parseAlpmDBEntry(reader io.Reader) (*parsedData, error) {
scanner := newScanner(reader) scanner := newScanner(reader)
metadata, err := parseDatabase(scanner) metadata, err := parseDatabase(scanner)
@ -119,7 +171,7 @@ func newScanner(reader io.Reader) *bufio.Scanner {
return scanner return scanner
} }
func getFileReader(path string, resolver file.Resolver) (io.Reader, error) { func getLocation(path string, resolver file.Resolver) (*file.Location, error) {
locs, err := resolver.FilesByPath(path) locs, err := resolver.FilesByPath(path)
if err != nil { if err != nil {
return nil, err return nil, err
@ -128,13 +180,11 @@ func getFileReader(path string, resolver file.Resolver) (io.Reader, error) {
if len(locs) == 0 { if len(locs) == 0 {
return nil, fmt.Errorf("could not find file: %s", path) return nil, fmt.Errorf("could not find file: %s", path)
} }
// TODO: Should we maybe check if we found the file
dbContentReader, err := resolver.FileContentsByLocation(locs[0]) if len(locs) > 1 {
if err != nil { log.WithFields("path", path).Trace("multiple files found for path, using first path")
return nil, err
} }
defer internal.CloseAndLogError(dbContentReader, locs[0].RealPath) return &locs[0], nil
return dbContentReader, nil
} }
func parseDatabase(b *bufio.Scanner) (*parsedData, error) { func parseDatabase(b *bufio.Scanner) (*parsedData, error) {
@ -157,9 +207,9 @@ func parseDatabase(b *bufio.Scanner) (*parsedData, error) {
case "files": case "files":
var files []map[string]string var files []map[string]string
for _, f := range strings.Split(value, "\n") { for _, f := range strings.Split(value, "\n") {
path := fmt.Sprintf("/%s", f) p := fmt.Sprintf("/%s", f)
if ok := ignoredFiles[path]; !ok { if ok := ignoredFiles[p]; !ok {
files = append(files, map[string]string{"path": path}) files = append(files, map[string]string{"path": p})
} }
} }
pkgFields[key] = files pkgFields[key] = files
@ -167,10 +217,10 @@ func parseDatabase(b *bufio.Scanner) (*parsedData, error) {
var backup []map[string]interface{} var backup []map[string]interface{}
for _, f := range strings.Split(value, "\n") { for _, f := range strings.Split(value, "\n") {
fields := strings.SplitN(f, "\t", 2) fields := strings.SplitN(f, "\t", 2)
path := fmt.Sprintf("/%s", fields[0]) p := fmt.Sprintf("/%s", fields[0])
if ok := ignoredFiles[path]; !ok { if ok := ignoredFiles[p]; !ok {
backup = append(backup, map[string]interface{}{ backup = append(backup, map[string]interface{}{
"path": path, "path": p,
"digests": []file.Digest{{ "digests": []file.Digest{{
Algorithm: "md5", Algorithm: "md5",
Value: fields[1], Value: fields[1],
@ -178,6 +228,8 @@ func parseDatabase(b *bufio.Scanner) (*parsedData, error) {
} }
} }
pkgFields[key] = backup pkgFields[key] = backup
case "depends", "provides":
pkgFields[key] = processLibrarySpecs(value)
case "reason": case "reason":
fallthrough fallthrough
case "size": case "size":
@ -193,6 +245,19 @@ func parseDatabase(b *bufio.Scanner) (*parsedData, error) {
return parsePkgFiles(pkgFields) return parsePkgFiles(pkgFields)
} }
func processLibrarySpecs(value string) []string {
lines := strings.Split(value, "\n")
librarySpecs := make([]string, 0)
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
continue
}
librarySpecs = append(librarySpecs, line)
}
return librarySpecs
}
func parsePkgFiles(pkgFields map[string]interface{}) (*parsedData, error) { func parsePkgFiles(pkgFields map[string]interface{}) (*parsedData, error) {
var entry parsedData var entry parsedData
if err := mapstructure.Decode(pkgFields, &entry); err != nil { if err := mapstructure.Decode(pkgFields, &entry); err != nil {
@ -203,6 +268,10 @@ func parsePkgFiles(pkgFields map[string]interface{}) (*parsedData, error) {
entry.Backup = make([]pkg.AlpmFileRecord, 0) entry.Backup = make([]pkg.AlpmFileRecord, 0)
} }
if entry.Files == nil {
entry.Files = make([]pkg.AlpmFileRecord, 0)
}
if entry.Package == "" && len(entry.Files) == 0 && len(entry.Backup) == 0 { if entry.Package == "" && len(entry.Files) == 0 && len(entry.Backup) == 0 {
return nil, nil return nil, nil
} }

View File

@ -17,74 +17,120 @@ func TestDatabaseParser(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
fixture string fixture string
expected pkg.AlpmDBEntry expected *parsedData
}{ }{
{ {
name: "test alpm database parsing", name: "simple desc parsing",
fixture: "test-fixtures/files", fixture: "test-fixtures/files",
expected: pkg.AlpmDBEntry{ expected: &parsedData{
Backup: []pkg.AlpmFileRecord{ AlpmDBEntry: pkg.AlpmDBEntry{
{ Backup: []pkg.AlpmFileRecord{
Path: "/etc/pacman.conf", {
Digests: []file.Digest{{ Path: "/etc/pacman.conf",
Algorithm: "md5", Digests: []file.Digest{{
Value: "de541390e52468165b96511c4665bff4", Algorithm: "md5",
}}, Value: "de541390e52468165b96511c4665bff4",
}},
},
{
Path: "/etc/makepkg.conf",
Digests: []file.Digest{{
Algorithm: "md5",
Value: "79fce043df7dfc676ae5ecb903762d8b",
}},
},
}, },
{ Files: []pkg.AlpmFileRecord{
Path: "/etc/makepkg.conf", {
Digests: []file.Digest{{ Path: "/etc/",
Algorithm: "md5", },
Value: "79fce043df7dfc676ae5ecb903762d8b", {
}}, Path: "/etc/makepkg.conf",
},
{
Path: "/etc/pacman.conf",
},
{
Path: "/usr/",
},
{
Path: "/usr/bin/",
},
{
Path: "/usr/bin/makepkg",
},
{
Path: "/usr/bin/makepkg-template",
},
{
Path: "/usr/bin/pacman",
},
{
Path: "/usr/bin/pacman-conf",
},
{
Path: "/var/",
},
{
Path: "/var/cache/",
},
{
Path: "/var/cache/pacman/",
},
{
Path: "/var/cache/pacman/pkg/",
},
{
Path: "/var/lib/",
},
{
Path: "/var/lib/pacman/",
},
}, },
}, },
Files: []pkg.AlpmFileRecord{ },
{ },
Path: "/etc/", {
}, name: "with dependencies",
{ fixture: "test-fixtures/installed/var/lib/pacman/local/gmp-6.2.1-2/desc",
Path: "/etc/makepkg.conf", expected: &parsedData{
}, Licenses: "LGPL3\nGPL",
{ AlpmDBEntry: pkg.AlpmDBEntry{
Path: "/etc/pacman.conf", BasePackage: "gmp",
}, Package: "gmp",
{ Version: "6.2.1-2",
Path: "/usr/", Description: "A free library for arbitrary precision arithmetic",
}, Architecture: "x86_64",
{ Size: 1044438,
Path: "/usr/bin/", Packager: "Antonio Rojas <arojas@archlinux.org>",
}, URL: "https://gmplib.org/",
{ Validation: "pgp",
Path: "/usr/bin/makepkg", Reason: 1,
}, Files: []pkg.AlpmFileRecord{},
{ Backup: []pkg.AlpmFileRecord{},
Path: "/usr/bin/makepkg-template", Depends: []string{"gcc-libs", "sh", "libtree-sitter.so=1-64"},
}, },
{ },
Path: "/usr/bin/pacman", },
}, {
{ name: "with provides",
Path: "/usr/bin/pacman-conf", fixture: "test-fixtures/installed/var/lib/pacman/local/tree-sitter-0.22.6-1/desc",
}, expected: &parsedData{
{ Licenses: "MIT",
Path: "/var/", AlpmDBEntry: pkg.AlpmDBEntry{
}, BasePackage: "tree-sitter",
{ Package: "tree-sitter",
Path: "/var/cache/", Version: "0.22.6-1",
}, Description: "Incremental parsing library",
{ Architecture: "x86_64",
Path: "/var/cache/pacman/", Size: 223539,
}, Packager: "Daniel M. Capella <polyzen@archlinux.org>",
{ URL: "https://github.com/tree-sitter/tree-sitter",
Path: "/var/cache/pacman/pkg/", Validation: "pgp",
}, Reason: 1,
{ Files: []pkg.AlpmFileRecord{},
Path: "/var/lib/", Backup: []pkg.AlpmFileRecord{},
}, Provides: []string{"libtree-sitter.so=0-64"},
{
Path: "/var/lib/pacman/",
},
}, },
}, },
}, },
@ -101,13 +147,10 @@ func TestDatabaseParser(t *testing.T) {
entry, err := parseAlpmDBEntry(reader) entry, err := parseAlpmDBEntry(reader)
require.NoError(t, err) require.NoError(t, err)
if diff := cmp.Diff(entry.Files, test.expected.Files); diff != "" { if diff := cmp.Diff(test.expected, entry); diff != "" {
t.Errorf("Files mismatch (-want +got):\n%s", diff) t.Errorf("parsed data mismatch (-want +got):\n%s", diff)
} }
if diff := cmp.Diff(entry.Backup, test.expected.Backup); diff != "" {
t.Errorf("Backup mismatch (-want +got):\n%s", diff)
}
}) })
} }
} }

View File

@ -0,0 +1,38 @@
%NAME%
emacs
%VERSION%
29.3-3
%BASE%
emacs
%DESC%
The extensible, customizable, self-documenting real-time display editor
%URL%
https://www.gnu.org/software/emacs/emacs.html
%ARCH%
x86_64
%BUILDDATE%
1714249917
%INSTALLDATE%
1715026363
%PACKAGER%
Frederik Schwan <freswa@archlinux.org>
%SIZE%
126427862
%LICENSE%
GPL3
%VALIDATION%
pgp
%DEPENDS%
libtree-sitter.so=0-64

View File

@ -0,0 +1,8 @@
%NAME%
fuzzy
%VERSION%
1.2-3
%DEPENDS%
tree-sitter

View File

@ -0,0 +1,6 @@
%FILES%
etc/
etc/fuzzy.conf
%BACKUP%
etc/fuzzy.conf 79fce043df7dfc676ae5ecb903762d8b

View File

@ -0,0 +1,8 @@
%NAME%
madeup
%VERSION%
20.30-4
%DEPENDS%
libtree-sitter.so

View File

@ -0,0 +1,41 @@
%NAME%
tree-sitter
%VERSION%
0.22.6-1
%BASE%
tree-sitter
%DESC%
Incremental parsing library
%URL%
https://github.com/tree-sitter/tree-sitter
%ARCH%
x86_64
%BUILDDATE%
1714945746
%INSTALLDATE%
1715026360
%PACKAGER%
Daniel M. Capella <polyzen@archlinux.org>
%SIZE%
223539
%REASON%
1
%LICENSE%
MIT
%VALIDATION%
pgp
%PROVIDES%
libtree-sitter.so=0-64

View File

@ -8,8 +8,8 @@ import "github.com/quasilyte/go-ruleguard/dsl"
func resourceCleanup(m dsl.Matcher) { func resourceCleanup(m dsl.Matcher) {
m.Match(`$res, $err := $resolver.FileContentsByLocation($loc); if $*_ { $*_ }; $next`). m.Match(`$res, $err := $resolver.FileContentsByLocation($loc); if $*_ { $*_ }; $next`).
Where(m["res"].Type.Implements(`io.Closer`) && Where(m["res"].Type.Implements(`io.Closer`) &&
m["res"].Type.Implements(`io.Reader`) &&
m["err"].Type.Implements(`error`) && m["err"].Type.Implements(`error`) &&
m["res"].Type.Implements(`io.Closer`) &&
!m["next"].Text.Matches(`defer internal.CloseAndLogError`)). !m["next"].Text.Matches(`defer internal.CloseAndLogError`)).
Report(`please call "defer internal.CloseAndLogError($res, $loc.RealPath)" right after checking the error returned from $resolver.FileContentsByLocation.`) Report(`please call "defer internal.CloseAndLogError($res, $loc.RealPath)" right after checking the error returned from $resolver.FileContentsByLocation.`)
} }