diff --git a/Makefile b/Makefile index 492c107d7..3cc7a8ed1 100644 --- a/Makefile +++ b/Makefile @@ -32,9 +32,13 @@ test: unit integration ## Run all tests (currently unit & integration) help: @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "$(BOLD)$(CYAN)%-25s$(RESET)%s\n", $$1, $$2}' -ci-bootstrap: bootstrap +ci-bootstrap: ci-lib-dependencies bootstrap sudo apt install -y bc +ci-lib-dependencies: + # libdb5.3-dev and libssl-dev are required for Berkeley DB C bindings for RPM DB support + sudo apt install -y libdb5.3-dev libssl-dev + bootstrap: ## Download and install all project dependencies (+ prep tooling in the ./tmp dir) $(call title,Downloading dependencies) # prep temp dirs diff --git a/go.mod b/go.mod index 9c9f19d43..34a55f681 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 // indirect github.com/hashicorp/go-multierror v1.1.0 github.com/hashicorp/go-version v1.2.0 + github.com/knqyf263/go-rpmdb v0.0.0-20190501070121-10a1c42a10dc github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.3.1 github.com/sergi/go-diff v1.1.0 diff --git a/go.sum b/go.sum index f1212db41..f84821264 100644 --- a/go.sum +++ b/go.sum @@ -530,6 +530,10 @@ github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/knqyf263/berkeleydb v0.0.0-20190501065933-fafe01fb9662 h1:UGS0RbPHwXJkq8tcba8OD0nvVUWLf2h7uUJznuHPPB0= +github.com/knqyf263/berkeleydb v0.0.0-20190501065933-fafe01fb9662/go.mod h1:bu1CcN4tUtoRcI/B/RFHhxMNKFHVq/c3SV+UTyduoXg= +github.com/knqyf263/go-rpmdb v0.0.0-20190501070121-10a1c42a10dc h1:pumO9pqmRAjvic6oove22RGh9wDZQnj96XQjJSbSEPs= +github.com/knqyf263/go-rpmdb v0.0.0-20190501070121-10a1c42a10dc/go.mod h1:MrSSvdMpTSymaQWk1yFr9sxFSyQmKMj6jkbvGrchBV8= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= @@ -1087,6 +1091,7 @@ golang.org/x/tools v0.0.0-20200502202811-ed308ab3e770/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d h1:SR+e35rACZFBohNb4Om1ibX6N3iO0FtdbwqGSuD9dBU= golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/imgbom/cataloger/controller.go b/imgbom/cataloger/controller.go index 294a13b29..efc51abb0 100644 --- a/imgbom/cataloger/controller.go +++ b/imgbom/cataloger/controller.go @@ -4,6 +4,7 @@ import ( "github.com/anchore/imgbom/imgbom/cataloger/bundler" "github.com/anchore/imgbom/imgbom/cataloger/dpkg" "github.com/anchore/imgbom/imgbom/cataloger/python" + "github.com/anchore/imgbom/imgbom/cataloger/rpmdb" "github.com/anchore/imgbom/imgbom/event" "github.com/anchore/imgbom/imgbom/pkg" "github.com/anchore/imgbom/imgbom/scope" @@ -44,6 +45,7 @@ func newController() controller { ctrlr.add(dpkg.NewCataloger()) ctrlr.add(bundler.NewCataloger()) ctrlr.add(python.NewCataloger()) + ctrlr.add(rpmdb.NewCataloger()) return ctrlr } diff --git a/imgbom/cataloger/rpmdb/cataloger.go b/imgbom/cataloger/rpmdb/cataloger.go new file mode 100644 index 000000000..b460cc783 --- /dev/null +++ b/imgbom/cataloger/rpmdb/cataloger.go @@ -0,0 +1,34 @@ +package rpmdb + +import ( + "github.com/anchore/imgbom/imgbom/cataloger/common" + "github.com/anchore/imgbom/imgbom/pkg" + "github.com/anchore/imgbom/imgbom/scope" + "github.com/anchore/stereoscope/pkg/file" +) + +type Cataloger struct { + cataloger common.GenericCataloger +} + +func NewCataloger() *Cataloger { + pathParsers := map[string]common.ParserFn{ + "/var/lib/rpm/Packages": parseRpmDB, + } + + return &Cataloger{ + cataloger: common.NewGenericCataloger(pathParsers, nil), + } +} + +func (a *Cataloger) Name() string { + return "rpmdb-cataloger" +} + +func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference { + return a.cataloger.SelectFiles(resolver) +} + +func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) { + return a.cataloger.Catalog(contents, a.Name()) +} diff --git a/imgbom/cataloger/rpmdb/parse_rpmdb.go b/imgbom/cataloger/rpmdb/parse_rpmdb.go new file mode 100644 index 000000000..058629144 --- /dev/null +++ b/imgbom/cataloger/rpmdb/parse_rpmdb.go @@ -0,0 +1,62 @@ +package rpmdb + +import ( + "fmt" + "io" + "io/ioutil" + "os" + + "github.com/anchore/imgbom/imgbom/pkg" + "github.com/anchore/imgbom/internal" + "github.com/anchore/imgbom/internal/log" + rpmdb "github.com/knqyf263/go-rpmdb/pkg" +) + +func parseRpmDB(reader io.Reader) ([]pkg.Package, error) { + f, err := ioutil.TempFile("", internal.ApplicationName+"-rpmdb") + if err != nil { + return nil, fmt.Errorf("failed to create temp rpmdb file: %w", err) + } + + defer func() { + err = os.Remove(f.Name()) + if err != nil { + log.Errorf("failed to remove temp rpmdb file: %+v", err) + } + }() + + _, err = io.Copy(f, reader) + if err != nil { + return nil, fmt.Errorf("failed to copy rpmdb contents to temp file: %w", err) + } + + db := rpmdb.DB{} + err = db.Open(f.Name()) + if err != nil { + return nil, err + } + + pkgList, err := db.ListPackages() + if err != nil { + return nil, err + } + + allPkgs := make([]pkg.Package, 0) + + for _, entry := range pkgList { + p := pkg.Package{ + Name: entry.Name, + Version: entry.Version, + Type: pkg.RpmPkg, + Metadata: pkg.RpmMetadata{ + Epoch: entry.Epoch, + Arch: entry.Arch, + Release: entry.Release, + }, + } + + allPkgs = append(allPkgs, p) + } + + return allPkgs, nil +} diff --git a/imgbom/cataloger/rpmdb/parse_rpmdb_test.go b/imgbom/cataloger/rpmdb/parse_rpmdb_test.go new file mode 100644 index 000000000..d6c956f71 --- /dev/null +++ b/imgbom/cataloger/rpmdb/parse_rpmdb_test.go @@ -0,0 +1,50 @@ +package rpmdb + +import ( + "github.com/anchore/imgbom/imgbom/pkg" + "github.com/go-test/deep" + "os" + "testing" +) + +func TestParseRpmDB(t *testing.T) { + expected := map[string]pkg.Package{ + "dive": { + Name: "dive", + Version: "0.9.2", + Type: pkg.RpmPkg, + Metadata: pkg.RpmMetadata{ + Epoch: 0, + Arch: "x86_64", + Release: "1", + }, + }, + } + + fixture, err := os.Open("test-fixtures/Packages") + if err != nil { + t.Fatalf("failed to open fixture: %+v", err) + } + + actual, err := parseRpmDB(fixture) + if err != nil { + t.Fatalf("failed to parse rpmdb: %+v", err) + } + + if len(actual) != 1 { + for _, a := range actual { + t.Log(" ", a) + } + t.Fatalf("unexpected package count: %d!=%d", len(actual), 1) + } + + for _, a := range actual { + e := expected[a.Name] + diffs := deep.Equal(a, e) + if len(diffs) > 0 { + for _, d := range diffs { + t.Errorf("diff: %+v", d) + } + } + } +} diff --git a/imgbom/cataloger/rpmdb/test-fixtures/Packages b/imgbom/cataloger/rpmdb/test-fixtures/Packages new file mode 100644 index 000000000..ef499f2f8 Binary files /dev/null and b/imgbom/cataloger/rpmdb/test-fixtures/Packages differ diff --git a/imgbom/cataloger/rpmdb/test-fixtures/generate-fixture.sh b/imgbom/cataloger/rpmdb/test-fixtures/generate-fixture.sh new file mode 100755 index 000000000..80bf0996c --- /dev/null +++ b/imgbom/cataloger/rpmdb/test-fixtures/generate-fixture.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -eux + +docker create --name generate-rpmdb-fixture centos:latest sh -c 'tail -f /dev/null' + +function cleanup { + docker kill generate-rpmdb-fixture + docker rm generate-rpmdb-fixture +} +trap cleanup EXIT + +docker start generate-rpmdb-fixture +docker exec -i --tty=false generate-rpmdb-fixture bash <<-EOF + mkdir -p /scratch + cd /scratch + rpm --initdb --dbpath /scratch + curl -sSLO https://github.com/wagoodman/dive/releases/download/v0.9.2/dive_0.9.2_linux_amd64.rpm + rpm --dbpath /scratch -ivh dive_0.9.2_linux_amd64.rpm + rm dive_0.9.2_linux_amd64.rpm + rpm --dbpath /scratch -qa +EOF + +docker cp generate-rpmdb-fixture:/scratch/Packages . diff --git a/imgbom/pkg/language.go b/imgbom/pkg/language.go index 9995585ee..f308dd8ea 100644 --- a/imgbom/pkg/language.go +++ b/imgbom/pkg/language.go @@ -2,8 +2,8 @@ package pkg const ( UnknownLanguage Language = iota - Java - JavaScript + //Java + //JavaScript Python Ruby ) @@ -12,15 +12,15 @@ type Language uint var languageStr = []string{ "UnknownLanguage", - "java", - "javascript", + //"java", + //"javascript", "python", "ruby", } var AllLanguages = []Language{ - Java, - JavaScript, + //Java, + //JavaScript, Python, Ruby, } diff --git a/imgbom/pkg/metadata.go b/imgbom/pkg/metadata.go index b2057550c..cc0220f1b 100644 --- a/imgbom/pkg/metadata.go +++ b/imgbom/pkg/metadata.go @@ -8,3 +8,9 @@ type DpkgMetadata struct { Source string `mapstructure:"Source"` Version string `mapstructure:"Version"` } + +type RpmMetadata struct { + Epoch int `mapstructure:"Epoch"` + Arch string `mapstructure:"Arch"` + Release string `mapstructure:"Release"` +} diff --git a/imgbom/pkg/type.go b/imgbom/pkg/type.go index ce35aa12c..cd5554ab3 100644 --- a/imgbom/pkg/type.go +++ b/imgbom/pkg/type.go @@ -2,11 +2,11 @@ package pkg const ( UnknownPkg Type = iota - ApkPkg + //ApkPkg BundlerPkg DebPkg EggPkg - PacmanPkg + //PacmanPkg RpmPkg WheelPkg ) @@ -15,21 +15,21 @@ type Type uint var typeStr = []string{ "UnknownPackage", - "apk", + //"apk", "bundle", "deb", "egg", - "pacman", + //"pacman", "rpm", "wheel", } var AllPkgs = []Type{ - ApkPkg, + //ApkPkg, BundlerPkg, DebPkg, EggPkg, - PacmanPkg, + //PacmanPkg, RpmPkg, WheelPkg, } diff --git a/integration/fixture_image_language_pkgs_test.go b/integration/fixture_image_language_pkgs_test.go index 8d18e143b..321184cef 100644 --- a/integration/fixture_image_language_pkgs_test.go +++ b/integration/fixture_image_language_pkgs_test.go @@ -3,6 +3,7 @@ package integration import ( + "github.com/anchore/imgbom/internal" "testing" "github.com/anchore/go-testutils" @@ -12,7 +13,7 @@ import ( ) func TestLanguageImage(t *testing.T) { - img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-language-pkgs") + img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-pkg-coverage") defer cleanup() s, err := scope.NewImageScope(img, scope.AllLayersScope) @@ -27,6 +28,20 @@ func TestLanguageImage(t *testing.T) { pkgLanguage pkg.Language pkgInfo map[string]string }{ + { + name: "find rpmdb packages", + pkgType: pkg.RpmPkg, + pkgInfo: map[string]string{ + "dive": "0.9.2", + }, + }, + { + name: "find dpkg packages", + pkgType: pkg.DebPkg, + pkgInfo: map[string]string{ + "apt": "1.8.2", + }, + }, { name: "find python wheel packages", pkgType: pkg.WheelPkg, @@ -102,13 +117,28 @@ func TestLanguageImage(t *testing.T) { }, }, } + + observedLanguages := internal.NewStringSet() + definedLanguages := internal.NewStringSet() + for _, l := range pkg.AllLanguages { + definedLanguages.Add(l.String()) + } + + observedPkgs := internal.NewStringSet() + definedPkgs := internal.NewStringSet() + for _, p := range pkg.AllPkgs { + definedPkgs.Add(p.String()) + } + for _, c := range cases { t.Run(c.name, func(t *testing.T) { - pkgCount := 0 for a := range catalog.Enumerate(c.pkgType) { + observedLanguages.Add(a.Language.String()) + observedPkgs.Add(a.Type.String()) + expectedVersion, ok := c.pkgInfo[a.Name] if !ok { t.Errorf("unexpected package found: %s", a.Name) @@ -138,9 +168,17 @@ func TestLanguageImage(t *testing.T) { }) } + observedLanguages.Remove(pkg.UnknownLanguage.String()) + definedLanguages.Remove(pkg.UnknownLanguage.String()) + observedPkgs.Remove(pkg.UnknownPkg.String()) + definedPkgs.Remove(pkg.UnknownPkg.String()) + // ensure that integration test cases stay in sync with the available catalogers - if len(cataloger.Catalogers()) < len(cases) { - t.Fatalf("probably missed a cataloger during testing, double check that all catalogers are included in testing") + if len(observedLanguages) < len(definedLanguages) { + t.Errorf("language coverage incomplete (languages=%d, coverage=%d)", len(definedLanguages), len(observedLanguages)) } + if len(observedPkgs) < len(definedPkgs) { + t.Errorf("package coverage incomplete (packages=%d, coverage=%d)", len(definedPkgs), len(observedPkgs)) + } } diff --git a/integration/test-fixtures/image-language-pkgs/Dockerfile b/integration/test-fixtures/image-pkg-coverage/Dockerfile similarity index 100% rename from integration/test-fixtures/image-language-pkgs/Dockerfile rename to integration/test-fixtures/image-pkg-coverage/Dockerfile diff --git a/integration/test-fixtures/image-language-pkgs/Gemfile.lock b/integration/test-fixtures/image-pkg-coverage/Gemfile.lock similarity index 100% rename from integration/test-fixtures/image-language-pkgs/Gemfile.lock rename to integration/test-fixtures/image-pkg-coverage/Gemfile.lock diff --git a/integration/test-fixtures/image-language-pkgs/dist-info/METADATA b/integration/test-fixtures/image-pkg-coverage/dist-info/METADATA similarity index 100% rename from integration/test-fixtures/image-language-pkgs/dist-info/METADATA rename to integration/test-fixtures/image-pkg-coverage/dist-info/METADATA diff --git a/integration/test-fixtures/image-language-pkgs/egg-info/PKG-INFO b/integration/test-fixtures/image-pkg-coverage/egg-info/PKG-INFO similarity index 100% rename from integration/test-fixtures/image-language-pkgs/egg-info/PKG-INFO rename to integration/test-fixtures/image-pkg-coverage/egg-info/PKG-INFO diff --git a/integration/test-fixtures/image-pkg-coverage/var/lib/dpkg/status b/integration/test-fixtures/image-pkg-coverage/var/lib/dpkg/status new file mode 100644 index 000000000..da9beca38 --- /dev/null +++ b/integration/test-fixtures/image-pkg-coverage/var/lib/dpkg/status @@ -0,0 +1,35 @@ +Package: apt +Status: install ok installed +Priority: required +Section: admin +Installed-Size: 4064 +Maintainer: APT Development Team +Architecture: amd64 +Version: 1.8.2 +Source: apt-dev +Replaces: apt-transport-https (<< 1.5~alpha4~), apt-utils (<< 1.3~exp2~) +Provides: apt-transport-https (= 1.8.2) +Depends: adduser, gpgv | gpgv2 | gpgv1, debian-archive-keyring, libapt-pkg5.0 (>= 1.7.0~alpha3~), libc6 (>= 2.15), libgcc1 (>= 1:3.0), libgnutls30 (>= 3.6.6), libseccomp2 (>= 1.0.1), libstdc++6 (>= 5.2) +Recommends: ca-certificates +Suggests: apt-doc, aptitude | synaptic | wajig, dpkg-dev (>= 1.17.2), gnupg | gnupg2 | gnupg1, powermgmt-base +Breaks: apt-transport-https (<< 1.5~alpha4~), apt-utils (<< 1.3~exp2~), aptitude (<< 0.8.10) +Conffiles: + /etc/apt/apt.conf.d/01autoremove 76120d358bc9037bb6358e737b3050b5 + /etc/cron.daily/apt-compat 49e9b2cfa17849700d4db735d04244f3 + /etc/kernel/postinst.d/apt-auto-removal 4ad976a68f045517cf4696cec7b8aa3a + /etc/logrotate.d/apt 179f2ed4f85cbaca12fa3d69c2a4a1c3 +Description: commandline package manager + This package provides commandline tools for searching and + managing as well as querying information about packages + as a low-level access to all features of the libapt-pkg library. + . + These include: + * apt-get for retrieval of packages and information about them + from authenticated sources and for installation, upgrade and + removal of packages together with their dependencies + * apt-cache for querying available information about installed + as well as installable packages + * apt-cdrom to use removable media as a source for packages + * apt-config as an interface to the configuration settings + * apt-key as an interface to manage authentication keys + diff --git a/integration/test-fixtures/image-pkg-coverage/var/lib/rpm/Packages b/integration/test-fixtures/image-pkg-coverage/var/lib/rpm/Packages new file mode 100644 index 000000000..ef499f2f8 Binary files /dev/null and b/integration/test-fixtures/image-pkg-coverage/var/lib/rpm/Packages differ diff --git a/integration/test-fixtures/image-pkg-coverage/var/lib/rpm/generate-fixture.sh b/integration/test-fixtures/image-pkg-coverage/var/lib/rpm/generate-fixture.sh new file mode 100755 index 000000000..80bf0996c --- /dev/null +++ b/integration/test-fixtures/image-pkg-coverage/var/lib/rpm/generate-fixture.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -eux + +docker create --name generate-rpmdb-fixture centos:latest sh -c 'tail -f /dev/null' + +function cleanup { + docker kill generate-rpmdb-fixture + docker rm generate-rpmdb-fixture +} +trap cleanup EXIT + +docker start generate-rpmdb-fixture +docker exec -i --tty=false generate-rpmdb-fixture bash <<-EOF + mkdir -p /scratch + cd /scratch + rpm --initdb --dbpath /scratch + curl -sSLO https://github.com/wagoodman/dive/releases/download/v0.9.2/dive_0.9.2_linux_amd64.rpm + rpm --dbpath /scratch -ivh dive_0.9.2_linux_amd64.rpm + rm dive_0.9.2_linux_amd64.rpm + rpm --dbpath /scratch -qa +EOF + +docker cp generate-rpmdb-fixture:/scratch/Packages .