diff --git a/Makefile b/Makefile index 3b340ddbb..d4b9a6bc2 100644 --- a/Makefile +++ b/Makefile @@ -114,7 +114,7 @@ check-licenses: .PHONY: unit unit: ## Run unit tests (with coverage) $(call title,Running unit tests) - go test --race -coverprofile $(COVER_REPORT) ./... + go test -coverprofile $(COVER_REPORT) ./... @go tool cover -func $(COVER_REPORT) | grep total | awk '{print substr($$3, 1, length($$3)-1)}' > $(COVER_TOTAL) @echo "Coverage: $$(cat $(COVER_TOTAL))" @if [ $$(echo "$$(cat $(COVER_TOTAL)) >= $(COVERAGE_THRESHOLD)" | bc -l) -ne 1 ]; then echo "$(RED)$(BOLD)Failed coverage quality gate (> $(COVERAGE_THRESHOLD)%)$(RESET)" && false; fi diff --git a/imgbom/cataloger/apkdb/cataloger.go b/imgbom/cataloger/apkdb/cataloger.go new file mode 100644 index 000000000..27121937a --- /dev/null +++ b/imgbom/cataloger/apkdb/cataloger.go @@ -0,0 +1,34 @@ +package apkdb + +import ( + "github.com/anchore/imgbom/imgbom/cataloger/common" + "github.com/anchore/imgbom/imgbom/pkg" + "github.com/anchore/imgbom/imgbom/scope" + "github.com/anchore/stereoscope/pkg/file" +) + +type Cataloger struct { + cataloger common.GenericCataloger +} + +func NewCataloger() *Cataloger { + globParsers := map[string]common.ParserFn{ + "**/lib/apk/db/installed": parseApkDB, + } + + return &Cataloger{ + cataloger: common.NewGenericCataloger(nil, globParsers), + } +} + +func (a *Cataloger) Name() string { + return "apkdb-cataloger" +} + +func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference { + return a.cataloger.SelectFiles(resolver) +} + +func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) { + return a.cataloger.Catalog(contents, a.Name()) +} diff --git a/imgbom/cataloger/apkdb/parse_apk_db.go b/imgbom/cataloger/apkdb/parse_apk_db.go new file mode 100644 index 000000000..03a19f816 --- /dev/null +++ b/imgbom/cataloger/apkdb/parse_apk_db.go @@ -0,0 +1,99 @@ +package apkdb + +import ( + "bufio" + "fmt" + "io" + "strconv" + "strings" + + "github.com/mitchellh/mapstructure" + + "github.com/anchore/imgbom/imgbom/pkg" +) + +func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) { + packages := make([]pkg.Package, 0) + + scanner := bufio.NewScanner(reader) + onDoubleLF := func(data []byte, atEOF bool) (advance int, token []byte, err error) { + for i := 0; i < len(data); i++ { + if i > 0 && data[i-1] == '\n' && data[i] == '\n' { + return i + 1, data[:i-1], nil + } + } + if !atEOF { + return 0, nil, nil + } + // deliver the last token (which could be an empty string) + return 0, data, bufio.ErrFinalToken + } + + scanner.Split(onDoubleLF) + for scanner.Scan() { + metadata, err := parseApkDBEntry(strings.NewReader(scanner.Text())) + if err != nil { + return nil, err + } + if metadata != nil { + packages = append(packages, pkg.Package{ + Name: metadata.Package, + Version: metadata.Version, + Licenses: strings.Split(metadata.License, " "), + Type: pkg.ApkPkg, + Metadata: *metadata, + }) + } + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("failed to parse APK DB file: %w", err) + } + + return packages, nil +} + +func parseApkDBEntry(reader io.Reader) (*pkg.ApkMetadata, error) { + var entry pkg.ApkMetadata + pkgFields := make(map[string]interface{}) + files := make([]string, 0) + + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + line := scanner.Text() + fields := strings.SplitN(line, ":", 2) + if len(fields) != 2 { + continue + } + + key := fields[0] + value := strings.TrimSpace(fields[1]) + + switch key { + case "F": + // extract all file entries, don't store in map + files = append(files, value) + continue + case "I", "S": + // coerce to integer + iVal, err := strconv.Atoi(value) + if err != nil { + return nil, fmt.Errorf("failed to parse APK int: '%+v'", value) + } + pkgFields[key] = iVal + default: + pkgFields[key] = value + } + } + + if err := mapstructure.Decode(pkgFields, &entry); err != nil { + return nil, fmt.Errorf("unable to parse APK metadata: %w", err) + } + if entry.Package == "" { + return nil, nil + } + + entry.Files = files + + return &entry, nil +} diff --git a/imgbom/cataloger/apkdb/parse_apk_db_test.go b/imgbom/cataloger/apkdb/parse_apk_db_test.go new file mode 100644 index 000000000..1de4fe007 --- /dev/null +++ b/imgbom/cataloger/apkdb/parse_apk_db_test.go @@ -0,0 +1,156 @@ +package apkdb + +import ( + "bufio" + "os" + "testing" + + "github.com/go-test/deep" + + "github.com/anchore/imgbom/imgbom/pkg" +) + +func TestSinglePackage(t *testing.T) { + tests := []struct { + name string + expected pkg.ApkMetadata + }{ + { + name: "Test Single Package", + expected: pkg.ApkMetadata{ + Package: "musl-utils", + OriginPackage: "musl", + Version: "1.1.24-r2", + Description: "the musl c library (libc) implementation", + Maintainer: "Timo Teräs ", + License: "MIT BSD GPL2+", + Architecture: "x86_64", + URL: "https://musl.libc.org/", + Size: 37944, + InstalledSize: 151552, + PullDependencies: "scanelf so:libc.musl-x86_64.so.1", + PullChecksum: "Q1bTtF5526tETKfL+lnigzIDvm+2o=", + GitCommitOfAport: "4024cc3b29ad4c65544ad068b8f59172b5494306", + Files: []string{"sbin", "usr", "usr/bin"}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + file, err := os.Open("test-fixtures/single") + if err != nil { + t.Fatal("Unable to read test_fixtures/single: ", err) + } + defer func() { + err := file.Close() + if err != nil { + t.Fatal("closing file failed:", err) + } + }() + + reader := bufio.NewReader(file) + + entry, err := parseApkDBEntry(reader) + if err != nil { + t.Fatal("Unable to read file contents: ", err) + } + + if diff := deep.Equal(*entry, test.expected); diff != nil { + for _, d := range diff { + t.Errorf("diff: %+v", d) + } + } + }) + } +} + +func TestMultiplePackages(t *testing.T) { + tests := []struct { + fixture string + expected []pkg.Package + }{ + { + fixture: "test-fixtures/multiple", + expected: []pkg.Package{ + { + Name: "libc-utils", + Version: "0.7.2-r0", + Licenses: []string{"BSD"}, + Type: pkg.ApkPkg, + Metadata: pkg.ApkMetadata{ + Package: "libc-utils", + OriginPackage: "libc-dev", + Maintainer: "Natanael Copa ", + Version: "0.7.2-r0", + License: "BSD", + Architecture: "x86_64", + URL: "http://alpinelinux.org", + Description: "Meta package to pull in correct libc", + Size: 1175, + InstalledSize: 4096, + PullChecksum: "Q1p78yvTLG094tHE1+dToJGbmYzQE=", + GitCommitOfAport: "97b1c2842faa3bfa30f5811ffbf16d5ff9f1a479", + PullDependencies: "musl-utils", + Files: []string{}, + }, + }, + { + Name: "musl-utils", + Version: "1.1.24-r2", + Licenses: []string{"MIT", "BSD", "GPL2+"}, + Type: pkg.ApkPkg, + Metadata: pkg.ApkMetadata{ + Package: "musl-utils", + OriginPackage: "musl", + Version: "1.1.24-r2", + Description: "the musl c library (libc) implementation", + Maintainer: "Timo Teräs ", + License: "MIT BSD GPL2+", + Architecture: "x86_64", + URL: "https://musl.libc.org/", + Size: 37944, + InstalledSize: 151552, + PullDependencies: "scanelf so:libc.musl-x86_64.so.1", + PullChecksum: "Q1bTtF5526tETKfL+lnigzIDvm+2o=", + GitCommitOfAport: "4024cc3b29ad4c65544ad068b8f59172b5494306", + Files: []string{"sbin", "usr", "usr/bin"}, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.fixture, func(t *testing.T) { + file, err := os.Open(test.fixture) + if err != nil { + t.Fatal("Unable to read: ", err) + } + defer func() { + err := file.Close() + if err != nil { + t.Fatal("closing file failed:", err) + } + }() + + pkgs, err := parseApkDB(file.Name(), file) + if err != nil { + t.Fatal("Unable to read file contents: ", err) + } + + if len(pkgs) != 2 { + t.Fatalf("unexpected number of entries: %d", len(pkgs)) + } + + for idx, entry := range pkgs { + if diff := deep.Equal(entry, test.expected[idx]); diff != nil { + for _, d := range diff { + t.Errorf("diff: %+v", d) + } + } + } + + }) + } +} diff --git a/imgbom/cataloger/apkdb/test-fixtures/multiple b/imgbom/cataloger/apkdb/test-fixtures/multiple new file mode 100644 index 000000000..9ade5ff5a --- /dev/null +++ b/imgbom/cataloger/apkdb/test-fixtures/multiple @@ -0,0 +1,56 @@ +C:Q1p78yvTLG094tHE1+dToJGbmYzQE= +P:libc-utils +V:0.7.2-r0 +A:x86_64 +S:1175 +I:4096 +T:Meta package to pull in correct libc +U:http://alpinelinux.org +L:BSD +o:libc-dev +m:Natanael Copa +t:1575749004 +c:97b1c2842faa3bfa30f5811ffbf16d5ff9f1a479 +D:musl-utils + + + + + + + + +C:Q1bTtF5526tETKfL+lnigzIDvm+2o= +P:musl-utils +V:1.1.24-r2 +A:x86_64 +S:37944 +I:151552 +T:the musl c library (libc) implementation +U:https://musl.libc.org/ +L:MIT BSD GPL2+ +o:musl +m:Timo Teräs +t:1584790550 +c:4024cc3b29ad4c65544ad068b8f59172b5494306 +D:scanelf so:libc.musl-x86_64.so.1 +p:cmd:getconf cmd:getent cmd:iconv cmd:ldconfig cmd:ldd +r:libiconv +F:sbin +R:ldconfig +a:0:0:755 +Z:Q1Kja2+POZKxEkUOZqwSjC6kmaED4= +F:usr +F:usr/bin +R:iconv +a:0:0:755 +Z:Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY= +R:ldd +a:0:0:755 +Z:Q1yFAhGggmL7ERgbIA7KQxyTzf3ks= +R:getconf +a:0:0:755 +Z:Q1dAdYK8M/INibRQF5B3Rw7cmNDDA= +R:getent +a:0:0:755 +Z:Q1eR2Dz/WylabgbWMTkd2+hGmEya4= \ No newline at end of file diff --git a/imgbom/cataloger/apkdb/test-fixtures/single b/imgbom/cataloger/apkdb/test-fixtures/single new file mode 100644 index 000000000..5abf4f223 --- /dev/null +++ b/imgbom/cataloger/apkdb/test-fixtures/single @@ -0,0 +1,34 @@ +C:Q1bTtF5526tETKfL+lnigzIDvm+2o= +P:musl-utils +V:1.1.24-r2 +A:x86_64 +S:37944 +I:151552 +T:the musl c library (libc) implementation +U:https://musl.libc.org/ +L:MIT BSD GPL2+ +o:musl +m:Timo Teräs +t:1584790550 +c:4024cc3b29ad4c65544ad068b8f59172b5494306 +D:scanelf so:libc.musl-x86_64.so.1 +p:cmd:getconf cmd:getent cmd:iconv cmd:ldconfig cmd:ldd +r:libiconv +F:sbin +R:ldconfig +a:0:0:755 +Z:Q1Kja2+POZKxEkUOZqwSjC6kmaED4= +F:usr +F:usr/bin +R:iconv +a:0:0:755 +Z:Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY= +R:ldd +a:0:0:755 +Z:Q1yFAhGggmL7ERgbIA7KQxyTzf3ks= +R:getconf +a:0:0:755 +Z:Q1dAdYK8M/INibRQF5B3Rw7cmNDDA= +R:getent +a:0:0:755 +Z:Q1eR2Dz/WylabgbWMTkd2+hGmEya4= \ No newline at end of file diff --git a/imgbom/cataloger/controller.go b/imgbom/cataloger/controller.go index 90614aa8e..3c6976edb 100644 --- a/imgbom/cataloger/controller.go +++ b/imgbom/cataloger/controller.go @@ -1,6 +1,7 @@ package cataloger import ( + "github.com/anchore/imgbom/imgbom/cataloger/apkdb" "github.com/anchore/imgbom/imgbom/cataloger/bundler" "github.com/anchore/imgbom/imgbom/cataloger/dpkg" golang "github.com/anchore/imgbom/imgbom/cataloger/golang" @@ -50,6 +51,7 @@ func newController() controller { ctrlr.add(python.NewCataloger()) ctrlr.add(rpmdb.NewCataloger()) ctrlr.add(java.NewCataloger()) + ctrlr.add(apkdb.NewCataloger()) ctrlr.add(golang.NewCataloger()) ctrlr.add(npm.NewCataloger()) return ctrlr diff --git a/imgbom/pkg/metadata.go b/imgbom/pkg/metadata.go index 3dda9bbdf..d886aa06a 100644 --- a/imgbom/pkg/metadata.go +++ b/imgbom/pkg/metadata.go @@ -41,3 +41,21 @@ type JavaMetadata struct { PomProperties *PomProperties `mapstructure:"PomProperties"` Parent *Package } + +// source: https://wiki.alpinelinux.org/wiki/Apk_spec +type ApkMetadata struct { + Package string `mapstructure:"P"` + OriginPackage string `mapstructure:"o"` + Maintainer string `mapstructure:"m"` + Version string `mapstructure:"V"` + License string `mapstructure:"L"` + Architecture string `mapstructure:"A"` + URL string `mapstructure:"U"` + Description string `mapstructure:"T"` + Size int `mapstructure:"S"` + InstalledSize int `mapstructure:"I"` + PullDependencies string `mapstructure:"D"` + PullChecksum string `mapstructure:"C"` + GitCommitOfAport string `mapstructure:"c"` + Files []string +} diff --git a/imgbom/pkg/type.go b/imgbom/pkg/type.go index 0fa538d7e..9aa230073 100644 --- a/imgbom/pkg/type.go +++ b/imgbom/pkg/type.go @@ -2,7 +2,7 @@ package pkg const ( UnknownPkg Type = iota - //ApkPkg + ApkPkg BundlerPkg DebPkg EggPkg @@ -20,7 +20,7 @@ type Type uint var typeStr = []string{ "UnknownPackage", - //"apk", + "apk", "bundle", "deb", "egg", @@ -35,7 +35,7 @@ var typeStr = []string{ } var AllPkgs = []Type{ - //ApkPkg, + ApkPkg, BundlerPkg, DebPkg, EggPkg, diff --git a/test/integration/fixture_pkg_coverage_test.go b/test/integration/fixture_pkg_coverage_test.go index d921f9713..d070365c0 100644 --- a/test/integration/fixture_pkg_coverage_test.go +++ b/test/integration/fixture_pkg_coverage_test.go @@ -141,6 +141,15 @@ var cases = []struct { "unicorn": "4.8.3", }, }, + { + + name: "find apkdb packages", + pkgType: pkg.ApkPkg, + pkgInfo: map[string]string{ + "musl-utils": "1.1.24-r2", + "libc-utils": "0.7.2-r0", + }, + }, { name: "find golang modules", pkgType: pkg.GoModulePkg, diff --git a/test/integration/test-fixtures/image-pkg-coverage/lib/apk/db/installed b/test/integration/test-fixtures/image-pkg-coverage/lib/apk/db/installed new file mode 100644 index 000000000..67a633b40 --- /dev/null +++ b/test/integration/test-fixtures/image-pkg-coverage/lib/apk/db/installed @@ -0,0 +1,49 @@ +C:Q1p78yvTLG094tHE1+dToJGbmYzQE= +P:libc-utils +V:0.7.2-r0 +A:x86_64 +S:1175 +I:4096 +T:Meta package to pull in correct libc +U:http://alpinelinux.org +L:BSD +o:libc-dev +m:Natanael Copa +t:1575749004 +c:97b1c2842faa3bfa30f5811ffbf16d5ff9f1a479 +D:musl-utils + +C:Q1bTtF5526tETKfL+lnigzIDvm+2o= +P:musl-utils +V:1.1.24-r2 +A:x86_64 +S:37944 +I:151552 +T:the musl c library (libc) implementation +U:https://musl.libc.org/ +L:MIT BSD GPL2+ +o:musl +m:Timo Teräs +t:1584790550 +c:4024cc3b29ad4c65544ad068b8f59172b5494306 +D:scanelf so:libc.musl-x86_64.so.1 +p:cmd:getconf cmd:getent cmd:iconv cmd:ldconfig cmd:ldd +r:libiconv +F:sbin +R:ldconfig +a:0:0:755 +Z:Q1Kja2+POZKxEkUOZqwSjC6kmaED4= +F:usr +F:usr/bin +R:iconv +a:0:0:755 +Z:Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY= +R:ldd +a:0:0:755 +Z:Q1yFAhGggmL7ERgbIA7KQxyTzf3ks= +R:getconf +a:0:0:755 +Z:Q1dAdYK8M/INibRQF5B3Rw7cmNDDA= +R:getent +a:0:0:755 +Z:Q1eR2Dz/WylabgbWMTkd2+hGmEya4= \ No newline at end of file