From e9373eac961377ad49a23d04eca30979c3dc46a6 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Fri, 15 May 2020 16:44:01 -0400 Subject: [PATCH 1/9] initial implementation of dpkg parser Signed-off-by: Alfredo Deza --- imgbom/analyzer/dpkg/parser.go | 148 ++++++++++++++++++++ imgbom/analyzer/dpkg/parser_test.go | 130 +++++++++++++++++ imgbom/analyzer/dpkg/test-fixtures/multiple | 51 +++++++ imgbom/analyzer/dpkg/test-fixtures/single | 34 +++++ 4 files changed, 363 insertions(+) create mode 100644 imgbom/analyzer/dpkg/parser.go create mode 100644 imgbom/analyzer/dpkg/parser_test.go create mode 100644 imgbom/analyzer/dpkg/test-fixtures/multiple create mode 100644 imgbom/analyzer/dpkg/test-fixtures/single diff --git a/imgbom/analyzer/dpkg/parser.go b/imgbom/analyzer/dpkg/parser.go new file mode 100644 index 000000000..181a7ff8f --- /dev/null +++ b/imgbom/analyzer/dpkg/parser.go @@ -0,0 +1,148 @@ +package dpkg + +import ( + "bufio" + "fmt" + "io" + "strings" + + "github.com/mitchellh/mapstructure" +) + +// TODO: consider keeping the remaining values as an embedded map +type DpkgEntry struct { + Package string `mapstructure:"Package"` + Architecture string `mapstructure:"Architecture"` + DependsPkgs string `mapstructure:"Depends"` + InstalledSize string `mapstructure:"Installed-Size"` + Maintainer string `mapstructure:"Maintainer"` + Priority string `mapstructure:"Priority"` + ProvidesPkgs string `mapstructure:"Provides"` + RecommendsPkgs string `mapstructure:"Recommends"` + ReplacesPkgs string `mapstructure:"Replaces"` + Status string `mapstructure:"Status"` + SuggestsPkgs string `mapstructure:"Suggests"` + Version string `mapstructure:"Version"` + ConfigFiles string `mapstructure:"Conffiles"` +} + +// dpkg-query recognized fields +// Architecture +// Bugs +// Conffiles (internal) +// Config-Version (internal) +// Conflicts +// Breaks +// Depends +// Description +// Enhances +// Essential +// Filename (internal, front-end related) +// Homepage +// Installed-Size +// MD5sum (internal, front-end related) +// MSDOS-Filename (internal, front-end related) +// Maintainer +// Origin +// Package +// Pre-Depends +// Priority +// Provides +// Recommends +// Replaces +// Revision (obsolete) +// Section +// Size (internal, front-end related) +// Source +// Status (internal) +// Suggests +// Tag (usually not in .deb but in repository Packages files) +// Triggers-Awaited (internal) +// Triggers-Pending (internal) +// Version +// + +var EndOfPackages = fmt.Errorf("no more packages to read") + +func Read(reader io.Reader) (entry DpkgEntry, err error) { + buff := bufio.NewReader(reader) + dpkgFields := make(map[string]string) + var key string + + for { + line, ioerr := buff.ReadString('\n') + fmt.Printf("line:'%+v' err:'%+v'\n", line, ioerr) + if ioerr != nil { + if ioerr == io.EOF { + return DpkgEntry{}, EndOfPackages + } + return DpkgEntry{}, ioerr + } + + line = strings.TrimRight(line, "\n") + + // stop if there is no contents in line + if len(line) == 0 { + break + } + + switch { + case strings.HasPrefix(line, " "): + // a field-body continuation + if len(key) == 0 { + return DpkgEntry{}, fmt.Errorf("no match for continuation: line: '%s'", line) + } + + val, ok := dpkgFields[key] + if !ok { + return DpkgEntry{}, fmt.Errorf("no previous key exists, expecting: %s", key) + } + // concatenate onto previous value + val = fmt.Sprintf("%s\n %s", val, strings.TrimSpace(line)) + dpkgFields[key] = val + default: + // parse a new key + if i := strings.Index(line, ":"); i > 0 { + key = strings.TrimSpace(line[0:i]) + val := strings.TrimSpace(line[i+1:]) + + if _, ok := dpkgFields[key]; ok { + return DpkgEntry{}, fmt.Errorf("duplicate key discovered: %s", key) + } + + dpkgFields[key] = val + } else { + return DpkgEntry{}, fmt.Errorf("cannot parse field from line: '%s'", line) + } + } + } + + fmt.Println("OUTOFLOOP") + + // map -> struct + err = mapstructure.Decode(dpkgFields, &entry) + if err != nil { + return DpkgEntry{}, err + } + + return entry, nil +} + +func ReadAllDpkgEntries(reader io.Reader) ([]DpkgEntry, error) { + var entries = make([]DpkgEntry, 0) + + for { + // Read() until error + entry, err := Read(reader) + fmt.Printf("entry:'%+v'\n\terr:%+v\n", entry, err) + if err != nil { + if err == EndOfPackages { + break + } + return nil, err + } + entries = append(entries, entry) + } + + return entries, nil +} diff --git a/imgbom/analyzer/dpkg/parser_test.go b/imgbom/analyzer/dpkg/parser_test.go new file mode 100644 index 000000000..8da505fbb --- /dev/null +++ b/imgbom/analyzer/dpkg/parser_test.go @@ -0,0 +1,130 @@ +package dpkg + +import ( + "os" + "testing" + + "github.com/go-test/deep" +) + +func compareEntries(t *testing.T, left, right DpkgEntry) { + t.Helper() + if diff := deep.Equal(left, right); diff != nil { + t.Error(diff) + } +} + +func TestSinglePackage(t *testing.T) { + tests := []struct { + name string + expected DpkgEntry + }{ + { + name: "Test Single Package", + expected: DpkgEntry{ + Package: "apt", + Status: "install ok installed", + Priority: "required", + InstalledSize: "4064", + Maintainer: "APT Development Team ", + Architecture: "amd64", + Version: "1.8.2", + ReplacesPkgs: "apt-transport-https (<< 1.5~alpha4~), apt-utils (<< 1.3~exp2~)", + ProvidesPkgs: "apt-transport-https (= 1.8.2)", + DependsPkgs: "adduser, gpgv | gpgv2 | gpgv1, debian-archive-keyring, libapt-pkg5.0 (>= 1.7.0~alpha3~), libc6 (>= 2.15), libgcc1 (>= 1:3.0), libgnutls30 (>= 3.6.6), libseccomp2 (>= 1.0.1), libstdc++6 (>= 5.2)", + RecommendsPkgs: "ca-certificates", + SuggestsPkgs: "apt-doc, aptitude | synaptic | wajig, dpkg-dev (>= 1.17.2), gnupg | gnupg2 | gnupg1, powermgmt-base", + ConfigFiles: ` + /etc/apt/apt.conf.d/01autoremove 76120d358bc9037bb6358e737b3050b5 + /etc/cron.daily/apt-compat 49e9b2cfa17849700d4db735d04244f3 + /etc/kernel/postinst.d/apt-auto-removal 4ad976a68f045517cf4696cec7b8aa3a + /etc/logrotate.d/apt 179f2ed4f85cbaca12fa3d69c2a4a1c3`, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + file, err := os.Open("test-fixtures/single") + if err != nil { + t.Fatal("Unable to read test_fixtures/single: ", err) + } + + entry, err := Read(file) + if err != nil { + t.Fatal("Unable to read file contents: ", err) + } + + compareEntries(t, entry, test.expected) + + }) + } +} + +func TestMultiplePackage(t *testing.T) { + tests := []struct { + name string + expected []DpkgEntry + }{ + { + name: "Test Multiple Package", + expected: []DpkgEntry{ + { + Package: "tzdata", + Status: "install ok installed", + Priority: "required", + InstalledSize: "3036", + Maintainer: "GNU Libc Maintainers ", + Architecture: "all", + Version: "2020a-0+deb10u1", + ReplacesPkgs: "libc0.1, libc0.3, libc6, libc6.1", + ProvidesPkgs: "tzdata-buster", + DependsPkgs: "debconf (>= 0.5) | debconf-2.0", + }, + { + Package: "util-linux", + Status: "install ok installed", + Priority: "required", + InstalledSize: "4327", + Maintainer: "LaMont Jones ", + Architecture: "amd64", + Version: "2.33.1-0.1", + ReplacesPkgs: "bash-completion (<< 1:2.8), initscripts (<< 2.88dsf-59.2~), login (<< 1:4.5-1.1~), mount (<< 2.29.2-3~), s390-tools (<< 2.2.0-1~), setpriv (<< 2.32.1-0.2~), sysvinit-utils (<< 2.88dsf-59.1~)", + DependsPkgs: "fdisk, login (>= 1:4.5-1.1~)", + SuggestsPkgs: "dosfstools, kbd | console-tools, util-linux-locales", + ConfigFiles: ` + /etc/default/hwclock 3916544450533eca69131f894db0ca12 + /etc/init.d/hwclock.sh 1ca5c0743fa797ffa364db95bb8d8d8e + /etc/pam.d/runuser b8b44b045259525e0fae9e38fdb2aeeb + /etc/pam.d/runuser-l 2106ea05877e8913f34b2c77fa02be45 + /etc/pam.d/su ce6dcfda3b190a27a455bb38a45ff34a + /etc/pam.d/su-l 756fef5687fecc0d986e5951427b0c4f`, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + file, err := os.Open("test-fixtures/multiple") + if err != nil { + t.Fatal("Unable to read: ", err) + } + defer file.Close() + + entries, err := ReadAllDpkgEntries(file) + if err != nil { + t.Fatal("Unable to read file contents: ", err) + } + + if len(entries) != 2 { + t.Fatalf("unexpected number of entries: %d", len(entries)) + } + + for idx, entry := range entries { + compareEntries(t, entry, test.expected[idx]) + } + + }) + } +} diff --git a/imgbom/analyzer/dpkg/test-fixtures/multiple b/imgbom/analyzer/dpkg/test-fixtures/multiple new file mode 100644 index 000000000..8c515e695 --- /dev/null +++ b/imgbom/analyzer/dpkg/test-fixtures/multiple @@ -0,0 +1,51 @@ +Package: tzdata +Status: install ok installed +Priority: required +Section: localization +Installed-Size: 3036 +Maintainer: GNU Libc Maintainers +Architecture: all +Multi-Arch: foreign +Version: 2020a-0+deb10u1 +Replaces: libc0.1, libc0.3, libc6, libc6.1 +Provides: tzdata-buster +Depends: debconf (>= 0.5) | debconf-2.0 +Description: time zone and daylight-saving time data + This package contains data required for the implementation of + standard local time for many representative locations around the + globe. It is updated periodically to reflect changes made by + political bodies to time zone boundaries, UTC offsets, and + daylight-saving rules. +Homepage: https://www.iana.org/time-zones + +Package: util-linux +Essential: yes +Status: install ok installed +Priority: required +Section: utils +Installed-Size: 4327 +Maintainer: LaMont Jones +Architecture: amd64 +Multi-Arch: foreign +Version: 2.33.1-0.1 +Replaces: bash-completion (<< 1:2.8), initscripts (<< 2.88dsf-59.2~), login (<< 1:4.5-1.1~), mount (<< 2.29.2-3~), s390-tools (<< 2.2.0-1~), setpriv (<< 2.32.1-0.2~), sysvinit-utils (<< 2.88dsf-59.1~) +Depends: fdisk, login (>= 1:4.5-1.1~) +Pre-Depends: libaudit1 (>= 1:2.2.1), libblkid1 (>= 2.31.1), libc6 (>= 2.25), libcap-ng0 (>= 0.7.9), libmount1 (>= 2.25), libpam0g (>= 0.99.7.1), libselinux1 (>= 2.6-3~), libsmartcols1 (>= 2.33), libsystemd0, libtinfo6 (>= 6), libudev1 (>= 183), libuuid1 (>= 2.16), zlib1g (>= 1:1.1.4) +Suggests: dosfstools, kbd | console-tools, util-linux-locales +Breaks: bash-completion (<< 1:2.8), grml-debootstrap (<< 0.68), mount (<< 2.29.2-3~), s390-tools (<< 2.2.0-1~), setpriv (<< 2.32.1-0.2~), sysvinit-utils (<< 2.88dsf-59.4~) +Conffiles: + /etc/default/hwclock 3916544450533eca69131f894db0ca12 + /etc/init.d/hwclock.sh 1ca5c0743fa797ffa364db95bb8d8d8e + /etc/pam.d/runuser b8b44b045259525e0fae9e38fdb2aeeb + /etc/pam.d/runuser-l 2106ea05877e8913f34b2c77fa02be45 + /etc/pam.d/su ce6dcfda3b190a27a455bb38a45ff34a + /etc/pam.d/su-l 756fef5687fecc0d986e5951427b0c4f +Description: miscellaneous system utilities + This package contains a number of important utilities, most of which + are oriented towards maintenance of your system. Some of the more + important utilities included in this package allow you to view kernel + messages, create new filesystems, view block device information, + interface with real time clock, etc. + + + diff --git a/imgbom/analyzer/dpkg/test-fixtures/single b/imgbom/analyzer/dpkg/test-fixtures/single new file mode 100644 index 000000000..79590d82e --- /dev/null +++ b/imgbom/analyzer/dpkg/test-fixtures/single @@ -0,0 +1,34 @@ +Package: apt +Status: install ok installed +Priority: required +Section: admin +Installed-Size: 4064 +Maintainer: APT Development Team +Architecture: amd64 +Version: 1.8.2 +Replaces: apt-transport-https (<< 1.5~alpha4~), apt-utils (<< 1.3~exp2~) +Provides: apt-transport-https (= 1.8.2) +Depends: adduser, gpgv | gpgv2 | gpgv1, debian-archive-keyring, libapt-pkg5.0 (>= 1.7.0~alpha3~), libc6 (>= 2.15), libgcc1 (>= 1:3.0), libgnutls30 (>= 3.6.6), libseccomp2 (>= 1.0.1), libstdc++6 (>= 5.2) +Recommends: ca-certificates +Suggests: apt-doc, aptitude | synaptic | wajig, dpkg-dev (>= 1.17.2), gnupg | gnupg2 | gnupg1, powermgmt-base +Breaks: apt-transport-https (<< 1.5~alpha4~), apt-utils (<< 1.3~exp2~), aptitude (<< 0.8.10) +Conffiles: + /etc/apt/apt.conf.d/01autoremove 76120d358bc9037bb6358e737b3050b5 + /etc/cron.daily/apt-compat 49e9b2cfa17849700d4db735d04244f3 + /etc/kernel/postinst.d/apt-auto-removal 4ad976a68f045517cf4696cec7b8aa3a + /etc/logrotate.d/apt 179f2ed4f85cbaca12fa3d69c2a4a1c3 +Description: commandline package manager + This package provides commandline tools for searching and + managing as well as querying information about packages + as a low-level access to all features of the libapt-pkg library. + . + These include: + * apt-get for retrieval of packages and information about them + from authenticated sources and for installation, upgrade and + removal of packages together with their dependencies + * apt-cache for querying available information about installed + as well as installable packages + * apt-cdrom to use removable media as a source for packages + * apt-config as an interface to the configuration settings + * apt-key as an interface to manage authentication keys + From 09cf5b9862a57125723009e89f0cd93c2e6dc566 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Fri, 15 May 2020 16:44:25 -0400 Subject: [PATCH 2/9] update go.mod and go.sum Signed-off-by: Alfredo Deza --- go.sum | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.sum b/go.sum index 988ad505e..38291def2 100644 --- a/go.sum +++ b/go.sum @@ -133,6 +133,8 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.6 h1:UHSEyLZUwX9Qoi99vVwvewiMC8mM2bf7XEM2nqvzEn8= +github.com/go-test/deep v1.0.6/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= From d56d157e90ff57c4998a7cc55d6dc2358be28cd6 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Sat, 16 May 2020 07:59:34 -0400 Subject: [PATCH 3/9] create single dpkg bufio reader --- imgbom/analyzer/dpkg/parser.go | 77 ++++++++++++++--------------- imgbom/analyzer/dpkg/parser_test.go | 32 ++++++++---- 2 files changed, 61 insertions(+), 48 deletions(-) diff --git a/imgbom/analyzer/dpkg/parser.go b/imgbom/analyzer/dpkg/parser.go index 181a7ff8f..26717cd18 100644 --- a/imgbom/analyzer/dpkg/parser.go +++ b/imgbom/analyzer/dpkg/parser.go @@ -10,7 +10,7 @@ import ( ) // TODO: consider keeping the remaining values as an embedded map -type DpkgEntry struct { +type Entry struct { Package string `mapstructure:"Package"` Architecture string `mapstructure:"Architecture"` DependsPkgs string `mapstructure:"Depends"` @@ -60,29 +60,49 @@ type DpkgEntry struct { // Triggers-Awaited (internal) // Triggers-Pending (internal) // Version -// -var EndOfPackages = fmt.Errorf("no more packages to read") +var endOfPackages = fmt.Errorf("no more packages to read") -func Read(reader io.Reader) (entry DpkgEntry, err error) { - buff := bufio.NewReader(reader) +func ParseEntries(reader io.Reader) ([]Entry, error) { + buffedReader := bufio.NewReader(reader) + var entries = make([]Entry, 0) + + for { + entry, err := parseEntry(buffedReader) + if err != nil { + if err == endOfPackages { + break + } + return nil, err + } + entries = append(entries, entry) + } + + return entries, nil +} + + +func parseEntry(reader *bufio.Reader) (entry Entry, err error) { dpkgFields := make(map[string]string) var key string for { - line, ioerr := buff.ReadString('\n') - fmt.Printf("line:'%+v' err:'%+v'\n", line, ioerr) - if ioerr != nil { - if ioerr == io.EOF { - return DpkgEntry{}, EndOfPackages + line, err := reader.ReadString('\n') + if err != nil { + if err == io.EOF { + return Entry{}, endOfPackages } - return DpkgEntry{}, ioerr + return Entry{}, err } line = strings.TrimRight(line, "\n") - // stop if there is no contents in line + // empty line indicates end of entry if len(line) == 0 { + // if the entry has not started, keep parsing lines + if len(dpkgFields) == 0{ + continue + } break } @@ -90,12 +110,12 @@ func Read(reader io.Reader) (entry DpkgEntry, err error) { case strings.HasPrefix(line, " "): // a field-body continuation if len(key) == 0 { - return DpkgEntry{}, fmt.Errorf("no match for continuation: line: '%s'", line) + return Entry{}, fmt.Errorf("no match for continuation: line: '%s'", line) } val, ok := dpkgFields[key] if !ok { - return DpkgEntry{}, fmt.Errorf("no previous key exists, expecting: %s", key) + return Entry{}, fmt.Errorf("no previous key exists, expecting: %s", key) } // concatenate onto previous value val = fmt.Sprintf("%s\n %s", val, strings.TrimSpace(line)) @@ -107,42 +127,21 @@ func Read(reader io.Reader) (entry DpkgEntry, err error) { val := strings.TrimSpace(line[i+1:]) if _, ok := dpkgFields[key]; ok { - return DpkgEntry{}, fmt.Errorf("duplicate key discovered: %s", key) + return Entry{}, fmt.Errorf("duplicate key discovered: %s", key) } dpkgFields[key] = val } else { - return DpkgEntry{}, fmt.Errorf("cannot parse field from line: '%s'", line) + return Entry{}, fmt.Errorf("cannot parse field from line: '%s'", line) } } } - - fmt.Println("OUTOFLOOP") - - // map -> struct + err = mapstructure.Decode(dpkgFields, &entry) if err != nil { - return DpkgEntry{}, err + return Entry{}, err } return entry, nil } -func ReadAllDpkgEntries(reader io.Reader) ([]DpkgEntry, error) { - var entries = make([]DpkgEntry, 0) - - for { - // Read() until error - entry, err := Read(reader) - fmt.Printf("entry:'%+v'\n\terr:%+v\n", entry, err) - if err != nil { - if err == EndOfPackages { - break - } - return nil, err - } - entries = append(entries, entry) - } - - return entries, nil -} diff --git a/imgbom/analyzer/dpkg/parser_test.go b/imgbom/analyzer/dpkg/parser_test.go index 8da505fbb..0492b9e58 100644 --- a/imgbom/analyzer/dpkg/parser_test.go +++ b/imgbom/analyzer/dpkg/parser_test.go @@ -1,13 +1,14 @@ package dpkg import ( + "bufio" "os" "testing" "github.com/go-test/deep" ) -func compareEntries(t *testing.T, left, right DpkgEntry) { +func compareEntries(t *testing.T, left, right Entry) { t.Helper() if diff := deep.Equal(left, right); diff != nil { t.Error(diff) @@ -17,11 +18,11 @@ func compareEntries(t *testing.T, left, right DpkgEntry) { func TestSinglePackage(t *testing.T) { tests := []struct { name string - expected DpkgEntry + expected Entry }{ { name: "Test Single Package", - expected: DpkgEntry{ + expected: Entry{ Package: "apt", Status: "install ok installed", Priority: "required", @@ -49,8 +50,16 @@ func TestSinglePackage(t *testing.T) { if err != nil { t.Fatal("Unable to read test_fixtures/single: ", err) } + defer func() { + err := file.Close() + if err != nil { + panic(err) + } + }() - entry, err := Read(file) + reader := bufio.NewReader(file) + + entry, err := parseEntry(reader) if err != nil { t.Fatal("Unable to read file contents: ", err) } @@ -61,14 +70,14 @@ func TestSinglePackage(t *testing.T) { } } -func TestMultiplePackage(t *testing.T) { +func TestMultiplePackages(t *testing.T) { tests := []struct { name string - expected []DpkgEntry + expected []Entry }{ { name: "Test Multiple Package", - expected: []DpkgEntry{ + expected: []Entry{ { Package: "tzdata", Status: "install ok installed", @@ -110,9 +119,14 @@ func TestMultiplePackage(t *testing.T) { if err != nil { t.Fatal("Unable to read: ", err) } - defer file.Close() + defer func() { + err := file.Close() + if err != nil { + panic(err) + } + }() - entries, err := ReadAllDpkgEntries(file) + entries, err := ParseEntries(file) if err != nil { t.Fatal("Unable to read file contents: ", err) } From fa08a8fe6f31e6414e3dc45c7fd57cb6247ede77 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Tue, 19 May 2020 09:06:02 -0400 Subject: [PATCH 4/9] create a package to place analyzer metadata Signed-off-by: Alfredo Deza --- imgbom/pkg/metadata.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 imgbom/pkg/metadata.go diff --git a/imgbom/pkg/metadata.go b/imgbom/pkg/metadata.go new file mode 100644 index 000000000..811e4311c --- /dev/null +++ b/imgbom/pkg/metadata.go @@ -0,0 +1,20 @@ +package pkg + +// TODO: consider keeping the remaining values as an embedded map +// Available fields are described at http://manpages.ubuntu.com/manpages/xenial/man1/dpkg-query.1.html +// in the --showformat section +type DpkgMetadata struct { + Package string `mapstructure:"Package"` + Architecture string `mapstructure:"Architecture"` + DependsPkgs string `mapstructure:"Depends"` + InstalledSize string `mapstructure:"Installed-Size"` + Maintainer string `mapstructure:"Maintainer"` + Priority string `mapstructure:"Priority"` + ProvidesPkgs string `mapstructure:"Provides"` + RecommendsPkgs string `mapstructure:"Recommends"` + ReplacesPkgs string `mapstructure:"Replaces"` + Status string `mapstructure:"Status"` + SuggestsPkgs string `mapstructure:"Suggests"` + Version string `mapstructure:"Version"` + ConfigFiles string `mapstructure:"Conffiles"` +} From 5bc17310b6ddefe1f132cac0de3ebd00b14cd31f Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Tue, 19 May 2020 09:07:34 -0400 Subject: [PATCH 5/9] update dependencies Signed-off-by: Alfredo Deza --- go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.mod b/go.mod index a49b10de1..75ed9a058 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,10 @@ go 1.14 require ( github.com/anchore/stereoscope v0.0.0-20200518155435-f6c722e4572b + github.com/go-test/deep v1.0.6 github.com/golang/protobuf v1.4.2 // indirect github.com/hashicorp/go-multierror v1.1.0 + github.com/mitchellh/mapstructure v1.1.2 github.com/opencontainers/go-digest v1.0.0 // indirect github.com/sirupsen/logrus v1.6.0 // indirect github.com/spf13/cobra v1.0.0 From 61139007ca74d0f03c343c5af8a1fb5207995a8d Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Tue, 19 May 2020 09:08:31 -0400 Subject: [PATCH 6/9] create the dpkg analyzer Signed-off-by: Alfredo Deza --- imgbom/analyzer/dpkg/analyzer.go | 60 ++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 imgbom/analyzer/dpkg/analyzer.go diff --git a/imgbom/analyzer/dpkg/analyzer.go b/imgbom/analyzer/dpkg/analyzer.go new file mode 100644 index 000000000..0ee2994dc --- /dev/null +++ b/imgbom/analyzer/dpkg/analyzer.go @@ -0,0 +1,60 @@ +package dpkg + +import ( + "strings" + + "github.com/anchore/imgbom/imgbom/pkg" + "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/stereoscope/pkg/tree" +) + +type Analyzer struct { + selectedFiles []file.Reference +} + +func NewAnalyzer() *Analyzer { + return &Analyzer{} +} + +func (a *Analyzer) SelectFiles(trees []*tree.FileTree) []file.Reference { + files := make([]file.Reference, 0) + for _, tree := range trees { + // TODO: extract into function/slice/etc + file := tree.File("/var/lib/dpkg/status") + if file != nil { + files = append(files, *file) + } + } + + a.selectedFiles = files + + return files +} + +func (a *Analyzer) Analyze(contents map[file.Reference]string) ([]pkg.Package, error) { + packages := make([]pkg.Package, 0) + for _, reference := range a.selectedFiles { + content, ok := contents[reference] + if !ok { + // TODO: this needs handling + panic(reference) + } + + entries, err := ParseEntries(strings.NewReader(content)) + if err != nil { + // TODO: punt for now, we need to handle this + panic(err) + } + for _, entry := range entries { + packages = append(packages, pkg.Package{ + Name: entry.Package, + Version: entry.Version, + Type: pkg.DebPkg, + Source: []file.Reference{reference}, + Metadata: entry, + }) + } + + } + return packages, nil +} From 888cee3336d2b73539d9a38384a5356c726d957e Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Tue, 19 May 2020 09:09:05 -0400 Subject: [PATCH 7/9] replace dummy analyzer for the new dpkg one Signed-off-by: Alfredo Deza --- imgbom/analyzer/controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imgbom/analyzer/controller.go b/imgbom/analyzer/controller.go index 6c7f8d249..21431c8c4 100644 --- a/imgbom/analyzer/controller.go +++ b/imgbom/analyzer/controller.go @@ -1,7 +1,7 @@ package analyzer import ( - "github.com/anchore/imgbom/imgbom/analyzer/dummy" + "github.com/anchore/imgbom/imgbom/analyzer/dpkg" "github.com/anchore/imgbom/imgbom/pkg" "github.com/anchore/imgbom/imgbom/scope" "github.com/anchore/stereoscope/pkg/file" @@ -14,7 +14,7 @@ func init() { controllerInstance = controller{ analyzers: make([]Analyzer, 0), } - controllerInstance.add(dummy.NewAnalyzer()) + controllerInstance.add(dpkg.NewAnalyzer()) } func Analyze(s scope.Scope) (pkg.Catalog, error) { From ada11973b2e7d97f33222f002fc413695e2f5def Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Tue, 19 May 2020 09:09:37 -0400 Subject: [PATCH 8/9] use the new DpkgMetadata struct in the parser Signed-off-by: Alfredo Deza --- imgbom/analyzer/dpkg/parser.go | 79 ++++++---------------------------- 1 file changed, 13 insertions(+), 66 deletions(-) diff --git a/imgbom/analyzer/dpkg/parser.go b/imgbom/analyzer/dpkg/parser.go index 26717cd18..c1442b453 100644 --- a/imgbom/analyzer/dpkg/parser.go +++ b/imgbom/analyzer/dpkg/parser.go @@ -6,66 +6,15 @@ import ( "io" "strings" + "github.com/anchore/imgbom/imgbom/pkg" "github.com/mitchellh/mapstructure" ) -// TODO: consider keeping the remaining values as an embedded map -type Entry struct { - Package string `mapstructure:"Package"` - Architecture string `mapstructure:"Architecture"` - DependsPkgs string `mapstructure:"Depends"` - InstalledSize string `mapstructure:"Installed-Size"` - Maintainer string `mapstructure:"Maintainer"` - Priority string `mapstructure:"Priority"` - ProvidesPkgs string `mapstructure:"Provides"` - RecommendsPkgs string `mapstructure:"Recommends"` - ReplacesPkgs string `mapstructure:"Replaces"` - Status string `mapstructure:"Status"` - SuggestsPkgs string `mapstructure:"Suggests"` - Version string `mapstructure:"Version"` - ConfigFiles string `mapstructure:"Conffiles"` -} - -// dpkg-query recognized fields -// Architecture -// Bugs -// Conffiles (internal) -// Config-Version (internal) -// Conflicts -// Breaks -// Depends -// Description -// Enhances -// Essential -// Filename (internal, front-end related) -// Homepage -// Installed-Size -// MD5sum (internal, front-end related) -// MSDOS-Filename (internal, front-end related) -// Maintainer -// Origin -// Package -// Pre-Depends -// Priority -// Provides -// Recommends -// Replaces -// Revision (obsolete) -// Section -// Size (internal, front-end related) -// Source -// Status (internal) -// Suggests -// Tag (usually not in .deb but in repository Packages files) -// Triggers-Awaited (internal) -// Triggers-Pending (internal) -// Version - var endOfPackages = fmt.Errorf("no more packages to read") -func ParseEntries(reader io.Reader) ([]Entry, error) { +func ParseEntries(reader io.Reader) ([]pkg.DpkgMetadata, error) { buffedReader := bufio.NewReader(reader) - var entries = make([]Entry, 0) + var entries = make([]pkg.DpkgMetadata, 0) for { entry, err := parseEntry(buffedReader) @@ -81,8 +30,7 @@ func ParseEntries(reader io.Reader) ([]Entry, error) { return entries, nil } - -func parseEntry(reader *bufio.Reader) (entry Entry, err error) { +func parseEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err error) { dpkgFields := make(map[string]string) var key string @@ -90,9 +38,9 @@ func parseEntry(reader *bufio.Reader) (entry Entry, err error) { line, err := reader.ReadString('\n') if err != nil { if err == io.EOF { - return Entry{}, endOfPackages + return pkg.DpkgMetadata{}, endOfPackages } - return Entry{}, err + return pkg.DpkgMetadata{}, err } line = strings.TrimRight(line, "\n") @@ -100,7 +48,7 @@ func parseEntry(reader *bufio.Reader) (entry Entry, err error) { // empty line indicates end of entry if len(line) == 0 { // if the entry has not started, keep parsing lines - if len(dpkgFields) == 0{ + if len(dpkgFields) == 0 { continue } break @@ -110,12 +58,12 @@ func parseEntry(reader *bufio.Reader) (entry Entry, err error) { case strings.HasPrefix(line, " "): // a field-body continuation if len(key) == 0 { - return Entry{}, fmt.Errorf("no match for continuation: line: '%s'", line) + return pkg.DpkgMetadata{}, fmt.Errorf("no match for continuation: line: '%s'", line) } val, ok := dpkgFields[key] if !ok { - return Entry{}, fmt.Errorf("no previous key exists, expecting: %s", key) + return pkg.DpkgMetadata{}, fmt.Errorf("no previous key exists, expecting: %s", key) } // concatenate onto previous value val = fmt.Sprintf("%s\n %s", val, strings.TrimSpace(line)) @@ -127,21 +75,20 @@ func parseEntry(reader *bufio.Reader) (entry Entry, err error) { val := strings.TrimSpace(line[i+1:]) if _, ok := dpkgFields[key]; ok { - return Entry{}, fmt.Errorf("duplicate key discovered: %s", key) + return pkg.DpkgMetadata{}, fmt.Errorf("duplicate key discovered: %s", key) } dpkgFields[key] = val } else { - return Entry{}, fmt.Errorf("cannot parse field from line: '%s'", line) + return pkg.DpkgMetadata{}, fmt.Errorf("cannot parse field from line: '%s'", line) } } } - + err = mapstructure.Decode(dpkgFields, &entry) if err != nil { - return Entry{}, err + return pkg.DpkgMetadata{}, err } return entry, nil } - From 6fb3a876a89826198d388ac942889e2c09bcb9d7 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Tue, 19 May 2020 09:29:02 -0400 Subject: [PATCH 9/9] tests: replace Entry with DpkgMetadata name Signed-off-by: Alfredo Deza --- imgbom/analyzer/dpkg/parser_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/imgbom/analyzer/dpkg/parser_test.go b/imgbom/analyzer/dpkg/parser_test.go index 0492b9e58..0704c54ae 100644 --- a/imgbom/analyzer/dpkg/parser_test.go +++ b/imgbom/analyzer/dpkg/parser_test.go @@ -5,10 +5,11 @@ import ( "os" "testing" + "github.com/anchore/imgbom/imgbom/pkg" "github.com/go-test/deep" ) -func compareEntries(t *testing.T, left, right Entry) { +func compareEntries(t *testing.T, left, right pkg.DpkgMetadata) { t.Helper() if diff := deep.Equal(left, right); diff != nil { t.Error(diff) @@ -18,11 +19,11 @@ func compareEntries(t *testing.T, left, right Entry) { func TestSinglePackage(t *testing.T) { tests := []struct { name string - expected Entry + expected pkg.DpkgMetadata }{ { name: "Test Single Package", - expected: Entry{ + expected: pkg.DpkgMetadata{ Package: "apt", Status: "install ok installed", Priority: "required", @@ -73,11 +74,11 @@ func TestSinglePackage(t *testing.T) { func TestMultiplePackages(t *testing.T) { tests := []struct { name string - expected []Entry + expected []pkg.DpkgMetadata }{ { name: "Test Multiple Package", - expected: []Entry{ + expected: []pkg.DpkgMetadata{ { Package: "tzdata", Status: "install ok installed",