From 44c69f1f91e157392f77188ff4189d792ab92f5a Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 23 Jul 2020 08:17:30 -0400 Subject: [PATCH] add go.mod cataloger (#97) --- go.mod | 1 + go.sum | 1 + imgbom/cataloger/controller.go | 2 + imgbom/cataloger/golang/cataloger.go | 34 ++++++ imgbom/cataloger/golang/parse_go_mod.go | 63 +++++++++++ imgbom/cataloger/golang/parse_go_mod_test.go | 106 ++++++++++++++++++ .../golang/test-fixtures/many-packages | 21 ++++ .../golang/test-fixtures/one-package | 7 ++ imgbom/pkg/language.go | 3 + imgbom/pkg/type.go | 3 + integration/fixture_pkg_coverage_test.go | 8 ++ .../image-pkg-coverage/go/go.mod | 7 ++ 12 files changed, 256 insertions(+) create mode 100644 imgbom/cataloger/golang/cataloger.go create mode 100644 imgbom/cataloger/golang/parse_go_mod.go create mode 100644 imgbom/cataloger/golang/parse_go_mod_test.go create mode 100644 imgbom/cataloger/golang/test-fixtures/many-packages create mode 100644 imgbom/cataloger/golang/test-fixtures/one-package create mode 100644 integration/test-fixtures/image-pkg-coverage/go/go.mod diff --git a/go.mod b/go.mod index 886a74370..891fe88e3 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/knqyf263/go-rpmdb v0.0.0-20190501070121-10a1c42a10dc github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.3.1 + github.com/rogpeppe/go-internal v1.5.2 github.com/sergi/go-diff v1.1.0 github.com/spf13/cobra v1.0.0 github.com/spf13/viper v1.7.0 diff --git a/go.sum b/go.sum index 5726dea60..842c2c68a 100644 --- a/go.sum +++ b/go.sum @@ -696,6 +696,7 @@ github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uY github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.5.2 h1:qLvObTrvO/XRCqmkKxUlOBc48bI3efyDuAZe25QiF0w= github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= diff --git a/imgbom/cataloger/controller.go b/imgbom/cataloger/controller.go index fe508b398..bd3c488ac 100644 --- a/imgbom/cataloger/controller.go +++ b/imgbom/cataloger/controller.go @@ -3,6 +3,7 @@ package cataloger import ( "github.com/anchore/imgbom/imgbom/cataloger/bundler" "github.com/anchore/imgbom/imgbom/cataloger/dpkg" + golang "github.com/anchore/imgbom/imgbom/cataloger/golang" "github.com/anchore/imgbom/imgbom/cataloger/java" "github.com/anchore/imgbom/imgbom/cataloger/python" "github.com/anchore/imgbom/imgbom/cataloger/rpmdb" @@ -48,6 +49,7 @@ func newController() controller { ctrlr.add(python.NewCataloger()) ctrlr.add(rpmdb.NewCataloger()) ctrlr.add(java.NewCataloger()) + ctrlr.add(golang.NewCataloger()) return ctrlr } diff --git a/imgbom/cataloger/golang/cataloger.go b/imgbom/cataloger/golang/cataloger.go new file mode 100644 index 000000000..33f0de80a --- /dev/null +++ b/imgbom/cataloger/golang/cataloger.go @@ -0,0 +1,34 @@ +package golang + +import ( + "github.com/anchore/imgbom/imgbom/cataloger/common" + "github.com/anchore/imgbom/imgbom/pkg" + "github.com/anchore/imgbom/imgbom/scope" + "github.com/anchore/stereoscope/pkg/file" +) + +type Cataloger struct { + cataloger common.GenericCataloger +} + +func NewCataloger() *Cataloger { + globParsers := map[string]common.ParserFn{ + "**/go.mod": parseGoMod, + } + + return &Cataloger{ + cataloger: common.NewGenericCataloger(nil, globParsers), + } +} + +func (a *Cataloger) Name() string { + return "go-cataloger" +} + +func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference { + return a.cataloger.SelectFiles(resolver) +} + +func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) { + return a.cataloger.Catalog(contents, a.Name()) +} diff --git a/imgbom/cataloger/golang/parse_go_mod.go b/imgbom/cataloger/golang/parse_go_mod.go new file mode 100644 index 000000000..4ce327333 --- /dev/null +++ b/imgbom/cataloger/golang/parse_go_mod.go @@ -0,0 +1,63 @@ +package golang + +import ( + "fmt" + "io" + "io/ioutil" + "sort" + + "github.com/rogpeppe/go-internal/modfile" + + "github.com/anchore/imgbom/imgbom/pkg" +) + +func parseGoMod(path string, reader io.Reader) ([]pkg.Package, error) { + packages := make(map[string]pkg.Package) + + contents, err := ioutil.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("failed to read go module: %w", err) + } + + file, err := modfile.Parse(path, contents, nil) + if err != nil { + return nil, fmt.Errorf("failed to parse go module: %w", err) + } + + for _, m := range file.Require { + packages[m.Mod.Path] = pkg.Package{ + Name: m.Mod.Path, + Version: m.Mod.Version, + Language: pkg.Go, + Type: pkg.GoModulePkg, + } + } + + // remove any old packages and replace with new ones... + for _, m := range file.Replace { + packages[m.New.Path] = pkg.Package{ + Name: m.New.Path, + Version: m.New.Version, + Language: pkg.Go, + Type: pkg.GoModulePkg, + } + } + + // remove any packages from the exclude fields + for _, m := range file.Exclude { + delete(packages, m.Mod.Path) + } + + pkgsSlice := make([]pkg.Package, len(packages)) + idx := 0 + for _, p := range packages { + pkgsSlice[idx] = p + idx++ + } + + sort.SliceStable(pkgsSlice, func(i, j int) bool { + return pkgsSlice[i].Name < pkgsSlice[j].Name + }) + + return pkgsSlice, nil +} diff --git a/imgbom/cataloger/golang/parse_go_mod_test.go b/imgbom/cataloger/golang/parse_go_mod_test.go new file mode 100644 index 000000000..dff7a366e --- /dev/null +++ b/imgbom/cataloger/golang/parse_go_mod_test.go @@ -0,0 +1,106 @@ +package golang + +import ( + "os" + "testing" + + "github.com/go-test/deep" + + "github.com/anchore/imgbom/imgbom/pkg" +) + +func TestParseGoMod(t *testing.T) { + tests := []struct { + fixture string + expected map[string]pkg.Package + }{ + { + fixture: "test-fixtures/one-package", + expected: map[string]pkg.Package{ + "github.com/bmatcuk/doublestar": { + Name: "github.com/bmatcuk/doublestar", + Version: "v1.3.1", + Language: pkg.Go, + Type: pkg.GoModulePkg, + }, + }, + }, + { + + fixture: "test-fixtures/many-packages", + expected: map[string]pkg.Package{ + "github.com/anchore/go-testutils": { + Name: "github.com/anchore/go-testutils", + Version: "v0.0.0-20200624184116-66aa578126db", + Language: pkg.Go, + Type: pkg.GoModulePkg, + }, + "github.com/anchore/go-version": { + Name: "github.com/anchore/go-version", + Version: "v1.2.2-0.20200701162849-18adb9c92b9b", + Language: pkg.Go, + Type: pkg.GoModulePkg, + }, + "github.com/anchore/stereoscope": { + Name: "github.com/anchore/stereoscope", + Version: "v0.0.0-20200706164556-7cf39d7f4639", + Language: pkg.Go, + Type: pkg.GoModulePkg, + }, + "github.com/bmatcuk/doublestar": { + Name: "github.com/bmatcuk/doublestar", + Version: "v8.8.8", + Language: pkg.Go, + Type: pkg.GoModulePkg, + }, + "github.com/go-test/deep": { + Name: "github.com/go-test/deep", + Version: "v1.0.6", + Language: pkg.Go, + Type: pkg.GoModulePkg, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.fixture, func(t *testing.T) { + f, err := os.Open(test.fixture) + if err != nil { + t.Fatalf(err.Error()) + } + + actual, err := parseGoMod(test.fixture, f) + if err != nil { + t.Fatalf(err.Error()) + } + + if len(actual) != len(test.expected) { + t.Fatalf("unexpected length: %d", len(actual)) + } + + for _, a := range actual { + e, ok := test.expected[a.Name] + if !ok { + t.Errorf("extra package: %s", a.Name) + continue + } + + diffs := deep.Equal(a, e) + if len(diffs) > 0 { + t.Errorf("diffs found for %q", a.Name) + for _, d := range diffs { + t.Errorf("diff: %+v", d) + } + } + } + + if t.Failed() { + for _, a := range actual { + t.Logf("Found: %+v", a) + } + } + + }) + } +} diff --git a/imgbom/cataloger/golang/test-fixtures/many-packages b/imgbom/cataloger/golang/test-fixtures/many-packages new file mode 100644 index 000000000..d1c417880 --- /dev/null +++ b/imgbom/cataloger/golang/test-fixtures/many-packages @@ -0,0 +1,21 @@ +module ( + github.com/anchore/imgbom +) + +go 1.14 + +// github.com/bogus/package v10.10.10 + +require ( + github.com/adrg/xdg v0.2.1 + github.com/anchore/go-testutils v0.0.0-20200624184116-66aa578126db // github.com/bogus/package v10.10.10 + github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b + github.com/anchore/stereoscope v0.0.0-20200706164556-7cf39d7f4639 + //github.com/ignore/this v9.9.9 // indirect + github.com/bmatcuk/doublestar v1.3.1 // indirect + github.com/go-test/deep v1.0.6 // a comment +) + +replace github.com/bmatcuk/doublestar => github.com/bmatcuk/doublestar v8.8.8 + +exclude github.com/adrg/xdg v0.2.1 diff --git a/imgbom/cataloger/golang/test-fixtures/one-package b/imgbom/cataloger/golang/test-fixtures/one-package new file mode 100644 index 000000000..93da53fdf --- /dev/null +++ b/imgbom/cataloger/golang/test-fixtures/one-package @@ -0,0 +1,7 @@ +module github.com/anchore/imgbom + +go 1.14 + +require ( + github.com/bmatcuk/doublestar v1.3.1 +) diff --git a/imgbom/pkg/language.go b/imgbom/pkg/language.go index d0c9e67b5..b7c8c61e0 100644 --- a/imgbom/pkg/language.go +++ b/imgbom/pkg/language.go @@ -6,6 +6,7 @@ const ( //JavaScript Python Ruby + Go ) type Language uint @@ -16,6 +17,7 @@ var languageStr = []string{ //"javascript", "python", "ruby", + "go", } var AllLanguages = []Language{ @@ -23,6 +25,7 @@ var AllLanguages = []Language{ //JavaScript, Python, Ruby, + Go, } func (t Language) String() string { diff --git a/imgbom/pkg/type.go b/imgbom/pkg/type.go index c4b745b03..56d7ce8cd 100644 --- a/imgbom/pkg/type.go +++ b/imgbom/pkg/type.go @@ -12,6 +12,7 @@ const ( PythonRequirementsPkg JavaPkg JenkinsPluginPkg + GoModulePkg ) type Type uint @@ -28,6 +29,7 @@ var typeStr = []string{ "python-requirements", "java-archive", "jenkins-plugin", + "go-module", } var AllPkgs = []Type{ @@ -41,6 +43,7 @@ var AllPkgs = []Type{ PythonRequirementsPkg, JavaPkg, JenkinsPluginPkg, + GoModulePkg, } func (t Type) String() string { diff --git a/integration/fixture_pkg_coverage_test.go b/integration/fixture_pkg_coverage_test.go index 60ffc09f9..b81e8b735 100644 --- a/integration/fixture_pkg_coverage_test.go +++ b/integration/fixture_pkg_coverage_test.go @@ -133,6 +133,14 @@ var cases = []struct { "unicorn": "4.8.3", }, }, + { + name: "find golang modules", + pkgType: pkg.GoModulePkg, + pkgLanguage: pkg.Go, + pkgInfo: map[string]string{ + "github.com/bmatcuk/doublestar": "v1.3.1", + }, + }, } func TestPkgCoverageImage(t *testing.T) { diff --git a/integration/test-fixtures/image-pkg-coverage/go/go.mod b/integration/test-fixtures/image-pkg-coverage/go/go.mod new file mode 100644 index 000000000..93da53fdf --- /dev/null +++ b/integration/test-fixtures/image-pkg-coverage/go/go.mod @@ -0,0 +1,7 @@ +module github.com/anchore/imgbom + +go 1.14 + +require ( + github.com/bmatcuk/doublestar v1.3.1 +)