use common entry point for integration tests; refactor cmd pkg (#86)

This commit is contained in:
Alex Goodman 2020-07-17 15:16:33 -04:00 committed by GitHub
parent b5a353349f
commit 9e285fd0e2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 234 additions and 252 deletions

View File

@ -75,7 +75,7 @@ unit: ## Run unit tests (with coverage)
integration: ## Run integration tests
$(call title,Running integration tests)
go test -tags=integration ./integration
go test -v -tags=integration ./integration
integration/test-fixtures/tar-cache.key, integration-fingerprint:
find integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee integration/test-fixtures/tar-cache.fingerprint

View File

@ -1,50 +0,0 @@
package cmd
import (
"fmt"
"os"
"github.com/anchore/imgbom/imgbom/presenter"
"github.com/anchore/imgbom/imgbom/scope"
"github.com/anchore/imgbom/internal/config"
"github.com/spf13/viper"
)
var cliOpts = config.CliOnlyOptions{}
func setCliOptions() {
rootCmd.PersistentFlags().StringVarP(&cliOpts.ConfigPath, "config", "c", "", "application config file")
// scan options
flag := "scope"
rootCmd.Flags().StringP(
"scope", "s", scope.AllLayersScope.String(),
fmt.Sprintf("selection of layers to catalog, options=%v", scope.Options))
if err := viper.BindPFlag(flag, rootCmd.Flags().Lookup(flag)); err != nil {
fmt.Printf("unable to bind flag '%s': %+v", flag, err)
os.Exit(1)
}
// output & formatting options
flag = "output"
rootCmd.Flags().StringP(
flag, "o", presenter.TextPresenter.String(),
fmt.Sprintf("report output formatter, options=%v", presenter.Options),
)
if err := viper.BindPFlag(flag, rootCmd.Flags().Lookup(flag)); err != nil {
fmt.Printf("unable to bind flag '%s': %+v", flag, err)
os.Exit(1)
}
flag = "quiet"
rootCmd.Flags().BoolP(
flag, "q", false,
"suppress all logging output",
)
if err := viper.BindPFlag(flag, rootCmd.Flags().Lookup(flag)); err != nil {
fmt.Printf("unable to bind flag '%s': %+v", flag, err)
os.Exit(1)
}
rootCmd.Flags().CountVarP(&cliOpts.Verbosity, "verbose", "v", "increase verbosity (-v = info, -vv = debug)")
}

View File

@ -1,14 +1,122 @@
package cmd
import (
"fmt"
"os"
"github.com/spf13/cobra"
"github.com/anchore/imgbom/imgbom/presenter"
"github.com/anchore/imgbom/imgbom/scope"
"github.com/anchore/imgbom/imgbom"
"github.com/anchore/imgbom/internal/config"
"github.com/anchore/imgbom/internal/format"
"github.com/anchore/imgbom/internal/log"
"github.com/anchore/imgbom/internal/logger"
"github.com/anchore/stereoscope"
"github.com/spf13/viper"
"github.com/wagoodman/go-partybus"
"gopkg.in/yaml.v2"
)
var appConfig *config.Application
var eventBus *partybus.Bus
var eventSubscription *partybus.Subscription
var cliOpts = config.CliOnlyOptions{}
func init() {
setGlobalCliOptions()
cobra.OnInitialize(
initAppConfig,
initLogging,
logAppConfig,
initEventBus,
)
}
func Execute() {
if err := rootCmd.Execute(); err != nil {
log.Errorf("could not start application: %w", err)
os.Exit(1)
}
}
func setGlobalCliOptions() {
rootCmd.PersistentFlags().StringVarP(&cliOpts.ConfigPath, "config", "c", "", "application config file")
// scan options
flag := "scope"
rootCmd.Flags().StringP(
"scope", "s", scope.AllLayersScope.String(),
fmt.Sprintf("selection of layers to catalog, options=%v", scope.Options))
if err := viper.BindPFlag(flag, rootCmd.Flags().Lookup(flag)); err != nil {
fmt.Printf("unable to bind flag '%s': %+v", flag, err)
os.Exit(1)
}
// output & formatting options
flag = "output"
rootCmd.Flags().StringP(
flag, "o", presenter.TextPresenter.String(),
fmt.Sprintf("report output formatter, options=%v", presenter.Options),
)
if err := viper.BindPFlag(flag, rootCmd.Flags().Lookup(flag)); err != nil {
fmt.Printf("unable to bind flag '%s': %+v", flag, err)
os.Exit(1)
}
flag = "quiet"
rootCmd.Flags().BoolP(
flag, "q", false,
"suppress all logging output",
)
if err := viper.BindPFlag(flag, rootCmd.Flags().Lookup(flag)); err != nil {
fmt.Printf("unable to bind flag '%s': %+v", flag, err)
os.Exit(1)
}
rootCmd.Flags().CountVarP(&cliOpts.Verbosity, "verbose", "v", "increase verbosity (-v = info, -vv = debug)")
}
func initAppConfig() {
cfg, err := config.LoadConfigFromFile(viper.GetViper(), &cliOpts)
if err != nil {
fmt.Printf("failed to load application config: \n\t%+v\n", err)
os.Exit(1)
}
appConfig = cfg
}
func initLogging() {
config := logger.LogConfig{
EnableConsole: (appConfig.Log.FileLocation == "" || appConfig.CliOptions.Verbosity > 0) && !appConfig.Quiet,
EnableFile: appConfig.Log.FileLocation != "",
Level: appConfig.Log.LevelOpt,
Structured: appConfig.Log.Structured,
FileLocation: appConfig.Log.FileLocation,
}
logWrapper := logger.NewZapLogger(config)
imgbom.SetLogger(logWrapper)
stereoscope.SetLogger(logWrapper)
}
func logAppConfig() {
appCfgStr, err := yaml.Marshal(&appConfig)
if err != nil {
log.Debugf("Could not display application config: %+v", err)
} else {
log.Debugf("Application config:\n%+v", format.Magenta.Format(string(appCfgStr)))
}
}
func initEventBus() {
eventBus = partybus.NewBus()
eventSubscription = eventBus.Subscribe()
stereoscope.SetBus(eventBus)
imgbom.SetBus(eventBus)
}

View File

@ -1,61 +0,0 @@
package cmd
import (
"fmt"
"os"
"github.com/anchore/imgbom/imgbom"
"github.com/anchore/imgbom/internal/config"
"github.com/anchore/imgbom/internal/format"
"github.com/anchore/imgbom/internal/log"
"github.com/anchore/imgbom/internal/logger"
"github.com/anchore/stereoscope"
"github.com/spf13/viper"
"github.com/wagoodman/go-partybus"
"gopkg.in/yaml.v2"
)
var appConfig *config.Application
var eventBus *partybus.Bus
var eventSubscription *partybus.Subscription
func initAppConfig() {
cfg, err := config.LoadConfigFromFile(viper.GetViper(), &cliOpts)
if err != nil {
fmt.Printf("failed to load application config: \n\t%+v\n", err)
os.Exit(1)
}
appConfig = cfg
}
func initLogging() {
config := logger.LogConfig{
EnableConsole: (appConfig.Log.FileLocation == "" || appConfig.CliOptions.Verbosity > 0) && !appConfig.Quiet,
EnableFile: appConfig.Log.FileLocation != "",
Level: appConfig.Log.LevelOpt,
Structured: appConfig.Log.Structured,
FileLocation: appConfig.Log.FileLocation,
}
logWrapper := logger.NewZapLogger(config)
imgbom.SetLogger(logWrapper)
stereoscope.SetLogger(logWrapper)
}
func logAppConfig() {
appCfgStr, err := yaml.Marshal(&appConfig)
if err != nil {
log.Debugf("Could not display application config: %+v", err)
} else {
log.Debugf("Application config:\n%+v", format.Magenta.Format(string(appCfgStr)))
}
}
func initEventBus() {
eventBus = partybus.NewBus()
eventSubscription = eventBus.Subscribe()
stereoscope.SetBus(eventBus)
imgbom.SetBus(eventBus)
}

View File

@ -9,7 +9,6 @@ import (
"github.com/anchore/imgbom/imgbom/presenter"
"github.com/anchore/imgbom/internal"
"github.com/anchore/imgbom/internal/bus"
"github.com/anchore/imgbom/internal/log"
"github.com/anchore/imgbom/internal/ui"
"github.com/spf13/cobra"
"github.com/wagoodman/go-partybus"
@ -33,46 +32,20 @@ Supports the following image sources:
},
}
func init() {
setCliOptions()
cobra.OnInitialize(
initAppConfig,
initLogging,
logAppConfig,
initEventBus,
)
}
func startWorker(userInput string) <-chan error {
errs := make(chan error)
go func() {
defer close(errs)
s, cleanup, err := imgbom.NewScope(userInput, appConfig.ScopeOpt)
defer cleanup()
catalog, scope, _, err := imgbom.Catalog(userInput, appConfig.ScopeOpt)
if err != nil {
log.Errorf("could not produce catalog: %w", err)
}
log.Info("Identifying Distro")
distro := imgbom.IdentifyDistro(s)
if distro == nil {
log.Errorf("error identifying distro")
} else {
log.Infof(" Distro: %s", distro)
}
log.Info("Creating the Catalog")
catalog, err := imgbom.Catalog(s)
if err != nil {
log.Errorf("could not produce catalog: %w", err)
errs <- fmt.Errorf("failed to catalog input: %+v", err)
return
}
bus.Publish(partybus.Event{
Type: event.CatalogerFinished,
Value: presenter.GetPresenter(appConfig.PresenterOpt, s, catalog),
Value: presenter.GetPresenter(appConfig.PresenterOpt, *scope, catalog),
})
}()
return errs

View File

@ -12,10 +12,10 @@ type Distro struct {
RawVersion string
}
// NewUnknownDistro creates a standardized UnkownDistro with a "0.0.0" version
// NewUnknownDistro creates a standardized Distro object for unidentifiable distros
func NewUnknownDistro() Distro {
return Distro{
Type: UnknownDistro,
Type: UnknownDistroType,
}
}

View File

@ -13,7 +13,9 @@ import (
type parseFunc func(string) *Distro
// Identify parses distro-specific files to determine distro metadata like version and release
func Identify(s scope.Scope) *Distro {
func Identify(s scope.Scope) Distro {
distro := NewUnknownDistro()
identityFiles := map[file.Path]parseFunc{
"/etc/os-release": parseOsRelease,
// Debian and Debian-based distros have the same contents linked from this path
@ -25,7 +27,7 @@ func Identify(s scope.Scope) *Distro {
refs, err := s.FilesByPath(path)
if err != nil {
log.Errorf("unable to get path refs from %s: %s", path, err)
return nil
break
}
if len(refs) == 0 {
@ -51,21 +53,17 @@ func Identify(s scope.Scope) *Distro {
continue
}
distro := fn(content)
if distro == nil {
continue
if candidateDistro := fn(content); candidateDistro != nil {
distro = *candidateDistro
break
}
return distro
}
}
// TODO: is it useful to know partially detected distros? where the ID is known but not the version (and viceversa?)
distro := NewUnknownDistro()
return &distro
return distro
}
func assembleDistro(name, version string) *Distro {
func assemble(name, version string) *Distro {
distroType, ok := Mappings[name]
// Both distro and version must be present
@ -99,7 +97,7 @@ func parseOsRelease(contents string) *Distro {
}
}
return assembleDistro(id, vers)
return assemble(id, vers)
}
var busyboxVersionMatcher = regexp.MustCompile(`BusyBox v[\d\.]+`)
@ -109,7 +107,7 @@ func parseBusyBox(contents string) *Distro {
for _, match := range matches {
parts := strings.Split(match, " ")
version := strings.ReplaceAll(parts[1], "v", "")
distro := assembleDistro("busybox", version)
distro := assemble("busybox", version)
if distro != nil {
return distro
}

View File

@ -24,12 +24,12 @@ func TestIdentifyDistro(t *testing.T) {
{
fixture: "test-fixtures/os/empty",
name: "No OS files",
Type: UnknownDistro,
Type: UnknownDistroType,
},
{
fixture: "test-fixtures/os/unmatchable",
name: "Unmatchable distro",
Type: UnknownDistro,
Type: UnknownDistroType,
},
}

View File

@ -1,7 +1,7 @@
package distro
const (
UnknownDistro Type = iota
UnknownDistroType Type = iota
Debian
Ubuntu
RedHat
@ -22,7 +22,7 @@ const (
type Type int
var distroStr = []string{
"UnknownDistro",
"UnknownDistroType",
"debian",
"ubuntu",
"redhat",

View File

@ -10,60 +10,33 @@ import (
"github.com/anchore/imgbom/imgbom/scope"
"github.com/anchore/imgbom/internal/bus"
"github.com/anchore/imgbom/internal/log"
"github.com/anchore/stereoscope"
"github.com/anchore/stereoscope/pkg/image"
"github.com/wagoodman/go-partybus"
)
func IdentifyDistro(s scope.Scope) *distro.Distro {
func Catalog(userInput string, scoptOpt scope.Option) (*pkg.Catalog, *scope.Scope, *distro.Distro, error) {
s, cleanup, err := scope.NewScope(userInput, scoptOpt)
defer cleanup()
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to create scope: %w", err)
}
d := IdentifyDistro(s)
catalog, err := CatalogFromScope(s)
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to produce catalog: %w", err)
}
return catalog, &s, &d, nil
}
func IdentifyDistro(s scope.Scope) distro.Distro {
log.Info("Identifying Distro")
return distro.Identify(s)
}
// NewScope produces a Scope based on userInput like dir:// or image:tag
func NewScope(userInput string, o scope.Option) (scope.Scope, func(), error) {
protocol := NewProtocol(userInput)
log.Debugf("protocol: %+v", protocol)
switch protocol.Type {
case DirProtocol:
// populate the scope object for dir
s, err := GetScopeFromDir(protocol.Value, o)
if err != nil {
return scope.Scope{}, func() {}, fmt.Errorf("could not populate scope from path (%s): %w", protocol.Value, err)
}
return s, func() {}, nil
case ImageProtocol:
log.Infof("Fetching image '%s'", userInput)
img, err := stereoscope.GetImage(userInput)
cleanup := func() {
stereoscope.Cleanup()
}
if err != nil || img == nil {
return scope.Scope{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", userInput, err)
}
s, err := GetScopeFromImage(img, o)
if err != nil {
return scope.Scope{}, cleanup, fmt.Errorf("could not populate scope with image: %w", err)
}
return s, cleanup, nil
default:
return scope.Scope{}, func() {}, fmt.Errorf("unable to process input for scanning: '%s'", userInput)
}
}
func GetScopeFromDir(d string, o scope.Option) (scope.Scope, error) {
return scope.NewScopeFromDir(d, o)
}
func GetScopeFromImage(img *image.Image, o scope.Option) (scope.Scope, error) {
return scope.NewScopeFromImage(img, o)
}
func Catalog(s scope.Scope) (*pkg.Catalog, error) {
func CatalogFromScope(s scope.Scope) (*pkg.Catalog, error) {
log.Info("Building the Catalog")
return cataloger.Catalog(s)
}

View File

@ -1,4 +1,4 @@
package imgbom
package scope
import "strings"
@ -7,52 +7,53 @@ import "strings"
// and return an Option type.
const (
UnknownProtocol ProtocolType = iota
ImageProtocol
DirProtocol
// nolint:varcheck,deadcode
unknownProtocol protocolType = iota
imageProtocol
directoryProtocol
)
var optionStr = []string{
var protocolStr = []string{
"UnknownProtocol",
"image",
"dir",
"Image",
"Directory",
}
type ProtocolType int
type protocolType int
type Protocol struct {
Type ProtocolType
type protocol struct {
Type protocolType
Value string
}
func NewProtocol(userStr string) Protocol {
func newProtocol(userStr string) protocol {
candidates := strings.Split(userStr, "://")
switch len(candidates) {
case 2:
if strings.HasPrefix(userStr, "dir://") {
return Protocol{
Type: DirProtocol,
return protocol{
Type: directoryProtocol,
Value: strings.TrimPrefix(userStr, "dir://"),
}
}
// default to an Image for anything else since stereoscope can handle this
return Protocol{
Type: ImageProtocol,
return protocol{
Type: imageProtocol,
Value: userStr,
}
default:
return Protocol{
Type: ImageProtocol,
return protocol{
Type: imageProtocol,
Value: userStr,
}
}
}
func (o ProtocolType) String() string {
if int(o) >= len(optionStr) || o < 0 {
return optionStr[0]
func (o protocolType) String() string {
if int(o) >= len(protocolStr) || o < 0 {
return protocolStr[0]
}
return optionStr[o]
return protocolStr[o]
}

View File

@ -1,4 +1,4 @@
package imgbom
package scope
import "testing"
@ -6,31 +6,31 @@ func TestNewProtocol(t *testing.T) {
testCases := []struct {
desc string
input string
expType ProtocolType
expType protocolType
expValue string
}{
{
desc: "directory protocol",
input: "dir:///opt/",
expType: DirProtocol,
expType: directoryProtocol,
expValue: "/opt/",
},
{
desc: "unknown protocol",
input: "s4:///opt/",
expType: ImageProtocol,
expType: imageProtocol,
expValue: "s4:///opt/",
},
{
desc: "docker protocol",
input: "docker://ubuntu:20.04",
expType: ImageProtocol,
expType: imageProtocol,
expValue: "docker://ubuntu:20.04",
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
p := NewProtocol(test.input)
p := newProtocol(test.input)
if p.Type != test.expType {
t.Errorf("mismatched type in protocol: '%v' != '%v'", p.Type, test.expType)
}

View File

@ -3,7 +3,10 @@ package scope
import (
"fmt"
"github.com/anchore/stereoscope"
"github.com/anchore/imgbom/imgbom/scope/resolvers"
"github.com/anchore/imgbom/internal/log"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/image"
)
@ -23,6 +26,41 @@ type Scope struct {
DirSrc DirSource
}
// NewScope produces a Scope based on userInput like dir:// or image:tag
func NewScope(userInput string, o Option) (Scope, func(), error) {
protocol := newProtocol(userInput)
switch protocol.Type {
case directoryProtocol:
// populate the scope object for dir
s, err := NewScopeFromDir(protocol.Value, o)
if err != nil {
return Scope{}, func() {}, fmt.Errorf("could not populate scope from path (%s): %w", protocol.Value, err)
}
return s, func() {}, nil
case imageProtocol:
log.Infof("Fetching image '%s'", userInput)
img, err := stereoscope.GetImage(userInput)
cleanup := func() {
stereoscope.Cleanup()
}
if err != nil || img == nil {
return Scope{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", userInput, err)
}
s, err := NewScopeFromImage(img, o)
if err != nil {
return Scope{}, cleanup, fmt.Errorf("could not populate scope with image: %w", err)
}
return s, cleanup, nil
default:
return Scope{}, func() {}, fmt.Errorf("unable to process input for scanning: '%s'", userInput)
}
}
func NewScopeFromDir(path string, option Option) (Scope, error) {
return Scope{
Option: option,

View File

@ -5,24 +5,25 @@ package integration
import (
"testing"
"github.com/anchore/go-testutils"
"github.com/anchore/imgbom/imgbom"
"github.com/anchore/go-testutils"
"github.com/anchore/imgbom/imgbom/distro"
"github.com/anchore/imgbom/imgbom/scope"
"github.com/go-test/deep"
)
func TestDistroImage(t *testing.T) {
img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-distro-id")
fixtureImageName := "image-distro-id"
_, cleanup := testutils.GetFixtureImage(t, "docker-archive", fixtureImageName)
tarPath := testutils.GetFixtureImageTarPath(t, fixtureImageName)
defer cleanup()
s, err := imgbom.GetScopeFromImage(img, scope.AllLayersScope)
_, _, actualDistro, err := imgbom.Catalog("docker-archive://"+tarPath, scope.AllLayersScope)
if err != nil {
t.Fatalf("could not populate scope with image: %+v", err)
t.Fatalf("failed to catalog image: %+v", err)
}
actual := imgbom.IdentifyDistro(s)
if actual == nil {
if actualDistro == nil {
t.Fatalf("could not find distro")
}
@ -31,7 +32,7 @@ func TestDistroImage(t *testing.T) {
t.Fatalf("could not create distro: %+v", err)
}
diffs := deep.Equal(*actual, expected)
diffs := deep.Equal(*actualDistro, expected)
if len(diffs) != 0 {
for _, d := range diffs {
t.Errorf("found distro difference: %+v", d)

View File

@ -5,21 +5,22 @@ package integration
import (
"testing"
"github.com/anchore/imgbom/imgbom"
"github.com/anchore/imgbom/internal"
"github.com/anchore/go-testutils"
"github.com/anchore/imgbom/imgbom/cataloger"
"github.com/anchore/imgbom/imgbom/pkg"
"github.com/anchore/imgbom/imgbom/scope"
)
func TestLanguageImage(t *testing.T) {
img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-pkg-coverage")
func TestPkgCoverageImage(t *testing.T) {
fixtureImageName := "image-pkg-coverage"
_, cleanup := testutils.GetFixtureImage(t, "docker-archive", fixtureImageName)
tarPath := testutils.GetFixtureImageTarPath(t, fixtureImageName)
defer cleanup()
s, err := scope.NewScopeFromImage(img, scope.AllLayersScope)
catalog, err := cataloger.Catalog(s)
catalog, _, _, err := imgbom.Catalog("docker-archive://"+tarPath, scope.AllLayersScope)
if err != nil {
t.Fatalf("failed to catalog image: %+v", err)
}