Upgrade binary test fixtures management (#2444)

* test: strip fixtures of any execution permissions

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>

* chore: add lint check for large files

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>

* add helper script to capture binary snippets

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* chore: update scripts and add new dir output for snippets

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>

* test: update erlang test to new generated format

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>

* test: update memcached to new generator pattern

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>

* test: update openjdk to named version

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>

* test: move openjdk lts to versioned folder

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>

* test: rename unversioned java to versioned folders

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>

* test: migrate bash fixture to new snippet workflow

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>

* test: update script to size 600 bytes

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>

* test: update go classifier to new snippet workflow

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>

* test: move haproxy new new snippet

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>

* test: add flatter haproxy example

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>

* test: update tests to new pattern

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>

* test: final version of snippet script

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>

* [wip] download bin helpers

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add manager for binary cataloger test fixtures

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add remaining binary cataloger patterns and snippets

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* adjust gitignore to be more permissive to snippets

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add rust darwin snippets

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* skip tests that are missing full binaries

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address PR feedback

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add tests for binary test fixture manager

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* highlight rows that do not have binaries or snippets

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* bump fixture limit to 1K (found exceptions when adding snippets)

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add redis and postgres snippets

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* improve formating of fixture listing

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Christopher Angelo Phillips
2024-01-05 21:40:03 +00:00
committed by GitHub
co-authored by Alex Goodman
parent a16a4ad6c9
commit 7182f5b519
158 changed files with 3539 additions and 500 deletions
@@ -0,0 +1,40 @@
package cli
import (
"github.com/spf13/cobra"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/cli/commands"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config"
)
// list all managed binaries (in ./bin, organized by 'name-version/platform/binary')
// manager list binaries
// list all managed snippets (in ./snippets, same organization as ./bin: 'name-version/platform/binary' where each bin is a snippet)
// manager list snippets
// download all binaries (to ./bin)
// manager download [--name <name>] [--version <version>]
// capture snippet from a binary identified by offset
// manager capture snippet --binary <binary> --offset <offset> --length <length>
func New() (*cobra.Command, error) {
cfgP, err := config.Read()
if err != nil {
return nil, err
}
cfg := *cfgP
root := commands.Root(cfg)
root.AddCommand(
commands.List(cfg),
commands.Download(cfg),
commands.AddSnippet(cfg),
commands.WriteSnippet(cfg),
)
return root, nil
}
@@ -0,0 +1,109 @@
package commands
import (
"fmt"
"os"
"os/exec"
"strings"
"github.com/anmitsu/go-shlex"
"github.com/spf13/cobra"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/ui"
)
func AddSnippet(appConfig config.Application) *cobra.Command {
var binaryPath, searchPattern string
var length, prefixLength int
cmd := &cobra.Command{
Use: "add-snippet",
Short: "capture snippets from binaries",
Args: cobra.NoArgs,
PreRunE: func(cmd *cobra.Command, args []string) error {
candidates, err := internal.ListAllBinaries(appConfig)
if err != nil {
return fmt.Errorf("unable to list binaries: %w", err)
}
// launch the UI to pick a path
var binaryPaths []string
for _, k := range internal.NewLogicalEntryKeys(candidates) {
info := candidates[k]
if info.BinaryPath != "" {
binaryPaths = append(binaryPaths, info.BinaryPath)
}
}
binaryPath, err = ui.PromptSelectBinary(binaryPaths)
if err != nil {
return err
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
name, version, _, err := inferInfoFromBinaryPath(appConfig, binaryPath)
if err != nil {
return fmt.Errorf("unable to infer name and version from binary path: %w", err)
}
if searchPattern == "" {
searchPattern = strings.ReplaceAll(version, ".", `\\.`)
}
return runAddSnippet(binaryPath, name, version, searchPattern, length, prefixLength)
},
}
cmd.Flags().StringVar(&searchPattern, "search-for", "", "the pattern to search for in the binary (defaults to the version)")
cmd.Flags().IntVar(&length, "length", 100, "the length of the snippet to capture")
cmd.Flags().IntVar(&prefixLength, "prefix-length", 20, "number of bytes before the search hit to capture")
return cmd
}
func runAddSnippet(binaryPath, name, version, searchPattern string, length, prefixLength int) error {
// invoke ./capture-snippet.sh <path-to-binary> <version> [--search-for <pattern>] [--length <length>] [--prefix-length <prefix_length>]"
cmd := exec.Command("./capture-snippet.sh", binaryPath, version)
var args []string
if searchPattern != "" {
args = append(args, "--search-for", searchPattern)
}
if name != "" {
args = append(args, "--group", name)
}
if length > 0 {
args = append(args, fmt.Sprintf("--length %d", length))
}
if prefixLength > 0 {
args = append(args, fmt.Sprintf("--prefix-length %d", prefixLength))
}
var err error
args, err = shlex.Split(strings.Join(args, " "), true)
if err != nil {
return fmt.Errorf("failed to parse arguments: %w", err)
}
cmd.Args = append(cmd.Args, args...)
fmt.Printf("running: %s\n", strings.Join(cmd.Args, " "))
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Stdin = os.Stdin
if err := cmd.Start(); err != nil {
return fmt.Errorf("failed to start command: %w", err)
}
if err := cmd.Wait(); err != nil {
return fmt.Errorf("command execution failed: %w", err)
}
return nil
}
@@ -0,0 +1,79 @@
package commands
import (
"fmt"
"github.com/spf13/cobra"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config"
)
func Download(appConfig config.Application) *cobra.Command {
var configs []config.BinaryFromImage
var skipSnippets bool
cmd := &cobra.Command{
Use: "download",
Short: "download binaries [name@version ...]",
PreRunE: func(cmd *cobra.Command, args []string) error {
if len(args) > 0 {
for _, arg := range args {
binaryFromImageCfg := appConfig.GetBinaryFromImage(arg, "")
if binaryFromImageCfg == nil {
return fmt.Errorf("no config found for %q", arg)
}
configs = append(configs, *binaryFromImageCfg)
}
} else {
configs = appConfig.FromImages
}
if skipSnippets {
var err error
configs, err = configsWithoutSnippets(appConfig, configs)
if err != nil {
return err
}
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
for _, binaryFromImageCfg := range configs {
if err := internal.DownloadFromImage(appConfig.DownloadPath, binaryFromImageCfg); err != nil {
return err
}
}
if len(configs) == 0 {
fmt.Println("no binaries to download")
}
return nil
},
}
cmd.Flags().BoolVarP(&skipSnippets, "skip-if-covered-by-snippet", "s", false, "skip downloading entries already covered by snippets")
return cmd
}
func configsWithoutSnippets(appConfig config.Application, configs []config.BinaryFromImage) ([]config.BinaryFromImage, error) {
entries, err := internal.ListAllEntries(appConfig)
if err != nil {
return nil, err
}
var filtered []config.BinaryFromImage
for _, cfg := range configs {
if entries.BinaryFromImageHasSnippet(cfg) {
continue
}
filtered = append(filtered, cfg)
}
return filtered, nil
}
@@ -0,0 +1,137 @@
package commands
import (
"fmt"
"strings"
"github.com/charmbracelet/lipgloss"
"github.com/jedib0t/go-pretty/v6/table"
"github.com/spf13/cobra"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config"
)
func List(appConfig config.Application) *cobra.Command {
var showPaths bool
cmd := &cobra.Command{
Use: "list",
Short: "list managed binaries and managed/unmanaged snippets",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
return runList(appConfig, showPaths)
},
}
cmd.Flags().BoolVarP(&showPaths, "show-paths", "p", false, "show paths to binaries and snippets")
return cmd
}
func runList(appConfig config.Application, showPaths bool) error {
material, err := internal.ListAllEntries(appConfig)
if err != nil {
return err
}
report := renderCatalogerListTable(material, showPaths)
report += "\n Legend:\n"
report += errorStyle.Render(" !! indicates there is no binary or snippet for a configured entry") + "\n"
report += warningStyle.Render(" ! indicates that the binary is managed but there is no snippet") + "\n"
fmt.Println(report)
return nil
}
const yes = "yes"
func renderCatalogerListTable(material map[internal.LogicalEntryKey]internal.EntryInfo, showPaths bool) string {
t := table.NewWriter()
t.SetStyle(table.StyleLight)
t.AppendHeader(table.Row{"Group", "Version", "Platform", "Name", "Configured?", "Binary", "Snippet"})
keys := internal.NewLogicalEntryKeys(material)
for _, k := range keys {
info := material[k]
isConfigured := ""
if info.IsConfigured {
isConfigured = yes
}
bin := ""
snippet := ""
if showPaths {
bin = info.BinaryPath
snippet = info.SnippetPath
} else {
if info.BinaryPath != "" {
bin = yes
}
if info.SnippetPath != "" {
snippet = yes
}
}
var state displayState
if snippet == "" {
if bin == "" {
state = displayStateError
} else {
state = displayStateWarning
}
}
t.AppendRow(table.Row{
renderCell(k.OrgName, state, "!"),
renderCell(k.Version, state),
renderCell(displayPlatform(k.Platform), state),
renderCell(k.Filename, state),
renderCell(isConfigured, state),
renderCell(bin, state),
renderCell(snippet, state),
})
}
return t.Render()
}
type displayState string
const (
displayStateError displayState = "error"
displayStateWarning displayState = "warning"
)
var (
errorStyle = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("9"))
warningStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("11"))
stdStyle = lipgloss.NewStyle()
)
func renderCell(value string, state displayState, hints ...string) string {
hint := strings.Join(hints, "")
var prefix string
switch state {
case displayStateError:
return errorStyle.Render(strings.Repeat(hint, 2) + value)
case displayStateWarning:
if hint != "" {
prefix = " "
}
return warningStyle.Render(hint + prefix + value)
}
if hint != "" {
prefix = " "
}
return stdStyle.Render(prefix + value)
}
func displayPlatform(platform string) string {
return strings.ReplaceAll(platform, "-", "/")
}
@@ -0,0 +1,14 @@
package commands
import (
"github.com/spf13/cobra"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config"
)
func Root(_ config.Application) *cobra.Command {
return &cobra.Command{
Use: "manager",
Short: "manager is a tool for managing binaries and snippets",
}
}
@@ -0,0 +1,263 @@
package commands
import (
"debug/elf"
"debug/macho"
"debug/pe"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config"
)
func WriteSnippet(appConfig config.Application) *cobra.Command {
var offset, length int
var name, version string
var binaryPath string
cmd := &cobra.Command{
Use: "write-snippet [binary]",
Short: "capture snippets from binaries",
Args: cobra.ExactArgs(1),
PreRunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 && (name != "" || version != "") {
return fmt.Errorf("cannot provide name or version without a binary path")
}
binaryPath = args[0]
if _, err := os.Stat(binaryPath); err != nil {
return fmt.Errorf("unable to stat %q: %w", binaryPath, err)
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
platform, err := getPlatform(binaryPath)
if err != nil {
return fmt.Errorf("unable to get platform: %w", err)
}
snippetPath, err := getSnippetPath(appConfig, binaryPath, name, version, platform)
if err != nil {
return fmt.Errorf("unable to get snippet path: %w", err)
}
return runWriteSnippet(binaryPath, offset, length, snippetPath)
},
}
cmd.Flags().IntVar(&offset, "offset", -1, "the offset in the binary to start the snippet")
cmd.Flags().IntVar(&length, "length", 100, "the length of the snippet to capture")
cmd.Flags().StringVar(&name, "name", "", "the name of the snippet")
cmd.Flags().StringVar(&version, "version", "", "the version of the snippet")
return cmd
}
func runWriteSnippet(binaryPath string, offset, length int, snippetPath string) error {
f, err := os.Open(binaryPath)
if err != nil {
return fmt.Errorf("unable to open binary %q: %w", binaryPath, err)
}
n, err := f.Seek(int64(offset), io.SeekStart)
if err != nil {
return fmt.Errorf("unable to seek to offset %d: %w", offset, err)
}
if n != int64(offset) {
return fmt.Errorf("unexpectd to seek value: %d != %d", offset, n)
}
buf := make([]byte, length)
n2, err := f.Read(buf)
if err != nil {
return fmt.Errorf("unable to read %d bytes: %w", length, err)
}
if n2 != length {
return fmt.Errorf("unexpected read length: %d != %d", length, n2)
}
fileDigest, err := internal.Sha256SumFile(f)
if err != nil {
return err
}
metadata := internal.SnippetMetadata{
Name: filepath.Base(binaryPath),
Offset: offset,
Length: length,
SnippetSha256: internal.Sha256SumBytes(buf),
FileSha256: fileDigest,
}
metadataBytes, err := yaml.Marshal(metadata)
if err != nil {
return fmt.Errorf("unable to marshal metadata: %w", err)
}
splitter := []byte("\n### byte snippet to follow ###\n")
var finalBuf []byte
finalBuf = append(finalBuf, metadataBytes...)
finalBuf = append(finalBuf, splitter...)
finalBuf = append(finalBuf, buf...)
if err := os.MkdirAll(filepath.Dir(snippetPath), 0755); err != nil {
return fmt.Errorf("unable to create destination directory: %w", err)
}
if err := os.WriteFile(snippetPath, finalBuf, 0600); err != nil {
return fmt.Errorf("unable to write snippet: %w", err)
}
fmt.Printf("wrote snippet to %q\n", snippetPath)
return nil
}
func getSnippetPath(appConfig config.Application, binaryPath string, name, version, platform string) (string, error) {
binFilename := filepath.Base(binaryPath)
platform = config.PlatformAsValue(platform)
// if all values provided, use them
if name != "" && version != "" && platform != "" {
return filepath.Join(appConfig.SnippetPath, name, version, platform, binFilename), nil
}
// otherwise, try to infer them from the existing binary path
name, version, platform, err := inferInfoFromBinaryPath(appConfig, binaryPath)
if err != nil {
return "", err
}
return filepath.Join(appConfig.SnippetPath, name, version, platform, binFilename), nil
}
func inferInfoFromBinaryPath(appConfig config.Application, binaryPath string) (string, string, string, error) {
relativePath, err := filepath.Rel(appConfig.DownloadPath, binaryPath)
if err != nil {
return "", "", "", fmt.Errorf("unable to get relative path: %w", err)
}
// otherwise, try to infer them from the existing binary path
items := internal.SplitFilepath(relativePath)
if len(items) != 4 {
return "", "", "", fmt.Errorf("too few fields: %q", binaryPath)
}
name := items[0]
version := items[1]
platform := items[2]
return name, version, platform, nil
}
// getPlatform will return <os>-<arch> for the given binary path, where os can be "linux", "darwin", "windows",
// and arch can be "amd64", "arm64", "arm", etc.
func getPlatform(binaryPath string) (string, error) {
f, err := os.Open(binaryPath)
if err != nil {
return "", fmt.Errorf("unable to open binary %q: %w", binaryPath, err)
}
elfPlatform := getPlatformElf(f)
if elfPlatform != "" {
return elfPlatform, nil
}
macPlatform := getPlatformMac(f)
if macPlatform != "" {
return macPlatform, nil
}
winPlatform := getPlatformWindows(f)
if winPlatform != "" {
return winPlatform, nil
}
// attempt to infer from the path. It is possible to see invalid-looking binaries that are still something
// we'd like to detect.
items := internal.SplitFilepath(binaryPath)
if len(items) > 2 {
candidate := items[len(items)-2]
if strings.Contains(candidate, "linux") || strings.Contains(candidate, "darwin") || strings.Contains(candidate, "windows") {
return candidate, nil
}
}
return "", fmt.Errorf("unable to determine platform for %q", binaryPath)
}
const (
amd64 = "amd64"
arm64 = "arm64"
)
func getPlatformElf(f *os.File) string {
elfFile, err := elf.NewFile(f)
if err != nil {
return ""
}
var arch string
switch elfFile.Machine {
case elf.EM_X86_64:
arch = amd64
case elf.EM_AARCH64:
arch = arm64
// TODO...
default:
arch = fmt.Sprintf("unknown-%x", elfFile.Machine)
}
return fmt.Sprintf("linux-%s", arch)
}
func getPlatformMac(f *os.File) string {
machoFile, err := macho.NewFile(f)
if err != nil {
return ""
}
var arch string
switch machoFile.Cpu {
case macho.CpuAmd64:
arch = amd64
case macho.CpuArm64:
arch = arm64
// TODO...
default:
arch = fmt.Sprintf("unknown-%x", machoFile.Cpu)
}
return fmt.Sprintf("darwin-%s", arch)
}
func getPlatformWindows(f *os.File) string {
peFile, err := pe.NewFile(f)
if err != nil {
return ""
}
var arch string
switch peFile.Machine {
case pe.IMAGE_FILE_MACHINE_AMD64:
arch = amd64
case pe.IMAGE_FILE_MACHINE_ARM64:
arch = arm64
// TODO...
default:
arch = fmt.Sprintf("unknown-%x", peFile.Machine)
}
return fmt.Sprintf("windows-%s", arch)
}
@@ -0,0 +1,139 @@
package config
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/hashicorp/go-multierror"
"github.com/scylladb/go-set/strset"
"gopkg.in/yaml.v3"
)
const Path = "config.yaml"
type Application struct {
DownloadPath string `yaml:"download-path"`
SnippetPath string `yaml:"snippet-path"`
FromImages []BinaryFromImage `yaml:"from-images"`
}
func DefaultApplication() Application {
return Application{
DownloadPath: "bin",
}
}
func Read() (*Application, error) {
return read(Path)
}
func read(path string) (*Application, error) {
appConfig := DefaultApplication()
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
err = yaml.Unmarshal(data, &appConfig)
if err != nil {
return nil, err
}
if err := appConfig.Validate(); err != nil {
return nil, err
}
return &appConfig, nil
}
func (c Image) Key() string {
return fmt.Sprintf("%s:%s", c.Reference, c.Platform)
}
func (c Application) Validate() error {
set := strset.New()
var err error
for i, entry := range c.FromImages {
key := entry.Key()
if set.Has(key) {
err = multierror.Append(err, fmt.Errorf("duplicate entry %q", entry))
continue
}
set.Add(entry.Key())
if len(entry.PathsInImage) > 1 && entry.GenericName == "" {
err = multierror.Append(err, fmt.Errorf("specified multiple paths but no name for entry %d (%s)", i+1, key))
}
if entry.Name() == "" {
err = multierror.Append(err, fmt.Errorf("missing name for entry %d", i+1))
}
if entry.Version == "" {
err = multierror.Append(err, fmt.Errorf("missing version for entry %d", i+1))
}
if len(entry.Images) == 0 {
err = multierror.Append(err, fmt.Errorf("missing images for entry %d (%s)", i+1, key))
}
var imageSet = strset.New()
for j, image := range entry.Images {
imgKey := image.Key()
if imageSet.Has(imgKey) {
err = multierror.Append(err, fmt.Errorf("duplicate image %q for entry %d (%s)", image.Key(), i+1, key))
continue
}
imageSet.Add(imgKey)
if image.Reference == "" {
err = multierror.Append(err, fmt.Errorf("missing ref reference for entry %d (%s) image %d", i+1, key, j+1))
}
if image.Platform == "" {
err = multierror.Append(err, fmt.Errorf("missing platform for entry %d (%s) image %d", i+1, key, j+1))
}
}
if len(entry.PathsInImage) == 0 {
err = multierror.Append(err, fmt.Errorf("missing paths for entry %d (%s)", i+1, key))
}
}
return err
}
func (c Application) GetBinaryFromImage(name, version string) *BinaryFromImage {
if strings.Contains(name, "@") && version == "" {
parts := strings.Split(name, "@")
name = parts[0]
version = parts[1]
}
for _, entry := range c.FromImages {
if entry.Name() == name && entry.Version == version {
return &entry
}
}
return nil
}
func (c Application) GetBinaryFromImageByPath(storePath string) *BinaryFromImage {
// each key is the store path except for the root (e.g. bin or snippet)
entryByStorePath := make(map[string]BinaryFromImage)
for _, entry := range c.FromImages {
for _, path := range entry.AllStorePaths(c.DownloadPath) {
pathWithoutRoot := splitFilepath(path)[1:]
entryByStorePath[filepath.Join(pathWithoutRoot...)] = entry
}
}
pathWithoutRoot := filepath.Join(splitFilepath(storePath)[1:]...)
if entry, ok := entryByStorePath[pathWithoutRoot]; ok {
return &entry
}
return nil
}
func splitFilepath(path string) []string {
return strings.Split(path, string(filepath.Separator))
}
@@ -0,0 +1,67 @@
package config
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestReadAndValidate(t *testing.T) {
tests := []struct {
name string
expectErr bool
}{
{
name: "valid-1.yaml",
},
{
name: "valid-2.yaml",
},
{
name: "bad-implicit-name-collision.yaml",
expectErr: true,
},
{
name: "bad-missing-version.yaml",
expectErr: true,
},
{
name: "bad-missing-image.yaml",
expectErr: true,
},
{
name: "bad-no-name.yaml",
expectErr: true,
},
{
name: "bad-missing-image-platform.yaml",
expectErr: true,
},
{
name: "bad-missing-image-ref.yaml",
expectErr: true,
},
{
name: "bad-image-collision.yaml",
expectErr: true,
},
{
name: "bad-missing-paths.yaml",
expectErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
appConfig, err := read(filepath.Join("testdata", "app-configs", tt.name))
if tt.expectErr {
assert.Error(t, err)
} else {
assert.NoError(t, err)
assert.NotNil(t, appConfig)
assert.NoError(t, appConfig.Validate())
}
})
}
}
@@ -0,0 +1,80 @@
package config
import (
"crypto/sha256"
"fmt"
"path/filepath"
"strings"
"gopkg.in/yaml.v3"
)
type BinaryFromImage struct {
GenericName string `yaml:"name"`
Version string `yaml:"version"`
Images []Image `yaml:"images"`
PathsInImage []string `yaml:"paths"`
}
type Image struct {
Reference string `yaml:"ref"`
Platform string `yaml:"platform"`
}
func (c BinaryFromImage) Key() string {
return fmt.Sprintf("%s:%s", c.Name(), c.Version)
}
func (c BinaryFromImage) Name() string {
displayName := c.GenericName
if displayName == "" {
var path string
if len(c.PathsInImage) > 0 {
path = c.PathsInImage[0]
}
if path == "" {
return ""
}
return filepath.Base(path)
}
return displayName
}
func (c BinaryFromImage) AllStorePaths(dest string) []string {
var paths []string
for _, image := range c.Images {
paths = append(paths, c.AllStorePathsForImage(image, dest)...)
}
return paths
}
func (c BinaryFromImage) AllStorePathsForImage(image Image, dest string) []string {
var paths []string
platform := PlatformAsValue(image.Platform)
for _, path := range c.PathsInImage {
base := filepath.Base(path)
if path == "" {
base = ""
}
paths = append(paths, filepath.Join(dest, c.Name(), c.Version, platform, base))
}
return paths
}
func PlatformAsValue(platform string) string {
return strings.ReplaceAll(platform, "/", "-")
}
func (c BinaryFromImage) Fingerprint() string {
by, err := yaml.Marshal(c)
if err != nil {
panic(err)
}
hasher := sha256.New()
hasher.Write(by)
return fmt.Sprintf("%x", hasher.Sum(nil))
}
@@ -0,0 +1,191 @@
package config
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestKey(t *testing.T) {
tests := []struct {
name string
binary BinaryFromImage
expected string
}{
{
name: "gocase",
binary: BinaryFromImage{
GenericName: "test",
Version: "1.0",
},
expected: "test:1.0",
},
{
name: "binary name",
binary: BinaryFromImage{
Version: "1.0",
PathsInImage: []string{
"path/to/test",
},
},
expected: "test:1.0",
},
{
name: "first binary name",
binary: BinaryFromImage{
Version: "1.0",
PathsInImage: []string{
"path/to/test",
"path/to/nothing",
},
},
expected: "test:1.0",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, tt.binary.Key())
})
}
}
func TestName(t *testing.T) {
tests := []struct {
name string
binary BinaryFromImage
expected string
}{
{
name: "given name",
binary: BinaryFromImage{
GenericName: "given",
PathsInImage: []string{
"path/to/test",
},
},
expected: "given",
},
{
name: "binary name",
binary: BinaryFromImage{
PathsInImage: []string{
"path/to/test",
},
},
expected: "test",
},
{
name: "first binary name",
binary: BinaryFromImage{
PathsInImage: []string{
"path/to/test",
"path/to/nothing",
},
},
expected: "test",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, tt.binary.Name())
})
}
}
func TestAllStorePaths(t *testing.T) {
tests := []struct {
name string
binary BinaryFromImage
dest string
expected []string
}{
{
name: "gocase",
binary: BinaryFromImage{
GenericName: "test",
Version: "1.0",
Images: []Image{
{
Reference: "ref1",
Platform: "platform1",
},
{
Reference: "ref2",
Platform: "platform2",
},
},
PathsInImage: []string{
"path/to/test1",
"path/to/test2",
},
},
dest: "dest",
expected: []string{
"dest/test/1.0/platform1/test1",
"dest/test/1.0/platform1/test2",
"dest/test/1.0/platform2/test1",
"dest/test/1.0/platform2/test2",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, tt.binary.AllStorePaths(tt.dest))
})
}
}
func TestPlatformAsValue(t *testing.T) {
tests := []struct {
name string
platform string
expected string
}{
{
name: "gocase",
platform: "platform/test",
expected: "platform-test",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, PlatformAsValue(tt.platform))
})
}
}
func TestFingerprint(t *testing.T) {
tests := []struct {
name string
binary BinaryFromImage
expected string
}{
{
name: "gocase",
binary: BinaryFromImage{
GenericName: "test",
Version: "1.0",
Images: []Image{
{
Reference: "ref",
Platform: "platform",
},
},
PathsInImage: []string{
"path/to/test",
},
},
expected: "54ed081c07e4eba031afed4c04315cf96047822196473971be98d0769a0e3645",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, tt.binary.Fingerprint())
})
}
}
@@ -0,0 +1,14 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- name: busybox
version: 1.36.1
images:
- ref: busybox:1.36.1@sha256:058f0df5310fbbbfea7e81a3a3e2b4bf3452438ec841138d170e170adbbd27a4
platform: linux/amd64
- ref: busybox:1.36.1@sha256:058f0df5310fbbbfea7e81a3a3e2b4bf3452438ec841138d170e170adbbd27a4
platform: linux/amd64
paths:
- /bin/[
@@ -0,0 +1,19 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- name: bash # !important
version: 1.36.1
images:
- ref: busybox:1.36.1@sha256:058f0df5310fbbbfea7e81a3a3e2b4bf3452438ec841138d170e170adbbd27a4
platform: linux/amd64
paths:
- /bin/[
- version: 1.36.1 # !important
images:
- ref: bash:5.1.16@sha256:c7a903a541d8f5fe693cbe7f5ece18a1b6a03734c76092d2b153d7e98a964927
platform: linux/amd64
paths:
- /usr/local/bin/bash
@@ -0,0 +1,12 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- name: busybox
version: 1.36.1
images:
- ref: busybox:1.36.1@sha256:058f0df5310fbbbfea7e81a3a3e2b4bf3452438ec841138d170e170adbbd27a4
paths:
- /bin/[
@@ -0,0 +1,11 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- name: busybox
version: 1.36.1
images:
- platform: linux/amd64
paths:
- /bin/[
@@ -0,0 +1,9 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- name: busybox
version: 1.36.1
paths:
- /bin/[
@@ -0,0 +1,10 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- name: busybox
version: 1.36.1
images:
- ref: busybox:1.36.1@sha256:058f0df5310fbbbfea7e81a3a3e2b4bf3452438ec841138d170e170adbbd27a4
platform: linux/amd64
@@ -0,0 +1,10 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- images:
- ref: bash:5.1.16@sha256:c7a903a541d8f5fe693cbe7f5ece18a1b6a03734c76092d2b153d7e98a964927
platform: linux/amd64
paths:
- /usr/local/bin/bash
@@ -0,0 +1,12 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- version: 1.36.1
images:
- ref: busybox:1.36.1@sha256:058f0df5310fbbbfea7e81a3a3e2b4bf3452438ec841138d170e170adbbd27a4
platform: linux/amd64
paths:
- /bin/[
- /bin/busybox
@@ -0,0 +1,19 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- name: busybox
version: 1.36.1
images:
- ref: busybox:1.36.1@sha256:058f0df5310fbbbfea7e81a3a3e2b4bf3452438ec841138d170e170adbbd27a4
platform: linux/amd64
paths:
- /bin/[
- version: 5.1.16
images:
- ref: bash:5.1.16@sha256:c7a903a541d8f5fe693cbe7f5ece18a1b6a03734c76092d2b153d7e98a964927
platform: linux/amd64
paths:
- /usr/local/bin/bash
@@ -0,0 +1,19 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- name: bash # !important
version: 1.36.1
images:
- ref: busybox:1.36.1@sha256:058f0df5310fbbbfea7e81a3a3e2b4bf3452438ec841138d170e170adbbd27a4
platform: linux/amd64
paths:
- /bin/[
- version: 5.1.16
images:
- ref: bash:5.1.16@sha256:c7a903a541d8f5fe693cbe7f5ece18a1b6a03734c76092d2b153d7e98a964927
platform: linux/amd64
paths:
- /usr/local/bin/bash
@@ -0,0 +1,197 @@
package internal
import (
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"github.com/google/uuid"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/ui"
)
func DownloadFromImage(dest string, config config.BinaryFromImage) error {
t := ui.Title{Name: config.Name(), Version: config.Version}
t.Start()
hostPaths := config.AllStorePaths(dest)
if allPathsExist(hostPaths) {
if !isDownloadStale(config, hostPaths) {
t.Skip("already exists")
return nil
}
t.Update("stale, updating...")
}
if err := pullDockerImages(config.Images); err != nil {
return err
}
if err := copyBinariesFromDockerImages(config, dest); err != nil {
return fmt.Errorf("failed to copy binary for %s@%s: %v", config.Name(), config.Version, err)
}
return nil
}
func isDownloadStale(config config.BinaryFromImage, binaryPaths []string) bool {
currentFingerprint := config.Fingerprint()
for _, path := range binaryPaths {
fingerprintPath := path + ".fingerprint"
if _, err := os.Stat(fingerprintPath); err != nil {
// missing a fingerprint file means the download is stale
return true
}
writtenFingerprint, err := os.ReadFile(fingerprintPath)
if err != nil {
// missing a fingerprint file means the download is stale
return true
}
if string(writtenFingerprint) != currentFingerprint {
// the fingerprint file does not match the current fingerprint, so the download is stale
return true
}
}
return false
}
func allPathsExist(paths []string) bool {
for _, path := range paths {
if _, err := os.Stat(path); err != nil {
return false
}
}
return true
}
func pullDockerImages(images []config.Image) error {
for _, image := range images {
if err := pullDockerImage(image.Reference, image.Platform); err != nil {
return fmt.Errorf("failed to pull image %s for platform %s: %v", image.Reference, image.Platform, err)
}
}
return nil
}
type imageInspect struct {
OS string `json:"Os"`
Architecture string `json:"Architecture"`
}
func (i imageInspect) Platform() string {
return fmt.Sprintf("%s/%s", i.OS, i.Architecture)
}
func pullDockerImage(imageReference, platform string) error {
a := ui.Action{Msg: fmt.Sprintf("pull image %s (%s)", imageReference, platform)}
a.Start()
matches, _, _ := checkArchitecturesMatch(imageReference, platform)
if matches {
a.Skip(fmt.Sprintf("docker image already exists %q", imageReference))
return nil
}
cmd := exec.Command("docker", "pull", "--platform", platform, imageReference)
err := cmd.Run()
if err != nil {
a.Done(err)
return err
}
matches, gotPlatform, err := checkArchitecturesMatch(imageReference, platform)
if !matches && err == nil {
err = fmt.Errorf("image %q pulled but does not match expected platform %q != %q", imageReference, platform, gotPlatform)
}
a.Done(err)
return err
}
func checkArchitecturesMatch(imageReference, platform string) (bool, string, error) {
cmd := exec.Command("docker", "image", "inspect", imageReference)
out, err := cmd.CombinedOutput()
if err != nil {
return false, "", err
}
var inspect []imageInspect
if err := json.Unmarshal(out, &inspect); err != nil {
return false, "", fmt.Errorf("unable to unmarshal image inspect: %w", err)
}
if len(inspect) != 1 {
return false, "", fmt.Errorf("expected 1 image inspect, got %d", len(inspect))
}
gotPlatform := inspect[0].Platform()
return gotPlatform == platform, gotPlatform, nil
}
func copyBinariesFromDockerImages(config config.BinaryFromImage, destination string) (err error) {
for _, image := range config.Images {
if err := copyBinariesFromDockerImage(config, destination, image); err != nil {
return err
}
}
return nil
}
func copyBinariesFromDockerImage(config config.BinaryFromImage, destination string, image config.Image) (err error) {
containerName := fmt.Sprintf("%s-%s-%s", config.Name(), config.Version, uuid.New().String())
cmd := exec.Command("docker", "create", "--name", containerName, image.Reference)
if err = cmd.Run(); err != nil {
return err
}
defer func() {
cmd := exec.Command("docker", "rm", containerName)
cmd.Run() // nolint:errcheck
}()
for i, destinationPath := range config.AllStorePathsForImage(image, destination) {
path := config.PathsInImage[i]
if err := copyBinaryFromContainer(containerName, path, destinationPath, config.Fingerprint()); err != nil {
return err
}
}
return nil
}
func copyBinaryFromContainer(containerName, containerPath, destinationPath, fingerprint string) (err error) {
a := ui.Action{Msg: fmt.Sprintf("extract %s", containerPath)}
a.Start()
defer func() {
a.Done(err)
}()
if err := os.MkdirAll(filepath.Dir(destinationPath), 0755); err != nil {
return err
}
cmd := exec.Command("docker", "cp", fmt.Sprintf("%s:%s", containerName, containerPath), destinationPath) // nolint:gosec
// reason for gosec exception: this is for processing test fixtures only, not used in production
if err := cmd.Run(); err != nil {
return err
}
// capture fingerprint file
fingerprintPath := destinationPath + ".fingerprint"
if err := os.WriteFile(fingerprintPath, []byte(fingerprint), 0600); err != nil {
return fmt.Errorf("unable to write fingerprint file: %w", err)
}
return nil
}
@@ -0,0 +1,62 @@
package internal
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config"
)
func TestIsDownloadStale(t *testing.T) {
cases := []struct {
name string
fingerprint string
expected bool
}{
{
name: "no fingerprint",
fingerprint: "",
expected: true,
},
{
name: "fingerprint matches",
// this is the fingerprint for config in the loop body
fingerprint: "5177d458eaca031ea16fa707841043df2e31b89be6bae7ea41290aa32f0251a6",
expected: false,
},
{
name: "fingerprint does not match",
fingerprint: "fingerprint",
expected: true,
},
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
binaryPath := filepath.Join(t.TempDir(), "binary")
fh, err := os.Create(binaryPath + ".fingerprint")
require.NoError(t, err)
fh.Write([]byte(tt.fingerprint))
require.NoError(t, fh.Close())
cfg := config.BinaryFromImage{
GenericName: "name",
Version: "version",
Images: []config.Image{{
Reference: "ref",
Platform: "platform",
}},
PathsInImage: []string{"path"},
}
assert.Equal(t, tt.expected, isDownloadStale(cfg, []string{binaryPath}))
})
}
}
@@ -0,0 +1,197 @@
package internal
import (
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config"
)
type Entries map[LogicalEntryKey]EntryInfo
type EntryInfo struct {
IsConfigured bool
BinaryPath string
SnippetPath string
}
type LogicalEntryKey struct {
OrgName string
Version string
Platform string
Filename string
}
func (k LogicalEntryKey) Path() string {
return fmt.Sprintf("%s/%s/%s/%s", k.OrgName, k.Version, k.Platform, k.Filename)
}
type LogicalEntryKeys []LogicalEntryKey
func (l LogicalEntryKeys) Len() int {
return len(l)
}
func (l LogicalEntryKeys) Less(i, j int) bool {
if l[i].OrgName == l[j].OrgName {
if l[i].Version == l[j].Version {
if l[i].Platform == l[j].Platform {
return l[i].Filename < l[j].Filename
}
return l[i].Platform < l[j].Platform
}
return l[i].Version < l[j].Version
}
return l[i].OrgName < l[j].OrgName
}
func (l LogicalEntryKeys) Swap(i, j int) {
l[i], l[j] = l[j], l[i]
}
func NewLogicalEntryKeys(m map[LogicalEntryKey]EntryInfo) LogicalEntryKeys {
var keys LogicalEntryKeys
for k := range m {
keys = append(keys, k)
}
sort.Sort(keys)
return keys
}
func ListAllBinaries(appConfig config.Application) (Entries, error) {
binaries, err := allFilePaths(appConfig.DownloadPath)
if err != nil {
return nil, fmt.Errorf("unable to list binaries: %w", err)
}
cases := make(map[LogicalEntryKey]EntryInfo)
for _, storePath := range binaries {
isConfigured := appConfig.GetBinaryFromImageByPath(storePath) != nil
relativePath, err := filepath.Rel(appConfig.DownloadPath, storePath)
if err != nil {
return nil, fmt.Errorf("unable to get relative path for %q: %w", storePath, err)
}
key, err := getLogicalKey(relativePath)
if err != nil {
return nil, fmt.Errorf("unable to get logical key for binary %q: %w", storePath, err)
}
cases[*key] = EntryInfo{
IsConfigured: isConfigured,
BinaryPath: storePath,
}
}
return cases, nil
}
func ListAllEntries(appConfig config.Application) (Entries, error) {
snippets, err := allFilePaths(appConfig.SnippetPath)
if err != nil {
return nil, fmt.Errorf("unable to list snippets: %w", err)
}
cases, err := ListAllBinaries(appConfig)
if err != nil {
return nil, fmt.Errorf("unable to list binaries: %w", err)
}
// anything configured that isn't in the binaries list?
for _, cfg := range appConfig.FromImages {
for _, image := range cfg.Images {
for _, path := range cfg.AllStorePathsForImage(image, appConfig.DownloadPath) {
key := newLogicalEntryForImage(cfg, image, path)
if _, ok := cases[key]; ok {
continue
}
cases[key] = EntryInfo{
IsConfigured: true,
}
}
}
}
// correlate snippets to existing binaries and configurations (and add unmanaged ones)
for _, storePath := range snippets {
relativePath, err := filepath.Rel(appConfig.SnippetPath, storePath)
if err != nil {
return nil, fmt.Errorf("unable to get relative path for %q: %w", storePath, err)
}
key, err := getLogicalKey(relativePath)
if err != nil {
return nil, fmt.Errorf("unable to get logical key for snippet %q: %w", storePath, err)
}
if v, ok := cases[*key]; ok {
v.SnippetPath = storePath
cases[*key] = v
continue
}
cases[*key] = EntryInfo{
IsConfigured: false,
SnippetPath: storePath,
}
}
return cases, nil
}
func newLogicalEntryForImage(cfg config.BinaryFromImage, image config.Image, storePath string) LogicalEntryKey {
return LogicalEntryKey{
OrgName: cfg.Name(),
Version: cfg.Version,
Platform: config.PlatformAsValue(image.Platform),
Filename: filepath.Base(storePath),
}
}
func getLogicalKey(managedBinaryPath string) (*LogicalEntryKey, error) {
// infer the logical key from the path alone: name/version/platform/filename
items := SplitFilepath(managedBinaryPath)
if len(items) < 4 {
return nil, fmt.Errorf("invalid managed binary path: %q", managedBinaryPath)
}
return &LogicalEntryKey{
OrgName: items[0],
Version: items[1],
Platform: items[2],
Filename: filepath.Join(items[3:]...),
}, nil
}
func allFilePaths(root string) ([]string, error) {
var paths []string
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if info != nil && !info.IsDir() && !strings.HasSuffix(path, ".fingerprint") {
paths = append(paths, path)
}
return nil
})
if err != nil {
return nil, err
}
return paths, nil
}
func (e Entries) BinaryFromImageHasSnippet(cfg config.BinaryFromImage) bool {
// all paths for all images must have snippets to return true
for _, image := range cfg.Images {
for _, storePath := range cfg.AllStorePathsForImage(image, "") {
key := newLogicalEntryForImage(cfg, image, storePath)
if v, ok := e[key]; ok {
if v.SnippetPath == "" {
return false
}
}
}
}
return true
}
@@ -0,0 +1,116 @@
package internal
import (
"path/filepath"
"sort"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config"
)
func TestSortingLogicalEntryKeys(t *testing.T) {
keys := LogicalEntryKeys{
{OrgName: "Org1", Version: "1.0", Platform: "Linux", Filename: "fileB"},
{OrgName: "Org1", Version: "1.0", Platform: "Linux", Filename: "fileA"},
{OrgName: "Org2", Version: "1.0", Platform: "Linux", Filename: "fileA"},
{OrgName: "Org1", Version: "1.1", Platform: "Linux", Filename: "fileA"},
}
expected := LogicalEntryKeys{
{OrgName: "Org1", Version: "1.0", Platform: "Linux", Filename: "fileA"},
{OrgName: "Org1", Version: "1.0", Platform: "Linux", Filename: "fileB"},
{OrgName: "Org1", Version: "1.1", Platform: "Linux", Filename: "fileA"},
{OrgName: "Org2", Version: "1.0", Platform: "Linux", Filename: "fileA"},
}
sort.Sort(keys)
assert.Equal(t, expected, keys)
}
func TestListAllBinaries(t *testing.T) {
appConfig := config.Application{
DownloadPath: filepath.Join("testdata", "bin"),
SnippetPath: filepath.Join("testdata", "snippets"),
FromImages: []config.BinaryFromImage{
{
Version: "1.3.6",
Images: []config.Image{
{
Reference: "ref-1",
Platform: "linux/amd64",
},
// this makes it not configured
//{
// Reference: "ref-2",
// Platform: "linux/arm64",
//},
},
PathsInImage: []string{
"/usr/local/bin/busybox",
},
},
},
}
entries, err := ListAllBinaries(appConfig)
require.NoError(t, err)
require.Len(t, entries, 2)
assert.Equal(t,
Entries{
LogicalEntryKey{OrgName: "busybox", Version: "1.3.6", Platform: "linux-amd64", Filename: "busybox"}: EntryInfo{IsConfigured: true, BinaryPath: "testdata/bin/busybox/1.3.6/linux-amd64/busybox", SnippetPath: ""},
LogicalEntryKey{OrgName: "busybox", Version: "1.3.6", Platform: "linux-arm64", Filename: "busybox"}: EntryInfo{IsConfigured: false, BinaryPath: "testdata/bin/busybox/1.3.6/linux-arm64/busybox", SnippetPath: ""},
},
entries,
)
}
func TestListAllEntries(t *testing.T) {
appConfig := config.Application{
DownloadPath: filepath.Join("testdata", "bin"),
SnippetPath: filepath.Join("testdata", "snippets"),
FromImages: []config.BinaryFromImage{
{
Version: "1.3.6",
Images: []config.Image{
{
Reference: "ref-1",
Platform: "linux/amd64",
},
// this makes it not configured
//{
// Reference: "ref-2",
// Platform: "linux/arm64",
//},
},
PathsInImage: []string{
"/usr/local/bin/busybox",
},
},
},
}
entries, err := ListAllEntries(appConfig)
require.NoError(t, err)
require.Len(t, entries, 3)
assert.Equal(t,
Entries{
LogicalEntryKey{OrgName: "busybox", Version: "1.3.6", Platform: "linux-amd64", Filename: "busybox"}: EntryInfo{IsConfigured: true, BinaryPath: "testdata/bin/busybox/1.3.6/linux-amd64/busybox", SnippetPath: "testdata/snippets/busybox/1.3.6/linux-amd64/busybox"},
LogicalEntryKey{OrgName: "busybox", Version: "1.3.6", Platform: "linux-arm64", Filename: "busybox"}: EntryInfo{IsConfigured: false, BinaryPath: "testdata/bin/busybox/1.3.6/linux-arm64/busybox", SnippetPath: "testdata/snippets/busybox/1.3.6/linux-arm64/busybox"},
// note the standalone snippet!
LogicalEntryKey{OrgName: "postgres", Version: "9.6.10", Platform: "linux-amd64", Filename: "postgres"}: EntryInfo{IsConfigured: false, BinaryPath: "", SnippetPath: "testdata/snippets/postgres/9.6.10/linux-amd64/postgres"},
},
entries,
)
}
@@ -0,0 +1,40 @@
package internal
import (
"fmt"
"os"
"strings"
"gopkg.in/yaml.v3"
)
type SnippetMetadata struct {
Name string `yaml:"name"`
Offset int `yaml:"offset"`
Length int `yaml:"length"`
SnippetSha256 string `yaml:"snippetSha256"`
FileSha256 string `yaml:"fileSha256"`
}
func ReadSnippetMetadata(path string) (*SnippetMetadata, error) {
if path == "" {
return nil, nil
}
contents, err := os.ReadFile(path)
if err != nil {
return nil, err
}
fields := strings.Split(string(contents), "\n### byte snippet to follow ###\n")
if len(fields) != 2 {
return nil, fmt.Errorf("this is not a snippet")
}
var metadata SnippetMetadata
if err := yaml.Unmarshal([]byte(fields[0]), &metadata); err != nil {
return nil, err
}
return &metadata, nil
}
@@ -0,0 +1,70 @@
package internal
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestReadSnippetMetadata(t *testing.T) {
tests := []struct {
name string
content string
createFile bool
expectedError bool
expectedResult *SnippetMetadata
}{
{
name: "valid metadata",
content: "name: test\noffset: 10\nlength: 20\nsnippetSha256: abcd\nfileSha256: efgh\n### byte snippet to follow ###\n",
expectedError: false,
expectedResult: &SnippetMetadata{Name: "test", Offset: 10, Length: 20, SnippetSha256: "abcd", FileSha256: "efgh"},
},
{
name: "invalid format",
content: "invalid content",
expectedError: true,
},
{
name: "Empty content",
content: "",
createFile: true,
expectedError: true,
},
{
name: "no path",
expectedError: false,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
var path string
if tc.content != "" || tc.createFile {
path = createTestFile(t, tc.content)
}
result, err := ReadSnippetMetadata(path)
if tc.expectedError {
assert.Error(t, err)
} else {
require.NoError(t, err)
assert.Equal(t, tc.expectedResult, result)
}
})
}
}
func createTestFile(t *testing.T, content string) string {
t.Helper()
file, err := os.CreateTemp(t.TempDir(), "syft-test-snippetMetadata")
require.NoError(t, err)
if len(content) > 0 {
_, err = file.WriteString(content)
require.NoError(t, err)
}
require.NoError(t, file.Close())
return file.Name()
}
@@ -0,0 +1,8 @@
name: busybox
offset: 890794
length: 6
snippetSha256: 1cb0df35e77d61622940971e9dc8387719b37dbfe905034f7c89c730d45477c2
fileSha256: 5c44a88a3eaf375f58715829fe187b6eaeb2efcd41bdb77595ae732fddfd4959
### byte snippet to follow ###
fake 1
@@ -0,0 +1,8 @@
name: busybox
offset: 890794
length: 6
snippetSha256: 784ed38fdb1ad0498a159b15b710563ee43d1d88e361171b764e4f191888cbc0
fileSha256: eaf375f5db77595ae732fddfd45c44a88a39598715829fe187b6eaeb2efcd41b
### byte snippet to follow ###
fake 2
@@ -0,0 +1,8 @@
name: postgres
offset: 78994
length: 6
snippetSha256: 08b237bd6e2701f6c169a2ce8b293ba431c2016f5b9b28597241dffeff100941
fileSha256: efcd41bd715829fe187b6eaeb2ddfd4959b77595ae732f5c44a88a3eaf375f58
### byte snippet to follow ###
fake 3
@@ -0,0 +1,50 @@
package ui
import (
"errors"
"fmt"
"os/exec"
"strings"
)
type Action struct {
Msg string
}
func (a Action) Start() {
fmt.Printf(" • %s%s%s\n", purple+italic, a.Msg, reset)
}
func (a Action) Skip(newMsg ...string) {
if len(newMsg) > 0 {
// clear the line
goToPreviousLineStart()
// add a little extra to account for ansi escape codes (hack)
fmt.Printf("%s\n", strings.Repeat(" ", len(a.Msg)+10))
a.Msg = newMsg[0]
}
goToPreviousLineStart()
formatSkip(a.Msg)
}
func (a Action) Done(err error) {
goToPreviousLineStart()
if err != nil {
fmt.Printf(" %s✗%s %s%s%s\n", red+bold, reset, red, a.Msg, reset)
var exitError *exec.ExitError
if errors.As(err, &exitError) && len(exitError.Stderr) > 0 {
fmt.Printf(" %s├──%s %s%s%s\n", grey, reset, red, err, reset)
fmt.Printf(" %s└──%s %s%s%s\n", grey, reset, red, "stderr:", reset)
fmt.Println(string(exitError.Stderr))
} else {
fmt.Printf(" %s└──%s %s%s%s\n", grey, reset, red, err, reset)
}
return
}
fmt.Printf(" %s✔%s %s\n", green+bold, reset, a.Msg)
}
func formatSkip(msg string) {
fmt.Printf(" %s⏭%s %s%s%s\n", bold, reset, grey, msg, reset)
}
@@ -0,0 +1,17 @@
package ui
import "fmt"
const (
grey = "\033[90m"
reset = "\033[0m"
bold = "\033[1m"
red = "\033[31m"
italic = "\033[3m"
purple = "\033[95m" // hi variant
green = "\033[32m"
)
func goToPreviousLineStart() {
fmt.Printf("\033[F")
}
@@ -0,0 +1,108 @@
package ui
import (
"fmt"
"os"
"github.com/charmbracelet/bubbles/list"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
)
var quitTextStyle = lipgloss.NewStyle().Margin(1, 0, 2, 4)
type item string
func (i item) Title() string { return string(i) }
func (i item) Description() string { return "" }
func (i item) FilterValue() string { return string(i) }
type model struct {
list list.Model
choice string
quitting bool
}
func PromptSelectBinary(binaryPaths []string) (string, error) {
var items []list.Item
for _, p := range binaryPaths {
items = append(items, item(p))
}
d := list.NewDefaultDelegate()
d.ShowDescription = false
d.Styles.NormalTitle = d.Styles.NormalTitle.PaddingLeft(4)
d.Styles.SelectedTitle = d.Styles.SelectedTitle.PaddingLeft(3)
d.SetSpacing(0)
l := list.New(items, d, 80, 80)
l.Title = "Select a binary to capture a snippet from:"
l.SetShowStatusBar(false)
l.SetFilteringEnabled(true)
l.Styles.Title = lipgloss.NewStyle().Bold(true).MarginLeft(1)
l.Styles.PaginationStyle = list.DefaultStyles().PaginationStyle.PaddingLeft(4)
l.Styles.HelpStyle = list.DefaultStyles().HelpStyle.PaddingLeft(4).PaddingBottom(1)
m := model{list: l}
p := tea.NewProgram(m, tea.WithAltScreen())
fm, err := p.Run()
if err != nil {
fmt.Println("Error running program:", err)
os.Exit(1)
}
m = fm.(model)
if m.quitting {
return "", fmt.Errorf("cancelled")
}
if m.choice == "" {
return "", fmt.Errorf("no binary selected")
}
return m.choice, nil
}
func (m model) Init() tea.Cmd {
return nil
}
func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
switch msg := msg.(type) {
case tea.WindowSizeMsg:
m.list.SetWidth(msg.Width)
m.list.SetHeight(msg.Height)
return m, nil
case tea.KeyMsg:
switch keypress := msg.String(); keypress {
case "ctrl+c":
m.quitting = true
return m, tea.Quit
case "enter":
i, ok := m.list.SelectedItem().(item)
if ok {
m.choice = string(i)
}
return m, tea.Quit
}
}
var cmd tea.Cmd
m.list, cmd = m.list.Update(msg)
return m, cmd
}
func (m model) View() string {
if m.choice != "" {
return quitTextStyle.Render(fmt.Sprintf("Selected %q", m.choice))
}
if m.quitting {
return quitTextStyle.Render("Cancelled")
}
return "\n" + m.list.View()
}
@@ -0,0 +1,7 @@
package ui
import "fmt"
func RenderError(err error) string {
return fmt.Sprintf("%s%v%s", red, err, reset)
}
@@ -0,0 +1,33 @@
package ui
import (
"fmt"
"strings"
)
type Title struct {
Name, Version string
}
func (t Title) Start() {
t.start()
fmt.Println()
}
func (t Title) start() {
fmt.Printf("%s%s@%s%s", bold, t.Name, t.Version, reset)
}
func (t Title) Update(msg string) {
goToPreviousLineStart()
t.start()
fmt.Print(strings.Repeat(" ", 35-(len(t.Name)+len(t.Version))))
fmt.Printf(" %s⚠%s %s%s%s\n", bold, reset, italic+grey, msg, reset)
}
func (t Title) Skip(msg string) {
goToPreviousLineStart()
t.start()
fmt.Print(strings.Repeat(" ", 35-(len(t.Name)+len(t.Version))))
formatSkip(msg)
}
@@ -0,0 +1,33 @@
package internal
import (
"crypto/sha256"
"fmt"
"io"
"path/filepath"
"strings"
)
func SplitFilepath(path string) []string {
return strings.Split(path, string(filepath.Separator))
}
func Sha256SumFile(f io.ReadSeeker) (string, error) {
_, err := f.Seek(0, io.SeekStart)
if err != nil {
return "", fmt.Errorf("unable to seek to start of file: %w", err)
}
hasher := sha256.New()
_, err = io.Copy(hasher, f)
if err != nil {
return "", fmt.Errorf("unable to hash file: %w", err)
}
return fmt.Sprintf("%x", hasher.Sum(nil)), nil
}
func Sha256SumBytes(buf []byte) string {
hasher := sha256.New()
hasher.Write(buf)
return fmt.Sprintf("%x", hasher.Sum(nil))
}
@@ -0,0 +1,49 @@
package internal
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSplitFilepath(t *testing.T) {
tests := []struct {
path string
expected []string
}{
{path: "a/b/c", expected: []string{"a", "b", "c"}},
}
for _, test := range tests {
result := SplitFilepath(test.path)
assert.Equal(t, test.expected, result)
}
}
func TestSha256SumFile(t *testing.T) {
content := "test content"
reader := strings.NewReader(content)
hash, err := Sha256SumFile(reader)
require.NoError(t, err, "Sha256SumFile should not return an error")
expectedHash := Sha256SumBytes([]byte(content))
assert.Equal(t, expectedHash, hash)
}
func TestSha256SumBytes(t *testing.T) {
tests := []struct {
input []byte
expected string
}{
{input: []byte("hello"), expected: "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"},
}
for _, test := range tests {
result := Sha256SumBytes(test.input)
assert.Equal(t, test.expected, result)
}
}
@@ -0,0 +1,27 @@
package main
import (
"fmt"
"os"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/cli"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/ui"
)
func main() {
cmd, err := cli.New()
if err != nil {
exit(err)
}
if err := cmd.Execute(); err != nil {
exit(err)
}
}
func exit(err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", ui.RenderError(err))
}
os.Exit(1)
}
@@ -0,0 +1,115 @@
package testutil
import (
"fmt"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal"
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config"
)
// SnippetOrBinary returns the path to either the binary or the snippet for the given logical entry key.
// Note: this is intended to be used only within the context of the binary cataloger test fixtures. Any other
// use is unsupported. Path should be a logical path relative to the test-fixtures/classifiers directory (but does
// not specify the "bin" or "snippets" parent path... this is determined logically [snippets > binary unless told
// otherwise]). Path should also be to the directory containing the binary or snippets of interest (not the binaries
// or snippets itself).
func SnippetOrBinary(t *testing.T, path string, requireBinary bool) string {
t.Helper()
require.Len(t, internal.SplitFilepath(path), 3, "path must be a in the form <name>/<version>/<arch>")
// cd to test-fixtures directory and load the config
cwd, err := os.Getwd()
require.NoError(t, err)
require.NoError(t, os.Chdir("test-fixtures"))
defer func() {
require.NoError(t, os.Chdir(cwd))
}()
appConfig, err := config.Read()
require.NoError(t, err)
// find the first matching fixture path that matches the given requirements
entries, err := internal.ListAllEntries(*appConfig)
require.NoError(t, err)
var fixturePath string
for k, v := range entries {
if filepath.Dir(k.Path()) == path {
// prefer the snippet over the binary
if !requireBinary {
if v.SnippetPath != "" {
t.Logf("using snippet for %q", path)
require.NoError(t, validateSnippet(v.BinaryPath, v.SnippetPath))
fixturePath = v.SnippetPath
break
}
if v.BinaryPath != "" {
fixturePath = v.BinaryPath
break
}
t.Fatalf("no binary or snippet found for %q", path)
}
if v.BinaryPath != "" {
t.Logf("forcing the use of the original binary for %q", path)
fixturePath = v.BinaryPath
break
}
if v.SnippetPath != "" && !v.IsConfigured {
t.Skip("no binary found, but is covered by a snippet. Please add this case to the 'binary/test-fixtures/config.yaml' and recreate the snippet")
}
t.Fatalf("no binary found for %q", path)
}
}
if fixturePath == "" {
t.Fatalf("no fixture found for %q", path)
}
// this should be relative to the tests-fixtures directory and should be the directory containing the binary or
// snippet of interest (not the path to the binary or snippet itself)
return filepath.Join("test-fixtures", filepath.Dir(fixturePath))
}
func validateSnippet(binaryPath, snippetPath string) error {
// get a sha256 of the binary
if _, err := os.Stat(binaryPath); err != nil {
// no binary to validate against (this is ok)
return nil
}
metadata, err := internal.ReadSnippetMetadata(snippetPath)
if err != nil {
return err
}
if metadata == nil {
return nil
}
f, err := os.Open(binaryPath)
if err != nil {
return err
}
expected, err := internal.Sha256SumFile(f)
if err != nil {
return err
}
if expected != metadata.FileSha256 {
return fmt.Errorf("snippet shadows a binary with a different sha256 (want %q got %q)", expected, metadata.FileSha256)
}
return nil
}
@@ -0,0 +1,63 @@
package testutil
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
)
func TestValidateSnippet(t *testing.T) {
tests := []struct {
name string
binaryContent string
snippetContent string
expectError bool
}{
{
name: "valid",
binaryContent: "testBinary",
snippetContent: `name: bash
offset: 992758
length: 100
snippetSha256: 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824
fileSha256: 88ef99f28b69a6d8113cba62011e574a5afb1e6d8e0f884699e2ced91e4d910c
### byte snippet to follow ###
hello`,
expectError: false,
},
{
name: "invalid",
binaryContent: "testBinary",
snippetContent: `name: bash
offset: 992758
length: 100
snippetSha256: 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824
fileSha256: 011e574a8ef9584699e2ced9189f28b69a6d8113cba62e4d910cafb1e6d8e0f8
### byte snippet to follow ###
hello`,
expectError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
binaryFile, _ := os.CreateTemp(t.TempDir(), "binary")
binaryFile.WriteString(tt.binaryContent)
binaryFile.Close()
snippetFile, _ := os.CreateTemp(t.TempDir(), "snippet")
snippetFile.WriteString(tt.snippetContent)
snippetFile.Close()
err := validateSnippet(binaryFile.Name(), snippetFile.Name())
if tt.expectError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
})
}
}