chore: migrate fixtures to testdata (#4651)

* migrate fixtures to testdata

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix: correct broken symlinks after testdata migration

The migration from test-fixtures to testdata broke several symlinks:
- elf-test-fixtures symlinks pointed to old test-fixtures paths
- elf-test-fixtures needed to be renamed to elf-testdata
- image-pkg-coverage symlink pointed to test-fixtures instead of testdata

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix: handle missing classifiers/bin directory in Makefile

The clean-fingerprint target was failing when classifiers/bin doesn't
exist (e.g., on fresh clone without downloaded binaries).

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix: add gitignore negation for jar/zip fixtures in test/cli

The jar and zip files in test/cli/testdata/image-unknowns were being
gitignored by the root .gitignore patterns. This caused them to be
untracked and not included when building docker images in CI, resulting
in Test_Unknowns failures since the test expects errors from corrupt
archive files that weren't present.

Add a .gitignore in test/cli/testdata to negate the exclusions for
these specific test fixture files.

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* switch fixture cache to v2

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* test: update expected versions for rebuilt fixtures

Update test expectations for packages that have been updated in
upstream repositories when docker images are rebuilt:
- glibc: 2.42-r4 → 2.43-r1 (wolfi)
- php: 8.2.29 → 8.2.30 (ubuntu/apache)

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* upgrade go

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix: add go-shlex dependency for testdata manager tool

The manager tool in syft/pkg/cataloger/binary/testdata/ imports
go-shlex, but since it's in a testdata directory, Go doesn't track
its dependencies. This caused CI failures when go.mod didn't
explicitly list the dependency.

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor: move binary classifier manager to internal/

Move the manager tool from testdata/manager to internal/manager so
that Go properly tracks its dependencies. Code in testdata directories
is ignored by Go for dependency tracking, which caused CI failures
when go.mod didn't explicitly list transitive dependencies.

This is a cleaner solution than manually adding dependencies to go.mod
for code that happens to live in testdata.

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix: add gitignore negations for test fixtures blocked by root patterns

Multiple test fixtures were being blocked by root-level gitignore patterns
like bin/, *.jar, *.tar, and *.exe. This adds targeted .gitignore files with
negation patterns to allow these specific test fixtures to be tracked:

- syft/linux/testdata/os/busybox/bin/busybox (blocked by bin/)
- syft/pkg/cataloger/java/testdata/corrupt/example.{jar,tar} (blocked by *.jar, *.tar)
- syft/pkg/cataloger/binary/testdata/classifiers/snippets/go-version-hint/**/bin/go (blocked by bin/)
- syft/pkg/cataloger/bitnami/testdata/no-rel/.../bin/redis-server (blocked by bin/)

Also updates the bitnami test expectation to include the newly required
.gitignore files in the test fixture.

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* test: update glibc version expectation (2.43-r1 -> 2.43-r2)

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add capability drift check as unit step

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* dont clear test observations before drift detection

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* bump stereoscope commit to main

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
2026-03-06 19:42:04 +00:00
committed by GitHub
parent 35278f3d3d
commit b5e85c3ea5
1589 changed files with 1184 additions and 1058 deletions
@@ -0,0 +1,40 @@
package cli
import (
"github.com/spf13/cobra"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal/cli/commands"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal/config"
)
// list all managed binaries (in ./bin, organized by 'name-version/platform/binary')
// manager list binaries
// list all managed snippets (in ./snippets, same organization as ./bin: 'name-version/platform/binary' where each bin is a snippet)
// manager list snippets
// download all binaries (to ./bin)
// manager download [--name <name>] [--version <version>]
// capture snippet from a binary identified by offset
// manager capture snippet --binary <binary> --offset <offset> --length <length>
func New() (*cobra.Command, error) {
cfgP, err := config.Read()
if err != nil {
return nil, err
}
cfg := *cfgP
root := commands.Root(cfg)
root.AddCommand(
commands.List(cfg),
commands.Download(cfg),
commands.AddSnippet(cfg),
commands.WriteSnippet(cfg),
)
return root, nil
}
@@ -0,0 +1,109 @@
package commands
import (
"fmt"
"os"
"os/exec"
"strings"
"github.com/anmitsu/go-shlex"
"github.com/spf13/cobra"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal/config"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal/ui"
)
func AddSnippet(appConfig config.Application) *cobra.Command {
var binaryPath, searchPattern string
var length, prefixLength int
cmd := &cobra.Command{
Use: "add-snippet",
Short: "capture snippets from binaries",
Args: cobra.NoArgs,
PreRunE: func(_ *cobra.Command, _ []string) error {
candidates, err := internal.ListAllBinaries(appConfig)
if err != nil {
return fmt.Errorf("unable to list binaries: %w", err)
}
// launch the UI to pick a path
var binaryPaths []string
for _, k := range internal.NewLogicalEntryKeys(candidates) {
info := candidates[k]
if info.BinaryPath != "" {
binaryPaths = append(binaryPaths, info.BinaryPath)
}
}
binaryPath, err = ui.PromptSelectBinary(binaryPaths)
if err != nil {
return err
}
return nil
},
RunE: func(_ *cobra.Command, _ []string) error {
name, version, _, err := inferInfoFromBinaryPath(appConfig, binaryPath)
if err != nil {
return fmt.Errorf("unable to infer name and version from binary path: %w", err)
}
if searchPattern == "" {
searchPattern = strings.ReplaceAll(version, ".", `\\.`)
}
return runAddSnippet(binaryPath, name, version, searchPattern, length, prefixLength)
},
}
cmd.Flags().StringVar(&searchPattern, "search-for", "", "the pattern to search for in the binary (defaults to the version)")
cmd.Flags().IntVar(&length, "length", 100, "the length of the snippet to capture")
cmd.Flags().IntVar(&prefixLength, "prefix-length", 20, "number of bytes before the search hit to capture")
return cmd
}
func runAddSnippet(binaryPath, name, version, searchPattern string, length, prefixLength int) error {
// invoke ./capture-snippet.sh <path-to-binary> <version> [--search-for <pattern>] [--length <length>] [--prefix-length <prefix_length>]"
cmd := exec.Command("./capture-snippet.sh", binaryPath, version)
var args []string
if searchPattern != "" {
args = append(args, "--search-for", searchPattern)
}
if name != "" {
args = append(args, "--group", name)
}
if length > 0 {
args = append(args, fmt.Sprintf("--length %d", length))
}
if prefixLength > 0 {
args = append(args, fmt.Sprintf("--prefix-length %d", prefixLength))
}
var err error
args, err = shlex.Split(strings.Join(args, " "), true)
if err != nil {
return fmt.Errorf("failed to parse arguments: %w", err)
}
cmd.Args = append(cmd.Args, args...)
fmt.Printf("running: %s\n", strings.Join(cmd.Args, " "))
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Stdin = os.Stdin
if err := cmd.Start(); err != nil {
return fmt.Errorf("failed to start command: %w", err)
}
if err := cmd.Wait(); err != nil {
return fmt.Errorf("command execution failed: %w", err)
}
return nil
}
@@ -0,0 +1,79 @@
package commands
import (
"fmt"
"github.com/spf13/cobra"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal/config"
)
func Download(appConfig config.Application) *cobra.Command {
var configs []config.BinaryFromImage
var skipSnippets bool
cmd := &cobra.Command{
Use: "download",
Short: "download binaries [name@version ...]",
PreRunE: func(_ *cobra.Command, args []string) error {
if len(args) > 0 {
for _, arg := range args {
binaryFromImageCfg := appConfig.GetBinaryFromImage(arg, "")
if binaryFromImageCfg == nil {
return fmt.Errorf("no config found for %q", arg)
}
configs = append(configs, *binaryFromImageCfg)
}
} else {
configs = appConfig.FromImages
}
if skipSnippets {
var err error
configs, err = configsWithoutSnippets(appConfig, configs)
if err != nil {
return err
}
}
return nil
},
RunE: func(_ *cobra.Command, _ []string) error {
for _, binaryFromImageCfg := range configs {
if err := internal.DownloadFromImage(appConfig.DownloadPath, binaryFromImageCfg); err != nil {
return err
}
}
if len(configs) == 0 {
fmt.Println("no binaries to download")
}
return nil
},
}
cmd.Flags().BoolVarP(&skipSnippets, "skip-if-covered-by-snippet", "s", false, "skip downloading entries already covered by snippets")
return cmd
}
func configsWithoutSnippets(appConfig config.Application, configs []config.BinaryFromImage) ([]config.BinaryFromImage, error) {
entries, err := internal.ListAllEntries(appConfig)
if err != nil {
return nil, err
}
var filtered []config.BinaryFromImage
for _, cfg := range configs {
if entries.BinaryFromImageHasSnippet(cfg) {
continue
}
filtered = append(filtered, cfg)
}
return filtered, nil
}
@@ -0,0 +1,137 @@
package commands
import (
"fmt"
"strings"
"github.com/charmbracelet/lipgloss"
"github.com/jedib0t/go-pretty/v6/table"
"github.com/spf13/cobra"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal/config"
)
func List(appConfig config.Application) *cobra.Command {
var showPaths bool
cmd := &cobra.Command{
Use: "list",
Short: "list managed binaries and managed/unmanaged snippets",
Args: cobra.NoArgs,
RunE: func(_ *cobra.Command, _ []string) error {
return runList(appConfig, showPaths)
},
}
cmd.Flags().BoolVarP(&showPaths, "show-paths", "p", false, "show paths to binaries and snippets")
return cmd
}
func runList(appConfig config.Application, showPaths bool) error {
material, err := internal.ListAllEntries(appConfig)
if err != nil {
return err
}
report := renderCatalogerListTable(material, showPaths)
report += "\n Legend:\n"
report += errorStyle.Render(" !! indicates there is no binary or snippet for a configured entry") + "\n"
report += warningStyle.Render(" ! indicates that the binary is managed but there is no snippet") + "\n"
fmt.Println(report)
return nil
}
const yes = "yes"
func renderCatalogerListTable(material map[internal.LogicalEntryKey]internal.EntryInfo, showPaths bool) string {
t := table.NewWriter()
t.SetStyle(table.StyleLight)
t.AppendHeader(table.Row{"Group", "Version", "Platform", "Name", "Configured?", "Binary", "Snippet"})
keys := internal.NewLogicalEntryKeys(material)
for _, k := range keys {
info := material[k]
isConfigured := ""
if info.IsConfigured {
isConfigured = yes
}
bin := ""
snippet := ""
if showPaths {
bin = info.BinaryPath
snippet = info.SnippetPath
} else {
if info.BinaryPath != "" {
bin = yes
}
if info.SnippetPath != "" {
snippet = yes
}
}
var state displayState
if snippet == "" {
if bin == "" {
state = displayStateError
} else {
state = displayStateWarning
}
}
t.AppendRow(table.Row{
renderCell(k.OrgName, state, "!"),
renderCell(k.Version, state),
renderCell(displayPlatform(k.Platform), state),
renderCell(k.Filename, state),
renderCell(isConfigured, state),
renderCell(bin, state),
renderCell(snippet, state),
})
}
return t.Render()
}
type displayState string
const (
displayStateError displayState = "error"
displayStateWarning displayState = "warning"
)
var (
errorStyle = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("9"))
warningStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("11"))
stdStyle = lipgloss.NewStyle()
)
func renderCell(value string, state displayState, hints ...string) string {
hint := strings.Join(hints, "")
var prefix string
switch state {
case displayStateError:
return errorStyle.Render(strings.Repeat(hint, 2) + value)
case displayStateWarning:
if hint != "" {
prefix = " "
}
return warningStyle.Render(hint + prefix + value)
}
if hint != "" {
prefix = " "
}
return stdStyle.Render(prefix + value)
}
func displayPlatform(platform string) string {
return strings.ReplaceAll(platform, "-", "/")
}
@@ -0,0 +1,14 @@
package commands
import (
"github.com/spf13/cobra"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal/config"
)
func Root(_ config.Application) *cobra.Command {
return &cobra.Command{
Use: "manager",
Short: "manager is a tool for managing binaries and snippets",
}
}
@@ -0,0 +1,263 @@
package commands
import (
"debug/elf"
"debug/macho"
"debug/pe"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/spf13/cobra"
"go.yaml.in/yaml/v3"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal/config"
)
func WriteSnippet(appConfig config.Application) *cobra.Command {
var offset, length int
var name, version string
var binaryPath string
cmd := &cobra.Command{
Use: "write-snippet [binary]",
Short: "capture snippets from binaries",
Args: cobra.ExactArgs(1),
PreRunE: func(_ *cobra.Command, args []string) error {
if len(args) == 0 && (name != "" || version != "") {
return fmt.Errorf("cannot provide name or version without a binary path")
}
binaryPath = args[0]
if _, err := os.Stat(binaryPath); err != nil {
return fmt.Errorf("unable to stat %q: %w", binaryPath, err)
}
return nil
},
RunE: func(_ *cobra.Command, _ []string) error {
platform, err := getPlatform(binaryPath)
if err != nil {
return fmt.Errorf("unable to get platform: %w", err)
}
snippetPath, err := getSnippetPath(appConfig, binaryPath, name, version, platform)
if err != nil {
return fmt.Errorf("unable to get snippet path: %w", err)
}
return runWriteSnippet(binaryPath, offset, length, snippetPath)
},
}
cmd.Flags().IntVar(&offset, "offset", -1, "the offset in the binary to start the snippet")
cmd.Flags().IntVar(&length, "length", 100, "the length of the snippet to capture")
cmd.Flags().StringVar(&name, "name", "", "the name of the snippet")
cmd.Flags().StringVar(&version, "version", "", "the version of the snippet")
return cmd
}
func runWriteSnippet(binaryPath string, offset, length int, snippetPath string) error {
f, err := os.Open(binaryPath)
if err != nil {
return fmt.Errorf("unable to open binary %q: %w", binaryPath, err)
}
n, err := f.Seek(int64(offset), io.SeekStart)
if err != nil {
return fmt.Errorf("unable to seek to offset %d: %w", offset, err)
}
if n != int64(offset) {
return fmt.Errorf("unexpectd to seek value: %d != %d", offset, n)
}
buf := make([]byte, length)
n2, err := f.Read(buf)
if err != nil {
return fmt.Errorf("unable to read %d bytes: %w", length, err)
}
if n2 != length {
return fmt.Errorf("unexpected read length: %d != %d", length, n2)
}
fileDigest, err := internal.Sha256SumFile(f)
if err != nil {
return err
}
metadata := internal.SnippetMetadata{
Name: filepath.Base(binaryPath),
Offset: offset,
Length: length,
SnippetSha256: internal.Sha256SumBytes(buf),
FileSha256: fileDigest,
}
metadataBytes, err := yaml.Marshal(metadata)
if err != nil {
return fmt.Errorf("unable to marshal metadata: %w", err)
}
splitter := []byte("\n### byte snippet to follow ###\n")
var finalBuf []byte
finalBuf = append(finalBuf, metadataBytes...)
finalBuf = append(finalBuf, splitter...)
finalBuf = append(finalBuf, buf...)
if err := os.MkdirAll(filepath.Dir(snippetPath), 0755); err != nil {
return fmt.Errorf("unable to create destination directory: %w", err)
}
if err := os.WriteFile(snippetPath, finalBuf, 0600); err != nil {
return fmt.Errorf("unable to write snippet: %w", err)
}
fmt.Printf("wrote snippet to %q\n", snippetPath)
return nil
}
func getSnippetPath(appConfig config.Application, binaryPath string, name, version, platform string) (string, error) {
binFilename := filepath.Base(binaryPath)
platform = config.PlatformAsValue(platform)
// if all values provided, use them
if name != "" && version != "" && platform != "" {
return filepath.Join(appConfig.SnippetPath, name, version, platform, binFilename), nil
}
// otherwise, try to infer them from the existing binary path
name, version, platform, err := inferInfoFromBinaryPath(appConfig, binaryPath)
if err != nil {
return "", err
}
return filepath.Join(appConfig.SnippetPath, name, version, platform, binFilename), nil
}
func inferInfoFromBinaryPath(appConfig config.Application, binaryPath string) (string, string, string, error) {
relativePath, err := filepath.Rel(appConfig.DownloadPath, binaryPath)
if err != nil {
return "", "", "", fmt.Errorf("unable to get relative path: %w", err)
}
// otherwise, try to infer them from the existing binary path
items := internal.SplitFilepath(relativePath)
if len(items) != 4 {
return "", "", "", fmt.Errorf("too few fields: %q", binaryPath)
}
name := items[0]
version := items[1]
platform := items[2]
return name, version, platform, nil
}
// getPlatform will return <os>-<arch> for the given binary path, where os can be "linux", "darwin", "windows",
// and arch can be "amd64", "arm64", "arm", etc.
func getPlatform(binaryPath string) (string, error) {
f, err := os.Open(binaryPath)
if err != nil {
return "", fmt.Errorf("unable to open binary %q: %w", binaryPath, err)
}
elfPlatform := getPlatformElf(f)
if elfPlatform != "" {
return elfPlatform, nil
}
macPlatform := getPlatformMac(f)
if macPlatform != "" {
return macPlatform, nil
}
winPlatform := getPlatformWindows(f)
if winPlatform != "" {
return winPlatform, nil
}
// attempt to infer from the path. It is possible to see invalid-looking binaries that are still something
// we'd like to detect.
items := internal.SplitFilepath(binaryPath)
if len(items) > 2 {
candidate := items[len(items)-2]
if strings.Contains(candidate, "linux") || strings.Contains(candidate, "darwin") || strings.Contains(candidate, "windows") {
return candidate, nil
}
}
return "", fmt.Errorf("unable to determine platform for %q", binaryPath)
}
const (
amd64 = "amd64"
arm64 = "arm64"
)
func getPlatformElf(f *os.File) string {
elfFile, err := elf.NewFile(f)
if err != nil {
return ""
}
var arch string
switch elfFile.Machine {
case elf.EM_X86_64:
arch = amd64
case elf.EM_AARCH64:
arch = arm64
// TODO...
default:
arch = fmt.Sprintf("unknown-%x", elfFile.Machine)
}
return fmt.Sprintf("linux-%s", arch)
}
func getPlatformMac(f *os.File) string {
machoFile, err := macho.NewFile(f)
if err != nil {
return ""
}
var arch string
switch machoFile.Cpu {
case macho.CpuAmd64:
arch = amd64
case macho.CpuArm64:
arch = arm64
// TODO...
default:
arch = fmt.Sprintf("unknown-%x", machoFile.Cpu)
}
return fmt.Sprintf("darwin-%s", arch)
}
func getPlatformWindows(f *os.File) string {
peFile, err := pe.NewFile(f)
if err != nil {
return ""
}
var arch string
switch peFile.Machine {
case pe.IMAGE_FILE_MACHINE_AMD64:
arch = amd64
case pe.IMAGE_FILE_MACHINE_ARM64:
arch = arm64
// TODO...
default:
arch = fmt.Sprintf("unknown-%x", peFile.Machine)
}
return fmt.Sprintf("windows-%s", arch)
}
@@ -0,0 +1,139 @@
package config
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/hashicorp/go-multierror"
"github.com/scylladb/go-set/strset"
"go.yaml.in/yaml/v3"
)
const Path = "config.yaml"
type Application struct {
DownloadPath string `yaml:"download-path"`
SnippetPath string `yaml:"snippet-path"`
FromImages []BinaryFromImage `yaml:"from-images"`
}
func DefaultApplication() Application {
return Application{
DownloadPath: "bin",
}
}
func Read() (*Application, error) {
return read(Path)
}
func read(path string) (*Application, error) {
appConfig := DefaultApplication()
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
err = yaml.Unmarshal(data, &appConfig)
if err != nil {
return nil, err
}
if err := appConfig.Validate(); err != nil {
return nil, err
}
return &appConfig, nil
}
func (c Image) Key() string {
return fmt.Sprintf("%s:%s", c.Reference, c.Platform)
}
func (c Application) Validate() error {
set := strset.New()
var err error
for i, entry := range c.FromImages {
key := entry.Key()
if set.Has(key) {
err = multierror.Append(err, fmt.Errorf("duplicate entry %q", entry))
continue
}
set.Add(entry.Key())
if len(entry.PathsInImage) > 1 && entry.GenericName == "" {
err = multierror.Append(err, fmt.Errorf("specified multiple paths but no name for entry %d (%s)", i+1, key))
}
if entry.Name() == "" {
err = multierror.Append(err, fmt.Errorf("missing name for entry %d", i+1))
}
if entry.Version == "" {
err = multierror.Append(err, fmt.Errorf("missing version for entry %d", i+1))
}
if len(entry.Images) == 0 {
err = multierror.Append(err, fmt.Errorf("missing images for entry %d (%s)", i+1, key))
}
var imageSet = strset.New()
for j, image := range entry.Images {
imgKey := image.Key()
if imageSet.Has(imgKey) {
err = multierror.Append(err, fmt.Errorf("duplicate image %q for entry %d (%s)", image.Key(), i+1, key))
continue
}
imageSet.Add(imgKey)
if image.Reference == "" {
err = multierror.Append(err, fmt.Errorf("missing ref reference for entry %d (%s) image %d", i+1, key, j+1))
}
if image.Platform == "" {
err = multierror.Append(err, fmt.Errorf("missing platform for entry %d (%s) image %d", i+1, key, j+1))
}
}
if len(entry.PathsInImage) == 0 {
err = multierror.Append(err, fmt.Errorf("missing paths for entry %d (%s)", i+1, key))
}
}
return err
}
func (c Application) GetBinaryFromImage(name, version string) *BinaryFromImage {
if strings.Contains(name, "@") && version == "" {
parts := strings.Split(name, "@")
name = parts[0]
version = parts[1]
}
for _, entry := range c.FromImages {
if entry.Name() == name && entry.Version == version {
return &entry
}
}
return nil
}
func (c Application) GetBinaryFromImageByPath(storePath string) *BinaryFromImage {
// each key is the store path except for the root (e.g. bin or snippet)
entryByStorePath := make(map[string]BinaryFromImage)
for _, entry := range c.FromImages {
for _, path := range entry.AllStorePaths(c.DownloadPath) {
pathWithoutRoot := splitFilepath(path)[1:]
entryByStorePath[filepath.Join(pathWithoutRoot...)] = entry
}
}
pathWithoutRoot := filepath.Join(splitFilepath(storePath)[1:]...)
if entry, ok := entryByStorePath[pathWithoutRoot]; ok {
return &entry
}
return nil
}
func splitFilepath(path string) []string {
return strings.Split(path, string(filepath.Separator))
}
@@ -0,0 +1,67 @@
package config
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestReadAndValidate(t *testing.T) {
tests := []struct {
name string
expectErr bool
}{
{
name: "valid-1.yaml",
},
{
name: "valid-2.yaml",
},
{
name: "bad-implicit-name-collision.yaml",
expectErr: true,
},
{
name: "bad-missing-version.yaml",
expectErr: true,
},
{
name: "bad-missing-image.yaml",
expectErr: true,
},
{
name: "bad-no-name.yaml",
expectErr: true,
},
{
name: "bad-missing-image-platform.yaml",
expectErr: true,
},
{
name: "bad-missing-image-ref.yaml",
expectErr: true,
},
{
name: "bad-image-collision.yaml",
expectErr: true,
},
{
name: "bad-missing-paths.yaml",
expectErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
appConfig, err := read(filepath.Join("testdata", "app-configs", tt.name))
if tt.expectErr {
assert.Error(t, err)
} else {
assert.NoError(t, err)
assert.NotNil(t, appConfig)
assert.NoError(t, appConfig.Validate())
}
})
}
}
@@ -0,0 +1,80 @@
package config
import (
"fmt"
"path/filepath"
"strings"
"github.com/OneOfOne/xxhash"
"go.yaml.in/yaml/v3"
)
type BinaryFromImage struct {
GenericName string `yaml:"name"`
Version string `yaml:"version"`
Images []Image `yaml:"images"`
PathsInImage []string `yaml:"paths"`
}
type Image struct {
Reference string `yaml:"ref"`
Platform string `yaml:"platform"`
}
func (c BinaryFromImage) Key() string {
return fmt.Sprintf("%s:%s", c.Name(), c.Version)
}
func (c BinaryFromImage) Name() string {
displayName := c.GenericName
if displayName == "" {
var path string
if len(c.PathsInImage) > 0 {
path = c.PathsInImage[0]
}
if path == "" {
return ""
}
return filepath.Base(path)
}
return displayName
}
func (c BinaryFromImage) AllStorePaths(dest string) []string {
var paths []string
for _, image := range c.Images {
paths = append(paths, c.AllStorePathsForImage(image, dest)...)
}
return paths
}
func (c BinaryFromImage) AllStorePathsForImage(image Image, dest string) []string {
var paths []string
platform := PlatformAsValue(image.Platform)
for _, path := range c.PathsInImage {
base := filepath.Base(path)
if path == "" {
base = ""
}
paths = append(paths, filepath.Join(dest, c.Name(), c.Version, platform, base))
}
return paths
}
func PlatformAsValue(platform string) string {
return strings.ReplaceAll(platform, "/", "-")
}
func (c BinaryFromImage) Digest() string {
by, err := yaml.Marshal(c)
if err != nil {
panic(err)
}
hasher := xxhash.New64()
_, _ = hasher.Write(by)
return fmt.Sprintf("%x", hasher.Sum(nil))
}
@@ -0,0 +1,191 @@
package config
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestKey(t *testing.T) {
tests := []struct {
name string
binary BinaryFromImage
expected string
}{
{
name: "gocase",
binary: BinaryFromImage{
GenericName: "test",
Version: "1.0",
},
expected: "test:1.0",
},
{
name: "binary name",
binary: BinaryFromImage{
Version: "1.0",
PathsInImage: []string{
"path/to/test",
},
},
expected: "test:1.0",
},
{
name: "first binary name",
binary: BinaryFromImage{
Version: "1.0",
PathsInImage: []string{
"path/to/test",
"path/to/nothing",
},
},
expected: "test:1.0",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, tt.binary.Key())
})
}
}
func TestName(t *testing.T) {
tests := []struct {
name string
binary BinaryFromImage
expected string
}{
{
name: "given name",
binary: BinaryFromImage{
GenericName: "given",
PathsInImage: []string{
"path/to/test",
},
},
expected: "given",
},
{
name: "binary name",
binary: BinaryFromImage{
PathsInImage: []string{
"path/to/test",
},
},
expected: "test",
},
{
name: "first binary name",
binary: BinaryFromImage{
PathsInImage: []string{
"path/to/test",
"path/to/nothing",
},
},
expected: "test",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, tt.binary.Name())
})
}
}
func TestAllStorePaths(t *testing.T) {
tests := []struct {
name string
binary BinaryFromImage
dest string
expected []string
}{
{
name: "gocase",
binary: BinaryFromImage{
GenericName: "test",
Version: "1.0",
Images: []Image{
{
Reference: "ref1",
Platform: "platform1",
},
{
Reference: "ref2",
Platform: "platform2",
},
},
PathsInImage: []string{
"path/to/test1",
"path/to/test2",
},
},
dest: "dest",
expected: []string{
"dest/test/1.0/platform1/test1",
"dest/test/1.0/platform1/test2",
"dest/test/1.0/platform2/test1",
"dest/test/1.0/platform2/test2",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, tt.binary.AllStorePaths(tt.dest))
})
}
}
func TestPlatformAsValue(t *testing.T) {
tests := []struct {
name string
platform string
expected string
}{
{
name: "gocase",
platform: "platform/test",
expected: "platform-test",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, PlatformAsValue(tt.platform))
})
}
}
func TestDigest(t *testing.T) {
tests := []struct {
name string
binary BinaryFromImage
expected string
}{
{
name: "gocase",
binary: BinaryFromImage{
GenericName: "test",
Version: "1.0",
Images: []Image{
{
Reference: "ref",
Platform: "platform",
},
},
PathsInImage: []string{
"path/to/test",
},
},
expected: "fc25c48e3d2f01e3",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, tt.binary.Digest())
})
}
}
@@ -0,0 +1,14 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- name: busybox
version: 1.36.1
images:
- ref: busybox:1.36.1@sha256:058f0df5310fbbbfea7e81a3a3e2b4bf3452438ec841138d170e170adbbd27a4
platform: linux/amd64
- ref: busybox:1.36.1@sha256:058f0df5310fbbbfea7e81a3a3e2b4bf3452438ec841138d170e170adbbd27a4
platform: linux/amd64
paths:
- /bin/[
@@ -0,0 +1,19 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- name: bash # !important
version: 1.36.1
images:
- ref: busybox:1.36.1@sha256:058f0df5310fbbbfea7e81a3a3e2b4bf3452438ec841138d170e170adbbd27a4
platform: linux/amd64
paths:
- /bin/[
- version: 1.36.1 # !important
images:
- ref: bash:5.1.16@sha256:c7a903a541d8f5fe693cbe7f5ece18a1b6a03734c76092d2b153d7e98a964927
platform: linux/amd64
paths:
- /usr/local/bin/bash
@@ -0,0 +1,12 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- name: busybox
version: 1.36.1
images:
- ref: busybox:1.36.1@sha256:058f0df5310fbbbfea7e81a3a3e2b4bf3452438ec841138d170e170adbbd27a4
paths:
- /bin/[
@@ -0,0 +1,11 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- name: busybox
version: 1.36.1
images:
- platform: linux/amd64
paths:
- /bin/[
@@ -0,0 +1,9 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- name: busybox
version: 1.36.1
paths:
- /bin/[
@@ -0,0 +1,10 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- name: busybox
version: 1.36.1
images:
- ref: busybox:1.36.1@sha256:058f0df5310fbbbfea7e81a3a3e2b4bf3452438ec841138d170e170adbbd27a4
platform: linux/amd64
@@ -0,0 +1,10 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- images:
- ref: bash:5.1.16@sha256:c7a903a541d8f5fe693cbe7f5ece18a1b6a03734c76092d2b153d7e98a964927
platform: linux/amd64
paths:
- /usr/local/bin/bash
@@ -0,0 +1,12 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- version: 1.36.1
images:
- ref: busybox:1.36.1@sha256:058f0df5310fbbbfea7e81a3a3e2b4bf3452438ec841138d170e170adbbd27a4
platform: linux/amd64
paths:
- /bin/[
- /bin/busybox
@@ -0,0 +1,19 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- name: busybox
version: 1.36.1
images:
- ref: busybox:1.36.1@sha256:058f0df5310fbbbfea7e81a3a3e2b4bf3452438ec841138d170e170adbbd27a4
platform: linux/amd64
paths:
- /bin/[
- version: 5.1.16
images:
- ref: bash:5.1.16@sha256:c7a903a541d8f5fe693cbe7f5ece18a1b6a03734c76092d2b153d7e98a964927
platform: linux/amd64
paths:
- /usr/local/bin/bash
@@ -0,0 +1,19 @@
download-path: classifiers/bin
snippet-path: classifiers/snippets
from-images:
- name: bash # !important
version: 1.36.1
images:
- ref: busybox:1.36.1@sha256:058f0df5310fbbbfea7e81a3a3e2b4bf3452438ec841138d170e170adbbd27a4
platform: linux/amd64
paths:
- /bin/[
- version: 5.1.16
images:
- ref: bash:5.1.16@sha256:c7a903a541d8f5fe693cbe7f5ece18a1b6a03734c76092d2b153d7e98a964927
platform: linux/amd64
paths:
- /usr/local/bin/bash
@@ -0,0 +1,224 @@
package internal
import (
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/google/uuid"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal/config"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal/ui"
)
const digestFileSuffix = ".xxh64"
func DownloadFromImage(dest string, config config.BinaryFromImage) error {
t := ui.Title{Name: config.Name(), Version: config.Version}
t.Start()
hostPaths := config.AllStorePaths(dest)
if allPathsExist(hostPaths) {
if !isDownloadStale(config, hostPaths) {
t.Skip("already exists")
return nil
}
t.Update("stale, updating...")
}
if err := pullDockerImages(config.Images); err != nil {
return err
}
if err := copyBinariesFromDockerImages(config, dest); err != nil {
return fmt.Errorf("failed to copy binary for %s@%s: %v", config.Name(), config.Version, err)
}
return nil
}
func isDownloadStale(config config.BinaryFromImage, binaryPaths []string) bool {
currentDigest := config.Digest()
for _, path := range binaryPaths {
digestPath := path + digestFileSuffix
if _, err := os.Stat(digestPath); err != nil {
// missing a fingerprint file means the download is stale
return true
}
writtenDigest, err := os.ReadFile(digestPath)
if err != nil {
// missing a fingerprint file means the download is stale
return true
}
if string(writtenDigest) != currentDigest {
// the fingerprint file does not match the current fingerprint, so the download is stale
return true
}
}
return false
}
func allPathsExist(paths []string) bool {
for _, path := range paths {
if _, err := os.Stat(path); err != nil {
return false
}
}
return true
}
func pullDockerImages(images []config.Image) error {
for _, image := range images {
if err := pullDockerImage(image.Reference, image.Platform); err != nil {
return fmt.Errorf("failed to pull image %s for platform %s: %v", image.Reference, image.Platform, err)
}
}
return nil
}
type imageInspect struct {
OS string `json:"Os"`
Architecture string `json:"Architecture"`
}
func (i imageInspect) Platform() string {
return fmt.Sprintf("%s/%s", i.OS, i.Architecture)
}
func pullDockerImage(imageReference, platform string) error {
a := ui.Action{Msg: fmt.Sprintf("pull image %s (%s)", imageReference, platform)}
a.Start()
matches, _, _ := checkArchitecturesMatch(imageReference, platform)
if matches {
a.Skip(fmt.Sprintf("docker image already exists %q", imageReference))
return nil
}
cmd := exec.Command("docker", "pull", "--platform", platform, imageReference)
err := cmd.Run()
if err != nil {
// attach stderr to output message
var exitErr *exec.ExitError
if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
err = fmt.Errorf("pull failed: %w:\n%s", err, exitErr.Stderr)
}
a.Done(err)
return err
}
matches, gotPlatform, err := checkArchitecturesMatch(imageReference, platform)
if !matches && err == nil {
err = fmt.Errorf("image %q pulled but does not match expected platform %q != %q", imageReference, platform, gotPlatform)
}
a.Done(err)
return err
}
func checkArchitecturesMatch(imageReference, platform string) (bool, string, error) {
cmd := exec.Command("docker", "image", "inspect", imageReference)
out, err := cmd.CombinedOutput()
if err != nil {
return false, "", err
}
var inspect []imageInspect
if err := json.Unmarshal(out, &inspect); err != nil {
return false, "", fmt.Errorf("unable to unmarshal image inspect: %w", err)
}
if len(inspect) != 1 {
return false, "", fmt.Errorf("expected 1 image inspect, got %d", len(inspect))
}
gotPlatform := inspect[0].Platform()
return gotPlatform == platform, gotPlatform, nil
}
func copyBinariesFromDockerImages(config config.BinaryFromImage, destination string) (err error) {
for _, image := range config.Images {
if err := copyBinariesFromDockerImage(config, destination, image); err != nil {
return err
}
}
return nil
}
func copyBinariesFromDockerImage(config config.BinaryFromImage, destination string, image config.Image) (err error) {
containerName := fmt.Sprintf("%s-%s-%s", config.Name(), strings.ReplaceAll(config.Version, "+", "-"), uuid.New().String())
cmd := exec.Command("docker", "create", "--name", containerName, image.Reference)
if err = cmd.Run(); err != nil {
// attach stderr to output message
var exitErr *exec.ExitError
if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
err = fmt.Errorf("%w:\n%s", err, exitErr.Stderr)
}
return err
}
defer func() {
cmd := exec.Command("docker", "rm", containerName)
cmd.Run() //nolint:errcheck
}()
for i, destinationPath := range config.AllStorePathsForImage(image, destination) {
path := config.PathsInImage[i]
if err := copyBinaryFromContainer(containerName, path, destinationPath, config.Digest()); err != nil {
return err
}
}
return nil
}
func copyBinaryFromContainer(containerName, containerPath, destinationPath, digest string) (err error) {
a := ui.Action{Msg: fmt.Sprintf("extract %s", containerPath)}
a.Start()
defer func() {
a.Done(err)
}()
if err := os.MkdirAll(filepath.Dir(destinationPath), 0755); err != nil {
return err
}
cmd := exec.Command("docker", "cp", fmt.Sprintf("%s:%s", containerName, containerPath), destinationPath) //nolint:gosec
// reason for gosec exception: this is for processing test fixtures only, not used in production
if err := cmd.Run(); err != nil {
// attach stderr to output message
var exitErr *exec.ExitError
if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
err = fmt.Errorf("%w:\n%s", err, exitErr.Stderr)
}
return err
}
// ensure permissions are 600 for destination
if err := os.Chmod(destinationPath, 0600); err != nil {
return fmt.Errorf("unable to set permissions on file %q: %w", destinationPath, err)
}
// capture digest file
digestPath := destinationPath + digestFileSuffix
if err := os.WriteFile(digestPath, []byte(digest), 0600); err != nil {
return fmt.Errorf("unable to write digest file: %w", err)
}
return nil
}
@@ -0,0 +1,62 @@
package internal
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal/config"
)
func TestIsDownloadStale(t *testing.T) {
cases := []struct {
name string
digest string
expected bool
}{
{
name: "no digest",
digest: "",
expected: true,
},
{
name: "digest matches",
// this is the digest for config in the loop body
digest: "c9c8007f9c55c2f1",
expected: false,
},
{
name: "digest does not match",
digest: "bogus",
expected: true,
},
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
binaryPath := filepath.Join(t.TempDir(), "binary")
fh, err := os.Create(binaryPath + digestFileSuffix)
require.NoError(t, err)
fh.Write([]byte(tt.digest))
require.NoError(t, fh.Close())
cfg := config.BinaryFromImage{
GenericName: "name",
Version: "version",
Images: []config.Image{{
Reference: "ref",
Platform: "platform",
}},
PathsInImage: []string{"path"},
}
assert.Equal(t, tt.expected, isDownloadStale(cfg, []string{binaryPath}))
})
}
}
@@ -0,0 +1,215 @@
package internal
import (
"fmt"
"iter"
"os"
"path/filepath"
"slices"
"sort"
"strings"
"golang.org/x/exp/maps"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal/config"
)
type Entries map[LogicalEntryKey]EntryInfo
type EntryInfo struct {
IsConfigured bool
BinaryPath string
SnippetPath string
}
type LogicalEntryKey struct {
OrgName string
Version string
Platform string
Filename string
}
func (k LogicalEntryKey) Path() string {
return fmt.Sprintf("%s/%s/%s/%s", k.OrgName, k.Version, k.Platform, k.Filename)
}
type LogicalEntryKeys []LogicalEntryKey
func (l LogicalEntryKeys) Len() int {
return len(l)
}
func (l LogicalEntryKeys) Less(i, j int) bool {
if l[i].OrgName == l[j].OrgName {
if l[i].Version == l[j].Version {
if l[i].Platform == l[j].Platform {
return l[i].Filename < l[j].Filename
}
return l[i].Platform < l[j].Platform
}
return l[i].Version < l[j].Version
}
return l[i].OrgName < l[j].OrgName
}
func (l LogicalEntryKeys) Swap(i, j int) {
l[i], l[j] = l[j], l[i]
}
func NewLogicalEntryKeys(m map[LogicalEntryKey]EntryInfo) LogicalEntryKeys {
var keys LogicalEntryKeys
for k := range m {
keys = append(keys, k)
}
sort.Sort(keys)
return keys
}
func ListAllBinaries(appConfig config.Application) (Entries, error) {
binaries, err := allFilePaths(appConfig.DownloadPath)
if err != nil {
return nil, fmt.Errorf("unable to list binaries: %w", err)
}
cases := make(map[LogicalEntryKey]EntryInfo)
for _, storePath := range binaries {
isConfigured := appConfig.GetBinaryFromImageByPath(storePath) != nil
relativePath, err := filepath.Rel(appConfig.DownloadPath, storePath)
if err != nil {
return nil, fmt.Errorf("unable to get relative path for %q: %w", storePath, err)
}
key, err := getLogicalKey(relativePath)
if err != nil {
return nil, fmt.Errorf("unable to get logical key for binary %q: %w", storePath, err)
}
cases[*key] = EntryInfo{
IsConfigured: isConfigured,
BinaryPath: storePath,
}
}
return cases, nil
}
func ListAllEntries(appConfig config.Application) (Entries, error) {
snippets, err := allFilePaths(appConfig.SnippetPath)
if err != nil {
return nil, fmt.Errorf("unable to list snippets: %w", err)
}
cases, err := ListAllBinaries(appConfig)
if err != nil {
return nil, fmt.Errorf("unable to list binaries: %w", err)
}
// anything configured that isn't in the binaries list?
for _, cfg := range appConfig.FromImages {
for _, image := range cfg.Images {
for _, path := range cfg.AllStorePathsForImage(image, appConfig.DownloadPath) {
key := newLogicalEntryForImage(cfg, image, path)
if _, ok := cases[key]; ok {
continue
}
cases[key] = EntryInfo{
IsConfigured: true,
}
}
}
}
// correlate snippets to existing binaries and configurations (and add unmanaged ones)
for _, storePath := range snippets {
relativePath, err := filepath.Rel(appConfig.SnippetPath, storePath)
if err != nil {
return nil, fmt.Errorf("unable to get relative path for %q: %w", storePath, err)
}
key, err := getLogicalKey(relativePath)
if err != nil {
return nil, fmt.Errorf("unable to get logical key for snippet %q: %w", storePath, err)
}
if v, ok := cases[*key]; ok {
v.SnippetPath = storePath
cases[*key] = v
continue
}
cases[*key] = EntryInfo{
IsConfigured: false,
SnippetPath: storePath,
}
}
return cases, nil
}
func newLogicalEntryForImage(cfg config.BinaryFromImage, image config.Image, storePath string) LogicalEntryKey {
return LogicalEntryKey{
OrgName: cfg.Name(),
Version: cfg.Version,
Platform: config.PlatformAsValue(image.Platform),
Filename: filepath.Base(storePath),
}
}
func getLogicalKey(managedBinaryPath string) (*LogicalEntryKey, error) {
// infer the logical key from the path alone: name/version/platform/filename
items := SplitFilepath(managedBinaryPath)
if len(items) < 4 {
return nil, fmt.Errorf("invalid managed binary path: %q", managedBinaryPath)
}
return &LogicalEntryKey{
OrgName: items[0],
Version: items[1],
Platform: items[2],
Filename: filepath.Join(items[3:]...),
}, nil
}
func allFilePaths(root string) ([]string, error) {
var paths []string
err := filepath.Walk(root, func(path string, info os.FileInfo, _ error) error {
if info != nil && !info.IsDir() && !strings.HasSuffix(path, digestFileSuffix) && !strings.HasSuffix(path, ".DS_Store") {
paths = append(paths, path)
}
return nil
})
if err != nil {
return nil, err
}
return paths, nil
}
func (e Entries) BinaryFromImageHasSnippet(cfg config.BinaryFromImage) bool {
// all paths for all images must have snippets to return true
for _, image := range cfg.Images {
for _, storePath := range cfg.AllStorePathsForImage(image, "") {
key := newLogicalEntryForImage(cfg, image, storePath)
if v, ok := e[key]; ok {
if v.SnippetPath == "" {
return false
}
}
}
}
return true
}
func (e Entries) Sorted() iter.Seq2[LogicalEntryKey, EntryInfo] {
keys := maps.Keys(e)
slices.SortFunc(keys, func(a, b LogicalEntryKey) int {
return strings.Compare(a.Path(), b.Path())
})
return func(yield func(LogicalEntryKey, EntryInfo) bool) {
for _, k := range keys {
if !yield(k, e[k]) {
return
}
}
}
}
@@ -0,0 +1,116 @@
package internal
import (
"path/filepath"
"sort"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal/config"
)
func TestSortingLogicalEntryKeys(t *testing.T) {
keys := LogicalEntryKeys{
{OrgName: "Org1", Version: "1.0", Platform: "Linux", Filename: "fileB"},
{OrgName: "Org1", Version: "1.0", Platform: "Linux", Filename: "fileA"},
{OrgName: "Org2", Version: "1.0", Platform: "Linux", Filename: "fileA"},
{OrgName: "Org1", Version: "1.1", Platform: "Linux", Filename: "fileA"},
}
expected := LogicalEntryKeys{
{OrgName: "Org1", Version: "1.0", Platform: "Linux", Filename: "fileA"},
{OrgName: "Org1", Version: "1.0", Platform: "Linux", Filename: "fileB"},
{OrgName: "Org1", Version: "1.1", Platform: "Linux", Filename: "fileA"},
{OrgName: "Org2", Version: "1.0", Platform: "Linux", Filename: "fileA"},
}
sort.Sort(keys)
assert.Equal(t, expected, keys)
}
func TestListAllBinaries(t *testing.T) {
appConfig := config.Application{
DownloadPath: filepath.Join("testdata", "bin"),
SnippetPath: filepath.Join("testdata", "snippets"),
FromImages: []config.BinaryFromImage{
{
Version: "1.3.6",
Images: []config.Image{
{
Reference: "ref-1",
Platform: "linux/amd64",
},
// this makes it not configured
//{
// Reference: "ref-2",
// Platform: "linux/arm64",
//},
},
PathsInImage: []string{
"/usr/local/bin/busybox",
},
},
},
}
entries, err := ListAllBinaries(appConfig)
require.NoError(t, err)
require.Len(t, entries, 2)
assert.Equal(t,
Entries{
LogicalEntryKey{OrgName: "busybox", Version: "1.3.6", Platform: "linux-amd64", Filename: "busybox"}: EntryInfo{IsConfigured: true, BinaryPath: "testdata/bin/busybox/1.3.6/linux-amd64/busybox", SnippetPath: ""},
LogicalEntryKey{OrgName: "busybox", Version: "1.3.6", Platform: "linux-arm64", Filename: "busybox"}: EntryInfo{IsConfigured: false, BinaryPath: "testdata/bin/busybox/1.3.6/linux-arm64/busybox", SnippetPath: ""},
},
entries,
)
}
func TestListAllEntries(t *testing.T) {
appConfig := config.Application{
DownloadPath: filepath.Join("testdata", "bin"),
SnippetPath: filepath.Join("testdata", "snippets"),
FromImages: []config.BinaryFromImage{
{
Version: "1.3.6",
Images: []config.Image{
{
Reference: "ref-1",
Platform: "linux/amd64",
},
// this makes it not configured
//{
// Reference: "ref-2",
// Platform: "linux/arm64",
//},
},
PathsInImage: []string{
"/usr/local/bin/busybox",
},
},
},
}
entries, err := ListAllEntries(appConfig)
require.NoError(t, err)
require.Len(t, entries, 3)
assert.Equal(t,
Entries{
LogicalEntryKey{OrgName: "busybox", Version: "1.3.6", Platform: "linux-amd64", Filename: "busybox"}: EntryInfo{IsConfigured: true, BinaryPath: "testdata/bin/busybox/1.3.6/linux-amd64/busybox", SnippetPath: "testdata/snippets/busybox/1.3.6/linux-amd64/busybox"},
LogicalEntryKey{OrgName: "busybox", Version: "1.3.6", Platform: "linux-arm64", Filename: "busybox"}: EntryInfo{IsConfigured: false, BinaryPath: "testdata/bin/busybox/1.3.6/linux-arm64/busybox", SnippetPath: "testdata/snippets/busybox/1.3.6/linux-arm64/busybox"},
// note the standalone snippet!
LogicalEntryKey{OrgName: "postgres", Version: "9.6.10", Platform: "linux-amd64", Filename: "postgres"}: EntryInfo{IsConfigured: false, BinaryPath: "", SnippetPath: "testdata/snippets/postgres/9.6.10/linux-amd64/postgres"},
},
entries,
)
}
@@ -0,0 +1,40 @@
package internal
import (
"fmt"
"os"
"strings"
"go.yaml.in/yaml/v3"
)
type SnippetMetadata struct {
Name string `yaml:"name"`
Offset int `yaml:"offset"`
Length int `yaml:"length"`
SnippetSha256 string `yaml:"snippetSha256"`
FileSha256 string `yaml:"fileSha256"`
}
func ReadSnippetMetadata(path string) (*SnippetMetadata, error) {
if path == "" {
return nil, nil
}
contents, err := os.ReadFile(path)
if err != nil {
return nil, err
}
fields := strings.Split(string(contents), "\n### byte snippet to follow ###\n")
if len(fields) != 2 {
return nil, fmt.Errorf("this is not a snippet")
}
var metadata SnippetMetadata
if err := yaml.Unmarshal([]byte(fields[0]), &metadata); err != nil {
return nil, err
}
return &metadata, nil
}
@@ -0,0 +1,70 @@
package internal
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestReadSnippetMetadata(t *testing.T) {
tests := []struct {
name string
content string
createFile bool
expectedError bool
expectedResult *SnippetMetadata
}{
{
name: "valid metadata",
content: "name: test\noffset: 10\nlength: 20\nsnippetSha256: abcd\nfileSha256: efgh\n### byte snippet to follow ###\n",
expectedError: false,
expectedResult: &SnippetMetadata{Name: "test", Offset: 10, Length: 20, SnippetSha256: "abcd", FileSha256: "efgh"},
},
{
name: "invalid format",
content: "invalid content",
expectedError: true,
},
{
name: "Empty content",
content: "",
createFile: true,
expectedError: true,
},
{
name: "no path",
expectedError: false,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
var path string
if tc.content != "" || tc.createFile {
path = createTestFile(t, tc.content)
}
result, err := ReadSnippetMetadata(path)
if tc.expectedError {
assert.Error(t, err)
} else {
require.NoError(t, err)
assert.Equal(t, tc.expectedResult, result)
}
})
}
}
func createTestFile(t *testing.T, content string) string {
t.Helper()
file, err := os.CreateTemp(t.TempDir(), "syft-test-snippetMetadata")
require.NoError(t, err)
if len(content) > 0 {
_, err = file.WriteString(content)
require.NoError(t, err)
}
require.NoError(t, file.Close())
return file.Name()
}
@@ -0,0 +1 @@
fake contents 1
@@ -0,0 +1 @@
fake contents 2
@@ -0,0 +1,8 @@
name: busybox
offset: 890794
length: 6
snippetSha256: 1cb0df35e77d61622940971e9dc8387719b37dbfe905034f7c89c730d45477c2
fileSha256: 5c44a88a3eaf375f58715829fe187b6eaeb2efcd41bdb77595ae732fddfd4959
### byte snippet to follow ###
fake 1
@@ -0,0 +1,8 @@
name: busybox
offset: 890794
length: 6
snippetSha256: 784ed38fdb1ad0498a159b15b710563ee43d1d88e361171b764e4f191888cbc0
fileSha256: eaf375f5db77595ae732fddfd45c44a88a39598715829fe187b6eaeb2efcd41b
### byte snippet to follow ###
fake 2
@@ -0,0 +1,8 @@
name: postgres
offset: 78994
length: 6
snippetSha256: 08b237bd6e2701f6c169a2ce8b293ba431c2016f5b9b28597241dffeff100941
fileSha256: efcd41bd715829fe187b6eaeb2ddfd4959b77595ae732f5c44a88a3eaf375f58
### byte snippet to follow ###
fake 3
@@ -0,0 +1,50 @@
package ui
import (
"errors"
"fmt"
"os/exec"
"strings"
)
type Action struct {
Msg string
}
func (a Action) Start() {
fmt.Printf(" • %s%s%s\n", purple+italic, a.Msg, reset)
}
func (a Action) Skip(newMsg ...string) {
if len(newMsg) > 0 {
// clear the line
goToPreviousLineStart()
// add a little extra to account for ansi escape codes (hack)
fmt.Printf("%s\n", strings.Repeat(" ", len(a.Msg)+10))
a.Msg = newMsg[0]
}
goToPreviousLineStart()
formatSkip(a.Msg)
}
func (a Action) Done(err error) {
goToPreviousLineStart()
if err != nil {
fmt.Printf(" %s✗%s %s%s%s\n", red+bold, reset, red, a.Msg, reset)
var exitError *exec.ExitError
if errors.As(err, &exitError) && len(exitError.Stderr) > 0 {
fmt.Printf(" %s├──%s %s%s%s\n", grey, reset, red, err, reset)
fmt.Printf(" %s└──%s %s%s%s\n", grey, reset, red, "stderr:", reset)
fmt.Println(string(exitError.Stderr))
} else {
fmt.Printf(" %s└──%s %s%s%s\n", grey, reset, red, err, reset)
}
return
}
fmt.Printf(" %s✔%s %s\n", green+bold, reset, a.Msg)
}
func formatSkip(msg string) {
fmt.Printf(" %s⏭%s %s%s%s\n", bold, reset, grey, msg, reset)
}
@@ -0,0 +1,17 @@
package ui
import "fmt"
const (
grey = "\033[90m"
reset = "\033[0m"
bold = "\033[1m"
red = "\033[31m"
italic = "\033[3m"
purple = "\033[95m" // hi variant
green = "\033[32m"
)
func goToPreviousLineStart() {
fmt.Printf("\033[F")
}
@@ -0,0 +1,108 @@
package ui
import (
"fmt"
"os"
"github.com/charmbracelet/bubbles/list"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
)
var quitTextStyle = lipgloss.NewStyle().Margin(1, 0, 2, 4)
type item string
func (i item) Title() string { return string(i) }
func (i item) Description() string { return "" }
func (i item) FilterValue() string { return string(i) }
type model struct {
list list.Model
choice string
quitting bool
}
func PromptSelectBinary(binaryPaths []string) (string, error) {
var items []list.Item
for _, p := range binaryPaths {
items = append(items, item(p))
}
d := list.NewDefaultDelegate()
d.ShowDescription = false
d.Styles.NormalTitle = d.Styles.NormalTitle.PaddingLeft(4)
d.Styles.SelectedTitle = d.Styles.SelectedTitle.PaddingLeft(3)
d.SetSpacing(0)
l := list.New(items, d, 80, 80)
l.Title = "Select a binary to capture a snippet from:"
l.SetShowStatusBar(false)
l.SetFilteringEnabled(true)
l.Styles.Title = lipgloss.NewStyle().Bold(true).MarginLeft(1)
l.Styles.PaginationStyle = list.DefaultStyles().PaginationStyle.PaddingLeft(4)
l.Styles.HelpStyle = list.DefaultStyles().HelpStyle.PaddingLeft(4).PaddingBottom(1)
m := model{list: l}
p := tea.NewProgram(m, tea.WithAltScreen())
fm, err := p.Run()
if err != nil {
fmt.Println("Error running program:", err)
os.Exit(1)
}
m = fm.(model)
if m.quitting {
return "", fmt.Errorf("cancelled")
}
if m.choice == "" {
return "", fmt.Errorf("no binary selected")
}
return m.choice, nil
}
func (m model) Init() tea.Cmd {
return nil
}
func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
switch msg := msg.(type) {
case tea.WindowSizeMsg:
m.list.SetWidth(msg.Width)
m.list.SetHeight(msg.Height)
return m, nil
case tea.KeyMsg:
switch keypress := msg.String(); keypress {
case "ctrl+c":
m.quitting = true
return m, tea.Quit
case "enter":
i, ok := m.list.SelectedItem().(item)
if ok {
m.choice = string(i)
}
return m, tea.Quit
}
}
var cmd tea.Cmd
m.list, cmd = m.list.Update(msg)
return m, cmd
}
func (m model) View() string {
if m.choice != "" {
return quitTextStyle.Render(fmt.Sprintf("Selected %q", m.choice))
}
if m.quitting {
return quitTextStyle.Render("Cancelled")
}
return "\n" + m.list.View()
}
@@ -0,0 +1,7 @@
package ui
import "fmt"
func RenderError(err error) string {
return fmt.Sprintf("%s%v%s", red, err, reset)
}
@@ -0,0 +1,33 @@
package ui
import (
"fmt"
"strings"
)
type Title struct {
Name, Version string
}
func (t Title) Start() {
t.start()
fmt.Println()
}
func (t Title) start() {
fmt.Printf("%s%s@%s%s", bold, t.Name, t.Version, reset)
}
func (t Title) Update(msg string) {
goToPreviousLineStart()
t.start()
fmt.Print(strings.Repeat(" ", 35-(len(t.Name)+len(t.Version))))
fmt.Printf(" %s⚠%s %s%s%s\n", bold, reset, italic+grey, msg, reset)
}
func (t Title) Skip(msg string) {
goToPreviousLineStart()
t.start()
fmt.Print(strings.Repeat(" ", 35-(len(t.Name)+len(t.Version))))
formatSkip(msg)
}
@@ -0,0 +1,33 @@
package internal
import (
"crypto/sha256"
"fmt"
"io"
"path/filepath"
"strings"
)
func SplitFilepath(path string) []string {
return strings.Split(path, string(filepath.Separator))
}
func Sha256SumFile(f io.ReadSeeker) (string, error) {
_, err := f.Seek(0, io.SeekStart)
if err != nil {
return "", fmt.Errorf("unable to seek to start of file: %w", err)
}
hasher := sha256.New()
_, err = io.Copy(hasher, f)
if err != nil {
return "", fmt.Errorf("unable to hash file: %w", err)
}
return fmt.Sprintf("%x", hasher.Sum(nil)), nil
}
func Sha256SumBytes(buf []byte) string {
hasher := sha256.New()
hasher.Write(buf)
return fmt.Sprintf("%x", hasher.Sum(nil))
}
@@ -0,0 +1,49 @@
package internal
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSplitFilepath(t *testing.T) {
tests := []struct {
path string
expected []string
}{
{path: "a/b/c", expected: []string{"a", "b", "c"}},
}
for _, test := range tests {
result := SplitFilepath(test.path)
assert.Equal(t, test.expected, result)
}
}
func TestSha256SumFile(t *testing.T) {
content := "test content"
reader := strings.NewReader(content)
hash, err := Sha256SumFile(reader)
require.NoError(t, err, "Sha256SumFile should not return an error")
expectedHash := Sha256SumBytes([]byte(content))
assert.Equal(t, expectedHash, hash)
}
func TestSha256SumBytes(t *testing.T) {
tests := []struct {
input []byte
expected string
}{
{input: []byte("hello"), expected: "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"},
}
for _, test := range tests {
result := Sha256SumBytes(test.input)
assert.Equal(t, test.expected, result)
}
}
@@ -0,0 +1,27 @@
package main
import (
"fmt"
"os"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal/cli"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal/ui"
)
func main() {
cmd, err := cli.New()
if err != nil {
exit(err)
}
if err := cmd.Execute(); err != nil {
exit(err)
}
}
func exit(err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", ui.RenderError(err))
}
os.Exit(1)
}
@@ -0,0 +1,115 @@
package testutil
import (
"fmt"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal"
"github.com/anchore/syft/syft/pkg/cataloger/binary/internal/manager/internal/config"
)
// SnippetOrBinary returns the path to either the binary or the snippet for the given logical entry key.
// Note: this is intended to be used only within the context of the binary cataloger test fixtures. Any other
// use is unsupported. Path should be a logical path relative to the testdata/classifiers directory (but does
// not specify the "bin" or "snippets" parent path... this is determined logically [snippets > binary unless told
// otherwise]). Path should also be to the directory containing the binary or snippets of interest (not the binaries
// or snippets itself).
func SnippetOrBinary(t *testing.T, path string, requireBinary bool) string {
t.Helper()
require.Len(t, internal.SplitFilepath(path), 3, "path must be a in the form <name>/<version>/<arch>")
// cd to testdata directory and load the config
cwd, err := os.Getwd()
require.NoError(t, err)
require.NoError(t, os.Chdir("testdata"))
defer func() {
require.NoError(t, os.Chdir(cwd))
}()
appConfig, err := config.Read()
require.NoError(t, err)
// find the first matching fixture path that matches the given requirements
entries, err := internal.ListAllEntries(*appConfig)
require.NoError(t, err)
var fixturePath string
for k, v := range entries.Sorted() {
if filepath.Dir(k.Path()) == path {
// prefer the snippet over the binary
if !requireBinary {
if v.SnippetPath != "" {
t.Logf("using snippet for %q", path)
require.NoError(t, validateSnippet(v.BinaryPath, v.SnippetPath))
fixturePath = v.SnippetPath
break
}
if v.BinaryPath != "" {
fixturePath = v.BinaryPath
break
}
t.Fatalf("no binary or snippet found for %q", path)
}
if v.BinaryPath != "" {
t.Logf("forcing the use of the original binary for %q", path)
fixturePath = v.BinaryPath
break
}
if v.SnippetPath != "" && !v.IsConfigured {
t.Skip("no binary found, but is covered by a snippet. Please add this case to the 'binary/testdata/config.yaml' and recreate the snippet")
}
t.Fatalf("no binary found for %q", path)
}
}
if fixturePath == "" {
t.Fatalf("no fixture found for %q", path)
}
// this should be relative to the tests-fixtures directory and should be the directory containing the binary or
// snippet of interest (not the path to the binary or snippet itself)
return filepath.Join("testdata", filepath.Dir(fixturePath))
}
func validateSnippet(binaryPath, snippetPath string) error {
// get a sha256 of the binary
if _, err := os.Stat(binaryPath); err != nil {
// no binary to validate against (this is ok)
return nil
}
metadata, err := internal.ReadSnippetMetadata(snippetPath)
if err != nil {
return err
}
if metadata == nil {
return nil
}
f, err := os.Open(binaryPath)
if err != nil {
return err
}
expected, err := internal.Sha256SumFile(f)
if err != nil {
return err
}
if expected != metadata.FileSha256 {
return fmt.Errorf("snippet shadows a binary with a different sha256 (want %q got %q)", expected, metadata.FileSha256)
}
return nil
}
@@ -0,0 +1,63 @@
package testutil
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
)
func TestValidateSnippet(t *testing.T) {
tests := []struct {
name string
binaryContent string
snippetContent string
expectError bool
}{
{
name: "valid",
binaryContent: "testBinary",
snippetContent: `name: bash
offset: 992758
length: 100
snippetSha256: 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824
fileSha256: 88ef99f28b69a6d8113cba62011e574a5afb1e6d8e0f884699e2ced91e4d910c
### byte snippet to follow ###
hello`,
expectError: false,
},
{
name: "invalid",
binaryContent: "testBinary",
snippetContent: `name: bash
offset: 992758
length: 100
snippetSha256: 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824
fileSha256: 011e574a8ef9584699e2ced9189f28b69a6d8113cba62e4d910cafb1e6d8e0f8
### byte snippet to follow ###
hello`,
expectError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
binaryFile, _ := os.CreateTemp(t.TempDir(), "binary")
binaryFile.WriteString(tt.binaryContent)
binaryFile.Close()
snippetFile, _ := os.CreateTemp(t.TempDir(), "snippet")
snippetFile.WriteString(tt.snippetContent)
snippetFile.Close()
err := validateSnippet(binaryFile.Name(), snippetFile.Name())
if tt.expectError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
})
}
}