mirror of
https://github.com/anchore/syft.git
synced 2026-11-02 23:46:35 +01:00
Refactor source API (#1846)
* refactor source API and syft json source block Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update source detection and format test utils Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * generate list of all source metadata types Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * extract base and root normalization into helper functions Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * preserve syftjson model package name import ref Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * alias should not be a pointer Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <alex.goodman@anchore.com> Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
@@ -0,0 +1,13 @@
|
||||
package source
|
||||
|
||||
type Alias struct {
|
||||
Name string `json:"name" yaml:"name" mapstructure:"name"`
|
||||
Version string `json:"version" yaml:"version" mapstructure:"version"`
|
||||
}
|
||||
|
||||
func (a *Alias) IsEmpty() bool {
|
||||
if a == nil {
|
||||
return true
|
||||
}
|
||||
return a.Name == "" && a.Version == ""
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package source
|
||||
|
||||
// Description represents any static source data that helps describe "what" was cataloged.
|
||||
type Description struct {
|
||||
ID string `hash:"ignore"` // the id generated from the parent source struct
|
||||
Name string `hash:"ignore"`
|
||||
Version string `hash:"ignore"`
|
||||
Metadata interface{}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package source
|
||||
|
||||
import (
|
||||
"crypto"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/mitchellh/go-homedir"
|
||||
"github.com/spf13/afero"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
)
|
||||
|
||||
type detectedType string
|
||||
|
||||
const (
|
||||
// unknownType is the default scheme
|
||||
unknownType detectedType = "unknown-type"
|
||||
|
||||
// directoryType indicates the source being cataloged is a directory on the root filesystem
|
||||
directoryType detectedType = "directory-type"
|
||||
|
||||
// containerImageType indicates the source being cataloged is a container image
|
||||
containerImageType detectedType = "container-image-type"
|
||||
|
||||
// fileType indicates the source being cataloged is a single file
|
||||
fileType detectedType = "file-type"
|
||||
)
|
||||
|
||||
type sourceResolver func(string) (image.Source, string, error)
|
||||
|
||||
// Detection is an object that captures the detected user input regarding source location, scheme, and provider type.
|
||||
// It acts as a struct input for some source constructors.
|
||||
type Detection struct {
|
||||
detectedType detectedType
|
||||
imageSource image.Source
|
||||
location string
|
||||
}
|
||||
|
||||
func (d Detection) IsContainerImage() bool {
|
||||
return d.detectedType == containerImageType
|
||||
}
|
||||
|
||||
type DetectConfig struct {
|
||||
DefaultImageSource string
|
||||
}
|
||||
|
||||
func DefaultDetectConfig() DetectConfig {
|
||||
return DetectConfig{}
|
||||
}
|
||||
|
||||
// Detect generates a source Detection that can be used as an argument to generate a new source
|
||||
// from specific providers including a registry, with an explicit name.
|
||||
func Detect(userInput string, cfg DetectConfig) (*Detection, error) {
|
||||
fs := afero.NewOsFs()
|
||||
ty, src, location, err := detect(fs, image.DetectSource, userInput)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if src == image.UnknownSource {
|
||||
// only run for these two schemes
|
||||
// only check on packages command, attest we automatically try to pull from userInput
|
||||
switch ty {
|
||||
case containerImageType, unknownType:
|
||||
ty = containerImageType
|
||||
location = userInput
|
||||
if cfg.DefaultImageSource != "" {
|
||||
src = parseDefaultImageSource(cfg.DefaultImageSource)
|
||||
} else {
|
||||
src = image.DetermineDefaultImagePullSource(userInput)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// collect user input for downstream consumption
|
||||
return &Detection{
|
||||
detectedType: ty,
|
||||
imageSource: src,
|
||||
location: location,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type DetectionSourceConfig struct {
|
||||
Alias Alias
|
||||
RegistryOptions *image.RegistryOptions
|
||||
Platform *image.Platform
|
||||
Exclude ExcludeConfig
|
||||
DigestAlgorithms []crypto.Hash
|
||||
}
|
||||
|
||||
func DefaultDetectionSourceConfig() DetectionSourceConfig {
|
||||
return DetectionSourceConfig{
|
||||
DigestAlgorithms: []crypto.Hash{
|
||||
crypto.SHA256,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// NewSource produces a Source based on userInput like dir: or image:tag
|
||||
func (d Detection) NewSource(cfg DetectionSourceConfig) (Source, error) {
|
||||
var err error
|
||||
var src Source
|
||||
|
||||
if d.detectedType != containerImageType && cfg.Platform != nil {
|
||||
return nil, fmt.Errorf("cannot specify a platform for a non-image source")
|
||||
}
|
||||
|
||||
switch d.detectedType {
|
||||
case fileType:
|
||||
src, err = NewFromFile(
|
||||
FileConfig{
|
||||
Path: d.location,
|
||||
Exclude: cfg.Exclude,
|
||||
DigestAlgorithms: cfg.DigestAlgorithms,
|
||||
Alias: cfg.Alias,
|
||||
},
|
||||
)
|
||||
case directoryType:
|
||||
src, err = NewFromDirectory(
|
||||
DirectoryConfig{
|
||||
Path: d.location,
|
||||
Base: d.location,
|
||||
Exclude: cfg.Exclude,
|
||||
Alias: cfg.Alias,
|
||||
},
|
||||
)
|
||||
case containerImageType:
|
||||
src, err = NewFromStereoscopeImage(
|
||||
StereoscopeImageConfig{
|
||||
Reference: d.location,
|
||||
From: d.imageSource,
|
||||
Platform: cfg.Platform,
|
||||
RegistryOptions: cfg.RegistryOptions,
|
||||
Exclude: cfg.Exclude,
|
||||
Alias: cfg.Alias,
|
||||
},
|
||||
)
|
||||
default:
|
||||
err = fmt.Errorf("unable to process input for scanning")
|
||||
}
|
||||
|
||||
return src, err
|
||||
}
|
||||
|
||||
func detect(fs afero.Fs, imageSourceResolver sourceResolver, userInput string) (detectedType, image.Source, string, error) {
|
||||
switch {
|
||||
case strings.HasPrefix(userInput, "dir:"):
|
||||
dirLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "dir:"))
|
||||
if err != nil {
|
||||
return unknownType, image.UnknownSource, "", fmt.Errorf("unable to expand directory path: %w", err)
|
||||
}
|
||||
return directoryType, image.UnknownSource, dirLocation, nil
|
||||
|
||||
case strings.HasPrefix(userInput, "file:"):
|
||||
fileLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "file:"))
|
||||
if err != nil {
|
||||
return unknownType, image.UnknownSource, "", fmt.Errorf("unable to expand directory path: %w", err)
|
||||
}
|
||||
return fileType, image.UnknownSource, fileLocation, nil
|
||||
}
|
||||
|
||||
// try the most specific sources first and move out towards more generic sources.
|
||||
|
||||
// first: let's try the image detector, which has more scheme parsing internal to stereoscope
|
||||
src, imageSpec, err := imageSourceResolver(userInput)
|
||||
if err == nil && src != image.UnknownSource {
|
||||
return containerImageType, src, imageSpec, nil
|
||||
}
|
||||
|
||||
// next: let's try more generic sources (dir, file, etc.)
|
||||
location, err := homedir.Expand(userInput)
|
||||
if err != nil {
|
||||
return unknownType, image.UnknownSource, "", fmt.Errorf("unable to expand potential directory path: %w", err)
|
||||
}
|
||||
|
||||
fileMeta, err := fs.Stat(location)
|
||||
if err != nil {
|
||||
return unknownType, src, "", nil
|
||||
}
|
||||
|
||||
if fileMeta.IsDir() {
|
||||
return directoryType, src, location, nil
|
||||
}
|
||||
|
||||
return fileType, src, location, nil
|
||||
}
|
||||
|
||||
func parseDefaultImageSource(defaultImageSource string) image.Source {
|
||||
switch defaultImageSource {
|
||||
case "registry":
|
||||
return image.OciRegistrySource
|
||||
case "docker":
|
||||
return image.DockerDaemonSource
|
||||
case "podman":
|
||||
return image.PodmanDaemonSource
|
||||
default:
|
||||
return image.UnknownSource
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
)
|
||||
|
||||
func TestDetectScheme(t *testing.T) {
|
||||
func Test_Detect(t *testing.T) {
|
||||
type detectorResult struct {
|
||||
src image.Source
|
||||
ref string
|
||||
@@ -24,7 +24,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
dirs []string
|
||||
files []string
|
||||
detection detectorResult
|
||||
expectedScheme Scheme
|
||||
expectedScheme detectedType
|
||||
expectedLocation string
|
||||
}{
|
||||
{
|
||||
@@ -34,7 +34,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
src: image.DockerDaemonSource,
|
||||
ref: "wagoodman/dive:latest",
|
||||
},
|
||||
expectedScheme: ImageScheme,
|
||||
expectedScheme: containerImageType,
|
||||
expectedLocation: "wagoodman/dive:latest",
|
||||
},
|
||||
{
|
||||
@@ -44,7 +44,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
src: image.DockerDaemonSource,
|
||||
ref: "wagoodman/dive",
|
||||
},
|
||||
expectedScheme: ImageScheme,
|
||||
expectedScheme: containerImageType,
|
||||
expectedLocation: "wagoodman/dive",
|
||||
},
|
||||
{
|
||||
@@ -54,7 +54,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
src: image.OciRegistrySource,
|
||||
ref: "wagoodman/dive:latest",
|
||||
},
|
||||
expectedScheme: ImageScheme,
|
||||
expectedScheme: containerImageType,
|
||||
expectedLocation: "wagoodman/dive:latest",
|
||||
},
|
||||
{
|
||||
@@ -64,7 +64,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
src: image.DockerDaemonSource,
|
||||
ref: "wagoodman/dive:latest",
|
||||
},
|
||||
expectedScheme: ImageScheme,
|
||||
expectedScheme: containerImageType,
|
||||
expectedLocation: "wagoodman/dive:latest",
|
||||
},
|
||||
{
|
||||
@@ -74,7 +74,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
src: image.DockerDaemonSource,
|
||||
ref: "wagoodman/dive",
|
||||
},
|
||||
expectedScheme: ImageScheme,
|
||||
expectedScheme: containerImageType,
|
||||
expectedLocation: "wagoodman/dive",
|
||||
},
|
||||
{
|
||||
@@ -84,7 +84,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
src: image.DockerDaemonSource,
|
||||
ref: "latest",
|
||||
},
|
||||
expectedScheme: ImageScheme,
|
||||
expectedScheme: containerImageType,
|
||||
// we expected to be able to handle this case better, however, I don't see a way to do this
|
||||
// the user will need to provide more explicit input (docker:docker:latest)
|
||||
expectedLocation: "latest",
|
||||
@@ -96,7 +96,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
src: image.DockerDaemonSource,
|
||||
ref: "docker:latest",
|
||||
},
|
||||
expectedScheme: ImageScheme,
|
||||
expectedScheme: containerImageType,
|
||||
// we expected to be able to handle this case better, however, I don't see a way to do this
|
||||
// the user will need to provide more explicit input (docker:docker:latest)
|
||||
expectedLocation: "docker:latest",
|
||||
@@ -108,7 +108,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
src: image.OciTarballSource,
|
||||
ref: "some/path-to-file",
|
||||
},
|
||||
expectedScheme: ImageScheme,
|
||||
expectedScheme: containerImageType,
|
||||
expectedLocation: "some/path-to-file",
|
||||
},
|
||||
{
|
||||
@@ -119,7 +119,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
ref: "some/path-to-dir",
|
||||
},
|
||||
dirs: []string{"some/path-to-dir"},
|
||||
expectedScheme: ImageScheme,
|
||||
expectedScheme: containerImageType,
|
||||
expectedLocation: "some/path-to-dir",
|
||||
},
|
||||
{
|
||||
@@ -130,7 +130,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
ref: "",
|
||||
},
|
||||
dirs: []string{"some/path-to-dir"},
|
||||
expectedScheme: DirectoryScheme,
|
||||
expectedScheme: directoryType,
|
||||
expectedLocation: "some/path-to-dir",
|
||||
},
|
||||
{
|
||||
@@ -140,7 +140,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
src: image.DockerDaemonSource,
|
||||
ref: "some/path-to-dir",
|
||||
},
|
||||
expectedScheme: ImageScheme,
|
||||
expectedScheme: containerImageType,
|
||||
expectedLocation: "some/path-to-dir",
|
||||
},
|
||||
{
|
||||
@@ -150,7 +150,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
src: image.PodmanDaemonSource,
|
||||
ref: "something:latest",
|
||||
},
|
||||
expectedScheme: ImageScheme,
|
||||
expectedScheme: containerImageType,
|
||||
expectedLocation: "something:latest",
|
||||
},
|
||||
{
|
||||
@@ -161,7 +161,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
ref: "",
|
||||
},
|
||||
dirs: []string{"some/path-to-dir"},
|
||||
expectedScheme: DirectoryScheme,
|
||||
expectedScheme: directoryType,
|
||||
expectedLocation: "some/path-to-dir",
|
||||
},
|
||||
{
|
||||
@@ -172,7 +172,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
ref: "",
|
||||
},
|
||||
files: []string{"some/path-to-file"},
|
||||
expectedScheme: FileScheme,
|
||||
expectedScheme: fileType,
|
||||
expectedLocation: "some/path-to-file",
|
||||
},
|
||||
{
|
||||
@@ -183,7 +183,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
ref: "",
|
||||
},
|
||||
files: []string{"some/path-to-file"},
|
||||
expectedScheme: FileScheme,
|
||||
expectedScheme: fileType,
|
||||
expectedLocation: "some/path-to-file",
|
||||
},
|
||||
{
|
||||
@@ -193,7 +193,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
src: image.UnknownSource,
|
||||
ref: "",
|
||||
},
|
||||
expectedScheme: DirectoryScheme,
|
||||
expectedScheme: directoryType,
|
||||
expectedLocation: ".",
|
||||
},
|
||||
{
|
||||
@@ -203,7 +203,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
src: image.UnknownSource,
|
||||
ref: "",
|
||||
},
|
||||
expectedScheme: DirectoryScheme,
|
||||
expectedScheme: directoryType,
|
||||
expectedLocation: ".",
|
||||
},
|
||||
// we should support tilde expansion
|
||||
@@ -214,7 +214,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
src: image.OciDirectorySource,
|
||||
ref: "~/some-path",
|
||||
},
|
||||
expectedScheme: ImageScheme,
|
||||
expectedScheme: containerImageType,
|
||||
expectedLocation: "~/some-path",
|
||||
},
|
||||
{
|
||||
@@ -225,26 +225,26 @@ func TestDetectScheme(t *testing.T) {
|
||||
ref: "",
|
||||
},
|
||||
dirs: []string{"~/some-path"},
|
||||
expectedScheme: DirectoryScheme,
|
||||
expectedScheme: directoryType,
|
||||
expectedLocation: "~/some-path",
|
||||
},
|
||||
{
|
||||
name: "tilde-expansion-dir-explicit-exists",
|
||||
userInput: "dir:~/some-path",
|
||||
dirs: []string{"~/some-path"},
|
||||
expectedScheme: DirectoryScheme,
|
||||
expectedScheme: directoryType,
|
||||
expectedLocation: "~/some-path",
|
||||
},
|
||||
{
|
||||
name: "tilde-expansion-dir-explicit-dne",
|
||||
userInput: "dir:~/some-path",
|
||||
expectedScheme: DirectoryScheme,
|
||||
expectedScheme: directoryType,
|
||||
expectedLocation: "~/some-path",
|
||||
},
|
||||
{
|
||||
name: "tilde-expansion-dir-implicit-dne",
|
||||
userInput: "~/some-path",
|
||||
expectedScheme: UnknownScheme,
|
||||
expectedScheme: unknownType,
|
||||
expectedLocation: "",
|
||||
},
|
||||
}
|
||||
@@ -288,7 +288,7 @@ func TestDetectScheme(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
actualScheme, actualSource, actualLocation, err := DetectScheme(fs, imageDetector, test.userInput)
|
||||
actualScheme, actualSource, actualLocation, err := detect(fs, imageDetector, test.userInput)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err : %+v", err)
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package source
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
)
|
||||
|
||||
func artifactIDFromDigest(input string) artifact.ID {
|
||||
return artifact.ID(strings.TrimPrefix(input, "sha256:"))
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package source
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/internal/fileresolver"
|
||||
)
|
||||
|
||||
var _ Source = (*DirectorySource)(nil)
|
||||
|
||||
type DirectoryConfig struct {
|
||||
Path string
|
||||
Base string
|
||||
Exclude ExcludeConfig
|
||||
Alias Alias
|
||||
}
|
||||
|
||||
type DirectorySourceMetadata struct {
|
||||
Path string `json:"path" yaml:"path"`
|
||||
Base string `json:"-" yaml:"-"` // though this is important, for display purposes it leaks too much information (abs paths)
|
||||
}
|
||||
|
||||
type DirectorySource struct {
|
||||
id artifact.ID
|
||||
config DirectoryConfig
|
||||
resolver *fileresolver.Directory
|
||||
mutex *sync.Mutex
|
||||
}
|
||||
|
||||
func NewFromDirectoryPath(path string) (*DirectorySource, error) {
|
||||
cfg := DirectoryConfig{
|
||||
Path: path,
|
||||
}
|
||||
return NewFromDirectory(cfg)
|
||||
}
|
||||
|
||||
func NewFromDirectory(cfg DirectoryConfig) (*DirectorySource, error) {
|
||||
fi, err := os.Stat(cfg.Path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to stat path=%q: %w", cfg.Path, err)
|
||||
}
|
||||
|
||||
if !fi.IsDir() {
|
||||
return nil, fmt.Errorf("given path is not a directory: %q", cfg.Path)
|
||||
}
|
||||
|
||||
return &DirectorySource{
|
||||
id: deriveIDFromDirectory(cfg),
|
||||
config: cfg,
|
||||
mutex: &sync.Mutex{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// deriveIDFromDirectory generates an artifact ID from the given directory config. If an alias is provided, then
|
||||
// the artifact ID is derived exclusively from the alias name and version. Otherwise, the artifact ID is derived
|
||||
// from the path provided with an attempt to prune a prefix if a base is given. Since the contents of the directory
|
||||
// are not considered, there is no semantic meaning to the artifact ID -- this is why the alias is preferred without
|
||||
// consideration for the path.
|
||||
func deriveIDFromDirectory(cfg DirectoryConfig) artifact.ID {
|
||||
var info string
|
||||
if !cfg.Alias.IsEmpty() {
|
||||
// don't use any of the path information -- instead use the alias name and version as the artifact ID.
|
||||
// why? this allows the user to set a dependable stable value for the artifact ID in case the
|
||||
// scanning root changes (e.g. a user scans a directory, then moves it to a new location and scans again).
|
||||
info = fmt.Sprintf("%s@%s", cfg.Alias.Name, cfg.Alias.Version)
|
||||
} else {
|
||||
log.Warn("no explicit name and version provided for directory source, deriving artifact ID from the given path (which is not ideal)")
|
||||
info = cleanDirPath(cfg.Path, cfg.Base)
|
||||
}
|
||||
|
||||
return artifactIDFromDigest(digest.SHA256.FromString(filepath.Clean(info)).String())
|
||||
}
|
||||
|
||||
func cleanDirPath(path, base string) string {
|
||||
if path == base {
|
||||
return path
|
||||
}
|
||||
|
||||
if base != "" {
|
||||
cleanRoot, rootErr := fileresolver.NormalizeRootDirectory(path)
|
||||
cleanBase, baseErr := fileresolver.NormalizeBaseDirectory(base)
|
||||
|
||||
if rootErr == nil && baseErr == nil {
|
||||
// allows for normalizing inputs:
|
||||
// cleanRoot: /var/folders/8x/gw98pp6535s4r8drc374tb1r0000gn/T/TestDirectoryEncoder1121632790/001/some/path
|
||||
// cleanBase: /var/folders/8x/gw98pp6535s4r8drc374tb1r0000gn/T/TestDirectoryEncoder1121632790/001
|
||||
// normalized: some/path
|
||||
|
||||
relPath, err := filepath.Rel(cleanBase, cleanRoot)
|
||||
if err == nil {
|
||||
path = relPath
|
||||
}
|
||||
// this is odd, but this means we can't use base
|
||||
}
|
||||
// if the base is not a valid chroot, then just use the path as-is
|
||||
}
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
func (s DirectorySource) ID() artifact.ID {
|
||||
return s.id
|
||||
}
|
||||
|
||||
func (s DirectorySource) Describe() Description {
|
||||
name := cleanDirPath(s.config.Path, s.config.Base)
|
||||
version := ""
|
||||
if !s.config.Alias.IsEmpty() {
|
||||
a := s.config.Alias
|
||||
if a.Name != "" {
|
||||
name = a.Name
|
||||
}
|
||||
}
|
||||
return Description{
|
||||
ID: string(s.id),
|
||||
Name: name,
|
||||
Version: version,
|
||||
Metadata: DirectorySourceMetadata{
|
||||
Path: s.config.Path,
|
||||
Base: s.config.Base,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DirectorySource) FileResolver(_ Scope) (file.Resolver, error) {
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
|
||||
if s.resolver == nil {
|
||||
exclusionFunctions, err := getDirectoryExclusionFunctions(s.config.Path, s.config.Exclude.Paths)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res, err := fileresolver.NewFromDirectory(s.config.Path, s.config.Base, exclusionFunctions...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create directory resolver: %w", err)
|
||||
}
|
||||
|
||||
s.resolver = res
|
||||
}
|
||||
|
||||
return s.resolver, nil
|
||||
}
|
||||
|
||||
func (s *DirectorySource) Close() error {
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
s.resolver = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func getDirectoryExclusionFunctions(root string, exclusions []string) ([]fileresolver.PathIndexVisitor, error) {
|
||||
if len(exclusions) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// this is what directoryResolver.indexTree is doing to get the absolute path:
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// this handles Windows file paths by converting them to C:/something/else format
|
||||
root = filepath.ToSlash(root)
|
||||
|
||||
if !strings.HasSuffix(root, "/") {
|
||||
root += "/"
|
||||
}
|
||||
|
||||
var errors []string
|
||||
for idx, exclusion := range exclusions {
|
||||
// check exclusions for supported paths, these are all relative to the "scan root"
|
||||
if strings.HasPrefix(exclusion, "./") || strings.HasPrefix(exclusion, "*/") || strings.HasPrefix(exclusion, "**/") {
|
||||
exclusion = strings.TrimPrefix(exclusion, "./")
|
||||
exclusions[idx] = root + exclusion
|
||||
} else {
|
||||
errors = append(errors, exclusion)
|
||||
}
|
||||
}
|
||||
|
||||
if errors != nil {
|
||||
return nil, fmt.Errorf("invalid exclusion pattern(s): '%s' (must start with one of: './', '*/', or '**/')", strings.Join(errors, "', '"))
|
||||
}
|
||||
|
||||
return []fileresolver.PathIndexVisitor{
|
||||
func(path string, info os.FileInfo, _ error) error {
|
||||
for _, exclusion := range exclusions {
|
||||
// this is required to handle Windows filepaths
|
||||
path = filepath.ToSlash(path)
|
||||
matches, err := doublestar.Match(exclusion, path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if matches {
|
||||
if info != nil && info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return fileresolver.ErrSkipPath
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,560 @@
|
||||
package source
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/internal/fileresolver"
|
||||
)
|
||||
|
||||
func TestNewFromDirectory(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
input string
|
||||
expString string
|
||||
inputPaths []string
|
||||
expectedRefs int
|
||||
cxErr require.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
desc: "no paths exist",
|
||||
input: "foobar/",
|
||||
inputPaths: []string{"/opt/", "/other"},
|
||||
cxErr: require.Error,
|
||||
},
|
||||
{
|
||||
desc: "path detected",
|
||||
input: "test-fixtures",
|
||||
inputPaths: []string{"path-detected/.vimrc"},
|
||||
expectedRefs: 1,
|
||||
},
|
||||
{
|
||||
desc: "directory ignored",
|
||||
input: "test-fixtures",
|
||||
inputPaths: []string{"path-detected"},
|
||||
expectedRefs: 0,
|
||||
},
|
||||
{
|
||||
desc: "no files-by-path detected",
|
||||
input: "test-fixtures",
|
||||
inputPaths: []string{"no-path-detected"},
|
||||
expectedRefs: 0,
|
||||
},
|
||||
}
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
if test.cxErr == nil {
|
||||
test.cxErr = require.NoError
|
||||
}
|
||||
src, err := NewFromDirectory(DirectoryConfig{
|
||||
Path: test.input,
|
||||
})
|
||||
test.cxErr(t, err)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, src.Close())
|
||||
})
|
||||
assert.Equal(t, test.input, src.Describe().Metadata.(DirectorySourceMetadata).Path)
|
||||
|
||||
res, err := src.FileResolver(SquashedScope)
|
||||
require.NoError(t, err)
|
||||
|
||||
refs, err := res.FilesByPath(test.inputPaths...)
|
||||
require.NoError(t, err)
|
||||
|
||||
if len(refs) != test.expectedRefs {
|
||||
t.Errorf("unexpected number of refs returned: %d != %d", len(refs), test.expectedRefs)
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_DirectorySource_FilesByGlob(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
input string
|
||||
glob string
|
||||
expected int
|
||||
}{
|
||||
{
|
||||
input: "test-fixtures",
|
||||
desc: "no matches",
|
||||
glob: "bar/foo",
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/path-detected",
|
||||
desc: "a single match",
|
||||
glob: "**/*vimrc",
|
||||
expected: 1,
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/path-detected",
|
||||
desc: "multiple matches",
|
||||
glob: "**",
|
||||
expected: 2,
|
||||
},
|
||||
}
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
src, err := NewFromDirectory(DirectoryConfig{Path: test.input})
|
||||
require.NoError(t, err)
|
||||
|
||||
res, err := src.FileResolver(SquashedScope)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, src.Close())
|
||||
})
|
||||
|
||||
contents, err := res.FilesByGlob(test.glob)
|
||||
require.NoError(t, err)
|
||||
if len(contents) != test.expected {
|
||||
t.Errorf("unexpected number of files found by glob (%s): %d != %d", test.glob, len(contents), test.expected)
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_DirectorySource_Exclusions(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
input string
|
||||
glob string
|
||||
expected []string
|
||||
exclusions []string
|
||||
err bool
|
||||
}{
|
||||
{
|
||||
input: "test-fixtures/system_paths",
|
||||
desc: "exclude everything",
|
||||
glob: "**",
|
||||
expected: nil,
|
||||
exclusions: []string{"**/*"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/image-simple",
|
||||
desc: "a single path excluded",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
"Dockerfile",
|
||||
"file-1.txt",
|
||||
"file-2.txt",
|
||||
},
|
||||
exclusions: []string{"**/target/**"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/image-simple",
|
||||
desc: "exclude explicit directory relative to the root",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
"Dockerfile",
|
||||
"file-1.txt",
|
||||
"file-2.txt",
|
||||
//"target/really/nested/file-3.txt", // explicitly skipped
|
||||
},
|
||||
exclusions: []string{"./target"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/image-simple",
|
||||
desc: "exclude explicit file relative to the root",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
"Dockerfile",
|
||||
//"file-1.txt", // explicitly skipped
|
||||
"file-2.txt",
|
||||
"target/really/nested/file-3.txt",
|
||||
},
|
||||
exclusions: []string{"./file-1.txt"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/image-simple",
|
||||
desc: "exclude wildcard relative to the root",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
"Dockerfile",
|
||||
//"file-1.txt", // explicitly skipped
|
||||
//"file-2.txt", // explicitly skipped
|
||||
"target/really/nested/file-3.txt",
|
||||
},
|
||||
exclusions: []string{"./*.txt"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/image-simple",
|
||||
desc: "exclude files deeper",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
"Dockerfile",
|
||||
"file-1.txt",
|
||||
"file-2.txt",
|
||||
//"target/really/nested/file-3.txt", // explicitly skipped
|
||||
},
|
||||
exclusions: []string{"**/really/**"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/image-simple",
|
||||
desc: "files excluded with extension",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
"Dockerfile",
|
||||
//"file-1.txt", // explicitly skipped
|
||||
//"file-2.txt", // explicitly skipped
|
||||
//"target/really/nested/file-3.txt", // explicitly skipped
|
||||
},
|
||||
exclusions: []string{"**/*.txt"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/image-simple",
|
||||
desc: "keep files with different extensions",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
"Dockerfile",
|
||||
"file-1.txt",
|
||||
"file-2.txt",
|
||||
"target/really/nested/file-3.txt",
|
||||
},
|
||||
exclusions: []string{"**/target/**/*.jar"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/path-detected",
|
||||
desc: "file directly excluded",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
".vimrc",
|
||||
},
|
||||
exclusions: []string{"**/empty"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/path-detected",
|
||||
desc: "pattern error containing **/",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
".vimrc",
|
||||
},
|
||||
exclusions: []string{"/**/empty"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/path-detected",
|
||||
desc: "pattern error incorrect start",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
".vimrc",
|
||||
},
|
||||
exclusions: []string{"empty"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/path-detected",
|
||||
desc: "pattern error starting with /",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
".vimrc",
|
||||
},
|
||||
exclusions: []string{"/empty"},
|
||||
err: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
src, err := NewFromDirectory(DirectoryConfig{
|
||||
Path: test.input,
|
||||
Exclude: ExcludeConfig{
|
||||
Paths: test.exclusions,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, src.Close())
|
||||
})
|
||||
|
||||
if test.err {
|
||||
_, err = src.FileResolver(SquashedScope)
|
||||
require.Error(t, err)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
|
||||
res, err := src.FileResolver(SquashedScope)
|
||||
require.NoError(t, err)
|
||||
|
||||
locations, err := res.FilesByGlob(test.glob)
|
||||
require.NoError(t, err)
|
||||
|
||||
var actual []string
|
||||
for _, l := range locations {
|
||||
actual = append(actual, l.RealPath)
|
||||
}
|
||||
|
||||
assert.ElementsMatchf(t, test.expected, actual, "diff \n"+cmp.Diff(test.expected, actual))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_getDirectoryExclusionFunctions_crossPlatform(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
root string
|
||||
path string
|
||||
finfo os.FileInfo
|
||||
exclude string
|
||||
walkHint error
|
||||
}{
|
||||
{
|
||||
desc: "directory exclusion",
|
||||
root: "/",
|
||||
path: "/usr/var/lib",
|
||||
exclude: "**/var/lib",
|
||||
finfo: file.ManualInfo{ModeValue: os.ModeDir},
|
||||
walkHint: fs.SkipDir,
|
||||
},
|
||||
{
|
||||
desc: "no file info",
|
||||
root: "/",
|
||||
path: "/usr/var/lib",
|
||||
exclude: "**/var/lib",
|
||||
walkHint: fileresolver.ErrSkipPath,
|
||||
},
|
||||
// linux specific tests...
|
||||
{
|
||||
desc: "linux doublestar",
|
||||
root: "/usr",
|
||||
path: "/usr/var/lib/etc.txt",
|
||||
exclude: "**/*.txt",
|
||||
finfo: file.ManualInfo{},
|
||||
walkHint: fileresolver.ErrSkipPath,
|
||||
},
|
||||
{
|
||||
desc: "linux relative",
|
||||
root: "/usr/var/lib",
|
||||
path: "/usr/var/lib/etc.txt",
|
||||
exclude: "./*.txt",
|
||||
finfo: file.ManualInfo{},
|
||||
|
||||
walkHint: fileresolver.ErrSkipPath,
|
||||
},
|
||||
{
|
||||
desc: "linux one level",
|
||||
root: "/usr",
|
||||
path: "/usr/var/lib/etc.txt",
|
||||
exclude: "*/*.txt",
|
||||
finfo: file.ManualInfo{},
|
||||
walkHint: nil,
|
||||
},
|
||||
// NOTE: since these tests will run in linux and macOS, the windows paths will be
|
||||
// considered relative if they do not start with a forward slash and paths with backslashes
|
||||
// won't be modified by the filepath.ToSlash call, so these are emulating the result of
|
||||
// filepath.ToSlash usage
|
||||
|
||||
// windows specific tests...
|
||||
{
|
||||
desc: "windows doublestar",
|
||||
root: "/C:/User/stuff",
|
||||
path: "/C:/User/stuff/thing.txt",
|
||||
exclude: "**/*.txt",
|
||||
finfo: file.ManualInfo{},
|
||||
walkHint: fileresolver.ErrSkipPath,
|
||||
},
|
||||
{
|
||||
desc: "windows relative",
|
||||
root: "/C:/User/stuff",
|
||||
path: "/C:/User/stuff/thing.txt",
|
||||
exclude: "./*.txt",
|
||||
finfo: file.ManualInfo{},
|
||||
walkHint: fileresolver.ErrSkipPath,
|
||||
},
|
||||
{
|
||||
desc: "windows one level",
|
||||
root: "/C:/User/stuff",
|
||||
path: "/C:/User/stuff/thing.txt",
|
||||
exclude: "*/*.txt",
|
||||
finfo: file.ManualInfo{},
|
||||
walkHint: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
fns, err := getDirectoryExclusionFunctions(test.root, []string{test.exclude})
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, f := range fns {
|
||||
result := f(test.path, test.finfo, nil)
|
||||
require.Equal(t, test.walkHint, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_DirectorySource_FilesByPathDoesNotExist(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
input string
|
||||
path string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
input: "test-fixtures/path-detected",
|
||||
desc: "path does not exist",
|
||||
path: "foo",
|
||||
},
|
||||
}
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
src, err := NewFromDirectory(DirectoryConfig{Path: test.input})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, src.Close())
|
||||
})
|
||||
|
||||
res, err := src.FileResolver(SquashedScope)
|
||||
require.NoError(t, err)
|
||||
|
||||
refs, err := res.FilesByPath(test.path)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, refs, 0)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_DirectorySource_ID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg DirectoryConfig
|
||||
want artifact.ID
|
||||
wantErr require.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
cfg: DirectoryConfig{},
|
||||
wantErr: require.Error,
|
||||
},
|
||||
{
|
||||
name: "to a non-existent directory",
|
||||
cfg: DirectoryConfig{
|
||||
Path: "./test-fixtures/does-not-exist",
|
||||
},
|
||||
wantErr: require.Error,
|
||||
},
|
||||
{
|
||||
name: "with odd unclean path through non-existent directory",
|
||||
cfg: DirectoryConfig{Path: "test-fixtures/does-not-exist/../"},
|
||||
wantErr: require.Error,
|
||||
},
|
||||
{
|
||||
name: "to a file (not a directory)",
|
||||
cfg: DirectoryConfig{
|
||||
Path: "./test-fixtures/image-simple/Dockerfile",
|
||||
},
|
||||
wantErr: require.Error,
|
||||
},
|
||||
{
|
||||
name: "to dir with name and version",
|
||||
cfg: DirectoryConfig{
|
||||
Path: "./test-fixtures",
|
||||
Alias: Alias{
|
||||
Name: "name-me-that!",
|
||||
Version: "version-me-this!",
|
||||
},
|
||||
},
|
||||
want: artifact.ID("51a5f2a1536cf4b5220d4247814b07eec5862ab0547050f90e9ae216548ded7e"),
|
||||
},
|
||||
{
|
||||
name: "to different dir with name and version",
|
||||
cfg: DirectoryConfig{
|
||||
Path: "./test-fixtures/image-simple",
|
||||
Alias: Alias{
|
||||
Name: "name-me-that!",
|
||||
Version: "version-me-this!",
|
||||
},
|
||||
},
|
||||
// note: this must match the previous value because the alias should trump the path info
|
||||
want: artifact.ID("51a5f2a1536cf4b5220d4247814b07eec5862ab0547050f90e9ae216548ded7e"),
|
||||
},
|
||||
{
|
||||
name: "with path",
|
||||
cfg: DirectoryConfig{Path: "./test-fixtures"},
|
||||
want: artifact.ID("c2f936b0054dc6114fc02a3446bf8916bde8fdf87166a23aee22ea011b443522"),
|
||||
},
|
||||
{
|
||||
name: "with unclean path",
|
||||
cfg: DirectoryConfig{Path: "test-fixtures/image-simple/../"},
|
||||
want: artifact.ID("c2f936b0054dc6114fc02a3446bf8916bde8fdf87166a23aee22ea011b443522"),
|
||||
},
|
||||
{
|
||||
name: "other fields do not affect ID",
|
||||
cfg: DirectoryConfig{
|
||||
Path: "test-fixtures",
|
||||
Base: "a-base!",
|
||||
Exclude: ExcludeConfig{
|
||||
Paths: []string{"a", "b"},
|
||||
},
|
||||
},
|
||||
want: artifact.ID("c2f936b0054dc6114fc02a3446bf8916bde8fdf87166a23aee22ea011b443522"),
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.wantErr == nil {
|
||||
tt.wantErr = require.NoError
|
||||
}
|
||||
s, err := NewFromDirectory(tt.cfg)
|
||||
tt.wantErr(t, err)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
assert.Equalf(t, tt.want, s.ID(), "ID()")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_cleanDirPath(t *testing.T) {
|
||||
|
||||
abs, err := filepath.Abs("test-fixtures")
|
||||
require.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
base string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "abs path, abs base, base contained in path",
|
||||
path: filepath.Join(abs, "system_paths/outside_root"),
|
||||
base: abs,
|
||||
want: "system_paths/outside_root",
|
||||
},
|
||||
{
|
||||
name: "abs path, abs base, base not contained in path",
|
||||
path: "/var/folders/8x/gw98pp6535s4r8drc374tb1r0000gn/T/001/some/path",
|
||||
base: "/var/folders/8x/gw98pp6535s4r8drc374tb1r0000gn/T/002",
|
||||
want: "/var/folders/8x/gw98pp6535s4r8drc374tb1r0000gn/T/001/some/path",
|
||||
},
|
||||
{
|
||||
name: "path and base match",
|
||||
path: "/var/folders/8x/gw98pp6535s4r8drc374tb1r0000gn/T/001/some/path",
|
||||
base: "/var/folders/8x/gw98pp6535s4r8drc374tb1r0000gn/T/001/some/path",
|
||||
want: "/var/folders/8x/gw98pp6535s4r8drc374tb1r0000gn/T/001/some/path",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, cleanDirPath(tt.path, tt.base))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
//go:build windows
|
||||
// +build windows
|
||||
|
||||
// why the build tags? there is behavior from filepath.ToSlash() that must be tested, but can't be tested on non-windows
|
||||
// since the stdlib keeps this functionality behind a build tag (specifically filepath.Separator):
|
||||
// - https://github.com/golang/go/blob/3aea422e2cb8b1ec2e0c2774be97fe96c7299838/src/path/filepath/path.go#L224-L227
|
||||
// - https://github.com/golang/go/blob/3aea422e2cb8b1ec2e0c2774be97fe96c7299838/src/path/filepath/path.go#L63
|
||||
// - https://github.com/golang/go/blob/master/src/os/path_windows.go#L8
|
||||
//
|
||||
// It would be nice to extract this to simplify testing, however, we also need filepath.Abs(), which in windows
|
||||
// requires a specific syscall:
|
||||
// - https://github.com/golang/go/blob/3aea422e2cb8b1ec2e0c2774be97fe96c7299838/src/path/filepath/path_windows.go#L216
|
||||
// ... which means we can't extract this functionality without build tags.
|
||||
|
||||
package source
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Test_DirectorySource_crossPlatformExclusions(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
root string
|
||||
path string
|
||||
exclude string
|
||||
match bool
|
||||
}{
|
||||
{
|
||||
desc: "windows doublestar",
|
||||
root: "C:\\User\\stuff",
|
||||
path: "C:\\User\\stuff\\thing.txt",
|
||||
exclude: "**/*.txt",
|
||||
match: true,
|
||||
},
|
||||
{
|
||||
desc: "windows relative",
|
||||
root: "C:\\User\\stuff",
|
||||
path: "C:\\User\\stuff\\thing.txt",
|
||||
exclude: "./*.txt",
|
||||
match: true,
|
||||
},
|
||||
{
|
||||
desc: "windows one level",
|
||||
root: "C:\\User\\stuff",
|
||||
path: "C:\\User\\stuff\\thing.txt",
|
||||
exclude: "*/*.txt",
|
||||
match: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
fns, err := getDirectoryExclusionFunctions(test.root, []string{test.exclude})
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, f := range fns {
|
||||
result := f(test.path, nil, nil)
|
||||
require.Equal(t, test.match, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
package source
|
||||
|
||||
type ExcludeConfig struct {
|
||||
Paths []string
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package source
|
||||
|
||||
import (
|
||||
"crypto"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/mholt/archiver/v3"
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
||||
stereoFile "github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/internal/fileresolver"
|
||||
)
|
||||
|
||||
var _ Source = (*FileSource)(nil)
|
||||
|
||||
type FileConfig struct {
|
||||
Path string
|
||||
Exclude ExcludeConfig
|
||||
DigestAlgorithms []crypto.Hash
|
||||
Alias Alias
|
||||
}
|
||||
|
||||
type FileSourceMetadata struct {
|
||||
Path string `json:"path" yaml:"path"`
|
||||
Digests []file.Digest `json:"digests,omitempty" yaml:"digests,omitempty"`
|
||||
MIMEType string `json:"mimeType" yaml:"mimeType"`
|
||||
}
|
||||
|
||||
type FileSource struct {
|
||||
id artifact.ID
|
||||
digestForVersion string
|
||||
config FileConfig
|
||||
resolver *fileresolver.Directory
|
||||
mutex *sync.Mutex
|
||||
closer func() error
|
||||
digests []file.Digest
|
||||
mimeType string
|
||||
analysisPath string
|
||||
}
|
||||
|
||||
func NewFromFile(cfg FileConfig) (*FileSource, error) {
|
||||
fileMeta, err := os.Stat(cfg.Path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to stat path=%q: %w", cfg.Path, err)
|
||||
}
|
||||
|
||||
if fileMeta.IsDir() {
|
||||
return nil, fmt.Errorf("given path is a directory: %q", cfg.Path)
|
||||
}
|
||||
|
||||
analysisPath, cleanupFn := fileAnalysisPath(cfg.Path)
|
||||
|
||||
var digests []file.Digest
|
||||
if len(cfg.DigestAlgorithms) > 0 {
|
||||
fh, err := os.Open(cfg.Path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to open file=%q: %w", cfg.Path, err)
|
||||
}
|
||||
|
||||
defer fh.Close()
|
||||
|
||||
digests, err = file.NewDigestsFromFile(fh, cfg.DigestAlgorithms)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to calculate digests for file=%q: %w", cfg.Path, err)
|
||||
}
|
||||
}
|
||||
|
||||
fh, err := os.Open(cfg.Path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to open file=%q: %w", cfg.Path, err)
|
||||
}
|
||||
|
||||
defer fh.Close()
|
||||
|
||||
id, versionDigest := deriveIDFromFile(cfg)
|
||||
|
||||
return &FileSource{
|
||||
id: id,
|
||||
config: cfg,
|
||||
mutex: &sync.Mutex{},
|
||||
closer: cleanupFn,
|
||||
analysisPath: analysisPath,
|
||||
digestForVersion: versionDigest,
|
||||
digests: digests,
|
||||
mimeType: stereoFile.MIMEType(fh),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// deriveIDFromFile derives an artifact ID from the contents of a file. If an alias is provided, it will be included
|
||||
// in the ID derivation (along with contents). This way if the user scans the same item but is considered to be
|
||||
// logically different, then ID will express that.
|
||||
func deriveIDFromFile(cfg FileConfig) (artifact.ID, string) {
|
||||
d := digestOfFileContents(cfg.Path)
|
||||
info := d
|
||||
|
||||
if !cfg.Alias.IsEmpty() {
|
||||
// if the user provided an alias, we want to consider that in the artifact ID. This way if the user
|
||||
// scans the same item but is considered to be logically different, then ID will express that.
|
||||
info += fmt.Sprintf(":%s@%s", cfg.Alias.Name, cfg.Alias.Version)
|
||||
}
|
||||
|
||||
if d != "" {
|
||||
d = fmt.Sprintf("sha256:%s", d)
|
||||
}
|
||||
|
||||
return artifactIDFromDigest(digest.SHA256.FromString(info).String()), d
|
||||
}
|
||||
|
||||
func (s FileSource) ID() artifact.ID {
|
||||
return s.id
|
||||
}
|
||||
|
||||
func (s FileSource) Describe() Description {
|
||||
name := path.Base(s.config.Path)
|
||||
version := s.digestForVersion
|
||||
if !s.config.Alias.IsEmpty() {
|
||||
a := s.config.Alias
|
||||
if a.Name != "" {
|
||||
name = a.Name
|
||||
}
|
||||
|
||||
if a.Version != "" {
|
||||
version = a.Version
|
||||
}
|
||||
}
|
||||
return Description{
|
||||
ID: string(s.id),
|
||||
Name: name,
|
||||
Version: version,
|
||||
Metadata: FileSourceMetadata{
|
||||
Path: s.config.Path,
|
||||
Digests: s.digests,
|
||||
MIMEType: s.mimeType,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (s FileSource) FileResolver(_ Scope) (file.Resolver, error) {
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
|
||||
if s.resolver != nil {
|
||||
return s.resolver, nil
|
||||
}
|
||||
|
||||
exclusionFunctions, err := getDirectoryExclusionFunctions(s.analysisPath, s.config.Exclude.Paths)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fi, err := os.Stat(s.analysisPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to stat path=%q: %w", s.analysisPath, err)
|
||||
}
|
||||
isArchiveAnalysis := fi.IsDir()
|
||||
|
||||
absAnalysisPath, err := filepath.Abs(s.analysisPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to get absolute path for analysis path=%q: %w", s.analysisPath, err)
|
||||
}
|
||||
absParentDir := filepath.Dir(absAnalysisPath)
|
||||
|
||||
var res *fileresolver.Directory
|
||||
if isArchiveAnalysis {
|
||||
// this is an analysis of an archive file... we should scan the directory where the archive contents
|
||||
res, err = fileresolver.NewFromDirectory(s.analysisPath, "", exclusionFunctions...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create directory resolver: %w", err)
|
||||
}
|
||||
} else {
|
||||
// this is an analysis of a single file. We want to ultimately scan the directory that the file is in, but we
|
||||
// don't want to include any other files except this the given file.
|
||||
exclusionFunctions = append([]fileresolver.PathIndexVisitor{
|
||||
|
||||
// note: we should exclude these kinds of paths first before considering any other user-provided exclusions
|
||||
func(p string, info os.FileInfo, err error) error {
|
||||
if p == absParentDir {
|
||||
// this is the root directory... always include it
|
||||
return nil
|
||||
}
|
||||
|
||||
if filepath.Dir(p) != absParentDir {
|
||||
// we are no longer in the root directory containing the single file we want to scan...
|
||||
// we should skip the directory this path resides in entirely!
|
||||
return fs.SkipDir
|
||||
}
|
||||
|
||||
if path.Base(p) != path.Base(s.config.Path) {
|
||||
// we're in the root directory, but this is not the file we want to scan...
|
||||
// we should selectively skip this file (not the directory we're in).
|
||||
return fileresolver.ErrSkipPath
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}, exclusionFunctions...)
|
||||
|
||||
res, err = fileresolver.NewFromDirectory(absParentDir, absParentDir, exclusionFunctions...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create directory resolver: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
s.resolver = res
|
||||
|
||||
return s.resolver, nil
|
||||
}
|
||||
|
||||
func (s *FileSource) Close() error {
|
||||
if s.closer == nil {
|
||||
return nil
|
||||
}
|
||||
s.resolver = nil
|
||||
return s.closer()
|
||||
}
|
||||
|
||||
// fileAnalysisPath returns the path given, or in the case the path is an archive, the location where the archive
|
||||
// contents have been made available. A cleanup function is provided for any temp files created (if any).
|
||||
func fileAnalysisPath(path string) (string, func() error) {
|
||||
var analysisPath = path
|
||||
var cleanupFn = func() error { return nil }
|
||||
|
||||
// if the given file is an archive (as indicated by the file extension and not MIME type) then unarchive it and
|
||||
// use the contents as the source. Note: this does NOT recursively unarchive contents, only the given path is
|
||||
// unarchived.
|
||||
envelopedUnarchiver, err := archiver.ByExtension(path)
|
||||
if unarchiver, ok := envelopedUnarchiver.(archiver.Unarchiver); err == nil && ok {
|
||||
if tar, ok := unarchiver.(*archiver.Tar); ok {
|
||||
// when tar files are extracted, if there are multiple entries at the same
|
||||
// location, the last entry wins
|
||||
// NOTE: this currently does not display any messages if an overwrite happens
|
||||
tar.OverwriteExisting = true
|
||||
}
|
||||
unarchivedPath, tmpCleanup, err := unarchiveToTmp(path, unarchiver)
|
||||
if err != nil {
|
||||
log.Warnf("file could not be unarchived: %+v", err)
|
||||
} else {
|
||||
log.Debugf("source path is an archive")
|
||||
analysisPath = unarchivedPath
|
||||
}
|
||||
if tmpCleanup != nil {
|
||||
cleanupFn = tmpCleanup
|
||||
}
|
||||
}
|
||||
|
||||
return analysisPath, cleanupFn
|
||||
}
|
||||
|
||||
func digestOfFileContents(path string) string {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return digest.SHA256.FromString(path).String()
|
||||
}
|
||||
defer file.Close()
|
||||
di, err := digest.SHA256.FromReader(file)
|
||||
if err != nil {
|
||||
return digest.SHA256.FromString(path).String()
|
||||
}
|
||||
return di.String()
|
||||
}
|
||||
|
||||
func unarchiveToTmp(path string, unarchiver archiver.Unarchiver) (string, func() error, error) {
|
||||
tempDir, err := os.MkdirTemp("", "syft-archive-contents-")
|
||||
if err != nil {
|
||||
return "", func() error { return nil }, fmt.Errorf("unable to create tempdir for archive processing: %w", err)
|
||||
}
|
||||
|
||||
cleanupFn := func() error {
|
||||
return os.RemoveAll(tempDir)
|
||||
}
|
||||
|
||||
return tempDir, cleanupFn, unarchiver.Unarchive(path, tempDir)
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
package source
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
func TestNewFromFile(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
input string
|
||||
expString string
|
||||
testPathFn func(file.Resolver) ([]file.Location, error)
|
||||
expRefs int
|
||||
}{
|
||||
{
|
||||
desc: "path detected by glob",
|
||||
input: "test-fixtures/file-index-filter/.vimrc",
|
||||
testPathFn: func(resolver file.Resolver) ([]file.Location, error) {
|
||||
return resolver.FilesByGlob("**/.vimrc", "**/.2", "**/.1/*", "**/empty")
|
||||
},
|
||||
expRefs: 1,
|
||||
},
|
||||
{
|
||||
desc: "path detected by abs path",
|
||||
input: "test-fixtures/file-index-filter/.vimrc",
|
||||
testPathFn: func(resolver file.Resolver) ([]file.Location, error) {
|
||||
return resolver.FilesByPath("/.vimrc", "/.2", "/.1/something", "/empty")
|
||||
},
|
||||
expRefs: 1,
|
||||
},
|
||||
{
|
||||
desc: "path detected by relative path",
|
||||
input: "test-fixtures/file-index-filter/.vimrc",
|
||||
testPathFn: func(resolver file.Resolver) ([]file.Location, error) {
|
||||
return resolver.FilesByPath(".vimrc", "/.2", "/.1/something", "empty")
|
||||
},
|
||||
expRefs: 1,
|
||||
},
|
||||
}
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
src, err := NewFromFile(FileConfig{
|
||||
Path: test.input,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, src.Close())
|
||||
})
|
||||
|
||||
assert.Equal(t, test.input, src.Describe().Metadata.(FileSourceMetadata).Path)
|
||||
|
||||
res, err := src.FileResolver(SquashedScope)
|
||||
require.NoError(t, err)
|
||||
|
||||
refs, err := test.testPathFn(res)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, refs, test.expRefs)
|
||||
if test.expRefs == 1 {
|
||||
assert.Equal(t, path.Base(test.input), path.Base(refs[0].RealPath))
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFromFile_WithArchive(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
input string
|
||||
expString string
|
||||
inputPaths []string
|
||||
expRefs int
|
||||
layer2 bool
|
||||
contents string
|
||||
}{
|
||||
{
|
||||
desc: "path detected",
|
||||
input: "test-fixtures/path-detected",
|
||||
inputPaths: []string{"/.vimrc"},
|
||||
expRefs: 1,
|
||||
},
|
||||
{
|
||||
desc: "use first entry for duplicate paths",
|
||||
input: "test-fixtures/path-detected",
|
||||
inputPaths: []string{"/.vimrc"},
|
||||
expRefs: 1,
|
||||
layer2: true,
|
||||
contents: "Another .vimrc file",
|
||||
},
|
||||
}
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
archivePath := setupArchiveTest(t, test.input, test.layer2)
|
||||
|
||||
src, err := NewFromFile(FileConfig{
|
||||
Path: archivePath,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, src.Close())
|
||||
})
|
||||
|
||||
assert.Equal(t, archivePath, src.Describe().Metadata.(FileSourceMetadata).Path)
|
||||
|
||||
res, err := src.FileResolver(SquashedScope)
|
||||
require.NoError(t, err)
|
||||
|
||||
refs, err := res.FilesByPath(test.inputPaths...)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, refs, test.expRefs)
|
||||
|
||||
if test.contents != "" {
|
||||
reader, err := res.FileContentsByLocation(refs[0])
|
||||
require.NoError(t, err)
|
||||
|
||||
data, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, test.contents, string(data))
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// setupArchiveTest encapsulates common test setup work for tar file tests. It returns a cleanup function,
|
||||
// which should be called (typically deferred) by the caller, the path of the created tar archive, and an error,
|
||||
// which should trigger a fatal test failure in the consuming test. The returned cleanup function will never be nil
|
||||
// (even if there's an error), and it should always be called.
|
||||
func setupArchiveTest(t testing.TB, sourceDirPath string, layer2 bool) string {
|
||||
t.Helper()
|
||||
|
||||
archivePrefix, err := os.CreateTemp("", "syft-archive-TEST-")
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Cleanup(func() {
|
||||
assert.NoError(t, os.Remove(archivePrefix.Name()))
|
||||
})
|
||||
|
||||
destinationArchiveFilePath := archivePrefix.Name() + ".tar"
|
||||
t.Logf("archive path: %s", destinationArchiveFilePath)
|
||||
createArchive(t, sourceDirPath, destinationArchiveFilePath, layer2)
|
||||
|
||||
t.Cleanup(func() {
|
||||
assert.NoError(t, os.Remove(destinationArchiveFilePath))
|
||||
})
|
||||
|
||||
cwd, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Logf("running from: %s", cwd)
|
||||
|
||||
return destinationArchiveFilePath
|
||||
}
|
||||
|
||||
// createArchive creates a new archive file at destinationArchivePath based on the directory found at sourceDirPath.
|
||||
func createArchive(t testing.TB, sourceDirPath, destinationArchivePath string, layer2 bool) {
|
||||
t.Helper()
|
||||
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("unable to get cwd: %+v", err)
|
||||
}
|
||||
|
||||
cmd := exec.Command("./generate-tar-fixture-from-source-dir.sh", destinationArchivePath, path.Base(sourceDirPath))
|
||||
cmd.Dir = filepath.Join(cwd, "test-fixtures")
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatalf("unable to start generate zip fixture script: %+v", err)
|
||||
}
|
||||
|
||||
if err := cmd.Wait(); err != nil {
|
||||
if exiterr, ok := err.(*exec.ExitError); ok {
|
||||
// The program has exited with an exit code != 0
|
||||
|
||||
// This works on both Unix and Windows. Although package
|
||||
// syscall is generally platform dependent, WaitStatus is
|
||||
// defined for both Unix and Windows and in both cases has
|
||||
// an ExitStatus() method with the same signature.
|
||||
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
|
||||
if status.ExitStatus() != 0 {
|
||||
t.Fatalf("failed to generate fixture: rc=%d", status.ExitStatus())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
t.Fatalf("unable to get generate fixture script result: %+v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if layer2 {
|
||||
cmd = exec.Command("tar", "-rvf", destinationArchivePath, ".")
|
||||
cmd.Dir = filepath.Join(cwd, "test-fixtures", path.Base(sourceDirPath+"-2"))
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatalf("unable to start tar appending fixture script: %+v", err)
|
||||
}
|
||||
_ = cmd.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
func Test_FileSource_ID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg FileConfig
|
||||
want artifact.ID
|
||||
wantErr require.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
cfg: FileConfig{},
|
||||
wantErr: require.Error,
|
||||
},
|
||||
{
|
||||
name: "does not exist",
|
||||
cfg: FileConfig{
|
||||
Path: "./test-fixtures/does-not-exist",
|
||||
},
|
||||
wantErr: require.Error,
|
||||
},
|
||||
{
|
||||
name: "to dir",
|
||||
cfg: FileConfig{
|
||||
Path: "./test-fixtures/image-simple",
|
||||
},
|
||||
wantErr: require.Error,
|
||||
},
|
||||
{
|
||||
name: "with path",
|
||||
cfg: FileConfig{Path: "./test-fixtures/image-simple/Dockerfile"},
|
||||
want: artifact.ID("db7146472cf6d49b3ac01b42812fb60020b0b4898b97491b21bb690c808d5159"),
|
||||
},
|
||||
{
|
||||
name: "with path and alias",
|
||||
cfg: FileConfig{
|
||||
Path: "./test-fixtures/image-simple/Dockerfile",
|
||||
Alias: Alias{
|
||||
Name: "name-me-that!",
|
||||
Version: "version-me-this!",
|
||||
},
|
||||
},
|
||||
want: artifact.ID("3c713003305ac6605255cec8bf4ea649aa44b2b9a9f3a07bd683869d1363438a"),
|
||||
},
|
||||
{
|
||||
name: "other fields do not affect ID",
|
||||
cfg: FileConfig{
|
||||
Path: "test-fixtures/image-simple/Dockerfile",
|
||||
Exclude: ExcludeConfig{
|
||||
Paths: []string{"a", "b"},
|
||||
},
|
||||
},
|
||||
want: artifact.ID("db7146472cf6d49b3ac01b42812fb60020b0b4898b97491b21bb690c808d5159"),
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.wantErr == nil {
|
||||
tt.wantErr = require.NoError
|
||||
}
|
||||
s, err := NewFromFile(tt.cfg)
|
||||
tt.wantErr(t, err)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
assert.Equalf(t, tt.want, s.ID(), "ID()")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package source
|
||||
|
||||
import "github.com/anchore/stereoscope/pkg/image"
|
||||
|
||||
// ImageMetadata represents all static metadata that defines what a container image is. This is useful to later describe
|
||||
// "what" was cataloged without needing the more complicated stereoscope Image objects or FileResolver objects.
|
||||
type ImageMetadata struct {
|
||||
UserInput string `json:"userInput"`
|
||||
ID string `json:"imageID"`
|
||||
ManifestDigest string `json:"manifestDigest"`
|
||||
MediaType string `json:"mediaType"`
|
||||
Tags []string `json:"tags"`
|
||||
Size int64 `json:"imageSize"`
|
||||
Layers []LayerMetadata `json:"layers"`
|
||||
RawManifest []byte `json:"manifest"`
|
||||
RawConfig []byte `json:"config"`
|
||||
RepoDigests []string `json:"repoDigests"`
|
||||
Architecture string `json:"architecture"`
|
||||
Variant string `json:"architectureVariant,omitempty"`
|
||||
OS string `json:"os"`
|
||||
}
|
||||
|
||||
// LayerMetadata represents all static metadata that defines what a container image layer is.
|
||||
type LayerMetadata struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// NewImageMetadata creates a new ImageMetadata object populated from the given stereoscope Image object and user configuration.
|
||||
func NewImageMetadata(img *image.Image, userInput string) ImageMetadata {
|
||||
// populate artifacts...
|
||||
tags := make([]string, len(img.Metadata.Tags))
|
||||
for idx, tag := range img.Metadata.Tags {
|
||||
tags[idx] = tag.String()
|
||||
}
|
||||
theImg := ImageMetadata{
|
||||
ID: img.Metadata.ID,
|
||||
UserInput: userInput,
|
||||
ManifestDigest: img.Metadata.ManifestDigest,
|
||||
Size: img.Metadata.Size,
|
||||
MediaType: string(img.Metadata.MediaType),
|
||||
Tags: tags,
|
||||
Layers: make([]LayerMetadata, len(img.Layers)),
|
||||
RawConfig: img.Metadata.RawConfig,
|
||||
RawManifest: img.Metadata.RawManifest,
|
||||
RepoDigests: img.Metadata.RepoDigests,
|
||||
Architecture: img.Metadata.Architecture,
|
||||
Variant: img.Metadata.Variant,
|
||||
OS: img.Metadata.OS,
|
||||
}
|
||||
|
||||
// populate image metadata
|
||||
for idx, l := range img.Layers {
|
||||
theImg.Layers[idx] = LayerMetadata{
|
||||
MediaType: string(l.Metadata.MediaType),
|
||||
Digest: l.Metadata.Digest,
|
||||
Size: l.Metadata.Size,
|
||||
}
|
||||
}
|
||||
return theImg
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
package source
|
||||
|
||||
// Metadata represents any static source data that helps describe "what" was cataloged.
|
||||
type Metadata struct {
|
||||
ID string `hash:"ignore"` // the id generated from the parent source struct
|
||||
Scheme Scheme // the source data scheme type (directory or image)
|
||||
ImageMetadata ImageMetadata // all image info (image only)
|
||||
Path string // the root path to be cataloged (directory only)
|
||||
Base string // the base path to be cataloged (directory only)
|
||||
Name string
|
||||
Version string
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package source
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/mitchellh/go-homedir"
|
||||
"github.com/spf13/afero"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
)
|
||||
|
||||
// Scheme represents the optional prefixed string at the beginning of a user request (e.g. "docker:").
|
||||
type Scheme string
|
||||
|
||||
const (
|
||||
// UnknownScheme is the default scheme
|
||||
UnknownScheme Scheme = "UnknownScheme"
|
||||
// DirectoryScheme indicates the source being cataloged is a directory on the root filesystem
|
||||
DirectoryScheme Scheme = "DirectoryScheme"
|
||||
// ImageScheme indicates the source being cataloged is a container image
|
||||
ImageScheme Scheme = "ImageScheme"
|
||||
// FileScheme indicates the source being cataloged is a single file
|
||||
FileScheme Scheme = "FileScheme"
|
||||
)
|
||||
|
||||
var AllSchemes = []Scheme{
|
||||
DirectoryScheme,
|
||||
ImageScheme,
|
||||
FileScheme,
|
||||
}
|
||||
|
||||
func DetectScheme(fs afero.Fs, imageDetector sourceDetector, userInput string) (Scheme, image.Source, string, error) {
|
||||
switch {
|
||||
case strings.HasPrefix(userInput, "dir:"):
|
||||
dirLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "dir:"))
|
||||
if err != nil {
|
||||
return UnknownScheme, image.UnknownSource, "", fmt.Errorf("unable to expand directory path: %w", err)
|
||||
}
|
||||
return DirectoryScheme, image.UnknownSource, dirLocation, nil
|
||||
|
||||
case strings.HasPrefix(userInput, "file:"):
|
||||
fileLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "file:"))
|
||||
if err != nil {
|
||||
return UnknownScheme, image.UnknownSource, "", fmt.Errorf("unable to expand directory path: %w", err)
|
||||
}
|
||||
return FileScheme, image.UnknownSource, fileLocation, nil
|
||||
}
|
||||
|
||||
// try the most specific sources first and move out towards more generic sources.
|
||||
|
||||
// first: let's try the image detector, which has more scheme parsing internal to stereoscope
|
||||
source, imageSpec, err := imageDetector(userInput)
|
||||
if err == nil && source != image.UnknownSource {
|
||||
return ImageScheme, source, imageSpec, nil
|
||||
}
|
||||
|
||||
// next: let's try more generic sources (dir, file, etc.)
|
||||
location, err := homedir.Expand(userInput)
|
||||
if err != nil {
|
||||
return UnknownScheme, image.UnknownSource, "", fmt.Errorf("unable to expand potential directory path: %w", err)
|
||||
}
|
||||
|
||||
fileMeta, err := fs.Stat(location)
|
||||
if err != nil {
|
||||
return UnknownScheme, source, "", nil
|
||||
}
|
||||
|
||||
if fileMeta.IsDir() {
|
||||
return DirectoryScheme, source, location, nil
|
||||
}
|
||||
|
||||
return FileScheme, source, location, nil
|
||||
}
|
||||
@@ -10,7 +10,7 @@ const (
|
||||
UnknownScope Scope = "UnknownScope"
|
||||
// SquashedScope indicates to only catalog content visible from the squashed filesystem representation (what can be seen only within the container at runtime)
|
||||
SquashedScope Scope = "Squashed"
|
||||
// AllLayersScope indicates to catalog content on all layers, irregardless if it is visible from the container at runtime.
|
||||
// AllLayersScope indicates to catalog content on all layers, regardless if it is visible from the container at runtime.
|
||||
AllLayersScope Scope = "AllLayers"
|
||||
)
|
||||
|
||||
|
||||
+20
-606
@@ -6,628 +6,42 @@ within this package.
|
||||
package source
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
"github.com/mholt/archiver/v3"
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
"github.com/spf13/afero"
|
||||
|
||||
"github.com/anchore/stereoscope"
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/internal/fileresolver"
|
||||
)
|
||||
|
||||
// Source is an object that captures the data source to be cataloged, configuration, and a specific resolver used
|
||||
// in cataloging (based on the data source and configuration)
|
||||
type Source struct {
|
||||
id artifact.ID `hash:"ignore"`
|
||||
Image *image.Image `hash:"ignore"` // the image object to be cataloged (image only)
|
||||
Metadata Metadata
|
||||
directoryResolver *fileresolver.Directory `hash:"ignore"`
|
||||
path string
|
||||
base string
|
||||
mutex *sync.Mutex
|
||||
Exclusions []string `hash:"ignore"`
|
||||
type Source interface {
|
||||
artifact.Identifiable
|
||||
FileResolver(Scope) (file.Resolver, error)
|
||||
Describe() Description
|
||||
io.Closer
|
||||
}
|
||||
|
||||
// Input is an object that captures the detected user input regarding source location, scheme, and provider type.
|
||||
// It acts as a struct input for some source constructors.
|
||||
type Input struct {
|
||||
UserInput string
|
||||
Scheme Scheme
|
||||
ImageSource image.Source
|
||||
Location string
|
||||
Platform string
|
||||
Name string
|
||||
Version string
|
||||
type emptySource struct {
|
||||
description Description
|
||||
}
|
||||
|
||||
// ParseInput generates a source Input that can be used as an argument to generate a new source
|
||||
// from specific providers including a registry.
|
||||
func ParseInput(userInput string, platform string) (*Input, error) {
|
||||
return ParseInputWithName(userInput, platform, "", "")
|
||||
}
|
||||
|
||||
// ParseInputWithName generates a source Input that can be used as an argument to generate a new source
|
||||
// from specific providers including a registry, with an explicit name.
|
||||
func ParseInputWithName(userInput string, platform, name, defaultImageSource string) (*Input, error) {
|
||||
return ParseInputWithNameVersion(userInput, platform, name, "", defaultImageSource)
|
||||
}
|
||||
|
||||
// ParseInputWithNameVersion generates a source Input that can be used as an argument to generate a new source
|
||||
// from specific providers including a registry, with an explicit name and version.
|
||||
func ParseInputWithNameVersion(userInput, platform, name, version, defaultImageSource string) (*Input, error) {
|
||||
fs := afero.NewOsFs()
|
||||
scheme, source, location, err := DetectScheme(fs, image.DetectSource, userInput)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if source == image.UnknownSource {
|
||||
// only run for these two scheme
|
||||
// only check on packages command, attest we automatically try to pull from userInput
|
||||
switch scheme {
|
||||
case ImageScheme, UnknownScheme:
|
||||
scheme = ImageScheme
|
||||
location = userInput
|
||||
if defaultImageSource != "" {
|
||||
source = parseDefaultImageSource(defaultImageSource)
|
||||
} else {
|
||||
imagePullSource := image.DetermineDefaultImagePullSource(userInput)
|
||||
source = imagePullSource
|
||||
}
|
||||
if location == "" {
|
||||
location = userInput
|
||||
}
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
if scheme != ImageScheme && platform != "" {
|
||||
return nil, fmt.Errorf("cannot specify a platform for a non-image source")
|
||||
}
|
||||
|
||||
// collect user input for downstream consumption
|
||||
return &Input{
|
||||
UserInput: userInput,
|
||||
Scheme: scheme,
|
||||
ImageSource: source,
|
||||
Location: location,
|
||||
Platform: platform,
|
||||
Name: name,
|
||||
Version: version,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseDefaultImageSource(defaultImageSource string) image.Source {
|
||||
switch defaultImageSource {
|
||||
case "registry":
|
||||
return image.OciRegistrySource
|
||||
case "docker":
|
||||
return image.DockerDaemonSource
|
||||
case "podman":
|
||||
return image.PodmanDaemonSource
|
||||
default:
|
||||
return image.UnknownSource
|
||||
func FromDescription(d Description) Source {
|
||||
return &emptySource{
|
||||
description: d,
|
||||
}
|
||||
}
|
||||
|
||||
type sourceDetector func(string) (image.Source, string, error)
|
||||
|
||||
func NewFromRegistry(in Input, registryOptions *image.RegistryOptions, exclusions []string) (*Source, func(), error) {
|
||||
source, cleanupFn, err := generateImageSource(in, registryOptions)
|
||||
if source != nil {
|
||||
source.Exclusions = exclusions
|
||||
}
|
||||
return source, cleanupFn, err
|
||||
func (e emptySource) ID() artifact.ID {
|
||||
return artifact.ID(e.description.ID)
|
||||
}
|
||||
|
||||
// New produces a Source based on userInput like dir: or image:tag
|
||||
func New(in Input, registryOptions *image.RegistryOptions, exclusions []string) (*Source, func(), error) {
|
||||
var err error
|
||||
fs := afero.NewOsFs()
|
||||
var source *Source
|
||||
cleanupFn := func() {}
|
||||
|
||||
switch in.Scheme {
|
||||
case FileScheme:
|
||||
source, cleanupFn, err = generateFileSource(fs, in)
|
||||
case DirectoryScheme:
|
||||
source, cleanupFn, err = generateDirectorySource(fs, in)
|
||||
case ImageScheme:
|
||||
source, cleanupFn, err = generateImageSource(in, registryOptions)
|
||||
default:
|
||||
err = fmt.Errorf("unable to process input for scanning: %q", in.UserInput)
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
source.Exclusions = exclusions
|
||||
}
|
||||
|
||||
return source, cleanupFn, err
|
||||
func (e emptySource) FileResolver(_ Scope) (file.Resolver, error) {
|
||||
return nil, errors.New("no file resolver available for description-only source")
|
||||
}
|
||||
|
||||
func generateImageSource(in Input, registryOptions *image.RegistryOptions) (*Source, func(), error) {
|
||||
img, cleanup, err := getImageWithRetryStrategy(in, registryOptions)
|
||||
if err != nil || img == nil {
|
||||
return nil, cleanup, fmt.Errorf("could not fetch image %q: %w", in.Location, err)
|
||||
}
|
||||
|
||||
s, err := NewFromImageWithNameVersion(img, in.Location, in.Name, in.Version)
|
||||
if err != nil {
|
||||
return nil, cleanup, fmt.Errorf("could not populate source with image: %w", err)
|
||||
}
|
||||
|
||||
return &s, cleanup, nil
|
||||
func (e emptySource) Describe() Description {
|
||||
return e.description
|
||||
}
|
||||
|
||||
func parseScheme(userInput string) string {
|
||||
parts := strings.SplitN(userInput, ":", 2)
|
||||
if len(parts) < 2 {
|
||||
return ""
|
||||
}
|
||||
|
||||
return parts[0]
|
||||
}
|
||||
|
||||
func getImageWithRetryStrategy(in Input, registryOptions *image.RegistryOptions) (*image.Image, func(), error) {
|
||||
ctx := context.TODO()
|
||||
|
||||
var opts []stereoscope.Option
|
||||
if registryOptions != nil {
|
||||
opts = append(opts, stereoscope.WithRegistryOptions(*registryOptions))
|
||||
}
|
||||
|
||||
if in.Platform != "" {
|
||||
opts = append(opts, stereoscope.WithPlatform(in.Platform))
|
||||
}
|
||||
|
||||
img, err := stereoscope.GetImageFromSource(ctx, in.Location, in.ImageSource, opts...)
|
||||
cleanup := func() {
|
||||
if err := img.Cleanup(); err != nil {
|
||||
log.Warnf("unable to cleanup image=%q: %w", in.UserInput, err)
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
// Success on the first try!
|
||||
return img, cleanup, nil
|
||||
}
|
||||
|
||||
scheme := parseScheme(in.UserInput)
|
||||
if !(scheme == "docker" || scheme == "registry") {
|
||||
// Image retrieval failed, and we shouldn't retry it. It's most likely that the
|
||||
// user _did_ intend the parsed scheme, but there was a legitimate failure with
|
||||
// using the scheme to load the image. Alert the user to this failure, so they
|
||||
// can fix the problem.
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Maybe the user wanted "docker" or "registry" to refer to an _image name_
|
||||
// (e.g. "docker:latest"), not a scheme. We'll retry image retrieval with this
|
||||
// alternative interpretation, in an attempt to avoid unnecessary user friction.
|
||||
|
||||
log.Warnf(
|
||||
"scheme %q specified, but it coincides with a common image name; re-examining user input %q"+
|
||||
" without scheme parsing because image retrieval using scheme parsing was unsuccessful: %v",
|
||||
scheme,
|
||||
in.UserInput,
|
||||
err,
|
||||
)
|
||||
|
||||
// We need to determine the image source again, such that this determination
|
||||
// doesn't take scheme parsing into account.
|
||||
in.ImageSource = image.DetermineDefaultImagePullSource(in.UserInput)
|
||||
img, userInputErr := stereoscope.GetImageFromSource(ctx, in.UserInput, in.ImageSource, opts...)
|
||||
cleanup = func() {
|
||||
if err := img.Cleanup(); err != nil {
|
||||
log.Warnf("unable to cleanup image=%q: %w", in.UserInput, err)
|
||||
}
|
||||
}
|
||||
if userInputErr != nil {
|
||||
// Image retrieval failed on both tries, we will want to return both errors.
|
||||
return nil, nil, fmt.Errorf(
|
||||
"scheme %q specified; "+
|
||||
"image retrieval using scheme parsing (%s) was unsuccessful: %v; "+
|
||||
"image retrieval without scheme parsing (%s) was unsuccessful: %v",
|
||||
scheme,
|
||||
in.Location,
|
||||
err,
|
||||
in.UserInput,
|
||||
userInputErr,
|
||||
)
|
||||
}
|
||||
|
||||
return img, cleanup, nil
|
||||
}
|
||||
|
||||
func generateDirectorySource(fs afero.Fs, in Input) (*Source, func(), error) {
|
||||
fileMeta, err := fs.Stat(in.Location)
|
||||
if err != nil {
|
||||
return nil, func() {}, fmt.Errorf("unable to stat dir=%q: %w", in.Location, err)
|
||||
}
|
||||
|
||||
if !fileMeta.IsDir() {
|
||||
return nil, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", in.Location, err)
|
||||
}
|
||||
|
||||
s, err := NewFromDirectoryWithNameVersion(in.Location, in.Name, in.Version)
|
||||
if err != nil {
|
||||
return nil, func() {}, fmt.Errorf("could not populate source from path=%q: %w", in.Location, err)
|
||||
}
|
||||
|
||||
return &s, func() {}, nil
|
||||
}
|
||||
|
||||
func generateFileSource(fs afero.Fs, in Input) (*Source, func(), error) {
|
||||
fileMeta, err := fs.Stat(in.Location)
|
||||
if err != nil {
|
||||
return nil, func() {}, fmt.Errorf("unable to stat dir=%q: %w", in.Location, err)
|
||||
}
|
||||
|
||||
if fileMeta.IsDir() {
|
||||
return nil, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", in.Location, err)
|
||||
}
|
||||
|
||||
s, cleanupFn := NewFromFileWithNameVersion(in.Location, in.Name, in.Version)
|
||||
|
||||
return &s, cleanupFn, nil
|
||||
}
|
||||
|
||||
// NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively.
|
||||
func NewFromDirectory(path string) (Source, error) {
|
||||
return NewFromDirectoryWithName(path, "")
|
||||
}
|
||||
|
||||
// NewFromDirectoryWithName creates a new source object tailored to catalog a given filesystem directory recursively, with an explicitly provided name.
|
||||
func NewFromDirectoryWithName(path string, name string) (Source, error) {
|
||||
return NewFromDirectoryWithNameVersion(path, name, "")
|
||||
}
|
||||
|
||||
// NewFromDirectoryWithNameVersion creates a new source object tailored to catalog a given filesystem directory recursively, with an explicitly provided name.
|
||||
func NewFromDirectoryWithNameVersion(path string, name string, version string) (Source, error) {
|
||||
s := Source{
|
||||
mutex: &sync.Mutex{},
|
||||
Metadata: Metadata{
|
||||
Name: name,
|
||||
Version: version,
|
||||
Scheme: DirectoryScheme,
|
||||
Path: path,
|
||||
},
|
||||
path: path,
|
||||
}
|
||||
s.SetID()
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// NewFromDirectoryRoot creates a new source object tailored to catalog a given filesystem directory recursively.
|
||||
func NewFromDirectoryRoot(path string) (Source, error) {
|
||||
return NewFromDirectoryRootWithName(path, "")
|
||||
}
|
||||
|
||||
// NewFromDirectoryRootWithName creates a new source object tailored to catalog a given filesystem directory recursively, with an explicitly provided name.
|
||||
func NewFromDirectoryRootWithName(path string, name string) (Source, error) {
|
||||
return NewFromDirectoryRootWithNameVersion(path, name, "")
|
||||
}
|
||||
|
||||
// NewFromDirectoryRootWithNameVersion creates a new source object tailored to catalog a given filesystem directory recursively, with an explicitly provided name.
|
||||
func NewFromDirectoryRootWithNameVersion(path string, name string, version string) (Source, error) {
|
||||
s := Source{
|
||||
mutex: &sync.Mutex{},
|
||||
Metadata: Metadata{
|
||||
Name: name,
|
||||
Version: version,
|
||||
Scheme: DirectoryScheme,
|
||||
Path: path,
|
||||
Base: path,
|
||||
},
|
||||
path: path,
|
||||
base: path,
|
||||
}
|
||||
s.SetID()
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// NewFromFile creates a new source object tailored to catalog a file.
|
||||
func NewFromFile(path string) (Source, func()) {
|
||||
return NewFromFileWithName(path, "")
|
||||
}
|
||||
|
||||
// NewFromFileWithName creates a new source object tailored to catalog a file, with an explicitly provided name.
|
||||
func NewFromFileWithName(path string, name string) (Source, func()) {
|
||||
return NewFromFileWithNameVersion(path, name, "")
|
||||
}
|
||||
|
||||
// NewFromFileWithNameVersion creates a new source object tailored to catalog a file, with an explicitly provided name and version.
|
||||
func NewFromFileWithNameVersion(path string, name string, version string) (Source, func()) {
|
||||
analysisPath, cleanupFn := fileAnalysisPath(path)
|
||||
|
||||
s := Source{
|
||||
mutex: &sync.Mutex{},
|
||||
Metadata: Metadata{
|
||||
Name: name,
|
||||
Version: version,
|
||||
Scheme: FileScheme,
|
||||
Path: path,
|
||||
},
|
||||
path: analysisPath,
|
||||
}
|
||||
|
||||
s.SetID()
|
||||
return s, cleanupFn
|
||||
}
|
||||
|
||||
// fileAnalysisPath returns the path given, or in the case the path is an archive, the location where the archive
|
||||
// contents have been made available. A cleanup function is provided for any temp files created (if any).
|
||||
func fileAnalysisPath(path string) (string, func()) {
|
||||
var analysisPath = path
|
||||
var cleanupFn = func() {}
|
||||
|
||||
// if the given file is an archive (as indicated by the file extension and not MIME type) then unarchive it and
|
||||
// use the contents as the source. Note: this does NOT recursively unarchive contents, only the given path is
|
||||
// unarchived.
|
||||
envelopedUnarchiver, err := archiver.ByExtension(path)
|
||||
if unarchiver, ok := envelopedUnarchiver.(archiver.Unarchiver); err == nil && ok {
|
||||
if tar, ok := unarchiver.(*archiver.Tar); ok {
|
||||
// when tar files are extracted, if there are multiple entries at the same
|
||||
// location, the last entry wins
|
||||
// NOTE: this currently does not display any messages if an overwrite happens
|
||||
tar.OverwriteExisting = true
|
||||
}
|
||||
unarchivedPath, tmpCleanup, err := unarchiveToTmp(path, unarchiver)
|
||||
if err != nil {
|
||||
log.Warnf("file could not be unarchived: %+v", err)
|
||||
} else {
|
||||
log.Debugf("source path is an archive")
|
||||
analysisPath = unarchivedPath
|
||||
}
|
||||
if tmpCleanup != nil {
|
||||
cleanupFn = tmpCleanup
|
||||
}
|
||||
}
|
||||
|
||||
return analysisPath, cleanupFn
|
||||
}
|
||||
|
||||
// NewFromImage creates a new source object tailored to catalog a given container image, relative to the
|
||||
// option given (e.g. all-layers, squashed, etc)
|
||||
func NewFromImage(img *image.Image, userImageStr string) (Source, error) {
|
||||
return NewFromImageWithName(img, userImageStr, "")
|
||||
}
|
||||
|
||||
// NewFromImageWithName creates a new source object tailored to catalog a given container image, relative to the
|
||||
// option given (e.g. all-layers, squashed, etc), with an explicit name.
|
||||
func NewFromImageWithName(img *image.Image, userImageStr string, name string) (Source, error) {
|
||||
return NewFromImageWithNameVersion(img, userImageStr, name, "")
|
||||
}
|
||||
|
||||
// NewFromImageWithNameVersion creates a new source object tailored to catalog a given container image, relative to the
|
||||
// option given (e.g. all-layers, squashed, etc), with an explicit name and version.
|
||||
func NewFromImageWithNameVersion(img *image.Image, userImageStr string, name string, version string) (Source, error) {
|
||||
if img == nil {
|
||||
return Source{}, fmt.Errorf("no image given")
|
||||
}
|
||||
|
||||
s := Source{
|
||||
Image: img,
|
||||
Metadata: Metadata{
|
||||
Name: name,
|
||||
Version: version,
|
||||
Scheme: ImageScheme,
|
||||
ImageMetadata: NewImageMetadata(img, userImageStr),
|
||||
},
|
||||
}
|
||||
s.SetID()
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *Source) ID() artifact.ID {
|
||||
if s.id == "" {
|
||||
s.SetID()
|
||||
}
|
||||
return s.id
|
||||
}
|
||||
|
||||
func (s *Source) SetID() {
|
||||
var d string
|
||||
switch s.Metadata.Scheme {
|
||||
case DirectoryScheme:
|
||||
d = digest.FromString(s.Metadata.Path).String()
|
||||
case FileScheme:
|
||||
// attempt to use the digest of the contents of the file as the ID
|
||||
file, err := os.Open(s.Metadata.Path)
|
||||
if err != nil {
|
||||
d = digest.FromString(s.Metadata.Path).String()
|
||||
break
|
||||
}
|
||||
defer file.Close()
|
||||
di, err := digest.FromReader(file)
|
||||
if err != nil {
|
||||
d = digest.FromString(s.Metadata.Path).String()
|
||||
break
|
||||
}
|
||||
d = di.String()
|
||||
case ImageScheme:
|
||||
manifestDigest := digest.FromBytes(s.Metadata.ImageMetadata.RawManifest).String()
|
||||
if manifestDigest != "" {
|
||||
d = manifestDigest
|
||||
break
|
||||
}
|
||||
|
||||
// calcuate chain ID for image sources where manifestDigest is not available
|
||||
// https://github.com/opencontainers/image-spec/blob/main/config.md#layer-chainid
|
||||
d = calculateChainID(s.Metadata.ImageMetadata.Layers)
|
||||
if d == "" {
|
||||
// TODO what happens here if image has no layers?
|
||||
// Is this case possible
|
||||
d = digest.FromString(s.Metadata.ImageMetadata.UserInput).String()
|
||||
}
|
||||
default: // for UnknownScheme we hash the struct
|
||||
id, _ := artifact.IDByHash(s)
|
||||
d = string(id)
|
||||
}
|
||||
|
||||
s.id = artifact.ID(strings.TrimPrefix(d, "sha256:"))
|
||||
s.Metadata.ID = strings.TrimPrefix(d, "sha256:")
|
||||
}
|
||||
|
||||
func calculateChainID(lm []LayerMetadata) string {
|
||||
if len(lm) < 1 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// DiffID(L0) = digest of layer 0
|
||||
// https://github.com/anchore/stereoscope/blob/1b1b744a919964f38d14e1416fb3f25221b761ce/pkg/image/layer_metadata.go#L19-L32
|
||||
chainID := lm[0].Digest
|
||||
id := chain(chainID, lm[1:])
|
||||
|
||||
return id
|
||||
}
|
||||
|
||||
func chain(chainID string, layers []LayerMetadata) string {
|
||||
if len(layers) < 1 {
|
||||
return chainID
|
||||
}
|
||||
|
||||
chainID = digest.FromString(layers[0].Digest + " " + chainID).String()
|
||||
return chain(chainID, layers[1:])
|
||||
}
|
||||
|
||||
func (s *Source) FileResolver(scope Scope) (file.Resolver, error) {
|
||||
switch s.Metadata.Scheme {
|
||||
case DirectoryScheme, FileScheme:
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
if s.directoryResolver == nil {
|
||||
exclusionFunctions, err := getDirectoryExclusionFunctions(s.path, s.Exclusions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err := fileresolver.NewFromDirectory(s.path, s.base, exclusionFunctions...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create directory resolver: %w", err)
|
||||
}
|
||||
s.directoryResolver = res
|
||||
}
|
||||
return s.directoryResolver, nil
|
||||
case ImageScheme:
|
||||
var res file.Resolver
|
||||
var err error
|
||||
switch scope {
|
||||
case SquashedScope:
|
||||
res, err = fileresolver.NewFromContainerImageSquash(s.Image)
|
||||
case AllLayersScope:
|
||||
res, err = fileresolver.NewFromContainerImageAllLayers(s.Image)
|
||||
default:
|
||||
return nil, fmt.Errorf("bad image scope provided: %+v", scope)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// image tree contains all paths, so we filter out the excluded entries afterwards
|
||||
if len(s.Exclusions) > 0 {
|
||||
res = fileresolver.NewExcluding(res, getImageExclusionFunction(s.Exclusions))
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unable to determine FilePathResolver with current scheme=%q", s.Metadata.Scheme)
|
||||
}
|
||||
|
||||
func unarchiveToTmp(path string, unarchiver archiver.Unarchiver) (string, func(), error) {
|
||||
tempDir, err := os.MkdirTemp("", "syft-archive-contents-")
|
||||
if err != nil {
|
||||
return "", func() {}, fmt.Errorf("unable to create tempdir for archive processing: %w", err)
|
||||
}
|
||||
|
||||
cleanupFn := func() {
|
||||
if err := os.RemoveAll(tempDir); err != nil {
|
||||
log.Warnf("unable to cleanup archive tempdir: %+v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return tempDir, cleanupFn, unarchiver.Unarchive(path, tempDir)
|
||||
}
|
||||
|
||||
func getImageExclusionFunction(exclusions []string) func(string) bool {
|
||||
if len(exclusions) == 0 {
|
||||
return nil
|
||||
}
|
||||
// add subpath exclusions
|
||||
for _, exclusion := range exclusions {
|
||||
exclusions = append(exclusions, exclusion+"/**")
|
||||
}
|
||||
return func(path string) bool {
|
||||
for _, exclusion := range exclusions {
|
||||
matches, err := doublestar.Match(exclusion, path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if matches {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func getDirectoryExclusionFunctions(root string, exclusions []string) ([]fileresolver.PathIndexVisitor, error) {
|
||||
if len(exclusions) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// this is what Directory.indexTree is doing to get the absolute path:
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// this handles Windows file paths by converting them to C:/something/else format
|
||||
root = filepath.ToSlash(root)
|
||||
|
||||
if !strings.HasSuffix(root, "/") {
|
||||
root += "/"
|
||||
}
|
||||
|
||||
var errors []string
|
||||
for idx, exclusion := range exclusions {
|
||||
// check exclusions for supported paths, these are all relative to the "scan root"
|
||||
if strings.HasPrefix(exclusion, "./") || strings.HasPrefix(exclusion, "*/") || strings.HasPrefix(exclusion, "**/") {
|
||||
exclusion = strings.TrimPrefix(exclusion, "./")
|
||||
exclusions[idx] = root + exclusion
|
||||
} else {
|
||||
errors = append(errors, exclusion)
|
||||
}
|
||||
}
|
||||
|
||||
if errors != nil {
|
||||
return nil, fmt.Errorf("invalid exclusion pattern(s): '%s' (must start with one of: './', '*/', or '**/')", strings.Join(errors, "', '"))
|
||||
}
|
||||
|
||||
return []fileresolver.PathIndexVisitor{
|
||||
func(path string, info os.FileInfo, _ error) error {
|
||||
for _, exclusion := range exclusions {
|
||||
// this is required to handle Windows filepaths
|
||||
path = filepath.ToSlash(path)
|
||||
matches, err := doublestar.Match(exclusion, path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if matches {
|
||||
if info != nil && info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return fileresolver.ErrSkipPath
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}, nil
|
||||
func (e emptySource) Close() error {
|
||||
return nil // no-op
|
||||
}
|
||||
|
||||
@@ -1,920 +0,0 @@
|
||||
//go:build !windows
|
||||
// +build !windows
|
||||
|
||||
package source
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
"github.com/anchore/stereoscope/pkg/imagetest"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/internal/fileresolver"
|
||||
)
|
||||
|
||||
func TestParseInput(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
platform string
|
||||
expected Scheme
|
||||
errFn require.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "ParseInput parses a file input",
|
||||
input: "test-fixtures/image-simple/file-1.txt",
|
||||
expected: FileScheme,
|
||||
},
|
||||
{
|
||||
name: "errors out when using platform for non-image scheme",
|
||||
input: "test-fixtures/image-simple/file-1.txt",
|
||||
platform: "arm64",
|
||||
errFn: require.Error,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if test.errFn == nil {
|
||||
test.errFn = require.NoError
|
||||
}
|
||||
sourceInput, err := ParseInput(test.input, test.platform)
|
||||
test.errFn(t, err)
|
||||
if test.expected != "" {
|
||||
require.NotNil(t, sourceInput)
|
||||
assert.Equal(t, sourceInput.Scheme, test.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFromImageFails(t *testing.T) {
|
||||
t.Run("no image given", func(t *testing.T) {
|
||||
_, err := NewFromImage(nil, "")
|
||||
if err == nil {
|
||||
t.Errorf("expected an error condition but none was given")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestSetID(t *testing.T) {
|
||||
layer := image.NewLayer(nil)
|
||||
layer.Metadata = image.LayerMetadata{
|
||||
Digest: "sha256:6f4fb385d4e698647bf2a450749dfbb7bc2831ec9a730ef4046c78c08d468e89",
|
||||
}
|
||||
img := image.Image{
|
||||
Layers: []*image.Layer{layer},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input *Source
|
||||
expected artifact.ID
|
||||
}{
|
||||
{
|
||||
name: "source.SetID sets the ID for FileScheme",
|
||||
input: &Source{
|
||||
Metadata: Metadata{
|
||||
Scheme: FileScheme,
|
||||
Path: "test-fixtures/image-simple/file-1.txt",
|
||||
},
|
||||
},
|
||||
expected: artifact.ID("55096713247489add592ce977637be868497132b36d1e294a3831925ec64319a"),
|
||||
},
|
||||
{
|
||||
name: "source.SetID sets the ID for ImageScheme",
|
||||
input: &Source{
|
||||
Image: &img,
|
||||
Metadata: Metadata{
|
||||
Scheme: ImageScheme,
|
||||
},
|
||||
},
|
||||
expected: artifact.ID("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"),
|
||||
},
|
||||
{
|
||||
name: "source.SetID sets the ID for DirectoryScheme",
|
||||
input: &Source{
|
||||
Image: &img,
|
||||
Metadata: Metadata{
|
||||
Scheme: DirectoryScheme,
|
||||
Path: "test-fixtures/image-simple",
|
||||
},
|
||||
},
|
||||
expected: artifact.ID("91db61e5e0ae097ef764796ce85e442a93f2a03e5313d4c7307e9b413f62e8c4"),
|
||||
},
|
||||
{
|
||||
name: "source.SetID sets the ID for UnknownScheme",
|
||||
input: &Source{
|
||||
Image: &img,
|
||||
Metadata: Metadata{
|
||||
Scheme: UnknownScheme,
|
||||
Path: "test-fixtures/image-simple",
|
||||
},
|
||||
},
|
||||
expected: artifact.ID("9ee9e786412d6ae5"),
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
test.input.SetID()
|
||||
assert.Equal(t, test.expected, test.input.ID())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFromImage(t *testing.T) {
|
||||
layer := image.NewLayer(nil)
|
||||
img := image.Image{
|
||||
Layers: []*image.Layer{layer},
|
||||
}
|
||||
|
||||
t.Run("create a new source object from image", func(t *testing.T) {
|
||||
_, err := NewFromImage(&img, "")
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error when creating a new Locations from img: %+v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestNewFromDirectory(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
input string
|
||||
expString string
|
||||
inputPaths []string
|
||||
expectedRefs int
|
||||
expectedErr bool
|
||||
}{
|
||||
{
|
||||
desc: "no paths exist",
|
||||
input: "foobar/",
|
||||
inputPaths: []string{"/opt/", "/other"},
|
||||
expectedErr: true,
|
||||
},
|
||||
{
|
||||
desc: "path detected",
|
||||
input: "test-fixtures",
|
||||
inputPaths: []string{"path-detected/.vimrc"},
|
||||
expectedRefs: 1,
|
||||
},
|
||||
{
|
||||
desc: "directory ignored",
|
||||
input: "test-fixtures",
|
||||
inputPaths: []string{"path-detected"},
|
||||
expectedRefs: 0,
|
||||
},
|
||||
{
|
||||
desc: "no files-by-path detected",
|
||||
input: "test-fixtures",
|
||||
inputPaths: []string{"no-path-detected"},
|
||||
expectedRefs: 0,
|
||||
},
|
||||
}
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
src, err := NewFromDirectory(test.input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, test.input, src.Metadata.Path)
|
||||
|
||||
res, err := src.FileResolver(SquashedScope)
|
||||
if test.expectedErr {
|
||||
if err == nil {
|
||||
t.Fatal("expected an error when making the resolver but got none")
|
||||
}
|
||||
return
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
refs, err := res.FilesByPath(test.inputPaths...)
|
||||
if err != nil {
|
||||
t.Errorf("FilesByPath call produced an error: %+v", err)
|
||||
}
|
||||
if len(refs) != test.expectedRefs {
|
||||
t.Errorf("unexpected number of refs returned: %d != %d", len(refs), test.expectedRefs)
|
||||
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFromFile(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
input string
|
||||
expString string
|
||||
inputPaths []string
|
||||
expRefs int
|
||||
}{
|
||||
{
|
||||
desc: "path detected",
|
||||
input: "test-fixtures/path-detected",
|
||||
inputPaths: []string{"/.vimrc"},
|
||||
expRefs: 1,
|
||||
},
|
||||
}
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
src, cleanup := NewFromFile(test.input)
|
||||
if cleanup != nil {
|
||||
t.Cleanup(cleanup)
|
||||
}
|
||||
|
||||
assert.Equal(t, test.input, src.Metadata.Path)
|
||||
assert.Equal(t, src.Metadata.Path, src.path)
|
||||
|
||||
res, err := src.FileResolver(SquashedScope)
|
||||
require.NoError(t, err)
|
||||
|
||||
refs, err := res.FilesByPath(test.inputPaths...)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, refs, test.expRefs)
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFromFile_WithArchive(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
input string
|
||||
expString string
|
||||
inputPaths []string
|
||||
expRefs int
|
||||
layer2 bool
|
||||
contents string
|
||||
}{
|
||||
{
|
||||
desc: "path detected",
|
||||
input: "test-fixtures/path-detected",
|
||||
inputPaths: []string{"/.vimrc"},
|
||||
expRefs: 1,
|
||||
},
|
||||
{
|
||||
desc: "lest entry for duplicate paths",
|
||||
input: "test-fixtures/path-detected",
|
||||
inputPaths: []string{"/.vimrc"},
|
||||
expRefs: 1,
|
||||
layer2: true,
|
||||
contents: "Another .vimrc file",
|
||||
},
|
||||
}
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
archivePath := setupArchiveTest(t, test.input, test.layer2)
|
||||
|
||||
src, cleanup := NewFromFile(archivePath)
|
||||
if cleanup != nil {
|
||||
t.Cleanup(cleanup)
|
||||
}
|
||||
|
||||
assert.Equal(t, archivePath, src.Metadata.Path)
|
||||
assert.NotEqual(t, src.Metadata.Path, src.path)
|
||||
|
||||
res, err := src.FileResolver(SquashedScope)
|
||||
require.NoError(t, err)
|
||||
|
||||
refs, err := res.FilesByPath(test.inputPaths...)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, refs, test.expRefs)
|
||||
|
||||
if test.contents != "" {
|
||||
reader, err := res.FileContentsByLocation(refs[0])
|
||||
require.NoError(t, err)
|
||||
|
||||
data, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, test.contents, string(data))
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFromDirectoryShared(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
input string
|
||||
expString string
|
||||
notExist string
|
||||
inputPaths []string
|
||||
expRefs int
|
||||
}{
|
||||
{
|
||||
desc: "path detected",
|
||||
input: "test-fixtures",
|
||||
notExist: "foobar/",
|
||||
inputPaths: []string{"path-detected/.vimrc"},
|
||||
expRefs: 1,
|
||||
},
|
||||
{
|
||||
desc: "directory ignored",
|
||||
input: "test-fixtures",
|
||||
notExist: "foobar/",
|
||||
inputPaths: []string{"path-detected"},
|
||||
expRefs: 0,
|
||||
},
|
||||
{
|
||||
desc: "no files-by-path detected",
|
||||
input: "test-fixtures",
|
||||
notExist: "foobar/",
|
||||
inputPaths: []string{"no-path-detected"},
|
||||
expRefs: 0,
|
||||
},
|
||||
}
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
src, err := NewFromDirectory(test.input)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("could not create NewDirScope: %+v", err)
|
||||
}
|
||||
if src.Metadata.Path != test.input {
|
||||
t.Errorf("mismatched stringer: '%s' != '%s'", src.Metadata.Path, test.input)
|
||||
}
|
||||
|
||||
_, err = src.FileResolver(SquashedScope)
|
||||
assert.NoError(t, err)
|
||||
|
||||
src.Metadata.Path = test.notExist
|
||||
resolver, err := src.FileResolver(SquashedScope)
|
||||
assert.NoError(t, err)
|
||||
|
||||
refs, err := resolver.FilesByPath(test.inputPaths...)
|
||||
if err != nil {
|
||||
t.Errorf("FilesByPath call produced an error: %+v", err)
|
||||
}
|
||||
if len(refs) != test.expRefs {
|
||||
t.Errorf("unexpected number of refs returned: %d != %d", len(refs), test.expRefs)
|
||||
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesByPathDoesNotExist(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
input string
|
||||
path string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
input: "test-fixtures/path-detected",
|
||||
desc: "path does not exist",
|
||||
path: "foo",
|
||||
},
|
||||
}
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
src, err := NewFromDirectory(test.input)
|
||||
if err != nil {
|
||||
t.Errorf("could not create NewDirScope: %+v", err)
|
||||
}
|
||||
res, err := src.FileResolver(SquashedScope)
|
||||
if err != nil {
|
||||
t.Errorf("could not get resolver error: %+v", err)
|
||||
}
|
||||
refs, err := res.FilesByPath(test.path)
|
||||
if err != nil {
|
||||
t.Errorf("could not get file references from path: %s, %v", test.path, err)
|
||||
}
|
||||
|
||||
if len(refs) != 0 {
|
||||
t.Errorf("didnt' expect a ref, but got: %d", len(refs))
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesByGlob(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
input string
|
||||
glob string
|
||||
expected int
|
||||
}{
|
||||
{
|
||||
input: "test-fixtures",
|
||||
desc: "no matches",
|
||||
glob: "bar/foo",
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/path-detected",
|
||||
desc: "a single match",
|
||||
glob: "**/*vimrc",
|
||||
expected: 1,
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/path-detected",
|
||||
desc: "multiple matches",
|
||||
glob: "**",
|
||||
expected: 2,
|
||||
},
|
||||
}
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
src, err := NewFromDirectory(test.input)
|
||||
if err != nil {
|
||||
t.Errorf("could not create NewDirScope: %+v", err)
|
||||
}
|
||||
res, err := src.FileResolver(SquashedScope)
|
||||
if err != nil {
|
||||
t.Errorf("could not get resolver error: %+v", err)
|
||||
}
|
||||
contents, err := res.FilesByGlob(test.glob)
|
||||
if err != nil {
|
||||
t.Errorf("could not get files by glob: %s+v", err)
|
||||
}
|
||||
if len(contents) != test.expected {
|
||||
t.Errorf("unexpected number of files found by glob (%s): %d != %d", test.glob, len(contents), test.expected)
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectoryExclusions(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
input string
|
||||
glob string
|
||||
expected []string
|
||||
exclusions []string
|
||||
err bool
|
||||
}{
|
||||
{
|
||||
input: "test-fixtures/system_paths",
|
||||
desc: "exclude everything",
|
||||
glob: "**",
|
||||
expected: nil,
|
||||
exclusions: []string{"**/*"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/image-simple",
|
||||
desc: "a single path excluded",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
"Dockerfile",
|
||||
"file-1.txt",
|
||||
"file-2.txt",
|
||||
},
|
||||
exclusions: []string{"**/target/**"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/image-simple",
|
||||
desc: "exclude explicit directory relative to the root",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
"Dockerfile",
|
||||
"file-1.txt",
|
||||
"file-2.txt",
|
||||
//"target/really/nested/file-3.txt", // explicitly skipped
|
||||
},
|
||||
exclusions: []string{"./target"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/image-simple",
|
||||
desc: "exclude explicit file relative to the root",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
"Dockerfile",
|
||||
//"file-1.txt", // explicitly skipped
|
||||
"file-2.txt",
|
||||
"target/really/nested/file-3.txt",
|
||||
},
|
||||
exclusions: []string{"./file-1.txt"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/image-simple",
|
||||
desc: "exclude wildcard relative to the root",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
"Dockerfile",
|
||||
//"file-1.txt", // explicitly skipped
|
||||
//"file-2.txt", // explicitly skipped
|
||||
"target/really/nested/file-3.txt",
|
||||
},
|
||||
exclusions: []string{"./*.txt"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/image-simple",
|
||||
desc: "exclude files deeper",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
"Dockerfile",
|
||||
"file-1.txt",
|
||||
"file-2.txt",
|
||||
//"target/really/nested/file-3.txt", // explicitly skipped
|
||||
},
|
||||
exclusions: []string{"**/really/**"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/image-simple",
|
||||
desc: "files excluded with extension",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
"Dockerfile",
|
||||
//"file-1.txt", // explicitly skipped
|
||||
//"file-2.txt", // explicitly skipped
|
||||
//"target/really/nested/file-3.txt", // explicitly skipped
|
||||
},
|
||||
exclusions: []string{"**/*.txt"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/image-simple",
|
||||
desc: "keep files with different extensions",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
"Dockerfile",
|
||||
"file-1.txt",
|
||||
"file-2.txt",
|
||||
"target/really/nested/file-3.txt",
|
||||
},
|
||||
exclusions: []string{"**/target/**/*.jar"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/path-detected",
|
||||
desc: "file directly excluded",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
".vimrc",
|
||||
},
|
||||
exclusions: []string{"**/empty"},
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/path-detected",
|
||||
desc: "pattern error containing **/",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
".vimrc",
|
||||
},
|
||||
exclusions: []string{"/**/empty"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/path-detected",
|
||||
desc: "pattern error incorrect start",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
".vimrc",
|
||||
},
|
||||
exclusions: []string{"empty"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
input: "test-fixtures/path-detected",
|
||||
desc: "pattern error starting with /",
|
||||
glob: "**",
|
||||
expected: []string{
|
||||
".vimrc",
|
||||
},
|
||||
exclusions: []string{"/empty"},
|
||||
err: true,
|
||||
},
|
||||
}
|
||||
registryOpts := &image.RegistryOptions{}
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
sourceInput, err := ParseInput("dir:"+test.input, "")
|
||||
require.NoError(t, err)
|
||||
src, fn, err := New(*sourceInput, registryOpts, test.exclusions)
|
||||
defer fn()
|
||||
|
||||
if test.err {
|
||||
_, err = src.FileResolver(SquashedScope)
|
||||
if err == nil {
|
||||
t.Errorf("expected an error for patterns: %s", strings.Join(test.exclusions, " or "))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("could not create NewDirScope: %+v", err)
|
||||
}
|
||||
res, err := src.FileResolver(SquashedScope)
|
||||
if err != nil {
|
||||
t.Errorf("could not get resolver error: %+v", err)
|
||||
}
|
||||
locations, err := res.FilesByGlob(test.glob)
|
||||
if err != nil {
|
||||
t.Errorf("could not get files by glob: %s+v", err)
|
||||
}
|
||||
var actual []string
|
||||
for _, l := range locations {
|
||||
actual = append(actual, l.RealPath)
|
||||
}
|
||||
|
||||
sort.Strings(test.expected)
|
||||
sort.Strings(actual)
|
||||
|
||||
assert.Equal(t, test.expected, actual, "diff \n"+cmp.Diff(test.expected, actual))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageExclusions(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
input string
|
||||
glob string
|
||||
expected int
|
||||
exclusions []string
|
||||
}{
|
||||
// NOTE: in the Dockerfile, /target is moved to /, which makes /really a top-level dir
|
||||
{
|
||||
input: "image-simple",
|
||||
desc: "a single path excluded",
|
||||
glob: "**",
|
||||
expected: 2,
|
||||
exclusions: []string{"/really/**"},
|
||||
},
|
||||
{
|
||||
input: "image-simple",
|
||||
desc: "a directly referenced directory is excluded",
|
||||
glob: "**",
|
||||
expected: 2,
|
||||
exclusions: []string{"/really"},
|
||||
},
|
||||
{
|
||||
input: "image-simple",
|
||||
desc: "a partial directory is not excluded",
|
||||
glob: "**",
|
||||
expected: 3,
|
||||
exclusions: []string{"/reall"},
|
||||
},
|
||||
{
|
||||
input: "image-simple",
|
||||
desc: "exclude files deeper",
|
||||
glob: "**",
|
||||
expected: 2,
|
||||
exclusions: []string{"**/nested/**"},
|
||||
},
|
||||
{
|
||||
input: "image-simple",
|
||||
desc: "files excluded with extension",
|
||||
glob: "**",
|
||||
expected: 2,
|
||||
exclusions: []string{"**/*1.txt"},
|
||||
},
|
||||
{
|
||||
input: "image-simple",
|
||||
desc: "keep files with different extensions",
|
||||
glob: "**",
|
||||
expected: 3,
|
||||
exclusions: []string{"**/target/**/*.jar"},
|
||||
},
|
||||
{
|
||||
input: "image-simple",
|
||||
desc: "file directly excluded",
|
||||
glob: "**",
|
||||
expected: 2,
|
||||
exclusions: []string{"**/somefile-1.txt"}, // file-1 renamed to somefile-1 in Dockerfile
|
||||
},
|
||||
}
|
||||
registryOpts := &image.RegistryOptions{}
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
archiveLocation := imagetest.PrepareFixtureImage(t, "docker-archive", test.input)
|
||||
sourceInput, err := ParseInput(archiveLocation, "")
|
||||
require.NoError(t, err)
|
||||
src, fn, err := New(*sourceInput, registryOpts, test.exclusions)
|
||||
defer fn()
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("could not create NewDirScope: %+v", err)
|
||||
}
|
||||
res, err := src.FileResolver(SquashedScope)
|
||||
if err != nil {
|
||||
t.Errorf("could not get resolver error: %+v", err)
|
||||
}
|
||||
contents, err := res.FilesByGlob(test.glob)
|
||||
if err != nil {
|
||||
t.Errorf("could not get files by glob: %s+v", err)
|
||||
}
|
||||
if len(contents) != test.expected {
|
||||
t.Errorf("wrong number of files after exclusions (%s): %d != %d", test.glob, len(contents), test.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type dummyInfo struct {
|
||||
isDir bool
|
||||
}
|
||||
|
||||
func (d dummyInfo) Name() string {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (d dummyInfo) Size() int64 {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (d dummyInfo) Mode() fs.FileMode {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (d dummyInfo) ModTime() time.Time {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (d dummyInfo) IsDir() bool {
|
||||
return d.isDir
|
||||
}
|
||||
|
||||
func (d dummyInfo) Sys() any {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func Test_crossPlatformExclusions(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
root string
|
||||
path string
|
||||
finfo os.FileInfo
|
||||
exclude string
|
||||
walkHint error
|
||||
}{
|
||||
{
|
||||
desc: "directory exclusion",
|
||||
root: "/",
|
||||
path: "/usr/var/lib",
|
||||
exclude: "**/var/lib",
|
||||
finfo: dummyInfo{isDir: true},
|
||||
walkHint: fs.SkipDir,
|
||||
},
|
||||
{
|
||||
desc: "no file info",
|
||||
root: "/",
|
||||
path: "/usr/var/lib",
|
||||
exclude: "**/var/lib",
|
||||
walkHint: fileresolver.ErrSkipPath,
|
||||
},
|
||||
// linux specific tests...
|
||||
{
|
||||
desc: "linux doublestar",
|
||||
root: "/usr",
|
||||
path: "/usr/var/lib/etc.txt",
|
||||
exclude: "**/*.txt",
|
||||
finfo: dummyInfo{isDir: false},
|
||||
walkHint: fileresolver.ErrSkipPath,
|
||||
},
|
||||
{
|
||||
desc: "linux relative",
|
||||
root: "/usr/var/lib",
|
||||
path: "/usr/var/lib/etc.txt",
|
||||
exclude: "./*.txt",
|
||||
finfo: dummyInfo{isDir: false},
|
||||
|
||||
walkHint: fileresolver.ErrSkipPath,
|
||||
},
|
||||
{
|
||||
desc: "linux one level",
|
||||
root: "/usr",
|
||||
path: "/usr/var/lib/etc.txt",
|
||||
exclude: "*/*.txt",
|
||||
finfo: dummyInfo{isDir: false},
|
||||
walkHint: nil,
|
||||
},
|
||||
// NOTE: since these tests will run in linux and macOS, the windows paths will be
|
||||
// considered relative if they do not start with a forward slash and paths with backslashes
|
||||
// won't be modified by the filepath.ToSlash call, so these are emulating the result of
|
||||
// filepath.ToSlash usage
|
||||
|
||||
// windows specific tests...
|
||||
{
|
||||
desc: "windows doublestar",
|
||||
root: "/C:/User/stuff",
|
||||
path: "/C:/User/stuff/thing.txt",
|
||||
exclude: "**/*.txt",
|
||||
finfo: dummyInfo{isDir: false},
|
||||
walkHint: fileresolver.ErrSkipPath,
|
||||
},
|
||||
{
|
||||
desc: "windows relative",
|
||||
root: "/C:/User/stuff",
|
||||
path: "/C:/User/stuff/thing.txt",
|
||||
exclude: "./*.txt",
|
||||
finfo: dummyInfo{isDir: false},
|
||||
walkHint: fileresolver.ErrSkipPath,
|
||||
},
|
||||
{
|
||||
desc: "windows one level",
|
||||
root: "/C:/User/stuff",
|
||||
path: "/C:/User/stuff/thing.txt",
|
||||
exclude: "*/*.txt",
|
||||
finfo: dummyInfo{isDir: false},
|
||||
walkHint: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
fns, err := getDirectoryExclusionFunctions(test.root, []string{test.exclude})
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, f := range fns {
|
||||
result := f(test.path, test.finfo, nil)
|
||||
require.Equal(t, test.walkHint, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// createArchive creates a new archive file at destinationArchivePath based on the directory found at sourceDirPath.
|
||||
func createArchive(t testing.TB, sourceDirPath, destinationArchivePath string, layer2 bool) {
|
||||
t.Helper()
|
||||
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("unable to get cwd: %+v", err)
|
||||
}
|
||||
|
||||
cmd := exec.Command("./generate-tar-fixture-from-source-dir.sh", destinationArchivePath, path.Base(sourceDirPath))
|
||||
cmd.Dir = filepath.Join(cwd, "test-fixtures")
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatalf("unable to start generate zip fixture script: %+v", err)
|
||||
}
|
||||
|
||||
if err := cmd.Wait(); err != nil {
|
||||
if exiterr, ok := err.(*exec.ExitError); ok {
|
||||
// The program has exited with an exit code != 0
|
||||
|
||||
// This works on both Unix and Windows. Although package
|
||||
// syscall is generally platform dependent, WaitStatus is
|
||||
// defined for both Unix and Windows and in both cases has
|
||||
// an ExitStatus() method with the same signature.
|
||||
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
|
||||
if status.ExitStatus() != 0 {
|
||||
t.Fatalf("failed to generate fixture: rc=%d", status.ExitStatus())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
t.Fatalf("unable to get generate fixture script result: %+v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if layer2 {
|
||||
cmd = exec.Command("tar", "-rvf", destinationArchivePath, ".")
|
||||
cmd.Dir = filepath.Join(cwd, "test-fixtures", path.Base(sourceDirPath+"-2"))
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatalf("unable to start tar appending fixture script: %+v", err)
|
||||
}
|
||||
_ = cmd.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
// setupArchiveTest encapsulates common test setup work for tar file tests. It returns a cleanup function,
|
||||
// which should be called (typically deferred) by the caller, the path of the created tar archive, and an error,
|
||||
// which should trigger a fatal test failure in the consuming test. The returned cleanup function will never be nil
|
||||
// (even if there's an error), and it should always be called.
|
||||
func setupArchiveTest(t testing.TB, sourceDirPath string, layer2 bool) string {
|
||||
t.Helper()
|
||||
|
||||
archivePrefix, err := os.CreateTemp(t.TempDir(), "syft-archive-TEST-")
|
||||
require.NoError(t, err)
|
||||
|
||||
destinationArchiveFilePath := archivePrefix.Name() + ".tar"
|
||||
t.Logf("archive path: %s", destinationArchiveFilePath)
|
||||
createArchive(t, sourceDirPath, destinationArchiveFilePath, layer2)
|
||||
|
||||
cwd, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Logf("running from: %s", cwd)
|
||||
|
||||
return destinationArchiveFilePath
|
||||
}
|
||||
|
||||
func assertNoError(t testing.TB, fn func() error) func() {
|
||||
return func() {
|
||||
assert.NoError(t, fn())
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
//go:build windows
|
||||
// +build windows
|
||||
|
||||
package source
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Test_crossPlatformExclusions(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
root string
|
||||
path string
|
||||
exclude string
|
||||
match bool
|
||||
}{
|
||||
{
|
||||
desc: "windows doublestar",
|
||||
root: "C:\\User\\stuff",
|
||||
path: "C:\\User\\stuff\\thing.txt",
|
||||
exclude: "**/*.txt",
|
||||
match: true,
|
||||
},
|
||||
{
|
||||
desc: "windows relative",
|
||||
root: "C:\\User\\stuff",
|
||||
path: "C:\\User\\stuff\\thing.txt",
|
||||
exclude: "./*.txt",
|
||||
match: true,
|
||||
},
|
||||
{
|
||||
desc: "windows one level",
|
||||
root: "C:\\User\\stuff",
|
||||
path: "C:\\User\\stuff\\thing.txt",
|
||||
exclude: "*/*.txt",
|
||||
match: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
fns, err := getDirectoryExclusionFunctions(test.root, []string{test.exclude})
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, f := range fns {
|
||||
result := f(test.path, nil)
|
||||
require.Equal(t, test.match, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package source
|
||||
|
||||
import "github.com/anchore/stereoscope/pkg/image"
|
||||
|
||||
// StereoscopeImageSourceMetadata represents all static metadata that defines what a container image is. This is useful to later describe
|
||||
// "what" was cataloged without needing the more complicated stereoscope Image objects or FileResolver objects.
|
||||
type StereoscopeImageSourceMetadata struct {
|
||||
UserInput string `json:"userInput"`
|
||||
ID string `json:"imageID"`
|
||||
ManifestDigest string `json:"manifestDigest"`
|
||||
MediaType string `json:"mediaType"`
|
||||
Tags []string `json:"tags"`
|
||||
Size int64 `json:"imageSize"`
|
||||
Layers []StereoscopeLayerMetadata `json:"layers"`
|
||||
RawManifest []byte `json:"manifest"`
|
||||
RawConfig []byte `json:"config"`
|
||||
RepoDigests []string `json:"repoDigests"`
|
||||
Architecture string `json:"architecture"`
|
||||
Variant string `json:"architectureVariant,omitempty"`
|
||||
OS string `json:"os"`
|
||||
}
|
||||
|
||||
// StereoscopeLayerMetadata represents all static metadata that defines what a container image layer is.
|
||||
type StereoscopeLayerMetadata struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// NewStereoscopeImageMetadata creates a new ImageMetadata object populated from the given stereoscope Image object and user configuration.
|
||||
func NewStereoscopeImageMetadata(img *image.Image, userInput string) StereoscopeImageSourceMetadata {
|
||||
// populate artifacts...
|
||||
tags := make([]string, len(img.Metadata.Tags))
|
||||
for idx, tag := range img.Metadata.Tags {
|
||||
tags[idx] = tag.String()
|
||||
}
|
||||
theImg := StereoscopeImageSourceMetadata{
|
||||
ID: img.Metadata.ID,
|
||||
UserInput: userInput,
|
||||
ManifestDigest: img.Metadata.ManifestDigest,
|
||||
Size: img.Metadata.Size,
|
||||
MediaType: string(img.Metadata.MediaType),
|
||||
Tags: tags,
|
||||
Layers: make([]StereoscopeLayerMetadata, len(img.Layers)),
|
||||
RawConfig: img.Metadata.RawConfig,
|
||||
RawManifest: img.Metadata.RawManifest,
|
||||
RepoDigests: img.Metadata.RepoDigests,
|
||||
Architecture: img.Metadata.Architecture,
|
||||
Variant: img.Metadata.Variant,
|
||||
OS: img.Metadata.OS,
|
||||
}
|
||||
|
||||
// populate image metadata
|
||||
for idx, l := range img.Layers {
|
||||
theImg.Layers[idx] = StereoscopeLayerMetadata{
|
||||
MediaType: string(l.Metadata.MediaType),
|
||||
Digest: l.Metadata.Digest,
|
||||
Size: l.Metadata.Size,
|
||||
}
|
||||
}
|
||||
return theImg
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
package source
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
||||
"github.com/anchore/stereoscope"
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/internal/fileresolver"
|
||||
)
|
||||
|
||||
var _ Source = (*StereoscopeImageSource)(nil)
|
||||
|
||||
type StereoscopeImageConfig struct {
|
||||
Reference string
|
||||
From image.Source
|
||||
Platform *image.Platform
|
||||
RegistryOptions *image.RegistryOptions
|
||||
Exclude ExcludeConfig
|
||||
Alias Alias
|
||||
}
|
||||
|
||||
type StereoscopeImageSource struct {
|
||||
id artifact.ID
|
||||
config StereoscopeImageConfig
|
||||
image *image.Image
|
||||
metadata StereoscopeImageSourceMetadata
|
||||
}
|
||||
|
||||
func NewFromStereoscopeImageObject(img *image.Image, reference string, alias *Alias) (*StereoscopeImageSource, error) {
|
||||
var aliasVal Alias
|
||||
if !alias.IsEmpty() {
|
||||
aliasVal = *alias
|
||||
}
|
||||
cfg := StereoscopeImageConfig{
|
||||
Reference: reference,
|
||||
Alias: aliasVal,
|
||||
}
|
||||
metadata := imageMetadataFromStereoscopeImage(img, cfg.Reference)
|
||||
|
||||
return &StereoscopeImageSource{
|
||||
id: deriveIDFromStereoscopeImage(cfg.Alias, metadata),
|
||||
config: cfg,
|
||||
image: img,
|
||||
metadata: metadata,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func NewFromStereoscopeImage(cfg StereoscopeImageConfig) (*StereoscopeImageSource, error) {
|
||||
ctx := context.TODO()
|
||||
|
||||
var opts []stereoscope.Option
|
||||
if cfg.RegistryOptions != nil {
|
||||
opts = append(opts, stereoscope.WithRegistryOptions(*cfg.RegistryOptions))
|
||||
}
|
||||
|
||||
if cfg.Platform != nil {
|
||||
opts = append(opts, stereoscope.WithPlatform(cfg.Platform.String()))
|
||||
}
|
||||
|
||||
img, err := stereoscope.GetImageFromSource(ctx, cfg.Reference, cfg.From, opts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to load image: %w", err)
|
||||
}
|
||||
|
||||
metadata := imageMetadataFromStereoscopeImage(img, cfg.Reference)
|
||||
|
||||
return &StereoscopeImageSource{
|
||||
id: deriveIDFromStereoscopeImage(cfg.Alias, metadata),
|
||||
config: cfg,
|
||||
image: img,
|
||||
metadata: metadata,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s StereoscopeImageSource) ID() artifact.ID {
|
||||
return s.id
|
||||
}
|
||||
|
||||
func (s StereoscopeImageSource) Describe() Description {
|
||||
name := s.metadata.UserInput
|
||||
version := s.metadata.ManifestDigest
|
||||
|
||||
a := s.config.Alias
|
||||
if a.Name != "" {
|
||||
name = a.Name
|
||||
}
|
||||
|
||||
if a.Version != "" {
|
||||
version = a.Version
|
||||
}
|
||||
|
||||
return Description{
|
||||
ID: string(s.id),
|
||||
Name: name,
|
||||
Version: version,
|
||||
Metadata: s.metadata,
|
||||
}
|
||||
}
|
||||
|
||||
func (s StereoscopeImageSource) FileResolver(scope Scope) (file.Resolver, error) {
|
||||
var res file.Resolver
|
||||
var err error
|
||||
|
||||
switch scope {
|
||||
case SquashedScope:
|
||||
res, err = fileresolver.NewFromContainerImageSquash(s.image)
|
||||
case AllLayersScope:
|
||||
res, err = fileresolver.NewFromContainerImageAllLayers(s.image)
|
||||
default:
|
||||
return nil, fmt.Errorf("bad image scope provided: %+v", scope)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// image tree contains all paths, so we filter out the excluded entries afterward
|
||||
if len(s.config.Exclude.Paths) > 0 {
|
||||
res = fileresolver.NewExcludingDecorator(res, getImageExclusionFunction(s.config.Exclude.Paths))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (s StereoscopeImageSource) Close() error {
|
||||
if s.image == nil {
|
||||
return nil
|
||||
}
|
||||
return s.image.Cleanup()
|
||||
}
|
||||
|
||||
func imageMetadataFromStereoscopeImage(img *image.Image, reference string) StereoscopeImageSourceMetadata {
|
||||
tags := make([]string, len(img.Metadata.Tags))
|
||||
for idx, tag := range img.Metadata.Tags {
|
||||
tags[idx] = tag.String()
|
||||
}
|
||||
|
||||
layers := make([]StereoscopeLayerMetadata, len(img.Layers))
|
||||
for idx, l := range img.Layers {
|
||||
layers[idx] = StereoscopeLayerMetadata{
|
||||
MediaType: string(l.Metadata.MediaType),
|
||||
Digest: l.Metadata.Digest,
|
||||
Size: l.Metadata.Size,
|
||||
}
|
||||
}
|
||||
|
||||
return StereoscopeImageSourceMetadata{
|
||||
ID: img.Metadata.ID,
|
||||
UserInput: reference,
|
||||
ManifestDigest: img.Metadata.ManifestDigest,
|
||||
Size: img.Metadata.Size,
|
||||
MediaType: string(img.Metadata.MediaType),
|
||||
Tags: tags,
|
||||
Layers: layers,
|
||||
RawConfig: img.Metadata.RawConfig,
|
||||
RawManifest: img.Metadata.RawManifest,
|
||||
RepoDigests: img.Metadata.RepoDigests,
|
||||
Architecture: img.Metadata.Architecture,
|
||||
Variant: img.Metadata.Variant,
|
||||
OS: img.Metadata.OS,
|
||||
}
|
||||
}
|
||||
|
||||
// deriveIDFromStereoscopeImage derives an artifact ID from the given image metadata. The order of data precedence is:
|
||||
// 1. prefer a digest of the raw container image manifest
|
||||
// 2. if no manifest digest is available, calculate a chain ID from the image layer metadata
|
||||
// 3. if no layer metadata is available, use the user input string
|
||||
//
|
||||
// in all cases, if an alias is provided, it is additionally considered in the ID calculation. This allows for the
|
||||
// same image to be scanned multiple times with different aliases and be considered logically different.
|
||||
func deriveIDFromStereoscopeImage(alias Alias, metadata StereoscopeImageSourceMetadata) artifact.ID {
|
||||
var input string
|
||||
|
||||
if len(metadata.RawManifest) > 0 {
|
||||
input = digest.Canonical.FromBytes(metadata.RawManifest).String()
|
||||
} else {
|
||||
// calculate chain ID for image sources where manifestDigest is not available
|
||||
// https://github.com/opencontainers/image-spec/blob/main/config.md#layer-chainid
|
||||
input = calculateChainID(metadata.Layers)
|
||||
if input == "" {
|
||||
// TODO what happens here if image has no layers?
|
||||
// is this case possible?
|
||||
input = digest.Canonical.FromString(metadata.UserInput).String()
|
||||
}
|
||||
}
|
||||
|
||||
if !alias.IsEmpty() {
|
||||
// if the user provided an alias, we want to consider that in the artifact ID. This way if the user
|
||||
// scans the same item but is considered to be logically different, then ID will express that.
|
||||
aliasStr := fmt.Sprintf(":%s@%s", alias.Name, alias.Version)
|
||||
input = digest.Canonical.FromString(input + aliasStr).String()
|
||||
}
|
||||
|
||||
return artifactIDFromDigest(input)
|
||||
}
|
||||
|
||||
func calculateChainID(lm []StereoscopeLayerMetadata) string {
|
||||
if len(lm) < 1 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// DiffID(L0) = digest of layer 0
|
||||
// https://github.com/anchore/stereoscope/blob/1b1b744a919964f38d14e1416fb3f25221b761ce/pkg/image/layer_metadata.go#L19-L32
|
||||
chainID := lm[0].Digest
|
||||
id := chain(chainID, lm[1:])
|
||||
|
||||
return id
|
||||
}
|
||||
|
||||
func chain(chainID string, layers []StereoscopeLayerMetadata) string {
|
||||
if len(layers) < 1 {
|
||||
return chainID
|
||||
}
|
||||
|
||||
chainID = digest.Canonical.FromString(layers[0].Digest + " " + chainID).String()
|
||||
return chain(chainID, layers[1:])
|
||||
}
|
||||
|
||||
func getImageExclusionFunction(exclusions []string) func(string) bool {
|
||||
if len(exclusions) == 0 {
|
||||
return nil
|
||||
}
|
||||
// add subpath exclusions
|
||||
for _, exclusion := range exclusions {
|
||||
exclusions = append(exclusions, exclusion+"/**")
|
||||
}
|
||||
return func(path string) bool {
|
||||
for _, exclusion := range exclusions {
|
||||
matches, err := doublestar.Match(exclusion, path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if matches {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
package source
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
"github.com/anchore/stereoscope/pkg/imagetest"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
)
|
||||
|
||||
func Test_StereoscopeImage_Exclusions(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
input string
|
||||
glob string
|
||||
expected int
|
||||
exclusions []string
|
||||
}{
|
||||
// NOTE: in the Dockerfile, /target is moved to /, which makes /really a top-level dir
|
||||
{
|
||||
input: "image-simple",
|
||||
desc: "a single path excluded",
|
||||
glob: "**",
|
||||
expected: 2,
|
||||
exclusions: []string{"/really/**"},
|
||||
},
|
||||
{
|
||||
input: "image-simple",
|
||||
desc: "a directly referenced directory is excluded",
|
||||
glob: "**",
|
||||
expected: 2,
|
||||
exclusions: []string{"/really"},
|
||||
},
|
||||
{
|
||||
input: "image-simple",
|
||||
desc: "a partial directory is not excluded",
|
||||
glob: "**",
|
||||
expected: 3,
|
||||
exclusions: []string{"/reall"},
|
||||
},
|
||||
{
|
||||
input: "image-simple",
|
||||
desc: "exclude files deeper",
|
||||
glob: "**",
|
||||
expected: 2,
|
||||
exclusions: []string{"**/nested/**"},
|
||||
},
|
||||
{
|
||||
input: "image-simple",
|
||||
desc: "files excluded with extension",
|
||||
glob: "**",
|
||||
expected: 2,
|
||||
exclusions: []string{"**/*1.txt"},
|
||||
},
|
||||
{
|
||||
input: "image-simple",
|
||||
desc: "keep files with different extensions",
|
||||
glob: "**",
|
||||
expected: 3,
|
||||
exclusions: []string{"**/target/**/*.jar"},
|
||||
},
|
||||
{
|
||||
input: "image-simple",
|
||||
desc: "file directly excluded",
|
||||
glob: "**",
|
||||
expected: 2,
|
||||
exclusions: []string{"**/somefile-1.txt"}, // file-1 renamed to somefile-1 in Dockerfile
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range testCases {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
src, err := NewFromStereoscopeImage(
|
||||
StereoscopeImageConfig{
|
||||
Reference: strings.SplitN(imagetest.PrepareFixtureImage(t, "docker-archive", test.input), ":", 2)[1],
|
||||
From: image.DockerTarballSource,
|
||||
Exclude: ExcludeConfig{
|
||||
Paths: test.exclusions,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, src.Close())
|
||||
})
|
||||
|
||||
res, err := src.FileResolver(SquashedScope)
|
||||
require.NoError(t, err)
|
||||
|
||||
contents, err := res.FilesByGlob(test.glob)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, contents, test.expected)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_StereoscopeImageSource_ID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
alias Alias
|
||||
metadata StereoscopeImageSourceMetadata
|
||||
want artifact.ID
|
||||
}{
|
||||
{
|
||||
name: "use raw manifest over chain ID or user input",
|
||||
metadata: StereoscopeImageSourceMetadata{
|
||||
UserInput: "user-input",
|
||||
Layers: []StereoscopeLayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
RawManifest: []byte("raw-manifest"),
|
||||
},
|
||||
want: func() artifact.ID {
|
||||
hasher := sha256.New()
|
||||
hasher.Write([]byte("raw-manifest"))
|
||||
return artifact.ID(fmt.Sprintf("%x", hasher.Sum(nil)))
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "use chain ID over user input",
|
||||
metadata: StereoscopeImageSourceMetadata{
|
||||
//UserInput: "user-input",
|
||||
Layers: []StereoscopeLayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
},
|
||||
want: func() artifact.ID {
|
||||
metadata := []StereoscopeLayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
}
|
||||
return artifact.ID(strings.TrimPrefix(calculateChainID(metadata), "sha256:"))
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "use user input last",
|
||||
metadata: StereoscopeImageSourceMetadata{
|
||||
UserInput: "user-input",
|
||||
},
|
||||
want: func() artifact.ID {
|
||||
hasher := sha256.New()
|
||||
hasher.Write([]byte("user-input"))
|
||||
return artifact.ID(fmt.Sprintf("%x", hasher.Sum(nil)))
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "without alias (first)",
|
||||
metadata: StereoscopeImageSourceMetadata{
|
||||
UserInput: "user-input",
|
||||
Layers: []StereoscopeLayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
RawManifest: []byte("raw-manifest"),
|
||||
},
|
||||
want: "85298926ecd92ed57688f13039017160cd728f04dd0d2d10a10629007106f107",
|
||||
},
|
||||
{
|
||||
name: "always consider alias (first)",
|
||||
alias: Alias{
|
||||
Name: "alias",
|
||||
Version: "version",
|
||||
},
|
||||
metadata: StereoscopeImageSourceMetadata{
|
||||
UserInput: "user-input",
|
||||
Layers: []StereoscopeLayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
RawManifest: []byte("raw-manifest"),
|
||||
},
|
||||
want: "a8717e42449960c1dd4963f2f22bd69c7c105e7e82445be0a65aa1825d62ff0d",
|
||||
},
|
||||
{
|
||||
name: "without alias (last)",
|
||||
metadata: StereoscopeImageSourceMetadata{
|
||||
UserInput: "user-input",
|
||||
},
|
||||
want: "ab0dff627d80b9753193d7280bec8f45e8ec6b4cb0912c6fffcf7cd782d9739e",
|
||||
},
|
||||
{
|
||||
name: "always consider alias (last)",
|
||||
alias: Alias{
|
||||
Name: "alias",
|
||||
Version: "version",
|
||||
},
|
||||
metadata: StereoscopeImageSourceMetadata{
|
||||
UserInput: "user-input",
|
||||
},
|
||||
want: "fe86c0eecd5654d3c0c0b2176aa394aef6440347c241aa8d9b628dfdde4287cf",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, deriveIDFromStereoscopeImage(tt.alias, tt.metadata))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
content
|
||||
@@ -0,0 +1 @@
|
||||
something
|
||||
@@ -0,0 +1 @@
|
||||
Another .vimrc file
|
||||
Reference in New Issue
Block a user