Refactor source API (#1846)

* refactor source API and syft json source block

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update source detection and format test utils

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* generate list of all source metadata types

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* extract base and root normalization into helper functions

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* preserve syftjson model package name import ref

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* alias should not be a pointer

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
2023-06-30 14:19:16 +00:00
committed by GitHub
parent 608dbded06
commit 4da3be864f
126 changed files with 7384 additions and 3190 deletions
@@ -0,0 +1,165 @@
package fileresolver
import (
"fmt"
"os"
"path"
"path/filepath"
"strings"
"github.com/anchore/syft/syft/internal/windows"
)
// ChrootContext helps to modify path from a real filesystem to a chroot-like filesystem, taking into account
// the user given root, the base path (if any) to consider as the root, and the current working directory.
// Note: this only works on a real filesystem, not on a virtual filesystem (such as a stereoscope filetree).
type ChrootContext struct {
root string
base string
cwd string
cwdRelativeToRoot string
}
func NewChrootContextFromCWD(root, base string) (*ChrootContext, error) {
currentWD, err := os.Getwd()
if err != nil {
return nil, fmt.Errorf("could not get current working directory: %w", err)
}
return NewChrootContext(root, base, currentWD)
}
func NewChrootContext(root, base, cwd string) (*ChrootContext, error) {
cleanRoot, err := NormalizeRootDirectory(root)
if err != nil {
return nil, err
}
cleanBase, err := NormalizeBaseDirectory(base)
if err != nil {
return nil, err
}
chroot := &ChrootContext{
root: cleanRoot,
base: cleanBase,
cwd: cwd,
}
return chroot, chroot.ChangeDirectory(cwd)
}
func NormalizeRootDirectory(root string) (string, error) {
cleanRoot, err := filepath.EvalSymlinks(root)
if err != nil {
return "", fmt.Errorf("could not evaluate root=%q symlinks: %w", root, err)
}
return cleanRoot, nil
}
func NormalizeBaseDirectory(base string) (string, error) {
if base == "" {
return "", nil
}
cleanBase, err := filepath.EvalSymlinks(base)
if err != nil {
return "", fmt.Errorf("could not evaluate base=%q symlinks: %w", base, err)
}
return filepath.Abs(cleanBase)
}
// Root returns the root path with all symlinks evaluated.
func (r ChrootContext) Root() string {
return r.root
}
// Base returns the absolute base path with all symlinks evaluated.
func (r ChrootContext) Base() string {
return r.base
}
// ChangeRoot swaps the path for the chroot.
func (r *ChrootContext) ChangeRoot(dir string) error {
newR, err := NewChrootContext(dir, r.base, r.cwd)
if err != nil {
return fmt.Errorf("could not change root: %w", err)
}
*r = *newR
return nil
}
// ChangeDirectory changes the current working directory so that any relative paths passed
// into ToNativePath() and ToChrootPath() honor the new CWD. If the process changes the CWD in-flight, this should be
// called again to ensure correct functionality of ToNativePath() and ToChrootPath().
func (r *ChrootContext) ChangeDirectory(dir string) error {
var (
cwdRelativeToRoot string
err error
)
dir, err = filepath.Abs(dir)
if err != nil {
return fmt.Errorf("could not determine absolute path to CWD: %w", err)
}
if path.IsAbs(r.root) {
cwdRelativeToRoot, err = filepath.Rel(dir, r.root)
if err != nil {
return fmt.Errorf("could not determine given root path to CWD: %w", err)
}
} else {
cwdRelativeToRoot = filepath.Clean(r.root)
}
r.cwd = dir
r.cwdRelativeToRoot = cwdRelativeToRoot
return nil
}
// ToNativePath takes a path in the context of the chroot-like filesystem and converts it to a path in the underlying fs domain.
func (r ChrootContext) ToNativePath(chrootPath string) (string, error) {
responsePath := chrootPath
if filepath.IsAbs(responsePath) {
// don't allow input to potentially hop above root path
responsePath = path.Join(r.root, responsePath)
} else {
// ensure we take into account any relative difference between the root path and the CWD for relative requests
responsePath = path.Join(r.cwdRelativeToRoot, responsePath)
}
var err error
responsePath, err = filepath.Abs(responsePath)
if err != nil {
return "", err
}
return responsePath, nil
}
// ToChrootPath takes a path from the underlying fs domain and converts it to a path that is relative to the current root context.
func (r ChrootContext) ToChrootPath(nativePath string) string {
responsePath := nativePath
// check to see if we need to encode back to Windows from posix
if windows.HostRunningOnWindows() {
responsePath = windows.FromPosix(responsePath)
}
// clean references to the request path (either the root, or the base if set)
if filepath.IsAbs(responsePath) {
var prefix string
if r.base != "" {
prefix = r.base
} else {
// we need to account for the cwd relative to the running process and the given root for the directory resolver
prefix = filepath.Clean(filepath.Join(r.cwd, r.cwdRelativeToRoot))
prefix += string(filepath.Separator)
}
responsePath = strings.TrimPrefix(responsePath, prefix)
}
return responsePath
}
@@ -0,0 +1,481 @@
package fileresolver
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_ChrootContext_RequestResponse(t *testing.T) {
// /
// somewhere/
// outside.txt
// root-link -> ./
// path/
// to/
// abs-inside.txt -> /path/to/the/file.txt # absolute link to somewhere inside of the root
// rel-inside.txt -> ./the/file.txt # relative link to somewhere inside of the root
// the/
// file.txt
// abs-outside.txt -> /somewhere/outside.txt # absolute link to outside of the root
// rel-outside -> ../../../somewhere/outside.txt # relative link to outside of the root
//
testDir, err := os.Getwd()
require.NoError(t, err)
relative := filepath.Join("test-fixtures", "req-resp")
absolute := filepath.Join(testDir, relative)
absPathToTheFile := filepath.Join(absolute, "path", "to", "the", "file.txt")
absAbsInsidePath := filepath.Join(absolute, "path", "to", "abs-inside.txt")
absAbsOutsidePath := filepath.Join(absolute, "path", "to", "the", "abs-outside.txt")
absRelOutsidePath := filepath.Join(absolute, "path", "to", "the", "rel-outside.txt")
relViaLink := filepath.Join(relative, "root-link")
absViaLink := filepath.Join(absolute, "root-link")
absViaLinkPathToTheFile := filepath.Join(absViaLink, "path", "to", "the", "file.txt")
absViaLinkAbsOutsidePath := filepath.Join(absViaLink, "path", "to", "the", "abs-outside.txt")
absViaLinkRelOutsidePath := filepath.Join(absViaLink, "path", "to", "the", "rel-outside.txt")
relViaDoubleLink := filepath.Join(relative, "root-link", "root-link")
absViaDoubleLink := filepath.Join(absolute, "root-link", "root-link")
absViaDoubleLinkPathToTheFile := filepath.Join(absViaDoubleLink, "path", "to", "the", "file.txt")
absViaDoubleLinkRelOutsidePath := filepath.Join(absViaDoubleLink, "path", "to", "the", "rel-outside.txt")
cleanup := func() {
_ = os.Remove(absAbsInsidePath)
_ = os.Remove(absAbsOutsidePath)
}
// ensure the absolute symlinks are cleaned up from any previous runs
cleanup()
require.NoError(t, os.Symlink(filepath.Join(absolute, "path", "to", "the", "file.txt"), absAbsInsidePath))
require.NoError(t, os.Symlink(filepath.Join(absolute, "somewhere", "outside.txt"), absAbsOutsidePath))
t.Cleanup(cleanup)
cases := []struct {
name string
cwd string
root string
base string
input string
expectedNativePath string
expectedChrootPath string
}{
{
name: "relative root, relative request, direct",
root: relative,
input: "path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "abs root, relative request, direct",
root: absolute,
input: "path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "relative root, abs request, direct",
root: relative,
input: "/path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "abs root, abs request, direct",
root: absolute,
input: "/path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
// cwd within root...
{
name: "relative root, relative request, direct, cwd within root",
cwd: filepath.Join(relative, "path/to"),
root: "../../",
input: "path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "abs root, relative request, direct, cwd within root",
cwd: filepath.Join(relative, "path/to"),
root: absolute,
input: "path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "relative root, abs request, direct, cwd within root",
cwd: filepath.Join(relative, "path/to"),
root: "../../",
input: "/path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "abs root, abs request, direct, cwd within root",
cwd: filepath.Join(relative, "path/to"),
root: absolute,
input: "/path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
// cwd within symlink root...
{
name: "relative root, relative request, direct, cwd within symlink root",
cwd: relViaLink,
root: "./",
input: "path/to/the/file.txt",
expectedNativePath: absViaLinkPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "abs root, relative request, direct, cwd within symlink root",
cwd: relViaLink,
root: absViaLink,
input: "path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "relative root, abs request, direct, cwd within symlink root",
cwd: relViaLink,
root: "./",
input: "/path/to/the/file.txt",
expectedNativePath: absViaLinkPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "abs root, abs request, direct, cwd within symlink root",
cwd: relViaLink,
root: absViaLink,
input: "/path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
// cwd within symlink root, request nested within...
{
name: "relative root, relative nested request, direct, cwd within symlink root",
cwd: relViaLink,
root: "./path",
input: "to/the/file.txt",
expectedNativePath: absViaLinkPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "abs root, relative nested request, direct, cwd within symlink root",
cwd: relViaLink,
root: filepath.Join(absViaLink, "path"),
input: "to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "relative root, abs nested request, direct, cwd within symlink root",
cwd: relViaLink,
root: "./path",
input: "/to/the/file.txt",
expectedNativePath: absViaLinkPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "abs root, abs nested request, direct, cwd within symlink root",
cwd: relViaLink,
root: filepath.Join(absViaLink, "path"),
input: "/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
// cwd within DOUBLE symlink root...
{
name: "relative root, relative request, direct, cwd within (double) symlink root",
cwd: relViaDoubleLink,
root: "./",
input: "path/to/the/file.txt",
expectedNativePath: absViaDoubleLinkPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "abs root, relative request, direct, cwd within (double) symlink root",
cwd: relViaDoubleLink,
root: absViaDoubleLink,
input: "path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "relative root, abs request, direct, cwd within (double) symlink root",
cwd: relViaDoubleLink,
root: "./",
input: "/path/to/the/file.txt",
expectedNativePath: absViaDoubleLinkPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "abs root, abs request, direct, cwd within (double) symlink root",
cwd: relViaDoubleLink,
root: absViaDoubleLink,
input: "/path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
// cwd within DOUBLE symlink root, request nested within...
{
name: "relative root, relative nested request, direct, cwd within (double) symlink root",
cwd: relViaDoubleLink,
root: "./path",
input: "to/the/file.txt",
expectedNativePath: absViaDoubleLinkPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "abs root, relative nested request, direct, cwd within (double) symlink root",
cwd: relViaDoubleLink,
root: filepath.Join(absViaDoubleLink, "path"),
input: "to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "relative root, abs nested request, direct, cwd within (double) symlink root",
cwd: relViaDoubleLink,
root: "./path",
input: "/to/the/file.txt",
expectedNativePath: absViaDoubleLinkPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "abs root, abs nested request, direct, cwd within (double) symlink root",
cwd: relViaDoubleLink,
root: filepath.Join(absViaDoubleLink, "path"),
input: "/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
// cwd within DOUBLE symlink root, request nested DEEP within...
{
name: "relative root, relative nested request, direct, cwd deep within (double) symlink root",
cwd: filepath.Join(relViaDoubleLink, "path", "to"),
root: "../",
input: "to/the/file.txt",
expectedNativePath: absViaDoubleLinkPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "abs root, relative nested request, direct, cwd deep within (double) symlink root",
cwd: filepath.Join(relViaDoubleLink, "path", "to"),
root: filepath.Join(absViaDoubleLink, "path"),
input: "to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "relative root, abs nested request, direct, cwd deep within (double) symlink root",
cwd: filepath.Join(relViaDoubleLink, "path", "to"),
root: "../",
input: "/to/the/file.txt",
expectedNativePath: absViaDoubleLinkPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "abs root, abs nested request, direct, cwd deep within (double) symlink root",
cwd: filepath.Join(relViaDoubleLink, "path", "to"),
root: filepath.Join(absViaDoubleLink, "path"),
input: "/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
// link to outside of root cases...
{
name: "relative root, relative request, abs indirect (outside of root)",
root: filepath.Join(relative, "path"),
input: "to/the/abs-outside.txt",
expectedNativePath: absAbsOutsidePath,
expectedChrootPath: "to/the/abs-outside.txt",
},
{
name: "abs root, relative request, abs indirect (outside of root)",
root: filepath.Join(absolute, "path"),
input: "to/the/abs-outside.txt",
expectedNativePath: absAbsOutsidePath,
expectedChrootPath: "to/the/abs-outside.txt",
},
{
name: "relative root, abs request, abs indirect (outside of root)",
root: filepath.Join(relative, "path"),
input: "/to/the/abs-outside.txt",
expectedNativePath: absAbsOutsidePath,
expectedChrootPath: "to/the/abs-outside.txt",
},
{
name: "abs root, abs request, abs indirect (outside of root)",
root: filepath.Join(absolute, "path"),
input: "/to/the/abs-outside.txt",
expectedNativePath: absAbsOutsidePath,
expectedChrootPath: "to/the/abs-outside.txt",
},
{
name: "relative root, relative request, relative indirect (outside of root)",
root: filepath.Join(relative, "path"),
input: "to/the/rel-outside.txt",
expectedNativePath: absRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "abs root, relative request, relative indirect (outside of root)",
root: filepath.Join(absolute, "path"),
input: "to/the/rel-outside.txt",
expectedNativePath: absRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "relative root, abs request, relative indirect (outside of root)",
root: filepath.Join(relative, "path"),
input: "/to/the/rel-outside.txt",
expectedNativePath: absRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "abs root, abs request, relative indirect (outside of root)",
root: filepath.Join(absolute, "path"),
input: "/to/the/rel-outside.txt",
expectedNativePath: absRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
// link to outside of root cases... cwd within symlink root
{
name: "relative root, relative request, abs indirect (outside of root), cwd within symlink root",
cwd: relViaLink,
root: "path",
input: "to/the/abs-outside.txt",
expectedNativePath: absViaLinkAbsOutsidePath,
expectedChrootPath: "to/the/abs-outside.txt",
},
{
name: "abs root, relative request, abs indirect (outside of root), cwd within symlink root",
cwd: relViaLink,
root: filepath.Join(absolute, "path"),
input: "to/the/abs-outside.txt",
expectedNativePath: absAbsOutsidePath,
expectedChrootPath: "to/the/abs-outside.txt",
},
{
name: "relative root, abs request, abs indirect (outside of root), cwd within symlink root",
cwd: relViaLink,
root: "path",
input: "/to/the/abs-outside.txt",
expectedNativePath: absViaLinkAbsOutsidePath,
expectedChrootPath: "to/the/abs-outside.txt",
},
{
name: "abs root, abs request, abs indirect (outside of root), cwd within symlink root",
cwd: relViaLink,
root: filepath.Join(absolute, "path"),
input: "/to/the/abs-outside.txt",
expectedNativePath: absAbsOutsidePath,
expectedChrootPath: "to/the/abs-outside.txt",
},
{
name: "relative root, relative request, relative indirect (outside of root), cwd within symlink root",
cwd: relViaLink,
root: "path",
input: "to/the/rel-outside.txt",
expectedNativePath: absViaLinkRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "abs root, relative request, relative indirect (outside of root), cwd within symlink root",
cwd: relViaLink,
root: filepath.Join(absolute, "path"),
input: "to/the/rel-outside.txt",
expectedNativePath: absRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "relative root, abs request, relative indirect (outside of root), cwd within symlink root",
cwd: relViaLink,
root: "path",
input: "/to/the/rel-outside.txt",
expectedNativePath: absViaLinkRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "abs root, abs request, relative indirect (outside of root), cwd within symlink root",
cwd: relViaLink,
root: filepath.Join(absolute, "path"),
input: "/to/the/rel-outside.txt",
expectedNativePath: absRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "relative root, relative request, relative indirect (outside of root), cwd within DOUBLE symlink root",
cwd: relViaDoubleLink,
root: "path",
input: "to/the/rel-outside.txt",
expectedNativePath: absViaDoubleLinkRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "abs root, relative request, relative indirect (outside of root), cwd within DOUBLE symlink root",
cwd: relViaDoubleLink,
root: filepath.Join(absolute, "path"),
input: "to/the/rel-outside.txt",
expectedNativePath: absRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "relative root, abs request, relative indirect (outside of root), cwd within DOUBLE symlink root",
cwd: relViaDoubleLink,
root: "path",
input: "/to/the/rel-outside.txt",
expectedNativePath: absViaDoubleLinkRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "abs root, abs request, relative indirect (outside of root), cwd within DOUBLE symlink root",
cwd: relViaDoubleLink,
root: filepath.Join(absolute, "path"),
input: "/to/the/rel-outside.txt",
expectedNativePath: absRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
// we need to mimic a shell, otherwise we won't get a path within a symlink
targetPath := filepath.Join(testDir, c.cwd)
t.Setenv("PWD", filepath.Clean(targetPath))
require.NoError(t, err)
require.NoError(t, os.Chdir(targetPath))
t.Cleanup(func() {
require.NoError(t, os.Chdir(testDir))
})
chroot, err := NewChrootContextFromCWD(c.root, c.base)
require.NoError(t, err)
require.NotNil(t, chroot)
req, err := chroot.ToNativePath(c.input)
require.NoError(t, err)
assert.Equal(t, c.expectedNativePath, req, "native path different")
resp := chroot.ToChrootPath(req)
assert.Equal(t, c.expectedChrootPath, resp, "chroot path different")
})
}
}
@@ -6,6 +6,7 @@ import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/scylladb/go-set/strset"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -513,6 +514,26 @@ func Test_imageSquashResolver_resolvesLinks(t *testing.T) {
}
func compareLocations(t *testing.T, expected, actual []file.Location) {
t.Helper()
ignoreUnexported := cmpopts.IgnoreUnexported(file.LocationData{})
ignoreMetadata := cmpopts.IgnoreFields(file.LocationMetadata{}, "Annotations")
ignoreFS := cmpopts.IgnoreFields(file.Coordinates{}, "FileSystemID")
sort.Sort(file.Locations(expected))
sort.Sort(file.Locations(actual))
if d := cmp.Diff(expected, actual,
ignoreUnexported,
ignoreFS,
ignoreMetadata,
); d != "" {
t.Errorf("unexpected locations (-want +got):\n%s", d)
}
}
func TestSquashResolver_AllLocations(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-files-deleted")
+22 -113
View File
@@ -5,19 +5,14 @@ import (
"fmt"
"io"
"os"
"path"
"path/filepath"
"runtime"
"strings"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/internal/windows"
)
const WindowsOS = "windows"
var unixSystemRuntimePrefixes = []string{
"/proc",
"/dev",
@@ -30,14 +25,12 @@ var _ file.Resolver = (*Directory)(nil)
// Directory implements path and content access for the directory data source.
type Directory struct {
path string
base string
currentWdRelativeToRoot string
currentWd string
tree filetree.Reader
index filetree.IndexReader
searchContext filetree.Searcher
indexer *directoryIndexer
path string
chroot ChrootContext
tree filetree.Reader
index filetree.IndexReader
searchContext filetree.Searcher
indexer *directoryIndexer
}
func NewFromDirectory(root string, base string, pathFilters ...PathIndexVisitor) (*Directory, error) {
@@ -50,46 +43,20 @@ func NewFromDirectory(root string, base string, pathFilters ...PathIndexVisitor)
}
func newFromDirectoryWithoutIndex(root string, base string, pathFilters ...PathIndexVisitor) (*Directory, error) {
currentWD, err := os.Getwd()
chroot, err := NewChrootContextFromCWD(root, base)
if err != nil {
return nil, fmt.Errorf("could not get CWD: %w", err)
return nil, fmt.Errorf("unable to interpret chroot context: %w", err)
}
cleanRoot, err := filepath.EvalSymlinks(root)
if err != nil {
return nil, fmt.Errorf("could not evaluate root=%q symlinks: %w", root, err)
}
cleanBase := ""
if base != "" {
cleanBase, err = filepath.EvalSymlinks(base)
if err != nil {
return nil, fmt.Errorf("could not evaluate base=%q symlinks: %w", base, err)
}
cleanBase, err = filepath.Abs(cleanBase)
if err != nil {
return nil, err
}
}
var currentWdRelRoot string
if path.IsAbs(cleanRoot) {
currentWdRelRoot, err = filepath.Rel(currentWD, cleanRoot)
if err != nil {
return nil, fmt.Errorf("could not determine given root path to CWD: %w", err)
}
} else {
currentWdRelRoot = filepath.Clean(cleanRoot)
}
cleanRoot := chroot.Root()
cleanBase := chroot.Base()
return &Directory{
path: cleanRoot,
base: cleanBase,
currentWd: currentWD,
currentWdRelativeToRoot: currentWdRelRoot,
tree: filetree.New(),
index: filetree.NewIndex(),
indexer: newDirectoryIndexer(cleanRoot, cleanBase, pathFilters...),
path: cleanRoot,
chroot: *chroot,
tree: filetree.New(),
index: filetree.NewIndex(),
indexer: newDirectoryIndexer(cleanRoot, cleanBase, pathFilters...),
}, nil
}
@@ -110,43 +77,12 @@ func (r *Directory) buildIndex() error {
}
func (r Directory) requestPath(userPath string) (string, error) {
if filepath.IsAbs(userPath) {
// don't allow input to potentially hop above root path
userPath = path.Join(r.path, userPath)
} else {
// ensure we take into account any relative difference between the root path and the CWD for relative requests
userPath = path.Join(r.currentWdRelativeToRoot, userPath)
}
var err error
userPath, err = filepath.Abs(userPath)
if err != nil {
return "", err
}
return userPath, nil
return r.chroot.ToNativePath(userPath)
}
// responsePath takes a path from the underlying fs domain and converts it to a path that is relative to the root of the directory resolver.
func (r Directory) responsePath(path string) string {
// check to see if we need to encode back to Windows from posix
if runtime.GOOS == WindowsOS {
path = posixToWindows(path)
}
// clean references to the request path (either the root, or the base if set)
if filepath.IsAbs(path) {
var prefix string
if r.base != "" {
prefix = r.base
} else {
// we need to account for the cwd relative to the running process and the given root for the directory resolver
prefix = filepath.Clean(filepath.Join(r.currentWd, r.currentWdRelativeToRoot))
prefix += string(filepath.Separator)
}
path = strings.TrimPrefix(path, prefix)
}
return path
return r.chroot.ToChrootPath(path)
}
// HasPath indicates if the given path exists in the underlying source.
@@ -196,8 +132,8 @@ func (r Directory) FilesByPath(userPaths ...string) ([]file.Location, error) {
continue
}
if runtime.GOOS == WindowsOS {
userStrPath = windowsToPosix(userStrPath)
if windows.HostRunningOnWindows() {
userStrPath = windows.ToPosix(userStrPath)
}
if ref.HasReference() {
@@ -286,8 +222,8 @@ func (r Directory) FileContentsByLocation(location file.Location) (io.ReadCloser
// RealPath is posix so for windows directory resolver we need to translate
// to its true on disk path.
filePath := string(location.Reference().RealPath)
if runtime.GOOS == WindowsOS {
filePath = posixToWindows(filePath)
if windows.HostRunningOnWindows() {
filePath = windows.FromPosix(filePath)
}
return stereoscopeFile.NewLazyReadCloser(filePath), nil
@@ -338,30 +274,3 @@ func (r *Directory) FilesByMIMEType(types ...string) ([]file.Location, error) {
return uniqueLocations, nil
}
func windowsToPosix(windowsPath string) (posixPath string) {
// volume should be encoded at the start (e.g /c/<path>) where c is the volume
volumeName := filepath.VolumeName(windowsPath)
pathWithoutVolume := strings.TrimPrefix(windowsPath, volumeName)
volumeLetter := strings.ToLower(strings.TrimSuffix(volumeName, ":"))
// translate non-escaped backslash to forwardslash
translatedPath := strings.ReplaceAll(pathWithoutVolume, "\\", "/")
// always have `/` as the root... join all components, e.g.:
// convert: C:\\some\windows\Place
// into: /c/some/windows/Place
return path.Clean("/" + strings.Join([]string{volumeLetter, translatedPath}, "/"))
}
func posixToWindows(posixPath string) (windowsPath string) {
// decode the volume (e.g. /c/<path> --> C:\\) - There should always be a volume name.
pathFields := strings.Split(posixPath, "/")
volumeName := strings.ToUpper(pathFields[1]) + `:\\`
// translate non-escaped forward slashes into backslashes
remainingTranslatedPath := strings.Join(pathFields[2:], "\\")
// combine volume name and backslash components
return filepath.Clean(volumeName + remainingTranslatedPath)
}
@@ -7,7 +7,6 @@ import (
"os"
"path"
"path/filepath"
"runtime"
"strings"
"github.com/wagoodman/go-partybus"
@@ -19,6 +18,7 @@ import (
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/internal/windows"
)
type PathIndexVisitor func(string, os.FileInfo, error) error
@@ -263,8 +263,8 @@ func (r *directoryIndexer) indexPath(path string, info os.FileInfo, err error) (
}
// here we check to see if we need to normalize paths to posix on the way in coming from windows
if runtime.GOOS == WindowsOS {
path = windowsToPosix(path)
if windows.HostRunningOnWindows() {
path = windows.ToPosix(path)
}
newRoot, err := r.addPathToIndex(path, info)
+2 -2
View File
@@ -16,9 +16,9 @@ type excluding struct {
excludeFn excludeFn
}
// NewExcluding create a new resolver which wraps the provided delegate and excludes
// NewExcludingDecorator create a new resolver which wraps the provided delegate and excludes
// entries based on a provided path exclusion function
func NewExcluding(delegate file.Resolver, excludeFn excludeFn) file.Resolver {
func NewExcludingDecorator(delegate file.Resolver, excludeFn excludeFn) file.Resolver {
return &excluding{
delegate,
excludeFn,
@@ -56,7 +56,7 @@ func TestExcludingResolver(t *testing.T) {
resolver := &mockResolver{
locations: test.locations,
}
er := NewExcluding(resolver, test.excludeFn)
er := NewExcludingDecorator(resolver, test.excludeFn)
locations, _ := er.FilesByPath()
assert.ElementsMatch(t, locationPaths(locations), test.expected)
@@ -0,0 +1,2 @@
path/to/abs-inside.txt
path/to/the/abs-outside.txt
@@ -14,7 +14,6 @@ import (
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/scylladb/go-set/strset"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -1263,23 +1262,3 @@ func testWithTimeout(t *testing.T, timeout time.Duration, test func(*testing.T))
case <-done:
}
}
func compareLocations(t *testing.T, expected, actual []file.Location) {
t.Helper()
ignoreUnexported := cmpopts.IgnoreFields(file.LocationData{}, "ref")
ignoreMetadata := cmpopts.IgnoreFields(file.LocationMetadata{}, "Annotations")
ignoreFS := cmpopts.IgnoreFields(file.Coordinates{}, "FileSystemID")
sort.Sort(file.Locations(expected))
sort.Sort(file.Locations(actual))
if d := cmp.Diff(expected, actual,
ignoreUnexported,
ignoreFS,
ignoreMetadata,
); d != "" {
t.Errorf("unexpected locations (-want +got):\n%s", d)
}
}
+4
View File
@@ -0,0 +1,4 @@
package internal
//go:generate go run ./sourcemetadata/generate/main.go
//go:generate go run ./packagemetadata/generate/main.go
+1
View File
@@ -0,0 +1 @@
Please see [schema/json/README.md](../../../schema/json/README.md) for more information on the JSON schema files in this directory.
+169
View File
@@ -0,0 +1,169 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"github.com/invopop/jsonschema"
"github.com/anchore/syft/internal"
syftJsonModel "github.com/anchore/syft/syft/formats/syftjson/model"
"github.com/anchore/syft/syft/internal/packagemetadata"
)
/*
This method of creating the JSON schema only captures strongly typed fields for the purpose of integrations between syft
JSON output and integrations. The downside to this approach is that any values and types used on weakly typed fields
are not captured (empty interfaces). This means that pkg.Package.Metadata is not validated at this time. This approach
can be extended to include specific package metadata struct shapes in the future.
*/
func main() {
write(encode(build()))
}
func schemaID() jsonschema.ID {
// Today we do not host the schemas at this address, but per the JSON schema spec we should be referencing
// the schema by a URL in a domain we control. This is a placeholder for now.
return jsonschema.ID(fmt.Sprintf("anchore.io/schema/syft/json/%s", internal.JSONSchemaVersion))
}
func assembleTypeContainer(items []any) any {
structFields := make([]reflect.StructField, len(items))
for i, item := range items {
itemType := reflect.TypeOf(item)
fieldName := itemType.Name()
structFields[i] = reflect.StructField{
Name: fieldName,
Type: itemType,
}
}
structType := reflect.StructOf(structFields)
return reflect.New(structType).Elem().Interface()
}
func build() *jsonschema.Schema {
reflector := &jsonschema.Reflector{
BaseSchemaID: schemaID(),
AllowAdditionalProperties: true,
Namer: func(r reflect.Type) string {
return strings.TrimPrefix(r.Name(), "JSON")
},
}
pkgMetadataContainer := assembleTypeContainer(packagemetadata.AllTypes())
pkgMetadataContainerType := reflect.TypeOf(pkgMetadataContainer)
// srcMetadataContainer := assembleTypeContainer(sourcemetadata.AllTypes())
// srcMetadataContainerType := reflect.TypeOf(srcMetadataContainer)
documentSchema := reflector.ReflectFromType(reflect.TypeOf(&syftJsonModel.Document{}))
pkgMetadataSchema := reflector.ReflectFromType(reflect.TypeOf(pkgMetadataContainer))
// srcMetadataSchema := reflector.ReflectFromType(reflect.TypeOf(srcMetadataContainer))
// TODO: add source metadata types
// inject the definitions of all packages metadatas into the schema definitions
var metadataNames []string
for name, definition := range pkgMetadataSchema.Definitions {
if name == pkgMetadataContainerType.Name() {
// ignore the definition for the fake container
continue
}
documentSchema.Definitions[name] = definition
if strings.HasSuffix(name, "Metadata") {
metadataNames = append(metadataNames, name)
}
}
// ensure the generated list of names is stable between runs
sort.Strings(metadataNames)
var metadataTypes = []map[string]string{
// allow for no metadata to be provided
{"type": "null"},
}
for _, name := range metadataNames {
metadataTypes = append(metadataTypes, map[string]string{
"$ref": fmt.Sprintf("#/$defs/%s", name),
})
}
// set the "anyOf" field for Package.Metadata to be a conjunction of several types
documentSchema.Definitions["Package"].Properties.Set("metadata", map[string][]map[string]string{
"anyOf": metadataTypes,
})
return documentSchema
}
func encode(schema *jsonschema.Schema) []byte {
var newSchemaBuffer = new(bytes.Buffer)
enc := json.NewEncoder(newSchemaBuffer)
// prevent > and < from being escaped in the payload
enc.SetEscapeHTML(false)
enc.SetIndent("", " ")
err := enc.Encode(&schema)
if err != nil {
panic(err)
}
return newSchemaBuffer.Bytes()
}
func write(schema []byte) {
repoRoot, err := packagemetadata.RepoRoot()
if err != nil {
fmt.Println("unable to determine repo root")
os.Exit(1)
}
schemaPath := filepath.Join(repoRoot, "schema", "json", fmt.Sprintf("schema-%s.json", internal.JSONSchemaVersion))
if _, err := os.Stat(schemaPath); !os.IsNotExist(err) {
// check if the schema is the same...
existingFh, err := os.Open(schemaPath)
if err != nil {
panic(err)
}
existingSchemaBytes, err := io.ReadAll(existingFh)
if err != nil {
panic(err)
}
if bytes.Equal(existingSchemaBytes, schema) {
// the generated schema is the same, bail with no error :)
fmt.Println("No change to the existing schema!")
os.Exit(0)
}
// the generated schema is different, bail with error :(
fmt.Printf("Cowardly refusing to overwrite existing schema (%s)!\nSee the schema/json/README.md for how to increment\n", schemaPath)
os.Exit(1)
}
fh, err := os.Create(schemaPath)
if err != nil {
panic(err)
}
_, err = fh.Write(schema)
if err != nil {
panic(err)
}
defer fh.Close()
fmt.Printf("Wrote new schema to %q\n", schemaPath)
}
@@ -0,0 +1,150 @@
package packagemetadata
import (
"fmt"
"go/ast"
"go/parser"
"go/token"
"os/exec"
"path/filepath"
"sort"
"strings"
"unicode"
"github.com/scylladb/go-set/strset"
)
var metadataExceptions = strset.New(
"FileMetadata",
)
func DiscoverTypeNames() ([]string, error) {
root, err := RepoRoot()
if err != nil {
return nil, err
}
files, err := filepath.Glob(filepath.Join(root, "syft/pkg/*.go"))
if err != nil {
return nil, err
}
return findMetadataDefinitionNames(files...)
}
func RepoRoot() (string, error) {
root, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
if err != nil {
return "", fmt.Errorf("unable to find repo root dir: %+v", err)
}
absRepoRoot, err := filepath.Abs(strings.TrimSpace(string(root)))
if err != nil {
return "", fmt.Errorf("unable to get abs path to repo root: %w", err)
}
return absRepoRoot, nil
}
func findMetadataDefinitionNames(paths ...string) ([]string, error) {
names := strset.New()
usedNames := strset.New()
for _, path := range paths {
metadataDefinitions, usedTypeNames, err := findMetadataDefinitionNamesInFile(path)
if err != nil {
return nil, err
}
// useful for debugging...
// fmt.Println(path)
// fmt.Println("Defs:", metadataDefinitions)
// fmt.Println("Used Types:", usedTypeNames)
// fmt.Println()
names.Add(metadataDefinitions...)
usedNames.Add(usedTypeNames...)
}
// any definition that is used within another struct should not be considered a top-level metadata definition
names.Remove(usedNames.List()...)
strNames := names.List()
sort.Strings(strNames)
// note: 30 is a point-in-time gut check. This number could be updated if new metadata definitions are added, but is not required.
// it is really intended to catch any major issues with the generation process that would generate, say, 0 definitions.
if len(strNames) < 30 {
return nil, fmt.Errorf("not enough metadata definitions found (discovered: " + fmt.Sprintf("%d", len(strNames)) + ")")
}
return strNames, nil
}
func findMetadataDefinitionNamesInFile(path string) ([]string, []string, error) {
// set up the parser
fs := token.NewFileSet()
f, err := parser.ParseFile(fs, path, nil, parser.ParseComments)
if err != nil {
return nil, nil, err
}
var metadataDefinitions []string
var usedTypeNames []string
for _, decl := range f.Decls {
// check if the declaration is a type declaration
spec, ok := decl.(*ast.GenDecl)
if !ok || spec.Tok != token.TYPE {
continue
}
// loop over all types declared in the type declaration
for _, typ := range spec.Specs {
// check if the type is a struct type
spec, ok := typ.(*ast.TypeSpec)
if !ok || spec.Type == nil {
continue
}
structType, ok := spec.Type.(*ast.StructType)
if !ok {
continue
}
// check if the struct type ends with "Metadata"
name := spec.Name.String()
// only look for exported types that end with "Metadata"
if isMetadataTypeCandidate(name) {
// print the full declaration of the struct type
metadataDefinitions = append(metadataDefinitions, name)
usedTypeNames = append(usedTypeNames, typeNamesUsedInStruct(structType)...)
}
}
}
return metadataDefinitions, usedTypeNames, nil
}
func typeNamesUsedInStruct(structType *ast.StructType) []string {
// recursively find all type names used in the struct type
var names []string
for i := range structType.Fields.List {
// capture names of all of the types (not field names)
ast.Inspect(structType.Fields.List[i].Type, func(n ast.Node) bool {
ident, ok := n.(*ast.Ident)
if !ok {
return true
}
// add the type name to the list
names = append(names, ident.Name)
// continue inspecting
return true
})
}
return names
}
func isMetadataTypeCandidate(name string) bool {
return len(name) > 0 &&
strings.HasSuffix(name, "Metadata") &&
unicode.IsUpper(rune(name[0])) && // must be exported
!metadataExceptions.Has(name)
}
@@ -0,0 +1,55 @@
package main
import (
"fmt"
"os"
"github.com/dave/jennifer/jen"
"github.com/anchore/syft/syft/internal/packagemetadata"
)
// This program is invoked from syft/internal and generates packagemetadata/generated.go
const (
pkgImport = "github.com/anchore/syft/syft/pkg"
path = "packagemetadata/generated.go"
)
func main() {
typeNames, err := packagemetadata.DiscoverTypeNames()
if err != nil {
panic(fmt.Errorf("unable to get all metadata type names: %w", err))
}
fmt.Printf("updating package metadata type list with %+v types\n", len(typeNames))
f := jen.NewFile("packagemetadata")
f.HeaderComment("DO NOT EDIT: generated by syft/internal/packagemetadata/generate/main.go")
f.ImportName(pkgImport, "pkg")
f.Comment("AllTypes returns a list of all pkg metadata types that syft supports (that are represented in the pkg.Package.Metadata field).")
f.Func().Id("AllTypes").Params().Index().Any().BlockFunc(func(g *jen.Group) {
g.ReturnFunc(func(g *jen.Group) {
g.Index().Any().ValuesFunc(func(g *jen.Group) {
for _, typeName := range typeNames {
g.Qual(pkgImport, typeName).Values()
}
})
})
})
rendered := fmt.Sprintf("%#v", f)
fh, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
panic(fmt.Errorf("unable to open file: %w", err))
}
_, err = fh.WriteString(rendered)
if err != nil {
panic(fmt.Errorf("unable to write file: %w", err))
}
if err := fh.Close(); err != nil {
panic(fmt.Errorf("unable to close file: %w", err))
}
}
@@ -0,0 +1,10 @@
// DO NOT EDIT: generated by syft/internal/packagemetadata/generate/main.go
package packagemetadata
import "github.com/anchore/syft/syft/pkg"
// AllTypes returns a list of all pkg metadata types that syft supports (that are represented in the pkg.Package.Metadata field).
func AllTypes() []any {
return []any{pkg.AlpmMetadata{}, pkg.ApkMetadata{}, pkg.BinaryMetadata{}, pkg.CargoPackageMetadata{}, pkg.CocoapodsMetadata{}, pkg.ConanLockMetadata{}, pkg.ConanMetadata{}, pkg.DartPubMetadata{}, pkg.DotnetDepsMetadata{}, pkg.DpkgMetadata{}, pkg.GemMetadata{}, pkg.GolangBinMetadata{}, pkg.GolangModMetadata{}, pkg.HackageMetadata{}, pkg.JavaMetadata{}, pkg.KbPackageMetadata{}, pkg.LinuxKernelMetadata{}, pkg.LinuxKernelModuleMetadata{}, pkg.MixLockMetadata{}, pkg.NixStoreMetadata{}, pkg.NpmPackageJSONMetadata{}, pkg.NpmPackageLockJSONMetadata{}, pkg.PhpComposerJSONMetadata{}, pkg.PortageMetadata{}, pkg.PythonPackageMetadata{}, pkg.PythonPipfileLockMetadata{}, pkg.PythonRequirementsMetadata{}, pkg.RDescriptionFileMetadata{}, pkg.RebarLockMetadata{}, pkg.RpmMetadata{}}
}
+13
View File
@@ -0,0 +1,13 @@
package packagemetadata
import (
"reflect"
)
func AllNames() []string {
names := make([]string, 0)
for _, t := range AllTypes() {
names = append(names, reflect.TypeOf(t).Name())
}
return names
}
@@ -0,0 +1,25 @@
package packagemetadata
import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAllNames(t *testing.T) {
// note: this is a form of completion testing relative to the current code base.
expected, err := DiscoverTypeNames()
require.NoError(t, err)
actual := AllNames()
// ensure that the codebase (from ast analysis) reflects the latest code generated state
if !assert.ElementsMatch(t, expected, actual) {
t.Errorf("metadata types not fully represented: \n%s", cmp.Diff(expected, actual))
t.Log("did you add a new pkg.*Metadata type without updating the JSON schema?")
t.Log("if so, you need to update the schema version and regenerate the JSON schema (make generate-json-schema)")
}
}
@@ -0,0 +1,69 @@
package sourcemetadata
import (
"reflect"
"testing"
)
type CompletionTester struct {
saw []any
valid []any
ignore []any
}
func NewCompletionTester(t testing.TB, ignore ...any) *CompletionTester {
tester := &CompletionTester{
valid: AllTypes(),
ignore: ignore,
}
t.Cleanup(func() {
t.Helper()
tester.validate(t)
})
return tester
}
func (tr *CompletionTester) Tested(t testing.TB, m any) {
t.Helper()
if m == nil {
return
}
if len(tr.valid) == 0 {
t.Fatal("no valid metadata types to test against")
}
ty := reflect.TypeOf(m)
for _, v := range tr.valid {
if reflect.TypeOf(v) == ty {
tr.saw = append(tr.saw, m)
return
}
}
t.Fatalf("tested metadata type is not valid: %s", ty.Name())
}
func (tr *CompletionTester) validate(t testing.TB) {
t.Helper()
count := make(map[reflect.Type]int)
for _, m := range tr.saw {
count[reflect.TypeOf(m)]++
}
validations:
for _, v := range tr.valid {
ty := reflect.TypeOf(v)
for _, ignore := range tr.ignore {
if ty == reflect.TypeOf(ignore) {
// skip ignored types
continue validations
}
}
if c, exists := count[ty]; c == 0 || !exists {
t.Errorf("metadata type %s is not covered by a test", ty.Name())
}
}
}
@@ -0,0 +1,148 @@
package sourcemetadata
import (
"fmt"
"go/ast"
"go/parser"
"go/token"
"os/exec"
"path/filepath"
"sort"
"strings"
"unicode"
"github.com/scylladb/go-set/strset"
)
var metadataExceptions = strset.New()
func DiscoverTypeNames() ([]string, error) {
root, err := repoRoot()
if err != nil {
return nil, err
}
files, err := filepath.Glob(filepath.Join(root, "syft/source/*.go"))
if err != nil {
return nil, err
}
return findMetadataDefinitionNames(files...)
}
func repoRoot() (string, error) {
root, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
if err != nil {
return "", fmt.Errorf("unable to find repo root dir: %+v", err)
}
absRepoRoot, err := filepath.Abs(strings.TrimSpace(string(root)))
if err != nil {
return "", fmt.Errorf("unable to get abs path to repo root: %w", err)
}
return absRepoRoot, nil
}
func findMetadataDefinitionNames(paths ...string) ([]string, error) {
names := strset.New()
usedNames := strset.New()
for _, path := range paths {
metadataDefinitions, usedTypeNames, err := findMetadataDefinitionNamesInFile(path)
if err != nil {
return nil, err
}
// useful for debugging...
// fmt.Println(path)
// fmt.Println("Defs:", metadataDefinitions)
// fmt.Println("Used Types:", usedTypeNames)
// fmt.Println()
names.Add(metadataDefinitions...)
usedNames.Add(usedTypeNames...)
}
// any definition that is used within another struct should not be considered a top-level metadata definition
names.Remove(usedNames.List()...)
strNames := names.List()
sort.Strings(strNames)
// note: 3 is a point-in-time gut check. This number could be updated if new metadata definitions are added, but is not required.
// it is really intended to catch any major issues with the generation process that would generate, say, 0 definitions.
if len(strNames) < 3 {
return nil, fmt.Errorf("not enough metadata definitions found (discovered: " + fmt.Sprintf("%d", len(strNames)) + ")")
}
return strNames, nil
}
func findMetadataDefinitionNamesInFile(path string) ([]string, []string, error) {
// set up the parser
fs := token.NewFileSet()
f, err := parser.ParseFile(fs, path, nil, parser.ParseComments)
if err != nil {
return nil, nil, err
}
var metadataDefinitions []string
var usedTypeNames []string
for _, decl := range f.Decls {
// check if the declaration is a type declaration
spec, ok := decl.(*ast.GenDecl)
if !ok || spec.Tok != token.TYPE {
continue
}
// loop over all types declared in the type declaration
for _, typ := range spec.Specs {
// check if the type is a struct type
spec, ok := typ.(*ast.TypeSpec)
if !ok || spec.Type == nil {
continue
}
structType, ok := spec.Type.(*ast.StructType)
if !ok {
continue
}
// check if the struct type ends with "Metadata"
name := spec.Name.String()
// only look for exported types that end with "Metadata"
if isMetadataTypeCandidate(name) {
// print the full declaration of the struct type
metadataDefinitions = append(metadataDefinitions, name)
usedTypeNames = append(usedTypeNames, typeNamesUsedInStruct(structType)...)
}
}
}
return metadataDefinitions, usedTypeNames, nil
}
func typeNamesUsedInStruct(structType *ast.StructType) []string {
// recursively find all type names used in the struct type
var names []string
for i := range structType.Fields.List {
// capture names of all of the types (not field names)
ast.Inspect(structType.Fields.List[i].Type, func(n ast.Node) bool {
ident, ok := n.(*ast.Ident)
if !ok {
return true
}
// add the type name to the list
names = append(names, ident.Name)
// continue inspecting
return true
})
}
return names
}
func isMetadataTypeCandidate(name string) bool {
return len(name) > 0 &&
strings.HasSuffix(name, "Metadata") &&
unicode.IsUpper(rune(name[0])) && // must be exported
!metadataExceptions.Has(name)
}
@@ -0,0 +1,55 @@
package main
import (
"fmt"
"os"
"github.com/dave/jennifer/jen"
"github.com/anchore/syft/syft/internal/sourcemetadata"
)
// This program is invoked from syft/internal and generates sourcemetadata/generated.go
const (
srcImport = "github.com/anchore/syft/syft/source"
path = "sourcemetadata/generated.go"
)
func main() {
typeNames, err := sourcemetadata.DiscoverTypeNames()
if err != nil {
panic(fmt.Errorf("unable to get all metadata type names: %w", err))
}
fmt.Printf("updating source metadata type list with %+v types\n", len(typeNames))
f := jen.NewFile("sourcemetadata")
f.HeaderComment("DO NOT EDIT: generated by syft/internal/sourcemetadata/generate/main.go")
f.ImportName(srcImport, "source")
f.Comment("AllTypes returns a list of all source metadata types that syft supports (that are represented in the source.Description.Metadata field).")
f.Func().Id("AllTypes").Params().Index().Any().BlockFunc(func(g *jen.Group) {
g.ReturnFunc(func(g *jen.Group) {
g.Index().Any().ValuesFunc(func(g *jen.Group) {
for _, typeName := range typeNames {
g.Qual(srcImport, typeName).Values()
}
})
})
})
rendered := fmt.Sprintf("%#v", f)
fh, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
panic(fmt.Errorf("unable to open file: %w", err))
}
_, err = fh.WriteString(rendered)
if err != nil {
panic(fmt.Errorf("unable to write file: %w", err))
}
if err := fh.Close(); err != nil {
panic(fmt.Errorf("unable to close file: %w", err))
}
}
+10
View File
@@ -0,0 +1,10 @@
// DO NOT EDIT: generated by syft/internal/sourcemetadata/generate/main.go
package sourcemetadata
import "github.com/anchore/syft/syft/source"
// AllTypes returns a list of all source metadata types that syft supports (that are represented in the source.Description.Metadata field).
func AllTypes() []any {
return []any{source.DirectorySourceMetadata{}, source.FileSourceMetadata{}, source.StereoscopeImageSourceMetadata{}}
}
+41
View File
@@ -0,0 +1,41 @@
package sourcemetadata
import (
"reflect"
"strings"
"github.com/anchore/syft/syft/source"
)
var jsonNameFromType = map[reflect.Type][]string{
reflect.TypeOf(source.DirectorySourceMetadata{}): {"directory", "dir"},
reflect.TypeOf(source.FileSourceMetadata{}): {"file"},
reflect.TypeOf(source.StereoscopeImageSourceMetadata{}): {"image"},
}
func AllNames() []string {
names := make([]string, 0)
for _, t := range AllTypes() {
names = append(names, reflect.TypeOf(t).Name())
}
return names
}
func JSONName(metadata any) string {
if vs, exists := jsonNameFromType[reflect.TypeOf(metadata)]; exists {
return vs[0]
}
return ""
}
func ReflectTypeFromJSONName(name string) reflect.Type {
name = strings.ToLower(name)
for t, vs := range jsonNameFromType {
for _, v := range vs {
if v == name {
return t
}
}
}
return nil
}
@@ -0,0 +1,29 @@
package sourcemetadata
import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAllNames(t *testing.T) {
// note: this is a form of completion testing relative to the current code base.
expected, err := DiscoverTypeNames()
require.NoError(t, err)
actual := AllNames()
// ensure that the codebase (from ast analysis) reflects the latest code generated state
if !assert.ElementsMatch(t, expected, actual) {
t.Errorf("metadata types not fully represented: \n%s", cmp.Diff(expected, actual))
t.Log("did you add a new source.*Metadata type without updating the JSON schema?")
t.Log("if so, you need to update the schema version and regenerate the JSON schema (make generate-json-schema)")
}
for _, ty := range AllTypes() {
assert.NotEmpty(t, JSONName(ty), "metadata type %q does not have a JSON name", ty)
}
}
+41
View File
@@ -0,0 +1,41 @@
package windows
import (
"path"
"path/filepath"
"runtime"
"strings"
)
const windowsGoOS = "windows"
func HostRunningOnWindows() bool {
return runtime.GOOS == windowsGoOS
}
func ToPosix(windowsPath string) (posixPath string) {
// volume should be encoded at the start (e.g /c/<path>) where c is the volume
volumeName := filepath.VolumeName(windowsPath)
pathWithoutVolume := strings.TrimPrefix(windowsPath, volumeName)
volumeLetter := strings.ToLower(strings.TrimSuffix(volumeName, ":"))
// translate non-escaped backslash to forwardslash
translatedPath := strings.ReplaceAll(pathWithoutVolume, "\\", "/")
// always have `/` as the root... join all components, e.g.:
// convert: C:\\some\windows\Place
// into: /c/some/windows/Place
return path.Clean("/" + strings.Join([]string{volumeLetter, translatedPath}, "/"))
}
func FromPosix(posixPath string) (windowsPath string) {
// decode the volume (e.g. /c/<path> --> C:\\) - There should always be a volume name.
pathFields := strings.Split(posixPath, "/")
volumeName := strings.ToUpper(pathFields[1]) + `:\\`
// translate non-escaped forward slashes into backslashes
remainingTranslatedPath := strings.Join(pathFields[2:], "\\")
// combine volume name and backslash components
return filepath.Clean(volumeName + remainingTranslatedPath)
}