Compare commits

...
Author SHA1 Message Date
Will Murphy d54d20a563 use new atomic stager struct
Signed-off-by: Will Murphy <will.murphy@anchore.com>
2023-09-08 10:13:52 -04:00
Will Murphy abfd244dc5 test for race conditions
Signed-off-by: Will Murphy <will.murphy@anchore.com>
2023-09-08 08:42:19 -04:00
Will Murphy 8e9d1d5e91 Fix race in current stage fo go-progress
Signed-off-by: Will Murphy <will.murphy@anchore.com>
2023-09-08 07:48:42 -04:00
12 changed files with 41 additions and 57 deletions
+7 -2
View File
@@ -70,7 +70,7 @@ all: static-analysis test ## Run all linux-based checks (linting, license check,
static-analysis: check-go-mod-tidy lint check-licenses check-json-schema-drift ## Run all static analysis checks
.PHONY: test
test: unit integration validate-cyclonedx-schema benchmark cli ## Run all tests (currently unit, integration, linux compare, and cli tests)
test: unit race integration validate-cyclonedx-schema benchmark cli ## Run all tests (currently unit, integration, linux compare, and cli tests)
## Bootstrapping targets #################################
@@ -147,9 +147,14 @@ check-json-schema-drift:
.PHONY: unit
unit: $(TEMP_DIR) fixtures ## Run unit tests (with coverage)
$(call title,Running unit tests)
go test -coverprofile $(TEMP_DIR)/unit-coverage-details.txt $(shell go list ./... | grep -v anchore/syft/test)
go test -race -coverprofile $(TEMP_DIR)/unit-coverage-details.txt $(shell go list ./... | grep -v anchore/syft/test)
@.github/scripts/coverage.py $(COVERAGE_THRESHOLD) $(TEMP_DIR)/unit-coverage-details.txt
.PHONY: race
race:
$(call title,Running main with race detector enabled)
go run -race cmd/syft/main.go alpine:latest
.PHONY: integration
integration: ## Run integration tests
$(call title,Running integration tests)
+7 -7
View File
@@ -65,7 +65,7 @@ func (m *Handler) handleAttestationStarted(e partybus.Event) []tea.Model {
return nil
}
stage := progress.Stage{}
stage := progress.NewAtomicStage("")
tsk := m.newTaskProgress(
taskprogress.Title{
@@ -79,7 +79,7 @@ func (m *Handler) handleAttestationStarted(e partybus.Event) []tea.Model {
progress.Stager
}{
Progressable: prog,
Stager: &stage,
Stager: stage,
},
),
)
@@ -94,7 +94,7 @@ func (m *Handler) handleAttestationStarted(e partybus.Event) []tea.Model {
return []tea.Model{
tsk,
newLogFrame(newBackgroundLineReader(m.Running, reader, &stage), prog, borderStyle),
newLogFrame(newBackgroundLineReader(m.Running, reader, stage), prog, borderStyle),
}
}
@@ -108,7 +108,7 @@ func newLogFrame(reader *backgroundLineReader, prog progress.Progressable, borde
}
}
func newBackgroundLineReader(wg *sync.WaitGroup, reader io.Reader, stage *progress.Stage) *backgroundLineReader {
func newBackgroundLineReader(wg *sync.WaitGroup, reader io.Reader, stage *progress.AtomicStage) *backgroundLineReader {
r := &backgroundLineReader{
limit: 7,
lock: &sync.RWMutex{},
@@ -131,7 +131,7 @@ func newBackgroundLineReader(wg *sync.WaitGroup, reader io.Reader, stage *progre
return r
}
func (l *backgroundLineReader) read(reader io.Reader, stage *progress.Stage) {
func (l *backgroundLineReader) read(reader io.Reader, stage *progress.AtomicStage) {
s := bufio.NewScanner(reader)
for s.Scan() {
@@ -146,9 +146,9 @@ func (l *backgroundLineReader) read(reader io.Reader, stage *progress.Stage) {
if len(fields) == 2 {
present = fmt.Sprintf("transparency log index: %s", fields[1])
}
stage.Current = present
stage.Set(present)
} else if strings.Contains(text, "WARNING: skipping transparency log upload") {
stage.Current = "transparency log upload skipped"
stage.Set("transparency log upload skipped")
}
// only show the last X lines of the shell output
+2 -6
View File
@@ -33,9 +33,7 @@ func TestHandler_handleFetchImage(t *testing.T) {
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
Stager: progress.NewAtomicStage("current"),
}
return partybus.Event{
@@ -58,9 +56,7 @@ func TestHandler_handleFetchImage(t *testing.T) {
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
Stager: progress.NewAtomicStage("current"),
}
return partybus.Event{
@@ -33,9 +33,7 @@ func TestHandler_handleFileDigestsCatalogerStarted(t *testing.T) {
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
Stager: progress.NewAtomicStage("current"),
}
return partybus.Event{
@@ -57,9 +55,7 @@ func TestHandler_handleFileDigestsCatalogerStarted(t *testing.T) {
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
Stager: progress.NewAtomicStage("current"),
}
return partybus.Event{
+2 -6
View File
@@ -33,9 +33,7 @@ func TestHandler_handleFileIndexingStarted(t *testing.T) {
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
Stager: progress.NewAtomicStage("current"),
}
return partybus.Event{
@@ -58,9 +56,7 @@ func TestHandler_handleFileIndexingStarted(t *testing.T) {
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
Stager: progress.NewAtomicStage("current"),
}
return partybus.Event{
@@ -33,9 +33,7 @@ func TestHandler_handleFileMetadataCatalogerStarted(t *testing.T) {
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
Stager: progress.NewAtomicStage("current"),
}
return partybus.Event{
@@ -57,9 +55,7 @@ func TestHandler_handleFileMetadataCatalogerStarted(t *testing.T) {
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
Stager: progress.NewAtomicStage("current"),
}
return partybus.Event{
@@ -25,9 +25,7 @@ func TestHandler_handleSecretsCatalogerStarted(t *testing.T) {
{
name: "cataloging in progress",
eventFn: func(t *testing.T) partybus.Event {
stage := &progress.Stage{
Current: "current",
}
stage := progress.NewAtomicStage("current")
secretsDiscovered := progress.NewManual(-1)
secretsDiscovered.Set(64)
prog := progress.NewManual(72)
@@ -47,9 +45,7 @@ func TestHandler_handleSecretsCatalogerStarted(t *testing.T) {
{
name: "cataloging complete",
eventFn: func(t *testing.T) partybus.Event {
stage := &progress.Stage{
Current: "current",
}
stage := progress.NewAtomicStage("current")
secretsDiscovered := progress.NewManual(-1)
secretsDiscovered.Set(64)
prog := progress.NewManual(72)
+3 -3
View File
@@ -43,7 +43,7 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
stage, prog := digestsCatalogingProgress(int64(len(locations)))
for _, location := range locations {
stage.Current = location.RealPath
stage.Set(location.RealPath)
result, err := i.catalogLocation(resolver, location)
if errors.Is(err, ErrUndigestableFile) {
@@ -91,8 +91,8 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
return digests, nil
}
func digestsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) {
stage := &progress.Stage{}
func digestsCatalogingProgress(locations int64) (*progress.AtomicStage, *progress.Manual) {
stage := progress.NewAtomicStage("")
prog := progress.NewManual(locations)
bus.Publish(partybus.Event{
@@ -38,7 +38,7 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
stage, prog := metadataCatalogingProgress(int64(len(locations)))
for location := range locations {
stage.Current = location.RealPath
stage.Set(location.RealPath)
metadata, err := resolver.FileMetadataByLocation(location)
if err != nil {
return nil, err
@@ -52,8 +52,8 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
return results, nil
}
func metadataCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) {
stage := &progress.Stage{}
func metadataCatalogingProgress(locations int64) (*progress.AtomicStage, *progress.Manual) {
stage := progress.NewAtomicStage("")
prog := progress.NewManual(locations)
bus.Publish(partybus.Event{
+3 -3
View File
@@ -47,7 +47,7 @@ func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates][]file
locations := internal2.AllRegularFiles(resolver)
stage, prog, secretsDiscovered := secretsCatalogingProgress(int64(len(locations)))
for _, location := range locations {
stage.Current = location.RealPath
stage.Set(location.RealPath)
result, err := i.catalogLocation(resolver, location)
if internal.IsErrPathPermission(err) {
log.Debugf("secrets cataloger skipping - %+v", err)
@@ -139,8 +139,8 @@ type Monitor struct {
progress.Progressable
}
func secretsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual, *progress.Manual) {
stage := &progress.Stage{}
func secretsCatalogingProgress(locations int64) (*progress.AtomicStage, *progress.Manual, *progress.Manual) {
stage := progress.NewAtomicStage("")
secretsDiscovered := &progress.Manual{}
prog := progress.NewManual(locations)
@@ -58,7 +58,7 @@ func (r *directoryIndexer) build() (filetree.Reader, filetree.IndexReader, error
return r.tree, r.index, indexAllRoots(r.path, r.indexTree)
}
func indexAllRoots(root string, indexer func(string, *progress.Stage) ([]string, error)) error {
func indexAllRoots(root string, indexer func(string, *progress.AtomicStage) ([]string, error)) error {
// why account for multiple roots? To cover cases when there is a symlink that references above the root path,
// in which case we need to additionally index where the link resolves to. it's for this reason why the filetree
// must be relative to the root of the filesystem (and not just relative to the given path).
@@ -95,7 +95,7 @@ loop:
return nil
}
func (r *directoryIndexer) indexTree(root string, stager *progress.Stage) ([]string, error) {
func (r *directoryIndexer) indexTree(root string, stager *progress.AtomicStage) ([]string, error) {
log.WithFields("path", root).Trace("indexing filetree")
var roots []string
@@ -138,10 +138,9 @@ func (r *directoryIndexer) indexTree(root string, stager *progress.Stage) ([]str
err = filepath.Walk(root,
func(path string, info os.FileInfo, err error) error {
stager.Current = path
stager.Set(path)
newRoot, err := r.indexPath(path, info, err)
if err != nil {
return err
}
@@ -173,7 +172,7 @@ func isRealPath(root string) (bool, error) {
return rootParent == realRootParent, nil
}
func (r *directoryIndexer) indexBranch(root string, stager *progress.Stage) ([]string, error) {
func (r *directoryIndexer) indexBranch(root string, stager *progress.AtomicStage) ([]string, error) {
rootRealPath, err := filepath.EvalSymlinks(root)
if err != nil {
return nil, err
@@ -200,7 +199,7 @@ func (r *directoryIndexer) indexBranch(root string, stager *progress.Stage) ([]s
targetPath = p
}
stager.Current = targetPath
stager.Set(targetPath)
lstat, err := os.Lstat(targetPath)
newRoot, err := r.indexPath(targetPath, lstat, err)
@@ -465,8 +464,8 @@ func requireFileInfo(_ string, info os.FileInfo, _ error) error {
return nil
}
func indexingProgress(path string) (*progress.Stage, *progress.Manual) {
stage := &progress.Stage{}
func indexingProgress(path string) (*progress.AtomicStage, *progress.Manual) {
stage := progress.NewAtomicStage("")
prog := progress.NewManual(-1)
bus.Publish(partybus.Event{
@@ -22,7 +22,7 @@ type indexerMock struct {
additionalRoots map[string][]string
}
func (m *indexerMock) indexer(s string, _ *progress.Stage) ([]string, error) {
func (m *indexerMock) indexer(s string, _ *progress.AtomicStage) ([]string, error) {
m.observedRoots = append(m.observedRoots, s)
return m.additionalRoots[s], nil
}