Compare commits

..
Author SHA1 Message Date
Weston Steimel 00f3b44e6f fix(java): improve identification for org.neo4j.procedure artifacts
Signed-off-by: Weston Steimel <weston.steimel@proton.me>
2023-12-01 15:02:41 +00:00
William Murphy 13758260dd fix: bump fangs for ptr summarize fix (#2387)
Previously, pointers to primitive types in config summarization could be
printed literally (like "0x123aefef"). Pull in fangs to get the fix for
this.

Signed-off-by: Will Murphy <will.murphy@anchore.com>
2023-12-01 14:37:42 +00:00
Weston Steimel 11039f4b4e fix(java): improve identification for org.elasticsearch artifacts (#2383)
Signed-off-by: Weston Steimel <weston.steimel@proton.me>
2023-12-01 09:22:33 -05:00
Weston Steimel 413ffdb233 fix(java): improve identification for org.apache.geode artifacts (#2382)
Signed-off-by: Weston Steimel <weston.steimel@proton.me>
2023-12-01 12:48:15 +00:00
Weston Steimel e53fe51612 fix(java): improve identification for org.apache.tomcat.embed artifacts (#2381)
Signed-off-by: Weston Steimel <weston.steimel@proton.me>
2023-12-01 07:13:13 -05:00
Weston Steimel facbc486a8 fix(java): improve identification for io.projectreactor.netty artifacts (#2378)
Signed-off-by: Weston Steimel <weston.steimel@proton.me>
2023-12-01 07:09:06 -05:00
Weston Steimel 5d42a349e6 fix(java): improve identification for org.eclipse.platform artifacts (#2349)
Signed-off-by: Weston Steimel <weston.steimel@proton.me>
2023-11-30 16:02:03 -05:00
Alex Goodman 4adfbeb5f0 Generalize UI events for cataloging tasks (#2369)
* generalize ui events for cataloging tasks

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* moderate review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* incorporate review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* rename cataloger task progress object

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* migrate cataloger task fn to bus helper

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
2023-11-30 16:25:50 +00:00
anchore-actions-token-generator[bot]andspiffcs b943da6433 chore(deps): update tools to latest versions (#2376)
Signed-off-by: GitHub <noreply@github.com>
Co-authored-by: spiffcs <spiffcs@users.noreply.github.com>
2023-11-30 09:25:02 -05:00
dependabot[bot] e8119acf93 chore(deps): bump github.com/google/go-containerregistry (#2377)
Bumps [github.com/google/go-containerregistry](https://github.com/google/go-containerregistry) from 0.16.1 to 0.17.0.
- [Release notes](https://github.com/google/go-containerregistry/releases)
- [Changelog](https://github.com/google/go-containerregistry/blob/main/.goreleaser.yml)
- [Commits](https://github.com/google/go-containerregistry/compare/v0.16.1...v0.17.0)

---
updated-dependencies:
- dependency-name: github.com/google/go-containerregistry
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-30 09:24:25 -05:00
Laurent Goderre 06b9a79e3d chore: fix tests failing due to Mac Rosetta cache (#2374)
Signed-off-by: Laurent Goderre <laurent.goderre@docker.com>
2023-11-29 18:39:28 +00:00
Keith Zantow ef5c1651ef fix: improve dotnet portable executable identification (#2133)
Signed-off-by: Keith Zantow <kzantow@gmail.com>
2023-11-29 12:51:24 -05:00
39 changed files with 1533 additions and 1144 deletions
+1 -1
View File
@@ -83,7 +83,7 @@ tools:
- name: task
version:
want: v3.31.0
want: v3.32.0
method: github-release
with:
repo: go-task/task
@@ -1,16 +1,24 @@
[TestHandler_handleCatalogerTaskStarted/cataloging_task_in_progress - 1]
some task title [some value]
⠙ Cataloging contents
⠙ some task title ━━━━━━━━━━━━━━━━━━━━ [some stage]
---
[TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_in_progress - 1]
└── some task title [some value]
⠙ Cataloging contents
└── ⠙ some task title ━━━━━━━━━━━━━━━━━━━━ [some stage]
---
[TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_complete - 1]
✔ └── some task done [some value]
⠙ Cataloging contents
└── ✔ some task done [some stage]
---
[TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_complete_--_hide_stage - 1]
⠙ Cataloging contents
└── ✔ some task done
---
[TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_complete_with_removal - 1]
⠙ Cataloging contents
---
+10 -3
View File
@@ -100,18 +100,21 @@ func TestHandler_handleAttestationStarted(t *testing.T) {
Height: 80,
}
models := handler.Handle(event)
models, _ := handler.Handle(event)
require.Len(t, models, 2)
t.Run("task line", func(t *testing.T) {
tsk, ok := models[0].(taskprogress.Model)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
gotModel := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
got := gotModel.View()
t.Log(got)
snaps.MatchSnapshot(t, got)
})
@@ -119,11 +122,15 @@ func TestHandler_handleAttestationStarted(t *testing.T) {
t.Run("log", func(t *testing.T) {
log, ok := models[1].(attestLogFrame)
require.True(t, ok)
got := runModel(t, log, tt.iterations, attestLogFrameTickMsg{
gotModel := runModel(t, log, tt.iterations, attestLogFrameTickMsg{
Time: time.Now(),
Sequence: log.sequence,
ID: log.id,
}, log.reader.running)
got := gotModel.View()
t.Log(got)
snaps.MatchSnapshot(t, got)
})
+91 -40
View File
@@ -3,70 +3,121 @@ package ui
import (
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
"github.com/google/uuid"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress"
"github.com/anchore/bubbly/bubbles/tree"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event/monitor"
syftEventParsers "github.com/anchore/syft/syft/event/parsers"
)
var _ progress.Stager = (*catalogerTaskStageAdapter)(nil)
// we standardize how rows are instantiated to ensure consistency in the appearance across the UI
type taskModelFactory func(title taskprogress.Title, opts ...taskprogress.Option) taskprogress.Model
type catalogerTaskStageAdapter struct {
mon *monitor.CatalogerTask
var _ tea.Model = (*catalogerTaskModel)(nil)
type catalogerTaskModel struct {
model tree.Model
modelFactory taskModelFactory
}
func newCatalogerTaskStageAdapter(mon *monitor.CatalogerTask) *catalogerTaskStageAdapter {
return &catalogerTaskStageAdapter{
mon: mon,
func newCatalogerTaskTreeModel(f taskModelFactory) *catalogerTaskModel {
t := tree.NewModel()
t.Padding = " "
t.RootsWithoutPrefix = true
return &catalogerTaskModel{
modelFactory: f,
model: t,
}
}
func (c catalogerTaskStageAdapter) Stage() string {
return c.mon.GetValue()
type newCatalogerTaskRowEvent struct {
info monitor.GenericTask
prog progress.StagedProgressable
}
func (m *Handler) handleCatalogerTaskStarted(e partybus.Event) []tea.Model {
mon, err := syftEventParsers.ParseCatalogerTaskStarted(e)
if err != nil {
log.WithFields("error", err).Warn("unable to parse event")
return nil
func (cts catalogerTaskModel) Init() tea.Cmd {
return cts.model.Init()
}
func (cts catalogerTaskModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
event, ok := msg.(newCatalogerTaskRowEvent)
if !ok {
model, cmd := cts.model.Update(msg)
cts.model = model.(tree.Model)
return cts, cmd
}
var prefix string
if mon.SubStatus {
// TODO: support list of sub-statuses, not just a single leaf
prefix = "└── "
}
info, prog := event.info, event.prog
tsk := m.newTaskProgress(
tsk := cts.modelFactory(
taskprogress.Title{
// TODO: prefix should not be part of the title, but instead a separate field that is aware of the tree structure
Default: prefix + mon.Title,
Running: prefix + mon.Title,
Success: prefix + mon.TitleOnCompletion,
Default: info.Title.Default,
Running: info.Title.WhileRunning,
Success: info.Title.OnSuccess,
},
taskprogress.WithStagedProgressable(
struct {
progress.Stager
progress.Progressable
}{
Progressable: mon.GetMonitor(),
Stager: newCatalogerTaskStageAdapter(mon),
},
),
taskprogress.WithStagedProgressable(prog),
)
// TODO: this isn't ideal since the model stays around after it is no longer needed, but it works for now
tsk.HideOnSuccess = mon.RemoveOnCompletion
tsk.HideStageOnSuccess = false
tsk.HideProgressOnSuccess = false
if info.Context != "" {
tsk.Context = []string{info.Context}
}
tsk.TitleStyle = lipgloss.NewStyle()
// TODO: this is a hack to get the spinner to not show up, but ideally the component would support making the spinner optional
tsk.Spinner.Spinner.Frames = []string{" "}
tsk.HideOnSuccess = info.HideOnSuccess
tsk.HideStageOnSuccess = info.HideStageOnSuccess
tsk.HideProgressOnSuccess = true
return []tea.Model{tsk}
if info.ParentID != "" {
tsk.TitleStyle = lipgloss.NewStyle()
}
if err := cts.model.Add(info.ParentID, info.ID, tsk); err != nil {
log.WithFields("error", err).Error("unable to add cataloger task to tree model")
}
return cts, tsk.Init()
}
func (cts catalogerTaskModel) View() string {
return cts.model.View()
}
func (m *Handler) handleCatalogerTaskStarted(e partybus.Event) ([]tea.Model, tea.Cmd) {
mon, info, err := syftEventParsers.ParseCatalogerTaskStarted(e)
if err != nil {
log.WithFields("error", err).Warn("unable to parse event")
return nil, nil
}
var models []tea.Model
// only create the new cataloger task tree once to manage all cataloger task events
m.onNewCatalogerTask.Do(func() {
models = append(models, newCatalogerTaskTreeModel(m.newTaskProgress))
})
// we need to update the cataloger task model with a new row. We should never update the model outside of the
// bubbletea update-render event loop. Instead, we return a command that will be executed by the bubbletea runtime,
// producing a message that is passed to the cataloger task model. This is the prescribed way to update models
// in bubbletea.
if info.ID == "" {
// ID is optional from the consumer perspective, but required internally
info.ID = uuid.Must(uuid.NewRandom()).String()
}
cmd := func() tea.Msg {
// this message will cause the cataloger task model to add a new row to the output based on the given task
// information and progress data.
return newCatalogerTaskRowEvent{
info: *info,
prog: mon,
}
}
return models, cmd
}
+119 -44
View File
@@ -2,19 +2,22 @@ package ui
import (
"testing"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/gkampitakis/go-snaps/snaps"
"github.com/stretchr/testify/require"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress"
syftEvent "github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
)
func TestHandler_handleCatalogerTaskStarted(t *testing.T) {
title := monitor.Title{
Default: "some task title",
OnSuccess: "some task done",
}
tests := []struct {
name string
eventFn func(*testing.T) partybus.Event
@@ -23,99 +26,171 @@ func TestHandler_handleCatalogerTaskStarted(t *testing.T) {
{
name: "cataloging task in progress",
eventFn: func(t *testing.T) partybus.Event {
src := &monitor.CatalogerTask{
SubStatus: false,
RemoveOnCompletion: false,
Title: "some task title",
TitleOnCompletion: "some task done",
value := &monitor.CatalogerTaskProgress{
AtomicStage: progress.NewAtomicStage("some stage"),
Manual: progress.NewManual(100),
}
src.SetValue("some value")
value.Manual.Add(50)
return partybus.Event{
Type: syftEvent.CatalogerTaskStarted,
Source: src,
Type: syftEvent.CatalogerTaskStarted,
Source: monitor.GenericTask{
Title: title,
HideOnSuccess: false,
HideStageOnSuccess: false,
ID: "my-id",
},
Value: value,
}
},
},
{
name: "cataloging sub task in progress",
eventFn: func(t *testing.T) partybus.Event {
src := &monitor.CatalogerTask{
SubStatus: true,
RemoveOnCompletion: false,
Title: "some task title",
TitleOnCompletion: "some task done",
value := &monitor.CatalogerTaskProgress{
AtomicStage: progress.NewAtomicStage("some stage"),
Manual: progress.NewManual(100),
}
src.SetValue("some value")
value.Manual.Add(50)
return partybus.Event{
Type: syftEvent.CatalogerTaskStarted,
Source: src,
Type: syftEvent.CatalogerTaskStarted,
Source: monitor.GenericTask{
Title: title,
HideOnSuccess: false,
HideStageOnSuccess: false,
ID: "my-id",
ParentID: "top-level-task",
},
Value: value,
}
},
},
{
name: "cataloging sub task complete",
eventFn: func(t *testing.T) partybus.Event {
src := &monitor.CatalogerTask{
SubStatus: true,
RemoveOnCompletion: false,
Title: "some task title",
TitleOnCompletion: "some task done",
value := &monitor.CatalogerTaskProgress{
AtomicStage: progress.NewAtomicStage("some stage"),
Manual: progress.NewManual(100),
}
src.SetValue("some value")
src.SetCompleted()
value.SetCompleted()
return partybus.Event{
Type: syftEvent.CatalogerTaskStarted,
Source: src,
Type: syftEvent.CatalogerTaskStarted,
Source: monitor.GenericTask{
Title: title,
HideOnSuccess: false,
HideStageOnSuccess: false,
ID: "my-id",
ParentID: "top-level-task",
},
Value: value,
}
},
},
{
name: "cataloging sub task complete -- hide stage",
eventFn: func(t *testing.T) partybus.Event {
value := &monitor.CatalogerTaskProgress{
AtomicStage: progress.NewAtomicStage("some stage"),
Manual: progress.NewManual(100),
}
value.SetCompleted()
return partybus.Event{
Type: syftEvent.CatalogerTaskStarted,
Source: monitor.GenericTask{
Title: title,
HideOnSuccess: false,
HideStageOnSuccess: true,
ID: "my-id",
ParentID: "top-level-task",
},
Value: value,
}
},
},
{
name: "cataloging sub task complete with removal",
eventFn: func(t *testing.T) partybus.Event {
src := &monitor.CatalogerTask{
SubStatus: true,
RemoveOnCompletion: true,
Title: "some task title",
TitleOnCompletion: "some task done",
value := &monitor.CatalogerTaskProgress{
AtomicStage: progress.NewAtomicStage("some stage"),
Manual: progress.NewManual(100),
}
src.SetValue("some value")
src.SetCompleted()
value.SetCompleted()
return partybus.Event{
Type: syftEvent.CatalogerTaskStarted,
Source: src,
Type: syftEvent.CatalogerTaskStarted,
Source: monitor.GenericTask{
Title: title,
HideOnSuccess: true,
HideStageOnSuccess: false,
ID: "my-id",
ParentID: "top-level-task",
},
Value: value,
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event := tt.eventFn(t)
// need to be able to get the initial newCatalogerTaskRowEvent + initialize the nested taskprogress model
if tt.iterations == 0 {
tt.iterations = 2
}
e := tt.eventFn(t)
handler := New(DefaultHandlerConfig())
handler.WindowSize = tea.WindowSizeMsg{
Width: 100,
Height: 80,
}
models := handler.Handle(event)
info := monitor.GenericTask{
Title: monitor.Title{
Default: "Catalog contents",
WhileRunning: "Cataloging contents",
OnSuccess: "Cataloged contents",
},
ID: "top-level-task",
}
// note: this line / event is not under test, only needed to show a sub status
kickoffEvent := &monitor.CatalogerTaskProgress{
AtomicStage: progress.NewAtomicStage(""),
Manual: progress.NewManual(-1),
}
models, cmd := handler.Handle(
partybus.Event{
Type: syftEvent.CatalogerTaskStarted,
Source: info,
Value: progress.StagedProgressable(kickoffEvent),
},
)
require.Len(t, models, 1)
require.NotNil(t, cmd)
model := models[0]
tsk, ok := model.(taskprogress.Model)
tr, ok := model.(*catalogerTaskModel)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
gotModel := runModel(t, tr, tt.iterations, cmd())
models, cmd = handler.Handle(e)
require.Len(t, models, 0)
require.NotNil(t, cmd)
gotModel = runModel(t, gotModel, tt.iterations, cmd())
got := gotModel.View()
t.Log(got)
snaps.MatchSnapshot(t, got)
})
+5 -2
View File
@@ -80,18 +80,21 @@ func TestHandler_handleFetchImage(t *testing.T) {
Height: 80,
}
models := handler.Handle(event)
models, _ := handler.Handle(event)
require.Len(t, models, 1)
model := models[0]
tsk, ok := model.(taskprogress.Model)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
gotModel := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
got := gotModel.View()
t.Log(got)
snaps.MatchSnapshot(t, got)
})
@@ -1,28 +0,0 @@
package ui
import (
tea "github.com/charmbracelet/bubbletea"
"github.com/wagoodman/go-partybus"
"github.com/anchore/bubbly/bubbles/taskprogress"
"github.com/anchore/syft/internal/log"
syftEventParsers "github.com/anchore/syft/syft/event/parsers"
)
func (m *Handler) handleFileDigestsCatalogerStarted(e partybus.Event) []tea.Model {
prog, err := syftEventParsers.ParseFileDigestsCatalogingStarted(e)
if err != nil {
log.WithFields("error", err).Warn("unable to parse event")
return nil
}
tsk := m.newTaskProgress(
taskprogress.Title{
Default: "Catalog file digests",
Running: "Cataloging file digests",
Success: "Cataloged file digests",
}, taskprogress.WithStagedProgressable(prog),
)
return []tea.Model{tsk}
}
@@ -1,97 +0,0 @@
package ui
import (
"testing"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/gkampitakis/go-snaps/snaps"
"github.com/stretchr/testify/require"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress"
syftEvent "github.com/anchore/syft/syft/event"
)
func TestHandler_handleFileDigestsCatalogerStarted(t *testing.T) {
tests := []struct {
name string
eventFn func(*testing.T) partybus.Event
iterations int
}{
{
name: "cataloging in progress",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(50)
mon := struct {
progress.Progressable
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
}
return partybus.Event{
Type: syftEvent.FileDigestsCatalogerStarted,
Value: mon,
}
},
},
{
name: "cataloging complete",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(100)
prog.SetCompleted()
mon := struct {
progress.Progressable
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
}
return partybus.Event{
Type: syftEvent.FileDigestsCatalogerStarted,
Value: mon,
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event := tt.eventFn(t)
handler := New(DefaultHandlerConfig())
handler.WindowSize = tea.WindowSizeMsg{
Width: 100,
Height: 80,
}
models := handler.Handle(event)
require.Len(t, models, 1)
model := models[0]
tsk, ok := model.(taskprogress.Model)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
t.Log(got)
snaps.MatchSnapshot(t, got)
})
}
}
+5 -2
View File
@@ -80,18 +80,21 @@ func TestHandler_handleFileIndexingStarted(t *testing.T) {
Height: 80,
}
models := handler.Handle(event)
models, _ := handler.Handle(event)
require.Len(t, models, 1)
model := models[0]
tsk, ok := model.(taskprogress.Model)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
gotModel := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
got := gotModel.View()
t.Log(got)
snaps.MatchSnapshot(t, got)
})
@@ -1,29 +0,0 @@
package ui
import (
tea "github.com/charmbracelet/bubbletea"
"github.com/wagoodman/go-partybus"
"github.com/anchore/bubbly/bubbles/taskprogress"
"github.com/anchore/syft/internal/log"
syftEventParsers "github.com/anchore/syft/syft/event/parsers"
)
func (m *Handler) handleFileMetadataCatalogerStarted(e partybus.Event) []tea.Model {
prog, err := syftEventParsers.ParseFileMetadataCatalogingStarted(e)
if err != nil {
log.WithFields("error", err).Warn("unable to parse event")
return nil
}
tsk := m.newTaskProgress(
taskprogress.Title{
Default: "Catalog file metadata",
Running: "Cataloging file metadata",
Success: "Cataloged file metadata",
},
taskprogress.WithStagedProgressable(prog),
)
return []tea.Model{tsk}
}
@@ -1,97 +0,0 @@
package ui
import (
"testing"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/gkampitakis/go-snaps/snaps"
"github.com/stretchr/testify/require"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress"
syftEvent "github.com/anchore/syft/syft/event"
)
func TestHandler_handleFileMetadataCatalogerStarted(t *testing.T) {
tests := []struct {
name string
eventFn func(*testing.T) partybus.Event
iterations int
}{
{
name: "cataloging in progress",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(50)
mon := struct {
progress.Progressable
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
}
return partybus.Event{
Type: syftEvent.FileMetadataCatalogerStarted,
Value: mon,
}
},
},
{
name: "cataloging complete",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(100)
prog.SetCompleted()
mon := struct {
progress.Progressable
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
}
return partybus.Event{
Type: syftEvent.FileMetadataCatalogerStarted,
Value: mon,
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event := tt.eventFn(t)
handler := New(DefaultHandlerConfig())
handler.WindowSize = tea.WindowSizeMsg{
Width: 100,
Height: 80,
}
models := handler.Handle(event)
require.Len(t, models, 1)
model := models[0]
tsk, ok := model.(taskprogress.Model)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
t.Log(got)
snaps.MatchSnapshot(t, got)
})
}
}
@@ -1,87 +0,0 @@
package ui
import (
"fmt"
tea "github.com/charmbracelet/bubbletea"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress"
"github.com/anchore/syft/internal/log"
syftEventParsers "github.com/anchore/syft/syft/event/parsers"
"github.com/anchore/syft/syft/pkg/cataloger"
)
var _ progress.StagedProgressable = (*packageCatalogerProgressAdapter)(nil)
type packageCatalogerProgressAdapter struct {
monitor *cataloger.Monitor
monitors []progress.Monitorable
}
func newPackageCatalogerProgressAdapter(monitor *cataloger.Monitor) packageCatalogerProgressAdapter {
return packageCatalogerProgressAdapter{
monitor: monitor,
monitors: []progress.Monitorable{
monitor.FilesProcessed,
monitor.PackagesDiscovered,
},
}
}
func (p packageCatalogerProgressAdapter) Stage() string {
return fmt.Sprintf("%d packages", p.monitor.PackagesDiscovered.Current())
}
func (p packageCatalogerProgressAdapter) Current() int64 {
return p.monitor.PackagesDiscovered.Current()
}
func (p packageCatalogerProgressAdapter) Error() error {
completedMonitors := 0
for _, monitor := range p.monitors {
err := monitor.Error()
if err == nil {
continue
}
if progress.IsErrCompleted(err) {
completedMonitors++
continue
}
// something went wrong
return err
}
if completedMonitors == len(p.monitors) && len(p.monitors) > 0 {
return p.monitors[0].Error()
}
return nil
}
func (p packageCatalogerProgressAdapter) Size() int64 {
// this is an inherently unknown value (indeterminate total number of packages to discover)
return -1
}
func (m *Handler) handlePackageCatalogerStarted(e partybus.Event) []tea.Model {
monitor, err := syftEventParsers.ParsePackageCatalogerStarted(e)
if err != nil {
log.WithFields("error", err).Warn("unable to parse event")
return nil
}
tsk := m.newTaskProgress(
taskprogress.Title{
Default: "Catalog packages",
Running: "Cataloging packages",
Success: "Cataloged packages",
},
taskprogress.WithStagedProgressable(
newPackageCatalogerProgressAdapter(monitor),
),
)
tsk.HideStageOnSuccess = false
return []tea.Model{tsk}
}
@@ -1,133 +0,0 @@
package ui
import (
"testing"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/gkampitakis/go-snaps/snaps"
"github.com/stretchr/testify/require"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress"
syftEvent "github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/pkg/cataloger"
)
func TestHandler_handlePackageCatalogerStarted(t *testing.T) {
tests := []struct {
name string
eventFn func(*testing.T) partybus.Event
iterations int
}{
{
name: "cataloging in progress",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(50)
mon := cataloger.Monitor{
FilesProcessed: progress.NewManual(-1),
PackagesDiscovered: prog,
}
return partybus.Event{
Type: syftEvent.PackageCatalogerStarted,
Value: mon,
}
},
},
{
name: "cataloging only files complete",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(50)
files := progress.NewManual(-1)
files.SetCompleted()
mon := cataloger.Monitor{
FilesProcessed: files,
PackagesDiscovered: prog,
}
return partybus.Event{
Type: syftEvent.PackageCatalogerStarted,
Value: mon,
}
},
},
{
name: "cataloging only packages complete",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(100)
prog.SetCompleted()
files := progress.NewManual(-1)
mon := cataloger.Monitor{
FilesProcessed: files,
PackagesDiscovered: prog,
}
return partybus.Event{
Type: syftEvent.PackageCatalogerStarted,
Value: mon,
}
},
},
{
name: "cataloging complete",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(100)
prog.SetCompleted()
files := progress.NewManual(-1)
files.SetCompleted()
mon := cataloger.Monitor{
FilesProcessed: files,
PackagesDiscovered: prog,
}
return partybus.Event{
Type: syftEvent.PackageCatalogerStarted,
Value: mon,
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event := tt.eventFn(t)
handler := New(DefaultHandlerConfig())
handler.WindowSize = tea.WindowSizeMsg{
Width: 100,
Height: 80,
}
models := handler.Handle(event)
require.Len(t, models, 1)
model := models[0]
tsk, ok := model.(taskprogress.Model)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
t.Log(got)
snaps.MatchSnapshot(t, got)
})
}
}
+5 -2
View File
@@ -98,18 +98,21 @@ func TestHandler_handleReadImage(t *testing.T) {
Height: 80,
}
models := handler.Handle(event)
models, _ := handler.Handle(event)
require.Len(t, models, 1)
model := models[0]
tsk, ok := model.(taskprogress.Model)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
gotModel := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
got := gotModel.View()
t.Log(got)
snaps.MatchSnapshot(t, got)
})
+19 -13
View File
@@ -29,6 +29,8 @@ type Handler struct {
Config HandlerConfig
bubbly.EventHandler
onNewCatalogerTask *sync.Once
}
func DefaultHandlerConfig() HandlerConfig {
@@ -41,28 +43,32 @@ func New(cfg HandlerConfig) *Handler {
d := bubbly.NewEventDispatcher()
h := &Handler{
EventHandler: d,
Running: &sync.WaitGroup{},
Config: cfg,
EventHandler: d,
Running: &sync.WaitGroup{},
Config: cfg,
onNewCatalogerTask: &sync.Once{},
}
// register all supported event types with the respective handler functions
d.AddHandlers(map[partybus.EventType]bubbly.EventHandlerFn{
stereoscopeEvent.PullDockerImage: h.handlePullDockerImage,
stereoscopeEvent.PullContainerdImage: h.handlePullContainerdImage,
stereoscopeEvent.ReadImage: h.handleReadImage,
stereoscopeEvent.FetchImage: h.handleFetchImage,
syftEvent.PackageCatalogerStarted: h.handlePackageCatalogerStarted,
syftEvent.FileDigestsCatalogerStarted: h.handleFileDigestsCatalogerStarted,
syftEvent.FileMetadataCatalogerStarted: h.handleFileMetadataCatalogerStarted,
syftEvent.FileIndexingStarted: h.handleFileIndexingStarted,
syftEvent.AttestationStarted: h.handleAttestationStarted,
syftEvent.CatalogerTaskStarted: h.handleCatalogerTaskStarted,
stereoscopeEvent.PullDockerImage: simpleHandler(h.handlePullDockerImage),
stereoscopeEvent.PullContainerdImage: simpleHandler(h.handlePullContainerdImage),
stereoscopeEvent.ReadImage: simpleHandler(h.handleReadImage),
stereoscopeEvent.FetchImage: simpleHandler(h.handleFetchImage),
syftEvent.FileIndexingStarted: simpleHandler(h.handleFileIndexingStarted),
syftEvent.AttestationStarted: simpleHandler(h.handleAttestationStarted),
syftEvent.CatalogerTaskStarted: h.handleCatalogerTaskStarted,
})
return h
}
func simpleHandler(fn func(partybus.Event) []tea.Model) bubbly.EventHandlerFn {
return func(e partybus.Event) ([]tea.Model, tea.Cmd) {
return fn(e), nil
}
}
func (m *Handler) OnMessage(msg tea.Msg) {
if msg, ok := msg.(tea.WindowSizeMsg); ok {
m.WindowSize = msg
+13 -27
View File
@@ -4,12 +4,11 @@ import (
"reflect"
"sync"
"testing"
"unsafe"
tea "github.com/charmbracelet/bubbletea"
)
func runModel(t testing.TB, m tea.Model, iterations int, message tea.Msg, h ...*sync.WaitGroup) string {
func runModel(t testing.TB, m tea.Model, iterations int, message tea.Msg, h ...*sync.WaitGroup) tea.Model {
t.Helper()
if iterations == 0 {
iterations = 1
@@ -37,34 +36,21 @@ func runModel(t testing.TB, m tea.Model, iterations int, message tea.Msg, h ...*
cmd = tea.Batch(nextCmds...)
}
return m.View()
return m
}
func flatten(p tea.Msg) (msgs []tea.Msg) {
if reflect.TypeOf(p).Name() == "batchMsg" {
partials := extractBatchMessages(p)
for _, m := range partials {
msgs = append(msgs, flatten(m)...)
func flatten(ps ...tea.Msg) (msgs []tea.Msg) {
for _, p := range ps {
if bm, ok := p.(tea.BatchMsg); ok {
for _, m := range bm {
if m == nil {
continue
}
msgs = append(msgs, flatten(m())...)
}
} else {
msgs = []tea.Msg{p}
}
} else {
msgs = []tea.Msg{p}
}
return msgs
}
func extractBatchMessages(m tea.Msg) (ret []tea.Msg) {
sliceMsgType := reflect.SliceOf(reflect.TypeOf(tea.Cmd(nil)))
value := reflect.ValueOf(m) // note: this is technically unaddressable
// make our own instance that is addressable
valueCopy := reflect.New(value.Type()).Elem()
valueCopy.Set(value)
cmds := reflect.NewAt(sliceMsgType, unsafe.Pointer(valueCopy.UnsafeAddr())).Elem()
for i := 0; i < cmds.Len(); i++ {
item := cmds.Index(i)
r := item.Call(nil)
ret = append(ret, r[0].Interface().(tea.Msg))
}
return ret
}
+5 -1
View File
@@ -155,7 +155,11 @@ func (m *UI) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
return m, nil
}
for _, newModel := range m.handler.Handle(msg) {
models, cmd := m.handler.Handle(msg)
if cmd != nil {
cmds = append(cmds, cmd)
}
for _, newModel := range models {
if newModel == nil {
continue
}
+3 -3
View File
@@ -8,9 +8,9 @@ require (
github.com/Masterminds/sprig/v3 v3.2.3
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/acobaugh/osrelease v0.1.0
github.com/anchore/bubbly v0.0.0-20230801194016-acdb4981b461
github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9
github.com/anchore/clio v0.0.0-20231016125544-c98a83e1c7fc
github.com/anchore/fangs v0.0.0-20231103141714-84c94dc43a2e
github.com/anchore/fangs v0.0.0-20231201140849-5075d28d6d8b
github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a
github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
@@ -35,7 +35,7 @@ require (
github.com/go-git/go-git/v5 v5.10.1
github.com/go-test/deep v1.1.0
github.com/google/go-cmp v0.6.0
github.com/google/go-containerregistry v0.16.1
github.com/google/go-containerregistry v0.17.0
github.com/google/licensecheck v0.3.1
github.com/google/uuid v1.4.0
github.com/gookit/color v1.5.4
+6 -6
View File
@@ -89,12 +89,12 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/anchore/bubbly v0.0.0-20230801194016-acdb4981b461 h1:xGu4/uMWucwWV0YV3fpFIQZ6KVfS/Wfhmma8t0s0vRo=
github.com/anchore/bubbly v0.0.0-20230801194016-acdb4981b461/go.mod h1:Ger02eh5NpPm2IqkPAy396HU1KlK3BhOeCljDYXySSk=
github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9 h1:p0ZIe0htYOX284Y4axJaGBvXHU0VCCzLN5Wf5XbKStU=
github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9/go.mod h1:3ZsFB9tzW3vl4gEiUeuSOMDnwroWxIxJelOOHUp8dSw=
github.com/anchore/clio v0.0.0-20231016125544-c98a83e1c7fc h1:A1KFO+zZZmbNlz1+WKsCF0RKVx6XRoxsAG3lrqH9hUQ=
github.com/anchore/clio v0.0.0-20231016125544-c98a83e1c7fc/go.mod h1:QeWvNzxsrUNxcs6haQo3OtISfXUXW0qAuiG4EQiz0GU=
github.com/anchore/fangs v0.0.0-20231103141714-84c94dc43a2e h1:O8ZubApaSl7dRzKNvyfGq9cLIPLQ5v3Iz0Y3huHKCgg=
github.com/anchore/fangs v0.0.0-20231103141714-84c94dc43a2e/go.mod h1:yPsN3NUGhU5dcBtYBa1dMNzGu1yT5ZAfSjKq9DY4aV8=
github.com/anchore/fangs v0.0.0-20231201140849-5075d28d6d8b h1:L/djgY7ZbZ/38+wUtdkk398W3PIBJLkt1N8nU/7e47A=
github.com/anchore/fangs v0.0.0-20231201140849-5075d28d6d8b/go.mod h1:TLcE0RE5+8oIx2/NPWem/dq1DeaMoC+fPEH7hoSzPLo=
github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a h1:nJ2G8zWKASyVClGVgG7sfM5mwoZlZ2zYpIzN2OhjWkw=
github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a/go.mod h1:ubLFmlsv8/DFUQrZwY5syT5/8Er3ugSr4rDFwHsE3hg=
github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb h1:iDMnx6LIjtjZ46C0akqveX83WFzhpTD3eqOthawb5vU=
@@ -368,8 +368,8 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-containerregistry v0.16.1 h1:rUEt426sR6nyrL3gt+18ibRcvYpKYdpsa5ZW7MA08dQ=
github.com/google/go-containerregistry v0.16.1/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ=
github.com/google/go-containerregistry v0.17.0 h1:5p+zYs/R4VGHkhyvgWurWrpJ2hW4Vv9fQI+GzdcwXLk=
github.com/google/go-containerregistry v0.17.0/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/licensecheck v0.3.1 h1:QoxgoDkaeC4nFrtGN1jV7IPmDCHFNIVh54e5hSt6sPs=
github.com/google/licensecheck v0.3.1/go.mod h1:ORkR35t/JjW+emNKtfJDII0zlciG9JgbT7SmsohlHmY=
+17
View File
@@ -2,10 +2,12 @@ package bus
import (
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/clio"
"github.com/anchore/syft/internal/redact"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
)
func Exit() {
@@ -33,3 +35,18 @@ func Notify(message string) {
Value: message,
})
}
func StartCatalogerTask(info monitor.GenericTask, size int64, initialStage string) *monitor.CatalogerTaskProgress {
t := &monitor.CatalogerTaskProgress{
AtomicStage: progress.NewAtomicStage(initialStage),
Manual: progress.NewManual(size),
}
Publish(partybus.Event{
Type: event.CatalogerTaskStarted,
Source: info,
Value: progress.StagedProgressable(t),
})
return t
}
-9
View File
@@ -14,15 +14,6 @@ const (
// Events from the syft library
// PackageCatalogerStarted is a partybus event that occurs when the package cataloging has begun
PackageCatalogerStarted partybus.EventType = typePrefix + "-package-cataloger-started-event"
// FileMetadataCatalogerStarted is a partybus event that occurs when the file metadata cataloging has begun
FileMetadataCatalogerStarted partybus.EventType = typePrefix + "-file-metadata-cataloger-started-event"
// FileDigestsCatalogerStarted is a partybus event that occurs when the file digests cataloging has begun
FileDigestsCatalogerStarted partybus.EventType = typePrefix + "-file-digests-cataloger-started-event"
// FileIndexingStarted is a partybus event that occurs when the directory resolver begins indexing a filesystem
FileIndexingStarted partybus.EventType = typePrefix + "-file-indexing-started-event"
-55
View File
@@ -1,55 +0,0 @@
package monitor
import (
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/syft/event"
)
// TODO: this should be refactored to support read-only/write-only access using idioms of the progress lib
type CatalogerTask struct {
prog *progress.Manual
// Title
Title string
// TitleOnCompletion a string to use as title when completed
TitleOnCompletion string
// SubStatus indicates this progress should be rendered as a sub-item
SubStatus bool
// RemoveOnCompletion indicates this progress line will be removed when completed
RemoveOnCompletion bool
// value is the value to display -- not public as SetValue needs to be called to initialize this progress
value string
}
func (e *CatalogerTask) init() {
e.prog = progress.NewManual(-1)
bus.Publish(partybus.Event{
Type: event.CatalogerTaskStarted,
Source: e,
})
}
func (e *CatalogerTask) SetCompleted() {
if e.prog != nil {
e.prog.SetCompleted()
}
}
func (e *CatalogerTask) SetValue(value string) {
if e.prog == nil {
e.init()
}
e.value = value
}
func (e *CatalogerTask) GetValue() string {
return e.value
}
func (e *CatalogerTask) GetMonitor() *progress.Manual {
return e.prog
}
@@ -0,0 +1,10 @@
package monitor
import (
"github.com/wagoodman/go-progress"
)
type CatalogerTaskProgress struct {
*progress.AtomicStage
*progress.Manual
}
+15 -2
View File
@@ -18,6 +18,19 @@ type Title struct {
}
type GenericTask struct {
Title Title
Context string
// required fields
Title Title
// optional format fields
HideOnSuccess bool
HideStageOnSuccess bool
// optional fields
ID string
ParentID string
Context string
}
+12 -45
View File
@@ -12,7 +12,6 @@ import (
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/pkg/cataloger"
)
type ErrBadPayload struct {
@@ -40,45 +39,6 @@ func checkEventType(actual, expected partybus.EventType) error {
return nil
}
func ParsePackageCatalogerStarted(e partybus.Event) (*cataloger.Monitor, error) {
if err := checkEventType(e.Type, event.PackageCatalogerStarted); err != nil {
return nil, err
}
monitor, ok := e.Value.(cataloger.Monitor)
if !ok {
return nil, newPayloadErr(e.Type, "Value", e.Value)
}
return &monitor, nil
}
func ParseFileMetadataCatalogingStarted(e partybus.Event) (progress.StagedProgressable, error) {
if err := checkEventType(e.Type, event.FileMetadataCatalogerStarted); err != nil {
return nil, err
}
prog, ok := e.Value.(progress.StagedProgressable)
if !ok {
return nil, newPayloadErr(e.Type, "Value", e.Value)
}
return prog, nil
}
func ParseFileDigestsCatalogingStarted(e partybus.Event) (progress.StagedProgressable, error) {
if err := checkEventType(e.Type, event.FileDigestsCatalogerStarted); err != nil {
return nil, err
}
prog, ok := e.Value.(progress.StagedProgressable)
if !ok {
return nil, newPayloadErr(e.Type, "Value", e.Value)
}
return prog, nil
}
func ParseFileIndexingStarted(e partybus.Event) (string, progress.StagedProgressable, error) {
if err := checkEventType(e.Type, event.FileIndexingStarted); err != nil {
return "", nil, err
@@ -97,17 +57,24 @@ func ParseFileIndexingStarted(e partybus.Event) (string, progress.StagedProgress
return path, prog, nil
}
func ParseCatalogerTaskStarted(e partybus.Event) (*monitor.CatalogerTask, error) {
func ParseCatalogerTaskStarted(e partybus.Event) (progress.StagedProgressable, *monitor.GenericTask, error) {
if err := checkEventType(e.Type, event.CatalogerTaskStarted); err != nil {
return nil, err
return nil, nil, err
}
source, ok := e.Source.(*monitor.CatalogerTask)
var mon progress.StagedProgressable
source, ok := e.Source.(monitor.GenericTask)
if !ok {
return nil, newPayloadErr(e.Type, "Source", e.Source)
return nil, nil, newPayloadErr(e.Type, "Source", e.Source)
}
return source, nil
mon, ok = e.Value.(progress.StagedProgressable)
if !ok {
mon = nil
}
return mon, &source, nil
}
func ParseAttestationStartedEvent(e partybus.Event) (io.Reader, progress.Progressable, *monitor.GenericTask, error) {
+19 -19
View File
@@ -3,16 +3,16 @@ package filedigest
import (
"crypto"
"errors"
"fmt"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/dustin/go-humanize"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/bus"
intFile "github.com/anchore/syft/internal/file"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
intCataloger "github.com/anchore/syft/syft/file/cataloger/internal"
)
@@ -41,9 +41,11 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
}
}
stage, prog := digestsCatalogingProgress(int64(len(locations)))
prog := digestsCatalogingProgress(int64(len(locations)))
for _, location := range locations {
stage.Current = location.RealPath
prog.Increment()
prog.AtomicStage.Set(location.Path())
result, err := i.catalogLocation(resolver, location)
if errors.Is(err, ErrUndigestableFile) {
@@ -61,8 +63,12 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
prog.Increment()
results[location.Coordinates] = result
}
log.Debugf("file digests cataloger processed %d files", prog.Current())
prog.AtomicStage.Set(fmt.Sprintf("%s digests", humanize.Comma(prog.Current())))
prog.SetCompleted()
return results, nil
}
@@ -91,20 +97,14 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
return digests, nil
}
func digestsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) {
stage := &progress.Stage{}
prog := progress.NewManual(locations)
bus.Publish(partybus.Event{
Type: event.FileDigestsCatalogerStarted,
Value: struct {
progress.Stager
progress.Progressable
}{
Stager: progress.Stager(stage),
Progressable: prog,
func digestsCatalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
info := monitor.GenericTask{
Title: monitor.Title{
Default: "Catalog file digests",
WhileRunning: "Cataloging file digests",
OnSuccess: "Cataloged file digests",
},
})
}
return stage, prog
return bus.StartCatalogerTask(info, locations, "")
}
+20 -21
View File
@@ -1,12 +1,13 @@
package filemetadata
import (
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"fmt"
"github.com/dustin/go-humanize"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
)
@@ -20,7 +21,6 @@ func NewCataloger() *Cataloger {
func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordinates) (map[file.Coordinates]file.Metadata, error) {
results := make(map[file.Coordinates]file.Metadata)
var locations <-chan file.Location
if len(coordinates) == 0 {
locations = resolver.AllLocations()
} else {
@@ -43,36 +43,35 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
}()
}
stage, prog := metadataCatalogingProgress(int64(len(locations)))
prog := metadataCatalogingProgress(int64(len(locations)))
for location := range locations {
stage.Current = location.RealPath
prog.Increment()
prog.AtomicStage.Set(location.Path())
metadata, err := resolver.FileMetadataByLocation(location)
if err != nil {
return nil, err
}
results[location.Coordinates] = metadata
prog.Increment()
}
log.Debugf("file metadata cataloger processed %d files", prog.Current())
prog.AtomicStage.Set(fmt.Sprintf("%s locations", humanize.Comma(prog.Current())))
prog.SetCompleted()
return results, nil
}
func metadataCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) {
stage := &progress.Stage{}
prog := progress.NewManual(locations)
bus.Publish(partybus.Event{
Type: event.FileMetadataCatalogerStarted,
Value: struct {
progress.Stager
progress.Progressable
}{
Stager: progress.Stager(stage),
Progressable: prog,
func metadataCatalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
info := monitor.GenericTask{
Title: monitor.Title{
Default: "Catalog file metadata",
WhileRunning: "Cataloging file metadata",
OnSuccess: "Cataloged file metadata",
},
})
}
return stage, prog
return bus.StartCatalogerTask(info, locations, "")
}
@@ -640,6 +640,9 @@ func TestAllLayersResolver_AllLocations(t *testing.T) {
// this isn't important for the test, so we remove them.
paths.Remove("/proc", "/sys", "/dev", "/etc")
// Remove cache created by Mac Rosetta when emulating different arches
paths.Remove("/.cache/rosetta", "/.cache")
pathsList := paths.List()
sort.Strings(pathsList)
@@ -547,6 +547,9 @@ func TestSquashResolver_AllLocations(t *testing.T) {
// this isn't important for the test, so we remove them.
paths.Remove("/proc", "/sys", "/dev", "/etc")
// Remove cache created by Mac Rosetta when emulating different arches
paths.Remove("/.cache/rosetta", "/.cache")
pathsList := paths.List()
sort.Strings(pathsList)
@@ -69,18 +69,10 @@ var defaultClassifiers = []classifier{
{
Class: "java-binary-openjdk",
FileGlob: "**/java",
EvidenceMatcher: evidenceMatchers(
fileContentsVersionMatcher(
// covers linux/amd64
// [NUL]openjdk[NUL]java[NUL]0.0[NUL]11.0.17+8-LTS[NUL]
// [NUL]openjdk[NUL]java[NUL]1.8[NUL]1.8.0_352-b08[NUL]
`(?m)\x00openjdk\x00java\x00(?P<release>[0-9]+[.0-9]*)\x00(?P<version>[0-9]+[^\x00]+)\x00`,
),
fileContentsVersionMatcher(
// covers linux/arm64
`(?m)(?P<release>[0-9]+[.0-9]*)(\x00)+(?P<version>[0-9]+[^\x00]+)(\x00)+openjdk(\x00)+java\x00`,
),
),
EvidenceMatcher: fileContentsVersionMatcher(
// [NUL]openjdk[NUL]java[NUL]0.0[NUL]11.0.17+8-LTS[NUL]
// [NUL]openjdk[NUL]java[NUL]1.8[NUL]1.8.0_352-b08[NUL]
`(?m)\x00openjdk\x00java\x00(?P<release>[0-9]+[.0-9]*)\x00(?P<version>[0-9]+[^\x00]+)\x00`),
Package: "java",
PURL: mustPURL("pkg:generic/java@version"),
// TODO the updates might need to be part of the CPE, like: 1.8.0:update152
+29 -30
View File
@@ -6,14 +6,14 @@ import (
"runtime/debug"
"sync"
"github.com/dustin/go-humanize"
"github.com/hashicorp/go-multierror"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
@@ -35,21 +35,6 @@ type catalogResult struct {
Error error
}
// newMonitor creates a new Monitor object and publishes the object on the bus as a PackageCatalogerStarted event.
func newMonitor() (*progress.Manual, *progress.Manual) {
filesProcessed := progress.Manual{}
packagesDiscovered := progress.Manual{}
bus.Publish(partybus.Event{
Type: event.PackageCatalogerStarted,
Value: Monitor{
FilesProcessed: progress.Monitorable(&filesProcessed),
PackagesDiscovered: progress.Monitorable(&packagesDiscovered),
},
})
return &filesProcessed, &packagesDiscovered
}
func runCataloger(cataloger pkg.Cataloger, resolver file.Resolver) (catalogerResult *catalogResult, err error) {
// handle individual cataloger panics
defer func() {
@@ -101,6 +86,7 @@ func runCataloger(cataloger pkg.Cataloger, resolver file.Resolver) (catalogerRes
}
catalogerResult.Packages = append(catalogerResult.Packages, p)
}
catalogerResult.Relationships = append(catalogerResult.Relationships, relationships...)
log.WithFields("cataloger", cataloger.Name()).Trace("cataloging complete")
return catalogerResult, err
@@ -116,9 +102,7 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
catalog := pkg.NewCollection()
var allRelationships []artifact.Relationship
filesProcessed, packagesDiscovered := newMonitor()
defer filesProcessed.SetCompleted()
defer packagesDiscovered.SetCompleted()
prog := monitorPackageCatalogingTask()
// perform analysis, accumulating errors for each failed analysis
var errs error
@@ -131,10 +115,11 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
jobs := make(chan pkg.Cataloger, nCatalogers)
results := make(chan *catalogResult, nCatalogers)
discoveredPackages := make(chan int64, nCatalogers)
waitGroup := sync.WaitGroup{}
var totalPackagesDiscovered int64
for i := 0; i < parallelism; i++ {
waitGroup.Add(1)
@@ -148,20 +133,16 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
// ensure we set the error to be aggregated
result.Error = err
discoveredPackages <- result.Discovered
prog.Add(result.Discovered)
totalPackagesDiscovered += result.Discovered
count := humanize.Comma(totalPackagesDiscovered)
prog.AtomicStage.Set(fmt.Sprintf("%s packages", count))
results <- result
}
}()
}
// dynamically show updated discovered package status
go func() {
for discovered := range discoveredPackages {
packagesDiscovered.Add(discovered)
}
}()
// Enqueue the jobs
for _, cataloger := range catalogers {
jobs <- cataloger
@@ -171,7 +152,6 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
// Wait for the jobs to finish
waitGroup.Wait()
close(results)
close(discoveredPackages)
// collect the results
for result := range results {
@@ -186,6 +166,12 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
allRelationships = append(allRelationships, pkg.NewRelationships(catalog)...)
if errs != nil {
prog.SetError(errs)
} else {
prog.SetCompleted()
}
return catalog, allRelationships, errs
}
@@ -228,3 +214,16 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver
}
return relationships, nil
}
func monitorPackageCatalogingTask() *monitor.CatalogerTaskProgress {
info := monitor.GenericTask{
Title: monitor.Title{
Default: "Catalog packages",
WhileRunning: "Cataloging packages",
OnSuccess: "Cataloged packages",
},
HideOnSuccess: false,
}
return bus.StartCatalogerTask(info, -1, "")
}
+1 -1
View File
@@ -294,7 +294,7 @@ func GetManifestFieldGroupIDs(manifest *pkg.JavaManifest, fields []string) (grou
}
func cleanGroupID(groupID string) string {
return strings.TrimSpace(removeOSCIDirectives(groupID))
return strings.TrimSpace(strings.Split(removeOSCIDirectives(groupID), "#")[0])
}
func removeOSCIDirectives(groupID string) string {
File diff suppressed because it is too large Load Diff
@@ -161,6 +161,17 @@ func Test_groupIDsFromJavaPackage(t *testing.T) {
},
expects: []string{"io.jenkins-ci.plugin.thing"},
},
{
name: "clean # suffixes",
pkg: pkg.Package{
Metadata: pkg.JavaArchive{
PomProperties: &pkg.JavaPomProperties{
GroupID: "org.elasticsearch.plugin#parent-join;6.8.15",
},
},
},
expects: []string{"org.elasticsearch.plugin"},
},
{
name: "from artifactID",
pkg: pkg.Package{
@@ -58,23 +58,14 @@ func parseDotnetPortableExecutable(_ file.Resolver, _ *generic.Environment, f fi
func buildDotNetPackage(versionResources map[string]string, f file.LocationReadCloser) (dnpkg pkg.Package, err error) {
name := findName(versionResources)
if name == "" {
return dnpkg, fmt.Errorf("unable to find FileDescription, or ProductName in PE file: %s", f.RealPath)
return dnpkg, fmt.Errorf("unable to find PE name in file: %s", f.RealPath)
}
version := findVersion(versionResources)
if strings.TrimSpace(version) == "" {
return dnpkg, fmt.Errorf("unable to find FileVersion in PE file: %s", f.RealPath)
if version == "" {
return dnpkg, fmt.Errorf("unable to find PE version in file: %s", f.RealPath)
}
purl := packageurl.NewPackageURL(
packageurl.TypeNuget, // See explanation in syft/pkg/cataloger/dotnet/package.go as to why this was chosen.
"",
name,
version,
nil,
"",
).ToString()
metadata := pkg.DotnetPortableExecutableEntry{
AssemblyVersion: versionResources["Assembly Version"],
LegalCopyright: versionResources["LegalCopyright"],
@@ -91,7 +82,7 @@ func buildDotNetPackage(versionResources map[string]string, f file.LocationReadC
Locations: file.NewLocationSet(f.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
Type: pkg.DotnetPkg,
Language: pkg.Dotnet,
PURL: purl,
PURL: portableExecutablePackageURL(name, version),
Metadata: metadata,
}
@@ -100,30 +91,128 @@ func buildDotNetPackage(versionResources map[string]string, f file.LocationReadC
return dnpkg, nil
}
func findVersion(versionResources map[string]string) string {
for _, key := range []string{"FileVersion"} {
if version, ok := versionResources[key]; ok {
if strings.TrimSpace(version) == "" {
continue
}
fields := strings.Fields(version)
if len(fields) > 0 {
return fields[0]
}
func portableExecutablePackageURL(name, version string) string {
return packageurl.NewPackageURL(
packageurl.TypeNuget, // See explanation in syft/pkg/cataloger/dotnet/package.go as to why this was chosen.
"",
name,
version,
nil,
"",
).ToString()
}
func extractVersion(version string) string {
version = strings.TrimSpace(version)
out := ""
// some example versions are: "1, 0, 0, 0", "Release 73" or "4.7.4076.0 built by: NET472REL1LAST_B"
// so try to split it and take the first parts that look numeric
for i, f := range strings.Fields(version) {
// if the output already has a number but the current segment does not have a number,
// return what we found for the version
if containsNumber(out) && !containsNumber(f) {
return out
}
if i == 0 {
out = f
} else {
out += " " + f
}
}
return out
}
func findVersion(versionResources map[string]string) string {
productVersion := extractVersion(versionResources["ProductVersion"])
fileVersion := extractVersion(versionResources["FileVersion"])
if productVersion == "" {
return fileVersion
}
productVersionDetail := punctuationCount(productVersion)
fileVersionDetail := punctuationCount(fileVersion)
if containsNumber(productVersion) && productVersionDetail >= fileVersionDetail {
return productVersion
}
if containsNumber(fileVersion) && fileVersionDetail > 0 {
return fileVersion
}
if containsNumber(productVersion) {
return productVersion
}
if containsNumber(fileVersion) {
return fileVersion
}
return productVersion
}
func containsNumber(s string) bool {
return numberRegex.MatchString(s)
}
func punctuationCount(s string) int {
return len(versionPunctuationRegex.FindAllString(s, -1))
}
var (
// spaceRegex includes nbsp (#160) considered to be a space character
spaceRegex = regexp.MustCompile(`[\s\xa0]+`)
numberRegex = regexp.MustCompile(`\d`)
versionPunctuationRegex = regexp.MustCompile(`[.,]+`)
)
func findName(versionResources map[string]string) string {
// PE files found in the wild _not_ authored by Microsoft seem to use ProductName as a clear
// identifier of the software
nameFields := []string{"ProductName", "FileDescription", "InternalName", "OriginalFilename"}
if isMicrosoft(versionResources) {
// Microsoft seems to be consistent using the FileDescription, with a few that are blank and have
// fallbacks to ProductName last, as this is often something very broad like "Microsoft Windows"
nameFields = []string{"FileDescription", "InternalName", "OriginalFilename", "ProductName"}
}
for _, field := range nameFields {
value := spaceNormalize(versionResources[field])
if value == "" {
continue
}
return value
}
return ""
}
func findName(versionResources map[string]string) string {
for _, key := range []string{"FileDescription", "ProductName"} {
if name, ok := versionResources[key]; ok {
if strings.TrimSpace(name) == "" {
continue
}
trimmed := strings.TrimSpace(name)
return regexp.MustCompile(`[^a-zA-Z0-9.]+`).ReplaceAllString(trimmed, "")
}
// normalizes a string to a trimmed version with all contigous whitespace collapsed to a single space character
func spaceNormalize(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
return ""
// ensure valid utf8 text
value = strings.ToValidUTF8(value, "")
// consolidate all space characters
value = spaceRegex.ReplaceAllString(value, " ")
// remove other non-space, non-printable characters
value = regexp.MustCompile(`[\x00-\x1f]`).ReplaceAllString(value, "")
// consolidate all space characters again in case other non-printables were in-between
value = spaceRegex.ReplaceAllString(value, " ")
// finally, remove any remaining surrounding whitespace
value = strings.TrimSpace(value)
return value
}
func isMicrosoft(versionResources map[string]string) bool {
return strings.Contains(strings.ToLower(versionResources["CompanyName"]), "microsoft") ||
strings.Contains(strings.ToLower(versionResources["ProductName"]), "microsoft")
}
@@ -32,12 +32,8 @@ func TestParseDotnetPortableExecutable(t *testing.T) {
"Assembly Version": "3.14.2.11",
},
expectedPackage: pkg.Package{
Name: "ActiveDirectoryAuthenticationLibrary",
Version: "3.14.40721.0918",
Locations: file.NewLocationSet(file.NewLocation("").WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
Type: pkg.DotnetPkg,
Language: pkg.Dotnet,
PURL: "pkg:nuget/ActiveDirectoryAuthenticationLibrary@3.14.40721.0918",
Name: "Active Directory Authentication Library",
Version: "3.14.40721.0918",
Metadata: pkg.DotnetPortableExecutableEntry{
AssemblyVersion: "3.14.2.11",
LegalCopyright: "Copyright (c) Microsoft Corporation. All rights reserved.",
@@ -52,7 +48,7 @@ func TestParseDotnetPortableExecutable(t *testing.T) {
name: "dotnet package with malformed field and extended version",
versionResources: map[string]string{
"CompanyName": "Microsoft Corporation",
"FileDescription": "äbFileVersion",
"FileDescription": "äbFile\xa0\xa1Versi on",
"FileVersion": "4.6.25512.01 built by: dlab-DDVSOWINAGE016. Commit Hash: d0d5c7b49271cadb6d97de26d8e623e98abdc8db",
"InternalName": "äbFileVersion",
"LegalCopyright": "© Microsoft Corporation. All rights reserved.",
@@ -61,13 +57,9 @@ func TestParseDotnetPortableExecutable(t *testing.T) {
"ProductVersion": "4.6.25512.01 built by: dlab-DDVSOWINAGE016. Commit Hash: d0d5c7b49271cadb6d97de26d8e623e98abdc8db",
},
expectedPackage: pkg.Package{
Name: "bFileVersion",
Name: "äbFileVersi on",
Version: "4.6.25512.01",
Locations: file.NewLocationSet(
file.NewLocation("").WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
Type: pkg.DotnetPkg,
Language: pkg.Dotnet,
PURL: "pkg:nuget/bFileVersion@4.6.25512.01",
PURL: "pkg:nuget/%C3%A4bFileVersi%20on@4.6.25512.01",
Metadata: pkg.DotnetPortableExecutableEntry{
LegalCopyright: "© Microsoft Corporation. All rights reserved.",
InternalName: "äb\x01FileVersion",
@@ -77,16 +69,208 @@ func TestParseDotnetPortableExecutable(t *testing.T) {
},
},
},
{
name: "System.Data.Linq.dll",
versionResources: map[string]string{
"CompanyName": "Microsoft Corporation",
"FileDescription": "System.Data.Linq.dll",
"FileVersion": "4.7.3190.0 built by: NET472REL1LAST_C",
"InternalName": "System.Data.Linq.dll",
"LegalCopyright": "© Microsoft Corporation. All rights reserved.",
"OriginalFilename": "System.Data.Linq.dll",
"ProductName": "Microsoft® .NET Framework",
"ProductVersion": "4.7.3190.0",
},
expectedPackage: pkg.Package{
Name: "System.Data.Linq.dll",
Version: "4.7.3190.0",
},
},
{
name: "curl",
versionResources: map[string]string{
"CompanyName": "curl, https://curl.se/",
"FileDescription": "The curl executable",
"FileVersion": "8.4.0",
"InternalName": "curl",
"LegalCopyright": "© Daniel Stenberg, <daniel@haxx.se>.",
"OriginalFilename": "curl.exe",
"ProductName": "The curl executable",
"ProductVersion": "8.4.0",
},
expectedPackage: pkg.Package{
Name: "The curl executable",
Version: "8.4.0",
},
},
{
name: "Prometheus",
versionResources: map[string]string{
"AssemblyVersion": "8.0.0.0",
"CompanyName": "",
"FileDescription": "",
"FileVersion": "8.0.1",
"InternalName": "Prometheus.AspNetCore.dll",
"OriginalFilename": "Prometheus.AspNetCore.dll",
"ProductName": "",
"ProductVersion": "8.0.1",
},
expectedPackage: pkg.Package{
Name: "Prometheus.AspNetCore.dll",
Version: "8.0.1",
},
},
{
name: "Hidden Input",
versionResources: map[string]string{
"FileDescription": "Reads from stdin without leaking info to the terminal and outputs back to stdout",
"FileVersion": "1, 0, 0, 0",
"InternalName": "hiddeninput",
"LegalCopyright": "Jordi Boggiano - 2012",
"OriginalFilename": "hiddeninput.exe",
"ProductName": "Hidden Input",
"ProductVersion": "1, 0, 0, 0",
},
expectedPackage: pkg.Package{
Name: "Hidden Input",
Version: "1, 0, 0, 0",
},
},
{
name: "SQLite3",
versionResources: map[string]string{
"CompanyName": "SQLite Development Team",
"FileDescription": "SQLite is a software library that implements a self-contained, serverless, zero-configuration, transactional SQL database engine.",
"FileVersion": "3.23.2",
"InternalName": "sqlite3",
"LegalCopyright": "http://www.sqlite.org/copyright.html",
"ProductName": "SQLite",
"ProductVersion": "3.23.2",
},
expectedPackage: pkg.Package{
Name: "SQLite",
Version: "3.23.2",
},
},
{
name: "Brave Browser",
versionResources: map[string]string{
"CompanyName": "Brave Software, Inc.",
"FileDescription": "Brave Browser",
"FileVersion": "80.1.7.92",
"InternalName": "chrome_exe",
"LegalCopyright": "Copyright 2016 The Brave Authors. All rights reserved.",
"OriginalFilename": "chrome.exe",
"ProductName": "Brave Browser",
"ProductVersion": "80.1.7.92",
},
expectedPackage: pkg.Package{
Name: "Brave Browser",
Version: "80.1.7.92",
},
},
{
name: "Better product version",
versionResources: map[string]string{
"FileDescription": "Better version",
"FileVersion": "80.1.7",
"ProductVersion": "80.1.7.92",
},
expectedPackage: pkg.Package{
Name: "Better version",
Version: "80.1.7.92",
},
},
{
name: "Better file version",
versionResources: map[string]string{
"FileDescription": "Better version",
"FileVersion": "80.1.7.92",
"ProductVersion": "80.1.7",
},
expectedPackage: pkg.Package{
Name: "Better version",
Version: "80.1.7.92",
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
location := file.NewLocation("")
f := file.LocationReadCloser{
Location: file.NewLocation(""),
Location: location,
}
got, err := buildDotNetPackage(tc.versionResources, f)
assert.NoErrorf(t, err, "failed to build package from version resources: %+v", tc.versionResources)
// ignore certain metadata
if tc.expectedPackage.Metadata == nil {
got.Metadata = nil
}
// set known defaults
if tc.expectedPackage.Type == "" {
tc.expectedPackage.Type = pkg.DotnetPkg
}
if tc.expectedPackage.Language == "" {
tc.expectedPackage.Language = pkg.Dotnet
}
if tc.expectedPackage.PURL == "" {
tc.expectedPackage.PURL = portableExecutablePackageURL(tc.expectedPackage.Name, tc.expectedPackage.Version)
}
tc.expectedPackage.Locations = file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation))
pkgtest.AssertPackagesEqual(t, tc.expectedPackage, got)
})
}
}
func Test_extractVersion(t *testing.T) {
tests := []struct {
input string
expected string
}{
{
input: "1, 0, 0, 0",
expected: "1, 0, 0, 0",
},
{
input: "Release 73",
expected: "Release 73",
},
{
input: "4.7.4076.0 built by: NET472REL1LAST_B",
expected: "4.7.4076.0",
},
}
for _, test := range tests {
t.Run(test.input, func(t *testing.T) {
got := extractVersion(test.input)
assert.Equal(t, test.expected, got)
})
}
}
func Test_spaceNormalize(t *testing.T) {
tests := []struct {
input string
expected string
}{
{
expected: "some spaces apart",
input: " some spaces\n\t\t \n\rapart\n",
},
{
expected: "söme ¡nvalid characters",
input: "\rsöme \u0001¡nvalid\t characters\n",
},
}
for _, test := range tests {
t.Run(test.expected, func(t *testing.T) {
got := spaceNormalize(test.input)
assert.Equal(t, test.expected, got)
})
}
}
+18 -14
View File
@@ -20,6 +20,7 @@ import (
"github.com/go-git/go-git/v5/storage/memory"
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/licenses"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event/monitor"
@@ -31,7 +32,6 @@ import (
type goLicenses struct {
opts CatalogerConfig
localModCacheResolver file.WritableResolver
progress *monitor.CatalogerTask
lowerLicenseFileNames *strset.Set
}
@@ -39,11 +39,6 @@ func newGoLicenses(opts CatalogerConfig) goLicenses {
return goLicenses{
opts: opts,
localModCacheResolver: modCacheResolver(opts.LocalModCacheDir),
progress: &monitor.CatalogerTask{
SubStatus: true,
RemoveOnCompletion: true,
Title: "Downloading go mod",
},
lowerLicenseFileNames: strset.New(lowercaseLicenseFiles()...),
}
}
@@ -123,7 +118,16 @@ func (c *goLicenses) getLicensesFromRemote(moduleName, moduleVersion string) ([]
proxies := remotesForModule(c.opts.Proxies, c.opts.NoProxy, moduleName)
fsys, err := getModule(c.progress, proxies, moduleName, moduleVersion)
prog := bus.StartCatalogerTask(monitor.GenericTask{
Title: monitor.Title{
Default: "Download go mod",
WhileRunning: "Downloading go mod",
OnSuccess: "Downloaded go mod",
},
HideOnSuccess: true,
}, -1, "")
fsys, err := getModule(prog, proxies, moduleName, moduleVersion)
if err != nil {
return nil, err
}
@@ -206,7 +210,7 @@ func processCaps(s string) string {
})
}
func getModule(progress *monitor.CatalogerTask, proxies []string, moduleName, moduleVersion string) (fsys fs.FS, err error) {
func getModule(progress *monitor.CatalogerTaskProgress, proxies []string, moduleName, moduleVersion string) (fsys fs.FS, err error) {
for _, proxy := range proxies {
u, _ := url.Parse(proxy)
if proxy == "direct" {
@@ -218,7 +222,7 @@ func getModule(progress *monitor.CatalogerTask, proxies []string, moduleName, mo
fsys, err = getModuleProxy(progress, proxy, moduleName, moduleVersion)
case "file":
p := filepath.Join(u.Path, moduleName, "@v", moduleVersion)
progress.SetValue(fmt.Sprintf("file: %s", p))
progress.AtomicStage.Set(fmt.Sprintf("file: %s", p))
fsys = os.DirFS(p)
}
if fsys != nil {
@@ -228,9 +232,9 @@ func getModule(progress *monitor.CatalogerTask, proxies []string, moduleName, mo
return
}
func getModuleProxy(progress *monitor.CatalogerTask, proxy string, moduleName string, moduleVersion string) (out fs.FS, _ error) {
func getModuleProxy(progress *monitor.CatalogerTaskProgress, proxy string, moduleName string, moduleVersion string) (out fs.FS, _ error) {
u := fmt.Sprintf("%s/%s/@v/%s.zip", proxy, moduleName, moduleVersion)
progress.SetValue(u)
progress.AtomicStage.Set(u)
// get the module zip
resp, err := http.Get(u) //nolint:gosec
@@ -241,7 +245,7 @@ func getModuleProxy(progress *monitor.CatalogerTask, proxy string, moduleName st
if resp.StatusCode != http.StatusOK {
u = fmt.Sprintf("%s/%s/@v/%s.zip", proxy, strings.ToLower(moduleName), moduleVersion)
progress.SetValue(u)
progress.AtomicStage.Set(u)
// try lowercasing it; some packages have mixed casing that really messes up the proxy
resp, err = http.Get(u) //nolint:gosec
@@ -284,14 +288,14 @@ func findVersionPath(f fs.FS, dir string) string {
return ""
}
func getModuleRepository(progress *monitor.CatalogerTask, moduleName string, moduleVersion string) (fs.FS, error) {
func getModuleRepository(progress *monitor.CatalogerTaskProgress, moduleName string, moduleVersion string) (fs.FS, error) {
repoName := moduleName
parts := strings.Split(moduleName, "/")
if len(parts) > 2 {
repoName = fmt.Sprintf("%s/%s/%s", parts[0], parts[1], parts[2])
}
progress.SetValue(fmt.Sprintf("git: %s", repoName))
progress.AtomicStage.Set(fmt.Sprintf("git: %s", repoName))
f := memfs.New()
buf := &bytes.Buffer{}
+19 -7
View File
@@ -153,6 +153,22 @@ func extractNameFromApacheMavenBundlePlugin(manifest *pkg.JavaManifest) string {
return ""
}
func extractNameFromArchiveFilename(a archiveFilename) string {
if strings.Contains(a.name, ".") {
// special case: this *might* be a group id + artifact id. By convention artifact ids do not have "." in them;
// however, there are some specific exceptions like with the artifacts under
// https://repo1.maven.org/maven2/org/eclipse/platform/
if strings.HasPrefix(a.name, "org.eclipse.") {
return a.name
}
fields := strings.Split(a.name, ".")
return fields[len(fields)-1]
}
return a.name
}
func selectName(manifest *pkg.JavaManifest, filenameObj archiveFilename) string {
name := extractNameFromApacheMavenBundlePlugin(manifest)
if name != "" {
@@ -160,13 +176,9 @@ func selectName(manifest *pkg.JavaManifest, filenameObj archiveFilename) string
}
// the filename tends to be the next-best reference for the package name
if filenameObj.name != "" {
if strings.Contains(filenameObj.name, ".") {
// special case: this *might* be a group id + artifact id. By convention artifact ids do not have "." in them.
fields := strings.Split(filenameObj.name, ".")
return fields[len(fields)-1]
}
return filenameObj.name
name = extractNameFromArchiveFilename(filenameObj)
if name != "" {
return name
}
// remaining fields in the manifest is a bit of a free-for-all depending on the build tooling used and package maintainer preferences
@@ -209,6 +209,16 @@ func TestSelectName(t *testing.T) {
archive: newJavaArchiveFilename("/something/com.atlassian.gadgets.atlassian-gadgets-api.jar"),
expected: "atlassian-gadgets-api",
},
{
desc: "Skip stripping groupId prefix from archive filename for org.eclipse",
manifest: pkg.JavaManifest{
Main: map[string]string{
"Automatic-Module-Name": "org.eclipse.ant.core",
},
},
archive: newJavaArchiveFilename("/something/org.eclipse.ant.core-3.7.0.jar"),
expected: "org.eclipse.ant.core",
},
{
// example: pkg:maven/com.google.oauth-client/google-oauth-client@1.25.0
desc: "skip Apache Maven Bundle Plugin logic if symbolic name is same as vendor id",