From 5d48882a787ef55a8df8457bbb62f5bbe9974a8e Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 15 Sep 2023 14:51:21 -0400 Subject: [PATCH] Add GitHub actions and shared workflow usage catalogers (#2140) * add github actions usage cataloger Signed-off-by: Alex Goodman * update integration and cli tests with github actions sample Signed-off-by: Alex Goodman * add support for shared workflows Signed-off-by: Alex Goodman * split github actions usage cataloger Signed-off-by: Alex Goodman * add source explanation for github action types Signed-off-by: Alex Goodman * a github purl does not always mean the package is a github action Signed-off-by: Alex Goodman * keep github action catalogers as dir only catalogers Signed-off-by: Alex Goodman --------- Signed-off-by: Alex Goodman --- .../formats/common/spdxhelpers/source_info.go | 2 + .../common/spdxhelpers/source_info_test.go | 16 ++ syft/pkg/cataloger/cataloger.go | 5 + syft/pkg/cataloger/githubactions/cataloger.go | 16 ++ .../cataloger/githubactions/cataloger_test.go | 50 +++++ syft/pkg/cataloger/githubactions/package.go | 103 +++++++++ .../githubactions/parse_composite_action.go | 51 +++++ .../parse_composite_action_test.go | 35 +++ .../cataloger/githubactions/parse_workflow.go | 91 ++++++++ .../githubactions/parse_workflow_test.go | 88 ++++++++ .../test-fixtures/call-shared-workflow.yaml | 19 ++ .../test-fixtures/composite-action.yaml | 81 +++++++ .../.github/actions/bootstrap/action.yaml | 1 + .../.github/actions/unbootstrap/action.yml | 1 + .../glob/.github/workflows/release.yml | 1 + .../glob/.github/workflows/validations.yaml | 1 + .../test-fixtures/workflow-multi-job.yaml | 210 ++++++++++++++++++ syft/pkg/type.go | 65 +++--- syft/pkg/type_test.go | 1 + test/cli/packages_cmd_test.go | 14 +- .../catalog_packages_cases_test.go | 14 ++ test/integration/catalog_packages_test.go | 2 + .../.github/workflows/validations.yaml | 18 ++ 23 files changed, 853 insertions(+), 32 deletions(-) create mode 100644 syft/pkg/cataloger/githubactions/cataloger.go create mode 100644 syft/pkg/cataloger/githubactions/cataloger_test.go create mode 100644 syft/pkg/cataloger/githubactions/package.go create mode 100644 syft/pkg/cataloger/githubactions/parse_composite_action.go create mode 100644 syft/pkg/cataloger/githubactions/parse_composite_action_test.go create mode 100644 syft/pkg/cataloger/githubactions/parse_workflow.go create mode 100644 syft/pkg/cataloger/githubactions/parse_workflow_test.go create mode 100644 syft/pkg/cataloger/githubactions/test-fixtures/call-shared-workflow.yaml create mode 100644 syft/pkg/cataloger/githubactions/test-fixtures/composite-action.yaml create mode 100644 syft/pkg/cataloger/githubactions/test-fixtures/glob/.github/actions/bootstrap/action.yaml create mode 100644 syft/pkg/cataloger/githubactions/test-fixtures/glob/.github/actions/unbootstrap/action.yml create mode 100644 syft/pkg/cataloger/githubactions/test-fixtures/glob/.github/workflows/release.yml create mode 100644 syft/pkg/cataloger/githubactions/test-fixtures/glob/.github/workflows/validations.yaml create mode 100644 syft/pkg/cataloger/githubactions/test-fixtures/workflow-multi-job.yaml create mode 100644 test/integration/test-fixtures/image-pkg-coverage/pkgs/github-actions/.github/workflows/validations.yaml diff --git a/syft/formats/common/spdxhelpers/source_info.go b/syft/formats/common/spdxhelpers/source_info.go index 2ec80786e..c4942aeca 100644 --- a/syft/formats/common/spdxhelpers/source_info.go +++ b/syft/formats/common/spdxhelpers/source_info.go @@ -56,6 +56,8 @@ func SourceInfo(p pkg.Package) string { answer = "acquired package info from R-package DESCRIPTION file" case pkg.SwiftPkg: answer = "acquired package info from resolved Swift package manifest" + case pkg.GithubActionPkg, pkg.GithubActionWorkflowPkg: + answer = "acquired package info from GitHub Actions workflow file or composite action file" default: answer = "acquired package info from the following paths" } diff --git a/syft/formats/common/spdxhelpers/source_info_test.go b/syft/formats/common/spdxhelpers/source_info_test.go index 5b30b95fc..4fd221a9f 100644 --- a/syft/formats/common/spdxhelpers/source_info_test.go +++ b/syft/formats/common/spdxhelpers/source_info_test.go @@ -239,6 +239,22 @@ func Test_SourceInfo(t *testing.T) { "from resolved Swift package manifest", }, }, + { + input: pkg.Package{ + Type: pkg.GithubActionPkg, + }, + expected: []string{ + "from GitHub Actions workflow file or composite action file", + }, + }, + { + input: pkg.Package{ + Type: pkg.GithubActionWorkflowPkg, + }, + expected: []string{ + "from GitHub Actions workflow file or composite action file", + }, + }, } var pkgTypes []pkg.Type for _, test := range tests { diff --git a/syft/pkg/cataloger/cataloger.go b/syft/pkg/cataloger/cataloger.go index 2d3580024..c02c7e233 100644 --- a/syft/pkg/cataloger/cataloger.go +++ b/syft/pkg/cataloger/cataloger.go @@ -19,6 +19,7 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/dotnet" "github.com/anchore/syft/syft/pkg/cataloger/elixir" "github.com/anchore/syft/syft/pkg/cataloger/erlang" + "github.com/anchore/syft/syft/pkg/cataloger/githubactions" "github.com/anchore/syft/syft/pkg/cataloger/golang" "github.com/anchore/syft/syft/pkg/cataloger/haskell" "github.com/anchore/syft/syft/pkg/cataloger/java" @@ -74,6 +75,8 @@ func DirectoryCatalogers(cfg Config) []pkg.Cataloger { dotnet.NewDotnetPortableExecutableCataloger(), elixir.NewMixLockCataloger(), erlang.NewRebarLockCataloger(), + githubactions.NewActionUsageCataloger(), + githubactions.NewWorkflowUsageCataloger(), golang.NewGoModFileCataloger(cfg.Golang), golang.NewGoModuleBinaryCataloger(cfg.Golang), haskell.NewHackageCataloger(), @@ -110,6 +113,8 @@ func AllCatalogers(cfg Config) []pkg.Cataloger { dotnet.NewDotnetPortableExecutableCataloger(), elixir.NewMixLockCataloger(), erlang.NewRebarLockCataloger(), + githubactions.NewActionUsageCataloger(), + githubactions.NewWorkflowUsageCataloger(), golang.NewGoModFileCataloger(cfg.Golang), golang.NewGoModuleBinaryCataloger(cfg.Golang), haskell.NewHackageCataloger(), diff --git a/syft/pkg/cataloger/githubactions/cataloger.go b/syft/pkg/cataloger/githubactions/cataloger.go new file mode 100644 index 000000000..825d3942a --- /dev/null +++ b/syft/pkg/cataloger/githubactions/cataloger.go @@ -0,0 +1,16 @@ +package githubactions + +import "github.com/anchore/syft/syft/pkg/cataloger/generic" + +// NewActionUsageCataloger returns GitHub Actions used within workflows and composite actions. +func NewActionUsageCataloger() *generic.Cataloger { + return generic.NewCataloger("github-actions-usage-cataloger"). + WithParserByGlobs(parseWorkflowForActionUsage, "**/.github/workflows/*.yaml", "**/.github/workflows/*.yml"). + WithParserByGlobs(parseCompositeActionForActionUsage, "**/.github/actions/*/action.yml", "**/.github/actions/*/action.yaml") +} + +// NewWorkflowUsageCataloger returns shared workflows used within workflows. +func NewWorkflowUsageCataloger() *generic.Cataloger { + return generic.NewCataloger("github-action-workflow-usage-cataloger"). + WithParserByGlobs(parseWorkflowForWorkflowUsage, "**/.github/workflows/*.yaml", "**/.github/workflows/*.yml") +} diff --git a/syft/pkg/cataloger/githubactions/cataloger_test.go b/syft/pkg/cataloger/githubactions/cataloger_test.go new file mode 100644 index 000000000..f5866ef47 --- /dev/null +++ b/syft/pkg/cataloger/githubactions/cataloger_test.go @@ -0,0 +1,50 @@ +package githubactions + +import ( + "testing" + + "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" +) + +func TestCataloger_Globs(t *testing.T) { + tests := []struct { + name string + fixture string + cataloger *generic.Cataloger + expected []string + }{ + { + name: "obtain all workflow and composite action files", + fixture: "test-fixtures/glob", + cataloger: NewActionUsageCataloger(), + expected: []string{ + // composite actions + ".github/actions/bootstrap/action.yaml", + ".github/actions/unbootstrap/action.yml", + // workflows + ".github/workflows/release.yml", + ".github/workflows/validations.yaml", + }, + }, + { + name: "obtain all workflow files", + fixture: "test-fixtures/glob", + cataloger: NewWorkflowUsageCataloger(), + expected: []string{ + // workflows + ".github/workflows/release.yml", + ".github/workflows/validations.yaml", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + pkgtest.NewCatalogTester(). + FromDirectory(t, test.fixture). + ExpectsResolverContentQueries(test.expected). + TestCataloger(t, test.cataloger) + }) + } +} diff --git a/syft/pkg/cataloger/githubactions/package.go b/syft/pkg/cataloger/githubactions/package.go new file mode 100644 index 000000000..7d6341d3c --- /dev/null +++ b/syft/pkg/cataloger/githubactions/package.go @@ -0,0 +1,103 @@ +package githubactions + +import ( + "strings" + + "github.com/anchore/packageurl-go" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" +) + +func newPackageFromUsageStatement(use string, location file.Location) *pkg.Package { + name, version := parseStepUsageStatement(use) + + if name == "" { + log.WithFields("file", location.RealPath, "statement", use).Trace("unable to parse github action usage statement") + return nil + } + + if strings.Contains(name, ".github/workflows/") { + return newGithubActionWorkflowPackageUsage(name, version, location) + } + + return newGithubActionPackageUsage(name, version, location) +} + +func newGithubActionWorkflowPackageUsage(name, version string, workflowLocation file.Location) *pkg.Package { + p := &pkg.Package{ + Name: name, + Version: version, + Locations: file.NewLocationSet(workflowLocation.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)), + PURL: packageURL(name, version), + Type: pkg.GithubActionWorkflowPkg, + } + + p.SetID() + + return p +} + +func newGithubActionPackageUsage(name, version string, workflowLocation file.Location) *pkg.Package { + p := &pkg.Package{ + Name: name, + Version: version, + Locations: file.NewLocationSet(workflowLocation.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)), + PURL: packageURL(name, version), + Type: pkg.GithubActionPkg, + } + + p.SetID() + + return p +} + +func parseStepUsageStatement(use string) (string, string) { + // from octo-org/another-repo/.github/workflows/workflow.yml@v1 get octo-org/another-repo/.github/workflows/workflow.yml and v1 + // from ./.github/workflows/workflow-2.yml interpret as only the name + + // from actions/cache@v3 get actions/cache and v3 + + fields := strings.Split(use, "@") + switch len(fields) { + case 1: + return use, "" + case 2: + return fields[0], fields[1] + } + return "", "" +} + +func packageURL(name, version string) string { + var qualifiers packageurl.Qualifiers + var subPath string + var namespace string + + fields := strings.SplitN(name, "/", 3) + switch len(fields) { + case 1: + return "" + case 2: + namespace = fields[0] + name = fields[1] + case 3: + namespace = fields[0] + name = fields[1] + subPath = fields[2] + } + if namespace == "." { + // this is a local composite action, which is unclear how to represent in a PURL without more information + return "" + } + + // there isn't a github actions PURL but there is a github PURL type for referencing github repos, which is the + // next best thing until there is a supported type. + return packageurl.NewPackageURL( + packageurl.TypeGithub, + namespace, + name, + version, + qualifiers, + subPath, + ).ToString() +} diff --git a/syft/pkg/cataloger/githubactions/parse_composite_action.go b/syft/pkg/cataloger/githubactions/parse_composite_action.go new file mode 100644 index 000000000..0c27e32ce --- /dev/null +++ b/syft/pkg/cataloger/githubactions/parse_composite_action.go @@ -0,0 +1,51 @@ +package githubactions + +import ( + "fmt" + "io" + + "gopkg.in/yaml.v3" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" +) + +var _ generic.Parser = parseCompositeActionForActionUsage + +type compositeActionDef struct { + Runs compositeActionRunsDef `yaml:"runs"` +} + +type compositeActionRunsDef struct { + Steps []stepDef `yaml:"steps"` +} + +func parseCompositeActionForActionUsage(_ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + contents, err := io.ReadAll(reader) + if err != nil { + return nil, nil, fmt.Errorf("unable to read yaml composite action file: %w", err) + } + + var ca compositeActionDef + if err = yaml.Unmarshal(contents, &ca); err != nil { + return nil, nil, fmt.Errorf("unable to parse yaml composite action file: %w", err) + } + + // we use a collection to help with deduplication before raising to higher level processing + pkgs := pkg.NewCollection() + + for _, step := range ca.Runs.Steps { + if step.Uses == "" { + continue + } + + p := newPackageFromUsageStatement(step.Uses, reader.Location) + if p != nil { + pkgs.Add(*p) + } + } + + return pkgs.Sorted(), nil, nil +} diff --git a/syft/pkg/cataloger/githubactions/parse_composite_action_test.go b/syft/pkg/cataloger/githubactions/parse_composite_action_test.go new file mode 100644 index 000000000..e39e18e1a --- /dev/null +++ b/syft/pkg/cataloger/githubactions/parse_composite_action_test.go @@ -0,0 +1,35 @@ +package githubactions + +import ( + "testing" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" +) + +func Test_parseCompositeActionForActionUsage(t *testing.T) { + fixture := "test-fixtures/composite-action.yaml" + fixtureLocationSet := file.NewLocationSet(file.NewLocation(fixture).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)) + + expected := []pkg.Package{ + { + Name: "actions/setup-go", + Version: "v4", + Type: pkg.GithubActionPkg, + Locations: fixtureLocationSet, + PURL: "pkg:github/actions/setup-go@v4", + }, + { + Name: "actions/cache", + Version: "v3", + Type: pkg.GithubActionPkg, + Locations: fixtureLocationSet, + PURL: "pkg:github/actions/cache@v3", + }, + } + + var expectedRelationships []artifact.Relationship + pkgtest.TestFileParser(t, fixture, parseCompositeActionForActionUsage, expected, expectedRelationships) +} diff --git a/syft/pkg/cataloger/githubactions/parse_workflow.go b/syft/pkg/cataloger/githubactions/parse_workflow.go new file mode 100644 index 000000000..7460d87a7 --- /dev/null +++ b/syft/pkg/cataloger/githubactions/parse_workflow.go @@ -0,0 +1,91 @@ +package githubactions + +import ( + "fmt" + "io" + + "gopkg.in/yaml.v3" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" +) + +var ( + _ generic.Parser = parseWorkflowForActionUsage + _ generic.Parser = parseWorkflowForWorkflowUsage +) + +type workflowDef struct { + Jobs map[string]workflowJobDef `yaml:"jobs"` +} + +type workflowJobDef struct { + Uses string `yaml:"uses"` + Steps []stepDef `yaml:"steps"` +} + +type stepDef struct { + Name string `yaml:"name"` + Uses string `yaml:"uses"` + With struct { + Path string `yaml:"path"` + Key string `yaml:"key"` + } `yaml:"with"` +} + +func parseWorkflowForWorkflowUsage(_ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + contents, err := io.ReadAll(reader) + if err != nil { + return nil, nil, fmt.Errorf("unable to read yaml workflow file: %w", err) + } + + var wf workflowDef + if err = yaml.Unmarshal(contents, &wf); err != nil { + return nil, nil, fmt.Errorf("unable to parse yaml workflow file: %w", err) + } + + // we use a collection to help with deduplication before raising to higher level processing + pkgs := pkg.NewCollection() + + for _, job := range wf.Jobs { + if job.Uses != "" { + p := newPackageFromUsageStatement(job.Uses, reader.Location) + if p != nil { + pkgs.Add(*p) + } + } + } + + return pkgs.Sorted(), nil, nil +} + +func parseWorkflowForActionUsage(_ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + contents, err := io.ReadAll(reader) + if err != nil { + return nil, nil, fmt.Errorf("unable to read yaml workflow file: %w", err) + } + + var wf workflowDef + if err = yaml.Unmarshal(contents, &wf); err != nil { + return nil, nil, fmt.Errorf("unable to parse yaml workflow file: %w", err) + } + + // we use a collection to help with deduplication before raising to higher level processing + pkgs := pkg.NewCollection() + + for _, job := range wf.Jobs { + for _, step := range job.Steps { + if step.Uses == "" { + continue + } + p := newPackageFromUsageStatement(step.Uses, reader.Location) + if p != nil { + pkgs.Add(*p) + } + } + } + + return pkgs.Sorted(), nil, nil +} diff --git a/syft/pkg/cataloger/githubactions/parse_workflow_test.go b/syft/pkg/cataloger/githubactions/parse_workflow_test.go new file mode 100644 index 000000000..f5e5128b4 --- /dev/null +++ b/syft/pkg/cataloger/githubactions/parse_workflow_test.go @@ -0,0 +1,88 @@ +package githubactions + +import ( + "testing" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" +) + +func Test_parseWorkflowForActionUsage(t *testing.T) { + fixture := "test-fixtures/workflow-multi-job.yaml" + fixtureLocationSet := file.NewLocationSet(file.NewLocation(fixture).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)) + + expected := []pkg.Package{ + { + Name: "./.github/actions/bootstrap", + Version: "", + Type: pkg.GithubActionPkg, + Locations: fixtureLocationSet, + PURL: "", // don't have enough context without parsing the git origin, which still may not be accurate + }, + { + Name: "actions/cache", + Version: "v3", + Type: pkg.GithubActionPkg, + Locations: fixtureLocationSet, + PURL: "pkg:github/actions/cache@v3", + }, + { + Name: "actions/cache/restore", + Version: "v3", + Type: pkg.GithubActionPkg, + Locations: fixtureLocationSet, + PURL: "pkg:github/actions/cache@v3#restore", + }, + { + Name: "actions/cache/save", + Version: "v3", + Type: pkg.GithubActionPkg, + Locations: fixtureLocationSet, + PURL: "pkg:github/actions/cache@v3#save", + }, + { + Name: "actions/checkout", + Version: "v4", + Type: pkg.GithubActionPkg, + Locations: fixtureLocationSet, + PURL: "pkg:github/actions/checkout@v4", + }, + } + + var expectedRelationships []artifact.Relationship + pkgtest.TestFileParser(t, fixture, parseWorkflowForActionUsage, expected, expectedRelationships) +} + +func Test_parseWorkflowForWorkflowUsage(t *testing.T) { + fixture := "test-fixtures/call-shared-workflow.yaml" + fixtureLocationSet := file.NewLocationSet(file.NewLocation(fixture).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)) + + expected := []pkg.Package{ + { + Name: "octo-org/this-repo/.github/workflows/workflow-1.yml", + Version: "172239021f7ba04fe7327647b213799853a9eb89", + Type: pkg.GithubActionWorkflowPkg, + Locations: fixtureLocationSet, + PURL: "pkg:github/octo-org/this-repo@172239021f7ba04fe7327647b213799853a9eb89#.github/workflows/workflow-1.yml", + }, + { + Name: "./.github/workflows/workflow-2.yml", + Version: "", + Type: pkg.GithubActionWorkflowPkg, + Locations: fixtureLocationSet, + PURL: "", // don't have enough context without parsing the git origin, which still may not be accurate + }, + { + Name: "octo-org/another-repo/.github/workflows/workflow.yml", + Version: "v1", + Type: pkg.GithubActionWorkflowPkg, + Locations: fixtureLocationSet, + PURL: "pkg:github/octo-org/another-repo@v1#.github/workflows/workflow.yml", + }, + } + + var expectedRelationships []artifact.Relationship + pkgtest.TestFileParser(t, fixture, parseWorkflowForWorkflowUsage, expected, expectedRelationships) +} diff --git a/syft/pkg/cataloger/githubactions/test-fixtures/call-shared-workflow.yaml b/syft/pkg/cataloger/githubactions/test-fixtures/call-shared-workflow.yaml new file mode 100644 index 000000000..69061958b --- /dev/null +++ b/syft/pkg/cataloger/githubactions/test-fixtures/call-shared-workflow.yaml @@ -0,0 +1,19 @@ +jobs: + + call-workflow-1-in-local-repo: + uses: octo-org/this-repo/.github/workflows/workflow-1.yml@172239021f7ba04fe7327647b213799853a9eb89 + + call-workflow-2-in-local-repo: + uses: ./.github/workflows/workflow-2.yml + + call-workflow-in-another-repo: + uses: octo-org/another-repo/.github/workflows/workflow.yml@v1 + + + unit-test: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + + - name: Bootstrap environment + run: make unit diff --git a/syft/pkg/cataloger/githubactions/test-fixtures/composite-action.yaml b/syft/pkg/cataloger/githubactions/test-fixtures/composite-action.yaml new file mode 100644 index 000000000..b294c90ae --- /dev/null +++ b/syft/pkg/cataloger/githubactions/test-fixtures/composite-action.yaml @@ -0,0 +1,81 @@ +name: "Bootstrap" +description: "Bootstrap all tools and dependencies" +inputs: + go-version: + description: "Go version to install" + required: true + default: "1.21.x" + use-go-cache: + description: "Restore go cache" + required: true + default: "true" + cache-key-prefix: + description: "Prefix all cache keys with this value" + required: true + default: "831180ac25" + build-cache-key-prefix: + description: "Prefix build cache key with this value" + required: true + default: "f8b6d31dea" + bootstrap-apt-packages: + description: "Space delimited list of tools to install via apt" + default: "libxml2-utils" + +runs: + using: "composite" + steps: + - uses: actions/setup-go@v4 + with: + go-version: ${{ inputs.go-version }} + + - name: Restore tool cache + id: tool-cache + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/.tmp + key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-tool-${{ hashFiles('Makefile') }} + + # note: we need to keep restoring the go mod cache before bootstrapping tools since `go install` is used in + # some installations of project tools. + - name: Restore go module cache + id: go-mod-cache + if: inputs.use-go-cache == 'true' + uses: actions/cache@v3 + with: + path: | + ~/go/pkg/mod + key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-go-${{ inputs.go-version }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ inputs.cache-key-prefix }}-${{ runner.os }}-go-${{ inputs.go-version }}- + + - name: (cache-miss) Bootstrap project tools + shell: bash + if: steps.tool-cache.outputs.cache-hit != 'true' + run: make bootstrap-tools + + - name: Restore go build cache + id: go-cache + if: inputs.use-go-cache == 'true' + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + key: ${{ inputs.cache-key-prefix }}-${{ inputs.build-cache-key-prefix }}-${{ runner.os }}-go-${{ inputs.go-version }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ inputs.cache-key-prefix }}-${{ inputs.build-cache-key-prefix }}-${{ runner.os }}-go-${{ inputs.go-version }}- + + - name: (cache-miss) Bootstrap go dependencies + shell: bash + if: steps.go-mod-cache.outputs.cache-hit != 'true' && inputs.use-go-cache == 'true' + run: make bootstrap-go + + - name: Install apt packages + if: inputs.bootstrap-apt-packages != '' + shell: bash + run: | + DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y ${{ inputs.bootstrap-apt-packages }} + + - name: Create all cache fingerprints + shell: bash + run: make fingerprints + diff --git a/syft/pkg/cataloger/githubactions/test-fixtures/glob/.github/actions/bootstrap/action.yaml b/syft/pkg/cataloger/githubactions/test-fixtures/glob/.github/actions/bootstrap/action.yaml new file mode 100644 index 000000000..095e73c82 --- /dev/null +++ b/syft/pkg/cataloger/githubactions/test-fixtures/glob/.github/actions/bootstrap/action.yaml @@ -0,0 +1 @@ +# fake \ No newline at end of file diff --git a/syft/pkg/cataloger/githubactions/test-fixtures/glob/.github/actions/unbootstrap/action.yml b/syft/pkg/cataloger/githubactions/test-fixtures/glob/.github/actions/unbootstrap/action.yml new file mode 100644 index 000000000..095e73c82 --- /dev/null +++ b/syft/pkg/cataloger/githubactions/test-fixtures/glob/.github/actions/unbootstrap/action.yml @@ -0,0 +1 @@ +# fake \ No newline at end of file diff --git a/syft/pkg/cataloger/githubactions/test-fixtures/glob/.github/workflows/release.yml b/syft/pkg/cataloger/githubactions/test-fixtures/glob/.github/workflows/release.yml new file mode 100644 index 000000000..095e73c82 --- /dev/null +++ b/syft/pkg/cataloger/githubactions/test-fixtures/glob/.github/workflows/release.yml @@ -0,0 +1 @@ +# fake \ No newline at end of file diff --git a/syft/pkg/cataloger/githubactions/test-fixtures/glob/.github/workflows/validations.yaml b/syft/pkg/cataloger/githubactions/test-fixtures/glob/.github/workflows/validations.yaml new file mode 100644 index 000000000..095e73c82 --- /dev/null +++ b/syft/pkg/cataloger/githubactions/test-fixtures/glob/.github/workflows/validations.yaml @@ -0,0 +1 @@ +# fake \ No newline at end of file diff --git a/syft/pkg/cataloger/githubactions/test-fixtures/workflow-multi-job.yaml b/syft/pkg/cataloger/githubactions/test-fixtures/workflow-multi-job.yaml new file mode 100644 index 000000000..36502c133 --- /dev/null +++ b/syft/pkg/cataloger/githubactions/test-fixtures/workflow-multi-job.yaml @@ -0,0 +1,210 @@ +name: "Validations" + +on: + workflow_dispatch: + pull_request: + push: + branches: + - main + +jobs: + + Static-Analysis: + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + name: "Static analysis" + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + + - name: Bootstrap environment + uses: ./.github/actions/bootstrap + + - name: Run static analysis + run: make static-analysis + + + Unit-Test: + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + name: "Unit tests" + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + + - name: Bootstrap environment + uses: ./.github/actions/bootstrap + + - name: Restore Java test-fixture cache + uses: actions/cache@v3 + with: + path: syft/pkg/cataloger/java/test-fixtures/java-builds/packages + key: ${{ runner.os }}-unit-java-cache-${{ hashFiles( 'syft/pkg/cataloger/java/test-fixtures/java-builds/cache.fingerprint' ) }} + + - name: Restore RPM test-fixture cache + uses: actions/cache@v3 + with: + path: syft/pkg/cataloger/rpm/test-fixtures/rpms + key: ${{ runner.os }}-unit-rpm-cache-${{ hashFiles( 'syft/pkg/cataloger/rpm/test-fixtures/rpms.fingerprint' ) }} + + - name: Restore go binary test-fixture cache + uses: actions/cache@v3 + with: + path: syft/pkg/cataloger/golang/test-fixtures/archs/binaries + key: ${{ runner.os }}-unit-go-binaries-cache-${{ hashFiles( 'syft/pkg/cataloger/golang/test-fixtures/archs/binaries.fingerprint' ) }} + + - name: Restore binary cataloger test-fixture cache + uses: actions/cache@v3 + with: + path: syft/pkg/cataloger/binary/test-fixtures/classifiers/dynamic + key: ${{ runner.os }}-unit-binary-cataloger-cache-${{ hashFiles( 'syft/pkg/cataloger/binary/test-fixtures/cache.fingerprint' ) }} + + - name: Restore Kernel test-fixture cache + uses: actions/cache@v3 + with: + path: syft/pkg/cataloger/kernel/test-fixtures/cache + key: ${{ runner.os }}-unit-kernel-cache-${{ hashFiles( 'syft/pkg/cataloger/kernel/test-fixtures/cache.fingerprint' ) }} + + - name: Run unit tests + run: make unit + + + Integration-Test: + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + name: "Integration tests" + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + + - name: Bootstrap environment + uses: ./.github/actions/bootstrap + + - name: Validate syft output against the CycloneDX schema + run: make validate-cyclonedx-schema + + - name: Restore integration test cache + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/test/integration/test-fixtures/cache + key: ${{ runner.os }}-integration-test-cache-${{ hashFiles('test/integration/test-fixtures/cache.fingerprint') }} + + - name: Run integration tests + run: make integration + + + Build-Snapshot-Artifacts: + name: "Build snapshot artifacts" + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + + - name: Bootstrap environment + uses: ./.github/actions/bootstrap + with: + # why have another build cache key? We don't want unit/integration/etc test build caches to replace + # the snapshot build cache, which includes builds for all OSs and architectures. As long as this key is + # unique from the build-cache-key-prefix in other CI jobs, we should be fine. + # + # note: ideally this value should match what is used in release (just to help with build times). + build-cache-key-prefix: "snapshot" + bootstrap-apt-packages: "" + + - name: Build snapshot artifacts + run: make snapshot + + # why not use actions/upload-artifact? It is very slow (3 minutes to upload ~600MB of data, vs 10 seconds with this approach). + # see https://github.com/actions/upload-artifact/issues/199 for more info + - name: Upload snapshot artifacts + uses: actions/cache/save@v3 + with: + path: snapshot + key: snapshot-build-${{ github.run_id }} + + + Acceptance-Linux: + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + name: "Acceptance tests (Linux)" + needs: [Build-Snapshot-Artifacts] + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + + - name: Download snapshot build + uses: actions/cache/restore@v3 + with: + path: snapshot + key: snapshot-build-${{ github.run_id }} + + - name: Run comparison tests (Linux) + run: make compare-linux + + - name: Restore install.sh test image cache + id: install-test-image-cache + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/test/install/cache + key: ${{ runner.os }}-install-test-image-cache-${{ hashFiles('test/install/cache.fingerprint') }} + + - name: Load test image cache + if: steps.install-test-image-cache.outputs.cache-hit == 'true' + run: make install-test-cache-load + + - name: Run install.sh tests (Linux) + run: make install-test + + - name: (cache-miss) Create test image cache + if: steps.install-test-image-cache.outputs.cache-hit != 'true' + run: make install-test-cache-save + + + Acceptance-Mac: + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + name: "Acceptance tests (Mac)" + needs: [Build-Snapshot-Artifacts] + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + + - name: Download snapshot build + uses: actions/cache/restore@v3 + with: + path: snapshot + key: snapshot-build-${{ github.run_id }} + + - name: Restore docker image cache for compare testing + id: mac-compare-testing-cache + uses: actions/cache@v3 + with: + path: image.tar + key: ${{ runner.os }}-${{ hashFiles('test/compare/mac.sh') }} + + - name: Run comparison tests (Mac) + run: make compare-mac + + - name: Run install.sh tests (Mac) + run: make install-test-ci-mac + + + Cli-Linux: + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + name: "CLI tests (Linux)" + needs: [Build-Snapshot-Artifacts] + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + + - name: Bootstrap environment + uses: ./.github/actions/bootstrap + + - name: Restore CLI test-fixture cache + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/test/cli/test-fixtures/cache + key: ${{ runner.os }}-cli-test-cache-${{ hashFiles('test/cli/test-fixtures/cache.fingerprint') }} + + - name: Download snapshot build + uses: actions/cache/restore@v3 + with: + path: snapshot + key: snapshot-build-${{ github.run_id }} + + - name: Run CLI Tests (Linux) + run: make cli diff --git a/syft/pkg/type.go b/syft/pkg/type.go index e3ed3f4c1..788e584a5 100644 --- a/syft/pkg/type.go +++ b/syft/pkg/type.go @@ -9,34 +9,36 @@ type Type string const ( // the full set of supported packages - UnknownPkg Type = "UnknownPackage" - AlpmPkg Type = "alpm" - ApkPkg Type = "apk" - BinaryPkg Type = "binary" - CocoapodsPkg Type = "pod" - ConanPkg Type = "conan" - DartPubPkg Type = "dart-pub" - DebPkg Type = "deb" - DotnetPkg Type = "dotnet" - GemPkg Type = "gem" - GoModulePkg Type = "go-module" - GraalVMNativeImagePkg Type = "graalvm-native-image" - HackagePkg Type = "hackage" - HexPkg Type = "hex" - JavaPkg Type = "java-archive" - JenkinsPluginPkg Type = "jenkins-plugin" - KbPkg Type = "msrc-kb" - LinuxKernelPkg Type = "linux-kernel" - LinuxKernelModulePkg Type = "linux-kernel-module" - NixPkg Type = "nix" - NpmPkg Type = "npm" - PhpComposerPkg Type = "php-composer" - PortagePkg Type = "portage" - PythonPkg Type = "python" - Rpkg Type = "R-package" - RpmPkg Type = "rpm" - RustPkg Type = "rust-crate" - SwiftPkg Type = "swift" + UnknownPkg Type = "UnknownPackage" + AlpmPkg Type = "alpm" + ApkPkg Type = "apk" + BinaryPkg Type = "binary" + CocoapodsPkg Type = "pod" + ConanPkg Type = "conan" + DartPubPkg Type = "dart-pub" + DebPkg Type = "deb" + DotnetPkg Type = "dotnet" + GemPkg Type = "gem" + GithubActionPkg Type = "github-action" + GithubActionWorkflowPkg Type = "github-action-workflow" + GoModulePkg Type = "go-module" + GraalVMNativeImagePkg Type = "graalvm-native-image" + HackagePkg Type = "hackage" + HexPkg Type = "hex" + JavaPkg Type = "java-archive" + JenkinsPluginPkg Type = "jenkins-plugin" + KbPkg Type = "msrc-kb" + LinuxKernelPkg Type = "linux-kernel" + LinuxKernelModulePkg Type = "linux-kernel-module" + NixPkg Type = "nix" + NpmPkg Type = "npm" + PhpComposerPkg Type = "php-composer" + PortagePkg Type = "portage" + PythonPkg Type = "python" + Rpkg Type = "R-package" + RpmPkg Type = "rpm" + RustPkg Type = "rust-crate" + SwiftPkg Type = "swift" ) // AllPkgs represents all supported package types @@ -50,6 +52,8 @@ var AllPkgs = []Type{ DebPkg, DotnetPkg, GemPkg, + GithubActionPkg, + GithubActionWorkflowPkg, GoModulePkg, HackagePkg, HexPkg, @@ -70,6 +74,8 @@ var AllPkgs = []Type{ } // PackageURLType returns the PURL package type for the current package. +// +//nolint:funlen func (t Type) PackageURLType() string { switch t { case AlpmPkg: @@ -90,6 +96,9 @@ func (t Type) PackageURLType() string { return packageurl.TypeGem case HexPkg: return packageurl.TypeHex + case GithubActionPkg, GithubActionWorkflowPkg: + // note: this is not a real purl type, but it is the closest thing we have for now + return packageurl.TypeGithub case GoModulePkg: return packageurl.TypeGolang case HackagePkg: diff --git a/syft/pkg/type_test.go b/syft/pkg/type_test.go index 3da73184d..64d8e87c3 100644 --- a/syft/pkg/type_test.go +++ b/syft/pkg/type_test.go @@ -114,6 +114,7 @@ func TestTypeFromPURL(t *testing.T) { expectedTypes.Remove(string(PortagePkg)) expectedTypes.Remove(string(BinaryPkg)) expectedTypes.Remove(string(LinuxKernelModulePkg)) + expectedTypes.Remove(string(GithubActionPkg), string(GithubActionWorkflowPkg)) for _, test := range tests { t.Run(string(test.expected), func(t *testing.T) { diff --git a/test/cli/packages_cmd_test.go b/test/cli/packages_cmd_test.go index dc100ebeb..81ad258b1 100644 --- a/test/cli/packages_cmd_test.go +++ b/test/cli/packages_cmd_test.go @@ -6,6 +6,12 @@ import ( "testing" ) +const ( + // this is the number of packages that should be found in the image-pkg-coverage fixture image + // when analyzed with the squashed scope. + coverageImageSquashedPackageCount = 24 +) + func TestPackagesCmdFlags(t *testing.T) { hiddenPackagesImage := "docker-archive:" + getFixtureImage(t, "image-hidden-packages") coverageImage := "docker-archive:" + getFixtureImage(t, "image-pkg-coverage") @@ -114,7 +120,7 @@ func TestPackagesCmdFlags(t *testing.T) { name: "squashed-scope-flag", args: []string{"packages", "-o", "json", "-s", "squashed", coverageImage}, assertions: []traitAssertion{ - assertPackageCount(24), + assertPackageCount(coverageImageSquashedPackageCount), assertSuccessfulReturnCode, }, }, @@ -231,7 +237,7 @@ func TestPackagesCmdFlags(t *testing.T) { // the application config in the log matches that of what we expect to have been configured. assertInOutput("parallelism: 2"), assertInOutput("parallelism=2"), - assertPackageCount(24), + assertPackageCount(coverageImageSquashedPackageCount), assertSuccessfulReturnCode, }, }, @@ -242,7 +248,7 @@ func TestPackagesCmdFlags(t *testing.T) { // the application config in the log matches that of what we expect to have been configured. assertInOutput("parallelism: 1"), assertInOutput("parallelism=1"), - assertPackageCount(24), + assertPackageCount(coverageImageSquashedPackageCount), assertSuccessfulReturnCode, }, }, @@ -256,7 +262,7 @@ func TestPackagesCmdFlags(t *testing.T) { assertions: []traitAssertion{ assertNotInOutput("secret_password"), assertNotInOutput("secret_key_path"), - assertPackageCount(24), + assertPackageCount(coverageImageSquashedPackageCount), assertSuccessfulReturnCode, }, }, diff --git a/test/integration/catalog_packages_cases_test.go b/test/integration/catalog_packages_cases_test.go index 48b280a56..c88b2d559 100644 --- a/test/integration/catalog_packages_cases_test.go +++ b/test/integration/catalog_packages_cases_test.go @@ -368,6 +368,20 @@ var dirOnlyTestCases = []testCase{ "swift-numerics": "1.0.2", }, }, + { + name: "find github action packages (from usage in workflow files and composite actions)", + pkgType: pkg.GithubActionPkg, + pkgInfo: map[string]string{ + "actions/checkout": "v4", + }, + }, + { + name: "find github shared workflow calls (from usage in workflow files)", + pkgType: pkg.GithubActionWorkflowPkg, + pkgInfo: map[string]string{ + "octo-org/this-repo/.github/workflows/workflow-1.yml": "172239021f7ba04fe7327647b213799853a9eb89", + }, + }, } var commonTestCases = []testCase{ diff --git a/test/integration/catalog_packages_test.go b/test/integration/catalog_packages_test.go index 0b6ca54f0..50a8b06ba 100644 --- a/test/integration/catalog_packages_test.go +++ b/test/integration/catalog_packages_test.go @@ -96,6 +96,8 @@ func TestPkgCoverageImage(t *testing.T) { definedPkgs.Remove(string(pkg.LinuxKernelPkg)) definedPkgs.Remove(string(pkg.LinuxKernelModulePkg)) definedPkgs.Remove(string(pkg.SwiftPkg)) + definedPkgs.Remove(string(pkg.GithubActionPkg)) + definedPkgs.Remove(string(pkg.GithubActionWorkflowPkg)) var cases []testCase cases = append(cases, commonTestCases...) diff --git a/test/integration/test-fixtures/image-pkg-coverage/pkgs/github-actions/.github/workflows/validations.yaml b/test/integration/test-fixtures/image-pkg-coverage/pkgs/github-actions/.github/workflows/validations.yaml new file mode 100644 index 000000000..2c8c17526 --- /dev/null +++ b/test/integration/test-fixtures/image-pkg-coverage/pkgs/github-actions/.github/workflows/validations.yaml @@ -0,0 +1,18 @@ +name: "Validations" + +on: + workflow_dispatch: + pull_request: + push: + branches: + - main + +jobs: + call-workflow-1-in-local-repo: + uses: octo-org/this-repo/.github/workflows/workflow-1.yml@172239021f7ba04fe7327647b213799853a9eb89 + + Unit-Test: + name: "Unit tests" + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4