diff --git a/.github/workflows/acceptance-test.yaml b/.github/workflows/acceptance-test.yaml deleted file mode 100644 index 2fd79cad7..000000000 --- a/.github/workflows/acceptance-test.yaml +++ /dev/null @@ -1,142 +0,0 @@ -name: "Acceptance" -on: - workflow_dispatch: - push: - # ... only act on pushes to main - branches: - - main - # ... do not act on release tags - tags-ignore: - - v* - -env: - GO_VERSION: "1.14.x" - -jobs: - Build-Snapshot-Artifacts: - # though the release pipeline is running on mac for the signing step, we are skipping the signing step here and - # require a system with docker installed, which ubuntu-20.04 has by default (and mac does not for licensing reasons). - runs-on: ubuntu-20.04 - steps: - - uses: actions/setup-go@v2 - with: - go-version: ${{ env.GO_VERSION }} - - - uses: actions/checkout@v2 - - - name: Restore bootstrap cache - id: cache - uses: actions/cache@v2.1.3 - with: - path: | - ~/go/pkg/mod - ${{ github.workspace }}/.tmp - key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}-${{ hashFiles('Makefile') }} - restore-keys: | - ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}- - ${{ runner.os }}-go-${{ env.GO_VERSION }}- - - - name: Bootstrap project dependencies - if: steps.bootstrap-cache.outputs.cache-hit != 'true' - run: make bootstrap - - - name: Build snapshot artifacts - run: make snapshot - - - uses: actions/upload-artifact@v2 - with: - name: artifacts - path: snapshot/**/* - - - uses: 8398a7/action-slack@v3 - with: - status: ${{ job.status }} - fields: repo,workflow,job,commit,message,author - text: The syft acceptance tests have failed tragically! - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }} - if: ${{ failure() }} - - # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline - Acceptance-Linux: - needs: [Build-Snapshot-Artifacts] - # come Nov 30 2020 ubuntu-latest will be ubuntu-20.04, until then it needs to be explicitly referenced due to python 3.7 specific features being used - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - - uses: actions/download-artifact@v2 - with: - name: artifacts - path: snapshot - - - name: Run Acceptance Tests (Linux) - run: make acceptance-linux - - - uses: 8398a7/action-slack@v3 - with: - status: ${{ job.status }} - fields: repo,workflow,job,commit,message,author - text: The syft acceptance tests have failed tragically! - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }} - if: ${{ failure() }} - - # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline - Acceptance-Mac: - needs: [Build-Snapshot-Artifacts] - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - - uses: actions/download-artifact@v2 - with: - name: artifacts - path: snapshot - - - name: Run Acceptance Tests (Mac) - run: make acceptance-mac - - - uses: 8398a7/action-slack@v3 - with: - status: ${{ job.status }} - fields: repo,workflow,job,commit,message,author - text: The syft acceptance tests have failed tragically! - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }} - if: ${{ failure() }} - - # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline - Inline-Compare: - needs: [Build-Snapshot-Artifacts] - # come Nov 30 2020 ubuntu-latest will be ubuntu-20.04, until then it needs to be explicitly referenced due to python 3.7 specific features being used - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - - name: Fingerprint inline-compare sources - run: make compare-fingerprint - - - name: Restore inline reports cache - id: cache - uses: actions/cache@v2.1.3 - with: - path: ${{ github.workspace }}/test/inline-compare/inline-reports - key: inline-reports-${{ hashFiles('**/inline-compare.fingerprint') }} - - - uses: actions/download-artifact@v2 - with: - name: artifacts - path: snapshot - - - name: Compare Anchore inline-scan results against snapshot build output - run: make compare-snapshot - - - uses: 8398a7/action-slack@v3 - with: - status: ${{ job.status }} - fields: repo,workflow,job,commit,message,author - text: The syft acceptance tests have failed tragically! - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }} - if: ${{ failure() }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6e9393c3a..16d2500d1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,7 +9,7 @@ on: - "v*" env: - GO_VERSION: "1.14.x" + GO_VERSION: "1.16.x" jobs: wait-for-checks: @@ -29,17 +29,26 @@ jobs: id: static-analysis with: token: ${{ secrets.GITHUB_TOKEN }} - # This check name is defined as the circle-ci workflow name (in .circleci/config.yaml) - checkName: "Static-Analysis (1.x, ubuntu-latest)" + # This check name is defined as the github action job name (in .github/workflows/testing.yaml) + checkName: "Static-Analysis" ref: ${{ github.event.pull_request.head.sha || github.sha }} - - name: Check unit + integration results (latest go version) + - name: Check unit test results uses: fountainhead/action-wait-for-check@v1.0.0 - id: unit-integration + id: unit with: token: ${{ secrets.GITHUB_TOKEN }} - # This check name is defined as the circle-ci workflow name (in .circleci/config.yaml) - checkName: "Tests (1.x, ubuntu-latest)" + # This check name is defined as the github action job name (in .github/workflows/testing.yaml) + checkName: "Unit-Test" + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Check integration test results + uses: fountainhead/action-wait-for-check@v1.0.0 + id: integration + with: + token: ${{ secrets.GITHUB_TOKEN }} + # This check name is defined as the github action job name (in .github/workflows/testing.yaml) + checkName: "Integration-Test" ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Check acceptance test results (linux) @@ -47,7 +56,7 @@ jobs: id: acceptance-linux with: token: ${{ secrets.GITHUB_TOKEN }} - # This check name is defined as the github action job name (in .github/workflows/acceptance-test.yaml) + # This check name is defined as the github action job name (in .github/workflows/testing.yaml) checkName: "Acceptance-Linux" ref: ${{ github.event.pull_request.head.sha || github.sha }} @@ -56,27 +65,28 @@ jobs: id: acceptance-mac with: token: ${{ secrets.GITHUB_TOKEN }} - # This check name is defined as the github action job name (in .github/workflows/acceptance-test.yaml) + # This check name is defined as the github action job name (in .github/workflows/testing.yaml) checkName: "Acceptance-Mac" ref: ${{ github.event.pull_request.head.sha || github.sha }} - - name: Check inline comparison test results + - name: Check cli test results (linux) uses: fountainhead/action-wait-for-check@v1.0.0 - id: inline-compare + id: cli-linux with: token: ${{ secrets.GITHUB_TOKEN }} - # This check name is defined as the github action job name (in .github/workflows/acceptance-test.yaml) - checkName: "Inline-Compare" + # This check name is defined as the github action job name (in .github/workflows/testing.yaml) + checkName: "Cli-Linux" ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Quality gate - if: steps.static-analysis.outputs.conclusion != 'success' || steps.unit-integration.outputs.conclusion != 'success' || steps.inline-compare.outputs.conclusion != 'success' || steps.acceptance-linux.outputs.conclusion != 'success' || steps.acceptance-mac.outputs.conclusion != 'success' + if: steps.static-analysis.outputs.conclusion != 'success' || steps.unit.outputs.conclusion != 'success' || steps.integration.outputs.conclusion != 'success' || steps.cli-linux.outputs.conclusion != 'success' || steps.acceptance-linux.outputs.conclusion != 'success' || steps.acceptance-mac.outputs.conclusion != 'success' run: | echo "Static Analysis Status: ${{ steps.static-analysis.conclusion }}" - echo "Unit & Integration Test Status: ${{ steps.unit-integration.outputs.conclusion }}" + echo "Unit Test Status: ${{ steps.unit.outputs.conclusion }}" + echo "Integration Test Status: ${{ steps.integration.outputs.conclusion }}" echo "Acceptance Test (Linux) Status: ${{ steps.acceptance-linux.outputs.conclusion }}" echo "Acceptance Test (Mac) Status: ${{ steps.acceptance-mac.outputs.conclusion }}" - echo "Inline Compare Status: ${{ steps.inline-compare.outputs.conclusion }}" + echo "CLI Test (Linux) Status: ${{ steps.cli-linux.outputs.conclusion }}" false release: @@ -93,23 +103,32 @@ jobs: with: fetch-depth: 0 - # We are expecting this cache to have been created during the "Build-Snapshot-Artifacts" job in the "Acceptance" workflow. - - name: Restore bootstrap cache - id: cache + - name: Restore tool cache + id: tool-cache uses: actions/cache@v2.1.3 with: - path: | - ~/go/pkg/mod - ${{ github.workspace }}/.tmp - key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}-${{ hashFiles('Makefile') }} + path: ${{ github.workspace }}/.tmp + key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} + + - name: Restore go cache + id: go-cache + uses: actions/cache@v2.1.3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} restore-keys: | - ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}- ${{ runner.os }}-go-${{ env.GO_VERSION }}- - - name: Bootstrap project dependencies - if: steps.bootstrap-cache.outputs.cache-hit != 'true' + - name: (cache-miss) Bootstrap all project dependencies + if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' run: make bootstrap + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.TOOLBOX_DOCKER_USER }} + password: ${{ secrets.TOOLBOX_DOCKER_PASS }} + - name: Import GPG key id: import_gpg uses: crazy-max/ghaction-import-gpg@v2 diff --git a/.github/workflows/static-unit-integration.yaml b/.github/workflows/static-unit-integration.yaml deleted file mode 100644 index eac581c30..000000000 --- a/.github/workflows/static-unit-integration.yaml +++ /dev/null @@ -1,101 +0,0 @@ -name: "Static Analysis + Unit + Integration" -on: - workflow_dispatch: - push: - pull_request: -jobs: - Static-Analysis: - strategy: - matrix: - go-version: [1.x] - platform: [ubuntu-latest] - runs-on: ${{ matrix.platform }} - steps: - - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - - - uses: actions/checkout@v2 - - - name: Restore bootstrap cache - id: bootstrap-cache - uses: actions/cache@v2.1.3 - with: - path: | - ~/go/pkg/mod - ${{ github.workspace }}/.tmp - key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }}-${{ hashFiles('Makefile') }} - restore-keys: | - ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }}- - ${{ runner.os }}-go-${{ matrix.go-version }}- - - - name: Bootstrap project dependencies - if: steps.bootstrap-cache.outputs.cache-hit != 'true' - run: make bootstrap - - - name: Bootstrap CI dependencies - run: make ci-bootstrap - - - name: Run static analysis - run: make static-analysis - - Tests: - strategy: - matrix: - # test the lower bounds of support, and the latest available - go-version: [1.13.x, 1.x] - platform: [ubuntu-latest] - runs-on: ${{ matrix.platform }} - steps: - - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - - - uses: actions/checkout@v2 - - - name: Restore bootstrap cache - id: bootstrap-cache - uses: actions/cache@v2.1.3 - with: - path: | - ~/go/pkg/mod - ${{ github.workspace }}/.tmp - key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }}-${{ hashFiles('Makefile') }} - restore-keys: | - ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }}- - ${{ runner.os }}-go-${{ matrix.go-version }}- - - - name: Bootstrap project dependencies - if: steps.bootstrap-cache.outputs.cache-hit != 'true' - run: make bootstrap - - - name: Bootstrap CI dependencies - run: make ci-bootstrap - - - name: Build cache key for java test-fixture blobs (for unit tests) - run: make java-packages-fingerprint - - - name: Restore Java test-fixture cache - id: unit-java-cache - uses: actions/cache@v2.1.3 - with: - path: syft/cataloger/java/test-fixtures/java-builds/packages - key: ${{ runner.os }}-unit-java-cache-${{ hashFiles( 'syft/cataloger/java/test-fixtures/java-builds/packages.fingerprint' ) }} - - - name: Run unit tests - run: make unit - - - name: Validate syft output against the CycloneDX schema - run: make validate-cyclonedx-schema - - - name: Build key for tar cache - run: make integration-fingerprint - - - name: Restore integration test cache - uses: actions/cache@v2.1.3 - with: - path: ${{ github.workspace }}/integration/test-fixtures/cache - key: ${{ runner.os }}-integration-test-cache-${{ hashFiles('integration/test-fixtures/cache.fingerprint') }} - - - name: Run integration tests - run: make integration diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml new file mode 100644 index 000000000..73305c489 --- /dev/null +++ b/.github/workflows/validations.yaml @@ -0,0 +1,354 @@ +name: "Validations" +on: + workflow_dispatch: + push: + pull_request: + +env: + GO_VERSION: "1.16.x" + +jobs: + + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + Static-Analysis: + name: "Static analysis" + runs-on: ubuntu-20.04 + # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR) + # skip if this is a pull_request event on an internal PR (which is already covered by push events) + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + steps: + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - uses: actions/checkout@v2 + + - name: Restore tool cache + id: tool-cache + uses: actions/cache@v2.1.3 + with: + path: ${{ github.workspace }}/.tmp + key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} + + - name: Restore go cache + id: go-cache + uses: actions/cache@v2.1.3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ env.GO_VERSION }}- + + - name: (cache-miss) Bootstrap all project dependencies + if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' + run: make bootstrap + + - name: Bootstrap CI environment dependencies + run: make ci-bootstrap + + - name: Run static analysis + run: make static-analysis + + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + Unit-Test: + name: "Unit tests" + runs-on: ubuntu-20.04 + # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR) + # skip if this is a pull_request event on an internal PR (which is already covered by push events) + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + steps: + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - uses: actions/checkout@v2 + + - name: Restore tool cache + id: tool-cache + uses: actions/cache@v2.1.3 + with: + path: ${{ github.workspace }}/.tmp + key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} + + - name: Restore go cache + id: go-cache + uses: actions/cache@v2.1.3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ env.GO_VERSION }}- + + - name: (cache-miss) Bootstrap all project dependencies + if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' + run: make bootstrap + + - name: Bootstrap CI environment dependencies + run: make ci-bootstrap + + - name: Build cache key for java test-fixture blobs (for unit tests) + run: make java-packages-fingerprint + + - name: Restore Java test-fixture cache + id: unit-java-cache + uses: actions/cache@v2.1.3 + with: + path: syft/pkg/cataloger/java/test-fixtures/java-builds/packages + key: ${{ runner.os }}-unit-java-cache-${{ hashFiles( 'syft/pkg/cataloger/java/test-fixtures/java-builds/packages.fingerprint' ) }} + + - name: Run unit tests + run: make unit + + - uses: actions/upload-artifact@v2 + with: + name: unit-test-results + path: test/results/**/* + + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + Integration-Test: + name: "Integration tests" + runs-on: ubuntu-20.04 + # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR) + # skip if this is a pull_request event on an internal PR (which is already covered by push events) + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + steps: + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - uses: actions/checkout@v2 + + - name: Restore tool cache + id: tool-cache + uses: actions/cache@v2.1.3 + with: + path: ${{ github.workspace }}/.tmp + key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} + + - name: Restore go cache + id: go-cache + uses: actions/cache@v2.1.3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ env.GO_VERSION }}- + + - name: (cache-miss) Bootstrap all project dependencies + if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' + run: make bootstrap + + - name: Bootstrap CI environment dependencies + run: make ci-bootstrap + + - name: Validate syft output against the CycloneDX schema + run: make validate-cyclonedx-schema + + - name: Build key for tar cache + run: make integration-fingerprint + + - name: Restore integration test cache + uses: actions/cache@v2.1.3 + with: + path: ${{ github.workspace }}/test/integration/test-fixtures/cache + key: ${{ runner.os }}-integration-test-cache-${{ hashFiles('test/integration/test-fixtures/cache.fingerprint') }} + + - name: Run integration tests + run: make integration + + Benchmark-Test: + name: "Benchmark tests" + runs-on: ubuntu-20.04 + # note: we want benchmarks to run on pull_request events in order to publish results to a sticky comment, and + # we also want to run on push such that merges to main are recorded to the cache. For this reason we don't filter + # the job by event. + steps: + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - uses: actions/checkout@v2 + + - name: Restore tool cache + id: tool-cache + uses: actions/cache@v2.1.3 + with: + path: ${{ github.workspace }}/.tmp + key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} + + - name: Restore go cache + id: go-cache + uses: actions/cache@v2.1.3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ env.GO_VERSION }}- + + - name: (cache-miss) Bootstrap all project dependencies + if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' + run: make bootstrap + + - name: Bootstrap CI environment dependencies + run: make ci-bootstrap + + - name: Restore base benchmark result + uses: actions/cache@v2 + with: + path: test/results/benchmark-main.txt + # use base sha for PR or new commit hash for main push in benchmark result key + key: ${{ runner.os }}-bench-${{ (github.event.pull_request.base.sha != github.event.after) && github.event.pull_request.base.sha || github.event.after }} + + - name: Run benchmark tests + id: benchmark + run: | + REF_NAME=${GITHUB_REF##*/} make benchmark + echo "::set-output name=result::$(make show-benchstat)" + + - uses: actions/upload-artifact@v2 + with: + name: benchmark-test-results + path: test/results/**/* + + - name: Update PR benchmark results comment + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: benchmark + message: | + ### Benchmark Test Results + +
+ Benchmark results from the latest changes vs base branch + + ``` + ${{ steps.benchmark.outputs.result }} + ``` + +
+ + Build-Snapshot-Artifacts: + name: "Build snapshot artifacts" + runs-on: ubuntu-20.04 + # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR) + # skip if this is a pull_request event on an internal PR (which is already covered by push events) + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + steps: + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - uses: actions/checkout@v2 + + - name: Restore tool cache + id: tool-cache + uses: actions/cache@v2.1.3 + with: + path: ${{ github.workspace }}/.tmp + key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} + + - name: Restore go cache + id: go-cache + uses: actions/cache@v2.1.3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ env.GO_VERSION }}- + + - name: (cache-miss) Bootstrap all project dependencies + if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' + run: make bootstrap + + - name: Build snapshot artifacts + run: make snapshot + + - uses: actions/upload-artifact@v2 + with: + name: artifacts + path: snapshot/**/* + + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + Acceptance-Linux: + name: "Acceptance tests (Linux)" + needs: [Build-Snapshot-Artifacts] + runs-on: ubuntu-20.04 + # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR) + # skip if this is a pull_request event on an internal PR (which is already covered by push events) + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + steps: + - uses: actions/checkout@v2 + + - uses: actions/download-artifact@v2 + with: + name: artifacts + path: snapshot + + - name: Run Acceptance Tests (Linux) + run: make acceptance-linux + + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + Acceptance-Mac: + name: "Acceptance tests (Mac)" + needs: [Build-Snapshot-Artifacts] + runs-on: macos-latest + # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR) + # skip if this is a pull_request event on an internal PR (which is already covered by push events) + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + steps: + - uses: actions/checkout@v2 + + - uses: actions/download-artifact@v2 + with: + name: artifacts + path: snapshot + + - name: Restore docker image cache + id: go-cache + uses: actions/cache@v2.1.3 + with: + path: /tmp/image.tar + key: ${{ runner.os }}-${{ hashFiles('test/acceptance/mac.sh') }} + + - name: Run Acceptance Tests (Mac) + run: make acceptance-mac + + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + Cli-Linux: + name: "CLI tests (Linux)" + needs: [Build-Snapshot-Artifacts] + runs-on: ubuntu-20.04 + # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR) + # skip if this is a pull_request event on an internal PR (which is already covered by push events) + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + steps: + - uses: actions/checkout@v2 + + - name: Restore go cache + id: go-cache + uses: actions/cache@v2.1.3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ env.GO_VERSION }}- + + - name: (cache-miss) Bootstrap go dependencies + if: steps.go-cache.outputs.cache-hit != 'true' + run: make bootstrap-go + + - name: Build key for tar cache + run: make cli-fingerprint + + - name: Restore CLI test cache + uses: actions/cache@v2.1.3 + with: + path: ${{ github.workspace }}/test/cli/test-fixtures/cache + key: ${{ runner.os }}-cli-test-cache-${{ hashFiles('test/cli/test-fixtures/cache.fingerprint') }} + + - uses: actions/download-artifact@v2 + with: + name: artifacts + path: snapshot + + - name: Run CLI Tests (Linux) + run: make cli diff --git a/.gitignore b/.gitignore index 3edd0085e..5e4ca2bee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ CHANGELOG.md +/test/results /dist /snapshot .server/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml index bd8c7749e..2317d3bbf 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -92,18 +92,15 @@ brews: description: *description dockers: - - - binaries: - - syft - dockerfile: Dockerfile + - dockerfile: Dockerfile image_templates: - "anchore/syft:latest" - "anchore/syft:{{ .Tag }}" - "anchore/syft:v{{ .Major }}" - "anchore/syft:v{{ .Major }}.{{ .Minor }}" - build_flag_templates: - "--build-arg=BUILD_DATE={{.Date}}" - "--build-arg=BUILD_VERSION={{.Version}}" - "--build-arg=VCS_REF={{.FullCommit}}" - "--build-arg=VCS_URL={{.GitURL}}" + use_buildx: true diff --git a/Makefile b/Makefile index b085a7b76..fa3c5879e 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ BIN = syft TEMPDIR = ./.tmp -RESULTSDIR = $(TEMPDIR)/results -COVER_REPORT = $(RESULTSDIR)/cover.report -COVER_TOTAL = $(RESULTSDIR)/cover.total +RESULTSDIR = test/results +COVER_REPORT = $(RESULTSDIR)/unit-coverage-details.txt +COVER_TOTAL = $(RESULTSDIR)/unit-coverage-summary.txt LINTCMD = $(TEMPDIR)/golangci-lint run --tests=false --config .golangci.yaml ACC_TEST_IMAGE = centos:8.2.2004 ACC_DIR = ./test/acceptance @@ -15,17 +15,23 @@ RESET := $(shell tput -T linux sgr0) TITLE := $(BOLD)$(PURPLE) SUCCESS := $(BOLD)$(GREEN) # the quality gate lower threshold for unit test total % coverage (by function statements) -COVERAGE_THRESHOLD := 68 +COVERAGE_THRESHOLD := 70 # CI cache busting values; change these if you want CI to not use previous stored cache -COMPARE_CACHE_BUSTER="f7e689d76a9" -INTEGRATION_CACHE_BUSTER="789bacdf" -BOOTSTRAP_CACHE="789bacdf" +INTEGRATION_CACHE_BUSTER="88738d2f" +CLI_CACHE_BUSTER="789bacdf" +BOOTSTRAP_CACHE="c7afb99ad" ## Build variables DISTDIR=./dist SNAPSHOTDIR=./snapshot GITTREESTATE=$(if $(shell git status --porcelain),dirty,clean) -SNAPSHOT_CMD=$(shell realpath $(shell pwd)/$(SNAPSHOTDIR)/syft_linux_amd64/syft) +OS := $(shell uname) + +ifeq ($(OS),Darwin) + SNAPSHOT_CMD=$(shell realpath $(shell pwd)/$(SNAPSHOTDIR)/$(BIN)-macos_darwin_amd64/$(BIN)) +else + SNAPSHOT_CMD=$(shell realpath $(shell pwd)/$(SNAPSHOTDIR)/$(BIN)_linux_amd64/$(BIN)) +endif ifeq "$(strip $(VERSION))" "" override VERSION = $(shell git describe --always --tags --dirty) @@ -57,6 +63,10 @@ ifndef SNAPSHOTDIR $(error SNAPSHOTDIR is not set) endif +ifndef REF_NAME + REF_NAME = $(VERSION) +endif + define title @printf '$(TITLE)$(1)$(RESET)\n' endef @@ -68,7 +78,7 @@ all: clean static-analysis test ## Run all linux-based checks (linting, license @printf '$(SUCCESS)All checks pass!$(RESET)\n' .PHONY: test -test: unit validate-cyclonedx-schema integration acceptance-linux ## Run all tests (currently unit, integration, and linux acceptance tests) +test: unit validate-cyclonedx-schema integration benchmark acceptance-linux cli ## Run all tests (currently unit, integration, linux acceptance, and cli tests) .PHONY: help help: @@ -78,19 +88,30 @@ help: ci-bootstrap: DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y bc jq libxml2-utils -.PHONY: bootstrap -bootstrap: ## Download and install all go dependencies (+ prep tooling in the ./tmp dir) - $(call title,Bootstrapping dependencies) - @pwd - # prep temp dirs - mkdir -p $(TEMPDIR) +.PHONY: +ci-bootstrap-mac: + github_changelog_generator --version || sudo gem install github_changelog_generator + +$(RESULTSDIR): mkdir -p $(RESULTSDIR) - # install go dependencies + +$(TEMPDIR): + mkdir -p $(TEMPDIR) + +.PHONY: bootstrap-tools +bootstrap-tools: $(TEMPDIR) + GO111MODULE=off GOBIN=$(shell realpath $(TEMPDIR)) go get -u golang.org/x/perf/cmd/benchstat + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TEMPDIR)/ v1.26.0 + curl -sSfL https://raw.githubusercontent.com/wagoodman/go-bouncer/master/bouncer.sh | sh -s -- -b $(TEMPDIR)/ v0.2.0 + curl -sfL https://install.goreleaser.com/github.com/goreleaser/goreleaser.sh | sh -s -- -b $(TEMPDIR)/ v0.160.0 + +.PHONY: bootstrap-go +bootstrap-go: go mod download - # install utilities - [ -f "$(TEMPDIR)/golangci" ] || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TEMPDIR)/ v1.26.0 - [ -f "$(TEMPDIR)/bouncer" ] || curl -sSfL https://raw.githubusercontent.com/wagoodman/go-bouncer/master/bouncer.sh | sh -s -- -b $(TEMPDIR)/ v0.2.0 - [ -f "$(TEMPDIR)/goreleaser" ] || curl -sfL https://install.goreleaser.com/github.com/goreleaser/goreleaser.sh | sh -s -- -b $(TEMPDIR)/ v0.140.0 + +.PHONY: bootstrap +bootstrap: $(RESULTSDIR) bootstrap-go bootstrap-tools ## Download and install all go dependencies (+ prep tooling in the ./tmp dir) + $(call title,Bootstrapping dependencies) .PHONY: static-analysis static-analysis: lint check-licenses @@ -124,42 +145,50 @@ validate-cyclonedx-schema: cd schema/cyclonedx && make .PHONY: unit -unit: fixtures ## Run unit tests (with coverage) +unit: $(RESULTSDIR) fixtures ## Run unit tests (with coverage) $(call title,Running unit tests) - go test -coverprofile $(COVER_REPORT) $(shell go list ./... | grep -v anchore/syft/test) + go test -coverprofile $(COVER_REPORT) $(shell go list ./... | grep -v anchore/syft/test) @go tool cover -func $(COVER_REPORT) | grep total | awk '{print substr($$3, 1, length($$3)-1)}' > $(COVER_TOTAL) @echo "Coverage: $$(cat $(COVER_TOTAL))" @if [ $$(echo "$$(cat $(COVER_TOTAL)) >= $(COVERAGE_THRESHOLD)" | bc -l) -ne 1 ]; then echo "$(RED)$(BOLD)Failed coverage quality gate (> $(COVERAGE_THRESHOLD)%)$(RESET)" && false; fi +.PHONY: benchmark +benchmark: $(RESULTSDIR) ## Run benchmark tests and compare against the baseline (if available) + $(call title,Running benchmark tests) + go test -p 1 -run=^Benchmark -bench=. -count=5 -benchmem ./... | tee $(RESULTSDIR)/benchmark-$(REF_NAME).txt + (test -s $(RESULTSDIR)/benchmark-main.txt && \ + $(TEMPDIR)/benchstat $(RESULTSDIR)/benchmark-main.txt $(RESULTSDIR)/benchmark-$(REF_NAME).txt || \ + $(TEMPDIR)/benchstat $(RESULTSDIR)/benchmark-$(REF_NAME).txt) \ + | tee $(RESULTSDIR)/benchstat.txt + +.PHONY: show-benchstat +show-benchstat: + @cat $(RESULTSDIR)/benchstat.txt + .PHONY: integration integration: ## Run integration tests $(call title,Running integration tests) go test -v ./test/integration - # note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted integration-fingerprint: find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/cache.fingerprint && echo "$(INTEGRATION_CACHE_BUSTER)" >> test/integration/test-fixtures/cache.fingerprint .PHONY: java-packages-fingerprint java-packages-fingerprint: - @cd syft/cataloger/java/test-fixtures/java-builds && \ + @cd syft/pkg/cataloger/java/test-fixtures/java-builds && \ make packages.fingerprint .PHONY: fixtures fixtures: $(call title,Generating test fixtures) - cd syft/cataloger/java/test-fixtures/java-builds && make + cd syft/pkg/cataloger/java/test-fixtures/java-builds && make .PHONY: generate-json-schema generate-json-schema: ## Generate a new json schema cd schema/json && go run generate.go -.PHONY: clear-test-cache -clear-test-cache: ## Delete all test cache (built docker image tars) - find . -type f -wholename "**/test-fixtures/cache/*.tar" -delete - .PHONY: build build: $(SNAPSHOTDIR) ## Build release snapshot binaries and packages @@ -175,7 +204,7 @@ $(SNAPSHOTDIR): ## Build snapshot release binaries and packages # note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately .PHONY: acceptance-mac -acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binaries and packages (Mac) +acceptance-mac: $(RESULTSDIR) $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binaries and packages (Mac) $(call title,Running acceptance test: Run on Mac) $(ACC_DIR)/mac.sh \ $(SNAPSHOTDIR) \ @@ -187,22 +216,8 @@ acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binarie .PHONY: acceptance-linux acceptance-linux: acceptance-test-deb-package-install acceptance-test-rpm-package-install ## Run acceptance tests on build snapshot binaries and packages (Linux) -# note: this is used by CI to determine if the inline-scan report cache should be busted for the inline-compare tests -.PHONY: compare-fingerprint -compare-fingerprint: - find test/inline-compare/* -type f -exec md5sum {} + | grep -v '\-reports' | grep -v 'fingerprint' | awk '{print $1}' | sort | md5sum | tee test/inline-compare/inline-compare.fingerprint && echo "$(COMPARE_CACHE_BUSTER)" >> test/inline-compare/inline-compare.fingerprint - -.PHONY: compare-snapshot -compare-snapshot: $(SNAPSHOTDIR) ## Compare the reports of a run of a snapshot build of syft against inline-scan - chmod 755 $(SNAPSHOT_CMD) - @cd test/inline-compare && SYFT_CMD=$(SNAPSHOT_CMD) make - -.PHONY: compare -compare: ## Compare the reports of a run of a main-branch build of syft against inline-scan - @cd test/inline-compare && make - .PHONY: acceptance-test-deb-package-install -acceptance-test-deb-package-install: $(SNAPSHOTDIR) +acceptance-test-deb-package-install: $(RESULTSDIR) $(SNAPSHOTDIR) $(call title,Running acceptance test: DEB install) $(ACC_DIR)/deb.sh \ $(SNAPSHOTDIR) \ @@ -211,7 +226,7 @@ acceptance-test-deb-package-install: $(SNAPSHOTDIR) $(RESULTSDIR) .PHONY: acceptance-test-rpm-package-install -acceptance-test-rpm-package-install: $(SNAPSHOTDIR) +acceptance-test-rpm-package-install: $(RESULTSDIR) $(SNAPSHOTDIR) $(call title,Running acceptance test: RPM install) $(ACC_DIR)/rpm.sh \ $(SNAPSHOTDIR) \ @@ -219,6 +234,17 @@ acceptance-test-rpm-package-install: $(SNAPSHOTDIR) $(ACC_TEST_IMAGE) \ $(RESULTSDIR) +# note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted +cli-fingerprint: + find test/cli/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/cli/test-fixtures/cache.fingerprint && echo "$(CLI_CACHE_BUSTER)" >> test/cli/test-fixtures/cache.fingerprint + +.PHONY: cli +cli: $(SNAPSHOTDIR) ## Run CLI tests + chmod 755 "$(SNAPSHOT_CMD)" + $(SNAPSHOT_CMD) version + SYFT_BINARY_LOCATION='$(SNAPSHOT_CMD)' \ + go test -count=1 -v ./test/cli + .PHONY: changlog-release changelog-release: @echo "Last tag: $(SECOND_TO_LAST_TAG)" @@ -283,7 +309,7 @@ release: clean-dist changelog-release ## Build and publish final binaries and pa .PHONY: clean -clean: clean-dist clean-snapshot ## Remove previous builds and result reports +clean: clean-dist clean-snapshot clean-test-image-cache ## Remove previous builds, result reports, and test cache rm -rf $(RESULTSDIR)/* .PHONY: clean-snapshot @@ -293,3 +319,26 @@ clean-snapshot: .PHONY: clean-dist clean-dist: rm -rf $(DISTDIR) $(TEMPDIR)/goreleaser.yaml + +clean-test-image-cache: clean-test-image-tar-cache clean-test-image-docker-cache + +.PHONY: clear-test-image-tar-cache +clean-test-image-tar-cache: ## Delete all test cache (built docker image tars) + find . -type f -wholename "**/test-fixtures/cache/stereoscope-fixture-*.tar" -delete + +.PHONY: clear-test-image-docker-cache +clean-test-image-docker-cache: ## Purge all test docker images + docker images --format '{{.ID}} {{.Repository}}' | grep stereoscope-fixture- | awk '{print $$1}' | uniq | xargs docker rmi --force + +.PHONY: show-test-image-cache +show-test-image-cache: ## Show all docker and image tar cache + $(call title,Docker daemon cache) + @docker images --format '{{.ID}} {{.Repository}}:{{.Tag}}' | grep stereoscope-fixture- | sort + + $(call title,Tar cache) + @find . -type f -wholename "**/test-fixtures/cache/stereoscope-fixture-*.tar" | sort + +.PHONY: show-test-snapshots +show-test-snapshots: ## Show all test snapshots + $(call title,Test snapshots) + @find . -type f -wholename "**/test-fixtures/snapshot/*" | sort \ No newline at end of file diff --git a/README.md b/README.md index efcfbe7eb..a16433c2d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # syft -[![Static Analysis + Unit + Integration](https://github.com/anchore/syft/workflows/Static%20Analysis%20+%20Unit%20+%20Integration/badge.svg)](https://github.com/anchore/syft/actions?query=workflow%3A%22Static+Analysis+%2B+Unit+%2B+Integration%22) -[![Acceptance](https://github.com/anchore/syft/workflows/Acceptance/badge.svg)](https://github.com/anchore/syft/actions?query=workflow%3AAcceptance) +[![Validations](https://github.com/anchore/syft/workflows/validations.yaml/badge.svg)](https://github.com/anchore/syft/workflows/validations.yaml) [![Go Report Card](https://goreportcard.com/badge/github.com/anchore/syft)](https://goreportcard.com/report/github.com/anchore/syft) [![GitHub release](https://img.shields.io/github/release/anchore/syft.svg)](https://github.com/anchore/syft/releases/latest) [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/anchore/syft/blob/main/LICENSE) @@ -25,25 +24,30 @@ To generate an SBOM for a Docker or OCI image: syft ``` +**Note**: This is equivalent to specifying the `packages` subcommand: +``` +syft packages +``` + The above output includes only software that is visible in the container (i.e., the squashed representation of the image). To include software from all image layers in the SBOM, regardless of its presence in the final image, provide `--scope all-layers`: ``` -syft --scope all-layers +syft packages --scope all-layers ``` Syft can generate a SBOM from a variety of sources: ``` # catalog a container image archive (from the result of `docker image save ...`, `podman save ...`, or `skopeo copy` commands) -syft path/to/image.tar +syft packages path/to/image.tar # catalog a directory -syft path/to/dir +syft packages path/to/dir ``` The output format for Syft is configurable as well: ``` -syft -o +syft packages -o ``` Where the `format`s available are: @@ -85,10 +89,6 @@ Configuration options (example values are the default): # same as -o ; SYFT_OUTPUT env var output: "table" -# the search space to look for packages (options: all-layers, squashed) -# same as -s ; SYFT_SCOPE env var -scope: "squashed" - # suppress all output (except for the SBOM report) # same as -q ; SYFT_QUIET env var quiet: false @@ -97,6 +97,32 @@ quiet: false # same as SYFT_CHECK_FOR_APP_UPDATE env var check-for-app-update: true +# cataloging packages is exposed through the packages and power-user subcommands +package: + cataloger: + # enable/disable cataloging of packages + # SYFT_PACKAGE_CATALOGER_ENABLED env var + enabled: true + + # the search space to look for packages (options: all-layers, squashed) + # same as -s ; SYFT_PACKAGE_CATALOGER_SCOPE env var + scope: "squashed" + +# cataloging file metadata is exposed through the power-user subcommand +file-metadata: + cataloger: + # enable/disable cataloging of file metadata + # SYFT_FILE_METADATA_CATALOGER_ENABLED env var + enabled: true + + # the search space to look for file metadata (options: all-layers, squashed) + # SYFT_FILE_METADATA_CATALOGER_SCOPE env var + scope: "squashed" + + # the file digest algorithms to use when cataloging files (options: "sha256", "md5", "sha1") + # SYFT_FILE_METADATA_DIGESTS env var + digests: ["sha256"] + log: # use structured logging # same as SYFT_LOG_STRUCTURED env var @@ -110,11 +136,8 @@ log: # same as SYFT_LOG_FILE env var file: "" +# uploading package SBOM is exposed through the packages subcommand anchore: - # (feature-preview) enable uploading of results to Anchore Enterprise automatically (supported on Enterprise 3.0+) - # same as SYFT_ANCHORE_UPLOAD_ENABLED env var - upload-enabled: false - # (feature-preview) the Anchore Enterprise Host or URL to upload results to (supported on Enterprise 3.0+) # same as -H ; SYFT_ANCHORE_HOST env var host: "" diff --git a/cmd/check_for_application_update.go b/cmd/check_for_application_update.go new file mode 100644 index 000000000..409d6beb0 --- /dev/null +++ b/cmd/check_for_application_update.go @@ -0,0 +1,30 @@ +package cmd + +import ( + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/bus" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/internal/version" + "github.com/anchore/syft/syft/event" + "github.com/wagoodman/go-partybus" +) + +func checkForApplicationUpdate() { + if appConfig.CheckForAppUpdate { + isAvailable, newVersion, err := version.IsUpdateAvailable() + if err != nil { + // this should never stop the application + log.Errorf(err.Error()) + } + if isAvailable { + log.Infof("new version of %s is available: %s (current version is %s)", internal.ApplicationName, newVersion, version.FromBuild().Version) + + bus.Publish(partybus.Event{ + Type: event.AppUpdateAvailable, + Value: newVersion, + }) + } else { + log.Debugf("no new %s update available", internal.ApplicationName) + } + } +} diff --git a/cmd/cmd.go b/cmd/cmd.go index e813174a9..ac95e2d0a 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -4,28 +4,27 @@ import ( "fmt" "os" + "github.com/spf13/cobra" + "github.com/anchore/stereoscope" "github.com/anchore/syft/internal/config" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/logger" "github.com/anchore/syft/syft" - "github.com/anchore/syft/syft/presenter" - "github.com/anchore/syft/syft/source" "github.com/gookit/color" - "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/wagoodman/go-partybus" ) -var appConfig *config.Application -var eventBus *partybus.Bus -var eventSubscription *partybus.Subscription -var cliOpts = config.CliOnlyOptions{} +var ( + appConfig *config.Application + eventBus *partybus.Bus + eventSubscription *partybus.Subscription +) func init() { - setGlobalCliOptions() - cobra.OnInitialize( + initCmdAliasBindings, initAppConfig, initLogging, logAppConfig, @@ -35,109 +34,41 @@ func init() { func Execute() { if err := rootCmd.Execute(); err != nil { - fmt.Fprintln(os.Stderr, err.Error()) + fmt.Fprintln(os.Stderr, color.Red.Sprint(err.Error())) os.Exit(1) } } -func setGlobalCliOptions() { - rootCmd.PersistentFlags().StringVarP(&cliOpts.ConfigPath, "config", "c", "", "application config file") - - // scan options - flag := "scope" - rootCmd.Flags().StringP( - "scope", "s", source.SquashedScope.String(), - fmt.Sprintf("selection of layers to catalog, options=%v", source.AllScopes)) - if err := viper.BindPFlag(flag, rootCmd.Flags().Lookup(flag)); err != nil { - fmt.Printf("unable to bind flag '%s': %+v", flag, err) - os.Exit(1) +// we must setup the config-cli bindings first before the application configuration is parsed. However, this cannot +// be done without determining what the primary command that the config options should be bound to since there are +// shared concerns (the root-packages alias). +func initCmdAliasBindings() { + activeCmd, _, err := rootCmd.Find(os.Args[1:]) + if err != nil { + panic(err) } - setGlobalFormatOptions() - setGlobalUploadOptions() -} - -func setGlobalFormatOptions() { - // output & formatting options - flag := "output" - rootCmd.Flags().StringP( - flag, "o", string(presenter.TablePresenter), - fmt.Sprintf("report output formatter, options=%v", presenter.Options), - ) - if err := viper.BindPFlag(flag, rootCmd.Flags().Lookup(flag)); err != nil { - fmt.Printf("unable to bind flag '%s': %+v", flag, err) - os.Exit(1) - } - - flag = "quiet" - rootCmd.Flags().BoolP( - flag, "q", false, - "suppress all logging output", - ) - if err := viper.BindPFlag(flag, rootCmd.Flags().Lookup(flag)); err != nil { - fmt.Printf("unable to bind flag '%s': %+v", flag, err) - os.Exit(1) - } - - rootCmd.Flags().CountVarP(&cliOpts.Verbosity, "verbose", "v", "increase verbosity (-v = info, -vv = debug)") -} - -func setGlobalUploadOptions() { - flag := "host" - rootCmd.Flags().StringP( - flag, "H", "", - "the hostname or URL of the Anchore Enterprise instance to upload to", - ) - if err := viper.BindPFlag("anchore.host", rootCmd.Flags().Lookup(flag)); err != nil { - fmt.Printf("unable to bind flag '%s': %+v", flag, err) - os.Exit(1) - } - - flag = "username" - rootCmd.Flags().StringP( - flag, "u", "", - "the username to authenticate against Anchore Enterprise", - ) - if err := viper.BindPFlag("anchore.username", rootCmd.Flags().Lookup(flag)); err != nil { - fmt.Printf("unable to bind flag '%s': %+v", flag, err) - os.Exit(1) - } - - flag = "password" - rootCmd.Flags().StringP( - flag, "p", "", - "the password to authenticate against Anchore Enterprise", - ) - if err := viper.BindPFlag("anchore.password", rootCmd.Flags().Lookup(flag)); err != nil { - fmt.Printf("unable to bind flag '%s': %+v", flag, err) - os.Exit(1) - } - - flag = "dockerfile" - rootCmd.Flags().StringP( - flag, "d", "", - "include dockerfile for upload to Anchore Enterprise", - ) - if err := viper.BindPFlag("anchore.dockerfile", rootCmd.Flags().Lookup(flag)); err != nil { - fmt.Printf("unable to bind flag '#{flag}': #{err}") - os.Exit(1) - } - - flag = "overwrite-existing-image" - rootCmd.Flags().Bool( - flag, false, - "overwrite an existing image during the upload to Anchore Enterprise", - ) - if err := viper.BindPFlag("anchore.overwrite-existing-image", rootCmd.Flags().Lookup(flag)); err != nil { - fmt.Printf("unable to bind flag '#{flag}': #{err}") - os.Exit(1) + if activeCmd == packagesCmd || activeCmd == rootCmd { + // note: we need to lazily bind config options since they are shared between both the root command + // and the packages command. Otherwise there will be global viper state that is in contention. + // See for more details: https://github.com/spf13/viper/issues/233 . Additionally, the bindings must occur BEFORE + // reading the application configuration, which implies that it must be an initializer (or rewrite the command + // initialization structure against typical patterns used with cobra, which is somewhat extreme for a + // temporary alias) + if err = bindPackagesConfigOptions(activeCmd.Flags()); err != nil { + panic(err) + } + } else { + // even though the root command or packages command is NOT being run, we still need default bindings + // such that application config parsing passes. + if err = bindPackagesConfigOptions(packagesCmd.Flags()); err != nil { + panic(err) + } } } func initAppConfig() { - cfgVehicle := viper.GetViper() - wasHostnameSet := rootCmd.Flags().Changed("host") - cfg, err := config.LoadApplicationConfig(cfgVehicle, cliOpts, wasHostnameSet) + cfg, err := config.LoadApplicationConfig(viper.GetViper(), persistentOpts) if err != nil { fmt.Printf("failed to load application config: \n\t%+v\n", err) os.Exit(1) @@ -163,7 +94,7 @@ func initLogging() { } func logAppConfig() { - log.Debugf("Application config:\n%+v", color.Magenta.Sprint(appConfig.String())) + log.Debugf("application config:\n%+v", color.Magenta.Sprint(appConfig.String())) } func initEventBus() { diff --git a/cmd/completion.go b/cmd/completion.go index 7ec259b58..d71c0b4eb 100644 --- a/cmd/completion.go +++ b/cmd/completion.go @@ -1,15 +1,22 @@ package cmd import ( + "context" "os" + "strings" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/client" "github.com/spf13/cobra" ) // completionCmd represents the completion command var completionCmd = &cobra.Command{ - Use: "completion [bash|zsh|fish]", - Short: "Generate a shell completion for Syft (listing local docker images)", + Hidden: true, + Use: "completion [bash|zsh|fish]", + Short: "Generate a shell completion for Syft (listing local docker images)", Long: `To load completions (docker image list): Bash: @@ -63,3 +70,45 @@ $ syft completion fish > ~/.config/fish/completions/syft.fish func init() { rootCmd.AddCommand(completionCmd) } + +func dockerImageValidArgsFunction(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + // Since we use ValidArgsFunction, Cobra will call this AFTER having parsed all flags and arguments provided + dockerImageRepoTags, err := listLocalDockerImages(toComplete) + if err != nil { + // Indicates that an error occurred and completions should be ignored + return []string{"completion failed"}, cobra.ShellCompDirectiveError + } + if len(dockerImageRepoTags) == 0 { + return []string{"no docker images found"}, cobra.ShellCompDirectiveError + } + // ShellCompDirectiveDefault indicates that the shell will perform its default behavior after completions have + // been provided (without implying other possible directives) + return dockerImageRepoTags, cobra.ShellCompDirectiveDefault +} + +func listLocalDockerImages(prefix string) ([]string, error) { + var repoTags = make([]string, 0) + ctx := context.Background() + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return repoTags, err + } + + // Only want to return tagged images + imageListArgs := filters.NewArgs() + imageListArgs.Add("dangling", "false") + images, err := cli.ImageList(ctx, types.ImageListOptions{All: false, Filters: imageListArgs}) + if err != nil { + return repoTags, err + } + + for _, image := range images { + // image may have multiple tags + for _, tag := range image.RepoTags { + if strings.HasPrefix(tag, prefix) { + repoTags = append(repoTags, tag) + } + } + } + return repoTags, nil +} diff --git a/cmd/packages.go b/cmd/packages.go new file mode 100644 index 000000000..e19bee44e --- /dev/null +++ b/cmd/packages.go @@ -0,0 +1,270 @@ +package cmd + +import ( + "context" + "fmt" + "io/ioutil" + "os" + + "github.com/spf13/viper" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/anchore" + "github.com/anchore/syft/internal/bus" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/internal/presenter/packages" + "github.com/anchore/syft/internal/ui" + "github.com/anchore/syft/syft" + "github.com/anchore/syft/syft/distro" + "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" + "github.com/pkg/profile" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/wagoodman/go-partybus" +) + +const ( + packagesExample = ` {{.appName}} {{.command}} alpine:latest a summary of discovered packages + {{.appName}} {{.command}} alpine:latest -o json show all possible cataloging details + {{.appName}} {{.command}} alpine:latest -o cyclonedx show a CycloneDX SBOM + {{.appName}} {{.command}} alpine:latest -vv show verbose debug information + + Supports the following image sources: + {{.appName}} {{.command}} yourrepo/yourimage:tag defaults to using images from a Docker daemon + {{.appName}} {{.command}} path/to/a/file/or/dir a Docker tar, OCI tar, OCI directory, or generic filesystem directory + + You can also explicitly specify the scheme to use: + {{.appName}} {{.command}} docker:yourrepo/yourimage:tag explicitly use the Docker daemon + {{.appName}} {{.command}} docker-archive:path/to/yourimage.tar use a tarball from disk for archives created from "docker save" + {{.appName}} {{.command}} oci-archive:path/to/yourimage.tar use a tarball from disk for OCI archives (from Skopeo or otherwise) + {{.appName}} {{.command}} oci-dir:path/to/yourimage read directly from a path on disk for OCI layout directories (from Skopeo or otherwise) + {{.appName}} {{.command}} dir:path/to/yourproject read directly from a path on disk (any directory) +` +) + +var ( + packagesPresenterOpt packages.PresenterOption + packagesArgs = cobra.MinimumNArgs(1) + packagesCmd = &cobra.Command{ + Use: "packages [SOURCE]", + Short: "Generate a package SBOM", + Long: "Generate a packaged-based Software Bill Of Materials (SBOM) from container images and filesystems", + Example: internal.Tprintf(packagesExample, map[string]interface{}{ + "appName": internal.ApplicationName, + "command": "packages", + }), + Args: packagesArgs, + SilenceUsage: true, + SilenceErrors: true, + PreRunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + err := cmd.Help() + if err != nil { + return err + } + // silently exit + return fmt.Errorf("") + } + + // set the presenter + presenterOption := packages.ParsePresenterOption(appConfig.Output) + if presenterOption == packages.UnknownPresenterOption { + return fmt.Errorf("bad --output value '%s'", appConfig.Output) + } + packagesPresenterOpt = presenterOption + + if appConfig.Dev.ProfileCPU && appConfig.Dev.ProfileMem { + return fmt.Errorf("cannot profile CPU and memory simultaneously") + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + if appConfig.Dev.ProfileCPU { + defer profile.Start(profile.CPUProfile).Stop() + } else if appConfig.Dev.ProfileMem { + defer profile.Start(profile.MemProfile).Stop() + } + + return packagesExec(cmd, args) + }, + ValidArgsFunction: dockerImageValidArgsFunction, + } +) + +func init() { + setPackageFlags(packagesCmd.Flags()) + + rootCmd.AddCommand(packagesCmd) +} + +func setPackageFlags(flags *pflag.FlagSet) { + ///////// Formatting & Input options ////////////////////////////////////////////// + + flags.StringP( + "scope", "s", source.SquashedScope.String(), + fmt.Sprintf("selection of layers to catalog, options=%v", source.AllScopes)) + + flags.StringP( + "output", "o", string(packages.TablePresenterOption), + fmt.Sprintf("report output formatter, options=%v", packages.AllPresenters), + ) + + ///////// Upload options ////////////////////////////////////////////////////////// + flags.StringP( + "host", "H", "", + "the hostname or URL of the Anchore Enterprise instance to upload to", + ) + + flags.StringP( + "username", "u", "", + "the username to authenticate against Anchore Enterprise", + ) + + flags.StringP( + "password", "p", "", + "the password to authenticate against Anchore Enterprise", + ) + + flags.StringP( + "dockerfile", "d", "", + "include dockerfile for upload to Anchore Enterprise", + ) + + flags.Bool( + "overwrite-existing-image", false, + "overwrite an existing image during the upload to Anchore Enterprise", + ) +} + +func bindPackagesConfigOptions(flags *pflag.FlagSet) error { + ///////// Formatting & Input options ////////////////////////////////////////////// + + if err := viper.BindPFlag("package.cataloger.scope", flags.Lookup("scope")); err != nil { + return err + } + + if err := viper.BindPFlag("output", flags.Lookup("output")); err != nil { + return err + } + + ///////// Upload options ////////////////////////////////////////////////////////// + + if err := viper.BindPFlag("anchore.host", flags.Lookup("host")); err != nil { + return err + } + + if err := viper.BindPFlag("anchore.username", flags.Lookup("username")); err != nil { + return err + } + + if err := viper.BindPFlag("anchore.password", flags.Lookup("password")); err != nil { + return err + } + + if err := viper.BindPFlag("anchore.dockerfile", flags.Lookup("dockerfile")); err != nil { + return err + } + + if err := viper.BindPFlag("anchore.overwrite-existing-image", flags.Lookup("overwrite-existing-image")); err != nil { + return err + } + + return nil +} + +func packagesExec(_ *cobra.Command, args []string) error { + errs := packagesExecWorker(args[0]) + ux := ui.Select(appConfig.CliOptions.Verbosity > 0, appConfig.Quiet) + return ux(errs, eventSubscription) +} + +func packagesExecWorker(userInput string) <-chan error { + errs := make(chan error) + go func() { + defer close(errs) + + checkForApplicationUpdate() + + src, cleanup, err := source.New(userInput) + if err != nil { + errs <- fmt.Errorf("failed to determine image source: %+v", err) + return + } + defer cleanup() + + catalog, d, err := syft.CatalogPackages(src, appConfig.Package.Cataloger.ScopeOpt) + if err != nil { + errs <- fmt.Errorf("failed to catalog input: %+v", err) + return + } + + if appConfig.Anchore.Host != "" { + if err := runPackageSbomUpload(src, src.Metadata, catalog, d, appConfig.Package.Cataloger.ScopeOpt); err != nil { + errs <- err + return + } + } + + bus.Publish(partybus.Event{ + Type: event.PresenterReady, + Value: packages.Presenter(packagesPresenterOpt, packages.PresenterConfig{ + SourceMetadata: src.Metadata, + Catalog: catalog, + Distro: d, + Scope: appConfig.Package.Cataloger.ScopeOpt, + }), + }) + }() + return errs +} + +func runPackageSbomUpload(src source.Source, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, scope source.Scope) error { + log.Infof("uploading results to %s", appConfig.Anchore.Host) + + if src.Metadata.Scheme != source.ImageScheme { + return fmt.Errorf("unable to upload results: only images are supported") + } + + var dockerfileContents []byte + if appConfig.Anchore.Dockerfile != "" { + if _, err := os.Stat(appConfig.Anchore.Dockerfile); os.IsNotExist(err) { + return fmt.Errorf("unable dockerfile=%q does not exist: %w", appConfig.Anchore.Dockerfile, err) + } + + fh, err := os.Open(appConfig.Anchore.Dockerfile) + if err != nil { + return fmt.Errorf("unable to open dockerfile=%q: %w", appConfig.Anchore.Dockerfile, err) + } + + dockerfileContents, err = ioutil.ReadAll(fh) + if err != nil { + return fmt.Errorf("unable to read dockerfile=%q: %w", appConfig.Anchore.Dockerfile, err) + } + } + + c, err := anchore.NewClient(anchore.Configuration{ + BaseURL: appConfig.Anchore.Host, + Username: appConfig.Anchore.Username, + Password: appConfig.Anchore.Password, + }) + if err != nil { + return fmt.Errorf("failed to create anchore client: %+v", err) + } + + importCfg := anchore.ImportConfig{ + ImageMetadata: src.Image.Metadata, + SourceMetadata: s, + Catalog: catalog, + Distro: d, + Dockerfile: dockerfileContents, + OverwriteExistingUpload: appConfig.Anchore.OverwriteExistingImage, + Scope: scope, + } + + if err := c.Import(context.Background(), importCfg); err != nil { + return fmt.Errorf("failed to upload results to host=%s: %+v", appConfig.Anchore.Host, err) + } + return nil +} diff --git a/cmd/power_user.go b/cmd/power_user.go new file mode 100644 index 000000000..962470eba --- /dev/null +++ b/cmd/power_user.go @@ -0,0 +1,113 @@ +package cmd + +import ( + "fmt" + + "github.com/anchore/syft/internal" + + "github.com/anchore/syft/internal/bus" + "github.com/anchore/syft/internal/presenter/poweruser" + "github.com/anchore/syft/internal/ui" + "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/source" + "github.com/pkg/profile" + "github.com/spf13/cobra" + "github.com/wagoodman/go-partybus" +) + +const powerUserExample = ` {{.appName}} {{.command}} + + Only image sources are supported (e.g. docker: , docker-archive: , oci: , etc.), the directory source (dir:) is not supported. + + All behavior is controlled via application configuration and environment variables (see https://github.com/anchore/syft#configuration) +` + +var powerUserOpts = struct { + configPath string +}{} + +var powerUserCmd = &cobra.Command{ + Use: "power-user [IMAGE]", + Short: "Run bulk operations on container images", + Example: internal.Tprintf(powerUserExample, map[string]interface{}{ + "appName": internal.ApplicationName, + "command": "power-user", + }), + Args: cobra.ExactArgs(1), + Hidden: true, + SilenceUsage: true, + SilenceErrors: true, + PreRunE: func(cmd *cobra.Command, args []string) error { + if appConfig.Dev.ProfileCPU && appConfig.Dev.ProfileMem { + return fmt.Errorf("cannot profile CPU and memory simultaneously") + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + if appConfig.Dev.ProfileCPU { + defer profile.Start(profile.CPUProfile).Stop() + } else if appConfig.Dev.ProfileMem { + defer profile.Start(profile.MemProfile).Stop() + } + + return powerUserExec(cmd, args) + }, + ValidArgsFunction: dockerImageValidArgsFunction, +} + +func init() { + powerUserCmd.Flags().StringVarP(&powerUserOpts.configPath, "config", "c", "", "config file path with all power-user options") + + rootCmd.AddCommand(powerUserCmd) +} + +func powerUserExec(_ *cobra.Command, args []string) error { + errs := powerUserExecWorker(args[0]) + ux := ui.Select(appConfig.CliOptions.Verbosity > 0, appConfig.Quiet) + return ux(errs, eventSubscription) +} + +func powerUserExecWorker(userInput string) <-chan error { + errs := make(chan error) + go func() { + defer close(errs) + + tasks, err := powerUserTasks() + if err != nil { + errs <- err + return + } + + checkForApplicationUpdate() + + src, cleanup, err := source.New(userInput) + if err != nil { + errs <- err + return + } + defer cleanup() + + if src.Metadata.Scheme != source.ImageScheme { + errs <- fmt.Errorf("the power-user subcommand only allows for 'image' schemes, given %q", src.Metadata.Scheme) + return + } + + analysisResults := poweruser.JSONDocumentConfig{ + SourceMetadata: src.Metadata, + ApplicationConfig: *appConfig, + } + + for _, task := range tasks { + if err = task(&analysisResults, src); err != nil { + errs <- err + return + } + } + + bus.Publish(partybus.Event{ + Type: event.PresenterReady, + Value: poweruser.NewJSONPresenter(analysisResults), + }) + }() + return errs +} diff --git a/cmd/power_user_tasks.go b/cmd/power_user_tasks.go new file mode 100644 index 000000000..10070e548 --- /dev/null +++ b/cmd/power_user_tasks.go @@ -0,0 +1,103 @@ +package cmd + +import ( + "github.com/anchore/syft/internal/presenter/poweruser" + "github.com/anchore/syft/syft" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/source" +) + +type powerUserTask func(*poweruser.JSONDocumentConfig, source.Source) error + +func powerUserTasks() ([]powerUserTask, error) { + var tasks []powerUserTask + + generators := []func() (powerUserTask, error){ + catalogPackagesTask, + catalogFileMetadataTask, + catalogFileDigestTask, + } + + for _, generator := range generators { + task, err := generator() + if err != nil { + return nil, err + } + if task != nil { + tasks = append(tasks, task) + } + } + + return tasks, nil +} + +func catalogPackagesTask() (powerUserTask, error) { + if !appConfig.Package.Cataloger.Enabled { + return nil, nil + } + + task := func(results *poweruser.JSONDocumentConfig, src source.Source) error { + packageCatalog, theDistro, err := syft.CatalogPackages(src, appConfig.Package.Cataloger.ScopeOpt) + if err != nil { + return err + } + + results.PackageCatalog = packageCatalog + results.Distro = theDistro + + return nil + } + + return task, nil +} + +func catalogFileMetadataTask() (powerUserTask, error) { + if !appConfig.FileMetadata.Cataloger.Enabled { + return nil, nil + } + + metadataCataloger := file.NewMetadataCataloger() + + task := func(results *poweruser.JSONDocumentConfig, src source.Source) error { + resolver, err := src.FileResolver(appConfig.FileMetadata.Cataloger.ScopeOpt) + if err != nil { + return err + } + + result, err := metadataCataloger.Catalog(resolver) + if err != nil { + return err + } + results.FileMetadata = result + return nil + } + + return task, nil +} + +func catalogFileDigestTask() (powerUserTask, error) { + if !appConfig.FileMetadata.Cataloger.Enabled { + return nil, nil + } + + digestsCataloger, err := file.NewDigestsCataloger(appConfig.FileMetadata.Digests) + if err != nil { + return nil, err + } + + task := func(results *poweruser.JSONDocumentConfig, src source.Source) error { + resolver, err := src.FileResolver(appConfig.FileMetadata.Cataloger.ScopeOpt) + if err != nil { + return err + } + + result, err := digestsCataloger.Catalog(resolver) + if err != nil { + return err + } + results.FileDigests = result + return nil + } + + return task, nil +} diff --git a/cmd/root.go b/cmd/root.go index d89f8e739..cb131972e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,217 +1,45 @@ package cmd import ( - "context" "fmt" - "io/ioutil" "os" - "strings" - "github.com/pkg/profile" - - "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/anchore" - "github.com/anchore/syft/internal/bus" - "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/internal/ui" - "github.com/anchore/syft/internal/version" - "github.com/anchore/syft/syft" - "github.com/anchore/syft/syft/distro" - "github.com/anchore/syft/syft/event" - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/presenter" - "github.com/anchore/syft/syft/source" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/client" + "github.com/anchore/syft/internal/config" "github.com/spf13/cobra" - "github.com/wagoodman/go-partybus" + "github.com/spf13/viper" ) +var persistentOpts = config.CliOnlyOptions{} + +// rootCmd is currently an alias for the packages command var rootCmd = &cobra.Command{ - Use: fmt.Sprintf("%s [SOURCE]", internal.ApplicationName), - Short: "A tool for generating a Software Bill Of Materials (PackageSBOM) from container images and filesystems", - Long: internal.Tprintf(` -Supports the following image sources: - {{.appName}} yourrepo/yourimage:tag defaults to using images from a Docker daemon - {{.appName}} path/to/yourproject a Docker tar, OCI tar, OCI directory, or generic filesystem directory - -You can also explicitly specify the scheme to use: - {{.appName}} docker:yourrepo/yourimage:tag explicitly use the Docker daemon - {{.appName}} docker-archive:path/to/yourimage.tar use a tarball from disk for archives created from "docker save" - {{.appName}} oci-archive:path/to/yourimage.tar use a tarball from disk for OCI archives (from Podman or otherwise) - {{.appName}} oci-dir:path/to/yourimage read directly from a path on disk for OCI layout directories (from Skopeo or otherwise) - {{.appName}} dir:path/to/yourproject read directly from a path on disk (any directory) -`, map[string]interface{}{ - "appName": internal.ApplicationName, - }), - Args: cobra.MaximumNArgs(1), - Run: func(cmd *cobra.Command, args []string) { - if len(args) == 0 { - err := cmd.Help() - if err != nil { - log.Errorf(err.Error()) - os.Exit(1) - } - os.Exit(1) - } - - if appConfig.Dev.ProfileCPU && appConfig.Dev.ProfileMem { - log.Errorf("cannot profile CPU and memory simultaneously") - os.Exit(1) - } - - if appConfig.Dev.ProfileCPU { - defer profile.Start(profile.CPUProfile).Stop() - } else if appConfig.Dev.ProfileMem { - defer profile.Start(profile.MemProfile).Stop() - } - - err := doRunCmd(cmd, args) - - if err != nil { - log.Errorf(err.Error()) - os.Exit(1) - } - }, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - // Since we use ValidArgsFunction, Cobra will call this AFTER having parsed all flags and arguments provided - dockerImageRepoTags, err := ListLocalDockerImages(toComplete) - if err != nil { - // Indicates that an error occurred and completions should be ignored - return []string{"completion failed"}, cobra.ShellCompDirectiveError - } - if len(dockerImageRepoTags) == 0 { - return []string{"no docker images found"}, cobra.ShellCompDirectiveError - } - // ShellCompDirectiveDefault indicates that the shell will perform its default behavior after completions have - // been provided (without implying other possible directives) - return dockerImageRepoTags, cobra.ShellCompDirectiveDefault - }, + Short: packagesCmd.Short, + Long: packagesCmd.Long, + Args: packagesCmd.Args, + Example: packagesCmd.Example, + SilenceUsage: true, + SilenceErrors: true, + PreRunE: packagesCmd.PreRunE, + RunE: packagesCmd.RunE, + ValidArgsFunction: packagesCmd.ValidArgsFunction, } -func startWorker(userInput string) <-chan error { - errs := make(chan error) - go func() { - defer close(errs) +func init() { + // set universal flags + rootCmd.PersistentFlags().StringVarP(&persistentOpts.ConfigPath, "config", "c", "", "application config file") - if appConfig.CheckForAppUpdate { - isAvailable, newVersion, err := version.IsUpdateAvailable() - if err != nil { - log.Errorf(err.Error()) - } - if isAvailable { - log.Infof("new version of %s is available: %s", internal.ApplicationName, newVersion) + flag := "quiet" + rootCmd.PersistentFlags().BoolP( + flag, "q", false, + "suppress all logging output", + ) + if err := viper.BindPFlag(flag, rootCmd.PersistentFlags().Lookup(flag)); err != nil { + fmt.Printf("unable to bind flag '%s': %+v", flag, err) + os.Exit(1) + } - bus.Publish(partybus.Event{ - Type: event.AppUpdateAvailable, - Value: newVersion, - }) - } else { - log.Debugf("no new %s update available", internal.ApplicationName) - } - } + rootCmd.PersistentFlags().CountVarP(&persistentOpts.Verbosity, "verbose", "v", "increase verbosity (-v = info, -vv = debug)") - src, catalog, distro, err := syft.Catalog(userInput, appConfig.ScopeOpt) - if err != nil { - errs <- fmt.Errorf("failed to catalog input: %+v", err) - return - } - - if appConfig.Anchore.UploadEnabled { - if err := doImport(src, src.Metadata, catalog, distro); err != nil { - errs <- err - return - } - } - - bus.Publish(partybus.Event{ - Type: event.CatalogerFinished, - Value: presenter.GetPresenter(appConfig.PresenterOpt, src.Metadata, catalog, distro), - }) - }() - return errs -} - -func doImport(src source.Source, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro) error { - // TODO: ETUI element for this - log.Infof("uploading results to %s", appConfig.Anchore.Host) - - if src.Metadata.Scheme != source.ImageScheme { - return fmt.Errorf("unable to upload results: only images are supported") - } - - var dockerfileContents []byte - if appConfig.Anchore.Dockerfile != "" { - if _, err := os.Stat(appConfig.Anchore.Dockerfile); os.IsNotExist(err) { - return fmt.Errorf("unable to read dockerfile=%q: %w", appConfig.Anchore.Dockerfile, err) - } - - fh, err := os.Open(appConfig.Anchore.Dockerfile) - if err != nil { - return fmt.Errorf("unable to open dockerfile=%q: %w", appConfig.Anchore.Dockerfile, err) - } - - dockerfileContents, err = ioutil.ReadAll(fh) - if err != nil { - return fmt.Errorf("unable to read dockerfile=%q: %w", appConfig.Anchore.Dockerfile, err) - } - } - - c, err := anchore.NewClient(anchore.Configuration{ - BaseURL: appConfig.Anchore.Host, - Username: appConfig.Anchore.Username, - Password: appConfig.Anchore.Password, - }) - if err != nil { - return fmt.Errorf("unable to upload results: %w", err) - } - - importCfg := anchore.ImportConfig{ - ImageMetadata: src.Image.Metadata, - SourceMetadata: s, - Catalog: catalog, - Distro: d, - Dockerfile: dockerfileContents, - OverwriteExistingUpload: appConfig.Anchore.OverwriteExistingImage, - } - - if err := c.Import(context.Background(), importCfg); err != nil { - return fmt.Errorf("failed to upload results to host=%s: %+v", appConfig.Anchore.Host, err) - } - return nil -} - -func doRunCmd(_ *cobra.Command, args []string) error { - userInput := args[0] - errs := startWorker(userInput) - ux := ui.Select(appConfig.CliOptions.Verbosity > 0, appConfig.Quiet) - return ux(errs, eventSubscription) -} - -func ListLocalDockerImages(prefix string) ([]string, error) { - var repoTags = make([]string, 0) - ctx := context.Background() - cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - if err != nil { - return repoTags, err - } - - // Only want to return tagged images - imageListArgs := filters.NewArgs() - imageListArgs.Add("dangling", "false") - images, err := cli.ImageList(ctx, types.ImageListOptions{All: false, Filters: imageListArgs}) - if err != nil { - return repoTags, err - } - - for _, image := range images { - // image may have multiple tags - for _, tag := range image.RepoTags { - if strings.HasPrefix(tag, prefix) { - repoTags = append(repoTags, tag) - } - } - } - return repoTags, nil + // set common options that are not universal (package subcommand-alias specific) + setPackageFlags(rootCmd.Flags()) } diff --git a/cmd/version.go b/cmd/version.go index 8ceddca02..757132ec5 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -8,7 +8,6 @@ import ( "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/version" - "github.com/anchore/syft/syft/presenter" "github.com/spf13/cobra" ) @@ -21,7 +20,7 @@ var versionCmd = &cobra.Command{ } func init() { - versionCmd.Flags().StringVarP(&outputFormat, "output", "o", string(presenter.TextPresenter), "format to show version information (available=[text, json])") + versionCmd.Flags().StringVarP(&outputFormat, "output", "o", "text", "format to show version information (available=[text, json])") rootCmd.AddCommand(versionCmd) } diff --git a/go.mod b/go.mod index 038fe513e..5a9ded037 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,14 @@ module github.com/anchore/syft go 1.14 require ( + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/adrg/xdg v0.2.1 - github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 + github.com/alecthomas/jsonschema v0.0.0-20210301060011-54c507b6f074 github.com/anchore/client-go v0.0.0-20210222170800-9c70f9b80bcf github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b - github.com/anchore/stereoscope v0.0.0-20210201165248-e94c52b4052d + github.com/anchore/stereoscope v0.0.0-20210323182342-47b72675ff65 github.com/antihax/optional v1.0.0 github.com/bmatcuk/doublestar/v2 v2.0.4 github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible @@ -31,7 +32,9 @@ require ( github.com/sirupsen/logrus v1.6.0 github.com/spf13/afero v1.2.2 github.com/spf13/cobra v1.0.1-0.20200909172742-8a63648dd905 + github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.0 + github.com/stretchr/testify v1.6.0 github.com/wagoodman/go-partybus v0.0.0-20200526224238-eb215533f07d github.com/wagoodman/go-progress v0.0.0-20200731105512-1020f39e6240 github.com/wagoodman/jotframe v0.0.0-20200730190914-3517092dd163 diff --git a/go.sum b/go.sum index 625136e08..3ec96d0c7 100644 --- a/go.sum +++ b/go.sum @@ -96,10 +96,12 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/adrg/xdg v0.2.1 h1:VSVdnH7cQ7V+B33qSJHTCRlNgra1607Q8PzEmnvb2Ic= github.com/adrg/xdg v0.2.1/go.mod h1:ZuOshBmzV4Ta+s23hdfFZnBsdzmoR3US0d7ErpqSbTQ= -github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 h1:T3+cD5fYvuH36h7EZq+TDpm+d8a6FSD4pQsbmuGGQ8o= -github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= +github.com/alecthomas/jsonschema v0.0.0-20210301060011-54c507b6f074 h1:Lw9q+WyJLFOR+AULchS5/2GKfM+6gOh4szzizdfH3MU= +github.com/alecthomas/jsonschema v0.0.0-20210301060011-54c507b6f074/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -113,8 +115,8 @@ github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0v github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E= -github.com/anchore/stereoscope v0.0.0-20210201165248-e94c52b4052d h1:2hv5NOZ0fD8tPk1UdGiW9PHxmjBmBLL+sFlhLXjjKgo= -github.com/anchore/stereoscope v0.0.0-20210201165248-e94c52b4052d/go.mod h1:lhSEYyGLXTXMIFHAz7Ls/MNQ5EjYd5ziLxovKZp1xOs= +github.com/anchore/stereoscope v0.0.0-20210323182342-47b72675ff65 h1:r3tiir6UCgj/YeTqy4s2bfhZ9SuJYNlXx1Z9e/eLrbI= +github.com/anchore/stereoscope v0.0.0-20210323182342-47b72675ff65/go.mod h1:G7tFR0iI9r6AvibmXKA9v010pRS1IIJgd0t6fOMDxCw= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -737,7 +739,6 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g= github.com/vdemeester/k8s-pkg-credentialprovider v1.17.4/go.mod h1:inCTmtUdr5KJbreVojo06krnTgaeAz/Z7lynpPk/Q2c= github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/wagoodman/go-partybus v0.0.0-20200526224238-eb215533f07d h1:KOxOL6qpmqwoPloNwi+CEgc1ayjHNOFNrvoOmeDOjDg= diff --git a/internal/anchore/import.go b/internal/anchore/import.go index f6f62b861..17a679879 100644 --- a/internal/anchore/import.go +++ b/internal/anchore/import.go @@ -26,6 +26,7 @@ type ImportConfig struct { Distro *distro.Distro Dockerfile []byte OverwriteExistingUpload bool + Scope source.Scope } func importProgress(source string) (*progress.Stage, *progress.Manual) { @@ -71,7 +72,7 @@ func (c *Client) Import(ctx context.Context, cfg ImportConfig) error { prog.N++ sessionID := startOperation.Uuid - packageDigest, err := importPackageSBOM(authedCtx, c.client.ImportsApi, sessionID, cfg.SourceMetadata, cfg.Catalog, cfg.Distro, stage) + packageDigest, err := importPackageSBOM(authedCtx, c.client.ImportsApi, sessionID, cfg.SourceMetadata, cfg.Catalog, cfg.Distro, cfg.Scope, stage) if err != nil { return fmt.Errorf("failed to import Package SBOM: %w", err) } diff --git a/internal/anchore/import_package_sbom.go b/internal/anchore/import_package_sbom.go index f70a27e3d..02b832e7e 100644 --- a/internal/anchore/import_package_sbom.go +++ b/internal/anchore/import_package_sbom.go @@ -8,9 +8,9 @@ import ( "fmt" "net/http" - "github.com/wagoodman/go-progress" + "github.com/anchore/syft/internal/presenter/packages" - jsonPresenter "github.com/anchore/syft/syft/presenter/json" + "github.com/wagoodman/go-progress" "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/source" @@ -24,9 +24,9 @@ type packageSBOMImportAPI interface { ImportImagePackages(context.Context, string, external.ImagePackageManifest) (external.ImageImportContentResponse, *http.Response, error) } -func packageSbomModel(s source.Metadata, catalog *pkg.Catalog, d *distro.Distro) (*external.ImagePackageManifest, error) { +func packageSbomModel(s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, scope source.Scope) (*external.ImagePackageManifest, error) { var buf bytes.Buffer - pres := jsonPresenter.NewPresenter(catalog, s, d) + pres := packages.NewJSONPresenter(catalog, s, d, scope) err := pres.Present(&buf) if err != nil { return nil, fmt.Errorf("unable to serialize results: %w", err) @@ -41,11 +41,11 @@ func packageSbomModel(s source.Metadata, catalog *pkg.Catalog, d *distro.Distro) return &model, nil } -func importPackageSBOM(ctx context.Context, api packageSBOMImportAPI, sessionID string, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, stage *progress.Stage) (string, error) { +func importPackageSBOM(ctx context.Context, api packageSBOMImportAPI, sessionID string, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, scope source.Scope, stage *progress.Stage) (string, error) { log.Debug("importing package SBOM") stage.Current = "package SBOM" - model, err := packageSbomModel(s, catalog, d) + model, err := packageSbomModel(s, catalog, d, scope) if err != nil { return "", fmt.Errorf("unable to create PackageSBOM model: %w", err) } diff --git a/internal/anchore/import_package_sbom_test.go b/internal/anchore/import_package_sbom_test.go index becb74bec..cf97899ea 100644 --- a/internal/anchore/import_package_sbom_test.go +++ b/internal/anchore/import_package_sbom_test.go @@ -9,9 +9,9 @@ import ( "strings" "testing" - "github.com/wagoodman/go-progress" + "github.com/anchore/syft/internal/presenter/packages" - jsonPresenter "github.com/anchore/syft/syft/presenter/json" + "github.com/wagoodman/go-progress" "github.com/anchore/syft/syft/distro" @@ -38,7 +38,6 @@ func TestPackageSbomToModel(t *testing.T) { Scheme: source.ImageScheme, ImageMetadata: source.ImageMetadata{ UserInput: "user-in", - Scope: "scope!", Layers: []source.LayerMetadata{ { MediaType: "layer-metadata-type!", @@ -76,7 +75,7 @@ func TestPackageSbomToModel(t *testing.T) { c := pkg.NewCatalog(p) - model, err := packageSbomModel(m, c, &d) + model, err := packageSbomModel(m, c, &d, source.AllLayersScope) if err != nil { t.Fatalf("unable to generate model from source material: %+v", err) } @@ -89,19 +88,19 @@ func TestPackageSbomToModel(t *testing.T) { } var buf bytes.Buffer - pres := jsonPresenter.NewPresenter(c, m, &d) + pres := packages.NewJSONPresenter(c, m, &d, source.AllLayersScope) if err := pres.Present(&buf); err != nil { t.Fatalf("unable to get expected json: %+v", err) } // unmarshal expected result - var expectedDoc jsonPresenter.Document + var expectedDoc packages.JSONDocument if err := json.Unmarshal(buf.Bytes(), &expectedDoc); err != nil { t.Fatalf("unable to parse json doc: %+v", err) } // unmarshal actual result - var actualDoc jsonPresenter.Document + var actualDoc packages.JSONDocument if err := json.Unmarshal(modelJSON, &actualDoc); err != nil { t.Fatalf("unable to parse json doc: %+v", err) } @@ -178,10 +177,9 @@ func TestPackageSbomImport(t *testing.T) { }) m := source.Metadata{ - Scheme: "a-schema", + Scheme: source.ImageScheme, ImageMetadata: source.ImageMetadata{ UserInput: "user-in", - Scope: "scope!", Layers: nil, Size: 10, ManifestDigest: "sha256:digest!", @@ -192,7 +190,7 @@ func TestPackageSbomImport(t *testing.T) { d, _ := distro.NewDistro(distro.CentOS, "8.0", "") - theModel, err := packageSbomModel(m, catalog, &d) + theModel, err := packageSbomModel(m, catalog, &d, source.AllLayersScope) if err != nil { t.Fatalf("could not get sbom model: %+v", err) } @@ -231,7 +229,7 @@ func TestPackageSbomImport(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - digest, err := importPackageSBOM(context.TODO(), test.api, sessionID, m, catalog, &d, &progress.Stage{}) + digest, err := importPackageSBOM(context.TODO(), test.api, sessionID, m, catalog, &d, source.AllLayersScope, &progress.Stage{}) // validate error handling if err != nil && !test.expectsError { diff --git a/internal/config/anchore.go b/internal/config/anchore.go new file mode 100644 index 000000000..7b363baa0 --- /dev/null +++ b/internal/config/anchore.go @@ -0,0 +1,13 @@ +package config + +type anchore struct { + // upload options + Host string `yaml:"host" json:"host" mapstructure:"host"` // -H , hostname of the engine/enterprise instance to upload to (setting this value enables upload) + Path string `yaml:"path" json:"path" mapstructure:"path"` // override the engine/enterprise API upload path + // IMPORTANT: do not show the username in any YAML/JSON output (sensitive information) + Username string `yaml:"-" json:"-" mapstructure:"username"` // -u , username to authenticate upload + // IMPORTANT: do not show the password in any YAML/JSON output (sensitive information) + Password string `yaml:"-" json:"-" mapstructure:"password"` // -p , password to authenticate upload + Dockerfile string `yaml:"dockerfile" json:"dockerfile" mapstructure:"dockerfile"` // -d , dockerfile to attach for upload + OverwriteExistingImage bool `yaml:"overwrite-existing-image" json:"overwrite-existing-image" mapstructure:"overwrite-existing-image"` // --overwrite-existing-image , if any of the SBOM components have already been uploaded this flag will ensure they are overwritten with the current upload +} diff --git a/internal/config/application.go b/internal/config/application.go new file mode 100644 index 000000000..24c97d1a8 --- /dev/null +++ b/internal/config/application.go @@ -0,0 +1,196 @@ +package config + +import ( + "errors" + "fmt" + "path" + "strings" + + "github.com/anchore/syft/syft/source" + + "github.com/adrg/xdg" + "github.com/anchore/syft/internal" + "github.com/mitchellh/go-homedir" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "gopkg.in/yaml.v2" +) + +var ErrApplicationConfigNotFound = fmt.Errorf("application config not found") + +// Application is the main syft application configuration. +type Application struct { + ConfigPath string `yaml:",omitempty" json:"configPath"` // the location where the application config was read from (either from -c or discovered while loading) + Output string `yaml:"output" json:"output" mapstructure:"output"` // -o, the Presenter hint string to use for report formatting + Quiet bool `yaml:"quiet" json:"quiet" mapstructure:"quiet"` // -q, indicates to not show any status output to stderr (ETUI or logging UI) + Log logging `yaml:"log" json:"log" mapstructure:"log"` // all logging-related options + CliOptions CliOnlyOptions `yaml:"-" json:"-"` // all options only available through the CLI (not via env vars or config) + Dev Development `yaml:"dev" json:"dev" mapstructure:"dev"` + CheckForAppUpdate bool `yaml:"check-for-app-update" json:"check-for-app-update" mapstructure:"check-for-app-update"` // whether to check for an application update on start up or not + Anchore anchore `yaml:"anchore" json:"anchore" mapstructure:"anchore"` // options for interacting with Anchore Engine/Enterprise + Package Packages `yaml:"package" json:"package" mapstructure:"package"` + FileMetadata FileMetadata `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"` +} + +// LoadApplicationConfig populates the given viper object with application configuration discovered on disk +func LoadApplicationConfig(v *viper.Viper, cliOpts CliOnlyOptions) (*Application, error) { + // the user may not have a config, and this is OK, we can use the default config + default cobra cli values instead + setNonCliDefaultAppConfigValues(v) + if err := readConfig(v, cliOpts.ConfigPath); err != nil && !errors.Is(err, ErrApplicationConfigNotFound) { + return nil, err + } + + config := &Application{ + CliOptions: cliOpts, + } + + if err := v.Unmarshal(config); err != nil { + return nil, fmt.Errorf("unable to parse config: %w", err) + } + config.ConfigPath = v.ConfigFileUsed() + + if err := config.build(); err != nil { + return nil, fmt.Errorf("invalid application config: %w", err) + } + + return config, nil +} + +// build inflates simple config values into syft native objects (or other complex objects) after the config is fully read in. +func (cfg *Application) build() error { + if cfg.Quiet { + // TODO: this is bad: quiet option trumps all other logging options + // we should be able to quiet the console logging and leave file logging alone... + // ... this will be an enhancement for later + cfg.Log.LevelOpt = logrus.PanicLevel + } else { + if cfg.Log.Level != "" { + if cfg.CliOptions.Verbosity > 0 { + return fmt.Errorf("cannot explicitly set log level (cfg file or env var) and use -v flag together") + } + + lvl, err := logrus.ParseLevel(strings.ToLower(cfg.Log.Level)) + if err != nil { + return fmt.Errorf("bad log level configured (%q): %w", cfg.Log.Level, err) + } + // set the log level explicitly + cfg.Log.LevelOpt = lvl + } else { + // set the log level implicitly + switch v := cfg.CliOptions.Verbosity; { + case v == 1: + cfg.Log.LevelOpt = logrus.InfoLevel + case v >= 2: + cfg.Log.LevelOpt = logrus.DebugLevel + default: + cfg.Log.LevelOpt = logrus.WarnLevel + } + } + } + + if cfg.Anchore.Host == "" && cfg.Anchore.Dockerfile != "" { + return fmt.Errorf("cannot provide dockerfile option without enabling upload") + } + + for _, builder := range []func() error{ + cfg.Package.build, + cfg.FileMetadata.build, + } { + if err := builder(); err != nil { + return err + } + } + + return nil +} + +func (cfg Application) String() string { + // yaml is pretty human friendly (at least when compared to json) + appCfgStr, err := yaml.Marshal(&cfg) + + if err != nil { + return err.Error() + } + + return string(appCfgStr) +} + +// readConfig attempts to read the given config path from disk or discover an alternate store location +// nolint:funlen +func readConfig(v *viper.Viper, configPath string) error { + var err error + v.AutomaticEnv() + v.SetEnvPrefix(internal.ApplicationName) + // allow for nested options to be specified via environment variables + // e.g. pod.context = APPNAME_POD_CONTEXT + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_")) + + // use explicitly the given user config + if configPath != "" { + v.SetConfigFile(configPath) + if err := v.ReadInConfig(); err != nil { + return fmt.Errorf("unable to read application config=%q : %w", configPath, err) + } + // don't fall through to other options if the config path was explicitly provided + return nil + } + + // start searching for valid configs in order... + + // 1. look for ..yaml (in the current directory) + v.AddConfigPath(".") + v.SetConfigName("." + internal.ApplicationName) + if err = v.ReadInConfig(); err == nil { + return nil + } else if !errors.As(err, &viper.ConfigFileNotFoundError{}) { + return fmt.Errorf("unable to parse config=%q: %w", v.ConfigFileUsed(), err) + } + + // 2. look for ./config.yaml (in the current directory) + v.AddConfigPath("." + internal.ApplicationName) + v.SetConfigName("config") + if err = v.ReadInConfig(); err == nil { + return nil + } else if !errors.As(err, &viper.ConfigFileNotFoundError{}) { + return fmt.Errorf("unable to parse config=%q: %w", v.ConfigFileUsed(), err) + } + + // 3. look for ~/..yaml + home, err := homedir.Dir() + if err == nil { + v.AddConfigPath(home) + v.SetConfigName("." + internal.ApplicationName) + if err = v.ReadInConfig(); err == nil { + return nil + } else if !errors.As(err, &viper.ConfigFileNotFoundError{}) { + return fmt.Errorf("unable to parse config=%q: %w", v.ConfigFileUsed(), err) + } + } + + // 4. look for /config.yaml in xdg locations (starting with xdg home config dir, then moving upwards) + v.AddConfigPath(path.Join(xdg.ConfigHome, internal.ApplicationName)) + for _, dir := range xdg.ConfigDirs { + v.AddConfigPath(path.Join(dir, internal.ApplicationName)) + } + v.SetConfigName("config") + if err = v.ReadInConfig(); err == nil { + return nil + } else if !errors.As(err, &viper.ConfigFileNotFoundError{}) { + return fmt.Errorf("unable to parse config=%q: %w", v.ConfigFileUsed(), err) + } + + return ErrApplicationConfigNotFound +} + +// setNonCliDefaultAppConfigValues ensures that there are sane defaults for values that do not have CLI equivalent options (where there would already be a default value) +func setNonCliDefaultAppConfigValues(v *viper.Viper) { + v.SetDefault("anchore.path", "") + v.SetDefault("log.structured", false) + v.SetDefault("check-for-app-update", true) + v.SetDefault("dev.profile-cpu", false) + v.SetDefault("dev.profile-mem", false) + v.SetDefault("package.cataloger.enabled", true) + v.SetDefault("file-metadata.cataloger.enabled", true) + v.SetDefault("file-metadata.cataloger.scope", source.SquashedScope) + v.SetDefault("file-metadata.digests", []string{"sha256"}) +} diff --git a/internal/config/cataloger_options.go b/internal/config/cataloger_options.go new file mode 100644 index 000000000..72e4c0977 --- /dev/null +++ b/internal/config/cataloger_options.go @@ -0,0 +1,23 @@ +package config + +import ( + "fmt" + + "github.com/anchore/syft/syft/source" +) + +type catalogerOptions struct { + Enabled bool `yaml:"enabled" json:"enabled" mapstructure:"enabled"` + Scope string `yaml:"scope" json:"scope" mapstructure:"scope"` + ScopeOpt source.Scope `yaml:"-" json:"-"` +} + +func (cfg *catalogerOptions) build() error { + scopeOption := source.ParseScope(cfg.Scope) + if scopeOption == source.UnknownScope { + return fmt.Errorf("bad scope value %q", cfg.Scope) + } + cfg.ScopeOpt = scopeOption + + return nil +} diff --git a/internal/config/cli_only_options.go b/internal/config/cli_only_options.go new file mode 100644 index 000000000..5ece4d7eb --- /dev/null +++ b/internal/config/cli_only_options.go @@ -0,0 +1,7 @@ +package config + +// CliOnlyOptions are options that are in the application config in memory, but are only exposed via CLI switches (not from unmarshaling a config file) +type CliOnlyOptions struct { + ConfigPath string // -c. where the read config is on disk + Verbosity int // -v or -vv , controlling which UI (ETUI vs logging) and what the log level should be +} diff --git a/internal/config/config.go b/internal/config/config.go deleted file mode 100644 index b957ae588..000000000 --- a/internal/config/config.go +++ /dev/null @@ -1,228 +0,0 @@ -package config - -import ( - "fmt" - "path" - "strings" - - "github.com/adrg/xdg" - "github.com/anchore/syft/internal" - "github.com/anchore/syft/syft/presenter" - "github.com/anchore/syft/syft/source" - "github.com/mitchellh/go-homedir" - "github.com/sirupsen/logrus" - "github.com/spf13/viper" - "gopkg.in/yaml.v2" -) - -// Application is the main syft application configuration. -type Application struct { - ConfigPath string `yaml:",omitempty"` // the location where the application config was read from (either from -c or discovered while loading) - PresenterOpt presenter.Option `yaml:"-"` // -o, the native Presenter.Option to use for report formatting - Output string `yaml:"output" mapstructure:"output"` // -o, the Presenter hint string to use for report formatting - ScopeOpt source.Scope `yaml:"-"` // -s, the native source.Scope option to use for how to catalog the container image - Scope string `yaml:"scope" mapstructure:"scope"` // -s, the source.Scope string hint for how to catalog the container image - Quiet bool `yaml:"quiet" mapstructure:"quiet"` // -q, indicates to not show any status output to stderr (ETUI or logging UI) - Log logging `yaml:"log" mapstructure:"log"` // all logging-related options - CliOptions CliOnlyOptions `yaml:"-"` // all options only available through the CLI (not via env vars or config) - Dev Development `mapstructure:"dev"` - CheckForAppUpdate bool `yaml:"check-for-app-update" mapstructure:"check-for-app-update"` // whether to check for an application update on start up or not - Anchore anchore `yaml:"anchore" mapstructure:"anchore"` // options for interacting with Anchore Engine/Enterprise -} - -// CliOnlyOptions are options that are in the application config in memory, but are only exposed via CLI switches (not from unmarshaling a config file) -type CliOnlyOptions struct { - ConfigPath string // -c. where the read config is on disk - Verbosity int // -v or -vv , controlling which UI (ETUI vs logging) and what the log level should be -} - -// logging contains all logging-related configuration options available to the user via the application config. -type logging struct { - Structured bool `yaml:"structured" mapstructure:"structured"` // show all log entries as JSON formatted strings - LevelOpt logrus.Level `yaml:"level"` // the native log level object used by the logger - Level string `yaml:"-" mapstructure:"level"` // the log level string hint - FileLocation string `yaml:"file" mapstructure:"file"` // the file path to write logs to -} - -type anchore struct { - // upload options - UploadEnabled bool `yaml:"upload-enabled" mapstructure:"upload-enabled"` // whether to upload results to Anchore Engine/Enterprise (defaults to "false" unless there is the presence of -h CLI option) - Host string `yaml:"host" mapstructure:"host"` // -H , hostname of the engine/enterprise instance to upload to - Path string `yaml:"path" mapstructure:"path"` // override the engine/enterprise API upload path - Username string `yaml:"username" mapstructure:"username"` // -u , username to authenticate upload - Password string `yaml:"password" mapstructure:"password"` // -p , password to authenticate upload - Dockerfile string `yaml:"dockerfile" mapstructure:"dockerfile"` // -d , dockerfile to attach for upload - OverwriteExistingImage bool `yaml:"overwrite-existing-image" mapstructure:"overwrite-existing-image"` // --overwrite-existing-image , if any of the SBOM components have already been uploaded this flag will ensure they are overwritten with the current upload -} - -type Development struct { - ProfileCPU bool `mapstructure:"profile-cpu"` - ProfileMem bool `mapstructure:"profile-mem"` -} - -// LoadApplicationConfig populates the given viper object with application configuration discovered on disk -func LoadApplicationConfig(v *viper.Viper, cliOpts CliOnlyOptions, wasHostnameSet bool) (*Application, error) { - // the user may not have a config, and this is OK, we can use the default config + default cobra cli values instead - setNonCliDefaultValues(v) - _ = readConfig(v, cliOpts.ConfigPath) - - config := &Application{ - CliOptions: cliOpts, - } - - if err := v.Unmarshal(config); err != nil { - return nil, fmt.Errorf("unable to parse config: %w", err) - } - config.ConfigPath = v.ConfigFileUsed() - - if err := config.build(v, wasHostnameSet); err != nil { - return nil, fmt.Errorf("invalid config: %w", err) - } - - return config, nil -} - -// build inflates simple config values into syft native objects (or other complex objects) after the config is fully read in. -func (cfg *Application) build(v *viper.Viper, wasHostnameSet bool) error { - // set the presenter - presenterOption := presenter.ParseOption(cfg.Output) - if presenterOption == presenter.UnknownPresenter { - return fmt.Errorf("bad --output value '%s'", cfg.Output) - } - cfg.PresenterOpt = presenterOption - - // set the source - scopeOption := source.ParseScope(cfg.Scope) - if scopeOption == source.UnknownScope { - return fmt.Errorf("bad --scope value '%s'", cfg.Scope) - } - cfg.ScopeOpt = scopeOption - - if cfg.Quiet { - // TODO: this is bad: quiet option trumps all other logging options - // we should be able to quiet the console logging and leave file logging alone... - // ... this will be an enhancement for later - cfg.Log.LevelOpt = logrus.PanicLevel - } else { - if cfg.Log.Level != "" { - if cfg.CliOptions.Verbosity > 0 { - return fmt.Errorf("cannot explicitly set log level (cfg file or env var) and use -v flag together") - } - - lvl, err := logrus.ParseLevel(strings.ToLower(cfg.Log.Level)) - if err != nil { - return fmt.Errorf("bad log level configured (%q): %w", cfg.Log.Level, err) - } - // set the log level explicitly - cfg.Log.LevelOpt = lvl - } else { - // set the log level implicitly - switch v := cfg.CliOptions.Verbosity; { - case v == 1: - cfg.Log.LevelOpt = logrus.InfoLevel - case v >= 2: - cfg.Log.LevelOpt = logrus.DebugLevel - default: - cfg.Log.LevelOpt = logrus.WarnLevel - } - } - } - // check if upload should be done relative to the CLI and config behavior - if !v.IsSet("anchore.upload-enabled") && wasHostnameSet { - // we know the user didn't specify to upload in the config file and a --hostname option was provided (so set upload) - cfg.Anchore.UploadEnabled = true - } - - if !cfg.Anchore.UploadEnabled && cfg.Anchore.Dockerfile != "" { - return fmt.Errorf("cannot provide dockerfile option without enabling upload") - } - - return nil -} - -func (cfg Application) String() string { - // redact sensitive information - if cfg.Anchore.Username != "" { - cfg.Anchore.Username = "********" - } - - if cfg.Anchore.Password != "" { - cfg.Anchore.Password = "********" - } - - // yaml is pretty human friendly (at least when compared to json) - appCfgStr, err := yaml.Marshal(&cfg) - - if err != nil { - return err.Error() - } - - return string(appCfgStr) -} - -// readConfig attempts to read the given config path from disk or discover an alternate store location -func readConfig(v *viper.Viper, configPath string) error { - v.AutomaticEnv() - v.SetEnvPrefix(internal.ApplicationName) - // allow for nested options to be specified via environment variables - // e.g. pod.context = APPNAME_POD_CONTEXT - v.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_")) - - // use explicitly the given user config - if configPath != "" { - v.SetConfigFile(configPath) - if err := v.ReadInConfig(); err == nil { - return nil - } - // don't fall through to other options if this fails - return fmt.Errorf("unable to read config: %v", configPath) - } - - // start searching for valid configs in order... - - // 1. look for ..yaml (in the current directory) - v.AddConfigPath(".") - v.SetConfigName(internal.ApplicationName) - if err := v.ReadInConfig(); err == nil { - return nil - } - - // 2. look for ./config.yaml (in the current directory) - v.AddConfigPath("." + internal.ApplicationName) - v.SetConfigName("config") - if err := v.ReadInConfig(); err == nil { - return nil - } - - // 3. look for ~/..yaml - home, err := homedir.Dir() - if err == nil { - v.AddConfigPath(home) - v.SetConfigName("." + internal.ApplicationName) - if err := v.ReadInConfig(); err == nil { - return nil - } - } - - // 4. look for /config.yaml in xdg locations (starting with xdg home config dir, then moving upwards) - v.AddConfigPath(path.Join(xdg.ConfigHome, internal.ApplicationName)) - for _, dir := range xdg.ConfigDirs { - v.AddConfigPath(path.Join(dir, internal.ApplicationName)) - } - v.SetConfigName("config") - if err := v.ReadInConfig(); err == nil { - return nil - } - - return fmt.Errorf("application config not found") -} - -// setNonCliDefaultValues ensures that there are sane defaults for values that do not have CLI equivalent options (where there would already be a default value) -func setNonCliDefaultValues(v *viper.Viper) { - v.SetDefault("log.level", "") - v.SetDefault("log.file", "") - v.SetDefault("log.structured", false) - v.SetDefault("check-for-app-update", true) - v.SetDefault("dev.profile-cpu", false) - v.SetDefault("dev.profile-mem", false) -} diff --git a/internal/config/development.go b/internal/config/development.go new file mode 100644 index 000000000..dbb38ff08 --- /dev/null +++ b/internal/config/development.go @@ -0,0 +1,6 @@ +package config + +type Development struct { + ProfileCPU bool `yaml:"profile-cpu" json:"profile-cpu" mapstructure:"profile-cpu"` + ProfileMem bool `yaml:"profile-mem" json:"profile-mem" mapstructure:"profile-mem"` +} diff --git a/internal/config/file_metadata.go b/internal/config/file_metadata.go new file mode 100644 index 000000000..274fce24b --- /dev/null +++ b/internal/config/file_metadata.go @@ -0,0 +1,10 @@ +package config + +type FileMetadata struct { + Cataloger catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"` + Digests []string `yaml:"digests" json:"digests" mapstructure:"digests"` +} + +func (cfg *FileMetadata) build() error { + return cfg.Cataloger.build() +} diff --git a/internal/config/logging.go b/internal/config/logging.go new file mode 100644 index 000000000..53139419a --- /dev/null +++ b/internal/config/logging.go @@ -0,0 +1,11 @@ +package config + +import "github.com/sirupsen/logrus" + +// logging contains all logging-related configuration options available to the user via the application config. +type logging struct { + Structured bool `yaml:"structured" json:"structured" mapstructure:"structured"` // show all log entries as JSON formatted strings + LevelOpt logrus.Level `yaml:"-" json:"-"` // the native log level object used by the logger + Level string `yaml:"level" json:"level" mapstructure:"level"` // the log level string hint + FileLocation string `yaml:"file" json:"file-location" mapstructure:"file"` // the file path to write logs to +} diff --git a/internal/config/packages.go b/internal/config/packages.go new file mode 100644 index 000000000..8193ab04c --- /dev/null +++ b/internal/config/packages.go @@ -0,0 +1,9 @@ +package config + +type Packages struct { + Cataloger catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"` +} + +func (cfg *Packages) build() error { + return cfg.Cataloger.build() +} diff --git a/internal/constants.go b/internal/constants.go index 5cb49659a..b027f8502 100644 --- a/internal/constants.go +++ b/internal/constants.go @@ -6,5 +6,5 @@ const ( // JSONSchemaVersion is the current schema version output by the JSON presenter // This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment. - JSONSchemaVersion = "1.0.3" + JSONSchemaVersion = "1.0.4" ) diff --git a/internal/presenter/packages/cyclonedx_bom_descriptor.go b/internal/presenter/packages/cyclonedx_bom_descriptor.go new file mode 100644 index 000000000..eeaac61f7 --- /dev/null +++ b/internal/presenter/packages/cyclonedx_bom_descriptor.go @@ -0,0 +1,69 @@ +package packages + +import ( + "encoding/xml" + "time" + + "github.com/anchore/syft/syft/source" +) + +// Source: https://cyclonedx.org/ext/bom-descriptor/ + +// CycloneDxBomDescriptor represents all metadata surrounding the BOM report (such as when the BOM was made, with which tool, and the item being cataloged). +type CycloneDxBomDescriptor struct { + XMLName xml.Name `xml:"metadata"` + Timestamp string `xml:"timestamp,omitempty"` // The date and time (timestamp) when the document was created + Tools []CycloneDxBdTool `xml:"tools>tool"` // The tool used to create the BOM. + Component *CycloneDxBdComponent `xml:"component"` // The component that the BOM describes. +} + +// CycloneDxBdTool represents the tool that created the BOM report. +type CycloneDxBdTool struct { + XMLName xml.Name `xml:"tool"` + Vendor string `xml:"vendor,omitempty"` // The vendor of the tool used to create the BOM. + Name string `xml:"name,omitempty"` // The name of the tool used to create the BOM. + Version string `xml:"version,omitempty"` // The version of the tool used to create the BOM. + // TODO: hashes, author, manufacture, supplier + // TODO: add user-defined fields for the remaining build/version parameters +} + +// CycloneDxBdComponent represents the software/package being cataloged. +type CycloneDxBdComponent struct { + XMLName xml.Name `xml:"component"` + CycloneDxComponent +} + +// NewCycloneDxBomDescriptor returns a new CycloneDxBomDescriptor tailored for the current time and "syft" tool details. +func NewCycloneDxBomDescriptor(name, version string, srcMetadata source.Metadata) *CycloneDxBomDescriptor { + descriptor := CycloneDxBomDescriptor{ + XMLName: xml.Name{}, + Timestamp: time.Now().Format(time.RFC3339), + Tools: []CycloneDxBdTool{ + { + Vendor: "anchore", + Name: name, + Version: version, + }, + }, + } + + switch srcMetadata.Scheme { + case source.ImageScheme: + descriptor.Component = &CycloneDxBdComponent{ + CycloneDxComponent: CycloneDxComponent{ + Type: "container", + Name: srcMetadata.ImageMetadata.UserInput, + Version: srcMetadata.ImageMetadata.ManifestDigest, + }, + } + case source.DirectoryScheme: + descriptor.Component = &CycloneDxBdComponent{ + CycloneDxComponent: CycloneDxComponent{ + Type: "file", + Name: srcMetadata.Path, + }, + } + } + + return &descriptor +} diff --git a/internal/presenter/packages/cyclonedx_component.go b/internal/presenter/packages/cyclonedx_component.go new file mode 100644 index 000000000..9ff1877dd --- /dev/null +++ b/internal/presenter/packages/cyclonedx_component.go @@ -0,0 +1,27 @@ +package packages + +import "encoding/xml" + +// CycloneDxComponent represents a single element in the CycloneDX BOM +type CycloneDxComponent struct { + XMLName xml.Name `xml:"component"` + Type string `xml:"type,attr"` // Required; Describes if the component is a library, framework, application, container, operating system, firmware, hardware device, or file + Supplier string `xml:"supplier,omitempty"` // The organization that supplied the component. The supplier may often be the manufacture, but may also be a distributor or repackager. + Author string `xml:"author,omitempty"` // The person(s) or organization(s) that authored the component + Publisher string `xml:"publisher,omitempty"` // The person(s) or organization(s) that published the component + Group string `xml:"group,omitempty"` // The high-level classification that a project self-describes as. This will often be a shortened, single name of the company or project that produced the component, or the source package or domain name. + Name string `xml:"name"` // Required; The name of the component as defined by the project + Version string `xml:"version"` // Required; The version of the component as defined by the project + Description string `xml:"description,omitempty"` // A description of the component + Licenses *[]CycloneDxLicense `xml:"licenses>license"` // A node describing zero or more license names, SPDX license IDs or expressions + PackageURL string `xml:"purl,omitempty"` // Specifies the package-url (PackageURL). The purl, if specified, must be valid and conform to the specification defined at: https://github.com/package-url/purl-spec + // TODO: source, hashes, copyright, cpe, purl, swid, modified, pedigree, externalReferences + // TODO: add user-defined parameters for syft-specific values (image layer index, cataloger, location path, etc.) +} + +// CycloneDxLicense represents a single software license for a CycloneDxComponent +type CycloneDxLicense struct { + XMLName xml.Name `xml:"license"` + ID string `xml:"id,omitempty"` // A valid SPDX license ID + Name string `xml:"name,omitempty"` // If SPDX does not define the license used, this field may be used to provide the license name +} diff --git a/internal/presenter/packages/cyclonedx_document.go b/internal/presenter/packages/cyclonedx_document.go new file mode 100644 index 000000000..9278dc6d1 --- /dev/null +++ b/internal/presenter/packages/cyclonedx_document.go @@ -0,0 +1,57 @@ +package packages + +import ( + "encoding/xml" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/version" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" + "github.com/google/uuid" +) + +// Source: https://github.com/CycloneDX/specification + +// CycloneDxDocument represents a CycloneDX BOM CycloneDxDocument. +type CycloneDxDocument struct { + XMLName xml.Name `xml:"bom"` + XMLNs string `xml:"xmlns,attr"` + Version int `xml:"version,attr"` + SerialNumber string `xml:"serialNumber,attr"` + BomDescriptor *CycloneDxBomDescriptor `xml:"metadata"` // The BOM descriptor extension + Components []CycloneDxComponent `xml:"components>component"` // The BOM contents +} + +// NewCycloneDxDocument returns a CycloneDX CycloneDxDocument object populated with the catalog contents. +func NewCycloneDxDocument(catalog *pkg.Catalog, srcMetadata source.Metadata) CycloneDxDocument { + versionInfo := version.FromBuild() + + doc := CycloneDxDocument{ + XMLNs: "http://cyclonedx.org/schema/bom/1.2", + Version: 1, + SerialNumber: uuid.New().URN(), + BomDescriptor: NewCycloneDxBomDescriptor(internal.ApplicationName, versionInfo.Version, srcMetadata), + } + + // attach components + for p := range catalog.Enumerate() { + component := CycloneDxComponent{ + Type: "library", // TODO: this is not accurate + Name: p.Name, + Version: p.Version, + PackageURL: p.PURL, + } + var licenses []CycloneDxLicense + for _, licenseName := range p.Licenses { + licenses = append(licenses, CycloneDxLicense{ + Name: licenseName, + }) + } + if len(licenses) > 0 { + component.Licenses = &licenses + } + doc.Components = append(doc.Components, component) + } + + return doc +} diff --git a/syft/presenter/cyclonedx/presenter.go b/internal/presenter/packages/cyclonedx_presenter.go similarity index 57% rename from syft/presenter/cyclonedx/presenter.go rename to internal/presenter/packages/cyclonedx_presenter.go index 1c044c56c..f844fbf02 100644 --- a/syft/presenter/cyclonedx/presenter.go +++ b/internal/presenter/packages/cyclonedx_presenter.go @@ -1,7 +1,7 @@ /* Package cyclonedx is responsible for generating a CycloneDX XML report for the given container image or file system. */ -package cyclonedx +package packages import ( "encoding/xml" @@ -11,23 +11,23 @@ import ( "github.com/anchore/syft/syft/source" ) -// Presenter writes a CycloneDX report from the given Catalog and Locations contents -type Presenter struct { +// CycloneDxPresenter writes a CycloneDX report from the given Catalog and Locations contents +type CycloneDxPresenter struct { catalog *pkg.Catalog srcMetadata source.Metadata } -// NewPresenter creates a CycloneDX presenter from the given Catalog and Locations objects. -func NewPresenter(catalog *pkg.Catalog, srcMetadata source.Metadata) *Presenter { - return &Presenter{ +// NewCycloneDxPresenter creates a CycloneDX presenter from the given Catalog and Locations objects. +func NewCycloneDxPresenter(catalog *pkg.Catalog, srcMetadata source.Metadata) *CycloneDxPresenter { + return &CycloneDxPresenter{ catalog: catalog, srcMetadata: srcMetadata, } } // Present writes the CycloneDX report to the given io.Writer. -func (pres *Presenter) Present(output io.Writer) error { - bom := NewDocument(pres.catalog, pres.srcMetadata) +func (pres *CycloneDxPresenter) Present(output io.Writer) error { + bom := NewCycloneDxDocument(pres.catalog, pres.srcMetadata) encoder := xml.NewEncoder(output) encoder.Indent("", " ") diff --git a/syft/presenter/cyclonedx/presenter_test.go b/internal/presenter/packages/cyclonedx_presenter_test.go similarity index 93% rename from syft/presenter/cyclonedx/presenter_test.go rename to internal/presenter/packages/cyclonedx_presenter_test.go index 9ef9cbb24..46d3364f4 100644 --- a/syft/presenter/cyclonedx/presenter_test.go +++ b/internal/presenter/packages/cyclonedx_presenter_test.go @@ -1,4 +1,4 @@ -package cyclonedx +package packages import ( "bytes" @@ -62,7 +62,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) { t.Fatal(err) } - pres := NewPresenter(catalog, s.Metadata) + pres := NewCycloneDxPresenter(catalog, s.Metadata) // run presenter err = pres.Present(&buffer) @@ -93,8 +93,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) { var buffer bytes.Buffer catalog := pkg.NewCatalog() - img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-simple") - defer cleanup() + img := imagetest.GetFixtureImage(t, "docker-archive", "image-simple") _, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks) _, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks) @@ -125,7 +124,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) { PURL: "the-purl-2", }) - s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input") + s, err := source.NewFromImage(img, "user-image-input") if err != nil { t.Fatal(err) } @@ -138,7 +137,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) { // This value is sourced from the "version" node in "./test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden" s.Metadata.ImageMetadata.ManifestDigest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368" - pres := NewPresenter(catalog, s.Metadata) + pres := NewCycloneDxPresenter(catalog, s.Metadata) // run presenter err = pres.Present(&buffer) diff --git a/syft/presenter/json/distribution.go b/internal/presenter/packages/json_distribution.go similarity index 54% rename from syft/presenter/json/distribution.go rename to internal/presenter/packages/json_distribution.go index 07eb5ba23..5ba939c03 100644 --- a/syft/presenter/json/distribution.go +++ b/internal/presenter/packages/json_distribution.go @@ -1,21 +1,21 @@ -package json +package packages import "github.com/anchore/syft/syft/distro" -// Distribution provides information about a detected Linux Distribution. -type Distribution struct { +// JSONDistribution provides information about a detected Linux JSONDistribution. +type JSONDistribution struct { Name string `json:"name"` // Name of the Linux distribution Version string `json:"version"` // Version of the Linux distribution (major or major.minor version) IDLike string `json:"idLike"` // the ID_LIKE field found within the /etc/os-release file } -// NewDistribution creates a struct with the Linux distribution to be represented in JSON. -func NewDistribution(d *distro.Distro) Distribution { +// NewJSONDistribution creates a struct with the Linux distribution to be represented in JSON. +func NewJSONDistribution(d *distro.Distro) JSONDistribution { if d == nil { - return Distribution{} + return JSONDistribution{} } - return Distribution{ + return JSONDistribution{ Name: d.Name(), Version: d.FullVersion(), IDLike: d.IDLike, diff --git a/internal/presenter/packages/json_document.go b/internal/presenter/packages/json_document.go new file mode 100644 index 000000000..399d037ae --- /dev/null +++ b/internal/presenter/packages/json_document.go @@ -0,0 +1,62 @@ +package packages + +import ( + "fmt" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/version" + "github.com/anchore/syft/syft/distro" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +// JSONDocument represents the syft cataloging findings as a JSON document +type JSONDocument struct { + Artifacts []JSONPackage `json:"artifacts"` // Artifacts is the list of packages discovered and placed into the catalog + ArtifactRelationships []JSONRelationship `json:"artifactRelationships"` + Source JSONSource `json:"source"` // Source represents the original object that was cataloged + Distro JSONDistribution `json:"distro"` // Distro represents the Linux distribution that was detected from the source + Descriptor JSONDescriptor `json:"descriptor"` // Descriptor is a block containing self-describing information about syft + Schema JSONSchema `json:"schema"` // Schema is a block reserved for defining the version for the shape of this JSON document and where to find the schema document to validate the shape +} + +// NewJSONDocument creates and populates a new JSON document struct from the given cataloging results. +func NewJSONDocument(catalog *pkg.Catalog, srcMetadata source.Metadata, d *distro.Distro, scope source.Scope, configuration interface{}) (JSONDocument, error) { + src, err := NewJSONSource(srcMetadata, scope) + if err != nil { + return JSONDocument{}, err + } + + artifacts, err := NewJSONPackages(catalog) + if err != nil { + return JSONDocument{}, err + } + + return JSONDocument{ + Artifacts: artifacts, + ArtifactRelationships: newJSONRelationships(pkg.NewRelationships(catalog)), + Source: src, + Distro: NewJSONDistribution(d), + Descriptor: JSONDescriptor{ + Name: internal.ApplicationName, + Version: version.FromBuild().Version, + Configuration: configuration, + }, + Schema: JSONSchema{ + Version: internal.JSONSchemaVersion, + URL: fmt.Sprintf("https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-%s.json", internal.JSONSchemaVersion), + }, + }, nil +} + +// JSONDescriptor describes what created the document as well as surrounding metadata +type JSONDescriptor struct { + Name string `json:"name"` + Version string `json:"version"` + Configuration interface{} `json:"configuration,omitempty"` +} + +type JSONSchema struct { + Version string `json:"version"` + URL string `json:"url"` +} diff --git a/internal/presenter/packages/json_package.go b/internal/presenter/packages/json_package.go new file mode 100644 index 000000000..a2ea899dd --- /dev/null +++ b/internal/presenter/packages/json_package.go @@ -0,0 +1,71 @@ +package packages + +import ( + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +// JSONPackage represents a pkg.Package object specialized for JSON marshaling and unmarshaling. +type JSONPackage struct { + ID string `json:"id"` + Name string `json:"name"` + Version string `json:"version"` + Type string `json:"type"` + FoundBy string `json:"foundBy"` + Locations []source.Location `json:"locations"` + Licenses []string `json:"licenses"` + Language string `json:"language"` + CPEs []string `json:"cpes"` + PURL string `json:"purl"` + MetadataType string `json:"metadataType"` + Metadata interface{} `json:"metadata"` +} + +func NewJSONPackages(catalog *pkg.Catalog) ([]JSONPackage, error) { + artifacts := make([]JSONPackage, 0) + if catalog == nil { + return artifacts, nil + } + for _, p := range catalog.Sorted() { + art, err := NewJSONPackage(p) + if err != nil { + return nil, err + } + artifacts = append(artifacts, art) + } + return artifacts, nil +} + +// NewJSONPackage crates a new JSONPackage from the given pkg.Package. +func NewJSONPackage(p *pkg.Package) (JSONPackage, error) { + var cpes = make([]string, len(p.CPEs)) + for i, c := range p.CPEs { + cpes[i] = c.BindToFmtString() + } + + // ensure collections are never nil for presentation reasons + var locations = make([]source.Location, 0) + if p.Locations != nil { + locations = p.Locations + } + + var licenses = make([]string, 0) + if p.Licenses != nil { + licenses = p.Licenses + } + + return JSONPackage{ + ID: string(p.ID), + Name: p.Name, + Version: p.Version, + Type: string(p.Type), + FoundBy: p.FoundBy, + Locations: locations, + Licenses: licenses, + Language: string(p.Language), + CPEs: cpes, + PURL: p.PURL, + MetadataType: string(p.MetadataType), + Metadata: p.Metadata, + }, nil +} diff --git a/internal/presenter/packages/json_presenter.go b/internal/presenter/packages/json_presenter.go new file mode 100644 index 000000000..cbdc4d107 --- /dev/null +++ b/internal/presenter/packages/json_presenter.go @@ -0,0 +1,43 @@ +package packages + +import ( + "encoding/json" + "io" + + "github.com/anchore/syft/syft/distro" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +// JSONPresenter is a JSON presentation object for the syft results +type JSONPresenter struct { + catalog *pkg.Catalog + srcMetadata source.Metadata + distro *distro.Distro + scope source.Scope +} + +// NewJSONPresenter creates a new JSON presenter object for the given cataloging results. +func NewJSONPresenter(catalog *pkg.Catalog, s source.Metadata, d *distro.Distro, scope source.Scope) *JSONPresenter { + return &JSONPresenter{ + catalog: catalog, + srcMetadata: s, + distro: d, + scope: scope, + } +} + +// Present the catalog results to the given writer. +func (pres *JSONPresenter) Present(output io.Writer) error { + // we do not pass in configuration for backwards compatibility + doc, err := NewJSONDocument(pres.catalog, pres.srcMetadata, pres.distro, pres.scope, nil) + if err != nil { + return err + } + + enc := json.NewEncoder(output) + // prevent > and < from being escaped in the payload + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + return enc.Encode(&doc) +} diff --git a/syft/presenter/json/presenter_test.go b/internal/presenter/packages/json_presenter_test.go similarity index 89% rename from syft/presenter/json/presenter_test.go rename to internal/presenter/packages/json_presenter_test.go index f76215c93..582efdbe0 100644 --- a/syft/presenter/json/presenter_test.go +++ b/internal/presenter/packages/json_presenter_test.go @@ -1,4 +1,4 @@ -package json +package packages import ( "bytes" @@ -15,7 +15,7 @@ import ( "github.com/sergi/go-diff/diffmatchpatch" ) -var update = flag.Bool("update", false, "update the *.golden files for json presenters") +var updateJSONGoldenFiles = flag.Bool("update-json", false, "update the *.golden files for json presenters") func must(c pkg.CPE, e error) pkg.CPE { if e != nil { @@ -24,7 +24,7 @@ func must(c pkg.CPE, e error) pkg.CPE { return c } -func TestJsonDirsPresenter(t *testing.T) { +func TestJSONDirsPresenter(t *testing.T) { var buffer bytes.Buffer catalog := pkg.NewCatalog() @@ -75,7 +75,7 @@ func TestJsonDirsPresenter(t *testing.T) { if err != nil { t.Fatal(err) } - pres := NewPresenter(catalog, s.Metadata, d) + pres := NewJSONPresenter(catalog, s.Metadata, d, source.SquashedScope) // run presenter err = pres.Present(&buffer) @@ -84,7 +84,7 @@ func TestJsonDirsPresenter(t *testing.T) { } actual := buffer.Bytes() - if *update { + if *updateJSONGoldenFiles { testutils.UpdateGoldenFileContents(t, actual) } @@ -98,12 +98,12 @@ func TestJsonDirsPresenter(t *testing.T) { } -func TestJsonImgsPresenter(t *testing.T) { +func TestJSONImgsPresenter(t *testing.T) { var buffer bytes.Buffer testImage := "image-simple" - if *update { + if *updateJSONGoldenFiles { imagetest.UpdateGoldenFixtureImage(t, testImage) } @@ -158,9 +158,9 @@ func TestJsonImgsPresenter(t *testing.T) { // this is a hard coded value that is not given by the fixture helper and must be provided manually img.Metadata.ManifestDigest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368" - s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input") + s, err := source.NewFromImage(img, "user-image-input") var d *distro.Distro - pres := NewPresenter(catalog, s.Metadata, d) + pres := NewJSONPresenter(catalog, s.Metadata, d, source.SquashedScope) // run presenter err = pres.Present(&buffer) @@ -169,7 +169,7 @@ func TestJsonImgsPresenter(t *testing.T) { } actual := buffer.Bytes() - if *update { + if *updateJSONGoldenFiles { testutils.UpdateGoldenFileContents(t, actual) } diff --git a/syft/presenter/json/relationship.go b/internal/presenter/packages/json_relationship.go similarity index 63% rename from syft/presenter/json/relationship.go rename to internal/presenter/packages/json_relationship.go index 25eee2a3c..326e4a4a5 100644 --- a/syft/presenter/json/relationship.go +++ b/internal/presenter/packages/json_relationship.go @@ -1,18 +1,18 @@ -package json +package packages import "github.com/anchore/syft/syft/pkg" -type Relationship struct { +type JSONRelationship struct { Parent string `json:"parent"` Child string `json:"child"` Type string `json:"type"` Metadata interface{} `json:"metadata"` } -func newRelationships(relationships []pkg.Relationship) []Relationship { - result := make([]Relationship, len(relationships)) +func newJSONRelationships(relationships []pkg.Relationship) []JSONRelationship { + result := make([]JSONRelationship, len(relationships)) for i, r := range relationships { - result[i] = Relationship{ + result[i] = JSONRelationship{ Parent: string(r.Parent), Child: string(r.Child), Type: string(r.Type), diff --git a/internal/presenter/packages/json_source.go b/internal/presenter/packages/json_source.go new file mode 100644 index 000000000..d0c2a477f --- /dev/null +++ b/internal/presenter/packages/json_source.go @@ -0,0 +1,39 @@ +package packages + +import ( + "fmt" + + "github.com/anchore/syft/syft/source" +) + +// JSONSource object represents the thing that was cataloged +type JSONSource struct { + Type string `json:"type"` + Target interface{} `json:"target"` +} + +type JSONImageSource struct { + source.ImageMetadata + Scope source.Scope `json:"scope"` +} + +// NewJSONSource creates a new source object to be represented into JSON. +func NewJSONSource(src source.Metadata, scope source.Scope) (JSONSource, error) { + switch src.Scheme { + case source.ImageScheme: + return JSONSource{ + Type: "image", + Target: JSONImageSource{ + Scope: scope, + ImageMetadata: src.ImageMetadata, + }, + }, nil + case source.DirectoryScheme: + return JSONSource{ + Type: "directory", + Target: src.Path, + }, nil + default: + return JSONSource{}, fmt.Errorf("unsupported source: %q", src.Scheme) + } +} diff --git a/internal/presenter/packages/presenter.go b/internal/presenter/packages/presenter.go new file mode 100644 index 000000000..c73763b5c --- /dev/null +++ b/internal/presenter/packages/presenter.go @@ -0,0 +1,25 @@ +/* +Defines a Presenter interface for displaying catalog results to an io.Writer as well as a helper utility to obtain +a specific Presenter implementation given user configuration. +*/ +package packages + +import ( + "github.com/anchore/syft/internal/presenter" +) + +// Presenter returns a presenter for images or directories +func Presenter(option PresenterOption, config PresenterConfig) presenter.Presenter { + switch option { + case JSONPresenterOption: + return NewJSONPresenter(config.Catalog, config.SourceMetadata, config.Distro, config.Scope) + case TextPresenterOption: + return NewTextPresenter(config.Catalog, config.SourceMetadata) + case TablePresenterOption: + return NewTablePresenter(config.Catalog) + case CycloneDxPresenterOption: + return NewCycloneDxPresenter(config.Catalog, config.SourceMetadata) + default: + return nil + } +} diff --git a/internal/presenter/packages/presenter_config.go b/internal/presenter/packages/presenter_config.go new file mode 100644 index 000000000..d318e4288 --- /dev/null +++ b/internal/presenter/packages/presenter_config.go @@ -0,0 +1,14 @@ +package packages + +import ( + "github.com/anchore/syft/syft/distro" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +type PresenterConfig struct { + SourceMetadata source.Metadata + Catalog *pkg.Catalog + Distro *distro.Distro + Scope source.Scope +} diff --git a/internal/presenter/packages/presenter_option.go b/internal/presenter/packages/presenter_option.go new file mode 100644 index 000000000..9bfe1f6f2 --- /dev/null +++ b/internal/presenter/packages/presenter_option.go @@ -0,0 +1,35 @@ +package packages + +import "strings" + +const ( + UnknownPresenterOption PresenterOption = "UnknownPresenterOption" + JSONPresenterOption PresenterOption = "json" + TextPresenterOption PresenterOption = "text" + TablePresenterOption PresenterOption = "table" + CycloneDxPresenterOption PresenterOption = "cyclonedx" +) + +var AllPresenters = []PresenterOption{ + JSONPresenterOption, + TextPresenterOption, + TablePresenterOption, + CycloneDxPresenterOption, +} + +type PresenterOption string + +func ParsePresenterOption(userStr string) PresenterOption { + switch strings.ToLower(userStr) { + case string(JSONPresenterOption): + return JSONPresenterOption + case string(TextPresenterOption): + return TextPresenterOption + case string(TablePresenterOption): + return TablePresenterOption + case string(CycloneDxPresenterOption), "cyclone", "cyclone-dx": + return CycloneDxPresenterOption + default: + return UnknownPresenterOption + } +} diff --git a/syft/presenter/table/presenter.go b/internal/presenter/packages/table_presenter.go similarity index 86% rename from syft/presenter/table/presenter.go rename to internal/presenter/packages/table_presenter.go index c7303e8a5..aefaa7286 100644 --- a/syft/presenter/table/presenter.go +++ b/internal/presenter/packages/table_presenter.go @@ -1,4 +1,4 @@ -package table +package packages import ( "fmt" @@ -11,17 +11,17 @@ import ( "github.com/anchore/syft/syft/pkg" ) -type Presenter struct { +type TablePresenter struct { catalog *pkg.Catalog } -func NewPresenter(catalog *pkg.Catalog) *Presenter { - return &Presenter{ +func NewTablePresenter(catalog *pkg.Catalog) *TablePresenter { + return &TablePresenter{ catalog: catalog, } } -func (pres *Presenter) Present(output io.Writer) error { +func (pres *TablePresenter) Present(output io.Writer) error { rows := make([][]string, 0) columns := []string{"Name", "Version", "Type"} @@ -42,7 +42,7 @@ func (pres *Presenter) Present(output io.Writer) error { // sort by name, version, then type sort.SliceStable(rows, func(i, j int) bool { for col := 0; col < len(columns); col++ { - if rows[i][0] != rows[j][0] { + if rows[i][col] != rows[j][col] { return rows[i][col] < rows[j][col] } } @@ -73,7 +73,6 @@ func (pres *Presenter) Present(output io.Writer) error { func removeDuplicateRows(items [][]string) [][]string { seen := map[string][]string{} - // nolint:prealloc var result [][]string for _, v := range items { diff --git a/syft/presenter/table/presenter_test.go b/internal/presenter/packages/table_presenter_test.go similarity index 87% rename from syft/presenter/table/presenter_test.go rename to internal/presenter/packages/table_presenter_test.go index fb120494b..54e1db38e 100644 --- a/syft/presenter/table/presenter_test.go +++ b/internal/presenter/packages/table_presenter_test.go @@ -1,4 +1,4 @@ -package table +package packages import ( "bytes" @@ -16,7 +16,7 @@ import ( "github.com/sergi/go-diff/diffmatchpatch" ) -var update = flag.Bool("update", false, "update the *.golden files for table presenters") +var updateTablePresenterGoldenFiles = flag.Bool("update-table", false, "update the *.golden files for table presenters") func TestTablePresenter(t *testing.T) { var buffer bytes.Buffer @@ -24,8 +24,7 @@ func TestTablePresenter(t *testing.T) { testImage := "image-simple" catalog := pkg.NewCatalog() - img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", testImage) - defer cleanup() + img := imagetest.GetFixtureImage(t, "docker-archive", testImage) _, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks) _, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks) @@ -48,7 +47,7 @@ func TestTablePresenter(t *testing.T) { Type: pkg.DebPkg, }) - pres := NewPresenter(catalog) + pres := NewTablePresenter(catalog) // run presenter err := pres.Present(&buffer) @@ -57,7 +56,7 @@ func TestTablePresenter(t *testing.T) { } actual := buffer.Bytes() - if *update { + if *updateTablePresenterGoldenFiles { testutils.UpdateGoldenFileContents(t, actual) } diff --git a/syft/presenter/text/test-fixtures/image-simple/Dockerfile b/internal/presenter/packages/test-fixtures/image-simple/Dockerfile similarity index 100% rename from syft/presenter/text/test-fixtures/image-simple/Dockerfile rename to internal/presenter/packages/test-fixtures/image-simple/Dockerfile diff --git a/syft/presenter/cyclonedx/test-fixtures/image-simple/file-1.txt b/internal/presenter/packages/test-fixtures/image-simple/file-1.txt similarity index 100% rename from syft/presenter/cyclonedx/test-fixtures/image-simple/file-1.txt rename to internal/presenter/packages/test-fixtures/image-simple/file-1.txt diff --git a/syft/presenter/cyclonedx/test-fixtures/image-simple/file-2.txt b/internal/presenter/packages/test-fixtures/image-simple/file-2.txt similarity index 100% rename from syft/presenter/cyclonedx/test-fixtures/image-simple/file-2.txt rename to internal/presenter/packages/test-fixtures/image-simple/file-2.txt diff --git a/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden b/internal/presenter/packages/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden similarity index 100% rename from syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden rename to internal/presenter/packages/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden diff --git a/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden b/internal/presenter/packages/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden similarity index 100% rename from syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden rename to internal/presenter/packages/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden diff --git a/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden b/internal/presenter/packages/test-fixtures/snapshot/TestJSONDirsPresenter.golden similarity index 94% rename from syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden rename to internal/presenter/packages/test-fixtures/snapshot/TestJSONDirsPresenter.golden index 721fe5bac..b43c0e029 100644 --- a/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden +++ b/internal/presenter/packages/test-fixtures/snapshot/TestJSONDirsPresenter.golden @@ -60,6 +60,7 @@ } } ], + "artifactRelationships": [], "source": { "type": "directory", "target": "/some/path" @@ -74,8 +75,7 @@ "version": "[not provided]" }, "schema": { - "version": "1.0.3", - "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.3.json" - }, - "artifactRelationships": [] + "version": "1.0.4", + "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.4.json" + } } diff --git a/internal/presenter/packages/test-fixtures/snapshot/TestJSONImgsPresenter.golden b/internal/presenter/packages/test-fixtures/snapshot/TestJSONImgsPresenter.golden new file mode 100644 index 000000000..0c27b16d9 --- /dev/null +++ b/internal/presenter/packages/test-fixtures/snapshot/TestJSONImgsPresenter.golden @@ -0,0 +1,107 @@ +{ + "artifacts": [ + { + "id": "package-1-id", + "name": "package-1", + "version": "1.0.1", + "type": "python", + "foundBy": "the-cataloger-1", + "locations": [ + { + "path": "/somefile-1.txt", + "layerID": "sha256:3de16c5b8659a2e8d888b8ded8427be7a5686a3c8c4e4dd30de20f362827285b" + } + ], + "licenses": [ + "MIT" + ], + "language": "python", + "cpes": [ + "cpe:2.3:*:some:package:1:*:*:*:*:*:*:*" + ], + "purl": "a-purl-1", + "metadataType": "PythonPackageMetadata", + "metadata": { + "name": "package-1", + "version": "1.0.1", + "license": "", + "author": "", + "authorEmail": "", + "platform": "", + "sitePackagesRootPath": "" + } + }, + { + "id": "package-2-id", + "name": "package-2", + "version": "2.0.1", + "type": "deb", + "foundBy": "the-cataloger-2", + "locations": [ + { + "path": "/somefile-2.txt", + "layerID": "sha256:366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703" + } + ], + "licenses": [], + "language": "", + "cpes": [ + "cpe:2.3:*:some:package:2:*:*:*:*:*:*:*" + ], + "purl": "a-purl-2", + "metadataType": "DpkgMetadata", + "metadata": { + "package": "package-2", + "source": "", + "version": "2.0.1", + "sourceVersion": "", + "architecture": "", + "maintainer": "", + "installedSize": 0, + "files": null + } + } + ], + "artifactRelationships": [], + "source": { + "type": "image", + "target": { + "userInput": "user-image-input", + "imageID": "sha256:c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0", + "manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368", + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "tags": [ + "stereoscope-fixture-image-simple:85066c51088bdd274f7a89e99e00490f666c49e72ffc955707cd6e18f0e22c5b" + ], + "imageSize": 38, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:3de16c5b8659a2e8d888b8ded8427be7a5686a3c8c4e4dd30de20f362827285b", + "size": 22 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703", + "size": 16 + } + ], + "manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjoxNTg2LCJkaWdlc3QiOiJzaGEyNTY6YzJiNDZiNGViMDYyOTY5MzNiN2NmMDcyMjY4Mzk2NGU5ZWNiZDkzMjY1YjllZjZhZTk2NDJlMzk1MmFmYmJhMCJ9LCJsYXllcnMiOlt7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLCJzaXplIjoyMDQ4LCJkaWdlc3QiOiJzaGEyNTY6M2RlMTZjNWI4NjU5YTJlOGQ4ODhiOGRlZDg0MjdiZTdhNTY4NmEzYzhjNGU0ZGQzMGRlMjBmMzYyODI3Mjg1YiJ9LHsibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1NjozNjZhM2Y1NjUzZTM0NjczYjg3NTg5MWIwMjE2NDc0NDBkMDEyN2MyZWYwNDFlM2IxYTIyZGEyYTdkNGYzNzAzIn1dfQ==", + "config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJIb3N0bmFtZSI6IiIsIkRvbWFpbm5hbWUiOiIiLCJVc2VyIjoiIiwiQXR0YWNoU3RkaW4iOmZhbHNlLCJBdHRhY2hTdGRvdXQiOmZhbHNlLCJBdHRhY2hTdGRlcnIiOmZhbHNlLCJUdHkiOmZhbHNlLCJPcGVuU3RkaW4iOmZhbHNlLCJTdGRpbk9uY2UiOmZhbHNlLCJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiQ21kIjpudWxsLCJJbWFnZSI6InNoYTI1NjpkYWMyMTUwMzhjMDUwZTM1NzMwNTVlZmU4YTkwM2NkMWY5YmJkZmU0ZjlhZTlkODk5OTFjNTljY2M2OTA1MmU1IiwiVm9sdW1lcyI6bnVsbCwiV29ya2luZ0RpciI6IiIsIkVudHJ5cG9pbnQiOm51bGwsIk9uQnVpbGQiOm51bGwsIkxhYmVscyI6bnVsbH0sImNvbnRhaW5lcl9jb25maWciOnsiSG9zdG5hbWUiOiIiLCJEb21haW5uYW1lIjoiIiwiVXNlciI6IiIsIkF0dGFjaFN0ZGluIjpmYWxzZSwiQXR0YWNoU3Rkb3V0IjpmYWxzZSwiQXR0YWNoU3RkZXJyIjpmYWxzZSwiVHR5IjpmYWxzZSwiT3BlblN0ZGluIjpmYWxzZSwiU3RkaW5PbmNlIjpmYWxzZSwiRW52IjpbIlBBVEg9L3Vzci9sb2NhbC9zYmluOi91c3IvbG9jYWwvYmluOi91c3Ivc2JpbjovdXNyL2Jpbjovc2JpbjovYmluIl0sIkNtZCI6WyIvYmluL3NoIiwiLWMiLCIjKG5vcCkgQUREIGZpbGU6ZGYzYjc0NGY1NGE5YjE2YjliOWFlZDQwZTNlOThkOWNhMmI0OWY1YTc3ZDlmYThhOTc2OTBkN2JhZjU4ODgyMCBpbiAvc29tZWZpbGUtMi50eHQgIl0sIkltYWdlIjoic2hhMjU2OmRhYzIxNTAzOGMwNTBlMzU3MzA1NWVmZThhOTAzY2QxZjliYmRmZTRmOWFlOWQ4OTk5MWM1OWNjYzY5MDUyZTUiLCJWb2x1bWVzIjpudWxsLCJXb3JraW5nRGlyIjoiIiwiRW50cnlwb2ludCI6bnVsbCwiT25CdWlsZCI6bnVsbCwiTGFiZWxzIjpudWxsfSwiY3JlYXRlZCI6IjIwMjEtMDMtMjNUMTg6MTU6NTguODcyMjg5OFoiLCJkb2NrZXJfdmVyc2lvbiI6IjIwLjEwLjIiLCJoaXN0b3J5IjpbeyJjcmVhdGVkIjoiMjAyMS0wMy0yM1QxODoxNTo1OC42MTc3OTU2WiIsImNyZWF0ZWRfYnkiOiIvYmluL3NoIC1jICMobm9wKSBBREQgZmlsZTphYzMyZGEyM2Q1MWU4MDFmMDJmOTI0MTIzZWQzMDk5MGViM2YwZmVjMWI5ZWQ0ZjBiMDZjMjRlODhiOWMzNjk1IGluIC9zb21lZmlsZS0xLnR4dCAifSx7ImNyZWF0ZWQiOiIyMDIxLTAzLTIzVDE4OjE1OjU4Ljg3MjI4OThaIiwiY3JlYXRlZF9ieSI6Ii9iaW4vc2ggLWMgIyhub3ApIEFERCBmaWxlOmRmM2I3NDRmNTRhOWIxNmI5YjlhZWQ0MGUzZTk4ZDljYTJiNDlmNWE3N2Q5ZmE4YTk3NjkwZDdiYWY1ODg4MjAgaW4gL3NvbWVmaWxlLTIudHh0ICJ9XSwib3MiOiJsaW51eCIsInJvb3RmcyI6eyJ0eXBlIjoibGF5ZXJzIiwiZGlmZl9pZHMiOlsic2hhMjU2OjNkZTE2YzViODY1OWEyZThkODg4YjhkZWQ4NDI3YmU3YTU2ODZhM2M4YzRlNGRkMzBkZTIwZjM2MjgyNzI4NWIiLCJzaGEyNTY6MzY2YTNmNTY1M2UzNDY3M2I4NzU4OTFiMDIxNjQ3NDQwZDAxMjdjMmVmMDQxZTNiMWEyMmRhMmE3ZDRmMzcwMyJdfX0=", + "scope": "Squashed" + } + }, + "distro": { + "name": "", + "version": "", + "idLike": "" + }, + "descriptor": { + "name": "syft", + "version": "[not provided]" + }, + "schema": { + "version": "1.0.4", + "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.4.json" + } +} diff --git a/syft/presenter/table/test-fixtures/snapshot/TestTablePresenter.golden b/internal/presenter/packages/test-fixtures/snapshot/TestTablePresenter.golden similarity index 100% rename from syft/presenter/table/test-fixtures/snapshot/TestTablePresenter.golden rename to internal/presenter/packages/test-fixtures/snapshot/TestTablePresenter.golden diff --git a/syft/presenter/text/test-fixtures/snapshot/TestTextDirPresenter.golden b/internal/presenter/packages/test-fixtures/snapshot/TestTextDirPresenter.golden similarity index 100% rename from syft/presenter/text/test-fixtures/snapshot/TestTextDirPresenter.golden rename to internal/presenter/packages/test-fixtures/snapshot/TestTextDirPresenter.golden diff --git a/syft/presenter/text/test-fixtures/snapshot/TestTextImgPresenter.golden b/internal/presenter/packages/test-fixtures/snapshot/TestTextImgPresenter.golden similarity index 100% rename from syft/presenter/text/test-fixtures/snapshot/TestTextImgPresenter.golden rename to internal/presenter/packages/test-fixtures/snapshot/TestTextImgPresenter.golden diff --git a/syft/presenter/text/test-fixtures/snapshot/TestTextPresenter.golden b/internal/presenter/packages/test-fixtures/snapshot/TestTextPresenter.golden similarity index 100% rename from syft/presenter/text/test-fixtures/snapshot/TestTextPresenter.golden rename to internal/presenter/packages/test-fixtures/snapshot/TestTextPresenter.golden diff --git a/syft/presenter/json/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden b/internal/presenter/packages/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden similarity index 52% rename from syft/presenter/json/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden rename to internal/presenter/packages/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden index 6c23e82fd..10739912e 100644 Binary files a/syft/presenter/json/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden and b/internal/presenter/packages/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden differ diff --git a/internal/presenter/packages/test-fixtures/snapshot/stereoscope-fixture-packages-image-simple.golden b/internal/presenter/packages/test-fixtures/snapshot/stereoscope-fixture-packages-image-simple.golden new file mode 100644 index 000000000..b7d3fed18 Binary files /dev/null and b/internal/presenter/packages/test-fixtures/snapshot/stereoscope-fixture-packages-image-simple.golden differ diff --git a/syft/presenter/text/presenter.go b/internal/presenter/packages/text_presenter.go similarity index 77% rename from syft/presenter/text/presenter.go rename to internal/presenter/packages/text_presenter.go index 8e86fea47..44127da64 100644 --- a/syft/presenter/text/presenter.go +++ b/internal/presenter/packages/text_presenter.go @@ -1,4 +1,4 @@ -package text +package packages import ( "fmt" @@ -10,22 +10,22 @@ import ( "github.com/anchore/syft/syft/source" ) -// Presenter is a human-friendly text presenter to represent package and source data. -type Presenter struct { +// TextPresenter is a human-friendly text presenter to represent package and source data. +type TextPresenter struct { catalog *pkg.Catalog srcMetadata source.Metadata } -// NewPresenter creates a new presenter for the given set of catalog and image data. -func NewPresenter(catalog *pkg.Catalog, srcMetadata source.Metadata) *Presenter { - return &Presenter{ +// NewTextPresenter creates a new presenter for the given set of catalog and image data. +func NewTextPresenter(catalog *pkg.Catalog, srcMetadata source.Metadata) *TextPresenter { + return &TextPresenter{ catalog: catalog, srcMetadata: srcMetadata, } } // Present is a method that is in charge of writing to an output buffer -func (pres *Presenter) Present(output io.Writer) error { +func (pres *TextPresenter) Present(output io.Writer) error { // init the tabular writer w := new(tabwriter.Writer) w.Init(output, 0, 8, 0, '\t', tabwriter.AlignRight) diff --git a/syft/presenter/text/presenter_test.go b/internal/presenter/packages/text_presenter_test.go similarity index 86% rename from syft/presenter/text/presenter_test.go rename to internal/presenter/packages/text_presenter_test.go index ccac4b3d9..3633149cb 100644 --- a/syft/presenter/text/presenter_test.go +++ b/internal/presenter/packages/text_presenter_test.go @@ -1,4 +1,4 @@ -package text +package packages import ( "bytes" @@ -14,7 +14,7 @@ import ( "github.com/sergi/go-diff/diffmatchpatch" ) -var update = flag.Bool("update", false, "update the *.golden files for text presenters") +var updateTextPresenterGoldenFiles = flag.Bool("update-text", false, "update the *.golden files for text presenters") func TestTextDirPresenter(t *testing.T) { var buffer bytes.Buffer @@ -37,7 +37,7 @@ func TestTextDirPresenter(t *testing.T) { if err != nil { t.Fatalf("unable to create source: %+v", err) } - pres := NewPresenter(catalog, s.Metadata) + pres := NewTextPresenter(catalog, s.Metadata) // run presenter err = pres.Present(&buffer) @@ -46,7 +46,7 @@ func TestTextDirPresenter(t *testing.T) { } actual := buffer.Bytes() - if *update { + if *updateTextPresenterGoldenFiles { testutils.UpdateGoldenFileContents(t, actual) } @@ -69,8 +69,7 @@ func TestTextImgPresenter(t *testing.T) { var buffer bytes.Buffer catalog := pkg.NewCatalog() - img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-simple") - defer cleanup() + img := imagetest.GetFixtureImage(t, "docker-archive", "image-simple") _, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks) _, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks) @@ -102,18 +101,18 @@ func TestTextImgPresenter(t *testing.T) { l.Metadata.Digest = "sha256:ad8ecdc058976c07e7e347cb89fa9ad86a294b5ceaae6d09713fb035f84115abf3c4a2388a4af3aa60f13b94f4c6846930bdf53" } - s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input") + s, err := source.NewFromImage(img, "user-image-input") if err != nil { t.Fatal(err) } - pres := NewPresenter(catalog, s.Metadata) + pres := NewTextPresenter(catalog, s.Metadata) // run presenter err = pres.Present(&buffer) if err != nil { t.Fatal(err) } actual := buffer.Bytes() - if *update { + if *updateTextPresenterGoldenFiles { testutils.UpdateGoldenFileContents(t, actual) } diff --git a/internal/presenter/poweruser/json_document.go b/internal/presenter/poweruser/json_document.go new file mode 100644 index 000000000..856090a80 --- /dev/null +++ b/internal/presenter/poweruser/json_document.go @@ -0,0 +1,32 @@ +package poweruser + +import ( + "github.com/anchore/syft/internal/presenter/packages" +) + +type JSONDocument struct { + // note: poweruser.JSONDocument is meant to always be a superset of packages.JSONDocument, any additional fields + // here should be optional by supplying "omitempty" on these fields hint to the jsonschema generator to not + // require these fields. As an accepted rule in this repo all collections should still be initialized in the + // context of being used in a JSON document. + FileMetadata []JSONFileMetadata `json:"fileMetadata,omitempty"` + packages.JSONDocument +} + +// NewJSONDocument creates and populates a new JSON document struct from the given cataloging results. +func NewJSONDocument(config JSONDocumentConfig) (JSONDocument, error) { + pkgsDoc, err := packages.NewJSONDocument(config.PackageCatalog, config.SourceMetadata, config.Distro, config.ApplicationConfig.Package.Cataloger.ScopeOpt, config.ApplicationConfig) + if err != nil { + return JSONDocument{}, err + } + + fileMetadata, err := NewJSONFileMetadata(config.FileMetadata, config.FileDigests) + if err != nil { + return JSONDocument{}, err + } + + return JSONDocument{ + FileMetadata: fileMetadata, + JSONDocument: pkgsDoc, + }, nil +} diff --git a/internal/presenter/poweruser/json_document_config.go b/internal/presenter/poweruser/json_document_config.go new file mode 100644 index 000000000..fc2f053da --- /dev/null +++ b/internal/presenter/poweruser/json_document_config.go @@ -0,0 +1,18 @@ +package poweruser + +import ( + "github.com/anchore/syft/internal/config" + "github.com/anchore/syft/syft/distro" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +type JSONDocumentConfig struct { + ApplicationConfig config.Application + PackageCatalog *pkg.Catalog + FileMetadata map[source.Location]source.FileMetadata + FileDigests map[source.Location][]file.Digest + Distro *distro.Distro + SourceMetadata source.Metadata +} diff --git a/internal/presenter/poweruser/json_file_metadata.go b/internal/presenter/poweruser/json_file_metadata.go new file mode 100644 index 000000000..1b0a8d08e --- /dev/null +++ b/internal/presenter/poweruser/json_file_metadata.go @@ -0,0 +1,59 @@ +package poweruser + +import ( + "fmt" + "sort" + "strconv" + + "github.com/anchore/syft/syft/file" + + "github.com/anchore/syft/syft/source" +) + +type JSONFileMetadata struct { + Location source.Location `json:"location"` + Metadata JSONFileMetadataEntry `json:"metadata"` +} + +type JSONFileMetadataEntry struct { + Mode int `json:"mode"` + Type source.FileType `json:"type"` + UserID int `json:"userID"` + GroupID int `json:"groupID"` + Digests []file.Digest `json:"digests"` +} + +func NewJSONFileMetadata(data map[source.Location]source.FileMetadata, digests map[source.Location][]file.Digest) ([]JSONFileMetadata, error) { + results := make([]JSONFileMetadata, 0) + for location, metadata := range data { + mode, err := strconv.Atoi(fmt.Sprintf("%o", metadata.Mode)) + if err != nil { + return nil, fmt.Errorf("invalid mode found in file catalog @ location=%+v mode=%q: %w", location, metadata.Mode, err) + } + + digestResults := make([]file.Digest, 0) + if digestsForLocation, exists := digests[location]; exists { + digestResults = digestsForLocation + } + + results = append(results, JSONFileMetadata{ + Location: location, + Metadata: JSONFileMetadataEntry{ + Mode: mode, + Type: metadata.Type, + UserID: metadata.UserID, + GroupID: metadata.GroupID, + Digests: digestResults, + }, + }) + } + + // sort by real path then virtual path to ensure the result is stable across multiple runs + sort.SliceStable(results, func(i, j int) bool { + if results[i].Location.RealPath != results[j].Location.RealPath { + return results[i].Location.VirtualPath < results[j].Location.VirtualPath + } + return false + }) + return results, nil +} diff --git a/internal/presenter/poweruser/json_presenter.go b/internal/presenter/poweruser/json_presenter.go new file mode 100644 index 000000000..9432693f3 --- /dev/null +++ b/internal/presenter/poweruser/json_presenter.go @@ -0,0 +1,32 @@ +package poweruser + +import ( + "encoding/json" + "io" +) + +// JSONPresenter is a JSON presentation object for the syft results +type JSONPresenter struct { + config JSONDocumentConfig +} + +// NewJSONPresenter creates a new JSON presenter object for the given cataloging results. +func NewJSONPresenter(config JSONDocumentConfig) *JSONPresenter { + return &JSONPresenter{ + config: config, + } +} + +// Present the PackageCatalog results to the given writer. +func (p *JSONPresenter) Present(output io.Writer) error { + doc, err := NewJSONDocument(p.config) + if err != nil { + return err + } + + enc := json.NewEncoder(output) + // prevent > and < from being escaped in the payload + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + return enc.Encode(&doc) +} diff --git a/internal/presenter/presenter.go b/internal/presenter/presenter.go new file mode 100644 index 000000000..823a2e38d --- /dev/null +++ b/internal/presenter/presenter.go @@ -0,0 +1,9 @@ +package presenter + +import "io" + +// Presenter defines the expected behavior for an object responsible for displaying arbitrary input and processed data +// to a given io.Writer. +type Presenter interface { + Present(io.Writer) error +} diff --git a/internal/ui/common/event_handlers.go b/internal/ui/common/event_handlers.go index 756f74012..b601e6ac7 100644 --- a/internal/ui/common/event_handlers.go +++ b/internal/ui/common/event_handlers.go @@ -8,11 +8,11 @@ import ( "github.com/wagoodman/go-partybus" ) -// CatalogerFinishedHandler is a UI function for processing the CatalogerFinished bus event, displaying the catalog +// CatalogerPresenterReady is a UI function for processing the CatalogerFinished bus event, displaying the catalog // via the given presenter to stdout. -func CatalogerFinishedHandler(event partybus.Event) error { +func CatalogerPresenterReady(event partybus.Event) error { // show the report to stdout - pres, err := syftEventParsers.ParseCatalogerFinished(event) + pres, err := syftEventParsers.ParsePresenterReady(event) if err != nil { return fmt.Errorf("bad CatalogerFinished event: %w", err) } diff --git a/internal/ui/etui/ephemeral_tui.go b/internal/ui/etui/ephemeral_tui.go index 8aa9dec81..fa5a0a61b 100644 --- a/internal/ui/etui/ephemeral_tui.go +++ b/internal/ui/etui/ephemeral_tui.go @@ -129,7 +129,7 @@ eventLoop: log.Errorf("unable to show %s event: %+v", e.Type, err) } - case e.Type == syftEvent.CatalogerFinished: + case e.Type == syftEvent.PresenterReady: // we may have other background processes still displaying progress, wait for them to // finish before discontinuing dynamic content and showing the final report wg.Wait() @@ -146,7 +146,7 @@ eventLoop: fmt.Fprint(output, logBuffer.String()) } - if err := common.CatalogerFinishedHandler(e); err != nil { + if err := common.CatalogerPresenterReady(e); err != nil { log.Errorf("unable to show %s event: %+v", e.Type, err) } diff --git a/internal/ui/logger_output.go b/internal/ui/logger_output.go index 511300edf..d402958fe 100644 --- a/internal/ui/logger_output.go +++ b/internal/ui/logger_output.go @@ -24,8 +24,8 @@ eventLoop: } // ignore all events except for the final event - if e.Type == syftEvent.CatalogerFinished { - err := common.CatalogerFinishedHandler(e) + if e.Type == syftEvent.PresenterReady { + err := common.CatalogerPresenterReady(e) if err != nil { log.Errorf("unable to show catalog image finished event: %+v", err) } diff --git a/internal/version/build.go b/internal/version/build.go index 3d6702c0c..1313fa9a3 100644 --- a/internal/version/build.go +++ b/internal/version/build.go @@ -6,6 +6,7 @@ package version import ( "fmt" "runtime" + "strings" ) const valueNotProvided = "[not provided]" @@ -28,6 +29,13 @@ type Version struct { Platform string `json:"platform"` // GOOS and GOARCH at build-time } +func (v Version) IsProductionBuild() bool { + if strings.Contains(v.Version, "SNAPSHOT") || strings.Contains(v.Version, valueNotProvided) { + return false + } + return true +} + // FromBuild provides all version details func FromBuild() Version { return Version{ diff --git a/internal/version/update.go b/internal/version/update.go index 3aa13ef3a..7fd791e1c 100644 --- a/internal/version/update.go +++ b/internal/version/update.go @@ -20,13 +20,13 @@ var latestAppVersionURL = struct { // IsUpdateAvailable indicates if there is a newer application version available, and if so, what the new version is. func IsUpdateAvailable() (bool, string, error) { - currentVersionStr := FromBuild().Version - currentVersion, err := hashiVersion.NewVersion(currentVersionStr) + currentBuildInfo := FromBuild() + if !currentBuildInfo.IsProductionBuild() { + // don't allow for non-production builds to check for a version. + return false, "", nil + } + currentVersion, err := hashiVersion.NewVersion(currentBuildInfo.Version) if err != nil { - if currentVersionStr == valueNotProvided { - // this is the default build arg and should be ignored (this is not an error case) - return false, "", nil - } return false, "", fmt.Errorf("failed to parse current application version: %w", err) } diff --git a/internal/version/update_test.go b/internal/version/update_test.go index 10c34b738..c79750540 100644 --- a/internal/version/update_test.go +++ b/internal/version/update_test.go @@ -81,6 +81,15 @@ func TestIsUpdateAvailable(t *testing.T) { newVersion: "", err: false, }, + { + name: "SnapshotBuildVersion", + buildVersion: "2.0.0-SHAPSHOT-a78bf9c", + latestVersion: "1.0.0", + code: 200, + isAvailable: false, + newVersion: "", + err: false, + }, { name: "BadUpdateValidVersion", buildVersion: "1.0.0", diff --git a/schema/json/README.md b/schema/json/README.md index 957bb2bec..e9fe920db 100644 --- a/schema/json/README.md +++ b/schema/json/README.md @@ -1,10 +1,10 @@ # JSON Schema -This is the JSON schema for output from the JSON presenter (`syft -o json`). The required inputs for defining the JSON schema are as follows: +This is the JSON schema for output from the JSON presenters (`syft packages -o json` and `syft power-user `). The required inputs for defining the JSON schema are as follows: - the value of `internal.JSONSchemaVersion` that governs the schema filename -- the `Document` struct definition within `syft/presenters/json/document.go` that governs the overall document shape -- the `metadataContainer` struct definition within `schema/json/generate.go` that governs the allowable shapes of `pkg.Package.Metadata` +- the `Document` struct definition within `internal/presenters/poweruser/json_document.go` that governs the overall document shape +- the `artifactMetadataContainer` struct definition within `schema/json/generate.go` that governs the allowable shapes of `pkg.Package.Metadata` With regard to testing the JSON schema, integration test cases provided by the developer are used as examples to validate that JSON output from Syft is always valid relative to the `schema/json/schema-$VERSION.json` file. @@ -26,7 +26,7 @@ When adding a new `pkg.*Metadata` that is assigned to the `pkg.Package.Metadata` are done: - a new integration test case is added to `test/integration/pkg_cases_test.go` that exercises the new package type with the new metadata -- the new metadata struct is added to the `metadataContainer` struct within `schema/json/generate.go` +- the new metadata struct is added to the `artifactMetadataContainer` struct within `schema/json/generate.go` ## Generating a New Schema diff --git a/schema/json/generate.go b/schema/json/generate.go index 4896e1924..dd65c45e6 100644 --- a/schema/json/generate.go +++ b/schema/json/generate.go @@ -6,13 +6,14 @@ import ( "fmt" "io/ioutil" "os" + "reflect" "sort" "strings" "github.com/alecthomas/jsonschema" "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/presenter/poweruser" "github.com/anchore/syft/syft/pkg" - jsonPresenter "github.com/anchore/syft/syft/presenter/json" ) /* @@ -25,7 +26,7 @@ can be extended to include specific package metadata struct shapes in the future // This should represent all possible metadatas represented in the pkg.Package.Metadata field (an interface{}). // When a new package metadata definition is created it will need to be manually added here. The variable name does // not matter as long as it is exported. -type metadataContainer struct { +type artifactMetadataContainer struct { Apk pkg.ApkMetadata Dpkg pkg.DpkgMetadata Gem pkg.GemMetadata @@ -36,10 +37,23 @@ type metadataContainer struct { Cargo pkg.CargoPackageMetadata } -// nolint:funlen func main() { - metadataSchema := jsonschema.Reflect(&metadataContainer{}) - documentSchema := jsonschema.Reflect(&jsonPresenter.Document{}) + write(encode(build())) +} + +func build() *jsonschema.Schema { + reflector := &jsonschema.Reflector{ + AllowAdditionalProperties: true, + TypeNamer: func(r reflect.Type) string { + name := r.Name() + if strings.HasPrefix(name, "JSON") { + name = strings.TrimPrefix(name, "JSON") + } + return name + }, + } + documentSchema := reflector.ReflectFromType(reflect.TypeOf(&poweruser.JSONDocument{})) + metadataSchema := reflector.ReflectFromType(reflect.TypeOf(&artifactMetadataContainer{})) // TODO: inject source definitions @@ -47,7 +61,7 @@ func main() { var metadataNames []string for name, definition := range metadataSchema.Definitions { - if name == "metadataContainer" { + if name == "artifactMetadataContainer" { // ignore the definition for the fake container continue } @@ -71,22 +85,30 @@ func main() { } // set the "anyOf" field for Package.Metadata to be a conjunction of several types - documentSchema.Definitions["Package"].Properties.Set("metadata", map[string][]map[string]string{ + documentSchema.Definitions["Document"].Properties.Set("artifacts.metadata", map[string][]map[string]string{ "anyOf": metadataTypes, }) - filename := fmt.Sprintf("schema-%s.json", internal.JSONSchemaVersion) + return documentSchema +} +func encode(schema *jsonschema.Schema) []byte { var newSchemaBuffer = new(bytes.Buffer) enc := json.NewEncoder(newSchemaBuffer) // prevent > and < from being escaped in the payload enc.SetEscapeHTML(false) enc.SetIndent("", " ") - err := enc.Encode(&documentSchema) + err := enc.Encode(&schema) if err != nil { panic(err) } + return newSchemaBuffer.Bytes() +} + +func write(schema []byte) { + filename := fmt.Sprintf("schema-%s.json", internal.JSONSchemaVersion) + if _, err := os.Stat(filename); !os.IsNotExist(err) { // check if the schema is the same... existingFh, err := os.Open(filename) @@ -99,7 +121,7 @@ func main() { panic(err) } - if bytes.Equal(existingSchemaBytes, newSchemaBuffer.Bytes()) { + if bytes.Equal(existingSchemaBytes, schema) { // the generated schema is the same, bail with no error :) fmt.Println("No change to the existing schema!") os.Exit(0) @@ -115,7 +137,7 @@ func main() { panic(err) } - _, err = fh.Write(newSchemaBuffer.Bytes()) + _, err = fh.Write(schema) if err != nil { panic(err) } diff --git a/schema/json/schema-1.0.3.json b/schema/json/schema-1.0.3.json index de3d7f933..108ba7479 100644 --- a/schema/json/schema-1.0.3.json +++ b/schema/json/schema-1.0.3.json @@ -749,4 +749,4 @@ "type": "object" } } -} +} \ No newline at end of file diff --git a/schema/json/schema-1.0.4.json b/schema/json/schema-1.0.4.json new file mode 100644 index 000000000..04618439c --- /dev/null +++ b/schema/json/schema-1.0.4.json @@ -0,0 +1,830 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Document", + "definitions": { + "ApkFileRecord": { + "required": [ + "path" + ], + "properties": { + "path": { + "type": "string" + }, + "ownerUid": { + "type": "string" + }, + "ownerGid": { + "type": "string" + }, + "permissions": { + "type": "string" + }, + "checksum": { + "type": "string" + } + }, + "additionalProperties": true, + "type": "object" + }, + "ApkMetadata": { + "required": [ + "package", + "originPackage", + "maintainer", + "version", + "license", + "architecture", + "url", + "description", + "size", + "installedSize", + "pullDependencies", + "pullChecksum", + "gitCommitOfApkPort", + "files" + ], + "properties": { + "package": { + "type": "string" + }, + "originPackage": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "version": { + "type": "string" + }, + "license": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "url": { + "type": "string" + }, + "description": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "installedSize": { + "type": "integer" + }, + "pullDependencies": { + "type": "string" + }, + "pullChecksum": { + "type": "string" + }, + "gitCommitOfApkPort": { + "type": "string" + }, + "files": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/ApkFileRecord" + }, + "type": "array" + } + }, + "additionalProperties": true, + "type": "object" + }, + "CargoPackageMetadata": { + "required": [ + "name", + "version", + "source", + "checksum", + "dependencies" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "source": { + "type": "string" + }, + "checksum": { + "type": "string" + }, + "dependencies": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": true, + "type": "object" + }, + "Descriptor": { + "required": [ + "name", + "version" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "configuration": { + "additionalProperties": true + } + }, + "additionalProperties": true, + "type": "object" + }, + "Digest": { + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "additionalProperties": true, + "type": "object" + }, + "Distribution": { + "required": [ + "name", + "version", + "idLike" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "idLike": { + "type": "string" + } + }, + "additionalProperties": true, + "type": "object" + }, + "Document": { + "required": [ + "artifacts", + "artifactRelationships", + "source", + "distro", + "descriptor", + "schema" + ], + "properties": { + "fileMetadata": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/FileMetadata" + }, + "type": "array" + }, + "artifacts": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Package" + }, + "type": "array" + }, + "artifactRelationships": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Relationship" + }, + "type": "array" + }, + "source": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Source" + }, + "distro": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Distribution" + }, + "descriptor": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Descriptor" + }, + "schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Schema" + }, + "artifacts.metadata": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#/definitions/ApkMetadata" + }, + { + "$ref": "#/definitions/CargoPackageMetadata" + }, + { + "$ref": "#/definitions/DpkgMetadata" + }, + { + "$ref": "#/definitions/GemMetadata" + }, + { + "$ref": "#/definitions/JavaMetadata" + }, + { + "$ref": "#/definitions/NpmPackageJSONMetadata" + }, + { + "$ref": "#/definitions/PythonPackageMetadata" + }, + { + "$ref": "#/definitions/RpmdbMetadata" + } + ] + } + }, + "additionalProperties": true, + "type": "object" + }, + "DpkgFileRecord": { + "required": [ + "path", + "md5" + ], + "properties": { + "path": { + "type": "string" + }, + "md5": { + "type": "string" + } + }, + "additionalProperties": true, + "type": "object" + }, + "DpkgMetadata": { + "required": [ + "package", + "source", + "version", + "sourceVersion", + "architecture", + "maintainer", + "installedSize", + "files" + ], + "properties": { + "package": { + "type": "string" + }, + "source": { + "type": "string" + }, + "version": { + "type": "string" + }, + "sourceVersion": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "installedSize": { + "type": "integer" + }, + "files": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/DpkgFileRecord" + }, + "type": "array" + } + }, + "additionalProperties": true, + "type": "object" + }, + "FileMetadata": { + "required": [ + "location", + "metadata" + ], + "properties": { + "location": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Location" + }, + "metadata": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/FileMetadataEntry" + } + }, + "additionalProperties": true, + "type": "object" + }, + "FileMetadataEntry": { + "required": [ + "mode", + "type", + "userID", + "groupID", + "digests" + ], + "properties": { + "mode": { + "type": "integer" + }, + "type": { + "type": "string" + }, + "userID": { + "type": "integer" + }, + "groupID": { + "type": "integer" + }, + "digests": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Digest" + }, + "type": "array" + } + }, + "additionalProperties": true, + "type": "object" + }, + "GemMetadata": { + "required": [ + "name", + "version" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "files": { + "items": { + "type": "string" + }, + "type": "array" + }, + "authors": { + "items": { + "type": "string" + }, + "type": "array" + }, + "licenses": { + "items": { + "type": "string" + }, + "type": "array" + }, + "homepage": { + "type": "string" + } + }, + "additionalProperties": true, + "type": "object" + }, + "JavaManifest": { + "properties": { + "main": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "namedSections": { + "patternProperties": { + ".*": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "additionalProperties": true, + "type": "object" + }, + "JavaMetadata": { + "required": [ + "virtualPath" + ], + "properties": { + "virtualPath": { + "type": "string" + }, + "manifest": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/JavaManifest" + }, + "pomProperties": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/PomProperties" + } + }, + "additionalProperties": true, + "type": "object" + }, + "Location": { + "required": [ + "path" + ], + "properties": { + "path": { + "type": "string" + }, + "layerID": { + "type": "string" + } + }, + "additionalProperties": true, + "type": "object" + }, + "NpmPackageJSONMetadata": { + "required": [ + "author", + "licenses", + "homepage", + "description", + "url" + ], + "properties": { + "files": { + "items": { + "type": "string" + }, + "type": "array" + }, + "author": { + "type": "string" + }, + "licenses": { + "items": { + "type": "string" + }, + "type": "array" + }, + "homepage": { + "type": "string" + }, + "description": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "additionalProperties": true, + "type": "object" + }, + "Package": { + "required": [ + "id", + "name", + "version", + "type", + "foundBy", + "locations", + "licenses", + "language", + "cpes", + "purl", + "metadataType", + "metadata" + ], + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "type": { + "type": "string" + }, + "foundBy": { + "type": "string" + }, + "locations": { + "items": { + "$ref": "#/definitions/Location" + }, + "type": "array" + }, + "licenses": { + "items": { + "type": "string" + }, + "type": "array" + }, + "language": { + "type": "string" + }, + "cpes": { + "items": { + "type": "string" + }, + "type": "array" + }, + "purl": { + "type": "string" + }, + "metadataType": { + "type": "string" + }, + "metadata": { + "additionalProperties": true + } + }, + "additionalProperties": true, + "type": "object" + }, + "PomProperties": { + "required": [ + "path", + "name", + "groupId", + "artifactId", + "version", + "extraFields" + ], + "properties": { + "path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "groupId": { + "type": "string" + }, + "artifactId": { + "type": "string" + }, + "version": { + "type": "string" + }, + "extraFields": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "additionalProperties": true, + "type": "object" + }, + "PythonFileDigest": { + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "additionalProperties": true, + "type": "object" + }, + "PythonFileRecord": { + "required": [ + "path" + ], + "properties": { + "path": { + "type": "string" + }, + "digest": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/PythonFileDigest" + }, + "size": { + "type": "string" + } + }, + "additionalProperties": true, + "type": "object" + }, + "PythonPackageMetadata": { + "required": [ + "name", + "version", + "license", + "author", + "authorEmail", + "platform", + "sitePackagesRootPath" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "license": { + "type": "string" + }, + "author": { + "type": "string" + }, + "authorEmail": { + "type": "string" + }, + "platform": { + "type": "string" + }, + "files": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/PythonFileRecord" + }, + "type": "array" + }, + "sitePackagesRootPath": { + "type": "string" + }, + "topLevelPackages": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": true, + "type": "object" + }, + "Relationship": { + "required": [ + "parent", + "child", + "type", + "metadata" + ], + "properties": { + "parent": { + "type": "string" + }, + "child": { + "type": "string" + }, + "type": { + "type": "string" + }, + "metadata": { + "additionalProperties": true + } + }, + "additionalProperties": true, + "type": "object" + }, + "RpmdbFileRecord": { + "required": [ + "path", + "mode", + "size", + "sha256" + ], + "properties": { + "path": { + "type": "string" + }, + "mode": { + "type": "integer" + }, + "size": { + "type": "integer" + }, + "sha256": { + "type": "string" + } + }, + "additionalProperties": true, + "type": "object" + }, + "RpmdbMetadata": { + "required": [ + "name", + "version", + "epoch", + "architecture", + "release", + "sourceRpm", + "size", + "license", + "vendor", + "files" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "epoch": { + "type": "integer" + }, + "architecture": { + "type": "string" + }, + "release": { + "type": "string" + }, + "sourceRpm": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "license": { + "type": "string" + }, + "vendor": { + "type": "string" + }, + "files": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/RpmdbFileRecord" + }, + "type": "array" + } + }, + "additionalProperties": true, + "type": "object" + }, + "Schema": { + "required": [ + "version", + "url" + ], + "properties": { + "version": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "additionalProperties": true, + "type": "object" + }, + "Source": { + "required": [ + "type", + "target" + ], + "properties": { + "type": { + "type": "string" + }, + "target": { + "additionalProperties": true + } + }, + "additionalProperties": true, + "type": "object" + } + } +} diff --git a/syft/cataloger/common/generic_cataloger_test.go b/syft/cataloger/common/generic_cataloger_test.go deleted file mode 100644 index e390525d1..000000000 --- a/syft/cataloger/common/generic_cataloger_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package common - -import ( - "fmt" - "io" - "io/ioutil" - "strings" - "testing" - - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/source" -) - -type testResolverMock struct { - contents map[source.Location]io.ReadCloser -} - -func newTestResolver() *testResolverMock { - return &testResolverMock{ - contents: make(map[source.Location]io.ReadCloser), - } -} - -func (r testResolverMock) HasPath(path string) bool { - panic("not implemented") -} - -func (r *testResolverMock) FileContentsByLocation(_ source.Location) (io.ReadCloser, error) { - return nil, fmt.Errorf("not implemented") -} - -func (r *testResolverMock) MultipleFileContentsByLocation([]source.Location) (map[source.Location]io.ReadCloser, error) { - return r.contents, nil -} - -func (r *testResolverMock) FilesByPath(paths ...string) ([]source.Location, error) { - results := make([]source.Location, len(paths)) - - for idx, p := range paths { - results[idx] = source.NewLocation(p) - r.contents[results[idx]] = ioutil.NopCloser(strings.NewReader(fmt.Sprintf("%s file contents!", p))) - } - - return results, nil -} - -func (r *testResolverMock) FilesByGlob(_ ...string) ([]source.Location, error) { - path := "/a-path.txt" - location := source.NewLocation(path) - r.contents[location] = ioutil.NopCloser(strings.NewReader(fmt.Sprintf("%s file contents!", path))) - return []source.Location{location}, nil -} - -func (r *testResolverMock) RelativeFileByPath(_ source.Location, _ string) *source.Location { - panic(fmt.Errorf("not implemented")) - return nil -} - -func parser(_ string, reader io.Reader) ([]pkg.Package, error) { - contents, err := ioutil.ReadAll(reader) - if err != nil { - panic(err) - } - return []pkg.Package{ - { - Name: string(contents), - }, - }, nil -} - -func TestGenericCataloger(t *testing.T) { - - globParsers := map[string]ParserFn{ - "**a-path.txt": parser, - } - pathParsers := map[string]ParserFn{ - "/another-path.txt": parser, - "/last/path.txt": parser, - } - upstream := "some-other-cataloger" - resolver := newTestResolver() - cataloger := NewGenericCataloger(pathParsers, globParsers, upstream) - - expectedSelection := []string{"/last/path.txt", "/another-path.txt", "/a-path.txt"} - expectedPkgs := make(map[string]pkg.Package) - for _, path := range expectedSelection { - expectedPkgs[path] = pkg.Package{ - FoundBy: upstream, - Name: fmt.Sprintf("%s file contents!", path), - } - } - - actualPkgs, err := cataloger.Catalog(resolver) - if err != nil { - t.Fatalf("cataloger catalog action failed: %+v", err) - } - - if len(actualPkgs) != len(expectedPkgs) { - t.Fatalf("unexpected packages len: %d", len(actualPkgs)) - } - - for _, p := range actualPkgs { - ref := p.Locations[0] - exP, ok := expectedPkgs[ref.RealPath] - if !ok { - t.Errorf("missing expected pkg: ref=%+v", ref) - continue - } - - if p.FoundBy != exP.FoundBy { - t.Errorf("bad upstream: %s", p.FoundBy) - } - - if exP.Name != p.Name { - t.Errorf("bad contents mapping: %+v", p.Locations) - } - } -} diff --git a/syft/cataloger/deb/cataloger.go b/syft/cataloger/deb/cataloger.go deleted file mode 100644 index 49694fb38..000000000 --- a/syft/cataloger/deb/cataloger.go +++ /dev/null @@ -1,190 +0,0 @@ -/* -Package dpkg provides a concrete Cataloger implementation for Debian package DB status files. -*/ -package deb - -import ( - "fmt" - "io" - "path" - "path/filepath" - - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/source" -) - -const ( - md5sumsExt = ".md5sums" - docsPath = "/usr/share/doc" -) - -type Cataloger struct{} - -// NewDpkgdbCataloger returns a new Deb package cataloger object. -func NewDpkgdbCataloger() *Cataloger { - return &Cataloger{} -} - -// Name returns a string that uniquely describes a cataloger -func (c *Cataloger) Name() string { - return "dpkgdb-cataloger" -} - -// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files. -// nolint:funlen -func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { - dbFileMatches, err := resolver.FilesByGlob(pkg.DpkgDbGlob) - if err != nil { - return nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err) - } - - var results []pkg.Package - var pkgs []pkg.Package - for _, dbLocation := range dbFileMatches { - dbContents, err := resolver.FileContentsByLocation(dbLocation) - if err != nil { - return nil, err - } - - pkgs, err = parseDpkgStatus(dbContents) - if err != nil { - return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.RealPath, err) - } - - md5ContentsByName, md5RefsByName, err := fetchMd5Contents(resolver, dbLocation, pkgs) - if err != nil { - return nil, fmt.Errorf("unable to find dpkg md5 contents: %w", err) - } - - copyrightContentsByName, copyrightLocationByName, err := fetchCopyrightContents(resolver, dbLocation, pkgs) - if err != nil { - return nil, fmt.Errorf("unable to find dpkg copyright contents: %w", err) - } - - for i := range pkgs { - p := &pkgs[i] - p.FoundBy = c.Name() - p.Locations = []source.Location{dbLocation} - - metadata := p.Metadata.(pkg.DpkgMetadata) - - if md5Reader, ok := md5ContentsByName[md5Key(*p)]; ok { - // attach the file list - metadata.Files = parseDpkgMD5Info(md5Reader) - - // keep a record of the file where this was discovered - if ref, ok := md5RefsByName[md5Key(*p)]; ok { - p.Locations = append(p.Locations, ref) - } - } else { - // ensure the file list is an empty collection (not nil) - metadata.Files = make([]pkg.DpkgFileRecord, 0) - } - - // persist alterations - p.Metadata = metadata - - copyrightReader, ok := copyrightContentsByName[p.Name] - if ok { - // attach the licenses - p.Licenses = parseLicensesFromCopyright(copyrightReader) - - // keep a record of the file where this was discovered - if ref, ok := copyrightLocationByName[p.Name]; ok { - p.Locations = append(p.Locations, ref) - } - } - } - - results = append(results, pkgs...) - } - return results, nil -} - -func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) { - // fetch all MD5 file contents. This approach is more efficient than fetching each MD5 file one at a time - - var md5FileMatches []source.Location - var nameByRef = make(map[source.Location]string) - parentPath := filepath.Dir(dbLocation.RealPath) - - for _, p := range pkgs { - // look for /var/lib/dpkg/info/NAME:ARCH.md5sums - name := md5Key(p) - md5SumLocation := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "info", name+md5sumsExt)) - - if md5SumLocation == nil { - // the most specific key did not work, fallback to just the name - // look for /var/lib/dpkg/info/NAME.md5sums - md5SumLocation = resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "info", p.Name+md5sumsExt)) - } - // we should have at least one reference - if md5SumLocation != nil { - md5FileMatches = append(md5FileMatches, *md5SumLocation) - nameByRef[*md5SumLocation] = name - } - } - - // fetch the md5 contents - md5ContentsByLocation, err := resolver.MultipleFileContentsByLocation(md5FileMatches) - if err != nil { - return nil, nil, err - } - - // organize content results and refs by a combination of name and architecture - var contentsByName = make(map[string]io.Reader) - var locationByName = make(map[string]source.Location) - for location, contents := range md5ContentsByLocation { - name := nameByRef[location] - contentsByName[name] = contents - locationByName[name] = location - } - - return contentsByName, locationByName, nil -} - -func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) { - // fetch all copyright file contents. This approach is more efficient than fetching each copyright file one at a time - - var copyrightFileMatches []source.Location - var nameByLocation = make(map[source.Location]string) - for _, p := range pkgs { - // look for /usr/share/docs/NAME/copyright files - name := p.Name - copyrightPath := path.Join(docsPath, name, "copyright") - copyrightLocation := resolver.RelativeFileByPath(dbLocation, copyrightPath) - - // we may not have a copyright file for each package, ignore missing files - if copyrightLocation != nil { - copyrightFileMatches = append(copyrightFileMatches, *copyrightLocation) - nameByLocation[*copyrightLocation] = name - } - } - - // fetch the copyright contents - copyrightContentsByLocation, err := resolver.MultipleFileContentsByLocation(copyrightFileMatches) - if err != nil { - return nil, nil, err - } - - // organize content results and refs by package name - var contentsByName = make(map[string]io.Reader) - var refsByName = make(map[string]source.Location) - for location, contents := range copyrightContentsByLocation { - name := nameByLocation[location] - contentsByName[name] = contents - refsByName[name] = location - } - - return contentsByName, refsByName, nil -} - -func md5Key(p pkg.Package) string { - metadata := p.Metadata.(pkg.DpkgMetadata) - - contentKey := p.Name - if metadata.Architecture != "" && metadata.Architecture != "all" { - contentKey = contentKey + ":" + metadata.Architecture - } - return contentKey -} diff --git a/syft/cataloger/python/package_entry.go b/syft/cataloger/python/package_entry.go deleted file mode 100644 index f27aabae8..000000000 --- a/syft/cataloger/python/package_entry.go +++ /dev/null @@ -1,49 +0,0 @@ -package python - -import ( - "path/filepath" - - "github.com/anchore/syft/syft/source" -) - -type packageEntry struct { - Metadata source.FileData - FileRecord *source.FileData - TopPackage *source.FileData -} - -// newPackageEntry returns a new packageEntry to be processed relative to what information is available in the given FileResolver. -func newPackageEntry(resolver source.FileResolver, metadataLocation source.Location) *packageEntry { - // we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory - // or for an image... for an image the METADATA file may be present within multiple layers, so it is important - // to reconcile the RECORD path to the same layer (or a lower layer). The same is true with the top_level.txt file. - - // lets find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure) - recordPath := filepath.Join(filepath.Dir(metadataLocation.RealPath), "RECORD") - recordLocation := resolver.RelativeFileByPath(metadataLocation, recordPath) - - // a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages - parentDir := filepath.Dir(metadataLocation.RealPath) - topLevelPath := filepath.Join(parentDir, "top_level.txt") - topLevelLocation := resolver.RelativeFileByPath(metadataLocation, topLevelPath) - - // build an entry that will later be populated with contents when the request is executed - entry := &packageEntry{ - Metadata: source.FileData{ - Location: metadataLocation, - }, - } - - if recordLocation != nil { - entry.FileRecord = &source.FileData{ - Location: *recordLocation, - } - } - - if topLevelLocation != nil { - entry.TopPackage = &source.FileData{ - Location: *topLevelLocation, - } - } - return entry -} diff --git a/syft/distro/identify.go b/syft/distro/identify.go index fc6fe52bc..72be74b97 100644 --- a/syft/distro/identify.go +++ b/syft/distro/identify.go @@ -36,7 +36,7 @@ var identityFiles = []parseEntry{ } // Identify parses distro-specific files to determine distro metadata like version and release. -func Identify(resolver source.Resolver) *Distro { +func Identify(resolver source.FileResolver) *Distro { var distro *Distro identifyLoop: diff --git a/syft/distro/identify_test.go b/syft/distro/identify_test.go index 3e73aa2c6..d0d4470ee 100644 --- a/syft/distro/identify_test.go +++ b/syft/distro/identify_test.go @@ -99,7 +99,12 @@ func TestIdentifyDistro(t *testing.T) { t.Fatalf("unable to produce a new source for testing: %s", test.fixture) } - d := Identify(s.Resolver) + resolver, err := s.FileResolver(source.SquashedScope) + if err != nil { + t.Fatalf("unable to get resolver: %+v", err) + } + + d := Identify(resolver) if d == nil { if test.Type == UnknownDistroType { return diff --git a/syft/event/event.go b/syft/event/event.go index a88d38422..8c6faefc1 100644 --- a/syft/event/event.go +++ b/syft/event/event.go @@ -10,11 +10,11 @@ const ( // AppUpdateAvailable is a partybus event that occurs when an application update is available AppUpdateAvailable partybus.EventType = "syft-app-update-available" - // CatalogerStarted is a partybus event that occurs when the package cataloging has begun - CatalogerStarted partybus.EventType = "syft-cataloger-started-event" + // PackageCatalogerStarted is a partybus event that occurs when the package cataloging has begun + PackageCatalogerStarted partybus.EventType = "syft-cataloger-started-event" - // CatalogerFinished is a partybus event that occurs when the package cataloging has completed - CatalogerFinished partybus.EventType = "syft-cataloger-finished-event" + // PresenterReady is a partybus event that occurs when an analysis result is ready for final presentation + PresenterReady partybus.EventType = "syft-presenter-ready-event" // ImportStarted is a partybus event that occurs when an SBOM upload process has begun ImportStarted partybus.EventType = "syft-import-started-event" diff --git a/syft/event/parsers/parsers.go b/syft/event/parsers/parsers.go index 1eddcf245..a5d02a2f0 100644 --- a/syft/event/parsers/parsers.go +++ b/syft/event/parsers/parsers.go @@ -6,11 +6,12 @@ package parsers import ( "fmt" + "github.com/anchore/syft/internal/presenter" + "github.com/wagoodman/go-progress" - "github.com/anchore/syft/syft/cataloger" "github.com/anchore/syft/syft/event" - "github.com/anchore/syft/syft/presenter" + "github.com/anchore/syft/syft/pkg/cataloger" "github.com/wagoodman/go-partybus" ) @@ -40,7 +41,7 @@ func checkEventType(actual, expected partybus.EventType) error { } func ParseCatalogerStarted(e partybus.Event) (*cataloger.Monitor, error) { - if err := checkEventType(e.Type, event.CatalogerStarted); err != nil { + if err := checkEventType(e.Type, event.PackageCatalogerStarted); err != nil { return nil, err } @@ -52,8 +53,8 @@ func ParseCatalogerStarted(e partybus.Event) (*cataloger.Monitor, error) { return &monitor, nil } -func ParseCatalogerFinished(e partybus.Event) (presenter.Presenter, error) { - if err := checkEventType(e.Type, event.CatalogerFinished); err != nil { +func ParsePresenterReady(e partybus.Event) (presenter.Presenter, error) { + if err := checkEventType(e.Type, event.PresenterReady); err != nil { return nil, err } diff --git a/syft/file/digest.go b/syft/file/digest.go new file mode 100644 index 000000000..87b53dbb8 --- /dev/null +++ b/syft/file/digest.go @@ -0,0 +1,6 @@ +package file + +type Digest struct { + Algorithm string `json:"algorithm"` + Value string `json:"value"` +} diff --git a/syft/file/digest_cataloger.go b/syft/file/digest_cataloger.go new file mode 100644 index 000000000..10a5bba42 --- /dev/null +++ b/syft/file/digest_cataloger.go @@ -0,0 +1,96 @@ +package file + +import ( + "crypto" + "fmt" + "hash" + "io" + "strings" + + "github.com/anchore/syft/syft/source" +) + +var supportedHashAlgorithms = make(map[string]crypto.Hash) + +type DigestsCataloger struct { + hashes []crypto.Hash +} + +func init() { + for _, h := range []crypto.Hash{ + crypto.MD5, + crypto.SHA1, + crypto.SHA256, + } { + supportedHashAlgorithms[cleanAlgorithmName(h.String())] = h + } +} + +func NewDigestsCataloger(hashAlgorithms []string) (*DigestsCataloger, error) { + var hashes []crypto.Hash + for _, hashStr := range hashAlgorithms { + name := cleanAlgorithmName(hashStr) + hashObj, ok := supportedHashAlgorithms[name] + if !ok { + return nil, fmt.Errorf("unsupported hash algorithm: %s", hashStr) + } + hashes = append(hashes, hashObj) + } + + return &DigestsCataloger{ + hashes: hashes, + }, nil +} + +func (i *DigestsCataloger) Catalog(resolver source.FileResolver) (map[source.Location][]Digest, error) { + results := make(map[source.Location][]Digest) + for location := range resolver.AllLocations() { + result, err := i.catalogLocation(resolver, location) + if err != nil { + return nil, err + } + results[location] = result + } + return results, nil +} + +func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) ([]Digest, error) { + contentReader, err := resolver.FileContentsByLocation(location) + if err != nil { + return nil, err + } + defer contentReader.Close() + + // create a set of hasher objects tied together with a single writer to feed content into + hashers := make([]hash.Hash, len(i.hashes)) + writers := make([]io.Writer, len(i.hashes)) + for idx, hashObj := range i.hashes { + hashers[idx] = hashObj.New() + writers[idx] = hashers[idx] + } + + size, err := io.Copy(io.MultiWriter(writers...), contentReader) + if err != nil { + return nil, fmt.Errorf("unable to observe contents of %+v: %+v", location.RealPath, err) + } + + result := make([]Digest, len(i.hashes)) + if size > 0 { + // only capture digests when there is content. It is important to do this based on SIZE and not + // FILE TYPE. The reasoning is that it is possible for a tar to be crafted with a header-only + // file type but a body is still allowed. + for idx, hasher := range hashers { + result[idx] = Digest{ + Algorithm: cleanAlgorithmName(i.hashes[idx].String()), + Value: fmt.Sprintf("%+x", hasher.Sum(nil)), + } + } + } + + return result, nil +} + +func cleanAlgorithmName(name string) string { + lower := strings.ToLower(name) + return strings.Replace(lower, "-", "", -1) +} diff --git a/syft/file/digest_cataloger_test.go b/syft/file/digest_cataloger_test.go new file mode 100644 index 000000000..ad21f9184 --- /dev/null +++ b/syft/file/digest_cataloger_test.go @@ -0,0 +1,98 @@ +package file + +import ( + "crypto" + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/anchore/syft/syft/source" +) + +func testDigests(t testing.TB, files []string, hashes ...crypto.Hash) map[source.Location][]Digest { + digests := make(map[source.Location][]Digest) + + for _, f := range files { + fh, err := os.Open(f) + if err != nil { + t.Fatalf("could not open %q : %+v", f, err) + } + b, err := ioutil.ReadAll(fh) + if err != nil { + t.Fatalf("could not read %q : %+v", f, err) + } + + for _, hash := range hashes { + h := hash.New() + h.Write(b) + digests[source.NewLocation(f)] = append(digests[source.NewLocation(f)], Digest{ + Algorithm: cleanAlgorithmName(hash.String()), + Value: fmt.Sprintf("%x", h.Sum(nil)), + }) + } + } + + return digests +} + +func TestDigestsCataloger(t *testing.T) { + files := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"} + + tests := []struct { + name string + algorithms []string + expected map[source.Location][]Digest + constructorErr bool + catalogErr bool + }{ + { + name: "bad algorithm", + algorithms: []string{"sha-nothing"}, + constructorErr: true, + }, + { + name: "unsupported algorithm", + algorithms: []string{"sha512"}, + constructorErr: true, + }, + { + name: "md5-sha1-sha256", + algorithms: []string{"md5"}, + expected: testDigests(t, files, crypto.MD5), + }, + { + name: "md5-sha1-sha256", + algorithms: []string{"md5", "sha1", "sha256"}, + expected: testDigests(t, files, crypto.MD5, crypto.SHA1, crypto.SHA256), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + c, err := NewDigestsCataloger(test.algorithms) + if err != nil && !test.constructorErr { + t.Fatalf("could not create cataloger (but should have been able to): %+v", err) + } else if err == nil && test.constructorErr { + t.Fatalf("expected constructor error but did not get one") + } else if test.constructorErr && err != nil { + return + } + + resolver := source.NewMockResolverForPaths(files...) + actual, err := c.Catalog(resolver) + if err != nil && !test.catalogErr { + t.Fatalf("could not catalog (but should have been able to): %+v", err) + } else if err == nil && test.catalogErr { + t.Fatalf("expected catalog error but did not get one") + } else if test.catalogErr && err != nil { + return + } + + assert.Equal(t, actual, test.expected, "mismatched digests") + + }) + } +} diff --git a/syft/file/metadata_cataloger.go b/syft/file/metadata_cataloger.go new file mode 100644 index 000000000..7ffd41168 --- /dev/null +++ b/syft/file/metadata_cataloger.go @@ -0,0 +1,25 @@ +package file + +import ( + "github.com/anchore/syft/syft/source" +) + +type MetadataCataloger struct { +} + +func NewMetadataCataloger() *MetadataCataloger { + return &MetadataCataloger{} +} + +func (i *MetadataCataloger) Catalog(resolver source.FileResolver) (map[source.Location]source.FileMetadata, error) { + results := make(map[source.Location]source.FileMetadata) + for location := range resolver.AllLocations() { + metadata, err := resolver.FileMetadataByLocation(location) + if err != nil { + return nil, err + } + + results[location] = metadata + } + return results, nil +} diff --git a/syft/file/metadata_cataloger_test.go b/syft/file/metadata_cataloger_test.go new file mode 100644 index 000000000..a6971d6b3 --- /dev/null +++ b/syft/file/metadata_cataloger_test.go @@ -0,0 +1,134 @@ +package file + +import ( + "flag" + "os" + "testing" + + "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/anchore/syft/syft/source" + "github.com/stretchr/testify/assert" +) + +var updateImageGoldenFiles = flag.Bool("update-image", false, "update the golden fixture images used for testing") + +func TestFileMetadataCataloger(t *testing.T) { + testImage := "image-file-type-mix" + + if *updateImageGoldenFiles { + imagetest.UpdateGoldenFixtureImage(t, testImage) + } + + img := imagetest.GetGoldenFixtureImage(t, testImage) + + c := NewMetadataCataloger() + + src, err := source.NewFromImage(img, "---") + if err != nil { + t.Fatalf("could not create source: %+v", err) + } + + resolver, err := src.FileResolver(source.SquashedScope) + if err != nil { + t.Fatalf("could not create resolver: %+v", err) + } + + actual, err := c.Catalog(resolver) + if err != nil { + t.Fatalf("could not catalog: %+v", err) + } + + tests := []struct { + path string + exists bool + expected source.FileMetadata + err bool + }{ + { + path: "/file-1.txt", + exists: true, + expected: source.FileMetadata{ + Mode: 0644, + Type: "regularFile", + UserID: 1, + GroupID: 2, + }, + }, + { + path: "/hardlink-1", + exists: true, + expected: source.FileMetadata{ + Mode: 0644, + Type: "hardLink", + UserID: 1, + GroupID: 2, + }, + }, + { + path: "/symlink-1", + exists: true, + expected: source.FileMetadata{ + Mode: 0777 | os.ModeSymlink, + Type: "symbolicLink", + UserID: 0, + GroupID: 0, + }, + }, + { + path: "/char-device-1", + exists: true, + expected: source.FileMetadata{ + Mode: 0644 | os.ModeDevice | os.ModeCharDevice, + Type: "characterDevice", + UserID: 0, + GroupID: 0, + }, + }, + { + path: "/block-device-1", + exists: true, + expected: source.FileMetadata{ + Mode: 0644 | os.ModeDevice, + Type: "blockDevice", + UserID: 0, + GroupID: 0, + }, + }, + { + path: "/fifo-1", + exists: true, + expected: source.FileMetadata{ + Mode: 0644 | os.ModeNamedPipe, + Type: "fifoNode", + UserID: 0, + GroupID: 0, + }, + }, + { + path: "/bin", + exists: true, + expected: source.FileMetadata{ + Mode: 0755 | os.ModeDir, + Type: "directory", + UserID: 0, + GroupID: 0, + }, + }, + } + + for _, test := range tests { + t.Run(test.path, func(t *testing.T) { + _, ref, err := img.SquashedTree().File(file.Path(test.path)) + if err != nil { + t.Fatalf("unable to get file: %+v", err) + } + + l := source.NewLocationFromImage(test.path, *ref, img) + + assert.Equal(t, actual[l], test.expected, "mismatched metadata") + + }) + } + +} diff --git a/syft/file/test-fixtures/a-path.txt b/syft/file/test-fixtures/a-path.txt new file mode 100644 index 000000000..67e954034 --- /dev/null +++ b/syft/file/test-fixtures/a-path.txt @@ -0,0 +1 @@ +test-fixtures/a-path.txt file contents! \ No newline at end of file diff --git a/syft/file/test-fixtures/another-path.txt b/syft/file/test-fixtures/another-path.txt new file mode 100644 index 000000000..0d654f8fe --- /dev/null +++ b/syft/file/test-fixtures/another-path.txt @@ -0,0 +1 @@ +test-fixtures/another-path.txt file contents! \ No newline at end of file diff --git a/syft/file/test-fixtures/image-file-type-mix/Dockerfile b/syft/file/test-fixtures/image-file-type-mix/Dockerfile new file mode 100644 index 000000000..d8f728587 --- /dev/null +++ b/syft/file/test-fixtures/image-file-type-mix/Dockerfile @@ -0,0 +1,11 @@ +FROM busybox:latest + +ADD file-1.txt . +RUN chmod 644 file-1.txt +RUN chown 1:2 file-1.txt +RUN ln -s file-1.txt symlink-1 +# note: hard links may behave inconsistently, this should be a golden image +RUN ln file-1.txt hardlink-1 +RUN mknod char-device-1 c 89 1 +RUN mknod block-device-1 b 0 1 +RUN mknod fifo-1 p diff --git a/syft/file/test-fixtures/image-file-type-mix/file-1.txt b/syft/file/test-fixtures/image-file-type-mix/file-1.txt new file mode 100644 index 000000000..d86db8155 --- /dev/null +++ b/syft/file/test-fixtures/image-file-type-mix/file-1.txt @@ -0,0 +1 @@ +file 1! \ No newline at end of file diff --git a/syft/file/test-fixtures/last/path.txt b/syft/file/test-fixtures/last/path.txt new file mode 100644 index 000000000..3d4a165ab --- /dev/null +++ b/syft/file/test-fixtures/last/path.txt @@ -0,0 +1 @@ +test-fixtures/last/path.txt file contents! \ No newline at end of file diff --git a/syft/file/test-fixtures/snapshot/stereoscope-fixture-image-file-type-mix.golden b/syft/file/test-fixtures/snapshot/stereoscope-fixture-image-file-type-mix.golden new file mode 100644 index 000000000..8acd4584c Binary files /dev/null and b/syft/file/test-fixtures/snapshot/stereoscope-fixture-image-file-type-mix.golden differ diff --git a/syft/lib.go b/syft/lib.go index 7f63e2210..9b9bfd965 100644 --- a/syft/lib.go +++ b/syft/lib.go @@ -17,32 +17,29 @@ Similar to the cataloging process, Linux distribution identification is also per package syft import ( - "encoding/json" "fmt" - "io" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/cataloger" "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/logger" "github.com/anchore/syft/syft/pkg" - jsonPresenter "github.com/anchore/syft/syft/presenter/json" + "github.com/anchore/syft/syft/pkg/cataloger" "github.com/anchore/syft/syft/source" "github.com/wagoodman/go-partybus" ) -// Catalog the given image from a particular perspective (e.g. squashed source, all-layers source). Returns the discovered -// set of packages, the identified Linux distribution, and the source object used to wrap the data source. -func Catalog(userInput string, scope source.Scope) (source.Source, *pkg.Catalog, *distro.Distro, error) { - theSource, cleanup, err := source.New(userInput, scope) - defer cleanup() +// CatalogPackages takes an inventory of packages from the given image from a particular perspective +// (e.g. squashed source, all-layers source). Returns the discovered set of packages, the identified Linux +// distribution, and the source object used to wrap the data source. +func CatalogPackages(src source.Source, scope source.Scope) (*pkg.Catalog, *distro.Distro, error) { + resolver, err := src.FileResolver(scope) if err != nil { - return source.Source{}, nil, nil, err + return nil, nil, fmt.Errorf("unable to determine FileResolver while cataloging packages: %w", err) } // find the distro - theDistro := distro.Identify(theSource.Resolver) + theDistro := distro.Identify(resolver) if theDistro != nil { log.Infof("identified distro: %s", theDistro.String()) } else { @@ -51,7 +48,7 @@ func Catalog(userInput string, scope source.Scope) (source.Source, *pkg.Catalog, // conditionally use the correct set of loggers based on the input type (container image or directory) var catalogers []cataloger.Cataloger - switch theSource.Metadata.Scheme { + switch src.Metadata.Scheme { case source.ImageScheme: log.Info("cataloging image") catalogers = cataloger.ImageCatalogers() @@ -59,46 +56,15 @@ func Catalog(userInput string, scope source.Scope) (source.Source, *pkg.Catalog, log.Info("cataloging directory") catalogers = cataloger.DirectoryCatalogers() default: - return source.Source{}, nil, nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", theSource.Metadata.Scheme) + return nil, nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", src.Metadata.Scheme) } - catalog, err := cataloger.Catalog(theSource.Resolver, theDistro, catalogers...) + catalog, err := cataloger.Catalog(resolver, theDistro, catalogers...) if err != nil { - return source.Source{}, nil, nil, err + return nil, nil, err } - return theSource, catalog, theDistro, nil -} - -// CatalogFromJSON takes an existing syft report and generates native syft objects. -func CatalogFromJSON(reader io.Reader) (source.Metadata, *pkg.Catalog, *distro.Distro, error) { - var doc jsonPresenter.Document - var err error - decoder := json.NewDecoder(reader) - if err := decoder.Decode(&doc); err != nil { - return source.Metadata{}, nil, nil, err - } - - var pkgs = make([]pkg.Package, len(doc.Artifacts)) - for i, a := range doc.Artifacts { - pkgs[i], err = a.ToPackage() - if err != nil { - return source.Metadata{}, nil, nil, err - } - } - - catalog := pkg.NewCatalog(pkgs...) - - var theDistro *distro.Distro - if doc.Distro.Name != "" { - d, err := distro.NewDistro(distro.Type(doc.Distro.Name), doc.Distro.Version, doc.Distro.IDLike) - if err != nil { - return source.Metadata{}, nil, nil, err - } - theDistro = &d - } - - return doc.Source.ToSourceMetadata(), catalog, theDistro, nil + return catalog, theDistro, nil } // SetLogger sets the logger object used for all syft logging calls. diff --git a/syft/cataloger/apkdb/cataloger.go b/syft/pkg/cataloger/apkdb/cataloger.go similarity index 88% rename from syft/cataloger/apkdb/cataloger.go rename to syft/pkg/cataloger/apkdb/cataloger.go index 51ab165f7..6219eaaf7 100644 --- a/syft/cataloger/apkdb/cataloger.go +++ b/syft/pkg/cataloger/apkdb/cataloger.go @@ -4,8 +4,8 @@ Package apkdb provides a concrete Cataloger implementation for Alpine DB files. package apkdb import ( - "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/common" ) // NewApkdbCataloger returns a new Alpine DB cataloger object. diff --git a/syft/cataloger/apkdb/parse_apk_db.go b/syft/pkg/cataloger/apkdb/parse_apk_db.go similarity index 98% rename from syft/cataloger/apkdb/parse_apk_db.go rename to syft/pkg/cataloger/apkdb/parse_apk_db.go index 98c5d7c07..695553630 100644 --- a/syft/cataloger/apkdb/parse_apk_db.go +++ b/syft/pkg/cataloger/apkdb/parse_apk_db.go @@ -9,8 +9,8 @@ import ( "strings" "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/common" "github.com/mitchellh/mapstructure" ) diff --git a/syft/cataloger/apkdb/parse_apk_db_test.go b/syft/pkg/cataloger/apkdb/parse_apk_db_test.go similarity index 100% rename from syft/cataloger/apkdb/parse_apk_db_test.go rename to syft/pkg/cataloger/apkdb/parse_apk_db_test.go diff --git a/syft/cataloger/apkdb/test-fixtures/base b/syft/pkg/cataloger/apkdb/test-fixtures/base similarity index 100% rename from syft/cataloger/apkdb/test-fixtures/base rename to syft/pkg/cataloger/apkdb/test-fixtures/base diff --git a/syft/cataloger/apkdb/test-fixtures/extra-file-attributes b/syft/pkg/cataloger/apkdb/test-fixtures/extra-file-attributes similarity index 100% rename from syft/cataloger/apkdb/test-fixtures/extra-file-attributes rename to syft/pkg/cataloger/apkdb/test-fixtures/extra-file-attributes diff --git a/syft/cataloger/apkdb/test-fixtures/multiple b/syft/pkg/cataloger/apkdb/test-fixtures/multiple similarity index 100% rename from syft/cataloger/apkdb/test-fixtures/multiple rename to syft/pkg/cataloger/apkdb/test-fixtures/multiple diff --git a/syft/cataloger/apkdb/test-fixtures/single b/syft/pkg/cataloger/apkdb/test-fixtures/single similarity index 100% rename from syft/cataloger/apkdb/test-fixtures/single rename to syft/pkg/cataloger/apkdb/test-fixtures/single diff --git a/syft/cataloger/catalog.go b/syft/pkg/cataloger/catalog.go similarity index 92% rename from syft/cataloger/catalog.go rename to syft/pkg/cataloger/catalog.go index 27c10556d..962388844 100644 --- a/syft/cataloger/catalog.go +++ b/syft/pkg/cataloger/catalog.go @@ -18,13 +18,13 @@ type Monitor struct { PackagesDiscovered progress.Monitorable // the number of packages discovered from all registered catalogers } -// newMonitor creates a new Monitor object and publishes the object on the bus as a CatalogerStarted event. +// newMonitor creates a new Monitor object and publishes the object on the bus as a PackageCatalogerStarted event. func newMonitor() (*progress.Manual, *progress.Manual) { filesProcessed := progress.Manual{} packagesDiscovered := progress.Manual{} bus.Publish(partybus.Event{ - Type: event.CatalogerStarted, + Type: event.PackageCatalogerStarted, Value: Monitor{ FilesProcessed: progress.Monitorable(&filesProcessed), PackagesDiscovered: progress.Monitorable(&packagesDiscovered), @@ -37,7 +37,7 @@ func newMonitor() (*progress.Manual, *progress.Manual) { // In order to efficiently retrieve contents from a underlying container image the content fetch requests are // done in bulk. Specifically, all files of interest are collected from each catalogers and accumulated into a single // request. -func Catalog(resolver source.Resolver, theDistro *distro.Distro, catalogers ...Cataloger) (*pkg.Catalog, error) { +func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers ...Cataloger) (*pkg.Catalog, error) { catalog := pkg.NewCatalog() filesProcessed, packagesDiscovered := newMonitor() diff --git a/syft/cataloger/cataloger.go b/syft/pkg/cataloger/cataloger.go similarity index 78% rename from syft/cataloger/cataloger.go rename to syft/pkg/cataloger/cataloger.go index 28a418270..29eb9b933 100644 --- a/syft/cataloger/cataloger.go +++ b/syft/pkg/cataloger/cataloger.go @@ -6,16 +6,16 @@ catalogers defined in child packages as well as the interface definition to impl package cataloger import ( - "github.com/anchore/syft/syft/cataloger/apkdb" - "github.com/anchore/syft/syft/cataloger/deb" - "github.com/anchore/syft/syft/cataloger/golang" - "github.com/anchore/syft/syft/cataloger/java" - "github.com/anchore/syft/syft/cataloger/javascript" - "github.com/anchore/syft/syft/cataloger/python" - "github.com/anchore/syft/syft/cataloger/rpmdb" - "github.com/anchore/syft/syft/cataloger/ruby" - "github.com/anchore/syft/syft/cataloger/rust" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/apkdb" + "github.com/anchore/syft/syft/pkg/cataloger/deb" + "github.com/anchore/syft/syft/pkg/cataloger/golang" + "github.com/anchore/syft/syft/pkg/cataloger/java" + "github.com/anchore/syft/syft/pkg/cataloger/javascript" + "github.com/anchore/syft/syft/pkg/cataloger/python" + "github.com/anchore/syft/syft/pkg/cataloger/rpmdb" + "github.com/anchore/syft/syft/pkg/cataloger/ruby" + "github.com/anchore/syft/syft/pkg/cataloger/rust" "github.com/anchore/syft/syft/source" ) @@ -26,7 +26,7 @@ type Cataloger interface { // Name returns a string that uniquely describes a cataloger Name() string // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source. - Catalog(resolver source.Resolver) ([]pkg.Package, error) + Catalog(resolver source.FileResolver) ([]pkg.Package, error) } // ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages. diff --git a/syft/cataloger/common/generic_cataloger.go b/syft/pkg/cataloger/common/generic_cataloger.go similarity index 54% rename from syft/cataloger/common/generic_cataloger.go rename to syft/pkg/cataloger/common/generic_cataloger.go index c4783b176..5471a9129 100644 --- a/syft/cataloger/common/generic_cataloger.go +++ b/syft/pkg/cataloger/common/generic_cataloger.go @@ -4,7 +4,7 @@ Package common provides generic utilities used by multiple catalogers. package common import ( - "io" + "fmt" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/pkg" @@ -16,8 +16,6 @@ import ( type GenericCataloger struct { globParsers map[string]ParserFn pathParsers map[string]ParserFn - selectedFiles []source.Location - parsers map[source.Location]ParserFn upstreamCataloger string } @@ -26,8 +24,6 @@ func NewGenericCataloger(pathParsers map[string]ParserFn, globParsers map[string return &GenericCataloger{ globParsers: globParsers, pathParsers: pathParsers, - selectedFiles: make([]source.Location, 0), - parsers: make(map[source.Location]ParserFn), upstreamCataloger: upstreamCataloger, } } @@ -37,74 +33,22 @@ func (c *GenericCataloger) Name() string { return c.upstreamCataloger } -// register pairs a set of file references with a parser function for future cataloging (when the file contents are resolved) -func (c *GenericCataloger) register(files []source.Location, parser ParserFn) { - c.selectedFiles = append(c.selectedFiles, files...) - for _, f := range files { - c.parsers[f] = parser - } -} - -// clear deletes all registered file-reference-to-parser-function pairings from former SelectFiles() and register() calls -func (c *GenericCataloger) clear() { - c.selectedFiles = make([]source.Location, 0) - c.parsers = make(map[source.Location]ParserFn) -} - // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source. -func (c *GenericCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { - fileSelection := c.selectFiles(resolver) - contents, err := resolver.MultipleFileContentsByLocation(fileSelection) - if err != nil { - return nil, err - } - return c.catalog(contents) -} +func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) { + var packages []pkg.Package + parserByLocation := c.selectFiles(resolver) -// SelectFiles takes a set of file trees and resolves and file references of interest for future cataloging -func (c *GenericCataloger) selectFiles(resolver source.FileResolver) []source.Location { - // select by exact path - for path, parser := range c.pathParsers { - files, err := resolver.FilesByPath(path) + for location, parser := range parserByLocation { + content, err := resolver.FileContentsByLocation(location) if err != nil { - log.Warnf("cataloger failed to select files by path: %+v", err) - } - if files != nil { - c.register(files, parser) - } - } - - // select by glob pattern - for globPattern, parser := range c.globParsers { - fileMatches, err := resolver.FilesByGlob(globPattern) - if err != nil { - log.Warnf("failed to find files by glob: %s", globPattern) - } - if fileMatches != nil { - c.register(fileMatches, parser) - } - } - - return c.selectedFiles -} - -// catalog takes a set of file contents and uses any configured parser functions to resolve and return discovered packages -func (c *GenericCataloger) catalog(contents map[source.Location]io.ReadCloser) ([]pkg.Package, error) { - defer c.clear() - - packages := make([]pkg.Package, 0) - - for location, parser := range c.parsers { - content, ok := contents[location] - if !ok { - log.Warnf("cataloger '%s' missing file content: %+v", c.upstreamCataloger, location) - continue + // TODO: fail or log? + return nil, fmt.Errorf("unable to fetch contents for location=%v : %w", location, err) } entries, err := parser(location.RealPath, content) if err != nil { // TODO: should we fail? or only log? - log.Warnf("cataloger '%s' failed to parse entries (%+v): %+v", c.upstreamCataloger, location, err) + log.Warnf("cataloger '%s' failed to parse entries (location=%+v): %+v", c.upstreamCataloger, location, err) continue } @@ -115,6 +59,34 @@ func (c *GenericCataloger) catalog(contents map[source.Location]io.ReadCloser) ( packages = append(packages, entry) } } - return packages, nil } + +// SelectFiles takes a set of file trees and resolves and file references of interest for future cataloging +func (c *GenericCataloger) selectFiles(resolver source.FilePathResolver) map[source.Location]ParserFn { + var parserByLocation = make(map[source.Location]ParserFn) + + // select by exact path + for path, parser := range c.pathParsers { + files, err := resolver.FilesByPath(path) + if err != nil { + log.Warnf("cataloger failed to select files by path: %+v", err) + } + for _, f := range files { + parserByLocation[f] = parser + } + } + + // select by glob pattern + for globPattern, parser := range c.globParsers { + fileMatches, err := resolver.FilesByGlob(globPattern) + if err != nil { + log.Warnf("failed to find files by glob: %s", globPattern) + } + for _, f := range fileMatches { + parserByLocation[f] = parser + } + } + + return parserByLocation +} diff --git a/syft/pkg/cataloger/common/generic_cataloger_test.go b/syft/pkg/cataloger/common/generic_cataloger_test.go new file mode 100644 index 000000000..8c1f4215c --- /dev/null +++ b/syft/pkg/cataloger/common/generic_cataloger_test.go @@ -0,0 +1,73 @@ +package common + +import ( + "fmt" + "io" + "io/ioutil" + "testing" + + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +func parser(_ string, reader io.Reader) ([]pkg.Package, error) { + contents, err := ioutil.ReadAll(reader) + if err != nil { + panic(err) + } + return []pkg.Package{ + { + Name: string(contents), + }, + }, nil +} + +func TestGenericCataloger(t *testing.T) { + + globParsers := map[string]ParserFn{ + "**a-path.txt": parser, + } + pathParsers := map[string]ParserFn{ + "test-fixtures/another-path.txt": parser, + "test-fixtures/last/path.txt": parser, + } + upstream := "some-other-cataloger" + + expectedSelection := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"} + resolver := source.NewMockResolverForPaths(expectedSelection...) + cataloger := NewGenericCataloger(pathParsers, globParsers, upstream) + + expectedPkgs := make(map[string]pkg.Package) + for _, path := range expectedSelection { + expectedPkgs[path] = pkg.Package{ + FoundBy: upstream, + Name: fmt.Sprintf("%s file contents!", path), + } + } + + actualPkgs, err := cataloger.Catalog(resolver) + if err != nil { + t.Fatalf("cataloger catalog action failed: %+v", err) + } + + if len(actualPkgs) != len(expectedPkgs) { + t.Fatalf("unexpected packages len: %d", len(actualPkgs)) + } + + for _, p := range actualPkgs { + ref := p.Locations[0] + exP, ok := expectedPkgs[ref.RealPath] + if !ok { + t.Errorf("missing expected pkg: ref=%+v", ref) + continue + } + + if p.FoundBy != exP.FoundBy { + t.Errorf("bad upstream: %s", p.FoundBy) + } + + if exP.Name != p.Name { + t.Errorf("bad contents mapping: %+v", p.Locations) + } + } +} diff --git a/syft/cataloger/common/parser.go b/syft/pkg/cataloger/common/parser.go similarity index 100% rename from syft/cataloger/common/parser.go rename to syft/pkg/cataloger/common/parser.go diff --git a/syft/pkg/cataloger/common/test-fixtures/a-path.txt b/syft/pkg/cataloger/common/test-fixtures/a-path.txt new file mode 100644 index 000000000..67e954034 --- /dev/null +++ b/syft/pkg/cataloger/common/test-fixtures/a-path.txt @@ -0,0 +1 @@ +test-fixtures/a-path.txt file contents! \ No newline at end of file diff --git a/syft/pkg/cataloger/common/test-fixtures/another-path.txt b/syft/pkg/cataloger/common/test-fixtures/another-path.txt new file mode 100644 index 000000000..0d654f8fe --- /dev/null +++ b/syft/pkg/cataloger/common/test-fixtures/another-path.txt @@ -0,0 +1 @@ +test-fixtures/another-path.txt file contents! \ No newline at end of file diff --git a/syft/pkg/cataloger/common/test-fixtures/last/path.txt b/syft/pkg/cataloger/common/test-fixtures/last/path.txt new file mode 100644 index 000000000..3d4a165ab --- /dev/null +++ b/syft/pkg/cataloger/common/test-fixtures/last/path.txt @@ -0,0 +1 @@ +test-fixtures/last/path.txt file contents! \ No newline at end of file diff --git a/syft/cataloger/cpe.go b/syft/pkg/cataloger/cpe.go similarity index 100% rename from syft/cataloger/cpe.go rename to syft/pkg/cataloger/cpe.go diff --git a/syft/cataloger/cpe_specificity.go b/syft/pkg/cataloger/cpe_specificity.go similarity index 100% rename from syft/cataloger/cpe_specificity.go rename to syft/pkg/cataloger/cpe_specificity.go diff --git a/syft/cataloger/cpe_test.go b/syft/pkg/cataloger/cpe_test.go similarity index 100% rename from syft/cataloger/cpe_test.go rename to syft/pkg/cataloger/cpe_test.go diff --git a/syft/pkg/cataloger/deb/cataloger.go b/syft/pkg/cataloger/deb/cataloger.go new file mode 100644 index 000000000..34c3df3ac --- /dev/null +++ b/syft/pkg/cataloger/deb/cataloger.go @@ -0,0 +1,156 @@ +/* +Package dpkg provides a concrete Cataloger implementation for Debian package DB status files. +*/ +package deb + +import ( + "fmt" + "io" + "path" + "path/filepath" + + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +const ( + md5sumsExt = ".md5sums" + docsPath = "/usr/share/doc" +) + +type Cataloger struct{} + +// NewDpkgdbCataloger returns a new Deb package cataloger object. +func NewDpkgdbCataloger() *Cataloger { + return &Cataloger{} +} + +// Name returns a string that uniquely describes a cataloger +func (c *Cataloger) Name() string { + return "dpkgdb-cataloger" +} + +// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files. +// nolint:funlen +func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) { + dbFileMatches, err := resolver.FilesByGlob(pkg.DpkgDbGlob) + if err != nil { + return nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err) + } + + var results []pkg.Package + var pkgs []pkg.Package + for _, dbLocation := range dbFileMatches { + dbContents, err := resolver.FileContentsByLocation(dbLocation) + if err != nil { + return nil, err + } + + pkgs, err = parseDpkgStatus(dbContents) + if err != nil { + return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.RealPath, err) + } + + for i := range pkgs { + p := &pkgs[i] + p.FoundBy = c.Name() + p.Locations = []source.Location{dbLocation} + + metadata := p.Metadata.(pkg.DpkgMetadata) + + md5Reader, md5Location, err := fetchMd5Contents(resolver, dbLocation, p) + if err != nil { + return nil, fmt.Errorf("unable to find dpkg md5 contents: %w", err) + } + + if md5Reader != nil { + // attach the file list + metadata.Files = parseDpkgMD5Info(md5Reader) + + // keep a record of the file where this was discovered + if md5Location != nil { + p.Locations = append(p.Locations, *md5Location) + } + } else { + // ensure the file list is an empty collection (not nil) + metadata.Files = make([]pkg.DpkgFileRecord, 0) + } + + // persist alterations + p.Metadata = metadata + + // get license information from the copyright file + copyrightReader, copyrightLocation, err := fetchCopyrightContents(resolver, dbLocation, p) + if err != nil { + return nil, fmt.Errorf("unable to find dpkg copyright contents: %w", err) + } + + if copyrightReader != nil { + // attach the licenses + p.Licenses = parseLicensesFromCopyright(copyrightReader) + + // keep a record of the file where this was discovered + if copyrightLocation != nil { + p.Locations = append(p.Locations, *copyrightLocation) + } + } + } + + results = append(results, pkgs...) + } + return results, nil +} + +func fetchMd5Contents(resolver source.FileResolver, dbLocation source.Location, p *pkg.Package) (io.Reader, *source.Location, error) { + parentPath := filepath.Dir(dbLocation.RealPath) + + // look for /var/lib/dpkg/info/NAME:ARCH.md5sums + name := md5Key(p) + md5SumLocation := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "info", name+md5sumsExt)) + + if md5SumLocation == nil { + // the most specific key did not work, fallback to just the name + // look for /var/lib/dpkg/info/NAME.md5sums + md5SumLocation = resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "info", p.Name+md5sumsExt)) + } + + // this is unexpected, but not a show-stopper + if md5SumLocation == nil { + return nil, nil, nil + } + + reader, err := resolver.FileContentsByLocation(*md5SumLocation) + if err != nil { + return nil, nil, fmt.Errorf("failed to fetch deb md5 contents (%+v): %w", p, err) + } + return reader, md5SumLocation, nil +} + +func fetchCopyrightContents(resolver source.FileResolver, dbLocation source.Location, p *pkg.Package) (io.Reader, *source.Location, error) { + // look for /usr/share/docs/NAME/copyright files + name := p.Name + copyrightPath := path.Join(docsPath, name, "copyright") + copyrightLocation := resolver.RelativeFileByPath(dbLocation, copyrightPath) + + // we may not have a copyright file for each package, ignore missing files + if copyrightLocation == nil { + return nil, nil, nil + } + + reader, err := resolver.FileContentsByLocation(*copyrightLocation) + if err != nil { + return nil, nil, fmt.Errorf("failed to fetch deb copyright contents (%+v): %w", p, err) + } + + return reader, copyrightLocation, nil +} + +func md5Key(p *pkg.Package) string { + metadata := p.Metadata.(pkg.DpkgMetadata) + + contentKey := p.Name + if metadata.Architecture != "" && metadata.Architecture != "all" { + contentKey = contentKey + ":" + metadata.Architecture + } + return contentKey +} diff --git a/syft/cataloger/deb/cataloger_test.go b/syft/pkg/cataloger/deb/cataloger_test.go similarity index 89% rename from syft/cataloger/deb/cataloger_test.go rename to syft/pkg/cataloger/deb/cataloger_test.go index d7e9079b5..b856a0cb9 100644 --- a/syft/cataloger/deb/cataloger_test.go +++ b/syft/pkg/cataloger/deb/cataloger_test.go @@ -51,17 +51,21 @@ func TestDpkgCataloger(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-dpkg") - defer cleanup() + img := imagetest.GetFixtureImage(t, "docker-archive", "image-dpkg") - s, err := source.NewFromImage(img, source.SquashedScope, "") + s, err := source.NewFromImage(img, "") if err != nil { t.Fatal(err) } c := NewDpkgdbCataloger() - actual, err := c.Catalog(s.Resolver) + resolver, err := s.FileResolver(source.SquashedScope) + if err != nil { + t.Errorf("could not get resolver error: %+v", err) + } + + actual, err := c.Catalog(resolver) if err != nil { t.Fatalf("failed to catalog: %+v", err) } diff --git a/syft/cataloger/deb/parse_copyright.go b/syft/pkg/cataloger/deb/parse_copyright.go similarity index 100% rename from syft/cataloger/deb/parse_copyright.go rename to syft/pkg/cataloger/deb/parse_copyright.go diff --git a/syft/cataloger/deb/parse_copyright_test.go b/syft/pkg/cataloger/deb/parse_copyright_test.go similarity index 100% rename from syft/cataloger/deb/parse_copyright_test.go rename to syft/pkg/cataloger/deb/parse_copyright_test.go diff --git a/syft/cataloger/deb/parse_dpkg_info_files.go b/syft/pkg/cataloger/deb/parse_dpkg_info_files.go similarity index 100% rename from syft/cataloger/deb/parse_dpkg_info_files.go rename to syft/pkg/cataloger/deb/parse_dpkg_info_files.go diff --git a/syft/cataloger/deb/parse_dpkg_info_files_test.go b/syft/pkg/cataloger/deb/parse_dpkg_info_files_test.go similarity index 100% rename from syft/cataloger/deb/parse_dpkg_info_files_test.go rename to syft/pkg/cataloger/deb/parse_dpkg_info_files_test.go diff --git a/syft/cataloger/deb/parse_dpkg_status.go b/syft/pkg/cataloger/deb/parse_dpkg_status.go similarity index 100% rename from syft/cataloger/deb/parse_dpkg_status.go rename to syft/pkg/cataloger/deb/parse_dpkg_status.go diff --git a/syft/cataloger/deb/parse_dpkg_status_test.go b/syft/pkg/cataloger/deb/parse_dpkg_status_test.go similarity index 100% rename from syft/cataloger/deb/parse_dpkg_status_test.go rename to syft/pkg/cataloger/deb/parse_dpkg_status_test.go diff --git a/syft/cataloger/deb/test-fixtures/copyright/libaudit-common b/syft/pkg/cataloger/deb/test-fixtures/copyright/libaudit-common similarity index 100% rename from syft/cataloger/deb/test-fixtures/copyright/libaudit-common rename to syft/pkg/cataloger/deb/test-fixtures/copyright/libaudit-common diff --git a/syft/cataloger/deb/test-fixtures/copyright/liblzma5 b/syft/pkg/cataloger/deb/test-fixtures/copyright/liblzma5 similarity index 100% rename from syft/cataloger/deb/test-fixtures/copyright/liblzma5 rename to syft/pkg/cataloger/deb/test-fixtures/copyright/liblzma5 diff --git a/syft/cataloger/deb/test-fixtures/copyright/python b/syft/pkg/cataloger/deb/test-fixtures/copyright/python similarity index 100% rename from syft/cataloger/deb/test-fixtures/copyright/python rename to syft/pkg/cataloger/deb/test-fixtures/copyright/python diff --git a/syft/cataloger/deb/test-fixtures/copyright/trilicense b/syft/pkg/cataloger/deb/test-fixtures/copyright/trilicense similarity index 100% rename from syft/cataloger/deb/test-fixtures/copyright/trilicense rename to syft/pkg/cataloger/deb/test-fixtures/copyright/trilicense diff --git a/syft/cataloger/deb/test-fixtures/image-dpkg/Dockerfile b/syft/pkg/cataloger/deb/test-fixtures/image-dpkg/Dockerfile similarity index 100% rename from syft/cataloger/deb/test-fixtures/image-dpkg/Dockerfile rename to syft/pkg/cataloger/deb/test-fixtures/image-dpkg/Dockerfile diff --git a/syft/cataloger/deb/test-fixtures/image-dpkg/usr/share/doc/libpam-runtime/copyright b/syft/pkg/cataloger/deb/test-fixtures/image-dpkg/usr/share/doc/libpam-runtime/copyright similarity index 100% rename from syft/cataloger/deb/test-fixtures/image-dpkg/usr/share/doc/libpam-runtime/copyright rename to syft/pkg/cataloger/deb/test-fixtures/image-dpkg/usr/share/doc/libpam-runtime/copyright diff --git a/syft/cataloger/deb/test-fixtures/image-dpkg/var/lib/dpkg/info/libpam-runtime.md5sums b/syft/pkg/cataloger/deb/test-fixtures/image-dpkg/var/lib/dpkg/info/libpam-runtime.md5sums similarity index 100% rename from syft/cataloger/deb/test-fixtures/image-dpkg/var/lib/dpkg/info/libpam-runtime.md5sums rename to syft/pkg/cataloger/deb/test-fixtures/image-dpkg/var/lib/dpkg/info/libpam-runtime.md5sums diff --git a/syft/cataloger/deb/test-fixtures/image-dpkg/var/lib/dpkg/status b/syft/pkg/cataloger/deb/test-fixtures/image-dpkg/var/lib/dpkg/status similarity index 100% rename from syft/cataloger/deb/test-fixtures/image-dpkg/var/lib/dpkg/status rename to syft/pkg/cataloger/deb/test-fixtures/image-dpkg/var/lib/dpkg/status diff --git a/syft/cataloger/deb/test-fixtures/info/zlib1g.md5sums b/syft/pkg/cataloger/deb/test-fixtures/info/zlib1g.md5sums similarity index 100% rename from syft/cataloger/deb/test-fixtures/info/zlib1g.md5sums rename to syft/pkg/cataloger/deb/test-fixtures/info/zlib1g.md5sums diff --git a/syft/cataloger/deb/test-fixtures/status/multiple b/syft/pkg/cataloger/deb/test-fixtures/status/multiple similarity index 100% rename from syft/cataloger/deb/test-fixtures/status/multiple rename to syft/pkg/cataloger/deb/test-fixtures/status/multiple diff --git a/syft/cataloger/deb/test-fixtures/status/single b/syft/pkg/cataloger/deb/test-fixtures/status/single similarity index 100% rename from syft/cataloger/deb/test-fixtures/status/single rename to syft/pkg/cataloger/deb/test-fixtures/status/single diff --git a/syft/cataloger/golang/cataloger.go b/syft/pkg/cataloger/golang/cataloger.go similarity index 87% rename from syft/cataloger/golang/cataloger.go rename to syft/pkg/cataloger/golang/cataloger.go index 268bc1cd0..34f33a595 100644 --- a/syft/cataloger/golang/cataloger.go +++ b/syft/pkg/cataloger/golang/cataloger.go @@ -4,7 +4,7 @@ Package golang provides a concrete Cataloger implementation for go.mod files. package golang import ( - "github.com/anchore/syft/syft/cataloger/common" + "github.com/anchore/syft/syft/pkg/cataloger/common" ) // NewGoModCataloger returns a new Go module cataloger object. diff --git a/syft/cataloger/golang/parse_go_mod.go b/syft/pkg/cataloger/golang/parse_go_mod.go similarity index 100% rename from syft/cataloger/golang/parse_go_mod.go rename to syft/pkg/cataloger/golang/parse_go_mod.go diff --git a/syft/cataloger/golang/parse_go_mod_test.go b/syft/pkg/cataloger/golang/parse_go_mod_test.go similarity index 100% rename from syft/cataloger/golang/parse_go_mod_test.go rename to syft/pkg/cataloger/golang/parse_go_mod_test.go diff --git a/syft/cataloger/golang/test-fixtures/many-packages b/syft/pkg/cataloger/golang/test-fixtures/many-packages similarity index 100% rename from syft/cataloger/golang/test-fixtures/many-packages rename to syft/pkg/cataloger/golang/test-fixtures/many-packages diff --git a/syft/cataloger/golang/test-fixtures/one-package b/syft/pkg/cataloger/golang/test-fixtures/one-package similarity index 100% rename from syft/cataloger/golang/test-fixtures/one-package rename to syft/pkg/cataloger/golang/test-fixtures/one-package diff --git a/syft/cataloger/java/archive_filename.go b/syft/pkg/cataloger/java/archive_filename.go similarity index 100% rename from syft/cataloger/java/archive_filename.go rename to syft/pkg/cataloger/java/archive_filename.go diff --git a/syft/cataloger/java/archive_filename_test.go b/syft/pkg/cataloger/java/archive_filename_test.go similarity index 100% rename from syft/cataloger/java/archive_filename_test.go rename to syft/pkg/cataloger/java/archive_filename_test.go diff --git a/syft/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go similarity index 99% rename from syft/cataloger/java/archive_parser.go rename to syft/pkg/cataloger/java/archive_parser.go index 05446ae08..711ca0d1b 100644 --- a/syft/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -9,8 +9,8 @@ import ( "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/file" - "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/common" ) // integrity check diff --git a/syft/cataloger/java/archive_parser_test.go b/syft/pkg/cataloger/java/archive_parser_test.go similarity index 100% rename from syft/cataloger/java/archive_parser_test.go rename to syft/pkg/cataloger/java/archive_parser_test.go diff --git a/syft/cataloger/java/cataloger.go b/syft/pkg/cataloger/java/cataloger.go similarity index 89% rename from syft/cataloger/java/cataloger.go rename to syft/pkg/cataloger/java/cataloger.go index 35d776e93..2285c6474 100644 --- a/syft/cataloger/java/cataloger.go +++ b/syft/pkg/cataloger/java/cataloger.go @@ -4,7 +4,7 @@ Package java provides a concrete Cataloger implementation for Java archives (jar package java import ( - "github.com/anchore/syft/syft/cataloger/common" + "github.com/anchore/syft/syft/pkg/cataloger/common" ) // NewJavaCataloger returns a new Java archive cataloger object. diff --git a/syft/cataloger/java/java_manifest.go b/syft/pkg/cataloger/java/java_manifest.go similarity index 100% rename from syft/cataloger/java/java_manifest.go rename to syft/pkg/cataloger/java/java_manifest.go diff --git a/syft/cataloger/java/java_manifest_test.go b/syft/pkg/cataloger/java/java_manifest_test.go similarity index 100% rename from syft/cataloger/java/java_manifest_test.go rename to syft/pkg/cataloger/java/java_manifest_test.go diff --git a/syft/cataloger/java/pom_properties.go b/syft/pkg/cataloger/java/pom_properties.go similarity index 100% rename from syft/cataloger/java/pom_properties.go rename to syft/pkg/cataloger/java/pom_properties.go diff --git a/syft/cataloger/java/pom_properties_test.go b/syft/pkg/cataloger/java/pom_properties_test.go similarity index 100% rename from syft/cataloger/java/pom_properties_test.go rename to syft/pkg/cataloger/java/pom_properties_test.go diff --git a/syft/cataloger/java/save_archive_to_tmp.go b/syft/pkg/cataloger/java/save_archive_to_tmp.go similarity index 100% rename from syft/cataloger/java/save_archive_to_tmp.go rename to syft/pkg/cataloger/java/save_archive_to_tmp.go diff --git a/syft/cataloger/java/test-fixtures/java-builds/.gitignore b/syft/pkg/cataloger/java/test-fixtures/java-builds/.gitignore similarity index 87% rename from syft/cataloger/java/test-fixtures/java-builds/.gitignore rename to syft/pkg/cataloger/java/test-fixtures/java-builds/.gitignore index 1685225cc..b954422b0 100644 --- a/syft/cataloger/java/test-fixtures/java-builds/.gitignore +++ b/syft/pkg/cataloger/java/test-fixtures/java-builds/.gitignore @@ -1,4 +1,4 @@ -/packages/* +/packages/sb *.fingerprint # maven when running in a volume may spit out directories like this **/\?/ diff --git a/syft/cataloger/java/test-fixtures/java-builds/Makefile b/syft/pkg/cataloger/java/test-fixtures/java-builds/Makefile similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/Makefile rename to syft/pkg/cataloger/java/test-fixtures/java-builds/Makefile diff --git a/syft/cataloger/java/test-fixtures/java-builds/build-example-java-app-gradle.sh b/syft/pkg/cataloger/java/test-fixtures/java-builds/build-example-java-app-gradle.sh similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/build-example-java-app-gradle.sh rename to syft/pkg/cataloger/java/test-fixtures/java-builds/build-example-java-app-gradle.sh diff --git a/syft/cataloger/java/test-fixtures/java-builds/build-example-java-app-maven.sh b/syft/pkg/cataloger/java/test-fixtures/java-builds/build-example-java-app-maven.sh similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/build-example-java-app-maven.sh rename to syft/pkg/cataloger/java/test-fixtures/java-builds/build-example-java-app-maven.sh diff --git a/syft/cataloger/java/test-fixtures/java-builds/build-example-jenkins-plugin.sh b/syft/pkg/cataloger/java/test-fixtures/java-builds/build-example-jenkins-plugin.sh similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/build-example-jenkins-plugin.sh rename to syft/pkg/cataloger/java/test-fixtures/java-builds/build-example-jenkins-plugin.sh diff --git a/syft/cataloger/java/test-fixtures/java-builds/build-example-sb-app-nestedjar.sh b/syft/pkg/cataloger/java/test-fixtures/java-builds/build-example-sb-app-nestedjar.sh similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/build-example-sb-app-nestedjar.sh rename to syft/pkg/cataloger/java/test-fixtures/java-builds/build-example-sb-app-nestedjar.sh diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-java-app/.gitignore b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-java-app/.gitignore similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-java-app/.gitignore rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-java-app/.gitignore diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-java-app/build.gradle b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-java-app/build.gradle similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-java-app/build.gradle rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-java-app/build.gradle diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-java-app/pom.xml b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-java-app/pom.xml similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-java-app/pom.xml rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-java-app/pom.xml diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-java-app/src/main/java/hello/Greeter.java b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-java-app/src/main/java/hello/Greeter.java similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-java-app/src/main/java/hello/Greeter.java rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-java-app/src/main/java/hello/Greeter.java diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-java-app/src/main/java/hello/HelloWorld.java b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-java-app/src/main/java/hello/HelloWorld.java similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-java-app/src/main/java/hello/HelloWorld.java rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-java-app/src/main/java/hello/HelloWorld.java diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/pom.xml b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/pom.xml similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/pom.xml rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/pom.xml diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/java/io/jenkins/plugins/sample/HelloWorldBuilder.java b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/java/io/jenkins/plugins/sample/HelloWorldBuilder.java similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/java/io/jenkins/plugins/sample/HelloWorldBuilder.java rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/java/io/jenkins/plugins/sample/HelloWorldBuilder.java diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/index.jelly b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/index.jelly similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/index.jelly rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/index.jelly diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/config.jelly b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/config.jelly similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/config.jelly rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/config.jelly diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/config.properties b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/config.properties similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/config.properties rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/config.properties diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/config_fr.properties b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/config_fr.properties similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/config_fr.properties rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/config_fr.properties diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/help-name.html b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/help-name.html similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/help-name.html rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/help-name.html diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/help-name_fr.html b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/help-name_fr.html similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/help-name_fr.html rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/help-name_fr.html diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/help-useFrench.html b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/help-useFrench.html similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/help-useFrench.html rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/help-useFrench.html diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/help-useFrench_fr.html b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/help-useFrench_fr.html similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/help-useFrench_fr.html rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/HelloWorldBuilder/help-useFrench_fr.html diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/Messages.properties b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/Messages.properties similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/Messages.properties rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/Messages.properties diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/Messages_fr.properties b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/Messages_fr.properties similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/Messages_fr.properties rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-jenkins-plugin/src/main/resources/io/jenkins/plugins/sample/Messages_fr.properties diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-sb-app/.gitignore b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-sb-app/.gitignore similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-sb-app/.gitignore rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-sb-app/.gitignore diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-sb-app/pom.xml b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-sb-app/pom.xml similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-sb-app/pom.xml rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-sb-app/pom.xml diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-sb-app/src/main/java/com/example/springboot/Application.java b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-sb-app/src/main/java/com/example/springboot/Application.java similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-sb-app/src/main/java/com/example/springboot/Application.java rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-sb-app/src/main/java/com/example/springboot/Application.java diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-sb-app/src/main/java/com/example/springboot/HelloController.java b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-sb-app/src/main/java/com/example/springboot/HelloController.java similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-sb-app/src/main/java/com/example/springboot/HelloController.java rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-sb-app/src/main/java/com/example/springboot/HelloController.java diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-sb-app/src/test/java/com/example/springboot/HelloControllerIT.java b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-sb-app/src/test/java/com/example/springboot/HelloControllerIT.java similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-sb-app/src/test/java/com/example/springboot/HelloControllerIT.java rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-sb-app/src/test/java/com/example/springboot/HelloControllerIT.java diff --git a/syft/cataloger/java/test-fixtures/java-builds/example-sb-app/src/test/java/com/example/springboot/HelloControllerTest.java b/syft/pkg/cataloger/java/test-fixtures/java-builds/example-sb-app/src/test/java/com/example/springboot/HelloControllerTest.java similarity index 100% rename from syft/cataloger/java/test-fixtures/java-builds/example-sb-app/src/test/java/com/example/springboot/HelloControllerTest.java rename to syft/pkg/cataloger/java/test-fixtures/java-builds/example-sb-app/src/test/java/com/example/springboot/HelloControllerTest.java diff --git a/syft/cataloger/java/test-fixtures/manifest/continuation b/syft/pkg/cataloger/java/test-fixtures/manifest/continuation similarity index 100% rename from syft/cataloger/java/test-fixtures/manifest/continuation rename to syft/pkg/cataloger/java/test-fixtures/manifest/continuation diff --git a/syft/cataloger/java/test-fixtures/manifest/extra-info b/syft/pkg/cataloger/java/test-fixtures/manifest/extra-info similarity index 100% rename from syft/cataloger/java/test-fixtures/manifest/extra-info rename to syft/pkg/cataloger/java/test-fixtures/manifest/extra-info diff --git a/syft/cataloger/java/test-fixtures/manifest/small b/syft/pkg/cataloger/java/test-fixtures/manifest/small similarity index 100% rename from syft/cataloger/java/test-fixtures/manifest/small rename to syft/pkg/cataloger/java/test-fixtures/manifest/small diff --git a/syft/cataloger/java/test-fixtures/manifest/standard-info b/syft/pkg/cataloger/java/test-fixtures/manifest/standard-info similarity index 100% rename from syft/cataloger/java/test-fixtures/manifest/standard-info rename to syft/pkg/cataloger/java/test-fixtures/manifest/standard-info diff --git a/syft/cataloger/java/test-fixtures/manifest/version-with-date b/syft/pkg/cataloger/java/test-fixtures/manifest/version-with-date similarity index 100% rename from syft/cataloger/java/test-fixtures/manifest/version-with-date rename to syft/pkg/cataloger/java/test-fixtures/manifest/version-with-date diff --git a/syft/cataloger/java/test-fixtures/pom/colon-delimited-with-equals.pom.properties b/syft/pkg/cataloger/java/test-fixtures/pom/colon-delimited-with-equals.pom.properties similarity index 79% rename from syft/cataloger/java/test-fixtures/pom/colon-delimited-with-equals.pom.properties rename to syft/pkg/cataloger/java/test-fixtures/pom/colon-delimited-with-equals.pom.properties index a189c75a6..68721e96b 100644 --- a/syft/cataloger/java/test-fixtures/pom/colon-delimited-with-equals.pom.properties +++ b/syft/pkg/cataloger/java/test-fixtures/pom/colon-delimited-with-equals.pom.properties @@ -2,4 +2,4 @@ #Tue Jul 07 18:59:56 GMT 2020 groupId:org.anchore artifactId: example-java=app-maven -version: 0.1.0=something +version: 0.1.0=something \ No newline at end of file diff --git a/syft/cataloger/java/test-fixtures/pom/colon-delimited.pom.properties b/syft/pkg/cataloger/java/test-fixtures/pom/colon-delimited.pom.properties similarity index 86% rename from syft/cataloger/java/test-fixtures/pom/colon-delimited.pom.properties rename to syft/pkg/cataloger/java/test-fixtures/pom/colon-delimited.pom.properties index 4069b275c..a32d166e6 100644 --- a/syft/cataloger/java/test-fixtures/pom/colon-delimited.pom.properties +++ b/syft/pkg/cataloger/java/test-fixtures/pom/colon-delimited.pom.properties @@ -2,4 +2,4 @@ #Tue Jul 07 18:59:56 GMT 2020 groupId:org.anchore artifactId: example-java-app-maven -version: 0.1.0 +version: 0.1.0 \ No newline at end of file diff --git a/syft/cataloger/java/test-fixtures/pom/equals-delimited-with-colons.pom.properties b/syft/pkg/cataloger/java/test-fixtures/pom/equals-delimited-with-colons.pom.properties similarity index 79% rename from syft/cataloger/java/test-fixtures/pom/equals-delimited-with-colons.pom.properties rename to syft/pkg/cataloger/java/test-fixtures/pom/equals-delimited-with-colons.pom.properties index 7cea7ae6a..144b78b81 100644 --- a/syft/cataloger/java/test-fixtures/pom/equals-delimited-with-colons.pom.properties +++ b/syft/pkg/cataloger/java/test-fixtures/pom/equals-delimited-with-colons.pom.properties @@ -2,4 +2,4 @@ #Tue Jul 07 18:59:56 GMT 2020 groupId=org.anchore artifactId= example-java:app-maven -version= 0.1.0:something +version= 0.1.0:something \ No newline at end of file diff --git a/syft/cataloger/java/test-fixtures/pom/extra.pom.properties b/syft/pkg/cataloger/java/test-fixtures/pom/extra.pom.properties similarity index 100% rename from syft/cataloger/java/test-fixtures/pom/extra.pom.properties rename to syft/pkg/cataloger/java/test-fixtures/pom/extra.pom.properties diff --git a/syft/cataloger/java/test-fixtures/pom/small.pom.properties b/syft/pkg/cataloger/java/test-fixtures/pom/small.pom.properties similarity index 100% rename from syft/cataloger/java/test-fixtures/pom/small.pom.properties rename to syft/pkg/cataloger/java/test-fixtures/pom/small.pom.properties diff --git a/syft/cataloger/javascript/cataloger.go b/syft/pkg/cataloger/javascript/cataloger.go similarity index 94% rename from syft/cataloger/javascript/cataloger.go rename to syft/pkg/cataloger/javascript/cataloger.go index fe709d458..791554a5d 100644 --- a/syft/cataloger/javascript/cataloger.go +++ b/syft/pkg/cataloger/javascript/cataloger.go @@ -4,7 +4,7 @@ Package javascript provides a concrete Cataloger implementation for JavaScript e package javascript import ( - "github.com/anchore/syft/syft/cataloger/common" + "github.com/anchore/syft/syft/pkg/cataloger/common" ) // NewJavascriptPackageCataloger returns a new JavaScript cataloger object based on detection of npm based packages. diff --git a/syft/cataloger/javascript/parse_package_json.go b/syft/pkg/cataloger/javascript/parse_package_json.go similarity index 97% rename from syft/cataloger/javascript/parse_package_json.go rename to syft/pkg/cataloger/javascript/parse_package_json.go index 5a4fbadc9..d5ff50142 100644 --- a/syft/cataloger/javascript/parse_package_json.go +++ b/syft/pkg/cataloger/javascript/parse_package_json.go @@ -13,8 +13,8 @@ import ( "github.com/mitchellh/mapstructure" - "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/common" ) // integrity check @@ -161,7 +161,7 @@ func licensesFromJSON(p PackageJSON) ([]string, error) { return nil, fmt.Errorf("unable to parse license field: %w", err) } -// parsePackageJson parses a package.json and returns the discovered JavaScript packages. +// parsePackageJSON parses a package.json and returns the discovered JavaScript packages. func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { packages := make([]pkg.Package, 0) dec := json.NewDecoder(reader) diff --git a/syft/cataloger/javascript/parse_package_json_test.go b/syft/pkg/cataloger/javascript/parse_package_json_test.go similarity index 100% rename from syft/cataloger/javascript/parse_package_json_test.go rename to syft/pkg/cataloger/javascript/parse_package_json_test.go diff --git a/syft/cataloger/javascript/parse_package_lock.go b/syft/pkg/cataloger/javascript/parse_package_lock.go similarity index 96% rename from syft/cataloger/javascript/parse_package_lock.go rename to syft/pkg/cataloger/javascript/parse_package_lock.go index 4018cbf63..c4f3a8f99 100644 --- a/syft/cataloger/javascript/parse_package_lock.go +++ b/syft/pkg/cataloger/javascript/parse_package_lock.go @@ -5,8 +5,8 @@ import ( "fmt" "io" - "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/common" ) // integrity check diff --git a/syft/cataloger/javascript/parse_package_lock_test.go b/syft/pkg/cataloger/javascript/parse_package_lock_test.go similarity index 100% rename from syft/cataloger/javascript/parse_package_lock_test.go rename to syft/pkg/cataloger/javascript/parse_package_lock_test.go diff --git a/syft/cataloger/javascript/parse_yarn_lock.go b/syft/pkg/cataloger/javascript/parse_yarn_lock.go similarity index 97% rename from syft/cataloger/javascript/parse_yarn_lock.go rename to syft/pkg/cataloger/javascript/parse_yarn_lock.go index 5e18c5aff..078610d47 100644 --- a/syft/cataloger/javascript/parse_yarn_lock.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock.go @@ -8,8 +8,8 @@ import ( "strings" "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/common" ) // integrity check diff --git a/syft/cataloger/javascript/parse_yarn_lock_test.go b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go similarity index 100% rename from syft/cataloger/javascript/parse_yarn_lock_test.go rename to syft/pkg/cataloger/javascript/parse_yarn_lock_test.go diff --git a/syft/cataloger/javascript/test-fixtures/pkg-json/package-license-object.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-license-object.json similarity index 100% rename from syft/cataloger/javascript/test-fixtures/pkg-json/package-license-object.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-license-object.json diff --git a/syft/cataloger/javascript/test-fixtures/pkg-json/package-license-objects.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-license-objects.json similarity index 100% rename from syft/cataloger/javascript/test-fixtures/pkg-json/package-license-objects.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-license-objects.json diff --git a/syft/cataloger/javascript/test-fixtures/pkg-json/package-nested-author.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-nested-author.json similarity index 100% rename from syft/cataloger/javascript/test-fixtures/pkg-json/package-nested-author.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-nested-author.json diff --git a/syft/cataloger/javascript/test-fixtures/pkg-json/package-no-license.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-no-license.json similarity index 100% rename from syft/cataloger/javascript/test-fixtures/pkg-json/package-no-license.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-no-license.json diff --git a/syft/cataloger/javascript/test-fixtures/pkg-json/package-partial.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-partial.json similarity index 100% rename from syft/cataloger/javascript/test-fixtures/pkg-json/package-partial.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-partial.json diff --git a/syft/cataloger/javascript/test-fixtures/pkg-json/package-repo-string.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-repo-string.json similarity index 100% rename from syft/cataloger/javascript/test-fixtures/pkg-json/package-repo-string.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-repo-string.json diff --git a/syft/cataloger/javascript/test-fixtures/pkg-json/package.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package.json similarity index 100% rename from syft/cataloger/javascript/test-fixtures/pkg-json/package.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package.json diff --git a/syft/cataloger/javascript/test-fixtures/pkg-lock/package-lock.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/package-lock.json similarity index 100% rename from syft/cataloger/javascript/test-fixtures/pkg-lock/package-lock.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/package-lock.json diff --git a/syft/cataloger/javascript/test-fixtures/yarn/yarn.lock b/syft/pkg/cataloger/javascript/test-fixtures/yarn/yarn.lock similarity index 100% rename from syft/cataloger/javascript/test-fixtures/yarn/yarn.lock rename to syft/pkg/cataloger/javascript/test-fixtures/yarn/yarn.lock diff --git a/syft/cataloger/package_url.go b/syft/pkg/cataloger/package_url.go similarity index 100% rename from syft/cataloger/package_url.go rename to syft/pkg/cataloger/package_url.go diff --git a/syft/cataloger/package_url_test.go b/syft/pkg/cataloger/package_url_test.go similarity index 100% rename from syft/cataloger/package_url_test.go rename to syft/pkg/cataloger/package_url_test.go diff --git a/syft/cataloger/python/index_cataloger.go b/syft/pkg/cataloger/python/index_cataloger.go similarity index 92% rename from syft/cataloger/python/index_cataloger.go rename to syft/pkg/cataloger/python/index_cataloger.go index 620892b22..4c821a289 100644 --- a/syft/cataloger/python/index_cataloger.go +++ b/syft/pkg/cataloger/python/index_cataloger.go @@ -4,7 +4,7 @@ Package python provides a concrete Cataloger implementation for Python ecosystem package python import ( - "github.com/anchore/syft/syft/cataloger/common" + "github.com/anchore/syft/syft/pkg/cataloger/common" ) // NewPythonIndexCataloger returns a new cataloger for python packages referenced from poetry lock files, requirements.txt files, and setup.py files. diff --git a/syft/cataloger/python/package_cataloger.go b/syft/pkg/cataloger/python/package_cataloger.go similarity index 50% rename from syft/cataloger/python/package_cataloger.go rename to syft/pkg/cataloger/python/package_cataloger.go index af8360792..980c4b699 100644 --- a/syft/cataloger/python/package_cataloger.go +++ b/syft/pkg/cataloger/python/package_cataloger.go @@ -3,6 +3,7 @@ package python import ( "bufio" "fmt" + "path/filepath" "github.com/anchore/syft/syft/pkg" @@ -28,64 +29,33 @@ func (c *PackageCataloger) Name() string { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations. -func (c *PackageCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { - entries, err := c.getPackageEntries(resolver) - if err != nil { - return nil, err +func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) { + var fileMatches []source.Location + + for _, glob := range []string{eggMetadataGlob, wheelMetadataGlob, eggFileMetadataGlob} { + matches, err := resolver.FilesByGlob(glob) + if err != nil { + return nil, fmt.Errorf("failed to find files by glob: %s", glob) + } + fileMatches = append(fileMatches, matches...) } - var packages []pkg.Package - for _, entry := range entries { - p, err := c.catalogEggOrWheel(entry) + var pkgs []pkg.Package + for _, location := range fileMatches { + p, err := c.catalogEggOrWheel(resolver, location) if err != nil { - return nil, fmt.Errorf("unable to catalog python package=%+v: %w", entry.Metadata.Location.RealPath, err) + return nil, fmt.Errorf("unable to catalog python package=%+v: %w", location.RealPath, err) } if p != nil { - packages = append(packages, *p) + pkgs = append(pkgs, *p) } } - - return packages, nil -} - -// getPackageEntries fetches the contents for all python packages within the given resolver. -func (c *PackageCataloger) getPackageEntries(resolver source.Resolver) ([]*packageEntry, error) { - var metadataLocations []source.Location - - // find all primary record paths - matches, err := resolver.FilesByGlob(eggMetadataGlob, eggFileMetadataGlob, wheelMetadataGlob) - if err != nil { - return nil, fmt.Errorf("failed to find files by glob: %w", err) - } - metadataLocations = append(metadataLocations, matches...) - - // for every primary record path, craft all secondary record paths and build a request object to gather all file contents for each record - requester := source.NewContentRequester() - entries := make([]*packageEntry, len(metadataLocations)) - for i, metadataLocation := range metadataLocations { - // build the entry to process (holding only path information) - entry := newPackageEntry(resolver, metadataLocation) - - // populate the data onto the requester object - requester.Add(&entry.Metadata) - if entry.FileRecord != nil { - requester.Add(entry.FileRecord) - } - if entry.TopPackage != nil { - requester.Add(entry.TopPackage) - } - - // keep track of the entry for later package processing - entries[i] = entry - } - - // return the set of entries and execute the request for fetching contents - return entries, requester.Execute(resolver) + return pkgs, nil } // catalogEggOrWheel takes the primary metadata file reference and returns the python package it represents. -func (c *PackageCataloger) catalogEggOrWheel(entry *packageEntry) (*pkg.Package, error) { - metadata, sources, err := c.assembleEggOrWheelMetadata(entry) +func (c *PackageCataloger) catalogEggOrWheel(resolver source.FileResolver, metadataLocation source.Location) (*pkg.Package, error) { + metadata, sources, err := c.assembleEggOrWheelMetadata(resolver, metadataLocation) if err != nil { return nil, err } @@ -114,45 +84,26 @@ func (c *PackageCataloger) catalogEggOrWheel(entry *packageEntry) (*pkg.Package, }, nil } -// assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from. -func (c *PackageCataloger) assembleEggOrWheelMetadata(entry *packageEntry) (*pkg.PythonPackageMetadata, []source.Location, error) { - var sources = []source.Location{entry.Metadata.Location} - - metadata, err := parseWheelOrEggMetadata(entry.Metadata.Location.RealPath, entry.Metadata.Contents) - if err != nil { - return nil, nil, err - } - - // attach any python files found for the given wheel/egg installation - r, s, err := c.processRecordFiles(entry.FileRecord) - if err != nil { - return nil, nil, err - } - sources = append(sources, s...) - metadata.Files = r - - // attach any top-level package names found for the given wheel/egg installation - p, s, err := c.processTopLevelPackages(entry.TopPackage) - if err != nil { - return nil, nil, err - } - sources = append(sources, s...) - metadata.TopLevelPackages = p - - return &metadata, sources, nil -} - -// processRecordFiles takes a corresponding RECORD file for the given python package metadata file and returns the set of file records contained. -func (c *PackageCataloger) processRecordFiles(entry *source.FileData) (files []pkg.PythonFileRecord, sources []source.Location, err error) { +// fetchRecordFiles finds a corresponding RECORD file for the given python package metadata file and returns the set of file records contained. +func (c *PackageCataloger) fetchRecordFiles(resolver source.FileResolver, metadataLocation source.Location) (files []pkg.PythonFileRecord, sources []source.Location, err error) { // we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory // or for an image... for an image the METADATA file may be present within multiple layers, so it is important // to reconcile the RECORD path to the same layer (or the next adjacent lower layer). - if entry != nil { - sources = append(sources, entry.Location) + // lets find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure) + recordPath := filepath.Join(filepath.Dir(metadataLocation.RealPath), "RECORD") + recordRef := resolver.RelativeFileByPath(metadataLocation, recordPath) + + if recordRef != nil { + sources = append(sources, *recordRef) + + recordContents, err := resolver.FileContentsByLocation(*recordRef) + if err != nil { + return nil, nil, err + } // parse the record contents - records, err := parseWheelOrEggRecord(entry.Contents) + records, err := parseWheelOrEggRecord(recordContents) if err != nil { return nil, nil, err } @@ -162,15 +113,25 @@ func (c *PackageCataloger) processRecordFiles(entry *source.FileData) (files []p return files, sources, nil } -// processTopLevelPackages takes a corresponding top_level.txt file for the given python package metadata file and returns the set of package names contained. -func (c *PackageCataloger) processTopLevelPackages(entry *source.FileData) (pkgs []string, sources []source.Location, err error) { - if entry == nil { +// fetchTopLevelPackages finds a corresponding top_level.txt file for the given python package metadata file and returns the set of package names contained. +func (c *PackageCataloger) fetchTopLevelPackages(resolver source.FileResolver, metadataLocation source.Location) (pkgs []string, sources []source.Location, err error) { + // a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages + parentDir := filepath.Dir(metadataLocation.RealPath) + topLevelPath := filepath.Join(parentDir, "top_level.txt") + topLevelLocation := resolver.RelativeFileByPath(metadataLocation, topLevelPath) + + if topLevelLocation == nil { return nil, nil, nil } - sources = append(sources, entry.Location) + sources = append(sources, *topLevelLocation) - scanner := bufio.NewScanner(entry.Contents) + topLevelContents, err := resolver.FileContentsByLocation(*topLevelLocation) + if err != nil { + return nil, nil, err + } + + scanner := bufio.NewScanner(topLevelContents) for scanner.Scan() { pkgs = append(pkgs, scanner.Text()) } @@ -181,3 +142,36 @@ func (c *PackageCataloger) processTopLevelPackages(entry *source.FileData) (pkgs return pkgs, sources, nil } + +// assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from. +func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.FileResolver, metadataLocation source.Location) (*pkg.PythonPackageMetadata, []source.Location, error) { + var sources = []source.Location{metadataLocation} + + metadataContents, err := resolver.FileContentsByLocation(metadataLocation) + if err != nil { + return nil, nil, err + } + + metadata, err := parseWheelOrEggMetadata(metadataLocation.RealPath, metadataContents) + if err != nil { + return nil, nil, err + } + + // attach any python files found for the given wheel/egg installation + r, s, err := c.fetchRecordFiles(resolver, metadataLocation) + if err != nil { + return nil, nil, err + } + sources = append(sources, s...) + metadata.Files = r + + // attach any top-level package names found for the given wheel/egg installation + p, s, err := c.fetchTopLevelPackages(resolver, metadataLocation) + if err != nil { + return nil, nil, err + } + sources = append(sources, s...) + metadata.TopLevelPackages = p + + return &metadata, sources, nil +} diff --git a/syft/cataloger/python/package_cataloger_test.go b/syft/pkg/cataloger/python/package_cataloger_test.go similarity index 100% rename from syft/cataloger/python/package_cataloger_test.go rename to syft/pkg/cataloger/python/package_cataloger_test.go diff --git a/syft/cataloger/python/parse_poetry_lock.go b/syft/pkg/cataloger/python/parse_poetry_lock.go similarity index 92% rename from syft/cataloger/python/parse_poetry_lock.go rename to syft/pkg/cataloger/python/parse_poetry_lock.go index 6b08098a4..b6981a1b0 100644 --- a/syft/cataloger/python/parse_poetry_lock.go +++ b/syft/pkg/cataloger/python/parse_poetry_lock.go @@ -4,8 +4,8 @@ import ( "fmt" "io" - "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/common" "github.com/pelletier/go-toml" ) diff --git a/syft/cataloger/python/parse_poetry_lock_test.go b/syft/pkg/cataloger/python/parse_poetry_lock_test.go similarity index 100% rename from syft/cataloger/python/parse_poetry_lock_test.go rename to syft/pkg/cataloger/python/parse_poetry_lock_test.go diff --git a/syft/cataloger/python/parse_requirements.go b/syft/pkg/cataloger/python/parse_requirements.go similarity index 97% rename from syft/cataloger/python/parse_requirements.go rename to syft/pkg/cataloger/python/parse_requirements.go index 27ec43cc8..b206224dd 100644 --- a/syft/cataloger/python/parse_requirements.go +++ b/syft/pkg/cataloger/python/parse_requirements.go @@ -6,8 +6,8 @@ import ( "io" "strings" - "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/common" ) // integrity check diff --git a/syft/cataloger/python/parse_requirements_test.go b/syft/pkg/cataloger/python/parse_requirements_test.go similarity index 100% rename from syft/cataloger/python/parse_requirements_test.go rename to syft/pkg/cataloger/python/parse_requirements_test.go diff --git a/syft/cataloger/python/parse_setup.go b/syft/pkg/cataloger/python/parse_setup.go similarity index 95% rename from syft/cataloger/python/parse_setup.go rename to syft/pkg/cataloger/python/parse_setup.go index 337c436e7..e1b0c39ce 100644 --- a/syft/cataloger/python/parse_setup.go +++ b/syft/pkg/cataloger/python/parse_setup.go @@ -6,8 +6,8 @@ import ( "regexp" "strings" - "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/common" ) // integrity check diff --git a/syft/cataloger/python/parse_setup_test.go b/syft/pkg/cataloger/python/parse_setup_test.go similarity index 100% rename from syft/cataloger/python/parse_setup_test.go rename to syft/pkg/cataloger/python/parse_setup_test.go diff --git a/syft/cataloger/python/parse_wheel_egg_metadata.go b/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go similarity index 100% rename from syft/cataloger/python/parse_wheel_egg_metadata.go rename to syft/pkg/cataloger/python/parse_wheel_egg_metadata.go diff --git a/syft/cataloger/python/parse_wheel_egg_metadata_test.go b/syft/pkg/cataloger/python/parse_wheel_egg_metadata_test.go similarity index 100% rename from syft/cataloger/python/parse_wheel_egg_metadata_test.go rename to syft/pkg/cataloger/python/parse_wheel_egg_metadata_test.go diff --git a/syft/cataloger/python/parse_wheel_egg_record.go b/syft/pkg/cataloger/python/parse_wheel_egg_record.go similarity index 100% rename from syft/cataloger/python/parse_wheel_egg_record.go rename to syft/pkg/cataloger/python/parse_wheel_egg_record.go diff --git a/syft/cataloger/python/parse_wheel_egg_record_test.go b/syft/pkg/cataloger/python/parse_wheel_egg_record_test.go similarity index 100% rename from syft/cataloger/python/parse_wheel_egg_record_test.go rename to syft/pkg/cataloger/python/parse_wheel_egg_record_test.go diff --git a/syft/cataloger/python/poetry_metadata.go b/syft/pkg/cataloger/python/poetry_metadata.go similarity index 100% rename from syft/cataloger/python/poetry_metadata.go rename to syft/pkg/cataloger/python/poetry_metadata.go diff --git a/syft/cataloger/python/poetry_metadata_package.go b/syft/pkg/cataloger/python/poetry_metadata_package.go similarity index 100% rename from syft/cataloger/python/poetry_metadata_package.go rename to syft/pkg/cataloger/python/poetry_metadata_package.go diff --git a/syft/cataloger/python/test-fixtures/Python-2.7.egg-info b/syft/pkg/cataloger/python/test-fixtures/Python-2.7.egg-info similarity index 100% rename from syft/cataloger/python/test-fixtures/Python-2.7.egg-info rename to syft/pkg/cataloger/python/test-fixtures/Python-2.7.egg-info diff --git a/syft/cataloger/python/test-fixtures/dist-info/METADATA b/syft/pkg/cataloger/python/test-fixtures/dist-info/METADATA similarity index 100% rename from syft/cataloger/python/test-fixtures/dist-info/METADATA rename to syft/pkg/cataloger/python/test-fixtures/dist-info/METADATA diff --git a/syft/cataloger/python/test-fixtures/dist-info/RECORD b/syft/pkg/cataloger/python/test-fixtures/dist-info/RECORD similarity index 100% rename from syft/cataloger/python/test-fixtures/dist-info/RECORD rename to syft/pkg/cataloger/python/test-fixtures/dist-info/RECORD diff --git a/syft/cataloger/python/test-fixtures/dist-info/top_level.txt b/syft/pkg/cataloger/python/test-fixtures/dist-info/top_level.txt similarity index 100% rename from syft/cataloger/python/test-fixtures/dist-info/top_level.txt rename to syft/pkg/cataloger/python/test-fixtures/dist-info/top_level.txt diff --git a/syft/cataloger/python/test-fixtures/egg-info/PKG-INFO b/syft/pkg/cataloger/python/test-fixtures/egg-info/PKG-INFO similarity index 100% rename from syft/cataloger/python/test-fixtures/egg-info/PKG-INFO rename to syft/pkg/cataloger/python/test-fixtures/egg-info/PKG-INFO diff --git a/syft/cataloger/python/test-fixtures/egg-info/PKG-INFO-INVALID b/syft/pkg/cataloger/python/test-fixtures/egg-info/PKG-INFO-INVALID similarity index 100% rename from syft/cataloger/python/test-fixtures/egg-info/PKG-INFO-INVALID rename to syft/pkg/cataloger/python/test-fixtures/egg-info/PKG-INFO-INVALID diff --git a/syft/cataloger/python/test-fixtures/egg-info/RECORD b/syft/pkg/cataloger/python/test-fixtures/egg-info/RECORD similarity index 100% rename from syft/cataloger/python/test-fixtures/egg-info/RECORD rename to syft/pkg/cataloger/python/test-fixtures/egg-info/RECORD diff --git a/syft/cataloger/python/test-fixtures/egg-info/top_level.txt b/syft/pkg/cataloger/python/test-fixtures/egg-info/top_level.txt similarity index 100% rename from syft/cataloger/python/test-fixtures/egg-info/top_level.txt rename to syft/pkg/cataloger/python/test-fixtures/egg-info/top_level.txt diff --git a/syft/cataloger/python/test-fixtures/partial.dist-info/METADATA b/syft/pkg/cataloger/python/test-fixtures/partial.dist-info/METADATA similarity index 100% rename from syft/cataloger/python/test-fixtures/partial.dist-info/METADATA rename to syft/pkg/cataloger/python/test-fixtures/partial.dist-info/METADATA diff --git a/syft/cataloger/python/test-fixtures/poetry/poetry.lock b/syft/pkg/cataloger/python/test-fixtures/poetry/poetry.lock similarity index 100% rename from syft/cataloger/python/test-fixtures/poetry/poetry.lock rename to syft/pkg/cataloger/python/test-fixtures/poetry/poetry.lock diff --git a/syft/cataloger/python/test-fixtures/requires/requirements.txt b/syft/pkg/cataloger/python/test-fixtures/requires/requirements.txt similarity index 100% rename from syft/cataloger/python/test-fixtures/requires/requirements.txt rename to syft/pkg/cataloger/python/test-fixtures/requires/requirements.txt diff --git a/syft/cataloger/python/test-fixtures/setup/setup.py b/syft/pkg/cataloger/python/test-fixtures/setup/setup.py similarity index 100% rename from syft/cataloger/python/test-fixtures/setup/setup.py rename to syft/pkg/cataloger/python/test-fixtures/setup/setup.py diff --git a/syft/cataloger/python/test-fixtures/test.egg-info b/syft/pkg/cataloger/python/test-fixtures/test.egg-info similarity index 100% rename from syft/cataloger/python/test-fixtures/test.egg-info rename to syft/pkg/cataloger/python/test-fixtures/test.egg-info diff --git a/syft/cataloger/rpmdb/cataloger.go b/syft/pkg/cataloger/rpmdb/cataloger.go similarity index 93% rename from syft/cataloger/rpmdb/cataloger.go rename to syft/pkg/cataloger/rpmdb/cataloger.go index 6f7a93288..eca4c60c7 100644 --- a/syft/cataloger/rpmdb/cataloger.go +++ b/syft/pkg/cataloger/rpmdb/cataloger.go @@ -25,7 +25,7 @@ func (c *Cataloger) Name() string { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation. -func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) { +func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) { fileMatches, err := resolver.FilesByGlob(pkg.RpmDbGlob) if err != nil { return nil, fmt.Errorf("failed to find rpmdb's by glob: %w", err) diff --git a/syft/cataloger/rpmdb/parse_rpmdb.go b/syft/pkg/cataloger/rpmdb/parse_rpmdb.go similarity index 90% rename from syft/cataloger/rpmdb/parse_rpmdb.go rename to syft/pkg/cataloger/rpmdb/parse_rpmdb.go index f67a123e7..103de4f5f 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb.go +++ b/syft/pkg/cataloger/rpmdb/parse_rpmdb.go @@ -14,7 +14,7 @@ import ( ) // parseApkDb parses an "Packages" RPM DB and returns the Packages listed within it. -func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader io.Reader) ([]pkg.Package, error) { +func parseRpmDB(resolver source.FilePathResolver, dbLocation source.Location, reader io.Reader) ([]pkg.Package, error) { f, err := ioutil.TempFile("", internal.ApplicationName+"-rpmdb") if err != nil { return nil, fmt.Errorf("failed to create temp rpmdb file: %w", err) @@ -72,7 +72,7 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader return allPkgs, nil } -func extractRpmdbFileRecords(resolver source.FileResolver, entry *rpmdb.PackageInfo) []pkg.RpmdbFileRecord { +func extractRpmdbFileRecords(resolver source.FilePathResolver, entry *rpmdb.PackageInfo) []pkg.RpmdbFileRecord { var records = make([]pkg.RpmdbFileRecord, 0) for _, record := range entry.Files { diff --git a/syft/cataloger/rpmdb/parse_rpmdb_test.go b/syft/pkg/cataloger/rpmdb/parse_rpmdb_test.go similarity index 100% rename from syft/cataloger/rpmdb/parse_rpmdb_test.go rename to syft/pkg/cataloger/rpmdb/parse_rpmdb_test.go diff --git a/syft/cataloger/rpmdb/test-fixtures/Packages b/syft/pkg/cataloger/rpmdb/test-fixtures/Packages similarity index 100% rename from syft/cataloger/rpmdb/test-fixtures/Packages rename to syft/pkg/cataloger/rpmdb/test-fixtures/Packages diff --git a/syft/cataloger/rpmdb/test-fixtures/generate-fixture.sh b/syft/pkg/cataloger/rpmdb/test-fixtures/generate-fixture.sh similarity index 100% rename from syft/cataloger/rpmdb/test-fixtures/generate-fixture.sh rename to syft/pkg/cataloger/rpmdb/test-fixtures/generate-fixture.sh diff --git a/syft/cataloger/ruby/catalogers.go b/syft/pkg/cataloger/ruby/catalogers.go similarity index 94% rename from syft/cataloger/ruby/catalogers.go rename to syft/pkg/cataloger/ruby/catalogers.go index e9d9b7cec..6e7499fce 100644 --- a/syft/cataloger/ruby/catalogers.go +++ b/syft/pkg/cataloger/ruby/catalogers.go @@ -4,7 +4,7 @@ Package bundler provides a concrete Cataloger implementation for Ruby Gemfile.lo package ruby import ( - "github.com/anchore/syft/syft/cataloger/common" + "github.com/anchore/syft/syft/pkg/cataloger/common" ) // NewGemFileLockCataloger returns a new Bundler cataloger object tailored for parsing index-oriented files (e.g. Gemfile.lock). diff --git a/syft/cataloger/ruby/parse_gemfile_lock.go b/syft/pkg/cataloger/ruby/parse_gemfile_lock.go similarity index 96% rename from syft/cataloger/ruby/parse_gemfile_lock.go rename to syft/pkg/cataloger/ruby/parse_gemfile_lock.go index 23a4a7756..b27b3ba66 100644 --- a/syft/cataloger/ruby/parse_gemfile_lock.go +++ b/syft/pkg/cataloger/ruby/parse_gemfile_lock.go @@ -6,8 +6,8 @@ import ( "strings" "github.com/anchore/syft/internal" - "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/common" ) // integrity check diff --git a/syft/cataloger/ruby/parse_gemfile_lock_test.go b/syft/pkg/cataloger/ruby/parse_gemfile_lock_test.go similarity index 100% rename from syft/cataloger/ruby/parse_gemfile_lock_test.go rename to syft/pkg/cataloger/ruby/parse_gemfile_lock_test.go diff --git a/syft/cataloger/ruby/parse_gemspec.go b/syft/pkg/cataloger/ruby/parse_gemspec.go similarity index 98% rename from syft/cataloger/ruby/parse_gemspec.go rename to syft/pkg/cataloger/ruby/parse_gemspec.go index 239d2a4eb..fc99f6710 100644 --- a/syft/cataloger/ruby/parse_gemspec.go +++ b/syft/pkg/cataloger/ruby/parse_gemspec.go @@ -12,8 +12,8 @@ import ( "github.com/mitchellh/mapstructure" - "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/common" ) // integrity check @@ -53,7 +53,6 @@ var postProcessors = map[string]postProcessor{ } func processList(s string) []string { - // nolint:prealloc var results []string for _, item := range strings.Split(s, ",") { results = append(results, strings.Trim(item, "\" ")) diff --git a/syft/cataloger/ruby/parse_gemspec_test.go b/syft/pkg/cataloger/ruby/parse_gemspec_test.go similarity index 100% rename from syft/cataloger/ruby/parse_gemspec_test.go rename to syft/pkg/cataloger/ruby/parse_gemspec_test.go diff --git a/syft/cataloger/ruby/test-fixtures/Gemfile.lock b/syft/pkg/cataloger/ruby/test-fixtures/Gemfile.lock similarity index 100% rename from syft/cataloger/ruby/test-fixtures/Gemfile.lock rename to syft/pkg/cataloger/ruby/test-fixtures/Gemfile.lock diff --git a/syft/cataloger/ruby/test-fixtures/bundler.gemspec b/syft/pkg/cataloger/ruby/test-fixtures/bundler.gemspec similarity index 100% rename from syft/cataloger/ruby/test-fixtures/bundler.gemspec rename to syft/pkg/cataloger/ruby/test-fixtures/bundler.gemspec diff --git a/syft/cataloger/rust/cargo_metadata.go b/syft/pkg/cataloger/rust/cargo_metadata.go similarity index 100% rename from syft/cataloger/rust/cargo_metadata.go rename to syft/pkg/cataloger/rust/cargo_metadata.go diff --git a/syft/cataloger/rust/cataloger.go b/syft/pkg/cataloger/rust/cataloger.go similarity index 88% rename from syft/cataloger/rust/cataloger.go rename to syft/pkg/cataloger/rust/cataloger.go index bd2625c5d..df0f9ee40 100644 --- a/syft/cataloger/rust/cataloger.go +++ b/syft/pkg/cataloger/rust/cataloger.go @@ -4,7 +4,7 @@ Package rust provides a concrete Cataloger implementation for Cargo.lock files. package rust import ( - "github.com/anchore/syft/syft/cataloger/common" + "github.com/anchore/syft/syft/pkg/cataloger/common" ) // NewCargoLockCataloger returns a new Rust Cargo lock file cataloger object. diff --git a/syft/cataloger/rust/parse_cargo_lock.go b/syft/pkg/cataloger/rust/parse_cargo_lock.go similarity index 92% rename from syft/cataloger/rust/parse_cargo_lock.go rename to syft/pkg/cataloger/rust/parse_cargo_lock.go index 2ec0b834a..ccb7dafd8 100644 --- a/syft/cataloger/rust/parse_cargo_lock.go +++ b/syft/pkg/cataloger/rust/parse_cargo_lock.go @@ -4,8 +4,8 @@ import ( "fmt" "io" - "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/common" "github.com/pelletier/go-toml" ) diff --git a/syft/cataloger/rust/parse_cargo_lock_test.go b/syft/pkg/cataloger/rust/parse_cargo_lock_test.go similarity index 100% rename from syft/cataloger/rust/parse_cargo_lock_test.go rename to syft/pkg/cataloger/rust/parse_cargo_lock_test.go diff --git a/syft/cataloger/rust/test-fixtures/Cargo.lock b/syft/pkg/cataloger/rust/test-fixtures/Cargo.lock similarity index 100% rename from syft/cataloger/rust/test-fixtures/Cargo.lock rename to syft/pkg/cataloger/rust/test-fixtures/Cargo.lock diff --git a/syft/pkg/ownership_by_files_relationship.go b/syft/pkg/ownership_by_files_relationship.go index 0151ee2b4..f235557a4 100644 --- a/syft/pkg/ownership_by_files_relationship.go +++ b/syft/pkg/ownership_by_files_relationship.go @@ -45,6 +45,10 @@ func ownershipByFilesRelationships(catalog *Catalog) []Relationship { func findOwnershipByFilesRelationships(catalog *Catalog) map[ID]map[ID]*strset.Set { var relationships = make(map[ID]map[ID]*strset.Set) + if catalog == nil { + return relationships + } + for _, candidateOwnerPkg := range catalog.Sorted() { if candidateOwnerPkg.Metadata == nil { continue diff --git a/syft/presenter/cyclonedx/bom-descriptor.go b/syft/presenter/cyclonedx/bom-descriptor.go deleted file mode 100644 index a9a3301dd..000000000 --- a/syft/presenter/cyclonedx/bom-descriptor.go +++ /dev/null @@ -1,69 +0,0 @@ -package cyclonedx - -import ( - "encoding/xml" - "time" - - "github.com/anchore/syft/syft/source" -) - -// Source: https://cyclonedx.org/ext/bom-descriptor/ - -// BomDescriptor represents all metadata surrounding the BOM report (such as when the BOM was made, with which tool, and the item being cataloged). -type BomDescriptor struct { - XMLName xml.Name `xml:"metadata"` - Timestamp string `xml:"timestamp,omitempty"` // The date and time (timestamp) when the document was created - Tools []BdTool `xml:"tools>tool"` // The tool used to create the BOM. - Component *BdComponent `xml:"component"` // The component that the BOM describes. -} - -// BdTool represents the tool that created the BOM report. -type BdTool struct { - XMLName xml.Name `xml:"tool"` - Vendor string `xml:"vendor,omitempty"` // The vendor of the tool used to create the BOM. - Name string `xml:"name,omitempty"` // The name of the tool used to create the BOM. - Version string `xml:"version,omitempty"` // The version of the tool used to create the BOM. - // TODO: hashes, author, manufacture, supplier - // TODO: add user-defined fields for the remaining build/version parameters -} - -// BdComponent represents the software/package being cataloged. -type BdComponent struct { - XMLName xml.Name `xml:"component"` - Component -} - -// NewBomDescriptor returns a new BomDescriptor tailored for the current time and "syft" tool details. -func NewBomDescriptor(name, version string, srcMetadata source.Metadata) *BomDescriptor { - descriptor := BomDescriptor{ - XMLName: xml.Name{}, - Timestamp: time.Now().Format(time.RFC3339), - Tools: []BdTool{ - { - Vendor: "anchore", - Name: name, - Version: version, - }, - }, - } - - switch srcMetadata.Scheme { - case source.ImageScheme: - descriptor.Component = &BdComponent{ - Component: Component{ - Type: "container", - Name: srcMetadata.ImageMetadata.UserInput, - Version: srcMetadata.ImageMetadata.ManifestDigest, - }, - } - case source.DirectoryScheme: - descriptor.Component = &BdComponent{ - Component: Component{ - Type: "file", - Name: srcMetadata.Path, - }, - } - } - - return &descriptor -} diff --git a/syft/presenter/cyclonedx/component.go b/syft/presenter/cyclonedx/component.go deleted file mode 100644 index e540ef9ea..000000000 --- a/syft/presenter/cyclonedx/component.go +++ /dev/null @@ -1,27 +0,0 @@ -package cyclonedx - -import "encoding/xml" - -// Component represents a single element in the CycloneDX BOM -type Component struct { - XMLName xml.Name `xml:"component"` - Type string `xml:"type,attr"` // Required; Describes if the component is a library, framework, application, container, operating system, firmware, hardware device, or file - Supplier string `xml:"supplier,omitempty"` // The organization that supplied the component. The supplier may often be the manufacture, but may also be a distributor or repackager. - Author string `xml:"author,omitempty"` // The person(s) or organization(s) that authored the component - Publisher string `xml:"publisher,omitempty"` // The person(s) or organization(s) that published the component - Group string `xml:"group,omitempty"` // The high-level classification that a project self-describes as. This will often be a shortened, single name of the company or project that produced the component, or the source package or domain name. - Name string `xml:"name"` // Required; The name of the component as defined by the project - Version string `xml:"version"` // Required; The version of the component as defined by the project - Description string `xml:"description,omitempty"` // A description of the component - Licenses *[]License `xml:"licenses>license"` // A node describing zero or more license names, SPDX license IDs or expressions - PackageURL string `xml:"purl,omitempty"` // Specifies the package-url (PackageURL). The purl, if specified, must be valid and conform to the specification defined at: https://github.com/package-url/purl-spec - // TODO: source, hashes, copyright, cpe, purl, swid, modified, pedigree, externalReferences - // TODO: add user-defined parameters for syft-specific values (image layer index, cataloger, location path, etc.) -} - -// License represents a single software license for a Component -type License struct { - XMLName xml.Name `xml:"license"` - ID string `xml:"id,omitempty"` // A valid SPDX license ID - Name string `xml:"name,omitempty"` // If SPDX does not define the license used, this field may be used to provide the license name -} diff --git a/syft/presenter/cyclonedx/document.go b/syft/presenter/cyclonedx/document.go deleted file mode 100644 index 8791d3f9c..000000000 --- a/syft/presenter/cyclonedx/document.go +++ /dev/null @@ -1,57 +0,0 @@ -package cyclonedx - -import ( - "encoding/xml" - - "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/version" - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/source" - "github.com/google/uuid" -) - -// Source: https://github.com/CycloneDX/specification - -// Document represents a CycloneDX BOM Document. -type Document struct { - XMLName xml.Name `xml:"bom"` - XMLNs string `xml:"xmlns,attr"` - Version int `xml:"version,attr"` - SerialNumber string `xml:"serialNumber,attr"` - BomDescriptor *BomDescriptor `xml:"metadata"` // The BOM descriptor extension - Components []Component `xml:"components>component"` // The BOM contents -} - -// NewDocumentFromCatalog returns a CycloneDX Document object populated with the catalog contents. -func NewDocument(catalog *pkg.Catalog, srcMetadata source.Metadata) Document { - versionInfo := version.FromBuild() - - doc := Document{ - XMLNs: "http://cyclonedx.org/schema/bom/1.2", - Version: 1, - SerialNumber: uuid.New().URN(), - BomDescriptor: NewBomDescriptor(internal.ApplicationName, versionInfo.Version, srcMetadata), - } - - // attach components - for p := range catalog.Enumerate() { - component := Component{ - Type: "library", // TODO: this is not accurate - Name: p.Name, - Version: p.Version, - PackageURL: p.PURL, - } - var licenses []License - for _, licenseName := range p.Licenses { - licenses = append(licenses, License{ - Name: licenseName, - }) - } - if len(licenses) > 0 { - component.Licenses = &licenses - } - doc.Components = append(doc.Components, component) - } - - return doc -} diff --git a/syft/presenter/cyclonedx/test-fixtures/image-simple/Dockerfile b/syft/presenter/cyclonedx/test-fixtures/image-simple/Dockerfile deleted file mode 100644 index 62fb151e4..000000000 --- a/syft/presenter/cyclonedx/test-fixtures/image-simple/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -# Note: changes to this file will result in updating several test values. Consider making a new image fixture instead of editing this one. -FROM scratch -ADD file-1.txt /somefile-1.txt -ADD file-2.txt /somefile-2.txt -# note: adding a directory will behave differently on docker engine v18 vs v19 -ADD target / diff --git a/syft/presenter/cyclonedx/test-fixtures/image-simple/target/really/nested/file-3.txt b/syft/presenter/cyclonedx/test-fixtures/image-simple/target/really/nested/file-3.txt deleted file mode 100644 index f85472c93..000000000 --- a/syft/presenter/cyclonedx/test-fixtures/image-simple/target/really/nested/file-3.txt +++ /dev/null @@ -1,2 +0,0 @@ -another file! -with lines... \ No newline at end of file diff --git a/syft/presenter/json/descriptor.go b/syft/presenter/json/descriptor.go deleted file mode 100644 index de07721fc..000000000 --- a/syft/presenter/json/descriptor.go +++ /dev/null @@ -1,7 +0,0 @@ -package json - -// Descriptor describes what created the document as well as surrounding metadata -type Descriptor struct { - Name string `json:"name"` - Version string `json:"version"` -} diff --git a/syft/presenter/json/document.go b/syft/presenter/json/document.go deleted file mode 100644 index 5f04cdfcf..000000000 --- a/syft/presenter/json/document.go +++ /dev/null @@ -1,54 +0,0 @@ -package json - -import ( - "fmt" - - "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/version" - "github.com/anchore/syft/syft/distro" - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/source" -) - -// Document represents the syft cataloging findings as a JSON document -type Document struct { - Artifacts []Package `json:"artifacts"` // Artifacts is the list of packages discovered and placed into the catalog - Source Source `json:"source"` // Source represents the original object that was cataloged - Distro Distribution `json:"distro"` // Distro represents the Linux distribution that was detected from the source - Descriptor Descriptor `json:"descriptor"` // Descriptor is a block containing self-describing information about syft - Schema Schema `json:"schema"` // Schema is a block reserved for defining the version for the shape of this JSON document and where to find the schema document to validate the shape - ArtifactRelationships []Relationship `json:"artifactRelationships"` -} - -// NewDocument creates and populates a new JSON document struct from the given cataloging results. -func NewDocument(catalog *pkg.Catalog, srcMetadata source.Metadata, d *distro.Distro) (Document, error) { - src, err := NewSource(srcMetadata) - if err != nil { - return Document{}, nil - } - - doc := Document{ - Artifacts: make([]Package, 0), - Source: src, - Distro: NewDistribution(d), - Descriptor: Descriptor{ - Name: internal.ApplicationName, - Version: version.FromBuild().Version, - }, - Schema: Schema{ - Version: internal.JSONSchemaVersion, - URL: fmt.Sprintf("https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-%s.json", internal.JSONSchemaVersion), - }, - ArtifactRelationships: newRelationships(pkg.NewRelationships(catalog)), - } - - for _, p := range catalog.Sorted() { - art, err := NewPackage(p) - if err != nil { - return Document{}, err - } - doc.Artifacts = append(doc.Artifacts, art) - } - - return doc, nil -} diff --git a/syft/presenter/json/package.go b/syft/presenter/json/package.go deleted file mode 100644 index 456af01ba..000000000 --- a/syft/presenter/json/package.go +++ /dev/null @@ -1,181 +0,0 @@ -package json - -import ( - "encoding/json" - "fmt" - - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/source" -) - -// Package represents a pkg.Package object specialized for JSON marshaling and unmarshaling. -type Package struct { - packageBasicMetadata - packageCustomMetadata -} - -// packageBasicMetadata contains non-ambiguous values (type-wise) from pkg.Package. -type packageBasicMetadata struct { - ID string `json:"id"` - Name string `json:"name"` - Version string `json:"version"` - Type string `json:"type"` - FoundBy string `json:"foundBy"` - Locations []source.Location `json:"locations"` - Licenses []string `json:"licenses"` - Language string `json:"language"` - CPEs []string `json:"cpes"` - PURL string `json:"purl"` -} - -// packageCustomMetadata contains ambiguous values (type-wise) from pkg.Package. -type packageCustomMetadata struct { - MetadataType string `json:"metadataType"` - Metadata interface{} `json:"metadata"` -} - -// packageMetadataUnpacker is all values needed from Package to disambiguate ambiguous fields during json unmarshaling. -type packageMetadataUnpacker struct { - MetadataType string `json:"metadataType"` - Metadata json.RawMessage `json:"metadata"` -} - -// NewPackage crates a new Package from the given pkg.Package. -func NewPackage(p *pkg.Package) (Package, error) { - var cpes = make([]string, len(p.CPEs)) - for i, c := range p.CPEs { - cpes[i] = c.BindToFmtString() - } - - // ensure collections are never nil for presentation reasons - - var locations = make([]source.Location, 0) - if p.Locations != nil { - locations = p.Locations - } - - var licenses = make([]string, 0) - if p.Licenses != nil { - licenses = p.Licenses - } - - return Package{ - packageBasicMetadata: packageBasicMetadata{ - ID: string(p.ID), - Name: p.Name, - Version: p.Version, - Type: string(p.Type), - FoundBy: p.FoundBy, - Locations: locations, - Licenses: licenses, - Language: string(p.Language), - CPEs: cpes, - PURL: p.PURL, - }, - packageCustomMetadata: packageCustomMetadata{ - MetadataType: string(p.MetadataType), - Metadata: p.Metadata, - }, - }, nil -} - -// ToPackage generates a pkg.Package from the current Package. -func (a Package) ToPackage() (pkg.Package, error) { - var cpes = make([]pkg.CPE, len(a.CPEs)) - var err error - for i, c := range a.CPEs { - cpes[i], err = pkg.NewCPE(c) - if err != nil { - return pkg.Package{}, fmt.Errorf("unable to parse CPE from JSON package: %w", err) - } - } - return pkg.Package{ - // does not include found-by and locations - ID: pkg.ID(a.ID), - Name: a.Name, - Version: a.Version, - FoundBy: a.FoundBy, - Licenses: a.Licenses, - Language: pkg.Language(a.Language), - Locations: a.Locations, - CPEs: cpes, - PURL: a.PURL, - Type: pkg.Type(a.Type), - MetadataType: pkg.MetadataType(a.MetadataType), - Metadata: a.Metadata, - }, nil -} - -// UnmarshalJSON is a custom unmarshaller for handling basic values and values with ambiguous types. -// nolint:funlen -func (a *Package) UnmarshalJSON(b []byte) error { - var basic packageBasicMetadata - if err := json.Unmarshal(b, &basic); err != nil { - return err - } - a.packageBasicMetadata = basic - - var unpacker packageMetadataUnpacker - if err := json.Unmarshal(b, &unpacker); err != nil { - return err - } - - a.MetadataType = unpacker.MetadataType - - switch pkg.MetadataType(a.MetadataType) { - case pkg.RpmdbMetadataType: - var payload pkg.RpmdbMetadata - if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { - return err - } - a.Metadata = payload - case pkg.PythonPackageMetadataType: - var payload pkg.PythonPackageMetadata - if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { - return err - } - a.Metadata = payload - case pkg.DpkgMetadataType: - var payload pkg.DpkgMetadata - if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { - return err - } - a.Metadata = payload - case pkg.ApkMetadataType: - var payload pkg.ApkMetadata - if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { - return err - } - a.Metadata = payload - case pkg.JavaMetadataType: - var payload pkg.JavaMetadata - if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { - return err - } - a.Metadata = payload - case pkg.NpmPackageJSONMetadataType: - var payload pkg.NpmPackageJSONMetadata - if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { - return err - } - a.Metadata = payload - case pkg.GemMetadataType: - var payload pkg.GemMetadata - if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { - return err - } - a.Metadata = payload - case pkg.RustCargoPackageMetadataType: - var payload pkg.CargoPackageMetadata - if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil { - return err - } - a.Metadata = payload - case "": - // there may be packages with no metadata, which is OK - default: - return fmt.Errorf("unsupported package metadata type: %+v", a.MetadataType) - } - - return nil -} diff --git a/syft/presenter/json/presenter.go b/syft/presenter/json/presenter.go deleted file mode 100644 index 845ecd1bf..000000000 --- a/syft/presenter/json/presenter.go +++ /dev/null @@ -1,40 +0,0 @@ -package json - -import ( - "encoding/json" - "io" - - "github.com/anchore/syft/syft/distro" - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/source" -) - -// Presenter is a JSON presentation object for the syft results -type Presenter struct { - catalog *pkg.Catalog - srcMetadata source.Metadata - distro *distro.Distro -} - -// NewPresenter creates a new JSON presenter object for the given cataloging results. -func NewPresenter(catalog *pkg.Catalog, s source.Metadata, d *distro.Distro) *Presenter { - return &Presenter{ - catalog: catalog, - srcMetadata: s, - distro: d, - } -} - -// Present the catalog results to the given writer. -func (pres *Presenter) Present(output io.Writer) error { - doc, err := NewDocument(pres.catalog, pres.srcMetadata, pres.distro) - if err != nil { - return err - } - - enc := json.NewEncoder(output) - // prevent > and < from being escaped in the payload - enc.SetEscapeHTML(false) - enc.SetIndent("", " ") - return enc.Encode(&doc) -} diff --git a/syft/presenter/json/schema.go b/syft/presenter/json/schema.go deleted file mode 100644 index d7f8d27ec..000000000 --- a/syft/presenter/json/schema.go +++ /dev/null @@ -1,6 +0,0 @@ -package json - -type Schema struct { - Version string `json:"version"` - URL string `json:"url"` -} diff --git a/syft/presenter/json/source.go b/syft/presenter/json/source.go deleted file mode 100644 index 7a5ad6ad2..000000000 --- a/syft/presenter/json/source.go +++ /dev/null @@ -1,75 +0,0 @@ -package json - -import ( - "encoding/json" - "fmt" - - "github.com/anchore/syft/syft/source" -) - -// Source object represents the thing that was cataloged -type Source struct { - Type string `json:"type"` - Target interface{} `json:"target"` -} - -// sourceUnpacker is used to unmarshal Source objects -type sourceUnpacker struct { - Type string `json:"type"` - Target json.RawMessage `json:"target"` -} - -// NewSource creates a new source object to be represented into JSON. -func NewSource(src source.Metadata) (Source, error) { - switch src.Scheme { - case source.ImageScheme: - return Source{ - Type: "image", - Target: src.ImageMetadata, - }, nil - case source.DirectoryScheme: - return Source{ - Type: "directory", - Target: src.Path, - }, nil - default: - return Source{}, fmt.Errorf("unsupported source: %T", src) - } -} - -// UnmarshalJSON populates a source object from JSON bytes. -func (s *Source) UnmarshalJSON(b []byte) error { - var unpacker sourceUnpacker - if err := json.Unmarshal(b, &unpacker); err != nil { - return err - } - - s.Type = unpacker.Type - - switch s.Type { - case "image": - var payload source.ImageMetadata - if err := json.Unmarshal(unpacker.Target, &payload); err != nil { - return err - } - s.Target = payload - default: - return fmt.Errorf("unsupported package metadata type: %+v", s.Type) - } - - return nil -} - -// ToSourceMetadata takes a source object represented from JSON and creates a source.Metadata object. -func (s *Source) ToSourceMetadata() source.Metadata { - var metadata source.Metadata - switch s.Type { - case "directory": - metadata.Scheme = source.DirectoryScheme - metadata.Path = s.Target.(string) - case "image": - metadata.Scheme = source.ImageScheme - metadata.ImageMetadata = s.Target.(source.ImageMetadata) - } - return metadata -} diff --git a/syft/presenter/json/test-fixtures/image-simple/Dockerfile b/syft/presenter/json/test-fixtures/image-simple/Dockerfile deleted file mode 100644 index 62fb151e4..000000000 --- a/syft/presenter/json/test-fixtures/image-simple/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -# Note: changes to this file will result in updating several test values. Consider making a new image fixture instead of editing this one. -FROM scratch -ADD file-1.txt /somefile-1.txt -ADD file-2.txt /somefile-2.txt -# note: adding a directory will behave differently on docker engine v18 vs v19 -ADD target / diff --git a/syft/presenter/json/test-fixtures/image-simple/file-1.txt b/syft/presenter/json/test-fixtures/image-simple/file-1.txt deleted file mode 100644 index 985d3408e..000000000 --- a/syft/presenter/json/test-fixtures/image-simple/file-1.txt +++ /dev/null @@ -1 +0,0 @@ -this file has contents \ No newline at end of file diff --git a/syft/presenter/json/test-fixtures/image-simple/file-2.txt b/syft/presenter/json/test-fixtures/image-simple/file-2.txt deleted file mode 100644 index 396d08bbc..000000000 --- a/syft/presenter/json/test-fixtures/image-simple/file-2.txt +++ /dev/null @@ -1 +0,0 @@ -file-2 contents! \ No newline at end of file diff --git a/syft/presenter/json/test-fixtures/image-simple/target/really/nested/file-3.txt b/syft/presenter/json/test-fixtures/image-simple/target/really/nested/file-3.txt deleted file mode 100644 index f85472c93..000000000 --- a/syft/presenter/json/test-fixtures/image-simple/target/really/nested/file-3.txt +++ /dev/null @@ -1,2 +0,0 @@ -another file! -with lines... \ No newline at end of file diff --git a/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden b/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden deleted file mode 100644 index 7d9e4889d..000000000 --- a/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden +++ /dev/null @@ -1,112 +0,0 @@ -{ - "artifacts": [ - { - "id": "package-1-id", - "name": "package-1", - "version": "1.0.1", - "type": "python", - "foundBy": "the-cataloger-1", - "locations": [ - { - "path": "/somefile-1.txt", - "layerID": "sha256:2abd6dc7d143b384177da9a30f5311eb1f6c4e920eb6e218ec32b58c1a8806b1" - } - ], - "licenses": [ - "MIT" - ], - "language": "python", - "cpes": [ - "cpe:2.3:*:some:package:1:*:*:*:*:*:*:*" - ], - "purl": "a-purl-1", - "metadataType": "PythonPackageMetadata", - "metadata": { - "name": "package-1", - "version": "1.0.1", - "license": "", - "author": "", - "authorEmail": "", - "platform": "", - "sitePackagesRootPath": "" - } - }, - { - "id": "package-2-id", - "name": "package-2", - "version": "2.0.1", - "type": "deb", - "foundBy": "the-cataloger-2", - "locations": [ - { - "path": "/somefile-2.txt", - "layerID": "sha256:63574b0cbd9fcfc799d82a449c0975ee3f3560747b0957f042938bdec902c4cc" - } - ], - "licenses": [], - "language": "", - "cpes": [ - "cpe:2.3:*:some:package:2:*:*:*:*:*:*:*" - ], - "purl": "a-purl-2", - "metadataType": "DpkgMetadata", - "metadata": { - "package": "package-2", - "source": "", - "version": "2.0.1", - "sourceVersion": "", - "architecture": "", - "maintainer": "", - "installedSize": 0, - "files": null - } - } - ], - "source": { - "type": "image", - "target": { - "userInput": "user-image-input", - "imageID": "sha256:0a7326bbb1c3c467969c7a73deb6366b1e34eb034616b97f8614ad662cad1ce1", - "manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368", - "mediaType": "application/vnd.docker.distribution.manifest.v2+json", - "tags": [ - "stereoscope-fixture-image-simple:04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7" - ], - "imageSize": 65, - "scope": "Squashed", - "layers": [ - { - "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "digest": "sha256:2abd6dc7d143b384177da9a30f5311eb1f6c4e920eb6e218ec32b58c1a8806b1", - "size": 22 - }, - { - "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "digest": "sha256:63574b0cbd9fcfc799d82a449c0975ee3f3560747b0957f042938bdec902c4cc", - "size": 16 - }, - { - "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "digest": "sha256:b102400548006e8dfa3abf6e2b2e6f1c440e29936d91a36e145736e2fd8cf0a1", - "size": 27 - } - ], - "manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjoxODA4LCJkaWdlc3QiOiJzaGEyNTY6MGE3MzI2YmJiMWMzYzQ2Nzk2OWM3YTczZGViNjM2NmIxZTM0ZWIwMzQ2MTZiOTdmODYxNGFkNjYyY2FkMWNlMSJ9LCJsYXllcnMiOlt7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLCJzaXplIjoyMDQ4LCJkaWdlc3QiOiJzaGEyNTY6MmFiZDZkYzdkMTQzYjM4NDE3N2RhOWEzMGY1MzExZWIxZjZjNGU5MjBlYjZlMjE4ZWMzMmI1OGMxYTg4MDZiMSJ9LHsibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1Njo2MzU3NGIwY2JkOWZjZmM3OTlkODJhNDQ5YzA5NzVlZTNmMzU2MDc0N2IwOTU3ZjA0MjkzOGJkZWM5MDJjNGNjIn0seyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwic2l6ZSI6MzU4NCwiZGlnZXN0Ijoic2hhMjU2OmIxMDI0MDA1NDgwMDZlOGRmYTNhYmY2ZTJiMmU2ZjFjNDQwZTI5OTM2ZDkxYTM2ZTE0NTczNmUyZmQ4Y2YwYTEifV19", - "config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJIb3N0bmFtZSI6IiIsIkRvbWFpbm5hbWUiOiIiLCJVc2VyIjoiIiwiQXR0YWNoU3RkaW4iOmZhbHNlLCJBdHRhY2hTdGRvdXQiOmZhbHNlLCJBdHRhY2hTdGRlcnIiOmZhbHNlLCJUdHkiOmZhbHNlLCJPcGVuU3RkaW4iOmZhbHNlLCJTdGRpbk9uY2UiOmZhbHNlLCJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiQ21kIjpudWxsLCJJbWFnZSI6InNoYTI1Njo4YjlhNTM1MmM3YzU3M2RmNTkxYTYwMzA0NzIxMmU3ZmYzOGI3YjRiYzZiMzk1NmI3Zjk2Nzk0MWU3NGMyMjhkIiwiVm9sdW1lcyI6bnVsbCwiV29ya2luZ0RpciI6IiIsIkVudHJ5cG9pbnQiOm51bGwsIk9uQnVpbGQiOm51bGwsIkxhYmVscyI6bnVsbH0sImNvbnRhaW5lcl9jb25maWciOnsiSG9zdG5hbWUiOiIiLCJEb21haW5uYW1lIjoiIiwiVXNlciI6IiIsIkF0dGFjaFN0ZGluIjpmYWxzZSwiQXR0YWNoU3Rkb3V0IjpmYWxzZSwiQXR0YWNoU3RkZXJyIjpmYWxzZSwiVHR5IjpmYWxzZSwiT3BlblN0ZGluIjpmYWxzZSwiU3RkaW5PbmNlIjpmYWxzZSwiRW52IjpbIlBBVEg9L3Vzci9sb2NhbC9zYmluOi91c3IvbG9jYWwvYmluOi91c3Ivc2JpbjovdXNyL2Jpbjovc2JpbjovYmluIl0sIkNtZCI6WyIvYmluL3NoIiwiLWMiLCIjKG5vcCkgQUREIGRpcjpjOTM3YzZhYTUwODkwN2UyODUwOWI2NDRhMTJmOGQ2YzY3ZDM0ZTk2OWY4M2IxNGRlZTkzZWExN2Q3NjkwMjhhIGluIC8gIl0sIkltYWdlIjoic2hhMjU2OjhiOWE1MzUyYzdjNTczZGY1OTFhNjAzMDQ3MjEyZTdmZjM4YjdiNGJjNmIzOTU2YjdmOTY3OTQxZTc0YzIyOGQiLCJWb2x1bWVzIjpudWxsLCJXb3JraW5nRGlyIjoiIiwiRW50cnlwb2ludCI6bnVsbCwiT25CdWlsZCI6bnVsbCwiTGFiZWxzIjpudWxsfSwiY3JlYXRlZCI6IjIwMjEtMDMtMjJUMTM6MDg6MzUuNDkyOTI1MzAzWiIsImRvY2tlcl92ZXJzaW9uIjoiMjAuMTAuMiIsImhpc3RvcnkiOlt7ImNyZWF0ZWQiOiIyMDIxLTAzLTIyVDEzOjA4OjM1LjA5NDMxMjQ3NloiLCJjcmVhdGVkX2J5IjoiL2Jpbi9zaCAtYyAjKG5vcCkgQUREIGZpbGU6YWMzMmRhMjNkNTFlODAxZjAyZjkyNDEyM2VkMzA5OTBlYjNmMGZlYzFiOWVkNGYwYjA2YzI0ZTg4YjljMzY5NSBpbiAvc29tZWZpbGUtMS50eHQgIn0seyJjcmVhdGVkIjoiMjAyMS0wMy0yMlQxMzowODozNS4yODk5MjIwNTFaIiwiY3JlYXRlZF9ieSI6Ii9iaW4vc2ggLWMgIyhub3ApIEFERCBmaWxlOmRmM2I3NDRmNTRhOWIxNmI5YjlhZWQ0MGUzZTk4ZDljYTJiNDlmNWE3N2Q5ZmE4YTk3NjkwZDdiYWY1ODg4MjAgaW4gL3NvbWVmaWxlLTIudHh0ICJ9LHsiY3JlYXRlZCI6IjIwMjEtMDMtMjJUMTM6MDg6MzUuNDkyOTI1MzAzWiIsImNyZWF0ZWRfYnkiOiIvYmluL3NoIC1jICMobm9wKSBBREQgZGlyOmM5MzdjNmFhNTA4OTA3ZTI4NTA5YjY0NGExMmY4ZDZjNjdkMzRlOTY5ZjgzYjE0ZGVlOTNlYTE3ZDc2OTAyOGEgaW4gLyAifV0sIm9zIjoibGludXgiLCJyb290ZnMiOnsidHlwZSI6ImxheWVycyIsImRpZmZfaWRzIjpbInNoYTI1NjoyYWJkNmRjN2QxNDNiMzg0MTc3ZGE5YTMwZjUzMTFlYjFmNmM0ZTkyMGViNmUyMThlYzMyYjU4YzFhODgwNmIxIiwic2hhMjU2OjYzNTc0YjBjYmQ5ZmNmYzc5OWQ4MmE0NDljMDk3NWVlM2YzNTYwNzQ3YjA5NTdmMDQyOTM4YmRlYzkwMmM0Y2MiLCJzaGEyNTY6YjEwMjQwMDU0ODAwNmU4ZGZhM2FiZjZlMmIyZTZmMWM0NDBlMjk5MzZkOTFhMzZlMTQ1NzM2ZTJmZDhjZjBhMSJdfX0=" - } - }, - "distro": { - "name": "", - "version": "", - "idLike": "" - }, - "descriptor": { - "name": "syft", - "version": "[not provided]" - }, - "schema": { - "version": "1.0.3", - "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.3.json" - }, - "artifactRelationships": [] -} diff --git a/syft/presenter/option.go b/syft/presenter/option.go deleted file mode 100644 index fe9b125d9..000000000 --- a/syft/presenter/option.go +++ /dev/null @@ -1,35 +0,0 @@ -package presenter - -import "strings" - -const ( - UnknownPresenter Option = "UnknownPresenter" - JSONPresenter Option = "json" - TextPresenter Option = "text" - TablePresenter Option = "table" - CycloneDxPresenter Option = "cyclonedx" -) - -var Options = []Option{ - JSONPresenter, - TextPresenter, - TablePresenter, - CycloneDxPresenter, -} - -type Option string - -func ParseOption(userStr string) Option { - switch strings.ToLower(userStr) { - case string(JSONPresenter): - return JSONPresenter - case string(TextPresenter): - return TextPresenter - case string(TablePresenter): - return TablePresenter - case string(CycloneDxPresenter), "cyclone", "cyclone-dx": - return CycloneDxPresenter - default: - return UnknownPresenter - } -} diff --git a/syft/presenter/presenter.go b/syft/presenter/presenter.go deleted file mode 100644 index 8bf3e46fb..000000000 --- a/syft/presenter/presenter.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Defines a Presenter interface for displaying catalog results to an io.Writer as well as a helper utility to obtain -a specific Presenter implementation given user configuration. -*/ -package presenter - -import ( - "io" - - "github.com/anchore/syft/syft/distro" - - "github.com/anchore/syft/syft/presenter/cyclonedx" - - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/presenter/json" - "github.com/anchore/syft/syft/presenter/table" - "github.com/anchore/syft/syft/presenter/text" - "github.com/anchore/syft/syft/source" -) - -// Presenter defines the expected behavior for an object responsible for displaying arbitrary input and processed data -// to a given io.Writer. -type Presenter interface { - Present(io.Writer) error -} - -// GetPresenter returns a presenter for images or directories -func GetPresenter(option Option, srcMetadata source.Metadata, catalog *pkg.Catalog, d *distro.Distro) Presenter { - switch option { - case JSONPresenter: - return json.NewPresenter(catalog, srcMetadata, d) - case TextPresenter: - return text.NewPresenter(catalog, srcMetadata) - case TablePresenter: - return table.NewPresenter(catalog) - case CycloneDxPresenter: - return cyclonedx.NewPresenter(catalog, srcMetadata) - default: - return nil - } -} diff --git a/syft/presenter/table/test-fixtures/image-simple/Dockerfile b/syft/presenter/table/test-fixtures/image-simple/Dockerfile deleted file mode 100644 index 62fb151e4..000000000 --- a/syft/presenter/table/test-fixtures/image-simple/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -# Note: changes to this file will result in updating several test values. Consider making a new image fixture instead of editing this one. -FROM scratch -ADD file-1.txt /somefile-1.txt -ADD file-2.txt /somefile-2.txt -# note: adding a directory will behave differently on docker engine v18 vs v19 -ADD target / diff --git a/syft/presenter/table/test-fixtures/image-simple/file-1.txt b/syft/presenter/table/test-fixtures/image-simple/file-1.txt deleted file mode 100644 index 985d3408e..000000000 --- a/syft/presenter/table/test-fixtures/image-simple/file-1.txt +++ /dev/null @@ -1 +0,0 @@ -this file has contents \ No newline at end of file diff --git a/syft/presenter/table/test-fixtures/image-simple/file-2.txt b/syft/presenter/table/test-fixtures/image-simple/file-2.txt deleted file mode 100644 index 396d08bbc..000000000 --- a/syft/presenter/table/test-fixtures/image-simple/file-2.txt +++ /dev/null @@ -1 +0,0 @@ -file-2 contents! \ No newline at end of file diff --git a/syft/presenter/table/test-fixtures/image-simple/target/really/nested/file-3.txt b/syft/presenter/table/test-fixtures/image-simple/target/really/nested/file-3.txt deleted file mode 100644 index f85472c93..000000000 --- a/syft/presenter/table/test-fixtures/image-simple/target/really/nested/file-3.txt +++ /dev/null @@ -1,2 +0,0 @@ -another file! -with lines... \ No newline at end of file diff --git a/syft/presenter/text/test-fixtures/image-simple/file-1.txt b/syft/presenter/text/test-fixtures/image-simple/file-1.txt deleted file mode 100644 index 985d3408e..000000000 --- a/syft/presenter/text/test-fixtures/image-simple/file-1.txt +++ /dev/null @@ -1 +0,0 @@ -this file has contents \ No newline at end of file diff --git a/syft/presenter/text/test-fixtures/image-simple/file-2.txt b/syft/presenter/text/test-fixtures/image-simple/file-2.txt deleted file mode 100644 index 396d08bbc..000000000 --- a/syft/presenter/text/test-fixtures/image-simple/file-2.txt +++ /dev/null @@ -1 +0,0 @@ -file-2 contents! \ No newline at end of file diff --git a/syft/presenter/text/test-fixtures/snapshot/TestJsonPresenter.golden b/syft/presenter/text/test-fixtures/snapshot/TestJsonPresenter.golden deleted file mode 100644 index ab2fd8317..000000000 --- a/syft/presenter/text/test-fixtures/snapshot/TestJsonPresenter.golden +++ /dev/null @@ -1 +0,0 @@ -{"artifacts":[{"name":"package-1","version":"1.0.1","type":"deb","cataloger":"","sources":[],"metadata":null},{"name":"package-2","version":"2.0.1","type":"deb","cataloger":"","sources":[],"metadata":null}],"Source":"/some/path"} \ No newline at end of file diff --git a/syft/source/all_layers_resolver.go b/syft/source/all_layers_resolver.go index f2f248ee2..b446eccbf 100644 --- a/syft/source/all_layers_resolver.go +++ b/syft/source/all_layers_resolver.go @@ -2,10 +2,8 @@ package source import ( "archive/tar" - "bytes" "fmt" "io" - "io/ioutil" "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/filetree" @@ -13,16 +11,16 @@ import ( "github.com/anchore/syft/internal/log" ) -var _ Resolver = (*AllLayersResolver)(nil) +var _ FileResolver = (*allLayersResolver)(nil) -// AllLayersResolver implements path and content access for the AllLayers source option for container image data sources. -type AllLayersResolver struct { +// allLayersResolver implements path and content access for the AllLayers source option for container image data sources. +type allLayersResolver struct { img *image.Image layers []int } -// NewAllLayersResolver returns a new resolver from the perspective of all image layers for the given image. -func NewAllLayersResolver(img *image.Image) (*AllLayersResolver, error) { +// newAllLayersResolver returns a new resolver from the perspective of all image layers for the given image. +func newAllLayersResolver(img *image.Image) (*allLayersResolver, error) { if len(img.Layers) == 0 { return nil, fmt.Errorf("the image does not contain any layers") } @@ -31,14 +29,14 @@ func NewAllLayersResolver(img *image.Image) (*AllLayersResolver, error) { for idx := range img.Layers { layers = append(layers, idx) } - return &AllLayersResolver{ + return &allLayersResolver{ img: img, layers: layers, }, nil } // HasPath indicates if the given path exists in the underlying source. -func (r *AllLayersResolver) HasPath(path string) bool { +func (r *allLayersResolver) HasPath(path string) bool { p := file.Path(path) for _, layerIdx := range r.layers { tree := r.img.Layers[layerIdx].Tree @@ -49,7 +47,7 @@ func (r *AllLayersResolver) HasPath(path string) bool { return false } -func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.ReferenceSet, layerIdx int) ([]file.Reference, error) { +func (r *allLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.ReferenceSet, layerIdx int) ([]file.Reference, error) { uniqueFiles := make([]file.Reference, 0) // since there is potentially considerable work for each symlink/hardlink that needs to be resolved, let's check to see if this is a symlink/hardlink first @@ -80,7 +78,7 @@ func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.Ref } // FilesByPath returns all file.References that match the given paths from any layer in the image. -func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) { +func (r *allLayersResolver) FilesByPath(paths ...string) ([]Location, error) { uniqueFileIDs := file.NewFileReferenceSet() uniqueLocations := make([]Location, 0) @@ -123,7 +121,7 @@ func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) { // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. // nolint:gocognit -func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error) { +func (r *allLayersResolver) FilesByGlob(patterns ...string) ([]Location, error) { uniqueFileIDs := file.NewFileReferenceSet() uniqueLocations := make([]Location, 0) @@ -164,7 +162,7 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error) // RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. // This is helpful when attempting to find a file that is in the same layer or lower as another file. -func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) *Location { +func (r *allLayersResolver) RelativeFileByPath(location Location, path string) *Location { entry, err := r.img.FileCatalog.Get(location.ref) if err != nil { return nil @@ -184,55 +182,26 @@ func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) * return &relativeLocation } -// MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a -// file.Reference is a path relative to a particular layer. -func (r *AllLayersResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) { - return mapLocationRefs(r.img.MultipleFileContentsByRef, locations) -} - // FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer. // If the path does not exist an error is returned. -func (r *AllLayersResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) { +func (r *allLayersResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) { return r.img.FileContentsByRef(location.ref) } -type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]io.ReadCloser, error) - -func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]io.ReadCloser, error) { - var fileRefs = make([]file.Reference, len(locations)) - var locationByRefs = make(map[file.Reference][]Location) - var results = make(map[Location]io.ReadCloser) - - for i, location := range locations { - locationByRefs[location.ref] = append(locationByRefs[location.ref], location) - fileRefs[i] = location.ref - } - - contentsByRef, err := callback(fileRefs...) - if err != nil { - return nil, err - } - - // TODO: this is not tested, we need a test case that covers a mapLocationRefs which has multiple Locations with the same reference in the request. The io.Reader should be copied. - for ref, content := range contentsByRef { - mappedLocations := locationByRefs[ref] - switch { - case len(mappedLocations) > 1: - // TODO: fixme... this can lead to lots of unexpected memory usage in unusual circumstances (cache is not leveraged for large files). - // stereoscope wont duplicate content requests if the caller asks for the same file multiple times... thats up to the caller - contentsBytes, err := ioutil.ReadAll(content) - if err != nil { - return nil, fmt.Errorf("unable to read ref=%+v :%w", ref, err) +func (r *allLayersResolver) AllLocations() <-chan Location { + results := make(chan Location) + go func() { + defer close(results) + for _, layerIdx := range r.layers { + tree := r.img.Layers[layerIdx].Tree + for _, ref := range tree.AllFiles(file.AllTypes...) { + results <- NewLocationFromImage(string(ref.RealPath), ref, r.img) } - for _, loc := range mappedLocations { - results[loc] = ioutil.NopCloser(bytes.NewReader(contentsBytes)) - } - - case len(mappedLocations) == 1: - results[locationByRefs[ref][0]] = content - default: - return nil, fmt.Errorf("unexpected ref-location count=%d for ref=%v", len(mappedLocations), ref) } - } - return results, nil + }() + return results +} + +func (r *allLayersResolver) FileMetadataByLocation(location Location) (FileMetadata, error) { + return fileMetadataByLocation(r.img, location) } diff --git a/syft/source/all_layers_resolver_test.go b/syft/source/all_layers_resolver_test.go index 54b50c38b..421a6663d 100644 --- a/syft/source/all_layers_resolver_test.go +++ b/syft/source/all_layers_resolver_test.go @@ -82,10 +82,9 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") - defer cleanup() + img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") - resolver, err := NewAllLayersResolver(img) + resolver, err := newAllLayersResolver(img) if err != nil { t.Fatalf("could not create resolver: %+v", err) } @@ -201,10 +200,9 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") - defer cleanup() + img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") - resolver, err := NewAllLayersResolver(img) + resolver, err := newAllLayersResolver(img) if err != nil { t.Fatalf("could not create resolver: %+v", err) } diff --git a/syft/source/content_requester.go b/syft/source/content_requester.go deleted file mode 100644 index ef99513c0..000000000 --- a/syft/source/content_requester.go +++ /dev/null @@ -1,56 +0,0 @@ -package source - -import "sync" - -// ContentRequester is an object tailored for taking source.Location objects which file contents will be resolved -// upon invoking Execute(). -type ContentRequester struct { - request map[Location][]*FileData - lock sync.Mutex -} - -// NewContentRequester creates a new ContentRequester object with the given initial request data. -func NewContentRequester(data ...*FileData) *ContentRequester { - requester := &ContentRequester{ - request: make(map[Location][]*FileData), - } - for _, d := range data { - requester.Add(d) - } - return requester -} - -// Add appends a new single FileData containing a source.Location to later have the contents fetched and stored within -// the given FileData object. -func (r *ContentRequester) Add(data *FileData) { - r.lock.Lock() - defer r.lock.Unlock() - - r.request[data.Location] = append(r.request[data.Location], data) -} - -// Execute takes the previously provided source.Location's and resolves the file contents, storing the results within -// the previously provided FileData objects. -func (r *ContentRequester) Execute(resolver ContentResolver) error { - r.lock.Lock() - defer r.lock.Unlock() - - var locations = make([]Location, len(r.request)) - idx := 0 - for l := range r.request { - locations[idx] = l - idx++ - } - - response, err := resolver.MultipleFileContentsByLocation(locations) - if err != nil { - return err - } - - for l, contents := range response { - for i := range r.request[l] { - r.request[l][i].Contents = contents - } - } - return nil -} diff --git a/syft/source/content_requester_test.go b/syft/source/content_requester_test.go deleted file mode 100644 index f46498e63..000000000 --- a/syft/source/content_requester_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package source - -import ( - "io/ioutil" - "testing" - - "github.com/anchore/stereoscope/pkg/imagetest" - "github.com/sergi/go-diff/diffmatchpatch" -) - -func TestContentRequester(t *testing.T) { - tests := []struct { - fixture string - expectedContents map[string]string - }{ - { - fixture: "image-simple", - expectedContents: map[string]string{ - "/somefile-1.txt": "this file has contents", - "/somefile-2.txt": "file-2 contents!", - "/really/nested/file-3.txt": "another file!\nwith lines...", - }, - }, - } - - for _, test := range tests { - t.Run(test.fixture, func(t *testing.T) { - img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-simple") - defer cleanup() - - resolver, err := NewAllLayersResolver(img) - if err != nil { - t.Fatalf("could not create resolver: %+v", err) - } - - var data []*FileData - for path := range test.expectedContents { - - locations, err := resolver.FilesByPath(path) - if err != nil { - t.Fatalf("could not build request: %+v", err) - } - if len(locations) != 1 { - t.Fatalf("bad resolver paths: %+v", locations) - } - - data = append(data, &FileData{ - Location: locations[0], - }) - } - - if err := NewContentRequester(data...).Execute(resolver); err != nil { - t.Fatalf("could not execute request: %+v", err) - } - - for _, entry := range data { - if expected, ok := test.expectedContents[entry.Location.RealPath]; ok { - actualBytes, err := ioutil.ReadAll(entry.Contents) - if err != nil { - t.Fatalf("could not read %q: %+v", entry.Location.RealPath, err) - } - for expected != string(actualBytes) { - t.Errorf("mismatched contents for %q", entry.Location.RealPath) - dmp := diffmatchpatch.New() - diffs := dmp.DiffMain(expected, string(actualBytes), true) - t.Errorf("diff: %s", dmp.DiffPrettyText(diffs)) - } - continue - } - t.Errorf("could not find %q", entry.Location.RealPath) - } - }) - } -} diff --git a/syft/source/directory_resolver.go b/syft/source/directory_resolver.go index 7ef68bae8..9645cfee7 100644 --- a/syft/source/directory_resolver.go +++ b/syft/source/directory_resolver.go @@ -12,35 +12,39 @@ import ( "github.com/bmatcuk/doublestar/v2" ) -var _ Resolver = (*DirectoryResolver)(nil) +var _ FileResolver = (*directoryResolver)(nil) -// DirectoryResolver implements path and content access for the directory data source. -type DirectoryResolver struct { - Path string +// directoryResolver implements path and content access for the directory data source. +type directoryResolver struct { + path string } -func (r DirectoryResolver) requestPath(userPath string) string { +func newDirectoryResolver(path string) *directoryResolver { + return &directoryResolver{path: path} +} + +func (r directoryResolver) requestPath(userPath string) string { fullPath := userPath if filepath.IsAbs(fullPath) { // a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is - fullPath = path.Join(r.Path, fullPath) + fullPath = path.Join(r.path, fullPath) } return fullPath } // HasPath indicates if the given path exists in the underlying source. -func (r *DirectoryResolver) HasPath(userPath string) bool { +func (r *directoryResolver) HasPath(userPath string) bool { _, err := os.Stat(r.requestPath(userPath)) return !os.IsNotExist(err) } // Stringer to represent a directory path data source -func (r DirectoryResolver) String() string { - return fmt.Sprintf("dir:%s", r.Path) +func (r directoryResolver) String() string { + return fmt.Sprintf("dir:%s", r.path) } // FilesByPath returns all file.References that match the given paths from the directory. -func (r DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error) { +func (r directoryResolver) FilesByPath(userPaths ...string) ([]Location, error) { var references = make([]Location, 0) for _, userPath := range userPaths { @@ -64,11 +68,11 @@ func (r DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error) } // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. -func (r DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) { +func (r directoryResolver) FilesByGlob(patterns ...string) ([]Location, error) { result := make([]Location, 0) for _, pattern := range patterns { - pathPattern := path.Join(r.Path, pattern) + pathPattern := path.Join(r.path, pattern) pathMatches, err := doublestar.Glob(pathPattern) if err != nil { return nil, err @@ -93,8 +97,8 @@ func (r DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) { // RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. // This is helpful when attempting to find a file that is in the same layer or lower as another file. For the -// DirectoryResolver, this is a simple path lookup. -func (r *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Location { +// directoryResolver, this is a simple path lookup. +func (r *directoryResolver) RelativeFileByPath(_ Location, path string) *Location { paths, err := r.FilesByPath(path) if err != nil { return nil @@ -106,17 +110,51 @@ func (r *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Locatio return &paths[0] } -// MultipleFileContentsByLocation returns the file contents for all file.References relative a directory. -func (r DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) { - refContents := make(map[Location]io.ReadCloser) - for _, location := range locations { - refContents[location] = file.NewDeferredReadCloser(location.RealPath) - } - return refContents, nil -} - // FileContentsByLocation fetches file contents for a single file reference relative to a directory. // If the path does not exist an error is returned. -func (r DirectoryResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) { - return file.NewDeferredReadCloser(location.RealPath), nil +func (r directoryResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) { + return file.NewLazyReadCloser(location.RealPath), nil +} + +func (r *directoryResolver) AllLocations() <-chan Location { + results := make(chan Location) + go func() { + defer close(results) + err := filepath.Walk(r.path, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + results <- NewLocation(path) + return nil + }) + if err != nil { + log.Errorf("unable to walk path=%q : %+v", r.path, err) + } + }() + return results +} + +func (r *directoryResolver) FileMetadataByLocation(location Location) (FileMetadata, error) { + fi, err := os.Stat(location.RealPath) + if err != nil { + return FileMetadata{}, err + } + + // best effort + ty := UnknownFileType + switch { + case fi.Mode().IsDir(): + ty = Directory + case fi.Mode().IsRegular(): + ty = RegularFile + } + + return FileMetadata{ + Mode: fi.Mode(), + Type: ty, + // unsupported across platforms + UserID: -1, + GroupID: -1, + }, nil } diff --git a/syft/source/directory_resolver_test.go b/syft/source/directory_resolver_test.go index dc88876c7..db0ccd0c1 100644 --- a/syft/source/directory_resolver_test.go +++ b/syft/source/directory_resolver_test.go @@ -57,7 +57,7 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - resolver := DirectoryResolver{c.root} + resolver := directoryResolver{c.root} hasPath := resolver.HasPath(c.input) if !c.forcePositiveHasPath { @@ -112,7 +112,7 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - resolver := DirectoryResolver{"test-fixtures"} + resolver := directoryResolver{"test-fixtures"} refs, err := resolver.FilesByPath(c.input...) if err != nil { @@ -126,59 +126,9 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) { } } -func TestDirectoryResolver_MultipleFileContentsByRef(t *testing.T) { - cases := []struct { - name string - input []string - refCount int - contents []string - }{ - { - name: "gets multiple file contents", - input: []string{"test-fixtures/image-symlinks/file-1.txt", "test-fixtures/image-symlinks/file-2.txt"}, - refCount: 2, - }, - { - name: "skips non-existing files", - input: []string{"test-fixtures/image-symlinks/bogus.txt", "test-fixtures/image-symlinks/file-1.txt"}, - refCount: 1, - }, - { - name: "does not return anything for non-existing directories", - input: []string{"test-fixtures/non-existing/bogus.txt", "test-fixtures/non-existing/file-1.txt"}, - refCount: 0, - }, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - locations := make([]Location, 0) - resolver := DirectoryResolver{"test-fixtures"} - - for _, p := range c.input { - newRefs, err := resolver.FilesByPath(p) - if err != nil { - t.Errorf("could not generate locations: %+v", err) - } - for _, ref := range newRefs { - locations = append(locations, ref) - } - } - - contents, err := resolver.MultipleFileContentsByLocation(locations) - if err != nil { - t.Fatalf("unable to generate file contents by ref: %+v", err) - } - if len(contents) != c.refCount { - t.Errorf("unexpected number of locations produced: %d != %d", len(contents), c.refCount) - } - - }) - } -} - func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) { t.Run("finds multiple matching files", func(t *testing.T) { - resolver := DirectoryResolver{"test-fixtures"} + resolver := directoryResolver{"test-fixtures"} refs, err := resolver.FilesByGlob("image-symlinks/file*") if err != nil { @@ -195,7 +145,7 @@ func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) { func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) { t.Run("finds multiple matching files", func(t *testing.T) { - resolver := DirectoryResolver{"test-fixtures/image-symlinks"} + resolver := directoryResolver{"test-fixtures/image-symlinks"} refs, err := resolver.FilesByGlob("**/*.txt") if err != nil { @@ -212,7 +162,7 @@ func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) { func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) { t.Run("finds multiple matching files", func(t *testing.T) { - resolver := DirectoryResolver{"test-fixtures"} + resolver := directoryResolver{"test-fixtures"} refs, err := resolver.FilesByGlob("image-symlinks/*1.txt") if err != nil { t.Fatalf("could not use resolver: %+v, %+v", err, refs) diff --git a/syft/source/file_data.go b/syft/source/file_data.go deleted file mode 100644 index bd3d0c849..000000000 --- a/syft/source/file_data.go +++ /dev/null @@ -1,8 +0,0 @@ -package source - -import "io" - -type FileData struct { - Location Location - Contents io.ReadCloser -} diff --git a/syft/source/file_metadata.go b/syft/source/file_metadata.go new file mode 100644 index 000000000..45c6a877a --- /dev/null +++ b/syft/source/file_metadata.go @@ -0,0 +1,28 @@ +package source + +import ( + "os" + + "github.com/anchore/stereoscope/pkg/image" +) + +type FileMetadata struct { + Mode os.FileMode + Type FileType + UserID int + GroupID int +} + +func fileMetadataByLocation(img *image.Image, location Location) (FileMetadata, error) { + entry, err := img.FileCatalog.Get(location.ref) + if err != nil { + return FileMetadata{}, err + } + + return FileMetadata{ + Mode: entry.Metadata.Mode, + Type: newFileTypeFromTarHeaderTypeFlag(entry.Metadata.TypeFlag), + UserID: entry.Metadata.UserID, + GroupID: entry.Metadata.GroupID, + }, nil +} diff --git a/syft/source/file_resolver.go b/syft/source/file_resolver.go new file mode 100644 index 000000000..331820af2 --- /dev/null +++ b/syft/source/file_resolver.go @@ -0,0 +1,39 @@ +package source + +import ( + "io" +) + +// FileResolver is an interface that encompasses how to get specific file references and file contents for a generic data source. +type FileResolver interface { + FileContentResolver + FilePathResolver + FileLocationResolver + FileMetadataResolver +} + +// FileContentResolver knows how to get file content for a given Location +type FileContentResolver interface { + FileContentsByLocation(Location) (io.ReadCloser, error) +} + +type FileMetadataResolver interface { + FileMetadataByLocation(Location) (FileMetadata, error) +} + +// FilePathResolver knows how to get a Location for given string paths and globs +type FilePathResolver interface { + // HasPath indicates if the given path exists in the underlying source. + HasPath(string) bool + // FilesByPath fetches a set of file references which have the given path (for an image, there may be multiple matches) + FilesByPath(paths ...string) ([]Location, error) + // FilesByGlob fetches a set of file references which the given glob matches + FilesByGlob(patterns ...string) ([]Location, error) + // RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. + // This is helpful when attempting to find a file that is in the same layer or lower as another file. + RelativeFileByPath(_ Location, path string) *Location +} + +type FileLocationResolver interface { + AllLocations() <-chan Location +} diff --git a/syft/source/file_type.go b/syft/source/file_type.go new file mode 100644 index 000000000..3718b12c4 --- /dev/null +++ b/syft/source/file_type.go @@ -0,0 +1,34 @@ +package source + +const ( + UnknownFileType FileType = "unknownFileType" + RegularFile FileType = "regularFile" + HardLink FileType = "hardLink" + SymbolicLink FileType = "symbolicLink" + CharacterDevice FileType = "characterDevice" + BlockDevice FileType = "blockDevice" + Directory FileType = "directory" + FIFONode FileType = "fifoNode" +) + +type FileType string + +func newFileTypeFromTarHeaderTypeFlag(flag byte) FileType { + switch flag { + case '0', '\x00': + return RegularFile + case '1': + return HardLink + case '2': + return SymbolicLink + case '3': + return CharacterDevice + case '4': + return BlockDevice + case '5': + return Directory + case '6': + return FIFONode + } + return UnknownFileType +} diff --git a/syft/source/image_metadata.go b/syft/source/image_metadata.go index a00803caa..38a351ca3 100644 --- a/syft/source/image_metadata.go +++ b/syft/source/image_metadata.go @@ -3,7 +3,7 @@ package source import "github.com/anchore/stereoscope/pkg/image" // ImageMetadata represents all static metadata that defines what a container image is. This is useful to later describe -// "what" was cataloged without needing the more complicated stereoscope Image objects or Resolver objects. +// "what" was cataloged without needing the more complicated stereoscope Image objects or FileResolver objects. type ImageMetadata struct { UserInput string `json:"userInput"` ID string `json:"imageID"` @@ -11,7 +11,6 @@ type ImageMetadata struct { MediaType string `json:"mediaType"` Tags []string `json:"tags"` Size int64 `json:"imageSize"` - Scope Scope `json:"scope"` // specific perspective to catalog Layers []LayerMetadata `json:"layers"` RawManifest []byte `json:"manifest"` RawConfig []byte `json:"config"` @@ -25,7 +24,7 @@ type LayerMetadata struct { } // NewImageMetadata creates a new ImageMetadata object populated from the given stereoscope Image object and user configuration. -func NewImageMetadata(img *image.Image, userInput string, scope Scope) ImageMetadata { +func NewImageMetadata(img *image.Image, userInput string) ImageMetadata { // populate artifacts... tags := make([]string, len(img.Metadata.Tags)) for idx, tag := range img.Metadata.Tags { @@ -34,7 +33,6 @@ func NewImageMetadata(img *image.Image, userInput string, scope Scope) ImageMeta theImg := ImageMetadata{ ID: img.Metadata.ID, UserInput: userInput, - Scope: scope, ManifestDigest: img.Metadata.ManifestDigest, Size: img.Metadata.Size, MediaType: string(img.Metadata.MediaType), diff --git a/syft/source/image_squash_resolver.go b/syft/source/image_squash_resolver.go index 137d498b5..de51edf11 100644 --- a/syft/source/image_squash_resolver.go +++ b/syft/source/image_squash_resolver.go @@ -9,28 +9,31 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) -var _ Resolver = (*ImageSquashResolver)(nil) +var _ FileResolver = (*imageSquashResolver)(nil) -// ImageSquashResolver implements path and content access for the Squashed source option for container image data sources. -type ImageSquashResolver struct { +// imageSquashResolver implements path and content access for the Squashed source option for container image data sources. +type imageSquashResolver struct { img *image.Image } -// NewImageSquashResolver returns a new resolver from the perspective of the squashed representation for the given image. -func NewImageSquashResolver(img *image.Image) (*ImageSquashResolver, error) { +// newImageSquashResolver returns a new resolver from the perspective of the squashed representation for the given image. +func newImageSquashResolver(img *image.Image) (*imageSquashResolver, error) { if img.SquashedTree() == nil { return nil, fmt.Errorf("the image does not have have a squashed tree") } - return &ImageSquashResolver{img: img}, nil + + return &imageSquashResolver{ + img: img, + }, nil } // HasPath indicates if the given path exists in the underlying source. -func (r *ImageSquashResolver) HasPath(path string) bool { +func (r *imageSquashResolver) HasPath(path string) bool { return r.img.SquashedTree().HasPath(file.Path(path)) } // FilesByPath returns all file.References that match the given paths within the squashed representation of the image. -func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) { +func (r *imageSquashResolver) FilesByPath(paths ...string) ([]Location, error) { uniqueFileIDs := file.NewFileReferenceSet() uniqueLocations := make([]Location, 0) @@ -74,7 +77,7 @@ func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) { } // FilesByGlob returns all file.References that match the given path glob pattern within the squashed representation of the image. -func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error) { +func (r *imageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error) { uniqueFileIDs := file.NewFileReferenceSet() uniqueLocations := make([]Location, 0) @@ -88,7 +91,9 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error // don't consider directories (special case: there is no path information for /) if result.MatchPath == "/" { continue - } else if r.img.FileCatalog.Exists(result.Reference) { + } + + if r.img.FileCatalog.Exists(result.Reference) { metadata, err := r.img.FileCatalog.Get(result.Reference) if err != nil { return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", result.MatchPath, err) @@ -116,8 +121,8 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error // RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. // This is helpful when attempting to find a file that is in the same layer or lower as another file. For the -// ImageSquashResolver, this is a simple path lookup. -func (r *ImageSquashResolver) RelativeFileByPath(_ Location, path string) *Location { +// imageSquashResolver, this is a simple path lookup. +func (r *imageSquashResolver) RelativeFileByPath(_ Location, path string) *Location { paths, err := r.FilesByPath(path) if err != nil { return nil @@ -129,14 +134,23 @@ func (r *ImageSquashResolver) RelativeFileByPath(_ Location, path string) *Locat return &paths[0] } -// MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a -// file.Reference is a path relative to a particular layer, in this case only from the squashed representation. -func (r *ImageSquashResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) { - return mapLocationRefs(r.img.MultipleFileContentsByRef, locations) -} - // FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer. // If the path does not exist an error is returned. -func (r *ImageSquashResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) { +func (r *imageSquashResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) { return r.img.FileContentsByRef(location.ref) } + +func (r *imageSquashResolver) AllLocations() <-chan Location { + results := make(chan Location) + go func() { + defer close(results) + for _, ref := range r.img.SquashedTree().AllFiles(file.AllTypes...) { + results <- NewLocationFromImage(string(ref.RealPath), ref, r.img) + } + }() + return results +} + +func (r *imageSquashResolver) FileMetadataByLocation(location Location) (FileMetadata, error) { + return fileMetadataByLocation(r.img, location) +} diff --git a/syft/source/image_squash_resolver_test.go b/syft/source/image_squash_resolver_test.go index a866f0a91..4d8c1556f 100644 --- a/syft/source/image_squash_resolver_test.go +++ b/syft/source/image_squash_resolver_test.go @@ -62,10 +62,9 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") - defer cleanup() + img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") - resolver, err := NewImageSquashResolver(img) + resolver, err := newImageSquashResolver(img) if err != nil { t.Fatalf("could not create resolver: %+v", err) } @@ -179,10 +178,9 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") - defer cleanup() + img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") - resolver, err := NewImageSquashResolver(img) + resolver, err := newImageSquashResolver(img) if err != nil { t.Fatalf("could not create resolver: %+v", err) } diff --git a/syft/source/location.go b/syft/source/location.go index 50083166a..532b01623 100644 --- a/syft/source/location.go +++ b/syft/source/location.go @@ -45,6 +45,14 @@ func NewLocationFromImage(virtualPath string, ref file.Reference, img *image.Ima } } +func NewLocationFromReference(ref file.Reference) Location { + return Location{ + VirtualPath: string(ref.RealPath), + RealPath: string(ref.RealPath), + ref: ref, + } +} + func (l Location) String() string { str := "" if l.ref.ID() != 0 { diff --git a/syft/source/mock_resolver.go b/syft/source/mock_resolver.go index ba815f3c3..b574ccd14 100644 --- a/syft/source/mock_resolver.go +++ b/syft/source/mock_resolver.go @@ -8,13 +8,14 @@ import ( "github.com/anchore/syft/internal/file" ) -var _ Resolver = (*MockResolver)(nil) +var _ FileResolver = (*MockResolver)(nil) -// MockResolver implements the Resolver interface and is intended for use *only in test code*. +// MockResolver implements the FileResolver interface and is intended for use *only in test code*. // It provides an implementation that can resolve local filesystem paths using only a provided discrete list of file // paths, which are typically paths to test fixtures. type MockResolver struct { Locations []Location + Metadata map[Location]FileMetadata } // NewMockResolverForPaths creates a new MockResolver, where the only resolvable @@ -28,6 +29,15 @@ func NewMockResolverForPaths(paths ...string) *MockResolver { return &MockResolver{Locations: locations} } +func NewMockResolverForPathsWithMetadata(metadata map[Location]FileMetadata) *MockResolver { + var locations []Location + for p := range metadata { + locations = append(locations, p) + } + + return &MockResolver{Locations: locations, Metadata: metadata} +} + // HasPath indicates if the given path exists in the underlying source. func (r MockResolver) HasPath(path string) bool { for _, l := range r.Locations { @@ -55,20 +65,6 @@ func (r MockResolver) FileContentsByLocation(location Location) (io.ReadCloser, return nil, fmt.Errorf("no file for location: %v", location) } -// MultipleFileContentsByLocation returns the file contents for all specified Locations. -func (r MockResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) { - results := make(map[Location]io.ReadCloser) - for _, l := range locations { - contents, err := r.FileContentsByLocation(l) - if err != nil { - return nil, err - } - results[l] = contents - } - - return results, nil -} - // FilesByPath returns all Locations that match the given paths. func (r MockResolver) FilesByPath(paths ...string) ([]Location, error) { var results []Location @@ -110,3 +106,18 @@ func (r MockResolver) RelativeFileByPath(_ Location, path string) *Location { return &paths[0] } + +func (r MockResolver) AllLocations() <-chan Location { + results := make(chan Location) + go func() { + defer close(results) + for _, l := range r.Locations { + results <- l + } + }() + return results +} + +func (r MockResolver) FileMetadataByLocation(Location) (FileMetadata, error) { + panic("not implemented") +} diff --git a/syft/source/resolver.go b/syft/source/resolver.go deleted file mode 100644 index 019536aca..000000000 --- a/syft/source/resolver.go +++ /dev/null @@ -1,46 +0,0 @@ -package source - -import ( - "fmt" - "io" - - "github.com/anchore/stereoscope/pkg/image" -) - -// Resolver is an interface that encompasses how to get specific file references and file contents for a generic data source. -type Resolver interface { - ContentResolver - FileResolver -} - -// ContentResolver knows how to get file content for given file.References -type ContentResolver interface { - FileContentsByLocation(Location) (io.ReadCloser, error) - // TODO: it is possible to be given duplicate locations that will be overridden in the map (key), a subtle problem that coule easily be misued. - MultipleFileContentsByLocation([]Location) (map[Location]io.ReadCloser, error) -} - -// FileResolver knows how to get a Location for given string paths and globs -type FileResolver interface { - // HasPath indicates if the given path exists in the underlying source. - HasPath(path string) bool - // FilesByPath fetches a set of file references which have the given path (for an image, there may be multiple matches) - FilesByPath(paths ...string) ([]Location, error) - // FilesByGlob fetches a set of file references which the given glob matches - FilesByGlob(patterns ...string) ([]Location, error) - // RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. - // This is helpful when attempting to find a file that is in the same layer or lower as another file. - RelativeFileByPath(_ Location, path string) *Location -} - -// getImageResolver returns the appropriate resolve for a container image given the source option -func getImageResolver(img *image.Image, scope Scope) (Resolver, error) { - switch scope { - case SquashedScope: - return NewImageSquashResolver(img) - case AllLayersScope: - return NewAllLayersResolver(img) - default: - return nil, fmt.Errorf("bad scope provided: %+v", scope) - } -} diff --git a/syft/source/source.go b/syft/source/source.go index 182a87f71..b1e081db6 100644 --- a/syft/source/source.go +++ b/syft/source/source.go @@ -18,7 +18,6 @@ import ( // Source is an object that captures the data source to be cataloged, configuration, and a specific resolver used // in cataloging (based on the data source and configuration) type Source struct { - Resolver Resolver // a Resolver object to use in file path/glob resolution and file contents resolution Image *image.Image // the image object to be cataloged (image only) Metadata Metadata } @@ -26,7 +25,7 @@ type Source struct { type sourceDetector func(string) (image.Source, string, error) // New produces a Source based on userInput like dir: or image:tag -func New(userInput string, o Scope) (Source, func(), error) { +func New(userInput string) (Source, func(), error) { fs := afero.NewOsFs() parsedScheme, location, err := detectScheme(fs, image.DetectSource, userInput) if err != nil { @@ -60,7 +59,7 @@ func New(userInput string, o Scope) (Source, func(), error) { return Source{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", location, err) } - s, err := NewFromImage(img, o, location) + s, err := NewFromImage(img, location) if err != nil { return Source{}, cleanup, fmt.Errorf("could not populate source with image: %w", err) } @@ -73,9 +72,6 @@ func New(userInput string, o Scope) (Source, func(), error) { // NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively. func NewFromDirectory(path string) (Source, error) { return Source{ - Resolver: &DirectoryResolver{ - Path: path, - }, Metadata: Metadata{ Scheme: DirectoryScheme, Path: path, @@ -85,22 +81,33 @@ func NewFromDirectory(path string) (Source, error) { // NewFromImage creates a new source object tailored to catalog a given container image, relative to the // option given (e.g. all-layers, squashed, etc) -func NewFromImage(img *image.Image, scope Scope, userImageStr string) (Source, error) { +func NewFromImage(img *image.Image, userImageStr string) (Source, error) { if img == nil { return Source{}, fmt.Errorf("no image given") } - resolver, err := getImageResolver(img, scope) - if err != nil { - return Source{}, fmt.Errorf("could not determine file resolver: %w", err) - } - return Source{ - Resolver: resolver, - Image: img, + Image: img, Metadata: Metadata{ Scheme: ImageScheme, - ImageMetadata: NewImageMetadata(img, userImageStr, scope), + ImageMetadata: NewImageMetadata(img, userImageStr), }, }, nil } + +func (s Source) FileResolver(scope Scope) (FileResolver, error) { + switch s.Metadata.Scheme { + case DirectoryScheme: + return newDirectoryResolver(s.Metadata.Path), nil + case ImageScheme: + switch scope { + case SquashedScope: + return newImageSquashResolver(s.Image) + case AllLayersScope: + return newAllLayersResolver(s.Image) + default: + return nil, fmt.Errorf("bad image scope provided: %+v", scope) + } + } + return nil, fmt.Errorf("unable to determine FilePathResolver with current scheme=%q", s.Metadata.Scheme) +} diff --git a/syft/source/source_test.go b/syft/source/source_test.go index 35117a3f1..72a97653b 100644 --- a/syft/source/source_test.go +++ b/syft/source/source_test.go @@ -1,7 +1,6 @@ package source import ( - "io/ioutil" "os" "testing" @@ -12,18 +11,7 @@ import ( func TestNewFromImageFails(t *testing.T) { t.Run("no image given", func(t *testing.T) { - _, err := NewFromImage(nil, AllLayersScope, "") - if err == nil { - t.Errorf("expected an error condition but none was given") - } - }) -} - -func TestNewFromImageUnknownOption(t *testing.T) { - img := image.Image{} - - t.Run("unknown option is an error", func(t *testing.T) { - _, err := NewFromImage(&img, UnknownScope, "") + _, err := NewFromImage(nil, "") if err == nil { t.Errorf("expected an error condition but none was given") } @@ -37,7 +25,7 @@ func TestNewFromImage(t *testing.T) { } t.Run("create a new source object from image", func(t *testing.T) { - _, err := NewFromImage(&img, AllLayersScope, "") + _, err := NewFromImage(&img, "") if err != nil { t.Errorf("unexpected error when creating a new Locations from img: %+v", err) } @@ -87,8 +75,11 @@ func TestNewFromDirectory(t *testing.T) { if src.Metadata.Path != test.input { t.Errorf("mismatched stringer: '%s' != '%s'", src.Metadata.Path, test.input) } - - refs, err := src.Resolver.FilesByPath(test.inputPaths...) + resolver, err := src.FileResolver(SquashedScope) + if err != nil { + t.Errorf("could not get resolver error: %+v", err) + } + refs, err := resolver.FilesByPath(test.inputPaths...) if err != nil { t.Errorf("FilesByPath call produced an error: %+v", err) } @@ -101,58 +92,6 @@ func TestNewFromDirectory(t *testing.T) { } } -func TestMultipleFileContentsByLocation(t *testing.T) { - testCases := []struct { - desc string - input string - path string - expected string - }{ - { - input: "test-fixtures/path-detected", - desc: "empty file", - path: "test-fixtures/path-detected/empty", - expected: "", - }, - { - input: "test-fixtures/path-detected", - desc: "file has contents", - path: "test-fixtures/path-detected/.vimrc", - expected: "\" A .vimrc file\n", - }, - } - for _, test := range testCases { - t.Run(test.desc, func(t *testing.T) { - p, err := NewFromDirectory(test.input) - if err != nil { - t.Errorf("could not create NewDirScope: %+v", err) - } - locations, err := p.Resolver.FilesByPath(test.path) - if err != nil { - t.Errorf("could not get file references from path: %s, %v", test.path, err) - } - - if len(locations) != 1 { - t.Fatalf("expected a single location to be generated but got: %d", len(locations)) - } - location := locations[0] - - contents, err := p.Resolver.MultipleFileContentsByLocation([]Location{location}) - contentReader := contents[location] - - content, err := ioutil.ReadAll(contentReader) - if err != nil { - t.Fatalf("cannot read contents: %+v", err) - } - - if string(content) != test.expected { - t.Errorf("unexpected contents from file: '%s' != '%s'", content, test.expected) - } - - }) - } -} - func TestFilesByPathDoesNotExist(t *testing.T) { testCases := []struct { desc string @@ -168,11 +107,15 @@ func TestFilesByPathDoesNotExist(t *testing.T) { } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { - p, err := NewFromDirectory(test.input) + src, err := NewFromDirectory(test.input) if err != nil { t.Errorf("could not create NewDirScope: %+v", err) } - refs, err := p.Resolver.FilesByPath(test.path) + resolver, err := src.FileResolver(SquashedScope) + if err != nil { + t.Errorf("could not get resolver error: %+v", err) + } + refs, err := resolver.FilesByPath(test.path) if err != nil { t.Errorf("could not get file references from path: %s, %v", test.path, err) } @@ -213,12 +156,15 @@ func TestFilesByGlob(t *testing.T) { } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { - p, err := NewFromDirectory(test.input) + src, err := NewFromDirectory(test.input) if err != nil { t.Errorf("could not create NewDirScope: %+v", err) } - - contents, err := p.Resolver.FilesByGlob(test.glob) + resolver, err := src.FileResolver(SquashedScope) + if err != nil { + t.Errorf("could not get resolver error: %+v", err) + } + contents, err := resolver.FilesByGlob(test.glob) if len(contents) != test.expected { t.Errorf("unexpected number of files found by glob (%s): %d != %d", test.glob, len(contents), test.expected) diff --git a/test/acceptance/deb.sh b/test/acceptance/deb.sh index afe7eadcc..46793472d 100755 --- a/test/acceptance/deb.sh +++ b/test/acceptance/deb.sh @@ -42,7 +42,7 @@ docker run --rm \ /bin/bash -x -c "\ DEBIAN_FRONTEND=noninteractive apt install ${DISTDIR}/syft_*_linux_amd64.deb -y && \ syft version && \ - syft ${TEST_IMAGE} -vv -o json > ${REPORT} \ + syft packages ${TEST_IMAGE} -vv -o json > ${REPORT} \ " # keep the generated report around diff --git a/test/acceptance/mac.sh b/test/acceptance/mac.sh index fade10d76..b55a488f7 100755 --- a/test/acceptance/mac.sh +++ b/test/acceptance/mac.sh @@ -33,14 +33,14 @@ trap cleanup EXIT skopeo --version || brew install skopeo # fetch test image -skopeo --override-os linux copy docker://docker.io/${TEST_IMAGE} docker-archive:${TEST_IMAGE_TAR} +[[ -f ${TEST_IMAGE_TAR} ]] || skopeo --override-os linux copy "docker://docker.io/${TEST_IMAGE}" "docker-archive:${TEST_IMAGE_TAR}" ls -alh ${TEST_IMAGE_TAR} # run syft SYFT_PATH="${DISTDIR}/syft-macos_darwin_amd64/syft" chmod 755 "${SYFT_PATH}" "${SYFT_PATH}" version -SYFT_CHECK_FOR_APP_UPDATE=0 "${SYFT_PATH}" docker-archive://${TEST_IMAGE_TAR} -vv -o json > "${REPORT}" +SYFT_CHECK_FOR_APP_UPDATE=0 "${SYFT_PATH}" packages docker-archive://${TEST_IMAGE_TAR} -vv -o json > "${REPORT}" # keep the generated report around mkdir -p ${RESULTSDIR} diff --git a/test/acceptance/rpm.sh b/test/acceptance/rpm.sh index 062375cb0..53d6800c0 100755 --- a/test/acceptance/rpm.sh +++ b/test/acceptance/rpm.sh @@ -41,7 +41,7 @@ docker run --rm \ /bin/bash -x -c "\ rpm -ivh ${DISTDIR}/syft_*_linux_amd64.rpm && \ syft version && \ - syft ${TEST_IMAGE} -vv -o json > ${REPORT} \ + syft packages ${TEST_IMAGE} -vv -o json > ${REPORT} \ " # keep the generated report around diff --git a/test/cli/json_schema_test.go b/test/cli/json_schema_test.go new file mode 100644 index 000000000..669123639 --- /dev/null +++ b/test/cli/json_schema_test.go @@ -0,0 +1,91 @@ +package cli + +import ( + "fmt" + "path" + "strings" + "testing" + + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/anchore/syft/internal" + "github.com/xeipuuv/gojsonschema" +) + +// this is the path to the json schema directory relative to the root of the repo +const jsonSchemaPath = "schema/json" + +func TestJSONSchema(t *testing.T) { + + imageFixture := func(t *testing.T) string { + fixtureImageName := "image-pkg-coverage" + imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) + tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) + return "docker-archive:" + tarPath + } + + tests := []struct { + name string + subcommand string + args []string + fixture func(*testing.T) string + }{ + { + name: "packages:image:docker-archive:pkg-coverage", + subcommand: "packages", + args: []string{"-o", "json"}, + fixture: imageFixture, + }, + { + name: "power-user:image:docker-archive:pkg-coverage", + subcommand: "power-user", + fixture: imageFixture, + }, + { + name: "packages:dir:pkg-coverage", + subcommand: "packages", + args: []string{"-o", "json"}, + fixture: func(t *testing.T) string { + return "dir:test-fixtures/image-pkg-coverage" + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fixtureRef := test.fixture(t) + args := []string{ + test.subcommand, fixtureRef, "-q", + } + for _, a := range test.args { + args = append(args, a) + } + + _, stdout, _ := runSyftCommand(t, nil, args...) + + if len(strings.Trim(stdout, "\n ")) < 100 { + t.Fatalf("bad syft output: %q", stdout) + } + + validateAgainstV1Schema(t, stdout) + }) + } +} + +func validateAgainstV1Schema(t testing.TB, json string) { + fullSchemaPath := path.Join(repoRoot(t), jsonSchemaPath, fmt.Sprintf("schema-%s.json", internal.JSONSchemaVersion)) + schemaLoader := gojsonschema.NewReferenceLoader(fmt.Sprintf("file://%s", fullSchemaPath)) + documentLoader := gojsonschema.NewStringLoader(json) + + result, err := gojsonschema.Validate(schemaLoader, documentLoader) + if err != nil { + t.Fatal("unable to validate json schema:", err.Error()) + } + + if !result.Valid() { + t.Errorf("failed json schema validation:") + t.Errorf("JSON:\n%s\n", json) + for _, desc := range result.Errors() { + t.Errorf(" - %s\n", desc) + } + } +} diff --git a/test/cli/packages_cmd_test.go b/test/cli/packages_cmd_test.go new file mode 100644 index 000000000..e5c28dfbb --- /dev/null +++ b/test/cli/packages_cmd_test.go @@ -0,0 +1,139 @@ +package cli + +import ( + "strings" + "testing" + + "github.com/anchore/syft/syft/source" +) + +func TestPackagesCmdFlags(t *testing.T) { + request := "docker-archive:" + getFixtureImage(t, "image-pkg-coverage") + + tests := []struct { + name string + args []string + env map[string]string + assertions []traitAssertion + }{ + { + name: "json-output-flag", + args: []string{"packages", "-o", "json", request}, + assertions: []traitAssertion{ + assertJsonReport, + assertSource(source.SquashedScope), + assertSuccessfulReturnCode, + }, + }, + { + name: "output-env-binding", + env: map[string]string{ + "SYFT_OUTPUT": "json", + }, + args: []string{"packages", request}, + assertions: []traitAssertion{ + assertJsonReport, + assertSuccessfulReturnCode, + }, + }, + { + name: "table-output-flag", + args: []string{"packages", "-o", "table", request}, + assertions: []traitAssertion{ + assertTableReport, + assertSuccessfulReturnCode, + }, + }, + { + name: "default-output-flag", + args: []string{"packages", request}, + assertions: []traitAssertion{ + assertTableReport, + assertSuccessfulReturnCode, + }, + }, + { + name: "squashed-scope-flag", + args: []string{"packages", "-o", "json", "-s", "squashed", request}, + assertions: []traitAssertion{ + assertSource(source.SquashedScope), + assertSuccessfulReturnCode, + }, + }, + { + name: "all-layers-scope-flag", + args: []string{"packages", "-o", "json", "-s", "all-layers", request}, + assertions: []traitAssertion{ + assertSource(source.AllLayersScope), + assertSuccessfulReturnCode, + }, + }, + { + name: "packages-scope-env-binding", + env: map[string]string{ + "SYFT_PACKAGE_CATALOGER_SCOPE": "all-layers", + }, + args: []string{"packages", "-o", "json", request}, + assertions: []traitAssertion{ + assertSource(source.AllLayersScope), + assertSuccessfulReturnCode, + }, + }, + { + name: "attempt-upload-on-cli-switches", + args: []string{"packages", "-vv", "-H", "localhost:8080", "-u", "the-username", "-d", "test-fixtures/image-pkg-coverage/Dockerfile", "--overwrite-existing-image", request}, + env: map[string]string{ + "SYFT_ANCHORE_PATH": "path/to/api", + "SYFT_ANCHORE_PASSWORD": "the-password", + }, + assertions: []traitAssertion{ + // we cannot easily assert a successful upload behavior, so instead we are doing the next best thing + // and asserting that the parsed configuration has the expected values and we see log entries + // indicating an upload attempt. + assertNotInOutput("the-username"), + assertNotInOutput("the-password"), + assertInOutput("uploading results to localhost:8080"), + assertInOutput(`dockerfile: test-fixtures/image-pkg-coverage/Dockerfile`), + assertInOutput(`overwrite-existing-image: true`), + assertInOutput(`path: path/to/api`), + assertInOutput(`host: localhost:8080`), + assertFailingReturnCode, // upload can't go anywhere, so if this passes that would be surprising + }, + }, + { + name: "dockerfile-without-upload-is-invalid", + args: []string{"packages", "-vv", "-d", "test-fixtures/image-pkg-coverage/Dockerfile", request}, + assertions: []traitAssertion{ + + assertNotInOutput("uploading results to localhost:8080"), + assertInOutput("invalid application config: cannot provide dockerfile option without enabling upload"), + assertFailingReturnCode, + }, + }, + { + name: "attempt-upload-with-env-host-set", + args: []string{"packages", "-vv", request}, + env: map[string]string{ + "SYFT_ANCHORE_HOST": "localhost:8080", + }, + assertions: []traitAssertion{ + assertInOutput("uploading results to localhost:8080"), + assertFailingReturnCode, // upload can't go anywhere, so if this passes that would be surprising + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd, stdout, stderr := runSyftCommand(t, test.env, test.args...) + for _, traitFn := range test.assertions { + traitFn(t, stdout, stderr, cmd.ProcessState.ExitCode()) + } + if t.Failed() { + t.Log("STDOUT:\n", stdout) + t.Log("STDERR:\n", stderr) + t.Log("COMMAND:", strings.Join(cmd.Args, " ")) + } + }) + } +} diff --git a/test/cli/power_user_cmd_test.go b/test/cli/power_user_cmd_test.go new file mode 100644 index 000000000..603c1a18d --- /dev/null +++ b/test/cli/power_user_cmd_test.go @@ -0,0 +1,50 @@ +package cli + +import ( + "strings" + "testing" +) + +func TestPowerUserCmdFlags(t *testing.T) { + request := "docker-archive:" + getFixtureImage(t, "image-pkg-coverage") + + tests := []struct { + name string + args []string + env map[string]string + assertions []traitAssertion + }{ + { + name: "json-output-flag-fails", + args: []string{"power-user", "-o", "json", request}, + assertions: []traitAssertion{ + assertFailingReturnCode, + }, + }, + { + name: "default-results", + args: []string{"power-user", request}, + assertions: []traitAssertion{ + assertNotInOutput(" command is deprecated"), // only the root command should be deprecated + assertInOutput(`"type": "regularFile"`), // proof of file-metadata data + assertInOutput(`"algorithm": "sha256"`), // proof of file-metadata default digest algorithm of sha256 + assertInOutput(`"metadataType": "ApkMetadata"`), // proof of package artifacts data + assertSuccessfulReturnCode, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd, stdout, stderr := runSyftCommand(t, test.env, test.args...) + for _, traitFn := range test.assertions { + traitFn(t, stdout, stderr, cmd.ProcessState.ExitCode()) + } + if t.Failed() { + t.Log("STDOUT:\n", stdout) + t.Log("STDERR:\n", stderr) + t.Log("COMMAND:", strings.Join(cmd.Args, " ")) + } + }) + } +} diff --git a/test/cli/root_cmd_test.go b/test/cli/root_cmd_test.go new file mode 100644 index 000000000..ec46ed132 --- /dev/null +++ b/test/cli/root_cmd_test.go @@ -0,0 +1,116 @@ +package cli + +import ( + "strings" + "testing" + + "github.com/sergi/go-diff/diffmatchpatch" +) + +func TestRootCmdAliasesToPackagesSubcommand(t *testing.T) { + request := "docker-archive:" + getFixtureImage(t, "image-pkg-coverage") + + tests := []struct { + name string + env map[string]string + assertions []traitAssertion + }{ + { + name: "go-case", + assertions: []traitAssertion{ + assertTableReport, + assertSuccessfulReturnCode, + }, + }, + { + name: "respond-to-output-binding", + env: map[string]string{ + "SYFT_OUTPUT": "text", + }, + assertions: []traitAssertion{ + assertInOutput("[Image]"), + assertSuccessfulReturnCode, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + aliasCmd, aliasStdout, aliasStderr := runSyftCommand(t, test.env, request) + for _, traitFn := range test.assertions { + traitFn(t, aliasStdout, aliasStderr, aliasCmd.ProcessState.ExitCode()) + } + + pkgCmd, pkgsStdout, pkgsStderr := runSyftCommand(t, test.env, "packages", request) + for _, traitFn := range test.assertions { + traitFn(t, pkgsStdout, pkgsStderr, pkgCmd.ProcessState.ExitCode()) + } + + if aliasStdout != pkgsStdout { + t.Errorf("packages and root command should have same report output but do not!") + dmp := diffmatchpatch.New() + diffs := dmp.DiffMain(aliasStdout, pkgsStdout, true) + t.Error(dmp.DiffPrettyText(diffs)) + } + }) + } +} + +func TestPersistentFlags(t *testing.T) { + request := "docker-archive:" + getFixtureImage(t, "image-pkg-coverage") + + tests := []struct { + name string + args []string + env map[string]string + assertions []traitAssertion + }{ + { + name: "quiet-flag", + // note: the root command will always show the deprecation warning, so the packages command is used instead + args: []string{"packages", "-q", request}, + assertions: []traitAssertion{ + func(tb testing.TB, stdout, stderr string, rc int) { + // ensure there is no status + if len(stderr) != 0 { + tb.Errorf("should have seen no stderr output, got %d bytes", len(stderr)) + } + // ensure there is still a report + if len(stdout) == 0 { + tb.Errorf("should have seen a report on stdout, got nothing") + } + }, + }, + }, + { + name: "info-log-flag", + args: []string{"-v", request}, + assertions: []traitAssertion{ + assertLoggingLevel("info"), + assertSuccessfulReturnCode, + }, + }, + { + name: "debug-log-flag", + args: []string{"-vv", request}, + assertions: []traitAssertion{ + assertLoggingLevel("debug"), + assertSuccessfulReturnCode, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd, stdout, stderr := runSyftCommand(t, test.env, test.args...) + for _, traitFn := range test.assertions { + traitFn(t, stdout, stderr, cmd.ProcessState.ExitCode()) + } + if t.Failed() { + t.Log("STDOUT:\n", stdout) + t.Log("STDERR:\n", stderr) + t.Log("COMMAND:", strings.Join(cmd.Args, " ")) + } + }) + } +} diff --git a/test/cli/test-fixtures/image-pkg-coverage b/test/cli/test-fixtures/image-pkg-coverage new file mode 120000 index 000000000..155332274 --- /dev/null +++ b/test/cli/test-fixtures/image-pkg-coverage @@ -0,0 +1 @@ +../../integration/test-fixtures/image-pkg-coverage \ No newline at end of file diff --git a/test/cli/trait_assertions_test.go b/test/cli/trait_assertions_test.go new file mode 100644 index 000000000..1eb1b807e --- /dev/null +++ b/test/cli/trait_assertions_test.go @@ -0,0 +1,82 @@ +package cli + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" + "testing" + + "github.com/acarl005/stripansi" + "github.com/anchore/syft/syft/source" +) + +type traitAssertion func(tb testing.TB, stdout, stderr string, rc int) + +func assertJsonReport(tb testing.TB, stdout, _ string, _ int) { + var data interface{} + + if err := json.Unmarshal([]byte(stdout), &data); err != nil { + tb.Errorf("expected to find a JSON report, but was unmarshalable: %+v", err) + } +} + +func assertTableReport(tb testing.TB, stdout, _ string, _ int) { + if !strings.Contains(stdout, "NAME") || !strings.Contains(stdout, "VERSION") || !strings.Contains(stdout, "TYPE") { + tb.Errorf("expected to find a table report, but did not") + } +} + +func assertSource(scope source.Scope) traitAssertion { + return func(tb testing.TB, stdout, stderr string, rc int) { + // we can only verify source with the json report + assertJsonReport(tb, stdout, stderr, rc) + + if !strings.Contains(stdout, fmt.Sprintf(`"scope": "%s"`, scope.String())) { + tb.Errorf("JSON report did not indicate the %q scope", scope) + } + } +} + +func assertLoggingLevel(level string) traitAssertion { + // match examples: + // "[0000] INFO" + // "[0012] DEBUG" + logPattern := regexp.MustCompile(`(?m)^\[\d\d\d\d\]\s+` + strings.ToUpper(level)) + return func(tb testing.TB, _, stderr string, _ int) { + if !logPattern.MatchString(stripansi.Strip(stderr)) { + tb.Errorf("output did not indicate the %q logging level", level) + } + } +} + +func assertNotInOutput(data string) traitAssertion { + return func(tb testing.TB, stdout, stderr string, _ int) { + if strings.Contains(stripansi.Strip(stderr), data) { + tb.Errorf("data=%q was found in stderr, but should not have been there", data) + } + if strings.Contains(stripansi.Strip(stdout), data) { + tb.Errorf("data=%q was found in stdout, but should not have been there", data) + } + } +} + +func assertInOutput(data string) traitAssertion { + return func(tb testing.TB, stdout, stderr string, _ int) { + if !strings.Contains(stripansi.Strip(stderr), data) && !strings.Contains(stripansi.Strip(stdout), data) { + tb.Errorf("data=%q was NOT found in any output, but should have been there", data) + } + } +} + +func assertFailingReturnCode(tb testing.TB, _, _ string, rc int) { + if rc == 0 { + tb.Errorf("expected a failure but got rc=%d", rc) + } +} + +func assertSuccessfulReturnCode(tb testing.TB, _, _ string, rc int) { + if rc != 0 { + tb.Errorf("expected no failure but got rc=%d", rc) + } +} diff --git a/test/cli/utils_test.go b/test/cli/utils_test.go new file mode 100644 index 000000000..a12525d47 --- /dev/null +++ b/test/cli/utils_test.go @@ -0,0 +1,76 @@ +package cli + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/anchore/stereoscope/pkg/imagetest" +) + +func getFixtureImage(t testing.TB, fixtureImageName string) string { + imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) + return imagetest.GetFixtureImageTarPath(t, fixtureImageName) +} + +func runSyftCommand(t testing.TB, env map[string]string, args ...string) (*exec.Cmd, string, string) { + cmd := getSyftCommand(t, args...) + if env != nil { + var envList []string + for key, val := range env { + if key == "" { + continue + } + envList = append(envList, fmt.Sprintf("%s=%s", key, val)) + } + cmd.Env = envList + } + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + // ignore errors since this may be what the test expects + cmd.Run() + + return cmd, stdout.String(), stderr.String() +} + +func getSyftCommand(t testing.TB, args ...string) *exec.Cmd { + + var binaryLocation string + if os.Getenv("SYFT_BINARY_LOCATION") != "" { + // SYFT_BINARY_LOCATION is the absolute path to the snapshot binary + binaryLocation = os.Getenv("SYFT_BINARY_LOCATION") + } else { + // note: there is a subtle - vs _ difference between these versions + switch runtime.GOOS { + case "darwin": + binaryLocation = path.Join(repoRoot(t), fmt.Sprintf("snapshot/syft-macos_darwin_%s/syft", runtime.GOARCH)) + case "linux": + binaryLocation = path.Join(repoRoot(t), fmt.Sprintf("snapshot/syft_linux_%s/syft", runtime.GOARCH)) + default: + t.Fatalf("unsupported OS: %s", runtime.GOOS) + } + + } + return exec.Command(binaryLocation, args...) +} + +func repoRoot(t testing.TB) string { + t.Helper() + root, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + if err != nil { + t.Fatalf("unable to find repo root dir: %+v", err) + } + absRepoRoot, err := filepath.Abs(strings.TrimSpace(string(root))) + if err != nil { + t.Fatal("unable to get abs path to repo root:", err) + } + return absRepoRoot +} diff --git a/test/inline-compare/.gitignore b/test/inline-compare/.gitignore deleted file mode 100644 index 1bdfd8fd4..000000000 --- a/test/inline-compare/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.json -*.pyc -inline-reports \ No newline at end of file diff --git a/test/inline-compare/Makefile b/test/inline-compare/Makefile deleted file mode 100644 index 945b01698..000000000 --- a/test/inline-compare/Makefile +++ /dev/null @@ -1,49 +0,0 @@ -ifndef SYFT_CMD - SYFT_CMD = go run ../../main.go -endif - -IMAGE_CLEAN = $(shell basename $(COMPARE_IMAGE) | tr ":" "_" ) -SYFT_DIR = syft-reports -SYFT_REPORT = $(SYFT_DIR)/$(IMAGE_CLEAN).json -INLINE_DIR = inline-reports -INLINE_REPORT = $(INLINE_DIR)/$(IMAGE_CLEAN)-content-os.json - -ifndef SYFT_DIR - $(error SYFT_DIR is not set) -endif - -ifndef INLINE_DIR - $(error INLINE_DIR is not set) -endif - -.PHONY: all -.DEFAULT_GOAL := -all: clean-syft - ./compare-all.sh - -.PHONY: compare-image -compare-image: $(SYFT_REPORT) $(INLINE_REPORT) - ./compare.py $(COMPARE_IMAGE) - -.PHONY: gather-image -gather-image: $(SYFT_REPORT) $(INLINE_REPORT) - -$(INLINE_REPORT): - echo "Creating $(INLINE_REPORT)..." - mkdir -p $(INLINE_DIR) - curl -s https://ci-tools.anchore.io/inline_scan-v0.7.0 | bash -s -- -p -r $(COMPARE_IMAGE) - mv anchore-reports/* $(INLINE_DIR)/ - rmdir anchore-reports - -$(SYFT_REPORT): - echo "Creating $(SYFT_REPORT)..." - mkdir -p $(SYFT_DIR) - $(SYFT_CMD) $(COMPARE_IMAGE) -o json > $(SYFT_REPORT) - -.PHONY: clean -clean: clean-syft - rm -f $(INLINE_DIR)/* - -.PHONY: clean-syft -clean-syft: - rm -f $(SYFT_DIR)/* \ No newline at end of file diff --git a/test/inline-compare/compare-all.sh b/test/inline-compare/compare-all.sh deleted file mode 100755 index 146afe9b2..000000000 --- a/test/inline-compare/compare-all.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -eu - -images=("debian:10.5" "centos:8.2.2004" "rails:5.0.1" "alpine:3.12.0" "anchore/test_images:java" "anchore/test_images:py38" "anchore/anchore-engine:v0.8.2" "jenkins/jenkins:2.249.2-lts-jdk11" ) - -# gather all image analyses -for img in "${images[@]}"; do - echo "Gathering facts for $img" - COMPARE_IMAGE=${img} make gather-image -done - -# compare all results -for img in "${images[@]}"; do - echo "Comparing results for $img" - COMPARE_IMAGE=${img} make compare-image -done diff --git a/test/inline-compare/compare.py b/test/inline-compare/compare.py deleted file mode 100755 index 6dc5bb655..000000000 --- a/test/inline-compare/compare.py +++ /dev/null @@ -1,234 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys -import difflib -import collections - -import utils.package -from utils.format import Colors, print_rows -from utils.inline import InlineScan -from utils.syft import Syft - -DEFAULT_QUALITY_GATE_THRESHOLD = 0.95 -INDENT = " " - -PACKAGE_QUALITY_GATE = collections.defaultdict(lambda: DEFAULT_QUALITY_GATE_THRESHOLD, **{}) -METADATA_QUALITY_GATE = collections.defaultdict(lambda: DEFAULT_QUALITY_GATE_THRESHOLD, **{ - # syft is better at detecting package versions in specific cases, leading to a drop in matching metadata - "anchore/test_images:java": 0.61, - "jenkins/jenkins:2.249.2-lts-jdk11": 0.85, -}) - -# We additionally fail if an image is above a particular threshold. Why? We expect the lower threshold to be 90%, -# however additional functionality in grype is still being implemented, so this threshold may not be able to be met. -# In these cases the IMAGE_QUALITY_GATE is set to a lower value to allow the test to pass for known issues. Once these -# issues/enhancements are done we want to ensure that the lower threshold is bumped up to catch regression. The only way -# to do this is to select an upper threshold for images with known threshold values, so we have a failure that -# loudly indicates the lower threshold should be bumped. -PACKAGE_UPPER_THRESHOLD = collections.defaultdict(lambda: 1, **{}) -METADATA_UPPER_THRESHOLD = collections.defaultdict(lambda: 1, **{ - # syft is better at detecting package versions in specific cases, leading to a drop in matching metadata - "anchore/test_images:java": 0.65, - "jenkins/jenkins:2.249.2-lts-jdk11": 0.9, -}) - - -def report(image, analysis): - if analysis.extra_packages: - rows = [] - print( - Colors.bold + "Syft found extra packages:", - Colors.reset, - "Syft discovered packages that Inline did not", - ) - for package in sorted(list(analysis.extra_packages)): - rows.append([INDENT, repr(package)]) - print_rows(rows) - print() - - if analysis.missing_packages: - rows = [] - print( - Colors.bold + "Syft missed packages:", - Colors.reset, - "Inline discovered packages that Syft did not", - ) - for package in sorted(list(analysis.missing_packages)): - rows.append([INDENT, repr(package)]) - print_rows(rows) - print() - - if analysis.missing_metadata: - print( - Colors.bold + "Syft mismatched metadata:", - Colors.reset, - "the packages between Syft and Inline are the same, the metadata is not", - ) - for inline_metadata_pair in sorted(list(analysis.missing_metadata)): - pkg, metadata = inline_metadata_pair - if pkg not in analysis.syft_data.metadata[pkg.type]: - continue - syft_metadata_item = analysis.syft_data.metadata[pkg.type][pkg] - - diffs = difflib.ndiff([repr(syft_metadata_item)], [repr(metadata)]) - - print(INDENT + "for: " + repr(pkg), "(top is syft, bottom is inline)") - print(INDENT+INDENT+("\n"+INDENT+INDENT).join(list(diffs))) - - if not analysis.missing_metadata: - print( - INDENT, - "There are mismatches, but only due to packages Syft did not find (but inline did).\n", - ) - - if analysis.similar_missing_packages: - rows = [] - print( - Colors.bold + "Probably pairings of missing/extra packages:", - Colors.reset, - "to aid in troubleshooting missed/extra packages", - ) - for similar_packages in analysis.similar_missing_packages: - rows.append( - [ - INDENT, - repr(similar_packages.pkg), - "--->", - repr(similar_packages.missed), - ] - ) - print_rows(rows) - print() - - show_probable_mismatches = analysis.unmatched_missing_packages and analysis.extra_packages and len(analysis.unmatched_missing_packages) != len(analysis.missing_packages) - - if show_probable_mismatches: - rows = [] - print( - Colors.bold + "Probably missed packages:", - Colors.reset, - "a probable pair was not found", - ) - for p in analysis.unmatched_missing_packages: - rows.append([INDENT, repr(p)]) - print_rows(rows) - print() - - print(Colors.bold + "Summary:", Colors.reset, image) - print(" Inline Packages : %d" % len(analysis.inline_data.packages)) - print(" Syft Packages : %d" % len(analysis.syft_data.packages)) - print( - " (extra) : %d (note: this is ignored by the quality gate!)" - % len(analysis.extra_packages) - ) - print(" (missing) : %d" % len(analysis.missing_packages)) - print() - - if show_probable_mismatches: - print( - " Probable Package Matches : %d (matches not made, but were probably found by both Inline and Syft)" - % len(analysis.similar_missing_packages) - ) - print( - " Probable Packages Matched : %2.3f %% (%d/%d packages)" - % ( - analysis.percent_probable_overlapping_packages, - len(analysis.overlapping_packages) - + len(analysis.similar_missing_packages), - len(analysis.inline_data.packages), - ) - ) - print( - " Probable Packages Missing : %d " - % len(analysis.unmatched_missing_packages) - ) - print() - print( - " Baseline Packages Matched : %2.3f %% (%d/%d packages)" - % ( - analysis.percent_overlapping_packages, - len(analysis.overlapping_packages), - len(analysis.inline_data.packages), - ) - ) - print( - " Baseline Metadata Matched : %2.3f %% (%d/%d metadata)" - % ( - analysis.percent_overlapping_metadata, - len(analysis.overlapping_metadata), - len(analysis.inline_metadata), - ) - ) - - -def enforce_quality_gate(title, actual_value, lower_gate_value, upper_gate_value): - - if actual_value < lower_gate_value: - print( - Colors.bold - + " %s Quality Gate:\t" % title - + Colors.FG.red - + "FAIL (is not >= %d %%)" % lower_gate_value, - Colors.reset, - ) - return False - elif actual_value > upper_gate_value: - print( - Colors.bold - + " %s Quality Gate:\t" % title - + Colors.FG.orange - + "FAIL (lower threshold is artificially low and should be updated)", - Colors.reset, - ) - return False - - print( - Colors.bold - + " %s Quality Gate:\t" % title - + Colors.FG.green - + "Pass (>= %d %%)" % lower_gate_value, - Colors.reset, - ) - - return True - -def main(image): - cwd = os.path.dirname(os.path.abspath(__file__)) - - # parse the inline-scan and syft reports on disk - inline = InlineScan(image=image, report_dir=os.path.join(cwd, "inline-reports")) - syft = Syft(image=image, report_dir=os.path.join(cwd, "syft-reports")) - - # analyze the raw data to generate all derivative data for the report and quality gate - analysis = utils.package.Analysis( - syft_data=syft.packages(), inline_data=inline.packages() - ) - - # show some useful report data for debugging / warm fuzzies - report(image, analysis) - - # enforce a quality gate based on the comparison of package values and metadata values - success = True - success &= enforce_quality_gate( - title="Package", - actual_value=analysis.percent_overlapping_packages, - lower_gate_value=PACKAGE_QUALITY_GATE[image] * 100, - upper_gate_value=PACKAGE_UPPER_THRESHOLD[image] * 100 - ) - success &= enforce_quality_gate( - title="Metadata", - actual_value=analysis.percent_overlapping_metadata, - lower_gate_value=METADATA_QUALITY_GATE[image] * 100, - upper_gate_value=METADATA_UPPER_THRESHOLD[image] * 100 - ) - - if not success: - return 1 - return 0 - -if __name__ == "__main__": - if len(sys.argv) != 2: - sys.exit("provide an image") - - rc = main(sys.argv[1]) - sys.exit(rc) diff --git a/test/inline-compare/utils/__init__.py b/test/inline-compare/utils/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test/inline-compare/utils/format.py b/test/inline-compare/utils/format.py deleted file mode 100644 index e376bf820..000000000 --- a/test/inline-compare/utils/format.py +++ /dev/null @@ -1,46 +0,0 @@ -class Colors: - reset = "\033[0m" - bold = "\033[01m" - disable = "\033[02m" - underline = "\033[04m" - reverse = "\033[07m" - strikethrough = "\033[09m" - invisible = "\033[08m" - - class FG: - black = "\033[30m" - red = "\033[31m" - green = "\033[32m" - orange = "\033[33m" - blue = "\033[34m" - purple = "\033[35m" - cyan = "\033[36m" - lightgrey = "\033[37m" - darkgrey = "\033[90m" - lightred = "\033[91m" - lightgreen = "\033[92m" - yellow = "\033[93m" - lightblue = "\033[94m" - pink = "\033[95m" - lightcyan = "\033[96m" - - class BG: - black = "\033[40m" - red = "\033[41m" - green = "\033[42m" - orange = "\033[43m" - blue = "\033[44m" - purple = "\033[45m" - cyan = "\033[46m" - lightgrey = "\033[47m" - - -def print_rows(rows): - if not rows: - return - widths = [] - for col, _ in enumerate(rows[0]): - width = max(len(row[col]) for row in rows) + 2 # padding - widths.append(width) - for row in rows: - print("".join(word.ljust(widths[col_idx]) for col_idx, word in enumerate(row))) diff --git a/test/inline-compare/utils/image.py b/test/inline-compare/utils/image.py deleted file mode 100644 index 8b2d3818a..000000000 --- a/test/inline-compare/utils/image.py +++ /dev/null @@ -1,5 +0,0 @@ -import os - - -def clean(image: str) -> str: - return os.path.basename(image.replace(":", "_")) diff --git a/test/inline-compare/utils/inline.py b/test/inline-compare/utils/inline.py deleted file mode 100644 index 781d6a72a..000000000 --- a/test/inline-compare/utils/inline.py +++ /dev/null @@ -1,142 +0,0 @@ -import os -import re -import json -import collections - -import utils.package -import utils.image - - -class InlineScan: - """ - Class for parsing inlnie-scan output files into a set of packages and package metadata. - """ - report_tmpl = "{image}-{report}.json" - - def __init__(self, image, report_dir): - self.report_dir = report_dir - self.image = image - - def packages(self): - python_packages, python_metadata = self._python_packages() - gem_packages, gem_metadata = self._gem_packages() - java_packages, java_metadata = self._java_packages() - npm_packages, npm_metadata = self._npm_packages() - os_packages, os_metadata = self._os_packages() - - packages = ( - python_packages | os_packages | gem_packages | java_packages | npm_packages - ) - metadata = { - **python_metadata, - **os_metadata, - **gem_metadata, - **java_metadata, - **npm_metadata, - } - - return utils.package.Info(packages=frozenset(packages), metadata=metadata) - - def _report_path(self, report): - return os.path.join( - self.report_dir, - self.report_tmpl.format(image=utils.image.clean(self.image), report=report), - ) - - def _enumerate_section(self, report, section): - report_path = self._report_path(report=report) - os_report_path = self._report_path(report="content-os") - - if os.path.exists(os_report_path) and not os.path.exists(report_path): - # if the OS report is there but the target report is not, that is engine's way of saying "no findings" - return - - with open(report_path) as json_file: - data = json.load(json_file) - for entry in data[section]: - yield entry - - def _java_packages(self): - packages = set() - metadata = collections.defaultdict(dict) - for entry in self._enumerate_section(report="content-java", section="content"): - # normalize to pseudo-inline - pkg_type = entry["type"].lower() - if pkg_type in ("java-jar", "java-war", "java-ear"): - pkg_type = "java-?ar" - elif pkg_type in ("java-jpi", "java-hpi"): - pkg_type = "java-?pi" - - # this would usually be "package" but this would not be able to account for duplicate dependencies in - # nested jars of the same name. Fallback to the package name if there is no given location - name = entry["location"] - - # replace fields with "N/A" with None - for k, v in dict(entry).items(): - if v in ("", "N/A"): - entry[k] = None - - pkg = utils.package.Package( - name=name, - type=pkg_type, - ) - packages.add(pkg) - - metadata[pkg.type][pkg] = utils.package.Metadata( - version=entry["maven-version"], - ) - - return packages, metadata - - def _npm_packages(self): - packages = set() - metadata = collections.defaultdict(dict) - for entry in self._enumerate_section(report="content-npm", section="content"): - pkg = utils.package.Package( - name=entry["package"], - type=entry["type"].lower(), - ) - packages.add(pkg) - metadata[pkg.type][pkg] = utils.package.Metadata(version=entry["version"]) - - return packages, metadata - - def _python_packages(self): - packages = set() - metadata = collections.defaultdict(dict) - for entry in self._enumerate_section( - report="content-python", section="content" - ): - pkg = utils.package.Package( - name=entry["package"], - type=entry["type"].lower(), - ) - packages.add(pkg) - metadata[pkg.type][pkg] = utils.package.Metadata(version=entry["version"]) - - return packages, metadata - - def _gem_packages(self): - packages = set() - metadata = collections.defaultdict(dict) - for entry in self._enumerate_section(report="content-gem", section="content"): - pkg = utils.package.Package( - name=entry["package"], - type=entry["type"].lower(), - ) - packages.add(pkg) - metadata[pkg.type][pkg] = utils.package.Metadata(version=entry["version"]) - - return packages, metadata - - def _os_packages(self): - packages = set() - metadata = collections.defaultdict(dict) - for entry in self._enumerate_section(report="content-os", section="content"): - pkg = utils.package.Package( - name=entry["package"], type=entry["type"].lower() - ) - packages.add(pkg) - metadata[pkg.type][pkg] = utils.package.Metadata(version=entry["version"]) - - return packages, metadata diff --git a/test/inline-compare/utils/package.py b/test/inline-compare/utils/package.py deleted file mode 100644 index 13118a4b5..000000000 --- a/test/inline-compare/utils/package.py +++ /dev/null @@ -1,146 +0,0 @@ -import difflib -import collections -import dataclasses -from typing import Set, FrozenSet, Tuple, Any, List - -Metadata = collections.namedtuple("Metadata", "version") -Package = collections.namedtuple("Package", "name type") -Info = collections.namedtuple("Info", "packages metadata") - -SimilarPackages = collections.namedtuple("SimilarPackages", "pkg missed") -ProbableMatch = collections.namedtuple("ProbableMatch", "pkg ratio") - - -@dataclasses.dataclass() -class Analysis: - """ - A package metadata analysis class. When given the raw syft and inline data, all necessary derivative information - needed to do a comparison of package and metadata is performed, allowing callers to interpret the results - """ - - # all raw data from the inline scan and syft reports - syft_data: Info - inline_data: Info - - # all derivative information (derived from the raw data above) - overlapping_packages: FrozenSet[Package] = dataclasses.field(init=False) - extra_packages: FrozenSet[Package] = dataclasses.field(init=False) - missing_packages: FrozenSet[Package] = dataclasses.field(init=False) - - inline_metadata: Set[Tuple[Any, Any]] = dataclasses.field(init=False) - missing_metadata: Set[Tuple[Any, Any]] = dataclasses.field(init=False) - overlapping_metadata: Set[Tuple[Any, Any]] = dataclasses.field(init=False) - - similar_missing_packages: List[Package] = dataclasses.field(init=False) - unmatched_missing_packages: List[Package] = dataclasses.field(init=False) - - def __post_init__(self): - if not self.valid(): - raise RuntimeError("invalid data given") - - # basic sets derived from package information - self.overlapping_packages = self.syft_data.packages & self.inline_data.packages - self.extra_packages = self.syft_data.packages - self.inline_data.packages - self.missing_packages = self.inline_data.packages - self.syft_data.packages - - # basic sets derived from metadata information - self.inline_metadata = self._inline_metadata() - self.overlapping_metadata = self._overlapping_metadata() - self.missing_metadata = self.inline_metadata - self.overlapping_metadata - - # try to account for potential false negatives by pairing extra packages discovered only by syft with missing - # packages discovered only by inline scan. - ( - similar_missing_packages, - unmatched_missing_packages, - ) = self._pair_similar_packages(self.extra_packages, self.missing_packages) - self.similar_missing_packages = similar_missing_packages - self.unmatched_missing_packages = unmatched_missing_packages - - def valid(self) -> bool: - # we are purposefully selecting test images that are guaranteed to have packages (this should never happen). - # ... if it does, then this analysis is not valid! - return bool(self.inline_data.packages) - - def _inline_metadata(self): - """ - Returns the set of inline scan metadata paired with the corresponding package info. - """ - inline_metadata_set = set() - for package in self.inline_data.packages: - metadata = self.inline_data.metadata[package.type][package] - inline_metadata_set.add((package, metadata)) - return inline_metadata_set - - def _overlapping_metadata(self): - """ - Returns the metadata which has been found similar between both syft and inline scan. - """ - syft_overlap_metadata_set = set() - for package in self.syft_data.packages: - metadata = self.syft_data.metadata[package.type][package] - # we only want to really count mismatched metadata for packages that are at least found by inline - if package in self.inline_data.metadata.get(package.type, []): - syft_overlap_metadata_set.add((package, metadata)) - - return syft_overlap_metadata_set & self.inline_metadata - - @staticmethod - def _pair_similar_packages(extra_packages, missing_packages, similar_threshold=0.7): - """ - Try to account for potential false negatives by pairing extra packages discovered only by syft with missing - packages discovered only by inline scan. - """ - matches = collections.defaultdict(set) - found = {} - for s in extra_packages: - for i in missing_packages: - ratio = difflib.SequenceMatcher(None, s.name, i.name).ratio() - if ratio >= similar_threshold: - if i in found: - # only allow for an inline package to be paired once - if ratio < found[i]: - continue - else: - matches[s].discard(i) - - # persist the result - found[i] = ratio - matches[s].add(i) - - results = [] - for s, i_set in matches.items(): - missed = tuple([ProbableMatch(pkg=i, ratio=found[i]) for i in i_set]) - results.append(SimilarPackages(pkg=s, missed=missed)) - - not_found = [i for i in missing_packages if i not in found] - - return sorted(results, key=lambda x: x.pkg), sorted( - not_found, key=lambda x: x.name - ) - - @property - def percent_overlapping_packages(self): - """Returns a percentage representing how many packages that were found relative to the number of expected""" - return ( - float(len(self.overlapping_packages)) - / float(len(self.inline_data.packages)) - ) * 100.0 - - @property - def percent_overlapping_metadata(self): - """Returns a percentage representing how many matching metdata that were found relative to the number of expected""" - return ( - float(len(self.overlapping_metadata)) / float(len(self.inline_metadata)) - ) * 100.0 - - @property - def percent_probable_overlapping_packages(self): - """ - Returns a percentage representing how many packages that were found relative to the number of expected after - considering pairing of missing packages with extra packages in a fuzzy match. - """ - return ( - float(len(self.overlapping_packages) + len(self.similar_missing_packages)) - / float(len(self.inline_data.packages)) - ) * 100.0 diff --git a/test/inline-compare/utils/syft.py b/test/inline-compare/utils/syft.py deleted file mode 100644 index 51c7e0ba0..000000000 --- a/test/inline-compare/utils/syft.py +++ /dev/null @@ -1,68 +0,0 @@ -import os -import json -import collections - -import utils.package -import utils.image - - -class Syft: - """ - Class for parsing syft output into a set of packages and package metadata. - """ - report_tmpl = "{image}.json" - - def __init__(self, image, report_dir): - self.report_path = os.path.join( - report_dir, self.report_tmpl.format(image=utils.image.clean(image)) - ) - - def _enumerate_section(self, section): - with open(self.report_path) as json_file: - data = json.load(json_file) - for entry in data[section]: - yield entry - - def packages(self): - packages = set() - metadata = collections.defaultdict(dict) - for entry in self._enumerate_section(section="artifacts"): - - # normalize to inline - pkg_type = entry["type"].lower() - if pkg_type in ("wheel", "egg", "python"): - pkg_type = "python" - elif pkg_type in ("deb",): - pkg_type = "dpkg" - elif pkg_type in ("java-archive",): - # normalize to pseudo-inline - pkg_type = "java-?ar" - elif pkg_type in ("jenkins-plugin",): - # normalize to pseudo-inline - pkg_type = "java-?pi" - elif pkg_type in ("apk",): - pkg_type = "apkg" - - name = entry["name"] - version = entry["version"] - - if "java" in pkg_type: - # we need to use the virtual path instead of the name to account for nested dependencies with the same - # package name (but potentially different metadata) - name = entry.get("metadata", {}).get("virtualPath") - - elif pkg_type == "apkg": - # inline scan strips off the release from the version, which should be normalized here - fields = entry["version"].split("-") - version = "-".join(fields[:-1]) - - pkg = utils.package.Package( - name=name, - type=pkg_type, - ) - - packages.add(pkg) - - metadata[pkg.type][pkg] = utils.package.Metadata(version=version) - - return utils.package.Info(packages=frozenset(packages), metadata=metadata) diff --git a/test/integration/pkg_cases_test.go b/test/integration/catalog_packages_cases_test.go similarity index 100% rename from test/integration/pkg_cases_test.go rename to test/integration/catalog_packages_cases_test.go diff --git a/test/integration/pkg_coverage_test.go b/test/integration/catalog_packages_test.go similarity index 79% rename from test/integration/pkg_coverage_test.go rename to test/integration/catalog_packages_test.go index d676c7644..33df9c172 100644 --- a/test/integration/pkg_coverage_test.go +++ b/test/integration/catalog_packages_test.go @@ -3,25 +3,54 @@ package integration import ( "testing" + "github.com/anchore/syft/syft/distro" + "github.com/anchore/syft/syft/pkg/cataloger" + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/anchore/syft/syft/source" + "github.com/go-test/deep" "github.com/anchore/syft/internal" - "github.com/anchore/syft/syft" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/source" ) -func TestPkgCoverageImage(t *testing.T) { +func BenchmarkImagePackageCatalogers(b *testing.B) { fixtureImageName := "image-pkg-coverage" - _, cleanup := imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) - tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) - defer cleanup() + imagetest.GetFixtureImage(b, "docker-archive", fixtureImageName) + tarPath := imagetest.GetFixtureImageTarPath(b, fixtureImageName) - _, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) - if err != nil { - t.Fatalf("failed to catalog image: %+v", err) + var pc *pkg.Catalog + for _, c := range cataloger.ImageCatalogers() { + // in case of future alteration where state is persisted, assume no dependency is safe to reuse + theSource, cleanupSource, err := source.New("docker-archive:" + tarPath) + b.Cleanup(cleanupSource) + if err != nil { + b.Fatalf("unable to get source: %+v", err) + } + + resolver, err := theSource.FileResolver(source.SquashedScope) + if err != nil { + b.Fatalf("unable to get resolver: %+v", err) + } + + theDistro := distro.Identify(resolver) + + b.Run(c.Name(), func(b *testing.B) { + for i := 0; i < b.N; i++ { + pc, err = cataloger.Catalog(resolver, theDistro, c) + if err != nil { + b.Fatalf("failure during benchmark: %+v", err) + } + } + }) + + b.Logf("catalog for %q number of packages: %d", c.Name(), pc.PackageCount()) } +} + +func TestPkgCoverageImage(t *testing.T) { + catalog, _, _ := catalogFixtureImage(t, "image-pkg-coverage") observedLanguages := internal.NewStringSet() definedLanguages := internal.NewStringSet() @@ -100,11 +129,7 @@ func TestPkgCoverageImage(t *testing.T) { } func TestPkgCoverageDirectory(t *testing.T) { - _, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.SquashedScope) - - if err != nil { - t.Errorf("unable to create source from dir: %+v", err) - } + catalog, _, _ := catalogDirectory(t, "test-fixtures/image-pkg-coverage") observedLanguages := internal.NewStringSet() definedLanguages := internal.NewStringSet() diff --git a/test/integration/distro_test.go b/test/integration/distro_test.go index 51d888e75..a54c472f3 100644 --- a/test/integration/distro_test.go +++ b/test/integration/distro_test.go @@ -3,23 +3,12 @@ package integration import ( "testing" - "github.com/anchore/stereoscope/pkg/imagetest" - "github.com/anchore/syft/syft" "github.com/anchore/syft/syft/distro" - "github.com/anchore/syft/syft/source" "github.com/go-test/deep" ) func TestDistroImage(t *testing.T) { - fixtureImageName := "image-distro-id" - _, cleanup := imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) - tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) - defer cleanup() - - _, _, actualDistro, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) - if err != nil { - t.Fatalf("failed to catalog image: %+v", err) - } + _, actualDistro, _ := catalogFixtureImage(t, "image-distro-id") expected, err := distro.NewDistro(distro.Busybox, "1.31.1", "") if err != nil { diff --git a/test/integration/document_import_test.go b/test/integration/document_import_test.go deleted file mode 100644 index cd80d1ca0..000000000 --- a/test/integration/document_import_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package integration - -import ( - "bytes" - "strings" - "testing" - - "github.com/anchore/stereoscope/pkg/imagetest" - "github.com/anchore/syft/syft" - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/presenter/json" - "github.com/anchore/syft/syft/source" - "github.com/go-test/deep" -) - -func TestCatalogFromJSON(t *testing.T) { - - // ensure each of our fixture images results in roughly the same shape when: - // generate json -> import json -> assert packages and distro are the same (except for select fields) - - tests := []struct { - fixture string - }{ - { - fixture: "image-pkg-coverage", - }, - } - - for _, test := range tests { - t.Run(test.fixture, func(t *testing.T) { - _, cleanup := imagetest.GetFixtureImage(t, "docker-archive", test.fixture) - tarPath := imagetest.GetFixtureImageTarPath(t, test.fixture) - defer cleanup() - - expectedSource, expectedCatalog, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) - if err != nil { - t.Fatalf("failed to catalog image: %+v", err) - } - - var buf bytes.Buffer - jsonPres := json.NewPresenter(expectedCatalog, expectedSource.Metadata, expectedDistro) - if err = jsonPres.Present(&buf); err != nil { - t.Fatalf("failed to write to presenter: %+v", err) - } - - sourceMetadata, actualCatalog, actualDistro, err := syft.CatalogFromJSON(&buf) - if err != nil { - t.Fatalf("failed to import document: %+v", err) - } - - for _, d := range deep.Equal(sourceMetadata, expectedSource.Metadata) { - t.Errorf(" image metadata diff: %+v", d) - } - - for _, d := range deep.Equal(actualDistro, expectedDistro) { - t.Errorf(" distro diff: %+v", d) - } - - var actualPackages, expectedPackages []*pkg.Package - - for _, p := range expectedCatalog.Sorted() { - expectedPackages = append(expectedPackages, p) - } - - for _, p := range actualCatalog.Sorted() { - actualPackages = append(actualPackages, p) - } - - if len(actualPackages) != len(expectedPackages) { - t.Fatalf("mismatched package length: %d != %d", len(actualPackages), len(expectedPackages)) - } - - for i, e := range expectedPackages { - a := actualPackages[i] - - // omit fields that should be missing - if e.MetadataType == pkg.JavaMetadataType { - metadata := e.Metadata.(pkg.JavaMetadata) - metadata.Parent = nil - e.Metadata = metadata - } - - // ignore the virtual path on the location for now - for l := range a.Locations { - a.Locations[l].VirtualPath = "" - e.Locations[l].VirtualPath = "" - } - - for _, d := range deep.Equal(a, e) { - // ignore errors for empty collections vs nil for select fields - if strings.Contains(d, "[] != ") { - continue - } - t.Errorf(" package %d (name=%s) diff: %+v", i, e.Name, d) - } - } - - }) - } - -} diff --git a/test/integration/java_no_main_package_test.go b/test/integration/java_no_main_package_test.go deleted file mode 100644 index e330a78fb..000000000 --- a/test/integration/java_no_main_package_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package integration - -import ( - "github.com/anchore/stereoscope/pkg/imagetest" - "github.com/anchore/syft/syft" - "github.com/anchore/syft/syft/source" - "testing" -) - -func TestJavaNoMainPackage(t *testing.T) { // Regression: https://github.com/anchore/syft/issues/252 - fixtureImageName := "image-java-no-main-package" - _, cleanup := imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) - tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) - defer cleanup() - - _, _, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) - if err != nil { - t.Fatalf("failed to catalog image: %+v", err) - } -} diff --git a/test/integration/json_schema_test.go b/test/integration/json_schema_test.go deleted file mode 100644 index 022c9db54..000000000 --- a/test/integration/json_schema_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package integration - -import ( - "bytes" - "fmt" - "os/exec" - "path" - "path/filepath" - "strings" - "testing" - - "github.com/anchore/syft/internal" - - "github.com/anchore/syft/syft/distro" - "github.com/anchore/syft/syft/presenter" - - "github.com/anchore/stereoscope/pkg/imagetest" - "github.com/anchore/syft/syft" - "github.com/anchore/syft/syft/source" - "github.com/xeipuuv/gojsonschema" -) - -const jsonSchemaPath = "schema/json" - -func repoRoot(t *testing.T) string { - t.Helper() - repoRoot, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() - if err != nil { - t.Fatalf("unable to find repo root dir: %+v", err) - } - absRepoRoot, err := filepath.Abs(strings.TrimSpace(string(repoRoot))) - if err != nil { - t.Fatal("unable to get abs path to repo root:", err) - } - return absRepoRoot -} - -func validateAgainstV1Schema(t *testing.T, json string) { - fullSchemaPath := path.Join(repoRoot(t), jsonSchemaPath, fmt.Sprintf("schema-%s.json", internal.JSONSchemaVersion)) - schemaLoader := gojsonschema.NewReferenceLoader(fmt.Sprintf("file://%s", fullSchemaPath)) - documentLoader := gojsonschema.NewStringLoader(json) - - result, err := gojsonschema.Validate(schemaLoader, documentLoader) - if err != nil { - t.Fatal("unable to validate json schema:", err.Error()) - } - - if !result.Valid() { - t.Errorf("failed json schema validation:") - t.Errorf("JSON:\n%s\n", json) - for _, desc := range result.Errors() { - t.Errorf(" - %s\n", desc) - } - } -} - -func TestJsonSchemaImg(t *testing.T) { - fixtureImageName := "image-pkg-coverage" - _, cleanup := imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) - tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) - defer cleanup() - - src, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) - if err != nil { - t.Fatalf("failed to catalog image: %+v", err) - } - - output := bytes.NewBufferString("") - - d, err := distro.NewDistro(distro.CentOS, "5", "rhel fedora") - if err != nil { - t.Fatalf("bad distro: %+v", err) - } - - p := presenter.GetPresenter(presenter.JSONPresenter, src.Metadata, catalog, &d) - if p == nil { - t.Fatal("unable to get presenter") - } - - err = p.Present(output) - if err != nil { - t.Fatalf("unable to present: %+v", err) - } - - validateAgainstV1Schema(t, output.String()) - -} - -func TestJsonSchemaDirs(t *testing.T) { - src, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.SquashedScope) - if err != nil { - t.Errorf("unable to create source from dir: %+v", err) - } - - output := bytes.NewBufferString("") - - d, err := distro.NewDistro(distro.CentOS, "5", "rhel fedora") - if err != nil { - t.Fatalf("bad distro: %+v", err) - } - - p := presenter.GetPresenter(presenter.JSONPresenter, src.Metadata, catalog, &d) - if p == nil { - t.Fatal("unable to get presenter") - } - - err = p.Present(output) - if err != nil { - t.Fatalf("unable to present: %+v", err) - } - - validateAgainstV1Schema(t, output.String()) -} diff --git a/test/integration/package_ownership_relationship_test.go b/test/integration/package_ownership_relationship_test.go index 490408ccc..dee87a6d2 100644 --- a/test/integration/package_ownership_relationship_test.go +++ b/test/integration/package_ownership_relationship_test.go @@ -5,11 +5,7 @@ import ( "encoding/json" "testing" - "github.com/anchore/stereoscope/pkg/imagetest" - "github.com/anchore/syft/syft" - "github.com/anchore/syft/syft/presenter" - jsonPresenter "github.com/anchore/syft/syft/presenter/json" - "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/internal/presenter/packages" ) func TestPackageOwnershipRelationships(t *testing.T) { @@ -25,27 +21,24 @@ func TestPackageOwnershipRelationships(t *testing.T) { for _, test := range tests { t.Run(test.fixture, func(t *testing.T) { - _, cleanup := imagetest.GetFixtureImage(t, "docker-archive", test.fixture) - tarPath := imagetest.GetFixtureImageTarPath(t, test.fixture) - defer cleanup() + catalog, d, src := catalogFixtureImage(t, test.fixture) - src, catalog, d, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) - if err != nil { - t.Fatalf("failed to catalog image: %+v", err) - } - - p := presenter.GetPresenter(presenter.JSONPresenter, src.Metadata, catalog, d) + p := packages.Presenter(packages.JSONPresenterOption, packages.PresenterConfig{ + SourceMetadata: src.Metadata, + Catalog: catalog, + Distro: d, + }) if p == nil { t.Fatal("unable to get presenter") } output := bytes.NewBufferString("") - err = p.Present(output) + err := p.Present(output) if err != nil { t.Fatalf("unable to present: %+v", err) } - var doc jsonPresenter.Document + var doc packages.JSONDocument decoder := json.NewDecoder(output) if err := decoder.Decode(&doc); err != nil { t.Fatalf("unable to decode json doc: %+v", err) diff --git a/test/integration/regression_test.go b/test/integration/regression_apk_scanner_buffer_size_test.go similarity index 52% rename from test/integration/regression_test.go rename to test/integration/regression_apk_scanner_buffer_size_test.go index 34f8f2ff8..5aaa4f24d 100644 --- a/test/integration/regression_test.go +++ b/test/integration/regression_apk_scanner_buffer_size_test.go @@ -4,25 +4,12 @@ import ( "testing" "github.com/anchore/syft/syft/pkg" - - "github.com/anchore/stereoscope/pkg/imagetest" - "github.com/anchore/syft/syft" - "github.com/anchore/syft/syft/source" ) func TestRegression212ApkBufferSize(t *testing.T) { // This is a regression test for issue #212 (https://github.com/anchore/syft/issues/212) in which the apk db could // not be processed due to a scanner buffer that was too small - - fixtureImageName := "image-large-apk-data" - _, cleanup := imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) - tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) - defer cleanup() - - _, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) - if err != nil { - t.Fatalf("failed to catalog image: %+v", err) - } + catalog, _, _ := catalogFixtureImage(t, "image-large-apk-data") expectedPkgs := 58 actualPkgs := 0 diff --git a/test/integration/regression_java_no_main_package_test.go b/test/integration/regression_java_no_main_package_test.go new file mode 100644 index 000000000..a5f41ddaf --- /dev/null +++ b/test/integration/regression_java_no_main_package_test.go @@ -0,0 +1,9 @@ +package integration + +import ( + "testing" +) + +func TestRegressionJavaNoMainPackage(t *testing.T) { // Regression: https://github.com/anchore/syft/issues/252 + catalogFixtureImage(t, "image-java-no-main-package") +} diff --git a/test/integration/utils_test.go b/test/integration/utils_test.go new file mode 100644 index 000000000..ad194b273 --- /dev/null +++ b/test/integration/utils_test.go @@ -0,0 +1,44 @@ +package integration + +import ( + "testing" + + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/anchore/syft/syft" + "github.com/anchore/syft/syft/distro" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +func catalogFixtureImage(t *testing.T, fixtureImageName string) (*pkg.Catalog, *distro.Distro, source.Source) { + imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) + tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) + + theSource, cleanupSource, err := source.New("docker-archive:" + tarPath) + t.Cleanup(cleanupSource) + if err != nil { + t.Fatalf("unable to get source: %+v", err) + } + + pkgCatalog, actualDistro, err := syft.CatalogPackages(theSource, source.SquashedScope) + if err != nil { + t.Fatalf("failed to catalog image: %+v", err) + } + + return pkgCatalog, actualDistro, theSource +} + +func catalogDirectory(t *testing.T, dir string) (*pkg.Catalog, *distro.Distro, source.Source) { + theSource, cleanupSource, err := source.New("dir:" + dir) + t.Cleanup(cleanupSource) + if err != nil { + t.Fatalf("unable to get source: %+v", err) + } + + pkgCatalog, actualDistro, err := syft.CatalogPackages(theSource, source.AllLayersScope) + if err != nil { + t.Fatalf("failed to catalog image: %+v", err) + } + + return pkgCatalog, actualDistro, theSource +} diff --git a/ui/event_handlers.go b/ui/event_handlers.go index 6f56ac697..103a4dd95 100644 --- a/ui/event_handlers.go +++ b/ui/event_handlers.go @@ -268,8 +268,8 @@ func ReadImageHandler(ctx context.Context, fr *frame.Frame, event partybus.Event return nil } -// CatalogerStartedHandler periodically writes catalog statistics to a single line. -func CatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error { +// PackageCatalogerStartedHandler periodically writes catalog statistics to a single line. +func PackageCatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error { monitor, err := syftEventParsers.ParseCatalogerStarted(event) if err != nil { return fmt.Errorf("bad %s event: %w", event.Type, err) diff --git a/ui/handler.go b/ui/handler.go index 7149ffd46..edf2e5ead 100644 --- a/ui/handler.go +++ b/ui/handler.go @@ -15,7 +15,7 @@ import ( "github.com/wagoodman/jotframe/pkg/frame" ) -// Handler is an aggregated event handler for the set of supported events (PullDockerImage, ReadImage, FetchImage, CatalogerStarted) +// Handler is an aggregated event handler for the set of supported events (PullDockerImage, ReadImage, FetchImage, PackageCatalogerStarted) type Handler struct { } @@ -27,7 +27,7 @@ func NewHandler() *Handler { // RespondsTo indicates if the handler is capable of handling the given event. func (r *Handler) RespondsTo(event partybus.Event) bool { switch event.Type { - case stereoscopeEvent.PullDockerImage, stereoscopeEvent.ReadImage, stereoscopeEvent.FetchImage, syftEvent.CatalogerStarted, syftEvent.ImportStarted: + case stereoscopeEvent.PullDockerImage, stereoscopeEvent.ReadImage, stereoscopeEvent.FetchImage, syftEvent.PackageCatalogerStarted, syftEvent.ImportStarted: return true default: return false @@ -46,8 +46,8 @@ func (r *Handler) Handle(ctx context.Context, fr *frame.Frame, event partybus.Ev case stereoscopeEvent.FetchImage: return FetchImageHandler(ctx, fr, event, wg) - case syftEvent.CatalogerStarted: - return CatalogerStartedHandler(ctx, fr, event, wg) + case syftEvent.PackageCatalogerStarted: + return PackageCatalogerStartedHandler(ctx, fr, event, wg) case syftEvent.ImportStarted: return ImportStartedHandler(ctx, fr, event, wg)