mirror of
https://github.com/anchore/syft.git
synced 2026-10-28 04:52:23 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c5e3b631ac | ||
|
|
05b9d35ee6 | ||
|
|
555376e250 | ||
|
|
fee878028a | ||
|
|
5370daf027 | ||
|
|
8d838b18a8 | ||
|
|
bf2d5ed87e | ||
|
|
a350e5889b | ||
|
|
e4393e7a3d | ||
|
|
26d1975870 | ||
|
|
babb09b3a4 | ||
|
|
b207bc8ee2 | ||
|
|
5b5fa7ec90 | ||
|
|
6d730d24dd | ||
|
|
b22fd987db | ||
|
|
3b4d4b186f | ||
|
|
407769e88c | ||
|
|
1416e3cb7a | ||
|
|
0ccfee03f1 | ||
|
|
bb1facbf81 | ||
|
|
ae7cd6bbb7 | ||
|
|
7bcdafe745 | ||
|
|
4576c081b9 | ||
|
|
d5779a9822 | ||
|
|
9ec3ad58c8 | ||
|
|
176dfdd9c1 | ||
|
|
0f6288881b | ||
|
|
6a49717314 | ||
|
|
06641cfda2 | ||
|
|
a5537943fa | ||
|
|
0030880e74 | ||
|
|
fc8b431ea6 | ||
|
|
bb70b0b43e | ||
|
|
ee0a02621a | ||
|
|
33c27c4f3d | ||
|
|
133d180eec | ||
|
|
37b96a241b | ||
|
|
7f4e8ab97d | ||
|
|
76446abd7d | ||
|
|
c2799b35d8 | ||
|
|
d475e6280a | ||
|
|
e299a5355f | ||
|
|
359212e8ee | ||
|
|
52e719dcb8 | ||
|
|
13c289eb7e | ||
|
|
c1fa701602 | ||
|
|
183b8f79d0 | ||
|
|
558781eed6 | ||
|
|
60f525af18 | ||
|
|
8dbb095fc6 | ||
|
|
d0d7e849c9 |
@@ -1,4 +1,4 @@
|
||||
name: 'Acceptance'
|
||||
name: "Acceptance"
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
@@ -16,7 +16,6 @@ jobs:
|
||||
Build-Snapshot-Artifacts:
|
||||
runs-on: macos-latest # We're creating these snapshot builds on macOS to be consistent with our release workflow's build process, which also takes place on macOS (due to code signing requirements).
|
||||
steps:
|
||||
|
||||
- uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
@@ -25,7 +24,7 @@ jobs:
|
||||
|
||||
- name: Restore bootstrap cache
|
||||
id: cache
|
||||
uses: actions/cache@v2
|
||||
uses: actions/cache@v2.1.3
|
||||
with:
|
||||
path: |
|
||||
~/go/pkg/mod
|
||||
@@ -75,11 +74,10 @@ jobs:
|
||||
|
||||
# Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
|
||||
Acceptance-Linux:
|
||||
needs: [ Build-Snapshot-Artifacts ]
|
||||
needs: [Build-Snapshot-Artifacts]
|
||||
# come Nov 30 2020 ubuntu-latest will be ubuntu-20.04, until then it needs to be explicitly referenced due to python 3.7 specific features being used
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- uses: actions/download-artifact@v2
|
||||
@@ -101,10 +99,9 @@ jobs:
|
||||
|
||||
# Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
|
||||
Acceptance-Mac:
|
||||
needs: [ Build-Snapshot-Artifacts ]
|
||||
needs: [Build-Snapshot-Artifacts]
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- uses: actions/download-artifact@v2
|
||||
@@ -126,11 +123,10 @@ jobs:
|
||||
|
||||
# Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
|
||||
Inline-Compare:
|
||||
needs: [ Build-Snapshot-Artifacts ]
|
||||
needs: [Build-Snapshot-Artifacts]
|
||||
# come Nov 30 2020 ubuntu-latest will be ubuntu-20.04, until then it needs to be explicitly referenced due to python 3.7 specific features being used
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Fingerprint inline-compare sources
|
||||
@@ -138,7 +134,7 @@ jobs:
|
||||
|
||||
- name: Restore inline reports cache
|
||||
id: cache
|
||||
uses: actions/cache@v2
|
||||
uses: actions/cache@v2.1.3
|
||||
with:
|
||||
path: ${{ github.workspace }}/test/inline-compare/inline-reports
|
||||
key: inline-reports-${{ hashFiles('**/inline-compare.fingerprint') }}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
name: 'Release'
|
||||
name: "Release"
|
||||
on:
|
||||
push:
|
||||
# take no actions on push to any branch...
|
||||
branches-ignore:
|
||||
- '**'
|
||||
- "**"
|
||||
# ... only act on release tags
|
||||
tags:
|
||||
- 'v*'
|
||||
- "v*"
|
||||
|
||||
env:
|
||||
GO_VERSION: "1.14.x"
|
||||
@@ -15,7 +15,6 @@ jobs:
|
||||
wait-for-checks:
|
||||
runs-on: ubuntu-latest # This OS choice is arbitrary. None of the steps in this job are specific to either Linux or macOS.
|
||||
steps:
|
||||
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
# we don't want to release commits that have been pushed and tagged, but not necessarily merged onto main
|
||||
@@ -81,10 +80,9 @@ jobs:
|
||||
false
|
||||
|
||||
release:
|
||||
needs: [ wait-for-checks ]
|
||||
needs: [wait-for-checks]
|
||||
runs-on: macos-latest # Due to our code signing process, it's vital that we run our release steps on macOS.
|
||||
steps:
|
||||
|
||||
- uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
@@ -96,7 +94,7 @@ jobs:
|
||||
# We are expecting this cache to have been created during the "Build-Snapshot-Artifacts" job in the "Acceptance" workflow.
|
||||
- name: Restore bootstrap cache
|
||||
id: cache
|
||||
uses: actions/cache@v2
|
||||
uses: actions/cache@v2.1.3
|
||||
with:
|
||||
path: |
|
||||
~/go/pkg/mod
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
name: 'Static Analysis + Unit + Integration'
|
||||
name: "Static Analysis + Unit + Integration"
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
pull_request:
|
||||
jobs:
|
||||
|
||||
Static-Analysis:
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -12,7 +11,6 @@ jobs:
|
||||
platform: [ubuntu-latest]
|
||||
runs-on: ${{ matrix.platform }}
|
||||
steps:
|
||||
|
||||
- uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
@@ -21,7 +19,7 @@ jobs:
|
||||
|
||||
- name: Restore bootstrap cache
|
||||
id: bootstrap-cache
|
||||
uses: actions/cache@v2
|
||||
uses: actions/cache@v2.1.3
|
||||
with:
|
||||
path: |
|
||||
~/go/pkg/mod
|
||||
@@ -49,7 +47,6 @@ jobs:
|
||||
platform: [ubuntu-latest]
|
||||
runs-on: ${{ matrix.platform }}
|
||||
steps:
|
||||
|
||||
- uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
@@ -58,7 +55,7 @@ jobs:
|
||||
|
||||
- name: Restore bootstrap cache
|
||||
id: bootstrap-cache
|
||||
uses: actions/cache@v2
|
||||
uses: actions/cache@v2.1.3
|
||||
with:
|
||||
path: |
|
||||
~/go/pkg/mod
|
||||
@@ -80,7 +77,7 @@ jobs:
|
||||
|
||||
- name: Restore Java test-fixture cache
|
||||
id: unit-java-cache
|
||||
uses: actions/cache@v2
|
||||
uses: actions/cache@v2.1.3
|
||||
with:
|
||||
path: syft/cataloger/java/test-fixtures/java-builds/packages
|
||||
key: ${{ runner.os }}-unit-java-cache-${{ hashFiles( 'syft/cataloger/java/test-fixtures/java-builds/packages.fingerprint' ) }}
|
||||
@@ -95,7 +92,7 @@ jobs:
|
||||
run: make integration-fingerprint
|
||||
|
||||
- name: Restore integration test cache
|
||||
uses: actions/cache@v2
|
||||
uses: actions/cache@v2.1.3
|
||||
with:
|
||||
path: ${{ github.workspace }}/integration/test-fixtures/cache
|
||||
key: ${{ runner.os }}-integration-test-cache-${{ hashFiles('integration/test-fixtures/cache.fingerprint') }}
|
||||
|
||||
+2
-2
@@ -31,7 +31,6 @@ linters:
|
||||
- misspell
|
||||
- nakedret
|
||||
- nolintlint
|
||||
- prealloc
|
||||
- rowserrcheck
|
||||
- scopelint
|
||||
- staticcheck
|
||||
@@ -55,5 +54,6 @@ linters:
|
||||
# - lll # without a way to specify per-line exception cases, this is not usable
|
||||
# - maligned # this is an excellent linter, but tricky to optimize and we are not sensitive to memory layout optimizations
|
||||
# - nestif
|
||||
# - prealloc # following this rule isn't consistently a good idea, as it sometimes forces unnecessary allocations that result in less idiomatic code
|
||||
# - testpackage
|
||||
# - wsl
|
||||
# - wsl
|
||||
|
||||
@@ -16,6 +16,10 @@ TITLE := $(BOLD)$(PURPLE)
|
||||
SUCCESS := $(BOLD)$(GREEN)
|
||||
# the quality gate lower threshold for unit test total % coverage (by function statements)
|
||||
COVERAGE_THRESHOLD := 68
|
||||
# CI cache busting values; change these if you want CI to not use previous stored cache
|
||||
COMPARE_CACHE_BUSTER="f7e689d76a9"
|
||||
INTEGRATION_CACHE_BUSTER="789bacdf"
|
||||
BOOTSTRAP_CACHE="789bacdf"
|
||||
|
||||
## Build variables
|
||||
DISTDIR=./dist
|
||||
@@ -140,7 +144,7 @@ integration: ## Run integration tests
|
||||
|
||||
# note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted
|
||||
integration-fingerprint:
|
||||
find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/cache.fingerprint
|
||||
find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/cache.fingerprint && echo "$(INTEGRATION_CACHE_BUSTER)" >> test/integration/test-fixtures/cache.fingerprint
|
||||
|
||||
.PHONY: java-packages-fingerprint
|
||||
java-packages-fingerprint:
|
||||
@@ -173,6 +177,7 @@ $(SNAPSHOTDIR): ## Build snapshot release binaries and packages
|
||||
BUILD_GIT_TREE_STATE=$(GITTREESTATE) \
|
||||
$(TEMPDIR)/goreleaser release --skip-publish --rm-dist --snapshot --config $(TEMPDIR)/goreleaser.yaml
|
||||
|
||||
# note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately
|
||||
.PHONY: acceptance-mac
|
||||
acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binaries and packages (Mac)
|
||||
$(call title,Running acceptance test: Run on Mac)
|
||||
@@ -182,13 +187,14 @@ acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binarie
|
||||
$(ACC_TEST_IMAGE) \
|
||||
$(RESULTSDIR)
|
||||
|
||||
# note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately
|
||||
.PHONY: acceptance-linux
|
||||
acceptance-linux: acceptance-test-deb-package-install acceptance-test-rpm-package-install ## Run acceptance tests on build snapshot binaries and packages (Linux)
|
||||
|
||||
# note: this is used by CI to determine if the inline-scan report cache should be busted for the inline-compare tests
|
||||
.PHONY: compare-fingerprint
|
||||
compare-fingerprint:
|
||||
find test/inline-compare/* -type f -exec md5sum {} + | grep -v '\-reports' | grep -v 'fingerprint' | awk '{print $1}' | sort | md5sum | tee test/inline-compare/inline-compare.fingerprint
|
||||
find test/inline-compare/* -type f -exec md5sum {} + | grep -v '\-reports' | grep -v 'fingerprint' | awk '{print $1}' | sort | md5sum | tee test/inline-compare/inline-compare.fingerprint && echo "$(COMPARE_CACHE_BUSTER)" >> test/inline-compare/inline-compare.fingerprint
|
||||
|
||||
.PHONY: compare-snapshot
|
||||
compare-snapshot: $(SNAPSHOTDIR) ## Compare the reports of a run of a snapshot build of syft against inline-scan
|
||||
|
||||
+2
-11
@@ -158,22 +158,13 @@ func doImport(src source.Source, s source.Metadata, catalog *pkg.Catalog, d *dis
|
||||
}
|
||||
}
|
||||
|
||||
var scheme string
|
||||
var hostname = appConfig.Anchore.Host
|
||||
urlFields := strings.Split(hostname, "://")
|
||||
if len(urlFields) > 1 {
|
||||
scheme = urlFields[0]
|
||||
hostname = urlFields[1]
|
||||
}
|
||||
|
||||
c, err := anchore.NewClient(anchore.Configuration{
|
||||
Hostname: hostname,
|
||||
BaseURL: appConfig.Anchore.Host,
|
||||
Username: appConfig.Anchore.Username,
|
||||
Password: appConfig.Anchore.Password,
|
||||
Scheme: scheme,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create anchore client: %+v", err)
|
||||
return fmt.Errorf("unable to upload results: %w", err)
|
||||
}
|
||||
|
||||
importCfg := anchore.ImportConfig{
|
||||
|
||||
@@ -9,9 +9,9 @@ require (
|
||||
github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12
|
||||
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
|
||||
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
|
||||
github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e
|
||||
github.com/anchore/stereoscope v0.0.0-20210201165248-e94c52b4052d
|
||||
github.com/antihax/optional v1.0.0
|
||||
github.com/bmatcuk/doublestar v1.3.3
|
||||
github.com/bmatcuk/doublestar/v2 v2.0.4
|
||||
github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible
|
||||
github.com/dustin/go-humanize v1.0.0
|
||||
github.com/facebookincubator/nvdtools v0.1.4
|
||||
|
||||
@@ -126,8 +126,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
|
||||
github.com/anchore/client-go v0.0.0-20201210022459-59e7a0749c74 h1:9kkKTIyXJC+/syUcY6KWxFoJZJ+GWwrIscF+gBY067k=
|
||||
github.com/anchore/client-go v0.0.0-20201210022459-59e7a0749c74/go.mod h1:FaODhIA06mxO1E6R32JE0TL1JWZZkmjRIAd4ULvHUKk=
|
||||
github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238 h1:/iI+1cj1a27ow0wj378pPJIm8sCSy6I21Tz6oLbLDQY=
|
||||
github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238/go.mod h1:FaODhIA06mxO1E6R32JE0TL1JWZZkmjRIAd4ULvHUKk=
|
||||
github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 h1:xbeIbn5F52JVx3RUIajxCj8b0y+9lywspql4sFhcxWQ=
|
||||
@@ -136,8 +134,14 @@ github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0v
|
||||
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ=
|
||||
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods=
|
||||
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
|
||||
github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e h1:vHUqHTvH9/oxdDDh1fxS9Ls9gWGytKO7XbbzcQ9MBwI=
|
||||
github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e/go.mod h1:/dHAFjYflH/1tzhdHAcnMCjprMch+YzHJKi59m/1KCM=
|
||||
github.com/anchore/stereoscope v0.0.0-20210104203718-4c1d1bd9a255 h1:Ng7BDr9PQTCztANogjfEdEjjWUylhlPyZPhtarIGo00=
|
||||
github.com/anchore/stereoscope v0.0.0-20210104203718-4c1d1bd9a255/go.mod h1:BMdPL0QEIYfpjQ3M7sHYZvuh6+vcomqF3TMHL8gr6Vw=
|
||||
github.com/anchore/stereoscope v0.0.0-20210105000809-428eda0b2ec6 h1:JWpsV/8x1fuCYjJmNjT43cVFblLTpO/ISDnePukiTNw=
|
||||
github.com/anchore/stereoscope v0.0.0-20210105000809-428eda0b2ec6/go.mod h1:BMdPL0QEIYfpjQ3M7sHYZvuh6+vcomqF3TMHL8gr6Vw=
|
||||
github.com/anchore/stereoscope v0.0.0-20210105001222-7beea73cb7e5 h1:NGRfS6BZKElgiMbqdoH9iQn+6oxT7CJdZYrqgwvGkWY=
|
||||
github.com/anchore/stereoscope v0.0.0-20210105001222-7beea73cb7e5/go.mod h1:BMdPL0QEIYfpjQ3M7sHYZvuh6+vcomqF3TMHL8gr6Vw=
|
||||
github.com/anchore/stereoscope v0.0.0-20210201165248-e94c52b4052d h1:2hv5NOZ0fD8tPk1UdGiW9PHxmjBmBLL+sFlhLXjjKgo=
|
||||
github.com/anchore/stereoscope v0.0.0-20210201165248-e94c52b4052d/go.mod h1:lhSEYyGLXTXMIFHAz7Ls/MNQ5EjYd5ziLxovKZp1xOs=
|
||||
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
|
||||
github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
@@ -168,8 +172,8 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm
|
||||
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4=
|
||||
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
|
||||
github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
|
||||
github.com/bmatcuk/doublestar v1.3.3 h1:pVP1d49CcQQaNOl+PI6sPybIrIOD/6sux31PFdmhTH0=
|
||||
github.com/bmatcuk/doublestar v1.3.3/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
|
||||
github.com/bmatcuk/doublestar/v2 v2.0.4 h1:6I6oUiT/sU27eE2OFcWqBhL1SwjyvQuOssxT4a1yidI=
|
||||
github.com/bmatcuk/doublestar/v2 v2.0.4/go.mod h1:QMmcs3H2AUQICWhfzLXz+IYln8lRQmTZRptLie8RgRw=
|
||||
github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U=
|
||||
github.com/bombsimon/wsl/v2 v2.2.0/go.mod h1:Azh8c3XGEJl9LyX0/sFC+CKMc7Ssgua0g+6abzXN4Pg=
|
||||
github.com/bombsimon/wsl/v3 v3.0.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
|
||||
@@ -847,6 +851,7 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
|
||||
github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
|
||||
github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
|
||||
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
|
||||
github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g=
|
||||
github.com/vdemeester/k8s-pkg-credentialprovider v1.17.4/go.mod h1:inCTmtUdr5KJbreVojo06krnTgaeAz/Z7lynpPk/Q2c=
|
||||
github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
|
||||
github.com/wagoodman/go-partybus v0.0.0-20200526224238-eb215533f07d h1:KOxOL6qpmqwoPloNwi+CEgc1ayjHNOFNrvoOmeDOjDg=
|
||||
|
||||
@@ -2,7 +2,11 @@ package anchore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/anchore/client-go/pkg/external"
|
||||
"github.com/anchore/syft/internal"
|
||||
@@ -10,11 +14,10 @@ import (
|
||||
)
|
||||
|
||||
type Configuration struct {
|
||||
Hostname string
|
||||
BaseURL string
|
||||
Username string
|
||||
Password string
|
||||
UserAgent string
|
||||
Scheme string
|
||||
}
|
||||
|
||||
type Client struct {
|
||||
@@ -29,16 +32,16 @@ func NewClient(cfg Configuration) (*Client, error) {
|
||||
cfg.UserAgent = fmt.Sprintf("%s / %s %s", internal.ApplicationName, versionInfo.Version, versionInfo.Platform)
|
||||
}
|
||||
|
||||
if cfg.Scheme == "" {
|
||||
cfg.Scheme = "https"
|
||||
baseURL, err := prepareBaseURLForClient(cfg.BaseURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create client: %w", err)
|
||||
}
|
||||
|
||||
return &Client{
|
||||
config: cfg,
|
||||
client: external.NewAPIClient(&external.Configuration{
|
||||
Host: cfg.Hostname,
|
||||
BasePath: baseURL,
|
||||
UserAgent: cfg.UserAgent,
|
||||
Scheme: cfg.Scheme,
|
||||
}),
|
||||
}, nil
|
||||
}
|
||||
@@ -56,3 +59,57 @@ func (c *Client) newRequestContext(parentContext context.Context) context.Contex
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
var ErrInvalidBaseURLInput = errors.New("invalid base URL input")
|
||||
|
||||
func prepareBaseURLForClient(baseURL string) (string, error) {
|
||||
if err := checkBaseURLInput(baseURL); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
scheme, urlWithoutScheme := splitSchemeFromURL(baseURL)
|
||||
|
||||
if scheme == "" {
|
||||
scheme = "http"
|
||||
}
|
||||
|
||||
urlWithoutScheme = path.Clean(urlWithoutScheme)
|
||||
|
||||
const requiredSuffix = "v1"
|
||||
if path.Base(urlWithoutScheme) != requiredSuffix {
|
||||
urlWithoutScheme = path.Join(urlWithoutScheme, requiredSuffix)
|
||||
}
|
||||
|
||||
preparedBaseURL := scheme + "://" + urlWithoutScheme
|
||||
return preparedBaseURL, nil
|
||||
}
|
||||
|
||||
func checkBaseURLInput(url string) error {
|
||||
if url == "" {
|
||||
return ErrInvalidBaseURLInput
|
||||
}
|
||||
|
||||
firstCharacter := rune(url[0])
|
||||
if !(unicode.IsLetter(firstCharacter)) {
|
||||
return ErrInvalidBaseURLInput
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func splitSchemeFromURL(url string) (scheme, urlWithoutScheme string) {
|
||||
if hasScheme(url) {
|
||||
urlParts := strings.SplitN(url, "://", 2)
|
||||
scheme = urlParts[0]
|
||||
urlWithoutScheme = urlParts[1]
|
||||
return
|
||||
}
|
||||
|
||||
return "", url
|
||||
}
|
||||
|
||||
func hasScheme(url string) bool {
|
||||
parts := strings.Split(url, "://")
|
||||
|
||||
return len(parts) > 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
package anchore
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestHasScheme(t *testing.T) {
|
||||
cases := []struct {
|
||||
url string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
url: "http://localhost",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
url: "https://anchore.com:8443",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
url: "google.com",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
url: "",
|
||||
expected: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range cases {
|
||||
t.Run(testCase.url, func(t *testing.T) {
|
||||
result := hasScheme(testCase.url)
|
||||
|
||||
if testCase.expected != result {
|
||||
t.Errorf("expected %t but got %t", testCase.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareBaseURLForClient(t *testing.T) {
|
||||
cases := []struct {
|
||||
inputURL string
|
||||
expectedURL string
|
||||
expectedErr error
|
||||
}{
|
||||
{
|
||||
inputURL: "",
|
||||
expectedURL: "",
|
||||
expectedErr: ErrInvalidBaseURLInput,
|
||||
},
|
||||
{
|
||||
inputURL: "localhost",
|
||||
expectedURL: "http://localhost/v1",
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
inputURL: "https://localhost",
|
||||
expectedURL: "https://localhost/v1",
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
inputURL: "https://localhost/",
|
||||
expectedURL: "https://localhost/v1",
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
inputURL: "https://localhost/v1/",
|
||||
expectedURL: "https://localhost/v1",
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
inputURL: "https://localhost/v1//",
|
||||
expectedURL: "https://localhost/v1",
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
inputURL: "http://something.com/platform/v1/services/anchore",
|
||||
expectedURL: "http://something.com/platform/v1/services/anchore/v1",
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
inputURL: "my-host:8228",
|
||||
expectedURL: "http://my-host:8228/v1",
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
inputURL: "v1/v1",
|
||||
expectedURL: "http://v1/v1",
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
inputURL: "/v1",
|
||||
expectedURL: "",
|
||||
expectedErr: ErrInvalidBaseURLInput,
|
||||
},
|
||||
{
|
||||
inputURL: "/imports/images",
|
||||
expectedURL: "",
|
||||
expectedErr: ErrInvalidBaseURLInput,
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range cases {
|
||||
t.Run(testCase.inputURL, func(t *testing.T) {
|
||||
resultURL, err := prepareBaseURLForClient(testCase.inputURL)
|
||||
if err != testCase.expectedErr {
|
||||
t.Errorf("expected err to be '%v' but got '%v'", testCase.expectedErr, err)
|
||||
}
|
||||
|
||||
if resultURL != testCase.expectedURL {
|
||||
t.Errorf("expected URL to be '%v' but got '%v'", testCase.expectedURL, resultURL)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckBaseURLInput(t *testing.T) {
|
||||
cases := []struct {
|
||||
input string
|
||||
expected error
|
||||
}{
|
||||
{
|
||||
input: "",
|
||||
expected: ErrInvalidBaseURLInput,
|
||||
},
|
||||
{
|
||||
input: "x",
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
input: "localhost:8000",
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
input: ":80",
|
||||
expected: ErrInvalidBaseURLInput,
|
||||
},
|
||||
{
|
||||
input: "/v1",
|
||||
expected: ErrInvalidBaseURLInput,
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range cases {
|
||||
t.Run(testCase.input, func(t *testing.T) {
|
||||
resultErr := checkBaseURLInput(testCase.input)
|
||||
|
||||
if testCase.expected != resultErr {
|
||||
t.Errorf("expected err to be '%v' but got '%v'", testCase.expected, resultErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitSchemeFromURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
input string
|
||||
expectedScheme string
|
||||
expectedURLWithoutScheme string
|
||||
}{
|
||||
{
|
||||
input: "",
|
||||
expectedScheme: "",
|
||||
expectedURLWithoutScheme: "",
|
||||
},
|
||||
{
|
||||
input: "localhost",
|
||||
expectedScheme: "",
|
||||
expectedURLWithoutScheme: "localhost",
|
||||
},
|
||||
{
|
||||
input: "https://anchore.com/path",
|
||||
expectedScheme: "https",
|
||||
expectedURLWithoutScheme: "anchore.com/path",
|
||||
},
|
||||
{
|
||||
input: "tcp://host:1234",
|
||||
expectedScheme: "tcp",
|
||||
expectedURLWithoutScheme: "host:1234",
|
||||
},
|
||||
{
|
||||
input: "/hello",
|
||||
expectedScheme: "",
|
||||
expectedURLWithoutScheme: "/hello",
|
||||
},
|
||||
{
|
||||
input: "://host",
|
||||
expectedScheme: "",
|
||||
expectedURLWithoutScheme: "host",
|
||||
},
|
||||
{
|
||||
input: "http//localhost",
|
||||
expectedScheme: "",
|
||||
expectedURLWithoutScheme: "http//localhost",
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range cases {
|
||||
t.Run(testCase.input, func(t *testing.T) {
|
||||
resultScheme, resultURLWithoutScheme := splitSchemeFromURL(testCase.input)
|
||||
|
||||
if testCase.expectedScheme != resultScheme {
|
||||
t.Errorf("expected scheme to be '%s' but got '%s'", testCase.expectedScheme, resultScheme)
|
||||
}
|
||||
|
||||
if testCase.expectedURLWithoutScheme != resultURLWithoutScheme {
|
||||
t.Errorf("expected urlWithoutScheme to be '%s' but got '%s'", testCase.expectedURLWithoutScheme, resultURLWithoutScheme)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -51,7 +51,7 @@ func importProgress(source string) (*progress.Stage, *progress.Manual) {
|
||||
|
||||
// nolint:funlen
|
||||
func (c *Client) Import(ctx context.Context, cfg ImportConfig) error {
|
||||
stage, prog := importProgress(c.config.Hostname)
|
||||
stage, prog := importProgress(c.config.BaseURL)
|
||||
|
||||
ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Second*30)
|
||||
defer cancel()
|
||||
|
||||
@@ -61,7 +61,7 @@ func TestPackageSbomToModel(t *testing.T) {
|
||||
FoundBy: "foundBy",
|
||||
Locations: []source.Location{
|
||||
{
|
||||
Path: "path",
|
||||
RealPath: "path",
|
||||
FileSystemID: "layerID",
|
||||
},
|
||||
},
|
||||
@@ -139,7 +139,7 @@ func TestPackageSbomImport(t *testing.T) {
|
||||
FoundBy: "foundBy",
|
||||
Locations: []source.Location{
|
||||
{
|
||||
Path: "path",
|
||||
RealPath: "path",
|
||||
FileSystemID: "layerID",
|
||||
},
|
||||
},
|
||||
|
||||
@@ -23,7 +23,9 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync"
|
||||
"syscall"
|
||||
|
||||
"github.com/anchore/syft/internal/logger"
|
||||
|
||||
@@ -102,6 +104,8 @@ func OutputToEphemeralTUI(workerErrs <-chan error, subscription *partybus.Subscr
|
||||
ctx := context.Background()
|
||||
syftUIHandler := ui.NewHandler()
|
||||
|
||||
signals := interruptingSignals()
|
||||
|
||||
eventLoop:
|
||||
for {
|
||||
select {
|
||||
@@ -154,8 +158,23 @@ eventLoop:
|
||||
log.Errorf("cancelled (%+v)", err)
|
||||
}
|
||||
break eventLoop
|
||||
case <-signals:
|
||||
break eventLoop
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func interruptingSignals() chan os.Signal {
|
||||
c := make(chan os.Signal, 1) // Note: A buffered channel is recommended for this; see https://golang.org/pkg/os/signal/#Notify
|
||||
|
||||
interruptions := []os.Signal{
|
||||
syscall.SIGINT,
|
||||
syscall.SIGTERM,
|
||||
}
|
||||
|
||||
signal.Notify(c, interruptions...)
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ Package common provides generic utilities used by multiple catalogers.
|
||||
package common
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"io"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
@@ -89,7 +89,7 @@ func (c *GenericCataloger) selectFiles(resolver source.FileResolver) []source.Lo
|
||||
}
|
||||
|
||||
// catalog takes a set of file contents and uses any configured parser functions to resolve and return discovered packages
|
||||
func (c *GenericCataloger) catalog(contents map[source.Location]string) ([]pkg.Package, error) {
|
||||
func (c *GenericCataloger) catalog(contents map[source.Location]io.ReadCloser) ([]pkg.Package, error) {
|
||||
defer c.clear()
|
||||
|
||||
packages := make([]pkg.Package, 0)
|
||||
@@ -101,7 +101,7 @@ func (c *GenericCataloger) catalog(contents map[source.Location]string) ([]pkg.P
|
||||
continue
|
||||
}
|
||||
|
||||
entries, err := parser(location.Path, strings.NewReader(content))
|
||||
entries, err := parser(location.RealPath, content)
|
||||
if err != nil {
|
||||
// TODO: should we fail? or only log?
|
||||
log.Warnf("cataloger '%s' failed to parse entries (location=%+v): %+v", c.upstreamCataloger, location, err)
|
||||
|
||||
@@ -4,28 +4,32 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/anchore/syft/syft/source"
|
||||
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
type testResolverMock struct {
|
||||
contents map[source.Location]string
|
||||
contents map[source.Location]io.ReadCloser
|
||||
}
|
||||
|
||||
func newTestResolver() *testResolverMock {
|
||||
return &testResolverMock{
|
||||
contents: make(map[source.Location]string),
|
||||
contents: make(map[source.Location]io.ReadCloser),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *testResolverMock) FileContentsByLocation(_ source.Location) (string, error) {
|
||||
return "", fmt.Errorf("not implemented")
|
||||
func (r testResolverMock) HasPath(path string) bool {
|
||||
panic("not implemented")
|
||||
}
|
||||
|
||||
func (r *testResolverMock) MultipleFileContentsByLocation([]source.Location) (map[source.Location]string, error) {
|
||||
func (r *testResolverMock) FileContentsByLocation(_ source.Location) (io.ReadCloser, error) {
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func (r *testResolverMock) MultipleFileContentsByLocation([]source.Location) (map[source.Location]io.ReadCloser, error) {
|
||||
return r.contents, nil
|
||||
}
|
||||
|
||||
@@ -34,7 +38,7 @@ func (r *testResolverMock) FilesByPath(paths ...string) ([]source.Location, erro
|
||||
|
||||
for idx, p := range paths {
|
||||
results[idx] = source.NewLocation(p)
|
||||
r.contents[results[idx]] = fmt.Sprintf("%s file contents!", p)
|
||||
r.contents[results[idx]] = ioutil.NopCloser(strings.NewReader(fmt.Sprintf("%s file contents!", p)))
|
||||
}
|
||||
|
||||
return results, nil
|
||||
@@ -43,7 +47,7 @@ func (r *testResolverMock) FilesByPath(paths ...string) ([]source.Location, erro
|
||||
func (r *testResolverMock) FilesByGlob(_ ...string) ([]source.Location, error) {
|
||||
path := "/a-path.txt"
|
||||
location := source.NewLocation(path)
|
||||
r.contents[location] = fmt.Sprintf("%s file contents!", path)
|
||||
r.contents[location] = ioutil.NopCloser(strings.NewReader(fmt.Sprintf("%s file contents!", path)))
|
||||
return []source.Location{location}, nil
|
||||
}
|
||||
|
||||
@@ -97,7 +101,7 @@ func TestGenericCataloger(t *testing.T) {
|
||||
|
||||
for _, p := range actualPkgs {
|
||||
ref := p.Locations[0]
|
||||
exP, ok := expectedPkgs[ref.Path]
|
||||
exP, ok := expectedPkgs[ref.RealPath]
|
||||
if !ok {
|
||||
t.Errorf("missing expected pkg: ref=%+v", ref)
|
||||
continue
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"io"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
@@ -47,9 +46,9 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pkgs, err = parseDpkgStatus(strings.NewReader(dbContents))
|
||||
pkgs, err = parseDpkgStatus(dbContents)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.Path, err)
|
||||
return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.RealPath, err)
|
||||
}
|
||||
|
||||
md5ContentsByName, md5RefsByName, err := fetchMd5Contents(resolver, dbLocation, pkgs)
|
||||
@@ -57,7 +56,7 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
|
||||
return nil, fmt.Errorf("unable to find dpkg md5 contents: %w", err)
|
||||
}
|
||||
|
||||
copyrightContentsByName, copyrightRefsByName, err := fetchCopyrightContents(resolver, dbLocation, pkgs)
|
||||
copyrightContentsByName, copyrightLocationByName, err := fetchCopyrightContents(resolver, dbLocation, pkgs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to find dpkg copyright contents: %w", err)
|
||||
}
|
||||
@@ -91,7 +90,7 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
|
||||
p.Licenses = parseLicensesFromCopyright(copyrightReader)
|
||||
|
||||
// keep a record of the file where this was discovered
|
||||
if ref, ok := copyrightRefsByName[p.Name]; ok {
|
||||
if ref, ok := copyrightLocationByName[p.Name]; ok {
|
||||
p.Locations = append(p.Locations, ref)
|
||||
}
|
||||
}
|
||||
@@ -105,20 +104,17 @@ func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs
|
||||
|
||||
var md5FileMatches []source.Location
|
||||
var nameByRef = make(map[source.Location]string)
|
||||
parentPath := filepath.Dir(dbLocation.Path)
|
||||
parentPath := filepath.Dir(dbLocation.RealPath)
|
||||
|
||||
for _, p := range pkgs {
|
||||
// look for /var/lib/dpkg/info/NAME:ARCH.md5sums
|
||||
name := md5Key(p)
|
||||
md5sumPath := path.Join(parentPath, "info", name+md5sumsExt)
|
||||
md5SumLocation := resolver.RelativeFileByPath(dbLocation, md5sumPath)
|
||||
md5SumLocation := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "info", name+md5sumsExt))
|
||||
|
||||
if md5SumLocation == nil {
|
||||
// the most specific key did not work, fallback to just the name
|
||||
// look for /var/lib/dpkg/info/NAME.md5sums
|
||||
name := p.Name
|
||||
md5sumPath := path.Join(parentPath, "info", name+md5sumsExt)
|
||||
md5SumLocation = resolver.RelativeFileByPath(dbLocation, md5sumPath)
|
||||
md5SumLocation = resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "info", p.Name+md5sumsExt))
|
||||
}
|
||||
// we should have at least one reference
|
||||
if md5SumLocation != nil {
|
||||
@@ -135,14 +131,14 @@ func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs
|
||||
|
||||
// organize content results and refs by a combination of name and architecture
|
||||
var contentsByName = make(map[string]io.Reader)
|
||||
var refsByName = make(map[string]source.Location)
|
||||
var locationByName = make(map[string]source.Location)
|
||||
for location, contents := range md5ContentsByLocation {
|
||||
name := nameByRef[location]
|
||||
contentsByName[name] = strings.NewReader(contents)
|
||||
refsByName[name] = location
|
||||
contentsByName[name] = contents
|
||||
locationByName[name] = location
|
||||
}
|
||||
|
||||
return contentsByName, refsByName, nil
|
||||
return contentsByName, locationByName, nil
|
||||
}
|
||||
|
||||
func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) {
|
||||
@@ -174,7 +170,7 @@ func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location
|
||||
var refsByName = make(map[string]source.Location)
|
||||
for location, contents := range copyrightContentsByLocation {
|
||||
name := nameByLocation[location]
|
||||
contentsByName[name] = strings.NewReader(contents)
|
||||
contentsByName[name] = contents
|
||||
refsByName[name] = location
|
||||
}
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ func TestDpkgCataloger(t *testing.T) {
|
||||
img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-dpkg")
|
||||
defer cleanup()
|
||||
|
||||
s, err := source.NewFromImage(img, source.AllLayersScope, "")
|
||||
s, err := source.NewFromImage(img, source.SquashedScope, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -79,7 +79,7 @@ func TestDpkgCataloger(t *testing.T) {
|
||||
// we will test the sources separately
|
||||
var sourcesList = make([]string, len(a.Locations))
|
||||
for i, s := range a.Locations {
|
||||
sourcesList[i] = s.Path
|
||||
sourcesList[i] = s.RealPath
|
||||
}
|
||||
a.Locations = nil
|
||||
|
||||
|
||||
@@ -178,36 +178,40 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([
|
||||
return nil, fmt.Errorf("failed to parse pom.properties (%s): %w", j.virtualPath, err)
|
||||
}
|
||||
|
||||
if propsObj != nil {
|
||||
if propsObj.Version != "" && propsObj.ArtifactID != "" {
|
||||
// TODO: if there is no parentPkg (no java manifest) one of these poms could be the parent. We should discover the right parent and attach the correct info accordingly to each discovered package
|
||||
if propsObj == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// keep the artifact name within the virtual path if this package does not match the parent package
|
||||
vPathSuffix := ""
|
||||
if !strings.HasPrefix(propsObj.ArtifactID, parentPkg.Name) {
|
||||
vPathSuffix += ":" + propsObj.ArtifactID
|
||||
}
|
||||
virtualPath := j.virtualPath + vPathSuffix
|
||||
if propsObj.Version != "" && propsObj.ArtifactID != "" {
|
||||
// TODO: if there is no parentPkg (no java manifest) one of these poms could be the parent. We should discover the right parent and attach the correct info accordingly to each discovered package
|
||||
|
||||
// discovered props = new package
|
||||
p := pkg.Package{
|
||||
Name: propsObj.ArtifactID,
|
||||
Version: propsObj.Version,
|
||||
Language: pkg.Java,
|
||||
Type: pkg.JavaPkg,
|
||||
MetadataType: pkg.JavaMetadataType,
|
||||
Metadata: pkg.JavaMetadata{
|
||||
VirtualPath: virtualPath,
|
||||
PomProperties: propsObj,
|
||||
Parent: parentPkg,
|
||||
},
|
||||
}
|
||||
// keep the artifact name within the virtual path if this package does not match the parent package
|
||||
vPathSuffix := ""
|
||||
if parentPkg != nil && !strings.HasPrefix(propsObj.ArtifactID, parentPkg.Name) {
|
||||
vPathSuffix += ":" + propsObj.ArtifactID
|
||||
}
|
||||
virtualPath := j.virtualPath + vPathSuffix
|
||||
|
||||
pkgKey := uniquePkgKey(&p)
|
||||
// discovered props = new package
|
||||
p := pkg.Package{
|
||||
Name: propsObj.ArtifactID,
|
||||
Version: propsObj.Version,
|
||||
Language: pkg.Java,
|
||||
Type: pkg.JavaPkg,
|
||||
MetadataType: pkg.JavaMetadataType,
|
||||
Metadata: pkg.JavaMetadata{
|
||||
VirtualPath: virtualPath,
|
||||
PomProperties: propsObj,
|
||||
Parent: parentPkg,
|
||||
},
|
||||
}
|
||||
|
||||
// the name/version pair matches...
|
||||
matchesParentPkg := pkgKey == parentKey
|
||||
pkgKey := uniquePkgKey(&p)
|
||||
|
||||
// the name/version pair matches...
|
||||
matchesParentPkg := pkgKey == parentKey
|
||||
|
||||
if parentPkg != nil {
|
||||
// the virtual path matches...
|
||||
matchesParentPkg = matchesParentPkg || parentPkg.Metadata.(pkg.JavaMetadata).VirtualPath == virtualPath
|
||||
|
||||
@@ -231,11 +235,13 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([
|
||||
parentMetadata.PomProperties = propsObj
|
||||
parentPkg.Metadata = parentMetadata
|
||||
}
|
||||
} else if !j.discoveredPkgs.Contains(pkgKey) {
|
||||
// only keep packages we haven't seen yet (and are not related to the parent package)
|
||||
pkgs = append(pkgs, p)
|
||||
}
|
||||
}
|
||||
|
||||
if !matchesParentPkg && !j.discoveredPkgs.Contains(pkgKey) {
|
||||
// only keep packages we haven't seen yet (and are not related to the parent package)
|
||||
pkgs = append(pkgs, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
return pkgs, nil
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"io"
|
||||
"regexp"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
|
||||
"github.com/mitchellh/mapstructure"
|
||||
@@ -172,6 +174,11 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
return nil, fmt.Errorf("failed to parse package.json file: %w", err)
|
||||
}
|
||||
|
||||
if !p.hasNameAndVersionValues() {
|
||||
log.Debug("encountered package.json file without a name and/or version field, ignoring this file")
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
licenses, err := licensesFromJSON(p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse package.json file: %w", err)
|
||||
@@ -195,3 +202,7 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
|
||||
return packages, nil
|
||||
}
|
||||
|
||||
func (p PackageJSON) hasNameAndVersionValues() bool {
|
||||
return p.Name != "" && p.Version != ""
|
||||
}
|
||||
|
||||
@@ -142,3 +142,20 @@ func TestParsePackageJSON(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePackageJSON_Partial(t *testing.T) { // see https://github.com/anchore/syft/issues/311
|
||||
const fixtureFile = "test-fixtures/pkg-json/package-partial.json"
|
||||
fixture, err := os.Open(fixtureFile)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parsePackageJSON("", fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse package-lock.json: %+v", err)
|
||||
}
|
||||
|
||||
if actualCount := len(actual); actualCount != 0 {
|
||||
t.Errorf("no packages should've been returned (but got %d packages)", actualCount)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"sideEffects": false,
|
||||
"module": "../../esm/fp/isSaturday/index.js",
|
||||
"typings": "../../typings.d.ts"
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package python
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
|
||||
@@ -39,7 +38,7 @@ func (c *PackageCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, err
|
||||
for _, entry := range entries {
|
||||
p, err := c.catalogEggOrWheel(entry)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to catalog python package=%+v: %w", entry.Metadata.Location.Path, err)
|
||||
return nil, fmt.Errorf("unable to catalog python package=%+v: %w", entry.Metadata.Location.RealPath, err)
|
||||
}
|
||||
if p != nil {
|
||||
packages = append(packages, *p)
|
||||
@@ -91,6 +90,12 @@ func (c *PackageCataloger) catalogEggOrWheel(entry *packageEntry) (*pkg.Package,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// This can happen for Python 2.7 where it is reported from an egg-info, but Python is
|
||||
// the actual runtime, it isn't a "package". The special-casing here allows to skip it
|
||||
if metadata.Name == "Python" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var licenses []string
|
||||
if metadata.License != "" {
|
||||
licenses = []string{metadata.License}
|
||||
@@ -113,7 +118,7 @@ func (c *PackageCataloger) catalogEggOrWheel(entry *packageEntry) (*pkg.Package,
|
||||
func (c *PackageCataloger) assembleEggOrWheelMetadata(entry *packageEntry) (*pkg.PythonPackageMetadata, []source.Location, error) {
|
||||
var sources = []source.Location{entry.Metadata.Location}
|
||||
|
||||
metadata, err := parseWheelOrEggMetadata(entry.Metadata.Location.Path, strings.NewReader(entry.Metadata.Contents))
|
||||
metadata, err := parseWheelOrEggMetadata(entry.Metadata.Location.RealPath, entry.Metadata.Contents)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -147,7 +152,7 @@ func (c *PackageCataloger) processRecordFiles(entry *source.FileData) (files []p
|
||||
sources = append(sources, entry.Location)
|
||||
|
||||
// parse the record contents
|
||||
records, err := parseWheelOrEggRecord(strings.NewReader(entry.Contents))
|
||||
records, err := parseWheelOrEggRecord(entry.Contents)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -165,7 +170,7 @@ func (c *PackageCataloger) processTopLevelPackages(entry *source.FileData) (pkgs
|
||||
|
||||
sources = append(sources, entry.Location)
|
||||
|
||||
scanner := bufio.NewScanner(strings.NewReader(entry.Contents))
|
||||
scanner := bufio.NewScanner(entry.Contents)
|
||||
for scanner.Scan() {
|
||||
pkgs = append(pkgs, scanner.Text())
|
||||
}
|
||||
|
||||
@@ -1,154 +1,27 @@
|
||||
package python
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/anchore/syft/internal/file"
|
||||
|
||||
"github.com/anchore/syft/syft/source"
|
||||
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
"github.com/go-test/deep"
|
||||
)
|
||||
|
||||
// TODO: make this generic (based on maps of source.FileData) and make a generic mock to move to the source pkg
|
||||
type pythonTestResolverMock struct {
|
||||
metadataReader io.Reader
|
||||
recordReader io.Reader
|
||||
topLevelReader io.Reader
|
||||
metadataRef *source.Location
|
||||
recordRef *source.Location
|
||||
topLevelRef *source.Location
|
||||
contents map[source.Location]string
|
||||
}
|
||||
|
||||
func newTestResolver(metaPath, recordPath, topPath string) *pythonTestResolverMock {
|
||||
metadataReader, err := os.Open(metaPath)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("failed to open metadata: %+v", err))
|
||||
}
|
||||
|
||||
var recordReader io.Reader
|
||||
if recordPath != "" {
|
||||
recordReader, err = os.Open(recordPath)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("failed to open record: %+v", err))
|
||||
}
|
||||
}
|
||||
|
||||
var topLevelReader io.Reader
|
||||
if topPath != "" {
|
||||
topLevelReader, err = os.Open(topPath)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("failed to open top level: %+v", err))
|
||||
}
|
||||
}
|
||||
|
||||
var recordRef *source.Location
|
||||
if recordReader != nil {
|
||||
ref := source.NewLocation("test-fixtures/dist-info/RECORD")
|
||||
recordRef = &ref
|
||||
}
|
||||
var topLevelRef *source.Location
|
||||
if topLevelReader != nil {
|
||||
ref := source.NewLocation("test-fixtures/dist-info/top_level.txt")
|
||||
topLevelRef = &ref
|
||||
}
|
||||
metadataRef := source.NewLocation("test-fixtures/dist-info/METADATA")
|
||||
return &pythonTestResolverMock{
|
||||
recordReader: recordReader,
|
||||
metadataReader: metadataReader,
|
||||
topLevelReader: topLevelReader,
|
||||
metadataRef: &metadataRef,
|
||||
recordRef: recordRef,
|
||||
topLevelRef: topLevelRef,
|
||||
contents: make(map[source.Location]string),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *pythonTestResolverMock) FileContentsByLocation(location source.Location) (string, error) {
|
||||
switch {
|
||||
case r.topLevelRef != nil && location.Path == r.topLevelRef.Path:
|
||||
b, err := ioutil.ReadAll(r.topLevelReader)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(b), nil
|
||||
case location.Path == r.metadataRef.Path:
|
||||
b, err := ioutil.ReadAll(r.metadataReader)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(b), nil
|
||||
case location.Path == r.recordRef.Path:
|
||||
b, err := ioutil.ReadAll(r.recordReader)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
return "", fmt.Errorf("invalid value given")
|
||||
}
|
||||
|
||||
func (r *pythonTestResolverMock) MultipleFileContentsByLocation(locations []source.Location) (map[source.Location]string, error) {
|
||||
var results = make(map[source.Location]string)
|
||||
var err error
|
||||
for _, l := range locations {
|
||||
results[l], err = r.FileContentsByLocation(l)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (r *pythonTestResolverMock) FilesByPath(_ ...string) ([]source.Location, error) {
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func (r *pythonTestResolverMock) FilesByGlob(patterns ...string) ([]source.Location, error) {
|
||||
var results []source.Location
|
||||
for _, pattern := range patterns {
|
||||
for _, l := range []*source.Location{r.topLevelRef, r.metadataRef, r.recordRef} {
|
||||
if l == nil {
|
||||
continue
|
||||
}
|
||||
if file.GlobMatch(pattern, l.Path) {
|
||||
results = append(results, *l)
|
||||
}
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
func (r *pythonTestResolverMock) RelativeFileByPath(_ source.Location, path string) *source.Location {
|
||||
switch {
|
||||
case strings.Contains(path, "RECORD"):
|
||||
return r.recordRef
|
||||
case strings.Contains(path, "top_level.txt"):
|
||||
return r.topLevelRef
|
||||
default:
|
||||
panic(fmt.Errorf("invalid RelativeFileByPath value given: %q", path))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPythonPackageWheelCataloger(t *testing.T) {
|
||||
tests := []struct {
|
||||
MetadataFixture string
|
||||
RecordFixture string
|
||||
TopLevelFixture string
|
||||
ExpectedPackage pkg.Package
|
||||
name string
|
||||
fixtures []string
|
||||
expectedPackage pkg.Package
|
||||
}{
|
||||
{
|
||||
MetadataFixture: "test-fixtures/egg-info/PKG-INFO",
|
||||
RecordFixture: "test-fixtures/egg-info/RECORD",
|
||||
TopLevelFixture: "test-fixtures/egg-info/top_level.txt",
|
||||
ExpectedPackage: pkg.Package{
|
||||
name: "egg-info directory",
|
||||
fixtures: []string{
|
||||
"test-fixtures/egg-info/PKG-INFO",
|
||||
"test-fixtures/egg-info/RECORD",
|
||||
"test-fixtures/egg-info/top_level.txt",
|
||||
},
|
||||
expectedPackage: pkg.Package{
|
||||
Name: "requests",
|
||||
Version: "2.22.0",
|
||||
Type: pkg.PythonPkg,
|
||||
@@ -177,10 +50,13 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
|
||||
},
|
||||
},
|
||||
{
|
||||
MetadataFixture: "test-fixtures/dist-info/METADATA",
|
||||
RecordFixture: "test-fixtures/dist-info/RECORD",
|
||||
TopLevelFixture: "test-fixtures/dist-info/top_level.txt",
|
||||
ExpectedPackage: pkg.Package{
|
||||
name: "dist-info directory",
|
||||
fixtures: []string{
|
||||
"test-fixtures/dist-info/METADATA",
|
||||
"test-fixtures/dist-info/RECORD",
|
||||
"test-fixtures/dist-info/top_level.txt",
|
||||
},
|
||||
expectedPackage: pkg.Package{
|
||||
Name: "Pygments",
|
||||
Version: "2.6.1",
|
||||
Type: pkg.PythonPkg,
|
||||
@@ -210,8 +86,9 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
|
||||
{
|
||||
// in cases where the metadata file is available and the record is not we should still record there is a package
|
||||
// additionally empty top_level.txt files should not result in an error
|
||||
MetadataFixture: "test-fixtures/partial.dist-info/METADATA",
|
||||
ExpectedPackage: pkg.Package{
|
||||
name: "partial dist-info directory",
|
||||
fixtures: []string{"test-fixtures/partial.dist-info/METADATA"},
|
||||
expectedPackage: pkg.Package{
|
||||
Name: "Pygments",
|
||||
Version: "2.6.1",
|
||||
Type: pkg.PythonPkg,
|
||||
@@ -231,8 +108,9 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
|
||||
},
|
||||
},
|
||||
{
|
||||
MetadataFixture: "test-fixtures/test.egg-info",
|
||||
ExpectedPackage: pkg.Package{
|
||||
name: "egg-info regular file",
|
||||
fixtures: []string{"test-fixtures/test.egg-info"},
|
||||
expectedPackage: pkg.Package{
|
||||
Name: "requests",
|
||||
Version: "2.22.0",
|
||||
Type: pkg.PythonPkg,
|
||||
@@ -254,19 +132,15 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.MetadataFixture, func(t *testing.T) {
|
||||
resolver := newTestResolver(test.MetadataFixture, test.RecordFixture, test.TopLevelFixture)
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
resolver := source.NewMockResolverForPaths(test.fixtures...)
|
||||
|
||||
// note that the source is the record ref created by the resolver mock... attach the expected values
|
||||
test.ExpectedPackage.Locations = []source.Location{*resolver.metadataRef}
|
||||
if resolver.recordRef != nil {
|
||||
test.ExpectedPackage.Locations = append(test.ExpectedPackage.Locations, *resolver.recordRef)
|
||||
locations, err := resolver.FilesByPath(test.fixtures...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if resolver.topLevelRef != nil {
|
||||
test.ExpectedPackage.Locations = append(test.ExpectedPackage.Locations, *resolver.topLevelRef)
|
||||
}
|
||||
// end patching expected values with runtime data...
|
||||
test.expectedPackage.Locations = locations
|
||||
|
||||
actual, err := NewPythonPackageCataloger().Catalog(resolver)
|
||||
if err != nil {
|
||||
@@ -274,13 +148,37 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
|
||||
}
|
||||
|
||||
if len(actual) != 1 {
|
||||
t.Fatalf("unexpected length: %d", len(actual))
|
||||
t.Fatalf("unexpected number of packages: %d", len(actual))
|
||||
}
|
||||
|
||||
for _, d := range deep.Equal(actual[0], test.ExpectedPackage) {
|
||||
for _, d := range deep.Equal(actual[0], test.expectedPackage) {
|
||||
t.Errorf("diff: %+v", d)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIgnorePackage(t *testing.T) {
|
||||
tests := []struct {
|
||||
MetadataFixture string
|
||||
}{
|
||||
{
|
||||
MetadataFixture: "test-fixtures/Python-2.7.egg-info",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.MetadataFixture, func(t *testing.T) {
|
||||
resolver := source.NewMockResolverForPaths(test.MetadataFixture)
|
||||
|
||||
actual, err := NewPythonPackageCataloger().Catalog(resolver)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to catalog python package: %+v", err)
|
||||
}
|
||||
|
||||
if len(actual) != 0 {
|
||||
t.Fatalf("Expected 0 packages but found: %d", len(actual))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,11 +19,11 @@ func newPackageEntry(resolver source.FileResolver, metadataLocation source.Locat
|
||||
// to reconcile the RECORD path to the same layer (or a lower layer). The same is true with the top_level.txt file.
|
||||
|
||||
// lets find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure)
|
||||
recordPath := filepath.Join(filepath.Dir(metadataLocation.Path), "RECORD")
|
||||
recordPath := filepath.Join(filepath.Dir(metadataLocation.RealPath), "RECORD")
|
||||
recordLocation := resolver.RelativeFileByPath(metadataLocation, recordPath)
|
||||
|
||||
// a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages
|
||||
parentDir := filepath.Dir(metadataLocation.Path)
|
||||
parentDir := filepath.Dir(metadataLocation.RealPath)
|
||||
topLevelPath := filepath.Join(parentDir, "top_level.txt")
|
||||
topLevelLocation := resolver.RelativeFileByPath(metadataLocation, topLevelPath)
|
||||
|
||||
|
||||
@@ -7,6 +7,9 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/internal/file"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
|
||||
"github.com/mitchellh/mapstructure"
|
||||
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
@@ -54,7 +57,7 @@ func parseWheelOrEggMetadata(path string, reader io.Reader) (pkg.PythonPackageMe
|
||||
|
||||
fields[key] = val
|
||||
} else {
|
||||
return pkg.PythonPackageMetadata{}, fmt.Errorf("cannot parse field from line: '%s'", line)
|
||||
log.Warnf("cannot parse field from path: %q from line: %q", path, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -70,11 +73,28 @@ func parseWheelOrEggMetadata(path string, reader io.Reader) (pkg.PythonPackageMe
|
||||
|
||||
// add additional metadata not stored in the egg/wheel metadata file
|
||||
|
||||
metadata.SitePackagesRootPath = filepath.Clean(filepath.Join(filepath.Dir(path), ".."))
|
||||
metadata.SitePackagesRootPath = determineSitePackagesRootPath(path)
|
||||
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
// isEggRegularFile determines if the specified path is the regular file variant
|
||||
// of egg metadata (as opposed to a directory that contains more metadata
|
||||
// files).
|
||||
func isEggRegularFile(path string) bool {
|
||||
return file.GlobMatch(eggFileMetadataGlob, path)
|
||||
}
|
||||
|
||||
// determineSitePackagesRootPath returns the path of the site packages root,
|
||||
// given the egg metadata file or directory specified in the path.
|
||||
func determineSitePackagesRootPath(path string) string {
|
||||
if isEggRegularFile(path) {
|
||||
return filepath.Clean(filepath.Dir(path))
|
||||
}
|
||||
|
||||
return filepath.Clean(filepath.Dir(filepath.Dir(path)))
|
||||
}
|
||||
|
||||
// handleFieldBodyContinuation returns the updated value for the specified field after processing the specified line.
|
||||
// If the continuation cannot be processed, it returns an error.
|
||||
func handleFieldBodyContinuation(key, line string, fields map[string]string) (string, error) {
|
||||
|
||||
@@ -56,5 +56,98 @@ func TestParseWheelEggMetadata(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRegularEggFile(t *testing.T) {
|
||||
cases := []struct {
|
||||
path string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
"/usr/lib64/python2.6/site-packages/M2Crypto-0.20.2-py2.6.egg-info",
|
||||
true,
|
||||
},
|
||||
{
|
||||
"/usr/lib64/python2.6/site-packages/M2Crypto-0.20.2-py2.6.egg-info/PKG-INFO",
|
||||
false,
|
||||
},
|
||||
{
|
||||
"/usr/lib64/python2.6/site-packages/M2Crypto-0.20.2-py2.6.dist-info/METADATA",
|
||||
false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.path, func(t *testing.T) {
|
||||
actual := isEggRegularFile(c.path)
|
||||
|
||||
if actual != c.expected {
|
||||
t.Errorf("expected %t but got %t", c.expected, actual)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetermineSitePackagesRootPath(t *testing.T) {
|
||||
cases := []struct {
|
||||
inputPath string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
inputPath: "/usr/lib64/python2.6/site-packages/ethtool-0.6-py2.6.egg-info",
|
||||
expected: "/usr/lib64/python2.6/site-packages",
|
||||
},
|
||||
{
|
||||
inputPath: "/usr/lib/python2.7/dist-packages/configobj-5.0.6.egg-info/top_level.txt",
|
||||
expected: "/usr/lib/python2.7/dist-packages",
|
||||
},
|
||||
{
|
||||
inputPath: "/usr/lib/python2.7/dist-packages/six-1.10.0.egg-info/PKG-INFO",
|
||||
expected: "/usr/lib/python2.7/dist-packages",
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.inputPath, func(t *testing.T) {
|
||||
actual := determineSitePackagesRootPath(c.inputPath)
|
||||
|
||||
if actual != c.expected {
|
||||
t.Errorf("expected %s but got %s", c.expected, actual)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseWheelEggMetadataInvalid(t *testing.T) {
|
||||
tests := []struct {
|
||||
Fixture string
|
||||
ExpectedMetadata pkg.PythonPackageMetadata
|
||||
}{
|
||||
{
|
||||
Fixture: "test-fixtures/egg-info/PKG-INFO-INVALID",
|
||||
ExpectedMetadata: pkg.PythonPackageMetadata{
|
||||
Name: "mxnet",
|
||||
Version: "1.8.0",
|
||||
SitePackagesRootPath: "test-fixtures",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Fixture, func(t *testing.T) {
|
||||
fixture, err := os.Open(test.Fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parseWheelOrEggMetadata(test.Fixture, fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse: %+v", err)
|
||||
}
|
||||
|
||||
for _, d := range deep.Equal(actual, test.ExpectedMetadata) {
|
||||
t.Errorf("diff: %+v", d)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
Metadata-Version: 1.1
|
||||
Name: Python
|
||||
Version: 2.7.13
|
||||
Summary: A high-level object-oriented programming language
|
||||
Home-page: http://www.python.org/2.7
|
||||
Author: Guido van Rossum and the Python community
|
||||
Author-email: python-dev@python.org
|
||||
License: PSF license
|
||||
Description: Python is an interpreted, interactive, object-oriented programming
|
||||
language. It is often compared to Tcl, Perl, Scheme or Java.
|
||||
|
||||
Python combines remarkable power with very clear syntax. It has
|
||||
modules, classes, exceptions, very high level dynamic data types, and
|
||||
dynamic typing. There are interfaces to many system calls and
|
||||
libraries, as well as to various windowing systems (X11, Motif, Tk,
|
||||
Mac, MFC). New built-in modules are easily written in C or C++. Python
|
||||
is also usable as an extension language for applications that need a
|
||||
programmable interface.
|
||||
|
||||
The Python implementation is portable: it runs on many brands of UNIX,
|
||||
on Windows, DOS, OS/2, Mac, Amiga... If your favorite system isn't
|
||||
listed here, it may still be supported, if there's a C compiler for
|
||||
it. Ask around on comp.lang.python -- or just try compiling Python
|
||||
yourself.
|
||||
Platform: Many
|
||||
Classifier: Development Status :: 6 - Mature
|
||||
Classifier: License :: OSI Approved :: Python Software Foundation License
|
||||
Classifier: Natural Language :: English
|
||||
Classifier: Programming Language :: C
|
||||
Classifier: Programming Language :: Python
|
||||
Classifier: Topic :: Software Development
|
||||
@@ -0,0 +1,69 @@
|
||||
Metadata-Version: 1.1
|
||||
Name: mxnet
|
||||
Version: 1.8.0
|
||||
Summary: <!--- Licensed to the Apache Software Foundation (ASF) under one -->
|
||||
<!--- or more contributor license agreements. See the NOTICE file -->
|
||||
<!--- distributed with this work for additional information -->
|
||||
<!--- regarding copyright ownership. The ASF licenses this file -->
|
||||
<!--- to you under the Apache License, Version 2.0 (the -->
|
||||
<!--- "License"); you may not use this file except in compliance -->
|
||||
<!--- with the License. You may obtain a copy of the License at -->
|
||||
|
||||
<!--- http://www.apache.org/licenses/LICENSE-2.0 -->
|
||||
|
||||
<!--- Unless required by applicable law or agreed to in writing, -->
|
||||
<!--- software distributed under the License is distributed on an -->
|
||||
<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
|
||||
<!--- KIND, either express or implied. See the License for the -->
|
||||
<!--- specific language governing permissions and limitations -->
|
||||
<!--- under the License. -->
|
||||
|
||||
MXNet Python Package
|
||||
====================
|
||||
This directory and nested files contain MXNet Python package and language binding.
|
||||
|
||||
## Installation
|
||||
To install MXNet Python package, visit MXNet [Install Instruction](https://mxnet.apache.org/get_started)
|
||||
|
||||
|
||||
## Running the unit tests
|
||||
|
||||
For running unit tests, you will need the [nose PyPi package](https://pypi.python.org/pypi/nose). To install:
|
||||
```bash
|
||||
pip install --upgrade nose
|
||||
```
|
||||
|
||||
Once ```nose``` is installed, run the following from MXNet root directory (please make sure the installation path of ```nosetests``` is included in your ```$PATH``` environment variable):
|
||||
```
|
||||
nosetests tests/python/unittest
|
||||
nosetests tests/python/train
|
||||
|
||||
```
|
||||
|
||||
Home-page: https://github.com/apache/incubator-mxnet
|
||||
Author: UNKNOWN
|
||||
Author-email: UNKNOWN
|
||||
License: UNKNOWN
|
||||
Description: UNKNOWN
|
||||
Platform: UNKNOWN
|
||||
Classifier: Development Status :: 5 - Production/Stable
|
||||
Classifier: Intended Audience :: Developers
|
||||
Classifier: Intended Audience :: Education
|
||||
Classifier: Intended Audience :: Science/Research
|
||||
Classifier: License :: OSI Approved :: Apache Software License
|
||||
Classifier: Programming Language :: C++
|
||||
Classifier: Programming Language :: Cython
|
||||
Classifier: Programming Language :: Other
|
||||
Classifier: Programming Language :: Perl
|
||||
Classifier: Programming Language :: Python
|
||||
Classifier: Programming Language :: Python :: 2.7
|
||||
Classifier: Programming Language :: Python :: 3.4
|
||||
Classifier: Programming Language :: Python :: 3.5
|
||||
Classifier: Programming Language :: Python :: 3.6
|
||||
Classifier: Programming Language :: Python :: Implementation :: CPython
|
||||
Classifier: Topic :: Scientific/Engineering
|
||||
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
||||
Classifier: Topic :: Scientific/Engineering :: Mathematics
|
||||
Classifier: Topic :: Software Development
|
||||
Classifier: Topic :: Software Development :: Libraries
|
||||
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
||||
@@ -5,7 +5,6 @@ package rpmdb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
@@ -37,14 +36,14 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
|
||||
|
||||
var pkgs []pkg.Package
|
||||
for _, location := range fileMatches {
|
||||
dbContents, err := resolver.FileContentsByLocation(location)
|
||||
dbContentReader, err := resolver.FileContentsByLocation(location)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pkgs, err = parseRpmDB(resolver, location, strings.NewReader(dbContents))
|
||||
pkgs, err = parseRpmDB(resolver, location, dbContentReader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.Path, err)
|
||||
return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.RealPath, err)
|
||||
}
|
||||
}
|
||||
return pkgs, nil
|
||||
|
||||
@@ -45,11 +45,6 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader
|
||||
allPkgs := make([]pkg.Package, 0)
|
||||
|
||||
for _, entry := range pkgList {
|
||||
records, err := extractRpmdbFileRecords(resolver, entry)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
p := pkg.Package{
|
||||
Name: entry.Name,
|
||||
Version: fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does, instead of fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch)
|
||||
@@ -67,7 +62,7 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader
|
||||
Vendor: entry.Vendor,
|
||||
License: entry.License,
|
||||
Size: entry.Size,
|
||||
Files: records,
|
||||
Files: extractRpmdbFileRecords(resolver, entry),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -77,25 +72,19 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader
|
||||
return allPkgs, nil
|
||||
}
|
||||
|
||||
func extractRpmdbFileRecords(resolver source.FileResolver, entry *rpmdb.PackageInfo) ([]pkg.RpmdbFileRecord, error) {
|
||||
func extractRpmdbFileRecords(resolver source.FileResolver, entry *rpmdb.PackageInfo) []pkg.RpmdbFileRecord {
|
||||
var records = make([]pkg.RpmdbFileRecord, 0)
|
||||
|
||||
for _, record := range entry.Files {
|
||||
refs, err := resolver.FilesByPath(record.Path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resolve path=%+v: %w", record.Path, err)
|
||||
}
|
||||
//only persist RPMDB file records which exist in the image/directory, otherwise ignore them
|
||||
if len(refs) == 0 {
|
||||
continue
|
||||
if resolver.HasPath(record.Path) {
|
||||
records = append(records, pkg.RpmdbFileRecord{
|
||||
Path: record.Path,
|
||||
Mode: pkg.RpmdbFileMode(record.Mode),
|
||||
Size: int(record.Size),
|
||||
SHA256: record.SHA256,
|
||||
})
|
||||
}
|
||||
|
||||
records = append(records, pkg.RpmdbFileRecord{
|
||||
Path: record.Path,
|
||||
Mode: pkg.RpmdbFileMode(record.Mode),
|
||||
Size: int(record.Size),
|
||||
SHA256: record.SHA256,
|
||||
})
|
||||
}
|
||||
return records, nil
|
||||
return records
|
||||
}
|
||||
|
||||
@@ -21,6 +21,10 @@ func newTestFileResolver(ignorePaths bool) *rpmdbTestFileResolverMock {
|
||||
}
|
||||
}
|
||||
|
||||
func (r rpmdbTestFileResolverMock) HasPath(path string) bool {
|
||||
return !r.ignorePaths
|
||||
}
|
||||
|
||||
func (r *rpmdbTestFileResolverMock) FilesByPath(paths ...string) ([]source.Location, error) {
|
||||
if r.ignorePaths {
|
||||
// act as if no paths exist
|
||||
|
||||
@@ -19,7 +19,7 @@ func NewGemFileLockCataloger() *common.GenericCataloger {
|
||||
// NewGemSpecCataloger returns a new Bundler cataloger object tailored for detecting installations of gems (e.g. Gemspec).
|
||||
func NewGemSpecCataloger() *common.GenericCataloger {
|
||||
globParsers := map[string]common.ParserFn{
|
||||
"**/specifications/*.gemspec": parseGemSpecEntries,
|
||||
"**/specifications/**/*.gemspec": parseGemSpecEntries,
|
||||
}
|
||||
|
||||
return common.NewGenericCataloger(nil, globParsers, "ruby-gemspec-cataloger")
|
||||
|
||||
+28
-21
@@ -1,6 +1,7 @@
|
||||
package distro
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
@@ -16,28 +17,28 @@ type parseEntry struct {
|
||||
fn parseFunc
|
||||
}
|
||||
|
||||
var identityFiles = []parseEntry{
|
||||
{
|
||||
// most distros provide a link at this location
|
||||
path: "/etc/os-release",
|
||||
fn: parseOsRelease,
|
||||
},
|
||||
{
|
||||
// standard location for rhel & debian distros
|
||||
path: "/usr/lib/os-release",
|
||||
fn: parseOsRelease,
|
||||
},
|
||||
{
|
||||
// check for busybox (important to check this last since other distros contain the busybox binary)
|
||||
path: "/bin/busybox",
|
||||
fn: parseBusyBox,
|
||||
},
|
||||
}
|
||||
|
||||
// Identify parses distro-specific files to determine distro metadata like version and release.
|
||||
func Identify(resolver source.Resolver) *Distro {
|
||||
var distro *Distro
|
||||
|
||||
identityFiles := []parseEntry{
|
||||
{
|
||||
// most distros provide a link at this location
|
||||
path: "/etc/os-release",
|
||||
fn: parseOsRelease,
|
||||
},
|
||||
{
|
||||
// standard location for rhel & debian distros
|
||||
path: "/usr/lib/os-release",
|
||||
fn: parseOsRelease,
|
||||
},
|
||||
{
|
||||
// check for busybox (important to check this last since other distros contain the busybox binary)
|
||||
path: "/bin/busybox",
|
||||
fn: parseBusyBox,
|
||||
},
|
||||
}
|
||||
|
||||
identifyLoop:
|
||||
for _, entry := range identityFiles {
|
||||
locations, err := resolver.FilesByPath(entry.path)
|
||||
@@ -52,19 +53,25 @@ identifyLoop:
|
||||
}
|
||||
|
||||
for _, location := range locations {
|
||||
content, err := resolver.FileContentsByLocation(location)
|
||||
contentReader, err := resolver.FileContentsByLocation(location)
|
||||
|
||||
if err != nil {
|
||||
log.Debugf("unable to get contents from %s: %s", entry.path, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if content == "" {
|
||||
content, err := ioutil.ReadAll(contentReader)
|
||||
if err != nil {
|
||||
log.Errorf("unable to read %q: %+v", location.RealPath, err)
|
||||
break
|
||||
}
|
||||
|
||||
if len(content) == 0 {
|
||||
log.Debugf("no contents in file, skipping: %s", entry.path)
|
||||
continue
|
||||
}
|
||||
|
||||
if candidateDistro := entry.fn(content); candidateDistro != nil {
|
||||
if candidateDistro := entry.fn(string(content)); candidateDistro != nil {
|
||||
distro = candidateDistro
|
||||
break identifyLoop
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/filetree"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/imagetest"
|
||||
|
||||
"github.com/anchore/go-testutils"
|
||||
@@ -28,7 +30,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) {
|
||||
Type: pkg.DebPkg,
|
||||
FoundBy: "the-cataloger-1",
|
||||
Locations: []source.Location{
|
||||
{Path: "/some/path/pkg1"},
|
||||
{RealPath: "/some/path/pkg1"},
|
||||
},
|
||||
Metadata: pkg.DpkgMetadata{
|
||||
Package: "package1",
|
||||
@@ -42,7 +44,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) {
|
||||
Type: pkg.DebPkg,
|
||||
FoundBy: "the-cataloger-2",
|
||||
Locations: []source.Location{
|
||||
{Path: "/some/path/pkg1"},
|
||||
{RealPath: "/some/path/pkg1"},
|
||||
},
|
||||
Licenses: []string{
|
||||
"MIT",
|
||||
@@ -94,12 +96,15 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
|
||||
img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-simple")
|
||||
defer cleanup()
|
||||
|
||||
_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
|
||||
_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
|
||||
|
||||
// populate catalog with test data
|
||||
catalog.Add(pkg.Package{
|
||||
Name: "package1",
|
||||
Version: "1.0.1",
|
||||
Locations: []source.Location{
|
||||
source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
|
||||
source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
|
||||
},
|
||||
Type: pkg.RpmPkg,
|
||||
FoundBy: "the-cataloger-1",
|
||||
@@ -109,7 +114,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
|
||||
Name: "package2",
|
||||
Version: "2.0.1",
|
||||
Locations: []source.Location{
|
||||
source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
|
||||
source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
|
||||
},
|
||||
Type: pkg.RpmPkg,
|
||||
FoundBy: "the-cataloger-2",
|
||||
@@ -120,7 +125,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
|
||||
PURL: "the-purl-2",
|
||||
})
|
||||
|
||||
s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input")
|
||||
s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<bom xmlns="http://cyclonedx.org/schema/bom/1.2" version="1" serialNumber="urn:uuid:553b2601-a7b0-4461-bce9-fd56e2e2a4be">
|
||||
<bom xmlns="http://cyclonedx.org/schema/bom/1.2" version="1" serialNumber="urn:uuid:544b9f08-b39d-4d0c-8723-32609567e8ed">
|
||||
<metadata>
|
||||
<timestamp>2020-12-01T22:19:00-05:00</timestamp>
|
||||
<timestamp>2020-12-28T13:56:29-05:00</timestamp>
|
||||
<tools>
|
||||
<tool>
|
||||
<vendor>anchore</vendor>
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<bom xmlns="http://cyclonedx.org/schema/bom/1.2" version="1" serialNumber="urn:uuid:4a54c180-9469-4513-a1cb-af3fe1e83738">
|
||||
<bom xmlns="http://cyclonedx.org/schema/bom/1.2" version="1" serialNumber="urn:uuid:e1515688-2fd6-422a-acb2-97405a44cdfe">
|
||||
<metadata>
|
||||
<timestamp>2020-12-01T22:19:00-05:00</timestamp>
|
||||
<timestamp>2020-12-28T13:56:29-05:00</timestamp>
|
||||
<tools>
|
||||
<tool>
|
||||
<vendor>anchore</vendor>
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"flag"
|
||||
"testing"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/filetree"
|
||||
|
||||
"github.com/anchore/go-testutils"
|
||||
"github.com/anchore/stereoscope/pkg/imagetest"
|
||||
"github.com/anchore/syft/syft/distro"
|
||||
@@ -34,7 +36,7 @@ func TestJsonDirsPresenter(t *testing.T) {
|
||||
Type: pkg.PythonPkg,
|
||||
FoundBy: "the-cataloger-1",
|
||||
Locations: []source.Location{
|
||||
{Path: "/some/path/pkg1"},
|
||||
{RealPath: "/some/path/pkg1"},
|
||||
},
|
||||
Language: pkg.Python,
|
||||
MetadataType: pkg.PythonPackageMetadataType,
|
||||
@@ -54,7 +56,7 @@ func TestJsonDirsPresenter(t *testing.T) {
|
||||
Type: pkg.DebPkg,
|
||||
FoundBy: "the-cataloger-2",
|
||||
Locations: []source.Location{
|
||||
{Path: "/some/path/pkg1"},
|
||||
{RealPath: "/some/path/pkg1"},
|
||||
},
|
||||
MetadataType: pkg.DpkgMetadataType,
|
||||
Metadata: pkg.DpkgMetadata{
|
||||
@@ -106,12 +108,15 @@ func TestJsonImgsPresenter(t *testing.T) {
|
||||
catalog := pkg.NewCatalog()
|
||||
img := imagetest.GetGoldenFixtureImage(t, testImage)
|
||||
|
||||
_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
|
||||
_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
|
||||
|
||||
// populate catalog with test data
|
||||
catalog.Add(pkg.Package{
|
||||
Name: "package-1",
|
||||
Version: "1.0.1",
|
||||
Locations: []source.Location{
|
||||
source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
|
||||
source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
|
||||
},
|
||||
Type: pkg.PythonPkg,
|
||||
FoundBy: "the-cataloger-1",
|
||||
@@ -131,7 +136,7 @@ func TestJsonImgsPresenter(t *testing.T) {
|
||||
Name: "package-2",
|
||||
Version: "2.0.1",
|
||||
Locations: []source.Location{
|
||||
source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
|
||||
source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
|
||||
},
|
||||
Type: pkg.DebPkg,
|
||||
FoundBy: "the-cataloger-2",
|
||||
@@ -149,7 +154,7 @@ func TestJsonImgsPresenter(t *testing.T) {
|
||||
// this is a hard coded value that is not given by the fixture helper and must be provided manually
|
||||
img.Metadata.ManifestDigest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368"
|
||||
|
||||
s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input")
|
||||
s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input")
|
||||
var d *distro.Distro
|
||||
pres := NewPresenter(catalog, s.Metadata, d)
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@
|
||||
"stereoscope-fixture-image-simple:04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7"
|
||||
],
|
||||
"imageSize": 65,
|
||||
"scope": "AllLayers",
|
||||
"scope": "Squashed",
|
||||
"layers": [
|
||||
{
|
||||
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"flag"
|
||||
"testing"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/filetree"
|
||||
|
||||
"github.com/go-test/deep"
|
||||
|
||||
"github.com/anchore/go-testutils"
|
||||
@@ -25,12 +27,15 @@ func TestTablePresenter(t *testing.T) {
|
||||
img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", testImage)
|
||||
defer cleanup()
|
||||
|
||||
_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
|
||||
_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
|
||||
|
||||
// populate catalog with test data
|
||||
catalog.Add(pkg.Package{
|
||||
Name: "package-1",
|
||||
Version: "1.0.1",
|
||||
Locations: []source.Location{
|
||||
source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
|
||||
source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
|
||||
},
|
||||
Type: pkg.DebPkg,
|
||||
})
|
||||
@@ -38,7 +43,7 @@ func TestTablePresenter(t *testing.T) {
|
||||
Name: "package-2",
|
||||
Version: "2.0.1",
|
||||
Locations: []source.Location{
|
||||
source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
|
||||
source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
|
||||
},
|
||||
Type: pkg.DebPkg,
|
||||
})
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"flag"
|
||||
"testing"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/filetree"
|
||||
|
||||
"github.com/anchore/go-testutils"
|
||||
"github.com/anchore/stereoscope/pkg/imagetest"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
@@ -70,12 +72,15 @@ func TestTextImgPresenter(t *testing.T) {
|
||||
img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-simple")
|
||||
defer cleanup()
|
||||
|
||||
_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
|
||||
_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
|
||||
|
||||
// populate catalog with test data
|
||||
catalog.Add(pkg.Package{
|
||||
Name: "package-1",
|
||||
Version: "1.0.1",
|
||||
Locations: []source.Location{
|
||||
source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
|
||||
source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
|
||||
},
|
||||
FoundBy: "dpkg",
|
||||
Type: pkg.DebPkg,
|
||||
@@ -84,7 +89,7 @@ func TestTextImgPresenter(t *testing.T) {
|
||||
Name: "package-2",
|
||||
Version: "2.0.1",
|
||||
Locations: []source.Location{
|
||||
source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
|
||||
source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
|
||||
},
|
||||
FoundBy: "dpkg",
|
||||
Metadata: PackageInfo{Name: "package-2", Version: "1.0.2"},
|
||||
@@ -97,7 +102,7 @@ func TestTextImgPresenter(t *testing.T) {
|
||||
l.Metadata.Digest = "sha256:ad8ecdc058976c07e7e347cb89fa9ad86a294b5ceaae6d09713fb035f84115abf3c4a2388a4af3aa60f13b94f4c6846930bdf53"
|
||||
}
|
||||
|
||||
s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input")
|
||||
s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -2,10 +2,15 @@ package source
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/stereoscope/pkg/filetree"
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
)
|
||||
|
||||
var _ Resolver = (*AllLayersResolver)(nil)
|
||||
@@ -32,6 +37,18 @@ func NewAllLayersResolver(img *image.Image) (*AllLayersResolver, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// HasPath indicates if the given path exists in the underlying source.
|
||||
func (r *AllLayersResolver) HasPath(path string) bool {
|
||||
p := file.Path(path)
|
||||
for _, layerIdx := range r.layers {
|
||||
tree := r.img.Layers[layerIdx].Tree
|
||||
if tree.HasPath(p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.ReferenceSet, layerIdx int) ([]file.Reference, error) {
|
||||
uniqueFiles := make([]file.Reference, 0)
|
||||
|
||||
@@ -70,19 +87,22 @@ func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) {
|
||||
for _, path := range paths {
|
||||
for idx, layerIdx := range r.layers {
|
||||
tree := r.img.Layers[layerIdx].Tree
|
||||
ref := tree.File(file.Path(path))
|
||||
_, ref, err := tree.File(file.Path(path), filetree.FollowBasenameLinks, filetree.DoNotFollowDeadBasenameLinks)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ref == nil {
|
||||
// no file found, keep looking through layers
|
||||
continue
|
||||
}
|
||||
|
||||
// don't consider directories (special case: there is no path information for /)
|
||||
if ref.Path == "/" {
|
||||
if ref.RealPath == "/" {
|
||||
continue
|
||||
} else if r.img.FileCatalog.Exists(*ref) {
|
||||
metadata, err := r.img.FileCatalog.Get(*ref)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
|
||||
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.RealPath, err)
|
||||
}
|
||||
if metadata.Metadata.IsDir {
|
||||
continue
|
||||
@@ -94,7 +114,7 @@ func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) {
|
||||
return nil, err
|
||||
}
|
||||
for _, result := range results {
|
||||
uniqueLocations = append(uniqueLocations, NewLocationFromImage(result, r.img))
|
||||
uniqueLocations = append(uniqueLocations, NewLocationFromImage(path, result, r.img))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -109,31 +129,31 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error)
|
||||
|
||||
for _, pattern := range patterns {
|
||||
for idx, layerIdx := range r.layers {
|
||||
refs, err := r.img.Layers[layerIdx].Tree.FilesByGlob(pattern)
|
||||
results, err := r.img.Layers[layerIdx].Tree.FilesByGlob(pattern, filetree.DoNotFollowDeadBasenameLinks)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
|
||||
}
|
||||
|
||||
for _, ref := range refs {
|
||||
for _, result := range results {
|
||||
// don't consider directories (special case: there is no path information for /)
|
||||
if ref.Path == "/" {
|
||||
if result.RealPath == "/" {
|
||||
continue
|
||||
} else if r.img.FileCatalog.Exists(ref) {
|
||||
metadata, err := r.img.FileCatalog.Get(ref)
|
||||
} else if r.img.FileCatalog.Exists(result.Reference) {
|
||||
metadata, err := r.img.FileCatalog.Get(result.Reference)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
|
||||
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", result.MatchPath, err)
|
||||
}
|
||||
if metadata.Metadata.IsDir {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
results, err := r.fileByRef(ref, uniqueFileIDs, idx)
|
||||
refResults, err := r.fileByRef(result.Reference, uniqueFileIDs, idx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, result := range results {
|
||||
uniqueLocations = append(uniqueLocations, NewLocationFromImage(result, r.img))
|
||||
for _, refResult := range refResults {
|
||||
uniqueLocations = append(uniqueLocations, NewLocationFromImage(string(result.MatchPath), refResult, r.img))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -150,37 +170,41 @@ func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) *
|
||||
return nil
|
||||
}
|
||||
|
||||
relativeRef := entry.Source.SquashedTree.File(file.Path(path))
|
||||
if relativeRef == nil {
|
||||
exists, relativeRef, err := entry.Layer.SquashedTree.File(file.Path(path), filetree.FollowBasenameLinks)
|
||||
if err != nil {
|
||||
log.Errorf("failed to find path=%q in squash: %+w", path, err)
|
||||
return nil
|
||||
}
|
||||
if !exists && relativeRef == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
relativeLocation := NewLocationFromImage(*relativeRef, r.img)
|
||||
relativeLocation := NewLocationFromImage(path, *relativeRef, r.img)
|
||||
|
||||
return &relativeLocation
|
||||
}
|
||||
|
||||
// MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a
|
||||
// file.Reference is a path relative to a particular layer.
|
||||
func (r *AllLayersResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
|
||||
func (r *AllLayersResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
|
||||
return mapLocationRefs(r.img.MultipleFileContentsByRef, locations)
|
||||
}
|
||||
|
||||
// FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer.
|
||||
// If the path does not exist an error is returned.
|
||||
func (r *AllLayersResolver) FileContentsByLocation(location Location) (string, error) {
|
||||
func (r *AllLayersResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
|
||||
return r.img.FileContentsByRef(location.ref)
|
||||
}
|
||||
|
||||
type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]string, error)
|
||||
type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]io.ReadCloser, error)
|
||||
|
||||
func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]string, error) {
|
||||
func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]io.ReadCloser, error) {
|
||||
var fileRefs = make([]file.Reference, len(locations))
|
||||
var locationByRefs = make(map[file.Reference]Location)
|
||||
var results = make(map[Location]string)
|
||||
var locationByRefs = make(map[file.Reference][]Location)
|
||||
var results = make(map[Location]io.ReadCloser)
|
||||
|
||||
for i, location := range locations {
|
||||
locationByRefs[location.ref] = location
|
||||
locationByRefs[location.ref] = append(locationByRefs[location.ref], location)
|
||||
fileRefs[i] = location.ref
|
||||
}
|
||||
|
||||
@@ -189,8 +213,26 @@ func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Lo
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// TODO: this is not tested, we need a test case that covers a mapLocationRefs which has multiple Locations with the same reference in the request. The io.Reader should be copied.
|
||||
for ref, content := range contentsByRef {
|
||||
results[locationByRefs[ref]] = content
|
||||
mappedLocations := locationByRefs[ref]
|
||||
switch {
|
||||
case len(mappedLocations) > 1:
|
||||
// TODO: fixme... this can lead to lots of unexpected memory usage in unusual circumstances (cache is not leveraged for large files).
|
||||
// stereoscope wont duplicate content requests if the caller asks for the same file multiple times... thats up to the caller
|
||||
contentsBytes, err := ioutil.ReadAll(content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to read ref=%+v :%w", ref, err)
|
||||
}
|
||||
for _, loc := range mappedLocations {
|
||||
results[loc] = ioutil.NopCloser(bytes.NewReader(contentsBytes))
|
||||
}
|
||||
|
||||
case len(mappedLocations) == 1:
|
||||
results[locationByRefs[ref][0]] = content
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected ref-location count=%d for ref=%v", len(mappedLocations), ref)
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
@@ -13,9 +13,10 @@ type resolution struct {
|
||||
|
||||
func TestAllLayersResolver_FilesByPath(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
linkPath string
|
||||
resolutions []resolution
|
||||
name string
|
||||
linkPath string
|
||||
resolutions []resolution
|
||||
forcePositiveHasPath bool
|
||||
}{
|
||||
{
|
||||
name: "link with previous data",
|
||||
@@ -41,10 +42,6 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
|
||||
name: "link with overridden data",
|
||||
linkPath: "/link-2",
|
||||
resolutions: []resolution{
|
||||
{
|
||||
layer: 3,
|
||||
path: "/link-2",
|
||||
},
|
||||
{
|
||||
layer: 4,
|
||||
path: "/file-2.txt",
|
||||
@@ -70,19 +67,17 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "dead link",
|
||||
linkPath: "/link-dead",
|
||||
resolutions: []resolution{
|
||||
{
|
||||
layer: 8,
|
||||
path: "/link-dead",
|
||||
},
|
||||
},
|
||||
name: "dead link",
|
||||
linkPath: "/link-dead",
|
||||
resolutions: []resolution{},
|
||||
forcePositiveHasPath: true,
|
||||
},
|
||||
{
|
||||
name: "ignore directories",
|
||||
linkPath: "/bin",
|
||||
resolutions: []resolution{},
|
||||
// directories don't resolve BUT do exist
|
||||
forcePositiveHasPath: true,
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
@@ -95,6 +90,17 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
|
||||
t.Fatalf("could not create resolver: %+v", err)
|
||||
}
|
||||
|
||||
hasPath := resolver.HasPath(c.linkPath)
|
||||
if !c.forcePositiveHasPath {
|
||||
if len(c.resolutions) > 0 && !hasPath {
|
||||
t.Errorf("expected HasPath() to indicate existance, but did not")
|
||||
} else if len(c.resolutions) == 0 && hasPath {
|
||||
t.Errorf("expeced HasPath() to NOT indicate existance, but does")
|
||||
}
|
||||
} else if !hasPath {
|
||||
t.Errorf("expected HasPath() to indicate existance, but did not (force path)")
|
||||
}
|
||||
|
||||
refs, err := resolver.FilesByPath(c.linkPath)
|
||||
if err != nil {
|
||||
t.Fatalf("could not use resolver: %+v", err)
|
||||
@@ -107,8 +113,12 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
|
||||
for idx, actual := range refs {
|
||||
expected := c.resolutions[idx]
|
||||
|
||||
if actual.Path != expected.path {
|
||||
t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
|
||||
if string(actual.ref.RealPath) != expected.path {
|
||||
t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), expected.path)
|
||||
}
|
||||
|
||||
if expected.path != "" && string(actual.ref.RealPath) != actual.RealPath {
|
||||
t.Errorf("we should always prefer real paths over ones with links")
|
||||
}
|
||||
|
||||
entry, err := img.FileCatalog.Get(actual.ref)
|
||||
@@ -116,8 +126,8 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
|
||||
t.Fatalf("failed to get metadata: %+v", err)
|
||||
}
|
||||
|
||||
if entry.Source.Metadata.Index != expected.layer {
|
||||
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
|
||||
if entry.Layer.Metadata.Index != expected.layer {
|
||||
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, expected.layer)
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -132,7 +142,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
name: "link with previous data",
|
||||
glob: "**ink-1",
|
||||
glob: "**/*ink-1",
|
||||
resolutions: []resolution{
|
||||
{
|
||||
layer: 1,
|
||||
@@ -142,7 +152,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "link with in layer data",
|
||||
glob: "**nk-within",
|
||||
glob: "**/*nk-within",
|
||||
resolutions: []resolution{
|
||||
{
|
||||
layer: 5,
|
||||
@@ -152,12 +162,8 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "link with overridden data",
|
||||
glob: "**ink-2",
|
||||
glob: "**/*ink-2",
|
||||
resolutions: []resolution{
|
||||
{
|
||||
layer: 3,
|
||||
path: "/link-2",
|
||||
},
|
||||
{
|
||||
layer: 4,
|
||||
path: "/file-2.txt",
|
||||
@@ -170,7 +176,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "indirect link (with overridden data)",
|
||||
glob: "**nk-indirect",
|
||||
glob: "**/*nk-indirect",
|
||||
resolutions: []resolution{
|
||||
{
|
||||
layer: 4,
|
||||
@@ -183,14 +189,9 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "dead link",
|
||||
glob: "**k-dead",
|
||||
resolutions: []resolution{
|
||||
{
|
||||
layer: 8,
|
||||
path: "/link-dead",
|
||||
},
|
||||
},
|
||||
name: "dead link",
|
||||
glob: "**/*k-dead",
|
||||
resolutions: []resolution{},
|
||||
},
|
||||
{
|
||||
name: "ignore directories",
|
||||
@@ -220,8 +221,12 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
|
||||
for idx, actual := range refs {
|
||||
expected := c.resolutions[idx]
|
||||
|
||||
if actual.Path != expected.path {
|
||||
t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
|
||||
if string(actual.ref.RealPath) != expected.path {
|
||||
t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), expected.path)
|
||||
}
|
||||
|
||||
if expected.path != "" && string(actual.ref.RealPath) != actual.RealPath {
|
||||
t.Errorf("we should always prefer real paths over ones with links")
|
||||
}
|
||||
|
||||
entry, err := img.FileCatalog.Get(actual.ref)
|
||||
@@ -229,8 +234,8 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
|
||||
t.Fatalf("failed to get metadata: %+v", err)
|
||||
}
|
||||
|
||||
if entry.Source.Metadata.Index != expected.layer {
|
||||
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
|
||||
if entry.Layer.Metadata.Index != expected.layer {
|
||||
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, expected.layer)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package source
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/imagetest"
|
||||
@@ -53,16 +54,20 @@ func TestContentRequester(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, entry := range data {
|
||||
if expected, ok := test.expectedContents[entry.Location.Path]; ok {
|
||||
for expected != entry.Contents {
|
||||
t.Errorf("mismatched contents for %q", entry.Location.Path)
|
||||
if expected, ok := test.expectedContents[entry.Location.RealPath]; ok {
|
||||
actualBytes, err := ioutil.ReadAll(entry.Contents)
|
||||
if err != nil {
|
||||
t.Fatalf("could not read %q: %+v", entry.Location.RealPath, err)
|
||||
}
|
||||
for expected != string(actualBytes) {
|
||||
t.Errorf("mismatched contents for %q", entry.Location.RealPath)
|
||||
dmp := diffmatchpatch.New()
|
||||
diffs := dmp.DiffMain(expected, entry.Contents, true)
|
||||
diffs := dmp.DiffMain(expected, string(actualBytes), true)
|
||||
t.Errorf("diff: %s", dmp.DiffPrettyText(diffs))
|
||||
}
|
||||
continue
|
||||
}
|
||||
t.Errorf("could not find %q", entry.Location.Path)
|
||||
t.Errorf("could not find %q", entry.Location.RealPath)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2,13 +2,14 @@ package source
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/bmatcuk/doublestar"
|
||||
"github.com/bmatcuk/doublestar/v2"
|
||||
)
|
||||
|
||||
var _ Resolver = (*DirectoryResolver)(nil)
|
||||
@@ -18,27 +19,37 @@ type DirectoryResolver struct {
|
||||
Path string
|
||||
}
|
||||
|
||||
func (r DirectoryResolver) requestPath(userPath string) string {
|
||||
fullPath := userPath
|
||||
if filepath.IsAbs(fullPath) {
|
||||
// a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is
|
||||
fullPath = path.Join(r.Path, fullPath)
|
||||
}
|
||||
return fullPath
|
||||
}
|
||||
|
||||
// HasPath indicates if the given path exists in the underlying source.
|
||||
func (r *DirectoryResolver) HasPath(userPath string) bool {
|
||||
_, err := os.Stat(r.requestPath(userPath))
|
||||
return !os.IsNotExist(err)
|
||||
}
|
||||
|
||||
// Stringer to represent a directory path data source
|
||||
func (s DirectoryResolver) String() string {
|
||||
return fmt.Sprintf("dir:%s", s.Path)
|
||||
func (r DirectoryResolver) String() string {
|
||||
return fmt.Sprintf("dir:%s", r.Path)
|
||||
}
|
||||
|
||||
// FilesByPath returns all file.References that match the given paths from the directory.
|
||||
func (s DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error) {
|
||||
func (r DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error) {
|
||||
var references = make([]Location, 0)
|
||||
|
||||
for _, userPath := range userPaths {
|
||||
userStrPath := userPath
|
||||
|
||||
if filepath.IsAbs(userStrPath) {
|
||||
// a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is
|
||||
userStrPath = path.Join(s.Path, userStrPath)
|
||||
}
|
||||
userStrPath := r.requestPath(userPath)
|
||||
fileMeta, err := os.Stat(userStrPath)
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
log.Errorf("path (%s) is not valid: %v", userStrPath, err)
|
||||
log.Errorf("path (%r) is not valid: %v", userStrPath, err)
|
||||
}
|
||||
|
||||
// don't consider directories
|
||||
@@ -53,11 +64,11 @@ func (s DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error)
|
||||
}
|
||||
|
||||
// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
|
||||
func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
|
||||
func (r DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
|
||||
result := make([]Location, 0)
|
||||
|
||||
for _, pattern := range patterns {
|
||||
pathPattern := path.Join(s.Path, pattern)
|
||||
pathPattern := path.Join(r.Path, pattern)
|
||||
pathMatches, err := doublestar.Glob(pathPattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -83,8 +94,8 @@ func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
|
||||
// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
|
||||
// This is helpful when attempting to find a file that is in the same layer or lower as another file. For the
|
||||
// DirectoryResolver, this is a simple path lookup.
|
||||
func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Location {
|
||||
paths, err := s.FilesByPath(path)
|
||||
func (r *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Location {
|
||||
paths, err := r.FilesByPath(path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
@@ -96,35 +107,16 @@ func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Locatio
|
||||
}
|
||||
|
||||
// MultipleFileContentsByLocation returns the file contents for all file.References relative a directory.
|
||||
func (s DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
|
||||
refContents := make(map[Location]string)
|
||||
func (r DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
|
||||
refContents := make(map[Location]io.ReadCloser)
|
||||
for _, location := range locations {
|
||||
contents, err := fileContents(location.Path)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not read contents of file: %s", location.Path)
|
||||
}
|
||||
refContents[location] = string(contents)
|
||||
refContents[location] = file.NewDeferredReadCloser(location.RealPath)
|
||||
}
|
||||
return refContents, nil
|
||||
}
|
||||
|
||||
// FileContentsByLocation fetches file contents for a single file reference relative to a directory.
|
||||
// If the path does not exist an error is returned.
|
||||
func (s DirectoryResolver) FileContentsByLocation(location Location) (string, error) {
|
||||
contents, err := fileContents(location.Path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("could not read contents of file: %s", location.Path)
|
||||
}
|
||||
|
||||
return string(contents), nil
|
||||
}
|
||||
|
||||
func fileContents(path string) ([]byte, error) {
|
||||
contents, err := ioutil.ReadFile(path)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return contents, nil
|
||||
func (r DirectoryResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
|
||||
return file.NewDeferredReadCloser(location.RealPath), nil
|
||||
}
|
||||
|
||||
@@ -6,11 +6,12 @@ import (
|
||||
|
||||
func TestDirectoryResolver_FilesByPath(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
root string
|
||||
input string
|
||||
expected string
|
||||
refCount int
|
||||
name string
|
||||
root string
|
||||
input string
|
||||
expected string
|
||||
refCount int
|
||||
forcePositiveHasPath bool
|
||||
}{
|
||||
{
|
||||
name: "finds a file (relative)",
|
||||
@@ -47,15 +48,28 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
|
||||
refCount: 1,
|
||||
},
|
||||
{
|
||||
name: "directories ignored",
|
||||
root: "./test-fixtures/",
|
||||
input: "/image-symlinks",
|
||||
refCount: 0,
|
||||
name: "directories ignored",
|
||||
root: "./test-fixtures/",
|
||||
input: "/image-symlinks",
|
||||
refCount: 0,
|
||||
forcePositiveHasPath: true,
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
resolver := DirectoryResolver{c.root}
|
||||
|
||||
hasPath := resolver.HasPath(c.input)
|
||||
if !c.forcePositiveHasPath {
|
||||
if c.refCount != 0 && !hasPath {
|
||||
t.Errorf("expected HasPath() to indicate existance, but did not")
|
||||
} else if c.refCount == 0 && hasPath {
|
||||
t.Errorf("expeced HasPath() to NOT indicate existance, but does")
|
||||
}
|
||||
} else if !hasPath {
|
||||
t.Errorf("expected HasPath() to indicate existance, but did not (force path)")
|
||||
}
|
||||
|
||||
refs, err := resolver.FilesByPath(c.input)
|
||||
if err != nil {
|
||||
t.Fatalf("could not use resolver: %+v, %+v", err, refs)
|
||||
@@ -66,8 +80,8 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, actual := range refs {
|
||||
if actual.Path != c.expected {
|
||||
t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.expected)
|
||||
if actual.RealPath != c.expected {
|
||||
t.Errorf("bad resolve path: '%s'!='%s'", actual.RealPath, c.expected)
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -171,8 +185,9 @@ func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
|
||||
t.Fatalf("could not use resolver: %+v, %+v", err, refs)
|
||||
}
|
||||
|
||||
if len(refs) != 2 {
|
||||
t.Errorf("unexpected number of refs: %d != 2", len(refs))
|
||||
expected := 2
|
||||
if len(refs) != expected {
|
||||
t.Errorf("unexpected number of refs: %d != %d", len(refs), expected)
|
||||
}
|
||||
|
||||
})
|
||||
@@ -187,8 +202,9 @@ func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) {
|
||||
t.Fatalf("could not use resolver: %+v, %+v", err, refs)
|
||||
}
|
||||
|
||||
if len(refs) != 4 {
|
||||
t.Errorf("unexpected number of refs: %d != 4", len(refs))
|
||||
expected := 6
|
||||
if len(refs) != expected {
|
||||
t.Errorf("unexpected number of refs: %d != %d", len(refs), expected)
|
||||
}
|
||||
|
||||
})
|
||||
@@ -202,8 +218,9 @@ func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) {
|
||||
t.Fatalf("could not use resolver: %+v, %+v", err, refs)
|
||||
}
|
||||
|
||||
if len(refs) != 1 {
|
||||
t.Errorf("unexpected number of refs: %d != 1", len(refs))
|
||||
expected := 1
|
||||
if len(refs) != expected {
|
||||
t.Errorf("unexpected number of refs: %d != %d", len(refs), expected)
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package source
|
||||
|
||||
import "io"
|
||||
|
||||
type FileData struct {
|
||||
Location Location
|
||||
Contents string
|
||||
Contents io.ReadCloser
|
||||
}
|
||||
|
||||
@@ -2,8 +2,10 @@ package source
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/stereoscope/pkg/filetree"
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
)
|
||||
|
||||
@@ -22,6 +24,11 @@ func NewImageSquashResolver(img *image.Image) (*ImageSquashResolver, error) {
|
||||
return &ImageSquashResolver{img: img}, nil
|
||||
}
|
||||
|
||||
// HasPath indicates if the given path exists in the underlying source.
|
||||
func (r *ImageSquashResolver) HasPath(path string) bool {
|
||||
return r.img.SquashedTree().HasPath(file.Path(path))
|
||||
}
|
||||
|
||||
// FilesByPath returns all file.References that match the given paths within the squashed representation of the image.
|
||||
func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
|
||||
uniqueFileIDs := file.NewFileReferenceSet()
|
||||
@@ -29,19 +36,22 @@ func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
|
||||
|
||||
for _, path := range paths {
|
||||
tree := r.img.SquashedTree()
|
||||
ref := tree.File(file.Path(path))
|
||||
_, ref, err := tree.File(file.Path(path), filetree.FollowBasenameLinks)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ref == nil {
|
||||
// no file found, keep looking through layers
|
||||
continue
|
||||
}
|
||||
|
||||
// don't consider directories (special case: there is no path information for /)
|
||||
if ref.Path == "/" {
|
||||
if ref.RealPath == "/" {
|
||||
continue
|
||||
} else if r.img.FileCatalog.Exists(*ref) {
|
||||
metadata, err := r.img.FileCatalog.Get(*ref)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
|
||||
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.RealPath, err)
|
||||
}
|
||||
if metadata.Metadata.IsDir {
|
||||
continue
|
||||
@@ -56,7 +66,7 @@ func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
|
||||
|
||||
if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) {
|
||||
uniqueFileIDs.Add(*resolvedRef)
|
||||
uniqueLocations = append(uniqueLocations, NewLocationFromImage(*resolvedRef, r.img))
|
||||
uniqueLocations = append(uniqueLocations, NewLocationFromImage(path, *resolvedRef, r.img))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,28 +79,28 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error
|
||||
uniqueLocations := make([]Location, 0)
|
||||
|
||||
for _, pattern := range patterns {
|
||||
refs, err := r.img.SquashedTree().FilesByGlob(pattern)
|
||||
results, err := r.img.SquashedTree().FilesByGlob(pattern)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
|
||||
}
|
||||
|
||||
for _, ref := range refs {
|
||||
for _, result := range results {
|
||||
// don't consider directories (special case: there is no path information for /)
|
||||
if ref.Path == "/" {
|
||||
if result.MatchPath == "/" {
|
||||
continue
|
||||
} else if r.img.FileCatalog.Exists(ref) {
|
||||
metadata, err := r.img.FileCatalog.Get(ref)
|
||||
} else if r.img.FileCatalog.Exists(result.Reference) {
|
||||
metadata, err := r.img.FileCatalog.Get(result.Reference)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
|
||||
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", result.MatchPath, err)
|
||||
}
|
||||
if metadata.Metadata.IsDir {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
resolvedLocations, err := r.FilesByPath(string(ref.Path))
|
||||
resolvedLocations, err := r.FilesByPath(string(result.MatchPath))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find files by path (ref=%+v): %w", ref, err)
|
||||
return nil, fmt.Errorf("failed to find files by path (result=%+v): %w", result, err)
|
||||
}
|
||||
for _, resolvedLocation := range resolvedLocations {
|
||||
if !uniqueFileIDs.Contains(resolvedLocation.ref) {
|
||||
@@ -121,12 +131,12 @@ func (r *ImageSquashResolver) RelativeFileByPath(_ Location, path string) *Locat
|
||||
|
||||
// MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a
|
||||
// file.Reference is a path relative to a particular layer, in this case only from the squashed representation.
|
||||
func (r *ImageSquashResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
|
||||
func (r *ImageSquashResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
|
||||
return mapLocationRefs(r.img.MultipleFileContentsByRef, locations)
|
||||
}
|
||||
|
||||
// FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer.
|
||||
// If the path does not exist an error is returned.
|
||||
func (r *ImageSquashResolver) FileContentsByLocation(location Location) (string, error) {
|
||||
func (r *ImageSquashResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
|
||||
return r.img.FileContentsByRef(location.ref)
|
||||
}
|
||||
|
||||
@@ -8,10 +8,11 @@ import (
|
||||
|
||||
func TestImageSquashResolver_FilesByPath(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
linkPath string
|
||||
resolveLayer uint
|
||||
resolvePath string
|
||||
name string
|
||||
linkPath string
|
||||
resolveLayer uint
|
||||
resolvePath string
|
||||
forcePositiveHasPath bool
|
||||
}{
|
||||
{
|
||||
name: "link with previous data",
|
||||
@@ -41,12 +42,22 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
|
||||
name: "dead link",
|
||||
linkPath: "/link-dead",
|
||||
resolveLayer: 8,
|
||||
resolvePath: "/link-dead",
|
||||
resolvePath: "",
|
||||
// the path should exist, even if the link is dead
|
||||
forcePositiveHasPath: true,
|
||||
},
|
||||
{
|
||||
name: "ignore directories",
|
||||
linkPath: "/bin",
|
||||
resolvePath: "",
|
||||
// the path should exist, even if we ignore it
|
||||
forcePositiveHasPath: true,
|
||||
},
|
||||
{
|
||||
name: "parent is a link (with overridden data)",
|
||||
linkPath: "/parent-link/file-4.txt",
|
||||
resolveLayer: 11,
|
||||
resolvePath: "/parent/file-4.txt",
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
@@ -59,6 +70,17 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
|
||||
t.Fatalf("could not create resolver: %+v", err)
|
||||
}
|
||||
|
||||
hasPath := resolver.HasPath(c.linkPath)
|
||||
if !c.forcePositiveHasPath {
|
||||
if c.resolvePath != "" && !hasPath {
|
||||
t.Errorf("expected HasPath() to indicate existance, but did not")
|
||||
} else if c.resolvePath == "" && hasPath {
|
||||
t.Errorf("expeced HasPath() to NOT indicate existance, but does")
|
||||
}
|
||||
} else if !hasPath {
|
||||
t.Errorf("expected HasPath() to indicate existance, but did not (force path)")
|
||||
}
|
||||
|
||||
refs, err := resolver.FilesByPath(c.linkPath)
|
||||
if err != nil {
|
||||
t.Fatalf("could not use resolver: %+v", err)
|
||||
@@ -80,8 +102,12 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
|
||||
|
||||
actual := refs[0]
|
||||
|
||||
if actual.Path != c.resolvePath {
|
||||
t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
|
||||
if string(actual.ref.RealPath) != c.resolvePath {
|
||||
t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), c.resolvePath)
|
||||
}
|
||||
|
||||
if c.resolvePath != "" && string(actual.ref.RealPath) != actual.RealPath {
|
||||
t.Errorf("we should always prefer real paths over ones with links")
|
||||
}
|
||||
|
||||
entry, err := img.FileCatalog.Get(actual.ref)
|
||||
@@ -89,8 +115,8 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
|
||||
t.Fatalf("failed to get metadata: %+v", err)
|
||||
}
|
||||
|
||||
if entry.Source.Metadata.Index != c.resolveLayer {
|
||||
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
|
||||
if entry.Layer.Metadata.Index != c.resolveLayer {
|
||||
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, c.resolveLayer)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -105,39 +131,51 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
name: "link with previous data",
|
||||
glob: "**link-1",
|
||||
glob: "**/link-1",
|
||||
resolveLayer: 1,
|
||||
resolvePath: "/file-1.txt",
|
||||
},
|
||||
{
|
||||
name: "link with in layer data",
|
||||
glob: "**link-within",
|
||||
glob: "**/link-within",
|
||||
resolveLayer: 5,
|
||||
resolvePath: "/file-3.txt",
|
||||
},
|
||||
{
|
||||
name: "link with overridden data",
|
||||
glob: "**link-2",
|
||||
glob: "**/link-2",
|
||||
resolveLayer: 7,
|
||||
resolvePath: "/file-2.txt",
|
||||
},
|
||||
{
|
||||
name: "indirect link (with overridden data)",
|
||||
glob: "**link-indirect",
|
||||
glob: "**/link-indirect",
|
||||
resolveLayer: 7,
|
||||
resolvePath: "/file-2.txt",
|
||||
},
|
||||
{
|
||||
name: "dead link",
|
||||
glob: "**link-dead",
|
||||
resolveLayer: 8,
|
||||
resolvePath: "/link-dead",
|
||||
name: "dead link",
|
||||
glob: "**/link-dead",
|
||||
// dead links are dead! they shouldn't match on globs
|
||||
resolvePath: "",
|
||||
},
|
||||
{
|
||||
name: "ignore directories",
|
||||
glob: "**/bin",
|
||||
resolvePath: "",
|
||||
},
|
||||
{
|
||||
name: "parent without link",
|
||||
glob: "**/parent/*.txt",
|
||||
resolveLayer: 11,
|
||||
resolvePath: "/parent/file-4.txt",
|
||||
},
|
||||
{
|
||||
name: "parent is a link (override)",
|
||||
glob: "**/parent-link/file-4.txt",
|
||||
resolveLayer: 11,
|
||||
resolvePath: "/parent/file-4.txt",
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
@@ -170,8 +208,12 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
|
||||
|
||||
actual := refs[0]
|
||||
|
||||
if actual.Path != c.resolvePath {
|
||||
t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
|
||||
if string(actual.ref.RealPath) != c.resolvePath {
|
||||
t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), c.resolvePath)
|
||||
}
|
||||
|
||||
if c.resolvePath != "" && string(actual.ref.RealPath) != actual.RealPath {
|
||||
t.Errorf("we should always prefer real paths over ones with links")
|
||||
}
|
||||
|
||||
entry, err := img.FileCatalog.Get(actual.ref)
|
||||
@@ -179,8 +221,8 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
|
||||
t.Fatalf("failed to get metadata: %+v", err)
|
||||
}
|
||||
|
||||
if entry.Source.Metadata.Index != c.resolveLayer {
|
||||
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
|
||||
if entry.Layer.Metadata.Index != c.resolveLayer {
|
||||
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, c.resolveLayer)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+12
-8
@@ -7,9 +7,11 @@ import (
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
)
|
||||
|
||||
// Location represents a path relative to a particular filesystem.
|
||||
// Location represents a path relative to a particular filesystem resolved to a specific file.Reference. This struct is used as a key
|
||||
// in content fetching to uniquely identify a file relative to a request (the VirtualPath).
|
||||
type Location struct {
|
||||
Path string `json:"path"` // The string path of the location (e.g. /etc/hosts)
|
||||
RealPath string `json:"path"` // The path where all path ancestors have no hardlinks / symlinks
|
||||
VirtualPath string `json:"-"` // The path to the file which may or may not have hardlinks / symlinks
|
||||
FileSystemID string `json:"layerID,omitempty"` // An ID representing the filesystem. For container images this is a layer digest, directories or root filesystem this is blank.
|
||||
ref file.Reference // The file reference relative to the stereoscope.FileCatalog that has more information about this location.
|
||||
}
|
||||
@@ -17,24 +19,26 @@ type Location struct {
|
||||
// NewLocation creates a new Location representing a path without denoting a filesystem or FileCatalog reference.
|
||||
func NewLocation(path string) Location {
|
||||
return Location{
|
||||
Path: path,
|
||||
RealPath: path,
|
||||
}
|
||||
}
|
||||
|
||||
// NewLocationFromImage creates a new Location representing the given path (extracted from the ref) relative to the given image.
|
||||
func NewLocationFromImage(ref file.Reference, img *image.Image) Location {
|
||||
func NewLocationFromImage(virtualPath string, ref file.Reference, img *image.Image) Location {
|
||||
entry, err := img.FileCatalog.Get(ref)
|
||||
if err != nil {
|
||||
log.Warnf("unable to find file catalog entry for ref=%+v", ref)
|
||||
return Location{
|
||||
Path: string(ref.Path),
|
||||
ref: ref,
|
||||
VirtualPath: virtualPath,
|
||||
RealPath: string(ref.RealPath),
|
||||
ref: ref,
|
||||
}
|
||||
}
|
||||
|
||||
return Location{
|
||||
Path: string(ref.Path),
|
||||
FileSystemID: entry.Source.Metadata.Digest,
|
||||
VirtualPath: virtualPath,
|
||||
RealPath: string(ref.RealPath),
|
||||
FileSystemID: entry.Layer.Metadata.Digest,
|
||||
ref: ref,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
package source
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/anchore/syft/internal/file"
|
||||
)
|
||||
|
||||
var _ Resolver = (*MockResolver)(nil)
|
||||
|
||||
// MockResolver implements the Resolver interface and is intended for use *only in test code*.
|
||||
// It provides an implementation that can resolve local filesystem paths using only a provided discrete list of file
|
||||
// paths, which are typically paths to test fixtures.
|
||||
type MockResolver struct {
|
||||
Locations []Location
|
||||
}
|
||||
|
||||
// NewMockResolverForPaths creates a new MockResolver, where the only resolvable
|
||||
// files are those specified by the supplied paths.
|
||||
func NewMockResolverForPaths(paths ...string) *MockResolver {
|
||||
var locations []Location
|
||||
for _, p := range paths {
|
||||
locations = append(locations, NewLocation(p))
|
||||
}
|
||||
|
||||
return &MockResolver{Locations: locations}
|
||||
}
|
||||
|
||||
// HasPath indicates if the given path exists in the underlying source.
|
||||
func (r MockResolver) HasPath(path string) bool {
|
||||
for _, l := range r.Locations {
|
||||
if l.RealPath == path {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// String returns the string representation of the MockResolver.
|
||||
func (r MockResolver) String() string {
|
||||
return fmt.Sprintf("mock:(%s,...)", r.Locations[0].RealPath)
|
||||
}
|
||||
|
||||
// FileContentsByLocation fetches file contents for a single location. If the
|
||||
// path does not exist, an error is returned.
|
||||
func (r MockResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
|
||||
for _, l := range r.Locations {
|
||||
if l == location {
|
||||
return os.Open(location.RealPath)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("no file for location: %v", location)
|
||||
}
|
||||
|
||||
// MultipleFileContentsByLocation returns the file contents for all specified Locations.
|
||||
func (r MockResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
|
||||
results := make(map[Location]io.ReadCloser)
|
||||
for _, l := range locations {
|
||||
contents, err := r.FileContentsByLocation(l)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results[l] = contents
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// FilesByPath returns all Locations that match the given paths.
|
||||
func (r MockResolver) FilesByPath(paths ...string) ([]Location, error) {
|
||||
var results []Location
|
||||
for _, p := range paths {
|
||||
for _, location := range r.Locations {
|
||||
if p == location.RealPath {
|
||||
results = append(results, NewLocation(p))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// FilesByGlob returns all Locations that match the given path glob pattern.
|
||||
func (r MockResolver) FilesByGlob(patterns ...string) ([]Location, error) {
|
||||
var results []Location
|
||||
for _, pattern := range patterns {
|
||||
for _, location := range r.Locations {
|
||||
if file.GlobMatch(pattern, location.RealPath) {
|
||||
results = append(results, location)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// RelativeFileByPath returns a single Location for the given path.
|
||||
func (r MockResolver) RelativeFileByPath(_ Location, path string) *Location {
|
||||
paths, err := r.FilesByPath(path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(paths) < 1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &paths[0]
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package source
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
)
|
||||
@@ -14,13 +15,15 @@ type Resolver interface {
|
||||
|
||||
// ContentResolver knows how to get file content for given file.References
|
||||
type ContentResolver interface {
|
||||
FileContentsByLocation(Location) (string, error)
|
||||
MultipleFileContentsByLocation([]Location) (map[Location]string, error)
|
||||
// TODO: we should consider refactoring to return a set of io.Readers or file.Openers instead of the full contents themselves (allow for optional buffering).
|
||||
FileContentsByLocation(Location) (io.ReadCloser, error)
|
||||
// TODO: it is possible to be given duplicate locations that will be overridden in the map (key), a subtle problem that coule easily be misued.
|
||||
MultipleFileContentsByLocation([]Location) (map[Location]io.ReadCloser, error)
|
||||
}
|
||||
|
||||
// FileResolver knows how to get file.References for given string paths and globs
|
||||
// FileResolver knows how to get a Location for given string paths and globs
|
||||
type FileResolver interface {
|
||||
// HasPath indicates if the given path exists in the underlying source.
|
||||
HasPath(path string) bool
|
||||
// FilesByPath fetches a set of file references which have the given path (for an image, there may be multiple matches)
|
||||
FilesByPath(paths ...string) ([]Location, error)
|
||||
// FilesByGlob fetches a set of file references which the given glob matches
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package source
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
@@ -137,9 +138,14 @@ func TestMultipleFileContentsByLocation(t *testing.T) {
|
||||
location := locations[0]
|
||||
|
||||
contents, err := p.Resolver.MultipleFileContentsByLocation([]Location{location})
|
||||
content := contents[location]
|
||||
contentReader := contents[location]
|
||||
|
||||
if content != test.expected {
|
||||
content, err := ioutil.ReadAll(contentReader)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot read contents: %+v", err)
|
||||
}
|
||||
|
||||
if string(content) != test.expected {
|
||||
t.Errorf("unexpected contents from file: '%s' != '%s'", content, test.expected)
|
||||
}
|
||||
|
||||
|
||||
@@ -21,4 +21,12 @@ RUN ln -s ./link-2 link-indirect
|
||||
ADD new-file-2.txt file-2.txt
|
||||
|
||||
# LAYER 8: dead link
|
||||
RUN ln -s ./i-dont-exist.txt link-dead
|
||||
RUN ln -s ./i-dont-exist.txt link-dead
|
||||
|
||||
# LAYER 9: add the parent dir
|
||||
ADD parent /parent
|
||||
# LAYER 10: parent is a symlink
|
||||
RUN ln -s /parent parent-link
|
||||
|
||||
# LAYER 11: parent is a symlink and the child target is overridden
|
||||
COPY new-file-4.txt /parent-link/file-4.txt
|
||||
@@ -0,0 +1 @@
|
||||
override file 4!
|
||||
@@ -0,0 +1 @@
|
||||
the 4th file!
|
||||
@@ -16,7 +16,7 @@ func TestDistroImage(t *testing.T) {
|
||||
tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
|
||||
defer cleanup()
|
||||
|
||||
_, _, actualDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope)
|
||||
_, _, actualDistro, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to catalog image: %+v", err)
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ func TestCatalogFromJSON(t *testing.T) {
|
||||
tarPath := imagetest.GetFixtureImageTarPath(t, test.fixture)
|
||||
defer cleanup()
|
||||
|
||||
expectedSource, expectedCatalog, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope)
|
||||
expectedSource, expectedCatalog, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to catalog image: %+v", err)
|
||||
}
|
||||
@@ -81,6 +81,12 @@ func TestCatalogFromJSON(t *testing.T) {
|
||||
e.Metadata = metadata
|
||||
}
|
||||
|
||||
// ignore the virtual path on the location for now
|
||||
for l := range a.Locations {
|
||||
a.Locations[l].VirtualPath = ""
|
||||
e.Locations[l].VirtualPath = ""
|
||||
}
|
||||
|
||||
for _, d := range deep.Equal(a, e) {
|
||||
// ignore errors for empty collections vs nil for select fields
|
||||
// TODO: this is brittle, but not dangerously so. We should still find a better way to do this.
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package integration
|
||||
|
||||
import (
|
||||
"github.com/anchore/stereoscope/pkg/imagetest"
|
||||
"github.com/anchore/syft/syft"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestJavaNoMainPackage(t *testing.T) { // Regression: https://github.com/anchore/syft/issues/252
|
||||
fixtureImageName := "image-java-no-main-package"
|
||||
_, cleanup := imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName)
|
||||
tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
|
||||
defer cleanup()
|
||||
|
||||
_, _, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to catalog image: %+v", err)
|
||||
}
|
||||
}
|
||||
@@ -60,7 +60,7 @@ func TestJsonSchemaImg(t *testing.T) {
|
||||
tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
|
||||
defer cleanup()
|
||||
|
||||
src, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope)
|
||||
src, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to catalog image: %+v", err)
|
||||
}
|
||||
@@ -87,7 +87,7 @@ func TestJsonSchemaImg(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestJsonSchemaDirs(t *testing.T) {
|
||||
src, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope)
|
||||
src, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.SquashedScope)
|
||||
if err != nil {
|
||||
t.Errorf("unable to create source from dir: %+v", err)
|
||||
}
|
||||
|
||||
@@ -15,7 +15,10 @@ var imageOnlyTestCases = []testCase{
|
||||
pkgType: pkg.GemPkg,
|
||||
pkgLanguage: pkg.Ruby,
|
||||
pkgInfo: map[string]string{
|
||||
// specifications in the root specification directory
|
||||
"bundler": "2.1.4",
|
||||
// specifications in named directories
|
||||
"unbundler": "3.1.4",
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -18,7 +18,7 @@ func TestPkgCoverageImage(t *testing.T) {
|
||||
tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
|
||||
defer cleanup()
|
||||
|
||||
_, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope)
|
||||
_, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to catalog image: %+v", err)
|
||||
}
|
||||
@@ -100,7 +100,7 @@ func TestPkgCoverageImage(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPkgCoverageDirectory(t *testing.T) {
|
||||
_, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope)
|
||||
_, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.SquashedScope)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("unable to create source from dir: %+v", err)
|
||||
|
||||
@@ -24,7 +24,7 @@ func TestRegression212ApkBufferSize(t *testing.T) {
|
||||
t.Fatalf("failed to catalog image: %+v", err)
|
||||
}
|
||||
|
||||
expectedPkgs := 57
|
||||
expectedPkgs := 58
|
||||
actualPkgs := 0
|
||||
for range catalog.Enumerate(pkg.ApkPkg) {
|
||||
actualPkgs += 1
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
FROM jenkins:2.60.3
|
||||
|
||||
USER root
|
||||
|
||||
WORKDIR /usr/share/jenkins
|
||||
|
||||
RUN mkdir tmp
|
||||
|
||||
WORKDIR /usr/share/jenkins/tmp
|
||||
|
||||
RUN apt-get update 2>&1 > /dev/null && apt-get install -y less zip 2>&1 > /dev/null
|
||||
|
||||
RUN unzip ../jenkins.war 2>&1 > /dev/null
|
||||
|
||||
RUN rm -f ./META-INF/MANIFEST.MF
|
||||
|
||||
WORKDIR /usr/share/jenkins
|
||||
|
||||
RUN rm -rf jenkins.war
|
||||
|
||||
RUN cd ./tmp && zip -r ../jenkins.war . && cd ..
|
||||
|
||||
RUN rm -rf ./tmp
|
||||
@@ -1,2 +1,5 @@
|
||||
FROM alpine:latest
|
||||
RUN apk add tzdata vim alpine-sdk
|
||||
FROM alpine@sha256:d9a7354e3845ea8466bb00b22224d9116b183e594527fb5b6c3d30bc01a20378
|
||||
RUN apk add --no-cache \
|
||||
tzdata=2021a-r0 \
|
||||
vim=8.2.2320-r0 \
|
||||
alpine-sdk=1.0-r0
|
||||
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
# frozen_string_literal: true
|
||||
# -*- encoding: utf-8 -*-
|
||||
# stub: unbundler 2.1.4 ruby lib
|
||||
|
||||
Gem::Specification.new do |s|
|
||||
s.name = "unbundler".freeze
|
||||
s.version = "3.1.4"
|
||||
|
||||
s.required_rubygems_version = Gem::Requirement.new(">= 2.5.2".freeze) if s.respond_to? :required_rubygems_version=
|
||||
s.require_paths = ["lib".freeze]
|
||||
s.authors = ["Andr\u00E9 Arko".freeze, "Samuel Giddins".freeze, "Colby Swandale".freeze, "Hiroshi Shibata".freeze, "David Rodr\u00EDguez".freeze, "Grey Baker".f
|
||||
s.bindir = "exe".freeze
|
||||
s.date = "2020-01-05"
|
||||
s.description = "Bundler manages an application's dependencies through its entire life, across many machines, systematically and repeatably".freeze
|
||||
s.email = ["team@unbundler.io".freeze]
|
||||
s.executables = ["unbundle".freeze, "unbundler".freeze]
|
||||
s.files = ["exe/unbundle".freeze, "exe/unbundler".freeze]
|
||||
s.homepage = "https://unbundler.io".freeze
|
||||
s.licenses = ["MIT".freeze]
|
||||
s.required_ruby_version = Gem::Requirement.new(">= 2.3.0".freeze)
|
||||
s.rubygems_version = "3.1.2".freeze
|
||||
s.summary = "The best way to manage your application's dependencies".freeze
|
||||
|
||||
s.installed_by_version = "3.1.2" if s.respond_to? :installed_by_version
|
||||
end
|
||||
Reference in New Issue
Block a user