From b69259534d5af990686bb976cb2a65ea99abbcc6 Mon Sep 17 00:00:00 2001 From: Avi Deitcher Date: Fri, 14 Apr 2023 22:13:29 +0300 Subject: [PATCH] feat: Support scanning license files in golang packages over the network (#1630) Signed-off-by: Avi Deitcher Signed-off-by: Keith Zantow Signed-off-by: Alex Goodman Co-authored-by: Keith Zantow Co-authored-by: Alex Goodman --- README.md | 14 + go.mod | 17 +- go.sum | 77 +- internal/config/application.go | 10 +- internal/config/golang.go | 6 + syft/event/cataloger_task.go | 52 ++ syft/event/event.go | 3 + syft/event/parsers/parsers.go | 13 + syft/pkg/cataloger/golang/billy_adapter.go | 87 ++ .../cataloger/golang/billy_adapter_test.go | 33 + syft/pkg/cataloger/golang/cataloger.go | 41 +- syft/pkg/cataloger/golang/licenses.go | 250 ++++-- syft/pkg/cataloger/golang/licenses_test.go | 128 ++- syft/pkg/cataloger/golang/options.go | 114 +++ syft/pkg/cataloger/golang/options_test.go | 99 +++ .../cataloger/golang/parse_go_binary_test.go | 2 +- syft/pkg/cataloger/golang/subfs.go | 117 +++ syft/pkg/cataloger/golang/subfs_test.go | 27 + .../golang/test-fixtures/repo/LICENSE | 7 + .../github.com/someorg/somepkg@version/a-file | 1 + .../someorg/somepkg@version/subdir/subfile | 1 + syft/source/directory_resolver_test.go | 9 +- syft/source/empty_resolver.go | 45 ++ syft/source/file_resolver.go | 6 + syft/source/mock_resolver.go | 4 + syft/source/unindexed_directory_resolver.go | 553 +++++++++++++ .../unindexed_directory_resolver_test.go | 744 ++++++++++++++++++ ui/event_handlers.go | 111 ++- ui/handler.go | 6 +- 29 files changed, 2475 insertions(+), 102 deletions(-) create mode 100644 syft/event/cataloger_task.go create mode 100644 syft/pkg/cataloger/golang/billy_adapter.go create mode 100644 syft/pkg/cataloger/golang/billy_adapter_test.go create mode 100644 syft/pkg/cataloger/golang/options.go create mode 100644 syft/pkg/cataloger/golang/options_test.go create mode 100644 syft/pkg/cataloger/golang/subfs.go create mode 100644 syft/pkg/cataloger/golang/subfs_test.go create mode 100644 syft/pkg/cataloger/golang/test-fixtures/repo/LICENSE create mode 100644 syft/pkg/cataloger/golang/test-fixtures/zip-fs/github.com/someorg/somepkg@version/a-file create mode 100644 syft/pkg/cataloger/golang/test-fixtures/zip-fs/github.com/someorg/somepkg@version/subdir/subfile create mode 100644 syft/source/empty_resolver.go create mode 100644 syft/source/unindexed_directory_resolver.go create mode 100644 syft/source/unindexed_directory_resolver_test.go diff --git a/README.md b/README.md index 7dff47bcf..f00f0979d 100644 --- a/README.md +++ b/README.md @@ -515,6 +515,20 @@ golang: # SYFT_GOLANG_LOCAL_MOD_CACHE_DIR env var local-mod-cache-dir: "" + # search for go package licences by retrieving the package from a network proxy + # SYFT_GOLANG_SEARCH_REMOTE_LICENSES env var + search-remote-licenses: false + + # remote proxy to use when retrieving go packages from the network, + # if unset this defaults to $GOPROXY followed by https://proxy.golang.org + # SYFT_GOLANG_PROXY env var + proxy: "" + + # specifies packages which should not be fetched by proxy + # if unset this defaults to $GONOPROXY + # SYFT_GOLANG_NOPROXY env var + no-proxy: "" + linux-kernel: # whether to catalog linux kernel modules found within lib/modules/** directories # SYFT_LINUX_KERNEL_CATALOG_MODULES env var diff --git a/go.mod b/go.mod index b902f0bd7..9069ceaf1 100644 --- a/go.mod +++ b/go.mod @@ -55,6 +55,8 @@ require ( github.com/anchore/stereoscope v0.0.0-20230406143206-e95d60a265e3 github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da github.com/docker/docker v23.0.3+incompatible + github.com/go-git/go-billy/v5 v5.4.1 + github.com/go-git/go-git/v5 v5.6.1 github.com/google/go-containerregistry v0.14.0 github.com/google/licensecheck v0.3.1 github.com/invopop/jsonschema v0.7.0 @@ -71,8 +73,11 @@ require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/Microsoft/go-winio v0.6.0 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect + github.com/acomagu/bufpipe v1.0.4 // indirect github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect github.com/becheran/wildmatch-go v1.0.0 // indirect + github.com/cloudflare/circl v1.1.0 // indirect github.com/containerd/containerd v1.6.18 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -82,8 +87,10 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect + github.com/emirpasic/gods v1.18.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gabriel-vasile/mimetype v1.4.0 // indirect + github.com/go-git/gcfg v1.5.0 // indirect github.com/go-restruct/restruct v1.2.0-alpha // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -92,9 +99,11 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/huandu/xstrings v1.3.3 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect - github.com/imdario/mergo v0.3.12 // indirect + github.com/imdario/mergo v0.3.13 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.16.0 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 // indirect @@ -109,11 +118,13 @@ require ( github.com/opencontainers/image-spec v1.1.0-rc2 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/rogpeppe/go-internal v1.8.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect + github.com/skeema/knownhosts v1.1.0 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/stretchr/objx v0.5.0 // indirect @@ -123,6 +134,7 @@ require ( github.com/therootcompany/xz v1.0.1 // indirect github.com/ulikunitz/xz v0.5.10 // indirect github.com/vbatts/tar-split v0.11.2 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect @@ -136,6 +148,7 @@ require ( google.golang.org/grpc v1.52.0 // indirect google.golang.org/protobuf v1.29.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect lukechampine.com/uint128 v1.1.1 // indirect modernc.org/cc/v3 v3.36.0 // indirect modernc.org/ccgo/v3 v3.16.6 // indirect @@ -153,7 +166,7 @@ require ( // go: warning: github.com/andybalholm/brotli@v1.0.1: retracted by module author: occasional panics and data corruption github.com/andybalholm/brotli v1.0.4 // indirect github.com/pkg/errors v0.9.1 // indirect - golang.org/x/crypto v0.3.0 // indirect + golang.org/x/crypto v0.6.0 // indirect ) retract ( diff --git a/go.sum b/go.sum index b2ce2a707..3e4d9ca6f 100644 --- a/go.sum +++ b/go.sum @@ -65,13 +65,18 @@ github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7Y github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= +github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/acobaugh/osrelease v0.1.0 h1:Yb59HQDGGNhCj4suHaFQQfBps5wyoKLSSX/J/+UifRE= github.com/acobaugh/osrelease v0.1.0/go.mod h1:4bFEs0MtgHNHBrmHCt67gNisnabCRAlzdVasCEGHTWY= +github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= +github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -96,12 +101,16 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/becheran/wildmatch-go v1.0.0 h1:mE3dGGkTmpKtT4Z+88t8RStG40yN9T+kFEGj2PZFSzA= github.com/becheran/wildmatch-go v1.0.0/go.mod h1:gbMvj0NtVdJ15Mg/mH9uxk2R1QCistMyU7d9KFzroX4= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -111,6 +120,7 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bmatcuk/doublestar/v4 v4.6.0 h1:HTuxyug8GyFbRkrffIpzNCSK4luc0TY3wzXvzIZhEXc= github.com/bmatcuk/doublestar/v4 v4.6.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -122,6 +132,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -141,6 +153,7 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -166,6 +179,8 @@ github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdf github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -192,6 +207,17 @@ github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbS github.com/gabriel-vasile/mimetype v1.4.0 h1:Cn9dkdYsMIu56tGho+fqzh7XmvY2YyGU0FnbhiOsEro= github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= +github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= +github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4= +github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= +github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ= +github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= +github.com/go-git/go-git/v5 v5.6.1 h1:q4ZRqQl4pR/ZJHc1L5CFjGA1a10u76aV1iC+nh+bHsk= +github.com/go-git/go-git/v5 v5.6.1/go.mod h1:mvyoL6Unz0PiTQrGQfSfiLFhBH1c1e84ylC2MDs4ee8= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -343,13 +369,16 @@ github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/invopop/jsonschema v0.7.0 h1:2vgQcBz1n256N+FpX3Jq7Y17AjYt46Ig3zIWyy770So= github.com/invopop/jsonschema v0.7.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -362,6 +391,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -379,16 +410,20 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= +github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -437,6 +472,7 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -446,6 +482,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -464,6 +501,8 @@ github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= +github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -517,6 +556,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/skeema/knownhosts v1.1.0 h1:Wvr9V0MxhjRbl3f9nMnKnFfiWTJmtECJ9Njkea3ysW0= +github.com/skeema/knownhosts v1.1.0/go.mod h1:sKFq3RD6/TKZkSWn8boUbDC7Qkgcv+8XXijpFO6roag= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spdx/gordf v0.0.0-20201111095634-7098f93598fb/go.mod h1:uKWaldnbMnjsSAXRurWqqrdyZen1R7kxl8TkmWk2OyM= github.com/spdx/tools-golang v0.5.0 h1:/fqihV2Jna7fmow65dHpgKNsilgLK7ICpd2tkCnPEyY= @@ -584,6 +625,8 @@ github.com/wagoodman/go-progress v0.0.0-20230301185719-21920a456ad5 h1:lwgTsTy18 github.com/wagoodman/go-progress v0.0.0-20230301185719-21920a456ad5/go.mod h1:jLXFoL31zFaHKAAyZUh+sxiTDFe1L1ZHrcK2T1itVKA= github.com/wagoodman/jotframe v0.0.0-20211129225309-56b0d0a4aebb h1:Yz6VVOcLuWLAHYlJzTw7JKnWxdV/WXpug2X0quEzRnY= github.com/wagoodman/jotframe v0.0.0-20211129225309-56b0d0a4aebb/go.mod h1:nDi3BAC5nEbVbg+WSJDHLbjHv0ZToq8nMPA97XMxF3E= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -617,6 +660,7 @@ go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -631,9 +675,14 @@ golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= +golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -674,6 +723,7 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -721,7 +771,11 @@ golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -798,6 +852,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -826,16 +881,24 @@ golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -848,6 +911,7 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -913,6 +977,7 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1076,11 +1141,15 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1092,6 +1161,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= @@ -1139,6 +1209,7 @@ modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/z v1.5.1 h1:RTNHdsrOpeoSeOF4FbzTo8gBYByaJ5xT7NgZ9ZqRiJM= modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/internal/config/application.go b/internal/config/application.go index 98dec82c8..1a37594f3 100644 --- a/internal/config/application.go +++ b/internal/config/application.go @@ -74,10 +74,12 @@ func (cfg Application) ToCatalogerConfig() cataloger.Config { }, Catalogers: cfg.Catalogers, Parallelism: cfg.Parallelism, - Golang: golangCataloger.GoCatalogerOpts{ - SearchLocalModCacheLicenses: cfg.Golang.SearchLocalModCacheLicenses, - LocalModCacheDir: cfg.Golang.LocalModCacheDir, - }, + Golang: golangCataloger.NewGoCatalogerOpts(). + WithSearchLocalModCacheLicenses(cfg.Golang.SearchLocalModCacheLicenses). + WithLocalModCacheDir(cfg.Golang.LocalModCacheDir). + WithSearchRemoteLicenses(cfg.Golang.SearchRemoteLicenses). + WithProxy(cfg.Golang.Proxy). + WithNoProxy(cfg.Golang.NoProxy), LinuxKernel: kernel.LinuxCatalogerConfig{ CatalogModules: cfg.LinuxKernel.CatalogModules, }, diff --git a/internal/config/golang.go b/internal/config/golang.go index 29ab13db1..56ebbed08 100644 --- a/internal/config/golang.go +++ b/internal/config/golang.go @@ -5,9 +5,15 @@ import "github.com/spf13/viper" type golang struct { SearchLocalModCacheLicenses bool `json:"search-local-mod-cache-licenses" yaml:"search-local-mod-cache-licenses" mapstructure:"search-local-mod-cache-licenses"` LocalModCacheDir string `json:"local-mod-cache-dir" yaml:"local-mod-cache-dir" mapstructure:"local-mod-cache-dir"` + SearchRemoteLicenses bool `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"` + Proxy string `json:"proxy" yaml:"proxy" mapstructure:"proxy"` + NoProxy string `json:"no-proxy" yaml:"no-proxy" mapstructure:"no-proxy"` } func (cfg golang) loadDefaultValues(v *viper.Viper) { v.SetDefault("golang.search-local-mod-cache-licenses", false) v.SetDefault("golang.local-mod-cache-dir", "") + v.SetDefault("golang.search-remote-licenses", false) + v.SetDefault("golang.proxy", "") + v.SetDefault("golang.no-proxy", "") } diff --git a/syft/event/cataloger_task.go b/syft/event/cataloger_task.go new file mode 100644 index 000000000..49fa1cdee --- /dev/null +++ b/syft/event/cataloger_task.go @@ -0,0 +1,52 @@ +package event + +import ( + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + + "github.com/anchore/syft/internal/bus" +) + +type CatalogerTask struct { + prog *progress.Manual + // Title + Title string + // TitleOnCompletion a string to use as title when completed + TitleOnCompletion string + // SubStatus indicates this progress should be rendered as a sub-item + SubStatus bool + // RemoveOnCompletion indicates this progress line will be removed when completed + RemoveOnCompletion bool + // value is the value to display -- not public as SetValue needs to be called to initialize this progress + value string +} + +func (e *CatalogerTask) init() { + e.prog = progress.NewManual(-1) + + bus.Publish(partybus.Event{ + Type: CatalogerTaskStarted, + Source: e, + }) +} + +func (e *CatalogerTask) SetCompleted() { + if e.prog != nil { + e.prog.SetCompleted() + } +} + +func (e *CatalogerTask) SetValue(value string) { + if e.prog == nil { + e.init() + } + e.value = value +} + +func (e *CatalogerTask) GetValue() string { + return e.value +} + +func (e *CatalogerTask) GetMonitor() *progress.Manual { + return e.prog +} diff --git a/syft/event/event.go b/syft/event/event.go index 621fa716d..1e9e6f400 100644 --- a/syft/event/event.go +++ b/syft/event/event.go @@ -34,4 +34,7 @@ const ( // AttestationStarted is a partybus event that occurs when starting an SBOM attestation process AttestationStarted partybus.EventType = "syft-attestation-started-event" + + // CatalogerTaskStarted is a partybus event that occurs when starting a task within a cataloger + CatalogerTaskStarted partybus.EventType = "syft-cataloger-task-started" ) diff --git a/syft/event/parsers/parsers.go b/syft/event/parsers/parsers.go index 630038428..e7a3d703d 100644 --- a/syft/event/parsers/parsers.go +++ b/syft/event/parsers/parsers.go @@ -111,6 +111,19 @@ func ParseFileIndexingStarted(e partybus.Event) (string, progress.StagedProgress return path, prog, nil } +func ParseCatalogerTaskStarted(e partybus.Event) (*event.CatalogerTask, error) { + if err := checkEventType(e.Type, event.CatalogerTaskStarted); err != nil { + return nil, err + } + + source, ok := e.Source.(*event.CatalogerTask) + if !ok { + return nil, newPayloadErr(e.Type, "Source", e.Source) + } + + return source, nil +} + func ParseExit(e partybus.Event) (func() error, error) { if err := checkEventType(e.Type, event.Exit); err != nil { return nil, err diff --git a/syft/pkg/cataloger/golang/billy_adapter.go b/syft/pkg/cataloger/golang/billy_adapter.go new file mode 100644 index 000000000..e570cdb0b --- /dev/null +++ b/syft/pkg/cataloger/golang/billy_adapter.go @@ -0,0 +1,87 @@ +package golang + +import ( + "io/fs" + "os" + + "github.com/go-git/go-billy/v5" +) + +// billyFSAdapter is a fs.FS, fs.ReadDirFS, and fs.StatFS wrapping what billyfs returns +type billyFSAdapter struct { + fs billy.Filesystem +} + +func (b billyFSAdapter) Stat(name string) (fs.FileInfo, error) { + return b.fs.Stat(name) +} + +func (b billyFSAdapter) ReadDir(name string) (out []fs.DirEntry, _ error) { + entries, err := b.fs.ReadDir(name) + if err != nil { + return nil, err + } + for _, e := range entries { + out = append(out, billyDirEntry{fi: e}) + } + return +} + +func (b billyFSAdapter) Open(name string) (fs.File, error) { + f, err := b.fs.Open(name) + if err != nil { + return nil, err + } + fi, err := b.Stat(name) + if err != nil { + return nil, err + } + return billyFile{f: f, fi: fi}, nil +} + +var _ fs.FS = (*billyFSAdapter)(nil) +var _ fs.ReadDirFS = (*billyFSAdapter)(nil) +var _ fs.StatFS = (*billyFSAdapter)(nil) + +// billyFile is a fs.File wrapping what billyfs returns +type billyFile struct { + f billy.File + fi fs.FileInfo +} + +func (b billyFile) Stat() (fs.FileInfo, error) { + return b.fi, nil +} + +func (b billyFile) Read(i []byte) (int, error) { + return b.f.Read(i) +} + +func (b billyFile) Close() error { + return b.f.Close() +} + +var _ fs.File = (*billyFile)(nil) + +// billyDirEntry is a fs.DirEntry wrapping what billyfs returns +type billyDirEntry struct { + fi os.FileInfo +} + +func (b billyDirEntry) Name() string { + return b.fi.Name() +} + +func (b billyDirEntry) IsDir() bool { + return b.fi.IsDir() +} + +func (b billyDirEntry) Type() fs.FileMode { + return b.fi.Mode() +} + +func (b billyDirEntry) Info() (fs.FileInfo, error) { + return b.fi, nil +} + +var _ fs.DirEntry = (*billyDirEntry)(nil) diff --git a/syft/pkg/cataloger/golang/billy_adapter_test.go b/syft/pkg/cataloger/golang/billy_adapter_test.go new file mode 100644 index 000000000..f684de9cf --- /dev/null +++ b/syft/pkg/cataloger/golang/billy_adapter_test.go @@ -0,0 +1,33 @@ +package golang + +import ( + "io/fs" + "os" + "testing" + + "github.com/go-git/go-git/v5" + "github.com/stretchr/testify/require" +) + +func Test_billyFSAdapter(t *testing.T) { + r, err := git.PlainInit("test-fixtures/repo", false) + + t.Cleanup(func() { + _ = os.RemoveAll("test-fixtures/repo/.git") + }) + + wt, err := r.Worktree() + require.NoError(t, err) + f := billyFSAdapter{ + fs: wt.Filesystem, + } + + found := "" + err = fs.WalkDir(f, ".", func(path string, d fs.DirEntry, err error) error { + found = path + return nil + }) + require.NoError(t, err) + + require.Equal(t, "LICENSE", found) +} diff --git a/syft/pkg/cataloger/golang/cataloger.go b/syft/pkg/cataloger/golang/cataloger.go index 2e64ad451..d28a9ed9a 100644 --- a/syft/pkg/cataloger/golang/cataloger.go +++ b/syft/pkg/cataloger/golang/cataloger.go @@ -5,28 +5,47 @@ package golang import ( "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/source" ) -type GoCatalogerOpts struct { - SearchLocalModCacheLicenses bool - LocalModCacheDir string -} - // NewGoModFileCataloger returns a new Go module cataloger object. -func NewGoModFileCataloger(opts GoCatalogerOpts) *generic.Cataloger { +func NewGoModFileCataloger(opts GoCatalogerOpts) pkg.Cataloger { c := goModCataloger{ licenses: newGoLicenses(opts), } - return generic.NewCataloger("go-mod-file-cataloger"). - WithParserByGlobs(c.parseGoModFile, "**/go.mod") + return &progressingCataloger{ + progress: c.licenses.progress, + cataloger: generic.NewCataloger("go-mod-file-cataloger"). + WithParserByGlobs(c.parseGoModFile, "**/go.mod"), + } } // NewGoModuleBinaryCataloger returns a new Golang cataloger object. -func NewGoModuleBinaryCataloger(opts GoCatalogerOpts) *generic.Cataloger { +func NewGoModuleBinaryCataloger(opts GoCatalogerOpts) pkg.Cataloger { c := goBinaryCataloger{ licenses: newGoLicenses(opts), } - return generic.NewCataloger("go-module-binary-cataloger"). - WithParserByMimeTypes(c.parseGoBinary, internal.ExecutableMIMETypeSet.List()...) + return &progressingCataloger{ + progress: c.licenses.progress, + cataloger: generic.NewCataloger("go-module-binary-cataloger"). + WithParserByMimeTypes(c.parseGoBinary, internal.ExecutableMIMETypeSet.List()...), + } +} + +type progressingCataloger struct { + progress *event.CatalogerTask + cataloger *generic.Cataloger +} + +func (p *progressingCataloger) Name() string { + return p.cataloger.Name() +} + +func (p *progressingCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { + defer p.progress.SetCompleted() + return p.cataloger.Catalog(resolver) } diff --git a/syft/pkg/cataloger/golang/licenses.go b/syft/pkg/cataloger/golang/licenses.go index 9d74e8c0e..bfffb9f7f 100644 --- a/syft/pkg/cataloger/golang/licenses.go +++ b/syft/pkg/cataloger/golang/licenses.go @@ -1,105 +1,158 @@ package golang import ( + "archive/zip" + "bytes" "fmt" + "io" + "io/fs" + "net/http" + "net/url" "os" "path" + "path/filepath" "regexp" "strings" - "github.com/mitchellh/go-homedir" + "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/storage/memory" "github.com/anchore/syft/internal/licenses" "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/source" ) type goLicenses struct { - searchLocalModCacheLicenses bool - localModCacheResolver source.FileResolver + opts GoCatalogerOpts + localModCacheResolver source.WritableFileResolver + progress *event.CatalogerTask } func newGoLicenses(opts GoCatalogerOpts) goLicenses { return goLicenses{ - searchLocalModCacheLicenses: opts.SearchLocalModCacheLicenses, - localModCacheResolver: modCacheResolver(opts.LocalModCacheDir), + opts: opts, + localModCacheResolver: modCacheResolver(opts.localModCacheDir), + progress: &event.CatalogerTask{ + SubStatus: true, + RemoveOnCompletion: true, + Title: "Downloading go mod", + }, } } -func defaultGoPath() string { - goPath := os.Getenv("GOPATH") - - if goPath == "" { - homeDir, err := homedir.Dir() - if err != nil { - log.Debug("unable to determine user home dir: %v", err) - } else { - goPath = path.Join(homeDir, "go") +func remotesForModule(proxies []string, noProxy []string, module string) []string { + for _, pattern := range noProxy { + if matched, err := path.Match(pattern, module); err == nil && matched { + // matched to be direct for this module + return directProxiesOnly } } - return goPath + return proxies } -// resolver needs to be shared between mod file & binary scanners so it's only scanned once -var modCacheResolvers = map[string]source.FileResolver{} - -func modCacheResolver(modCacheDir string) source.FileResolver { - if modCacheDir == "" { - goPath := defaultGoPath() - if goPath != "" { - modCacheDir = path.Join(goPath, "pkg", "mod") - } - } - - if r, ok := modCacheResolvers[modCacheDir]; ok { - return r - } - - var r source.FileResolver +func modCacheResolver(modCacheDir string) source.WritableFileResolver { + var r source.WritableFileResolver if modCacheDir == "" { log.Trace("unable to determine mod cache directory, skipping mod cache resolver") - r = source.NewMockResolverForPaths() + r = source.EmptyResolver{} } else { stat, err := os.Stat(modCacheDir) if os.IsNotExist(err) || stat == nil || !stat.IsDir() { log.Tracef("unable to open mod cache directory: %s, skipping mod cache resolver", modCacheDir) - r = source.NewMockResolverForPaths() + r = source.EmptyResolver{} } else { - r = source.NewDeferredResolverFromSource(func() (source.Source, error) { - return source.NewFromDirectory(modCacheDir) - }) + r = source.NewUnindexedDirectoryResolver(modCacheDir) } } - modCacheResolvers[modCacheDir] = r - return r } func (c *goLicenses) getLicenses(resolver source.FileResolver, moduleName, moduleVersion string) (licenses []string, err error) { - moduleName = processCaps(moduleName) - licenses, err = findLicenses(resolver, - fmt.Sprintf(`**/go/pkg/mod/%s@%s/*`, moduleName, moduleVersion), + fmt.Sprintf(`**/go/pkg/mod/%s@%s/*`, processCaps(moduleName), moduleVersion), ) - - if c.searchLocalModCacheLicenses && err == nil && len(licenses) == 0 { - // if we're running against a directory on the filesystem, it may not include the - // user's homedir / GOPATH, so we defer to using the localModCacheResolver - licenses, err = findLicenses(c.localModCacheResolver, - fmt.Sprintf(`**/%s@%s/*`, moduleName, moduleVersion), - ) + if err != nil || len(licenses) > 0 { + return requireCollection(licenses), err } - // always return a non-nil slice + // look in the local host mod cache... + licenses, err = c.getLicensesFromLocal(moduleName, moduleVersion) + if err != nil || len(licenses) > 0 { + return requireCollection(licenses), err + } + + // we did not find it yet and remote searching was enabled + licenses, err = c.getLicensesFromRemote(moduleName, moduleVersion) + return requireCollection(licenses), err +} + +func (c *goLicenses) getLicensesFromLocal(moduleName, moduleVersion string) ([]string, error) { + if !c.opts.searchLocalModCacheLicenses { + return nil, nil + } + + // if we're running against a directory on the filesystem, it may not include the + // user's homedir / GOPATH, so we defer to using the localModCacheResolver + return findLicenses(c.localModCacheResolver, moduleSearchGlob(moduleName, moduleVersion)) +} + +func (c *goLicenses) getLicensesFromRemote(moduleName, moduleVersion string) ([]string, error) { + if !c.opts.searchRemoteLicenses { + return nil, nil + } + + proxies := remotesForModule(c.opts.proxies, c.opts.noProxy, moduleName) + + fsys, err := getModule(c.progress, proxies, moduleName, moduleVersion) + if err != nil { + return nil, err + } + + dir := moduleDir(moduleName, moduleVersion) + + // populate the mod cache with the results + err = fs.WalkDir(fsys, ".", func(filePath string, d fs.DirEntry, err error) error { + if err != nil { + log.Debug(err) + return nil + } + if d.IsDir() { + return nil + } + f, err := fsys.Open(filePath) + if err != nil { + return err + } + return c.localModCacheResolver.Write(source.NewLocation(path.Join(dir, filePath)), f) + }) + + if err != nil { + log.Tracef("remote proxy walk failed for: %s", moduleName) + } + + return findLicenses(c.localModCacheResolver, moduleSearchGlob(moduleName, moduleVersion)) +} + +func moduleDir(moduleName, moduleVersion string) string { + return fmt.Sprintf("%s@%s", processCaps(moduleName), moduleVersion) +} + +func moduleSearchGlob(moduleName, moduleVersion string) string { + return fmt.Sprintf("%s/*", moduleDir(moduleName, moduleVersion)) +} + +func requireCollection(licenses []string) []string { if licenses == nil { - licenses = []string{} + return []string{} } - - return + return licenses } func findLicenses(resolver source.FileResolver, globMatch string) (out []string, err error) { @@ -138,3 +191,96 @@ func processCaps(s string) string { return "!" + strings.ToLower(s) }) } + +func getModule(progress *event.CatalogerTask, proxies []string, moduleName, moduleVersion string) (fsys fs.FS, err error) { + for _, proxy := range proxies { + u, _ := url.Parse(proxy) + if proxy == "direct" { + fsys, err = getModuleRepository(progress, moduleName, moduleVersion) + continue + } + switch u.Scheme { + case "https", "http": + fsys, err = getModuleProxy(progress, proxy, moduleName, moduleVersion) + case "file": + p := filepath.Join(u.Path, moduleName, "@v", moduleVersion) + progress.SetValue(fmt.Sprintf("file: %s", p)) + fsys = os.DirFS(p) + } + if fsys != nil { + break + } + } + return +} + +func getModuleProxy(progress *event.CatalogerTask, proxy string, moduleName string, moduleVersion string) (out fs.FS, _ error) { + u := fmt.Sprintf("%s/%s/@v/%s.zip", proxy, moduleName, moduleVersion) + progress.SetValue(u) + // get the module zip + resp, err := http.Get(u) //nolint:gosec + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + u = fmt.Sprintf("%s/%s/@v/%s.zip", proxy, strings.ToLower(moduleName), moduleVersion) + progress.SetValue(u) + // try lowercasing it; some packages have mixed casing that really messes up the proxy + resp, err = http.Get(u) //nolint:gosec + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get module zip: %s", resp.Status) + } + } + // read the zip + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + out, err = zip.NewReader(bytes.NewReader(b), resp.ContentLength) + versionPath := findVersionPath(out, ".") + out = getSubFS(out, versionPath) + return out, err +} + +func findVersionPath(f fs.FS, dir string) string { + list, _ := fs.ReadDir(f, dir) + for _, entry := range list { + name := entry.Name() + if strings.Contains(name, "@") { + return name + } + found := findVersionPath(f, path.Join(dir, name)) + if found != "" { + return path.Join(name, found) + } + } + return "" +} + +func getModuleRepository(progress *event.CatalogerTask, moduleName string, moduleVersion string) (fs.FS, error) { + repoName := moduleName + parts := strings.Split(moduleName, "/") + if len(parts) > 2 { + repoName = fmt.Sprintf("%s/%s/%s", parts[0], parts[1], parts[2]) + } + progress.SetValue(fmt.Sprintf("git: %s", repoName)) + f := memfs.New() + buf := &bytes.Buffer{} + _, err := git.Clone(memory.NewStorage(), f, &git.CloneOptions{ + URL: fmt.Sprintf("https://%s", repoName), + ReferenceName: plumbing.NewTagReferenceName(moduleVersion), // FIXME version might be a SHA + SingleBranch: true, + Depth: 1, + Progress: buf, + }) + if err != nil { + return nil, fmt.Errorf("%w -- %s", err, buf.String()) + } + + return billyFSAdapter{fs: f}, nil +} diff --git a/syft/pkg/cataloger/golang/licenses_test.go b/syft/pkg/cataloger/golang/licenses_test.go index 4003fe310..fa361004a 100644 --- a/syft/pkg/cataloger/golang/licenses_test.go +++ b/syft/pkg/cataloger/golang/licenses_test.go @@ -1,8 +1,14 @@ package golang import ( + "archive/zip" + "bytes" + "fmt" + "net/http" + "net/http/httptest" "os" "path" + "strings" "testing" "github.com/stretchr/testify/require" @@ -10,7 +16,7 @@ import ( "github.com/anchore/syft/syft/source" ) -func Test_LicenseSearch(t *testing.T) { +func Test_LocalLicenseSearch(t *testing.T) { tests := []struct { name string version string @@ -34,10 +40,85 @@ func Test_LicenseSearch(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { l := newGoLicenses(GoCatalogerOpts{ - SearchLocalModCacheLicenses: true, - LocalModCacheDir: path.Join(wd, "test-fixtures", "licenses"), + searchLocalModCacheLicenses: true, + localModCacheDir: path.Join(wd, "test-fixtures", "licenses", "pkg", "mod"), }) - licenses, err := l.getLicenses(source.MockResolver{}, test.name, test.version) + licenses, err := l.getLicenses(source.EmptyResolver{}, test.name, test.version) + require.NoError(t, err) + + require.Len(t, licenses, 1) + + require.Equal(t, test.expected, licenses[0]) + }) + } +} + +func Test_RemoteProxyLicenseSearch(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + buf := &bytes.Buffer{} + uri := strings.TrimPrefix(strings.TrimSuffix(r.RequestURI, ".zip"), "/") + + parts := strings.Split(uri, "/@v/") + modPath := parts[0] + modVersion := parts[1] + + wd, err := os.Getwd() + require.NoError(t, err) + testDir := path.Join(wd, "test-fixtures", "licenses", "pkg", "mod", processCaps(modPath)+"@"+modVersion) + + archive := zip.NewWriter(buf) + + entries, err := os.ReadDir(testDir) + require.NoError(t, err) + for _, f := range entries { + // the zip files downloaded contain a path to the repo that somewhat matches where it ends up on disk, + // so prefix entries with something similar + writer, err := archive.Create(path.Join("github.com/something/some@version", f.Name())) + require.NoError(t, err) + contents, err := os.ReadFile(path.Join(testDir, f.Name())) + require.NoError(t, err) + _, err = writer.Write(contents) + require.NoError(t, err) + } + + err = archive.Close() + require.NoError(t, err) + + w.Header().Add("Content-Length", fmt.Sprintf("%d", buf.Len())) + + _, err = w.Write(buf.Bytes()) + require.NoError(t, err) + })) + defer server.Close() + + tests := []struct { + name string + version string + expected string + }{ + { + name: "github.com/someorg/somename", + version: "v0.3.2", + expected: "Apache-2.0", + }, + { + name: "github.com/CapORG/CapProject", + version: "v4.111.5", + expected: "MIT", + }, + } + + modDir := path.Join(t.TempDir()) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + l := newGoLicenses(GoCatalogerOpts{ + searchRemoteLicenses: true, + proxies: []string{server.URL}, + localModCacheDir: modDir, + }) + + licenses, err := l.getLicenses(source.EmptyResolver{}, test.name, test.version) require.NoError(t, err) require.Len(t, licenses, 1) @@ -74,3 +155,42 @@ func Test_processCaps(t *testing.T) { }) } } + +func Test_remotesForModule(t *testing.T) { + allProxies := []string{"https://somewhere.org", "direct"} + directProxy := []string{"direct"} + + tests := []struct { + module string + noProxy string + expected []string + }{ + { + module: "github.com/anchore/syft", + expected: allProxies, + }, + { + module: "github.com/anchore/sbom-action", + noProxy: "*/anchore/*", + expected: directProxy, + }, + { + module: "github.com/anchore/sbom-action", + noProxy: "*/user/mod,*/anchore/sbom-action", + expected: directProxy, + }, + } + + for _, test := range tests { + t.Run(test.module, func(t *testing.T) { + got := remotesForModule(allProxies, strings.Split(test.noProxy, ","), test.module) + require.Equal(t, test.expected, got) + }) + } +} + +func Test_findVersionPath(t *testing.T) { + f := os.DirFS("test-fixtures/zip-fs") + vp := findVersionPath(f, ".") + require.Equal(t, "github.com/someorg/somepkg@version", vp) +} diff --git a/syft/pkg/cataloger/golang/options.go b/syft/pkg/cataloger/golang/options.go new file mode 100644 index 000000000..ee602d0a3 --- /dev/null +++ b/syft/pkg/cataloger/golang/options.go @@ -0,0 +1,114 @@ +package golang + +import ( + "os" + "path" + "strings" + + "github.com/mitchellh/go-homedir" + + "github.com/anchore/syft/internal/log" +) + +const ( + defaultProxies = "https://proxy.golang.org,direct" + directProxyOnly = "direct" +) + +var ( + directProxiesOnly = []string{directProxyOnly} +) + +type GoCatalogerOpts struct { + searchLocalModCacheLicenses bool + localModCacheDir string + searchRemoteLicenses bool + proxies []string + noProxy []string +} + +func (g GoCatalogerOpts) WithSearchLocalModCacheLicenses(input bool) GoCatalogerOpts { + g.searchLocalModCacheLicenses = input + return g +} + +func (g GoCatalogerOpts) WithLocalModCacheDir(input string) GoCatalogerOpts { + if input == "" { + return g + } + g.localModCacheDir = input + return g +} + +func (g GoCatalogerOpts) WithSearchRemoteLicenses(input bool) GoCatalogerOpts { + g.searchRemoteLicenses = input + return g +} + +func (g GoCatalogerOpts) WithProxy(input string) GoCatalogerOpts { + if input == "" { + return g + } + if input == "off" { + input = directProxyOnly + } + g.proxies = strings.Split(input, ",") + return g +} + +func (g GoCatalogerOpts) WithNoProxy(input string) GoCatalogerOpts { + if input == "" { + return g + } + g.noProxy = strings.Split(input, ",") + return g +} + +// NewGoCatalogerOpts create a GoCatalogerOpts with default options, which includes: +// - setting the default remote proxy if none is provided +// - setting the default no proxy if none is provided +// - setting the default local module cache dir if none is provided +func NewGoCatalogerOpts() GoCatalogerOpts { + g := GoCatalogerOpts{} + + // first process the proxy settings + if len(g.proxies) == 0 { + goProxy := os.Getenv("GOPROXY") + if goProxy == "" { + goProxy = defaultProxies + } + g = g.WithProxy(goProxy) + } + + // next process the gonoproxy settings + if len(g.noProxy) == 0 { + goPrivate := os.Getenv("GOPRIVATE") + goNoProxy := os.Getenv("GONOPROXY") + // we only use the env var if it was not set explicitly + if goPrivate != "" { + g.noProxy = append(g.noProxy, strings.Split(goPrivate, ",")...) + } + + // next process the goprivate settings; we always add those + if goNoProxy != "" { + g.noProxy = append(g.noProxy, strings.Split(goNoProxy, ",")...) + } + } + + if g.localModCacheDir == "" { + goPath := os.Getenv("GOPATH") + + if goPath == "" { + homeDir, err := homedir.Dir() + if err != nil { + log.Debug("unable to determine user home dir: %v", err) + } else { + goPath = path.Join(homeDir, "go") + } + } + if goPath != "" { + g.localModCacheDir = path.Join(goPath, "pkg", "mod") + } + } + return g +} diff --git a/syft/pkg/cataloger/golang/options_test.go b/syft/pkg/cataloger/golang/options_test.go new file mode 100644 index 000000000..a5f710c63 --- /dev/null +++ b/syft/pkg/cataloger/golang/options_test.go @@ -0,0 +1,99 @@ +package golang + +import ( + "testing" + + "github.com/mitchellh/go-homedir" + "github.com/stretchr/testify/assert" +) + +func Test_Options(t *testing.T) { + type opts struct { + local bool + cacheDir string + remote bool + proxy string + noProxy string + } + + homedirCacheDisabled := homedir.DisableCache + homedir.DisableCache = true + t.Cleanup(func() { + homedir.DisableCache = homedirCacheDisabled + }) + + allEnv := map[string]string{ + "HOME": "/usr/home", + "GOPATH": "", + "GOPROXY": "", + "GOPRIVATE": "", + "GONOPROXY": "", + } + + tests := []struct { + name string + env map[string]string + opts opts + expected GoCatalogerOpts + }{ + { + name: "set via env defaults", + env: map[string]string{ + "GOPATH": "/go", + "GOPROXY": "https://my.proxy", + "GOPRIVATE": "my.private", + "GONOPROXY": "no.proxy", + }, + opts: opts{}, + expected: GoCatalogerOpts{ + searchLocalModCacheLicenses: false, + localModCacheDir: "/go/pkg/mod", + searchRemoteLicenses: false, + proxies: []string{"https://my.proxy"}, + noProxy: []string{"my.private", "no.proxy"}, + }, + }, + { + name: "set via configuration", + env: map[string]string{ + "GOPATH": "/go", + "GOPROXY": "https://my.proxy", + "GOPRIVATE": "my.private", + "GONOPROXY": "no.proxy", + }, + opts: opts{ + local: true, + cacheDir: "/go-cache", + remote: true, + proxy: "https://alt.proxy,direct", + noProxy: "alt.no.proxy", + }, + expected: GoCatalogerOpts{ + searchLocalModCacheLicenses: true, + localModCacheDir: "/go-cache", + searchRemoteLicenses: true, + proxies: []string{"https://alt.proxy", "direct"}, + noProxy: []string{"alt.no.proxy"}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + for k, v := range allEnv { + t.Setenv(k, v) + } + for k, v := range test.env { + t.Setenv(k, v) + } + got := NewGoCatalogerOpts(). + WithSearchLocalModCacheLicenses(test.opts.local). + WithLocalModCacheDir(test.opts.cacheDir). + WithSearchRemoteLicenses(test.opts.remote). + WithProxy(test.opts.proxy). + WithNoProxy(test.opts.noProxy) + + assert.Equal(t, test.expected, got) + }) + } +} diff --git a/syft/pkg/cataloger/golang/parse_go_binary_test.go b/syft/pkg/cataloger/golang/parse_go_binary_test.go index c9d78a2ea..4fe87c2ec 100644 --- a/syft/pkg/cataloger/golang/parse_go_binary_test.go +++ b/syft/pkg/cataloger/golang/parse_go_binary_test.go @@ -510,7 +510,7 @@ func TestBuildGoPkgInfo(t *testing.T) { ) c := goBinaryCataloger{} - pkgs := c.buildGoPkgInfo(source.NewMockResolverForPaths(), location, test.mod, test.arch) + pkgs := c.buildGoPkgInfo(source.EmptyResolver{}, location, test.mod, test.arch) assert.Equal(t, test.expected, pkgs) }) } diff --git a/syft/pkg/cataloger/golang/subfs.go b/syft/pkg/cataloger/golang/subfs.go new file mode 100644 index 000000000..a30d30b31 --- /dev/null +++ b/syft/pkg/cataloger/golang/subfs.go @@ -0,0 +1,117 @@ +package golang + +import ( + "io/fs" + "path" + "time" +) + +type subFS struct { + root string + f fs.FS +} + +func getSubFS(f fs.FS, root string) fs.FS { + if s, ok := f.(fs.SubFS); ok { + s, err := s.Sub(root) + if err != nil { + return s + } + } + + return newSubFS(f, root) +} + +func newSubFS(f fs.FS, root string) fs.FS { + return subFS{ + root: root, + f: f, + } +} + +func (s subFS) Open(name string) (fs.File, error) { + if name == "." { + return rootFile{ + s: s, + }, nil + } + return s.f.Open(path.Join(s.root, name)) +} + +type rootFile struct { + s subFS +} + +func (r rootFile) Name() string { + return "." +} + +func (r rootFile) Size() int64 { + return 0 +} + +func (r rootFile) Mode() fs.FileMode { + return fs.ModePerm +} + +func (r rootFile) ModTime() time.Time { + return time.Now() +} + +func (r rootFile) IsDir() bool { + return true +} + +func (r rootFile) Sys() any { + return nil +} + +func (r rootFile) ReadDir(_ int) ([]fs.DirEntry, error) { + return fs.ReadDir(r.s.f, r.s.root) +} + +func (r rootFile) Stat() (fs.FileInfo, error) { + return r, nil +} + +func (r rootFile) Read(_ []byte) (int, error) { + panic("not implemented") +} + +func (r rootFile) Close() error { + return nil +} + +var _ fs.File = (*rootFile)(nil) +var _ fs.ReadDirFile = (*rootFile)(nil) +var _ fs.FileInfo = (*rootFile)(nil) + +type subFsFileInfo struct { + fi fs.FileInfo +} + +func (s subFsFileInfo) Name() string { + return s.fi.Name() +} + +func (s subFsFileInfo) Size() int64 { + return s.fi.Size() +} + +func (s subFsFileInfo) Mode() fs.FileMode { + return s.fi.Mode() +} + +func (s subFsFileInfo) ModTime() time.Time { + return s.fi.ModTime() +} + +func (s subFsFileInfo) IsDir() bool { + return s.fi.IsDir() +} + +func (s subFsFileInfo) Sys() any { + return s.fi.Sys() +} + +var _ fs.FileInfo = (*subFsFileInfo)(nil) diff --git a/syft/pkg/cataloger/golang/subfs_test.go b/syft/pkg/cataloger/golang/subfs_test.go new file mode 100644 index 000000000..a940f66f8 --- /dev/null +++ b/syft/pkg/cataloger/golang/subfs_test.go @@ -0,0 +1,27 @@ +package golang + +import ( + "io/fs" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_NewSubFS(t *testing.T) { + f := os.DirFS("test-fixtures/zip-fs") + f = newSubFS(f, "github.com/someorg/somepkg@version") + var names []string + err := fs.WalkDir(f, ".", func(path string, d fs.DirEntry, err error) error { + names = append(names, path) + return nil + }) + require.NoError(t, err) + expected := []string{ + ".", + "a-file", + "subdir", + "subdir/subfile", + } + require.Equal(t, expected, names) +} diff --git a/syft/pkg/cataloger/golang/test-fixtures/repo/LICENSE b/syft/pkg/cataloger/golang/test-fixtures/repo/LICENSE new file mode 100644 index 000000000..6622914d4 --- /dev/null +++ b/syft/pkg/cataloger/golang/test-fixtures/repo/LICENSE @@ -0,0 +1,7 @@ +Copyright (c) 2022 +Acme, inc. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +THIS SOFTWARE IS PROVIDED BY [Name of Organization] “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL [Name of Organisation] BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/syft/pkg/cataloger/golang/test-fixtures/zip-fs/github.com/someorg/somepkg@version/a-file b/syft/pkg/cataloger/golang/test-fixtures/zip-fs/github.com/someorg/somepkg@version/a-file new file mode 100644 index 000000000..aeef42651 --- /dev/null +++ b/syft/pkg/cataloger/golang/test-fixtures/zip-fs/github.com/someorg/somepkg@version/a-file @@ -0,0 +1 @@ +a file \ No newline at end of file diff --git a/syft/pkg/cataloger/golang/test-fixtures/zip-fs/github.com/someorg/somepkg@version/subdir/subfile b/syft/pkg/cataloger/golang/test-fixtures/zip-fs/github.com/someorg/somepkg@version/subdir/subfile new file mode 100644 index 000000000..cdcb28483 --- /dev/null +++ b/syft/pkg/cataloger/golang/test-fixtures/zip-fs/github.com/someorg/somepkg@version/subdir/subfile @@ -0,0 +1 @@ +another file \ No newline at end of file diff --git a/syft/source/directory_resolver_test.go b/syft/source/directory_resolver_test.go index c976fc343..0a752ba41 100644 --- a/syft/source/directory_resolver_test.go +++ b/syft/source/directory_resolver_test.go @@ -923,16 +923,19 @@ func TestDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) { resolver, err := newDirectoryResolver("./test-fixtures/symlinks-prune-indexing", "") require.NoError(t, err) - allRealPaths := resolver.tree.AllRealPaths() + var allRealPaths []file.Path + for l := range resolver.AllLocations() { + allRealPaths = append(allRealPaths, file.Path(l.RealPath)) + } pathSet := file.NewPathSet(allRealPaths...) assert.False(t, - pathSet.Contains("/before-path/file.txt"), + pathSet.Contains("before-path/file.txt"), "symlink destinations should only be indexed at their real path, not through their virtual (symlinked) path", ) assert.False(t, - pathSet.Contains("/a-path/file.txt"), + pathSet.Contains("a-path/file.txt"), "symlink destinations should only be indexed at their real path, not through their virtual (symlinked) path", ) diff --git a/syft/source/empty_resolver.go b/syft/source/empty_resolver.go new file mode 100644 index 000000000..72c9331dd --- /dev/null +++ b/syft/source/empty_resolver.go @@ -0,0 +1,45 @@ +package source + +import ( + "io" +) + +type EmptyResolver struct{} + +func (e EmptyResolver) FileContentsByLocation(_ Location) (io.ReadCloser, error) { + return nil, nil +} + +func (e EmptyResolver) HasPath(_ string) bool { + return false +} + +func (e EmptyResolver) FilesByPath(_ ...string) ([]Location, error) { + return nil, nil +} + +func (e EmptyResolver) FilesByGlob(_ ...string) ([]Location, error) { + return nil, nil +} + +func (e EmptyResolver) FilesByMIMEType(_ ...string) ([]Location, error) { + return nil, nil +} + +func (e EmptyResolver) RelativeFileByPath(_ Location, _ string) *Location { + return nil +} + +func (e EmptyResolver) AllLocations() <-chan Location { + return nil +} + +func (e EmptyResolver) FileMetadataByLocation(_ Location) (FileMetadata, error) { + return FileMetadata{}, nil +} + +func (e EmptyResolver) Write(_ Location, _ io.Reader) error { + return nil +} + +var _ WritableFileResolver = (*EmptyResolver)(nil) diff --git a/syft/source/file_resolver.go b/syft/source/file_resolver.go index 63b5dc90b..414be6373 100644 --- a/syft/source/file_resolver.go +++ b/syft/source/file_resolver.go @@ -57,3 +57,9 @@ type FileLocationResolver interface { // - returns locations for any file or directory AllLocations() <-chan Location } + +type WritableFileResolver interface { + FileResolver + + Write(location Location, reader io.Reader) error +} diff --git a/syft/source/mock_resolver.go b/syft/source/mock_resolver.go index 89960d6bd..74cffac34 100644 --- a/syft/source/mock_resolver.go +++ b/syft/source/mock_resolver.go @@ -204,3 +204,7 @@ func (r MockResolver) FilesByBasenameGlob(_ ...string) ([]Location, error) { // TODO implement me panic("implement me") } + +func (r MockResolver) Write(_ Location, _ io.Reader) error { + return nil +} diff --git a/syft/source/unindexed_directory_resolver.go b/syft/source/unindexed_directory_resolver.go new file mode 100644 index 000000000..e0b74d2ed --- /dev/null +++ b/syft/source/unindexed_directory_resolver.go @@ -0,0 +1,553 @@ +package source + +import ( + "fmt" + "io" + "io/fs" + "os" + "path" + "sort" + "strings" + "time" + + "github.com/bmatcuk/doublestar/v4" + "github.com/mitchellh/go-homedir" + "github.com/spf13/afero" + "golang.org/x/exp/slices" + + "github.com/anchore/syft/internal/log" +) + +type UnindexedDirectoryResolver struct { + ls afero.Lstater + lr afero.LinkReader + base string + dir string + fs afero.Fs +} + +func NewUnindexedDirectoryResolver(dir string) WritableFileResolver { + return NewUnindexedDirectoryResolverFS(afero.NewOsFs(), dir, "") +} + +func NewUnindexedDirectoryResolverRooted(dir string, base string) WritableFileResolver { + return NewUnindexedDirectoryResolverFS(afero.NewOsFs(), dir, base) +} + +func NewUnindexedDirectoryResolverFS(fs afero.Fs, dir string, base string) WritableFileResolver { + ls, ok := fs.(afero.Lstater) + if !ok { + panic(fmt.Sprintf("unable to get afero.Lstater interface from: %+v", fs)) + } + lr, ok := fs.(afero.LinkReader) + if !ok { + panic(fmt.Sprintf("unable to get afero.Lstater interface from: %+v", fs)) + } + expanded, err := homedir.Expand(dir) + if err == nil { + dir = expanded + } + if base != "" { + expanded, err = homedir.Expand(base) + if err == nil { + base = expanded + } + } + wd, err := os.Getwd() + if err == nil { + if !path.IsAbs(dir) { + dir = path.Clean(path.Join(wd, dir)) + } + if base != "" && !path.IsAbs(base) { + base = path.Clean(path.Join(wd, base)) + } + } + return UnindexedDirectoryResolver{ + base: base, + dir: dir, + fs: fs, + ls: ls, + lr: lr, + } +} + +func (u UnindexedDirectoryResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) { + p := u.absPath(u.scrubInputPath(location.RealPath)) + f, err := u.fs.Open(p) + if err != nil { + return nil, err + } + fi, err := f.Stat() + if err != nil { + return nil, err + } + if fi.IsDir() { + return nil, fmt.Errorf("unable to get contents of directory: %s", location.RealPath) + } + return f, nil +} + +// - full symlink resolution should be performed on all requests +// - returns locations for any file or directory +func (u UnindexedDirectoryResolver) HasPath(p string) bool { + locs, err := u.filesByPath(true, true, p) + return err == nil && len(locs) > 0 +} + +func (u UnindexedDirectoryResolver) canLstat(p string) bool { + _, _, err := u.ls.LstatIfPossible(u.absPath(p)) + return err == nil +} + +func (u UnindexedDirectoryResolver) isRegularFile(p string) bool { + fi, _, err := u.ls.LstatIfPossible(u.absPath(p)) + return err == nil && !fi.IsDir() +} + +func (u UnindexedDirectoryResolver) scrubInputPath(p string) string { + if path.IsAbs(p) { + p = p[1:] + } + return path.Clean(p) +} + +func (u UnindexedDirectoryResolver) scrubResolutionPath(p string) string { + if u.base != "" { + if path.IsAbs(p) { + p = p[1:] + } + for strings.HasPrefix(p, "../") { + p = p[3:] + } + } + return path.Clean(p) +} + +func (u UnindexedDirectoryResolver) absPath(p string) string { + if u.base != "" { + if path.IsAbs(p) { + p = p[1:] + } + for strings.HasPrefix(p, "../") { + p = p[3:] + } + p = path.Join(u.base, p) + return path.Clean(p) + } + if path.IsAbs(p) { + return p + } + return path.Clean(path.Join(u.dir, p)) +} + +// - full symlink resolution should be performed on all requests +// - only returns locations to files (NOT directories) +func (u UnindexedDirectoryResolver) FilesByPath(paths ...string) (out []Location, _ error) { + return u.filesByPath(true, false, paths...) +} + +func (u UnindexedDirectoryResolver) filesByPath(resolveLinks bool, includeDirs bool, paths ...string) (out []Location, _ error) { + // sort here for stable output + sort.Strings(paths) +nextPath: + for _, p := range paths { + p = u.scrubInputPath(p) + if u.canLstat(p) && (includeDirs || u.isRegularFile(p)) { + l := u.newLocation(p, resolveLinks) + if l == nil { + continue + } + // only include the first entry we find + for i := range out { + existing := &out[i] + if existing.RealPath == l.RealPath { + if l.VirtualPath == "" { + existing.VirtualPath = "" + } + continue nextPath + } + } + out = append(out, *l) + } + } + return +} + +// - full symlink resolution should be performed on all requests +// - if multiple paths to the same file are found, the best single match should be returned +// - only returns locations to files (NOT directories) +func (u UnindexedDirectoryResolver) FilesByGlob(patterns ...string) (out []Location, _ error) { + return u.filesByGlob(true, false, patterns...) +} + +func (u UnindexedDirectoryResolver) filesByGlob(resolveLinks bool, includeDirs bool, patterns ...string) (out []Location, _ error) { + f := unindexedDirectoryResolverFS{ + u: u, + } + var paths []string + for _, p := range patterns { + opts := []doublestar.GlobOption{doublestar.WithNoFollow()} + if !includeDirs { + opts = append(opts, doublestar.WithFilesOnly()) + } + found, err := doublestar.Glob(f, p, opts...) + if err != nil { + return nil, err + } + paths = append(paths, found...) + } + return u.filesByPath(resolveLinks, includeDirs, paths...) +} + +func (u UnindexedDirectoryResolver) FilesByMIMEType(_ ...string) ([]Location, error) { + panic("FilesByMIMEType unsupported") +} + +// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. +// This is helpful when attempting to find a file that is in the same layer or lower as another file. +func (u UnindexedDirectoryResolver) RelativeFileByPath(l Location, p string) *Location { + p = path.Clean(path.Join(l.RealPath, p)) + locs, err := u.filesByPath(true, false, p) + if err != nil || len(locs) == 0 { + return nil + } + l = locs[0] + p = l.RealPath + if u.isRegularFile(p) { + return u.newLocation(p, true) + } + return nil +} + +// - NO symlink resolution should be performed on results +// - returns locations for any file or directory +func (u UnindexedDirectoryResolver) AllLocations() <-chan Location { + out := make(chan Location) + go func() { + defer close(out) + err := afero.Walk(u.fs, u.absPath("."), func(p string, info fs.FileInfo, err error) error { + p = strings.TrimPrefix(p, u.dir) + if p == "" { + return nil + } + p = strings.TrimPrefix(p, "/") + out <- Location{ + LocationData: LocationData{ + Coordinates: Coordinates{ + RealPath: p, + }, + }, + } + return nil + }) + if err != nil { + log.Debug(err) + } + }() + return out +} + +func (u UnindexedDirectoryResolver) FileMetadataByLocation(_ Location) (FileMetadata, error) { + panic("FileMetadataByLocation unsupported") +} + +func (u UnindexedDirectoryResolver) Write(location Location, reader io.Reader) error { + filePath := location.RealPath + if path.IsAbs(filePath) { + filePath = filePath[1:] + } + absPath := u.absPath(filePath) + return afero.WriteReader(u.fs, absPath, reader) +} + +var _ FileResolver = (*UnindexedDirectoryResolver)(nil) +var _ WritableFileResolver = (*UnindexedDirectoryResolver)(nil) + +func (u UnindexedDirectoryResolver) newLocation(filePath string, resolveLinks bool) *Location { + filePath = path.Clean(filePath) + + virtualPath := "" + realPath := filePath + + if resolveLinks { + paths := u.resolveLinks(filePath) + if len(paths) > 1 { + realPath = paths[len(paths)-1] + if realPath != path.Clean(filePath) { + virtualPath = paths[0] + } + } + if len(paths) == 0 { + // this file does not exist, don't return a location + return nil + } + } + + return &Location{ + LocationData: LocationData{ + Coordinates: Coordinates{ + RealPath: realPath, + }, + VirtualPath: virtualPath, + }, + } +} + +//nolint:gocognit +func (u UnindexedDirectoryResolver) resolveLinks(filePath string) []string { + var visited []string + + out := []string{} + + resolvedPath := "" + + parts := strings.Split(filePath, "/") + for i := 0; i < len(parts); i++ { + part := parts[i] + if resolvedPath == "" { + resolvedPath = part + } else { + resolvedPath = path.Clean(path.Join(resolvedPath, part)) + } + resolvedPath = u.scrubResolutionPath(resolvedPath) + if resolvedPath == ".." { + resolvedPath = "" + continue + } + + absPath := u.absPath(resolvedPath) + if slices.Contains(visited, absPath) { + return nil // circular links can't resolve + } + visited = append(visited, absPath) + + fi, wasLstat, err := u.ls.LstatIfPossible(absPath) + if fi == nil || err != nil { + // this file does not exist + return nil + } + + for wasLstat && u.isSymlink(fi) { + next, err := u.lr.ReadlinkIfPossible(absPath) + if err == nil { + if !path.IsAbs(next) { + next = path.Clean(path.Join(path.Dir(resolvedPath), next)) + } + next = u.scrubResolutionPath(next) + absPath = u.absPath(next) + if slices.Contains(visited, absPath) { + return nil // circular links can't resolve + } + visited = append(visited, absPath) + + fi, wasLstat, err = u.ls.LstatIfPossible(absPath) + if fi == nil || err != nil { + // this file does not exist + return nil + } + if i < len(parts) { + out = append(out, path.Join(resolvedPath, path.Join(parts[i+1:]...))) + } + if u.base != "" && path.IsAbs(next) { + next = next[1:] + } + resolvedPath = next + } + } + } + + out = append(out, resolvedPath) + + return out +} + +func (u UnindexedDirectoryResolver) isSymlink(fi os.FileInfo) bool { + return fi.Mode().Type()&fs.ModeSymlink == fs.ModeSymlink +} + +// ------------------------- fs.FS ------------------------------ + +// unindexedDirectoryResolverFS wraps the UnindexedDirectoryResolver as a fs.FS, fs.ReadDirFS, and fs.StatFS +type unindexedDirectoryResolverFS struct { + u UnindexedDirectoryResolver +} + +// resolve takes a virtual path and returns the resolved absolute or relative path and file info +func (f unindexedDirectoryResolverFS) resolve(filePath string) (resolved string, fi fs.FileInfo, err error) { + parts := strings.Split(filePath, "/") + var visited []string + for i, part := range parts { + if i > 0 { + resolved = path.Clean(path.Join(resolved, part)) + } else { + resolved = part + } + abs := f.u.absPath(resolved) + fi, _, err = f.u.ls.LstatIfPossible(abs) + if err != nil { + return + } + for f.u.isSymlink(fi) { + if slices.Contains(visited, resolved) { + return resolved, fi, fmt.Errorf("link cycle detected at: %s", f.u.absPath(resolved)) + } + visited = append(visited, resolved) + link, err := f.u.lr.ReadlinkIfPossible(abs) + if err != nil { + return resolved, fi, err + } + if !path.IsAbs(link) { + link = path.Clean(path.Join(path.Dir(abs), link)) + link = strings.TrimPrefix(link, abs) + } else if f.u.base != "" { + link = path.Clean(path.Join(f.u.base, link[1:])) + } + resolved = link + abs = f.u.absPath(resolved) + fi, _, err = f.u.ls.LstatIfPossible(abs) + if err != nil { + return resolved, fi, err + } + } + } + return resolved, fi, err +} + +func (f unindexedDirectoryResolverFS) ReadDir(name string) (out []fs.DirEntry, _ error) { + p, _, err := f.resolve(name) + if err != nil { + return nil, err + } + entries, err := afero.ReadDir(f.u.fs, f.u.absPath(p)) + if err != nil { + return nil, err + } + for _, e := range entries { + isDir := e.IsDir() + _, fi, _ := f.resolve(path.Join(name, e.Name())) + if fi != nil && fi.IsDir() { + isDir = true + } + out = append(out, unindexedDirectoryResolverDirEntry{ + unindexedDirectoryResolverFileInfo: newFsFileInfo(f.u, e.Name(), isDir, e), + }) + } + return out, nil +} + +func (f unindexedDirectoryResolverFS) Stat(name string) (fs.FileInfo, error) { + fi, err := f.u.fs.Stat(f.u.absPath(name)) + if err != nil { + return nil, err + } + return newFsFileInfo(f.u, name, fi.IsDir(), fi), nil +} + +func (f unindexedDirectoryResolverFS) Open(name string) (fs.File, error) { + _, err := f.u.fs.Open(f.u.absPath(name)) + if err != nil { + return nil, err + } + + return unindexedDirectoryResolverFile{ + u: f.u, + path: name, + }, nil +} + +var _ fs.FS = (*unindexedDirectoryResolverFS)(nil) +var _ fs.StatFS = (*unindexedDirectoryResolverFS)(nil) +var _ fs.ReadDirFS = (*unindexedDirectoryResolverFS)(nil) + +type unindexedDirectoryResolverDirEntry struct { + unindexedDirectoryResolverFileInfo +} + +func (f unindexedDirectoryResolverDirEntry) Name() string { + return f.name +} + +func (f unindexedDirectoryResolverDirEntry) IsDir() bool { + return f.isDir +} + +func (f unindexedDirectoryResolverDirEntry) Type() fs.FileMode { + return f.mode +} + +func (f unindexedDirectoryResolverDirEntry) Info() (fs.FileInfo, error) { + return f, nil +} + +var _ fs.DirEntry = (*unindexedDirectoryResolverDirEntry)(nil) + +type unindexedDirectoryResolverFile struct { + u UnindexedDirectoryResolver + path string +} + +func (f unindexedDirectoryResolverFile) Stat() (fs.FileInfo, error) { + fi, err := f.u.fs.Stat(f.u.absPath(f.path)) + if err != nil { + return nil, err + } + return newFsFileInfo(f.u, fi.Name(), fi.IsDir(), fi), nil +} + +func (f unindexedDirectoryResolverFile) Read(_ []byte) (int, error) { + panic("Read not implemented") +} + +func (f unindexedDirectoryResolverFile) Close() error { + panic("Close not implemented") +} + +var _ fs.File = (*unindexedDirectoryResolverFile)(nil) + +type unindexedDirectoryResolverFileInfo struct { + u UnindexedDirectoryResolver + name string + size int64 + mode fs.FileMode + modTime time.Time + isDir bool + sys any +} + +func newFsFileInfo(u UnindexedDirectoryResolver, name string, isDir bool, fi os.FileInfo) unindexedDirectoryResolverFileInfo { + return unindexedDirectoryResolverFileInfo{ + u: u, + name: name, + size: fi.Size(), + mode: fi.Mode() & ^fs.ModeSymlink, // pretend nothing is a symlink + modTime: fi.ModTime(), + isDir: isDir, + // sys: fi.Sys(), // what values does this hold? + } +} + +func (f unindexedDirectoryResolverFileInfo) Name() string { + return f.name +} + +func (f unindexedDirectoryResolverFileInfo) Size() int64 { + return f.size +} + +func (f unindexedDirectoryResolverFileInfo) Mode() fs.FileMode { + return f.mode +} + +func (f unindexedDirectoryResolverFileInfo) ModTime() time.Time { + return f.modTime +} + +func (f unindexedDirectoryResolverFileInfo) IsDir() bool { + return f.isDir +} + +func (f unindexedDirectoryResolverFileInfo) Sys() any { + return f.sys +} + +var _ fs.FileInfo = (*unindexedDirectoryResolverFileInfo)(nil) diff --git a/syft/source/unindexed_directory_resolver_test.go b/syft/source/unindexed_directory_resolver_test.go new file mode 100644 index 000000000..f6b158671 --- /dev/null +++ b/syft/source/unindexed_directory_resolver_test.go @@ -0,0 +1,744 @@ +//go:build !windows +// +build !windows + +package source + +import ( + "io" + "os" + "path" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/scylladb/go-set/strset" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/stereoscope/pkg/file" +) + +func Test_UnindexedDirectoryResolver_Basic(t *testing.T) { + wd, err := os.Getwd() + require.NoError(t, err) + + r := NewUnindexedDirectoryResolver(path.Join(wd, "test-fixtures")) + locations, err := r.FilesByGlob("image-symlinks/*") + require.NoError(t, err) + require.Len(t, locations, 5) +} + +func Test_UnindexedDirectoryResolver_FilesByPath_relativeRoot(t *testing.T) { + cases := []struct { + name string + relativeRoot string + input string + expected []string + }{ + { + name: "should find a file from an absolute input", + relativeRoot: "./test-fixtures/", + input: "/image-symlinks/file-1.txt", + expected: []string{ + "image-symlinks/file-1.txt", + }, + }, + { + name: "should find a file from a relative path", + relativeRoot: "./test-fixtures/", + input: "image-symlinks/file-1.txt", + expected: []string{ + "image-symlinks/file-1.txt", + }, + }, + { + name: "should find a file from a relative path (root above cwd)", + relativeRoot: "../", + input: "sbom/sbom.go", + expected: []string{ + "sbom/sbom.go", + }, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + resolver := NewUnindexedDirectoryResolver(c.relativeRoot) + + refs, err := resolver.FilesByPath(c.input) + require.NoError(t, err) + assert.Len(t, refs, len(c.expected)) + s := strset.New() + for _, actual := range refs { + s.Add(actual.RealPath) + } + assert.ElementsMatch(t, c.expected, s.List()) + }) + } +} + +func Test_UnindexedDirectoryResolver_FilesByPath_absoluteRoot(t *testing.T) { + cases := []struct { + name string + relativeRoot string + input string + expected []string + }{ + { + name: "should find a file from an absolute input", + relativeRoot: "./test-fixtures/", + input: "/image-symlinks/file-1.txt", + expected: []string{ + "image-symlinks/file-1.txt", + }, + }, + { + name: "should find a file from a relative path", + relativeRoot: "./test-fixtures/", + input: "image-symlinks/file-1.txt", + expected: []string{ + "image-symlinks/file-1.txt", + }, + }, + { + name: "should find a file from a relative path (root above cwd)", + relativeRoot: "../", + input: "sbom/sbom.go", + expected: []string{ + "sbom/sbom.go", + }, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + // note: this test is all about asserting correct functionality when the given analysis path + // is an absolute path + absRoot, err := filepath.Abs(c.relativeRoot) + require.NoError(t, err) + + resolver := NewUnindexedDirectoryResolver(absRoot) + assert.NoError(t, err) + + refs, err := resolver.FilesByPath(c.input) + require.NoError(t, err) + assert.Len(t, refs, len(c.expected)) + s := strset.New() + for _, actual := range refs { + s.Add(actual.RealPath) + } + assert.ElementsMatch(t, c.expected, s.List()) + }) + } +} + +func Test_UnindexedDirectoryResolver_FilesByPath(t *testing.T) { + cases := []struct { + name string + root string + input string + expected string + refCount int + forcePositiveHasPath bool + }{ + { + name: "finds a file (relative)", + root: "./test-fixtures/", + input: "image-symlinks/file-1.txt", + expected: "image-symlinks/file-1.txt", + refCount: 1, + }, + { + name: "finds a file with relative indirection", + root: "./test-fixtures/../test-fixtures", + input: "image-symlinks/file-1.txt", + expected: "image-symlinks/file-1.txt", + refCount: 1, + }, + { + name: "managed non-existing files (relative)", + root: "./test-fixtures/", + input: "test-fixtures/image-symlinks/bogus.txt", + refCount: 0, + }, + { + name: "finds a file (absolute)", + root: "./test-fixtures/", + input: "/image-symlinks/file-1.txt", + expected: "image-symlinks/file-1.txt", + refCount: 1, + }, + { + name: "directories ignored", + root: "./test-fixtures/", + input: "/image-symlinks", + refCount: 0, + forcePositiveHasPath: true, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + resolver := NewUnindexedDirectoryResolver(c.root) + + hasPath := resolver.HasPath(c.input) + if !c.forcePositiveHasPath { + if c.refCount != 0 && !hasPath { + t.Errorf("expected HasPath() to indicate existence, but did not") + } else if c.refCount == 0 && hasPath { + t.Errorf("expected HasPath() to NOT indicate existence, but does") + } + } else if !hasPath { + t.Errorf("expected HasPath() to indicate existence, but did not (force path)") + } + + refs, err := resolver.FilesByPath(c.input) + require.NoError(t, err) + assert.Len(t, refs, c.refCount) + for _, actual := range refs { + assert.Equal(t, c.expected, actual.RealPath) + } + }) + } +} + +func Test_UnindexedDirectoryResolver_MultipleFilesByPath(t *testing.T) { + cases := []struct { + name string + input []string + refCount int + }{ + { + name: "finds multiple files", + input: []string{"image-symlinks/file-1.txt", "image-symlinks/file-2.txt"}, + refCount: 2, + }, + { + name: "skips non-existing files", + input: []string{"image-symlinks/bogus.txt", "image-symlinks/file-1.txt"}, + refCount: 1, + }, + { + name: "does not return anything for non-existing directories", + input: []string{"non-existing/bogus.txt", "non-existing/file-1.txt"}, + refCount: 0, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + resolver := NewUnindexedDirectoryResolver("./test-fixtures") + refs, err := resolver.FilesByPath(c.input...) + assert.NoError(t, err) + + if len(refs) != c.refCount { + t.Errorf("unexpected number of refs: %d != %d", len(refs), c.refCount) + } + }) + } +} + +func Test_UnindexedDirectoryResolver_FilesByGlobMultiple(t *testing.T) { + resolver := NewUnindexedDirectoryResolver("./test-fixtures") + refs, err := resolver.FilesByGlob("**/image-symlinks/file*") + assert.NoError(t, err) + + assert.Len(t, refs, 2) +} + +func Test_UnindexedDirectoryResolver_FilesByGlobRecursive(t *testing.T) { + resolver := NewUnindexedDirectoryResolver("./test-fixtures/image-symlinks") + refs, err := resolver.FilesByGlob("**/*.txt") + assert.NoError(t, err) + assert.Len(t, refs, 6) +} + +func Test_UnindexedDirectoryResolver_FilesByGlobSingle(t *testing.T) { + resolver := NewUnindexedDirectoryResolver("./test-fixtures") + refs, err := resolver.FilesByGlob("**/image-symlinks/*1.txt") + assert.NoError(t, err) + + assert.Len(t, refs, 1) + assert.Equal(t, "image-symlinks/file-1.txt", refs[0].RealPath) +} + +func Test_UnindexedDirectoryResolver_FilesByPath_ResolvesSymlinks(t *testing.T) { + + tests := []struct { + name string + fixture string + }{ + { + name: "one degree", + fixture: "link_to_new_readme", + }, + { + name: "two degrees", + fixture: "link_to_link_to_new_readme", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + resolver := NewUnindexedDirectoryResolver("./test-fixtures/symlinks-simple") + + refs, err := resolver.FilesByPath(test.fixture) + require.NoError(t, err) + require.Len(t, refs, 1) + + reader, err := resolver.FileContentsByLocation(refs[0]) + require.NoError(t, err) + + actual, err := io.ReadAll(reader) + require.NoError(t, err) + + expected, err := os.ReadFile("test-fixtures/symlinks-simple/readme") + require.NoError(t, err) + + require.Equal(t, string(expected), string(actual)) + }) + } +} + +func Test_UnindexedDirectoryResolverDoesNotIgnoreRelativeSystemPaths(t *testing.T) { + // let's make certain that "dev/place" is not ignored, since it is not "/dev/place" + resolver := NewUnindexedDirectoryResolver("test-fixtures/system_paths/target") + + // all paths should be found (non filtering matches a path) + locations, err := resolver.FilesByGlob("**/place") + assert.NoError(t, err) + // 4: within target/ + // 1: target/link --> relative path to "place" // NOTE: this is filtered out since it not unique relative to outside_root/link_target/place + // 1: outside_root/link_target/place + assert.Len(t, locations, 5) + + // ensure that symlink indexing outside of root worked + testLocation := "../outside_root/link_target/place" + ok := false + for _, location := range locations { + if strings.HasSuffix(location.RealPath, testLocation) { + ok = true + } + } + + if !ok { + t.Fatalf("could not find test location=%q", testLocation) + } +} + +func Test_UnindexedDirectoryResover_IndexingNestedSymLinks(t *testing.T) { + resolver := NewUnindexedDirectoryResolver("./test-fixtures/symlinks-simple") + + // check that we can get the real path + locations, err := resolver.FilesByPath("./readme") + require.NoError(t, err) + assert.Len(t, locations, 1) + + // check that we can access the same file via 1 symlink + locations, err = resolver.FilesByPath("./link_to_new_readme") + require.NoError(t, err) + require.Len(t, locations, 1) + assert.Equal(t, "readme", locations[0].RealPath) + assert.Equal(t, "link_to_new_readme", locations[0].VirtualPath) + + // check that we can access the same file via 2 symlinks + locations, err = resolver.FilesByPath("./link_to_link_to_new_readme") + require.NoError(t, err) + require.Len(t, locations, 1) + assert.Equal(t, "readme", locations[0].RealPath) + assert.Equal(t, "link_to_link_to_new_readme", locations[0].VirtualPath) + + // check that we can access the same file via 2 symlinks + locations, err = resolver.FilesByGlob("**/link_*") + require.NoError(t, err) + require.Len(t, locations, 1) // you would think this is 2, however, they point to the same file, and glob only returns unique files + + // returned locations can be in any order + expectedVirtualPaths := []string{ + "link_to_link_to_new_readme", + //"link_to_new_readme", // we filter out this one because the first symlink resolves to the same file + } + + expectedRealPaths := []string{ + "readme", + } + + actualRealPaths := strset.New() + actualVirtualPaths := strset.New() + for _, a := range locations { + actualVirtualPaths.Add(a.VirtualPath) + actualRealPaths.Add(a.RealPath) + } + + assert.ElementsMatch(t, expectedVirtualPaths, actualVirtualPaths.List()) + assert.ElementsMatch(t, expectedRealPaths, actualRealPaths.List()) +} + +func Test_UnindexedDirectoryResover_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) { + resolver := NewUnindexedDirectoryResolver("./test-fixtures/symlinks-multiple-roots/root") + + // check that we can get the real path + locations, err := resolver.FilesByPath("./readme") + require.NoError(t, err) + assert.Len(t, locations, 1) + + // check that we can access the same file via 2 symlinks (link_to_link_to_readme -> link_to_readme -> readme) + locations, err = resolver.FilesByPath("./link_to_link_to_readme") + require.NoError(t, err) + assert.Len(t, locations, 1) + + // something looks wrong here + t.Failed() +} + +func Test_UnindexedDirectoryResover_RootViaSymlink(t *testing.T) { + resolver := NewUnindexedDirectoryResolver("./test-fixtures/symlinked-root/nested/link-root") + + locations, err := resolver.FilesByPath("./file1.txt") + require.NoError(t, err) + assert.Len(t, locations, 1) + + locations, err = resolver.FilesByPath("./nested/file2.txt") + require.NoError(t, err) + assert.Len(t, locations, 1) + + locations, err = resolver.FilesByPath("./nested/linked-file1.txt") + require.NoError(t, err) + assert.Len(t, locations, 1) +} + +func Test_UnindexedDirectoryResolver_FileContentsByLocation(t *testing.T) { + cwd, err := os.Getwd() + require.NoError(t, err) + + r := NewUnindexedDirectoryResolver(path.Join(cwd, "test-fixtures/image-simple")) + require.NoError(t, err) + + tests := []struct { + name string + location Location + expects string + err bool + }{ + { + name: "use file reference for content requests", + location: NewLocation("file-1.txt"), + expects: "this file has contents", + }, + { + name: "error on empty file reference", + location: NewLocationFromDirectory("doesn't matter", file.Reference{}), + err: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + actual, err := r.FileContentsByLocation(test.location) + if test.err { + require.Error(t, err) + return + } + + require.NoError(t, err) + if test.expects != "" { + b, err := io.ReadAll(actual) + require.NoError(t, err) + assert.Equal(t, test.expects, string(b)) + } + }) + } +} + +func Test_UnindexedDirectoryResover_SymlinkLoopWithGlobsShouldResolve(t *testing.T) { + test := func(t *testing.T) { + resolver := NewUnindexedDirectoryResolver("./test-fixtures/symlinks-loop") + + locations, err := resolver.FilesByGlob("**/file.target") + require.NoError(t, err) + + require.Len(t, locations, 1) + assert.Equal(t, "devices/loop0/file.target", locations[0].RealPath) + } + + testWithTimeout(t, 5*time.Second, test) +} + +func Test_UnindexedDirectoryResolver_FilesByPath_baseRoot(t *testing.T) { + cases := []struct { + name string + root string + input string + expected []string + }{ + { + name: "should find the base file", + root: "./test-fixtures/symlinks-base/", + input: "./base", + expected: []string{ + "base", + }, + }, + { + name: "should follow a link with a pivoted root", + root: "./test-fixtures/symlinks-base/", + input: "./foo", + expected: []string{ + "base", + }, + }, + { + name: "should follow a relative link with extra parents", + root: "./test-fixtures/symlinks-base/", + input: "./bar", + expected: []string{ + "base", + }, + }, + { + name: "should follow an absolute link with extra parents", + root: "./test-fixtures/symlinks-base/", + input: "./baz", + expected: []string{ + "base", + }, + }, + { + name: "should follow an absolute link with extra parents", + root: "./test-fixtures/symlinks-base/", + input: "./sub/link", + expected: []string{ + "sub/item", + }, + }, + { + name: "should follow chained pivoted link", + root: "./test-fixtures/symlinks-base/", + input: "./chain", + expected: []string{ + "base", + }, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + resolver := NewUnindexedDirectoryResolverRooted(c.root, c.root) + + refs, err := resolver.FilesByPath(c.input) + require.NoError(t, err) + assert.Len(t, refs, len(c.expected)) + s := strset.New() + for _, actual := range refs { + s.Add(actual.RealPath) + } + assert.ElementsMatch(t, c.expected, s.List()) + }) + } + +} + +func Test_UnindexedDirectoryResolver_resolvesLinks(t *testing.T) { + tests := []struct { + name string + runner func(FileResolver) []Location + expected []Location + }{ + { + name: "by glob to links", + runner: func(resolver FileResolver) []Location { + // links are searched, but resolve to the real files + // for that reason we need to place **/ in front (which is not the same for other resolvers) + actualLocations, err := resolver.FilesByGlob("**/*ink-*") + assert.NoError(t, err) + return actualLocations + }, + expected: []Location{ + NewVirtualLocation("file-1.txt", "link-1"), + NewVirtualLocation("file-2.txt", "link-2"), + // we already have this real file path via another link, so only one is returned + // NewVirtualLocation("file-2.txt", "link-indirect"), + NewVirtualLocation("file-3.txt", "link-within"), + }, + }, + { + name: "by basename", + runner: func(resolver FileResolver) []Location { + // links are searched, but resolve to the real files + actualLocations, err := resolver.FilesByGlob("**/file-2.txt") + assert.NoError(t, err) + return actualLocations + }, + expected: []Location{ + // this has two copies in the base image, which overwrites the same location + NewLocation("file-2.txt"), + }, + }, + { + name: "by basename glob", + runner: func(resolver FileResolver) []Location { + // links are searched, but resolve to the real files + actualLocations, err := resolver.FilesByGlob("**/file-?.txt") + assert.NoError(t, err) + return actualLocations + }, + expected: []Location{ + NewLocation("file-1.txt"), + NewLocation("file-2.txt"), + NewLocation("file-3.txt"), + NewLocation("parent/file-4.txt"), + }, + }, + { + name: "by basename glob to links", + runner: func(resolver FileResolver) []Location { + actualLocations, err := resolver.FilesByGlob("**/link-*") + assert.NoError(t, err) + return actualLocations + }, + expected: []Location{ + NewVirtualLocationFromDirectory("file-1.txt", "link-1", file.Reference{RealPath: "file-1.txt"}), + NewVirtualLocationFromDirectory("file-2.txt", "link-2", file.Reference{RealPath: "file-2.txt"}), + // we already have this real file path via another link, so only one is returned + //NewVirtualLocationFromDirectory("file-2.txt", "link-indirect", file.Reference{RealPath: "file-2.txt"}), + NewVirtualLocationFromDirectory("file-3.txt", "link-within", file.Reference{RealPath: "file-3.txt"}), + }, + }, + { + name: "by extension", + runner: func(resolver FileResolver) []Location { + // links are searched, but resolve to the real files + actualLocations, err := resolver.FilesByGlob("**/*.txt") + assert.NoError(t, err) + return actualLocations + }, + expected: []Location{ + NewLocation("file-1.txt"), + NewLocation("file-2.txt"), + NewLocation("file-3.txt"), + NewLocation("parent/file-4.txt"), + }, + }, + { + name: "by path to degree 1 link", + runner: func(resolver FileResolver) []Location { + // links resolve to the final file + actualLocations, err := resolver.FilesByPath("/link-2") + assert.NoError(t, err) + return actualLocations + }, + expected: []Location{ + // we have multiple copies across layers + NewVirtualLocation("file-2.txt", "link-2"), + }, + }, + { + name: "by path to degree 2 link", + runner: func(resolver FileResolver) []Location { + // multiple links resolves to the final file + actualLocations, err := resolver.FilesByPath("/link-indirect") + assert.NoError(t, err) + return actualLocations + }, + expected: []Location{ + // we have multiple copies across layers + NewVirtualLocation("file-2.txt", "link-indirect"), + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + resolver := NewUnindexedDirectoryResolver("./test-fixtures/symlinks-from-image-symlinks-fixture") + + actual := test.runner(resolver) + + compareLocations(t, test.expected, actual) + }) + } +} + +func Test_UnindexedDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) { + resolver := NewUnindexedDirectoryResolver("./test-fixtures/symlinks-prune-indexing") + + allLocations := resolver.AllLocations() + var allRealPaths []file.Path + for l := range allLocations { + allRealPaths = append(allRealPaths, file.Path(l.RealPath)) + } + pathSet := file.NewPathSet(allRealPaths...) + + assert.False(t, + pathSet.Contains("before-path/file.txt"), + "symlink destinations should only be indexed at their real path, not through their virtual (symlinked) path", + ) + + assert.False(t, + pathSet.Contains("a-path/file.txt"), + "symlink destinations should only be indexed at their real path, not through their virtual (symlinked) path", + ) +} + +func Test_UnindexedDirectoryResolver_FilesContents_errorOnDirRequest(t *testing.T) { + resolver := NewUnindexedDirectoryResolver("./test-fixtures/system_paths") + + dirLoc := NewLocation("arg/foo") + + reader, err := resolver.FileContentsByLocation(dirLoc) + require.Error(t, err) + require.Nil(t, reader) +} + +func Test_UnindexedDirectoryResolver_AllLocations(t *testing.T) { + resolver := NewUnindexedDirectoryResolver("./test-fixtures/symlinks-from-image-symlinks-fixture") + + paths := strset.New() + for loc := range resolver.AllLocations() { + if strings.HasPrefix(loc.RealPath, "/") { + // ignore outside of the fixture root for now + continue + } + paths.Add(loc.RealPath) + } + expected := []string{ + "file-1.txt", + "file-2.txt", + "file-3.txt", + "link-1", + "link-2", + "link-dead", + "link-indirect", + "link-within", + "parent", + "parent-link", + "parent/file-4.txt", + } + + pathsList := paths.List() + sort.Strings(pathsList) + + assert.ElementsMatchf(t, expected, pathsList, "expected all paths to be indexed, but found different paths: \n%s", cmp.Diff(expected, paths.List())) +} + +func Test_WritableUnindexedDirectoryResolver(t *testing.T) { + tmpdir := t.TempDir() + + p := "some/path/file" + c := "some contents" + + dr := NewUnindexedDirectoryResolver(tmpdir) + + locations, err := dr.FilesByPath(p) + require.NoError(t, err) + require.Len(t, locations, 0) + + err = dr.Write(NewLocation(p), strings.NewReader(c)) + require.NoError(t, err) + + locations, err = dr.FilesByPath(p) + require.NoError(t, err) + require.Len(t, locations, 1) + + reader, err := dr.FileContentsByLocation(locations[0]) + require.NoError(t, err) + bytes, err := io.ReadAll(reader) + require.Equal(t, c, string(bytes)) +} diff --git a/ui/event_handlers.go b/ui/event_handlers.go index 9a015c216..66f6277d0 100644 --- a/ui/event_handlers.go +++ b/ui/event_handlers.go @@ -29,7 +29,8 @@ const maxBarWidth = 50 const statusSet = components.SpinnerDotSet const completedStatus = "✔" const failedStatus = "✘" -const tileFormat = color.Bold +const titleFormat = color.Bold +const subTitleFormat = color.Normal const interval = 150 * time.Millisecond // StatusTitleColumn is the column index in a given row where status text will be displayed. @@ -42,6 +43,7 @@ var ( dockerPullExtractColor = color.White dockerPullStageChars = strings.Split("▁▃▄▅▆▇█", "") statusTitleTemplate = fmt.Sprintf(" %%s %%-%ds ", StatusTitleColumn) + subStatusTitleTemplate = fmt.Sprintf(" └── %%-%ds ", StatusTitleColumn-3) ) // startProcess is a helper function for providing common elements for long-running UI elements (such as a @@ -83,7 +85,7 @@ func formatDockerPullPhase(phase docker.PullPhase, inputStr string) string { func formatDockerImagePullStatus(pullStatus *docker.PullStatus, spinner *components.Spinner, line *frame.Line) { var size, current uint64 - title := tileFormat.Sprint("Pulling image") + title := titleFormat.Sprint("Pulling image") layers := pullStatus.Layers() status := make(map[docker.LayerID]docker.LayerState) @@ -180,7 +182,7 @@ func PullDockerImageHandler(ctx context.Context, fr *frame.Frame, event partybus if pullStatus.Complete() { spin := color.Green.Sprint(completedStatus) - title := tileFormat.Sprint("Pulled image") + title := titleFormat.Sprint("Pulled image") _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate, spin, title)) } }() @@ -202,7 +204,7 @@ func FetchImageHandler(ctx context.Context, fr *frame.Frame, event partybus.Even formatter, spinner := startProcess() stream := progress.Stream(ctx, prog, interval) - title := tileFormat.Sprint("Loading image") + title := titleFormat.Sprint("Loading image") formatFn := func(p progress.Progress) { progStr, err := formatter.Format(p) @@ -224,7 +226,7 @@ func FetchImageHandler(ctx context.Context, fr *frame.Frame, event partybus.Even } spin := color.Green.Sprint(completedStatus) - title = tileFormat.Sprint("Loaded image") + title = titleFormat.Sprint("Loaded image") _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate, spin, title)) }() return err @@ -246,7 +248,7 @@ func ReadImageHandler(ctx context.Context, fr *frame.Frame, event partybus.Event formatter, spinner := startProcess() stream := progress.Stream(ctx, prog, interval) - title := tileFormat.Sprint("Parsing image") + title := titleFormat.Sprint("Parsing image") formatFn := func(p progress.Progress) { progStr, err := formatter.Format(p) @@ -267,7 +269,7 @@ func ReadImageHandler(ctx context.Context, fr *frame.Frame, event partybus.Event } spin := color.Green.Sprint(completedStatus) - title = tileFormat.Sprint("Parsed image") + title = titleFormat.Sprint("Parsed image") _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate, spin, title)) }() @@ -290,7 +292,7 @@ func PackageCatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event _, spinner := startProcess() stream := progress.StreamMonitors(ctx, []progress.Monitorable{monitor.FilesProcessed, monitor.PackagesDiscovered}, interval) - title := tileFormat.Sprint("Cataloging packages") + title := titleFormat.Sprint("Cataloging packages") formatFn := func(p int64) { spin := color.Magenta.Sprint(spinner.Next()) @@ -307,7 +309,7 @@ func PackageCatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event } spin := color.Green.Sprint(completedStatus) - title = tileFormat.Sprint("Cataloged packages") + title = titleFormat.Sprint("Cataloged packages") auxInfo := auxInfoFormat.Sprintf("[%d packages]", monitor.PackagesDiscovered.Current()) _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s", spin, title, auxInfo)) }() @@ -330,7 +332,7 @@ func SecretsCatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event formatter, spinner := startProcess() stream := progress.Stream(ctx, prog, interval) - title := tileFormat.Sprint("Cataloging secrets") + title := titleFormat.Sprint("Cataloging secrets") formatFn := func(p progress.Progress) { progStr, err := formatter.Format(p) @@ -352,7 +354,7 @@ func SecretsCatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event } spin := color.Green.Sprint(completedStatus) - title = tileFormat.Sprint("Cataloged secrets") + title = titleFormat.Sprint("Cataloged secrets") auxInfo := auxInfoFormat.Sprintf("[%d secrets]", prog.SecretsDiscovered.Current()) _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s", spin, title, auxInfo)) }() @@ -376,7 +378,7 @@ func FileMetadataCatalogerStartedHandler(ctx context.Context, fr *frame.Frame, e formatter, spinner := startProcess() stream := progress.Stream(ctx, prog, interval) - title := tileFormat.Sprint("Cataloging file metadata") + title := titleFormat.Sprint("Cataloging file metadata") formatFn := func(p progress.Progress) { progStr, err := formatter.Format(p) @@ -397,7 +399,7 @@ func FileMetadataCatalogerStartedHandler(ctx context.Context, fr *frame.Frame, e } spin := color.Green.Sprint(completedStatus) - title = tileFormat.Sprint("Cataloged file metadata") + title = titleFormat.Sprint("Cataloged file metadata") _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate, spin, title)) }() return err @@ -418,7 +420,7 @@ func FileIndexingStartedHandler(ctx context.Context, fr *frame.Frame, event part _, spinner := startProcess() stream := progress.Stream(ctx, prog, interval) - title := tileFormat.Sprintf("Indexing %s", path) + title := titleFormat.Sprintf("Indexing %s", path) formatFn := func(_ progress.Progress) { spin := color.Magenta.Sprint(spinner.Next()) @@ -439,7 +441,7 @@ func FileIndexingStartedHandler(ctx context.Context, fr *frame.Frame, event part } spin := color.Green.Sprint(completedStatus) - title = tileFormat.Sprintf("Indexed %s", path) + title = titleFormat.Sprintf("Indexed %s", path) _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate, spin, title)) }() return err @@ -462,7 +464,7 @@ func FileDigestsCatalogerStartedHandler(ctx context.Context, fr *frame.Frame, ev formatter, spinner := startProcess() stream := progress.Stream(ctx, prog, interval) - title := tileFormat.Sprint("Cataloging file digests") + title := titleFormat.Sprint("Cataloging file digests") formatFn := func(p progress.Progress) { progStr, err := formatter.Format(p) @@ -483,7 +485,7 @@ func FileDigestsCatalogerStartedHandler(ctx context.Context, fr *frame.Frame, ev } spin := color.Green.Sprint(completedStatus) - title = tileFormat.Sprint("Cataloged file digests") + title = titleFormat.Sprint("Cataloged file digests") _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate, spin, title)) }() return err @@ -504,7 +506,7 @@ func ImportStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.E formatter, spinner := startProcess() stream := progress.Stream(ctx, prog, interval) - title := tileFormat.Sprint("Uploading image") + title := titleFormat.Sprint("Uploading image") formatFn := func(p progress.Progress) { progStr, err := formatter.Format(p) @@ -526,7 +528,7 @@ func ImportStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.E } spin := color.Green.Sprint(completedStatus) - title = tileFormat.Sprint("Uploaded image") + title = titleFormat.Sprint("Uploaded image") auxInfo := auxInfoFormat.Sprintf("[%s]", host) _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s", spin, title, auxInfo)) }() @@ -550,7 +552,7 @@ func AttestationStartedHandler(ctx context.Context, fr *frame.Frame, event party _, spinner := startProcess() - title := tileFormat.Sprintf(taskInfo.Title.WhileRunning) + title := titleFormat.Sprintf(taskInfo.Title.WhileRunning) s := bufio.NewScanner(reader) l := list.New() @@ -569,7 +571,7 @@ func AttestationStartedHandler(ctx context.Context, fr *frame.Frame, event party spin = color.Red.Sprint(failedStatus) aux = prog.Error().Error() } else { - title = tileFormat.Sprintf(taskInfo.Title.OnSuccess) + title = titleFormat.Sprintf(taskInfo.Title.OnSuccess) } auxInfo := auxInfoFormat.Sprintf("[%s]", aux) @@ -648,3 +650,70 @@ func AttestationStartedHandler(ctx context.Context, fr *frame.Frame, event party }() return nil } + +// CatalogerTaskStartedHandler shows the intermittent progress for a cataloger subprocess messages +func CatalogerTaskStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error { + prog, err := syftEventParsers.ParseCatalogerTaskStarted(event) + if err != nil { + return fmt.Errorf("bad %s event: %w", event.Type, err) + } + + line, err := fr.Append() + if err != nil { + return err + } + wg.Add(1) + + stream := progress.Stream(ctx, prog.GetMonitor(), interval) + + _, spinner := startProcess() + + formatLine := func(complete bool, auxInfo string) string { + title := prog.Title + if complete && prog.TitleOnCompletion != "" { + title = prog.TitleOnCompletion + } + if prog.SubStatus { + title = subTitleFormat.Sprintf("%s", title) + if auxInfo == "" { + return fmt.Sprintf(subStatusTitleTemplate, title) + } + return fmt.Sprintf(subStatusTitleTemplate+"%s", title, auxInfo) + } + + spin := color.Magenta.Sprint(spinner.Next()) + if complete { + spin = color.Green.Sprint(completedStatus) + } + title = titleFormat.Sprintf("%s", title) + if auxInfo == "" { + return fmt.Sprintf(statusTitleTemplate, spin, title) + } + return fmt.Sprintf(statusTitleTemplate+"%s", spin, title, auxInfo) + } + + formatFn := func() { + if err != nil { + _, _ = io.WriteString(line, fmt.Sprintf("Error: %+v", err)) + } else { + auxInfo := auxInfoFormat.Sprintf("[%s]", internal.TruncateMiddleEllipsis(prog.GetValue(), 100)) + _, _ = io.WriteString(line, formatLine(false, auxInfo)) + } + } + + go func() { + defer wg.Done() + + formatFn() + for range stream { + formatFn() + } + + if prog.RemoveOnCompletion { + _ = fr.Remove(line) + } else { + _, _ = io.WriteString(line, formatLine(true, "")) + } + }() + return err +} diff --git a/ui/handler.go b/ui/handler.go index 217372531..b5e3a8fd6 100644 --- a/ui/handler.go +++ b/ui/handler.go @@ -37,7 +37,8 @@ func (r *Handler) RespondsTo(event partybus.Event) bool { syftEvent.FileMetadataCatalogerStarted, syftEvent.FileIndexingStarted, syftEvent.ImportStarted, - syftEvent.AttestationStarted: + syftEvent.AttestationStarted, + syftEvent.CatalogerTaskStarted: return true default: return false @@ -76,6 +77,9 @@ func (r *Handler) Handle(ctx context.Context, fr *frame.Frame, event partybus.Ev case syftEvent.AttestationStarted: return AttestationStartedHandler(ctx, fr, event, wg) + + case syftEvent.CatalogerTaskStarted: + return CatalogerTaskStartedHandler(ctx, fr, event, wg) } return nil }