From d76c868481afb533e8db969bd69186bfdfa77611 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 19 Nov 2021 09:16:25 -0500 Subject: [PATCH] Catalog archive contents for single-file input (#637) * add first-level archive processing when input is a file Signed-off-by: Alex Goodman * add license exception for github.com/xi2/xz Signed-off-by: Alex Goodman * always return cleanup function Signed-off-by: Alex Goodman * change source.NewFromFile log entry to warn Signed-off-by: Alex Goodman * ensure file source always has cleanup function Signed-off-by: Alex Goodman * ensure we are always preferring the unarchive cleanup function for source Signed-off-by: Alex Goodman --- .bouncer.yaml | 5 + Makefile | 2 +- cmd/packages.go | 4 +- cmd/power_user.go | 4 +- go.mod | 4 + go.sum | 24 +++ syft/distro/identify.go | 2 +- syft/source/source.go | 82 +++++++-- syft/source/source_test.go | 162 ++++++++++++++++++ .../generate-tar-fixture-from-source-dir.sh | 9 + 10 files changed, 276 insertions(+), 22 deletions(-) create mode 100755 syft/source/test-fixtures/generate-tar-fixture-from-source-dir.sh diff --git a/.bouncer.yaml b/.bouncer.yaml index a8c89ba34..c89d68f98 100644 --- a/.bouncer.yaml +++ b/.bouncer.yaml @@ -15,3 +15,8 @@ ignore-packages: # * GNU General Public License, version 2.0 or later (GPL-2.0-or-later). # (we choose Apache-2.0) - github.com/spdx/tools-golang + + # from: https://github.com/xi2/xz/blob/master/LICENSE + # All these files have been put into the public domain. + # You can do whatever you want with these files. + - github.com/xi2/xz diff --git a/Makefile b/Makefile index 650b77f46..b5cdb3cd0 100644 --- a/Makefile +++ b/Makefile @@ -135,7 +135,7 @@ lint-fix: ## Auto-format all source code + run golangci lint fixers go mod tidy .PHONY: check-licenses -check-licenses: +check-licenses: ## Ensure transitive dependencies are compliant with the current license policy $(TEMPDIR)/bouncer check check-go-mod-tidy: diff --git a/cmd/packages.go b/cmd/packages.go index a11c078b1..1f0140ab3 100644 --- a/cmd/packages.go +++ b/cmd/packages.go @@ -251,7 +251,9 @@ func packagesExecWorker(userInput string) <-chan error { errs <- fmt.Errorf("failed to determine image source: %w", err) return } - defer cleanup() + if cleanup != nil { + defer cleanup() + } catalog, relationships, d, err := syft.CatalogPackages(src, appConfig.Package.Cataloger.ScopeOpt) if err != nil { diff --git a/cmd/power_user.go b/cmd/power_user.go index a65e54415..54a696f88 100644 --- a/cmd/power_user.go +++ b/cmd/power_user.go @@ -108,7 +108,9 @@ func powerUserExecWorker(userInput string) <-chan error { errs <- err return } - defer cleanup() + if cleanup != nil { + defer cleanup() + } s := sbom.SBOM{ Source: src.Metadata, diff --git a/go.mod b/go.mod index 265fd3f64..c287930b9 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,9 @@ require ( github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29 github.com/anchore/stereoscope v0.0.0-20211116152349-7e4e1b56a15d + // we are hinting brotli to latest due to warning when installing archiver v3: + // go: warning: github.com/andybalholm/brotli@v1.0.1: retracted by module author: occasional panics and data corruption + github.com/andybalholm/brotli v1.0.4 // indirect github.com/antihax/optional v1.0.0 github.com/bmatcuk/doublestar/v2 v2.0.4 github.com/docker/docker v20.10.10+incompatible @@ -26,6 +29,7 @@ require ( github.com/hashicorp/go-version v1.2.0 github.com/jinzhu/copier v0.3.2 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect + github.com/mholt/archiver/v3 v3.5.1 github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/mitchellh/mapstructure v1.4.1 diff --git a/go.sum b/go.sum index 8421eb726..5cd1281b4 100644 --- a/go.sum +++ b/go.sum @@ -111,6 +111,9 @@ github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29/go.mod h1:Oc github.com/anchore/stereoscope v0.0.0-20211116152349-7e4e1b56a15d h1:ieKMmJk1IcmdviUv/KGO1DAk0yFxoGwDpwGq8ckVeVU= github.com/anchore/stereoscope v0.0.0-20211116152349-7e4e1b56a15d/go.mod h1:Zzs5pLx2ZtUctlER7bNDAucDmsc8RO2g/oL3for1tVA= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -302,6 +305,9 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -403,6 +409,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -530,10 +538,15 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -571,6 +584,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= +github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= @@ -606,6 +621,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -658,6 +675,8 @@ github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrap github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM= +github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -782,6 +801,9 @@ github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.9 h1:RsKRIA2MO8x56wkkcd3LbtcE/uMszhb6DpRf+3uwa3I= +github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -816,6 +838,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/syft/distro/identify.go b/syft/distro/identify.go index f4d894d64..1756aa6f5 100644 --- a/syft/distro/identify.go +++ b/syft/distro/identify.go @@ -60,7 +60,7 @@ identifyLoop: } if len(locations) == 0 { - log.Debugf("No Refs found from path: %s", entry.path) + log.Debugf("path not found: %s", entry.path) continue } diff --git a/syft/source/source.go b/syft/source/source.go index 6f2ec4a86..d01550e22 100644 --- a/syft/source/source.go +++ b/syft/source/source.go @@ -7,11 +7,14 @@ package source import ( "fmt" + "io/ioutil" + "os" "sync" "github.com/anchore/stereoscope" "github.com/anchore/stereoscope/pkg/image" "github.com/anchore/syft/internal/log" + "github.com/mholt/archiver/v3" "github.com/spf13/afero" ) @@ -19,9 +22,10 @@ import ( // in cataloging (based on the data source and configuration) type Source struct { Image *image.Image // the image object to be cataloged (image only) - DirectoryResolver *directoryResolver Metadata Metadata - Mutex *sync.Mutex + directoryResolver *directoryResolver + path string + mutex *sync.Mutex } type sourceDetector func(string) (image.Source, string, error) @@ -97,34 +101,61 @@ func generateFileSource(fs afero.Fs, location string) (*Source, func(), error) { return &Source{}, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", location, err) } - s, err := NewFromFile(location) - if err != nil { - return &Source{}, func() {}, fmt.Errorf("could not populate source from path=%q: %w", location, err) - } + s, cleanupFn := NewFromFile(location) - return &s, func() {}, nil + return &s, cleanupFn, nil } // NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively. func NewFromDirectory(path string) (Source, error) { return Source{ - Mutex: &sync.Mutex{}, + mutex: &sync.Mutex{}, Metadata: Metadata{ Scheme: DirectoryScheme, Path: path, }, + path: path, }, nil } -// NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively. -func NewFromFile(path string) (Source, error) { +// NewFromFile creates a new source object tailored to catalog a file. +func NewFromFile(path string) (Source, func()) { + analysisPath, cleanupFn := fileAnalysisPath(path) + return Source{ - Mutex: &sync.Mutex{}, + mutex: &sync.Mutex{}, Metadata: Metadata{ Scheme: FileScheme, Path: path, }, - }, nil + path: analysisPath, + }, cleanupFn +} + +// fileAnalysisPath returns the path given, or in the case the path is an archive, the location where the archive +// contents have been made available. A cleanup function is provided for any temp files created (if any). +func fileAnalysisPath(path string) (string, func()) { + var analysisPath = path + var cleanupFn = func() {} + + // if the given file is an archive (as indicated by the file extension and not MIME type) then unarchive it and + // use the contents as the source. Note: this does NOT recursively unarchive contents, only the given path is + // unarchived. + envelopedUnarchiver, err := archiver.ByExtension(path) + if unarchiver, ok := envelopedUnarchiver.(archiver.Unarchiver); err == nil && ok { + unarchivedPath, tmpCleanup, err := unarchiveToTmp(path, unarchiver) + if err != nil { + log.Warnf("file could not be unarchived: %+v", err) + } else { + log.Debugf("source path is an archive") + analysisPath = unarchivedPath + } + if tmpCleanup != nil { + cleanupFn = tmpCleanup + } + } + + return analysisPath, cleanupFn } // NewFromImage creates a new source object tailored to catalog a given container image, relative to the @@ -146,16 +177,16 @@ func NewFromImage(img *image.Image, userImageStr string) (Source, error) { func (s *Source) FileResolver(scope Scope) (FileResolver, error) { switch s.Metadata.Scheme { case DirectoryScheme, FileScheme: - s.Mutex.Lock() - defer s.Mutex.Unlock() - if s.DirectoryResolver == nil { - resolver, err := newDirectoryResolver(s.Metadata.Path) + s.mutex.Lock() + defer s.mutex.Unlock() + if s.directoryResolver == nil { + resolver, err := newDirectoryResolver(s.path) if err != nil { return nil, err } - s.DirectoryResolver = resolver + s.directoryResolver = resolver } - return s.DirectoryResolver, nil + return s.directoryResolver, nil case ImageScheme: switch scope { case SquashedScope: @@ -168,3 +199,18 @@ func (s *Source) FileResolver(scope Scope) (FileResolver, error) { } return nil, fmt.Errorf("unable to determine FilePathResolver with current scheme=%q", s.Metadata.Scheme) } + +func unarchiveToTmp(path string, unarchiver archiver.Unarchiver) (string, func(), error) { + tempDir, err := ioutil.TempDir("", "syft-archive-contents-") + if err != nil { + return "", func() {}, fmt.Errorf("unable to create tempdir for archive processing: %w", err) + } + + cleanupFn := func() { + if err := os.RemoveAll(tempDir); err != nil { + log.Warnf("unable to cleanup archive tempdir: %+v", err) + } + } + + return tempDir, cleanupFn, unarchiver.Unarchive(path, tempDir) +} diff --git a/syft/source/source_test.go b/syft/source/source_test.go index fe096bf43..1b3dbd4f9 100644 --- a/syft/source/source_test.go +++ b/syft/source/source_test.go @@ -1,8 +1,16 @@ package source import ( + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "syscall" "testing" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/assert" "github.com/anchore/stereoscope/pkg/image" @@ -89,6 +97,80 @@ func TestNewFromDirectory(t *testing.T) { } } +func TestNewFromFile(t *testing.T) { + testCases := []struct { + desc string + input string + expString string + inputPaths []string + expRefs int + }{ + { + desc: "path detected", + input: "test-fixtures/path-detected", + inputPaths: []string{"/.vimrc"}, + expRefs: 1, + }, + } + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + src, cleanup := NewFromFile(test.input) + if cleanup != nil { + t.Cleanup(cleanup) + } + + assert.Equal(t, test.input, src.Metadata.Path) + assert.Equal(t, src.Metadata.Path, src.path) + + resolver, err := src.FileResolver(SquashedScope) + require.NoError(t, err) + + refs, err := resolver.FilesByPath(test.inputPaths...) + require.NoError(t, err) + assert.Len(t, refs, test.expRefs) + + }) + } +} + +func TestNewFromFile_WithArchive(t *testing.T) { + testCases := []struct { + desc string + input string + expString string + inputPaths []string + expRefs int + }{ + { + desc: "path detected", + input: "test-fixtures/path-detected", + inputPaths: []string{"/.vimrc"}, + expRefs: 1, + }, + } + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + archivePath := setupArchiveTest(t, test.input) + + src, cleanup := NewFromFile(archivePath) + if cleanup != nil { + t.Cleanup(cleanup) + } + + assert.Equal(t, archivePath, src.Metadata.Path) + assert.NotEqual(t, src.Metadata.Path, src.path) + + resolver, err := src.FileResolver(SquashedScope) + require.NoError(t, err) + + refs, err := resolver.FilesByPath(test.inputPaths...) + require.NoError(t, err) + assert.Len(t, refs, test.expRefs) + + }) + } +} + func TestNewFromDirectoryShared(t *testing.T) { testCases := []struct { desc string @@ -232,3 +314,83 @@ func TestFilesByGlob(t *testing.T) { }) } } + +// createArchive creates a new archive file at destinationArchivePath based on the directory found at sourceDirPath. +func createArchive(t testing.TB, sourceDirPath, destinationArchivePath string) { + t.Helper() + + cwd, err := os.Getwd() + if err != nil { + t.Fatalf("unable to get cwd: %+v", err) + } + + cmd := exec.Command("./generate-tar-fixture-from-source-dir.sh", destinationArchivePath, path.Base(sourceDirPath)) + cmd.Dir = filepath.Join(cwd, "test-fixtures") + + if err := cmd.Start(); err != nil { + t.Fatalf("unable to start generate zip fixture script: %+v", err) + } + + if err := cmd.Wait(); err != nil { + if exiterr, ok := err.(*exec.ExitError); ok { + // The program has exited with an exit code != 0 + + // This works on both Unix and Windows. Although package + // syscall is generally platform dependent, WaitStatus is + // defined for both Unix and Windows and in both cases has + // an ExitStatus() method with the same signature. + if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { + if status.ExitStatus() != 0 { + t.Fatalf("failed to generate fixture: rc=%d", status.ExitStatus()) + } + } + } else { + t.Fatalf("unable to get generate fixture script result: %+v", err) + } + } + +} + +// setupArchiveTest encapsulates common test setup work for tar file tests. It returns a cleanup function, +// which should be called (typically deferred) by the caller, the path of the created tar archive, and an error, +// which should trigger a fatal test failure in the consuming test. The returned cleanup function will never be nil +// (even if there's an error), and it should always be called. +func setupArchiveTest(t testing.TB, sourceDirPath string) string { + t.Helper() + + archivePrefix, err := ioutil.TempFile("", "syft-archive-TEST-") + require.NoError(t, err) + + t.Cleanup( + assertNoError(t, + func() error { + return os.Remove(archivePrefix.Name()) + }, + ), + ) + + destinationArchiveFilePath := archivePrefix.Name() + ".tar" + t.Logf("archive path: %s", destinationArchiveFilePath) + createArchive(t, sourceDirPath, destinationArchiveFilePath) + + t.Cleanup( + assertNoError(t, + func() error { + return os.Remove(destinationArchiveFilePath) + }, + ), + ) + + cwd, err := os.Getwd() + require.NoError(t, err) + + t.Logf("running from: %s", cwd) + + return destinationArchiveFilePath +} + +func assertNoError(t testing.TB, fn func() error) func() { + return func() { + assert.NoError(t, fn()) + } +} diff --git a/syft/source/test-fixtures/generate-tar-fixture-from-source-dir.sh b/syft/source/test-fixtures/generate-tar-fixture-from-source-dir.sh new file mode 100755 index 000000000..922941d36 --- /dev/null +++ b/syft/source/test-fixtures/generate-tar-fixture-from-source-dir.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eux + +# $1 —— absolute path to destination file, should end with .tar +# $2 —— absolute path to directory from which to add entries to the archive + +pushd "$2" + tar -cvf "$1" . +popd