Merge pull request #348 from anchore/fix-java-pom-properties-parse-delimiter

Allow for multiple pom.properties delimiters

.github/workflows/acceptance-test.yaml

@@ -14,7 +14,9 @@ env:
 
 jobs:
   Build-Snapshot-Artifacts:
-    runs-on: macos-latest # We're creating these snapshot builds on macOS to be consistent with our release workflow's build process, which also takes place on macOS (due to code signing requirements).
+    # though the release pipeline is running on mac for the signing step, we are skipping the signing step here and
+    # require a system with docker installed, which ubuntu-20.04 has by default (and mac does not for licensing reasons).
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/setup-go@v2
         with:
@@ -38,25 +40,8 @@ jobs:
         if: steps.bootstrap-cache.outputs.cache-hit != 'true'
         run: make bootstrap
 
-      - name: Import GPG key
-        id: import_gpg
-        uses: crazy-max/ghaction-import-gpg@v2
-        env:
-          GPG_PRIVATE_KEY: ${{ secrets.SIGNING_GPG_PRIVATE_KEY }}
-          PASSPHRASE: ${{ secrets.SIGNING_GPG_PASSPHRASE }}
-
-      - name: GPG signing info
-        run: |
-          echo "fingerprint: ${{ steps.import_gpg.outputs.fingerprint }}"
-          echo "keyid:       ${{ steps.import_gpg.outputs.keyid }}"
-          echo "name:        ${{ steps.import_gpg.outputs.name }}"
-          echo "email:       ${{ steps.import_gpg.outputs.email }}"
-
       - name: Build snapshot artifacts
         run: make snapshot
-        env:
-          GPG_PRIVATE_KEY: ${{ secrets.SIGNING_GPG_PRIVATE_KEY }}
-          PASSPHRASE: ${{ secrets.SIGNING_GPG_PASSPHRASE }}
 
       - uses: actions/upload-artifact@v2
         with:

.github/workflows/release.yaml

@@ -83,6 +83,8 @@ jobs:
     needs: [wait-for-checks]
     runs-on: macos-latest # Due to our code signing process, it's vital that we run our release steps on macOS.
     steps:
+      - uses: docker-practice/actions-setup-docker@v1
+
       - uses: actions/setup-go@v2
         with:
           go-version: ${{ env.GO_VERSION }}
@@ -104,6 +106,10 @@ jobs:
             ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}-
             ${{ runner.os }}-go-${{ env.GO_VERSION }}-
 
+      - name: Bootstrap project dependencies
+        if: steps.bootstrap-cache.outputs.cache-hit != 'true'
+        run: make bootstrap
+
       - name: Import GPG key
         id: import_gpg
         uses: crazy-max/ghaction-import-gpg@v2

.goreleaser.yaml

@@ -90,3 +90,20 @@ brews:
       name: homebrew-syft
     homepage: *website
     description: *description
+
+dockers:
+  -
+    binaries:
+      - syft
+    dockerfile: Dockerfile
+    image_templates:
+      - "anchore/syft:latest"
+      - "anchore/syft:{{ .Tag }}"
+      - "anchore/syft:v{{ .Major }}"
+      - "anchore/syft:v{{ .Major }}.{{ .Minor }}"
+
+    build_flag_templates:
+      - "--build-arg=BUILD_DATE={{.Date}}"
+      - "--build-arg=BUILD_VERSION={{.Version}}"
+      - "--build-arg=VCS_REF={{.FullCommit}}"
+      - "--build-arg=VCS_URL={{.GitURL}}"

Dockerfile

@@ -0,0 +1,28 @@
+FROM alpine:latest AS build
+
+RUN apk --no-cache add ca-certificates
+
+FROM scratch
+# needed for version check HTTPS request
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+
+# create the /tmp dir, which is needed for image content cache
+WORKDIR /tmp
+
+COPY syft /
+
+ARG BUILD_DATE
+ARG BUILD_VERSION
+ARG VCS_REF
+ARG VCS_URL
+
+LABEL org.label-schema.schema-version="1.0"
+LABEL org.label-schema.build-date=$BUILD_DATE
+LABEL org.label-schema.name="syft"
+LABEL org.label-schema.description="CLI tool and library for generating a Software Bill of Materials from container images and filesystems"
+LABEL org.label-schema.vcs-url=$VCS_URL
+LABEL org.label-schema.vcs-ref=$VCS_REF
+LABEL org.label-schema.vendor="Anchore, Inc."
+LABEL org.label-schema.version=$BUILD_VERSION
+
+ENTRYPOINT ["/syft"]

Makefile

@@ -78,10 +78,6 @@ help:
 ci-bootstrap:
 	DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y bc jq libxml2-utils
 
-.PHONY:
-ci-bootstrap-mac:
-	github_changelog_generator --version || sudo gem install github_changelog_generator
-
 .PHONY: bootstrap
 bootstrap: ## Download and install all go dependencies (+ prep tooling in the ./tmp dir)
 	$(call title,Bootstrapping dependencies)
@@ -175,7 +171,7 @@ $(SNAPSHOTDIR): ## Build snapshot release binaries and packages
 
 	# build release snapshots
 	BUILD_GIT_TREE_STATE=$(GITTREESTATE) \
-	$(TEMPDIR)/goreleaser release --skip-publish --rm-dist --snapshot --config $(TEMPDIR)/goreleaser.yaml
+	$(TEMPDIR)/goreleaser release --skip-publish --skip-sign --rm-dist --snapshot --config $(TEMPDIR)/goreleaser.yaml
 
 # note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately
 .PHONY: acceptance-mac
@@ -226,8 +222,9 @@ acceptance-test-rpm-package-install: $(SNAPSHOTDIR)
 .PHONY: changlog-release
 changelog-release:
 	@echo "Last tag: $(SECOND_TO_LAST_TAG)"
-	@echo "Current tag: $(VERSION)"
-	@github_changelog_generator \
+	@docker run --rm \
+		-v "$(shell pwd)":/usr/local/src/your-app \
+		ferrarimarco/github-changelog-generator \
 		--user anchore \
 		--project $(BIN) \
 		-t ${GITHUB_TOKEN} \
@@ -259,7 +256,7 @@ changelog-unreleased: ## show the current changelog that will be produced on the
 			/CHANGELOG.md
 
 .PHONY: release
-release: clean-dist ci-bootstrap-mac changelog-release ## Build and publish final binaries and packages. Intended to be run only on macOS.
+release: clean-dist changelog-release ## Build and publish final binaries and packages. Intended to be run only on macOS.
 	$(call title,Publishing release artifacts)
 
 	# Prepare for macOS-specific signing process
@@ -284,6 +281,7 @@ release: clean-dist ci-bootstrap-mac changelog-release ## Build and publish fina
 	# upload the version file that supports the application version update check (excluding pre-releases)
 	.github/scripts/update-version-file.sh "$(DISTDIR)" "$(VERSION)"
 
+
 .PHONY: clean
 clean: clean-dist clean-snapshot ## Remove previous builds and result reports
 	rm -rf $(RESULTSDIR)/*

syft/cataloger/common/generic_cataloger.go

@@ -104,7 +104,7 @@ func (c *GenericCataloger) catalog(contents map[source.Location]io.ReadCloser) (
 		entries, err := parser(location.RealPath, content)
 		if err != nil {
 			// TODO: should we fail? or only log?
-			log.Warnf("cataloger '%s' failed to parse entries (location=%+v): %+v", c.upstreamCataloger, location, err)
+			log.Warnf("cataloger '%s' failed to parse entries (%+v): %+v", c.upstreamCataloger, location, err)
 			continue
 		}
 

syft/cataloger/deb/cataloger.go

@@ -38,6 +38,7 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 		return nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err)
 	}
 
+	var results []pkg.Package
 	var pkgs []pkg.Package
 	for _, dbLocation := range dbFileMatches {
 		dbContents, err := resolver.FileContentsByLocation(dbLocation)
@@ -94,8 +95,10 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 				}
 			}
 		}
+
+		results = append(results, pkgs...)
 	}
-	return pkgs, nil
+	return results, nil
 }
 
 func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) {

syft/cataloger/java/archive_parser.go

@@ -142,7 +142,8 @@ func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
 	manifestContents := contents[manifestMatches[0]]
 	manifest, err := parseJavaManifest(j.archivePath, strings.NewReader(manifestContents))
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse java manifest (%s): %w", j.virtualPath, err)
+		log.Warnf("failed to parse java manifest (%s): %+v", j.virtualPath, err)
+		return nil, nil
 	}
 
 	return &pkg.Package{
@@ -175,7 +176,8 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([
 	for propsPath, propsContents := range contents {
 		propsObj, err := parsePomProperties(propsPath, strings.NewReader(propsContents))
 		if err != nil {
-			return nil, fmt.Errorf("failed to parse pom.properties (%s): %w", j.virtualPath, err)
+			log.Warnf("failed to parse pom.properties (%s): %+v", j.virtualPath, err)
+			continue
 		}
 
 		if propsObj == nil {

syft/cataloger/java/pom_properties.go

@@ -24,7 +24,7 @@ func parsePomProperties(path string, reader io.Reader) (*pkg.PomProperties, erro
 			continue
 		}
 
-		idx := strings.Index(line, "=")
+		idx := strings.IndexAny(line, "=:")
 		if idx == -1 {
 			return nil, fmt.Errorf("unable to split pom.properties key-value pairs: %q", line)
 		}

syft/cataloger/java/pom_properties_test.go

@@ -11,11 +11,9 @@ import (
 
 func TestParseJavaPomProperties(t *testing.T) {
 	tests := []struct {
-		fixture  string
 		expected pkg.PomProperties
 	}{
 		{
-			fixture: "test-fixtures/pom/small.pom.properties",
 			expected: pkg.PomProperties{
 				Path:       "test-fixtures/pom/small.pom.properties",
 				GroupID:    "org.anchore",
@@ -25,7 +23,6 @@ func TestParseJavaPomProperties(t *testing.T) {
 			},
 		},
 		{
-			fixture: "test-fixtures/pom/extra.pom.properties",
 			expected: pkg.PomProperties{
 				Path:       "test-fixtures/pom/extra.pom.properties",
 				GroupID:    "org.anchore",
@@ -38,11 +35,38 @@ func TestParseJavaPomProperties(t *testing.T) {
 				},
 			},
 		},
+		{
+			expected: pkg.PomProperties{
+				Path:       "test-fixtures/pom/colon-delimited.pom.properties",
+				GroupID:    "org.anchore",
+				ArtifactID: "example-java-app-maven",
+				Version:    "0.1.0",
+				Extra:      map[string]string{},
+			},
+		},
+		{
+			expected: pkg.PomProperties{
+				Path:       "test-fixtures/pom/equals-delimited-with-colons.pom.properties",
+				GroupID:    "org.anchore",
+				ArtifactID: "example-java:app-maven",
+				Version:    "0.1.0:something",
+				Extra:      map[string]string{},
+			},
+		},
+		{
+			expected: pkg.PomProperties{
+				Path:       "test-fixtures/pom/colon-delimited-with-equals.pom.properties",
+				GroupID:    "org.anchore",
+				ArtifactID: "example-java=app-maven",
+				Version:    "0.1.0=something",
+				Extra:      map[string]string{},
+			},
+		},
 	}
 
 	for _, test := range tests {
-		t.Run(test.fixture, func(t *testing.T) {
-			fixture, err := os.Open(test.fixture)
+		t.Run(test.expected.Path, func(t *testing.T) {
+			fixture, err := os.Open(test.expected.Path)
 			if err != nil {
 				t.Fatalf("could not open fixture: %+v", err)
 			}

syft/cataloger/java/test-fixtures/pom/colon-delimited-with-equals.pom.properties

@@ -0,0 +1,5 @@
+#Generated by Maven
+#Tue Jul 07 18:59:56 GMT 2020
+groupId:org.anchore
+artifactId: example-java=app-maven
+version:   0.1.0=something

syft/cataloger/java/test-fixtures/pom/colon-delimited.pom.properties

@@ -0,0 +1,5 @@
+#Generated by Maven
+#Tue Jul 07 18:59:56 GMT 2020
+groupId:org.anchore
+artifactId: example-java-app-maven
+version:   0.1.0

syft/cataloger/java/test-fixtures/pom/equals-delimited-with-colons.pom.properties

@@ -0,0 +1,5 @@
+#Generated by Maven
+#Tue Jul 07 18:59:56 GMT 2020
+groupId=org.anchore
+artifactId= example-java:app-maven
+version=   0.1.0:something

syft/cataloger/java/test-fixtures/pom/small.pom.properties

@@ -1,5 +1,5 @@
 #Generated by Maven
 #Tue Jul 07 18:59:56 GMT 2020
 groupId=org.anchore
-artifactId=example-java-app-maven
-version=0.1.0
+artifactId= example-java-app-maven
+version=  0.1.0

syft/distro/identify_test.go

@@ -75,6 +75,11 @@ func TestIdentifyDistro(t *testing.T) {
 			Type:    OpenSuseLeap,
 			Version: "15.2.0",
 		},
+		{
+			fixture: "test-fixtures/os/photon",
+			Type:    Photon,
+			Version: "2.0.0",
+		},
 		{
 			fixture: "test-fixtures/os/arch",
 			Type:    ArchLinux,

syft/distro/test-fixtures/os/photon/etc/os-release

@@ -0,0 +1,8 @@
+NAME="VMware Photon OS"
+VERSION="2.0"
+ID=photon
+VERSION_ID=2.0
+PRETTY_NAME="VMware Photon OS/Linux"
+ANSI_COLOR="1;34"
+HOME_URL="https://vmware.github.io/photon/"
+BUG_REPORT_URL="https://github.com/vmware/photon/issues" 

syft/distro/type.go

@@ -17,6 +17,7 @@ const (
 	OracleLinux       Type = "oraclelinux"
 	ArchLinux         Type = "archlinux"
 	OpenSuseLeap      Type = "opensuseleap"
+	Photon            Type = "photon"
 )
 
 // All contains all Linux distribution options
@@ -32,6 +33,7 @@ var All = []Type{
 	OracleLinux,
 	ArchLinux,
 	OpenSuseLeap,
+	Photon,
 }
 
 // IDMapping connects a distro ID like "ubuntu" to a Distro type
@@ -47,6 +49,7 @@ var IDMapping = map[string]Type{
 	"ol":            OracleLinux,
 	"arch":          ArchLinux,
 	"opensuse-leap": OpenSuseLeap,
+	"photon":        Photon,
 }
 
 // String returns the string representation of the given Linux distribution.

syft/pkg/catalog.go

@@ -7,12 +7,6 @@ import (
 	"github.com/anchore/syft/internal/log"
 )
 
-var globsForbiddenFromBeingOwned = []string{
-	ApkDbGlob,
-	DpkgDbGlob,
-	RpmDbGlob,
-}
-
 // Catalog represents a collection of Packages.
 type Catalog struct {
 	byID      map[ID]*Package

syft/pkg/dpkg_metadata.go

@@ -8,7 +8,7 @@ import (
 	"github.com/scylladb/go-set/strset"
 )
 
-const DpkgDbGlob = "**/var/lib/dpkg/status"
+const DpkgDbGlob = "**/var/lib/dpkg/{status,status.d/**}"
 
 var _ fileOwner = (*DpkgMetadata)(nil)
 

syft/pkg/ownership_by_files_relationship.go

@@ -6,6 +6,16 @@ import (
 	"github.com/scylladb/go-set/strset"
 )
 
+var globsForbiddenFromBeingOwned = []string{
+	// any OS DBs should automatically be ignored to prevent cyclic issues (e.g. the "rpm" RPM owns the path to the
+	// RPM DB, so if not ignored that package would own all other packages on the system).
+	ApkDbGlob,
+	DpkgDbGlob,
+	RpmDbGlob,
+	// DEB packages share common copyright info between, this does not mean that sharing these paths implies ownership.
+	"/usr/share/doc/**/copyright",
+}
+
 type ownershipByFilesMetadata struct {
 	Files []string `json:"files"`
 }

syft/source/location.go

@@ -1,6 +1,8 @@
 package source
 
 import (
+	"fmt"
+
 	"github.com/anchore/syft/internal/log"
 
 	"github.com/anchore/stereoscope/pkg/file"
@@ -42,3 +44,21 @@ func NewLocationFromImage(virtualPath string, ref file.Reference, img *image.Ima
 		ref:          ref,
 	}
 }
+
+func (l Location) String() string {
+	str := ""
+	if l.ref.ID() != 0 {
+		str += fmt.Sprintf("id=%d ", l.ref.ID())
+	}
+
+	str += fmt.Sprintf("RealPath=%q", l.RealPath)
+
+	if l.VirtualPath != "" {
+		str += fmt.Sprintf(" VirtualPath=%q", l.VirtualPath)
+	}
+
+	if l.FileSystemID != "" {
+		str += fmt.Sprintf(" Layer=%q", l.FileSystemID)
+	}
+	return fmt.Sprintf("Location<%s>", str)
+}

test/integration/pkg_cases_test.go

@@ -144,7 +144,9 @@ var commonTestCases = []testCase{
 		name:    "find dpkg packages",
 		pkgType: pkg.DebPkg,
 		pkgInfo: map[string]string{
-			"apt": "1.8.2",
+			"apt":     "1.8.2",
+			"dash":    "0.5.8-2.4",
+			"netbase": "5.4",
 		},
 	},
 	{

test/integration/test-fixtures/image-pkg-coverage/var/lib/dpkg/status.d/dash

@@ -0,0 +1,18 @@
+Package: dash
+Version: 0.5.8-2.4
+Architecture: amd64
+Essential: yes
+Maintainer: Gerrit Pape <pape@smarden.org>
+Installed-Size: 204
+Pre-Depends: libc6 (>= 2.14)
+Depends: debianutils (>= 2.15), dpkg (>= 1.15.0)
+Section: shells
+Priority: required
+Homepage: http://gondor.apana.org.au/~herbert/dash/
+Description: POSIX-compliant shell
+ The Debian Almquist Shell (dash) is a POSIX-compliant shell derived
+ from ash.
+ .
+ Since it executes scripts faster than bash, and has fewer library
+ dependencies (making it more robust against software or hardware
+ failures), it is used as the default system shell on Debian systems.

test/integration/test-fixtures/image-pkg-coverage/var/lib/dpkg/status.d/netbase

@@ -0,0 +1,11 @@
+Package: netbase
+Version: 5.4
+Architecture: all
+Maintainer: Marco d'Itri <md@linux.it>
+Installed-Size: 44
+Section: admin
+Priority: important
+Multi-Arch: foreign
+Description: Basic TCP/IP networking system
+ This package provides the necessary infrastructure for basic TCP/IP based
+ networking.