Merge pull request #328 from anchore/issue-327

Be lenient on invalid fields in PKG-INFO

.github/workflows/acceptance-test.yaml

@@ -1,4 +1,4 @@
-name: 'Acceptance'
+name: "Acceptance"
 on:
   workflow_dispatch:
   push:
@@ -16,7 +16,6 @@ jobs:
   Build-Snapshot-Artifacts:
     runs-on: macos-latest # We're creating these snapshot builds on macOS to be consistent with our release workflow's build process, which also takes place on macOS (due to code signing requirements).
     steps:
-
       - uses: actions/setup-go@v2
         with:
           go-version: ${{ env.GO_VERSION }}
@@ -25,7 +24,7 @@ jobs:
 
       - name: Restore bootstrap cache
         id: cache
-        uses: actions/cache@v2
+        uses: actions/cache@v2.1.3
         with:
           path: |
             ~/go/pkg/mod
@@ -75,11 +74,10 @@ jobs:
 
   # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
   Acceptance-Linux:
-    needs: [ Build-Snapshot-Artifacts ]
+    needs: [Build-Snapshot-Artifacts]
     # come Nov 30 2020 ubuntu-latest will be ubuntu-20.04, until then it needs to be explicitly referenced due to python 3.7 specific features being used
     runs-on: ubuntu-20.04
     steps:
-
       - uses: actions/checkout@v2
 
       - uses: actions/download-artifact@v2
@@ -101,10 +99,9 @@ jobs:
 
   # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
   Acceptance-Mac:
-    needs: [ Build-Snapshot-Artifacts ]
+    needs: [Build-Snapshot-Artifacts]
     runs-on: macos-latest
     steps:
-
       - uses: actions/checkout@v2
 
       - uses: actions/download-artifact@v2
@@ -126,11 +123,10 @@ jobs:
 
   # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
   Inline-Compare:
-    needs: [ Build-Snapshot-Artifacts ]
+    needs: [Build-Snapshot-Artifacts]
     # come Nov 30 2020 ubuntu-latest will be ubuntu-20.04, until then it needs to be explicitly referenced due to python 3.7 specific features being used
     runs-on: ubuntu-20.04
     steps:
-
       - uses: actions/checkout@v2
 
       - name: Fingerprint inline-compare sources
@@ -138,7 +134,7 @@ jobs:
 
       - name: Restore inline reports cache
         id: cache
-        uses: actions/cache@v2
+        uses: actions/cache@v2.1.3
         with:
           path: ${{ github.workspace }}/test/inline-compare/inline-reports
           key: inline-reports-${{ hashFiles('**/inline-compare.fingerprint') }}

.github/workflows/release.yaml

@@ -1,12 +1,12 @@
-name: 'Release'
+name: "Release"
 on:
   push:
     # take no actions on push to any branch...
     branches-ignore:
-      - '**'
+      - "**"
     # ... only act on release tags
     tags:
-      - 'v*'
+      - "v*"
 
 env:
   GO_VERSION: "1.14.x"
@@ -15,7 +15,6 @@ jobs:
   wait-for-checks:
     runs-on: ubuntu-latest # This OS choice is arbitrary. None of the steps in this job are specific to either Linux or macOS.
     steps:
-
       - uses: actions/checkout@v2
 
       # we don't want to release commits that have been pushed and tagged, but not necessarily merged onto main
@@ -81,10 +80,9 @@ jobs:
           false
 
   release:
-    needs: [ wait-for-checks ]
+    needs: [wait-for-checks]
     runs-on: macos-latest # Due to our code signing process, it's vital that we run our release steps on macOS.
     steps:
-
       - uses: actions/setup-go@v2
         with:
           go-version: ${{ env.GO_VERSION }}
@@ -96,7 +94,7 @@ jobs:
       # We are expecting this cache to have been created during the "Build-Snapshot-Artifacts" job in the "Acceptance" workflow.
       - name: Restore bootstrap cache
         id: cache
-        uses: actions/cache@v2
+        uses: actions/cache@v2.1.3
         with:
           path: |
             ~/go/pkg/mod

.github/workflows/static-unit-integration.yaml

@@ -1,10 +1,9 @@
-name: 'Static Analysis + Unit + Integration'
+name: "Static Analysis + Unit + Integration"
 on:
   workflow_dispatch:
   push:
   pull_request:
 jobs:
-
   Static-Analysis:
     strategy:
       matrix:
@@ -12,7 +11,6 @@ jobs:
         platform: [ubuntu-latest]
     runs-on: ${{ matrix.platform }}
     steps:
-
       - uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.go-version }}
@@ -21,7 +19,7 @@ jobs:
 
       - name: Restore bootstrap cache
         id: bootstrap-cache
-        uses: actions/cache@v2
+        uses: actions/cache@v2.1.3
         with:
           path: |
             ~/go/pkg/mod
@@ -49,7 +47,6 @@ jobs:
         platform: [ubuntu-latest]
     runs-on: ${{ matrix.platform }}
     steps:
-
       - uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.go-version }}
@@ -58,7 +55,7 @@ jobs:
 
       - name: Restore bootstrap cache
         id: bootstrap-cache
-        uses: actions/cache@v2
+        uses: actions/cache@v2.1.3
         with:
           path: |
             ~/go/pkg/mod
@@ -80,7 +77,7 @@ jobs:
 
       - name: Restore Java test-fixture cache
         id: unit-java-cache
-        uses: actions/cache@v2
+        uses: actions/cache@v2.1.3
         with:
           path: syft/cataloger/java/test-fixtures/java-builds/packages
           key: ${{ runner.os }}-unit-java-cache-${{ hashFiles( 'syft/cataloger/java/test-fixtures/java-builds/packages.fingerprint' ) }}
@@ -95,7 +92,7 @@ jobs:
         run: make integration-fingerprint
 
       - name: Restore integration test cache
-        uses: actions/cache@v2
+        uses: actions/cache@v2.1.3
         with:
           path: ${{ github.workspace }}/integration/test-fixtures/cache
           key: ${{ runner.os }}-integration-test-cache-${{ hashFiles('integration/test-fixtures/cache.fingerprint') }}

Makefile

@@ -19,6 +19,7 @@ COVERAGE_THRESHOLD := 68
 # CI cache busting values; change these if you want CI to not use previous stored cache
 COMPARE_CACHE_BUSTER="f7e689d76a9"
 INTEGRATION_CACHE_BUSTER="789bacdf"
+BOOTSTRAP_CACHE="789bacdf"
 
 ## Build variables
 DISTDIR=./dist

cmd/root.go

@@ -158,22 +158,13 @@ func doImport(src source.Source, s source.Metadata, catalog *pkg.Catalog, d *dis
 		}
 	}
 
-	var scheme string
-	var hostname = appConfig.Anchore.Host
-	urlFields := strings.Split(hostname, "://")
-	if len(urlFields) > 1 {
-		scheme = urlFields[0]
-		hostname = urlFields[1]
-	}
-
 	c, err := anchore.NewClient(anchore.Configuration{
-		Hostname: hostname,
+		BaseURL:  appConfig.Anchore.Host,
 		Username: appConfig.Anchore.Username,
 		Password: appConfig.Anchore.Password,
-		Scheme:   scheme,
 	})
 	if err != nil {
-		return fmt.Errorf("failed to create anchore client: %+v", err)
+		return fmt.Errorf("unable to upload results: %w", err)
 	}
 
 	importCfg := anchore.ImportConfig{

internal/anchore/client.go

@@ -2,7 +2,11 @@ package anchore
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"path"
+	"strings"
+	"unicode"
 
 	"github.com/anchore/client-go/pkg/external"
 	"github.com/anchore/syft/internal"
@@ -10,11 +14,10 @@ import (
 )
 
 type Configuration struct {
-	Hostname  string
+	BaseURL   string
 	Username  string
 	Password  string
 	UserAgent string
-	Scheme    string
 }
 
 type Client struct {
@@ -29,16 +32,16 @@ func NewClient(cfg Configuration) (*Client, error) {
 		cfg.UserAgent = fmt.Sprintf("%s / %s %s", internal.ApplicationName, versionInfo.Version, versionInfo.Platform)
 	}
 
-	if cfg.Scheme == "" {
-		cfg.Scheme = "https"
+	baseURL, err := prepareBaseURLForClient(cfg.BaseURL)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create client: %w", err)
 	}
 
 	return &Client{
 		config: cfg,
 		client: external.NewAPIClient(&external.Configuration{
-			Host:      cfg.Hostname,
+			BasePath:  baseURL,
 			UserAgent: cfg.UserAgent,
-			Scheme:    cfg.Scheme,
 		}),
 	}, nil
 }
@@ -56,3 +59,57 @@ func (c *Client) newRequestContext(parentContext context.Context) context.Contex
 		},
 	)
 }
+
+var ErrInvalidBaseURLInput = errors.New("invalid base URL input")
+
+func prepareBaseURLForClient(baseURL string) (string, error) {
+	if err := checkBaseURLInput(baseURL); err != nil {
+		return "", err
+	}
+
+	scheme, urlWithoutScheme := splitSchemeFromURL(baseURL)
+
+	if scheme == "" {
+		scheme = "http"
+	}
+
+	urlWithoutScheme = path.Clean(urlWithoutScheme)
+
+	const requiredSuffix = "v1"
+	if path.Base(urlWithoutScheme) != requiredSuffix {
+		urlWithoutScheme = path.Join(urlWithoutScheme, requiredSuffix)
+	}
+
+	preparedBaseURL := scheme + "://" + urlWithoutScheme
+	return preparedBaseURL, nil
+}
+
+func checkBaseURLInput(url string) error {
+	if url == "" {
+		return ErrInvalidBaseURLInput
+	}
+
+	firstCharacter := rune(url[0])
+	if !(unicode.IsLetter(firstCharacter)) {
+		return ErrInvalidBaseURLInput
+	}
+
+	return nil
+}
+
+func splitSchemeFromURL(url string) (scheme, urlWithoutScheme string) {
+	if hasScheme(url) {
+		urlParts := strings.SplitN(url, "://", 2)
+		scheme = urlParts[0]
+		urlWithoutScheme = urlParts[1]
+		return
+	}
+
+	return "", url
+}
+
+func hasScheme(url string) bool {
+	parts := strings.Split(url, "://")
+
+	return len(parts) > 1
+}

internal/anchore/client_test.go

@@ -0,0 +1,210 @@
+package anchore
+
+import "testing"
+
+func TestHasScheme(t *testing.T) {
+	cases := []struct {
+		url      string
+		expected bool
+	}{
+		{
+			url:      "http://localhost",
+			expected: true,
+		},
+		{
+			url:      "https://anchore.com:8443",
+			expected: true,
+		},
+		{
+			url:      "google.com",
+			expected: false,
+		},
+		{
+			url:      "",
+			expected: false,
+		},
+	}
+
+	for _, testCase := range cases {
+		t.Run(testCase.url, func(t *testing.T) {
+			result := hasScheme(testCase.url)
+
+			if testCase.expected != result {
+				t.Errorf("expected %t but got %t", testCase.expected, result)
+			}
+		})
+	}
+}
+
+func TestPrepareBaseURLForClient(t *testing.T) {
+	cases := []struct {
+		inputURL    string
+		expectedURL string
+		expectedErr error
+	}{
+		{
+			inputURL:    "",
+			expectedURL: "",
+			expectedErr: ErrInvalidBaseURLInput,
+		},
+		{
+			inputURL:    "localhost",
+			expectedURL: "http://localhost/v1",
+			expectedErr: nil,
+		},
+		{
+			inputURL:    "https://localhost",
+			expectedURL: "https://localhost/v1",
+			expectedErr: nil,
+		},
+		{
+			inputURL:    "https://localhost/",
+			expectedURL: "https://localhost/v1",
+			expectedErr: nil,
+		},
+		{
+			inputURL:    "https://localhost/v1/",
+			expectedURL: "https://localhost/v1",
+			expectedErr: nil,
+		},
+		{
+			inputURL:    "https://localhost/v1//",
+			expectedURL: "https://localhost/v1",
+			expectedErr: nil,
+		},
+		{
+			inputURL:    "http://something.com/platform/v1/services/anchore",
+			expectedURL: "http://something.com/platform/v1/services/anchore/v1",
+			expectedErr: nil,
+		},
+		{
+			inputURL:    "my-host:8228",
+			expectedURL: "http://my-host:8228/v1",
+			expectedErr: nil,
+		},
+		{
+			inputURL:    "v1/v1",
+			expectedURL: "http://v1/v1",
+			expectedErr: nil,
+		},
+		{
+			inputURL:    "/v1",
+			expectedURL: "",
+			expectedErr: ErrInvalidBaseURLInput,
+		},
+		{
+			inputURL:    "/imports/images",
+			expectedURL: "",
+			expectedErr: ErrInvalidBaseURLInput,
+		},
+	}
+
+	for _, testCase := range cases {
+		t.Run(testCase.inputURL, func(t *testing.T) {
+			resultURL, err := prepareBaseURLForClient(testCase.inputURL)
+			if err != testCase.expectedErr {
+				t.Errorf("expected err to be '%v' but got '%v'", testCase.expectedErr, err)
+			}
+
+			if resultURL != testCase.expectedURL {
+				t.Errorf("expected URL to be '%v' but got '%v'", testCase.expectedURL, resultURL)
+			}
+		})
+	}
+}
+
+func TestCheckBaseURLInput(t *testing.T) {
+	cases := []struct {
+		input    string
+		expected error
+	}{
+		{
+			input:    "",
+			expected: ErrInvalidBaseURLInput,
+		},
+		{
+			input:    "x",
+			expected: nil,
+		},
+		{
+			input:    "localhost:8000",
+			expected: nil,
+		},
+		{
+			input:    ":80",
+			expected: ErrInvalidBaseURLInput,
+		},
+		{
+			input:    "/v1",
+			expected: ErrInvalidBaseURLInput,
+		},
+	}
+
+	for _, testCase := range cases {
+		t.Run(testCase.input, func(t *testing.T) {
+			resultErr := checkBaseURLInput(testCase.input)
+
+			if testCase.expected != resultErr {
+				t.Errorf("expected err to be '%v' but got '%v'", testCase.expected, resultErr)
+			}
+		})
+	}
+}
+
+func TestSplitSchemeFromURL(t *testing.T) {
+	cases := []struct {
+		input                    string
+		expectedScheme           string
+		expectedURLWithoutScheme string
+	}{
+		{
+			input:                    "",
+			expectedScheme:           "",
+			expectedURLWithoutScheme: "",
+		},
+		{
+			input:                    "localhost",
+			expectedScheme:           "",
+			expectedURLWithoutScheme: "localhost",
+		},
+		{
+			input:                    "https://anchore.com/path",
+			expectedScheme:           "https",
+			expectedURLWithoutScheme: "anchore.com/path",
+		},
+		{
+			input:                    "tcp://host:1234",
+			expectedScheme:           "tcp",
+			expectedURLWithoutScheme: "host:1234",
+		},
+		{
+			input:                    "/hello",
+			expectedScheme:           "",
+			expectedURLWithoutScheme: "/hello",
+		},
+		{
+			input:                    "://host",
+			expectedScheme:           "",
+			expectedURLWithoutScheme: "host",
+		},
+		{
+			input:                    "http//localhost",
+			expectedScheme:           "",
+			expectedURLWithoutScheme: "http//localhost",
+		},
+	}
+
+	for _, testCase := range cases {
+		t.Run(testCase.input, func(t *testing.T) {
+			resultScheme, resultURLWithoutScheme := splitSchemeFromURL(testCase.input)
+
+			if testCase.expectedScheme != resultScheme {
+				t.Errorf("expected scheme to be '%s' but got '%s'", testCase.expectedScheme, resultScheme)
+			}
+
+			if testCase.expectedURLWithoutScheme != resultURLWithoutScheme {
+				t.Errorf("expected urlWithoutScheme to be '%s' but got '%s'", testCase.expectedURLWithoutScheme, resultURLWithoutScheme)
+			}
+		})
+	}
+}

internal/anchore/import.go

@@ -51,7 +51,7 @@ func importProgress(source string) (*progress.Stage, *progress.Manual) {
 
 // nolint:funlen
 func (c *Client) Import(ctx context.Context, cfg ImportConfig) error {
-	stage, prog := importProgress(c.config.Hostname)
+	stage, prog := importProgress(c.config.BaseURL)
 
 	ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()

internal/ui/etui/ephemeral_tui.go

@@ -23,7 +23,9 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"os/signal"
 	"sync"
+	"syscall"
 
 	"github.com/anchore/syft/internal/logger"
 
@@ -102,6 +104,8 @@ func OutputToEphemeralTUI(workerErrs <-chan error, subscription *partybus.Subscr
 	ctx := context.Background()
 	syftUIHandler := ui.NewHandler()
 
+	signals := interruptingSignals()
+
 eventLoop:
 	for {
 		select {
@@ -154,8 +158,23 @@ eventLoop:
 				log.Errorf("cancelled (%+v)", err)
 			}
 			break eventLoop
+		case <-signals:
+			break eventLoop
 		}
 	}
 
 	return nil
 }
+
+func interruptingSignals() chan os.Signal {
+	c := make(chan os.Signal, 1) // Note: A buffered channel is recommended for this; see https://golang.org/pkg/os/signal/#Notify
+
+	interruptions := []os.Signal{
+		syscall.SIGINT,
+		syscall.SIGTERM,
+	}
+
+	signal.Notify(c, interruptions...)
+
+	return c
+}

syft/cataloger/python/parse_wheel_egg_metadata.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/anchore/syft/internal/file"
+	"github.com/anchore/syft/internal/log"
 
 	"github.com/mitchellh/mapstructure"
 
@@ -56,7 +57,7 @@ func parseWheelOrEggMetadata(path string, reader io.Reader) (pkg.PythonPackageMe
 
 				fields[key] = val
 			} else {
-				return pkg.PythonPackageMetadata{}, fmt.Errorf("cannot parse field from line: '%s'", line)
+				log.Warnf("cannot parse field from path: %q from line: %q", path, line)
 			}
 		}
 	}

syft/cataloger/python/parse_wheel_egg_metadata_test.go

@@ -117,3 +117,37 @@ func TestDetermineSitePackagesRootPath(t *testing.T) {
 		})
 	}
 }
+
+func TestParseWheelEggMetadataInvalid(t *testing.T) {
+	tests := []struct {
+		Fixture          string
+		ExpectedMetadata pkg.PythonPackageMetadata
+	}{
+		{
+			Fixture: "test-fixtures/egg-info/PKG-INFO-INVALID",
+			ExpectedMetadata: pkg.PythonPackageMetadata{
+				Name:                 "mxnet",
+				Version:              "1.8.0",
+				SitePackagesRootPath: "test-fixtures",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Fixture, func(t *testing.T) {
+			fixture, err := os.Open(test.Fixture)
+			if err != nil {
+				t.Fatalf("failed to open fixture: %+v", err)
+			}
+
+			actual, err := parseWheelOrEggMetadata(test.Fixture, fixture)
+			if err != nil {
+				t.Fatalf("failed to parse: %+v", err)
+			}
+
+			for _, d := range deep.Equal(actual, test.ExpectedMetadata) {
+				t.Errorf("diff: %+v", d)
+			}
+		})
+	}
+}

syft/cataloger/python/test-fixtures/egg-info/PKG-INFO-INVALID

@@ -0,0 +1,69 @@
+Metadata-Version: 1.1
+Name: mxnet
+Version: 1.8.0
+Summary: <!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+MXNet Python Package
+====================
+This directory and nested files contain MXNet Python package and language binding.
+
+## Installation
+To install MXNet Python package, visit MXNet [Install Instruction](https://mxnet.apache.org/get_started)
+
+
+## Running the unit tests
+
+For running unit tests, you will need the [nose PyPi package](https://pypi.python.org/pypi/nose). To install:
+```bash
+pip install --upgrade nose
+```
+
+Once ```nose``` is installed, run the following from MXNet root directory (please make sure the installation path of ```nosetests``` is included in your ```$PATH``` environment variable):
+```
+nosetests tests/python/unittest
+nosetests tests/python/train
+
+```
+
+Home-page: https://github.com/apache/incubator-mxnet
+Author: UNKNOWN
+Author-email: UNKNOWN
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: C++
+Classifier: Programming Language :: Cython
+Classifier: Programming Language :: Other
+Classifier: Programming Language :: Perl
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Mathematics
+Classifier: Topic :: Software Development
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Libraries :: Python Modules