From efcd8a8b9a1e5264ebb1d5bd87381124677e9d0a Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 18 Mar 2021 08:59:28 -0400 Subject: [PATCH] update integration and acceptance tests for new resolvers Signed-off-by: Alex Goodman --- test/acceptance/deb.sh | 2 +- test/acceptance/mac.sh | 2 +- test/acceptance/rpm.sh | 2 +- ...test.go => catalog_packages_cases_test.go} | 0 ...erage_test.go => catalog_packages_test.go} | 53 +++++--- test/integration/distro_test.go | 13 +- test/integration/document_import_test.go | 101 ---------------- test/integration/java_no_main_package_test.go | 20 ---- test/integration/json_schema_test.go | 113 ------------------ .../package_ownership_relationship_test.go | 25 ++-- ...egression_apk_scanner_buffer_size_test.go} | 15 +-- .../regression_java_no_main_package_test.go | 9 ++ test/integration/utils_test.go | 44 +++++++ 13 files changed, 106 insertions(+), 293 deletions(-) rename test/integration/{pkg_cases_test.go => catalog_packages_cases_test.go} (100%) rename test/integration/{pkg_coverage_test.go => catalog_packages_test.go} (79%) delete mode 100644 test/integration/document_import_test.go delete mode 100644 test/integration/java_no_main_package_test.go delete mode 100644 test/integration/json_schema_test.go rename test/integration/{regression_test.go => regression_apk_scanner_buffer_size_test.go} (52%) create mode 100644 test/integration/regression_java_no_main_package_test.go create mode 100644 test/integration/utils_test.go diff --git a/test/acceptance/deb.sh b/test/acceptance/deb.sh index afe7eadcc..46793472d 100755 --- a/test/acceptance/deb.sh +++ b/test/acceptance/deb.sh @@ -42,7 +42,7 @@ docker run --rm \ /bin/bash -x -c "\ DEBIAN_FRONTEND=noninteractive apt install ${DISTDIR}/syft_*_linux_amd64.deb -y && \ syft version && \ - syft ${TEST_IMAGE} -vv -o json > ${REPORT} \ + syft packages ${TEST_IMAGE} -vv -o json > ${REPORT} \ " # keep the generated report around diff --git a/test/acceptance/mac.sh b/test/acceptance/mac.sh index fade10d76..e6c955bc7 100755 --- a/test/acceptance/mac.sh +++ b/test/acceptance/mac.sh @@ -40,7 +40,7 @@ ls -alh ${TEST_IMAGE_TAR} SYFT_PATH="${DISTDIR}/syft-macos_darwin_amd64/syft" chmod 755 "${SYFT_PATH}" "${SYFT_PATH}" version -SYFT_CHECK_FOR_APP_UPDATE=0 "${SYFT_PATH}" docker-archive://${TEST_IMAGE_TAR} -vv -o json > "${REPORT}" +SYFT_CHECK_FOR_APP_UPDATE=0 "${SYFT_PATH}" packages docker-archive://${TEST_IMAGE_TAR} -vv -o json > "${REPORT}" # keep the generated report around mkdir -p ${RESULTSDIR} diff --git a/test/acceptance/rpm.sh b/test/acceptance/rpm.sh index 062375cb0..53d6800c0 100755 --- a/test/acceptance/rpm.sh +++ b/test/acceptance/rpm.sh @@ -41,7 +41,7 @@ docker run --rm \ /bin/bash -x -c "\ rpm -ivh ${DISTDIR}/syft_*_linux_amd64.rpm && \ syft version && \ - syft ${TEST_IMAGE} -vv -o json > ${REPORT} \ + syft packages ${TEST_IMAGE} -vv -o json > ${REPORT} \ " # keep the generated report around diff --git a/test/integration/pkg_cases_test.go b/test/integration/catalog_packages_cases_test.go similarity index 100% rename from test/integration/pkg_cases_test.go rename to test/integration/catalog_packages_cases_test.go diff --git a/test/integration/pkg_coverage_test.go b/test/integration/catalog_packages_test.go similarity index 79% rename from test/integration/pkg_coverage_test.go rename to test/integration/catalog_packages_test.go index d676c7644..33df9c172 100644 --- a/test/integration/pkg_coverage_test.go +++ b/test/integration/catalog_packages_test.go @@ -3,25 +3,54 @@ package integration import ( "testing" + "github.com/anchore/syft/syft/distro" + "github.com/anchore/syft/syft/pkg/cataloger" + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/anchore/syft/syft/source" + "github.com/go-test/deep" "github.com/anchore/syft/internal" - "github.com/anchore/syft/syft" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/source" ) -func TestPkgCoverageImage(t *testing.T) { +func BenchmarkImagePackageCatalogers(b *testing.B) { fixtureImageName := "image-pkg-coverage" - _, cleanup := imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) - tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) - defer cleanup() + imagetest.GetFixtureImage(b, "docker-archive", fixtureImageName) + tarPath := imagetest.GetFixtureImageTarPath(b, fixtureImageName) - _, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) - if err != nil { - t.Fatalf("failed to catalog image: %+v", err) + var pc *pkg.Catalog + for _, c := range cataloger.ImageCatalogers() { + // in case of future alteration where state is persisted, assume no dependency is safe to reuse + theSource, cleanupSource, err := source.New("docker-archive:" + tarPath) + b.Cleanup(cleanupSource) + if err != nil { + b.Fatalf("unable to get source: %+v", err) + } + + resolver, err := theSource.FileResolver(source.SquashedScope) + if err != nil { + b.Fatalf("unable to get resolver: %+v", err) + } + + theDistro := distro.Identify(resolver) + + b.Run(c.Name(), func(b *testing.B) { + for i := 0; i < b.N; i++ { + pc, err = cataloger.Catalog(resolver, theDistro, c) + if err != nil { + b.Fatalf("failure during benchmark: %+v", err) + } + } + }) + + b.Logf("catalog for %q number of packages: %d", c.Name(), pc.PackageCount()) } +} + +func TestPkgCoverageImage(t *testing.T) { + catalog, _, _ := catalogFixtureImage(t, "image-pkg-coverage") observedLanguages := internal.NewStringSet() definedLanguages := internal.NewStringSet() @@ -100,11 +129,7 @@ func TestPkgCoverageImage(t *testing.T) { } func TestPkgCoverageDirectory(t *testing.T) { - _, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.SquashedScope) - - if err != nil { - t.Errorf("unable to create source from dir: %+v", err) - } + catalog, _, _ := catalogDirectory(t, "test-fixtures/image-pkg-coverage") observedLanguages := internal.NewStringSet() definedLanguages := internal.NewStringSet() diff --git a/test/integration/distro_test.go b/test/integration/distro_test.go index 51d888e75..a54c472f3 100644 --- a/test/integration/distro_test.go +++ b/test/integration/distro_test.go @@ -3,23 +3,12 @@ package integration import ( "testing" - "github.com/anchore/stereoscope/pkg/imagetest" - "github.com/anchore/syft/syft" "github.com/anchore/syft/syft/distro" - "github.com/anchore/syft/syft/source" "github.com/go-test/deep" ) func TestDistroImage(t *testing.T) { - fixtureImageName := "image-distro-id" - _, cleanup := imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) - tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) - defer cleanup() - - _, _, actualDistro, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) - if err != nil { - t.Fatalf("failed to catalog image: %+v", err) - } + _, actualDistro, _ := catalogFixtureImage(t, "image-distro-id") expected, err := distro.NewDistro(distro.Busybox, "1.31.1", "") if err != nil { diff --git a/test/integration/document_import_test.go b/test/integration/document_import_test.go deleted file mode 100644 index cd80d1ca0..000000000 --- a/test/integration/document_import_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package integration - -import ( - "bytes" - "strings" - "testing" - - "github.com/anchore/stereoscope/pkg/imagetest" - "github.com/anchore/syft/syft" - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/presenter/json" - "github.com/anchore/syft/syft/source" - "github.com/go-test/deep" -) - -func TestCatalogFromJSON(t *testing.T) { - - // ensure each of our fixture images results in roughly the same shape when: - // generate json -> import json -> assert packages and distro are the same (except for select fields) - - tests := []struct { - fixture string - }{ - { - fixture: "image-pkg-coverage", - }, - } - - for _, test := range tests { - t.Run(test.fixture, func(t *testing.T) { - _, cleanup := imagetest.GetFixtureImage(t, "docker-archive", test.fixture) - tarPath := imagetest.GetFixtureImageTarPath(t, test.fixture) - defer cleanup() - - expectedSource, expectedCatalog, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) - if err != nil { - t.Fatalf("failed to catalog image: %+v", err) - } - - var buf bytes.Buffer - jsonPres := json.NewPresenter(expectedCatalog, expectedSource.Metadata, expectedDistro) - if err = jsonPres.Present(&buf); err != nil { - t.Fatalf("failed to write to presenter: %+v", err) - } - - sourceMetadata, actualCatalog, actualDistro, err := syft.CatalogFromJSON(&buf) - if err != nil { - t.Fatalf("failed to import document: %+v", err) - } - - for _, d := range deep.Equal(sourceMetadata, expectedSource.Metadata) { - t.Errorf(" image metadata diff: %+v", d) - } - - for _, d := range deep.Equal(actualDistro, expectedDistro) { - t.Errorf(" distro diff: %+v", d) - } - - var actualPackages, expectedPackages []*pkg.Package - - for _, p := range expectedCatalog.Sorted() { - expectedPackages = append(expectedPackages, p) - } - - for _, p := range actualCatalog.Sorted() { - actualPackages = append(actualPackages, p) - } - - if len(actualPackages) != len(expectedPackages) { - t.Fatalf("mismatched package length: %d != %d", len(actualPackages), len(expectedPackages)) - } - - for i, e := range expectedPackages { - a := actualPackages[i] - - // omit fields that should be missing - if e.MetadataType == pkg.JavaMetadataType { - metadata := e.Metadata.(pkg.JavaMetadata) - metadata.Parent = nil - e.Metadata = metadata - } - - // ignore the virtual path on the location for now - for l := range a.Locations { - a.Locations[l].VirtualPath = "" - e.Locations[l].VirtualPath = "" - } - - for _, d := range deep.Equal(a, e) { - // ignore errors for empty collections vs nil for select fields - if strings.Contains(d, "[] != ") { - continue - } - t.Errorf(" package %d (name=%s) diff: %+v", i, e.Name, d) - } - } - - }) - } - -} diff --git a/test/integration/java_no_main_package_test.go b/test/integration/java_no_main_package_test.go deleted file mode 100644 index e330a78fb..000000000 --- a/test/integration/java_no_main_package_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package integration - -import ( - "github.com/anchore/stereoscope/pkg/imagetest" - "github.com/anchore/syft/syft" - "github.com/anchore/syft/syft/source" - "testing" -) - -func TestJavaNoMainPackage(t *testing.T) { // Regression: https://github.com/anchore/syft/issues/252 - fixtureImageName := "image-java-no-main-package" - _, cleanup := imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) - tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) - defer cleanup() - - _, _, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) - if err != nil { - t.Fatalf("failed to catalog image: %+v", err) - } -} diff --git a/test/integration/json_schema_test.go b/test/integration/json_schema_test.go deleted file mode 100644 index 022c9db54..000000000 --- a/test/integration/json_schema_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package integration - -import ( - "bytes" - "fmt" - "os/exec" - "path" - "path/filepath" - "strings" - "testing" - - "github.com/anchore/syft/internal" - - "github.com/anchore/syft/syft/distro" - "github.com/anchore/syft/syft/presenter" - - "github.com/anchore/stereoscope/pkg/imagetest" - "github.com/anchore/syft/syft" - "github.com/anchore/syft/syft/source" - "github.com/xeipuuv/gojsonschema" -) - -const jsonSchemaPath = "schema/json" - -func repoRoot(t *testing.T) string { - t.Helper() - repoRoot, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() - if err != nil { - t.Fatalf("unable to find repo root dir: %+v", err) - } - absRepoRoot, err := filepath.Abs(strings.TrimSpace(string(repoRoot))) - if err != nil { - t.Fatal("unable to get abs path to repo root:", err) - } - return absRepoRoot -} - -func validateAgainstV1Schema(t *testing.T, json string) { - fullSchemaPath := path.Join(repoRoot(t), jsonSchemaPath, fmt.Sprintf("schema-%s.json", internal.JSONSchemaVersion)) - schemaLoader := gojsonschema.NewReferenceLoader(fmt.Sprintf("file://%s", fullSchemaPath)) - documentLoader := gojsonschema.NewStringLoader(json) - - result, err := gojsonschema.Validate(schemaLoader, documentLoader) - if err != nil { - t.Fatal("unable to validate json schema:", err.Error()) - } - - if !result.Valid() { - t.Errorf("failed json schema validation:") - t.Errorf("JSON:\n%s\n", json) - for _, desc := range result.Errors() { - t.Errorf(" - %s\n", desc) - } - } -} - -func TestJsonSchemaImg(t *testing.T) { - fixtureImageName := "image-pkg-coverage" - _, cleanup := imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) - tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) - defer cleanup() - - src, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) - if err != nil { - t.Fatalf("failed to catalog image: %+v", err) - } - - output := bytes.NewBufferString("") - - d, err := distro.NewDistro(distro.CentOS, "5", "rhel fedora") - if err != nil { - t.Fatalf("bad distro: %+v", err) - } - - p := presenter.GetPresenter(presenter.JSONPresenter, src.Metadata, catalog, &d) - if p == nil { - t.Fatal("unable to get presenter") - } - - err = p.Present(output) - if err != nil { - t.Fatalf("unable to present: %+v", err) - } - - validateAgainstV1Schema(t, output.String()) - -} - -func TestJsonSchemaDirs(t *testing.T) { - src, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.SquashedScope) - if err != nil { - t.Errorf("unable to create source from dir: %+v", err) - } - - output := bytes.NewBufferString("") - - d, err := distro.NewDistro(distro.CentOS, "5", "rhel fedora") - if err != nil { - t.Fatalf("bad distro: %+v", err) - } - - p := presenter.GetPresenter(presenter.JSONPresenter, src.Metadata, catalog, &d) - if p == nil { - t.Fatal("unable to get presenter") - } - - err = p.Present(output) - if err != nil { - t.Fatalf("unable to present: %+v", err) - } - - validateAgainstV1Schema(t, output.String()) -} diff --git a/test/integration/package_ownership_relationship_test.go b/test/integration/package_ownership_relationship_test.go index 490408ccc..dee87a6d2 100644 --- a/test/integration/package_ownership_relationship_test.go +++ b/test/integration/package_ownership_relationship_test.go @@ -5,11 +5,7 @@ import ( "encoding/json" "testing" - "github.com/anchore/stereoscope/pkg/imagetest" - "github.com/anchore/syft/syft" - "github.com/anchore/syft/syft/presenter" - jsonPresenter "github.com/anchore/syft/syft/presenter/json" - "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/internal/presenter/packages" ) func TestPackageOwnershipRelationships(t *testing.T) { @@ -25,27 +21,24 @@ func TestPackageOwnershipRelationships(t *testing.T) { for _, test := range tests { t.Run(test.fixture, func(t *testing.T) { - _, cleanup := imagetest.GetFixtureImage(t, "docker-archive", test.fixture) - tarPath := imagetest.GetFixtureImageTarPath(t, test.fixture) - defer cleanup() + catalog, d, src := catalogFixtureImage(t, test.fixture) - src, catalog, d, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) - if err != nil { - t.Fatalf("failed to catalog image: %+v", err) - } - - p := presenter.GetPresenter(presenter.JSONPresenter, src.Metadata, catalog, d) + p := packages.Presenter(packages.JSONPresenterOption, packages.PresenterConfig{ + SourceMetadata: src.Metadata, + Catalog: catalog, + Distro: d, + }) if p == nil { t.Fatal("unable to get presenter") } output := bytes.NewBufferString("") - err = p.Present(output) + err := p.Present(output) if err != nil { t.Fatalf("unable to present: %+v", err) } - var doc jsonPresenter.Document + var doc packages.JSONDocument decoder := json.NewDecoder(output) if err := decoder.Decode(&doc); err != nil { t.Fatalf("unable to decode json doc: %+v", err) diff --git a/test/integration/regression_test.go b/test/integration/regression_apk_scanner_buffer_size_test.go similarity index 52% rename from test/integration/regression_test.go rename to test/integration/regression_apk_scanner_buffer_size_test.go index 34f8f2ff8..5aaa4f24d 100644 --- a/test/integration/regression_test.go +++ b/test/integration/regression_apk_scanner_buffer_size_test.go @@ -4,25 +4,12 @@ import ( "testing" "github.com/anchore/syft/syft/pkg" - - "github.com/anchore/stereoscope/pkg/imagetest" - "github.com/anchore/syft/syft" - "github.com/anchore/syft/syft/source" ) func TestRegression212ApkBufferSize(t *testing.T) { // This is a regression test for issue #212 (https://github.com/anchore/syft/issues/212) in which the apk db could // not be processed due to a scanner buffer that was too small - - fixtureImageName := "image-large-apk-data" - _, cleanup := imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) - tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) - defer cleanup() - - _, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) - if err != nil { - t.Fatalf("failed to catalog image: %+v", err) - } + catalog, _, _ := catalogFixtureImage(t, "image-large-apk-data") expectedPkgs := 58 actualPkgs := 0 diff --git a/test/integration/regression_java_no_main_package_test.go b/test/integration/regression_java_no_main_package_test.go new file mode 100644 index 000000000..a5f41ddaf --- /dev/null +++ b/test/integration/regression_java_no_main_package_test.go @@ -0,0 +1,9 @@ +package integration + +import ( + "testing" +) + +func TestRegressionJavaNoMainPackage(t *testing.T) { // Regression: https://github.com/anchore/syft/issues/252 + catalogFixtureImage(t, "image-java-no-main-package") +} diff --git a/test/integration/utils_test.go b/test/integration/utils_test.go new file mode 100644 index 000000000..ad194b273 --- /dev/null +++ b/test/integration/utils_test.go @@ -0,0 +1,44 @@ +package integration + +import ( + "testing" + + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/anchore/syft/syft" + "github.com/anchore/syft/syft/distro" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +func catalogFixtureImage(t *testing.T, fixtureImageName string) (*pkg.Catalog, *distro.Distro, source.Source) { + imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) + tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) + + theSource, cleanupSource, err := source.New("docker-archive:" + tarPath) + t.Cleanup(cleanupSource) + if err != nil { + t.Fatalf("unable to get source: %+v", err) + } + + pkgCatalog, actualDistro, err := syft.CatalogPackages(theSource, source.SquashedScope) + if err != nil { + t.Fatalf("failed to catalog image: %+v", err) + } + + return pkgCatalog, actualDistro, theSource +} + +func catalogDirectory(t *testing.T, dir string) (*pkg.Catalog, *distro.Distro, source.Source) { + theSource, cleanupSource, err := source.New("dir:" + dir) + t.Cleanup(cleanupSource) + if err != nil { + t.Fatalf("unable to get source: %+v", err) + } + + pkgCatalog, actualDistro, err := syft.CatalogPackages(theSource, source.AllLayersScope) + if err != nil { + t.Fatalf("failed to catalog image: %+v", err) + } + + return pkgCatalog, actualDistro, theSource +}