feat: basic Conda ecosystem support (#4002)

----------------------------------------------------------------
Signed-off-by: Simeon Stoykov <simeon.stoykov@quantco.com>
Signed-off-by: Christopher Angelo Phillips <32073428+spiffcs@users.noreply.github.com>
Co-authored-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com>
This commit is contained in:
Simeon Stoykov
2025-08-19 22:37:27 -04:00
committed by GitHub
co-authored by Christopher Phillips
parent 8e51e8d995
commit a433045d51
18 changed files with 3988 additions and 2 deletions
@@ -18,6 +18,7 @@ func Test_OriginatorSupplier(t *testing.T) {
pkg.ConanV2LockEntry{}, // the field Username might be the username of either the package originator or the supplier (unclear currently)
pkg.ConanfileEntry{},
pkg.ConaninfoEntry{},
pkg.CondaMetaPackage{},
pkg.DartPubspecLockEntry{},
pkg.DartPubspec{},
pkg.DotnetDepsEntry{},
@@ -48,6 +48,8 @@ func SourceInfo(p pkg.Package) string {
answer = "acquired package info from installed cocoapods manifest file"
case pkg.ConanPkg:
answer = "acquired package info from conan manifest"
case pkg.CondaPkg:
answer = "acquired package info from conda metadata"
case pkg.PortagePkg:
answer = "acquired package info from portage DB"
case pkg.HackagePkg:
@@ -191,6 +191,14 @@ func Test_SourceInfo(t *testing.T) {
"from conan manifest",
},
},
{
input: pkg.Package{
Type: pkg.CondaPkg,
},
expected: []string{
"from conda metadata",
},
},
{
input: pkg.Package{
Type: pkg.PortagePkg,
@@ -16,6 +16,7 @@ func AllTypes() []any {
pkg.ConanV2LockEntry{},
pkg.ConanfileEntry{},
pkg.ConaninfoEntry{},
pkg.CondaMetaPackage{},
pkg.DartPubspec{},
pkg.DartPubspecLockEntry{},
pkg.DotnetDepsEntry{},
+1
View File
@@ -119,6 +119,7 @@ var jsonTypes = makeJSONTypes(
jsonNames(pkg.LuaRocksPackage{}, "luarocks-package"),
jsonNames(pkg.TerraformLockProviderEntry{}, "terraform-lock-provider-entry"),
jsonNames(pkg.DotnetPackagesLockEntry{}, "dotnet-packages-lock-entry"),
jsonNames(pkg.CondaMetaPackage{}, "conda-metadata-entry", "CondaPackageMetadata"),
)
func expandLegacyNameVariants(names ...string) []string {
+46
View File
@@ -0,0 +1,46 @@
/*
Package conda provides a concrete Cataloger implementation for packages within the Conda ecosystem.
*/
package conda
import (
"context"
"encoding/json"
"fmt"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
)
// NewCondaMetaCataloger returns a new cataloger object for Conda environments by parsing the package metadata files in conda-meta.
func NewCondaMetaCataloger() pkg.Cataloger {
return generic.NewCataloger("conda-meta-cataloger").
WithParserByGlobs(parseCondaMetaJSON, "**/conda-meta/*.json")
}
func parseCondaMetaJSON(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
dec := json.NewDecoder(reader)
var meta pkg.CondaMetaPackage
if err := dec.Decode(&meta); err != nil {
return nil, nil, fmt.Errorf("failed to parse conda-meta package file at %s: %w", reader.RealPath, err)
}
p := pkg.Package{
Name: meta.Name,
Version: meta.Version,
Locations: file.NewLocationSet(reader.Location),
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocationsWithContext(ctx, meta.License, reader.Location),
),
Language: pkg.UnknownLanguage,
Type: pkg.CondaPkg,
Metadata: meta,
}
p.SetID()
return []pkg.Package{
p,
}, nil, nil
}
+250
View File
@@ -0,0 +1,250 @@
package conda
import (
"context"
"strings"
"testing"
"github.com/go-test/deep"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
)
func Test_CondaCataloger(t *testing.T) {
ctx := context.TODO()
tests := []struct {
name string
fixture string
expectedPackages []pkg.Package
wantErr require.ErrorAssertionFunc
}{
{
name: "multiple packages in conda meta (python, c binaries, ...)",
fixture: "test-fixtures/conda-meta-python-c-etc",
wantErr: require.NoError,
expectedPackages: []pkg.Package{
{
Name: "jupyterlab",
Version: "4.4.3",
FoundBy: "conda-meta-cataloger",
Locations: file.NewLocationSet(
file.NewLocation("conda-meta/jupyterlab-4.4.3-pyhd8ed1ab_0.json"),
),
Language: pkg.UnknownLanguage,
Type: pkg.CondaPkg,
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocationsWithContext(ctx, "BSD-3-Clause", file.NewLocation("conda-meta/jupyterlab-4.4.3-pyhd8ed1ab_0.json")),
),
Metadata: pkg.CondaMetaPackage{
Name: "jupyterlab",
Version: "4.4.3",
Build: "pyhd8ed1ab_0",
BuildNumber: 0,
Channel: "https://conda.anaconda.org/conda-forge/",
Subdir: "noarch",
Noarch: "python",
License: "BSD-3-Clause",
LicenseFamily: "BSD",
MD5: "4861a0c2a5a5d0481a450a9dfaf9febe",
SHA256: "fc0235a71d852734fe92183a78cb91827367573450eba82465ae522c64230736",
Size: 8236973,
Timestamp: 1748273017680,
Filename: "jupyterlab-4.4.3-pyhd8ed1ab_0.conda",
URL: "https://conda.anaconda.org/conda-forge/noarch/jupyterlab-4.4.3-pyhd8ed1ab_0.conda",
ExtractedPackageDir: "/Users/example-user/Library/Caches/rattler/cache/pkgs/jupyterlab-4.4.3-pyhd8ed1ab_0",
Depends: []string{
"async-lru >=1.0.0",
"httpx >=0.25.0",
"importlib-metadata >=4.8.3",
"ipykernel >=6.5.0",
"jinja2 >=3.0.3",
"jupyter-lsp >=2.0.0",
"jupyter_core",
"jupyter_server >=2.4.0,<3",
"jupyterlab_server >=2.27.1,<3",
"notebook-shim >=0.2",
"packaging",
"python >=3.9",
"setuptools >=41.1.0",
"tomli >=1.2.2",
"tornado >=6.2.0",
"traitlets",
},
Files: []string{
"lib/python3.13/site-packages/jupyterlab/schemas/@jupyterlab/apputils-extension/kernels-settings.json",
"lib/python3.13/site-packages/jupyterlab/schemas/@jupyterlab/apputils-extension/notification.json",
},
PathsData: &pkg.CondaPathsData{
PathsVersion: 1,
Paths: []pkg.CondaPathData{
{
Path: "lib/python3.13/site-packages/jupyterlab/schemas/@jupyterlab/apputils-extension/kernels-settings.json",
PathType: "hardlink",
SHA256: "081a7e126deffbcd596863f3349a19416fbbe1fd570ab392270315f7cf5a8c27",
SHA256InPrefix: "081a7e126deffbcd596863f3349a19416fbbe1fd570ab392270315f7cf5a8c27",
SizeInBytes: 935,
},
{
Path: "lib/python3.13/site-packages/jupyterlab/schemas/@jupyterlab/apputils-extension/notification.json",
PathType: "hardlink",
SHA256: "f9f42636592f62cdd03e3d5552b020811e3f8be6fc47c03d5a92396941b8d5d8",
SHA256InPrefix: "f9f42636592f62cdd03e3d5552b020811e3f8be6fc47c03d5a92396941b8d5d8",
SizeInBytes: 1565,
},
},
},
Link: &pkg.CondaLink{
Source: "/Users/example-user/Library/Caches/rattler/cache/pkgs/jupyterlab-4.4.3-pyhd8ed1ab_0",
Type: 1,
},
},
},
{
Name: "zlib",
Version: "1.2.11",
FoundBy: "conda-meta-cataloger",
Locations: file.NewLocationSet(
file.NewLocation("conda-meta/zlib-1.2.11-h90dfc92_1014.json"),
),
Language: pkg.UnknownLanguage,
Type: pkg.CondaPkg,
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocationsWithContext(ctx, "Zlib", file.NewLocation("conda-meta/zlib-1.2.11-h90dfc92_1014.json")),
),
Metadata: pkg.CondaMetaPackage{
Arch: "arm64",
Name: "zlib",
Version: "1.2.11",
Build: "h90dfc92_1014",
BuildNumber: 1014,
Channel: "https://conda.anaconda.org/conda-forge/",
Subdir: "osx-arm64",
Noarch: "",
License: "Zlib",
LicenseFamily: "Other",
MD5: "348a30b1350c9d91a4dbf05f5e46e0bb",
SHA256: "a70c028fd3b9af1d7ea3d7099d810f3d2588096237bb472db331a51a36f931c0",
Size: 86757,
Timestamp: 1648307332172,
Filename: "zlib-1.2.11-h90dfc92_1014.tar.bz2",
URL: "https://conda.anaconda.org/conda-forge/osx-arm64/zlib-1.2.11-h90dfc92_1014.tar.bz2",
ExtractedPackageDir: "/Users/example-user/Library/Caches/rattler/cache/pkgs/zlib-1.2.11-h90dfc92_1014",
Depends: []string{
"libzlib 1.2.11 h90dfc92_1014",
},
Files: []string{
"include/zconf.h",
"include/zlib.h",
"lib/pkgconfig/zlib.pc",
"lib/libz.a",
"lib/libz.dylib",
},
PathsData: &pkg.CondaPathsData{
PathsVersion: 1,
Paths: []pkg.CondaPathData{
{
Path: "include/zconf.h",
PathType: "hardlink",
SHA256: "77304005ceb5f0d03ad4c37eb8386a10866e4ceeb204f7c3b6599834c7319541",
SHA256InPrefix: "77304005ceb5f0d03ad4c37eb8386a10866e4ceeb204f7c3b6599834c7319541",
SizeInBytes: 16262,
},
{
Path: "include/zlib.h",
PathType: "hardlink",
SHA256: "4ddc82b4af931ab55f44d977bde81bfbc4151b5dcdccc03142831a301b5ec3c8",
SHA256InPrefix: "4ddc82b4af931ab55f44d977bde81bfbc4151b5dcdccc03142831a301b5ec3c8",
SizeInBytes: 96239,
},
{
Path: "lib/pkgconfig/zlib.pc",
PathType: "hardlink",
SHA256: "357773df3c44a5ebd77fdadd0869b5b06394bbf556c2d6c9736dd53e9df3b2c2",
SHA256InPrefix: "5b4eb6062f97875eaadb3b6c7cf8cfeff3808798ecbf2bfc095f18dcecc509bf",
SizeInBytes: 285,
},
{
Path: "lib/libz.a",
PathType: "hardlink",
SHA256: "40c056a5d8155d9b3f42adfe35f7fc6e5fa15cc6588ffad0f09fe67517feada0",
SHA256InPrefix: "40c056a5d8155d9b3f42adfe35f7fc6e5fa15cc6588ffad0f09fe67517feada0",
SizeInBytes: 107128,
},
{
Path: "lib/libz.dylib",
PathType: "softlink",
SHA256: "67ed489e2f378880f72fb1c0d1cc916e184b9632eccf2b5c1b34ddf01ed1701c",
SHA256InPrefix: "09e47dbc60aa970e153913fe84551ad7f5aa51c21907591340a0cc999adab859",
SizeInBytes: 122478,
},
},
},
Link: &pkg.CondaLink{
Source: "/Users/example-user/Library/Caches/rattler/cache/pkgs/zlib-1.2.11-h90dfc92_1014",
Type: 1,
},
},
},
},
},
{
name: "badly formatted conda meta json file",
fixture: "test-fixtures/conda-meta-bad-json",
expectedPackages: nil,
wantErr: func(t require.TestingT, err error, msgAndArgs ...interface{}) {
require.Error(t, err)
require.Contains(t, err.Error(), "failed to parse conda-meta package file at conda-meta/package-1.2.3-pyhd8ed1ab_0.json")
require.Contains(t, err.Error(), "invalid character")
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
pkgtest.NewCatalogTester().
FromDirectory(t, test.fixture).
Expects(test.expectedPackages, nil).
WithErrorAssertion(test.wantErr).
TestCataloger(t, NewCondaMetaCataloger())
})
}
}
func TestCondaMetaPackageMetadata_FileOwner(t *testing.T) {
tests := []struct {
metadata pkg.CondaMetaPackage
expected []string
}{
{
metadata: pkg.CondaMetaPackage{
Files: []string{
"include/zconf.h",
"include/zlib.h",
"lib/pkgconfig/zlib.pc",
"lib/libz.a",
"lib/libz.dylib",
},
},
expected: []string{
"include/zconf.h",
"include/zlib.h",
"lib/libz.a",
"lib/libz.dylib",
"lib/pkgconfig/zlib.pc",
},
},
}
for _, test := range tests {
t.Run(strings.Join(test.expected, ","), func(t *testing.T) {
actual := test.metadata.OwnedFiles()
for _, d := range deep.Equal(test.expected, actual) {
t.Errorf("diff: %+v", d)
}
})
}
}
@@ -0,0 +1 @@
not json, will cause a parsing error
@@ -0,0 +1,63 @@
{
"build": "pyhd8ed1ab_0",
"build_number": 0,
"depends": [
"async-lru >=1.0.0",
"httpx >=0.25.0",
"importlib-metadata >=4.8.3",
"ipykernel >=6.5.0",
"jinja2 >=3.0.3",
"jupyter-lsp >=2.0.0",
"jupyter_core",
"jupyter_server >=2.4.0,<3",
"jupyterlab_server >=2.27.1,<3",
"notebook-shim >=0.2",
"packaging",
"python >=3.9",
"setuptools >=41.1.0",
"tomli >=1.2.2",
"tornado >=6.2.0",
"traitlets"
],
"license": "BSD-3-Clause",
"license_family": "BSD",
"md5": "4861a0c2a5a5d0481a450a9dfaf9febe",
"name": "jupyterlab",
"noarch": "python",
"sha256": "fc0235a71d852734fe92183a78cb91827367573450eba82465ae522c64230736",
"size": 8236973,
"subdir": "noarch",
"timestamp": 1748273017680,
"version": "4.4.3",
"fn": "jupyterlab-4.4.3-pyhd8ed1ab_0.conda",
"url": "https://conda.anaconda.org/conda-forge/noarch/jupyterlab-4.4.3-pyhd8ed1ab_0.conda",
"channel": "https://conda.anaconda.org/conda-forge/",
"extracted_package_dir": "/Users/example-user/Library/Caches/rattler/cache/pkgs/jupyterlab-4.4.3-pyhd8ed1ab_0",
"files": [
"lib/python3.13/site-packages/jupyterlab/schemas/@jupyterlab/apputils-extension/kernels-settings.json",
"lib/python3.13/site-packages/jupyterlab/schemas/@jupyterlab/apputils-extension/notification.json"
],
"paths_data": {
"paths_version": 1,
"paths": [
{
"_path": "lib/python3.13/site-packages/jupyterlab/schemas/@jupyterlab/apputils-extension/kernels-settings.json",
"path_type": "hardlink",
"sha256": "081a7e126deffbcd596863f3349a19416fbbe1fd570ab392270315f7cf5a8c27",
"sha256_in_prefix": "081a7e126deffbcd596863f3349a19416fbbe1fd570ab392270315f7cf5a8c27",
"size_in_bytes": 935
},
{
"_path": "lib/python3.13/site-packages/jupyterlab/schemas/@jupyterlab/apputils-extension/notification.json",
"path_type": "hardlink",
"sha256": "f9f42636592f62cdd03e3d5552b020811e3f8be6fc47c03d5a92396941b8d5d8",
"sha256_in_prefix": "f9f42636592f62cdd03e3d5552b020811e3f8be6fc47c03d5a92396941b8d5d8",
"size_in_bytes": 1565
}
]
},
"link": {
"source": "/Users/example-user/Library/Caches/rattler/cache/pkgs/jupyterlab-4.4.3-pyhd8ed1ab_0",
"type": 1
}
}
@@ -0,0 +1,75 @@
{
"arch": "arm64",
"build": "h90dfc92_1014",
"build_number": 1014,
"depends": [
"libzlib 1.2.11 h90dfc92_1014"
],
"license": "Zlib",
"license_family": "Other",
"md5": "348a30b1350c9d91a4dbf05f5e46e0bb",
"name": "zlib",
"platform": "osx",
"sha256": "a70c028fd3b9af1d7ea3d7099d810f3d2588096237bb472db331a51a36f931c0",
"size": 86757,
"subdir": "osx-arm64",
"timestamp": 1648307332172,
"version": "1.2.11",
"fn": "zlib-1.2.11-h90dfc92_1014.tar.bz2",
"url": "https://conda.anaconda.org/conda-forge/osx-arm64/zlib-1.2.11-h90dfc92_1014.tar.bz2",
"channel": "https://conda.anaconda.org/conda-forge/",
"extracted_package_dir": "/Users/example-user/Library/Caches/rattler/cache/pkgs/zlib-1.2.11-h90dfc92_1014",
"files": [
"include/zconf.h",
"include/zlib.h",
"lib/pkgconfig/zlib.pc",
"lib/libz.a",
"lib/libz.dylib"
],
"paths_data": {
"paths_version": 1,
"paths": [
{
"_path": "include/zconf.h",
"path_type": "hardlink",
"sha256": "77304005ceb5f0d03ad4c37eb8386a10866e4ceeb204f7c3b6599834c7319541",
"sha256_in_prefix": "77304005ceb5f0d03ad4c37eb8386a10866e4ceeb204f7c3b6599834c7319541",
"size_in_bytes": 16262
},
{
"_path": "include/zlib.h",
"path_type": "hardlink",
"sha256": "4ddc82b4af931ab55f44d977bde81bfbc4151b5dcdccc03142831a301b5ec3c8",
"sha256_in_prefix": "4ddc82b4af931ab55f44d977bde81bfbc4151b5dcdccc03142831a301b5ec3c8",
"size_in_bytes": 96239
},
{
"_path": "lib/pkgconfig/zlib.pc",
"path_type": "hardlink",
"sha256": "357773df3c44a5ebd77fdadd0869b5b06394bbf556c2d6c9736dd53e9df3b2c2",
"sha256_in_prefix": "5b4eb6062f97875eaadb3b6c7cf8cfeff3808798ecbf2bfc095f18dcecc509bf",
"size_in_bytes": 285,
"file_mode": "text",
"prefix_placeholder": "/Users/runner/miniforge3/conda-bld/zlib-split_1648307175973/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehol"
},
{
"_path": "lib/libz.a",
"path_type": "hardlink",
"sha256": "40c056a5d8155d9b3f42adfe35f7fc6e5fa15cc6588ffad0f09fe67517feada0",
"sha256_in_prefix": "40c056a5d8155d9b3f42adfe35f7fc6e5fa15cc6588ffad0f09fe67517feada0",
"size_in_bytes": 107128
},
{
"_path": "lib/libz.dylib",
"path_type": "softlink",
"sha256": "67ed489e2f378880f72fb1c0d1cc916e184b9632eccf2b5c1b34ddf01ed1701c",
"sha256_in_prefix": "09e47dbc60aa970e153913fe84551ad7f5aa51c21907591340a0cc999adab859",
"size_in_bytes": 122478
}
]
},
"link": {
"source": "/Users/example-user/Library/Caches/rattler/cache/pkgs/zlib-1.2.11-h90dfc92_1014",
"type": 1
}
}
+61
View File
@@ -0,0 +1,61 @@
package pkg
import (
"sort"
"github.com/scylladb/go-set/strset"
)
type CondaPathData struct {
Path string `json:"_path"`
PathType string `json:"path_type"`
SHA256 string `json:"sha256"`
SHA256InPrefix string `json:"sha256_in_prefix"`
SizeInBytes int64 `json:"size_in_bytes"`
}
type CondaPathsData struct {
PathsVersion int `json:"paths_version"`
Paths []CondaPathData `json:"paths"`
}
type CondaLink struct {
Source string `json:"source"`
Type int `json:"type"`
}
type CondaMetaPackage struct {
Arch string `json:"arch,omitempty"`
Name string `json:"name"`
Version string `json:"version"`
Build string `json:"build"`
BuildNumber int `json:"build_number"`
Channel string `json:"channel,omitempty"`
Subdir string `json:"subdir,omitempty"`
Noarch string `json:"noarch,omitempty"`
License string `json:"license,omitempty"`
LicenseFamily string `json:"license_family,omitempty"`
MD5 string `json:"md5,omitempty"`
SHA256 string `json:"sha256,omitempty"`
Size int64 `json:"size,omitempty"`
Timestamp int64 `json:"timestamp,omitempty"`
Filename string `json:"fn,omitempty"`
URL string `json:"url,omitempty"`
ExtractedPackageDir string `json:"extracted_package_dir,omitempty"`
Depends []string `json:"depends,omitempty"`
Files []string `json:"files,omitempty"`
PathsData *CondaPathsData `json:"paths_data,omitempty"`
Link *CondaLink `json:"link,omitempty"`
}
func (m CondaMetaPackage) OwnedFiles() (result []string) {
s := strset.New()
for _, f := range m.Files {
if f != "" {
s.Add(f)
}
}
result = s.List()
sort.Strings(result)
return result
}
+6
View File
@@ -16,6 +16,7 @@ const (
BitnamiPkg Type = "bitnami"
CocoapodsPkg Type = "pod"
ConanPkg Type = "conan"
CondaPkg Type = "conda"
DartPubPkg Type = "dart-pub"
DebPkg Type = "deb"
DotnetPkg Type = "dotnet"
@@ -59,6 +60,7 @@ var AllPkgs = []Type{
BitnamiPkg,
CocoapodsPkg,
ConanPkg,
CondaPkg,
DartPubPkg,
DebPkg,
DotnetPkg,
@@ -109,6 +111,8 @@ func (t Type) PackageURLType() string {
return packageurl.TypeCocoapods
case ConanPkg:
return packageurl.TypeConan
case CondaPkg:
return packageurl.TypeGeneric
case DartPubPkg:
return packageurl.TypePub
case DebPkg:
@@ -206,6 +210,8 @@ func TypeByName(name string) Type {
return GemPkg
case "cargo", "crate":
return RustPkg
case "conda":
return CondaPkg
case packageurl.TypePub:
return DartPubPkg
case "dotnet": // here to support legacy use cases
+5
View File
@@ -130,6 +130,11 @@ func TestTypeFromPURL(t *testing.T) {
purl: "pkg:opam/ocaml-base-compiler@5.2.0",
expected: OpamPkg,
},
{
name: "conda",
purl: "pkg:generic/conda@1.2.3",
expected: CondaPkg,
},
}
var pkgTypes = strset.New()