fix: rebar lock file decoding panic (#1628)

This commit is contained in:
Keith Zantow 2023-03-01 10:08:29 -05:00 committed by GitHub
parent 24584a4d27
commit 2e6e3b0c74
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 651 additions and 153 deletions

View File

@ -0,0 +1,235 @@
package erlang
import (
"bytes"
"fmt"
"io"
"strings"
"unicode"
)
type erlangNode struct {
value interface{}
}
func (e erlangNode) Slice() []erlangNode {
out, ok := e.value.([]erlangNode)
if ok {
return out
}
return []erlangNode{}
}
func (e erlangNode) String() string {
out, ok := e.value.(string)
if ok {
return out
}
return ""
}
func (e erlangNode) Get(index int) erlangNode {
s := e.Slice()
if len(s) > index {
return s[index]
}
return erlangNode{}
}
func node(value interface{}) erlangNode {
return erlangNode{
value: value,
}
}
// parseErlang basic parser for erlang, used by rebar.lock
func parseErlang(reader io.Reader) (erlangNode, error) {
data, err := io.ReadAll(reader)
if err != nil {
return node(nil), err
}
out := erlangNode{
value: []erlangNode{},
}
i := 0
for i < len(data) {
item, err := parseErlangBlock(data, &i)
if err != nil {
return node(nil), fmt.Errorf("%w\n%s", err, printError(data, i))
}
skipWhitespace(data, &i)
if i, ok := item.value.(string); ok && i == "." {
continue
}
out.value = append(out.value.([]erlangNode), item)
}
return out, nil
}
func printError(data []byte, i int) string {
line := 1
char := 1
prev := []string{}
curr := bytes.Buffer{}
for idx, c := range data {
if c == '\n' {
prev = append(prev, curr.String())
curr.Reset()
if idx >= i {
break
} else {
line++
}
char = 1
continue
}
if idx < i {
char++
}
curr.WriteByte(c)
}
l1 := fmt.Sprintf("%d", line-1)
l2 := fmt.Sprintf("%d", line)
if len(l1) < len(l2) {
l1 = " " + l1
}
sep := ": "
lines := ""
if len(prev) > 1 {
lines += fmt.Sprintf("%s%s%s\n", l1, sep, prev[len(prev)-2])
}
if len(prev) > 0 {
lines += fmt.Sprintf("%s%s%s\n", l2, sep, prev[len(prev)-1])
}
pointer := strings.Repeat(" ", len(l2)+len(sep)+char-1) + "^"
return fmt.Sprintf("line: %v, char: %v\n%s%s", line, char, lines, pointer)
}
func skipWhitespace(data []byte, i *int) {
for *i < len(data) && isWhitespace(data[*i]) {
*i++
}
}
func parseErlangBlock(data []byte, i *int) (erlangNode, error) {
block, err := parseErlangNode(data, i)
if err != nil {
return node(nil), err
}
skipWhitespace(data, i)
*i++ // skip the trailing .
return block, nil
}
func parseErlangNode(data []byte, i *int) (erlangNode, error) {
skipWhitespace(data, i)
c := data[*i]
switch c {
case '[', '{':
return parseErlangList(data, i)
case '"':
return parseErlangString(data, i)
case '<':
return parseErlangAngleString(data, i)
}
if isLiteral(c) {
return parseErlangLiteral(data, i)
}
return erlangNode{}, fmt.Errorf("invalid literal character: %s", string(c))
}
func isWhitespace(c byte) bool {
return unicode.IsSpace(rune(c))
}
func isLiteral(c byte) bool {
r := rune(c)
return unicode.IsNumber(r) || unicode.IsLetter(r) || r == '.' || r == '_'
}
func parseErlangLiteral(data []byte, i *int) (erlangNode, error) {
var buf bytes.Buffer
for *i < len(data) {
c := data[*i]
if isLiteral(c) {
buf.WriteByte(c)
} else {
break
}
*i++
}
return node(buf.String()), nil
}
func parseErlangAngleString(data []byte, i *int) (erlangNode, error) {
*i += 2
out, err := parseErlangString(data, i)
*i += 2
return out, err
}
func parseErlangString(data []byte, i *int) (erlangNode, error) {
delim := data[*i]
*i++
var buf bytes.Buffer
for *i < len(data) {
c := data[*i]
if c == delim {
*i++
return node(buf.String()), nil
}
if c == '\\' {
*i++
if len(data) >= *i {
return node(nil), fmt.Errorf("invalid escape without closed string at %d", *i)
}
c = data[*i]
}
buf.WriteByte(c)
*i++
}
return node(buf.String()), nil
}
func parseErlangList(data []byte, i *int) (erlangNode, error) {
*i++
out := erlangNode{
value: []erlangNode{},
}
for *i < len(data) {
item, err := parseErlangNode(data, i)
if err != nil {
return node(nil), err
}
out.value = append(out.value.([]erlangNode), item)
skipWhitespace(data, i)
c := data[*i]
switch c {
case ',':
*i++
continue
case ']', '}':
*i++
return out, nil
default:
return node(nil), fmt.Errorf("unexpected character: %s", string(c))
}
}
return out, nil
}

View File

@ -0,0 +1,72 @@
package erlang
import (
"bytes"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_parseErlang(t *testing.T) {
tests := []struct {
name string
content string
wantErr require.ErrorAssertionFunc
}{
{
name: "basic valid content",
content: `
{"1.2.0",
[{<<"bcrypt">>,{pkg,<<"bcrypt">>,<<"1.1.5">>},0},
{<<"bson">>,
{git,"https://github.com/comtihon/bson-erlang",
{ref,"14308ab927cfa69324742c3de720578094e0bb19"}},
1},
{<<"syslog">>,{pkg,<<"syslog">>,<<"1.1.0">>},0},
{<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.0">>},1},
{<<"vernemq_dev">>,
{git,"https://github.com/vernemq/vernemq_dev.git",
{ref,"6d622aa8c901ae7777433aef2bd049e380c474a6"}},
0}]
}.
[
{pkg_hash,[
{<<"bcrypt">>, <<"A6763BD4E1AF46D34776F85B7995E63A02978DE110C077E9570ED17006E03386">>},
{<<"unicode_util_compat">>, <<"BC84380C9AB48177092F43AC89E4DFA2C6D62B40B8BD132B1059ECC7232F9A78">>}]},
{pkg_hash_ext,[
{<<"bcrypt">>, <<"3418821BC17CE6E96A4A77D1A88D7485BF783E212069FACFC79510AFBFF95352">>},
{<<"unicode_util_compat">>, <<"25EEE6D67DF61960CF6A794239566599B09E17E668D3700247BC498638152521">>}]}
].`,
},
{
name: "invalid string content",
wantErr: require.Error,
content: `
{"1.2.0
">>},
].`,
},
{
name: "invalid content",
wantErr: require.Error,
content: `
{"1.2.0"}.
].`,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
value, err := parseErlang(bytes.NewReader([]byte(test.content)))
if test.wantErr == nil {
require.NoError(t, err)
} else {
test.wantErr(t, err)
}
assert.IsType(t, erlangNode{}, value)
})
}
}

View File

@ -1,12 +1,6 @@
package erlang package erlang
import ( import (
"bufio"
"errors"
"fmt"
"io"
"regexp"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
@ -14,57 +8,44 @@ import (
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/source"
) )
// integrity check // parseRebarLock parses a rebar.lock and returns the discovered Elixir packages.
var _ generic.Parser = parseRebarLock //
//nolint:funlen
var rebarLockDelimiter = regexp.MustCompile(`[\[{<">},: \]\n]+`)
// parseMixLock parses a mix.lock and returns the discovered Elixir packages.
func parseRebarLock(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { func parseRebarLock(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
r := bufio.NewReader(reader) doc, err := parseErlang(reader)
if err != nil {
return nil, nil, err
}
pkgMap := make(map[string]*pkg.Package) pkgMap := make(map[string]*pkg.Package)
var names []string // rebar.lock structure is:
loop: // [
for { // ["version", [
line, err := r.ReadString('\n') // [<<"package-name">>, ["version-type", "version"]...
switch { // ],
case errors.Is(io.EOF, err): // [
break loop // [pkg_hash, [
case err != nil: // [<<"package-name">>, <<"package-hash">>]
// TODO: return partial result and warn // ],
return nil, nil, fmt.Errorf("failed to parse rebar.lock file: %w", err) // [pkg_hash_ext, [
} // [<<"package-name">>, <<"package-hash">>]
tokens := rebarLockDelimiter.Split(line, -1) // ]
if len(tokens) < 4 { // ]
continue // ]
}
if len(tokens) < 5 {
name, hash := tokens[1], tokens[2]
sourcePkg := pkgMap[name]
metadata, ok := sourcePkg.Metadata.(pkg.RebarLockMetadata)
if !ok {
log.WithFields("package", name).Warn("unable to extract rebar.lock metadata to add hash metadata")
continue
}
if metadata.PkgHash == "" { versions := doc.Get(0)
metadata.PkgHash = hash deps := versions.Get(1)
} else {
metadata.PkgHashExt = hash
}
sourcePkg.Metadata = metadata
continue
}
name, version := tokens[1], tokens[4]
sourcePkg := pkg.Package{ for _, dep := range deps.Slice() {
Name: name, name := dep.Get(0).String()
Version: version, versionNode := dep.Get(1)
Language: pkg.Erlang, versionType := versionNode.Get(0).String()
Type: pkg.HexPkg, version := versionNode.Get(2).String()
MetadataType: pkg.RebarLockMetadataType,
// capture git hashes if no version specified
if versionType == "git" {
version = versionNode.Get(2).Get(1).String()
} }
p := newPackage(pkg.RebarLockMetadata{ p := newPackage(pkg.RebarLockMetadata{
@ -72,15 +53,45 @@ loop:
Version: version, Version: version,
}) })
names = append(names, name) pkgMap[name] = &p
pkgMap[sourcePkg.Name] = &p }
hashes := doc.Get(1)
for _, hashStruct := range hashes.Slice() {
hashType := hashStruct.Get(0).String()
for _, hashValue := range hashStruct.Get(1).Slice() {
name := hashValue.Get(0).String()
hash := hashValue.Get(1).String()
sourcePkg := pkgMap[name]
if sourcePkg == nil {
log.WithFields("package", name).Warn("unable find source package")
continue
}
metadata, ok := sourcePkg.Metadata.(pkg.RebarLockMetadata)
if !ok {
log.WithFields("package", name).Warn("unable to extract rebar.lock metadata to add hash metadata")
continue
}
switch hashType {
case "pkg_hash":
metadata.PkgHash = hash
case "pkg_hash_ext":
metadata.PkgHashExt = hash
}
sourcePkg.Metadata = metadata
}
} }
var packages []pkg.Package var packages []pkg.Package
for _, name := range names { for _, p := range pkgMap {
p := pkgMap[name]
p.SetID() p.SetID()
packages = append(packages, *p) packages = append(packages, *p)
} }
return packages, nil, nil return packages, nil, nil
} }
// integrity check
var _ generic.Parser = parseRebarLock

View File

@ -9,7 +9,13 @@ import (
) )
func TestParseRebarLock(t *testing.T) { func TestParseRebarLock(t *testing.T) {
expected := []pkg.Package{ tests := []struct {
fixture string
expected []pkg.Package
}{
{
fixture: "test-fixtures/rebar.lock",
expected: []pkg.Package{
{ {
Name: "certifi", Name: "certifi",
Version: "2.9.0", Version: "2.9.0",
@ -108,12 +114,154 @@ func TestParseRebarLock(t *testing.T) {
PkgHashExt: "25EEE6D67DF61960CF6A794239566599B09E17E668D3700247BC498638152521", PkgHashExt: "25EEE6D67DF61960CF6A794239566599B09E17E668D3700247BC498638152521",
}, },
}, },
},
},
{
fixture: "test-fixtures/rebar-2.lock",
expected: []pkg.Package{
//[{<<"bcrypt">>,{pkg,<<"bcrypt">>,<<"1.1.5">>},0},
// {<<"bcrypt">>, <<"A6763BD4E1AF46D34776F85B7995E63A02978DE110C077E9570ED17006E03386">>},
// {<<"bcrypt">>, <<"3418821BC17CE6E96A4A77D1A88D7485BF783E212069FACFC79510AFBFF95352">>},
{
Name: "bcrypt",
Version: "1.1.5",
Language: pkg.Erlang,
Type: pkg.HexPkg,
PURL: "pkg:hex/bcrypt@1.1.5",
MetadataType: pkg.RebarLockMetadataType,
Metadata: pkg.RebarLockMetadata{
Name: "bcrypt",
Version: "1.1.5",
PkgHash: "A6763BD4E1AF46D34776F85B7995E63A02978DE110C077E9570ED17006E03386",
PkgHashExt: "3418821BC17CE6E96A4A77D1A88D7485BF783E212069FACFC79510AFBFF95352",
},
},
// {<<"bson">>,
// {git,"https://github.com/comtihon/bson-erlang",
// {ref,"14308ab927cfa69324742c3de720578094e0bb19"}},
// 1},
{
Name: "bson",
Version: "14308ab927cfa69324742c3de720578094e0bb19",
Language: pkg.Erlang,
Type: pkg.HexPkg,
PURL: "pkg:hex/bson@14308ab927cfa69324742c3de720578094e0bb19",
MetadataType: pkg.RebarLockMetadataType,
Metadata: pkg.RebarLockMetadata{
Name: "bson",
Version: "14308ab927cfa69324742c3de720578094e0bb19",
},
},
// {<<"certifi">>,{pkg,<<"certifi">>,<<"2.9.0">>},1},
// {<<"certifi">>, <<"6F2A475689DD47F19FB74334859D460A2DC4E3252A3324BD2111B8F0429E7E21">>}, {<<"stdout_formatter">>, <<"EC24868D8619757A68F0798357C7190807A1CFC42CE90C18C23760E59249A21A">>},
// {<<"certifi">>, <<"266DA46BDB06D6C6D35FDE799BCB28D36D985D424AD7C08B5BB48F5B5CDD4641">>},
{
Name: "certifi",
Version: "2.9.0",
Language: pkg.Erlang,
Type: pkg.HexPkg,
PURL: "pkg:hex/certifi@2.9.0",
MetadataType: pkg.RebarLockMetadataType,
Metadata: pkg.RebarLockMetadata{
Name: "certifi",
Version: "2.9.0",
PkgHash: "6F2A475689DD47F19FB74334859D460A2DC4E3252A3324BD2111B8F0429E7E21",
PkgHashExt: "266DA46BDB06D6C6D35FDE799BCB28D36D985D424AD7C08B5BB48F5B5CDD4641",
},
},
// {<<"stdout_formatter">>,{pkg,<<"stdout_formatter">>,<<"0.2.3">>},0},
// {<<"stdout_formatter">>, <<"EC24868D8619757A68F0798357C7190807A1CFC42CE90C18C23760E59249A21A">>},
// {<<"stdout_formatter">>, <<"6B9CAAD8930006F9BB35680C5D3311917AC67690C3AF1BA018623324C015ABE5">>},
{
Name: "stdout_formatter",
Version: "0.2.3",
Language: pkg.Erlang,
Type: pkg.HexPkg,
PURL: "pkg:hex/stdout_formatter@0.2.3",
MetadataType: pkg.RebarLockMetadataType,
Metadata: pkg.RebarLockMetadata{
Name: "stdout_formatter",
Version: "0.2.3",
PkgHash: "EC24868D8619757A68F0798357C7190807A1CFC42CE90C18C23760E59249A21A",
PkgHashExt: "6B9CAAD8930006F9BB35680C5D3311917AC67690C3AF1BA018623324C015ABE5",
},
},
// {<<"swc">>,
// {git,"https://github.com/vernemq/ServerWideClocks.git",
// {ref,"4835239dca5a5f4ac7202dd94d7effcaa617d575"}},
// 0},
{
Name: "swc",
Version: "4835239dca5a5f4ac7202dd94d7effcaa617d575",
Language: pkg.Erlang,
Type: pkg.HexPkg,
PURL: "pkg:hex/swc@4835239dca5a5f4ac7202dd94d7effcaa617d575",
MetadataType: pkg.RebarLockMetadataType,
Metadata: pkg.RebarLockMetadata{
Name: "swc",
Version: "4835239dca5a5f4ac7202dd94d7effcaa617d575",
},
},
// {<<"syslog">>,{pkg,<<"syslog">>,<<"1.1.0">>},0},
// {<<"syslog">>, <<"6419A232BEA84F07B56DC575225007FFE34D9FDC91ABE6F1B2F254FD71D8EFC2">>},
// {<<"syslog">>, <<"4C6A41373C7E20587BE33EF841D3DE6F3BEBA08519809329ECC4D27B15B659E1">>},
{
Name: "syslog",
Version: "1.1.0",
Language: pkg.Erlang,
Type: pkg.HexPkg,
PURL: "pkg:hex/syslog@1.1.0",
MetadataType: pkg.RebarLockMetadataType,
Metadata: pkg.RebarLockMetadata{
Name: "syslog",
Version: "1.1.0",
PkgHash: "6419A232BEA84F07B56DC575225007FFE34D9FDC91ABE6F1B2F254FD71D8EFC2",
PkgHashExt: "4C6A41373C7E20587BE33EF841D3DE6F3BEBA08519809329ECC4D27B15B659E1",
},
},
// {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.0">>},1},
// {<<"unicode_util_compat">>, <<"BC84380C9AB48177092F43AC89E4DFA2C6D62B40B8BD132B1059ECC7232F9A78">>}]},
// {<<"unicode_util_compat">>, <<"25EEE6D67DF61960CF6A794239566599B09E17E668D3700247BC498638152521">>}]}
{
Name: "unicode_util_compat",
Version: "0.7.0",
Language: pkg.Erlang,
Type: pkg.HexPkg,
PURL: "pkg:hex/unicode_util_compat@0.7.0",
MetadataType: pkg.RebarLockMetadataType,
Metadata: pkg.RebarLockMetadata{
Name: "unicode_util_compat",
Version: "0.7.0",
PkgHash: "BC84380C9AB48177092F43AC89E4DFA2C6D62B40B8BD132B1059ECC7232F9A78",
PkgHashExt: "25EEE6D67DF61960CF6A794239566599B09E17E668D3700247BC498638152521",
},
},
// {<<"vernemq_dev">>,
// {git,"https://github.com/vernemq/vernemq_dev.git",
// {ref,"6d622aa8c901ae7777433aef2bd049e380c474a6"}},
// 0}]}.
{
Name: "vernemq_dev",
Version: "6d622aa8c901ae7777433aef2bd049e380c474a6",
Language: pkg.Erlang,
Type: pkg.HexPkg,
PURL: "pkg:hex/vernemq_dev@6d622aa8c901ae7777433aef2bd049e380c474a6",
MetadataType: pkg.RebarLockMetadataType,
Metadata: pkg.RebarLockMetadata{
Name: "vernemq_dev",
Version: "6d622aa8c901ae7777433aef2bd049e380c474a6",
},
},
},
},
} }
fixture := "test-fixtures/rebar.lock" for _, test := range tests {
t.Run(test.fixture, func(t *testing.T) {
// TODO: relationships are not under test // TODO: relationships are not under test
var expectedRelationships []artifact.Relationship var expectedRelationships []artifact.Relationship
pkgtest.TestFileParser(t, fixture, parseRebarLock, expected, expectedRelationships) pkgtest.TestFileParser(t, test.fixture, parseRebarLock, test.expected, expectedRelationships)
})
}
} }

View File

@ -0,0 +1,32 @@
{"1.2.0",
[{<<"bcrypt">>,{pkg,<<"bcrypt">>,<<"1.1.5">>},0},
{<<"bson">>,
{git,"https://github.com/comtihon/bson-erlang",
{ref,"14308ab927cfa69324742c3de720578094e0bb19"}},
1},
{<<"certifi">>,{pkg,<<"certifi">>,<<"2.9.0">>},1},
{<<"stdout_formatter">>,{pkg,<<"stdout_formatter">>,<<"0.2.3">>},0},
{<<"swc">>,
{git,"https://github.com/vernemq/ServerWideClocks.git",
{ref,"4835239dca5a5f4ac7202dd94d7effcaa617d575"}},
0},
{<<"syslog">>,{pkg,<<"syslog">>,<<"1.1.0">>},0},
{<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.0">>},1},
{<<"vernemq_dev">>,
{git,"https://github.com/vernemq/vernemq_dev.git",
{ref,"6d622aa8c901ae7777433aef2bd049e380c474a6"}},
0}]}.
[
{pkg_hash,[
{<<"bcrypt">>, <<"A6763BD4E1AF46D34776F85B7995E63A02978DE110C077E9570ED17006E03386">>},
{<<"certifi">>, <<"6F2A475689DD47F19FB74334859D460A2DC4E3252A3324BD2111B8F0429E7E21">>},
{<<"stdout_formatter">>, <<"EC24868D8619757A68F0798357C7190807A1CFC42CE90C18C23760E59249A21A">>},
{<<"syslog">>, <<"6419A232BEA84F07B56DC575225007FFE34D9FDC91ABE6F1B2F254FD71D8EFC2">>},
{<<"unicode_util_compat">>, <<"BC84380C9AB48177092F43AC89E4DFA2C6D62B40B8BD132B1059ECC7232F9A78">>}]},
{pkg_hash_ext,[
{<<"bcrypt">>, <<"3418821BC17CE6E96A4A77D1A88D7485BF783E212069FACFC79510AFBFF95352">>},
{<<"certifi">>, <<"266DA46BDB06D6C6D35FDE799BCB28D36D985D424AD7C08B5BB48F5B5CDD4641">>},
{<<"stdout_formatter">>, <<"6B9CAAD8930006F9BB35680C5D3311917AC67690C3AF1BA018623324C015ABE5">>},
{<<"syslog">>, <<"4C6A41373C7E20587BE33EF841D3DE6F3BEBA08519809329ECC4D27B15B659E1">>},
{<<"unicode_util_compat">>, <<"25EEE6D67DF61960CF6A794239566599B09E17E668D3700247BC498638152521">>}]}
].