mirror of
https://github.com/anchore/syft.git
synced 2026-10-30 14:02:26 +01:00
Use reader when scanning for package versions over reading entire binary into memory (#3558)
* use streaming readers Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * replace redis search patterns Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * address PR feedback Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
@@ -1,6 +1,11 @@
|
||||
package internal
|
||||
|
||||
import "regexp"
|
||||
import (
|
||||
"io"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
const readerChunkSize = 1024 * 1024
|
||||
|
||||
// MatchNamedCaptureGroups takes a regular expression and string and returns all of the named capture group results in a map.
|
||||
// This is only for the first match in the regex. Callers shouldn't be providing regexes with multiple capture groups with the same name.
|
||||
@@ -32,6 +37,89 @@ func MatchNamedCaptureGroups(regEx *regexp.Regexp, content string) map[string]st
|
||||
return results
|
||||
}
|
||||
|
||||
// MatchNamedCaptureGroupsFromReader matches named capture groups from a reader, assuming the pattern fits within
|
||||
// 1.5x the reader chunk size (1MB * 1.5).
|
||||
func MatchNamedCaptureGroupsFromReader(re *regexp.Regexp, r io.Reader) (map[string]string, error) {
|
||||
results := make(map[string]string)
|
||||
_, err := processReaderInChunks(r, readerChunkSize, matchNamedCaptureGroupsHandler(re, results))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(results) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// MatchAnyFromReader matches any of the provided regular expressions from a reader, assuming the pattern fits within
|
||||
// 1.5x the reader chunk size (1MB * 1.5).
|
||||
func MatchAnyFromReader(r io.Reader, res ...*regexp.Regexp) (bool, error) {
|
||||
return processReaderInChunks(r, readerChunkSize, matchAnyHandler(res))
|
||||
}
|
||||
|
||||
func matchNamedCaptureGroupsHandler(re *regexp.Regexp, results map[string]string) func(data []byte) (bool, error) {
|
||||
return func(data []byte) (bool, error) {
|
||||
if match := re.FindSubmatch(data); match != nil {
|
||||
groupNames := re.SubexpNames()
|
||||
for i, name := range groupNames {
|
||||
if i > 0 && name != "" {
|
||||
results[name] = string(match[i])
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
func matchAnyHandler(res []*regexp.Regexp) func(data []byte) (bool, error) {
|
||||
return func(data []byte) (bool, error) {
|
||||
for _, re := range res {
|
||||
if re.Match(data) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// processReaderInChunks reads from the provided reader in chunks and calls the provided handler with each chunk + portion of the previous neighboring chunk.
|
||||
// Note that we only overlap the last half of the previous chunk with the current chunk to avoid missing matches that span chunk boundaries.
|
||||
func processReaderInChunks(rdr io.Reader, chunkSize int, handler func(data []byte) (bool, error)) (bool, error) {
|
||||
half := chunkSize / 2
|
||||
bufSize := chunkSize + half
|
||||
buf := make([]byte, bufSize)
|
||||
lastRead := 0
|
||||
|
||||
for {
|
||||
offset := half
|
||||
if lastRead < half {
|
||||
offset = lastRead
|
||||
}
|
||||
start := half - offset
|
||||
if lastRead > 0 {
|
||||
copy(buf[start:], buf[half+offset:half+lastRead])
|
||||
}
|
||||
n, err := rdr.Read(buf[half:])
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
|
||||
// process the combined data with the handler
|
||||
matched, handlerErr := handler(buf[start : half+n])
|
||||
if handlerErr != nil {
|
||||
return false, handlerErr
|
||||
}
|
||||
if matched {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
lastRead = n
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func isEmptyMap(m map[string]string) bool {
|
||||
if len(m) == 0 {
|
||||
return true
|
||||
|
||||
@@ -2,9 +2,11 @@ package internal
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMatchCaptureGroups(t *testing.T) {
|
||||
@@ -68,3 +70,149 @@ func TestMatchCaptureGroups(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchNamedCaptureGroupsFromReader(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
pattern string
|
||||
input string
|
||||
want map[string]string
|
||||
wantErr require.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "match single group",
|
||||
pattern: `(?P<key>[^1-9]+)`,
|
||||
input: "key",
|
||||
want: map[string]string{"key": "key"},
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "match multiple groups",
|
||||
pattern: `(?P<key>[^1-9]+):(?P<value>\w+)`,
|
||||
input: "key:value",
|
||||
want: map[string]string{"key": "key", "value": "value"},
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "no match",
|
||||
pattern: `(?P<key>[^1-9]+)`,
|
||||
input: "2345",
|
||||
want: nil,
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "error empty reader",
|
||||
pattern: `(?P<key>\w+)`,
|
||||
input: "",
|
||||
want: nil,
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
re := regexp.MustCompile(tt.pattern)
|
||||
r := strings.NewReader(tt.input)
|
||||
got, err := MatchNamedCaptureGroupsFromReader(re, r)
|
||||
tt.wantErr(t, err)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchAnyFromReader(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
patterns []*regexp.Regexp
|
||||
want bool
|
||||
wantErr require.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "match single pattern",
|
||||
input: "hello world",
|
||||
patterns: []*regexp.Regexp{regexp.MustCompile(`hello`)},
|
||||
want: true,
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "match multiple patterns",
|
||||
input: "test case",
|
||||
patterns: []*regexp.Regexp{regexp.MustCompile(`case`), regexp.MustCompile(`test`)},
|
||||
want: true,
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "no match",
|
||||
input: "nothing here",
|
||||
patterns: []*regexp.Regexp{regexp.MustCompile(`absent`)},
|
||||
want: false,
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "error empty reader",
|
||||
input: "",
|
||||
patterns: []*regexp.Regexp{regexp.MustCompile(`match`)},
|
||||
want: false,
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := strings.NewReader(tt.input)
|
||||
got, err := MatchAnyFromReader(r, tt.patterns...)
|
||||
tt.wantErr(t, err)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessReaderInChunks_ChunkBoundaries(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
chunkSize int
|
||||
expectedCalls []string
|
||||
returnOnChunk int
|
||||
wantErr require.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "go case",
|
||||
input: "123456789012345",
|
||||
chunkSize: 4,
|
||||
returnOnChunk: 2,
|
||||
expectedCalls: []string{"1234", "345678", "789012"},
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "no match",
|
||||
input: "123456789012345",
|
||||
chunkSize: 4,
|
||||
returnOnChunk: -1,
|
||||
expectedCalls: []string{"1234", "345678", "789012", "12345"},
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var actualCalls []string
|
||||
var current int
|
||||
handler := func(data []byte) (bool, error) {
|
||||
actualCalls = append(actualCalls, string(data))
|
||||
if current == tt.returnOnChunk {
|
||||
return true, nil
|
||||
}
|
||||
current++
|
||||
return false, nil
|
||||
}
|
||||
r := strings.NewReader(tt.input)
|
||||
got, err := processReaderInChunks(r, tt.chunkSize, handler)
|
||||
tt.wantErr(t, err)
|
||||
if tt.returnOnChunk == -1 {
|
||||
assert.False(t, got)
|
||||
} else {
|
||||
assert.True(t, got)
|
||||
}
|
||||
assert.Equal(t, tt.expectedCalls, actualCalls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user