Use reader when scanning for package versions over reading entire binary into memory (#3558)

* use streaming readers

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* replace redis search patterns

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address PR feedback

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

internal/regex_helpers.go

@@ -1,6 +1,11 @@
 package internal
 
-import "regexp"
+import (
+	"io"
+	"regexp"
+)
+
+const readerChunkSize = 1024 * 1024
 
 // MatchNamedCaptureGroups takes a regular expression and string and returns all of the named capture group results in a map.
 // This is only for the first match in the regex. Callers shouldn't be providing regexes with multiple capture groups with the same name.
@@ -32,6 +37,89 @@ func MatchNamedCaptureGroups(regEx *regexp.Regexp, content string) map[string]st
 	return results
 }
 
+// MatchNamedCaptureGroupsFromReader matches named capture groups from a reader, assuming the pattern fits within
+// 1.5x the reader chunk size (1MB * 1.5).
+func MatchNamedCaptureGroupsFromReader(re *regexp.Regexp, r io.Reader) (map[string]string, error) {
+	results := make(map[string]string)
+	_, err := processReaderInChunks(r, readerChunkSize, matchNamedCaptureGroupsHandler(re, results))
+	if err != nil {
+		return nil, err
+	}
+	if len(results) == 0 {
+		return nil, nil
+	}
+	return results, nil
+}
+
+// MatchAnyFromReader matches any of the provided regular expressions from a reader, assuming the pattern fits within
+// 1.5x the reader chunk size (1MB * 1.5).
+func MatchAnyFromReader(r io.Reader, res ...*regexp.Regexp) (bool, error) {
+	return processReaderInChunks(r, readerChunkSize, matchAnyHandler(res))
+}
+
+func matchNamedCaptureGroupsHandler(re *regexp.Regexp, results map[string]string) func(data []byte) (bool, error) {
+	return func(data []byte) (bool, error) {
+		if match := re.FindSubmatch(data); match != nil {
+			groupNames := re.SubexpNames()
+			for i, name := range groupNames {
+				if i > 0 && name != "" {
+					results[name] = string(match[i])
+				}
+			}
+			return true, nil
+		}
+		return false, nil
+	}
+}
+
+func matchAnyHandler(res []*regexp.Regexp) func(data []byte) (bool, error) {
+	return func(data []byte) (bool, error) {
+		for _, re := range res {
+			if re.Match(data) {
+				return true, nil
+			}
+		}
+		return false, nil
+	}
+}
+
+// processReaderInChunks reads from the provided reader in chunks and calls the provided handler with each chunk + portion of the previous neighboring chunk.
+// Note that we only overlap the last half of the previous chunk with the current chunk to avoid missing matches that span chunk boundaries.
+func processReaderInChunks(rdr io.Reader, chunkSize int, handler func(data []byte) (bool, error)) (bool, error) {
+	half := chunkSize / 2
+	bufSize := chunkSize + half
+	buf := make([]byte, bufSize)
+	lastRead := 0
+
+	for {
+		offset := half
+		if lastRead < half {
+			offset = lastRead
+		}
+		start := half - offset
+		if lastRead > 0 {
+			copy(buf[start:], buf[half+offset:half+lastRead])
+		}
+		n, err := rdr.Read(buf[half:])
+		if err != nil {
+			break
+		}
+
+		// process the combined data with the handler
+		matched, handlerErr := handler(buf[start : half+n])
+		if handlerErr != nil {
+			return false, handlerErr
+		}
+		if matched {
+			return true, nil
+		}
+
+		lastRead = n
+	}
+
+	return false, nil
+}
+
 func isEmptyMap(m map[string]string) bool {
 	if len(m) == 0 {
 		return true