Files

68 lines
1.6 KiB
Go

package llm
import (
"log"
"strings"
"sync"
)
// WorkerFunc is a generic function type for processing a chunk of items.
type WorkerFunc[T any, R any] func(chunk []T) (R, error)
// ProcessInChunks is a generic function that processes a slice of items in parallel chunks.
// It takes a slice of items, a chunk size, and a worker function to process each chunk.
// It returns a slice of results from the successful chunks and a map of failed items, keyed by the error reason.
func ProcessInChunks[T any, R any](
items []T,
chunkSize int,
worker func(chunk []T) (R, error),
) ([]R, map[string][]T) {
var wg sync.WaitGroup
var mu sync.Mutex
results := make([]R, 0)
failedChunks := make(map[string][]T)
for i := 0; i < len(items); i += chunkSize {
end := min(i+chunkSize, len(items))
chunk := items[i:end]
wg.Add(1)
go func(chunk []T) {
defer wg.Done()
result, err := worker(chunk)
if err != nil {
log.Printf("A chunk failed during parallel processing: %v", err)
// Determine a simple error key for bucketing failed chunks.
errorKey := "Processing Error"
if strings.Contains(err.Error(), "DEADLINE_EXCEEDED") {
errorKey = "Processing Timeout"
} else if strings.Contains(strings.ToLower(err.Error()), "json") {
errorKey = "Invalid Response"
}
mu.Lock()
failedChunks[errorKey] = append(failedChunks[errorKey], chunk...)
mu.Unlock()
return
}
mu.Lock()
results = append(results, result)
mu.Unlock()
}(chunk)
}
wg.Wait()
return results, failedChunks
}
func min(a, b int) int {
if a < b {
return a
}
return b
}