Files
eleith 07870d00ee refactor: enforce strict user categories in AI view
- Remove default category injection in config to distinguish between explicit user categories and auto-mode.
- Update AI prompts:
    - Rename initialGroupingPrompt to openGroupingPrompt.
    - Add strictGroupingPrompt for enforcing user-defined categories.
- Update AI grouping logic:
    - Use strictGroupingPrompt and skip consolidation if categories are provided.
    - Use openGroupingPrompt and perform consolidation if categories are empty.
- Add Uncategorized group merging logic.
- Add tests for strict user mode and merging logic.
2025-12-17 14:16:15 -08:00

649 lines
20 KiB
Go

package view
import (
"bytes"
"context"
"encoding/json"
"fmt"
"log"
"miniflux-digest/internal/app/services"
"miniflux-digest/internal/config"
"miniflux-digest/internal/llm"
"miniflux-digest/internal/models"
"miniflux-digest/internal/utils"
"sort"
"strings"
"sync"
"text/template"
"google.golang.org/genai"
)
const (
MaxEntryContentLengthForLLM = 250
MaxEntriesPerJob = 150
)
type llmEntry struct {
ID int64 `json:"id"`
Title string `json:"title"`
URL string `json:"url"`
Content string `json:"content"`
FeedTitle string `json:"feed_title"`
}
func GroupAIEntries(ctx context.Context, entries []*models.Entry, llmService services.LLMService, categories []config.ConfigCategory) (map[string][]*models.Entry, map[int64]bool, map[string][]*models.Entry) {
const chunkSize = 200
// Prepare for processing
entryMap := make(map[int64]*models.Entry)
for _, entry := range entries {
entryMap[entry.ID] = entry
}
var promptTemplateStr string
if len(categories) > 0 {
promptTemplateStr = strictGroupingPrompt
} else {
promptTemplateStr = openGroupingPrompt
}
promptTemplate, err := template.New("grouping").Parse(promptTemplateStr)
if err != nil {
failedChunks := make(map[string][]*models.Entry)
failedChunks["Prompt template parsing failed"] = entries
return nil, nil, failedChunks
}
workerConfig := llmWorkerConfig{
promptTemplate: promptTemplate,
promptData: categories,
responseSchema: InitialGroupingResponseSchema,
prepareEntries: func(entries []*models.Entry) any {
return prepareLLMEntries(entries)
},
}
worker := createLLMWorker[*InitialGroupingResponse](ctx, llmService, workerConfig, "")
// Run the parallel processing
chunkResponses, failedChunks := llm.ProcessInChunks(entries, chunkSize, worker)
// Process the results
rawGroups := make(map[string][]*models.Entry)
groupedEntryIDs := make(map[int64]bool)
for _, response := range chunkResponses {
for _, group := range response.Groups {
if _, ok := rawGroups[group.Title]; !ok {
rawGroups[group.Title] = []*models.Entry{}
}
for _, entryID := range group.EntryIDs {
if _, exists := groupedEntryIDs[entryID]; exists {
continue
}
if entry, ok := entryMap[entryID]; ok {
rawGroups[group.Title] = append(rawGroups[group.Title], entry)
groupedEntryIDs[entryID] = true
}
}
}
}
return rawGroups, groupedEntryIDs, failedChunks
}
func consolidatePrimaryGroups(ctx context.Context, rawGroups map[string][]*models.Entry, llmService services.LLMService) ([]*models.PrimaryGroupDigestData, error) {
var initialTitles []string
for title := range rawGroups {
initialTitles = append(initialTitles, title)
}
if len(initialTitles) <= 10 { // No need to consolidate if we already have a small number of groups
return convertRawGroupsToDigestData(rawGroups), nil
}
tmpl, err := template.New("consolidation").Parse(consolidationPrompt)
if err != nil {
return nil, fmt.Errorf("failed to parse consolidation prompt template: %w", err)
}
var prompt bytes.Buffer
if err := tmpl.Execute(&prompt, initialTitles); err != nil {
return nil, fmt.Errorf("failed to execute consolidation prompt template: %w", err)
}
var response ConsolidationResponse
if err := llmService.GenerateContentWithResponse(ctx, prompt.String(), ConsolidationResponseSchema, &response); err != nil {
// Fallback: If LLM fails, use the original, unconsolidated groups
log.Printf("LLM consolidation failed, falling back to original groups: %v", err)
return convertRawGroupsToDigestData(rawGroups), nil
}
// Merge groups based on consolidation response
consolidatedGroups := make(map[string][]*models.Entry)
mappedOldTitles := make(map[string]bool)
for _, cg := range response.ConsolidatedGroups {
if _, ok := consolidatedGroups[cg.NewTitle]; !ok {
consolidatedGroups[cg.NewTitle] = []*models.Entry{}
}
for _, oldTitle := range cg.OldTitles {
if entries, ok := rawGroups[oldTitle]; ok {
consolidatedGroups[cg.NewTitle] = append(consolidatedGroups[cg.NewTitle], entries...)
mappedOldTitles[oldTitle] = true
}
}
}
// Add any groups that the LLM may have missed
for oldTitle, entries := range rawGroups {
if !mappedOldTitles[oldTitle] {
consolidatedGroups[oldTitle] = entries
}
}
return convertRawGroupsToDigestData(consolidatedGroups), nil
}
func convertRawGroupsToDigestData(rawGroups map[string][]*models.Entry) []*models.PrimaryGroupDigestData {
var digestData []*models.PrimaryGroupDigestData
currentPrimaryGroupID := int64(1)
for title, entries := range rawGroups {
if len(entries) == 0 {
continue
}
newPGDD := &models.PrimaryGroupDigestData{
ID: currentPrimaryGroupID,
Title: title,
Slug: utils.Slugify(title),
SubGroups: []*models.EntryGroup{
{
Title: title, // Initial sub-group is the same as the primary group
Entries: entries,
Slug: utils.Slugify(title),
},
},
Summary: "",
TotalEntries: len(entries),
TotalFeeds: getUniqueFeedIDs(entries),
}
digestData = append(digestData, newPGDD)
currentPrimaryGroupID++
}
return digestData
}
func getSummaryForGroup(ctx context.Context, groupTitle string, entries []*models.Entry, llmService services.LLMService) (string, error) {
llmResponseWorker := createLLMWorker[SummaryResponse](ctx, llmService, llmWorkerConfig{
promptFormat: summaryPrompt,
responseSchema: SummaryResponseSchema,
prepareEntries: func(entries []*models.Entry) any {
return prepareLLMEntries(entries)
},
}, groupTitle)
summaryWorker := func(chunk []*models.Entry) (string, error) {
resp, err := llmResponseWorker(chunk)
if err != nil {
return "", err
}
return resp.Summary, nil
}
partialSummaries, failedChunks := llm.ProcessInChunks(entries, MaxEntriesPerJob, summaryWorker)
if len(partialSummaries) == 0 {
return "", fmt.Errorf("all summary chunks failed for group '%s'", groupTitle)
}
// Reduce step: summarize the summaries
summariesString := strings.Join(partialSummaries, "\n\n---\n\n")
finalSummaryPromptStr := fmt.Sprintf(summaryOfSummariesPrompt, summariesString)
var finalSummaryResponse SummaryResponse
if err := llmService.GenerateContentWithResponse(ctx, finalSummaryPromptStr, SummaryResponseSchema, &finalSummaryResponse); err != nil {
return "", fmt.Errorf("summary of summaries failed: %w", err)
}
finalSummary := finalSummaryResponse.Summary
var failedItems []*models.Entry
for _, items := range failedChunks {
failedItems = append(failedItems, items...)
}
if len(failedItems) > 0 {
finalSummary += fmt.Sprintf("\n\n(Note: %d entries could not be included in the summary due to processing errors.)", len(failedItems))
}
return finalSummary, nil
}
func getSubGroupsForGroup(ctx context.Context, groupTitle string, entries []*models.Entry, llmService services.LLMService) ([]*models.EntryGroup, error) {
entryMap := make(map[int64]*models.Entry)
for _, entry := range entries {
entryMap[entry.ID] = entry
}
subGroupWorker := createLLMWorker[*SubGroupingResponse](ctx, llmService, llmWorkerConfig{
promptFormat: subGroupingPrompt,
responseSchema: SubGroupingResponseSchema,
prepareEntries: func(entries []*models.Entry) any {
return prepareLLMEntries(entries)
},
}, groupTitle)
partialSubGroupResponses, failedChunks := llm.ProcessInChunks(entries, MaxEntriesPerJob, subGroupWorker)
if len(partialSubGroupResponses) == 0 {
return nil, fmt.Errorf("all sub-grouping chunks failed for group '%s'", groupTitle)
}
// Reduce step: merge sub-groups
mergedSubGroups := make(map[string][]*models.Entry)
allGroupedEntryIDs := make(map[int64]bool)
for _, response := range partialSubGroupResponses {
for _, llmSubGroup := range response.SubGroups {
if _, ok := mergedSubGroups[llmSubGroup.Title]; !ok {
mergedSubGroups[llmSubGroup.Title] = []*models.Entry{}
}
for _, entryID := range llmSubGroup.EntryIDs {
if _, exists := allGroupedEntryIDs[entryID]; exists {
continue // Avoid duplicating entries if they appear in multiple chunk responses
}
if entry, ok := entryMap[entryID]; ok {
mergedSubGroups[llmSubGroup.Title] = append(mergedSubGroups[llmSubGroup.Title], entry)
allGroupedEntryIDs[entryID] = true
}
}
}
}
var finalSubGroups []*models.EntryGroup
for title, groupEntries := range mergedSubGroups {
if len(groupEntries) > 0 {
finalSubGroups = append(finalSubGroups, &models.EntryGroup{
Title: title,
Entries: groupEntries,
Slug: utils.Slugify(title),
TotalEntries: len(groupEntries),
TotalFeeds: getUniqueFeedIDs(groupEntries),
})
}
}
// Handle all ungrouped entries from all chunks
var allUngroupedEntries []*models.Entry
for _, entry := range entries {
if !allGroupedEntryIDs[entry.ID] {
allUngroupedEntries = append(allUngroupedEntries, entry)
}
}
// Collect all failed items from the map into a single slice
var allFailedItems []*models.Entry
for _, items := range failedChunks {
allFailedItems = append(allFailedItems, items...)
}
// Add items from failed chunks to a separate group
if len(allFailedItems) > 0 {
finalSubGroups = append(finalSubGroups, &models.EntryGroup{
Title: "Uncategorized - Processing Failed",
Entries: allFailedItems,
Slug: "uncategorized-processing-failed",
TotalEntries: len(allFailedItems),
TotalFeeds: getUniqueFeedIDs(allFailedItems),
})
}
if len(allUngroupedEntries) > 0 {
// Find existing "Uncategorized" group to append to, if it exists
var uncategorizedGroup *models.EntryGroup
for _, sg := range finalSubGroups {
if sg.Title == "Uncategorized" {
uncategorizedGroup = sg
break
}
}
if uncategorizedGroup != nil {
uncategorizedGroup.Entries = append(uncategorizedGroup.Entries, allUngroupedEntries...)
} else {
finalSubGroups = append(finalSubGroups, &models.EntryGroup{
Title: "Uncategorized",
Entries: allUngroupedEntries,
Slug: "uncategorized",
TotalEntries: len(allUngroupedEntries),
TotalFeeds: getUniqueFeedIDs(allUngroupedEntries),
})
}
}
return finalSubGroups, nil
}
func ProcessPrimaryGroupWithAI(ctx context.Context, pg *models.PrimaryGroup, llmService services.LLMService) (string, []*models.EntryGroup, error) {
entriesToProcess := pg.Entries
// If the group is large, process in chunks concurrently.
var finalSummary string
var finalSubGroups []*models.EntryGroup
var errSummary, errSubgroup error
var wg sync.WaitGroup
wg.Add(2)
// Goroutine for Summarization
go func() {
defer wg.Done()
finalSummary, errSummary = getSummaryForGroup(ctx, pg.Title, entriesToProcess, llmService)
}()
// Goroutine for Sub-grouping
go func() {
defer wg.Done()
finalSubGroups, errSubgroup = getSubGroupsForGroup(ctx, pg.Title, entriesToProcess, llmService)
}()
wg.Wait()
// Combine errors if any. For now, we just log them but don't halt everything.
if errSummary != nil {
log.Printf("Error generating summary for group '%s': %v", pg.Title, errSummary)
}
if errSubgroup != nil {
log.Printf("Error generating sub-groups for group '%s': %v", pg.Title, errSubgroup)
// If sub-grouping completely failed, create a single group as a fallback.
if finalSubGroups == nil {
finalSubGroups = []*models.EntryGroup{{
Title: "Uncategorized",
Entries: entriesToProcess,
Slug: "uncategorized",
TotalEntries: len(entriesToProcess),
TotalFeeds: getUniqueFeedIDs(entriesToProcess),
}}
}
}
return finalSummary, finalSubGroups, nil
}
func getUniqueFeedIDs(entries []*models.Entry) int {
uniqueFeedIDs := make(map[int64]bool)
for _, entry := range entries {
uniqueFeedIDs[entry.FeedID] = true
}
return len(uniqueFeedIDs)
}
func prepareLLMEntries(entries []*models.Entry) []llmEntry {
var llmEntries []llmEntry
for _, entry := range entries {
content := entry.Content
if len(content) > MaxEntryContentLengthForLLM {
content = content[:MaxEntryContentLengthForLLM]
}
llmEntries = append(llmEntries, llmEntry{
ID: entry.ID,
Title: entry.Title,
URL: entry.URL,
Content: content,
FeedTitle: entry.FeedTitle,
})
}
return llmEntries
}
func BuildDigestDataByAI(entries []*models.Entry, ctx context.Context, llmService services.LLMService, categories []config.ConfigCategory) []*models.PrimaryGroupDigestData {
rawGroups, groupedEntryIDs, failedChunks, fallbackGroups := initialAIGGrouping(ctx, entries, llmService, categories)
if fallbackGroups != nil {
return fallbackGroups
}
var highLevelGroups []*models.PrimaryGroupDigestData
if len(categories) == 0 {
// Auto Mode: Consolidate the LLM's open-ended groups
highLevelGroups = consolidateGroups(ctx, rawGroups, llmService)
} else {
// User Mode: Use raw groups directly (strict adherence)
highLevelGroups = convertRawGroupsToDigestData(rawGroups)
}
highLevelGroups = handleFailedAndUngroupedEntries(highLevelGroups, entries, groupedEntryIDs, failedChunks)
allPrimaryGroups := processHighLevelGroupsConcurrently(ctx, highLevelGroups, llmService)
sortPrimaryGroups(allPrimaryGroups)
return allPrimaryGroups
}
func initialAIGGrouping(ctx context.Context, entries []*models.Entry, llmService services.LLMService, categories []config.ConfigCategory) (map[string][]*models.Entry, map[int64]bool, map[string][]*models.Entry, []*models.PrimaryGroupDigestData) {
rawGroups, groupedEntryIDs, failedChunks := GroupAIEntries(ctx, entries, llmService, categories)
// If all chunks failed and we have no raw groups, fall back to category grouping.
if len(rawGroups) == 0 && len(failedChunks) > 0 {
log.Printf("All AI grouping chunks failed, falling back to category grouping.")
categoryGroups := BuildDigestDataByCategory(entries)
if len(categoryGroups) == 1 && len(entries) > 1 {
return nil, nil, nil, BuildDigestDataByDate(entries)
}
return nil, nil, nil, categoryGroups
}
return rawGroups, groupedEntryIDs, failedChunks, nil
}
func consolidateGroups(ctx context.Context, rawGroups map[string][]*models.Entry, llmService services.LLMService) []*models.PrimaryGroupDigestData {
highLevelGroups, err := consolidatePrimaryGroups(ctx, rawGroups, llmService)
if err != nil {
// This case should ideally not be hit if consolidatePrimaryGroups has a fallback
log.Printf("AI group consolidation failed unexpectedly: %v", err)
highLevelGroups = convertRawGroupsToDigestData(rawGroups)
}
return highLevelGroups
}
func handleFailedAndUngroupedEntries(highLevelGroups []*models.PrimaryGroupDigestData, allEntries []*models.Entry, groupedEntryIDs map[int64]bool, failedChunks map[string][]*models.Entry) []*models.PrimaryGroupDigestData {
failedEntryIDs := make(map[int64]bool)
for reason, chunkEntries := range failedChunks {
title := fmt.Sprintf("Uncategorized - %s", reason)
highLevelGroups = append(highLevelGroups, &models.PrimaryGroupDigestData{
ID: int64(len(highLevelGroups) + 1),
Title: title,
Slug: utils.Slugify(title),
SubGroups: []*models.EntryGroup{
{
Title: title,
Entries: chunkEntries,
Slug: utils.Slugify(title),
TotalEntries: len(chunkEntries),
TotalFeeds: getUniqueFeedIDs(chunkEntries),
},
},
Summary: "These entries could not be processed by the AI during the initial grouping.",
TotalEntries: len(chunkEntries),
TotalFeeds: getUniqueFeedIDs(chunkEntries),
})
for _, entry := range chunkEntries {
failedEntryIDs[entry.ID] = true
}
}
// Handle entries that were not grouped by the initial LLM call
var otherUngroupedEntries []*models.Entry
for _, entry := range allEntries {
if !groupedEntryIDs[entry.ID] && !failedEntryIDs[entry.ID] {
otherUngroupedEntries = append(otherUngroupedEntries, entry)
}
}
if len(otherUngroupedEntries) > 0 {
var merged bool
for _, group := range highLevelGroups {
if group.Title == "Uncategorized" {
merged = true
// Find or create the "Uncategorized" subgroup
var subGroup *models.EntryGroup
for _, sg := range group.SubGroups {
if sg.Title == "Uncategorized" {
subGroup = sg
break
}
}
if subGroup == nil {
subGroup = &models.EntryGroup{
Title: "Uncategorized",
Slug: "uncategorized",
}
group.SubGroups = append(group.SubGroups, subGroup)
}
subGroup.Entries = append(subGroup.Entries, otherUngroupedEntries...)
subGroup.TotalEntries = len(subGroup.Entries)
subGroup.TotalFeeds = getUniqueFeedIDs(subGroup.Entries)
// Update primary group totals
var allEntries []*models.Entry
for _, sg := range group.SubGroups {
allEntries = append(allEntries, sg.Entries...)
}
group.TotalEntries = len(allEntries)
group.TotalFeeds = getUniqueFeedIDs(allEntries)
break
}
}
if !merged {
highLevelGroups = append(highLevelGroups, &models.PrimaryGroupDigestData{
ID: int64(len(highLevelGroups) + 1),
Title: "Uncategorized",
Slug: "uncategorized",
SubGroups: []*models.EntryGroup{
{
Title: "Uncategorized",
Entries: otherUngroupedEntries,
Slug: "uncategorized",
TotalEntries: len(otherUngroupedEntries),
TotalFeeds: getUniqueFeedIDs(otherUngroupedEntries),
},
},
Summary: "",
TotalEntries: len(otherUngroupedEntries),
TotalFeeds: getUniqueFeedIDs(otherUngroupedEntries),
})
}
}
return highLevelGroups
}
func processHighLevelGroupsConcurrently(ctx context.Context, highLevelGroups []*models.PrimaryGroupDigestData, llmService services.LLMService) []*models.PrimaryGroupDigestData {
var allPrimaryGroups []*models.PrimaryGroupDigestData
var wg sync.WaitGroup
var mu sync.Mutex
for _, hlg := range highLevelGroups {
wg.Add(1)
go func(hlg *models.PrimaryGroupDigestData) {
defer wg.Done()
var allEntriesInHlg []*models.Entry
for _, subGroup := range hlg.SubGroups {
allEntriesInHlg = append(allEntriesInHlg, subGroup.Entries...)
}
pg := &models.PrimaryGroup{
ID: hlg.ID,
Title: hlg.Title,
Entries: allEntriesInHlg,
}
summary, subGroups, err := ProcessPrimaryGroupWithAI(ctx, pg, llmService)
if err != nil {
log.Printf("Failed to process primary group '%s' with AI: %v. Using original group.", pg.Title, err)
hlg.SubGroups = []*models.EntryGroup{
{
Title: pg.Title,
Entries: pg.Entries,
Slug: utils.Slugify(pg.Title),
TotalEntries: len(pg.Entries),
TotalFeeds: getUniqueFeedIDs(pg.Entries),
},
}
} else {
hlg.Summary = summary
hlg.SubGroups = subGroups
}
mu.Lock()
allPrimaryGroups = append(allPrimaryGroups, hlg)
mu.Unlock()
}(hlg)
}
wg.Wait()
return allPrimaryGroups
}
func sortPrimaryGroups(allPrimaryGroups []*models.PrimaryGroupDigestData) {
sort.Slice(allPrimaryGroups, func(i, j int) bool {
return allPrimaryGroups[i].Title < allPrimaryGroups[j].Title
})
}
// llmWorkerConfig holds configuration for creating an LLM worker function.
type llmWorkerConfig struct {
promptTemplate *template.Template
promptData any // Data to pass to template execution
promptFormat string // For prompts that use Sprintf
responseSchema *genai.Schema
prepareEntries func([]*models.Entry) any // Function to prepare entries for LLM
}
// createLLMWorker creates a worker function for llm.ProcessInChunks.
func createLLMWorker[R any](
ctx context.Context,
llmService services.LLMService,
config llmWorkerConfig,
groupTitle string, // Optional, for prompts that need it
) llm.WorkerFunc[*models.Entry, R] {
return func(chunk []*models.Entry) (R, error) {
var zero R // Initialize a zero value of type R
// 1. Prepare LLM entries
var llmEntriesJSON []byte
if config.prepareEntries != nil {
prepared := config.prepareEntries(chunk)
var err error
llmEntriesJSON, err = json.MarshalIndent(prepared, "", " ")
if err != nil {
return zero, fmt.Errorf("failed to marshal entries to JSON: %w", err)
}
}
// 2. Construct the prompt
var prompt bytes.Buffer
if config.promptTemplate != nil {
if err := config.promptTemplate.Execute(&prompt, config.promptData); err != nil {
return zero, fmt.Errorf("failed to execute prompt template: %w", err)
}
} else if config.promptFormat != "" {
prompt.WriteString(fmt.Sprintf(config.promptFormat, groupTitle))
}
if len(llmEntriesJSON) > 0 {
prompt.Write(llmEntriesJSON)
}
// 3. Call LLM service
var response R
if err := llmService.GenerateContentWithResponse(ctx, prompt.String(), config.responseSchema, &response); err != nil {
return zero, err
}
return response, nil
}
}