diff --git a/.gitignore b/.gitignore index 99bfde7..f224fad 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ # Gemini CLI state /.gemini/ + +PROJECT.md \ No newline at end of file diff --git a/internal/digest/digest_test.go b/internal/digest/digest_test.go index d5884e8..59b5cb0 100644 --- a/internal/digest/digest_test.go +++ b/internal/digest/digest_test.go @@ -9,6 +9,7 @@ import ( "sort" "testing" + "github.com/stretchr/testify/assert" "google.golang.org/genai" ) @@ -38,13 +39,13 @@ func TestDigestService_BuildDigestData(t *testing.T) { case view.GroupingResponseSchema: return []byte(`{ "groups": [ - { - "title": "AI News", - "entry_ids": [1, 3] - }, { "title": "Go Lang", "entry_ids": [2, 4] + }, + { + "title": "AI News", + "entry_ids": [1, 3] } ] }`), nil @@ -121,6 +122,9 @@ func TestDigestService_BuildDigestData(t *testing.T) { if len(overviewData.PrimaryGroups) != 2 { t.Errorf("Expected 2 primary groups, got %d", len(overviewData.PrimaryGroups)) } + // Assert on sorted order + assert.Equal(t, "AI News", overviewData.PrimaryGroups[0].Title) + assert.Equal(t, "Go Lang", overviewData.PrimaryGroups[1].Title) }) t.Run("view=ai - GroupAIEntries error fallback", func(t *testing.T) { @@ -188,4 +192,4 @@ func TestDigestService_BuildDigestData(t *testing.T) { t.Errorf("Expected a single 'Uncategorized' sub-group on error, got %+v", group.SubGroups) } }) -} +} \ No newline at end of file diff --git a/internal/digest/view/ai.go b/internal/digest/view/ai.go index f223b51..82730b2 100644 --- a/internal/digest/view/ai.go +++ b/internal/digest/view/ai.go @@ -7,7 +7,7 @@ import ( "miniflux-digest/internal/app/services" "miniflux-digest/internal/models" "miniflux-digest/internal/utils" - + "sync" "sort" "log" @@ -385,44 +385,55 @@ func BuildDigestDataByAI(entries []*models.Entry, ctx context.Context, llmServic } var allPrimaryGroups []*models.PrimaryGroupDigestData + var wg sync.WaitGroup + var mu sync.Mutex for _, hlg := range highLevelGroups { - var allEntriesInHlg []*models.Entry - for _, subGroup := range hlg.SubGroups { - allEntriesInHlg = append(allEntriesInHlg, subGroup.Entries...) - } + wg.Add(1) + go func(hlg *models.PrimaryGroupDigestData) { + defer wg.Done() - pg := &models.PrimaryGroup{ - ID: hlg.ID, - Title: hlg.Title, - Entries: allEntriesInHlg, - } - - summary, subGroups, err := ProcessPrimaryGroupWithAI(ctx, pg, llmService) - if err != nil { - log.Printf("Failed to process primary group '%s' with AI: %v. Using original group.", pg.Title, err) - // If processing fails, use the original primary group's entries as a single sub-group - hlg.SubGroups = []*models.EntryGroup{ - { - Title: pg.Title, - Entries: pg.Entries, - Slug: utils.Slugify(pg.Title), - TotalEntries: len(pg.Entries), - TotalFeeds: getUniqueFeedIDs(pg.Entries), - }, + var allEntriesInHlg []*models.Entry + for _, subGroup := range hlg.SubGroups { + allEntriesInHlg = append(allEntriesInHlg, subGroup.Entries...) } - } else { - hlg.Summary = summary - hlg.SubGroups = subGroups - } - allPrimaryGroups = append(allPrimaryGroups, hlg) + pg := &models.PrimaryGroup{ + ID: hlg.ID, + Title: hlg.Title, + Entries: allEntriesInHlg, + } + + summary, subGroups, err := ProcessPrimaryGroupWithAI(ctx, pg, llmService) + if err != nil { + log.Printf("Failed to process primary group '%s' with AI: %v. Using original group.", pg.Title, err) + // If processing fails, use the original primary group's entries as a single sub-group + hlg.SubGroups = []*models.EntryGroup{ + { + Title: pg.Title, + Entries: pg.Entries, + Slug: utils.Slugify(pg.Title), + TotalEntries: len(pg.Entries), + TotalFeeds: getUniqueFeedIDs(pg.Entries), + }, + } + } else { + hlg.Summary = summary + hlg.SubGroups = subGroups + } + + mu.Lock() + allPrimaryGroups = append(allPrimaryGroups, hlg) + mu.Unlock() + }(hlg) } + wg.Wait() + sort.Slice(allPrimaryGroups, func(i, j int) bool { return allPrimaryGroups[i].Title < allPrimaryGroups[j].Title }) return allPrimaryGroups -} +} \ No newline at end of file diff --git a/internal/digest/view/ai_test.go b/internal/digest/view/ai_test.go index f10e489..2189c96 100644 --- a/internal/digest/view/ai_test.go +++ b/internal/digest/view/ai_test.go @@ -95,6 +95,49 @@ func TestGroupAIEntries(t *testing.T) { _, err := GroupAIEntries(context.Background(), entries, llmService) assert.Error(t, err) }) + + t.Run("handles chunking", func(t *testing.T) { + var largeEntries []*models.Entry + for i := 0; i < 1005; i++ { + largeEntries = append(largeEntries, &models.Entry{ID: int64(i)}) + } + + callCount := 0 + llmService := &mockLLMService{ + GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + callCount++ + var resp GroupingResponse + if callCount == 1 { // First chunk + resp.Groups = []struct { + Title string `json:"title"` + EntryIDs []int64 `json:"entry_ids"` + }{ + {Title: "Chunk 1 Group", EntryIDs: []int64{0, 1}}, + } + } else { // Second chunk + resp.Groups = []struct { + Title string `json:"title"` + EntryIDs []int64 `json:"entry_ids"` + }{ + {Title: "Chunk 2 Group", EntryIDs: []int64{1000, 1001}}, + } + } + mockJSON, _ := json.Marshal(resp) + return mockJSON, nil + }, + } + + groups, err := GroupAIEntries(context.Background(), largeEntries, llmService) + assert.NoError(t, err) + assert.Equal(t, 2, callCount, "Expected LLM to be called twice") + + assert.Len(t, groups, 3, "Expected 2 groups from chunks and 1 uncategorized") + assert.NotNil(t, testutil.FindPrimaryGroup(groups, "Chunk 1 Group")) + assert.NotNil(t, testutil.FindPrimaryGroup(groups, "Chunk 2 Group")) + uncategorized := testutil.FindPrimaryGroup(groups, "Uncategorized") + assert.NotNil(t, uncategorized) + assert.Equal(t, 1001, uncategorized.TotalEntries, "Expected 1001 entries to be uncategorized (1005 total - 4 grouped)") + }) } func TestProcessPrimaryGroupWithAI(t *testing.T) { @@ -216,7 +259,7 @@ func TestBuildDigestDataByAI(t *testing.T) { Groups: []struct { Title string `json:"title"` EntryIDs []int64 `json:"entry_ids"` - }{{Title: "AI Group 1", EntryIDs: []int64{1, 2}}}, + }{{Title: "AI Group 2", EntryIDs: []int64{3}}, {Title: "AI Group 1", EntryIDs: []int64{1, 2}}}, } groupingJSON, _ := json.Marshal(groupingResp) @@ -249,11 +292,15 @@ func TestBuildDigestDataByAI(t *testing.T) { groups := BuildDigestDataByAI(entries, context.Background(), llmService) assert.Len(t, groups, 2) - aiGroup := testutil.FindPrimaryGroup(groups, "AI Group 1") - assert.NotNil(t, aiGroup) - assert.Equal(t, "AI summary", aiGroup.Summary) - assert.Len(t, aiGroup.SubGroups, 1) - assert.Equal(t, "AI Sub-group", aiGroup.SubGroups[0].Title) + // Assertions are now made on a sorted slice + assert.Equal(t, "AI Group 1", groups[0].Title) + assert.Equal(t, "AI Group 2", groups[1].Title) + + aiGroup1 := groups[0] + assert.NotNil(t, aiGroup1) + assert.Equal(t, "AI summary", aiGroup1.Summary) + assert.Len(t, aiGroup1.SubGroups, 1) + assert.Equal(t, "AI Sub-group", aiGroup1.SubGroups[0].Title) }) t.Run("fallback to category grouping", func(t *testing.T) {