From 306b6f5d9d882656b519ec49decdb7c02524c7cb Mon Sep 17 00:00:00 2001 From: eleith Date: Fri, 5 Sep 2025 19:09:34 +0000 Subject: [PATCH] feat(ai): Implement multi-call AI grouping and summarization This commit enhances the AI view to be more reliable and performant. The AI digest generation now makes multiple calls to the LLM to create its own primary and sub-groups, replacing the previous single-call, category-based approach. The overview summary has been removed in favor of per-group summaries to manage context length. To improve reliability, the LLM calls for summarization and sub-grouping have been separated, and the timeout handling has been refactored to use a per-try timeout, making it more resilient to transient server errors. The email template has also been updated to include the primary group summaries, aligning its purpose with the overview HTML page. --- go.mod | 4 + go.sum | 2 + internal/app/services/llm.go | 4 + internal/digest/digest.go | 94 +--- internal/digest/digest_test.go | 243 ++++------ internal/digest/group_by/date_test.go | 23 - internal/digest/group_by/llm.go | 238 ---------- internal/digest/view/ai.go | 428 ++++++++++++++++++ internal/digest/view/ai_test.go | 288 ++++++++++++ .../digest/{group_by => view}/category.go | 15 +- internal/digest/view/category_test.go | 42 ++ internal/digest/{group_by => view}/date.go | 20 +- internal/digest/view/date_test.go | 48 ++ internal/digest/{group_by => view}/feed.go | 6 +- internal/digest/view/feed_test.go | 61 +++ internal/digest/view/view.go | 75 +++ internal/email/email.go | 2 +- internal/email/email_test.go | 2 +- internal/llm/gemini.go | 29 +- internal/models/models.go | 13 +- internal/processor/processor_test.go | 2 +- internal/templates/email.gotxt | 24 +- internal/templates/overview.gohtml | 7 +- internal/templates/templates.go | 2 +- internal/templates/templates_test.go | 2 +- internal/testutil/mocks.go | 35 +- internal/utils/slugify.go | 14 + scripts/preview/main.go | 2 +- 28 files changed, 1164 insertions(+), 561 deletions(-) delete mode 100644 internal/digest/group_by/date_test.go delete mode 100644 internal/digest/group_by/llm.go create mode 100644 internal/digest/view/ai.go create mode 100644 internal/digest/view/ai_test.go rename internal/digest/{group_by => view}/category.go (65%) create mode 100644 internal/digest/view/category_test.go rename internal/digest/{group_by => view}/date.go (84%) create mode 100644 internal/digest/view/date_test.go rename internal/digest/{group_by => view}/feed.go (91%) create mode 100644 internal/digest/view/feed_test.go create mode 100644 internal/digest/view/view.go diff --git a/go.mod b/go.mod index be8f4bd..1726ddd 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/knadh/koanf/providers/file v1.2.0 github.com/knadh/koanf/v2 v2.2.2 github.com/robfig/cron/v3 v3.0.1 + github.com/stretchr/testify v1.10.0 github.com/tdewolff/minify/v2 v2.23.8 github.com/wneessen/go-mail v0.6.2 google.golang.org/genai v1.17.0 @@ -21,6 +22,7 @@ require ( cloud.google.com/go v0.116.0 // indirect cloud.google.com/go/auth v0.9.3 // indirect cloud.google.com/go/compute/metadata v0.5.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.8 // indirect github.com/go-playground/locales v0.14.1 // indirect @@ -38,7 +40,9 @@ require ( github.com/leodido/go-urn v1.4.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/tdewolff/parse/v2 v2.8.1 // indirect go.opencensus.io v0.24.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect diff --git a/go.sum b/go.sum index b472d2c..174e599 100644 --- a/go.sum +++ b/go.sum @@ -98,6 +98,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= diff --git a/internal/app/services/llm.go b/internal/app/services/llm.go index 13e27fa..3ab154d 100644 --- a/internal/app/services/llm.go +++ b/internal/app/services/llm.go @@ -6,6 +6,10 @@ import ( "google.golang.org/genai" ) +// TODO: To make timeouts customizable per-call, consider adding an optional timeout +// parameter to the GenerateContent method, e.g.: +// GenerateContent(ctx context.Context, prompt string, schema *genai.Schema, timeout ...time.Duration) ([]byte, error) + type LLMService interface { GenerateContent(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) } diff --git a/internal/digest/digest.go b/internal/digest/digest.go index 76a8fd5..12ffe9c 100644 --- a/internal/digest/digest.go +++ b/internal/digest/digest.go @@ -2,14 +2,10 @@ package digest import ( "miniflux-digest/internal/app/services" + "miniflux-digest/internal/digest/view" "miniflux-digest/internal/models" - "miniflux-digest/internal/digest/group_by" - "sort" - "time" ) - - func NewDigestService(llmService services.LLMService) services.DigestService { return &digestServiceImpl{llmService: llmService} } @@ -18,88 +14,6 @@ type digestServiceImpl struct { llmService services.LLMService } -func (s *digestServiceImpl) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string) *models.OverviewTemplateData { - iconsSlice := make([]*models.FeedIcon, 0, len(icons)) - for _, icon := range icons { - iconsSlice = append(iconsSlice, icon) - } - - var allPrimaryGroups []*models.PrimaryGroupDigestData - var overallDigestSummary *string - - switch view { - case "date": - primaryGroups := group_by.GroupByDate(entries) - allPrimaryGroups, overallDigestSummary = group_by.SubGroupByFeed(primaryGroups) - case "category": - primaryGroups := group_by.GroupByCategory(entries) - allPrimaryGroups, overallDigestSummary = group_by.SubGroupByFeed(primaryGroups) - sort.Slice(allPrimaryGroups, func(i, j int) bool { - return allPrimaryGroups[i].Title < allPrimaryGroups[j].Title - }) - case "ai": - primaryGroups := group_by.GroupByCategory(entries) - allPrimaryGroups, overallDigestSummary = group_by.SubGroupByAI(primaryGroups, s.llmService) - sort.Slice(allPrimaryGroups, func(i, j int) bool { - return allPrimaryGroups[i].Title < allPrimaryGroups[j].Title - }) - default: - primaryGroups := group_by.GroupByCategory(entries) - allPrimaryGroups, overallDigestSummary = group_by.SubGroupByFeed(primaryGroups) - sort.Slice(allPrimaryGroups, func(i, j int) bool { - return allPrimaryGroups[i].Title < allPrimaryGroups[j].Title - }) - } - - if view == "date" || view == "category" { - for _, pg := range allPrimaryGroups { - for _, sg := range pg.SubGroups { - sort.Slice(sg.Entries, func(i, j int) bool { - return sg.Entries[i].Date.Before(sg.Entries[j].Date) - }) - } - } - } - - for _, pg := range allPrimaryGroups { - pgTotalEntries := 0 - pgUniqueFeedIDs := make(map[int64]bool) - for _, subGroup := range pg.SubGroups { - pgTotalEntries += len(subGroup.Entries) - for _, entry := range subGroup.Entries { - pgUniqueFeedIDs[entry.FeedID] = true - } - } - pg.TotalEntries = pgTotalEntries - pg.TotalFeeds = len(pgUniqueFeedIDs) - } - - var allEntryGroups []*models.EntryGroup - for _, pg := range allPrimaryGroups { - allEntryGroups = append(allEntryGroups, pg.SubGroups...) - } - - uniqueFeedIDs := make(map[int64]bool) - for _, entry := range entries { - uniqueFeedIDs[entry.FeedID] = true - } - - return &models.OverviewTemplateData{ - Entries: entries, - GeneratedDate: time.Now(), - FeedIcons: iconsSlice, - PrimaryGroups: allPrimaryGroups, - EntryGroups: allEntryGroups, - OverviewSummary: safeDerefString(overallDigestSummary), - TotalEntries: len(entries), - TotalFeeds: len(uniqueFeedIDs), - MinifluxHost: minifluxHost, - } -} - -func safeDerefString(s *string) string { - if s == nil { - return "" - } - return *s -} +func (s *digestServiceImpl) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, viewType string, minifluxHost string) *models.OverviewTemplateData { + return view.BuildDigestDataForView(entries, icons, viewType, minifluxHost, s.llmService) +} \ No newline at end of file diff --git a/internal/digest/digest_test.go b/internal/digest/digest_test.go index 9d87c4d..d5884e8 100644 --- a/internal/digest/digest_test.go +++ b/internal/digest/digest_test.go @@ -3,11 +3,11 @@ package digest import ( "context" "errors" + "miniflux-digest/internal/digest/view" "miniflux-digest/internal/models" "miniflux-digest/internal/testutil" "sort" "testing" - "miniflux-digest/internal/digest/group_by" "google.golang.org/genai" ) @@ -23,24 +23,6 @@ func (m *mockLLMService) GenerateContent(ctx context.Context, prompt string, sch return nil, errors.New("GenerateContentFunc not implemented") } -func findPrimaryGroup(groups []*models.PrimaryGroupDigestData, title string) *models.PrimaryGroupDigestData { - for _, group := range groups { - if group.Title == title { - return group - } - } - return nil -} - -func findSubGroup(primaryGroup *models.PrimaryGroupDigestData, title string) *models.EntryGroup { - for _, group := range primaryGroup.SubGroups { - if group.Title == title { - return group - } - } - return nil -} - func createLLMGrouperMockEntries() []*models.Entry { return []*models.Entry{ {ID: 1, Title: "Entry 1", FeedID: 100, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A"}, @@ -49,119 +31,34 @@ func createLLMGrouperMockEntries() []*models.Entry { } } -func TestDayGrouper_GroupEntries(t *testing.T) { - entries := testutil.CreateMockEntries(0) - primaryGroupsMap := group_by.GroupByCategory(entries) - - groups, summary := group_by.SubGroupByDay(primaryGroupsMap) - - if summary != nil { - t.Errorf("Expected an empty summary for DayGrouper, got %q", *summary) - } - - if len(groups) != 3 { - t.Fatalf("Expected 3 primary groups, got %d", len(groups)) - } - - catAGroup := findPrimaryGroup(groups, "Category A") - if catAGroup == nil || len(catAGroup.SubGroups) != 1 { - t.Fatalf("Incorrect sub-groups for Category A: %+v", catAGroup) - } - - subGroup := findSubGroup(catAGroup, "Jan 2, 2024") - if subGroup == nil || len(subGroup.Entries) != 4 { - t.Fatalf("Incorrect entries for sub-group: %+v", subGroup) - } -} - -func TestFeedGrouper_GroupEntries(t *testing.T) { - entries := testutil.CreateMockEntries(0) - primaryGroupsMap := group_by.GroupByCategory(entries) - - groups, summary := group_by.SubGroupByFeed(primaryGroupsMap) - - if summary != nil { - t.Errorf("Expected an empty summary for FeedGrouper, got %q", *summary) - } - - if len(groups) != 3 { - t.Fatalf("Expected 3 primary groups, got %d", len(groups)) - } - - catAGroup := findPrimaryGroup(groups, "Category A") - if catAGroup == nil || len(catAGroup.SubGroups) != 3 { - t.Fatalf("Incorrect sub-groups for Category A: %+v", catAGroup) - } -} - -func TestLLMGrouper_GroupEntries(t *testing.T) { - entries := createLLMGrouperMockEntries() - primaryGroupsMap := group_by.GroupByCategory(entries) - - mockLLM := &mockLLMService{ - GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { - return []byte(`{ - "overview": "This is a summary of all entries.", - "primary_group_summaries": [ - { - "primary_group_id": 1, - "summary": "This is a summary of Category A." - } - ], - "sub_groups": [ - { - "title": "Sub-group 1", - "entry_ids": [1, 2] - } - ] - }`), - nil - }, - } - - groups, summary := group_by.SubGroupByAI(primaryGroupsMap, mockLLM) - - if summary == nil || *summary != "This is a summary of all entries." { - t.Errorf("Expected summary 'This is a summary of all entries.', got %q", *summary) - } - - if len(groups) != 1 { - t.Fatalf("Expected 1 group, got %d", len(groups)) - } - - catA := findPrimaryGroup(groups, "Category A") - if catA == nil || len(catA.SubGroups) != 2 { - t.Fatalf("Incorrect number of sub-groups for Category A: %+v", catA) - } - - if catA.Summary != "This is a summary of Category A." { - t.Errorf("Expected summary 'This is a summary of Category A.', got %q", catA.Summary) - } - - subGroup1 := findSubGroup(catA, "Sub-group 1") - if subGroup1 == nil || len(subGroup1.Entries) != 2 { - t.Fatalf("Incorrect entries for sub-group 1: %+v", subGroup1) - } - - uncategorized := findSubGroup(catA, "Uncategorized") - if uncategorized == nil || len(uncategorized.Entries) != 1 { - t.Fatalf("Incorrect entries for Uncategorized sub-group: %+v", uncategorized) - } -} - func TestDigestService_BuildDigestData(t *testing.T) { mockLLM := &mockLLMService{ GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { - return []byte(`{ - "overview": "This is a summary of all entries.", - "primary_group_summaries": [], - "sub_groups": [ - { - "title": "Sub-group 1", - "entry_ids": [1, 2] - } - ] - }`), nil + switch schema { + case view.GroupingResponseSchema: + return []byte(`{ + "groups": [ + { + "title": "AI News", + "entry_ids": [1, 3] + }, + { + "title": "Go Lang", + "entry_ids": [2, 4] + } + ] + }`), nil + case view.SummaryResponseSchema: + return []byte(`{"summary": "This is a summary."}`), nil + case view.SubGroupingResponseSchema: + return []byte(`{ + "sub_groups": [ + {"title": "Sub-group 1", "entry_ids": [1]} + ] + }`), nil + default: + return nil, errors.New("unexpected LLM call") + } }, } digestService := NewDigestService(mockLLM) @@ -180,11 +77,11 @@ func TestDigestService_BuildDigestData(t *testing.T) { if len(overviewData.PrimaryGroups) != 3 { t.Errorf("Expected 3 primary groups, got %d", len(overviewData.PrimaryGroups)) } - dateGroup := findPrimaryGroup(overviewData.PrimaryGroups, "Jan 1, 2024") + dateGroup := testutil.FindPrimaryGroup(overviewData.PrimaryGroups, "Jan 1, 2024") if dateGroup == nil { t.Fatalf("Did not find expected primary group 'Jan 1, 2024'") } - if len(dateGroup.SubGroups) != 2 { + if len(dateGroup.SubGroups) != 1 { t.Fatalf("Incorrect sub-groups for Jan 1, 2024: %+v", dateGroup) } firstSubGroup := dateGroup.SubGroups[0] @@ -203,7 +100,7 @@ func TestDigestService_BuildDigestData(t *testing.T) { if len(overviewData.PrimaryGroups) != 3 { t.Errorf("Expected 3 primary groups, got %d", len(overviewData.PrimaryGroups)) } - catAGroup := findPrimaryGroup(overviewData.PrimaryGroups, "Category A") + catAGroup := testutil.FindPrimaryGroup(overviewData.PrimaryGroups, "Category A") if catAGroup == nil || len(catAGroup.SubGroups) != 3 { t.Fatalf("Incorrect sub-groups for Category A: %+v", catAGroup) } @@ -221,30 +118,74 @@ func TestDigestService_BuildDigestData(t *testing.T) { if overviewData == nil { t.Fatal("overviewData is nil") } - if overviewData.OverviewSummary != "This is a summary of all entries." { - t.Errorf("Expected summary, got %q", overviewData.OverviewSummary) + if len(overviewData.PrimaryGroups) != 2 { + t.Errorf("Expected 2 primary groups, got %d", len(overviewData.PrimaryGroups)) + } + }) + + t.Run("view=ai - GroupAIEntries error fallback", func(t *testing.T) { + mockLLM := &mockLLMService{ + GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + if schema == view.GroupingResponseSchema { + return nil, errors.New("mock LLM grouping error") + } + return nil, errors.New("unexpected LLM call") + }, + } + digestService := NewDigestService(mockLLM) + aiEntries := createLLMGrouperMockEntries() + overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test") + + if overviewData == nil { + t.Fatal("overviewData is nil") + } + if len(overviewData.PrimaryGroups) != 1 { + t.Errorf("Expected 1 primary group from date fallback, got %d", len(overviewData.PrimaryGroups)) + } + dateGroup := testutil.FindPrimaryGroup(overviewData.PrimaryGroups, "Jan 1, 0001") + if dateGroup == nil { + t.Fatalf("Did not find expected primary group 'Jan 1, 0001' after date fallback") + } + }) + + t.Run("view=ai - sub-grouping error", func(t *testing.T) { + mockLLM := &mockLLMService{ + GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + if schema == view.GroupingResponseSchema { + return []byte(`{ + "groups": [ + { + "title": "Test Group", + "entry_ids": [1, 2, 3] + } + ] + }`), nil + } + if schema == view.SummaryResponseSchema { + return []byte(`{"summary": "This is a summary."}`), nil + } + if schema == view.SubGroupingResponseSchema { + return nil, errors.New("mock LLM sub-grouping error") + } + return nil, errors.New("unexpected LLM call") + }, + } + digestService := NewDigestService(mockLLM) + aiEntries := createLLMGrouperMockEntries() + overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test") + + if overviewData == nil { + t.Fatal("overviewData is nil") } if len(overviewData.PrimaryGroups) != 1 { t.Errorf("Expected 1 primary group, got %d", len(overviewData.PrimaryGroups)) } - }) - - t.Run("view=ai fallback", func(t *testing.T) { - llmErrorService := NewDigestService(&mockLLMService{ - GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { - return nil, errors.New("LLM service error") - }, - }) - overviewData := llmErrorService.BuildDigestData(entries, icons, "ai", "http://miniflux.test") - if overviewData == nil { - t.Fatal("overviewData is nil") + group := overviewData.PrimaryGroups[0] + if group.Summary != "This is a summary." { + t.Errorf("Expected group summary to be present, got %q", group.Summary) } - if overviewData.OverviewSummary != "" { - t.Errorf("Expected empty summary on fallback, got %q", overviewData.OverviewSummary) - } - catAGroup := findPrimaryGroup(overviewData.PrimaryGroups, "Category A") - if catAGroup == nil || len(catAGroup.SubGroups) != 3 { - t.Fatalf("Incorrect sub-groups for Category A on fallback: %+v", catAGroup) + if len(group.SubGroups) != 1 || group.SubGroups[0].Title != "Uncategorized" { + t.Errorf("Expected a single 'Uncategorized' sub-group on error, got %+v", group.SubGroups) } }) } diff --git a/internal/digest/group_by/date_test.go b/internal/digest/group_by/date_test.go deleted file mode 100644 index a6681dc..0000000 --- a/internal/digest/group_by/date_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package group_by - -import ( - "miniflux-digest/internal/testutil" - "testing" -) - -func TestGroupByDate(t *testing.T) { - entries := testutil.CreateMockEntries(0) - groups := GroupByDate(entries) - - if len(groups) != 3 { - t.Fatalf("Expected 3 date groups, got %d", len(groups)) - } - - if groups[0].Title != "Jan 1, 2024" { - t.Errorf("Expected first group to be Jan 1, 2024, got %s", groups[0].Title) - } - - if len(groups[0].Entries) != 3 { - t.Errorf("Expected 3 entries in first group, got %d", len(groups[0].Entries)) - } -} diff --git a/internal/digest/group_by/llm.go b/internal/digest/group_by/llm.go deleted file mode 100644 index 2f59ce5..0000000 --- a/internal/digest/group_by/llm.go +++ /dev/null @@ -1,238 +0,0 @@ -package group_by - -import ( - "context" - "encoding/json" - "log" - "miniflux-digest/internal/app/services" - "miniflux-digest/internal/models" - "miniflux-digest/internal/utils" - "time" - - "google.golang.org/genai" -) - -const ( - LLMTimeout = 10 * time.Minute -) - -type LLMResponse struct { - Overview string `json:"overview"` - PrimaryGroupSummaries []struct { - PrimaryGroupID int64 `json:"primary_group_id"` - Summary string `json:"summary"` - } `json:"primary_group_summaries"` - SubGroups []struct { - Title string `json:"title"` - EntryIDs []int64 `json:"entry_ids"` - } `json:"sub_groups"` -} - - - -type llmEntry struct { - ID int64 `json:"id"` - Title string `json:"title"` - URL string `json:"url"` - Content string `json:"content"` - FeedTitle string `json:"feed_title"` - PrimaryGroupID int64 `json:"primary_group_id"` -} - -const llmPrompt = `You will be given a large list of unread entry JSON Objects from a RSS feed reader. these entries help a user follow updates from blogs, social networks, and news sources. - -Your task is to generate the metadata according to the instructions below: - - - an 'overview' string field - - this is a 3-5 sentence overview for ALL entries, less than 250 words. - - the overview is really a brief overview. you are not to summarize every single entry - - the overview could highlight critical entries or summarize the main topics or both, you decide - - To improve readability, please insert two line breaks (\n\n) after every 1-2 sentences. - - - a list of 'primary_group_summaries' objects - - the 'primary_group_id' will be in every entry id - - the summary is really a brief overview using all entries associated to the primary group - - you don't need to summarize every single entry - - entries are only ever are associated with one primary group id - - this is a 2-3 sentence overview, less than 150 words. - - the overview should help the user decide whether to skim all entries in this group fast or slow and what to narrow in on. - - the overview could highlight critical entries or summarize the main topics or both whatever helps the user skip, skim or find the most important entries - - To improve readability, please insert two line breaks (\n\n) after every 1-2 sentences. - -- 'sub_groups': - - The sub_groups field should contain a list of subgroup objects. - - Each subgroup should have a 'title' that you come up with (generally 2-3 words, max 70 characters). - - Each subgroup should have an array of 'entry_ids'. An entry can only belong to one subgroup. - - The purpose of subgroups is to group related entries to help the user skim efficiently. - - For each entry, you must attempt to group it using only one of the strategies below - - Aim for sub_groups that have more than 2 entries but less than 20 entries. - - not all groups are created the same. below are a list groups from best to worst. do your best to fit entries in one of these types. - - Priority 1: Exact Duplicates. Group entries that are effectively identical or point to the same content. Goal: De-duplication. - - Priority 2: Specific Topic/Theme. Group entries about the same narrow subject (e.g., a specific product launch, a single news event). Goal: Thematic exploration. - - Priority 3: General Topic/Theme. Group entries under broader categories like 'Science', 'Technology', or 'Business'. Goal: General organization. - - Priority 4: Source. Only if an entry cannot be grouped by topic, group it with other entries from the same website or publication. Goal: Source-based review. - -Your goal is to help the user get through their unread entries as efficiently as possible. The metadata you generate will be used to create a digest that allows the user to quickly understand and navigate their unread entries. - -Below is the list of entries: ----------------------------- -` - -var llmResponseSchema = &genai.Schema{ - Type: genai.TypeObject, - Properties: map[string]*genai.Schema{ - "overview": { - Type: genai.TypeString, - }, - "primary_group_summaries": { - Type: genai.TypeArray, - Items: &genai.Schema{ - Type: genai.TypeObject, - Properties: map[string]*genai.Schema{ - "primary_group_id": { - Type: genai.TypeInteger, - }, - "summary": { - Type: genai.TypeString, - }, - }, - }, - }, - "sub_groups": { - Type: genai.TypeArray, - Items: &genai.Schema{ - Type: genai.TypeObject, - Properties: map[string]*genai.Schema{ - "title": { - Type: genai.TypeString, - }, - "entry_ids": { - Type: genai.TypeArray, - Items: &genai.Schema{ - Type: genai.TypeInteger, - }, - }, - }, - }, - }, - }, -} - -func SubGroupByAI(pgs []*models.PrimaryGroup, llmService services.LLMService) ([]*models.PrimaryGroupDigestData, *string) { - var llmEntries []llmEntry - entryMap := make(map[int64]*models.Entry) - primaryGroupMap := make(map[int64]*models.PrimaryGroup) - - for _, pg := range pgs { - primaryGroupMap[pg.ID] = pg - for _, entry := range pg.Entries { - content := entry.Content - if len(content) > 1000 { - content = content[:1000] - } - llmEntries = append(llmEntries, llmEntry{ - ID: entry.ID, - Title: entry.Title, - URL: entry.URL, - Content: content, - PrimaryGroupID: pg.ID, - }) - entryMap[entry.ID] = entry - } - } - - entriesJSON, err := json.MarshalIndent(llmEntries, "", " ") - if err != nil { - return fallbackToFeedGrouper(pgs) - } - - prompt := llmPrompt + string(entriesJSON) - - ctx, cancel := context.WithTimeout(context.Background(), LLMTimeout) - defer cancel() - - llmResponse, err := llmService.GenerateContent(ctx, prompt, llmResponseSchema) - if err != nil { - log.Printf("LLM service failed, falling back to feed grouping: %v\n", err) - return fallbackToFeedGrouper(pgs) - } - - var response LLMResponse - if err := json.Unmarshal(llmResponse, &response); err != nil { - log.Printf("Failed to parse LLM response, falling back to feed grouping: %v\n", err) - log.Printf("Raw LLM response:\n%s", llmResponse) - return fallbackToFeedGrouper(pgs) - } - - finalPrimaryGroupsMap := make(map[int64]*models.PrimaryGroupDigestData) - for _, pg := range pgs { - finalPrimaryGroupsMap[pg.ID] = &models.PrimaryGroupDigestData{ - ID: pg.ID, - Title: pg.Title, - Slug: utils.Slugify(pg.Title), - SubGroups: []*models.EntryGroup{}, - } - } - - for _, summary := range response.PrimaryGroupSummaries { - if pg, ok := finalPrimaryGroupsMap[summary.PrimaryGroupID]; ok { - pg.Summary = summary.Summary - } - } - - groupedEntryIDs := make(map[int64]bool) - for _, llmSubGroup := range response.SubGroups { - subGroupEntriesByPrimaryGroup := make(map[int64][]*models.Entry) - - for _, entryID := range llmSubGroup.EntryIDs { - if _, exists := groupedEntryIDs[entryID]; exists { - continue - } - if entry, ok := entryMap[entryID]; ok { - primaryGroupID := entry.GroupID - subGroupEntriesByPrimaryGroup[primaryGroupID] = append(subGroupEntriesByPrimaryGroup[primaryGroupID], entry) - groupedEntryIDs[entryID] = true - } - } - - for primaryGroupID, entriesInGroup := range subGroupEntriesByPrimaryGroup { - if pg, ok := finalPrimaryGroupsMap[primaryGroupID]; ok { - if len(entriesInGroup) > 0 { - pg.SubGroups = append(pg.SubGroups, &models.EntryGroup{ - Title: llmSubGroup.Title, - Entries: entriesInGroup, - Slug: utils.Slugify(llmSubGroup.Title), - }) - } - } - } - } - - for _, pg := range pgs { - var ungroupedInThisPrimaryGroup []*models.Entry - for _, entry := range pg.Entries { - if !groupedEntryIDs[entry.ID] { - ungroupedInThisPrimaryGroup = append(ungroupedInThisPrimaryGroup, entry) - } - } - - if len(ungroupedInThisPrimaryGroup) > 0 { - finalPrimaryGroupsMap[pg.ID].SubGroups = append(finalPrimaryGroupsMap[pg.ID].SubGroups, &models.EntryGroup{ - Title: "Uncategorized", - Entries: ungroupedInThisPrimaryGroup, - Slug: "uncategorized", - }) - } - } - - var finalPrimaryGroups []*models.PrimaryGroupDigestData - for _, pg := range finalPrimaryGroupsMap { - finalPrimaryGroups = append(finalPrimaryGroups, pg) - } - - return finalPrimaryGroups, &response.Overview -} - -func fallbackToFeedGrouper(pgs []*models.PrimaryGroup) ([]*models.PrimaryGroupDigestData, *string) { - return SubGroupByFeed(pgs) -} diff --git a/internal/digest/view/ai.go b/internal/digest/view/ai.go new file mode 100644 index 0000000..f223b51 --- /dev/null +++ b/internal/digest/view/ai.go @@ -0,0 +1,428 @@ +package view + +import ( + "context" + "encoding/json" + "fmt" + "miniflux-digest/internal/app/services" + "miniflux-digest/internal/models" + "miniflux-digest/internal/utils" + + "sort" + "log" + + "google.golang.org/genai" +) + +const ( + MaxEntryContentLengthForLLM = 1000 + MaxEntriesForSummarization = 1000 +) + +type GroupingResponse struct { + Groups []struct { + Title string `json:"title"` + EntryIDs []int64 `json:"entry_ids"` + } `json:"groups"` +} + +var GroupingResponseSchema = &genai.Schema{ + Type: genai.TypeObject, + Properties: map[string]*genai.Schema{ + "groups": { + Type: genai.TypeArray, + Items: &genai.Schema{ + Type: genai.TypeObject, + Properties: map[string]*genai.Schema{ + "title": { + Type: genai.TypeString, + }, + "entry_ids": { + Type: genai.TypeArray, + Items: &genai.Schema{ + Type: genai.TypeInteger, + }, + }, + }, + }, + }, + }, +} + +const groupingPrompt = `You will be given a list of unread JSON entry objects from an RSS feed reader. Each entry includes its ID, Title, URL, Content, and the Title of the Feed it came from. + +Your task is to group these entries based on their content. Follow these instructions: + +- Group entries that are about the same specific topic or event. +- Create group titles that are concise and descriptive (2-5 words). +- An entry can only belong to one group. +- Aim for groups with 2 to 20 entries. +- If an entry cannot be grouped, leave it out of the response. + +Your goal is to create meaningful groups that help the user quickly understand the main themes in their feed. + +Below is the list of entries: +---------------------------- +` + +type llmEntry struct { + ID int64 `json:"id"` + Title string `json:"title"` + URL string `json:"url"` + Content string `json:"content"` + FeedTitle string `json:"feed_title"` +} + +func GroupAIEntries(ctx context.Context, entries []*models.Entry, llmService services.LLMService) ([]*models.PrimaryGroupDigestData, error) { + const chunkSize = 1000 + + entryMap := make(map[int64]*models.Entry) + for _, entry := range entries { + entryMap[entry.ID] = entry + } + + groupedEntryIDs := make(map[int64]bool) + var allPrimaryGroupDigestData []*models.PrimaryGroupDigestData + primaryGroupMap := make(map[string]*models.PrimaryGroupDigestData) + currentPrimaryGroupID := int64(1) + + for i := 0; i < len(entries); i += chunkSize { + end := min(i + chunkSize, len(entries)) + chunk := entries[i:end] + + var llmEntries []llmEntry + for _, entry := range chunk { + content := entry.Content + if len(content) > MaxEntryContentLengthForLLM { + content = content[:MaxEntryContentLengthForLLM] + } + llmEntries = append(llmEntries, llmEntry{ + ID: entry.ID, + Title: entry.Title, + URL: entry.URL, + Content: content, + FeedTitle: entry.FeedTitle, + }) + } + + entriesJSON, err := json.MarshalIndent(llmEntries, "", " ") + if err != nil { + return nil, fmt.Errorf("failed to marshal entries to JSON: %w", err) + } + + prompt := groupingPrompt + string(entriesJSON) + + llmResponse, err := llmService.GenerateContent(ctx, prompt, GroupingResponseSchema) + if err != nil { + log.Printf("LLM call for 'primary-grouping' failed on chunk %d-%d: %v", i, end, err) + return nil, fmt.Errorf("LLM service failed for chunk %d-%d: %w", i, end, err) + } + + var response GroupingResponse + if err := json.Unmarshal(llmResponse, &response); err != nil { + return nil, fmt.Errorf("failed to parse LLM response for chunk %d-%d: %w", i, end, err) + } + + for _, group := range response.Groups { + var groupEntries []*models.Entry + for _, entryID := range group.EntryIDs { + if _, exists := groupedEntryIDs[entryID]; exists { + continue + } + if entry, ok := entryMap[entryID]; ok { + groupEntries = append(groupEntries, entry) + groupedEntryIDs[entryID] = true + } + } + + if len(groupEntries) > 0 { + if existingPGDD, ok := primaryGroupMap[group.Title]; ok { + existingPGDD.SubGroups[0].Entries = append(existingPGDD.SubGroups[0].Entries, groupEntries...) + existingPGDD.TotalEntries += len(groupEntries) + existingPGDD.TotalFeeds = getUniqueFeedIDs(existingPGDD.SubGroups[0].Entries) + } else { + newPGDD := &models.PrimaryGroupDigestData{ + ID: currentPrimaryGroupID, + Title: group.Title, + Slug: utils.Slugify(group.Title), + SubGroups: []*models.EntryGroup{ + { + Title: group.Title, + Entries: groupEntries, + Slug: utils.Slugify(group.Title), + }, + }, + Summary: "", + TotalEntries: len(groupEntries), + TotalFeeds: getUniqueFeedIDs(groupEntries), + } + allPrimaryGroupDigestData = append(allPrimaryGroupDigestData, newPGDD) + primaryGroupMap[group.Title] = newPGDD + currentPrimaryGroupID++ + } + } + } + } + + var ungroupedEntries []*models.Entry + + for _, entry := range entries { + if !groupedEntryIDs[entry.ID] { + ungroupedEntries = append(ungroupedEntries, entry) + } + } + + if len(ungroupedEntries) > 0 { + allPrimaryGroupDigestData = append(allPrimaryGroupDigestData, &models.PrimaryGroupDigestData{ + ID: currentPrimaryGroupID, + Title: "Uncategorized", + Slug: "uncategorized", + SubGroups: []*models.EntryGroup{ + { + Title: "Uncategorized", + Entries: ungroupedEntries, + Slug: "uncategorized", + }, + }, + Summary: "", + TotalEntries: len(ungroupedEntries), + TotalFeeds: getUniqueFeedIDs(ungroupedEntries), + }) + } + + return allPrimaryGroupDigestData, nil +} + +type AIGroupSummaryResponse struct { + Summary string `json:"summary"` +} + +type AISubGroupingResponse struct { + SubGroups []struct { + Title string `json:"title"` + EntryIDs []int64 `json:"entry_ids"` + } `json:"sub_groups"` +} + +var SummaryResponseSchema = &genai.Schema{ + Type: genai.TypeObject, + Properties: map[string]*genai.Schema{ + "summary": { + Type: genai.TypeString, + }, + }, +} + +var SubGroupingResponseSchema = &genai.Schema{ + Type: genai.TypeObject, + Properties: map[string]*genai.Schema{ + "sub_groups": { + Type: genai.TypeArray, + Items: &genai.Schema{ + Type: genai.TypeObject, + Properties: map[string]*genai.Schema{ + "title": { + Type: genai.TypeString, + }, + "entry_ids": { + Type: genai.TypeArray, + Items: &genai.Schema{ + Type: genai.TypeInteger, + }, + }, + }, + }, + }, + }, +} + +const summaryPrompt = `You will be given a list of JSON entry objects from a primary group titled '%s'. Generate a 2-3 sentence summary (less than 150 words) of the main topics. To improve readability, insert two line breaks (\n\n) after every 1-2 sentences. + +Below is the list of entries for the primary group: +---------------------------- +` + +const subGroupingPrompt = `You will be given a list of JSON entry objects from a primary group titled '%s'. Your task is to identify relevant sub-groups within it. + +- Create sub-groups for entries about the same specific topic or event. +- Create concise, descriptive titles (2-5 words) for each sub-group. +- An entry can only belong to one sub-group. +- Aim for sub-groups with 2 to 20 entries. +- If an entry cannot be sub-grouped, leave it out of the response. + +Your goal is to create meaningful sub-groups that help the user navigate the entries. + +Below is the list of entries for the primary group: +---------------------------- +` + +func ProcessPrimaryGroupWithAI(ctx context.Context, pg *models.PrimaryGroup, llmService services.LLMService) (string, []*models.EntryGroup, error) { + var llmEntries []llmEntry + entriesToProcess := pg.Entries + if len(entriesToProcess) > MaxEntriesForSummarization { + entriesToProcess = entriesToProcess[:MaxEntriesForSummarization] + } + + entryMap := make(map[int64]*models.Entry) + for _, entry := range entriesToProcess { + entryMap[entry.ID] = entry + content := entry.Content + if len(content) > MaxEntryContentLengthForLLM { + content = content[:MaxEntryContentLengthForLLM] + } + llmEntries = append(llmEntries, llmEntry{ + ID: entry.ID, + Title: entry.Title, + URL: entry.URL, + Content: content, + FeedTitle: entry.FeedTitle, + }) + } + + entriesJSON, err := json.MarshalIndent(llmEntries, "", " ") + if err != nil { + return "", nil, fmt.Errorf("failed to marshal entries to JSON for AI processing: %w", err) + } + + // 1. Get Summary + var summary string + summaryPromptFmt := fmt.Sprintf(summaryPrompt, pg.Title) + summaryFullPrompt := summaryPromptFmt + string(entriesJSON) + + summaryResponseBytes, err := llmService.GenerateContent(ctx, summaryFullPrompt, SummaryResponseSchema) + if err != nil { + log.Printf("LLM service failed during summarization for primary group '%s': %v", pg.Title, err) + } else { + var summaryResponse AIGroupSummaryResponse + if err := json.Unmarshal(summaryResponseBytes, &summaryResponse); err != nil { + log.Printf("Failed to parse LLM summarization response for primary group '%s': %v", pg.Title, err) + } else { + summary = summaryResponse.Summary + } + } + + // 2. Get Sub-groups + var subGroups []*models.EntryGroup + subGroupingPromptFmt := fmt.Sprintf(subGroupingPrompt, pg.Title) + subGroupingFullPrompt := subGroupingPromptFmt + string(entriesJSON) + + subGroupingResponseBytes, err := llmService.GenerateContent(ctx, subGroupingFullPrompt, SubGroupingResponseSchema) + if err != nil { + log.Printf("LLM service failed during sub-grouping for primary group '%s': %v. Creating single 'Uncategorized' group.", pg.Title, err) + } else { + var subGroupingResponse AISubGroupingResponse + if err := json.Unmarshal(subGroupingResponseBytes, &subGroupingResponse); err != nil { + log.Printf("Failed to parse LLM sub-grouping response for primary group '%s': %v. Creating single 'Uncategorized' group.", pg.Title, err) + } else { + groupedEntryIDs := make(map[int64]bool) + for _, llmSubGroup := range subGroupingResponse.SubGroups { + var currentSubGroupEntries []*models.Entry + for _, entryID := range llmSubGroup.EntryIDs { + if entry, ok := entryMap[entryID]; ok { + currentSubGroupEntries = append(currentSubGroupEntries, entry) + groupedEntryIDs[entryID] = true + } + } + if len(currentSubGroupEntries) > 0 { + subGroups = append(subGroups, &models.EntryGroup{ + Title: llmSubGroup.Title, + Entries: currentSubGroupEntries, + Slug: utils.Slugify(llmSubGroup.Title), + TotalEntries: len(currentSubGroupEntries), + TotalFeeds: getUniqueFeedIDs(currentSubGroupEntries), + }) + } + } + // Handle ungrouped entries + var ungroupedEntries []*models.Entry + for _, entry := range entriesToProcess { + if !groupedEntryIDs[entry.ID] { + ungroupedEntries = append(ungroupedEntries, entry) + } + } + if len(ungroupedEntries) > 0 { + subGroups = append(subGroups, &models.EntryGroup{ + Title: "Uncategorized", + Entries: ungroupedEntries, + Slug: "uncategorized", + TotalEntries: len(ungroupedEntries), + TotalFeeds: getUniqueFeedIDs(ungroupedEntries), + }) + } + } + } + + // Fallback if sub-grouping failed or returned no groups + if len(subGroups) == 0 { + subGroups = append(subGroups, &models.EntryGroup{ + Title: "Uncategorized", + Entries: entriesToProcess, + Slug: "uncategorized", + TotalEntries: len(entriesToProcess), + TotalFeeds: getUniqueFeedIDs(entriesToProcess), + }) + } + + return summary, subGroups, nil +} + +func getUniqueFeedIDs(entries []*models.Entry) int { + uniqueFeedIDs := make(map[int64]bool) + for _, entry := range entries { + uniqueFeedIDs[entry.FeedID] = true + } + return len(uniqueFeedIDs) +} + +func BuildDigestDataByAI(entries []*models.Entry, ctx context.Context, llmService services.LLMService) []*models.PrimaryGroupDigestData { + highLevelGroups, err := GroupAIEntries(ctx, entries, llmService) + if err != nil { + categoryGroups := BuildDigestDataByCategory(entries) + if len(categoryGroups) == 1 && len(entries) > 1 { + return BuildDigestDataByDate(entries) + } + return categoryGroups + } + + var allPrimaryGroups []*models.PrimaryGroupDigestData + + for _, hlg := range highLevelGroups { + var allEntriesInHlg []*models.Entry + for _, subGroup := range hlg.SubGroups { + allEntriesInHlg = append(allEntriesInHlg, subGroup.Entries...) + } + + pg := &models.PrimaryGroup{ + ID: hlg.ID, + Title: hlg.Title, + Entries: allEntriesInHlg, + } + + summary, subGroups, err := ProcessPrimaryGroupWithAI(ctx, pg, llmService) + if err != nil { + log.Printf("Failed to process primary group '%s' with AI: %v. Using original group.", pg.Title, err) + // If processing fails, use the original primary group's entries as a single sub-group + hlg.SubGroups = []*models.EntryGroup{ + { + Title: pg.Title, + Entries: pg.Entries, + Slug: utils.Slugify(pg.Title), + TotalEntries: len(pg.Entries), + TotalFeeds: getUniqueFeedIDs(pg.Entries), + }, + } + } else { + hlg.Summary = summary + hlg.SubGroups = subGroups + } + + allPrimaryGroups = append(allPrimaryGroups, hlg) + } + + sort.Slice(allPrimaryGroups, func(i, j int) bool { + return allPrimaryGroups[i].Title < allPrimaryGroups[j].Title + }) + + + return allPrimaryGroups +} diff --git a/internal/digest/view/ai_test.go b/internal/digest/view/ai_test.go new file mode 100644 index 0000000..f10e489 --- /dev/null +++ b/internal/digest/view/ai_test.go @@ -0,0 +1,288 @@ + +package view + +import ( + "context" + "encoding/json" + "errors" + + "miniflux-digest/internal/models" + "miniflux-digest/internal/testutil" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "google.golang.org/genai" +) + +// mockLLMService is a mock implementation of the LLMService for testing. +type mockLLMService struct { + GenerateContentFunc func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) +} + +func (m *mockLLMService) GenerateContent(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + if m.GenerateContentFunc != nil { + return m.GenerateContentFunc(ctx, prompt, schema) + } + return nil, errors.New("mockLLMService.GenerateContentFunc is not implemented") +} + +func TestGroupAIEntries(t *testing.T) { + entries := []*models.Entry{ + {ID: 1, Title: "Entry 1", Content: "Content 1", FeedTitle: "Feed 1"}, + {ID: 2, Title: "Entry 2", Content: "Content 2", FeedTitle: "Feed 2"}, + {ID: 3, Title: "Entry 3", Content: "Content 3", FeedTitle: "Feed 1"}, + {ID: 4, Title: "Entry 4", Content: "Content 4", FeedTitle: "Feed 3"}, + } + + t.Run("happy path", func(t *testing.T) { + mockResp := GroupingResponse{ + Groups: []struct { + Title string `json:"title"` + EntryIDs []int64 `json:"entry_ids"` + }{ + {Title: "Group 1", EntryIDs: []int64{1, 2}}, + {Title: "Group 2", EntryIDs: []int64{3}}, + }, + } + mockJSON, _ := json.Marshal(mockResp) + + llmService := &mockLLMService{ + GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + return mockJSON, nil + }, + } + + groups, err := GroupAIEntries(context.Background(), entries, llmService) + assert.NoError(t, err) + assert.NotNil(t, groups) + assert.Len(t, groups, 3, "Expected 2 grouped and 1 uncategorized group") + + group1 := testutil.FindPrimaryGroup(groups, "Group 1") + assert.NotNil(t, group1) + assert.Len(t, group1.SubGroups[0].Entries, 2) + assert.Equal(t, 2, group1.TotalEntries) + + group2 := testutil.FindPrimaryGroup(groups, "Group 2") + assert.NotNil(t, group2) + assert.Len(t, group2.SubGroups[0].Entries, 1) + assert.Equal(t, 1, group2.TotalEntries) + + uncategorized := testutil.FindPrimaryGroup(groups, "Uncategorized") + assert.NotNil(t, uncategorized) + assert.Len(t, uncategorized.SubGroups[0].Entries, 1) + assert.Equal(t, int64(4), uncategorized.SubGroups[0].Entries[0].ID) + }) + + t.Run("llm service error", func(t *testing.T) { + llmService := &mockLLMService{ + GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + return nil, errors.New("llm error") + }, + } + + _, err := GroupAIEntries(context.Background(), entries, llmService) + assert.Error(t, err) + }) + + t.Run("llm returns invalid json", func(t *testing.T) { + llmService := &mockLLMService{ + GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + return []byte("invalid json"), nil + }, + } + + _, err := GroupAIEntries(context.Background(), entries, llmService) + assert.Error(t, err) + }) +} + +func TestProcessPrimaryGroupWithAI(t *testing.T) { + pg := &models.PrimaryGroup{ + ID: 1, + Title: "Test Primary Group", + Entries: []*models.Entry{ + {ID: 1, Title: "Entry 1", Content: "Content 1"}, + {ID: 2, Title: "Entry 2", Content: "Content 2"}, + {ID: 3, Title: "Entry 3", Content: "Content 3"}, + }, + } + + t.Run("happy path", func(t *testing.T) { + summaryResp := AIGroupSummaryResponse{Summary: "This is a summary."} + summaryJSON, _ := json.Marshal(summaryResp) + + subGroupingResp := AISubGroupingResponse{ + SubGroups: []struct { + Title string `json:"title"` + EntryIDs []int64 `json:"entry_ids"` + }{ + {Title: "Sub-group 1", EntryIDs: []int64{1, 2}}, + }, + } + subGroupingJSON, _ := json.Marshal(subGroupingResp) + + llmService := &mockLLMService{ + GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + if schema == SummaryResponseSchema { + return summaryJSON, nil + } + if schema == SubGroupingResponseSchema { + return subGroupingJSON, nil + } + return nil, errors.New("unexpected schema in happy path test") + }, + } + + summary, subGroups, err := ProcessPrimaryGroupWithAI(context.Background(), pg, llmService) + assert.NoError(t, err) + assert.Equal(t, "This is a summary.", summary) + assert.Len(t, subGroups, 2, "Expected 1 sub-group and 1 uncategorized") + + sg1 := subGroups[0] + assert.Equal(t, "Sub-group 1", sg1.Title) + assert.Len(t, sg1.Entries, 2) + + uncategorized := subGroups[1] + assert.Equal(t, "Uncategorized", uncategorized.Title) + assert.Len(t, uncategorized.Entries, 1) + assert.Equal(t, int64(3), uncategorized.Entries[0].ID) + }) + + t.Run("summarization fails", func(t *testing.T) { + subGroupingResp := AISubGroupingResponse{ + SubGroups: []struct { + Title string `json:"title"` + EntryIDs []int64 `json:"entry_ids"` + }{ + {Title: "Sub-group 1", EntryIDs: []int64{1, 2, 3}}, + }, + } + subGroupingJSON, _ := json.Marshal(subGroupingResp) + + llmService := &mockLLMService{ + GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + if schema == SummaryResponseSchema { + return nil, errors.New("summarization failed") + } + if schema == SubGroupingResponseSchema { + return subGroupingJSON, nil + } + return nil, errors.New("unexpected schema") + }, + } + + summary, subGroups, err := ProcessPrimaryGroupWithAI(context.Background(), pg, llmService) + assert.NoError(t, err) + assert.Empty(t, summary, "Summary should be empty when summarization fails") + assert.Len(t, subGroups, 1, "Should still get sub-groups even if summarization fails") + assert.Equal(t, "Sub-group 1", subGroups[0].Title) + }) + + t.Run("sub-grouping fails", func(t *testing.T) { + summaryResp := AIGroupSummaryResponse{Summary: "This is a summary."} + summaryJSON, _ := json.Marshal(summaryResp) + + llmService := &mockLLMService{ + GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + if schema == SummaryResponseSchema { + return summaryJSON, nil + } + if schema == SubGroupingResponseSchema { + return nil, errors.New("sub-grouping failed") + } + return nil, errors.New("unexpected schema") + }, + } + + summary, subGroups, err := ProcessPrimaryGroupWithAI(context.Background(), pg, llmService) + assert.NoError(t, err) + assert.Equal(t, "This is a summary.", summary) + assert.Len(t, subGroups, 1, "Should have one fallback 'Uncategorized' group") + assert.Equal(t, "Uncategorized", subGroups[0].Title) + assert.Len(t, subGroups[0].Entries, 3, "All entries should be in the fallback group") + }) +} + +func TestBuildDigestDataByAI(t *testing.T) { + entries := []*models.Entry{ + {ID: 1, Title: "Entry 1", Content: "Content 1", FeedTitle: "Feed 1", GroupID: 101, GroupTitle: "Category 1", Date: time.Now()}, + {ID: 2, Title: "Entry 2", Content: "Content 2", FeedTitle: "Feed 2", GroupID: 101, GroupTitle: "Category 1", Date: time.Now().Add(-1 * time.Hour)}, + {ID: 3, Title: "Entry 3", Content: "Content 3", FeedTitle: "Feed 1", GroupID: 102, GroupTitle: "Category 2", Date: time.Now().Add(-2 * time.Hour)}, + } + + t.Run("happy path", func(t *testing.T) { + groupingResp := GroupingResponse{ + Groups: []struct { + Title string `json:"title"` + EntryIDs []int64 `json:"entry_ids"` + }{{Title: "AI Group 1", EntryIDs: []int64{1, 2}}}, + } + groupingJSON, _ := json.Marshal(groupingResp) + + summaryResp := AIGroupSummaryResponse{Summary: "AI summary"} + summaryJSON, _ := json.Marshal(summaryResp) + + subGroupingResp := AISubGroupingResponse{ + SubGroups: []struct { + Title string `json:"title"` + EntryIDs []int64 `json:"entry_ids"` + }{{Title: "AI Sub-group", EntryIDs: []int64{1, 2}}}, + } + subGroupingJSON, _ := json.Marshal(subGroupingResp) + + llmService := &mockLLMService{ + GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + if schema == GroupingResponseSchema { + return groupingJSON, nil + } + if schema == SummaryResponseSchema { + return summaryJSON, nil + } + if schema == SubGroupingResponseSchema { + return subGroupingJSON, nil + } + return nil, errors.New("unexpected schema") + }, + } + + groups := BuildDigestDataByAI(entries, context.Background(), llmService) + assert.Len(t, groups, 2) + + aiGroup := testutil.FindPrimaryGroup(groups, "AI Group 1") + assert.NotNil(t, aiGroup) + assert.Equal(t, "AI summary", aiGroup.Summary) + assert.Len(t, aiGroup.SubGroups, 1) + assert.Equal(t, "AI Sub-group", aiGroup.SubGroups[0].Title) + }) + + t.Run("fallback to category grouping", func(t *testing.T) { + llmService := &mockLLMService{ + GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + return nil, errors.New("llm error") + }, + } + + groups := BuildDigestDataByAI(entries, context.Background(), llmService) + assert.Len(t, groups, 2, "Should fall back to 2 categories") + cat1 := testutil.FindPrimaryGroup(groups, "Category 1") + assert.NotNil(t, cat1) + cat2 := testutil.FindPrimaryGroup(groups, "Category 2") + assert.NotNil(t, cat2) + }) + + t.Run("fallback to date grouping", func(t *testing.T) { + singleCategoryEntries := []*models.Entry{ + {ID: 1, Title: "Entry 1", GroupID: 101, GroupTitle: "Category 1", Date: time.Now()}, + {ID: 2, Title: "Entry 2", GroupID: 101, GroupTitle: "Category 1", Date: time.Now().Add(-25 * time.Hour)}, + } + llmService := &mockLLMService{ + GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + return nil, errors.New("llm error") + }, + } + + groups := BuildDigestDataByAI(singleCategoryEntries, context.Background(), llmService) + assert.Len(t, groups, 2, "Should fall back to 2 date groups") + }) +} diff --git a/internal/digest/group_by/category.go b/internal/digest/view/category.go similarity index 65% rename from internal/digest/group_by/category.go rename to internal/digest/view/category.go index 9c7c718..744eaa9 100644 --- a/internal/digest/group_by/category.go +++ b/internal/digest/view/category.go @@ -1,7 +1,8 @@ -package group_by +package view import ( "miniflux-digest/internal/models" + "sort" ) func GroupByCategory(entries []*models.Entry) []*models.PrimaryGroup { @@ -33,3 +34,15 @@ func GroupByCategory(entries []*models.Entry) []*models.PrimaryGroup { return result } + +func BuildDigestDataByCategory(entries []*models.Entry) []*models.PrimaryGroupDigestData { + primaryGroups := GroupByCategory(entries) + + allPrimaryGroups := SubGroupByFeed(primaryGroups) + + sort.Slice(allPrimaryGroups, func(i, j int) bool { + return allPrimaryGroups[i].Title < allPrimaryGroups[j].Title + }) + + return allPrimaryGroups +} \ No newline at end of file diff --git a/internal/digest/view/category_test.go b/internal/digest/view/category_test.go new file mode 100644 index 0000000..e6aaf19 --- /dev/null +++ b/internal/digest/view/category_test.go @@ -0,0 +1,42 @@ +package view + +import ( + "miniflux-digest/internal/models" // Added this line + "miniflux-digest/internal/testutil" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGroupByCategory(t *testing.T) { + entries := testutil.CreateMockEntries(0) // Use mock entries + + groups := GroupByCategory(entries) + + assert.Len(t, groups, 3, "Expected 3 category groups") + + // Check "Category A" group + catAGroup := findPrimaryGroup(groups, "Category A") + assert.NotNil(t, catAGroup, "Category A group not found") + assert.Len(t, catAGroup.Entries, 4, "Expected 4 entries in Category A group") + + // Check "Category B" group + catBGroup := findPrimaryGroup(groups, "Category B") + assert.NotNil(t, catBGroup, "Category B group not found") + assert.Len(t, catBGroup.Entries, 3, "Expected 3 entries in Category B group") + + // Check "Category C" group + catCGroup := findPrimaryGroup(groups, "Category C") + assert.NotNil(t, catCGroup, "Category C group not found") + assert.Len(t, catCGroup.Entries, 2, "Expected 2 entries in Category C group") +} + +// findPrimaryGroup is a helper function to find a primary group by title +func findPrimaryGroup(groups []*models.PrimaryGroup, title string) *models.PrimaryGroup { + for _, group := range groups { + if group.Title == title { + return group + } + } + return nil +} \ No newline at end of file diff --git a/internal/digest/group_by/date.go b/internal/digest/view/date.go similarity index 84% rename from internal/digest/group_by/date.go rename to internal/digest/view/date.go index c030ee2..88eb7e0 100644 --- a/internal/digest/group_by/date.go +++ b/internal/digest/view/date.go @@ -1,4 +1,4 @@ -package group_by +package view import ( "miniflux-digest/internal/models" @@ -39,7 +39,7 @@ const ( DayGroupTitleLayout = "Jan 2, 2006" ) -func SubGroupByDay(pgs []*models.PrimaryGroup) ([]*models.PrimaryGroupDigestData, *string) { +func SubGroupByDay(pgs []*models.PrimaryGroup) []*models.PrimaryGroupDigestData { var allPrimaryGroups []*models.PrimaryGroupDigestData for _, pg := range pgs { @@ -87,5 +87,17 @@ func SubGroupByDay(pgs []*models.PrimaryGroup) ([]*models.PrimaryGroupDigestData }) } - return allPrimaryGroups, nil -} \ No newline at end of file + return allPrimaryGroups +} + +func BuildDigestDataByDate(entries []*models.Entry) []*models.PrimaryGroupDigestData { + primaryGroups := GroupByDate(entries) + + allPrimaryGroups := SubGroupByDay(primaryGroups) + + sort.Slice(allPrimaryGroups, func(i, j int) bool { + return allPrimaryGroups[i].Title < allPrimaryGroups[j].Title + }) + + return allPrimaryGroups +} diff --git a/internal/digest/view/date_test.go b/internal/digest/view/date_test.go new file mode 100644 index 0000000..6f5cc75 --- /dev/null +++ b/internal/digest/view/date_test.go @@ -0,0 +1,48 @@ +package view + +import ( + + "miniflux-digest/internal/testutil" + + "testing" +) + + + +func TestGroupByDate(t *testing.T) { + entries := testutil.CreateMockEntries(0) + groups := GroupByDate(entries) + + if len(groups) != 3 { + t.Fatalf("Expected 3 date groups, got %d", len(groups)) + } + + if groups[0].Title != "Jan 1, 2024" { + t.Errorf("Expected first group to be Jan 1, 2024, got %s", groups[0].Title) + } + + if len(groups[0].Entries) != 3 { + t.Errorf("Expected 3 entries in first group, got %d", len(groups[0].Entries)) + } +} + +func TestDayGrouper_GroupEntries(t *testing.T) { + entries := testutil.CreateMockEntries(0) + primaryGroupsMap := GroupByCategory(entries) + + groups := SubGroupByDay(primaryGroupsMap) + + if len(groups) != 3 { + t.Fatalf("Expected 3 primary groups, got %d", len(groups)) + } + + catAGroup := testutil.FindPrimaryGroup(groups, "Category A") + if catAGroup == nil || len(catAGroup.SubGroups) != 1 { + t.Fatalf("Incorrect sub-groups for Category A: %+v", catAGroup) + } + + subGroup := testutil.FindSubGroup(catAGroup, "Jan 2, 2024") + if subGroup == nil || len(subGroup.Entries) != 4 { + t.Fatalf("Incorrect entries for sub-group: %+v", subGroup) + } +} \ No newline at end of file diff --git a/internal/digest/group_by/feed.go b/internal/digest/view/feed.go similarity index 91% rename from internal/digest/group_by/feed.go rename to internal/digest/view/feed.go index 76313bc..443d576 100644 --- a/internal/digest/group_by/feed.go +++ b/internal/digest/view/feed.go @@ -1,4 +1,4 @@ -package group_by +package view import ( "miniflux-digest/internal/models" @@ -6,7 +6,7 @@ import ( "sort" ) -func SubGroupByFeed(pgs []*models.PrimaryGroup) ([]*models.PrimaryGroupDigestData, *string) { +func SubGroupByFeed(pgs []*models.PrimaryGroup) []*models.PrimaryGroupDigestData { var allPrimaryGroups []*models.PrimaryGroupDigestData for _, pg := range pgs { @@ -53,5 +53,5 @@ func SubGroupByFeed(pgs []*models.PrimaryGroup) ([]*models.PrimaryGroupDigestDat }) } - return allPrimaryGroups, nil + return allPrimaryGroups } diff --git a/internal/digest/view/feed_test.go b/internal/digest/view/feed_test.go new file mode 100644 index 0000000..b1a567c --- /dev/null +++ b/internal/digest/view/feed_test.go @@ -0,0 +1,61 @@ +package view + +import ( + "miniflux-digest/internal/models" + "miniflux-digest/internal/testutil" + + "testing" + + "github.com/stretchr/testify/assert" +) + + + +func TestSubGroupByFeed(t *testing.T) { + entries := testutil.CreateMockEntries(0) + primaryGroups := GroupByCategory(entries) // Use GroupByCategory to get initial primary groups + + groups := SubGroupByFeed(primaryGroups) + assert.Len(t, groups, 3, "Expected 3 primary groups after SubGroupByFeed") + + // Check "Category A" group + catAGroup := findPrimaryGroupDigestData(groups, "Category A") + assert.NotNil(t, catAGroup, "Category A group not found") + assert.Len(t, catAGroup.SubGroups, 3, "Expected 3 sub-groups for Category A") // Feed A, Feed C, Feed D + + // Check "Category B" group + catBGroup := findPrimaryGroupDigestData(groups, "Category B") + assert.NotNil(t, catBGroup, "Category B group not found") + assert.Len(t, catBGroup.SubGroups, 2, "Expected 2 sub-groups for Category B") // Feed B, Feed E + + // Check "Category C" group + catCGroup := findPrimaryGroupDigestData(groups, "Category C") + assert.NotNil(t, catCGroup, "Category C group not found") + assert.Len(t, catCGroup.SubGroups, 2, "Expected 2 sub-groups for Category C") // Feed F, Feed G +} + +// findPrimaryGroupDigestData is a helper function to find a PrimaryGroupDigestData by title +func findPrimaryGroupDigestData(groups []*models.PrimaryGroupDigestData, title string) *models.PrimaryGroupDigestData { + for _, group := range groups { + if group.Title == title { + return group + } + } + return nil +} + +func TestFeedGrouper_GroupEntries(t *testing.T) { + entries := testutil.CreateMockEntries(0) + primaryGroupsMap := GroupByCategory(entries) + + groups := SubGroupByFeed(primaryGroupsMap) + + if len(groups) != 3 { + t.Fatalf("Expected 3 primary groups, got %d", len(groups)) + } + + catAGroup := testutil.FindPrimaryGroup(groups, "Category A") + if catAGroup == nil || len(catAGroup.SubGroups) != 3 { + t.Fatalf("Incorrect sub-groups for Category A: %+v", catAGroup) + } +} diff --git a/internal/digest/view/view.go b/internal/digest/view/view.go new file mode 100644 index 0000000..8d22beb --- /dev/null +++ b/internal/digest/view/view.go @@ -0,0 +1,75 @@ +package view + +import ( + "context" + "miniflux-digest/internal/app/services" + "miniflux-digest/internal/models" + "sort" + "time" +) + +func BuildDigestDataForView(entries []*models.Entry, icons map[int64]*models.FeedIcon, viewType string, minifluxHost string, llmService services.LLMService) *models.OverviewTemplateData { + iconsSlice := make([]*models.FeedIcon, 0, len(icons)) + for _, icon := range icons { + iconsSlice = append(iconsSlice, icon) + } + + var resultPrimaryGroups []*models.PrimaryGroupDigestData + + + switch viewType { + case "date": + resultPrimaryGroups = BuildDigestDataByDate(entries) + case "category", "default": + resultPrimaryGroups = BuildDigestDataByCategory(entries) + case "ai": + resultPrimaryGroups = BuildDigestDataByAI(entries, context.Background(), llmService) + } + + if viewType == "date" || viewType == "category" { + for _, pg := range resultPrimaryGroups { + for _, sg := range pg.SubGroups { + sort.Slice(sg.Entries, func(i, j int) bool { + return sg.Entries[i].Date.Before(sg.Entries[j].Date) + }) + } + } + } + + for _, pg := range resultPrimaryGroups { + pgTotalEntries := 0 + pgUniqueFeedIDs := make(map[int64]bool) + for _, subGroup := range pg.SubGroups { + pgTotalEntries += len(subGroup.Entries) + for _, entry := range subGroup.Entries { + pgUniqueFeedIDs[entry.FeedID] = true + } + } + pg.TotalEntries = pgTotalEntries + pg.TotalFeeds = len(pgUniqueFeedIDs) + } + + var allEntryGroups []*models.EntryGroup + for _, pg := range resultPrimaryGroups { + allEntryGroups = append(allEntryGroups, pg.SubGroups...) + } + + uniqueFeedIDs := make(map[int64]bool) + for _, entry := range entries { + uniqueFeedIDs[entry.FeedID] = true + } + + return &models.OverviewTemplateData{ + Entries: entries, + GeneratedDate: time.Now(), + FeedIcons: iconsSlice, + PrimaryGroups: resultPrimaryGroups, + EntryGroups: allEntryGroups, + + TotalEntries: len(entries), + TotalFeeds: len(uniqueFeedIDs), + MinifluxHost: minifluxHost, + } +} + + diff --git a/internal/email/email.go b/internal/email/email.go index abca738..ae9aa62 100644 --- a/internal/email/email.go +++ b/internal/email/email.go @@ -46,7 +46,7 @@ func (s *EmailServiceImpl) Send(cfg *config.Config, overviewFile *os.File, group emailData := templates.EmailTemplateData{ OverviewTemplateData: *data, URL: overviewURL, - Summary: data.OverviewSummary, + } var body bytes.Buffer diff --git a/internal/email/email_test.go b/internal/email/email_test.go index b086306..b6a5061 100644 --- a/internal/email/email_test.go +++ b/internal/email/email_test.go @@ -56,7 +56,7 @@ func TestSend(t *testing.T) { data := models.OverviewTemplateData{ Entries: testutil.NewMockEntries(), FeedIcons: testutil.NewMockFeedIcons(), - OverviewSummary: "Test summary", + EntryGroups: []*models.EntryGroup{ testutil.NewMockEntryGroup(), }, diff --git a/internal/llm/gemini.go b/internal/llm/gemini.go index 8e56d73..bb8d399 100644 --- a/internal/llm/gemini.go +++ b/internal/llm/gemini.go @@ -13,6 +13,7 @@ import ( const ( Model = "gemini-2.5-flash" maxRetries = 3 + perTryTimeout = 3 * time.Minute Temperature float32 = 0.4 ) @@ -42,36 +43,38 @@ func NewGeminiService(apiKey string) (services.LLMService, error) { } func (s *GeminiService) generateContentWithRetry(ctx context.Context, prompt string, schema *genai.Schema) (*genai.GenerateContentResponse, error) { - var resp *genai.GenerateContentResponse - var err error + var lastErr error - for i := range maxRetries { - resp, err = s.client.GenerateContent(ctx, s.modelName, genai.Text(prompt), &genai.GenerateContentConfig{ + for i := 1; i <= maxRetries; i++ { + tryCtx, tryCancel := context.WithTimeout(ctx, perTryTimeout) + resp, err := s.client.GenerateContent(tryCtx, s.modelName, genai.Text(prompt), &genai.GenerateContentConfig{ ResponseMIMEType: "application/json", ResponseSchema: schema, Temperature: genai.Ptr(Temperature), }) + tryCancel() if err == nil { return resp, nil } - log.Printf("LLM call failed: %v", err) + lastErr = err + + log.Printf("LLM attempt %d/%d failed: %v", i, maxRetries, err) var apiError genai.APIError - if errors.As(err, &apiError) { - log.Printf("LLM API error: Code=%d, Status=%s, Message=%s", apiError.Code, apiError.Status, apiError.Message) - if apiError.Code == 503 || apiError.Code == 500 { - log.Printf("Retrying LLM call (%d/%d)...", i+1, maxRetries) - s.sleep(time.Second * time.Duration(i+1)) + if errors.As(err, &apiError) && (apiError.Code == 503 || apiError.Code == 500) { + if i < maxRetries { + log.Printf("Retrying after server error...") + s.sleep(time.Second * time.Duration(i)) continue } } - return nil, err + break } - return nil, err + return nil, lastErr } func (s *GeminiService) GenerateContent(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { @@ -100,4 +103,4 @@ func (s *GeminiService) GenerateContent(ctx context.Context, prompt string, sche } return jsonData, nil -} +} \ No newline at end of file diff --git a/internal/models/models.go b/internal/models/models.go index e4337a3..7b5018e 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -23,6 +23,18 @@ type Entry struct { Date time.Time } +type Feed struct { + ID int64 + Title string + Description string +} + +type Category struct { + ID int64 + Title string + Description string +} + type FeedIcon struct { FeedID int64 Data string @@ -34,7 +46,6 @@ type OverviewTemplateData struct { FeedIcons []*FeedIcon EntryGroups []*EntryGroup PrimaryGroups []*PrimaryGroupDigestData - OverviewSummary string TotalEntries int TotalFeeds int MinifluxHost string diff --git a/internal/processor/processor_test.go b/internal/processor/processor_test.go index 4d2d087..895fccd 100644 --- a/internal/processor/processor_test.go +++ b/internal/processor/processor_test.go @@ -61,7 +61,7 @@ func TestProcessDigest_Success(t *testing.T) { 101: {FeedID: 101, Data: "icon1"}, 102: {FeedID: 102, Data: "icon2"}, } - expectedOverviewData := &models.OverviewTemplateData{OverviewSummary: "Test Digest"} + expectedOverviewData := &models.OverviewTemplateData{} mockMinifluxClient := &mockMinifluxClientService{ getAllUnreadEntriesFunc: func() ([]*models.Entry, error) { diff --git a/internal/templates/email.gotxt b/internal/templates/email.gotxt index 9d14e89..e4ca713 100644 --- a/internal/templates/email.gotxt +++ b/internal/templates/email.gotxt @@ -1,19 +1,19 @@ -Hello, +Your digest for {{ .GeneratedDate.Format "Jan 2, 2006" }} is ready with {{.TotalEntries}} entries from {{.TotalFeeds}} feeds. -Your daily digest of {{ .Entries | len }} entries from {{ .EntryGroups | len }} feeds is ready. +---------------------------------------- -{{ if .OverviewSummary -}} -Overall Summary: -{{ .OverviewSummary }} +{{range .PrimaryGroups -}} +{{ .Title }} +{{ if .Summary -}} +{{ .Summary }} +{{- else -}} +(no summary) {{- end }} -You have {{.TotalEntries}} entries from {{.TotalFeeds}} feeds. +{{end}}---------------------------------------- {{ if .URL -}} -You can view the full digest online: -{{ .URL }} -{{- end }} - -{{ if .EntryGroups -}} -The individual digests are attached to this email for offline reading. +The individual HTML digests for each group are attached, or you can view the full digest online: {{ .URL }} +{{- else -}} +The individual HTML digests for each group are attached. {{- end }} diff --git a/internal/templates/overview.gohtml b/internal/templates/overview.gohtml index ace2d46..a980748 100644 --- a/internal/templates/overview.gohtml +++ b/internal/templates/overview.gohtml @@ -17,12 +17,7 @@

- {{if .OverviewSummary}} -
-

Summary

-
{{textToHTML .OverviewSummary}}
-
- {{end}} +

Grouped Digests

diff --git a/internal/templates/templates.go b/internal/templates/templates.go index 34565a9..e70a73a 100644 --- a/internal/templates/templates.go +++ b/internal/templates/templates.go @@ -11,7 +11,7 @@ import ( type EmailTemplateData struct { models.OverviewTemplateData URL string - Summary string + } //go:embed *.gohtml *.gotxt diff --git a/internal/templates/templates_test.go b/internal/templates/templates_test.go index 906de7e..0ebcaed 100644 --- a/internal/templates/templates_test.go +++ b/internal/templates/templates_test.go @@ -73,7 +73,7 @@ func TestOverviewTemplateExecution(t *testing.T) { Entries: testutil.NewMockEntries(), GeneratedDate: time.Now(), FeedIcons: testutil.NewMockFeedIcons(), - OverviewSummary: "This is a test summary.", + TotalEntries: 5, TotalFeeds: 2, MinifluxHost: "http://localhost:8080", diff --git a/internal/testutil/mocks.go b/internal/testutil/mocks.go index 7fe7b32..3467544 100644 --- a/internal/testutil/mocks.go +++ b/internal/testutil/mocks.go @@ -12,24 +12,16 @@ import ( type MockMinifluxClient struct { services.MinifluxClientService - - - - + + FeedIconFunc func(feedID int64) (*miniflux.FeedIcon, error) - + GetAllUnreadEntriesFunc func() (*miniflux.Entries, error) UpdateEntriesFunc func(entryIDs []int64, status string) error } - - - - - - func (m *MockMinifluxClient) FeedIcon(feedID int64) (*miniflux.FeedIcon, error) { if m.FeedIconFunc != nil { return m.FeedIconFunc(feedID) @@ -39,8 +31,6 @@ func (m *MockMinifluxClient) FeedIcon(feedID int64) (*miniflux.FeedIcon, error) - - func (m *MockMinifluxClient) GetAllUnreadEntries() (*miniflux.Entries, error) { if m.GetAllUnreadEntriesFunc != nil { return m.GetAllUnreadEntriesFunc() @@ -86,3 +76,22 @@ func (m *MockDigestService) BuildDigestData(category *miniflux.Category, entries } return nil } + +// Helper functions for tests +func FindPrimaryGroup(groups []*models.PrimaryGroupDigestData, title string) *models.PrimaryGroupDigestData { + for _, group := range groups { + if group.Title == title { + return group + } + } + return nil +} + +func FindSubGroup(primaryGroup *models.PrimaryGroupDigestData, title string) *models.EntryGroup { + for _, group := range primaryGroup.SubGroups { + if group.Title == title { + return group + } + } + return nil +} \ No newline at end of file diff --git a/internal/utils/slugify.go b/internal/utils/slugify.go index 88bb51e..b9935bf 100644 --- a/internal/utils/slugify.go +++ b/internal/utils/slugify.go @@ -17,3 +17,17 @@ func Slugify(input string) string { return slug } + +// UniqueWords extracts unique words from a string, converts them to lowercase. +func UniqueWords(input string) []string { + words := strings.Fields(strings.ToLower(input)) + uniqueMap := make(map[string]bool) + for _, word := range words { + uniqueMap[word] = true + } + uniqueSlice := make([]string, 0, len(uniqueMap)) + for word := range uniqueMap { + uniqueSlice = append(uniqueSlice, word) + } + return uniqueSlice +} \ No newline at end of file diff --git a/scripts/preview/main.go b/scripts/preview/main.go index f52122b..5e67200 100644 --- a/scripts/preview/main.go +++ b/scripts/preview/main.go @@ -93,7 +93,7 @@ func generateMockDigest(cfg *config.Config) *models.OverviewTemplateData { data.PrimaryGroups[2].Summary = "This is another mock summary for a different group, showing that not all groups have summaries." } - data.OverviewSummary = "This is a mock overview summary to see how it looks on the page. It should provide a high-level overview of all the digests included below." + return data } -- 2.54.0