gemini/feature/multi-digest-architecture #61

Merged
eleith merged 18 commits from gemini/feature/multi-digest-architecture into main 2025-12-17 23:15:33 +00:00
34 changed files with 1285 additions and 276 deletions

No files matched your search

-2
View File
@@ -14,5 +14,3 @@
# Gemini CLI state
/.gemini/
PROJECT.md
+15
View File
@@ -13,6 +13,21 @@ and delivered to the web and your inbox.
These entries can be sorted, summarized and group to enable skimming and to
decrease time spent catching up.
## Architecture
### Multi-Digest Engine
The application is designed to support multiple, distinct digests running in
parallel or on different schedules.
- **One Run, Many Outputs:** The system processes multiple defined digests.
- **Stack-Based Priority:** The order of digests in the configuration array
determines priority. The first digest (index 0) has the highest priority. If
an item matches Digest X (Index 0) and Digest Y (Index 1), it belongs to
Digest X.
- **Deterministic Filtering:** Inclusion is based on rules (Feed Title,
Category Title, Site URL, Entry URL) using Exact, Prefix, or Regex matching.
## Development Workflow
### Writing Code
+9 -6
View File
@@ -24,11 +24,12 @@ Customize the digest view, schedule and more. Oh my.
> many of the following features are _optional_
* ⏰ Automated scheduling (ex: daily, weekly, ever third friday)
* 📚 **Multi-Digest Support:** Configure multiple digests (e.g., "Daily Tech", "Weekly Social") with unique schedules and content filters.
* 🌞 Dark and light themes
* 📥 Fetches all unread entries
* 📧 Delivers personalized HTML digests via email
* 🛜 Archives HTML digests for static web serving
* 🤖 Summarize and group entries for faster skimming
* 🤖 Summarize and group entries for faster skimming (AI view)
* ✅ Automatically marks entries as read
* 🧹 Manages storage by purging older digests
* ♻️ Wash, rinse, repeat
@@ -77,10 +78,14 @@ Customize the digest view, schedule and more. Oh my.
A `config.yaml` file is required for operation.
Create this file in the root of the project directory and edit it.
Create this file in the root of the project directory. You can start by copying the [example](config.yaml.example):
See the [config.yaml.example](config.yaml.example) to learn about
requirements, defaults and other options.
```bash
cp config.yaml.example config.yaml
```
**📚 Documentation:**
For a detailed reference of all available options, including how to configure multiple digests and filters, please read [docs/CONFIG.md](docs/CONFIG.md).
### Run
@@ -116,8 +121,6 @@ project's license.
* support more LLM providers
* allow user to customize prompts
* support explicit feed/category sources
* support multiple digests
* support text/html email content
* fix web scrolling bug when selecting item content
+27 -21
View File
@@ -23,22 +23,24 @@ const (
ArchiveCleanupDays = 21
)
func digestJob(application *app.App, source string) {
log.Printf("Starting digest job from source: %s", source)
overviewFile, groupedEntryFiles, data, err := processor.ProcessDigest(application)
func digestJob(application *app.App, digestIndex int, source string) {
digestConfig := application.Config.Digests[digestIndex]
log.Printf("Starting digest job for '%s' from source: %s", digestConfig.Title, source)
overviewFile, groupedEntryFiles, data, err := processor.ProcessDigest(application, digestIndex)
if err != nil {
log.Printf("Error processing digest: %v", err)
log.Printf("Error processing digest '%s': %v", digestConfig.Title, err)
return
}
emailSent := application.Config.Smtp.Host != ""
if emailSent {
if err := application.EmailService.Send(application.Config, overviewFile, groupedEntryFiles, data); err != nil {
log.Printf("Error sending digest email: %v", err)
if err := application.EmailService.Send(application.Config.Smtp, digestConfig, overviewFile, groupedEntryFiles, data); err != nil {
log.Printf("Error sending digest email for '%s': %v", digestConfig.Title, err)
}
}
log.Printf("Digest produced at %s: entries=%d, folder=%s, email_sent=%t, source=%s",
log.Printf("Digest '%s' produced at %s: entries=%d, folder=%s, email_sent=%t, source=%s",
digestConfig.Title,
data.GeneratedDate.Format(time.RFC3339),
len(data.Entries),
overviewFile.Name(),
@@ -47,20 +49,24 @@ func digestJob(application *app.App, source string) {
)
}
func registerDigestJob(application *app.App, scheduler gocron.Scheduler) {
_, err := scheduler.NewJob(
gocron.CronJob(application.Config.Digest.Schedule, true),
gocron.NewTask(func() {
digestJob(application, "scheduler")
}),
)
func registerDigestJobs(application *app.App, scheduler gocron.Scheduler) {
for i, d := range application.Config.Digests {
digestConfig := d // Capture loop variable
digestIndex := i
_, err := scheduler.NewJob(
gocron.CronJob(digestConfig.Schedule, true),
gocron.NewTask(func() {
digestJob(application, digestIndex, "scheduler")
}),
)
if err != nil {
log.Fatalf("Error creating job: %v", err)
}
if err != nil {
log.Fatalf("Error creating job for digest '%s': %v", digestConfig.Title, err)
}
if application.Config.Digest.RunOnStartup {
go digestJob(application, "startup")
if digestConfig.RunOnStartup {
go digestJob(application, digestIndex, "startup")
}
}
}
@@ -93,7 +99,7 @@ func main() {
scheduler.Start()
registerDigestJob(application, scheduler)
registerDigestJobs(application, scheduler)
registerArchiveCleanupJob(application, scheduler)
go webserver.ListenAndServe(webserver.ArchiveBasePath, webserver.Port)
@@ -129,4 +135,4 @@ func initServices(cfg *config.Config) (*app.App, error) {
func initScheduler() (gocron.Scheduler, error) {
return gocron.NewScheduler()
}
}
+31 -29
View File
@@ -1,36 +1,38 @@
miniflux:
host: "YOUR_MINIFLUX_URL"
api_token: "YOUR_MINIFLUX_API_API_KEY"
host: "https://reader.miniflux.app"
api_token: "YOUR_MINIFLUX_API_TOKEN"
smtp:
host: "YOUR_SMTP_HOST"
host: "smtp.example.com"
port: 587
user: "YOUR_SMTP_USERNAME"
password: "YOUR_SMTP_PASSWORD"
digest:
email:
to: "RECIPIENT@example.com"
from: "SENDER@example.com"
# cron syntax also supported
schedule: "@every 24h"
# default http://localhost:8080, URL where HTML archives will be served
host: "https://your-digest-host.com"
# default true, compresses digest html
compress: true
# default false | true
mark_as_read: true
# default false | true
run_on_startup: false
# default date | category | ai
view: date
user: "smtp_user"
password: "smtp_password"
ai:
# gemini api key
api_key: "YOUR_GEMINI_API_KEY"
digests:
- title: "Daily Tech"
schedule: "0 8 * * *"
host: "https://digest.example.com"
view: "ai"
mark_as_read: true
email:
to: "recipient@example.com"
from: "sender@example.com"
filters:
feed_titles: ["Hacker News", "Ars Technica"]
category_titles: ["Technology"]
categories:
- title: "AI & LLMs"
description: "Artificial intelligence, machine learning, and LLM news."
- title: "Programming"
description: "Software development, languages, and tools."
- title: "Weekly Catch-All"
schedule: "@weekly"
host: "https://digest.example.com"
view: "date"
email:
to: "recipient@example.com"
from: "sender@example.com"
+95
View File
@@ -0,0 +1,95 @@
# Configuration Reference
The `miniflux-digest` application is configured via a YAML file (default: `config.yaml`). This document details all available options, their default values, and validation rules.
## Root Object
| Field | Type | Required | Description |
| :--- | :--- | :--- | :--- |
| `miniflux` | Object | **Yes** | Connection settings for your Miniflux instance. |
| `smtp` | Object | No | SMTP server settings for sending emails. |
| `ai` | Object | Conditional | Required if any digest uses `view: ai`. |
| `digests` | Array | **Yes** | A list of digest configurations. At least one must be defined. |
---
## Miniflux (`miniflux`)
| Field | Type | Description |
| :--- | :--- | :--- |
| `host` | String | **Required.** The full URL to your Miniflux instance (e.g., `https://reader.miniflux.app`). |
| `api_token` | String | **Required.** Your Miniflux API Key. Create one in Miniflux under **Settings > API Keys**. |
## SMTP (`smtp`)
Required only if you want to receive digests via email.
| Field | Type | Default | Description |
| :--- | :--- | :--- | :--- |
| `host` | String | No | The hostname of your SMTP server (e.g., `smtp.gmail.com`). **Required** if any digest has email configured. |
| `port` | Int | `587` | The SMTP port (1-65535). |
| `user` | String | - | SMTP username. |
| `password` | String | - | SMTP password. |
## AI (`ai`)
| Field | Type | Description |
| :--- | :--- | :--- |
| `api_key` | String | **Required if using AI View.** Your Google Gemini API Key. |
---
## Digests (`digests`)
You can define multiple digests. The order matters!
### Priority & Ownership
The application processes digests in the order they appear in the configuration list.
* **First Match Wins:** The first digest that matches an entry "claims" it.
* **No Duplicates:** An entry will never appear in more than one digest.
* **Catch-All:** It is common to define specific digests first (e.g., "Tech News") and a broad, filter-less digest last to catch everything else.
### Digest Object
| Field | Type | Required | Description |
| :--- | :--- | :--- | :--- |
| `title` | String | **Yes** | The display title of the digest. Must be unique after "slugification" (e.g., "My Digest" and "My-Digest" conflict). |
| `schedule` | String | **Yes** | When to run the digest. Supports Cron syntax (`0 8 * * *`) or Go-cron descriptors (`@daily`, `@weekly`, `@every 2h`). |
| `host` | String | No | The base URL where the static HTML archives will be hosted (e.g., `https://digest.myserver.com`). **Required** for correct links in emails. |
| `view` | String | No | One of: `date`, `category`, `ai`. Controls how entries are grouped and presented. Default: `date`. |
| `email` | Object | No | Email settings for this specific digest. |
| `filters` | Object | No | Rules for including entries in this digest. |
| `categories`| Array | No | **(AI View Only)** Custom categories to guide the LLM. |
| `compress` | Bool | No | If `true` (default), minifies the generated HTML. |
| `mark_as_read`| Bool | No | If `true`, marks entries as read in Miniflux *after* processing. Default: `false`. |
| `run_on_startup`| Bool | No | If `true`, runs the digest job immediately when the application starts. Default: `false`. |
### Email Object (`email`)
| Field | Type | Description |
| :--- | :--- | :--- |
| `to` | String | Recipient email address. |
| `from` | String | Sender email address. |
### Filters Object (`filters`)
All conditions within a list are **OR** (e.g., matching *any* feed title).
Conditions between different fields are **AND** (e.g., matching feed title *AND* site URL).
| Field | Type | Matching Logic | Description |
| :--- | :--- | :--- | :--- |
| `feed_titles` | String[] | Exact | Matches the exact title of the RSS feed. |
| `category_titles` | String[] | Exact | Matches the exact title of the Miniflux category. |
| `site_urls` | String[] | Prefix | Matches if the feed's site URL starts with this string. |
| `entry_urls` | String[] | Prefix | Matches if the entry's URL starts with this string. |
| `feed_title_patterns` | String[] | Regex | Go-flavor Regex match against feed title. |
| `category_title_patterns` | String[] | Regex | Go-flavor Regex match against category title. |
| `site_url_patterns` | String[] | Regex | Go-flavor Regex match against site URL. |
| `entry_url_patterns` | String[] | Regex | Go-flavor Regex match against entry URL. |
### Categories Object (`categories`)
Used only when `view: ai`. Provide a list of objects with `title` and `description` to help the AI categorize content.
If you do not provide this block, the AI will automatically determine the best primary groups for your content.
-1
View File
@@ -6,7 +6,6 @@ require (
github.com/go-co-op/gocron/v2 v2.16.2
github.com/go-playground/validator/v10 v10.27.0
github.com/knadh/koanf/parsers/yaml v1.1.0
github.com/knadh/koanf/providers/confmap v1.0.0
github.com/knadh/koanf/providers/file v1.2.0
github.com/knadh/koanf/v2 v2.2.2
github.com/robfig/cron/v3 v3.0.1
-2
View File
@@ -70,8 +70,6 @@ github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpb
github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI=
github.com/knadh/koanf/parsers/yaml v1.1.0 h1:3ltfm9ljprAHt4jxgeYLlFPmUaunuCgu1yILuTXRdM4=
github.com/knadh/koanf/parsers/yaml v1.1.0/go.mod h1:HHmcHXUrp9cOPcuC+2wrr44GTUB0EC+PyfN3HZD9tFg=
github.com/knadh/koanf/providers/confmap v1.0.0 h1:mHKLJTE7iXEys6deO5p6olAiZdG5zwp8Aebir+/EaRE=
github.com/knadh/koanf/providers/confmap v1.0.0/go.mod h1:txHYHiI2hAtF0/0sCmcuol4IDcuQbKTybiB1nOcUo1A=
github.com/knadh/koanf/providers/file v1.2.0 h1:hrUJ6Y9YOA49aNu/RSYzOTFlqzXSCpmYIDXI7OJU6+U=
github.com/knadh/koanf/providers/file v1.2.0/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA=
github.com/knadh/koanf/v2 v2.2.2 h1:ghbduIkpFui3L587wavneC9e3WIliCgiCgdxYO/wd7A=
+2 -2
View File
@@ -21,7 +21,7 @@ func (m *mockArchiveService) CleanArchive(maxAge time.Duration) {}
type mockEmailService struct{}
func (m *mockEmailService) Send(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error {
func (m *mockEmailService) Send(smtpConfig config.ConfigSmtp, digestConfig config.ConfigDigest, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error {
return nil
}
@@ -39,7 +39,7 @@ func (m *mockMinifluxClientService) UpdateEntries(entryIDs []int64, status strin
type mockDigestService struct{}
func (m *mockDigestService) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string) *models.OverviewTemplateData {
func (m *mockDigestService) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData {
return nil
}
+3
View File
@@ -41,14 +41,17 @@ func (m *MinifluxClientWrapper) GetAllUnreadEntries() ([]*models.Entry, error) {
categoryTitle = entry.Feed.Category.Title
}
var feedTitle string
var siteURL string
if entry.Feed != nil {
feedTitle = entry.Feed.Title
siteURL = entry.Feed.SiteURL
}
entries = append(entries, &models.Entry{
ID: entry.ID,
Title: entry.Title,
URL: entry.URL,
SiteURL: siteURL,
Content: entry.Content,
FeedID: entry.FeedID,
FeedTitle: feedTitle,
+2 -1
View File
@@ -1,9 +1,10 @@
package services
import (
"miniflux-digest/internal/config"
"miniflux-digest/internal/models"
)
type DigestService interface {
BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string) *models.OverviewTemplateData
BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData
}
+1 -1
View File
@@ -7,5 +7,5 @@ import (
)
type EmailService interface {
Send(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error
Send(smtpConfig config.ConfigSmtp, digestConfig config.ConfigDigest, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error
}
+2 -1
View File
@@ -65,7 +65,8 @@ func (s *ArchiveServiceImpl) makeGroupedEntriesArchiveFile(data *models.PrimaryG
}
func (s *ArchiveServiceImpl) makeOverviewArchiveFile(data *models.OverviewTemplateData) (*os.File, string, error) {
dateFolderPath := fmt.Sprintf("%s/%s", s.ArchiveBaseDir, data.GeneratedDate.Format("2006-01-02"))
digestSlug := data.DigestSlug
dateFolderPath := fmt.Sprintf("%s/%s/%s", s.ArchiveBaseDir, digestSlug, data.GeneratedDate.Format("2006-01-02"))
filename := fmt.Sprintf("%s/index.html", dateFolderPath)
err := os.MkdirAll(dateFolderPath, 0755)
+4 -3
View File
@@ -131,6 +131,7 @@ func TestMakeOverviewArchiveFile(t *testing.T) {
archiveService := NewArchiveService(tempDir, &mockHTMLTemplate{}, &mockHTMLTemplate{})
data := &models.OverviewTemplateData{
DigestSlug: "test-digest",
GeneratedDate: time.Date(2024, time.January, 1, 0, 0, 0, 0, time.UTC),
}
@@ -147,12 +148,12 @@ func TestMakeOverviewArchiveFile(t *testing.T) {
}
}()
expectedPath := filepath.Join(tempDir, "2024-01-01", "index.html")
expectedPath := filepath.Join(tempDir, "test-digest", "2024-01-01", "index.html")
if _, err := os.Stat(expectedPath); os.IsNotExist(err) {
t.Errorf("Expected file %s to exist", expectedPath)
}
if dateFolderPath != filepath.Join(tempDir, "2024-01-01") {
t.Errorf("Expected date folder path %s, got %s", filepath.Join(tempDir, "2024-01-01"), dateFolderPath)
if dateFolderPath != filepath.Join(tempDir, "test-digest", "2024-01-01") {
t.Errorf("Expected date folder path %s, got %s", filepath.Join(tempDir, "test-digest", "2024-01-01"), dateFolderPath)
}
}
+93 -25
View File
@@ -3,10 +3,13 @@ package config
import (
"errors"
"fmt"
"regexp"
"miniflux-digest/internal/utils"
"github.com/go-playground/validator/v10"
"github.com/go-viper/mapstructure/v2"
"github.com/knadh/koanf/parsers/yaml"
"github.com/knadh/koanf/providers/confmap"
"github.com/knadh/koanf/providers/file"
"github.com/knadh/koanf/v2"
"github.com/robfig/cron/v3"
@@ -41,14 +44,33 @@ type ConfigSmtp struct {
Password string `koanf:"password"`
}
type ConfigCategory struct {
Title string `koanf:"title" validate:"required"`
Description string `koanf:"description"`
}
type ConfigFilters struct {
FeedTitles []string `koanf:"feed_titles"`
CategoryTitles []string `koanf:"category_titles"`
FeedURLs []string `koanf:"feed_urls"`
EntryURLs []string `koanf:"entry_urls"`
FeedTitlePatterns []string `koanf:"feed_title_patterns"`
CategoryTitlePatterns []string `koanf:"category_title_patterns"`
FeedURLPatterns []string `koanf:"feed_url_patterns"`
EntryURLPatterns []string `koanf:"entry_url_patterns"`
}
type ConfigDigest struct {
Title string `koanf:"title" validate:"required"`
Email ConfigDigestEmail `koanf:"email"`
Schedule string `koanf:"schedule" validate:"gocron"`
Host string `koanf:"host"`
Compress bool `koanf:"compress"`
View string `koanf:"view" validate:"oneof=date category ai"`
MarkAsRead bool `koanf:"mark_as_read"`
Schedule string `koanf:"schedule" validate:"required,gocron"`
Host string `koanf:"host" validate:"omitempty,url"`
Compress *bool `koanf:"compress"`
View string `koanf:"view" validate:"oneof=date category ai"`
MarkAsRead bool `koanf:"mark_as_read"`
RunOnStartup bool `koanf:"run_on_startup"`
Filters ConfigFilters `koanf:"filters"`
Categories []ConfigCategory `koanf:"categories" validate:"dive"`
}
type ConfigAI struct {
@@ -58,7 +80,7 @@ type ConfigAI struct {
type Config struct {
Miniflux ConfigMiniflux `koanf:"miniflux"`
Smtp ConfigSmtp `koanf:"smtp"`
Digest ConfigDigest `koanf:"digest"`
Digests []ConfigDigest `koanf:"digests" validate:"min=1,dive"`
AI ConfigAI `koanf:"ai"`
}
@@ -72,8 +94,47 @@ func (c *Config) Validate() error {
validate.RegisterStructValidation(func(sl validator.StructLevel) {
cfg := sl.Current().Interface().(Config)
if cfg.Digest.View == "ai" && cfg.AI.ApiKey == "" {
sl.ReportError(cfg.AI.ApiKey, "AI.ApiKey", "ApiKey", "required_if", "Digest.View is 'ai'")
slugs := make(map[string]string)
for _, digest := range cfg.Digests {
if digest.View == "ai" && cfg.AI.ApiKey == "" {
sl.ReportError(cfg.AI.ApiKey, "AI.ApiKey", "ApiKey", "required_if", "Digest.View is 'ai'")
}
// Email Validation dependencies
if digest.Email.To != "" {
if cfg.Smtp.Host == "" {
sl.ReportError(cfg.Smtp.Host, "Smtp.Host", "Host", "required_if_email", fmt.Sprintf("SMTP Host is required because digest '%s' has email configured", digest.Title))
}
if digest.Host == "" {
sl.ReportError(digest.Host, "Digests.Host", "Host", "required_if_email", fmt.Sprintf("Digest Host is required because digest '%s' has email configured (needed for links)", digest.Title))
}
}
slug := utils.Slugify(digest.Title)
if slug == "" {
sl.ReportError(digest.Title, "Digests.Title", "Title", "invalid_slug", fmt.Sprintf("Digest title '%s' results in an empty slug", digest.Title))
}
if originalTitle, exists := slugs[slug]; exists {
sl.ReportError(digest.Title, "Digests.Title", "Title", "unique_slug", fmt.Sprintf("Digest title '%s' conflicts with '%s' (both slugify to '%s')", digest.Title, originalTitle, slug))
} else {
slugs[slug] = digest.Title
}
// Validate regex patterns
validatePatterns := func(patterns []string, fieldName string) {
for _, p := range patterns {
if _, err := regexp.Compile(p); err != nil {
sl.ReportError(digest.Filters, fieldName, fieldName, "regex", fmt.Sprintf("Invalid regex '%s': %v", p, err))
}
}
}
validatePatterns(digest.Filters.FeedTitlePatterns, "Filters.FeedTitlePatterns")
validatePatterns(digest.Filters.CategoryTitlePatterns, "Filters.CategoryTitlePatterns")
validatePatterns(digest.Filters.FeedURLPatterns, "Filters.FeedURLPatterns")
validatePatterns(digest.Filters.EntryURLPatterns, "Filters.EntryURLPatterns")
}
}, Config{})
@@ -94,33 +155,40 @@ func Load(path string) (*Config, error) {
k := koanf.New(".")
parser := yaml.Parser()
if err := setDefaultValues(k); err != nil {
return nil, err
}
if err := k.Load(file.Provider(path), parser); err != nil {
return nil, err
}
cfg := &Config{}
if err := k.Unmarshal("", &cfg); err != nil {
if err := k.UnmarshalWithConf("", &cfg, koanf.UnmarshalConf{
DecoderConfig: &mapstructure.DecoderConfig{
ErrorUnused: true,
TagName: "koanf",
ZeroFields: true,
Result: &cfg,
},
}); err != nil {
return nil, err
}
// Apply defaults
if cfg.Smtp.Port == 0 {
cfg.Smtp.Port = 587
}
for i := range cfg.Digests {
if cfg.Digests[i].Compress == nil {
def := true
cfg.Digests[i].Compress = &def
}
if cfg.Digests[i].View == "" {
cfg.Digests[i].View = "date"
}
}
if err := cfg.Validate(); err != nil {
return nil, err
}
return cfg, nil
}
func setDefaultValues(k *koanf.Koanf) error {
return k.Load(confmap.Provider(map[string]any{
"digest.compress": true,
"digest.view": "date",
"digest.host": "http://localhost:8080",
"digest.schedule": "@weekly",
"digest.mark_as_read": true,
"digest.run_on_startup": false,
}, "."), nil)
}
+220 -41
View File
@@ -21,9 +21,12 @@ func TestLoad(t *testing.T) {
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digest": map[string]any{
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "@daily",
},
"host": "http://localhost:8080",
"view": "category",
}},
"ai": map[string]any{
"api_key": "dummy-key",
},
@@ -36,9 +39,44 @@ func TestLoad(t *testing.T) {
"miniflux": map[string]any{
"api_token": "test-token",
},
"digest": map[string]any{
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "@daily",
}},
},
wantErr: true,
},
{
name: "valid config missing view defaults to date",
config: map[string]any{
"miniflux": map[string]any{
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "@daily",
"host": "http://localhost:8080",
}},
},
wantErr: false,
},
{
name: "email configured but missing smtp host",
config: map[string]any{
"miniflux": map[string]any{
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "@daily",
"host": "http://localhost:8080",
"view": "category",
"email": map[string]any{
"to": "test@example.com",
},
}},
},
wantErr: true,
},
@@ -48,9 +86,12 @@ func TestLoad(t *testing.T) {
"miniflux": map[string]any{
"host": "miniflux.example.com",
},
"digest": map[string]any{
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "@daily",
},
"host": "http://localhost:8080",
"view": "category",
}},
},
wantErr: true,
},
@@ -61,9 +102,12 @@ func TestLoad(t *testing.T) {
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digest": map[string]any{
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "@daily",
},
"host": "http://localhost:8080",
"view": "category",
}},
"smtp": map[string]any{
"port": 65536,
},
@@ -77,9 +121,12 @@ func TestLoad(t *testing.T) {
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digest": map[string]any{
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "@daily",
},
"host": "http://localhost:8080",
"view": "category",
}},
"smtp": map[string]any{
"port": 587,
},
@@ -96,12 +143,15 @@ func TestLoad(t *testing.T) {
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digest": map[string]any{
"schedule": "@daily",
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "@daily",
"host": "http://localhost:8080",
"view": "category",
"email": map[string]any{
"to": "invalid-email",
},
},
}},
},
wantErr: true,
},
@@ -112,12 +162,15 @@ func TestLoad(t *testing.T) {
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digest": map[string]any{
"schedule": "@daily",
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "@daily",
"host": "http://localhost:8080",
"view": "category",
"email": map[string]any{
"from": "another-invalid-email",
},
},
}},
},
wantErr: true,
},
@@ -128,12 +181,18 @@ func TestLoad(t *testing.T) {
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digest": map[string]any{
"schedule": "@daily",
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "@daily",
"host": "http://localhost:8080",
"view": "category",
"email": map[string]any{
"to": "test@example.com",
"from": "sender@example.com",
},
}},
"smtp": map[string]any{
"host": "smtp.example.com",
},
"ai": map[string]any{
"api_key": "dummy-key",
@@ -148,9 +207,12 @@ func TestLoad(t *testing.T) {
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digest": map[string]any{
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "* * 1 * *",
},
"host": "http://localhost:8080",
"view": "category",
}},
"ai": map[string]any{
"api_key": "dummy-key",
},
@@ -164,9 +226,12 @@ func TestLoad(t *testing.T) {
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digest": map[string]any{
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "* * * * * * *",
},
"host": "http://localhost:8080",
"view": "category",
}},
},
wantErr: true,
},
@@ -177,9 +242,12 @@ func TestLoad(t *testing.T) {
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digest": map[string]any{
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "@every 1h30m",
},
"host": "http://localhost:8080",
"view": "category",
}},
"ai": map[string]any{
"api_key": "dummy-key",
},
@@ -193,13 +261,16 @@ func TestLoad(t *testing.T) {
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digest": map[string]any{
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "@every bad-duration",
},
"host": "http://localhost:8080",
"view": "category",
}},
},
wantErr: true,
},
{
name: "invalid view value",
config: map[string]any{
@@ -207,10 +278,12 @@ func TestLoad(t *testing.T) {
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digest": map[string]any{
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "@daily",
"view": "magic",
},
"host": "http://localhost:8080",
"view": "magic",
}},
},
wantErr: true,
},
@@ -221,10 +294,123 @@ func TestLoad(t *testing.T) {
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digest": map[string]any{
"schedule": "@daily",
"view": "ai",
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "@daily",
"host": "http://localhost:8080",
"view": "ai",
}},
},
wantErr: true,
},
{
name: "missing digests",
config: map[string]any{
"miniflux": map[string]any{
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digests": []map[string]any{},
},
wantErr: true,
},
{
name: "duplicate digest slugs",
config: map[string]any{
"miniflux": map[string]any{
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digests": []map[string]any{
{
"title": "My Digest",
"schedule": "@daily",
"host": "http://localhost:8080",
"view": "category",
},
{
"title": "My-Digest",
"schedule": "@daily",
"host": "http://localhost:8080",
"view": "category",
},
},
},
wantErr: true,
},
{
name: "invalid regex pattern",
config: map[string]any{
"miniflux": map[string]any{
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "@daily",
"host": "http://localhost:8080",
"view": "category",
"filters": map[string]any{
"feed_title_patterns": []string{"(unclosed parenthesis"},
},
}},
},
wantErr: true,
},
{
name: "invalid category missing title",
config: map[string]any{
"miniflux": map[string]any{
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "@daily",
"host": "http://localhost:8080",
"view": "ai",
"categories": []map[string]any{
{"description": "No title here"},
},
}},
"ai": map[string]any{
"api_key": "dummy-key",
},
},
wantErr: true,
},
{
name: "unknown filter field",
config: map[string]any{
"miniflux": map[string]any{
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digests": []map[string]any{{
"title": "Daily Digest",
"schedule": "@daily",
"host": "http://localhost:8080",
"view": "category",
"filters": map[string]any{
"feed_title": []string{"Some Feed"}, // Incorrect field name
},
}},
},
wantErr: true,
},
{
name: "invalid digest title empty slug",
config: map[string]any{
"miniflux": map[string]any{
"host": "miniflux.example.com",
"api_token": "test-token",
},
"digests": []map[string]any{{
"title": "!!!",
"schedule": "@daily",
"host": "http://localhost:8080",
"view": "category",
}},
},
wantErr: true,
},
@@ -252,19 +438,12 @@ func TestLoad(t *testing.T) {
t.Fatalf("Failed to write dummy config file: %v", err)
}
cfg, err := Load(configPath)
_, err = Load(configPath)
if (err != nil) != tt.wantErr {
t.Errorf("Load() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !tt.wantErr && tt.name == "default digest.schedule is @weekly" {
if cfg.Digest.Schedule != "@weekly" {
t.Errorf("Expected digest.schedule to be @weekly, got %s", cfg.Digest.Schedule)
}
}
})
}
}
}
+3 -2
View File
@@ -2,6 +2,7 @@ package digest
import (
"miniflux-digest/internal/app/services"
"miniflux-digest/internal/config"
"miniflux-digest/internal/digest/view"
"miniflux-digest/internal/models"
)
@@ -14,6 +15,6 @@ type digestServiceImpl struct {
llmService services.LLMService
}
func (s *digestServiceImpl) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, viewType string, minifluxHost string, digestHost string) *models.OverviewTemplateData {
return view.BuildDigestDataForView(entries, icons, viewType, minifluxHost, digestHost, s.llmService)
func (s *digestServiceImpl) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, viewType string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData {
return view.BuildDigestDataForView(entries, icons, viewType, minifluxHost, digestHost, s.llmService, categories, digestTitle)
}
+5 -5
View File
@@ -89,7 +89,7 @@ func TestDigestService_BuildDigestData(t *testing.T) {
}
t.Run("view=date", func(t *testing.T) {
overviewData := digestService.BuildDigestData(entries, icons, "date", "http://miniflux.test", "")
overviewData := digestService.BuildDigestData(entries, icons, "date", "http://miniflux.test", "", nil, "")
if overviewData == nil {
t.Fatal("overviewData is nil")
}
@@ -112,7 +112,7 @@ func TestDigestService_BuildDigestData(t *testing.T) {
})
t.Run("view=category", func(t *testing.T) {
overviewData := digestService.BuildDigestData(entries, icons, "category", "http://miniflux.test", "")
overviewData := digestService.BuildDigestData(entries, icons, "category", "http://miniflux.test", "", nil, "")
if overviewData == nil {
t.Fatal("overviewData is nil")
}
@@ -133,7 +133,7 @@ func TestDigestService_BuildDigestData(t *testing.T) {
t.Run("view=ai", func(t *testing.T) {
aiEntries := createLLMGrouperMockEntries()
overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "")
overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "", nil, "")
if overviewData == nil {
t.Fatal("overviewData is nil")
}
@@ -156,7 +156,7 @@ func TestDigestService_BuildDigestData(t *testing.T) {
}
digestService := NewDigestService(mockLLM)
aiEntries := createLLMGrouperMockEntries()
overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "")
overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "", nil, "")
if overviewData == nil {
t.Fatal("overviewData is nil")
@@ -200,7 +200,7 @@ func TestDigestService_BuildDigestData(t *testing.T) {
}
digestService := NewDigestService(mockLLM)
aiEntries := createLLMGrouperMockEntries()
overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "")
overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "", nil, "")
if overviewData == nil {
t.Fatal("overviewData is nil")
+83 -29
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"log"
"miniflux-digest/internal/app/services"
"miniflux-digest/internal/config"
"miniflux-digest/internal/llm"
"miniflux-digest/internal/models"
"miniflux-digest/internal/utils"
@@ -33,7 +34,7 @@ type llmEntry struct {
func GroupAIEntries(ctx context.Context, entries []*models.Entry, llmService services.LLMService) (map[string][]*models.Entry, map[int64]bool, map[string][]*models.Entry) {
func GroupAIEntries(ctx context.Context, entries []*models.Entry, llmService services.LLMService, categories []config.ConfigCategory) (map[string][]*models.Entry, map[int64]bool, map[string][]*models.Entry) {
const chunkSize = 200
// Prepare for processing
@@ -42,7 +43,14 @@ func GroupAIEntries(ctx context.Context, entries []*models.Entry, llmService ser
entryMap[entry.ID] = entry
}
promptTemplate, err := template.New("grouping").Parse(initialGroupingPrompt)
var promptTemplateStr string
if len(categories) > 0 {
promptTemplateStr = strictGroupingPrompt
} else {
promptTemplateStr = openGroupingPrompt
}
promptTemplate, err := template.New("grouping").Parse(promptTemplateStr)
if err != nil {
failedChunks := make(map[string][]*models.Entry)
failedChunks["Prompt template parsing failed"] = entries
@@ -51,8 +59,9 @@ func GroupAIEntries(ctx context.Context, entries []*models.Entry, llmService ser
workerConfig := llmWorkerConfig{
promptTemplate: promptTemplate,
promptData: categories,
responseSchema: InitialGroupingResponseSchema,
prepareEntries: func(entries []*models.Entry) interface{} {
prepareEntries: func(entries []*models.Entry) any {
return prepareLLMEntries(entries)
},
}
@@ -171,7 +180,7 @@ func getSummaryForGroup(ctx context.Context, groupTitle string, entries []*model
llmResponseWorker := createLLMWorker[SummaryResponse](ctx, llmService, llmWorkerConfig{
promptFormat: summaryPrompt,
responseSchema: SummaryResponseSchema,
prepareEntries: func(entries []*models.Entry) interface{} {
prepareEntries: func(entries []*models.Entry) any {
return prepareLLMEntries(entries)
},
}, groupTitle)
@@ -220,7 +229,7 @@ func getSubGroupsForGroup(ctx context.Context, groupTitle string, entries []*mod
subGroupWorker := createLLMWorker[*SubGroupingResponse](ctx, llmService, llmWorkerConfig{
promptFormat: subGroupingPrompt,
responseSchema: SubGroupingResponseSchema,
prepareEntries: func(entries []*models.Entry) interface{} {
prepareEntries: func(entries []*models.Entry) any {
return prepareLLMEntries(entries)
},
}, groupTitle)
@@ -389,13 +398,20 @@ func prepareLLMEntries(entries []*models.Entry) []llmEntry {
return llmEntries
}
func BuildDigestDataByAI(entries []*models.Entry, ctx context.Context, llmService services.LLMService) []*models.PrimaryGroupDigestData {
rawGroups, groupedEntryIDs, failedChunks, fallbackGroups := initialAIGGrouping(ctx, entries, llmService)
func BuildDigestDataByAI(entries []*models.Entry, ctx context.Context, llmService services.LLMService, categories []config.ConfigCategory) []*models.PrimaryGroupDigestData {
rawGroups, groupedEntryIDs, failedChunks, fallbackGroups := initialAIGGrouping(ctx, entries, llmService, categories)
if fallbackGroups != nil {
return fallbackGroups
}
highLevelGroups := consolidateGroups(ctx, rawGroups, llmService)
var highLevelGroups []*models.PrimaryGroupDigestData
if len(categories) == 0 {
// Auto Mode: Consolidate the LLM's open-ended groups
highLevelGroups = consolidateGroups(ctx, rawGroups, llmService)
} else {
// User Mode: Use raw groups directly (strict adherence)
highLevelGroups = convertRawGroupsToDigestData(rawGroups)
}
highLevelGroups = handleFailedAndUngroupedEntries(highLevelGroups, entries, groupedEntryIDs, failedChunks)
@@ -406,8 +422,8 @@ func BuildDigestDataByAI(entries []*models.Entry, ctx context.Context, llmServic
return allPrimaryGroups
}
func initialAIGGrouping(ctx context.Context, entries []*models.Entry, llmService services.LLMService) (map[string][]*models.Entry, map[int64]bool, map[string][]*models.Entry, []*models.PrimaryGroupDigestData) {
rawGroups, groupedEntryIDs, failedChunks := GroupAIEntries(ctx, entries, llmService)
func initialAIGGrouping(ctx context.Context, entries []*models.Entry, llmService services.LLMService, categories []config.ConfigCategory) (map[string][]*models.Entry, map[int64]bool, map[string][]*models.Entry, []*models.PrimaryGroupDigestData) {
rawGroups, groupedEntryIDs, failedChunks := GroupAIEntries(ctx, entries, llmService, categories)
// If all chunks failed and we have no raw groups, fall back to category grouping.
if len(rawGroups) == 0 && len(failedChunks) > 0 {
@@ -466,23 +482,60 @@ func handleFailedAndUngroupedEntries(highLevelGroups []*models.PrimaryGroupDiges
}
if len(otherUngroupedEntries) > 0 {
highLevelGroups = append(highLevelGroups, &models.PrimaryGroupDigestData{
ID: int64(len(highLevelGroups) + 1),
Title: "Uncategorized",
Slug: "uncategorized",
SubGroups: []*models.EntryGroup{
{
Title: "Uncategorized",
Entries: otherUngroupedEntries,
Slug: "uncategorized",
TotalEntries: len(otherUngroupedEntries),
TotalFeeds: getUniqueFeedIDs(otherUngroupedEntries),
var merged bool
for _, group := range highLevelGroups {
if group.Title == "Uncategorized" {
merged = true
// Find or create the "Uncategorized" subgroup
var subGroup *models.EntryGroup
for _, sg := range group.SubGroups {
if sg.Title == "Uncategorized" {
subGroup = sg
break
}
}
if subGroup == nil {
subGroup = &models.EntryGroup{
Title: "Uncategorized",
Slug: "uncategorized",
}
group.SubGroups = append(group.SubGroups, subGroup)
}
subGroup.Entries = append(subGroup.Entries, otherUngroupedEntries...)
subGroup.TotalEntries = len(subGroup.Entries)
subGroup.TotalFeeds = getUniqueFeedIDs(subGroup.Entries)
// Update primary group totals
var allEntries []*models.Entry
for _, sg := range group.SubGroups {
allEntries = append(allEntries, sg.Entries...)
}
group.TotalEntries = len(allEntries)
group.TotalFeeds = getUniqueFeedIDs(allEntries)
break
}
}
if !merged {
highLevelGroups = append(highLevelGroups, &models.PrimaryGroupDigestData{
ID: int64(len(highLevelGroups) + 1),
Title: "Uncategorized",
Slug: "uncategorized",
SubGroups: []*models.EntryGroup{
{
Title: "Uncategorized",
Entries: otherUngroupedEntries,
Slug: "uncategorized",
TotalEntries: len(otherUngroupedEntries),
TotalFeeds: getUniqueFeedIDs(otherUngroupedEntries),
},
},
},
Summary: "",
TotalEntries: len(otherUngroupedEntries),
TotalFeeds: getUniqueFeedIDs(otherUngroupedEntries),
})
Summary: "",
TotalEntries: len(otherUngroupedEntries),
TotalFeeds: getUniqueFeedIDs(otherUngroupedEntries),
})
}
}
return highLevelGroups
}
@@ -544,9 +597,10 @@ func sortPrimaryGroups(allPrimaryGroups []*models.PrimaryGroupDigestData) {
// llmWorkerConfig holds configuration for creating an LLM worker function.
type llmWorkerConfig struct {
promptTemplate *template.Template
promptData any // Data to pass to template execution
promptFormat string // For prompts that use Sprintf
responseSchema *genai.Schema
prepareEntries func([]*models.Entry) interface{} // Function to prepare entries for LLM
prepareEntries func([]*models.Entry) any // Function to prepare entries for LLM
}
// createLLMWorker creates a worker function for llm.ProcessInChunks.
@@ -573,7 +627,7 @@ func createLLMWorker[R any](
// 2. Construct the prompt
var prompt bytes.Buffer
if config.promptTemplate != nil {
if err := config.promptTemplate.Execute(&prompt, nil); err != nil {
if err := config.promptTemplate.Execute(&prompt, config.promptData); err != nil {
return zero, fmt.Errorf("failed to execute prompt template: %w", err)
}
} else if config.promptFormat != "" {
@@ -591,4 +645,4 @@ func createLLMWorker[R any](
}
return response, nil
}
}
}
+19 -1
View File
@@ -2,7 +2,7 @@ package view
import "google.golang.org/genai"
const initialGroupingPrompt = `You are creating high-level primary groups for a digest.
const openGroupingPrompt = `You are creating high-level primary groups for a digest.
**Your Task:**
Group the entries below into broad, high-level primary groups.
@@ -20,6 +20,24 @@ Below is the list of entries:
----------------------------
`
const strictGroupingPrompt = `You are organizing a digest into specific categories.
**Your Task:**
Sort the entries below into the following categories:
{{range .}}
- **{{.Title}}**: {{.Description}}
{{end}}
**Rules:**
- You MUST use ONLY the categories listed above.
- Do NOT create new categories.
- If an entry does not fit well into any category, do NOT include it in a group (it will be handled as uncategorized later).
- An entry can only belong to one group.
Below is the list of entries:
----------------------------
`
const consolidationPrompt = `You are an expert content curator responsible for organizing a digest. You have been given a list of proposed primary group titles that were automatically generated. Your task is to ensure that this list is broad and high level and does not contain potentially overlapping topics. Each topic will later on be further subdivided into sub-groups, so if two topics are better served as sub-topics, they should be merged into one high level topic. Our goal is to have no more than 10-12 primary groups.
**Your Goal:**
+129 -6
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"miniflux-digest/internal/config"
"miniflux-digest/internal/models"
"miniflux-digest/internal/testutil"
"strings"
@@ -72,7 +73,7 @@ func TestGroupAIEntries(t *testing.T) {
},
}
rawGroups, groupedIDs, failedChunks := GroupAIEntries(context.Background(), entries, llmService)
rawGroups, groupedIDs, failedChunks := GroupAIEntries(context.Background(), entries, llmService, nil)
assert.Empty(t, failedChunks)
assert.NotNil(t, rawGroups)
assert.Len(t, rawGroups, 2, "Expected 2 grouped")
@@ -96,7 +97,7 @@ func TestGroupAIEntries(t *testing.T) {
},
}
_, _, failedChunks := GroupAIEntries(context.Background(), entries, llmService)
_, _, failedChunks := GroupAIEntries(context.Background(), entries, llmService, nil)
assert.NotEmpty(t, failedChunks)
})
@@ -107,7 +108,7 @@ func TestGroupAIEntries(t *testing.T) {
},
}
_, _, failedChunks := GroupAIEntries(context.Background(), entries, llmService)
_, _, failedChunks := GroupAIEntries(context.Background(), entries, llmService, nil)
assert.NotEmpty(t, failedChunks)
})
@@ -149,7 +150,7 @@ func TestGroupAIEntries(t *testing.T) {
},
}
rawGroups, groupedIDs, failedChunks := GroupAIEntries(context.Background(), largeEntries, llmService)
rawGroups, groupedIDs, failedChunks := GroupAIEntries(context.Background(), largeEntries, llmService, nil)
assert.Empty(t, failedChunks)
assert.Equal(t, 2, callCount, "Expected LLM to be called 2 times for 205 entries with chunk size 200")
@@ -450,7 +451,7 @@ func TestBuildDigestDataByAI(t *testing.T) {
},
}
groups := BuildDigestDataByAI(entries, context.Background(), llmService)
groups := BuildDigestDataByAI(entries, context.Background(), llmService, nil)
assert.Len(t, groups, 2, "Expected 1 consolidated group + 1 uncategorized")
techGroup := testutil.FindPrimaryGroup(groups, "Tech")
@@ -466,6 +467,128 @@ func TestBuildDigestDataByAI(t *testing.T) {
assert.Equal(t, int64(4), uncategorized.SubGroups[0].Entries[0].ID)
})
t.Run("strict user mode - no consolidation", func(t *testing.T) {
userCategories := []config.ConfigCategory{
{Title: "Strict Cat A"},
{Title: "Strict Cat B"},
}
// Entries 1 & 2 assigned to user categories
groupingResp := InitialGroupingResponse{
Groups: []struct {
Title string `json:"title"`
EntryIDs []int64 `json:"entry_ids"`
}{
{Title: "Strict Cat A", EntryIDs: []int64{1}},
{Title: "Strict Cat B", EntryIDs: []int64{2}},
},
}
groupingJSON, _ := json.Marshal(groupingResp)
llmService := &mockLLMService{
GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) {
if schema == ConsolidationResponseSchema {
t.Fatal("Consolidation should NOT be called in strict user mode")
}
if schema == InitialGroupingResponseSchema {
// Verify that strict prompt is used (implicitly via context of strict mode logic,
// but strict prompt text check would require inspecting 'prompt' arg which we can do)
assert.Contains(t, prompt, "Sort the entries below into the following categories")
return groupingJSON, nil
}
if schema == SummaryResponseSchema {
return json.Marshal(SummaryResponse{Summary: "Summary"})
}
if schema == SubGroupingResponseSchema {
// Return dummy subgroup
return json.Marshal(SubGroupingResponse{SubGroups: []struct {
Title string `json:"title"`
EntryIDs []int64 `json:"entry_ids"`
}{}})
}
return nil, nil
},
}
groups := BuildDigestDataByAI(entries, context.Background(), llmService, userCategories)
// Expected: Strict Cat A, Strict Cat B, Uncategorized (for 3 & 4)
assert.Len(t, groups, 3)
catA := testutil.FindPrimaryGroup(groups, "Strict Cat A")
assert.NotNil(t, catA)
catB := testutil.FindPrimaryGroup(groups, "Strict Cat B")
assert.NotNil(t, catB)
uncat := testutil.FindPrimaryGroup(groups, "Uncategorized")
assert.NotNil(t, uncat)
assert.Equal(t, 2, uncat.TotalEntries) // Entries 3 and 4
})
t.Run("merges ungrouped into existing uncategorized", func(t *testing.T) {
// Entry 1 & 2 -> "Uncategorized" via LLM
// Entry 3 -> Ungrouped (should be merged)
entries := []*models.Entry{
{ID: 1, Title: "E1"},
{ID: 2, Title: "E2"},
{ID: 3, Title: "E3"},
}
groupingResp := InitialGroupingResponse{
Groups: []struct {
Title string `json:"title"`
EntryIDs []int64 `json:"entry_ids"`
}{
{Title: "Uncategorized", EntryIDs: []int64{1, 2}},
},
}
groupingJSON, _ := json.Marshal(groupingResp)
// Mock Summary/Sub-grouping for the "Uncategorized" group
summaryResp := SummaryResponse{Summary: "Uncategorized stuff"}
summaryJSON, _ := json.Marshal(summaryResp)
subGroupingResp := SubGroupingResponse{
SubGroups: []struct {
Title string `json:"title"`
EntryIDs []int64 `json:"entry_ids"`
}{{Title: "Uncategorized", EntryIDs: []int64{1, 2}}},
}
subGroupingJSON, _ := json.Marshal(subGroupingResp)
llmService := &mockLLMService{
GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) {
switch schema {
case InitialGroupingResponseSchema:
return groupingJSON, nil
case ConsolidationResponseSchema:
// Should not be called if < 10 groups, but handle just in case
return nil, nil
case SummaryResponseSchema:
return summaryJSON, nil
case SubGroupingResponseSchema:
return subGroupingJSON, nil
default:
return nil, errors.New("unexpected schema")
}
},
}
groups := BuildDigestDataByAI(entries, context.Background(), llmService, nil)
require.Len(t, groups, 1, "Expected exactly 1 group (merged Uncategorized)")
uncatGroup := groups[0]
assert.Equal(t, "Uncategorized", uncatGroup.Title)
// Check that it contains all 3 entries (2 from LLM + 1 ungrouped)
var totalEntries int
for _, sg := range uncatGroup.SubGroups {
totalEntries += len(sg.Entries)
}
assert.Equal(t, 3, totalEntries, "Should contain all 3 entries")
})
t.Run("fallback to category grouping", func(t *testing.T) {
llmService := &mockLLMService{
GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) {
@@ -473,7 +596,7 @@ func TestBuildDigestDataByAI(t *testing.T) {
},
}
groups := BuildDigestDataByAI(entries, context.Background(), llmService)
groups := BuildDigestDataByAI(entries, context.Background(), llmService, nil)
assert.Len(t, groups, 3, "Should fall back to 3 categories")
})
+6 -2
View File
@@ -3,12 +3,14 @@ package view
import (
"context"
"miniflux-digest/internal/app/services"
"miniflux-digest/internal/config"
"miniflux-digest/internal/models"
"miniflux-digest/internal/utils"
"sort"
"time"
)
func BuildDigestDataForView(entries []*models.Entry, icons map[int64]*models.FeedIcon, viewType string, minifluxHost string, digestHost string, llmService services.LLMService) *models.OverviewTemplateData {
func BuildDigestDataForView(entries []*models.Entry, icons map[int64]*models.FeedIcon, viewType string, minifluxHost string, digestHost string, llmService services.LLMService, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData {
iconsSlice := make([]*models.FeedIcon, 0, len(icons))
for _, icon := range icons {
iconsSlice = append(iconsSlice, icon)
@@ -23,7 +25,7 @@ func BuildDigestDataForView(entries []*models.Entry, icons map[int64]*models.Fee
case "category", "default":
resultPrimaryGroups = BuildDigestDataByCategory(entries)
case "ai":
resultPrimaryGroups = BuildDigestDataByAI(entries, context.Background(), llmService)
resultPrimaryGroups = BuildDigestDataByAI(entries, context.Background(), llmService, categories)
}
if viewType == "date" || viewType == "category" {
@@ -60,6 +62,8 @@ func BuildDigestDataForView(entries []*models.Entry, icons map[int64]*models.Fee
}
return &models.OverviewTemplateData{
DigestTitle: digestTitle,
DigestSlug: utils.Slugify(digestTitle),
Entries: entries,
GeneratedDate: time.Now(),
FeedIcons: iconsSlice,
+9 -9
View File
@@ -17,29 +17,29 @@ type EmailServiceImpl struct{
EmailTemplate *template.Template
}
func (s *EmailServiceImpl) Send(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error {
func (s *EmailServiceImpl) Send(smtpConfig config.ConfigSmtp, digestConfig config.ConfigDigest, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error {
message := mail.NewMsg()
client, err := mail.NewClient(
cfg.Smtp.Host,
smtpConfig.Host,
mail.WithSMTPAuth(mail.SMTPAuthAutoDiscover),
mail.WithPort(cfg.Smtp.Port),
mail.WithUsername(cfg.Smtp.User),
mail.WithPassword(cfg.Smtp.Password))
mail.WithPort(smtpConfig.Port),
mail.WithUsername(smtpConfig.User),
mail.WithPassword(smtpConfig.Password))
if err != nil {
return err
}
if err := message.From(cfg.Digest.Email.From); err != nil {
if err := message.From(digestConfig.Email.From); err != nil {
return err
}
if err := message.To(cfg.Digest.Email.To); err != nil {
if err := message.To(digestConfig.Email.To); err != nil {
return err
}
subject := fmt.Sprintf("Your Miniflux Digest for %s", data.GeneratedDate.Format("January 2, 2006"))
overviewURL := fmt.Sprintf("%s/archive/%s/index.html", cfg.Digest.Host, data.GeneratedDate.Format("2006-01-02"))
overviewURL := fmt.Sprintf("%s/archive/%s/index.html", digestConfig.Host, data.GeneratedDate.Format("2006-01-02"))
message.Subject(subject)
@@ -60,4 +60,4 @@ func (s *EmailServiceImpl) Send(cfg *config.Config, overviewFile *os.File, group
}
return client.DialAndSend(message)
}
}
+13 -15
View File
@@ -11,20 +11,18 @@ import (
)
func TestSend(t *testing.T) {
cfg := &config.Config{
Smtp: config.ConfigSmtp{
Host: "localhost",
Port: 1025,
User: "test-user",
Password: "test-password",
},
Digest: config.ConfigDigest{
Email: config.ConfigDigestEmail{
To: "to@example.com",
From: "from@example.com",
},
Host: "https://example.com",
smtpConfig := config.ConfigSmtp{
Host: "localhost",
Port: 1025,
User: "test-user",
Password: "test-password",
}
digestConfig := config.ConfigDigest{
Email: config.ConfigDigestEmail{
To: "to@example.com",
From: "from@example.com",
},
Host: "https://example.com",
}
tmpFile, err := os.CreateTemp("", "test.html")
@@ -70,7 +68,7 @@ func TestSend(t *testing.T) {
emailService := &EmailServiceImpl{
EmailTemplate: templates.EmailTemplate,
}
err = emailService.Send(cfg, file, []*os.File{}, &data)
err = emailService.Send(smtpConfig, digestConfig, file, []*os.File{}, &data)
if err != nil {
// We expect an error because we are not running a real SMTP server.
// The important part is that the function attempts to connect.
@@ -95,4 +93,4 @@ func TestTextTemplateData(t *testing.T) {
if textData.URL != url {
t.Errorf("Expected URL to be %s, got %s", url, textData.URL)
}
}
}
+130
View File
@@ -0,0 +1,130 @@
package filter
import (
"regexp"
"strings"
"miniflux-digest/internal/config"
"miniflux-digest/internal/models"
)
type Matcher struct {
// Cache regex patterns to avoid recompiling
feedTitleRegex []*regexp.Regexp
categoryTitleRegex []*regexp.Regexp
siteURLRegex []*regexp.Regexp
entryURLRegex []*regexp.Regexp
}
func NewMatcher(cfg config.ConfigFilters) (*Matcher, error) {
m := &Matcher{}
var err error
m.feedTitleRegex, err = compileRegexes(cfg.FeedTitlePatterns)
if err != nil {
return nil, err
}
m.categoryTitleRegex, err = compileRegexes(cfg.CategoryTitlePatterns)
if err != nil {
return nil, err
}
m.siteURLRegex, err = compileRegexes(cfg.FeedURLPatterns)
if err != nil {
return nil, err
}
m.entryURLRegex, err = compileRegexes(cfg.EntryURLPatterns)
if err != nil {
return nil, err
}
return m, nil
}
func compileRegexes(patterns []string) ([]*regexp.Regexp, error) {
regexes := make([]*regexp.Regexp, 0, len(patterns))
for _, p := range patterns {
r, err := regexp.Compile(p)
if err != nil {
return nil, err
}
regexes = append(regexes, r)
}
return regexes, nil
}
// Matches returns true if the entry matches the filter criteria.
// Logic:
// 1. If no filters are defined, it's a match (default allow).
// 2. OR logic within a specific field (e.g., FeedTitles: ["A", "B"] -> match if A OR B).
// 3. AND logic between fields (e.g., FeedTitle: "A" AND SiteURL: "B" -> match only if both are true).
func (m *Matcher) Matches(entry *models.Entry, cfg config.ConfigFilters) bool {
// If all filters are empty, return true (default allow if "filters" block is missing/empty)
// However, usually we want at least one filter. But per spec "Stack-Based Priority",
// a digest might be a "catch-all" at the end.
if isEmpty(cfg) {
return true
}
if !matchString(entry.FeedTitle, cfg.FeedTitles, m.feedTitleRegex, true) {
return false
}
if !matchString(entry.GroupTitle, cfg.CategoryTitles, m.categoryTitleRegex, true) {
return false
}
if !matchString(entry.SiteURL, cfg.FeedURLs, m.siteURLRegex, false) {
return false
}
// EntryURL corresponds to entry.URL
if !matchString(entry.URL, cfg.EntryURLs, m.entryURLRegex, false) {
return false
}
return true
}
// matchString checks if value matches any of the exact strings or regex patterns.
// exactMatch: true for Full Match (Titles), false for Prefix Match (URLs).
func matchString(value string, exacts []string, patterns []*regexp.Regexp, exactMatch bool) bool {
// If no filters for this specific field are set, we "pass" this field.
// We only check for a match if there IS something to match against.
if len(exacts) == 0 && len(patterns) == 0 {
return true
}
for _, e := range exacts {
if exactMatch {
if value == e {
return true
}
} else {
if strings.HasPrefix(value, e) {
return true
}
}
}
for _, r := range patterns {
if r.MatchString(value) {
return true
}
}
return false
}
func isEmpty(cfg config.ConfigFilters) bool {
return len(cfg.FeedTitles) == 0 &&
len(cfg.CategoryTitles) == 0 &&
len(cfg.FeedURLs) == 0 &&
len(cfg.EntryURLs) == 0 &&
len(cfg.FeedTitlePatterns) == 0 &&
len(cfg.CategoryTitlePatterns) == 0 &&
len(cfg.FeedURLPatterns) == 0 &&
len(cfg.EntryURLPatterns) == 0
}
+118
View File
@@ -0,0 +1,118 @@
package filter
import (
"testing"
"miniflux-digest/internal/config"
"miniflux-digest/internal/models"
)
func TestMatcher_Matches(t *testing.T) {
tests := []struct {
name string
entry *models.Entry
filters config.ConfigFilters
wantMatch bool
}{
{
name: "Match exact feed title",
entry: &models.Entry{
FeedTitle: "Tech News", GroupTitle: "Some Category", SiteURL: "http://example.com",
URL: "http://example.com/post/1",
},
filters: config.ConfigFilters{
FeedTitles: []string{"Tech News"},
},
wantMatch: true,
},
{
name: "No match exact feed title",
entry: &models.Entry{
FeedTitle: "Cooking", GroupTitle: "Some Category", SiteURL: "http://example.com",
URL: "http://example.com/post/1",
},
filters: config.ConfigFilters{
FeedTitles: []string{"Tech News"},
},
wantMatch: false,
},
{
name: "Match prefix site url",
entry: &models.Entry{
FeedTitle: "A", GroupTitle: "Some Category", SiteURL: "https://github.com/user/repo",
URL: "http://example.com/post/1",
},
filters: config.ConfigFilters{
FeedURLs: []string{"https://github.com"},
},
wantMatch: true,
},
{
name: "No match prefix site url",
entry: &models.Entry{
FeedTitle: "A", GroupTitle: "Some Category", SiteURL: "https://google.com",
URL: "http://example.com/post/1",
},
filters: config.ConfigFilters{
FeedURLs: []string{"https://github.com"},
},
wantMatch: false,
},
{
name: "Match regex feed title",
entry: &models.Entry{
FeedTitle: "My Tech Blog", GroupTitle: "Some Category", SiteURL: "http://example.com",
URL: "http://example.com/post/1",
},
filters: config.ConfigFilters{
FeedTitlePatterns: []string{".*Tech.*"},
},
wantMatch: true,
},
{
name: "Mixed AND logic (Feed Title AND Category)",
entry: &models.Entry{
FeedTitle: "Tech News", GroupTitle: "Programming", SiteURL: "http://example.com",
URL: "http://example.com/post/1",
},
filters: config.ConfigFilters{
FeedTitles: []string{"Tech News"},
CategoryTitles: []string{"Programming"},
},
wantMatch: true,
},
{
name: "Mixed AND logic fail (Feed Title match, Category mismatch)",
entry: &models.Entry{
FeedTitle: "Tech News", GroupTitle: "Cooking", SiteURL: "http://example.com",
URL: "http://example.com/post/1",
},
filters: config.ConfigFilters{
FeedTitles: []string{"Tech News"},
CategoryTitles: []string{"Programming"},
},
wantMatch: false,
},
{
name: "Empty filters matches all",
entry: &models.Entry{
FeedTitle: "Any", GroupTitle: "Any", SiteURL: "Any",
URL: "Any",
},
filters: config.ConfigFilters{},
wantMatch: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
m, err := NewMatcher(tt.filters)
if err != nil {
t.Fatalf("Failed to create matcher: %v", err)
}
if got := m.Matches(tt.entry, tt.filters); got != tt.wantMatch {
t.Errorf("Matches() = %v, want %v", got, tt.wantMatch)
}
})
}
}
+39
View File
@@ -0,0 +1,39 @@
package manager
import (
"miniflux-digest/internal/config"
"miniflux-digest/internal/filter"
"miniflux-digest/internal/models"
)
type DigestManager struct {
digests []config.ConfigDigest
matchers []*filter.Matcher
}
func NewDigestManager(digests []config.ConfigDigest) (*DigestManager, error) {
matchers := make([]*filter.Matcher, len(digests))
for i, d := range digests {
m, err := filter.NewMatcher(d.Filters)
if err != nil {
return nil, err
}
matchers[i] = m
}
return &DigestManager{
digests: digests,
matchers: matchers,
}, nil
}
// GetOwningDigest returns the index of the first digest that matches the entry.
// Returns -1 if no digest matches.
func (dm *DigestManager) GetOwningDigest(entry *models.Entry) int {
for i, m := range dm.matchers {
if m.Matches(entry, dm.digests[i].Filters) {
return i
}
}
return -1
}
+84
View File
@@ -0,0 +1,84 @@
package manager
import (
"testing"
"miniflux-digest/internal/config"
"miniflux-digest/internal/models"
)
func TestGetOwningDigest(t *testing.T) {
digests := []config.ConfigDigest{
{
Title: "High Priority Tech",
Filters: config.ConfigFilters{
FeedTitles: []string{"Tech News"},
},
},
{
Title: "General News",
Filters: config.ConfigFilters{
// Empty filters match everything (catch-all)
},
},
}
dm, err := NewDigestManager(digests)
if err != nil {
t.Fatalf("Failed to create DigestManager: %v", err)
}
tests := []struct {
name string
entry *models.Entry
wantIndex int
}{
{
name: "Match first digest",
entry: &models.Entry{
FeedTitle: "Tech News",
SiteURL: "http://technews.com",
},
wantIndex: 0,
},
{
name: "Match second digest (catch-all)",
entry: &models.Entry{
FeedTitle: "Cooking Weekly",
SiteURL: "http://cooking.com",
},
wantIndex: 1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := dm.GetOwningDigest(tt.entry)
if got != tt.wantIndex {
t.Errorf("GetOwningDigest() = %v, want %v", got, tt.wantIndex)
}
})
}
}
func TestGetOwningDigest_NoMatch(t *testing.T) {
digests := []config.ConfigDigest{
{
Title: "Exclusive Club",
Filters: config.ConfigFilters{
FeedTitles: []string{"VIP"},
},
},
}
dm, err := NewDigestManager(digests)
if err != nil {
t.Fatalf("Failed to create DigestManager: %v", err)
}
entry := &models.Entry{FeedTitle: "Plebeian News"}
got := dm.GetOwningDigest(entry)
if got != -1 {
t.Errorf("GetOwningDigest() = %v, want -1", got)
}
}
+3
View File
@@ -14,6 +14,7 @@ type Entry struct {
ID int64
Title string
URL string
SiteURL string
Content string
FeedID int64
FeedTitle string
@@ -41,6 +42,8 @@ type FeedIcon struct {
}
type OverviewTemplateData struct {
DigestTitle string
DigestSlug string
Entries []*Entry
GeneratedDate time.Time
FeedIcons []*FeedIcon
+27 -9
View File
@@ -5,20 +5,36 @@ import (
"os"
"miniflux-digest/internal/app"
"miniflux-digest/internal/manager"
"miniflux-digest/internal/models"
miniflux "miniflux.app/v2/client"
)
func ProcessDigest(application *app.App) (*os.File, []*os.File, *models.OverviewTemplateData, error) {
func ProcessDigest(application *app.App, digestIndex int) (*os.File, []*os.File, *models.OverviewTemplateData, error) {
digestConfig := application.Config.Digests[digestIndex]
entries, err := application.MinifluxClientService.GetAllUnreadEntries()
if err != nil {
log.Printf("Error getting all unread entries: %v", err)
return nil, nil, nil, err
}
icons := make(map[int64]*models.FeedIcon)
digestManager, err := manager.NewDigestManager(application.Config.Digests)
if err != nil {
log.Printf("Error creating digest manager: %v", err)
return nil, nil, nil, err
}
var digestEntries []*models.Entry
for _, entry := range entries {
if digestManager.GetOwningDigest(entry) == digestIndex {
digestEntries = append(digestEntries, entry)
}
}
icons := make(map[int64]*models.FeedIcon)
for _, entry := range digestEntries {
if _, ok := icons[entry.FeedID]; !ok {
icon, err := application.MinifluxClientService.FeedIcon(entry.FeedID)
if err != nil {
@@ -33,22 +49,24 @@ func ProcessDigest(application *app.App) (*os.File, []*os.File, *models.Overview
}
data := application.DigestService.BuildDigestData(
entries,
digestEntries,
icons,
application.Config.Digest.View,
digestConfig.View,
application.Config.Miniflux.Host,
application.Config.Digest.Host,
digestConfig.Host,
digestConfig.Categories,
digestConfig.Title,
)
overviewFile, groupedEntryFiles, err := application.ArchiveService.MakeArchiveHTML(data, application.Config.Digest.Compress)
overviewFile, groupedEntryFiles, err := application.ArchiveService.MakeArchiveHTML(data, *digestConfig.Compress)
if err != nil {
log.Printf("Error generating archive HTML: %v", err)
log.Printf("Error generating HTML: %v", err)
return nil, nil, nil, err
}
if application.Config.Digest.MarkAsRead {
if digestConfig.MarkAsRead {
var entryIDs []int64
for _, entry := range entries {
for _, entry := range digestEntries {
entryIDs = append(entryIDs, entry.ID)
}
+11 -9
View File
@@ -30,11 +30,11 @@ func (m *mockMinifluxClientService) UpdateEntries(entryIDs []int64, status strin
}
type mockDigestService struct {
buildDigestDataFunc func(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string) *models.OverviewTemplateData
buildDigestDataFunc func(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData
}
func (m *mockDigestService) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string) *models.OverviewTemplateData {
return m.buildDigestDataFunc(entries, icons, view, minifluxHost, digestHost)
func (m *mockDigestService) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData {
return m.buildDigestDataFunc(entries, icons, view, minifluxHost, digestHost, categories, digestTitle)
}
type mockArchiveService struct {
@@ -78,7 +78,7 @@ func TestProcessDigest_Success(t *testing.T) {
},
}
mockDigestService := &mockDigestService{
buildDigestDataFunc: func(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string) *models.OverviewTemplateData {
buildDigestDataFunc: func(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData {
return expectedOverviewData
},
}
@@ -92,10 +92,12 @@ func TestProcessDigest_Success(t *testing.T) {
mockApp := app.NewApp(
app.WithConfig(&config.Config{
Digest: config.ConfigDigest{
MarkAsRead: true,
Compress: true,
View: "category",
Digests: []config.ConfigDigest{
{
MarkAsRead: true,
Compress: func() *bool { b := true; return &b }(),
View: "category",
},
},
Miniflux: config.ConfigMiniflux{Host: "http://miniflux.test"},
}),
@@ -104,7 +106,7 @@ func TestProcessDigest_Success(t *testing.T) {
app.WithArchiveService(mockArchiveService),
)
overviewFile, groupedEntryFiles, data, err := ProcessDigest(mockApp)
overviewFile, groupedEntryFiles, data, err := ProcessDigest(mockApp, 0)
if err != nil {
t.Fatalf("ProcessDigest returned an unexpected error: %v", err)
+25 -17
View File
@@ -81,24 +81,28 @@ type feedTemplate struct {
GroupTitle string
}
func createHardcodedMockEntries() []*models.Entry {
return []*models.Entry{
// Category A
{ID: 1, Title: "Entry 1", Date: time.Date(2024, 1, 2, 10, 0, 0, 0, time.UTC), FeedID: 100, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A", SiteURL: "www.example.com", URL: "www.example.com/1"},
{ID: 3, Title: "Entry 3", Date: time.Date(2024, 1, 2, 11, 0, 0, 0, time.UTC), FeedID: 100, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A", SiteURL: "www.example2.com", URL: "www.example2.com/1"},
{ID: 5, Title: "Entry 5", Date: time.Date(2024, 1, 2, 12, 0, 0, 0, time.UTC), FeedID: 300, FeedTitle: "Feed C", GroupID: 1, GroupTitle: "Category A", SiteURL: "www.example3.com", URL: "www.example3.com/1"},
{ID: 6, Title: "Entry 6", Date: time.Date(2024, 1, 2, 13, 0, 0, 0, time.UTC), FeedID: 400, FeedTitle: "Feed D", GroupID: 1, GroupTitle: "Category A", SiteURL: "www.example4.com", URL: "www.example4.com/1"},
// Category B
{ID: 2, Title: "Entry 2", Date: time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC), FeedID: 200, FeedTitle: "Feed B", GroupID: 2, GroupTitle: "Category B", SiteURL: "www.example5.com", URL: "www.example.com/1"},
{ID: 4, Title: "Entry 4", Content: "Content of entry 4", Date: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC), FeedID: 200, FeedTitle: "Feed B", GroupID: 2, GroupTitle: "Category B", SiteURL: "www.example5.com", URL: "www.example5.com/2"},
{ID: 7, Title: "Entry 7", Date: time.Date(2024, 1, 1, 14, 0, 0, 0, time.UTC), FeedID: 500, FeedTitle: "Feed E", GroupID: 2, GroupTitle: "Category B", SiteURL: "www.example5.com", URL: "www.example5.com/3"},
// Category C
{ID: 8, Title: "Entry 8", Date: time.Date(2024, 1, 3, 10, 0, 0, 0, time.UTC), FeedID: 600, FeedTitle: "Feed F", GroupID: 3, GroupTitle: "Category C", SiteURL: "www.example6.com", URL: "www.example7.com/1"},
{ID: 9, Title: "Entry 9", Date: time.Date(2024, 1, 3, 11, 0, 0, 0, time.UTC), FeedID: 700, FeedTitle: "Feed G", GroupID: 3, GroupTitle: "Category C", SiteURL: "www.example7.com", URL: "www.example7.com/1"},
}
}
func CreateMockEntries(n int) []*models.Entry {
if n == 0 {
return []*models.Entry{
// Category A
{ID: 1, Title: "Entry 1", Date: time.Date(2024, 1, 2, 10, 0, 0, 0, time.UTC), FeedID: 100, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A"},
{ID: 3, Title: "Entry 3", Date: time.Date(2024, 1, 2, 11, 0, 0, 0, time.UTC), FeedID: 100, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A"},
{ID: 5, Title: "Entry 5", Date: time.Date(2024, 1, 2, 12, 0, 0, 0, time.UTC), FeedID: 300, FeedTitle: "Feed C", GroupID: 1, GroupTitle: "Category A"},
{ID: 6, Title: "Entry 6", Date: time.Date(2024, 1, 2, 13, 0, 0, 0, time.UTC), FeedID: 400, FeedTitle: "Feed D", GroupID: 1, GroupTitle: "Category A"},
// Category B
{ID: 2, Title: "Entry 2", Date: time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC), FeedID: 200, FeedTitle: "Feed B", GroupID: 2, GroupTitle: "Category B"},
{ID: 4, Title: "Entry 4", Content: "Content of entry 4", Date: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC), FeedID: 200, FeedTitle: "Feed B", GroupID: 2, GroupTitle: "Category B"},
{ID: 7, Title: "Entry 7", Date: time.Date(2024, 1, 1, 14, 0, 0, 0, time.UTC), FeedID: 500, FeedTitle: "Feed E", GroupID: 2, GroupTitle: "Category B"},
// Category C
{ID: 8, Title: "Entry 8", Date: time.Date(2024, 1, 3, 10, 0, 0, 0, time.UTC), FeedID: 600, FeedTitle: "Feed F", GroupID: 3, GroupTitle: "Category C"},
{ID: 9, Title: "Entry 9", Date: time.Date(2024, 1, 3, 11, 0, 0, 0, time.UTC), FeedID: 700, FeedTitle: "Feed G", GroupID: 3, GroupTitle: "Category C"},
}
return createHardcodedMockEntries()
}
titleTemplates := []string{
@@ -134,10 +138,13 @@ func CreateMockEntries(n int) []*models.Entry {
commentsURL = fmt.Sprintf("https://example.com/comments/%d", i)
}
siteURL := fmt.Sprintf("https://www.example%d.com", template.FeedID)
entryURL := fmt.Sprintf("%s/item/%d", siteURL, i)
entry := &models.Entry{
ID: int64(i),
Title: fmt.Sprintf("Entry %d - %s", i, title),
URL: fmt.Sprintf("https://example.com/%d", i),
URL: entryURL,
CommentsURL: commentsURL,
Date: time.Now().Add(time.Duration(-i) * time.Hour),
Content: content,
@@ -145,6 +152,7 @@ func CreateMockEntries(n int) []*models.Entry {
FeedTitle: template.FeedTitle,
GroupID: template.GroupID,
GroupTitle: template.GroupTitle,
SiteURL: siteURL,
}
entries = append(entries, entry)
}
+6 -6
View File
@@ -58,21 +58,21 @@ func (m *MockArchiveService) CleanArchive(maxAge time.Duration) {}
type MockEmailService struct {
services.EmailService
SendFunc func(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error
SendFunc func(smtpConfig config.ConfigSmtp, digestConfig config.ConfigDigest, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error
}
func (m *MockEmailService) Send(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error {
return m.SendFunc(cfg, overviewFile, groupedEntryFiles, data)
func (m *MockEmailService) Send(smtpConfig config.ConfigSmtp, digestConfig config.ConfigDigest, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error {
return m.SendFunc(smtpConfig, digestConfig, overviewFile, groupedEntryFiles, data)
}
type MockDigestService struct {
services.DigestService
BuildDigestDataFunc func(category *miniflux.Category, entries *miniflux.Entries, icons map[int64]*models.FeedIcon, subGroupBy string, sortBy string, minifluxHost string) *models.OverviewTemplateData
BuildDigestDataFunc func(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData
}
func (m *MockDigestService) BuildDigestData(category *miniflux.Category, entries *miniflux.Entries, icons map[int64]*models.FeedIcon, subGroupBy string, sortBy string, minifluxHost string) *models.OverviewTemplateData {
func (m *MockDigestService) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData {
if m.BuildDigestDataFunc != nil {
return m.BuildDigestDataFunc(category, entries, icons, subGroupBy, sortBy, minifluxHost)
return m.BuildDigestDataFunc(entries, icons, view, minifluxHost, digestHost, categories, digestTitle)
}
return nil
}
+71 -31
View File
@@ -18,6 +18,7 @@ import (
"miniflux-digest/internal/digest"
"miniflux-digest/internal/email"
"miniflux-digest/internal/llm"
"miniflux-digest/internal/manager"
"miniflux-digest/internal/models"
"miniflux-digest/internal/templates"
"miniflux-digest/internal/testutil"
@@ -47,7 +48,7 @@ func openBrowser(url string) error {
return exec.Command(cmd, args...).Start()
}
func generateMockDigest(cfg *config.Config) *models.OverviewTemplateData {
func generateMockDigest(cfg *config.Config, digestIndex int) *models.OverviewTemplateData {
log.Println("generateDigestData: Starting...")
llmService, err := llm.NewGeminiService(cfg.AI.ApiKey)
@@ -55,6 +56,11 @@ func generateMockDigest(cfg *config.Config) *models.OverviewTemplateData {
log.Fatalf("Failed to create LLM service: %v", err)
}
digestManager, err := manager.NewDigestManager(cfg.Digests)
if err != nil {
log.Fatalf("Error creating digest manager: %v", err)
}
digestSvc := digest.NewDigestService(llmService)
log.Println("generateDigestData: DigestService initialized.")
@@ -62,8 +68,17 @@ func generateMockDigest(cfg *config.Config) *models.OverviewTemplateData {
log.Println("generateDigestData: Using mock data...")
entries = testutil.CreateMockEntries(200)
icons := make(map[int64]*models.FeedIcon)
var digestEntries []*models.Entry
digestConfig := cfg.Digests[digestIndex]
for _, entry := range entries {
if digestManager.GetOwningDigest(entry) == digestIndex {
digestEntries = append(digestEntries, entry)
}
}
icons := make(map[int64]*models.FeedIcon)
for _, entry := range digestEntries {
if _, ok := icons[entry.FeedID]; !ok {
switch entry.FeedID {
case 1:
@@ -80,26 +95,23 @@ func generateMockDigest(cfg *config.Config) *models.OverviewTemplateData {
log.Println("generateDigestData: Building digest data...")
data := digestSvc.BuildDigestData(
entries,
digestEntries,
icons,
cfg.Digest.View,
digestConfig.View,
cfg.Miniflux.Host,
cfg.Digest.Host,
digestConfig.Host,
digestConfig.Categories,
digestConfig.Title,
)
if len(data.PrimaryGroups) > 0 {
data.PrimaryGroups[0].Summary = "This is a mock summary for the first group to test the layout."
for i := range data.PrimaryGroups {
data.PrimaryGroups[i].Summary = fmt.Sprintf("This is a mock summary for group %d to test the layout.", i+1)
}
if len(data.PrimaryGroups) > 2 {
data.PrimaryGroups[2].Summary = "This is another mock summary for a different group, showing that not all groups have summaries."
}
return data
}
func generateMinifluxDigest(cfg *config.Config, minifluxClientService services.MinifluxClientService) *models.OverviewTemplateData {
func generateMinifluxDigest(cfg *config.Config, digestIndex int, minifluxClientService services.MinifluxClientService) *models.OverviewTemplateData {
log.Println("generateDigestData: Starting...")
llmService, err := llm.NewGeminiService(cfg.AI.ApiKey)
@@ -107,6 +119,11 @@ func generateMinifluxDigest(cfg *config.Config, minifluxClientService services.M
log.Fatalf("Failed to create LLM service: %v", err)
}
digestManager, err := manager.NewDigestManager(cfg.Digests)
if err != nil {
log.Fatalf("Error creating digest manager: %v", err)
}
digestSvc := digest.NewDigestService(llmService)
log.Println("generateDigestData: DigestService initialized.")
@@ -118,8 +135,17 @@ func generateMinifluxDigest(cfg *config.Config, minifluxClientService services.M
log.Fatalf("Failed to fetch entries: %v", err)
}
icons := make(map[int64]*models.FeedIcon)
var digestEntries []*models.Entry
digestConfig := cfg.Digests[digestIndex]
for _, entry := range entries {
if digestManager.GetOwningDigest(entry) == digestIndex {
digestEntries = append(digestEntries, entry)
}
}
icons := make(map[int64]*models.FeedIcon)
for _, entry := range digestEntries {
if _, ok := icons[entry.FeedID]; !ok {
icon, err := minifluxClientService.FeedIcon(entry.FeedID)
if err != nil {
@@ -132,11 +158,13 @@ func generateMinifluxDigest(cfg *config.Config, minifluxClientService services.M
log.Println("generateDigestData: Building digest data...")
return digestSvc.BuildDigestData(
entries,
digestEntries,
icons,
cfg.Digest.View,
digestConfig.View,
cfg.Miniflux.Host,
cfg.Digest.Host,
digestConfig.Host,
digestConfig.Categories,
digestConfig.Title,
)
}
@@ -157,22 +185,22 @@ func loadConfig() *config.Config {
return cfg
}
func generateAndArchiveHTML(cfg *config.Config, minifluxFlag bool) (*os.File, []*os.File, string, *models.OverviewTemplateData) {
func generateAndArchiveHTML(cfg *config.Config, digestIndex int, minifluxFlag bool) (*os.File, []*os.File, string, *models.OverviewTemplateData) {
var data *models.OverviewTemplateData
digestConfig := cfg.Digests[digestIndex]
if minifluxFlag {
minifluxClient := miniflux.NewClient(cfg.Miniflux.Host, cfg.Miniflux.ApiToken)
clientWrapper := app.NewMinifluxClientWrapper(minifluxClient)
data = generateMinifluxDigest(cfg, clientWrapper)
data = generateMinifluxDigest(cfg, digestIndex, clientWrapper)
} else {
data = generateMockDigest(cfg)
data = generateMockDigest(cfg, digestIndex)
}
log.Println("main: Digest data generated.")
archiveSvc := archive.NewArchiveService(webserver.ArchiveBasePath, templates.ArchiveTemplate, templates.OverviewTemplate)
overviewFile, groupedEntryFiles, err := archiveSvc.MakeArchiveHTML(data, cfg.Digest.Compress)
overviewFile, groupedEntryFiles, err := archiveSvc.MakeArchiveHTML(data, *digestConfig.Compress)
if err != nil {
log.Fatalf("Failed to generate HTML: %v", err)
}
@@ -186,7 +214,11 @@ func generateAndArchiveHTML(cfg *config.Config, minifluxFlag bool) (*os.File, []
log.Fatalf("Failed to get relative path: %v", err)
}
log.Printf("relativePath: %s", relativePath)
overviewURL := fmt.Sprintf("http://localhost%s/archive/%s", webserver.Port, relativePath)
overviewURL := fmt.Sprintf("http://localhost%s/%s", webserver.Port, relativePath)
if digestConfig.Host != "" {
overviewURL = fmt.Sprintf("%s/archive/%s", digestConfig.Host, relativePath)
}
log.Printf("overviewURL: %s", overviewURL)
return overviewFile, groupedEntryFiles, overviewURL, data
@@ -198,7 +230,8 @@ func handleEmail(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []
EmailTemplate: templates.EmailTemplate,
}
if err := emailSvc.Send(cfg, overviewFile, groupedEntryFiles, data); err != nil {
digestConfig := cfg.Digests[0]
if err := emailSvc.Send(cfg.Smtp, digestConfig, overviewFile, groupedEntryFiles, data); err != nil {
log.Fatalf("Failed to send email: %v", err)
}
log.Printf("Successfully generated and sent email.")
@@ -230,18 +263,25 @@ func main() {
emailFlag, minifluxFlag, htmlFlag := setupAndParseFlags()
cfg := loadConfig()
numDigest := len(cfg.Digests)
if emailFlag {
overviewFile, groupedEntryFiles, _, data := generateAndArchiveHTML(cfg, minifluxFlag)
handleEmail(cfg, overviewFile, groupedEntryFiles, data)
for digestIndex := range numDigest {
overviewFile, groupedEntryFiles, _, data := generateAndArchiveHTML(cfg, digestIndex, minifluxFlag)
handleEmail(cfg, overviewFile, groupedEntryFiles, data)
}
} else if htmlFlag {
_, _, overviewURL, _ := generateAndArchiveHTML(cfg, minifluxFlag)
handleOpenUrl(overviewURL)
for digestIndex := range numDigest {
_, _, overviewURL, _ := generateAndArchiveHTML(cfg, digestIndex, minifluxFlag)
handleOpenUrl(overviewURL)
}
} else if minifluxFlag {
_, _, overviewURL, _ := generateAndArchiveHTML(cfg, minifluxFlag)
handleOpenUrl(overviewURL)
for digestIndex := range numDigest {
_, _, overviewURL, _ := generateAndArchiveHTML(cfg, digestIndex, minifluxFlag)
handleOpenUrl(overviewURL)
}
} else {
log.Println("main: Starting web server")
handleWebServer()
}
}
}