diff --git a/.gitignore b/.gitignore index f2e73b1..b08f15e 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,3 @@ # Gemini CLI state /.gemini/ - -PROJECT.md diff --git a/AGENTS.md b/AGENTS.md index 7628d64..166d050 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,21 @@ and delivered to the web and your inbox. These entries can be sorted, summarized and group to enable skimming and to decrease time spent catching up. +## Architecture + +### Multi-Digest Engine + +The application is designed to support multiple, distinct digests running in +parallel or on different schedules. + +- **One Run, Many Outputs:** The system processes multiple defined digests. +- **Stack-Based Priority:** The order of digests in the configuration array + determines priority. The first digest (index 0) has the highest priority. If + an item matches Digest X (Index 0) and Digest Y (Index 1), it belongs to + Digest X. +- **Deterministic Filtering:** Inclusion is based on rules (Feed Title, + Category Title, Site URL, Entry URL) using Exact, Prefix, or Regex matching. + ## Development Workflow ### Writing Code diff --git a/README.md b/README.md index 467b49d..8795586 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,12 @@ Customize the digest view, schedule and more. Oh my. > many of the following features are _optional_ * โฐ Automated scheduling (ex: daily, weekly, ever third friday) +* ๐Ÿ“š **Multi-Digest Support:** Configure multiple digests (e.g., "Daily Tech", "Weekly Social") with unique schedules and content filters. * ๐ŸŒž Dark and light themes * ๐Ÿ“ฅ Fetches all unread entries * ๐Ÿ“ง Delivers personalized HTML digests via email * ๐Ÿ›œ Archives HTML digests for static web serving -* ๐Ÿค– Summarize and group entries for faster skimming +* ๐Ÿค– Summarize and group entries for faster skimming (AI view) * โœ… Automatically marks entries as read * ๐Ÿงน Manages storage by purging older digests * โ™ป๏ธ Wash, rinse, repeat @@ -77,10 +78,14 @@ Customize the digest view, schedule and more. Oh my. A `config.yaml` file is required for operation. - Create this file in the root of the project directory and edit it. + Create this file in the root of the project directory. You can start by copying the [example](config.yaml.example): - See the [config.yaml.example](config.yaml.example) to learn about - requirements, defaults and other options. + ```bash + cp config.yaml.example config.yaml + ``` + + **๐Ÿ“š Documentation:** + For a detailed reference of all available options, including how to configure multiple digests and filters, please read [docs/CONFIG.md](docs/CONFIG.md). ### Run @@ -116,8 +121,6 @@ project's license. * support more LLM providers * allow user to customize prompts -* support explicit feed/category sources -* support multiple digests * support text/html email content * fix web scrolling bug when selecting item content diff --git a/cmd/miniflux-digest/main.go b/cmd/miniflux-digest/main.go index f317456..abdd579 100644 --- a/cmd/miniflux-digest/main.go +++ b/cmd/miniflux-digest/main.go @@ -23,22 +23,24 @@ const ( ArchiveCleanupDays = 21 ) -func digestJob(application *app.App, source string) { - log.Printf("Starting digest job from source: %s", source) - overviewFile, groupedEntryFiles, data, err := processor.ProcessDigest(application) +func digestJob(application *app.App, digestIndex int, source string) { + digestConfig := application.Config.Digests[digestIndex] + log.Printf("Starting digest job for '%s' from source: %s", digestConfig.Title, source) + overviewFile, groupedEntryFiles, data, err := processor.ProcessDigest(application, digestIndex) if err != nil { - log.Printf("Error processing digest: %v", err) + log.Printf("Error processing digest '%s': %v", digestConfig.Title, err) return } emailSent := application.Config.Smtp.Host != "" if emailSent { - if err := application.EmailService.Send(application.Config, overviewFile, groupedEntryFiles, data); err != nil { - log.Printf("Error sending digest email: %v", err) + if err := application.EmailService.Send(application.Config.Smtp, digestConfig, overviewFile, groupedEntryFiles, data); err != nil { + log.Printf("Error sending digest email for '%s': %v", digestConfig.Title, err) } } - log.Printf("Digest produced at %s: entries=%d, folder=%s, email_sent=%t, source=%s", + log.Printf("Digest '%s' produced at %s: entries=%d, folder=%s, email_sent=%t, source=%s", + digestConfig.Title, data.GeneratedDate.Format(time.RFC3339), len(data.Entries), overviewFile.Name(), @@ -47,20 +49,24 @@ func digestJob(application *app.App, source string) { ) } -func registerDigestJob(application *app.App, scheduler gocron.Scheduler) { - _, err := scheduler.NewJob( - gocron.CronJob(application.Config.Digest.Schedule, true), - gocron.NewTask(func() { - digestJob(application, "scheduler") - }), - ) +func registerDigestJobs(application *app.App, scheduler gocron.Scheduler) { + for i, d := range application.Config.Digests { + digestConfig := d // Capture loop variable + digestIndex := i + _, err := scheduler.NewJob( + gocron.CronJob(digestConfig.Schedule, true), + gocron.NewTask(func() { + digestJob(application, digestIndex, "scheduler") + }), + ) - if err != nil { - log.Fatalf("Error creating job: %v", err) - } + if err != nil { + log.Fatalf("Error creating job for digest '%s': %v", digestConfig.Title, err) + } - if application.Config.Digest.RunOnStartup { - go digestJob(application, "startup") + if digestConfig.RunOnStartup { + go digestJob(application, digestIndex, "startup") + } } } @@ -93,7 +99,7 @@ func main() { scheduler.Start() - registerDigestJob(application, scheduler) + registerDigestJobs(application, scheduler) registerArchiveCleanupJob(application, scheduler) go webserver.ListenAndServe(webserver.ArchiveBasePath, webserver.Port) @@ -129,4 +135,4 @@ func initServices(cfg *config.Config) (*app.App, error) { func initScheduler() (gocron.Scheduler, error) { return gocron.NewScheduler() -} +} \ No newline at end of file diff --git a/config.yaml.example b/config.yaml.example index 5c1e181..0887f8e 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -1,36 +1,38 @@ miniflux: - host: "YOUR_MINIFLUX_URL" - api_token: "YOUR_MINIFLUX_API_API_KEY" + host: "https://reader.miniflux.app" + api_token: "YOUR_MINIFLUX_API_TOKEN" smtp: - host: "YOUR_SMTP_HOST" + host: "smtp.example.com" port: 587 - user: "YOUR_SMTP_USERNAME" - password: "YOUR_SMTP_PASSWORD" - -digest: - email: - to: "RECIPIENT@example.com" - from: "SENDER@example.com" - - # cron syntax also supported - schedule: "@every 24h" - - # default http://localhost:8080, URL where HTML archives will be served - host: "https://your-digest-host.com" - - # default true, compresses digest html - compress: true - - # default false | true - mark_as_read: true - - # default false | true - run_on_startup: false - - # default date | category | ai - view: date + user: "smtp_user" + password: "smtp_password" ai: - # gemini api key api_key: "YOUR_GEMINI_API_KEY" + +digests: + - title: "Daily Tech" + schedule: "0 8 * * *" + host: "https://digest.example.com" + view: "ai" + mark_as_read: true + email: + to: "recipient@example.com" + from: "sender@example.com" + filters: + feed_titles: ["Hacker News", "Ars Technica"] + category_titles: ["Technology"] + categories: + - title: "AI & LLMs" + description: "Artificial intelligence, machine learning, and LLM news." + - title: "Programming" + description: "Software development, languages, and tools." + + - title: "Weekly Catch-All" + schedule: "@weekly" + host: "https://digest.example.com" + view: "date" + email: + to: "recipient@example.com" + from: "sender@example.com" diff --git a/docs/CONFIG.md b/docs/CONFIG.md new file mode 100644 index 0000000..89416fd --- /dev/null +++ b/docs/CONFIG.md @@ -0,0 +1,95 @@ +# Configuration Reference + +The `miniflux-digest` application is configured via a YAML file (default: `config.yaml`). This document details all available options, their default values, and validation rules. + +## Root Object + +| Field | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| `miniflux` | Object | **Yes** | Connection settings for your Miniflux instance. | +| `smtp` | Object | No | SMTP server settings for sending emails. | +| `ai` | Object | Conditional | Required if any digest uses `view: ai`. | +| `digests` | Array | **Yes** | A list of digest configurations. At least one must be defined. | + +--- + +## Miniflux (`miniflux`) + +| Field | Type | Description | +| :--- | :--- | :--- | +| `host` | String | **Required.** The full URL to your Miniflux instance (e.g., `https://reader.miniflux.app`). | +| `api_token` | String | **Required.** Your Miniflux API Key. Create one in Miniflux under **Settings > API Keys**. | + +## SMTP (`smtp`) + +Required only if you want to receive digests via email. + +| Field | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `host` | String | No | The hostname of your SMTP server (e.g., `smtp.gmail.com`). **Required** if any digest has email configured. | +| `port` | Int | `587` | The SMTP port (1-65535). | +| `user` | String | - | SMTP username. | +| `password` | String | - | SMTP password. | + +## AI (`ai`) + +| Field | Type | Description | +| :--- | :--- | :--- | +| `api_key` | String | **Required if using AI View.** Your Google Gemini API Key. | + +--- + +## Digests (`digests`) + +You can define multiple digests. The order matters! + +### Priority & Ownership +The application processes digests in the order they appear in the configuration list. +* **First Match Wins:** The first digest that matches an entry "claims" it. +* **No Duplicates:** An entry will never appear in more than one digest. +* **Catch-All:** It is common to define specific digests first (e.g., "Tech News") and a broad, filter-less digest last to catch everything else. + +### Digest Object + +| Field | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| `title` | String | **Yes** | The display title of the digest. Must be unique after "slugification" (e.g., "My Digest" and "My-Digest" conflict). | +| `schedule` | String | **Yes** | When to run the digest. Supports Cron syntax (`0 8 * * *`) or Go-cron descriptors (`@daily`, `@weekly`, `@every 2h`). | +| `host` | String | No | The base URL where the static HTML archives will be hosted (e.g., `https://digest.myserver.com`). **Required** for correct links in emails. | +| `view` | String | No | One of: `date`, `category`, `ai`. Controls how entries are grouped and presented. Default: `date`. | +| `email` | Object | No | Email settings for this specific digest. | +| `filters` | Object | No | Rules for including entries in this digest. | +| `categories`| Array | No | **(AI View Only)** Custom categories to guide the LLM. | +| `compress` | Bool | No | If `true` (default), minifies the generated HTML. | +| `mark_as_read`| Bool | No | If `true`, marks entries as read in Miniflux *after* processing. Default: `false`. | +| `run_on_startup`| Bool | No | If `true`, runs the digest job immediately when the application starts. Default: `false`. | + +### Email Object (`email`) + +| Field | Type | Description | +| :--- | :--- | :--- | +| `to` | String | Recipient email address. | +| `from` | String | Sender email address. | + +### Filters Object (`filters`) + +All conditions within a list are **OR** (e.g., matching *any* feed title). +Conditions between different fields are **AND** (e.g., matching feed title *AND* site URL). + +| Field | Type | Matching Logic | Description | +| :--- | :--- | :--- | :--- | +| `feed_titles` | String[] | Exact | Matches the exact title of the RSS feed. | +| `category_titles` | String[] | Exact | Matches the exact title of the Miniflux category. | +| `site_urls` | String[] | Prefix | Matches if the feed's site URL starts with this string. | +| `entry_urls` | String[] | Prefix | Matches if the entry's URL starts with this string. | +| `feed_title_patterns` | String[] | Regex | Go-flavor Regex match against feed title. | +| `category_title_patterns` | String[] | Regex | Go-flavor Regex match against category title. | +| `site_url_patterns` | String[] | Regex | Go-flavor Regex match against site URL. | +| `entry_url_patterns` | String[] | Regex | Go-flavor Regex match against entry URL. | + +### Categories Object (`categories`) + +Used only when `view: ai`. Provide a list of objects with `title` and `description` to help the AI categorize content. + +If you do not provide this block, the AI will automatically determine the best primary groups for your content. + diff --git a/go.mod b/go.mod index 3644772..7838eae 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( github.com/go-co-op/gocron/v2 v2.16.2 github.com/go-playground/validator/v10 v10.27.0 github.com/knadh/koanf/parsers/yaml v1.1.0 - github.com/knadh/koanf/providers/confmap v1.0.0 github.com/knadh/koanf/providers/file v1.2.0 github.com/knadh/koanf/v2 v2.2.2 github.com/robfig/cron/v3 v3.0.1 diff --git a/go.sum b/go.sum index b472d2c..792a181 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,6 @@ github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpb github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= github.com/knadh/koanf/parsers/yaml v1.1.0 h1:3ltfm9ljprAHt4jxgeYLlFPmUaunuCgu1yILuTXRdM4= github.com/knadh/koanf/parsers/yaml v1.1.0/go.mod h1:HHmcHXUrp9cOPcuC+2wrr44GTUB0EC+PyfN3HZD9tFg= -github.com/knadh/koanf/providers/confmap v1.0.0 h1:mHKLJTE7iXEys6deO5p6olAiZdG5zwp8Aebir+/EaRE= -github.com/knadh/koanf/providers/confmap v1.0.0/go.mod h1:txHYHiI2hAtF0/0sCmcuol4IDcuQbKTybiB1nOcUo1A= github.com/knadh/koanf/providers/file v1.2.0 h1:hrUJ6Y9YOA49aNu/RSYzOTFlqzXSCpmYIDXI7OJU6+U= github.com/knadh/koanf/providers/file v1.2.0/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA= github.com/knadh/koanf/v2 v2.2.2 h1:ghbduIkpFui3L587wavneC9e3WIliCgiCgdxYO/wd7A= diff --git a/internal/app/app_test.go b/internal/app/app_test.go index c682d00..938e51e 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -21,7 +21,7 @@ func (m *mockArchiveService) CleanArchive(maxAge time.Duration) {} type mockEmailService struct{} -func (m *mockEmailService) Send(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error { +func (m *mockEmailService) Send(smtpConfig config.ConfigSmtp, digestConfig config.ConfigDigest, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error { return nil } @@ -39,7 +39,7 @@ func (m *mockMinifluxClientService) UpdateEntries(entryIDs []int64, status strin type mockDigestService struct{} -func (m *mockDigestService) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string) *models.OverviewTemplateData { +func (m *mockDigestService) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData { return nil } diff --git a/internal/app/miniflux_client_wrapper.go b/internal/app/miniflux_client_wrapper.go index 5d11945..7f35240 100644 --- a/internal/app/miniflux_client_wrapper.go +++ b/internal/app/miniflux_client_wrapper.go @@ -41,14 +41,17 @@ func (m *MinifluxClientWrapper) GetAllUnreadEntries() ([]*models.Entry, error) { categoryTitle = entry.Feed.Category.Title } var feedTitle string + var siteURL string if entry.Feed != nil { feedTitle = entry.Feed.Title + siteURL = entry.Feed.SiteURL } entries = append(entries, &models.Entry{ ID: entry.ID, Title: entry.Title, URL: entry.URL, + SiteURL: siteURL, Content: entry.Content, FeedID: entry.FeedID, FeedTitle: feedTitle, diff --git a/internal/app/services/digest.go b/internal/app/services/digest.go index 4c45f83..2680134 100644 --- a/internal/app/services/digest.go +++ b/internal/app/services/digest.go @@ -1,9 +1,10 @@ package services import ( + "miniflux-digest/internal/config" "miniflux-digest/internal/models" ) type DigestService interface { - BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string) *models.OverviewTemplateData + BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData } diff --git a/internal/app/services/email.go b/internal/app/services/email.go index dbad0b8..18cd4db 100644 --- a/internal/app/services/email.go +++ b/internal/app/services/email.go @@ -7,5 +7,5 @@ import ( ) type EmailService interface { - Send(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error + Send(smtpConfig config.ConfigSmtp, digestConfig config.ConfigDigest, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error } diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 905b4ba..7af73b4 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -65,7 +65,8 @@ func (s *ArchiveServiceImpl) makeGroupedEntriesArchiveFile(data *models.PrimaryG } func (s *ArchiveServiceImpl) makeOverviewArchiveFile(data *models.OverviewTemplateData) (*os.File, string, error) { - dateFolderPath := fmt.Sprintf("%s/%s", s.ArchiveBaseDir, data.GeneratedDate.Format("2006-01-02")) + digestSlug := data.DigestSlug + dateFolderPath := fmt.Sprintf("%s/%s/%s", s.ArchiveBaseDir, digestSlug, data.GeneratedDate.Format("2006-01-02")) filename := fmt.Sprintf("%s/index.html", dateFolderPath) err := os.MkdirAll(dateFolderPath, 0755) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 095b792..788aac1 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -131,6 +131,7 @@ func TestMakeOverviewArchiveFile(t *testing.T) { archiveService := NewArchiveService(tempDir, &mockHTMLTemplate{}, &mockHTMLTemplate{}) data := &models.OverviewTemplateData{ + DigestSlug: "test-digest", GeneratedDate: time.Date(2024, time.January, 1, 0, 0, 0, 0, time.UTC), } @@ -147,12 +148,12 @@ func TestMakeOverviewArchiveFile(t *testing.T) { } }() - expectedPath := filepath.Join(tempDir, "2024-01-01", "index.html") + expectedPath := filepath.Join(tempDir, "test-digest", "2024-01-01", "index.html") if _, err := os.Stat(expectedPath); os.IsNotExist(err) { t.Errorf("Expected file %s to exist", expectedPath) } - if dateFolderPath != filepath.Join(tempDir, "2024-01-01") { - t.Errorf("Expected date folder path %s, got %s", filepath.Join(tempDir, "2024-01-01"), dateFolderPath) + if dateFolderPath != filepath.Join(tempDir, "test-digest", "2024-01-01") { + t.Errorf("Expected date folder path %s, got %s", filepath.Join(tempDir, "test-digest", "2024-01-01"), dateFolderPath) } } diff --git a/internal/config/config.go b/internal/config/config.go index 27f1a0e..9e87ca8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,10 +3,13 @@ package config import ( "errors" "fmt" + "regexp" + + "miniflux-digest/internal/utils" "github.com/go-playground/validator/v10" + "github.com/go-viper/mapstructure/v2" "github.com/knadh/koanf/parsers/yaml" - "github.com/knadh/koanf/providers/confmap" "github.com/knadh/koanf/providers/file" "github.com/knadh/koanf/v2" "github.com/robfig/cron/v3" @@ -41,14 +44,33 @@ type ConfigSmtp struct { Password string `koanf:"password"` } +type ConfigCategory struct { + Title string `koanf:"title" validate:"required"` + Description string `koanf:"description"` +} + +type ConfigFilters struct { + FeedTitles []string `koanf:"feed_titles"` + CategoryTitles []string `koanf:"category_titles"` + FeedURLs []string `koanf:"feed_urls"` + EntryURLs []string `koanf:"entry_urls"` + FeedTitlePatterns []string `koanf:"feed_title_patterns"` + CategoryTitlePatterns []string `koanf:"category_title_patterns"` + FeedURLPatterns []string `koanf:"feed_url_patterns"` + EntryURLPatterns []string `koanf:"entry_url_patterns"` +} + type ConfigDigest struct { + Title string `koanf:"title" validate:"required"` Email ConfigDigestEmail `koanf:"email"` - Schedule string `koanf:"schedule" validate:"gocron"` - Host string `koanf:"host"` - Compress bool `koanf:"compress"` - View string `koanf:"view" validate:"oneof=date category ai"` - MarkAsRead bool `koanf:"mark_as_read"` + Schedule string `koanf:"schedule" validate:"required,gocron"` + Host string `koanf:"host" validate:"omitempty,url"` + Compress *bool `koanf:"compress"` + View string `koanf:"view" validate:"oneof=date category ai"` + MarkAsRead bool `koanf:"mark_as_read"` RunOnStartup bool `koanf:"run_on_startup"` + Filters ConfigFilters `koanf:"filters"` + Categories []ConfigCategory `koanf:"categories" validate:"dive"` } type ConfigAI struct { @@ -58,7 +80,7 @@ type ConfigAI struct { type Config struct { Miniflux ConfigMiniflux `koanf:"miniflux"` Smtp ConfigSmtp `koanf:"smtp"` - Digest ConfigDigest `koanf:"digest"` + Digests []ConfigDigest `koanf:"digests" validate:"min=1,dive"` AI ConfigAI `koanf:"ai"` } @@ -72,8 +94,47 @@ func (c *Config) Validate() error { validate.RegisterStructValidation(func(sl validator.StructLevel) { cfg := sl.Current().Interface().(Config) - if cfg.Digest.View == "ai" && cfg.AI.ApiKey == "" { - sl.ReportError(cfg.AI.ApiKey, "AI.ApiKey", "ApiKey", "required_if", "Digest.View is 'ai'") + slugs := make(map[string]string) + + for _, digest := range cfg.Digests { + if digest.View == "ai" && cfg.AI.ApiKey == "" { + sl.ReportError(cfg.AI.ApiKey, "AI.ApiKey", "ApiKey", "required_if", "Digest.View is 'ai'") + } + + // Email Validation dependencies + if digest.Email.To != "" { + if cfg.Smtp.Host == "" { + sl.ReportError(cfg.Smtp.Host, "Smtp.Host", "Host", "required_if_email", fmt.Sprintf("SMTP Host is required because digest '%s' has email configured", digest.Title)) + } + if digest.Host == "" { + sl.ReportError(digest.Host, "Digests.Host", "Host", "required_if_email", fmt.Sprintf("Digest Host is required because digest '%s' has email configured (needed for links)", digest.Title)) + } + } + + slug := utils.Slugify(digest.Title) + if slug == "" { + sl.ReportError(digest.Title, "Digests.Title", "Title", "invalid_slug", fmt.Sprintf("Digest title '%s' results in an empty slug", digest.Title)) + } + + if originalTitle, exists := slugs[slug]; exists { + sl.ReportError(digest.Title, "Digests.Title", "Title", "unique_slug", fmt.Sprintf("Digest title '%s' conflicts with '%s' (both slugify to '%s')", digest.Title, originalTitle, slug)) + } else { + slugs[slug] = digest.Title + } + + // Validate regex patterns + validatePatterns := func(patterns []string, fieldName string) { + for _, p := range patterns { + if _, err := regexp.Compile(p); err != nil { + sl.ReportError(digest.Filters, fieldName, fieldName, "regex", fmt.Sprintf("Invalid regex '%s': %v", p, err)) + } + } + } + + validatePatterns(digest.Filters.FeedTitlePatterns, "Filters.FeedTitlePatterns") + validatePatterns(digest.Filters.CategoryTitlePatterns, "Filters.CategoryTitlePatterns") + validatePatterns(digest.Filters.FeedURLPatterns, "Filters.FeedURLPatterns") + validatePatterns(digest.Filters.EntryURLPatterns, "Filters.EntryURLPatterns") } }, Config{}) @@ -94,33 +155,40 @@ func Load(path string) (*Config, error) { k := koanf.New(".") parser := yaml.Parser() - if err := setDefaultValues(k); err != nil { - return nil, err - } - if err := k.Load(file.Provider(path), parser); err != nil { return nil, err } cfg := &Config{} - if err := k.Unmarshal("", &cfg); err != nil { + if err := k.UnmarshalWithConf("", &cfg, koanf.UnmarshalConf{ + DecoderConfig: &mapstructure.DecoderConfig{ + ErrorUnused: true, + TagName: "koanf", + ZeroFields: true, + Result: &cfg, + }, + }); err != nil { return nil, err } + // Apply defaults + if cfg.Smtp.Port == 0 { + cfg.Smtp.Port = 587 + } + + for i := range cfg.Digests { + if cfg.Digests[i].Compress == nil { + def := true + cfg.Digests[i].Compress = &def + } + if cfg.Digests[i].View == "" { + cfg.Digests[i].View = "date" + } + } + if err := cfg.Validate(); err != nil { return nil, err } return cfg, nil } - -func setDefaultValues(k *koanf.Koanf) error { - return k.Load(confmap.Provider(map[string]any{ - "digest.compress": true, - "digest.view": "date", - "digest.host": "http://localhost:8080", - "digest.schedule": "@weekly", - "digest.mark_as_read": true, - "digest.run_on_startup": false, - }, "."), nil) -} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ed3c656..97179bc 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -21,9 +21,12 @@ func TestLoad(t *testing.T) { "host": "miniflux.example.com", "api_token": "test-token", }, - "digest": map[string]any{ + "digests": []map[string]any{{ + "title": "Daily Digest", "schedule": "@daily", - }, + "host": "http://localhost:8080", + "view": "category", + }}, "ai": map[string]any{ "api_key": "dummy-key", }, @@ -36,9 +39,44 @@ func TestLoad(t *testing.T) { "miniflux": map[string]any{ "api_token": "test-token", }, - "digest": map[string]any{ + "digests": []map[string]any{{ + "title": "Daily Digest", "schedule": "@daily", + }}, + }, + wantErr: true, + }, + { + name: "valid config missing view defaults to date", + config: map[string]any{ + "miniflux": map[string]any{ + "host": "miniflux.example.com", + "api_token": "test-token", }, + "digests": []map[string]any{{ + "title": "Daily Digest", + "schedule": "@daily", + "host": "http://localhost:8080", + }}, + }, + wantErr: false, + }, + { + name: "email configured but missing smtp host", + config: map[string]any{ + "miniflux": map[string]any{ + "host": "miniflux.example.com", + "api_token": "test-token", + }, + "digests": []map[string]any{{ + "title": "Daily Digest", + "schedule": "@daily", + "host": "http://localhost:8080", + "view": "category", + "email": map[string]any{ + "to": "test@example.com", + }, + }}, }, wantErr: true, }, @@ -48,9 +86,12 @@ func TestLoad(t *testing.T) { "miniflux": map[string]any{ "host": "miniflux.example.com", }, - "digest": map[string]any{ + "digests": []map[string]any{{ + "title": "Daily Digest", "schedule": "@daily", - }, + "host": "http://localhost:8080", + "view": "category", + }}, }, wantErr: true, }, @@ -61,9 +102,12 @@ func TestLoad(t *testing.T) { "host": "miniflux.example.com", "api_token": "test-token", }, - "digest": map[string]any{ + "digests": []map[string]any{{ + "title": "Daily Digest", "schedule": "@daily", - }, + "host": "http://localhost:8080", + "view": "category", + }}, "smtp": map[string]any{ "port": 65536, }, @@ -77,9 +121,12 @@ func TestLoad(t *testing.T) { "host": "miniflux.example.com", "api_token": "test-token", }, - "digest": map[string]any{ + "digests": []map[string]any{{ + "title": "Daily Digest", "schedule": "@daily", - }, + "host": "http://localhost:8080", + "view": "category", + }}, "smtp": map[string]any{ "port": 587, }, @@ -96,12 +143,15 @@ func TestLoad(t *testing.T) { "host": "miniflux.example.com", "api_token": "test-token", }, - "digest": map[string]any{ - "schedule": "@daily", + "digests": []map[string]any{{ + "title": "Daily Digest", + "schedule": "@daily", + "host": "http://localhost:8080", + "view": "category", "email": map[string]any{ "to": "invalid-email", }, - }, + }}, }, wantErr: true, }, @@ -112,12 +162,15 @@ func TestLoad(t *testing.T) { "host": "miniflux.example.com", "api_token": "test-token", }, - "digest": map[string]any{ - "schedule": "@daily", + "digests": []map[string]any{{ + "title": "Daily Digest", + "schedule": "@daily", + "host": "http://localhost:8080", + "view": "category", "email": map[string]any{ "from": "another-invalid-email", }, - }, + }}, }, wantErr: true, }, @@ -128,12 +181,18 @@ func TestLoad(t *testing.T) { "host": "miniflux.example.com", "api_token": "test-token", }, - "digest": map[string]any{ - "schedule": "@daily", + "digests": []map[string]any{{ + "title": "Daily Digest", + "schedule": "@daily", + "host": "http://localhost:8080", + "view": "category", "email": map[string]any{ "to": "test@example.com", "from": "sender@example.com", }, + }}, + "smtp": map[string]any{ + "host": "smtp.example.com", }, "ai": map[string]any{ "api_key": "dummy-key", @@ -148,9 +207,12 @@ func TestLoad(t *testing.T) { "host": "miniflux.example.com", "api_token": "test-token", }, - "digest": map[string]any{ + "digests": []map[string]any{{ + "title": "Daily Digest", "schedule": "* * 1 * *", - }, + "host": "http://localhost:8080", + "view": "category", + }}, "ai": map[string]any{ "api_key": "dummy-key", }, @@ -164,9 +226,12 @@ func TestLoad(t *testing.T) { "host": "miniflux.example.com", "api_token": "test-token", }, - "digest": map[string]any{ + "digests": []map[string]any{{ + "title": "Daily Digest", "schedule": "* * * * * * *", - }, + "host": "http://localhost:8080", + "view": "category", + }}, }, wantErr: true, }, @@ -177,9 +242,12 @@ func TestLoad(t *testing.T) { "host": "miniflux.example.com", "api_token": "test-token", }, - "digest": map[string]any{ + "digests": []map[string]any{{ + "title": "Daily Digest", "schedule": "@every 1h30m", - }, + "host": "http://localhost:8080", + "view": "category", + }}, "ai": map[string]any{ "api_key": "dummy-key", }, @@ -193,13 +261,16 @@ func TestLoad(t *testing.T) { "host": "miniflux.example.com", "api_token": "test-token", }, - "digest": map[string]any{ + "digests": []map[string]any{{ + "title": "Daily Digest", "schedule": "@every bad-duration", - }, + "host": "http://localhost:8080", + "view": "category", + }}, }, wantErr: true, }, - + { name: "invalid view value", config: map[string]any{ @@ -207,10 +278,12 @@ func TestLoad(t *testing.T) { "host": "miniflux.example.com", "api_token": "test-token", }, - "digest": map[string]any{ + "digests": []map[string]any{{ + "title": "Daily Digest", "schedule": "@daily", - "view": "magic", - }, + "host": "http://localhost:8080", + "view": "magic", + }}, }, wantErr: true, }, @@ -221,10 +294,123 @@ func TestLoad(t *testing.T) { "host": "miniflux.example.com", "api_token": "test-token", }, - "digest": map[string]any{ - "schedule": "@daily", - "view": "ai", + "digests": []map[string]any{{ + "title": "Daily Digest", + "schedule": "@daily", + "host": "http://localhost:8080", + "view": "ai", + }}, + }, + wantErr: true, + }, + { + name: "missing digests", + config: map[string]any{ + "miniflux": map[string]any{ + "host": "miniflux.example.com", + "api_token": "test-token", }, + "digests": []map[string]any{}, + }, + wantErr: true, + }, + { + name: "duplicate digest slugs", + config: map[string]any{ + "miniflux": map[string]any{ + "host": "miniflux.example.com", + "api_token": "test-token", + }, + "digests": []map[string]any{ + { + "title": "My Digest", + "schedule": "@daily", + "host": "http://localhost:8080", + "view": "category", + }, + { + "title": "My-Digest", + "schedule": "@daily", + "host": "http://localhost:8080", + "view": "category", + }, + }, + }, + wantErr: true, + }, + { + name: "invalid regex pattern", + config: map[string]any{ + "miniflux": map[string]any{ + "host": "miniflux.example.com", + "api_token": "test-token", + }, + "digests": []map[string]any{{ + "title": "Daily Digest", + "schedule": "@daily", + "host": "http://localhost:8080", + "view": "category", + "filters": map[string]any{ + "feed_title_patterns": []string{"(unclosed parenthesis"}, + }, + }}, + }, + wantErr: true, + }, + { + name: "invalid category missing title", + config: map[string]any{ + "miniflux": map[string]any{ + "host": "miniflux.example.com", + "api_token": "test-token", + }, + "digests": []map[string]any{{ + "title": "Daily Digest", + "schedule": "@daily", + "host": "http://localhost:8080", + "view": "ai", + "categories": []map[string]any{ + {"description": "No title here"}, + }, + }}, + "ai": map[string]any{ + "api_key": "dummy-key", + }, + }, + wantErr: true, + }, + { + name: "unknown filter field", + config: map[string]any{ + "miniflux": map[string]any{ + "host": "miniflux.example.com", + "api_token": "test-token", + }, + "digests": []map[string]any{{ + "title": "Daily Digest", + "schedule": "@daily", + "host": "http://localhost:8080", + "view": "category", + "filters": map[string]any{ + "feed_title": []string{"Some Feed"}, // Incorrect field name + }, + }}, + }, + wantErr: true, + }, + { + name: "invalid digest title empty slug", + config: map[string]any{ + "miniflux": map[string]any{ + "host": "miniflux.example.com", + "api_token": "test-token", + }, + "digests": []map[string]any{{ + "title": "!!!", + "schedule": "@daily", + "host": "http://localhost:8080", + "view": "category", + }}, }, wantErr: true, }, @@ -252,19 +438,12 @@ func TestLoad(t *testing.T) { t.Fatalf("Failed to write dummy config file: %v", err) } - cfg, err := Load(configPath) + _, err = Load(configPath) if (err != nil) != tt.wantErr { t.Errorf("Load() error = %v, wantErr %v", err, tt.wantErr) return } - - if !tt.wantErr && tt.name == "default digest.schedule is @weekly" { - if cfg.Digest.Schedule != "@weekly" { - t.Errorf("Expected digest.schedule to be @weekly, got %s", cfg.Digest.Schedule) - } - } }) } - -} +} \ No newline at end of file diff --git a/internal/digest/digest.go b/internal/digest/digest.go index 3ab30af..4f8656b 100644 --- a/internal/digest/digest.go +++ b/internal/digest/digest.go @@ -2,6 +2,7 @@ package digest import ( "miniflux-digest/internal/app/services" + "miniflux-digest/internal/config" "miniflux-digest/internal/digest/view" "miniflux-digest/internal/models" ) @@ -14,6 +15,6 @@ type digestServiceImpl struct { llmService services.LLMService } -func (s *digestServiceImpl) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, viewType string, minifluxHost string, digestHost string) *models.OverviewTemplateData { - return view.BuildDigestDataForView(entries, icons, viewType, minifluxHost, digestHost, s.llmService) +func (s *digestServiceImpl) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, viewType string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData { + return view.BuildDigestDataForView(entries, icons, viewType, minifluxHost, digestHost, s.llmService, categories, digestTitle) } \ No newline at end of file diff --git a/internal/digest/digest_test.go b/internal/digest/digest_test.go index 536e4d6..378540b 100644 --- a/internal/digest/digest_test.go +++ b/internal/digest/digest_test.go @@ -89,7 +89,7 @@ func TestDigestService_BuildDigestData(t *testing.T) { } t.Run("view=date", func(t *testing.T) { - overviewData := digestService.BuildDigestData(entries, icons, "date", "http://miniflux.test", "") + overviewData := digestService.BuildDigestData(entries, icons, "date", "http://miniflux.test", "", nil, "") if overviewData == nil { t.Fatal("overviewData is nil") } @@ -112,7 +112,7 @@ func TestDigestService_BuildDigestData(t *testing.T) { }) t.Run("view=category", func(t *testing.T) { - overviewData := digestService.BuildDigestData(entries, icons, "category", "http://miniflux.test", "") + overviewData := digestService.BuildDigestData(entries, icons, "category", "http://miniflux.test", "", nil, "") if overviewData == nil { t.Fatal("overviewData is nil") } @@ -133,7 +133,7 @@ func TestDigestService_BuildDigestData(t *testing.T) { t.Run("view=ai", func(t *testing.T) { aiEntries := createLLMGrouperMockEntries() - overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "") + overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "", nil, "") if overviewData == nil { t.Fatal("overviewData is nil") } @@ -156,7 +156,7 @@ func TestDigestService_BuildDigestData(t *testing.T) { } digestService := NewDigestService(mockLLM) aiEntries := createLLMGrouperMockEntries() - overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "") + overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "", nil, "") if overviewData == nil { t.Fatal("overviewData is nil") @@ -200,7 +200,7 @@ func TestDigestService_BuildDigestData(t *testing.T) { } digestService := NewDigestService(mockLLM) aiEntries := createLLMGrouperMockEntries() - overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "") + overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "", nil, "") if overviewData == nil { t.Fatal("overviewData is nil") diff --git a/internal/digest/view/ai.go b/internal/digest/view/ai.go index c63f72f..6388d19 100644 --- a/internal/digest/view/ai.go +++ b/internal/digest/view/ai.go @@ -7,6 +7,7 @@ import ( "fmt" "log" "miniflux-digest/internal/app/services" + "miniflux-digest/internal/config" "miniflux-digest/internal/llm" "miniflux-digest/internal/models" "miniflux-digest/internal/utils" @@ -33,7 +34,7 @@ type llmEntry struct { -func GroupAIEntries(ctx context.Context, entries []*models.Entry, llmService services.LLMService) (map[string][]*models.Entry, map[int64]bool, map[string][]*models.Entry) { +func GroupAIEntries(ctx context.Context, entries []*models.Entry, llmService services.LLMService, categories []config.ConfigCategory) (map[string][]*models.Entry, map[int64]bool, map[string][]*models.Entry) { const chunkSize = 200 // Prepare for processing @@ -42,7 +43,14 @@ func GroupAIEntries(ctx context.Context, entries []*models.Entry, llmService ser entryMap[entry.ID] = entry } - promptTemplate, err := template.New("grouping").Parse(initialGroupingPrompt) + var promptTemplateStr string + if len(categories) > 0 { + promptTemplateStr = strictGroupingPrompt + } else { + promptTemplateStr = openGroupingPrompt + } + + promptTemplate, err := template.New("grouping").Parse(promptTemplateStr) if err != nil { failedChunks := make(map[string][]*models.Entry) failedChunks["Prompt template parsing failed"] = entries @@ -51,8 +59,9 @@ func GroupAIEntries(ctx context.Context, entries []*models.Entry, llmService ser workerConfig := llmWorkerConfig{ promptTemplate: promptTemplate, + promptData: categories, responseSchema: InitialGroupingResponseSchema, - prepareEntries: func(entries []*models.Entry) interface{} { + prepareEntries: func(entries []*models.Entry) any { return prepareLLMEntries(entries) }, } @@ -171,7 +180,7 @@ func getSummaryForGroup(ctx context.Context, groupTitle string, entries []*model llmResponseWorker := createLLMWorker[SummaryResponse](ctx, llmService, llmWorkerConfig{ promptFormat: summaryPrompt, responseSchema: SummaryResponseSchema, - prepareEntries: func(entries []*models.Entry) interface{} { + prepareEntries: func(entries []*models.Entry) any { return prepareLLMEntries(entries) }, }, groupTitle) @@ -220,7 +229,7 @@ func getSubGroupsForGroup(ctx context.Context, groupTitle string, entries []*mod subGroupWorker := createLLMWorker[*SubGroupingResponse](ctx, llmService, llmWorkerConfig{ promptFormat: subGroupingPrompt, responseSchema: SubGroupingResponseSchema, - prepareEntries: func(entries []*models.Entry) interface{} { + prepareEntries: func(entries []*models.Entry) any { return prepareLLMEntries(entries) }, }, groupTitle) @@ -389,13 +398,20 @@ func prepareLLMEntries(entries []*models.Entry) []llmEntry { return llmEntries } -func BuildDigestDataByAI(entries []*models.Entry, ctx context.Context, llmService services.LLMService) []*models.PrimaryGroupDigestData { - rawGroups, groupedEntryIDs, failedChunks, fallbackGroups := initialAIGGrouping(ctx, entries, llmService) +func BuildDigestDataByAI(entries []*models.Entry, ctx context.Context, llmService services.LLMService, categories []config.ConfigCategory) []*models.PrimaryGroupDigestData { + rawGroups, groupedEntryIDs, failedChunks, fallbackGroups := initialAIGGrouping(ctx, entries, llmService, categories) if fallbackGroups != nil { return fallbackGroups } - highLevelGroups := consolidateGroups(ctx, rawGroups, llmService) + var highLevelGroups []*models.PrimaryGroupDigestData + if len(categories) == 0 { + // Auto Mode: Consolidate the LLM's open-ended groups + highLevelGroups = consolidateGroups(ctx, rawGroups, llmService) + } else { + // User Mode: Use raw groups directly (strict adherence) + highLevelGroups = convertRawGroupsToDigestData(rawGroups) + } highLevelGroups = handleFailedAndUngroupedEntries(highLevelGroups, entries, groupedEntryIDs, failedChunks) @@ -406,8 +422,8 @@ func BuildDigestDataByAI(entries []*models.Entry, ctx context.Context, llmServic return allPrimaryGroups } -func initialAIGGrouping(ctx context.Context, entries []*models.Entry, llmService services.LLMService) (map[string][]*models.Entry, map[int64]bool, map[string][]*models.Entry, []*models.PrimaryGroupDigestData) { - rawGroups, groupedEntryIDs, failedChunks := GroupAIEntries(ctx, entries, llmService) +func initialAIGGrouping(ctx context.Context, entries []*models.Entry, llmService services.LLMService, categories []config.ConfigCategory) (map[string][]*models.Entry, map[int64]bool, map[string][]*models.Entry, []*models.PrimaryGroupDigestData) { + rawGroups, groupedEntryIDs, failedChunks := GroupAIEntries(ctx, entries, llmService, categories) // If all chunks failed and we have no raw groups, fall back to category grouping. if len(rawGroups) == 0 && len(failedChunks) > 0 { @@ -466,23 +482,60 @@ func handleFailedAndUngroupedEntries(highLevelGroups []*models.PrimaryGroupDiges } if len(otherUngroupedEntries) > 0 { - highLevelGroups = append(highLevelGroups, &models.PrimaryGroupDigestData{ - ID: int64(len(highLevelGroups) + 1), - Title: "Uncategorized", - Slug: "uncategorized", - SubGroups: []*models.EntryGroup{ - { - Title: "Uncategorized", - Entries: otherUngroupedEntries, - Slug: "uncategorized", - TotalEntries: len(otherUngroupedEntries), - TotalFeeds: getUniqueFeedIDs(otherUngroupedEntries), + var merged bool + for _, group := range highLevelGroups { + if group.Title == "Uncategorized" { + merged = true + // Find or create the "Uncategorized" subgroup + var subGroup *models.EntryGroup + for _, sg := range group.SubGroups { + if sg.Title == "Uncategorized" { + subGroup = sg + break + } + } + if subGroup == nil { + subGroup = &models.EntryGroup{ + Title: "Uncategorized", + Slug: "uncategorized", + } + group.SubGroups = append(group.SubGroups, subGroup) + } + + subGroup.Entries = append(subGroup.Entries, otherUngroupedEntries...) + subGroup.TotalEntries = len(subGroup.Entries) + subGroup.TotalFeeds = getUniqueFeedIDs(subGroup.Entries) + + // Update primary group totals + var allEntries []*models.Entry + for _, sg := range group.SubGroups { + allEntries = append(allEntries, sg.Entries...) + } + group.TotalEntries = len(allEntries) + group.TotalFeeds = getUniqueFeedIDs(allEntries) + break + } + } + + if !merged { + highLevelGroups = append(highLevelGroups, &models.PrimaryGroupDigestData{ + ID: int64(len(highLevelGroups) + 1), + Title: "Uncategorized", + Slug: "uncategorized", + SubGroups: []*models.EntryGroup{ + { + Title: "Uncategorized", + Entries: otherUngroupedEntries, + Slug: "uncategorized", + TotalEntries: len(otherUngroupedEntries), + TotalFeeds: getUniqueFeedIDs(otherUngroupedEntries), + }, }, - }, - Summary: "", - TotalEntries: len(otherUngroupedEntries), - TotalFeeds: getUniqueFeedIDs(otherUngroupedEntries), - }) + Summary: "", + TotalEntries: len(otherUngroupedEntries), + TotalFeeds: getUniqueFeedIDs(otherUngroupedEntries), + }) + } } return highLevelGroups } @@ -544,9 +597,10 @@ func sortPrimaryGroups(allPrimaryGroups []*models.PrimaryGroupDigestData) { // llmWorkerConfig holds configuration for creating an LLM worker function. type llmWorkerConfig struct { promptTemplate *template.Template + promptData any // Data to pass to template execution promptFormat string // For prompts that use Sprintf responseSchema *genai.Schema - prepareEntries func([]*models.Entry) interface{} // Function to prepare entries for LLM + prepareEntries func([]*models.Entry) any // Function to prepare entries for LLM } // createLLMWorker creates a worker function for llm.ProcessInChunks. @@ -573,7 +627,7 @@ func createLLMWorker[R any]( // 2. Construct the prompt var prompt bytes.Buffer if config.promptTemplate != nil { - if err := config.promptTemplate.Execute(&prompt, nil); err != nil { + if err := config.promptTemplate.Execute(&prompt, config.promptData); err != nil { return zero, fmt.Errorf("failed to execute prompt template: %w", err) } } else if config.promptFormat != "" { @@ -591,4 +645,4 @@ func createLLMWorker[R any]( } return response, nil } -} \ No newline at end of file +} diff --git a/internal/digest/view/ai_prompts.go b/internal/digest/view/ai_prompts.go index 46a6495..ff12725 100644 --- a/internal/digest/view/ai_prompts.go +++ b/internal/digest/view/ai_prompts.go @@ -2,7 +2,7 @@ package view import "google.golang.org/genai" -const initialGroupingPrompt = `You are creating high-level primary groups for a digest. +const openGroupingPrompt = `You are creating high-level primary groups for a digest. **Your Task:** Group the entries below into broad, high-level primary groups. @@ -20,6 +20,24 @@ Below is the list of entries: ---------------------------- ` +const strictGroupingPrompt = `You are organizing a digest into specific categories. + +**Your Task:** +Sort the entries below into the following categories: +{{range .}} +- **{{.Title}}**: {{.Description}} +{{end}} + +**Rules:** +- You MUST use ONLY the categories listed above. +- Do NOT create new categories. +- If an entry does not fit well into any category, do NOT include it in a group (it will be handled as uncategorized later). +- An entry can only belong to one group. + +Below is the list of entries: +---------------------------- +` + const consolidationPrompt = `You are an expert content curator responsible for organizing a digest. You have been given a list of proposed primary group titles that were automatically generated. Your task is to ensure that this list is broad and high level and does not contain potentially overlapping topics. Each topic will later on be further subdivided into sub-groups, so if two topics are better served as sub-topics, they should be merged into one high level topic. Our goal is to have no more than 10-12 primary groups. **Your Goal:** diff --git a/internal/digest/view/ai_test.go b/internal/digest/view/ai_test.go index ca0711b..487fac3 100644 --- a/internal/digest/view/ai_test.go +++ b/internal/digest/view/ai_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "miniflux-digest/internal/config" "miniflux-digest/internal/models" "miniflux-digest/internal/testutil" "strings" @@ -72,7 +73,7 @@ func TestGroupAIEntries(t *testing.T) { }, } - rawGroups, groupedIDs, failedChunks := GroupAIEntries(context.Background(), entries, llmService) + rawGroups, groupedIDs, failedChunks := GroupAIEntries(context.Background(), entries, llmService, nil) assert.Empty(t, failedChunks) assert.NotNil(t, rawGroups) assert.Len(t, rawGroups, 2, "Expected 2 grouped") @@ -96,7 +97,7 @@ func TestGroupAIEntries(t *testing.T) { }, } - _, _, failedChunks := GroupAIEntries(context.Background(), entries, llmService) + _, _, failedChunks := GroupAIEntries(context.Background(), entries, llmService, nil) assert.NotEmpty(t, failedChunks) }) @@ -107,7 +108,7 @@ func TestGroupAIEntries(t *testing.T) { }, } - _, _, failedChunks := GroupAIEntries(context.Background(), entries, llmService) + _, _, failedChunks := GroupAIEntries(context.Background(), entries, llmService, nil) assert.NotEmpty(t, failedChunks) }) @@ -149,7 +150,7 @@ func TestGroupAIEntries(t *testing.T) { }, } - rawGroups, groupedIDs, failedChunks := GroupAIEntries(context.Background(), largeEntries, llmService) + rawGroups, groupedIDs, failedChunks := GroupAIEntries(context.Background(), largeEntries, llmService, nil) assert.Empty(t, failedChunks) assert.Equal(t, 2, callCount, "Expected LLM to be called 2 times for 205 entries with chunk size 200") @@ -450,7 +451,7 @@ func TestBuildDigestDataByAI(t *testing.T) { }, } - groups := BuildDigestDataByAI(entries, context.Background(), llmService) + groups := BuildDigestDataByAI(entries, context.Background(), llmService, nil) assert.Len(t, groups, 2, "Expected 1 consolidated group + 1 uncategorized") techGroup := testutil.FindPrimaryGroup(groups, "Tech") @@ -466,6 +467,128 @@ func TestBuildDigestDataByAI(t *testing.T) { assert.Equal(t, int64(4), uncategorized.SubGroups[0].Entries[0].ID) }) + t.Run("strict user mode - no consolidation", func(t *testing.T) { + userCategories := []config.ConfigCategory{ + {Title: "Strict Cat A"}, + {Title: "Strict Cat B"}, + } + + // Entries 1 & 2 assigned to user categories + groupingResp := InitialGroupingResponse{ + Groups: []struct { + Title string `json:"title"` + EntryIDs []int64 `json:"entry_ids"` + }{ + {Title: "Strict Cat A", EntryIDs: []int64{1}}, + {Title: "Strict Cat B", EntryIDs: []int64{2}}, + }, + } + groupingJSON, _ := json.Marshal(groupingResp) + + llmService := &mockLLMService{ + GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + if schema == ConsolidationResponseSchema { + t.Fatal("Consolidation should NOT be called in strict user mode") + } + if schema == InitialGroupingResponseSchema { + // Verify that strict prompt is used (implicitly via context of strict mode logic, + // but strict prompt text check would require inspecting 'prompt' arg which we can do) + assert.Contains(t, prompt, "Sort the entries below into the following categories") + return groupingJSON, nil + } + + if schema == SummaryResponseSchema { + return json.Marshal(SummaryResponse{Summary: "Summary"}) + } + if schema == SubGroupingResponseSchema { + // Return dummy subgroup + return json.Marshal(SubGroupingResponse{SubGroups: []struct { + Title string `json:"title"` + EntryIDs []int64 `json:"entry_ids"` + }{}}) + } + return nil, nil + }, + } + + groups := BuildDigestDataByAI(entries, context.Background(), llmService, userCategories) + + // Expected: Strict Cat A, Strict Cat B, Uncategorized (for 3 & 4) + assert.Len(t, groups, 3) + + catA := testutil.FindPrimaryGroup(groups, "Strict Cat A") + assert.NotNil(t, catA) + + catB := testutil.FindPrimaryGroup(groups, "Strict Cat B") + assert.NotNil(t, catB) + + uncat := testutil.FindPrimaryGroup(groups, "Uncategorized") + assert.NotNil(t, uncat) + assert.Equal(t, 2, uncat.TotalEntries) // Entries 3 and 4 + }) + + t.Run("merges ungrouped into existing uncategorized", func(t *testing.T) { + // Entry 1 & 2 -> "Uncategorized" via LLM + // Entry 3 -> Ungrouped (should be merged) + entries := []*models.Entry{ + {ID: 1, Title: "E1"}, + {ID: 2, Title: "E2"}, + {ID: 3, Title: "E3"}, + } + + groupingResp := InitialGroupingResponse{ + Groups: []struct { + Title string `json:"title"` + EntryIDs []int64 `json:"entry_ids"` + }{ + {Title: "Uncategorized", EntryIDs: []int64{1, 2}}, + }, + } + groupingJSON, _ := json.Marshal(groupingResp) + + // Mock Summary/Sub-grouping for the "Uncategorized" group + summaryResp := SummaryResponse{Summary: "Uncategorized stuff"} + summaryJSON, _ := json.Marshal(summaryResp) + subGroupingResp := SubGroupingResponse{ + SubGroups: []struct { + Title string `json:"title"` + EntryIDs []int64 `json:"entry_ids"` + }{{Title: "Uncategorized", EntryIDs: []int64{1, 2}}}, + } + subGroupingJSON, _ := json.Marshal(subGroupingResp) + + llmService := &mockLLMService{ + GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + switch schema { + case InitialGroupingResponseSchema: + return groupingJSON, nil + case ConsolidationResponseSchema: + // Should not be called if < 10 groups, but handle just in case + return nil, nil + case SummaryResponseSchema: + return summaryJSON, nil + case SubGroupingResponseSchema: + return subGroupingJSON, nil + default: + return nil, errors.New("unexpected schema") + } + }, + } + + groups := BuildDigestDataByAI(entries, context.Background(), llmService, nil) + + require.Len(t, groups, 1, "Expected exactly 1 group (merged Uncategorized)") + uncatGroup := groups[0] + assert.Equal(t, "Uncategorized", uncatGroup.Title) + + // Check that it contains all 3 entries (2 from LLM + 1 ungrouped) + var totalEntries int + for _, sg := range uncatGroup.SubGroups { + totalEntries += len(sg.Entries) + } + assert.Equal(t, 3, totalEntries, "Should contain all 3 entries") + }) + t.Run("fallback to category grouping", func(t *testing.T) { llmService := &mockLLMService{ GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { @@ -473,7 +596,7 @@ func TestBuildDigestDataByAI(t *testing.T) { }, } - groups := BuildDigestDataByAI(entries, context.Background(), llmService) + groups := BuildDigestDataByAI(entries, context.Background(), llmService, nil) assert.Len(t, groups, 3, "Should fall back to 3 categories") }) diff --git a/internal/digest/view/view.go b/internal/digest/view/view.go index 47f3171..1dc7826 100644 --- a/internal/digest/view/view.go +++ b/internal/digest/view/view.go @@ -3,12 +3,14 @@ package view import ( "context" "miniflux-digest/internal/app/services" + "miniflux-digest/internal/config" "miniflux-digest/internal/models" + "miniflux-digest/internal/utils" "sort" "time" ) -func BuildDigestDataForView(entries []*models.Entry, icons map[int64]*models.FeedIcon, viewType string, minifluxHost string, digestHost string, llmService services.LLMService) *models.OverviewTemplateData { +func BuildDigestDataForView(entries []*models.Entry, icons map[int64]*models.FeedIcon, viewType string, minifluxHost string, digestHost string, llmService services.LLMService, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData { iconsSlice := make([]*models.FeedIcon, 0, len(icons)) for _, icon := range icons { iconsSlice = append(iconsSlice, icon) @@ -23,7 +25,7 @@ func BuildDigestDataForView(entries []*models.Entry, icons map[int64]*models.Fee case "category", "default": resultPrimaryGroups = BuildDigestDataByCategory(entries) case "ai": - resultPrimaryGroups = BuildDigestDataByAI(entries, context.Background(), llmService) + resultPrimaryGroups = BuildDigestDataByAI(entries, context.Background(), llmService, categories) } if viewType == "date" || viewType == "category" { @@ -60,6 +62,8 @@ func BuildDigestDataForView(entries []*models.Entry, icons map[int64]*models.Fee } return &models.OverviewTemplateData{ + DigestTitle: digestTitle, + DigestSlug: utils.Slugify(digestTitle), Entries: entries, GeneratedDate: time.Now(), FeedIcons: iconsSlice, diff --git a/internal/email/email.go b/internal/email/email.go index ae9aa62..d0e59c1 100644 --- a/internal/email/email.go +++ b/internal/email/email.go @@ -17,29 +17,29 @@ type EmailServiceImpl struct{ EmailTemplate *template.Template } -func (s *EmailServiceImpl) Send(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error { +func (s *EmailServiceImpl) Send(smtpConfig config.ConfigSmtp, digestConfig config.ConfigDigest, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error { message := mail.NewMsg() client, err := mail.NewClient( - cfg.Smtp.Host, + smtpConfig.Host, mail.WithSMTPAuth(mail.SMTPAuthAutoDiscover), - mail.WithPort(cfg.Smtp.Port), - mail.WithUsername(cfg.Smtp.User), - mail.WithPassword(cfg.Smtp.Password)) + mail.WithPort(smtpConfig.Port), + mail.WithUsername(smtpConfig.User), + mail.WithPassword(smtpConfig.Password)) if err != nil { return err } - if err := message.From(cfg.Digest.Email.From); err != nil { + if err := message.From(digestConfig.Email.From); err != nil { return err } - if err := message.To(cfg.Digest.Email.To); err != nil { + if err := message.To(digestConfig.Email.To); err != nil { return err } subject := fmt.Sprintf("Your Miniflux Digest for %s", data.GeneratedDate.Format("January 2, 2006")) - overviewURL := fmt.Sprintf("%s/archive/%s/index.html", cfg.Digest.Host, data.GeneratedDate.Format("2006-01-02")) + overviewURL := fmt.Sprintf("%s/archive/%s/index.html", digestConfig.Host, data.GeneratedDate.Format("2006-01-02")) message.Subject(subject) @@ -60,4 +60,4 @@ func (s *EmailServiceImpl) Send(cfg *config.Config, overviewFile *os.File, group } return client.DialAndSend(message) -} +} \ No newline at end of file diff --git a/internal/email/email_test.go b/internal/email/email_test.go index b6a5061..d27d37b 100644 --- a/internal/email/email_test.go +++ b/internal/email/email_test.go @@ -11,20 +11,18 @@ import ( ) func TestSend(t *testing.T) { - cfg := &config.Config{ - Smtp: config.ConfigSmtp{ - Host: "localhost", - Port: 1025, - User: "test-user", - Password: "test-password", - }, - Digest: config.ConfigDigest{ - Email: config.ConfigDigestEmail{ - To: "to@example.com", - From: "from@example.com", - }, - Host: "https://example.com", + smtpConfig := config.ConfigSmtp{ + Host: "localhost", + Port: 1025, + User: "test-user", + Password: "test-password", + } + digestConfig := config.ConfigDigest{ + Email: config.ConfigDigestEmail{ + To: "to@example.com", + From: "from@example.com", }, + Host: "https://example.com", } tmpFile, err := os.CreateTemp("", "test.html") @@ -70,7 +68,7 @@ func TestSend(t *testing.T) { emailService := &EmailServiceImpl{ EmailTemplate: templates.EmailTemplate, } - err = emailService.Send(cfg, file, []*os.File{}, &data) + err = emailService.Send(smtpConfig, digestConfig, file, []*os.File{}, &data) if err != nil { // We expect an error because we are not running a real SMTP server. // The important part is that the function attempts to connect. @@ -95,4 +93,4 @@ func TestTextTemplateData(t *testing.T) { if textData.URL != url { t.Errorf("Expected URL to be %s, got %s", url, textData.URL) } -} +} \ No newline at end of file diff --git a/internal/filter/filter.go b/internal/filter/filter.go new file mode 100644 index 0000000..37e59f5 --- /dev/null +++ b/internal/filter/filter.go @@ -0,0 +1,130 @@ +package filter + +import ( + "regexp" + "strings" + + "miniflux-digest/internal/config" + "miniflux-digest/internal/models" +) + +type Matcher struct { + // Cache regex patterns to avoid recompiling + feedTitleRegex []*regexp.Regexp + categoryTitleRegex []*regexp.Regexp + siteURLRegex []*regexp.Regexp + entryURLRegex []*regexp.Regexp +} + +func NewMatcher(cfg config.ConfigFilters) (*Matcher, error) { + m := &Matcher{} + var err error + + m.feedTitleRegex, err = compileRegexes(cfg.FeedTitlePatterns) + if err != nil { + return nil, err + } + + m.categoryTitleRegex, err = compileRegexes(cfg.CategoryTitlePatterns) + if err != nil { + return nil, err + } + + m.siteURLRegex, err = compileRegexes(cfg.FeedURLPatterns) + if err != nil { + return nil, err + } + + m.entryURLRegex, err = compileRegexes(cfg.EntryURLPatterns) + if err != nil { + return nil, err + } + + return m, nil +} + +func compileRegexes(patterns []string) ([]*regexp.Regexp, error) { + regexes := make([]*regexp.Regexp, 0, len(patterns)) + for _, p := range patterns { + r, err := regexp.Compile(p) + if err != nil { + return nil, err + } + regexes = append(regexes, r) + } + return regexes, nil +} + +// Matches returns true if the entry matches the filter criteria. +// Logic: +// 1. If no filters are defined, it's a match (default allow). +// 2. OR logic within a specific field (e.g., FeedTitles: ["A", "B"] -> match if A OR B). +// 3. AND logic between fields (e.g., FeedTitle: "A" AND SiteURL: "B" -> match only if both are true). +func (m *Matcher) Matches(entry *models.Entry, cfg config.ConfigFilters) bool { + // If all filters are empty, return true (default allow if "filters" block is missing/empty) + // However, usually we want at least one filter. But per spec "Stack-Based Priority", + // a digest might be a "catch-all" at the end. + if isEmpty(cfg) { + return true + } + + if !matchString(entry.FeedTitle, cfg.FeedTitles, m.feedTitleRegex, true) { + return false + } + + if !matchString(entry.GroupTitle, cfg.CategoryTitles, m.categoryTitleRegex, true) { + return false + } + + if !matchString(entry.SiteURL, cfg.FeedURLs, m.siteURLRegex, false) { + return false + } + + // EntryURL corresponds to entry.URL + if !matchString(entry.URL, cfg.EntryURLs, m.entryURLRegex, false) { + return false + } + + return true +} + +// matchString checks if value matches any of the exact strings or regex patterns. +// exactMatch: true for Full Match (Titles), false for Prefix Match (URLs). +func matchString(value string, exacts []string, patterns []*regexp.Regexp, exactMatch bool) bool { + // If no filters for this specific field are set, we "pass" this field. + // We only check for a match if there IS something to match against. + if len(exacts) == 0 && len(patterns) == 0 { + return true + } + + for _, e := range exacts { + if exactMatch { + if value == e { + return true + } + } else { + if strings.HasPrefix(value, e) { + return true + } + } + } + + for _, r := range patterns { + if r.MatchString(value) { + return true + } + } + + return false +} + +func isEmpty(cfg config.ConfigFilters) bool { + return len(cfg.FeedTitles) == 0 && + len(cfg.CategoryTitles) == 0 && + len(cfg.FeedURLs) == 0 && + len(cfg.EntryURLs) == 0 && + len(cfg.FeedTitlePatterns) == 0 && + len(cfg.CategoryTitlePatterns) == 0 && + len(cfg.FeedURLPatterns) == 0 && + len(cfg.EntryURLPatterns) == 0 +} diff --git a/internal/filter/filter_test.go b/internal/filter/filter_test.go new file mode 100644 index 0000000..f4c737c --- /dev/null +++ b/internal/filter/filter_test.go @@ -0,0 +1,118 @@ +package filter + +import ( + "testing" + + "miniflux-digest/internal/config" + "miniflux-digest/internal/models" +) + +func TestMatcher_Matches(t *testing.T) { + tests := []struct { + name string + entry *models.Entry + filters config.ConfigFilters + wantMatch bool + }{ + { + name: "Match exact feed title", + entry: &models.Entry{ + FeedTitle: "Tech News", GroupTitle: "Some Category", SiteURL: "http://example.com", + URL: "http://example.com/post/1", + }, + filters: config.ConfigFilters{ + FeedTitles: []string{"Tech News"}, + }, + wantMatch: true, + }, + { + name: "No match exact feed title", + entry: &models.Entry{ + FeedTitle: "Cooking", GroupTitle: "Some Category", SiteURL: "http://example.com", + URL: "http://example.com/post/1", + }, + filters: config.ConfigFilters{ + FeedTitles: []string{"Tech News"}, + }, + wantMatch: false, + }, + { + name: "Match prefix site url", + entry: &models.Entry{ + FeedTitle: "A", GroupTitle: "Some Category", SiteURL: "https://github.com/user/repo", + URL: "http://example.com/post/1", + }, + filters: config.ConfigFilters{ + FeedURLs: []string{"https://github.com"}, + }, + wantMatch: true, + }, + { + name: "No match prefix site url", + entry: &models.Entry{ + FeedTitle: "A", GroupTitle: "Some Category", SiteURL: "https://google.com", + URL: "http://example.com/post/1", + }, + filters: config.ConfigFilters{ + FeedURLs: []string{"https://github.com"}, + }, + wantMatch: false, + }, + { + name: "Match regex feed title", + entry: &models.Entry{ + FeedTitle: "My Tech Blog", GroupTitle: "Some Category", SiteURL: "http://example.com", + URL: "http://example.com/post/1", + }, + filters: config.ConfigFilters{ + FeedTitlePatterns: []string{".*Tech.*"}, + }, + wantMatch: true, + }, + { + name: "Mixed AND logic (Feed Title AND Category)", + entry: &models.Entry{ + FeedTitle: "Tech News", GroupTitle: "Programming", SiteURL: "http://example.com", + URL: "http://example.com/post/1", + }, + filters: config.ConfigFilters{ + FeedTitles: []string{"Tech News"}, + CategoryTitles: []string{"Programming"}, + }, + wantMatch: true, + }, + { + name: "Mixed AND logic fail (Feed Title match, Category mismatch)", + entry: &models.Entry{ + FeedTitle: "Tech News", GroupTitle: "Cooking", SiteURL: "http://example.com", + URL: "http://example.com/post/1", + }, + filters: config.ConfigFilters{ + FeedTitles: []string{"Tech News"}, + CategoryTitles: []string{"Programming"}, + }, + wantMatch: false, + }, + { + name: "Empty filters matches all", + entry: &models.Entry{ + FeedTitle: "Any", GroupTitle: "Any", SiteURL: "Any", + URL: "Any", + }, + filters: config.ConfigFilters{}, + wantMatch: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m, err := NewMatcher(tt.filters) + if err != nil { + t.Fatalf("Failed to create matcher: %v", err) + } + if got := m.Matches(tt.entry, tt.filters); got != tt.wantMatch { + t.Errorf("Matches() = %v, want %v", got, tt.wantMatch) + } + }) + } +} diff --git a/internal/manager/manager.go b/internal/manager/manager.go new file mode 100644 index 0000000..9be1288 --- /dev/null +++ b/internal/manager/manager.go @@ -0,0 +1,39 @@ +package manager + +import ( + "miniflux-digest/internal/config" + "miniflux-digest/internal/filter" + "miniflux-digest/internal/models" +) + +type DigestManager struct { + digests []config.ConfigDigest + matchers []*filter.Matcher +} + +func NewDigestManager(digests []config.ConfigDigest) (*DigestManager, error) { + matchers := make([]*filter.Matcher, len(digests)) + for i, d := range digests { + m, err := filter.NewMatcher(d.Filters) + if err != nil { + return nil, err + } + matchers[i] = m + } + + return &DigestManager{ + digests: digests, + matchers: matchers, + }, nil +} + +// GetOwningDigest returns the index of the first digest that matches the entry. +// Returns -1 if no digest matches. +func (dm *DigestManager) GetOwningDigest(entry *models.Entry) int { + for i, m := range dm.matchers { + if m.Matches(entry, dm.digests[i].Filters) { + return i + } + } + return -1 +} diff --git a/internal/manager/manager_test.go b/internal/manager/manager_test.go new file mode 100644 index 0000000..139ef6f --- /dev/null +++ b/internal/manager/manager_test.go @@ -0,0 +1,84 @@ +package manager + +import ( + "testing" + + "miniflux-digest/internal/config" + "miniflux-digest/internal/models" +) + +func TestGetOwningDigest(t *testing.T) { + digests := []config.ConfigDigest{ + { + Title: "High Priority Tech", + Filters: config.ConfigFilters{ + FeedTitles: []string{"Tech News"}, + }, + }, + { + Title: "General News", + Filters: config.ConfigFilters{ + // Empty filters match everything (catch-all) + }, + }, + } + + dm, err := NewDigestManager(digests) + if err != nil { + t.Fatalf("Failed to create DigestManager: %v", err) + } + + tests := []struct { + name string + entry *models.Entry + wantIndex int + }{ + { + name: "Match first digest", + entry: &models.Entry{ + FeedTitle: "Tech News", + SiteURL: "http://technews.com", + }, + wantIndex: 0, + }, + { + name: "Match second digest (catch-all)", + entry: &models.Entry{ + FeedTitle: "Cooking Weekly", + SiteURL: "http://cooking.com", + }, + wantIndex: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := dm.GetOwningDigest(tt.entry) + if got != tt.wantIndex { + t.Errorf("GetOwningDigest() = %v, want %v", got, tt.wantIndex) + } + }) + } +} + +func TestGetOwningDigest_NoMatch(t *testing.T) { + digests := []config.ConfigDigest{ + { + Title: "Exclusive Club", + Filters: config.ConfigFilters{ + FeedTitles: []string{"VIP"}, + }, + }, + } + + dm, err := NewDigestManager(digests) + if err != nil { + t.Fatalf("Failed to create DigestManager: %v", err) + } + + entry := &models.Entry{FeedTitle: "Plebeian News"} + got := dm.GetOwningDigest(entry) + if got != -1 { + t.Errorf("GetOwningDigest() = %v, want -1", got) + } +} diff --git a/internal/models/models.go b/internal/models/models.go index 64e1dc7..cdf4fd4 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -14,6 +14,7 @@ type Entry struct { ID int64 Title string URL string + SiteURL string Content string FeedID int64 FeedTitle string @@ -41,6 +42,8 @@ type FeedIcon struct { } type OverviewTemplateData struct { + DigestTitle string + DigestSlug string Entries []*Entry GeneratedDate time.Time FeedIcons []*FeedIcon diff --git a/internal/processor/processor.go b/internal/processor/processor.go index da7d266..48d7dd8 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -5,20 +5,36 @@ import ( "os" "miniflux-digest/internal/app" + "miniflux-digest/internal/manager" "miniflux-digest/internal/models" miniflux "miniflux.app/v2/client" ) -func ProcessDigest(application *app.App) (*os.File, []*os.File, *models.OverviewTemplateData, error) { +func ProcessDigest(application *app.App, digestIndex int) (*os.File, []*os.File, *models.OverviewTemplateData, error) { + digestConfig := application.Config.Digests[digestIndex] + entries, err := application.MinifluxClientService.GetAllUnreadEntries() if err != nil { log.Printf("Error getting all unread entries: %v", err) return nil, nil, nil, err } - icons := make(map[int64]*models.FeedIcon) + digestManager, err := manager.NewDigestManager(application.Config.Digests) + if err != nil { + log.Printf("Error creating digest manager: %v", err) + return nil, nil, nil, err + } + + var digestEntries []*models.Entry for _, entry := range entries { + if digestManager.GetOwningDigest(entry) == digestIndex { + digestEntries = append(digestEntries, entry) + } + } + + icons := make(map[int64]*models.FeedIcon) + for _, entry := range digestEntries { if _, ok := icons[entry.FeedID]; !ok { icon, err := application.MinifluxClientService.FeedIcon(entry.FeedID) if err != nil { @@ -33,22 +49,24 @@ func ProcessDigest(application *app.App) (*os.File, []*os.File, *models.Overview } data := application.DigestService.BuildDigestData( - entries, + digestEntries, icons, - application.Config.Digest.View, + digestConfig.View, application.Config.Miniflux.Host, - application.Config.Digest.Host, + digestConfig.Host, + digestConfig.Categories, + digestConfig.Title, ) - overviewFile, groupedEntryFiles, err := application.ArchiveService.MakeArchiveHTML(data, application.Config.Digest.Compress) + overviewFile, groupedEntryFiles, err := application.ArchiveService.MakeArchiveHTML(data, *digestConfig.Compress) if err != nil { - log.Printf("Error generating archive HTML: %v", err) + log.Printf("Error generating HTML: %v", err) return nil, nil, nil, err } - if application.Config.Digest.MarkAsRead { + if digestConfig.MarkAsRead { var entryIDs []int64 - for _, entry := range entries { + for _, entry := range digestEntries { entryIDs = append(entryIDs, entry.ID) } diff --git a/internal/processor/processor_test.go b/internal/processor/processor_test.go index 3c6399b..afd7b16 100644 --- a/internal/processor/processor_test.go +++ b/internal/processor/processor_test.go @@ -30,11 +30,11 @@ func (m *mockMinifluxClientService) UpdateEntries(entryIDs []int64, status strin } type mockDigestService struct { - buildDigestDataFunc func(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string) *models.OverviewTemplateData + buildDigestDataFunc func(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData } -func (m *mockDigestService) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string) *models.OverviewTemplateData { - return m.buildDigestDataFunc(entries, icons, view, minifluxHost, digestHost) +func (m *mockDigestService) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData { + return m.buildDigestDataFunc(entries, icons, view, minifluxHost, digestHost, categories, digestTitle) } type mockArchiveService struct { @@ -78,7 +78,7 @@ func TestProcessDigest_Success(t *testing.T) { }, } mockDigestService := &mockDigestService{ - buildDigestDataFunc: func(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string) *models.OverviewTemplateData { + buildDigestDataFunc: func(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData { return expectedOverviewData }, } @@ -92,10 +92,12 @@ func TestProcessDigest_Success(t *testing.T) { mockApp := app.NewApp( app.WithConfig(&config.Config{ - Digest: config.ConfigDigest{ - MarkAsRead: true, - Compress: true, - View: "category", + Digests: []config.ConfigDigest{ + { + MarkAsRead: true, + Compress: func() *bool { b := true; return &b }(), + View: "category", + }, }, Miniflux: config.ConfigMiniflux{Host: "http://miniflux.test"}, }), @@ -104,7 +106,7 @@ func TestProcessDigest_Success(t *testing.T) { app.WithArchiveService(mockArchiveService), ) - overviewFile, groupedEntryFiles, data, err := ProcessDigest(mockApp) + overviewFile, groupedEntryFiles, data, err := ProcessDigest(mockApp, 0) if err != nil { t.Fatalf("ProcessDigest returned an unexpected error: %v", err) diff --git a/internal/testutil/mock_data.go b/internal/testutil/mock_data.go index 44ed10a..cd09b3c 100644 --- a/internal/testutil/mock_data.go +++ b/internal/testutil/mock_data.go @@ -81,24 +81,28 @@ type feedTemplate struct { GroupTitle string } +func createHardcodedMockEntries() []*models.Entry { + return []*models.Entry{ + // Category A + {ID: 1, Title: "Entry 1", Date: time.Date(2024, 1, 2, 10, 0, 0, 0, time.UTC), FeedID: 100, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A", SiteURL: "www.example.com", URL: "www.example.com/1"}, + {ID: 3, Title: "Entry 3", Date: time.Date(2024, 1, 2, 11, 0, 0, 0, time.UTC), FeedID: 100, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A", SiteURL: "www.example2.com", URL: "www.example2.com/1"}, + {ID: 5, Title: "Entry 5", Date: time.Date(2024, 1, 2, 12, 0, 0, 0, time.UTC), FeedID: 300, FeedTitle: "Feed C", GroupID: 1, GroupTitle: "Category A", SiteURL: "www.example3.com", URL: "www.example3.com/1"}, + {ID: 6, Title: "Entry 6", Date: time.Date(2024, 1, 2, 13, 0, 0, 0, time.UTC), FeedID: 400, FeedTitle: "Feed D", GroupID: 1, GroupTitle: "Category A", SiteURL: "www.example4.com", URL: "www.example4.com/1"}, + + // Category B + {ID: 2, Title: "Entry 2", Date: time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC), FeedID: 200, FeedTitle: "Feed B", GroupID: 2, GroupTitle: "Category B", SiteURL: "www.example5.com", URL: "www.example.com/1"}, + {ID: 4, Title: "Entry 4", Content: "Content of entry 4", Date: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC), FeedID: 200, FeedTitle: "Feed B", GroupID: 2, GroupTitle: "Category B", SiteURL: "www.example5.com", URL: "www.example5.com/2"}, + {ID: 7, Title: "Entry 7", Date: time.Date(2024, 1, 1, 14, 0, 0, 0, time.UTC), FeedID: 500, FeedTitle: "Feed E", GroupID: 2, GroupTitle: "Category B", SiteURL: "www.example5.com", URL: "www.example5.com/3"}, + + // Category C + {ID: 8, Title: "Entry 8", Date: time.Date(2024, 1, 3, 10, 0, 0, 0, time.UTC), FeedID: 600, FeedTitle: "Feed F", GroupID: 3, GroupTitle: "Category C", SiteURL: "www.example6.com", URL: "www.example7.com/1"}, + {ID: 9, Title: "Entry 9", Date: time.Date(2024, 1, 3, 11, 0, 0, 0, time.UTC), FeedID: 700, FeedTitle: "Feed G", GroupID: 3, GroupTitle: "Category C", SiteURL: "www.example7.com", URL: "www.example7.com/1"}, + } +} + func CreateMockEntries(n int) []*models.Entry { if n == 0 { - return []*models.Entry{ - // Category A - {ID: 1, Title: "Entry 1", Date: time.Date(2024, 1, 2, 10, 0, 0, 0, time.UTC), FeedID: 100, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A"}, - {ID: 3, Title: "Entry 3", Date: time.Date(2024, 1, 2, 11, 0, 0, 0, time.UTC), FeedID: 100, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A"}, - {ID: 5, Title: "Entry 5", Date: time.Date(2024, 1, 2, 12, 0, 0, 0, time.UTC), FeedID: 300, FeedTitle: "Feed C", GroupID: 1, GroupTitle: "Category A"}, - {ID: 6, Title: "Entry 6", Date: time.Date(2024, 1, 2, 13, 0, 0, 0, time.UTC), FeedID: 400, FeedTitle: "Feed D", GroupID: 1, GroupTitle: "Category A"}, - - // Category B - {ID: 2, Title: "Entry 2", Date: time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC), FeedID: 200, FeedTitle: "Feed B", GroupID: 2, GroupTitle: "Category B"}, - {ID: 4, Title: "Entry 4", Content: "Content of entry 4", Date: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC), FeedID: 200, FeedTitle: "Feed B", GroupID: 2, GroupTitle: "Category B"}, - {ID: 7, Title: "Entry 7", Date: time.Date(2024, 1, 1, 14, 0, 0, 0, time.UTC), FeedID: 500, FeedTitle: "Feed E", GroupID: 2, GroupTitle: "Category B"}, - - // Category C - {ID: 8, Title: "Entry 8", Date: time.Date(2024, 1, 3, 10, 0, 0, 0, time.UTC), FeedID: 600, FeedTitle: "Feed F", GroupID: 3, GroupTitle: "Category C"}, - {ID: 9, Title: "Entry 9", Date: time.Date(2024, 1, 3, 11, 0, 0, 0, time.UTC), FeedID: 700, FeedTitle: "Feed G", GroupID: 3, GroupTitle: "Category C"}, - } + return createHardcodedMockEntries() } titleTemplates := []string{ @@ -134,10 +138,13 @@ func CreateMockEntries(n int) []*models.Entry { commentsURL = fmt.Sprintf("https://example.com/comments/%d", i) } + siteURL := fmt.Sprintf("https://www.example%d.com", template.FeedID) + entryURL := fmt.Sprintf("%s/item/%d", siteURL, i) + entry := &models.Entry{ ID: int64(i), Title: fmt.Sprintf("Entry %d - %s", i, title), - URL: fmt.Sprintf("https://example.com/%d", i), + URL: entryURL, CommentsURL: commentsURL, Date: time.Now().Add(time.Duration(-i) * time.Hour), Content: content, @@ -145,6 +152,7 @@ func CreateMockEntries(n int) []*models.Entry { FeedTitle: template.FeedTitle, GroupID: template.GroupID, GroupTitle: template.GroupTitle, + SiteURL: siteURL, } entries = append(entries, entry) } diff --git a/internal/testutil/mocks.go b/internal/testutil/mocks.go index a80d2e2..cf9d3c3 100644 --- a/internal/testutil/mocks.go +++ b/internal/testutil/mocks.go @@ -58,21 +58,21 @@ func (m *MockArchiveService) CleanArchive(maxAge time.Duration) {} type MockEmailService struct { services.EmailService - SendFunc func(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error + SendFunc func(smtpConfig config.ConfigSmtp, digestConfig config.ConfigDigest, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error } -func (m *MockEmailService) Send(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error { - return m.SendFunc(cfg, overviewFile, groupedEntryFiles, data) +func (m *MockEmailService) Send(smtpConfig config.ConfigSmtp, digestConfig config.ConfigDigest, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error { + return m.SendFunc(smtpConfig, digestConfig, overviewFile, groupedEntryFiles, data) } type MockDigestService struct { services.DigestService - BuildDigestDataFunc func(category *miniflux.Category, entries *miniflux.Entries, icons map[int64]*models.FeedIcon, subGroupBy string, sortBy string, minifluxHost string) *models.OverviewTemplateData + BuildDigestDataFunc func(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData } -func (m *MockDigestService) BuildDigestData(category *miniflux.Category, entries *miniflux.Entries, icons map[int64]*models.FeedIcon, subGroupBy string, sortBy string, minifluxHost string) *models.OverviewTemplateData { +func (m *MockDigestService) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData { if m.BuildDigestDataFunc != nil { - return m.BuildDigestDataFunc(category, entries, icons, subGroupBy, sortBy, minifluxHost) + return m.BuildDigestDataFunc(entries, icons, view, minifluxHost, digestHost, categories, digestTitle) } return nil } diff --git a/scripts/preview/main.go b/scripts/preview/main.go index 5136f54..5e84063 100644 --- a/scripts/preview/main.go +++ b/scripts/preview/main.go @@ -18,6 +18,7 @@ import ( "miniflux-digest/internal/digest" "miniflux-digest/internal/email" "miniflux-digest/internal/llm" + "miniflux-digest/internal/manager" "miniflux-digest/internal/models" "miniflux-digest/internal/templates" "miniflux-digest/internal/testutil" @@ -47,7 +48,7 @@ func openBrowser(url string) error { return exec.Command(cmd, args...).Start() } -func generateMockDigest(cfg *config.Config) *models.OverviewTemplateData { +func generateMockDigest(cfg *config.Config, digestIndex int) *models.OverviewTemplateData { log.Println("generateDigestData: Starting...") llmService, err := llm.NewGeminiService(cfg.AI.ApiKey) @@ -55,6 +56,11 @@ func generateMockDigest(cfg *config.Config) *models.OverviewTemplateData { log.Fatalf("Failed to create LLM service: %v", err) } + digestManager, err := manager.NewDigestManager(cfg.Digests) + if err != nil { + log.Fatalf("Error creating digest manager: %v", err) + } + digestSvc := digest.NewDigestService(llmService) log.Println("generateDigestData: DigestService initialized.") @@ -62,8 +68,17 @@ func generateMockDigest(cfg *config.Config) *models.OverviewTemplateData { log.Println("generateDigestData: Using mock data...") entries = testutil.CreateMockEntries(200) - icons := make(map[int64]*models.FeedIcon) + var digestEntries []*models.Entry + digestConfig := cfg.Digests[digestIndex] + for _, entry := range entries { + if digestManager.GetOwningDigest(entry) == digestIndex { + digestEntries = append(digestEntries, entry) + } + } + + icons := make(map[int64]*models.FeedIcon) + for _, entry := range digestEntries { if _, ok := icons[entry.FeedID]; !ok { switch entry.FeedID { case 1: @@ -80,26 +95,23 @@ func generateMockDigest(cfg *config.Config) *models.OverviewTemplateData { log.Println("generateDigestData: Building digest data...") data := digestSvc.BuildDigestData( - entries, + digestEntries, icons, - cfg.Digest.View, + digestConfig.View, cfg.Miniflux.Host, - cfg.Digest.Host, + digestConfig.Host, + digestConfig.Categories, + digestConfig.Title, ) - if len(data.PrimaryGroups) > 0 { - data.PrimaryGroups[0].Summary = "This is a mock summary for the first group to test the layout." + for i := range data.PrimaryGroups { + data.PrimaryGroups[i].Summary = fmt.Sprintf("This is a mock summary for group %d to test the layout.", i+1) } - if len(data.PrimaryGroups) > 2 { - data.PrimaryGroups[2].Summary = "This is another mock summary for a different group, showing that not all groups have summaries." - } - - return data } -func generateMinifluxDigest(cfg *config.Config, minifluxClientService services.MinifluxClientService) *models.OverviewTemplateData { +func generateMinifluxDigest(cfg *config.Config, digestIndex int, minifluxClientService services.MinifluxClientService) *models.OverviewTemplateData { log.Println("generateDigestData: Starting...") llmService, err := llm.NewGeminiService(cfg.AI.ApiKey) @@ -107,6 +119,11 @@ func generateMinifluxDigest(cfg *config.Config, minifluxClientService services.M log.Fatalf("Failed to create LLM service: %v", err) } + digestManager, err := manager.NewDigestManager(cfg.Digests) + if err != nil { + log.Fatalf("Error creating digest manager: %v", err) + } + digestSvc := digest.NewDigestService(llmService) log.Println("generateDigestData: DigestService initialized.") @@ -118,8 +135,17 @@ func generateMinifluxDigest(cfg *config.Config, minifluxClientService services.M log.Fatalf("Failed to fetch entries: %v", err) } - icons := make(map[int64]*models.FeedIcon) + var digestEntries []*models.Entry + digestConfig := cfg.Digests[digestIndex] + for _, entry := range entries { + if digestManager.GetOwningDigest(entry) == digestIndex { + digestEntries = append(digestEntries, entry) + } + } + + icons := make(map[int64]*models.FeedIcon) + for _, entry := range digestEntries { if _, ok := icons[entry.FeedID]; !ok { icon, err := minifluxClientService.FeedIcon(entry.FeedID) if err != nil { @@ -132,11 +158,13 @@ func generateMinifluxDigest(cfg *config.Config, minifluxClientService services.M log.Println("generateDigestData: Building digest data...") return digestSvc.BuildDigestData( - entries, + digestEntries, icons, - cfg.Digest.View, + digestConfig.View, cfg.Miniflux.Host, - cfg.Digest.Host, + digestConfig.Host, + digestConfig.Categories, + digestConfig.Title, ) } @@ -157,22 +185,22 @@ func loadConfig() *config.Config { return cfg } -func generateAndArchiveHTML(cfg *config.Config, minifluxFlag bool) (*os.File, []*os.File, string, *models.OverviewTemplateData) { +func generateAndArchiveHTML(cfg *config.Config, digestIndex int, minifluxFlag bool) (*os.File, []*os.File, string, *models.OverviewTemplateData) { var data *models.OverviewTemplateData + digestConfig := cfg.Digests[digestIndex] if minifluxFlag { minifluxClient := miniflux.NewClient(cfg.Miniflux.Host, cfg.Miniflux.ApiToken) clientWrapper := app.NewMinifluxClientWrapper(minifluxClient) - data = generateMinifluxDigest(cfg, clientWrapper) + data = generateMinifluxDigest(cfg, digestIndex, clientWrapper) } else { - data = generateMockDigest(cfg) + data = generateMockDigest(cfg, digestIndex) } log.Println("main: Digest data generated.") - archiveSvc := archive.NewArchiveService(webserver.ArchiveBasePath, templates.ArchiveTemplate, templates.OverviewTemplate) - overviewFile, groupedEntryFiles, err := archiveSvc.MakeArchiveHTML(data, cfg.Digest.Compress) + overviewFile, groupedEntryFiles, err := archiveSvc.MakeArchiveHTML(data, *digestConfig.Compress) if err != nil { log.Fatalf("Failed to generate HTML: %v", err) } @@ -186,7 +214,11 @@ func generateAndArchiveHTML(cfg *config.Config, minifluxFlag bool) (*os.File, [] log.Fatalf("Failed to get relative path: %v", err) } log.Printf("relativePath: %s", relativePath) - overviewURL := fmt.Sprintf("http://localhost%s/archive/%s", webserver.Port, relativePath) + + overviewURL := fmt.Sprintf("http://localhost%s/%s", webserver.Port, relativePath) + if digestConfig.Host != "" { + overviewURL = fmt.Sprintf("%s/archive/%s", digestConfig.Host, relativePath) + } log.Printf("overviewURL: %s", overviewURL) return overviewFile, groupedEntryFiles, overviewURL, data @@ -198,7 +230,8 @@ func handleEmail(cfg *config.Config, overviewFile *os.File, groupedEntryFiles [] EmailTemplate: templates.EmailTemplate, } - if err := emailSvc.Send(cfg, overviewFile, groupedEntryFiles, data); err != nil { + digestConfig := cfg.Digests[0] + if err := emailSvc.Send(cfg.Smtp, digestConfig, overviewFile, groupedEntryFiles, data); err != nil { log.Fatalf("Failed to send email: %v", err) } log.Printf("Successfully generated and sent email.") @@ -230,18 +263,25 @@ func main() { emailFlag, minifluxFlag, htmlFlag := setupAndParseFlags() cfg := loadConfig() + numDigest := len(cfg.Digests) if emailFlag { - overviewFile, groupedEntryFiles, _, data := generateAndArchiveHTML(cfg, minifluxFlag) - handleEmail(cfg, overviewFile, groupedEntryFiles, data) + for digestIndex := range numDigest { + overviewFile, groupedEntryFiles, _, data := generateAndArchiveHTML(cfg, digestIndex, minifluxFlag) + handleEmail(cfg, overviewFile, groupedEntryFiles, data) + } } else if htmlFlag { - _, _, overviewURL, _ := generateAndArchiveHTML(cfg, minifluxFlag) - handleOpenUrl(overviewURL) + for digestIndex := range numDigest { + _, _, overviewURL, _ := generateAndArchiveHTML(cfg, digestIndex, minifluxFlag) + handleOpenUrl(overviewURL) + } } else if minifluxFlag { - _, _, overviewURL, _ := generateAndArchiveHTML(cfg, minifluxFlag) - handleOpenUrl(overviewURL) + for digestIndex := range numDigest { + _, _, overviewURL, _ := generateAndArchiveHTML(cfg, digestIndex, minifluxFlag) + handleOpenUrl(overviewURL) + } } else { log.Println("main: Starting web server") handleWebServer() } -} \ No newline at end of file +}