gemini/feature/multi-digest-architecture #61
No files matched your search
@@ -14,5 +14,3 @@
|
||||
|
||||
# Gemini CLI state
|
||||
/.gemini/
|
||||
|
||||
PROJECT.md
|
||||
@@ -13,6 +13,21 @@ and delivered to the web and your inbox.
|
||||
These entries can be sorted, summarized and group to enable skimming and to
|
||||
decrease time spent catching up.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Multi-Digest Engine
|
||||
|
||||
The application is designed to support multiple, distinct digests running in
|
||||
parallel or on different schedules.
|
||||
|
||||
- **One Run, Many Outputs:** The system processes multiple defined digests.
|
||||
- **Stack-Based Priority:** The order of digests in the configuration array
|
||||
determines priority. The first digest (index 0) has the highest priority. If
|
||||
an item matches Digest X (Index 0) and Digest Y (Index 1), it belongs to
|
||||
Digest X.
|
||||
- **Deterministic Filtering:** Inclusion is based on rules (Feed Title,
|
||||
Category Title, Site URL, Entry URL) using Exact, Prefix, or Regex matching.
|
||||
|
||||
## Development Workflow
|
||||
|
||||
### Writing Code
|
||||
|
||||
@@ -24,11 +24,12 @@ Customize the digest view, schedule and more. Oh my.
|
||||
> many of the following features are _optional_
|
||||
|
||||
* ⏰ Automated scheduling (ex: daily, weekly, ever third friday)
|
||||
* 📚 **Multi-Digest Support:** Configure multiple digests (e.g., "Daily Tech", "Weekly Social") with unique schedules and content filters.
|
||||
* 🌞 Dark and light themes
|
||||
* 📥 Fetches all unread entries
|
||||
* 📧 Delivers personalized HTML digests via email
|
||||
* 🛜 Archives HTML digests for static web serving
|
||||
* 🤖 Summarize and group entries for faster skimming
|
||||
* 🤖 Summarize and group entries for faster skimming (AI view)
|
||||
* ✅ Automatically marks entries as read
|
||||
* 🧹 Manages storage by purging older digests
|
||||
* ♻️ Wash, rinse, repeat
|
||||
@@ -77,10 +78,14 @@ Customize the digest view, schedule and more. Oh my.
|
||||
|
||||
A `config.yaml` file is required for operation.
|
||||
|
||||
Create this file in the root of the project directory and edit it.
|
||||
Create this file in the root of the project directory. You can start by copying the [example](config.yaml.example):
|
||||
|
||||
See the [config.yaml.example](config.yaml.example) to learn about
|
||||
requirements, defaults and other options.
|
||||
```bash
|
||||
cp config.yaml.example config.yaml
|
||||
```
|
||||
|
||||
**📚 Documentation:**
|
||||
For a detailed reference of all available options, including how to configure multiple digests and filters, please read [docs/CONFIG.md](docs/CONFIG.md).
|
||||
|
||||
### Run
|
||||
|
||||
@@ -116,8 +121,6 @@ project's license.
|
||||
|
||||
* support more LLM providers
|
||||
* allow user to customize prompts
|
||||
* support explicit feed/category sources
|
||||
* support multiple digests
|
||||
* support text/html email content
|
||||
* fix web scrolling bug when selecting item content
|
||||
|
||||
|
||||
+27
-21
@@ -23,22 +23,24 @@ const (
|
||||
ArchiveCleanupDays = 21
|
||||
)
|
||||
|
||||
func digestJob(application *app.App, source string) {
|
||||
log.Printf("Starting digest job from source: %s", source)
|
||||
overviewFile, groupedEntryFiles, data, err := processor.ProcessDigest(application)
|
||||
func digestJob(application *app.App, digestIndex int, source string) {
|
||||
digestConfig := application.Config.Digests[digestIndex]
|
||||
log.Printf("Starting digest job for '%s' from source: %s", digestConfig.Title, source)
|
||||
overviewFile, groupedEntryFiles, data, err := processor.ProcessDigest(application, digestIndex)
|
||||
if err != nil {
|
||||
log.Printf("Error processing digest: %v", err)
|
||||
log.Printf("Error processing digest '%s': %v", digestConfig.Title, err)
|
||||
return
|
||||
}
|
||||
|
||||
emailSent := application.Config.Smtp.Host != ""
|
||||
if emailSent {
|
||||
if err := application.EmailService.Send(application.Config, overviewFile, groupedEntryFiles, data); err != nil {
|
||||
log.Printf("Error sending digest email: %v", err)
|
||||
if err := application.EmailService.Send(application.Config.Smtp, digestConfig, overviewFile, groupedEntryFiles, data); err != nil {
|
||||
log.Printf("Error sending digest email for '%s': %v", digestConfig.Title, err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("Digest produced at %s: entries=%d, folder=%s, email_sent=%t, source=%s",
|
||||
log.Printf("Digest '%s' produced at %s: entries=%d, folder=%s, email_sent=%t, source=%s",
|
||||
digestConfig.Title,
|
||||
data.GeneratedDate.Format(time.RFC3339),
|
||||
len(data.Entries),
|
||||
overviewFile.Name(),
|
||||
@@ -47,20 +49,24 @@ func digestJob(application *app.App, source string) {
|
||||
)
|
||||
}
|
||||
|
||||
func registerDigestJob(application *app.App, scheduler gocron.Scheduler) {
|
||||
_, err := scheduler.NewJob(
|
||||
gocron.CronJob(application.Config.Digest.Schedule, true),
|
||||
gocron.NewTask(func() {
|
||||
digestJob(application, "scheduler")
|
||||
}),
|
||||
)
|
||||
func registerDigestJobs(application *app.App, scheduler gocron.Scheduler) {
|
||||
for i, d := range application.Config.Digests {
|
||||
digestConfig := d // Capture loop variable
|
||||
digestIndex := i
|
||||
_, err := scheduler.NewJob(
|
||||
gocron.CronJob(digestConfig.Schedule, true),
|
||||
gocron.NewTask(func() {
|
||||
digestJob(application, digestIndex, "scheduler")
|
||||
}),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating job: %v", err)
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating job for digest '%s': %v", digestConfig.Title, err)
|
||||
}
|
||||
|
||||
if application.Config.Digest.RunOnStartup {
|
||||
go digestJob(application, "startup")
|
||||
if digestConfig.RunOnStartup {
|
||||
go digestJob(application, digestIndex, "startup")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,7 +99,7 @@ func main() {
|
||||
|
||||
scheduler.Start()
|
||||
|
||||
registerDigestJob(application, scheduler)
|
||||
registerDigestJobs(application, scheduler)
|
||||
registerArchiveCleanupJob(application, scheduler)
|
||||
|
||||
go webserver.ListenAndServe(webserver.ArchiveBasePath, webserver.Port)
|
||||
@@ -129,4 +135,4 @@ func initServices(cfg *config.Config) (*app.App, error) {
|
||||
|
||||
func initScheduler() (gocron.Scheduler, error) {
|
||||
return gocron.NewScheduler()
|
||||
}
|
||||
}
|
||||
+31
-29
@@ -1,36 +1,38 @@
|
||||
miniflux:
|
||||
host: "YOUR_MINIFLUX_URL"
|
||||
api_token: "YOUR_MINIFLUX_API_API_KEY"
|
||||
host: "https://reader.miniflux.app"
|
||||
api_token: "YOUR_MINIFLUX_API_TOKEN"
|
||||
|
||||
smtp:
|
||||
host: "YOUR_SMTP_HOST"
|
||||
host: "smtp.example.com"
|
||||
port: 587
|
||||
user: "YOUR_SMTP_USERNAME"
|
||||
password: "YOUR_SMTP_PASSWORD"
|
||||
|
||||
digest:
|
||||
email:
|
||||
to: "RECIPIENT@example.com"
|
||||
from: "SENDER@example.com"
|
||||
|
||||
# cron syntax also supported
|
||||
schedule: "@every 24h"
|
||||
|
||||
# default http://localhost:8080, URL where HTML archives will be served
|
||||
host: "https://your-digest-host.com"
|
||||
|
||||
# default true, compresses digest html
|
||||
compress: true
|
||||
|
||||
# default false | true
|
||||
mark_as_read: true
|
||||
|
||||
# default false | true
|
||||
run_on_startup: false
|
||||
|
||||
# default date | category | ai
|
||||
view: date
|
||||
user: "smtp_user"
|
||||
password: "smtp_password"
|
||||
|
||||
ai:
|
||||
# gemini api key
|
||||
api_key: "YOUR_GEMINI_API_KEY"
|
||||
|
||||
digests:
|
||||
- title: "Daily Tech"
|
||||
schedule: "0 8 * * *"
|
||||
host: "https://digest.example.com"
|
||||
view: "ai"
|
||||
mark_as_read: true
|
||||
email:
|
||||
to: "recipient@example.com"
|
||||
from: "sender@example.com"
|
||||
filters:
|
||||
feed_titles: ["Hacker News", "Ars Technica"]
|
||||
category_titles: ["Technology"]
|
||||
categories:
|
||||
- title: "AI & LLMs"
|
||||
description: "Artificial intelligence, machine learning, and LLM news."
|
||||
- title: "Programming"
|
||||
description: "Software development, languages, and tools."
|
||||
|
||||
- title: "Weekly Catch-All"
|
||||
schedule: "@weekly"
|
||||
host: "https://digest.example.com"
|
||||
view: "date"
|
||||
email:
|
||||
to: "recipient@example.com"
|
||||
from: "sender@example.com"
|
||||
@@ -0,0 +1,95 @@
|
||||
# Configuration Reference
|
||||
|
||||
The `miniflux-digest` application is configured via a YAML file (default: `config.yaml`). This document details all available options, their default values, and validation rules.
|
||||
|
||||
## Root Object
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
| :--- | :--- | :--- | :--- |
|
||||
| `miniflux` | Object | **Yes** | Connection settings for your Miniflux instance. |
|
||||
| `smtp` | Object | No | SMTP server settings for sending emails. |
|
||||
| `ai` | Object | Conditional | Required if any digest uses `view: ai`. |
|
||||
| `digests` | Array | **Yes** | A list of digest configurations. At least one must be defined. |
|
||||
|
||||
---
|
||||
|
||||
## Miniflux (`miniflux`)
|
||||
|
||||
| Field | Type | Description |
|
||||
| :--- | :--- | :--- |
|
||||
| `host` | String | **Required.** The full URL to your Miniflux instance (e.g., `https://reader.miniflux.app`). |
|
||||
| `api_token` | String | **Required.** Your Miniflux API Key. Create one in Miniflux under **Settings > API Keys**. |
|
||||
|
||||
## SMTP (`smtp`)
|
||||
|
||||
Required only if you want to receive digests via email.
|
||||
|
||||
| Field | Type | Default | Description |
|
||||
| :--- | :--- | :--- | :--- |
|
||||
| `host` | String | No | The hostname of your SMTP server (e.g., `smtp.gmail.com`). **Required** if any digest has email configured. |
|
||||
| `port` | Int | `587` | The SMTP port (1-65535). |
|
||||
| `user` | String | - | SMTP username. |
|
||||
| `password` | String | - | SMTP password. |
|
||||
|
||||
## AI (`ai`)
|
||||
|
||||
| Field | Type | Description |
|
||||
| :--- | :--- | :--- |
|
||||
| `api_key` | String | **Required if using AI View.** Your Google Gemini API Key. |
|
||||
|
||||
---
|
||||
|
||||
## Digests (`digests`)
|
||||
|
||||
You can define multiple digests. The order matters!
|
||||
|
||||
### Priority & Ownership
|
||||
The application processes digests in the order they appear in the configuration list.
|
||||
* **First Match Wins:** The first digest that matches an entry "claims" it.
|
||||
* **No Duplicates:** An entry will never appear in more than one digest.
|
||||
* **Catch-All:** It is common to define specific digests first (e.g., "Tech News") and a broad, filter-less digest last to catch everything else.
|
||||
|
||||
### Digest Object
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
| :--- | :--- | :--- | :--- |
|
||||
| `title` | String | **Yes** | The display title of the digest. Must be unique after "slugification" (e.g., "My Digest" and "My-Digest" conflict). |
|
||||
| `schedule` | String | **Yes** | When to run the digest. Supports Cron syntax (`0 8 * * *`) or Go-cron descriptors (`@daily`, `@weekly`, `@every 2h`). |
|
||||
| `host` | String | No | The base URL where the static HTML archives will be hosted (e.g., `https://digest.myserver.com`). **Required** for correct links in emails. |
|
||||
| `view` | String | No | One of: `date`, `category`, `ai`. Controls how entries are grouped and presented. Default: `date`. |
|
||||
| `email` | Object | No | Email settings for this specific digest. |
|
||||
| `filters` | Object | No | Rules for including entries in this digest. |
|
||||
| `categories`| Array | No | **(AI View Only)** Custom categories to guide the LLM. |
|
||||
| `compress` | Bool | No | If `true` (default), minifies the generated HTML. |
|
||||
| `mark_as_read`| Bool | No | If `true`, marks entries as read in Miniflux *after* processing. Default: `false`. |
|
||||
| `run_on_startup`| Bool | No | If `true`, runs the digest job immediately when the application starts. Default: `false`. |
|
||||
|
||||
### Email Object (`email`)
|
||||
|
||||
| Field | Type | Description |
|
||||
| :--- | :--- | :--- |
|
||||
| `to` | String | Recipient email address. |
|
||||
| `from` | String | Sender email address. |
|
||||
|
||||
### Filters Object (`filters`)
|
||||
|
||||
All conditions within a list are **OR** (e.g., matching *any* feed title).
|
||||
Conditions between different fields are **AND** (e.g., matching feed title *AND* site URL).
|
||||
|
||||
| Field | Type | Matching Logic | Description |
|
||||
| :--- | :--- | :--- | :--- |
|
||||
| `feed_titles` | String[] | Exact | Matches the exact title of the RSS feed. |
|
||||
| `category_titles` | String[] | Exact | Matches the exact title of the Miniflux category. |
|
||||
| `site_urls` | String[] | Prefix | Matches if the feed's site URL starts with this string. |
|
||||
| `entry_urls` | String[] | Prefix | Matches if the entry's URL starts with this string. |
|
||||
| `feed_title_patterns` | String[] | Regex | Go-flavor Regex match against feed title. |
|
||||
| `category_title_patterns` | String[] | Regex | Go-flavor Regex match against category title. |
|
||||
| `site_url_patterns` | String[] | Regex | Go-flavor Regex match against site URL. |
|
||||
| `entry_url_patterns` | String[] | Regex | Go-flavor Regex match against entry URL. |
|
||||
|
||||
### Categories Object (`categories`)
|
||||
|
||||
Used only when `view: ai`. Provide a list of objects with `title` and `description` to help the AI categorize content.
|
||||
|
||||
If you do not provide this block, the AI will automatically determine the best primary groups for your content.
|
||||
|
||||
@@ -6,7 +6,6 @@ require (
|
||||
github.com/go-co-op/gocron/v2 v2.16.2
|
||||
github.com/go-playground/validator/v10 v10.27.0
|
||||
github.com/knadh/koanf/parsers/yaml v1.1.0
|
||||
github.com/knadh/koanf/providers/confmap v1.0.0
|
||||
github.com/knadh/koanf/providers/file v1.2.0
|
||||
github.com/knadh/koanf/v2 v2.2.2
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
|
||||
@@ -70,8 +70,6 @@ github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpb
|
||||
github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI=
|
||||
github.com/knadh/koanf/parsers/yaml v1.1.0 h1:3ltfm9ljprAHt4jxgeYLlFPmUaunuCgu1yILuTXRdM4=
|
||||
github.com/knadh/koanf/parsers/yaml v1.1.0/go.mod h1:HHmcHXUrp9cOPcuC+2wrr44GTUB0EC+PyfN3HZD9tFg=
|
||||
github.com/knadh/koanf/providers/confmap v1.0.0 h1:mHKLJTE7iXEys6deO5p6olAiZdG5zwp8Aebir+/EaRE=
|
||||
github.com/knadh/koanf/providers/confmap v1.0.0/go.mod h1:txHYHiI2hAtF0/0sCmcuol4IDcuQbKTybiB1nOcUo1A=
|
||||
github.com/knadh/koanf/providers/file v1.2.0 h1:hrUJ6Y9YOA49aNu/RSYzOTFlqzXSCpmYIDXI7OJU6+U=
|
||||
github.com/knadh/koanf/providers/file v1.2.0/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA=
|
||||
github.com/knadh/koanf/v2 v2.2.2 h1:ghbduIkpFui3L587wavneC9e3WIliCgiCgdxYO/wd7A=
|
||||
|
||||
@@ -21,7 +21,7 @@ func (m *mockArchiveService) CleanArchive(maxAge time.Duration) {}
|
||||
|
||||
type mockEmailService struct{}
|
||||
|
||||
func (m *mockEmailService) Send(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error {
|
||||
func (m *mockEmailService) Send(smtpConfig config.ConfigSmtp, digestConfig config.ConfigDigest, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ func (m *mockMinifluxClientService) UpdateEntries(entryIDs []int64, status strin
|
||||
|
||||
type mockDigestService struct{}
|
||||
|
||||
func (m *mockDigestService) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string) *models.OverviewTemplateData {
|
||||
func (m *mockDigestService) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -41,14 +41,17 @@ func (m *MinifluxClientWrapper) GetAllUnreadEntries() ([]*models.Entry, error) {
|
||||
categoryTitle = entry.Feed.Category.Title
|
||||
}
|
||||
var feedTitle string
|
||||
var siteURL string
|
||||
if entry.Feed != nil {
|
||||
feedTitle = entry.Feed.Title
|
||||
siteURL = entry.Feed.SiteURL
|
||||
}
|
||||
|
||||
entries = append(entries, &models.Entry{
|
||||
ID: entry.ID,
|
||||
Title: entry.Title,
|
||||
URL: entry.URL,
|
||||
SiteURL: siteURL,
|
||||
Content: entry.Content,
|
||||
FeedID: entry.FeedID,
|
||||
FeedTitle: feedTitle,
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"miniflux-digest/internal/config"
|
||||
"miniflux-digest/internal/models"
|
||||
)
|
||||
|
||||
type DigestService interface {
|
||||
BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string) *models.OverviewTemplateData
|
||||
BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData
|
||||
}
|
||||
@@ -7,5 +7,5 @@ import (
|
||||
)
|
||||
|
||||
type EmailService interface {
|
||||
Send(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error
|
||||
Send(smtpConfig config.ConfigSmtp, digestConfig config.ConfigDigest, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error
|
||||
}
|
||||
@@ -65,7 +65,8 @@ func (s *ArchiveServiceImpl) makeGroupedEntriesArchiveFile(data *models.PrimaryG
|
||||
}
|
||||
|
||||
func (s *ArchiveServiceImpl) makeOverviewArchiveFile(data *models.OverviewTemplateData) (*os.File, string, error) {
|
||||
dateFolderPath := fmt.Sprintf("%s/%s", s.ArchiveBaseDir, data.GeneratedDate.Format("2006-01-02"))
|
||||
digestSlug := data.DigestSlug
|
||||
dateFolderPath := fmt.Sprintf("%s/%s/%s", s.ArchiveBaseDir, digestSlug, data.GeneratedDate.Format("2006-01-02"))
|
||||
filename := fmt.Sprintf("%s/index.html", dateFolderPath)
|
||||
err := os.MkdirAll(dateFolderPath, 0755)
|
||||
|
||||
|
||||
@@ -131,6 +131,7 @@ func TestMakeOverviewArchiveFile(t *testing.T) {
|
||||
archiveService := NewArchiveService(tempDir, &mockHTMLTemplate{}, &mockHTMLTemplate{})
|
||||
|
||||
data := &models.OverviewTemplateData{
|
||||
DigestSlug: "test-digest",
|
||||
GeneratedDate: time.Date(2024, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
}
|
||||
|
||||
@@ -147,12 +148,12 @@ func TestMakeOverviewArchiveFile(t *testing.T) {
|
||||
}
|
||||
}()
|
||||
|
||||
expectedPath := filepath.Join(tempDir, "2024-01-01", "index.html")
|
||||
expectedPath := filepath.Join(tempDir, "test-digest", "2024-01-01", "index.html")
|
||||
if _, err := os.Stat(expectedPath); os.IsNotExist(err) {
|
||||
t.Errorf("Expected file %s to exist", expectedPath)
|
||||
}
|
||||
if dateFolderPath != filepath.Join(tempDir, "2024-01-01") {
|
||||
t.Errorf("Expected date folder path %s, got %s", filepath.Join(tempDir, "2024-01-01"), dateFolderPath)
|
||||
if dateFolderPath != filepath.Join(tempDir, "test-digest", "2024-01-01") {
|
||||
t.Errorf("Expected date folder path %s, got %s", filepath.Join(tempDir, "test-digest", "2024-01-01"), dateFolderPath)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+93
-25
@@ -3,10 +3,13 @@ package config
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"miniflux-digest/internal/utils"
|
||||
|
||||
"github.com/go-playground/validator/v10"
|
||||
"github.com/go-viper/mapstructure/v2"
|
||||
"github.com/knadh/koanf/parsers/yaml"
|
||||
"github.com/knadh/koanf/providers/confmap"
|
||||
"github.com/knadh/koanf/providers/file"
|
||||
"github.com/knadh/koanf/v2"
|
||||
"github.com/robfig/cron/v3"
|
||||
@@ -41,14 +44,33 @@ type ConfigSmtp struct {
|
||||
Password string `koanf:"password"`
|
||||
}
|
||||
|
||||
type ConfigCategory struct {
|
||||
Title string `koanf:"title" validate:"required"`
|
||||
Description string `koanf:"description"`
|
||||
}
|
||||
|
||||
type ConfigFilters struct {
|
||||
FeedTitles []string `koanf:"feed_titles"`
|
||||
CategoryTitles []string `koanf:"category_titles"`
|
||||
FeedURLs []string `koanf:"feed_urls"`
|
||||
EntryURLs []string `koanf:"entry_urls"`
|
||||
FeedTitlePatterns []string `koanf:"feed_title_patterns"`
|
||||
CategoryTitlePatterns []string `koanf:"category_title_patterns"`
|
||||
FeedURLPatterns []string `koanf:"feed_url_patterns"`
|
||||
EntryURLPatterns []string `koanf:"entry_url_patterns"`
|
||||
}
|
||||
|
||||
type ConfigDigest struct {
|
||||
Title string `koanf:"title" validate:"required"`
|
||||
Email ConfigDigestEmail `koanf:"email"`
|
||||
Schedule string `koanf:"schedule" validate:"gocron"`
|
||||
Host string `koanf:"host"`
|
||||
Compress bool `koanf:"compress"`
|
||||
View string `koanf:"view" validate:"oneof=date category ai"`
|
||||
MarkAsRead bool `koanf:"mark_as_read"`
|
||||
Schedule string `koanf:"schedule" validate:"required,gocron"`
|
||||
Host string `koanf:"host" validate:"omitempty,url"`
|
||||
Compress *bool `koanf:"compress"`
|
||||
View string `koanf:"view" validate:"oneof=date category ai"`
|
||||
MarkAsRead bool `koanf:"mark_as_read"`
|
||||
RunOnStartup bool `koanf:"run_on_startup"`
|
||||
Filters ConfigFilters `koanf:"filters"`
|
||||
Categories []ConfigCategory `koanf:"categories" validate:"dive"`
|
||||
}
|
||||
|
||||
type ConfigAI struct {
|
||||
@@ -58,7 +80,7 @@ type ConfigAI struct {
|
||||
type Config struct {
|
||||
Miniflux ConfigMiniflux `koanf:"miniflux"`
|
||||
Smtp ConfigSmtp `koanf:"smtp"`
|
||||
Digest ConfigDigest `koanf:"digest"`
|
||||
Digests []ConfigDigest `koanf:"digests" validate:"min=1,dive"`
|
||||
AI ConfigAI `koanf:"ai"`
|
||||
}
|
||||
|
||||
@@ -72,8 +94,47 @@ func (c *Config) Validate() error {
|
||||
|
||||
validate.RegisterStructValidation(func(sl validator.StructLevel) {
|
||||
cfg := sl.Current().Interface().(Config)
|
||||
if cfg.Digest.View == "ai" && cfg.AI.ApiKey == "" {
|
||||
sl.ReportError(cfg.AI.ApiKey, "AI.ApiKey", "ApiKey", "required_if", "Digest.View is 'ai'")
|
||||
slugs := make(map[string]string)
|
||||
|
||||
for _, digest := range cfg.Digests {
|
||||
if digest.View == "ai" && cfg.AI.ApiKey == "" {
|
||||
sl.ReportError(cfg.AI.ApiKey, "AI.ApiKey", "ApiKey", "required_if", "Digest.View is 'ai'")
|
||||
}
|
||||
|
||||
// Email Validation dependencies
|
||||
if digest.Email.To != "" {
|
||||
if cfg.Smtp.Host == "" {
|
||||
sl.ReportError(cfg.Smtp.Host, "Smtp.Host", "Host", "required_if_email", fmt.Sprintf("SMTP Host is required because digest '%s' has email configured", digest.Title))
|
||||
}
|
||||
if digest.Host == "" {
|
||||
sl.ReportError(digest.Host, "Digests.Host", "Host", "required_if_email", fmt.Sprintf("Digest Host is required because digest '%s' has email configured (needed for links)", digest.Title))
|
||||
}
|
||||
}
|
||||
|
||||
slug := utils.Slugify(digest.Title)
|
||||
if slug == "" {
|
||||
sl.ReportError(digest.Title, "Digests.Title", "Title", "invalid_slug", fmt.Sprintf("Digest title '%s' results in an empty slug", digest.Title))
|
||||
}
|
||||
|
||||
if originalTitle, exists := slugs[slug]; exists {
|
||||
sl.ReportError(digest.Title, "Digests.Title", "Title", "unique_slug", fmt.Sprintf("Digest title '%s' conflicts with '%s' (both slugify to '%s')", digest.Title, originalTitle, slug))
|
||||
} else {
|
||||
slugs[slug] = digest.Title
|
||||
}
|
||||
|
||||
// Validate regex patterns
|
||||
validatePatterns := func(patterns []string, fieldName string) {
|
||||
for _, p := range patterns {
|
||||
if _, err := regexp.Compile(p); err != nil {
|
||||
sl.ReportError(digest.Filters, fieldName, fieldName, "regex", fmt.Sprintf("Invalid regex '%s': %v", p, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
validatePatterns(digest.Filters.FeedTitlePatterns, "Filters.FeedTitlePatterns")
|
||||
validatePatterns(digest.Filters.CategoryTitlePatterns, "Filters.CategoryTitlePatterns")
|
||||
validatePatterns(digest.Filters.FeedURLPatterns, "Filters.FeedURLPatterns")
|
||||
validatePatterns(digest.Filters.EntryURLPatterns, "Filters.EntryURLPatterns")
|
||||
}
|
||||
}, Config{})
|
||||
|
||||
@@ -94,33 +155,40 @@ func Load(path string) (*Config, error) {
|
||||
k := koanf.New(".")
|
||||
parser := yaml.Parser()
|
||||
|
||||
if err := setDefaultValues(k); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := k.Load(file.Provider(path), parser); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg := &Config{}
|
||||
if err := k.Unmarshal("", &cfg); err != nil {
|
||||
if err := k.UnmarshalWithConf("", &cfg, koanf.UnmarshalConf{
|
||||
DecoderConfig: &mapstructure.DecoderConfig{
|
||||
ErrorUnused: true,
|
||||
TagName: "koanf",
|
||||
ZeroFields: true,
|
||||
Result: &cfg,
|
||||
},
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Apply defaults
|
||||
if cfg.Smtp.Port == 0 {
|
||||
cfg.Smtp.Port = 587
|
||||
}
|
||||
|
||||
for i := range cfg.Digests {
|
||||
if cfg.Digests[i].Compress == nil {
|
||||
def := true
|
||||
cfg.Digests[i].Compress = &def
|
||||
}
|
||||
if cfg.Digests[i].View == "" {
|
||||
cfg.Digests[i].View = "date"
|
||||
}
|
||||
}
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func setDefaultValues(k *koanf.Koanf) error {
|
||||
return k.Load(confmap.Provider(map[string]any{
|
||||
"digest.compress": true,
|
||||
"digest.view": "date",
|
||||
"digest.host": "http://localhost:8080",
|
||||
"digest.schedule": "@weekly",
|
||||
"digest.mark_as_read": true,
|
||||
"digest.run_on_startup": false,
|
||||
}, "."), nil)
|
||||
}
|
||||
+220
-41
@@ -21,9 +21,12 @@ func TestLoad(t *testing.T) {
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digest": map[string]any{
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "@daily",
|
||||
},
|
||||
"host": "http://localhost:8080",
|
||||
"view": "category",
|
||||
}},
|
||||
"ai": map[string]any{
|
||||
"api_key": "dummy-key",
|
||||
},
|
||||
@@ -36,9 +39,44 @@ func TestLoad(t *testing.T) {
|
||||
"miniflux": map[string]any{
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digest": map[string]any{
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "@daily",
|
||||
}},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "valid config missing view defaults to date",
|
||||
config: map[string]any{
|
||||
"miniflux": map[string]any{
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "@daily",
|
||||
"host": "http://localhost:8080",
|
||||
}},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "email configured but missing smtp host",
|
||||
config: map[string]any{
|
||||
"miniflux": map[string]any{
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "@daily",
|
||||
"host": "http://localhost:8080",
|
||||
"view": "category",
|
||||
"email": map[string]any{
|
||||
"to": "test@example.com",
|
||||
},
|
||||
}},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
@@ -48,9 +86,12 @@ func TestLoad(t *testing.T) {
|
||||
"miniflux": map[string]any{
|
||||
"host": "miniflux.example.com",
|
||||
},
|
||||
"digest": map[string]any{
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "@daily",
|
||||
},
|
||||
"host": "http://localhost:8080",
|
||||
"view": "category",
|
||||
}},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
@@ -61,9 +102,12 @@ func TestLoad(t *testing.T) {
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digest": map[string]any{
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "@daily",
|
||||
},
|
||||
"host": "http://localhost:8080",
|
||||
"view": "category",
|
||||
}},
|
||||
"smtp": map[string]any{
|
||||
"port": 65536,
|
||||
},
|
||||
@@ -77,9 +121,12 @@ func TestLoad(t *testing.T) {
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digest": map[string]any{
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "@daily",
|
||||
},
|
||||
"host": "http://localhost:8080",
|
||||
"view": "category",
|
||||
}},
|
||||
"smtp": map[string]any{
|
||||
"port": 587,
|
||||
},
|
||||
@@ -96,12 +143,15 @@ func TestLoad(t *testing.T) {
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digest": map[string]any{
|
||||
"schedule": "@daily",
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "@daily",
|
||||
"host": "http://localhost:8080",
|
||||
"view": "category",
|
||||
"email": map[string]any{
|
||||
"to": "invalid-email",
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
@@ -112,12 +162,15 @@ func TestLoad(t *testing.T) {
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digest": map[string]any{
|
||||
"schedule": "@daily",
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "@daily",
|
||||
"host": "http://localhost:8080",
|
||||
"view": "category",
|
||||
"email": map[string]any{
|
||||
"from": "another-invalid-email",
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
@@ -128,12 +181,18 @@ func TestLoad(t *testing.T) {
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digest": map[string]any{
|
||||
"schedule": "@daily",
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "@daily",
|
||||
"host": "http://localhost:8080",
|
||||
"view": "category",
|
||||
"email": map[string]any{
|
||||
"to": "test@example.com",
|
||||
"from": "sender@example.com",
|
||||
},
|
||||
}},
|
||||
"smtp": map[string]any{
|
||||
"host": "smtp.example.com",
|
||||
},
|
||||
"ai": map[string]any{
|
||||
"api_key": "dummy-key",
|
||||
@@ -148,9 +207,12 @@ func TestLoad(t *testing.T) {
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digest": map[string]any{
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "* * 1 * *",
|
||||
},
|
||||
"host": "http://localhost:8080",
|
||||
"view": "category",
|
||||
}},
|
||||
"ai": map[string]any{
|
||||
"api_key": "dummy-key",
|
||||
},
|
||||
@@ -164,9 +226,12 @@ func TestLoad(t *testing.T) {
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digest": map[string]any{
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "* * * * * * *",
|
||||
},
|
||||
"host": "http://localhost:8080",
|
||||
"view": "category",
|
||||
}},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
@@ -177,9 +242,12 @@ func TestLoad(t *testing.T) {
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digest": map[string]any{
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "@every 1h30m",
|
||||
},
|
||||
"host": "http://localhost:8080",
|
||||
"view": "category",
|
||||
}},
|
||||
"ai": map[string]any{
|
||||
"api_key": "dummy-key",
|
||||
},
|
||||
@@ -193,13 +261,16 @@ func TestLoad(t *testing.T) {
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digest": map[string]any{
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "@every bad-duration",
|
||||
},
|
||||
"host": "http://localhost:8080",
|
||||
"view": "category",
|
||||
}},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
|
||||
|
||||
{
|
||||
name: "invalid view value",
|
||||
config: map[string]any{
|
||||
@@ -207,10 +278,12 @@ func TestLoad(t *testing.T) {
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digest": map[string]any{
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "@daily",
|
||||
"view": "magic",
|
||||
},
|
||||
"host": "http://localhost:8080",
|
||||
"view": "magic",
|
||||
}},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
@@ -221,10 +294,123 @@ func TestLoad(t *testing.T) {
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digest": map[string]any{
|
||||
"schedule": "@daily",
|
||||
"view": "ai",
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "@daily",
|
||||
"host": "http://localhost:8080",
|
||||
"view": "ai",
|
||||
}},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "missing digests",
|
||||
config: map[string]any{
|
||||
"miniflux": map[string]any{
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digests": []map[string]any{},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "duplicate digest slugs",
|
||||
config: map[string]any{
|
||||
"miniflux": map[string]any{
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digests": []map[string]any{
|
||||
{
|
||||
"title": "My Digest",
|
||||
"schedule": "@daily",
|
||||
"host": "http://localhost:8080",
|
||||
"view": "category",
|
||||
},
|
||||
{
|
||||
"title": "My-Digest",
|
||||
"schedule": "@daily",
|
||||
"host": "http://localhost:8080",
|
||||
"view": "category",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid regex pattern",
|
||||
config: map[string]any{
|
||||
"miniflux": map[string]any{
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "@daily",
|
||||
"host": "http://localhost:8080",
|
||||
"view": "category",
|
||||
"filters": map[string]any{
|
||||
"feed_title_patterns": []string{"(unclosed parenthesis"},
|
||||
},
|
||||
}},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid category missing title",
|
||||
config: map[string]any{
|
||||
"miniflux": map[string]any{
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "@daily",
|
||||
"host": "http://localhost:8080",
|
||||
"view": "ai",
|
||||
"categories": []map[string]any{
|
||||
{"description": "No title here"},
|
||||
},
|
||||
}},
|
||||
"ai": map[string]any{
|
||||
"api_key": "dummy-key",
|
||||
},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "unknown filter field",
|
||||
config: map[string]any{
|
||||
"miniflux": map[string]any{
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digests": []map[string]any{{
|
||||
"title": "Daily Digest",
|
||||
"schedule": "@daily",
|
||||
"host": "http://localhost:8080",
|
||||
"view": "category",
|
||||
"filters": map[string]any{
|
||||
"feed_title": []string{"Some Feed"}, // Incorrect field name
|
||||
},
|
||||
}},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid digest title empty slug",
|
||||
config: map[string]any{
|
||||
"miniflux": map[string]any{
|
||||
"host": "miniflux.example.com",
|
||||
"api_token": "test-token",
|
||||
},
|
||||
"digests": []map[string]any{{
|
||||
"title": "!!!",
|
||||
"schedule": "@daily",
|
||||
"host": "http://localhost:8080",
|
||||
"view": "category",
|
||||
}},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
@@ -252,19 +438,12 @@ func TestLoad(t *testing.T) {
|
||||
t.Fatalf("Failed to write dummy config file: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := Load(configPath)
|
||||
_, err = Load(configPath)
|
||||
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Load() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
if !tt.wantErr && tt.name == "default digest.schedule is @weekly" {
|
||||
if cfg.Digest.Schedule != "@weekly" {
|
||||
t.Errorf("Expected digest.schedule to be @weekly, got %s", cfg.Digest.Schedule)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package digest
|
||||
|
||||
import (
|
||||
"miniflux-digest/internal/app/services"
|
||||
"miniflux-digest/internal/config"
|
||||
"miniflux-digest/internal/digest/view"
|
||||
"miniflux-digest/internal/models"
|
||||
)
|
||||
@@ -14,6 +15,6 @@ type digestServiceImpl struct {
|
||||
llmService services.LLMService
|
||||
}
|
||||
|
||||
func (s *digestServiceImpl) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, viewType string, minifluxHost string, digestHost string) *models.OverviewTemplateData {
|
||||
return view.BuildDigestDataForView(entries, icons, viewType, minifluxHost, digestHost, s.llmService)
|
||||
func (s *digestServiceImpl) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, viewType string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData {
|
||||
return view.BuildDigestDataForView(entries, icons, viewType, minifluxHost, digestHost, s.llmService, categories, digestTitle)
|
||||
}
|
||||
@@ -89,7 +89,7 @@ func TestDigestService_BuildDigestData(t *testing.T) {
|
||||
}
|
||||
|
||||
t.Run("view=date", func(t *testing.T) {
|
||||
overviewData := digestService.BuildDigestData(entries, icons, "date", "http://miniflux.test", "")
|
||||
overviewData := digestService.BuildDigestData(entries, icons, "date", "http://miniflux.test", "", nil, "")
|
||||
if overviewData == nil {
|
||||
t.Fatal("overviewData is nil")
|
||||
}
|
||||
@@ -112,7 +112,7 @@ func TestDigestService_BuildDigestData(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("view=category", func(t *testing.T) {
|
||||
overviewData := digestService.BuildDigestData(entries, icons, "category", "http://miniflux.test", "")
|
||||
overviewData := digestService.BuildDigestData(entries, icons, "category", "http://miniflux.test", "", nil, "")
|
||||
if overviewData == nil {
|
||||
t.Fatal("overviewData is nil")
|
||||
}
|
||||
@@ -133,7 +133,7 @@ func TestDigestService_BuildDigestData(t *testing.T) {
|
||||
|
||||
t.Run("view=ai", func(t *testing.T) {
|
||||
aiEntries := createLLMGrouperMockEntries()
|
||||
overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "")
|
||||
overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "", nil, "")
|
||||
if overviewData == nil {
|
||||
t.Fatal("overviewData is nil")
|
||||
}
|
||||
@@ -156,7 +156,7 @@ func TestDigestService_BuildDigestData(t *testing.T) {
|
||||
}
|
||||
digestService := NewDigestService(mockLLM)
|
||||
aiEntries := createLLMGrouperMockEntries()
|
||||
overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "")
|
||||
overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "", nil, "")
|
||||
|
||||
if overviewData == nil {
|
||||
t.Fatal("overviewData is nil")
|
||||
@@ -200,7 +200,7 @@ func TestDigestService_BuildDigestData(t *testing.T) {
|
||||
}
|
||||
digestService := NewDigestService(mockLLM)
|
||||
aiEntries := createLLMGrouperMockEntries()
|
||||
overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "")
|
||||
overviewData := digestService.BuildDigestData(aiEntries, icons, "ai", "http://miniflux.test", "", nil, "")
|
||||
|
||||
if overviewData == nil {
|
||||
t.Fatal("overviewData is nil")
|
||||
|
||||
+83
-29
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"miniflux-digest/internal/app/services"
|
||||
"miniflux-digest/internal/config"
|
||||
"miniflux-digest/internal/llm"
|
||||
"miniflux-digest/internal/models"
|
||||
"miniflux-digest/internal/utils"
|
||||
@@ -33,7 +34,7 @@ type llmEntry struct {
|
||||
|
||||
|
||||
|
||||
func GroupAIEntries(ctx context.Context, entries []*models.Entry, llmService services.LLMService) (map[string][]*models.Entry, map[int64]bool, map[string][]*models.Entry) {
|
||||
func GroupAIEntries(ctx context.Context, entries []*models.Entry, llmService services.LLMService, categories []config.ConfigCategory) (map[string][]*models.Entry, map[int64]bool, map[string][]*models.Entry) {
|
||||
const chunkSize = 200
|
||||
|
||||
// Prepare for processing
|
||||
@@ -42,7 +43,14 @@ func GroupAIEntries(ctx context.Context, entries []*models.Entry, llmService ser
|
||||
entryMap[entry.ID] = entry
|
||||
}
|
||||
|
||||
promptTemplate, err := template.New("grouping").Parse(initialGroupingPrompt)
|
||||
var promptTemplateStr string
|
||||
if len(categories) > 0 {
|
||||
promptTemplateStr = strictGroupingPrompt
|
||||
} else {
|
||||
promptTemplateStr = openGroupingPrompt
|
||||
}
|
||||
|
||||
promptTemplate, err := template.New("grouping").Parse(promptTemplateStr)
|
||||
if err != nil {
|
||||
failedChunks := make(map[string][]*models.Entry)
|
||||
failedChunks["Prompt template parsing failed"] = entries
|
||||
@@ -51,8 +59,9 @@ func GroupAIEntries(ctx context.Context, entries []*models.Entry, llmService ser
|
||||
|
||||
workerConfig := llmWorkerConfig{
|
||||
promptTemplate: promptTemplate,
|
||||
promptData: categories,
|
||||
responseSchema: InitialGroupingResponseSchema,
|
||||
prepareEntries: func(entries []*models.Entry) interface{} {
|
||||
prepareEntries: func(entries []*models.Entry) any {
|
||||
return prepareLLMEntries(entries)
|
||||
},
|
||||
}
|
||||
@@ -171,7 +180,7 @@ func getSummaryForGroup(ctx context.Context, groupTitle string, entries []*model
|
||||
llmResponseWorker := createLLMWorker[SummaryResponse](ctx, llmService, llmWorkerConfig{
|
||||
promptFormat: summaryPrompt,
|
||||
responseSchema: SummaryResponseSchema,
|
||||
prepareEntries: func(entries []*models.Entry) interface{} {
|
||||
prepareEntries: func(entries []*models.Entry) any {
|
||||
return prepareLLMEntries(entries)
|
||||
},
|
||||
}, groupTitle)
|
||||
@@ -220,7 +229,7 @@ func getSubGroupsForGroup(ctx context.Context, groupTitle string, entries []*mod
|
||||
subGroupWorker := createLLMWorker[*SubGroupingResponse](ctx, llmService, llmWorkerConfig{
|
||||
promptFormat: subGroupingPrompt,
|
||||
responseSchema: SubGroupingResponseSchema,
|
||||
prepareEntries: func(entries []*models.Entry) interface{} {
|
||||
prepareEntries: func(entries []*models.Entry) any {
|
||||
return prepareLLMEntries(entries)
|
||||
},
|
||||
}, groupTitle)
|
||||
@@ -389,13 +398,20 @@ func prepareLLMEntries(entries []*models.Entry) []llmEntry {
|
||||
return llmEntries
|
||||
}
|
||||
|
||||
func BuildDigestDataByAI(entries []*models.Entry, ctx context.Context, llmService services.LLMService) []*models.PrimaryGroupDigestData {
|
||||
rawGroups, groupedEntryIDs, failedChunks, fallbackGroups := initialAIGGrouping(ctx, entries, llmService)
|
||||
func BuildDigestDataByAI(entries []*models.Entry, ctx context.Context, llmService services.LLMService, categories []config.ConfigCategory) []*models.PrimaryGroupDigestData {
|
||||
rawGroups, groupedEntryIDs, failedChunks, fallbackGroups := initialAIGGrouping(ctx, entries, llmService, categories)
|
||||
if fallbackGroups != nil {
|
||||
return fallbackGroups
|
||||
}
|
||||
|
||||
highLevelGroups := consolidateGroups(ctx, rawGroups, llmService)
|
||||
var highLevelGroups []*models.PrimaryGroupDigestData
|
||||
if len(categories) == 0 {
|
||||
// Auto Mode: Consolidate the LLM's open-ended groups
|
||||
highLevelGroups = consolidateGroups(ctx, rawGroups, llmService)
|
||||
} else {
|
||||
// User Mode: Use raw groups directly (strict adherence)
|
||||
highLevelGroups = convertRawGroupsToDigestData(rawGroups)
|
||||
}
|
||||
|
||||
highLevelGroups = handleFailedAndUngroupedEntries(highLevelGroups, entries, groupedEntryIDs, failedChunks)
|
||||
|
||||
@@ -406,8 +422,8 @@ func BuildDigestDataByAI(entries []*models.Entry, ctx context.Context, llmServic
|
||||
return allPrimaryGroups
|
||||
}
|
||||
|
||||
func initialAIGGrouping(ctx context.Context, entries []*models.Entry, llmService services.LLMService) (map[string][]*models.Entry, map[int64]bool, map[string][]*models.Entry, []*models.PrimaryGroupDigestData) {
|
||||
rawGroups, groupedEntryIDs, failedChunks := GroupAIEntries(ctx, entries, llmService)
|
||||
func initialAIGGrouping(ctx context.Context, entries []*models.Entry, llmService services.LLMService, categories []config.ConfigCategory) (map[string][]*models.Entry, map[int64]bool, map[string][]*models.Entry, []*models.PrimaryGroupDigestData) {
|
||||
rawGroups, groupedEntryIDs, failedChunks := GroupAIEntries(ctx, entries, llmService, categories)
|
||||
|
||||
// If all chunks failed and we have no raw groups, fall back to category grouping.
|
||||
if len(rawGroups) == 0 && len(failedChunks) > 0 {
|
||||
@@ -466,23 +482,60 @@ func handleFailedAndUngroupedEntries(highLevelGroups []*models.PrimaryGroupDiges
|
||||
}
|
||||
|
||||
if len(otherUngroupedEntries) > 0 {
|
||||
highLevelGroups = append(highLevelGroups, &models.PrimaryGroupDigestData{
|
||||
ID: int64(len(highLevelGroups) + 1),
|
||||
Title: "Uncategorized",
|
||||
Slug: "uncategorized",
|
||||
SubGroups: []*models.EntryGroup{
|
||||
{
|
||||
Title: "Uncategorized",
|
||||
Entries: otherUngroupedEntries,
|
||||
Slug: "uncategorized",
|
||||
TotalEntries: len(otherUngroupedEntries),
|
||||
TotalFeeds: getUniqueFeedIDs(otherUngroupedEntries),
|
||||
var merged bool
|
||||
for _, group := range highLevelGroups {
|
||||
if group.Title == "Uncategorized" {
|
||||
merged = true
|
||||
// Find or create the "Uncategorized" subgroup
|
||||
var subGroup *models.EntryGroup
|
||||
for _, sg := range group.SubGroups {
|
||||
if sg.Title == "Uncategorized" {
|
||||
subGroup = sg
|
||||
break
|
||||
}
|
||||
}
|
||||
if subGroup == nil {
|
||||
subGroup = &models.EntryGroup{
|
||||
Title: "Uncategorized",
|
||||
Slug: "uncategorized",
|
||||
}
|
||||
group.SubGroups = append(group.SubGroups, subGroup)
|
||||
}
|
||||
|
||||
subGroup.Entries = append(subGroup.Entries, otherUngroupedEntries...)
|
||||
subGroup.TotalEntries = len(subGroup.Entries)
|
||||
subGroup.TotalFeeds = getUniqueFeedIDs(subGroup.Entries)
|
||||
|
||||
// Update primary group totals
|
||||
var allEntries []*models.Entry
|
||||
for _, sg := range group.SubGroups {
|
||||
allEntries = append(allEntries, sg.Entries...)
|
||||
}
|
||||
group.TotalEntries = len(allEntries)
|
||||
group.TotalFeeds = getUniqueFeedIDs(allEntries)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !merged {
|
||||
highLevelGroups = append(highLevelGroups, &models.PrimaryGroupDigestData{
|
||||
ID: int64(len(highLevelGroups) + 1),
|
||||
Title: "Uncategorized",
|
||||
Slug: "uncategorized",
|
||||
SubGroups: []*models.EntryGroup{
|
||||
{
|
||||
Title: "Uncategorized",
|
||||
Entries: otherUngroupedEntries,
|
||||
Slug: "uncategorized",
|
||||
TotalEntries: len(otherUngroupedEntries),
|
||||
TotalFeeds: getUniqueFeedIDs(otherUngroupedEntries),
|
||||
},
|
||||
},
|
||||
},
|
||||
Summary: "",
|
||||
TotalEntries: len(otherUngroupedEntries),
|
||||
TotalFeeds: getUniqueFeedIDs(otherUngroupedEntries),
|
||||
})
|
||||
Summary: "",
|
||||
TotalEntries: len(otherUngroupedEntries),
|
||||
TotalFeeds: getUniqueFeedIDs(otherUngroupedEntries),
|
||||
})
|
||||
}
|
||||
}
|
||||
return highLevelGroups
|
||||
}
|
||||
@@ -544,9 +597,10 @@ func sortPrimaryGroups(allPrimaryGroups []*models.PrimaryGroupDigestData) {
|
||||
// llmWorkerConfig holds configuration for creating an LLM worker function.
|
||||
type llmWorkerConfig struct {
|
||||
promptTemplate *template.Template
|
||||
promptData any // Data to pass to template execution
|
||||
promptFormat string // For prompts that use Sprintf
|
||||
responseSchema *genai.Schema
|
||||
prepareEntries func([]*models.Entry) interface{} // Function to prepare entries for LLM
|
||||
prepareEntries func([]*models.Entry) any // Function to prepare entries for LLM
|
||||
}
|
||||
|
||||
// createLLMWorker creates a worker function for llm.ProcessInChunks.
|
||||
@@ -573,7 +627,7 @@ func createLLMWorker[R any](
|
||||
// 2. Construct the prompt
|
||||
var prompt bytes.Buffer
|
||||
if config.promptTemplate != nil {
|
||||
if err := config.promptTemplate.Execute(&prompt, nil); err != nil {
|
||||
if err := config.promptTemplate.Execute(&prompt, config.promptData); err != nil {
|
||||
return zero, fmt.Errorf("failed to execute prompt template: %w", err)
|
||||
}
|
||||
} else if config.promptFormat != "" {
|
||||
@@ -591,4 +645,4 @@ func createLLMWorker[R any](
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@ package view
|
||||
|
||||
import "google.golang.org/genai"
|
||||
|
||||
const initialGroupingPrompt = `You are creating high-level primary groups for a digest.
|
||||
const openGroupingPrompt = `You are creating high-level primary groups for a digest.
|
||||
|
||||
**Your Task:**
|
||||
Group the entries below into broad, high-level primary groups.
|
||||
@@ -20,6 +20,24 @@ Below is the list of entries:
|
||||
----------------------------
|
||||
`
|
||||
|
||||
const strictGroupingPrompt = `You are organizing a digest into specific categories.
|
||||
|
||||
**Your Task:**
|
||||
Sort the entries below into the following categories:
|
||||
{{range .}}
|
||||
- **{{.Title}}**: {{.Description}}
|
||||
{{end}}
|
||||
|
||||
**Rules:**
|
||||
- You MUST use ONLY the categories listed above.
|
||||
- Do NOT create new categories.
|
||||
- If an entry does not fit well into any category, do NOT include it in a group (it will be handled as uncategorized later).
|
||||
- An entry can only belong to one group.
|
||||
|
||||
Below is the list of entries:
|
||||
----------------------------
|
||||
`
|
||||
|
||||
const consolidationPrompt = `You are an expert content curator responsible for organizing a digest. You have been given a list of proposed primary group titles that were automatically generated. Your task is to ensure that this list is broad and high level and does not contain potentially overlapping topics. Each topic will later on be further subdivided into sub-groups, so if two topics are better served as sub-topics, they should be merged into one high level topic. Our goal is to have no more than 10-12 primary groups.
|
||||
|
||||
**Your Goal:**
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"miniflux-digest/internal/config"
|
||||
"miniflux-digest/internal/models"
|
||||
"miniflux-digest/internal/testutil"
|
||||
"strings"
|
||||
@@ -72,7 +73,7 @@ func TestGroupAIEntries(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
rawGroups, groupedIDs, failedChunks := GroupAIEntries(context.Background(), entries, llmService)
|
||||
rawGroups, groupedIDs, failedChunks := GroupAIEntries(context.Background(), entries, llmService, nil)
|
||||
assert.Empty(t, failedChunks)
|
||||
assert.NotNil(t, rawGroups)
|
||||
assert.Len(t, rawGroups, 2, "Expected 2 grouped")
|
||||
@@ -96,7 +97,7 @@ func TestGroupAIEntries(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
_, _, failedChunks := GroupAIEntries(context.Background(), entries, llmService)
|
||||
_, _, failedChunks := GroupAIEntries(context.Background(), entries, llmService, nil)
|
||||
assert.NotEmpty(t, failedChunks)
|
||||
})
|
||||
|
||||
@@ -107,7 +108,7 @@ func TestGroupAIEntries(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
_, _, failedChunks := GroupAIEntries(context.Background(), entries, llmService)
|
||||
_, _, failedChunks := GroupAIEntries(context.Background(), entries, llmService, nil)
|
||||
assert.NotEmpty(t, failedChunks)
|
||||
})
|
||||
|
||||
@@ -149,7 +150,7 @@ func TestGroupAIEntries(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
rawGroups, groupedIDs, failedChunks := GroupAIEntries(context.Background(), largeEntries, llmService)
|
||||
rawGroups, groupedIDs, failedChunks := GroupAIEntries(context.Background(), largeEntries, llmService, nil)
|
||||
assert.Empty(t, failedChunks)
|
||||
assert.Equal(t, 2, callCount, "Expected LLM to be called 2 times for 205 entries with chunk size 200")
|
||||
|
||||
@@ -450,7 +451,7 @@ func TestBuildDigestDataByAI(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
groups := BuildDigestDataByAI(entries, context.Background(), llmService)
|
||||
groups := BuildDigestDataByAI(entries, context.Background(), llmService, nil)
|
||||
assert.Len(t, groups, 2, "Expected 1 consolidated group + 1 uncategorized")
|
||||
|
||||
techGroup := testutil.FindPrimaryGroup(groups, "Tech")
|
||||
@@ -466,6 +467,128 @@ func TestBuildDigestDataByAI(t *testing.T) {
|
||||
assert.Equal(t, int64(4), uncategorized.SubGroups[0].Entries[0].ID)
|
||||
})
|
||||
|
||||
t.Run("strict user mode - no consolidation", func(t *testing.T) {
|
||||
userCategories := []config.ConfigCategory{
|
||||
{Title: "Strict Cat A"},
|
||||
{Title: "Strict Cat B"},
|
||||
}
|
||||
|
||||
// Entries 1 & 2 assigned to user categories
|
||||
groupingResp := InitialGroupingResponse{
|
||||
Groups: []struct {
|
||||
Title string `json:"title"`
|
||||
EntryIDs []int64 `json:"entry_ids"`
|
||||
}{
|
||||
{Title: "Strict Cat A", EntryIDs: []int64{1}},
|
||||
{Title: "Strict Cat B", EntryIDs: []int64{2}},
|
||||
},
|
||||
}
|
||||
groupingJSON, _ := json.Marshal(groupingResp)
|
||||
|
||||
llmService := &mockLLMService{
|
||||
GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) {
|
||||
if schema == ConsolidationResponseSchema {
|
||||
t.Fatal("Consolidation should NOT be called in strict user mode")
|
||||
}
|
||||
if schema == InitialGroupingResponseSchema {
|
||||
// Verify that strict prompt is used (implicitly via context of strict mode logic,
|
||||
// but strict prompt text check would require inspecting 'prompt' arg which we can do)
|
||||
assert.Contains(t, prompt, "Sort the entries below into the following categories")
|
||||
return groupingJSON, nil
|
||||
}
|
||||
|
||||
if schema == SummaryResponseSchema {
|
||||
return json.Marshal(SummaryResponse{Summary: "Summary"})
|
||||
}
|
||||
if schema == SubGroupingResponseSchema {
|
||||
// Return dummy subgroup
|
||||
return json.Marshal(SubGroupingResponse{SubGroups: []struct {
|
||||
Title string `json:"title"`
|
||||
EntryIDs []int64 `json:"entry_ids"`
|
||||
}{}})
|
||||
}
|
||||
return nil, nil
|
||||
},
|
||||
}
|
||||
|
||||
groups := BuildDigestDataByAI(entries, context.Background(), llmService, userCategories)
|
||||
|
||||
// Expected: Strict Cat A, Strict Cat B, Uncategorized (for 3 & 4)
|
||||
assert.Len(t, groups, 3)
|
||||
|
||||
catA := testutil.FindPrimaryGroup(groups, "Strict Cat A")
|
||||
assert.NotNil(t, catA)
|
||||
|
||||
catB := testutil.FindPrimaryGroup(groups, "Strict Cat B")
|
||||
assert.NotNil(t, catB)
|
||||
|
||||
uncat := testutil.FindPrimaryGroup(groups, "Uncategorized")
|
||||
assert.NotNil(t, uncat)
|
||||
assert.Equal(t, 2, uncat.TotalEntries) // Entries 3 and 4
|
||||
})
|
||||
|
||||
t.Run("merges ungrouped into existing uncategorized", func(t *testing.T) {
|
||||
// Entry 1 & 2 -> "Uncategorized" via LLM
|
||||
// Entry 3 -> Ungrouped (should be merged)
|
||||
entries := []*models.Entry{
|
||||
{ID: 1, Title: "E1"},
|
||||
{ID: 2, Title: "E2"},
|
||||
{ID: 3, Title: "E3"},
|
||||
}
|
||||
|
||||
groupingResp := InitialGroupingResponse{
|
||||
Groups: []struct {
|
||||
Title string `json:"title"`
|
||||
EntryIDs []int64 `json:"entry_ids"`
|
||||
}{
|
||||
{Title: "Uncategorized", EntryIDs: []int64{1, 2}},
|
||||
},
|
||||
}
|
||||
groupingJSON, _ := json.Marshal(groupingResp)
|
||||
|
||||
// Mock Summary/Sub-grouping for the "Uncategorized" group
|
||||
summaryResp := SummaryResponse{Summary: "Uncategorized stuff"}
|
||||
summaryJSON, _ := json.Marshal(summaryResp)
|
||||
subGroupingResp := SubGroupingResponse{
|
||||
SubGroups: []struct {
|
||||
Title string `json:"title"`
|
||||
EntryIDs []int64 `json:"entry_ids"`
|
||||
}{{Title: "Uncategorized", EntryIDs: []int64{1, 2}}},
|
||||
}
|
||||
subGroupingJSON, _ := json.Marshal(subGroupingResp)
|
||||
|
||||
llmService := &mockLLMService{
|
||||
GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) {
|
||||
switch schema {
|
||||
case InitialGroupingResponseSchema:
|
||||
return groupingJSON, nil
|
||||
case ConsolidationResponseSchema:
|
||||
// Should not be called if < 10 groups, but handle just in case
|
||||
return nil, nil
|
||||
case SummaryResponseSchema:
|
||||
return summaryJSON, nil
|
||||
case SubGroupingResponseSchema:
|
||||
return subGroupingJSON, nil
|
||||
default:
|
||||
return nil, errors.New("unexpected schema")
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
groups := BuildDigestDataByAI(entries, context.Background(), llmService, nil)
|
||||
|
||||
require.Len(t, groups, 1, "Expected exactly 1 group (merged Uncategorized)")
|
||||
uncatGroup := groups[0]
|
||||
assert.Equal(t, "Uncategorized", uncatGroup.Title)
|
||||
|
||||
// Check that it contains all 3 entries (2 from LLM + 1 ungrouped)
|
||||
var totalEntries int
|
||||
for _, sg := range uncatGroup.SubGroups {
|
||||
totalEntries += len(sg.Entries)
|
||||
}
|
||||
assert.Equal(t, 3, totalEntries, "Should contain all 3 entries")
|
||||
})
|
||||
|
||||
t.Run("fallback to category grouping", func(t *testing.T) {
|
||||
llmService := &mockLLMService{
|
||||
GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) {
|
||||
@@ -473,7 +596,7 @@ func TestBuildDigestDataByAI(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
groups := BuildDigestDataByAI(entries, context.Background(), llmService)
|
||||
groups := BuildDigestDataByAI(entries, context.Background(), llmService, nil)
|
||||
assert.Len(t, groups, 3, "Should fall back to 3 categories")
|
||||
})
|
||||
|
||||
|
||||
@@ -3,12 +3,14 @@ package view
|
||||
import (
|
||||
"context"
|
||||
"miniflux-digest/internal/app/services"
|
||||
"miniflux-digest/internal/config"
|
||||
"miniflux-digest/internal/models"
|
||||
"miniflux-digest/internal/utils"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
func BuildDigestDataForView(entries []*models.Entry, icons map[int64]*models.FeedIcon, viewType string, minifluxHost string, digestHost string, llmService services.LLMService) *models.OverviewTemplateData {
|
||||
func BuildDigestDataForView(entries []*models.Entry, icons map[int64]*models.FeedIcon, viewType string, minifluxHost string, digestHost string, llmService services.LLMService, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData {
|
||||
iconsSlice := make([]*models.FeedIcon, 0, len(icons))
|
||||
for _, icon := range icons {
|
||||
iconsSlice = append(iconsSlice, icon)
|
||||
@@ -23,7 +25,7 @@ func BuildDigestDataForView(entries []*models.Entry, icons map[int64]*models.Fee
|
||||
case "category", "default":
|
||||
resultPrimaryGroups = BuildDigestDataByCategory(entries)
|
||||
case "ai":
|
||||
resultPrimaryGroups = BuildDigestDataByAI(entries, context.Background(), llmService)
|
||||
resultPrimaryGroups = BuildDigestDataByAI(entries, context.Background(), llmService, categories)
|
||||
}
|
||||
|
||||
if viewType == "date" || viewType == "category" {
|
||||
@@ -60,6 +62,8 @@ func BuildDigestDataForView(entries []*models.Entry, icons map[int64]*models.Fee
|
||||
}
|
||||
|
||||
return &models.OverviewTemplateData{
|
||||
DigestTitle: digestTitle,
|
||||
DigestSlug: utils.Slugify(digestTitle),
|
||||
Entries: entries,
|
||||
GeneratedDate: time.Now(),
|
||||
FeedIcons: iconsSlice,
|
||||
|
||||
@@ -17,29 +17,29 @@ type EmailServiceImpl struct{
|
||||
EmailTemplate *template.Template
|
||||
}
|
||||
|
||||
func (s *EmailServiceImpl) Send(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error {
|
||||
func (s *EmailServiceImpl) Send(smtpConfig config.ConfigSmtp, digestConfig config.ConfigDigest, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error {
|
||||
message := mail.NewMsg()
|
||||
client, err := mail.NewClient(
|
||||
cfg.Smtp.Host,
|
||||
smtpConfig.Host,
|
||||
mail.WithSMTPAuth(mail.SMTPAuthAutoDiscover),
|
||||
mail.WithPort(cfg.Smtp.Port),
|
||||
mail.WithUsername(cfg.Smtp.User),
|
||||
mail.WithPassword(cfg.Smtp.Password))
|
||||
mail.WithPort(smtpConfig.Port),
|
||||
mail.WithUsername(smtpConfig.User),
|
||||
mail.WithPassword(smtpConfig.Password))
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := message.From(cfg.Digest.Email.From); err != nil {
|
||||
if err := message.From(digestConfig.Email.From); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := message.To(cfg.Digest.Email.To); err != nil {
|
||||
if err := message.To(digestConfig.Email.To); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
subject := fmt.Sprintf("Your Miniflux Digest for %s", data.GeneratedDate.Format("January 2, 2006"))
|
||||
overviewURL := fmt.Sprintf("%s/archive/%s/index.html", cfg.Digest.Host, data.GeneratedDate.Format("2006-01-02"))
|
||||
overviewURL := fmt.Sprintf("%s/archive/%s/index.html", digestConfig.Host, data.GeneratedDate.Format("2006-01-02"))
|
||||
|
||||
message.Subject(subject)
|
||||
|
||||
@@ -60,4 +60,4 @@ func (s *EmailServiceImpl) Send(cfg *config.Config, overviewFile *os.File, group
|
||||
}
|
||||
|
||||
return client.DialAndSend(message)
|
||||
}
|
||||
}
|
||||
@@ -11,20 +11,18 @@ import (
|
||||
)
|
||||
|
||||
func TestSend(t *testing.T) {
|
||||
cfg := &config.Config{
|
||||
Smtp: config.ConfigSmtp{
|
||||
Host: "localhost",
|
||||
Port: 1025,
|
||||
User: "test-user",
|
||||
Password: "test-password",
|
||||
},
|
||||
Digest: config.ConfigDigest{
|
||||
Email: config.ConfigDigestEmail{
|
||||
To: "to@example.com",
|
||||
From: "from@example.com",
|
||||
},
|
||||
Host: "https://example.com",
|
||||
smtpConfig := config.ConfigSmtp{
|
||||
Host: "localhost",
|
||||
Port: 1025,
|
||||
User: "test-user",
|
||||
Password: "test-password",
|
||||
}
|
||||
digestConfig := config.ConfigDigest{
|
||||
Email: config.ConfigDigestEmail{
|
||||
To: "to@example.com",
|
||||
From: "from@example.com",
|
||||
},
|
||||
Host: "https://example.com",
|
||||
}
|
||||
|
||||
tmpFile, err := os.CreateTemp("", "test.html")
|
||||
@@ -70,7 +68,7 @@ func TestSend(t *testing.T) {
|
||||
emailService := &EmailServiceImpl{
|
||||
EmailTemplate: templates.EmailTemplate,
|
||||
}
|
||||
err = emailService.Send(cfg, file, []*os.File{}, &data)
|
||||
err = emailService.Send(smtpConfig, digestConfig, file, []*os.File{}, &data)
|
||||
if err != nil {
|
||||
// We expect an error because we are not running a real SMTP server.
|
||||
// The important part is that the function attempts to connect.
|
||||
@@ -95,4 +93,4 @@ func TestTextTemplateData(t *testing.T) {
|
||||
if textData.URL != url {
|
||||
t.Errorf("Expected URL to be %s, got %s", url, textData.URL)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package filter
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"miniflux-digest/internal/config"
|
||||
"miniflux-digest/internal/models"
|
||||
)
|
||||
|
||||
type Matcher struct {
|
||||
// Cache regex patterns to avoid recompiling
|
||||
feedTitleRegex []*regexp.Regexp
|
||||
categoryTitleRegex []*regexp.Regexp
|
||||
siteURLRegex []*regexp.Regexp
|
||||
entryURLRegex []*regexp.Regexp
|
||||
}
|
||||
|
||||
func NewMatcher(cfg config.ConfigFilters) (*Matcher, error) {
|
||||
m := &Matcher{}
|
||||
var err error
|
||||
|
||||
m.feedTitleRegex, err = compileRegexes(cfg.FeedTitlePatterns)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
m.categoryTitleRegex, err = compileRegexes(cfg.CategoryTitlePatterns)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
m.siteURLRegex, err = compileRegexes(cfg.FeedURLPatterns)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
m.entryURLRegex, err = compileRegexes(cfg.EntryURLPatterns)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func compileRegexes(patterns []string) ([]*regexp.Regexp, error) {
|
||||
regexes := make([]*regexp.Regexp, 0, len(patterns))
|
||||
for _, p := range patterns {
|
||||
r, err := regexp.Compile(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
regexes = append(regexes, r)
|
||||
}
|
||||
return regexes, nil
|
||||
}
|
||||
|
||||
// Matches returns true if the entry matches the filter criteria.
|
||||
// Logic:
|
||||
// 1. If no filters are defined, it's a match (default allow).
|
||||
// 2. OR logic within a specific field (e.g., FeedTitles: ["A", "B"] -> match if A OR B).
|
||||
// 3. AND logic between fields (e.g., FeedTitle: "A" AND SiteURL: "B" -> match only if both are true).
|
||||
func (m *Matcher) Matches(entry *models.Entry, cfg config.ConfigFilters) bool {
|
||||
// If all filters are empty, return true (default allow if "filters" block is missing/empty)
|
||||
// However, usually we want at least one filter. But per spec "Stack-Based Priority",
|
||||
// a digest might be a "catch-all" at the end.
|
||||
if isEmpty(cfg) {
|
||||
return true
|
||||
}
|
||||
|
||||
if !matchString(entry.FeedTitle, cfg.FeedTitles, m.feedTitleRegex, true) {
|
||||
return false
|
||||
}
|
||||
|
||||
if !matchString(entry.GroupTitle, cfg.CategoryTitles, m.categoryTitleRegex, true) {
|
||||
return false
|
||||
}
|
||||
|
||||
if !matchString(entry.SiteURL, cfg.FeedURLs, m.siteURLRegex, false) {
|
||||
return false
|
||||
}
|
||||
|
||||
// EntryURL corresponds to entry.URL
|
||||
if !matchString(entry.URL, cfg.EntryURLs, m.entryURLRegex, false) {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// matchString checks if value matches any of the exact strings or regex patterns.
|
||||
// exactMatch: true for Full Match (Titles), false for Prefix Match (URLs).
|
||||
func matchString(value string, exacts []string, patterns []*regexp.Regexp, exactMatch bool) bool {
|
||||
// If no filters for this specific field are set, we "pass" this field.
|
||||
// We only check for a match if there IS something to match against.
|
||||
if len(exacts) == 0 && len(patterns) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, e := range exacts {
|
||||
if exactMatch {
|
||||
if value == e {
|
||||
return true
|
||||
}
|
||||
} else {
|
||||
if strings.HasPrefix(value, e) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, r := range patterns {
|
||||
if r.MatchString(value) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func isEmpty(cfg config.ConfigFilters) bool {
|
||||
return len(cfg.FeedTitles) == 0 &&
|
||||
len(cfg.CategoryTitles) == 0 &&
|
||||
len(cfg.FeedURLs) == 0 &&
|
||||
len(cfg.EntryURLs) == 0 &&
|
||||
len(cfg.FeedTitlePatterns) == 0 &&
|
||||
len(cfg.CategoryTitlePatterns) == 0 &&
|
||||
len(cfg.FeedURLPatterns) == 0 &&
|
||||
len(cfg.EntryURLPatterns) == 0
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package filter
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"miniflux-digest/internal/config"
|
||||
"miniflux-digest/internal/models"
|
||||
)
|
||||
|
||||
func TestMatcher_Matches(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
entry *models.Entry
|
||||
filters config.ConfigFilters
|
||||
wantMatch bool
|
||||
}{
|
||||
{
|
||||
name: "Match exact feed title",
|
||||
entry: &models.Entry{
|
||||
FeedTitle: "Tech News", GroupTitle: "Some Category", SiteURL: "http://example.com",
|
||||
URL: "http://example.com/post/1",
|
||||
},
|
||||
filters: config.ConfigFilters{
|
||||
FeedTitles: []string{"Tech News"},
|
||||
},
|
||||
wantMatch: true,
|
||||
},
|
||||
{
|
||||
name: "No match exact feed title",
|
||||
entry: &models.Entry{
|
||||
FeedTitle: "Cooking", GroupTitle: "Some Category", SiteURL: "http://example.com",
|
||||
URL: "http://example.com/post/1",
|
||||
},
|
||||
filters: config.ConfigFilters{
|
||||
FeedTitles: []string{"Tech News"},
|
||||
},
|
||||
wantMatch: false,
|
||||
},
|
||||
{
|
||||
name: "Match prefix site url",
|
||||
entry: &models.Entry{
|
||||
FeedTitle: "A", GroupTitle: "Some Category", SiteURL: "https://github.com/user/repo",
|
||||
URL: "http://example.com/post/1",
|
||||
},
|
||||
filters: config.ConfigFilters{
|
||||
FeedURLs: []string{"https://github.com"},
|
||||
},
|
||||
wantMatch: true,
|
||||
},
|
||||
{
|
||||
name: "No match prefix site url",
|
||||
entry: &models.Entry{
|
||||
FeedTitle: "A", GroupTitle: "Some Category", SiteURL: "https://google.com",
|
||||
URL: "http://example.com/post/1",
|
||||
},
|
||||
filters: config.ConfigFilters{
|
||||
FeedURLs: []string{"https://github.com"},
|
||||
},
|
||||
wantMatch: false,
|
||||
},
|
||||
{
|
||||
name: "Match regex feed title",
|
||||
entry: &models.Entry{
|
||||
FeedTitle: "My Tech Blog", GroupTitle: "Some Category", SiteURL: "http://example.com",
|
||||
URL: "http://example.com/post/1",
|
||||
},
|
||||
filters: config.ConfigFilters{
|
||||
FeedTitlePatterns: []string{".*Tech.*"},
|
||||
},
|
||||
wantMatch: true,
|
||||
},
|
||||
{
|
||||
name: "Mixed AND logic (Feed Title AND Category)",
|
||||
entry: &models.Entry{
|
||||
FeedTitle: "Tech News", GroupTitle: "Programming", SiteURL: "http://example.com",
|
||||
URL: "http://example.com/post/1",
|
||||
},
|
||||
filters: config.ConfigFilters{
|
||||
FeedTitles: []string{"Tech News"},
|
||||
CategoryTitles: []string{"Programming"},
|
||||
},
|
||||
wantMatch: true,
|
||||
},
|
||||
{
|
||||
name: "Mixed AND logic fail (Feed Title match, Category mismatch)",
|
||||
entry: &models.Entry{
|
||||
FeedTitle: "Tech News", GroupTitle: "Cooking", SiteURL: "http://example.com",
|
||||
URL: "http://example.com/post/1",
|
||||
},
|
||||
filters: config.ConfigFilters{
|
||||
FeedTitles: []string{"Tech News"},
|
||||
CategoryTitles: []string{"Programming"},
|
||||
},
|
||||
wantMatch: false,
|
||||
},
|
||||
{
|
||||
name: "Empty filters matches all",
|
||||
entry: &models.Entry{
|
||||
FeedTitle: "Any", GroupTitle: "Any", SiteURL: "Any",
|
||||
URL: "Any",
|
||||
},
|
||||
filters: config.ConfigFilters{},
|
||||
wantMatch: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
m, err := NewMatcher(tt.filters)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create matcher: %v", err)
|
||||
}
|
||||
if got := m.Matches(tt.entry, tt.filters); got != tt.wantMatch {
|
||||
t.Errorf("Matches() = %v, want %v", got, tt.wantMatch)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"miniflux-digest/internal/config"
|
||||
"miniflux-digest/internal/filter"
|
||||
"miniflux-digest/internal/models"
|
||||
)
|
||||
|
||||
type DigestManager struct {
|
||||
digests []config.ConfigDigest
|
||||
matchers []*filter.Matcher
|
||||
}
|
||||
|
||||
func NewDigestManager(digests []config.ConfigDigest) (*DigestManager, error) {
|
||||
matchers := make([]*filter.Matcher, len(digests))
|
||||
for i, d := range digests {
|
||||
m, err := filter.NewMatcher(d.Filters)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
matchers[i] = m
|
||||
}
|
||||
|
||||
return &DigestManager{
|
||||
digests: digests,
|
||||
matchers: matchers,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetOwningDigest returns the index of the first digest that matches the entry.
|
||||
// Returns -1 if no digest matches.
|
||||
func (dm *DigestManager) GetOwningDigest(entry *models.Entry) int {
|
||||
for i, m := range dm.matchers {
|
||||
if m.Matches(entry, dm.digests[i].Filters) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"miniflux-digest/internal/config"
|
||||
"miniflux-digest/internal/models"
|
||||
)
|
||||
|
||||
func TestGetOwningDigest(t *testing.T) {
|
||||
digests := []config.ConfigDigest{
|
||||
{
|
||||
Title: "High Priority Tech",
|
||||
Filters: config.ConfigFilters{
|
||||
FeedTitles: []string{"Tech News"},
|
||||
},
|
||||
},
|
||||
{
|
||||
Title: "General News",
|
||||
Filters: config.ConfigFilters{
|
||||
// Empty filters match everything (catch-all)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
dm, err := NewDigestManager(digests)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create DigestManager: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
entry *models.Entry
|
||||
wantIndex int
|
||||
}{
|
||||
{
|
||||
name: "Match first digest",
|
||||
entry: &models.Entry{
|
||||
FeedTitle: "Tech News",
|
||||
SiteURL: "http://technews.com",
|
||||
},
|
||||
wantIndex: 0,
|
||||
},
|
||||
{
|
||||
name: "Match second digest (catch-all)",
|
||||
entry: &models.Entry{
|
||||
FeedTitle: "Cooking Weekly",
|
||||
SiteURL: "http://cooking.com",
|
||||
},
|
||||
wantIndex: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := dm.GetOwningDigest(tt.entry)
|
||||
if got != tt.wantIndex {
|
||||
t.Errorf("GetOwningDigest() = %v, want %v", got, tt.wantIndex)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetOwningDigest_NoMatch(t *testing.T) {
|
||||
digests := []config.ConfigDigest{
|
||||
{
|
||||
Title: "Exclusive Club",
|
||||
Filters: config.ConfigFilters{
|
||||
FeedTitles: []string{"VIP"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
dm, err := NewDigestManager(digests)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create DigestManager: %v", err)
|
||||
}
|
||||
|
||||
entry := &models.Entry{FeedTitle: "Plebeian News"}
|
||||
got := dm.GetOwningDigest(entry)
|
||||
if got != -1 {
|
||||
t.Errorf("GetOwningDigest() = %v, want -1", got)
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ type Entry struct {
|
||||
ID int64
|
||||
Title string
|
||||
URL string
|
||||
SiteURL string
|
||||
Content string
|
||||
FeedID int64
|
||||
FeedTitle string
|
||||
@@ -41,6 +42,8 @@ type FeedIcon struct {
|
||||
}
|
||||
|
||||
type OverviewTemplateData struct {
|
||||
DigestTitle string
|
||||
DigestSlug string
|
||||
Entries []*Entry
|
||||
GeneratedDate time.Time
|
||||
FeedIcons []*FeedIcon
|
||||
|
||||
@@ -5,20 +5,36 @@ import (
|
||||
"os"
|
||||
|
||||
"miniflux-digest/internal/app"
|
||||
"miniflux-digest/internal/manager"
|
||||
"miniflux-digest/internal/models"
|
||||
|
||||
miniflux "miniflux.app/v2/client"
|
||||
)
|
||||
|
||||
func ProcessDigest(application *app.App) (*os.File, []*os.File, *models.OverviewTemplateData, error) {
|
||||
func ProcessDigest(application *app.App, digestIndex int) (*os.File, []*os.File, *models.OverviewTemplateData, error) {
|
||||
digestConfig := application.Config.Digests[digestIndex]
|
||||
|
||||
entries, err := application.MinifluxClientService.GetAllUnreadEntries()
|
||||
if err != nil {
|
||||
log.Printf("Error getting all unread entries: %v", err)
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
icons := make(map[int64]*models.FeedIcon)
|
||||
digestManager, err := manager.NewDigestManager(application.Config.Digests)
|
||||
if err != nil {
|
||||
log.Printf("Error creating digest manager: %v", err)
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
var digestEntries []*models.Entry
|
||||
for _, entry := range entries {
|
||||
if digestManager.GetOwningDigest(entry) == digestIndex {
|
||||
digestEntries = append(digestEntries, entry)
|
||||
}
|
||||
}
|
||||
|
||||
icons := make(map[int64]*models.FeedIcon)
|
||||
for _, entry := range digestEntries {
|
||||
if _, ok := icons[entry.FeedID]; !ok {
|
||||
icon, err := application.MinifluxClientService.FeedIcon(entry.FeedID)
|
||||
if err != nil {
|
||||
@@ -33,22 +49,24 @@ func ProcessDigest(application *app.App) (*os.File, []*os.File, *models.Overview
|
||||
}
|
||||
|
||||
data := application.DigestService.BuildDigestData(
|
||||
entries,
|
||||
digestEntries,
|
||||
icons,
|
||||
application.Config.Digest.View,
|
||||
digestConfig.View,
|
||||
application.Config.Miniflux.Host,
|
||||
application.Config.Digest.Host,
|
||||
digestConfig.Host,
|
||||
digestConfig.Categories,
|
||||
digestConfig.Title,
|
||||
)
|
||||
|
||||
overviewFile, groupedEntryFiles, err := application.ArchiveService.MakeArchiveHTML(data, application.Config.Digest.Compress)
|
||||
overviewFile, groupedEntryFiles, err := application.ArchiveService.MakeArchiveHTML(data, *digestConfig.Compress)
|
||||
if err != nil {
|
||||
log.Printf("Error generating archive HTML: %v", err)
|
||||
log.Printf("Error generating HTML: %v", err)
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
if application.Config.Digest.MarkAsRead {
|
||||
if digestConfig.MarkAsRead {
|
||||
var entryIDs []int64
|
||||
for _, entry := range entries {
|
||||
for _, entry := range digestEntries {
|
||||
entryIDs = append(entryIDs, entry.ID)
|
||||
}
|
||||
|
||||
|
||||
@@ -30,11 +30,11 @@ func (m *mockMinifluxClientService) UpdateEntries(entryIDs []int64, status strin
|
||||
}
|
||||
|
||||
type mockDigestService struct {
|
||||
buildDigestDataFunc func(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string) *models.OverviewTemplateData
|
||||
buildDigestDataFunc func(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData
|
||||
}
|
||||
|
||||
func (m *mockDigestService) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string) *models.OverviewTemplateData {
|
||||
return m.buildDigestDataFunc(entries, icons, view, minifluxHost, digestHost)
|
||||
func (m *mockDigestService) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData {
|
||||
return m.buildDigestDataFunc(entries, icons, view, minifluxHost, digestHost, categories, digestTitle)
|
||||
}
|
||||
|
||||
type mockArchiveService struct {
|
||||
@@ -78,7 +78,7 @@ func TestProcessDigest_Success(t *testing.T) {
|
||||
},
|
||||
}
|
||||
mockDigestService := &mockDigestService{
|
||||
buildDigestDataFunc: func(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string) *models.OverviewTemplateData {
|
||||
buildDigestDataFunc: func(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData {
|
||||
return expectedOverviewData
|
||||
},
|
||||
}
|
||||
@@ -92,10 +92,12 @@ func TestProcessDigest_Success(t *testing.T) {
|
||||
|
||||
mockApp := app.NewApp(
|
||||
app.WithConfig(&config.Config{
|
||||
Digest: config.ConfigDigest{
|
||||
MarkAsRead: true,
|
||||
Compress: true,
|
||||
View: "category",
|
||||
Digests: []config.ConfigDigest{
|
||||
{
|
||||
MarkAsRead: true,
|
||||
Compress: func() *bool { b := true; return &b }(),
|
||||
View: "category",
|
||||
},
|
||||
},
|
||||
Miniflux: config.ConfigMiniflux{Host: "http://miniflux.test"},
|
||||
}),
|
||||
@@ -104,7 +106,7 @@ func TestProcessDigest_Success(t *testing.T) {
|
||||
app.WithArchiveService(mockArchiveService),
|
||||
)
|
||||
|
||||
overviewFile, groupedEntryFiles, data, err := ProcessDigest(mockApp)
|
||||
overviewFile, groupedEntryFiles, data, err := ProcessDigest(mockApp, 0)
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("ProcessDigest returned an unexpected error: %v", err)
|
||||
|
||||
@@ -81,24 +81,28 @@ type feedTemplate struct {
|
||||
GroupTitle string
|
||||
}
|
||||
|
||||
func createHardcodedMockEntries() []*models.Entry {
|
||||
return []*models.Entry{
|
||||
// Category A
|
||||
{ID: 1, Title: "Entry 1", Date: time.Date(2024, 1, 2, 10, 0, 0, 0, time.UTC), FeedID: 100, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A", SiteURL: "www.example.com", URL: "www.example.com/1"},
|
||||
{ID: 3, Title: "Entry 3", Date: time.Date(2024, 1, 2, 11, 0, 0, 0, time.UTC), FeedID: 100, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A", SiteURL: "www.example2.com", URL: "www.example2.com/1"},
|
||||
{ID: 5, Title: "Entry 5", Date: time.Date(2024, 1, 2, 12, 0, 0, 0, time.UTC), FeedID: 300, FeedTitle: "Feed C", GroupID: 1, GroupTitle: "Category A", SiteURL: "www.example3.com", URL: "www.example3.com/1"},
|
||||
{ID: 6, Title: "Entry 6", Date: time.Date(2024, 1, 2, 13, 0, 0, 0, time.UTC), FeedID: 400, FeedTitle: "Feed D", GroupID: 1, GroupTitle: "Category A", SiteURL: "www.example4.com", URL: "www.example4.com/1"},
|
||||
|
||||
// Category B
|
||||
{ID: 2, Title: "Entry 2", Date: time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC), FeedID: 200, FeedTitle: "Feed B", GroupID: 2, GroupTitle: "Category B", SiteURL: "www.example5.com", URL: "www.example.com/1"},
|
||||
{ID: 4, Title: "Entry 4", Content: "Content of entry 4", Date: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC), FeedID: 200, FeedTitle: "Feed B", GroupID: 2, GroupTitle: "Category B", SiteURL: "www.example5.com", URL: "www.example5.com/2"},
|
||||
{ID: 7, Title: "Entry 7", Date: time.Date(2024, 1, 1, 14, 0, 0, 0, time.UTC), FeedID: 500, FeedTitle: "Feed E", GroupID: 2, GroupTitle: "Category B", SiteURL: "www.example5.com", URL: "www.example5.com/3"},
|
||||
|
||||
// Category C
|
||||
{ID: 8, Title: "Entry 8", Date: time.Date(2024, 1, 3, 10, 0, 0, 0, time.UTC), FeedID: 600, FeedTitle: "Feed F", GroupID: 3, GroupTitle: "Category C", SiteURL: "www.example6.com", URL: "www.example7.com/1"},
|
||||
{ID: 9, Title: "Entry 9", Date: time.Date(2024, 1, 3, 11, 0, 0, 0, time.UTC), FeedID: 700, FeedTitle: "Feed G", GroupID: 3, GroupTitle: "Category C", SiteURL: "www.example7.com", URL: "www.example7.com/1"},
|
||||
}
|
||||
}
|
||||
|
||||
func CreateMockEntries(n int) []*models.Entry {
|
||||
if n == 0 {
|
||||
return []*models.Entry{
|
||||
// Category A
|
||||
{ID: 1, Title: "Entry 1", Date: time.Date(2024, 1, 2, 10, 0, 0, 0, time.UTC), FeedID: 100, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A"},
|
||||
{ID: 3, Title: "Entry 3", Date: time.Date(2024, 1, 2, 11, 0, 0, 0, time.UTC), FeedID: 100, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A"},
|
||||
{ID: 5, Title: "Entry 5", Date: time.Date(2024, 1, 2, 12, 0, 0, 0, time.UTC), FeedID: 300, FeedTitle: "Feed C", GroupID: 1, GroupTitle: "Category A"},
|
||||
{ID: 6, Title: "Entry 6", Date: time.Date(2024, 1, 2, 13, 0, 0, 0, time.UTC), FeedID: 400, FeedTitle: "Feed D", GroupID: 1, GroupTitle: "Category A"},
|
||||
|
||||
// Category B
|
||||
{ID: 2, Title: "Entry 2", Date: time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC), FeedID: 200, FeedTitle: "Feed B", GroupID: 2, GroupTitle: "Category B"},
|
||||
{ID: 4, Title: "Entry 4", Content: "Content of entry 4", Date: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC), FeedID: 200, FeedTitle: "Feed B", GroupID: 2, GroupTitle: "Category B"},
|
||||
{ID: 7, Title: "Entry 7", Date: time.Date(2024, 1, 1, 14, 0, 0, 0, time.UTC), FeedID: 500, FeedTitle: "Feed E", GroupID: 2, GroupTitle: "Category B"},
|
||||
|
||||
// Category C
|
||||
{ID: 8, Title: "Entry 8", Date: time.Date(2024, 1, 3, 10, 0, 0, 0, time.UTC), FeedID: 600, FeedTitle: "Feed F", GroupID: 3, GroupTitle: "Category C"},
|
||||
{ID: 9, Title: "Entry 9", Date: time.Date(2024, 1, 3, 11, 0, 0, 0, time.UTC), FeedID: 700, FeedTitle: "Feed G", GroupID: 3, GroupTitle: "Category C"},
|
||||
}
|
||||
return createHardcodedMockEntries()
|
||||
}
|
||||
|
||||
titleTemplates := []string{
|
||||
@@ -134,10 +138,13 @@ func CreateMockEntries(n int) []*models.Entry {
|
||||
commentsURL = fmt.Sprintf("https://example.com/comments/%d", i)
|
||||
}
|
||||
|
||||
siteURL := fmt.Sprintf("https://www.example%d.com", template.FeedID)
|
||||
entryURL := fmt.Sprintf("%s/item/%d", siteURL, i)
|
||||
|
||||
entry := &models.Entry{
|
||||
ID: int64(i),
|
||||
Title: fmt.Sprintf("Entry %d - %s", i, title),
|
||||
URL: fmt.Sprintf("https://example.com/%d", i),
|
||||
URL: entryURL,
|
||||
CommentsURL: commentsURL,
|
||||
Date: time.Now().Add(time.Duration(-i) * time.Hour),
|
||||
Content: content,
|
||||
@@ -145,6 +152,7 @@ func CreateMockEntries(n int) []*models.Entry {
|
||||
FeedTitle: template.FeedTitle,
|
||||
GroupID: template.GroupID,
|
||||
GroupTitle: template.GroupTitle,
|
||||
SiteURL: siteURL,
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
|
||||
@@ -58,21 +58,21 @@ func (m *MockArchiveService) CleanArchive(maxAge time.Duration) {}
|
||||
|
||||
type MockEmailService struct {
|
||||
services.EmailService
|
||||
SendFunc func(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error
|
||||
SendFunc func(smtpConfig config.ConfigSmtp, digestConfig config.ConfigDigest, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error
|
||||
}
|
||||
|
||||
func (m *MockEmailService) Send(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error {
|
||||
return m.SendFunc(cfg, overviewFile, groupedEntryFiles, data)
|
||||
func (m *MockEmailService) Send(smtpConfig config.ConfigSmtp, digestConfig config.ConfigDigest, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error {
|
||||
return m.SendFunc(smtpConfig, digestConfig, overviewFile, groupedEntryFiles, data)
|
||||
}
|
||||
|
||||
type MockDigestService struct {
|
||||
services.DigestService
|
||||
BuildDigestDataFunc func(category *miniflux.Category, entries *miniflux.Entries, icons map[int64]*models.FeedIcon, subGroupBy string, sortBy string, minifluxHost string) *models.OverviewTemplateData
|
||||
BuildDigestDataFunc func(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData
|
||||
}
|
||||
|
||||
func (m *MockDigestService) BuildDigestData(category *miniflux.Category, entries *miniflux.Entries, icons map[int64]*models.FeedIcon, subGroupBy string, sortBy string, minifluxHost string) *models.OverviewTemplateData {
|
||||
func (m *MockDigestService) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, view string, minifluxHost string, digestHost string, categories []config.ConfigCategory, digestTitle string) *models.OverviewTemplateData {
|
||||
if m.BuildDigestDataFunc != nil {
|
||||
return m.BuildDigestDataFunc(category, entries, icons, subGroupBy, sortBy, minifluxHost)
|
||||
return m.BuildDigestDataFunc(entries, icons, view, minifluxHost, digestHost, categories, digestTitle)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
+71
-31
@@ -18,6 +18,7 @@ import (
|
||||
"miniflux-digest/internal/digest"
|
||||
"miniflux-digest/internal/email"
|
||||
"miniflux-digest/internal/llm"
|
||||
"miniflux-digest/internal/manager"
|
||||
"miniflux-digest/internal/models"
|
||||
"miniflux-digest/internal/templates"
|
||||
"miniflux-digest/internal/testutil"
|
||||
@@ -47,7 +48,7 @@ func openBrowser(url string) error {
|
||||
return exec.Command(cmd, args...).Start()
|
||||
}
|
||||
|
||||
func generateMockDigest(cfg *config.Config) *models.OverviewTemplateData {
|
||||
func generateMockDigest(cfg *config.Config, digestIndex int) *models.OverviewTemplateData {
|
||||
log.Println("generateDigestData: Starting...")
|
||||
|
||||
llmService, err := llm.NewGeminiService(cfg.AI.ApiKey)
|
||||
@@ -55,6 +56,11 @@ func generateMockDigest(cfg *config.Config) *models.OverviewTemplateData {
|
||||
log.Fatalf("Failed to create LLM service: %v", err)
|
||||
}
|
||||
|
||||
digestManager, err := manager.NewDigestManager(cfg.Digests)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating digest manager: %v", err)
|
||||
}
|
||||
|
||||
digestSvc := digest.NewDigestService(llmService)
|
||||
log.Println("generateDigestData: DigestService initialized.")
|
||||
|
||||
@@ -62,8 +68,17 @@ func generateMockDigest(cfg *config.Config) *models.OverviewTemplateData {
|
||||
log.Println("generateDigestData: Using mock data...")
|
||||
entries = testutil.CreateMockEntries(200)
|
||||
|
||||
icons := make(map[int64]*models.FeedIcon)
|
||||
var digestEntries []*models.Entry
|
||||
digestConfig := cfg.Digests[digestIndex]
|
||||
|
||||
for _, entry := range entries {
|
||||
if digestManager.GetOwningDigest(entry) == digestIndex {
|
||||
digestEntries = append(digestEntries, entry)
|
||||
}
|
||||
}
|
||||
|
||||
icons := make(map[int64]*models.FeedIcon)
|
||||
for _, entry := range digestEntries {
|
||||
if _, ok := icons[entry.FeedID]; !ok {
|
||||
switch entry.FeedID {
|
||||
case 1:
|
||||
@@ -80,26 +95,23 @@ func generateMockDigest(cfg *config.Config) *models.OverviewTemplateData {
|
||||
|
||||
log.Println("generateDigestData: Building digest data...")
|
||||
data := digestSvc.BuildDigestData(
|
||||
entries,
|
||||
digestEntries,
|
||||
icons,
|
||||
cfg.Digest.View,
|
||||
digestConfig.View,
|
||||
cfg.Miniflux.Host,
|
||||
cfg.Digest.Host,
|
||||
digestConfig.Host,
|
||||
digestConfig.Categories,
|
||||
digestConfig.Title,
|
||||
)
|
||||
|
||||
if len(data.PrimaryGroups) > 0 {
|
||||
data.PrimaryGroups[0].Summary = "This is a mock summary for the first group to test the layout."
|
||||
for i := range data.PrimaryGroups {
|
||||
data.PrimaryGroups[i].Summary = fmt.Sprintf("This is a mock summary for group %d to test the layout.", i+1)
|
||||
}
|
||||
if len(data.PrimaryGroups) > 2 {
|
||||
data.PrimaryGroups[2].Summary = "This is another mock summary for a different group, showing that not all groups have summaries."
|
||||
}
|
||||
|
||||
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
func generateMinifluxDigest(cfg *config.Config, minifluxClientService services.MinifluxClientService) *models.OverviewTemplateData {
|
||||
func generateMinifluxDigest(cfg *config.Config, digestIndex int, minifluxClientService services.MinifluxClientService) *models.OverviewTemplateData {
|
||||
log.Println("generateDigestData: Starting...")
|
||||
|
||||
llmService, err := llm.NewGeminiService(cfg.AI.ApiKey)
|
||||
@@ -107,6 +119,11 @@ func generateMinifluxDigest(cfg *config.Config, minifluxClientService services.M
|
||||
log.Fatalf("Failed to create LLM service: %v", err)
|
||||
}
|
||||
|
||||
digestManager, err := manager.NewDigestManager(cfg.Digests)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating digest manager: %v", err)
|
||||
}
|
||||
|
||||
digestSvc := digest.NewDigestService(llmService)
|
||||
log.Println("generateDigestData: DigestService initialized.")
|
||||
|
||||
@@ -118,8 +135,17 @@ func generateMinifluxDigest(cfg *config.Config, minifluxClientService services.M
|
||||
log.Fatalf("Failed to fetch entries: %v", err)
|
||||
}
|
||||
|
||||
icons := make(map[int64]*models.FeedIcon)
|
||||
var digestEntries []*models.Entry
|
||||
digestConfig := cfg.Digests[digestIndex]
|
||||
|
||||
for _, entry := range entries {
|
||||
if digestManager.GetOwningDigest(entry) == digestIndex {
|
||||
digestEntries = append(digestEntries, entry)
|
||||
}
|
||||
}
|
||||
|
||||
icons := make(map[int64]*models.FeedIcon)
|
||||
for _, entry := range digestEntries {
|
||||
if _, ok := icons[entry.FeedID]; !ok {
|
||||
icon, err := minifluxClientService.FeedIcon(entry.FeedID)
|
||||
if err != nil {
|
||||
@@ -132,11 +158,13 @@ func generateMinifluxDigest(cfg *config.Config, minifluxClientService services.M
|
||||
|
||||
log.Println("generateDigestData: Building digest data...")
|
||||
return digestSvc.BuildDigestData(
|
||||
entries,
|
||||
digestEntries,
|
||||
icons,
|
||||
cfg.Digest.View,
|
||||
digestConfig.View,
|
||||
cfg.Miniflux.Host,
|
||||
cfg.Digest.Host,
|
||||
digestConfig.Host,
|
||||
digestConfig.Categories,
|
||||
digestConfig.Title,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -157,22 +185,22 @@ func loadConfig() *config.Config {
|
||||
return cfg
|
||||
}
|
||||
|
||||
func generateAndArchiveHTML(cfg *config.Config, minifluxFlag bool) (*os.File, []*os.File, string, *models.OverviewTemplateData) {
|
||||
func generateAndArchiveHTML(cfg *config.Config, digestIndex int, minifluxFlag bool) (*os.File, []*os.File, string, *models.OverviewTemplateData) {
|
||||
var data *models.OverviewTemplateData
|
||||
digestConfig := cfg.Digests[digestIndex]
|
||||
|
||||
if minifluxFlag {
|
||||
minifluxClient := miniflux.NewClient(cfg.Miniflux.Host, cfg.Miniflux.ApiToken)
|
||||
clientWrapper := app.NewMinifluxClientWrapper(minifluxClient)
|
||||
data = generateMinifluxDigest(cfg, clientWrapper)
|
||||
data = generateMinifluxDigest(cfg, digestIndex, clientWrapper)
|
||||
} else {
|
||||
data = generateMockDigest(cfg)
|
||||
data = generateMockDigest(cfg, digestIndex)
|
||||
}
|
||||
|
||||
log.Println("main: Digest data generated.")
|
||||
|
||||
|
||||
archiveSvc := archive.NewArchiveService(webserver.ArchiveBasePath, templates.ArchiveTemplate, templates.OverviewTemplate)
|
||||
overviewFile, groupedEntryFiles, err := archiveSvc.MakeArchiveHTML(data, cfg.Digest.Compress)
|
||||
overviewFile, groupedEntryFiles, err := archiveSvc.MakeArchiveHTML(data, *digestConfig.Compress)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to generate HTML: %v", err)
|
||||
}
|
||||
@@ -186,7 +214,11 @@ func generateAndArchiveHTML(cfg *config.Config, minifluxFlag bool) (*os.File, []
|
||||
log.Fatalf("Failed to get relative path: %v", err)
|
||||
}
|
||||
log.Printf("relativePath: %s", relativePath)
|
||||
overviewURL := fmt.Sprintf("http://localhost%s/archive/%s", webserver.Port, relativePath)
|
||||
|
||||
overviewURL := fmt.Sprintf("http://localhost%s/%s", webserver.Port, relativePath)
|
||||
if digestConfig.Host != "" {
|
||||
overviewURL = fmt.Sprintf("%s/archive/%s", digestConfig.Host, relativePath)
|
||||
}
|
||||
log.Printf("overviewURL: %s", overviewURL)
|
||||
|
||||
return overviewFile, groupedEntryFiles, overviewURL, data
|
||||
@@ -198,7 +230,8 @@ func handleEmail(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []
|
||||
EmailTemplate: templates.EmailTemplate,
|
||||
}
|
||||
|
||||
if err := emailSvc.Send(cfg, overviewFile, groupedEntryFiles, data); err != nil {
|
||||
digestConfig := cfg.Digests[0]
|
||||
if err := emailSvc.Send(cfg.Smtp, digestConfig, overviewFile, groupedEntryFiles, data); err != nil {
|
||||
log.Fatalf("Failed to send email: %v", err)
|
||||
}
|
||||
log.Printf("Successfully generated and sent email.")
|
||||
@@ -230,18 +263,25 @@ func main() {
|
||||
emailFlag, minifluxFlag, htmlFlag := setupAndParseFlags()
|
||||
|
||||
cfg := loadConfig()
|
||||
numDigest := len(cfg.Digests)
|
||||
|
||||
if emailFlag {
|
||||
overviewFile, groupedEntryFiles, _, data := generateAndArchiveHTML(cfg, minifluxFlag)
|
||||
handleEmail(cfg, overviewFile, groupedEntryFiles, data)
|
||||
for digestIndex := range numDigest {
|
||||
overviewFile, groupedEntryFiles, _, data := generateAndArchiveHTML(cfg, digestIndex, minifluxFlag)
|
||||
handleEmail(cfg, overviewFile, groupedEntryFiles, data)
|
||||
}
|
||||
} else if htmlFlag {
|
||||
_, _, overviewURL, _ := generateAndArchiveHTML(cfg, minifluxFlag)
|
||||
handleOpenUrl(overviewURL)
|
||||
for digestIndex := range numDigest {
|
||||
_, _, overviewURL, _ := generateAndArchiveHTML(cfg, digestIndex, minifluxFlag)
|
||||
handleOpenUrl(overviewURL)
|
||||
}
|
||||
} else if minifluxFlag {
|
||||
_, _, overviewURL, _ := generateAndArchiveHTML(cfg, minifluxFlag)
|
||||
handleOpenUrl(overviewURL)
|
||||
for digestIndex := range numDigest {
|
||||
_, _, overviewURL, _ := generateAndArchiveHTML(cfg, digestIndex, minifluxFlag)
|
||||
handleOpenUrl(overviewURL)
|
||||
}
|
||||
} else {
|
||||
log.Println("main: Starting web server")
|
||||
handleWebServer()
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user