diff --git a/Makefile b/Makefile index 3872746..58a9ab7 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,17 @@ build: go build -mod=vendor -o miniflux-digest ./cmd/miniflux-digest +preview: + @trap "exit 0" INT; go run -mod=vendor ./scripts/preview/main.go + preview-html: - go run -mod=vendor ./scripts/preview/main.go + go run -mod=vendor ./scripts/preview/main.go --html preview-email: go run -mod=vendor ./scripts/preview/main.go --email preview-miniflux: -ifdef category - go run -mod=vendor ./scripts/preview/main.go --miniflux=${category} -else - @echo "use 'preview-miniflux category=' to preview a category from miniflux" -endif + go run -mod=vendor ./scripts/preview/main.go --miniflux test: go test -mod=vendor ./... ./cmd/miniflux-digest diff --git a/cmd/miniflux-digest/main.go b/cmd/miniflux-digest/main.go index 271adad..4c45470 100644 --- a/cmd/miniflux-digest/main.go +++ b/cmd/miniflux-digest/main.go @@ -1,11 +1,7 @@ package main import ( - "fmt" "log" - "math/rand" - "net/http" - "strings" "time" "github.com/go-co-op/gocron/v2" @@ -18,41 +14,30 @@ import ( "miniflux-digest/internal/email" "miniflux-digest/internal/llm" "miniflux-digest/internal/processor" + "miniflux-digest/internal/templates" + "miniflux-digest/internal/webserver" ) const ( JitterSeconds = 30 ArchiveCleanupDays = 21 - ArchiveBasePath = "web/miniflux-archive" - HealthCheckPort = ":8080" ) -func registerCategoryDigestJob(application *app.App, scheduler gocron.Scheduler, rawData *app.RawCategoryData) { - task := func(rawData *app.RawCategoryData) { - processor.CategoryDigestJob(application, rawData, application.Config.Digest.MarkAsRead) - } - - jitter := time.Duration(rand.Intn(JitterSeconds)) * time.Second - startTime := time.Now().Add(1*time.Minute + jitter) - - _, err := scheduler.NewJob( - gocron.OneTimeJob(gocron.OneTimeJobStartDateTime(startTime)), - gocron.NewTask(task, rawData), - ) +func digestJob(application *app.App) { + overviewFile, groupedEntryFiles, data, err := processor.ProcessAndSendDigest(application) if err != nil { - log.Printf("Error creating one-time job for category %d: %v", rawData.Category.ID, err) + log.Printf("Error processing digest: %v", err) + return + } + + if err := application.EmailService.Send(application.Config, overviewFile, groupedEntryFiles, data); err != nil { + log.Printf("Error sending digest email: %v", err) } } -func categoriesCheckJob(application *app.App, scheduler gocron.Scheduler) { - for rawData := range application.MinifluxClientService.StreamAllCategoryData() { - registerCategoryDigestJob(application, scheduler, rawData) - } -} - -func registerCategoriesCheckJob(application *app.App, scheduler gocron.Scheduler) { +func registerDigestJob(application *app.App, scheduler gocron.Scheduler) { _, err := scheduler.NewJob(gocron.CronJob(application.Config.Digest.Schedule, true), gocron.NewTask(func() { - categoriesCheckJob(application, scheduler) + digestJob(application) })) if err != nil { @@ -70,36 +55,6 @@ func registerArchiveCleanupJob(application *app.App, scheduler gocron.Scheduler) } } -func SetupServer(archiveBasePath string) *http.ServeMux { - mux := http.NewServeMux() - - mux.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - if _, err := fmt.Fprintf(w, "OK"); err != nil { - log.Printf("Error writing healthcheck response: %v", err) - } - }) - - fs := http.FileServer(http.Dir(archiveBasePath)) - mux.Handle("/archive/", http.StripPrefix("/archive/", fs)) - - return mux -} - -func requestSanitizerMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.Contains(r.URL.Path, "..") { - http.Error(w, "Bad Request", http.StatusBadRequest) - return - } - if strings.HasSuffix(r.URL.Path, "/") && len(r.URL.Path) > 1 { - http.NotFound(w, r) - return - } - next.ServeHTTP(w, r) - }) -} - func main() { cfg, err := config.Load("./config.yaml") @@ -123,21 +78,14 @@ func main() { } }() - registerCategoriesCheckJob(application, scheduler) + registerDigestJob(application, scheduler) registerArchiveCleanupJob(application, scheduler) if application.Config.Digest.RunOnStartup { - go categoriesCheckJob(application, scheduler) + go digestJob(application) } - go func() { - mux := SetupServer(ArchiveBasePath) - log.Printf("Internal web server starting on port %s", HealthCheckPort) - - if err := http.ListenAndServe(HealthCheckPort, requestSanitizerMiddleware(mux)); err != nil { - log.Fatalf("Internal web server failed to start: %v", err) - } - }() + go webserver.ListenAndServe(webserver.ArchiveBasePath, webserver.Port) scheduler.Start() @@ -149,11 +97,12 @@ func initServices(cfg *config.Config) (*app.App, error) { clientWrapper := app.NewMinifluxClientWrapper(minifluxClient) llmService, err := llm.NewGeminiService(cfg.AI.ApiKey) + if err != nil { return nil, err } - archiveSvc := archive.NewArchiveService(ArchiveBasePath) + archiveSvc := archive.NewArchiveService(webserver.ArchiveBasePath, templates.ArchiveTemplate, templates.OverviewTemplate) emailSvc := &email.EmailServiceImpl{} digestService := digest.NewDigestService(llmService) @@ -171,4 +120,4 @@ func initServices(cfg *config.Config) (*app.App, error) { func initScheduler() (gocron.Scheduler, error) { return gocron.NewScheduler() -} \ No newline at end of file +} diff --git a/cmd/miniflux-digest/main_test.go b/cmd/miniflux-digest/main_test.go index c22f56c..f681d45 100644 --- a/cmd/miniflux-digest/main_test.go +++ b/cmd/miniflux-digest/main_test.go @@ -7,6 +7,8 @@ import ( "os" "path/filepath" "testing" + + "miniflux-digest/internal/webserver" ) func setupTestArchive(t *testing.T) string { @@ -21,7 +23,6 @@ func setupTestArchive(t *testing.T) string { } }) - // Create a dummy file to serve categoryDir := filepath.Join(tmpDir, "test-category") if err := os.Mkdir(categoryDir, 0755); err != nil { t.Fatalf("Failed to create category dir: %v", err) @@ -32,15 +33,29 @@ func setupTestArchive(t *testing.T) string { t.Fatalf("Failed to write test file: %v", err) } + categoryWithIndexDir := filepath.Join(tmpDir, "test-category-with-index") + if err := os.Mkdir(categoryWithIndexDir, 0755); err != nil { + t.Fatalf("Failed to create category with index dir: %v", err) + } + indexPath := filepath.Join(categoryWithIndexDir, "index.html") + indexContent := "

Index File

" + if err := os.WriteFile(indexPath, []byte(indexContent), 0644); err != nil { + t.Fatalf("Failed to write index file: %v", err) + } + + categoryNoIndexDir := filepath.Join(tmpDir, "test-category-no-index") + if err := os.Mkdir(categoryNoIndexDir, 0755); err != nil { + t.Fatalf("Failed to create category no index dir: %v", err) + } + return tmpDir } func TestHealthCheckHandler(t *testing.T) { req := httptest.NewRequest("GET", "/healthcheck", nil) rr := httptest.NewRecorder() - mux := SetupServer("") // archive base path is not needed for this test - h := requestSanitizerMiddleware(mux) - h.ServeHTTP(rr, req) + mux := webserver.SetupServer("") + mux.ServeHTTP(rr, req) if status := rr.Code; status != http.StatusOK { t.Errorf("handler returned wrong status code: got %v want %v", @@ -56,12 +71,11 @@ func TestHealthCheckHandler(t *testing.T) { func TestServeArchiveFile_Success(t *testing.T) { archiveBasePath := setupTestArchive(t) - mux := SetupServer(archiveBasePath) + mux := webserver.SetupServer(archiveBasePath) req := httptest.NewRequest("GET", "/archive/test-category/test-file.html", nil) rr := httptest.NewRecorder() - h := requestSanitizerMiddleware(mux) - h.ServeHTTP(rr, req) + mux.ServeHTTP(rr, req) if status := rr.Code; status != http.StatusOK { t.Errorf("handler returned wrong status code: got %v want %v", @@ -81,12 +95,11 @@ func TestServeArchiveFile_Success(t *testing.T) { func TestServeArchiveFile_NotFound(t *testing.T) { archiveBasePath := setupTestArchive(t) - mux := SetupServer(archiveBasePath) + mux := webserver.SetupServer(archiveBasePath) req := httptest.NewRequest("GET", "/archive/test-category/not-found.html", nil) rr := httptest.NewRecorder() - h := requestSanitizerMiddleware(mux) - h.ServeHTTP(rr, req) + mux.ServeHTTP(rr, req) if status := rr.Code; status != http.StatusNotFound { t.Errorf("handler returned wrong status code: got %v want %v", @@ -96,32 +109,120 @@ func TestServeArchiveFile_NotFound(t *testing.T) { func TestServeArchiveFile_PathTraversal(t *testing.T) { archiveBasePath := setupTestArchive(t) - mux := SetupServer(archiveBasePath) + mux := webserver.SetupServer(archiveBasePath) + ts := httptest.NewServer(mux) + defer ts.Close() - // Attempt to access a file outside the archive base path - // The http.FileServer should prevent this, resulting in a 400 - req := httptest.NewRequest("GET", "/archive/../main_test.go", nil) - rr := httptest.NewRecorder() - h := requestSanitizerMiddleware(mux) - h.ServeHTTP(rr, req) + client := &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } - if status := rr.Code; status != http.StatusBadRequest { + req, err := http.NewRequest("GET", ts.URL+"/archive/../main_test.go", nil) + if err != nil { + t.Fatalf("Failed to create request: %v", err) + } + + resp, err := client.Do(req) + if err != nil { + t.Fatalf("Failed to perform request: %v", err) + } + defer func() { + if err := resp.Body.Close(); err != nil { + t.Errorf("Error closing response body: %v", err) + } + }() + + if resp.StatusCode != http.StatusMovedPermanently { t.Errorf("handler returned wrong status code for path traversal attempt: got %v want %v", - status, http.StatusBadRequest) + resp.StatusCode, http.StatusMovedPermanently) + } + + redirectURL, err := resp.Location() + if err != nil { + t.Fatalf("Failed to get redirect location: %v", err) + } + + req, err = http.NewRequest("GET", redirectURL.String(), nil) + if err != nil { + t.Fatalf("Failed to create redirect request: %v", err) + } + + clientWithRedirect := &http.Client{} + resp, err = clientWithRedirect.Do(req) + if err != nil { + t.Fatalf("Failed to perform redirect request: %v", err) + } + defer func() { + if err := resp.Body.Close(); err != nil { + t.Errorf("Error closing response body: %v", err) + } + }() + + if resp.StatusCode != http.StatusNotFound { + t.Errorf("handler returned wrong status code after redirect for path traversal attempt: got %v want %v", + resp.StatusCode, http.StatusNotFound) } } -func TestServeArchiveFile_DirectoryRequest(t *testing.T) { +func TestServeArchiveFile_DirectoryListingDisabled(t *testing.T) { archiveBasePath := setupTestArchive(t) - mux := SetupServer(archiveBasePath) + handler := webserver.SetupServer(archiveBasePath) - req := httptest.NewRequest("GET", "/archive/test-category/", nil) + if err := os.Mkdir(filepath.Join(archiveBasePath, "empty-dir"), 0755); err != nil { + t.Fatalf("Failed to create empty directory: %v", err) + } + + req := httptest.NewRequest("GET", "/archive/empty-dir/", nil) rr := httptest.NewRecorder() - h := requestSanitizerMiddleware(mux) - h.ServeHTTP(rr, req) + handler.ServeHTTP(rr, req) if status := rr.Code; status != http.StatusNotFound { - t.Errorf("handler returned wrong status code for directory request: got %v want %v", + t.Errorf("handler returned wrong status code for directory listing attempt: got %v want %v", status, http.StatusNotFound) } } + +func TestServeArchiveFile_NoDirectoryListingAndIndexHtml(t *testing.T) { + archiveBasePath := setupTestArchive(t) + mux := webserver.SetupServer(archiveBasePath) + + req1 := httptest.NewRequest("GET", "/archive/test-category-with-index/", nil) + rr1 := httptest.NewRecorder() + mux.ServeHTTP(rr1, req1) + + if status := rr1.Code; status != http.StatusOK { + t.Errorf("handler returned wrong status code for directory with index.html: got %v want %v", + status, http.StatusOK) + } + + expected1 := "

Index File

" + body1, err := io.ReadAll(rr1.Body) + if err != nil { + t.Fatalf("Failed to read response body: %v", err) + } + if string(body1) != expected1 { + t.Errorf("handler returned unexpected body for directory with index.html: got %q want %q", + string(body1), expected1) + } + + req2 := httptest.NewRequest("GET", "/archive/test-category-no-index/", nil) + rr2 := httptest.NewRecorder() + mux.ServeHTTP(rr2, req2) + + if status := rr2.Code; status != http.StatusNotFound { + t.Errorf("handler returned wrong status code for directory without index.html: got %v want %v", + status, http.StatusNotFound) + } +} + +func TestInitScheduler(t *testing.T) { + scheduler, err := initScheduler() + if err != nil { + t.Fatalf("initScheduler() returned an error: %v", err) + } + if scheduler == nil { + t.Fatal("initScheduler() returned a nil scheduler") + } +} diff --git a/config.yaml.example b/config.yaml.example index 20bb776..f509159 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -10,14 +10,16 @@ smtp: digest: email: - to: "RECIPIENT_EMAIL@example.com" + to: "RECIPIPIENT_EMAIL@example.com" from: "SENDER_EMAIL@example.com" schedule: "@every 24h" # Cron schedule for digest generation host: "https://your-digest-host.com" # URL where HTML archives will be served compress: true # Compress HTML before sending mark_as_read: true # Mark entries as read after sending run_on_startup: false # Run digest on startup - group_by: "day" # Group entries by "day" or "ai" + group_by: "category" # Group entries by "category" or "feed" + sub_group_by: "day" # Sub-group entries by "date", "feed", or "ai" + sort_by: "date" # Sort entries by "date", "title", or "ai" ai: api_key: "YOUR_GEMINI_API_KEY" diff --git a/internal/app/app.go b/internal/app/app.go index dffc2ee..8c3e305 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -1,17 +1,17 @@ package app import ( + "miniflux-digest/internal/app/services" "miniflux-digest/internal/config" - "miniflux-digest/internal/llm" ) type App struct { Config *config.Config - ArchiveService ArchiveService - EmailService EmailService - MinifluxClientService MinifluxClientService - DigestService DigestService - LLMService llm.LLMService + ArchiveService services.ArchiveService + EmailService services.EmailService + MinifluxClientService services.MinifluxClientService + DigestService services.DigestService + LLMService services.LLMService } type Option func(*App) @@ -30,31 +30,31 @@ func WithConfig(cfg *config.Config) Option { } } -func WithArchiveService(s ArchiveService) Option { +func WithArchiveService(s services.ArchiveService) Option { return func(a *App) { a.ArchiveService = s } } -func WithEmailService(s EmailService) Option { +func WithEmailService(s services.EmailService) Option { return func(a *App) { a.EmailService = s } } -func WithMinifluxClientService(s MinifluxClientService) Option { +func WithMinifluxClientService(s services.MinifluxClientService) Option { return func(a *App) { a.MinifluxClientService = s } } -func WithDigestService(s DigestService) Option { +func WithDigestService(s services.DigestService) Option { return func(a *App) { a.DigestService = s } } -func WithLLMService(s llm.LLMService) Option { +func WithLLMService(s services.LLMService) Option { // Changed return func(a *App) { a.LLMService = s } diff --git a/internal/app/app_test.go b/internal/app/app_test.go new file mode 100644 index 0000000..7be24f6 --- /dev/null +++ b/internal/app/app_test.go @@ -0,0 +1,106 @@ +package app + +import ( + "context" + "os" + "testing" + "time" + + "google.golang.org/genai" + + "miniflux-digest/internal/config" + "miniflux-digest/internal/models" +) + +// Mock services for testing +type mockArchiveService struct{} + +func (m *mockArchiveService) MakeArchiveHTML(data *models.OverviewTemplateData, compress bool) (*os.File, []*os.File, error) { + return nil, nil, nil +} +func (m *mockArchiveService) CleanArchive(maxAge time.Duration) {} + +type mockEmailService struct{} + +func (m *mockEmailService) Send(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error { + return nil +} + +type mockMinifluxClientService struct{} + +func (m *mockMinifluxClientService) FeedIcon(feedID int64) (*models.FeedIcon, error) { + return nil, nil +} +func (m *mockMinifluxClientService) GetAllUnreadEntries() ([]*models.Entry, error) { + return nil, nil +} +func (m *mockMinifluxClientService) UpdateEntries(entryIDs []int64, status string) error { + return nil +} + +type mockDigestService struct{} + +func (m *mockDigestService) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, groupBy string, subGroupBy string, sortBy string, minifluxHost string) *models.OverviewTemplateData { + return nil +} + +type mockLLMService struct{} + +func (m *mockLLMService) GenerateContent(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + return nil, nil +} + +func TestNewApp(t *testing.T) { + app := NewApp() + if app == nil { + t.Error("NewApp() should not return nil") + } +} + +func TestWithConfig(t *testing.T) { + cfg := &config.Config{} + app := NewApp(WithConfig(cfg)) + if app.Config != cfg { + t.Errorf("WithConfig() did not set the config correctly. Got %v, want %v", app.Config, cfg) + } +} + +func TestWithArchiveService(t *testing.T) { + mockSvc := &mockArchiveService{} + app := NewApp(WithArchiveService(mockSvc)) + if app.ArchiveService != mockSvc { + t.Errorf("WithArchiveService() did not set the archive service correctly. Got %v, want %v", app.ArchiveService, mockSvc) + } +} + +func TestWithEmailService(t *testing.T) { + mockSvc := &mockEmailService{} + app := NewApp(WithEmailService(mockSvc)) + if app.EmailService != mockSvc { + t.Errorf("WithEmailService() did not set the email service correctly. Got %v, want %v", app.EmailService, mockSvc) + } +} + +func TestWithMinifluxClientService(t *testing.T) { + mockSvc := &mockMinifluxClientService{} + app := NewApp(WithMinifluxClientService(mockSvc)) + if app.MinifluxClientService != mockSvc { + t.Errorf("WithMinifluxClientService() did not set the miniflux client service correctly. Got %v, want %v", app.MinifluxClientService, mockSvc) + } +} + +func TestWithDigestService(t *testing.T) { + mockSvc := &mockDigestService{} + app := NewApp(WithDigestService(mockSvc)) + if app.DigestService != mockSvc { + t.Errorf("WithDigestService() did not set the digest service correctly. Got %v, want %v", app.DigestService, mockSvc) + } +} + +func TestWithLLMService(t *testing.T) { + mockSvc := &mockLLMService{} + app := NewApp(WithLLMService(mockSvc)) + if app.LLMService != mockSvc { + t.Errorf("WithLLMService() did not set the LLM service correctly. Got %v, want %v", app.LLMService, mockSvc) + } +} diff --git a/internal/app/interfaces.go b/internal/app/interfaces.go deleted file mode 100644 index 225a829..0000000 --- a/internal/app/interfaces.go +++ /dev/null @@ -1,31 +0,0 @@ -package app - -import ( - "miniflux-digest/internal/config" - "miniflux-digest/internal/digest" - "miniflux-digest/internal/models" - "os" - "time" - miniflux "miniflux.app/v2/client" -) - -type ArchiveService interface { - MakeArchiveHTML(data *models.HTMLTemplateData, compress bool) (*os.File, error) - CleanArchive(maxAge time.Duration) -} - -type EmailService interface { - Send(cfg *config.Config, file *os.File, data *models.HTMLTemplateData) error -} - -type DigestService interface { - BuildDigestData(category *miniflux.Category, entries *miniflux.Entries, icons map[int64]*models.FeedIcon, groupBy digest.GroupingType, minifluxHost string) *models.HTMLTemplateData -} - -type MinifluxClientService interface { - MarkCategoryAsRead(categoryID int64) error - CategoryEntries(categoryID int64, filter *miniflux.Filter) (*miniflux.Entries, error) - CategoryFeeds(categoryID int64) ([]*miniflux.Feed, error) - FeedIcon(feedID int64) (*miniflux.FeedIcon, error) - StreamAllCategoryData() <-chan *RawCategoryData -} diff --git a/internal/app/miniflux_client_wrapper.go b/internal/app/miniflux_client_wrapper.go index ea88a99..5d11945 100644 --- a/internal/app/miniflux_client_wrapper.go +++ b/internal/app/miniflux_client_wrapper.go @@ -1,9 +1,6 @@ package app import ( - "fmt" - "log" - "miniflux-digest/internal/models" miniflux "miniflux.app/v2/client" ) @@ -16,111 +13,54 @@ func NewMinifluxClientWrapper(client *miniflux.Client) *MinifluxClientWrapper { return &MinifluxClientWrapper{client: client} } -func (m *MinifluxClientWrapper) MarkCategoryAsRead(categoryID int64) error { - return m.client.MarkCategoryAsRead(categoryID) -} - -func (m *MinifluxClientWrapper) categories() ([]*miniflux.Category, error) { - return m.client.Categories() -} - -func (m *MinifluxClientWrapper) CategoryEntries(categoryID int64, filter *miniflux.Filter) (*miniflux.Entries, error) { - entries, err := m.client.CategoryEntries(categoryID, filter) - if err != nil { - return nil, err - } - return &entries.Entries, nil -} - -func (m *MinifluxClientWrapper) CategoryFeeds(categoryID int64) ([]*miniflux.Feed, error) { - return m.client.CategoryFeeds(categoryID) -} - -func (m *MinifluxClientWrapper) FeedIcon(feedID int64) (*miniflux.FeedIcon, error) { - return m.client.FeedIcon(feedID) -} - -type RawCategoryData struct { - Category *miniflux.Category - Entries *miniflux.Entries - Feeds []*miniflux.Feed - Icons map[int64]*models.FeedIcon -} - -func (m *MinifluxClientWrapper) FetchRawCategoryData(categoryID int64) (*RawCategoryData, error) { - categories, err := m.categories() - if err != nil { - return nil, err - } - - var category *miniflux.Category - for _, c := range categories { - if c.ID == categoryID { - category = c - break - } - } - - if category == nil { - return nil, fmt.Errorf("category with ID %d not found", categoryID) - } - - entriesResult, err := m.client.CategoryEntries(category.ID, &miniflux.Filter{Status: miniflux.EntryStatusUnread}) - if err != nil { - return nil, err - } - - feeds, err := m.client.CategoryFeeds(category.ID) - feedIcons := make(map[int64]*models.FeedIcon) +func (m *MinifluxClientWrapper) FeedIcon(feedID int64) (*models.FeedIcon, error) { + icon, err := m.client.FeedIcon(feedID) if err != nil { return nil, err } - for _, feed := range feeds { - feedIcon, err := m.client.FeedIcon(feed.ID) - if err != nil { - log.Printf("Warning: failed to fetch icon for feed %d: %v", feed.ID, err) - continue - } - - feedIcons[feed.ID] = &models.FeedIcon{ - FeedID: feed.ID, - Data: feedIcon.Data, - } - } - - return &RawCategoryData{ - Category: category, - Entries: &entriesResult.Entries, - Feeds: feeds, - Icons: feedIcons, + return &models.FeedIcon{ + FeedID: feedID, + Data: icon.Data, }, nil } -func (m *MinifluxClientWrapper) StreamAllCategoryData() <-chan *RawCategoryData { - out := make(chan *RawCategoryData) +func (m *MinifluxClientWrapper) GetAllUnreadEntries() ([]*models.Entry, error) { + minifluxEntries, err := m.client.Entries(&miniflux.Filter{Status: miniflux.EntryStatusUnread}) + if err != nil { + return nil, err + } - go func() { - defer close(out) - - categories, err := m.categories() - - if err != nil { - log.Printf("Streamer failed to fetch categories: %v", err) - return + var entries []*models.Entry + for _, entry := range minifluxEntries.Entries { + var categoryID int64 + var categoryTitle string + if entry.Feed != nil && entry.Feed.Category != nil { + categoryID = entry.Feed.Category.ID + categoryTitle = entry.Feed.Category.Title + } + var feedTitle string + if entry.Feed != nil { + feedTitle = entry.Feed.Title } - for _, category := range categories { - data, err := m.FetchRawCategoryData(category.ID) - if err != nil { - log.Printf("Streamer failed to fetch data for category %q: %v", category.Title, err) - continue - } + entries = append(entries, &models.Entry{ + ID: entry.ID, + Title: entry.Title, + URL: entry.URL, + Content: entry.Content, + FeedID: entry.FeedID, + FeedTitle: feedTitle, + GroupID: categoryID, + GroupTitle: categoryTitle, + CommentsURL: entry.CommentsURL, + Date: entry.Date, + }) + } + return entries, nil +} - out <- data - } - }() - - return out -} \ No newline at end of file +func (m *MinifluxClientWrapper) UpdateEntries(entryIDs []int64, status string) error { + return m.client.UpdateEntries(entryIDs, status) +} diff --git a/internal/app/miniflux_client_wrapper_test.go b/internal/app/miniflux_client_wrapper_test.go index fd2b823..c317f3b 100644 --- a/internal/app/miniflux_client_wrapper_test.go +++ b/internal/app/miniflux_client_wrapper_test.go @@ -10,115 +10,7 @@ import ( miniflux "miniflux.app/v2/client" ) -func TestMinifluxClientWrapper_StreamAllCategoryData(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/v1/categories": - if _, err := fmt.Fprintln(w, `[{"id": 1, "title": "Test Category 1"}, {"id": 2, "title": "Test Category 2"}]`); err != nil { - panic(err) - } - case "/v1/categories/1/entries": - if _, err := fmt.Fprintln(w, `{"total": 1, "entries": [{"id": 101, "title": "Entry 1A"}]}`); err != nil { - panic(err) - } - case "/v1/categories/2/entries": - if _, err := fmt.Fprintln(w, `{"total": 1, "entries": [{"id": 102, "title": "Entry 2A"}]}`); err != nil { - panic(err) - } - case "/v1/categories/1/feeds", "/v1/categories/2/feeds": - if _, err := fmt.Fprintln(w, `[{"id": 1, "title": "Test Feed"}]`); err != nil { - panic(err) - } - case "/v1/feeds/1/icon": - if _, err := fmt.Fprintln(w, `{"data": "icon-data", "mime_type": "image/png"}`); err != nil { - panic(err) - } - default: - t.Fatalf("Unexpected request path: %s", r.URL.Path) - } - })) - defer server.Close() - client := miniflux.NewClient(server.URL, "test-token") - wrapper := app.NewMinifluxClientWrapper(client) - - dataChannel := wrapper.StreamAllCategoryData() - - var receivedCount int - for data := range dataChannel { - receivedCount++ - if data.Category == nil { - t.Errorf("Expected category data, but got nil category") - } - if data.Entries == nil { - t.Errorf("Expected entries data, but got nil entries") - } - } - - expectedCount := 2 - if receivedCount != expectedCount { - t.Errorf("Expected to receive data for %d categories, but got %d", expectedCount, receivedCount) - } -} - -func TestMinifluxClientWrapper_MarkCategoryAsRead(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPut { - t.Errorf("Expected PUT request, got %s", r.Method) - } - w.WriteHeader(http.StatusNoContent) - })) - defer server.Close() - - client := miniflux.NewClient(server.URL, "test-token") - wrapper := app.NewMinifluxClientWrapper(client) - - if err := wrapper.MarkCategoryAsRead(1); err != nil { - t.Errorf("Unexpected error: %v", err) - } -} - -func TestMinifluxClientWrapper_CategoryEntries(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if _, err := fmt.Fprintln(w, `{"total": 1, "entries": [{"id": 101, "title": "Entry 1A"}]}`); err != nil { - panic(err) - } - })) - defer server.Close() - - client := miniflux.NewClient(server.URL, "test-token") - wrapper := app.NewMinifluxClientWrapper(client) - - entries, err := wrapper.CategoryEntries(1, nil) - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - - if len(*entries) != 1 { - t.Errorf("Expected 1 entry, got %d", len(*entries)) - } -} - -func TestMinifluxClientWrapper_CategoryFeeds(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if _, err := fmt.Fprintln(w, `[{"id": 1, "title": "Test Feed"}]`); err != nil { - panic(err) - } - })) - defer server.Close() - - client := miniflux.NewClient(server.URL, "test-token") - wrapper := app.NewMinifluxClientWrapper(client) - - feeds, err := wrapper.CategoryFeeds(1) - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - - if len(feeds) != 1 { - t.Errorf("Expected 1 feed, got %d", len(feeds)) - } -} func TestMinifluxClientWrapper_FeedIcon(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/app/services/archive.go b/internal/app/services/archive.go new file mode 100644 index 0000000..3d05c63 --- /dev/null +++ b/internal/app/services/archive.go @@ -0,0 +1,12 @@ +package services + +import ( + "miniflux-digest/internal/models" + "os" + "time" +) + +type ArchiveService interface { + MakeArchiveHTML(data *models.OverviewTemplateData, compress bool) (*os.File, []*os.File, error) + CleanArchive(maxAge time.Duration) +} diff --git a/internal/app/services/digest.go b/internal/app/services/digest.go new file mode 100644 index 0000000..b2692f3 --- /dev/null +++ b/internal/app/services/digest.go @@ -0,0 +1,9 @@ +package services + +import ( + "miniflux-digest/internal/models" +) + +type DigestService interface { + BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, groupBy string, subGroupBy string, sortBy string, minifluxHost string) *models.OverviewTemplateData +} diff --git a/internal/app/services/email.go b/internal/app/services/email.go new file mode 100644 index 0000000..dbad0b8 --- /dev/null +++ b/internal/app/services/email.go @@ -0,0 +1,11 @@ +package services + +import ( + "miniflux-digest/internal/config" + "miniflux-digest/internal/models" + "os" +) + +type EmailService interface { + Send(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error +} diff --git a/internal/llm/interfaces.go b/internal/app/services/llm.go similarity index 75% rename from internal/llm/interfaces.go rename to internal/app/services/llm.go index 5f68ccb..13e27fa 100644 --- a/internal/llm/interfaces.go +++ b/internal/app/services/llm.go @@ -1,4 +1,4 @@ -package llm +package services import ( "context" @@ -7,5 +7,5 @@ import ( ) type LLMService interface { - GenerateContent(ctx context.Context, prompt string, schema *genai.Schema) (string, error) -} \ No newline at end of file + GenerateContent(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) +} diff --git a/internal/app/services/miniflux_client.go b/internal/app/services/miniflux_client.go new file mode 100644 index 0000000..2fc0c40 --- /dev/null +++ b/internal/app/services/miniflux_client.go @@ -0,0 +1,9 @@ +package services + +import "miniflux-digest/internal/models" + +type MinifluxClientService interface { + FeedIcon(feedID int64) (*models.FeedIcon, error) + GetAllUnreadEntries() ([]*models.Entry, error) + UpdateEntries(entryIDs []int64, status string) error +} diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 54f3b3b..7cc89de 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -6,30 +6,31 @@ import ( "io" "io/fs" "log" - "miniflux-digest/internal/app" "miniflux-digest/internal/digest" "miniflux-digest/internal/models" - "miniflux-digest/internal/templates" - "miniflux-digest/internal/utils" "os" "path/filepath" "time" ) -type ArchiveServiceImpl struct{ - ArchiveBaseDir string +type HTMLTemplate interface { + Execute(wr io.Writer, data any) error } -var _ app.ArchiveService = (*ArchiveServiceImpl)(nil) - -func NewArchiveService(archiveBaseDir string) *ArchiveServiceImpl { - return &ArchiveServiceImpl{ArchiveBaseDir: archiveBaseDir} +type ArchiveServiceImpl struct { + ArchiveBaseDir string + ArchiveTemplate HTMLTemplate + OverviewTemplate HTMLTemplate } -func (s *ArchiveServiceImpl) getHTML(data *models.HTMLTemplateData, compress bool) ([]byte, error) { +func NewArchiveService(archiveBaseDir string, archiveTemplate HTMLTemplate, overviewTemplate HTMLTemplate) *ArchiveServiceImpl { + return &ArchiveServiceImpl{ArchiveBaseDir: archiveBaseDir, ArchiveTemplate: archiveTemplate, OverviewTemplate: overviewTemplate} +} + +func (s *ArchiveServiceImpl) getHTML(template HTMLTemplate, data any, compress bool) ([]byte, error) { var buf bytes.Buffer - err := templates.ArchiveTemplate.Execute(&buf, data) + err := template.Execute(&buf, data) if err != nil { return nil, err } @@ -37,42 +38,104 @@ func (s *ArchiveServiceImpl) getHTML(data *models.HTMLTemplateData, compress boo return digest.MinifyHTML(buf.Bytes(), compress) } -func (s *ArchiveServiceImpl) makeArchiveFile(data *models.HTMLTemplateData) (*os.File, error) { - categorySlug := utils.Slugify(data.Category.Title) - categoryFolderPath := fmt.Sprintf("%s/%s", s.ArchiveBaseDir, categorySlug) - filename := fmt.Sprintf("%s/%s.html", categoryFolderPath, data.GeneratedDate.Format("2006-01-02")) - err := os.MkdirAll(categoryFolderPath, 0755) +func (s *ArchiveServiceImpl) makeGroupedEntriesArchiveFile(data *models.PrimaryGroupDigestData, dateFolderPath string) (*os.File, error) { + groupSlug := data.Slug + groupedFolderPath := filepath.Join(dateFolderPath, "digests") + if err := os.MkdirAll(groupedFolderPath, 0755); err != nil { + return nil, err + } + filename := fmt.Sprintf("%s/%s.html", groupedFolderPath, groupSlug) + file, err := os.Create(filename) + return file, err +} + +func (s *ArchiveServiceImpl) makeOverviewArchiveFile(data *models.OverviewTemplateData) (*os.File, string, error) { + dateFolderPath := fmt.Sprintf("%s/%s", s.ArchiveBaseDir, data.GeneratedDate.Format("2006-01-02")) + filename := fmt.Sprintf("%s/index.html", dateFolderPath) + err := os.MkdirAll(dateFolderPath, 0755) if err == nil { file, err := os.Create(filename) - return file, err + return file, dateFolderPath, err } - return nil, err + return nil, "", err } -func (s *ArchiveServiceImpl) MakeArchiveHTML(data *models.HTMLTemplateData, compress bool) (*os.File, error) { - file, err := s.makeArchiveFile(data) - +func (s *ArchiveServiceImpl) MakeArchiveHTML(data *models.OverviewTemplateData, compress bool) (*os.File, []*os.File, error) { + overviewFile, dateFolderPath, err := s.makeOverviewArchiveFile(data) if err != nil { - log.Printf("Error creating HTML file for category '%s': %v", data.Category.Title, err) - return nil, err + log.Printf("Error creating overview HTML file: %v", err) + return nil, nil, err + } + overviewHTML, err := s.getHTML(s.OverviewTemplate, data, compress) + if err != nil { + log.Printf("Error generating overview HTML: %v", err) + return overviewFile, nil, err + } + _, err = overviewFile.Write(overviewHTML) + if err != nil { + log.Printf("Error writing overview HTML to file: %v", err) } - htmlOutput, err := s.getHTML(data, compress) - + _, err = overviewFile.Seek(0, io.SeekStart) if err != nil { - log.Printf("Error generating HTML for category %s: %v", data.Category.Title, err) - return file, err + log.Printf("Error rewinding overview HTML file: %v", err) + return overviewFile, nil, err } - _, err = file.Write(htmlOutput) + var groupedEntryFiles []*os.File - if err != nil { - log.Printf("Error writing HTML to file for category '%s': %v", data.Category.Title, err) + feedIconsMap := make(map[int64]*models.FeedIcon) + for _, icon := range data.FeedIcons { + feedIconsMap[icon.FeedID] = icon } - return file, err + for _, primaryGroup := range data.PrimaryGroups { + totalEntriesInPrimaryGroup := 0 + for _, subGroup := range primaryGroup.SubGroups { + totalEntriesInPrimaryGroup += len(subGroup.Entries) + } + + groupIconsMap := make(map[int64]*models.FeedIcon) + for _, subGroup := range primaryGroup.SubGroups { + for _, entry := range subGroup.Entries { + if icon, ok := feedIconsMap[entry.FeedID]; ok { + groupIconsMap[entry.FeedID] = icon + } + } + } + + var groupIconsSlice []*models.FeedIcon + for _, icon := range groupIconsMap { + groupIconsSlice = append(groupIconsSlice, icon) + } + + groupedPageData := &models.GroupedDigestPageData{ + PrimaryGroup: primaryGroup, + FeedIcons: groupIconsSlice, + MinifluxHost: data.MinifluxHost, + GeneratedDate: data.GeneratedDate, + TotalEntries: totalEntriesInPrimaryGroup, + } + groupedEntriesFile, err := s.makeGroupedEntriesArchiveFile(primaryGroup, dateFolderPath) + if err != nil { + log.Printf("Error creating grouped entries HTML file: %v", err) + return nil, nil, err + } + groupedEntryFiles = append(groupedEntryFiles, groupedEntriesFile) + groupedEntriesHTML, err := s.getHTML(s.ArchiveTemplate, groupedPageData, compress) + if err != nil { + log.Printf("Error generating grouped entries HTML: %v", err) + return overviewFile, groupedEntryFiles, err + } + _, err = groupedEntriesFile.Write(groupedEntriesHTML) + if err != nil { + log.Printf("Error writing grouped entries HTML to file: %v", err) + } + } + + return overviewFile, groupedEntryFiles, err } func (s *ArchiveServiceImpl) removeOldArchiveFiles(maxAge time.Duration) { @@ -107,14 +170,52 @@ func (s *ArchiveServiceImpl) removeOldArchiveFiles(maxAge time.Duration) { } } +func (s *ArchiveServiceImpl) removeEmptyDirs() { + var dirs []string + + err := filepath.WalkDir(s.ArchiveBaseDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + dirs = append(dirs, path) + } + return nil + }) + + if err != nil { + log.Printf("Error walking directory tree: %v", err) + return + } + + for i := len(dirs) - 1; i >= 0; i-- { + dir := dirs[i] + // Don't try to remove the root archive directory + if dir == s.ArchiveBaseDir { + continue + } + + empty, err := s.isDirEmpty(dir) + if err != nil { + log.Printf("Warning: could not check if directory %s is empty: %v", dir, err) + continue + } + + if empty { + if err := os.Remove(dir); err != nil { + log.Printf("Warning: failed to delete empty directory %s: %v", dir, err) + } + } + } +} + func (s *ArchiveServiceImpl) isDirEmpty(name string) (bool, error) { f, err := os.Open(name) if err != nil { return false, err } - defer func() { - if err = f.Close(); err != nil { + if err := f.Close(); err != nil { log.Printf("Warning: failed to close directory %s: %v", name, err) } }() @@ -126,31 +227,7 @@ func (s *ArchiveServiceImpl) isDirEmpty(name string) (bool, error) { return false, err } -func (s *ArchiveServiceImpl) removeEmptyCategoryFolders() { - dirs, err := os.ReadDir(s.ArchiveBaseDir) - if err != nil { - log.Printf("Warning: could not read archive directory %s: %v", s.ArchiveBaseDir, err) - return - } - - for _, dir := range dirs { - if dir.IsDir() { - categoryPath := filepath.Join(s.ArchiveBaseDir, dir.Name()) - empty, err := s.isDirEmpty(categoryPath) - if err != nil { - log.Printf("Warning: could not check if directory %s is empty: %v", categoryPath, err) - continue - } - if empty { - if err := os.Remove(categoryPath); err != nil { - log.Printf("Warning: failed to delete empty directory %s: %v", categoryPath, err) - } - } - } - } -} - func (s *ArchiveServiceImpl) CleanArchive(maxAge time.Duration) { s.removeOldArchiveFiles(maxAge) - s.removeEmptyCategoryFolders() -} \ No newline at end of file + s.removeEmptyDirs() +} diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 239a266..095b792 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -1,9 +1,10 @@ package archive import ( + "io" "miniflux-digest/internal/models" + "miniflux-digest/internal/templates" "miniflux-digest/internal/testutil" - "miniflux-digest/internal/utils" "os" "path/filepath" "testing" @@ -11,14 +12,29 @@ import ( ) func TestGetHTML(t *testing.T) { - archiveService := NewArchiveService(t.TempDir()) - data := models.HTMLTemplateData{ - Category: testutil.NewMockCategory(), + archiveService := NewArchiveService(t.TempDir(), templates.OverviewTemplate, templates.ArchiveTemplate) + + mockSubGroup := &models.EntryGroup{ + Title: "Test SubGroup Title", + Summary: "Test SubGroup Summary", Entries: testutil.NewMockEntries(), - FeedIcons: testutil.NewMockFeedIcons(), + Slug: "test-subgroup-title", } - html, err := archiveService.getHTML(&data, true) + mockPrimaryGroup := &models.PrimaryGroupDigestData{ + Title: "Test Primary Group Title", + Slug: "test-primary-group-title", + SubGroups: []*models.EntryGroup{mockSubGroup}, + Summary: "Test Primary Group Summary", + } + + data := models.GroupedDigestPageData{ + PrimaryGroup: mockPrimaryGroup, + FeedIcons: testutil.NewMockFeedIcons(), + MinifluxHost: "http://localhost:8080", + } + + html, err := archiveService.getHTML(templates.ArchiveTemplate, &data, true) if err != nil { t.Fatalf("getHTML failed: %v", err) } @@ -28,68 +44,51 @@ func TestGetHTML(t *testing.T) { } func TestMakeArchiveFile(t *testing.T) { - // Create a temporary directory for the test tempDir := t.TempDir() - archiveService := NewArchiveService(tempDir) + archiveService := NewArchiveService(tempDir, templates.OverviewTemplate, templates.ArchiveTemplate) - data := models.HTMLTemplateData{ - Category: testutil.NewMockCategory(), + mockSubGroup := &models.EntryGroup{ + Title: "Test SubGroup Title", + Summary: "Test SubGroup Summary", Entries: testutil.NewMockEntries(), - FeedIcons: testutil.NewMockFeedIcons(), + Slug: "test-subgroup-title", } - file, err := archiveService.makeArchiveFile(&data) + data := models.PrimaryGroupDigestData{ + Title: "Test Primary Group Title", + Slug: "test-primary-group-title", + SubGroups: []*models.EntryGroup{mockSubGroup}, + Summary: "Test Primary Group Summary", + } + + dateFolderPath := filepath.Join(tempDir, time.Now().Format("2006-01-02")) + if err := os.MkdirAll(dateFolderPath, 0755); err != nil { + t.Fatalf("Failed to create date folder for test: %v", err) + } + + file, err := archiveService.makeGroupedEntriesArchiveFile(&data, dateFolderPath) if err != nil { t.Fatalf("makeArchiveFile failed: %v", err) } if file == nil { t.Fatal("Expected file to be non-nil") } - // Check if the file was created in the correct hardcoded path - expectedPath := filepath.Join(tempDir, utils.Slugify(data.Category.Title), data.GeneratedDate.Format("2006-01-02")+".html") + expectedPath := filepath.Join(dateFolderPath, "digests", data.Slug+".html") if _, err := os.Stat(expectedPath); os.IsNotExist(err) { t.Errorf("Expected file %s to exist", expectedPath) } } -func TestMakeArchiveHTML(t *testing.T) { - // Create a temporary directory for the test - tempDir := t.TempDir() - archiveService := NewArchiveService(tempDir) - - data := models.HTMLTemplateData{ - Category: testutil.NewMockCategory(), - Entries: testutil.NewMockEntries(), - FeedIcons: testutil.NewMockFeedIcons(), - } - file, err := archiveService.MakeArchiveHTML(&data, true) - if err != nil { - t.Fatalf("MakeArchiveHTML failed: %v", err) - } - if file == nil { - t.Fatal("Expected file to be non-nil") - } - info, err := file.Stat() - if err != nil { - t.Fatalf("Failed to stat file: %v", err) - } - if info.Size() == 0 { - t.Error("Expected file to have content") - } -} - func TestCleanArchive(t *testing.T) { - // Create a temporary directory for the test tempDir := t.TempDir() - archiveService := NewArchiveService(tempDir) + archiveService := NewArchiveService(tempDir, templates.OverviewTemplate, templates.ArchiveTemplate) - categorySlug := "test-category" - categoryPath := filepath.Join(tempDir, categorySlug) - if err := os.MkdirAll(categoryPath, 0755); err != nil { - t.Fatalf("Failed to create test directory: %v", err) + oldDirName := time.Now().Add(-48 * time.Hour).Format("2006-01-02") + oldDirPath := filepath.Join(tempDir, oldDirName) + if err := os.MkdirAll(oldDirPath, 0755); err != nil { + t.Fatalf("Failed to create old test directory: %v", err) } - - oldFilePath := filepath.Join(categoryPath, "old.html") + oldFilePath := filepath.Join(oldDirPath, "test.html") if err := os.WriteFile(oldFilePath, []byte("old"), 0644); err != nil { t.Fatalf("Failed to create old file: %v", err) } @@ -98,72 +97,130 @@ func TestCleanArchive(t *testing.T) { t.Fatalf("Failed to change file modification time: %v", err) } - newFilePath := filepath.Join(categoryPath, "new.html") + newDirName := time.Now().Format("2006-01-02") + newDirPath := filepath.Join(tempDir, newDirName) + if err := os.MkdirAll(newDirPath, 0755); err != nil { + t.Fatalf("Failed to create new test directory: %v", err) + } + newFilePath := filepath.Join(newDirPath, "test.html") if err := os.WriteFile(newFilePath, []byte("new"), 0644); err != nil { t.Fatalf("Failed to create new file: %v", err) } - archiveService.CleanArchive(24*time.Hour) + archiveService.CleanArchive(24 * time.Hour) - if _, err := os.Stat(oldFilePath); !os.IsNotExist(err) { - t.Error("Expected old file to be deleted") + if _, err := os.Stat(oldDirPath); !os.IsNotExist(err) { + t.Error("Expected old directory to be deleted") } - if _, err := os.Stat(newFilePath); os.IsNotExist(err) { - t.Error("Expected new file to be kept") - } - - if err := os.Remove(newFilePath); err != nil { - t.Fatalf("Failed to remove new file: %v", err) - } - - // Test removeEmptyCategoryFolders separately - emptyCategoryPath := filepath.Join(tempDir, "empty-category") - if err := os.MkdirAll(emptyCategoryPath, 0755); err != nil { - t.Fatalf("Failed to create empty test directory: %v", err) - } - archiveService.removeEmptyCategoryFolders() - - if _, err := os.Stat(emptyCategoryPath); !os.IsNotExist(err) { - t.Error("Expected empty category directory to be deleted") + if _, err := os.Stat(newDirPath); os.IsNotExist(err) { + t.Error("Expected new directory to be kept") } } -func TestIsDirEmpty(t *testing.T) { - archiveService := NewArchiveService(t.TempDir()) - tmpDir, err := os.MkdirTemp("", "test-empty-dir") +type mockHTMLTemplate struct { + executeFunc func(wr io.Writer, data any) error +} + +func (m *mockHTMLTemplate) Execute(wr io.Writer, data any) error { + return m.executeFunc(wr, data) +} + +func TestMakeOverviewArchiveFile(t *testing.T) { + tempDir := t.TempDir() + archiveService := NewArchiveService(tempDir, &mockHTMLTemplate{}, &mockHTMLTemplate{}) + + data := &models.OverviewTemplateData{ + GeneratedDate: time.Date(2024, time.January, 1, 0, 0, 0, 0, time.UTC), + } + + file, dateFolderPath, err := archiveService.makeOverviewArchiveFile(data) if err != nil { - t.Fatalf("Failed to create temp dir: %v", err) + t.Fatalf("makeOverviewArchiveFile failed: %v", err) + } + if file == nil { + t.Fatal("Expected file to be non-nil") } defer func() { - if err := os.RemoveAll(tmpDir); err != nil { - t.Errorf("Failed to remove temp dir: %v", err) + if err := file.Close(); err != nil { + t.Errorf("Error closing file: %v", err) } }() - empty, err := archiveService.isDirEmpty(tmpDir) - if err != nil { - t.Fatalf("isDirEmpty failed for empty dir: %v", err) + expectedPath := filepath.Join(tempDir, "2024-01-01", "index.html") + if _, err := os.Stat(expectedPath); os.IsNotExist(err) { + t.Errorf("Expected file %s to exist", expectedPath) } - if !empty { - t.Error("Expected directory to be empty") - } - - filePath := filepath.Join(tmpDir, "test.txt") - if err := os.WriteFile(filePath, []byte("test"), 0644); err != nil { - t.Fatalf("Failed to create test file: %v", err) - } - - empty, err = archiveService.isDirEmpty(tmpDir) - if err != nil { - t.Fatalf("isDirEmpty failed for non-empty dir: %v", err) - } - if empty { - t.Error("Expected directory to not be empty") - } - - _, err = archiveService.isDirEmpty("non-existent-dir") - if !os.IsNotExist(err) { - t.Errorf("Expected IsNotExist error for non-existent dir, got: %v", err) + if dateFolderPath != filepath.Join(tempDir, "2024-01-01") { + t.Errorf("Expected date folder path %s, got %s", filepath.Join(tempDir, "2024-01-01"), dateFolderPath) } } + +func TestMakeArchiveHTML_Success(t *testing.T) { + tempDir := t.TempDir() + + mockOverviewTemplate := &mockHTMLTemplate{ + executeFunc: func(wr io.Writer, data any) error { + _, err := wr.Write([]byte("overview html")) + return err + }, + } + mockArchiveTemplate := &mockHTMLTemplate{ + executeFunc: func(wr io.Writer, data any) error { + d, ok := data.(*models.GroupedDigestPageData) + if !ok { + t.Fatal("unexpected data type for archive template") + } + if len(d.FeedIcons) == 0 { + t.Error("expected feed icons to be present in archive template data") + } + _, err := wr.Write([]byte("grouped html")) + return err + }, + } + archiveService := NewArchiveService(tempDir, mockArchiveTemplate, mockOverviewTemplate) + + data := &models.OverviewTemplateData{ + GeneratedDate: time.Date(2024, time.January, 1, 0, 0, 0, 0, time.UTC), + PrimaryGroups: []*models.PrimaryGroupDigestData{ + { + Slug: "group-1", + SubGroups: []*models.EntryGroup{{Entries: []*models.Entry{{ID: 1, FeedID: 1}}}}, + }, + { + Slug: "group-2", + SubGroups: []*models.EntryGroup{{Entries: []*models.Entry{{ID: 2, FeedID: 2}}}}, + }, + }, + FeedIcons: []*models.FeedIcon{ + {FeedID: 1, Data: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="}, + {FeedID: 2, Data: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="}, + }, + } + + overviewFile, groupedEntryFiles, err := archiveService.MakeArchiveHTML(data, false) + if err != nil { + t.Fatalf("MakeArchiveHTML failed: %v", err) + } + if overviewFile == nil { + t.Fatal("Expected overviewFile to be non-nil") + } + if len(groupedEntryFiles) != 2 { + t.Errorf("Expected 2 groupedEntryFiles, got %d", len(groupedEntryFiles)) + } + + overviewContent, _ := os.ReadFile(overviewFile.Name()) + if string(overviewContent) != "overview html" { + t.Errorf("Expected overview content 'overview html', got %s", string(overviewContent)) + } + + for _, f := range groupedEntryFiles { + content, _ := os.ReadFile(f.Name()) + if string(content) != "grouped html" { + t.Errorf("Expected grouped content 'grouped html', got %s", string(content)) + } + } + + _ = overviewFile.Close() + _ = os.RemoveAll(filepath.Dir(overviewFile.Name())) +} diff --git a/internal/config/config.go b/internal/config/config.go index dee698f..9435504 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -10,8 +10,6 @@ import ( "github.com/knadh/koanf/providers/file" "github.com/knadh/koanf/v2" "github.com/robfig/cron/v3" - - "miniflux-digest/internal/digest" ) // https://github.com/go-co-op/gocron/issues/826 @@ -32,8 +30,8 @@ type ConfigMiniflux struct { } type ConfigDigestEmail struct { - To string `koanf:"to" validate:"omitempty,email"` - From string `koanf:"from" validate:"omitempty,email"` + To string `koanf:"to" validate:"omitempty,email"` + From string `koanf:"from" validate:"omitempty,email"` } type ConfigSmtp struct { @@ -44,13 +42,15 @@ type ConfigSmtp struct { } type ConfigDigest struct { - Email ConfigDigestEmail `koanf:"email"` - Schedule string `koanf:"schedule" validate:"gocron"` - Host string `koanf:"host"` - Compress bool `koanf:"compress"` - GroupBy digest.GroupingType `koanf:"group_by" validate:"omitempty,oneof=day feed ai"` - MarkAsRead bool `koanf:"mark_as_read"` - RunOnStartup bool `koanf:"run_on_startup"` + Email ConfigDigestEmail `koanf:"email"` + Schedule string `koanf:"schedule" validate:"gocron"` + Host string `koanf:"host"` + Compress bool `koanf:"compress"` + GroupBy string `koanf:"group_by" validate:"omitempty,oneof=category feed"` + SubGroupBy string `koanf:"sub_group_by" validate:"omitempty,oneof=date feed ai"` + SortBy string `koanf:"sort_by" validate:"omitempty,oneof=date ai"` + MarkAsRead bool `koanf:"mark_as_read"` + RunOnStartup bool `koanf:"run_on_startup"` } type ConfigAI struct { @@ -74,8 +74,8 @@ func (c *Config) Validate() error { validate.RegisterStructValidation(func(sl validator.StructLevel) { cfg := sl.Current().Interface().(Config) - if cfg.Digest.GroupBy == "ai" && cfg.AI.ApiKey == "" { - sl.ReportError(cfg.AI.ApiKey, "AI.ApiKey", "ApiKey", "required_if", "Digest.GroupBy is 'ai'") + if cfg.Digest.SubGroupBy == "ai" && cfg.AI.ApiKey == "" { + sl.ReportError(cfg.AI.ApiKey, "AI.ApiKey", "ApiKey", "required_if", "Digest.SubGroupBy is 'ai'") } }, Config{}) @@ -118,10 +118,12 @@ func Load(path string) (*Config, error) { func setDefaultValues(k *koanf.Koanf) error { return k.Load(confmap.Provider(map[string]any{ - "digest.compress": true, - "digest.group_by": "day", - "digest.schedule": "@weekly", - "digest.mark_as_read": true, + "digest.compress": true, + "digest.group_by": "category", + "digest.sub_group_by": "feed", + "digest.sort_by": "date", + "digest.schedule": "@weekly", + "digest.mark_as_read": true, "digest.run_on_startup": false, }, "."), nil) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 05cfd85..cba9863 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -24,6 +24,9 @@ func TestLoad(t *testing.T) { "digest": map[string]any{ "schedule": "@daily", }, + "ai": map[string]any{ + "api_key": "dummy-key", + }, }, wantErr: false, }, @@ -80,6 +83,9 @@ func TestLoad(t *testing.T) { "smtp": map[string]any{ "port": 587, }, + "ai": map[string]any{ + "api_key": "dummy-key", + }, }, wantErr: false, }, @@ -129,8 +135,11 @@ func TestLoad(t *testing.T) { "from": "sender@example.com", }, }, + "ai": map[string]any{ + "api_key": "dummy-key", + }, }, - wantErr: false, + wantErr: false, }, { name: "valid cron schedule", @@ -142,6 +151,9 @@ func TestLoad(t *testing.T) { "digest": map[string]any{ "schedule": "* * 1 * *", }, + "ai": map[string]any{ + "api_key": "dummy-key", + }, }, wantErr: false, }, @@ -168,6 +180,9 @@ func TestLoad(t *testing.T) { "digest": map[string]any{ "schedule": "@every 1h30m", }, + "ai": map[string]any{ + "api_key": "dummy-key", + }, }, wantErr: false, }, @@ -199,15 +214,15 @@ func TestLoad(t *testing.T) { wantErr: true, }, { - name: "missing ai.api_key when group_by is ai", + name: "missing ai.api_key when sub_group_by is ai", config: map[string]any{ "miniflux": map[string]any{ "host": "miniflux.example.com", "api_token": "test-token", }, "digest": map[string]any{ - "schedule": "@daily", - "group_by": "ai", + "schedule": "@daily", + "sub_group_by": "ai", }, }, wantErr: true, diff --git a/internal/digest/day_grouper.go b/internal/digest/day_grouper.go new file mode 100644 index 0000000..bded7b2 --- /dev/null +++ b/internal/digest/day_grouper.go @@ -0,0 +1,66 @@ +package digest + +import ( + "fmt" + "miniflux-digest/internal/models" + "miniflux-digest/internal/utils" + "sort" + "time" +) + +type DayGrouper struct{} + +func (g *DayGrouper) GroupEntries(pgs []*primaryGroup) ([]*models.PrimaryGroupDigestData, *string) { + var allPrimaryGroups []*models.PrimaryGroupDigestData + + for _, pg := range pgs { + entryGroupsMap := make(map[string]*models.EntryGroup) + for _, entry := range pg.Entries { + dateKey := entry.Date.Format(DayGroupLayout) + if _, ok := entryGroupsMap[dateKey]; !ok { + entryGroupsMap[dateKey] = &models.EntryGroup{ + Title: entry.Date.Format(DayGroupTitleLayout), + Entries: []*models.Entry{}, + Slug: utils.Slugify(entry.Date.Format(DayGroupTitleLayout)), + } + } + entryGroupsMap[dateKey].Entries = append(entryGroupsMap[dateKey].Entries, entry) + } + + sortedEntryGroups := make([]*models.EntryGroup, 0, len(entryGroupsMap)) + for _, group := range entryGroupsMap { + uniqueFeedIDs := make(map[int64]bool) + for _, entry := range group.Entries { + uniqueFeedIDs[entry.FeedID] = true + } + group.TotalEntries = len(group.Entries) + group.TotalFeeds = len(uniqueFeedIDs) + sortedEntryGroups = append(sortedEntryGroups, group) + } + + sort.Slice(sortedEntryGroups, func(i, j int) bool { + iDate, _ := time.Parse("Jan 2, 2006", sortedEntryGroups[i].Title) + jDate, _ := time.Parse("Jan 2, 2006", sortedEntryGroups[j].Title) + return iDate.Before(jDate) + }) + + for _, group := range sortedEntryGroups { + sort.Slice(group.Entries, func(i, j int) bool { + return group.Entries[i].Date.Before(group.Entries[j].Date) + }) + } + + for _, seg := range sortedEntryGroups { + seg.Title = fmt.Sprintf("%s - %s", pg.Title, seg.Title) + } + + allPrimaryGroups = append(allPrimaryGroups, &models.PrimaryGroupDigestData{ + ID: pg.ID, + Title: pg.Title, + Slug: utils.Slugify(pg.Title), + SubGroups: sortedEntryGroups, + }) + } + + return allPrimaryGroups, nil +} diff --git a/internal/digest/digest.go b/internal/digest/digest.go index 8565dba..a51aaf0 100644 --- a/internal/digest/digest.go +++ b/internal/digest/digest.go @@ -1,318 +1,114 @@ package digest import ( - "context" - "encoding/json" - "fmt" - "log" - "miniflux-digest/internal/llm" + "miniflux-digest/internal/app/services" "miniflux-digest/internal/models" "sort" "time" - "strings" - - "google.golang.org/genai" - miniflux "miniflux.app/v2/client" ) -type GroupingType string - -const ( - GroupingTypeDay GroupingType = "day" - GroupingTypeFeed GroupingType = "feed" -) - -const ( - LLMTimeout = 5 * time.Minute - DayGroupLayout = "2006-01-02" - DayGroupTitleLayout = "Jan 2, 2006" -) - -func (gt GroupingType) String() string { - return string(gt) +type primaryGroup struct { + ID int64 + Title string + Entries []*models.Entry } -type DigestService struct{ - LLMService llm.LLMService -} +func GroupEntries(entries []*models.Entry, groupBy string) []*primaryGroup { + groups := make(map[int64]*primaryGroup) -func NewDigestService(llmService llm.LLMService) *DigestService { - return &DigestService{LLMService: llmService} -} + for _, entry := range entries { + var groupID int64 + var groupTitle string -func NewGrouper(groupBy GroupingType, llmService llm.LLMService) Grouper { - switch groupBy { - case "ai": - return &LLMGrouper{LLMService: llmService} - case GroupingTypeFeed: - return &FeedGrouper{} - default: - return &DayGrouper{} + if groupBy == "feed" { + groupID = entry.FeedID + groupTitle = entry.FeedTitle + } else { // category + groupID = entry.GroupID + groupTitle = entry.GroupTitle + if groupTitle == "" { + groupTitle = "Uncategorized" + } + } + + if _, ok := groups[groupID]; !ok { + groups[groupID] = &primaryGroup{ + ID: groupID, + Title: groupTitle, + } + } + groups[groupID].Entries = append(groups[groupID].Entries, entry) } + + var result []*primaryGroup + for _, pg := range groups { + result = append(result, pg) + } + + return result } -func (s *DigestService) BuildDigestData(category *miniflux.Category, entries *miniflux.Entries, icons map[int64]*models.FeedIcon, groupBy GroupingType, minifluxHost string) *models.HTMLTemplateData { - // Convert map to slice +func NewDigestService(llmService services.LLMService) services.DigestService { + return &digestServiceImpl{llmService: llmService} +} + +type digestServiceImpl struct { + llmService services.LLMService +} + +func (s *digestServiceImpl) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, groupBy string, subGroupBy string, sortBy string, minifluxHost string) *models.OverviewTemplateData { iconsSlice := make([]*models.FeedIcon, 0, len(icons)) for _, icon := range icons { iconsSlice = append(iconsSlice, icon) } - // Group entries - grouper := NewGrouper(groupBy, s.LLMService) - entryGroups, summary := grouper.GroupEntries(entries) + primaryGroups := GroupEntries(entries, groupBy) + grouper := NewSubGrouper(subGroupBy, s.llmService) + allPrimaryGroups, overallDigestSummary := grouper.GroupEntries(primaryGroups) - return &models.HTMLTemplateData{ - Category: category, - Entries: entries, - GeneratedDate: time.Now(), - FeedIcons: iconsSlice, - EntryGroups: entryGroups, - Summary: summary, - MinifluxHost: minifluxHost, - } -} - -type Grouper interface { - GroupEntries(entries *miniflux.Entries) ([]*models.EntryGroup, string) -} - -type DayGrouper struct{} - -func (g *DayGrouper) GroupEntries(entries *miniflux.Entries) ([]*models.EntryGroup, string) { - entryGroupsMap := make(map[string]*models.EntryGroup) - for _, entry := range *entries { - dateKey := entry.Date.Format(DayGroupLayout) - if _, ok := entryGroupsMap[dateKey]; !ok { - entryGroupsMap[dateKey] = &models.EntryGroup{ - Title: entry.Date.Format(DayGroupTitleLayout), - Entries: []*miniflux.Entry{}, - } - } - entryGroupsMap[dateKey].Entries = append(entryGroupsMap[dateKey].Entries, entry) - } - - // Convert map to sorted slice of EntryGroups - sortedEntryGroups := make([]*models.EntryGroup, 0, len(entryGroupsMap)) - for _, group := range entryGroupsMap { - sortedEntryGroups = append(sortedEntryGroups, group) - } - - // Sort groups by date (older to newer) - sort.Slice(sortedEntryGroups, func(i, j int) bool { - // Dates are stored as strings, so we need to parse them back to time.Time - iDate, _ := time.Parse("Jan 2, 2006", sortedEntryGroups[i].Title) - jDate, _ := time.Parse("Jan 2, 2006", sortedEntryGroups[j].Title) - return iDate.Before(jDate) + sort.Slice(allPrimaryGroups, func(i, j int) bool { + return allPrimaryGroups[i].Title < allPrimaryGroups[j].Title }) - // Sort entries within each group by date (older to newer) - for _, group := range sortedEntryGroups { - sort.Slice(group.Entries, func(i, j int) bool { - return group.Entries[i].Date.Before(group.Entries[j].Date) - }) - } - - return sortedEntryGroups, fmt.Sprintf("You have %d entries from %d different days", len(*entries), len(sortedEntryGroups)) -} - -type FeedGrouper struct{} - -func (g *FeedGrouper) GroupEntries(entries *miniflux.Entries) ([]*models.EntryGroup, string) { - entryGroupsMap := make(map[int64]*models.EntryGroup) - for _, entry := range *entries { - if _, ok := entryGroupsMap[entry.FeedID]; !ok { - entryGroupsMap[entry.FeedID] = &models.EntryGroup{ - Title: entry.Feed.Title, - Entries: []*miniflux.Entry{}, + for _, primaryGroup := range allPrimaryGroups { + primaryGroupTotalEntries := 0 + primaryGroupUniqueFeedIDs := make(map[int64]bool) + for _, subGroup := range primaryGroup.SubGroups { + primaryGroupTotalEntries += len(subGroup.Entries) + for _, entry := range subGroup.Entries { + primaryGroupUniqueFeedIDs[entry.FeedID] = true } } - entryGroupsMap[entry.FeedID].Entries = append(entryGroupsMap[entry.FeedID].Entries, entry) + primaryGroup.TotalEntries = primaryGroupTotalEntries + primaryGroup.TotalFeeds = len(primaryGroupUniqueFeedIDs) } - // Convert map to slice of EntryGroups - entryGroups := make([]*models.EntryGroup, 0, len(entryGroupsMap)) - for _, group := range entryGroupsMap { - entryGroups = append(entryGroups, group) + var allEntryGroups []*models.EntryGroup + for _, primaryGroup := range allPrimaryGroups { + allEntryGroups = append(allEntryGroups, primaryGroup.SubGroups...) } - // Sort groups by feed title (alphabetically) - sort.Slice(entryGroups, func(i, j int) bool { - return entryGroups[i].Title < entryGroups[j].Title - }) - - // Sort entries within each group by date (older to newer) - for _, group := range entryGroups { - sort.Slice(group.Entries, func(i, j int) bool { - return group.Entries[i].Date.Before(group.Entries[j].Date) - }) + uniqueFeedIDs := make(map[int64]bool) + for _, entry := range entries { + uniqueFeedIDs[entry.FeedID] = true } - return entryGroups, fmt.Sprintf("You have %d entries from %d feeds", len(*entries), len(entryGroups)) + return &models.OverviewTemplateData{ + Entries: entries, + GeneratedDate: time.Now(), + FeedIcons: iconsSlice, + PrimaryGroups: allPrimaryGroups, + EntryGroups: allEntryGroups, + OverviewSummary: safeDerefString(overallDigestSummary), + TotalEntries: len(entries), + TotalFeeds: len(uniqueFeedIDs), + MinifluxHost: minifluxHost, + } } -type LLMGrouper struct { - LLMService llm.LLMService -} - -type LLMResponse struct { - Summary string `json:"summary"` - Groups []struct { - Title string `json:"title"` - Entries []int `json:"entries"` - } `json:"groups"` -} - -type llmEntry struct { - ID int64 `json:"id"` - Title string `json:"title"` - URL string `json:"url"` - Content string `json:"content"` - FeedTitle string `json:"feed_title"` -} - -const llmPrompt = `You are an expert content curator with a talent for identifying the most important and interesting information from a large volume of content. Your primary goal is to save the user time by providing a high-level, insightful overview of their content feeds. - -The content can come from a variety of sources, including news sites, blogs, forums like Reddit, and social media like Bluesky. Your tone should be that of a helpful assistant, not a news editor. - -Given the following list of entries, your task is to perform two distinct functions: - -1. **Create an insightful 'summary':** - * This must be a single, concise paragraph. - * Your task is to identify and highlight the most significant themes, trends, or critical events from the provided entries. - * **Do not** simply list the topics of every article. Instead, synthesize a compelling narrative. For example, you might point out a recurring theme across several articles or highlight a single entry if it represents a major, must-read development. Your summary should be opinionated and selective, giving the user a clear sense of what matters most. - -2. **Generate intelligent 'groups' for all entries:** - * Your goal is to cluster the entries into a set of meaningful, thematic groups that will help the user quickly navigate the content. - * The number of groups should be driven by the content itself. **Do not create a group for a single entry unless it represents a major, unique event.** The ideal number of groups is one that best helps a user skim the content. A group can contain many entries if they are all highly related. - * Group titles should be short, descriptive, and useful for skimming (e.g., "AI Industry News," "Project Updates," "Global Politics"). - * Within each group, you must rank the 'entries' by importance, with the most significant or actionable item appearing first. - * The goal of grouping is to help with reading, so if a few entries are about some similar topic, those are worth grouping because the user can just read one and skim the rest. - -Return the response as a JSON object according to the desired responseSchema. - -Below are the entries and other relevant metadata for this task: ------------------ - -` - - -var llmResponseSchema = &genai.Schema{ - Type: genai.TypeObject, - Properties: map[string]*genai.Schema{ - "summary": { - Type: genai.TypeString, - }, - "groups": { - Type: genai.TypeArray, - Items: &genai.Schema{ - Type: genai.TypeObject, - Properties: map[string]*genai.Schema{ - "title": { - Type: genai.TypeString, - }, - "entries": { - Type: genai.TypeArray, - Items: &genai.Schema{ - Type: genai.TypeInteger, - }, - }, - }, - }, - }, - }, -} - -func (g *LLMGrouper) GroupEntries(entries *miniflux.Entries) ([]*models.EntryGroup, string) { - llmEntries := make([]llmEntry, len(*entries)) - for i, entry := range *entries { - llmEntries[i] = llmEntry{ - ID: entry.ID, - Title: entry.Title, - URL: entry.URL, - Content: entry.Content, - FeedTitle: entry.Feed.Title, - } +func safeDerefString(s *string) string { + if s == nil { + return "" } - - entriesJSON, err := json.MarshalIndent(llmEntries, "", " ") - if err != nil { - return (&DayGrouper{}).GroupEntries(entries) - } - - prompt := llmPrompt + string(entriesJSON) - - ctx, cancel := context.WithTimeout(context.Background(), LLMTimeout) - defer cancel() - - llmResponse, err := g.LLMService.GenerateContent(ctx, prompt, llmResponseSchema) - - if err != nil { - log.Printf("LLM service failed, falling back to day grouping: %v\n", err) - return (&DayGrouper{}).GroupEntries(entries) - } - - var response LLMResponse - if err := json.Unmarshal([]byte(llmResponse), &response); err != nil { - log.Printf("Failed to parse LLM response, falling back to day grouping: %v\n", err) - return (&DayGrouper{}).GroupEntries(entries) - } - - entryMap := make(map[int64]*miniflux.Entry) - for _, entry := range *entries { - entryMap[entry.ID] = entry - } - - var entryGroups []*models.EntryGroup - groupedEntryIDs := make(map[int64]bool) - - for _, groupData := range response.Groups { - var groupEntries []*miniflux.Entry - for _, entryID := range groupData.Entries { - if _, exists := groupedEntryIDs[int64(entryID)]; !exists { - if entry, ok := entryMap[int64(entryID)]; ok { - groupEntries = append(groupEntries, entry) - groupedEntryIDs[int64(entryID)] = true - } - } - } - - entryGroups = append(entryGroups, &models.EntryGroup{ - Title: groupData.Title, - Entries: groupEntries, - }) - } - - var ungroupedEntries []*miniflux.Entry - for _, entry := range *entries { - if !groupedEntryIDs[entry.ID] { - ungroupedEntries = append(ungroupedEntries, entry) - } - } - - if len(ungroupedEntries) > 0 { - foundUncategorized := false - for _, group := range entryGroups { - if strings.EqualFold(group.Title, "Uncategorized") { - group.Entries = append(group.Entries, ungroupedEntries...) - foundUncategorized = true - break - } - } - if !foundUncategorized { - entryGroups = append(entryGroups, &models.EntryGroup{ - Title: "Uncategorized", - Entries: ungroupedEntries, - }) - } - } - - feedIDs := make(map[int64]bool) - for _, entry := range *entries { - feedIDs[entry.FeedID] = true - } - - statsSummary := fmt.Sprintf("You have %d entries from %d feeds.", len(*entries), len(feedIDs)) - - return entryGroups, fmt.Sprintf("%s\n\n%s", response.Summary, statsSummary) + return *s } diff --git a/internal/digest/digest_test.go b/internal/digest/digest_test.go index 53f4531..60ffea7 100644 --- a/internal/digest/digest_test.go +++ b/internal/digest/digest_test.go @@ -4,18 +4,24 @@ import ( "context" "errors" "miniflux-digest/internal/models" + "miniflux-digest/internal/testutil" "testing" - "time" "google.golang.org/genai" - miniflux "miniflux.app/v2/client" ) type mockLLMService struct { - GenerateContentFunc func(ctx context.Context, prompt string, schema *genai.Schema) (string, error) + GenerateContentFunc func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) } -func findGroup(groups []*models.EntryGroup, title string) *models.EntryGroup { +func (m *mockLLMService) GenerateContent(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + if m.GenerateContentFunc != nil { + return m.GenerateContentFunc(ctx, prompt, schema) + } + return nil, errors.New("GenerateContentFunc not implemented") +} + +func findPrimaryGroup(groups []*models.PrimaryGroupDigestData, title string) *models.PrimaryGroupDigestData { for _, group := range groups { if group.Title == title { return group @@ -24,291 +30,171 @@ func findGroup(groups []*models.EntryGroup, title string) *models.EntryGroup { return nil } -func (m *mockLLMService) GenerateContent(ctx context.Context, prompt string, schema *genai.Schema) (string, error) { - if m.GenerateContentFunc != nil { - return m.GenerateContentFunc(ctx, prompt, schema) +func findSubGroup(primaryGroup *models.PrimaryGroupDigestData, title string) *models.EntryGroup { + for _, group := range primaryGroup.SubGroups { + if group.Title == title { + return group + } } - return "", nil + return nil } -func createDayGrouperMockEntries() *miniflux.Entries { - return &miniflux.Entries{ - { - ID: 1, - Title: "Entry 1 - Jan 2", - Date: time.Date(2024, time.January, 2, 10, 0, 0, 0, time.UTC), - FeedID: 100, - Feed: &miniflux.Feed{ID: 100, Title: "Feed A"}, - }, - { - ID: 2, - Title: "Entry 2 - Jan 1", - Date: time.Date(2024, time.January, 1, 10, 0, 0, 0, time.UTC), - FeedID: 200, - Feed: &miniflux.Feed{ID: 200, Title: "Feed B"}, - }, - { - ID: 3, - Title: "Entry 3 - Jan 2", - Date: time.Date(2024, time.January, 2, 11, 0, 0, 0, time.UTC), - FeedID: 100, - Feed: &miniflux.Feed{ID: 100, Title: "Feed A"}, - }, - { - ID: 4, - Title: "Entry 4 - Jan 1", - Content: "Content of entry 4 about Go concurrency.", - Date: time.Date(2024, time.January, 1, 12, 0, 0, 0, time.UTC), - FeedID: 200, - Feed: &miniflux.Feed{ID: 200, Title: "Feed B"}, - }, - } -} - -func createFeedGrouperMockEntries() *miniflux.Entries { - return &miniflux.Entries{ - { - ID: 1, - Title: "Entry 1 - Feed A", - FeedID: 100, - Feed: &miniflux.Feed{ID: 100, Title: "Feed A"}, - Date: time.Date(2024, time.January, 1, 10, 0, 0, 0, time.UTC), - }, - { - ID: 2, - Title: "Entry 2 - Feed B", - FeedID: 200, - Feed: &miniflux.Feed{ID: 200, Title: "Feed B"}, - Date: time.Date(2024, time.January, 1, 11, 0, 0, 0, time.UTC), - }, - { - ID: 3, - Title: "Entry 3 - Feed A", - FeedID: 100, - Feed: &miniflux.Feed{ID: 100, Title: "Feed A"}, - Date: time.Date(2024, time.January, 1, 12, 0, 0, 0, time.UTC), - }, - { - ID: 4, - Title: "Entry 4 - Feed B", - FeedID: 200, - Feed: &miniflux.Feed{ID: 200, Title: "Feed B"}, - Date: time.Date(2024, time.January, 1, 13, 0, 0, 0, time.UTC), - }, +func createLLMGrouperMockEntries() []*models.Entry { + return []*models.Entry{ + {ID: 1, Title: "Entry 1", FeedID: 100, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A"}, + {ID: 2, Title: "Entry 2", FeedID: 100, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A"}, + {ID: 3, Title: "Entry 3", FeedID: 200, FeedTitle: "Feed B", GroupID: 1, GroupTitle: "Category A"}, } } func TestDayGrouper_GroupEntries(t *testing.T) { - entries := createDayGrouperMockEntries() + entries := testutil.CreateMockEntries(0) + primaryGroupsMap := GroupEntries(entries, "category") grouper := &DayGrouper{} - groups, summary := grouper.GroupEntries(entries) + groups, summary := grouper.GroupEntries(primaryGroupsMap) - if summary == "" { - t.Error("Expected a non-empty summary for DayGrouper") + if summary != nil { + t.Errorf("Expected an empty summary for DayGrouper, got %q", *summary) } - // Expect 2 groups: Jan 1, 2024 and Jan 2, 2024 - if len(groups) != 2 { - t.Fatalf("Expected 2 groups, got %d", len(groups)) + if len(groups) != 3 { + t.Fatalf("Expected 3 primary groups, got %d", len(groups)) } - // Check sorting (older to newer) - if groups[0].Title != "Jan 1, 2024" { - t.Errorf("Expected first group to be Jan 1, 2024, got %s", groups[0].Title) - } - if groups[1].Title != "Jan 2, 2024" { - t.Errorf("Expected second group to be Jan 2, 2024, got %s", groups[1].Title) + catAGroup := findPrimaryGroup(groups, "Category A") + if catAGroup == nil || len(catAGroup.SubGroups) != 1 { + t.Fatalf("Incorrect sub-groups for Category A: %+v", catAGroup) } - // Check entries within groups (older to newer) - jan1Group := findGroup(groups, "Jan 1, 2024") - if jan1Group == nil || len(jan1Group.Entries) != 2 || jan1Group.Entries[0].ID != 2 || jan1Group.Entries[1].ID != 4 { - t.Errorf("Incorrect entries for Jan 1, 2024 group: %+v", jan1Group) - } - - jan2Group := findGroup(groups, "Jan 2, 2024") - if jan2Group == nil || len(jan2Group.Entries) != 2 || jan2Group.Entries[0].ID != 1 || jan2Group.Entries[1].ID != 3 { - t.Errorf("Incorrect entries for Jan 2, 2024 group: %+v", jan2Group) + subGroup := findSubGroup(catAGroup, "Category A - Jan 2, 2024") + if subGroup == nil || len(subGroup.Entries) != 4 { + t.Fatalf("Incorrect entries for sub-group: %+v", subGroup) } } func TestFeedGrouper_GroupEntries(t *testing.T) { - entries := createFeedGrouperMockEntries() + entries := testutil.CreateMockEntries(0) + primaryGroupsMap := GroupEntries(entries, "category") grouper := &FeedGrouper{} - groups, summary := grouper.GroupEntries(entries) + groups, summary := grouper.GroupEntries(primaryGroupsMap) - if summary == "" { - t.Error("Expected a non-empty summary for FeedGrouper") + if summary != nil { + t.Errorf("Expected an empty summary for FeedGrouper, got %q", *summary) } - // Expect 2 groups: Feed A and Feed B - if len(groups) != 2 { - t.Fatalf("Expected 2 groups, got %d", len(groups)) + if len(groups) != 3 { + t.Fatalf("Expected 3 primary groups, got %d", len(groups)) } - // Find groups by title for easier assertion - feedAGroup := findGroup(groups, "Feed A") - if feedAGroup == nil || len(feedAGroup.Entries) != 2 || feedAGroup.Entries[0].ID != 1 || feedAGroup.Entries[1].ID != 3 { - t.Errorf("Incorrect entries for Feed A group: %+v", feedAGroup) - } - - feedBGroup := findGroup(groups, "Feed B") - if feedBGroup == nil || len(feedBGroup.Entries) != 2 || feedBGroup.Entries[0].ID != 2 || feedBGroup.Entries[1].ID != 4 { - t.Errorf("Incorrect entries for Feed B group: %+v", feedBGroup) - } -} - -func TestNewGrouper(t *testing.T) { - mockLLM := &mockLLMService{} - - if _, ok := NewGrouper(GroupingTypeDay, mockLLM).(*DayGrouper); !ok { - t.Error("Expected DayGrouper for 'day' grouping") - } - if _, ok := NewGrouper(GroupingTypeFeed, mockLLM).(*FeedGrouper); !ok { - t.Error("Expected FeedGrouper for 'feed' grouping") - } - if _, ok := NewGrouper("invalid", mockLLM).(*DayGrouper); !ok { - t.Error("Expected DayGrouper for invalid grouping") - } - if _, ok := NewGrouper("ai", mockLLM).(*LLMGrouper); !ok { - t.Error("Expected LLMGrouper for 'ai' grouping") + catAGroup := findPrimaryGroup(groups, "Category A") + if catAGroup == nil || len(catAGroup.SubGroups) != 3 { + t.Fatalf("Incorrect sub-groups for Category A: %+v", catAGroup) } } func TestLLMGrouper_GroupEntries(t *testing.T) { - entries := createDayGrouperMockEntries() - - expectedLLMResponse := `{ - "summary": "This is a summary of all entries.", - "groups": [ - { - "title": "Go Programming", - "entries": [1, 2, 4] - }, - { - "title": "Python Programming", - "entries": [3] - } - ] -}` + entries := createLLMGrouperMockEntries() + primaryGroupsMap := GroupEntries(entries, "category") mockLLM := &mockLLMService{ - GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) (string, error) { - return expectedLLMResponse, nil + GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + return []byte(`{ + "overview": "This is a summary of all entries.", + "primary_group_summaries": [ + { + "primary_group_id": 1, + "summary": "This is a summary of Category A." + } + ], + "sub_groups": [ + { + "title": "Sub-group 1", + "entry_ids": [1, 2] + } + ] + }`), + nil }, } grouper := &LLMGrouper{LLMService: mockLLM} - groups, summary := grouper.GroupEntries(entries) + groups, summary := grouper.GroupEntries(primaryGroupsMap) - expectedSummary := "This is a summary of all entries.\n\nYou have 4 entries from 2 feeds." - if summary != expectedSummary { - t.Errorf("Expected summary \"%s\", got \"%s\"", expectedSummary, summary) + if summary == nil || *summary != "This is a summary of all entries." { + t.Errorf("Expected summary 'This is a summary of all entries.', got %q", *summary) } - if len(groups) != 2 { - t.Fatalf("Expected 2 groups, got %d", len(groups)) + if len(groups) != 1 { + t.Fatalf("Expected 1 group, got %d", len(groups)) } - // Check group titles and entries - goGroup := findGroup(groups, "Go Programming") - if goGroup == nil || len(goGroup.Entries) != 3 || goGroup.Entries[0].ID != 1 || goGroup.Entries[1].ID != 2 || goGroup.Entries[2].ID != 4 { - t.Errorf("Incorrect Go Programming group: %+v", goGroup) + catA := findPrimaryGroup(groups, "Category A") + if catA == nil || len(catA.SubGroups) != 2 { // Sub-group 1 and Uncategorized + t.Fatalf("Incorrect number of sub-groups for Category A: %+v", catA) } - pythonGroup := findGroup(groups, "Python Programming") - if pythonGroup == nil || len(pythonGroup.Entries) != 1 || pythonGroup.Entries[0].ID != 3 { - t.Errorf("Incorrect Python Programming group: %+v", pythonGroup) + if catA.Summary != "This is a summary of Category A." { + t.Errorf("Expected summary 'This is a summary of Category A.', got %q", catA.Summary) } - // Test fallback to DayGrouper on LLM error - mockLLM.GenerateContentFunc = func(ctx context.Context, prompt string, schema *genai.Schema) (string, error) { - return "", errors.New("LLM API error") - } - groups, summary = grouper.GroupEntries(entries) - if len(groups) == 0 || summary == "" { - t.Error("Expected fallback to DayGrouper on LLM error") + subGroup1 := findSubGroup(catA, "Sub-group 1") + if subGroup1 == nil || len(subGroup1.Entries) != 2 { + t.Fatalf("Incorrect entries for sub-group 1: %+v", subGroup1) } - // Test fallback to DayGrouper on invalid JSON - mockLLM.GenerateContentFunc = func(ctx context.Context, prompt string, schema *genai.Schema) (string, error) { - return "invalid json", nil - } - groups, summary = grouper.GroupEntries(entries) - if len(groups) == 0 || summary == "" { - t.Error("Expected fallback to DayGrouper on invalid JSON") - } - - // Test ungrouped entries - expectedLLMResponseWithMissingEntry := `{ - "summary": "Summary with missing entry.", - "groups": [ - { - "title": "Go Programming", - "entries": [1, 2] - } - ] -}` - mockLLM.GenerateContentFunc = func(ctx context.Context, prompt string, schema *genai.Schema) (string, error) { - return expectedLLMResponseWithMissingEntry, nil - } - groups, _ = grouper.GroupEntries(entries) - - if len(groups) != 2 { - t.Fatalf("Expected 2 groups including uncategorized, got %d", len(groups)) - } - - uncategorizedGroup := findGroup(groups, "Uncategorized") - if uncategorizedGroup == nil || len(uncategorizedGroup.Entries) != 2 || uncategorizedGroup.Entries[0].ID != 3 || uncategorizedGroup.Entries[1].ID != 4 { - t.Errorf("Incorrect Uncategorized group: %+v", uncategorizedGroup) + uncategorized := findSubGroup(catA, "Uncategorized") + if uncategorized == nil || len(uncategorized.Entries) != 1 { // Entry 3 + t.Fatalf("Incorrect entries for Uncategorized sub-group: %+v", uncategorized) } } -func TestLLMGrouper_GroupEntries_WithDuplicateEntries(t *testing.T) { - entries := createDayGrouperMockEntries() +func TestDigestService_BuildDigestData_NonAI(t *testing.T) { + mockLLM := &mockLLMService{} + digestService := NewDigestService(mockLLM) - expectedLLMResponse := `{ - "summary": "This is a summary of all entries.", - "groups": [ - { - "title": "Go Programming", - "entries": [1, 2, 1, 4] - }, - { - "title": "Python Programming", - "entries": [3, 3] - } - ] -}` + entries := testutil.CreateMockEntries(0) + icons := map[int64]*models.FeedIcon{ + 101: {FeedID: 101, Data: "iconA"}, + 102: {FeedID: 102, Data: "iconB"}, + } + + overviewData := digestService.BuildDigestData(entries, icons, "category", "day", "date", "http://miniflux.test") + + if overviewData == nil { + t.Fatal("overviewData is nil") + } + if len(overviewData.PrimaryGroups) != 3 { + t.Errorf("Expected 3 primary groups, got %d", len(overviewData.PrimaryGroups)) + } +} + +func TestDigestService_BuildDigestData_LLMFallback(t *testing.T) { + entries := testutil.CreateMockEntries(0) + icons := map[int64]*models.FeedIcon{ + 100: {FeedID: 100, Data: "icon100"}, + 200: {FeedID: 200, Data: "icon200"}, + } mockLLM := &mockLLMService{ - GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) (string, error) { - return expectedLLMResponse, nil + GenerateContentFunc: func(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + return nil, errors.New("LLM service error during test") }, } + digestService := NewDigestService(mockLLM) - grouper := &LLMGrouper{LLMService: mockLLM} - groups, summary := grouper.GroupEntries(entries) + overviewData := digestService.BuildDigestData(entries, icons, "category", "ai", "date", "http://miniflux.test") - expectedSummary := "This is a summary of all entries.\n\nYou have 4 entries from 2 feeds." - if summary != expectedSummary { - t.Errorf("Expected summary \"%s\", got \"%s\"", expectedSummary, summary) + if overviewData == nil { + t.Fatal("overviewData is nil") } - if len(groups) != 2 { - t.Fatalf("Expected 2 groups, got %d", len(groups)) + if overviewData.OverviewSummary != "" { + t.Errorf("Expected OverviewSummary to be empty, got %q", overviewData.OverviewSummary) } - // Check group titles and entries - goGroup := findGroup(groups, "Go Programming") - if goGroup == nil || len(goGroup.Entries) != 3 { - t.Errorf("Incorrect Go Programming group: %+v", goGroup) - } - - pythonGroup := findGroup(groups, "Python Programming") - if pythonGroup == nil || len(pythonGroup.Entries) != 1 { - t.Errorf("Incorrect Python Programming group: %+v", pythonGroup) + if len(overviewData.PrimaryGroups) != 3 { // Category A, Category B, Category C + t.Fatalf("Expected 3 primary groups after fallback, got %d", len(overviewData.PrimaryGroups)) } } diff --git a/internal/digest/feed_grouper.go b/internal/digest/feed_grouper.go new file mode 100644 index 0000000..7d9a6eb --- /dev/null +++ b/internal/digest/feed_grouper.go @@ -0,0 +1,62 @@ +package digest + +import ( + "fmt" + "miniflux-digest/internal/models" + "miniflux-digest/internal/utils" + "sort" +) + +type FeedGrouper struct{} + +func (g *FeedGrouper) GroupEntries(pgs []*primaryGroup) ([]*models.PrimaryGroupDigestData, *string) { + var allPrimaryGroups []*models.PrimaryGroupDigestData + + for _, pg := range pgs { + entryGroupsMap := make(map[int64]*models.EntryGroup) + for _, entry := range pg.Entries { + if _, ok := entryGroupsMap[entry.FeedID]; !ok { + entryGroupsMap[entry.FeedID] = &models.EntryGroup{ + Title: entry.FeedTitle, + Entries: []*models.Entry{}, + Slug: utils.Slugify(entry.FeedTitle), + } + } + entryGroupsMap[entry.FeedID].Entries = append(entryGroupsMap[entry.FeedID].Entries, entry) + } + + entryGroups := make([]*models.EntryGroup, 0, len(entryGroupsMap)) + for _, group := range entryGroupsMap { + uniqueFeedIDs := make(map[int64]bool) + for _, entry := range group.Entries { + uniqueFeedIDs[entry.FeedID] = true + } + group.TotalEntries = len(group.Entries) + group.TotalFeeds = len(uniqueFeedIDs) + entryGroups = append(entryGroups, group) + } + + sort.Slice(entryGroups, func(i, j int) bool { + return entryGroups[i].Title < entryGroups[j].Title + }) + + for _, group := range entryGroups { + sort.Slice(group.Entries, func(i, j int) bool { + return group.Entries[i].Date.Before(group.Entries[j].Date) + }) + } + + for _, seg := range entryGroups { + seg.Title = fmt.Sprintf("%s - %s", pg.Title, seg.Title) + } + + allPrimaryGroups = append(allPrimaryGroups, &models.PrimaryGroupDigestData{ + ID: pg.ID, + Title: pg.Title, + Slug: utils.Slugify(pg.Title), + SubGroups: entryGroups, + }) + } + + return allPrimaryGroups, nil +} diff --git a/internal/digest/grouper.go b/internal/digest/grouper.go new file mode 100644 index 0000000..c31defa --- /dev/null +++ b/internal/digest/grouper.go @@ -0,0 +1,26 @@ +package digest + +import ( + "miniflux-digest/internal/app/services" + "miniflux-digest/internal/models" +) + +const ( + DayGroupLayout = "2006-01-02" + DayGroupTitleLayout = "Jan 2, 2006" +) + +type SubGrouper interface { + GroupEntries(pgs []*primaryGroup) ([]*models.PrimaryGroupDigestData, *string) +} + +func NewSubGrouper(subGroupBy string, llmService services.LLMService) SubGrouper { + switch subGroupBy { + case "ai": + return &LLMGrouper{LLMService: llmService} + case "feed": + return &FeedGrouper{} + default: // day + return &DayGrouper{} + } +} diff --git a/internal/digest/llm_grouper.go b/internal/digest/llm_grouper.go new file mode 100644 index 0000000..786e26b --- /dev/null +++ b/internal/digest/llm_grouper.go @@ -0,0 +1,241 @@ +package digest + +import ( + "context" + "encoding/json" + "log" + "miniflux-digest/internal/app/services" + "miniflux-digest/internal/models" + "miniflux-digest/internal/utils" + "time" + + "google.golang.org/genai" +) + +const ( + LLMTimeout = 10 * time.Minute +) + +type LLMResponse struct { + Overview string `json:"overview"` + PrimaryGroupSummaries []struct { + PrimaryGroupID int64 `json:"primary_group_id"` + Summary string `json:"summary"` + } `json:"primary_group_summaries"` + SubGroups []struct { + Title string `json:"title"` + EntryIDs []int64 `json:"entry_ids"` + } `json:"sub_groups"` +} + +type LLMGrouper struct { + LLMService services.LLMService +} + +type llmEntry struct { + ID int64 `json:"id"` + Title string `json:"title"` + URL string `json:"url"` + Content string `json:"content"` + FeedTitle string `json:"feed_title"` + PrimaryGroupID int64 `json:"primary_group_id"` +} + +const llmPrompt = `You will be given a large list of unread entry JSON Objects from a RSS feed reader. these entries help a user follow updates from blogs, social networks, and news sources. + +Your task is to generate the metadata according to the instructions below: + + - an 'overview' + - this is a 3-5 sentence overview for ALL entries, less than 250 words. + - the overview is really a brief overview. you are not to summarize every single entry + - The overview should be concise. don't go beyond 5 sentences. + - The overview should help the user quickly understand your overview of all entries in total + - the overview should help the user decide whether to skim fast or slow and what to narrow in on. + - the overview could highlight critical entries or summarize the main topics or both, you decide + + - a list of 'primary_group_summaries' + - the 'primary_group_id' will be found associated to one or more entry objects + - the summary is really a brief overview using all entries associated to the primary group + - you don't need to summarize every single entry + - entries are only ever are associated with one primary group id + - this is a 2-3 sentence overview, less than 150 words. + - The overview should be concise. + - the overview should help the user decide whether to skim all entries in this group fast or slow and what to narrow in on. + - the overview could highlight critical entries or summarize the main topics or both whatever helps the user skip, skim or find the most important entries + +- 'sub_groups': + - The sub_groups field should contain a list of subgroup objects + - Each subgroup should have a 'title' that you come up with + - a title is generally only 2-3 words, please no more than 70 characters total (including spaces) + - Each subgroup should have an array of 'entry_ids' + - an entry id can ONLY be associated to one sub_group + - all entry_ids in a sub_group must share the same group_id. + - The purpose of subgroups is to group related entries together to allow for faster skimming. The grouping can be based on topic, relevance, or any other criteria that you think would be helpful to the user. + - sometimes you'll want to subgroup entries because they are all about the same topic + - sometimes you'll want to subgroup entries because they are just basically leading to the same article itself + - sometimes you'll want to subgroup entries because they are just all complementing each other and worth reading together + +Your goal is to help the user get through their unread entries as efficiently as possible. The metadata you generate will be used to create a digest that allows the user to quickly understand and navigate their unread entries. + +Below is the list of entries: +---------------------------- +` + +var llmResponseSchema = &genai.Schema{ + Type: genai.TypeObject, + Properties: map[string]*genai.Schema{ + "overview": { + Type: genai.TypeString, + }, + "primary_group_summaries": { + Type: genai.TypeArray, + Items: &genai.Schema{ + Type: genai.TypeObject, + Properties: map[string]*genai.Schema{ + "primary_group_id": { + Type: genai.TypeInteger, + }, + "summary": { + Type: genai.TypeString, + }, + }, + }, + }, + "sub_groups": { + Type: genai.TypeArray, + Items: &genai.Schema{ + Type: genai.TypeObject, + Properties: map[string]*genai.Schema{ + "title": { + Type: genai.TypeString, + }, + "entry_ids": { + Type: genai.TypeArray, + Items: &genai.Schema{ + Type: genai.TypeInteger, + }, + }, + }, + }, + }, + }, +} + +func (g *LLMGrouper) GroupEntries(pgs []*primaryGroup) ([]*models.PrimaryGroupDigestData, *string) { + var llmEntries []llmEntry + entryMap := make(map[int64]*models.Entry) + primaryGroupMap := make(map[int64]*primaryGroup) + + for _, pg := range pgs { + primaryGroupMap[pg.ID] = pg + for _, entry := range pg.Entries { + content := entry.Content + if len(content) > 1000 { + content = content[:1000] + } + llmEntries = append(llmEntries, llmEntry{ + ID: entry.ID, + Title: entry.Title, + URL: entry.URL, + Content: content, + PrimaryGroupID: pg.ID, + }) + entryMap[entry.ID] = entry + } + } + + entriesJSON, err := json.MarshalIndent(llmEntries, "", " ") + if err != nil { + return fallbackToFeedGrouper(pgs) + } + + prompt := llmPrompt + string(entriesJSON) + + ctx, cancel := context.WithTimeout(context.Background(), LLMTimeout) + defer cancel() + + llmResponse, err := g.LLMService.GenerateContent(ctx, prompt, llmResponseSchema) + if err != nil { + log.Printf("LLM service failed, falling back to feed grouping: %v\n", err) + return fallbackToFeedGrouper(pgs) + } + + var response LLMResponse + if err := json.Unmarshal(llmResponse, &response); err != nil { + log.Printf("Failed to parse LLM response, falling back to feed grouping: %v\n", err) + log.Printf("Raw LLM response:\n%s", llmResponse) + return fallbackToFeedGrouper(pgs) + } + + finalPrimaryGroupsMap := make(map[int64]*models.PrimaryGroupDigestData) + for _, pg := range pgs { + finalPrimaryGroupsMap[pg.ID] = &models.PrimaryGroupDigestData{ + ID: pg.ID, + Title: pg.Title, + Slug: utils.Slugify(pg.Title), + SubGroups: []*models.EntryGroup{}, + } + } + + for _, summary := range response.PrimaryGroupSummaries { + if pg, ok := finalPrimaryGroupsMap[summary.PrimaryGroupID]; ok { + pg.Summary = summary.Summary + } + } + + groupedEntryIDs := make(map[int64]bool) + for _, llmSubGroup := range response.SubGroups { + subGroupEntriesByPrimaryGroup := make(map[int64][]*models.Entry) + + for _, entryID := range llmSubGroup.EntryIDs { + if _, exists := groupedEntryIDs[entryID]; exists { + continue + } + if entry, ok := entryMap[entryID]; ok { + primaryGroupID := entry.GroupID + subGroupEntriesByPrimaryGroup[primaryGroupID] = append(subGroupEntriesByPrimaryGroup[primaryGroupID], entry) + groupedEntryIDs[entryID] = true + } + } + + for primaryGroupID, entriesInGroup := range subGroupEntriesByPrimaryGroup { + if pg, ok := finalPrimaryGroupsMap[primaryGroupID]; ok { + if len(entriesInGroup) > 0 { + pg.SubGroups = append(pg.SubGroups, &models.EntryGroup{ + Title: llmSubGroup.Title, + Entries: entriesInGroup, + Slug: utils.Slugify(llmSubGroup.Title), + }) + } + } + } + } + + for _, pg := range pgs { + var ungroupedInThisPrimaryGroup []*models.Entry + for _, entry := range pg.Entries { + if !groupedEntryIDs[entry.ID] { + ungroupedInThisPrimaryGroup = append(ungroupedInThisPrimaryGroup, entry) + } + } + + if len(ungroupedInThisPrimaryGroup) > 0 { + finalPrimaryGroupsMap[pg.ID].SubGroups = append(finalPrimaryGroupsMap[pg.ID].SubGroups, &models.EntryGroup{ + Title: "Uncategorized", + Entries: ungroupedInThisPrimaryGroup, + Slug: "uncategorized", + }) + } + } + + var finalPrimaryGroups []*models.PrimaryGroupDigestData + for _, pg := range finalPrimaryGroupsMap { + finalPrimaryGroups = append(finalPrimaryGroups, pg) + } + + return finalPrimaryGroups, &response.Overview +} + +func fallbackToFeedGrouper(pgs []*primaryGroup) ([]*models.PrimaryGroupDigestData, *string) { + return (&FeedGrouper{}).GroupEntries(pgs) +} diff --git a/internal/email/email.go b/internal/email/email.go index 524c869..abca738 100644 --- a/internal/email/email.go +++ b/internal/email/email.go @@ -1,24 +1,23 @@ package email import ( + "bytes" "fmt" "os" - "path/filepath" + "text/template" "miniflux-digest/internal/config" - "miniflux-digest/internal/app" "miniflux-digest/internal/models" "miniflux-digest/internal/templates" "github.com/wneessen/go-mail" ) +type EmailServiceImpl struct{ + EmailTemplate *template.Template +} -type EmailServiceImpl struct{} - -var _ app.EmailService = (*EmailServiceImpl)(nil) - -func (s *EmailServiceImpl) Send(cfg *config.Config, file *os.File, data *models.HTMLTemplateData) error { +func (s *EmailServiceImpl) Send(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error { message := mail.NewMsg() client, err := mail.NewClient( cfg.Smtp.Host, @@ -39,24 +38,26 @@ func (s *EmailServiceImpl) Send(cfg *config.Config, file *os.File, data *models. return err } - subject := fmt.Sprintf("[miniflux digest] %s", data.Category.Title) - filename := filepath.Base(file.Name()) - dir := filepath.Base(filepath.Dir(file.Name())) - url := fmt.Sprintf("%s/%s/%s/%s", cfg.Digest.Host, "archive", dir, filename) - textData := templates.EmailTemplateData{ - HTMLTemplateData: *data, - URL: url, - Summary: data.Summary, - } + subject := fmt.Sprintf("Your Miniflux Digest for %s", data.GeneratedDate.Format("January 2, 2006")) + overviewURL := fmt.Sprintf("%s/archive/%s/index.html", cfg.Digest.Host, data.GeneratedDate.Format("2006-01-02")) message.Subject(subject) - message.AttachFile(file.Name(), mail.WithFileContentType("text/html")) - err = message.SetBodyTextTemplate(templates.EmailTemplate, textData) + emailData := templates.EmailTemplateData{ + OverviewTemplateData: *data, + URL: overviewURL, + Summary: data.OverviewSummary, + } - if err != nil { + var body bytes.Buffer + if err := s.EmailTemplate.Execute(&body, emailData); err != nil { return err } + message.SetBodyString(mail.TypeTextPlain, body.String()) + + for _, f := range groupedEntryFiles { + message.AttachFile(f.Name(), mail.WithFileContentType("text/html")) + } return client.DialAndSend(message) } diff --git a/internal/email/email_test.go b/internal/email/email_test.go index 805500b..b086306 100644 --- a/internal/email/email_test.go +++ b/internal/email/email_test.go @@ -20,10 +20,10 @@ func TestSend(t *testing.T) { }, Digest: config.ConfigDigest{ Email: config.ConfigDigestEmail{ - To: "to@example.com", - From: "from@example.com", + To: "to@example.com", + From: "from@example.com", }, - Host: "https://example.com", + Host: "https://example.com", }, } @@ -53,17 +53,24 @@ func TestSend(t *testing.T) { } }() - data := models.HTMLTemplateData{ - Category: testutil.NewMockCategory(), - Entries: testutil.NewMockEntries(), - FeedIcons: testutil.NewMockFeedIcons(), + data := models.OverviewTemplateData{ + Entries: testutil.NewMockEntries(), + FeedIcons: testutil.NewMockFeedIcons(), + OverviewSummary: "Test summary", + EntryGroups: []*models.EntryGroup{ + testutil.NewMockEntryGroup(), + }, + PrimaryGroups: []*models.PrimaryGroupDigestData{}, + MinifluxHost: "https://example.com", } // In a real scenario, you would use a mock SMTP server. // For this test, we are just checking if the function executes without error. // The go-mail library does not make it easy to mock the SMTP client. - emailService := &EmailServiceImpl{} - err = emailService.Send(cfg, file, &data) + emailService := &EmailServiceImpl{ + EmailTemplate: templates.EmailTemplate, + } + err = emailService.Send(cfg, file, []*os.File{}, &data) if err != nil { // We expect an error because we are not running a real SMTP server. // The important part is that the function attempts to connect. @@ -74,23 +81,18 @@ func TestSend(t *testing.T) { } func TestTextTemplateData(t *testing.T) { - htmlTemplateData := models.HTMLTemplateData{ - Category: testutil.NewMockCategory(), - Entries: testutil.NewMockEntries(), + htmlTemplateData := models.OverviewTemplateData{ + Entries: testutil.NewMockEntries(), FeedIcons: testutil.NewMockFeedIcons(), } url := "https://example.com" textData := templates.EmailTemplateData{ - HTMLTemplateData: htmlTemplateData, - URL: url, + OverviewTemplateData: htmlTemplateData, + URL: url, } if textData.URL != url { t.Errorf("Expected URL to be %s, got %s", url, textData.URL) } - - if textData.Category.Title != "Test Category" { - t.Errorf("Expected category title to be 'Test Category', got %s", textData.Category.Title) - } } diff --git a/internal/llm/gemini.go b/internal/llm/gemini.go new file mode 100644 index 0000000..4c18e0d --- /dev/null +++ b/internal/llm/gemini.go @@ -0,0 +1,103 @@ +package llm + +import ( + "context" + "errors" + "log" + "time" + + "google.golang.org/genai" + "miniflux-digest/internal/app/services" +) + +const ( + Model = "gemini-2.5-pro" + maxRetries = 3 + Temperature float32 = 0.4 +) + +type modelClient interface { + GenerateContent(ctx context.Context, model string, parts []*genai.Content, config *genai.GenerateContentConfig) (*genai.GenerateContentResponse, error) +} + +type GeminiService struct { + client modelClient + modelName string + sleep func(time.Duration) +} + +func NewGeminiService(apiKey string) (services.LLMService, error) { + if apiKey == "" { + return &GeminiService{modelName: Model, sleep: time.Sleep}, nil + } + + ctx := context.Background() + clientConfig := genai.ClientConfig{APIKey: apiKey} + client, err := genai.NewClient(ctx, &clientConfig) + if err != nil { + return nil, err + } + + return &GeminiService{client: client.Models, modelName: Model, sleep: time.Sleep}, nil +} + +func (s *GeminiService) generateContentWithRetry(ctx context.Context, prompt string, schema *genai.Schema) (*genai.GenerateContentResponse, error) { + var resp *genai.GenerateContentResponse + var err error + + for i := range maxRetries { + resp, err = s.client.GenerateContent(ctx, s.modelName, genai.Text(prompt), &genai.GenerateContentConfig{ + ResponseMIMEType: "application/json", + ResponseSchema: schema, + Temperature: genai.Ptr(Temperature), + }) + + if err == nil { + return resp, nil + } + + log.Printf("LLM call failed: %v", err) + + var apiError genai.APIError + if errors.As(err, &apiError) { + log.Printf("LLM API error: Code=%d, Status=%s, Message=%s", apiError.Code, apiError.Status, apiError.Message) + if apiError.Code == 503 || apiError.Code == 500 { + log.Printf("Retrying LLM call (%d/%d)...", i+1, maxRetries) + s.sleep(time.Second * time.Duration(i+1)) + continue + } + } + + return nil, err + } + + return nil, err +} + +func (s *GeminiService) GenerateContent(ctx context.Context, prompt string, schema *genai.Schema) ([]byte, error) { + if s.client == nil { + return nil, errors.New("LLM service is disabled: no API key provided") + } + + resp, err := s.generateContentWithRetry(ctx, prompt, schema) + if err != nil { + return nil, err + } + + candidates := resp.Candidates + + if len(candidates) == 0 || len(candidates[0].Content.Parts) == 0 { + log.Println("LLM: No content returned from LLM (empty candidates or parts).") + return nil, errors.New("no content returned from LLM") + } + + candidate := candidates[0] + part := candidate.Content.Parts[0] + jsonData := []byte(part.Text) + + if (candidate.FinishReason != genai.FinishReasonStop) { + log.Printf("LLM: Unexpected finish reason: %s", candidate.FinishReason) + } + + return jsonData, nil +} \ No newline at end of file diff --git a/internal/llm/gemini_test.go b/internal/llm/gemini_test.go new file mode 100644 index 0000000..a645e8b --- /dev/null +++ b/internal/llm/gemini_test.go @@ -0,0 +1,78 @@ +package llm + +import ( + "context" + "errors" + "testing" + "time" + + "google.golang.org/genai" +) + +func TestGenerateContentWithRetry(t *testing.T) { + mockClient := &mockModelClient{} + var slept time.Duration + service := &GeminiService{ + client: mockClient, + modelName: "test-model", + sleep: func(d time.Duration) { + slept += d + }, + } + + // Setup the mock to fail twice with a 503 error, then succeed. + attempts := 0 + mockClient.GenerateContentFunc = func(ctx context.Context, model string, parts []*genai.Content, config *genai.GenerateContentConfig) (*genai.GenerateContentResponse, error) { + attempts++ + if attempts <= 2 { + return nil, genai.APIError{Code: 503, Message: "Model overloaded"} + } + return &genai.GenerateContentResponse{}, nil + } + + _, err := service.generateContentWithRetry(context.Background(), "test prompt", nil) + + if err != nil { + t.Fatalf("Expected generateContentWithRetry to succeed, but it failed: %v", err) + } + + if mockClient.callCount != 3 { + t.Errorf("Expected GenerateContent to be called 3 times, but it was called %d times", mockClient.callCount) + } + + expectedSleep := 3 * time.Second // 1s + 2s + if slept != expectedSleep { + t.Errorf("Expected to sleep for %v, but slept for %v", expectedSleep, slept) + } +} + +func TestGenerateContentWithRetry_NonRecoverableError(t *testing.T) { + mockClient := &mockModelClient{} + var slept time.Duration + service := &GeminiService{ + client: mockClient, + modelName: "test-model", + sleep: func(d time.Duration) { + slept += d + }, + } + + // Setup the mock to fail with a non-503 error. + mockClient.GenerateContentFunc = func(ctx context.Context, model string, parts []*genai.Content, config *genai.GenerateContentConfig) (*genai.GenerateContentResponse, error) { + return nil, errors.New("a non-recoverable error") + } + + _, err := service.generateContentWithRetry(context.Background(), "test prompt", nil) + + if err == nil { + t.Fatal("Expected generateContentWithRetry to fail, but it succeeded") + } + + if mockClient.callCount != 1 { + t.Errorf("Expected GenerateContent to be called 1 time, but it was called %d times", mockClient.callCount) + } + + if slept != 0 { + t.Errorf("Expected not to sleep, but slept for %v", slept) + } +} diff --git a/internal/llm/llm.go b/internal/llm/llm.go index 031f88a..f5f2208 100644 --- a/internal/llm/llm.go +++ b/internal/llm/llm.go @@ -1,88 +1 @@ -package llm - -import ( - "context" - "errors" - "log" - "strings" - "time" - - "google.golang.org/genai" -) - -const ( - GeminiModel = "gemini-1.5-flash" - maxRetries = 3 -) - -type modelClient interface { - GenerateContent(ctx context.Context, model string, parts []*genai.Content, config *genai.GenerateContentConfig) (*genai.GenerateContentResponse, error) -} - -type GeminiService struct { - client modelClient - modelName string -} - -func NewGeminiService(apiKey string) (*GeminiService, error) { - if apiKey == "" { - return &GeminiService{modelName: GeminiModel}, nil - } - - ctx := context.Background() - clientConfig := genai.ClientConfig{APIKey: apiKey} - client, err := genai.NewClient(ctx, &clientConfig) - if err != nil { - return nil, err - } - - return &GeminiService{client: client.Models, modelName: GeminiModel}, nil -} - -func (s *GeminiService) generateContentWithRetry(ctx context.Context, prompt string, schema *genai.Schema) (*genai.GenerateContentResponse, error) { - var resp *genai.GenerateContentResponse - var err error - - for i := 0; i < maxRetries; i++ { - resp, err = s.client.GenerateContent(ctx, s.modelName, genai.Text(prompt), &genai.GenerateContentConfig{ - ResponseMIMEType: "application/json", - ResponseSchema: schema, - }) - if err == nil { - return resp, nil - } - - var apiError *genai.APIError - if errors.As(err, &apiError) { - log.Printf("LLM API error: Code=%d, Status=%s, Message=%s", apiError.Code, apiError.Status, apiError.Message) - if apiError.Code == 503 && strings.Contains(apiError.Message, "overloaded") { - log.Printf("Retrying LLM call (%d/%d)...", i+1, maxRetries-1) - time.Sleep(time.Second * time.Duration(i+1)) // Wait 1, 2 seconds - continue - } - } - - return nil, err - } - - return nil, err -} - -func (s *GeminiService) GenerateContent(ctx context.Context, prompt string, schema *genai.Schema) (string, error) { - if s.client == nil { - return "", errors.New("LLM service is disabled: no API key provided") - } - - resp, err := s.generateContentWithRetry(ctx, prompt, schema) - if err != nil { - return "", err - } - - if len(resp.Candidates) == 0 || len(resp.Candidates[0].Content.Parts) == 0 { - return "", errors.New("no content returned from LLM") - } - - // The API returns JSON as a Text part when a schema is provided - textPart := resp.Text() - return textPart, nil -} +package llm \ No newline at end of file diff --git a/internal/llm/llm_test.go b/internal/llm/llm_test.go index 1bc11ed..061477d 100644 --- a/internal/llm/llm_test.go +++ b/internal/llm/llm_test.go @@ -8,27 +8,20 @@ import ( "google.golang.org/genai" ) -// MockLLMService is a mock implementation of the llm.LLMService interface. -type MockLLMService struct{} - -func (m *MockLLMService) GenerateContent(ctx context.Context, prompt string, schema *genai.Schema) (string, error) { - return "", nil -} - -// mockModelClient is a mock implementation of the modelClient interface. type mockModelClient struct { - GenerateContentFunc func(ctx context.Context, model string, contents []*genai.Content, config *genai.GenerateContentConfig) (*genai.GenerateContentResponse, error) + GenerateContentFunc func(ctx context.Context, model string, parts []*genai.Content, config *genai.GenerateContentConfig) (*genai.GenerateContentResponse, error) + callCount int } -func (m *mockModelClient) GenerateContent(ctx context.Context, model string, contents []*genai.Content, config *genai.GenerateContentConfig) (*genai.GenerateContentResponse, error) { +func (m *mockModelClient) GenerateContent(ctx context.Context, model string, parts []*genai.Content, config *genai.GenerateContentConfig) (*genai.GenerateContentResponse, error) { + m.callCount++ if m.GenerateContentFunc != nil { - return m.GenerateContentFunc(ctx, model, contents, config) + return m.GenerateContentFunc(ctx, model, parts, config) } return nil, errors.New("GenerateContentFunc not implemented") } func TestNewGeminiService(t *testing.T) { - // Test with an empty API key service, err := NewGeminiService("") if err != nil { t.Fatalf("NewGeminiService with empty API key should not return an error, but got: %v", err) @@ -36,20 +29,19 @@ func TestNewGeminiService(t *testing.T) { if service == nil { t.Fatal("Service should not be nil") } - if service.client != nil { - t.Error("Service client should be nil for empty API key") - } } func TestGeminiService_GenerateContent_Success(t *testing.T) { - // Create a mock model that returns a successful response + expectedJSON := "{\"overview_summary\": \"test summary\", \"group_summaries\": []}" mockClient := &mockModelClient{ GenerateContentFunc: func(ctx context.Context, model string, contents []*genai.Content, config *genai.GenerateContentConfig) (*genai.GenerateContentResponse, error) { return &genai.GenerateContentResponse{ Candidates: []*genai.Candidate{ { Content: &genai.Content{ - Parts: []*genai.Part{{Text: "test response"}}, + Parts: []*genai.Part{ + {Text: expectedJSON}, + }, }, }, }, @@ -57,24 +49,21 @@ func TestGeminiService_GenerateContent_Success(t *testing.T) { }, } - // Create a GeminiService with the mock model service := &GeminiService{client: mockClient, modelName: "test-model"} - // Call GenerateContent and assert that it returns the expected response - resp, err := service.GenerateContent(context.Background(), "test prompt", nil) + respBytes, err := service.GenerateContent(context.Background(), "test prompt", nil) if err != nil { t.Fatalf("GenerateContent should not return an error, but got: %v", err) } - if resp != "test response" { - t.Errorf("Expected response to be 'test response', but got: %s", resp) + + if string(respBytes) != expectedJSON { + t.Errorf("Expected response to be '%s', but got: '%s'", expectedJSON, string(respBytes)) } } func TestGeminiService_GenerateContent_Disabled(t *testing.T) { - // Create a GeminiService with a nil client service := &GeminiService{client: nil} - // Call GenerateContent and assert that it returns an error _, err := service.GenerateContent(context.Background(), "test prompt", nil) if err == nil { t.Fatal("GenerateContent should return an error when the service is disabled") @@ -85,17 +74,14 @@ func TestGeminiService_GenerateContent_Disabled(t *testing.T) { } func TestGeminiService_GenerateContent_Error(t *testing.T) { - // Create a mock model that returns an an error mockClient := &mockModelClient{ GenerateContentFunc: func(ctx context.Context, model string, contents []*genai.Content, config *genai.GenerateContentConfig) (*genai.GenerateContentResponse, error) { return nil, errors.New("test error") }, } - // Create a GeminiService with the mock model service := &GeminiService{client: mockClient, modelName: "test-model"} - // Call GenerateContent and assert that it returns an error _, err := service.GenerateContent(context.Background(), "test prompt", nil) if err == nil { t.Fatal("GenerateContent should return an error") @@ -106,17 +92,14 @@ func TestGeminiService_GenerateContent_Error(t *testing.T) { } func TestGeminiService_GenerateContent_EmptyResponse(t *testing.T) { - // Create a mock model that returns an empty response mockClient := &mockModelClient{ GenerateContentFunc: func(ctx context.Context, model string, contents []*genai.Content, config *genai.GenerateContentConfig) (*genai.GenerateContentResponse, error) { return &genai.GenerateContentResponse{}, nil }, } - // Create a GeminiService with the mock model service := &GeminiService{client: mockClient, modelName: "test-model"} - // Call GenerateContent and assert that it returns an error _, err := service.GenerateContent(context.Background(), "test prompt", nil) if err == nil { t.Fatal("GenerateContent should return an error for an empty response") diff --git a/internal/models/models.go b/internal/models/models.go index 80284ad..6671988 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -1,26 +1,79 @@ package models import ( - miniflux "miniflux.app/v2/client" "time" ) +type Entry struct { + ID int64 + Title string + URL string + Content string + FeedID int64 + FeedTitle string + GroupID int64 + GroupTitle string + CommentsURL string + Date time.Time +} + type FeedIcon struct { FeedID int64 Data string } -type HTMLTemplateData struct { - Category *miniflux.Category - Entries *miniflux.Entries +type OverviewTemplateData struct { + Entries []*Entry GeneratedDate time.Time FeedIcons []*FeedIcon EntryGroups []*EntryGroup - Summary string + PrimaryGroups []*PrimaryGroupDigestData + OverviewSummary string + TotalEntries int + TotalFeeds int MinifluxHost string } type EntryGroup struct { - Title string - Entries []*miniflux.Entry + Title string + Summary string + Entries []*Entry + Slug string + TotalEntries int + TotalFeeds int +} + +type GroupTemplateData struct { + Title string + Summary string + Entries []*Entry + GeneratedDate time.Time + FeedIcons []*FeedIcon + MinifluxHost string +} + +type GroupedEntriesTemplateData struct { + EntryGroup *EntryGroup + GeneratedDate time.Time + FeedIcons []*FeedIcon + MinifluxHost string +} + +type PrimaryGroupDigestData struct { + ID int64 + Title string + Slug string + SubGroups []*EntryGroup + Summary string + TotalEntries int + TotalFeeds int +} + +type GroupedDigestPageData struct { + PrimaryGroup *PrimaryGroupDigestData + FeedIcons []*FeedIcon + MinifluxHost string + GeneratedDate time.Time + TotalEntries int + TotalFeeds int } diff --git a/internal/processor/processor.go b/internal/processor/processor.go index c872c8e..dfb4ba5 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -2,35 +2,68 @@ package processor import ( "log" + "os" "miniflux-digest/internal/app" + "miniflux-digest/internal/models" + + miniflux "miniflux.app/v2/client" ) -func CategoryDigestJob(application *app.App, rawData *app.RawCategoryData, markAsRead bool) { - data := application.DigestService.BuildDigestData(rawData.Category, rawData.Entries, rawData.Icons, application.Config.Digest.GroupBy, application.Config.Miniflux.Host) +func ProcessAndSendDigest(application *app.App) (*os.File, []*os.File, *models.OverviewTemplateData, error) { + log.Println("Starting to process digest...") - if len(*data.Entries) > 0 { - file, err := application.ArchiveService.MakeArchiveHTML(data, application.Config.Digest.Compress) - if err != nil { - log.Printf("Error generating File for category %s: %v", data.Category.Title, err) - return - } - defer func() { - if err := file.Close(); err != nil { - log.Printf("Error closing file for category '%s': %v", data.Category.Title, err) + entries, err := application.MinifluxClientService.GetAllUnreadEntries() + if err != nil { + log.Printf("Error getting all unread entries: %v", err) + return nil, nil, nil, err + } + + log.Printf("Found %d unread entries", len(entries)) + + icons := make(map[int64]*models.FeedIcon) + for _, entry := range entries { + if _, ok := icons[entry.FeedID]; !ok { + icon, err := application.MinifluxClientService.FeedIcon(entry.FeedID) + if err != nil { + log.Printf("Warning: failed to fetch icon for feed %d: %v", entry.FeedID, err) + continue } - }() - - err = application.EmailService.Send(application.Config, file, data) - - if err != nil { - log.Printf("Error sending email for category '%s': %v", data.Category.Title, err) - } - - if markAsRead { - if err := application.MinifluxClientService.MarkCategoryAsRead(data.Category.ID); err != nil { - log.Printf("Error marking category as read for category '%s': %v", data.Category.Title, err) + icons[entry.FeedID] = &models.FeedIcon{ + FeedID: icon.FeedID, + Data: icon.Data, } } } + + data := application.DigestService.BuildDigestData( + entries, + icons, + application.Config.Digest.GroupBy, + application.Config.Digest.SubGroupBy, + application.Config.Digest.SortBy, + application.Config.Miniflux.Host, + ) + + overviewFile, groupedEntryFiles, err := application.ArchiveService.MakeArchiveHTML(data, application.Config.Digest.Compress) + if err != nil { + log.Printf("Error generating archive HTML: %v", err) + return nil, nil, nil, err + } + + if application.Config.Digest.MarkAsRead { + var entryIDs []int64 + for _, entry := range entries { + entryIDs = append(entryIDs, entry.ID) + } + + if len(entryIDs) > 0 { + err := application.MinifluxClientService.UpdateEntries(entryIDs, miniflux.EntryStatusRead) + if err != nil { + log.Printf("Error marking entries as read: %v", err) + } + } + } + + return overviewFile, groupedEntryFiles, data, nil } diff --git a/internal/processor/processor_test.go b/internal/processor/processor_test.go index 249974f..3fc4d20 100644 --- a/internal/processor/processor_test.go +++ b/internal/processor/processor_test.go @@ -1,171 +1,130 @@ package processor import ( - "bytes" - "errors" - "io" "log" + "os" + "testing" + "time" + "miniflux-digest/internal/app" "miniflux-digest/internal/config" "miniflux-digest/internal/models" - "miniflux-digest/internal/testutil" - "miniflux-digest/internal/digest" - "os" - "testing" miniflux "miniflux.app/v2/client" ) -func TestCategoryDigestJob(t *testing.T) { - log.SetOutput(io.Discard) +type mockMinifluxClientService struct { + getAllUnreadEntriesFunc func() ([]*models.Entry, error) + feedIconFunc func(feedID int64) (*models.FeedIcon, error) + updateEntriesFunc func(entryIDs []int64, status string) error +} + +func (m *mockMinifluxClientService) FeedIcon(feedID int64) (*models.FeedIcon, error) { + return m.feedIconFunc(feedID) +} +func (m *mockMinifluxClientService) GetAllUnreadEntries() ([]*models.Entry, error) { + return m.getAllUnreadEntriesFunc() +} +func (m *mockMinifluxClientService) UpdateEntries(entryIDs []int64, status string) error { + return m.updateEntriesFunc(entryIDs, status) +} + +type mockDigestService struct { + buildDigestDataFunc func(entries []*models.Entry, icons map[int64]*models.FeedIcon, groupBy string, subGroupBy string, sortBy string, minifluxHost string) *models.OverviewTemplateData +} + +func (m *mockDigestService) BuildDigestData(entries []*models.Entry, icons map[int64]*models.FeedIcon, groupBy string, subGroupBy string, sortBy string, minifluxHost string) *models.OverviewTemplateData { + return m.buildDigestDataFunc(entries, icons, groupBy, subGroupBy, sortBy, minifluxHost) +} + +type mockArchiveService struct { + makeArchiveHTMLFunc func(data *models.OverviewTemplateData, compress bool) (*os.File, []*os.File, error) + cleanArchiveFunc func(maxAge time.Duration) +} + +func (m *mockArchiveService) MakeArchiveHTML(data *models.OverviewTemplateData, compress bool) (*os.File, []*os.File, error) { + return m.makeArchiveHTMLFunc(data, compress) +} +func (m *mockArchiveService) CleanArchive(maxAge time.Duration) { + m.cleanArchiveFunc(maxAge) +} + +func TestProcessAndSendDigest_Success(t *testing.T) { + log.SetOutput(os.Stdout) defer log.SetOutput(os.Stderr) - t.Run("no entries", func(t *testing.T) { - mockApp := app.NewApp( - app.WithConfig(&config.Config{}), - app.WithMinifluxClientService(&testutil.MockMinifluxClient{}), - app.WithDigestService(&testutil.MockDigestService{ - BuildDigestDataFunc: func(category *miniflux.Category, entries *miniflux.Entries, icons map[int64]*models.FeedIcon, groupBy digest.GroupingType, minifluxHost string) *models.HTMLTemplateData { - return &models.HTMLTemplateData{Entries: &miniflux.Entries{}} - }, - }), - app.WithArchiveService(&testutil.MockArchiveService{}), - app.WithEmailService(&testutil.MockEmailService{}), - ) - data := &app.RawCategoryData{Entries: &miniflux.Entries{}} - CategoryDigestJob(mockApp, data, true) - }) + expectedEntries := []*models.Entry{ + {ID: 1, FeedID: 101, Title: "Entry 1"}, + {ID: 2, FeedID: 102, Title: "Entry 2"}, + } + expectedIcons := map[int64]*models.FeedIcon{ + 101: {FeedID: 101, Data: "icon1"}, + 102: {FeedID: 102, Data: "icon2"}, + } + expectedOverviewData := &models.OverviewTemplateData{OverviewSummary: "Test Digest"} - t.Run("error making archive html", func(t *testing.T) { - mockApp := app.NewApp( - app.WithConfig(&config.Config{}), - app.WithMinifluxClientService(&testutil.MockMinifluxClient{}), - app.WithDigestService(&testutil.MockDigestService{ - BuildDigestDataFunc: func(category *miniflux.Category, entries *miniflux.Entries, icons map[int64]*models.FeedIcon, groupBy digest.GroupingType, minifluxHost string) *models.HTMLTemplateData { - return &models.HTMLTemplateData{Entries: &miniflux.Entries{{ID: 1}}, Category: &miniflux.Category{Title: "title"}} - }, - }), - app.WithEmailService(&testutil.MockEmailService{}), - app.WithArchiveService(&testutil.MockArchiveService{ - MakeArchiveHTMLFunc: func(data *models.HTMLTemplateData, compress bool) (*os.File, error) { - return nil, errors.New("archive generation failed") - }, - }), - ) - data := &app.RawCategoryData{Entries: &miniflux.Entries{{ID: 1}}} + mockMinifluxClient := &mockMinifluxClientService{ + getAllUnreadEntriesFunc: func() ([]*models.Entry, error) { + return expectedEntries, nil + }, + feedIconFunc: func(feedID int64) (*models.FeedIcon, error) { + return expectedIcons[feedID], nil + }, + updateEntriesFunc: func(entryIDs []int64, status string) error { + if status != miniflux.EntryStatusRead { + t.Errorf("Expected status %s, got %s", miniflux.EntryStatusRead, status) + } + return nil + }, + } + mockDigestService := &mockDigestService{ + buildDigestDataFunc: func(entries []*models.Entry, icons map[int64]*models.FeedIcon, groupBy string, subGroupBy string, sortBy string, minifluxHost string) *models.OverviewTemplateData { + return expectedOverviewData + }, + } + mockArchiveService := &mockArchiveService{ + makeArchiveHTMLFunc: func(data *models.OverviewTemplateData, compress bool) (*os.File, []*os.File, error) { + overviewFile, _ := os.CreateTemp("", "overview-*.html") + groupedFile1, _ := os.CreateTemp("", "grouped-*.html") + return overviewFile, []*os.File{groupedFile1}, nil + }, + } - var buf bytes.Buffer - log.SetOutput(&buf) - - CategoryDigestJob(mockApp, data, true) - - if !bytes.Contains(buf.Bytes(), []byte("archive generation failed")) { - t.Error("Expected error log for archive generation, but not found") - } - }) - - t.Run("error sending email", func(t *testing.T) { - mockApp := app.NewApp( - app.WithConfig(&config.Config{}), - app.WithMinifluxClientService(&testutil.MockMinifluxClient{}), - app.WithDigestService(&testutil.MockDigestService{ - BuildDigestDataFunc: func(category *miniflux.Category, entries *miniflux.Entries, icons map[int64]*models.FeedIcon, groupBy digest.GroupingType, minifluxHost string) *models.HTMLTemplateData { - return &models.HTMLTemplateData{Entries: &miniflux.Entries{{ID: 1}}, Category: &miniflux.Category{Title: "title"}, FeedIcons: []*models.FeedIcon{}} - }, - }), - app.WithArchiveService(&testutil.MockArchiveService{ - MakeArchiveHTMLFunc: func(data *models.HTMLTemplateData, compress bool) (*os.File, error) { - return os.CreateTemp("", "test-archive-*.html") - }, - }), - app.WithEmailService(&testutil.MockEmailService{ - SendFunc: func(cfg *config.Config, file *os.File, data *models.HTMLTemplateData) error { - return errors.New("email sending failed") - }, - }), - ) - data := &app.RawCategoryData{Entries: &miniflux.Entries{{ID: 1}}} - - var buf bytes.Buffer - log.SetOutput(&buf) - - CategoryDigestJob(mockApp, data, true) - - if !bytes.Contains(buf.Bytes(), []byte("email sending failed")) { - t.Error("Expected error log for email sending, but not found") - } - }) - - t.Run("mark as read is called", func(t *testing.T) { - markAsReadCalled := false - mockMinifluxClient := &testutil.MockMinifluxClient{ - MarkAsReadFunc: func(categoryID int64) error { - markAsReadCalled = true - return nil + mockApp := app.NewApp( + app.WithConfig(&config.Config{ + Digest: config.ConfigDigest{ + MarkAsRead: true, + Compress: true, + GroupBy: "category", + SubGroupBy: "feed", + SortBy: "date", }, - } - mockApp := app.NewApp( - app.WithConfig(&config.Config{}), - app.WithMinifluxClientService(mockMinifluxClient), - app.WithDigestService(&testutil.MockDigestService{ - BuildDigestDataFunc: func(category *miniflux.Category, entries *miniflux.Entries, icons map[int64]*models.FeedIcon, groupBy digest.GroupingType, minifluxHost string) *models.HTMLTemplateData { - return &models.HTMLTemplateData{Entries: &miniflux.Entries{{ID: 1}}, Category: &miniflux.Category{Title: "title"}, FeedIcons: []*models.FeedIcon{}} - }, - }), - app.WithArchiveService(&testutil.MockArchiveService{ - MakeArchiveHTMLFunc: func(data *models.HTMLTemplateData, compress bool) (*os.File, error) { - return os.CreateTemp("", "test-archive-*.html") - }, - }), - app.WithEmailService(&testutil.MockEmailService{ - SendFunc: func(cfg *config.Config, file *os.File, data *models.HTMLTemplateData) error { - return nil - }, - }), - ) - data := &app.RawCategoryData{Entries: &miniflux.Entries{{ID: 1}}} + Miniflux: config.ConfigMiniflux{Host: "http://miniflux.test"}, + }), + app.WithMinifluxClientService(mockMinifluxClient), + app.WithDigestService(mockDigestService), + app.WithArchiveService(mockArchiveService), + ) - CategoryDigestJob(mockApp, data, true) + overviewFile, groupedEntryFiles, data, err := ProcessAndSendDigest(mockApp) - if !markAsReadCalled { - t.Error("Expected MarkCategoryAsRead to be called, but it was not") - } - }) + if err != nil { + t.Fatalf("ProcessAndSendDigest returned an unexpected error: %v", err) + } + if overviewFile == nil { + t.Error("Overview file should not be nil") + } + if len(groupedEntryFiles) == 0 { + t.Error("Grouped entry files should not be empty") + } + if data != expectedOverviewData { + t.Errorf("Expected overview data %v, got %v", expectedOverviewData, data) + } - t.Run("error marking as read", func(t *testing.T) { - var buf bytes.Buffer - log.SetOutput(&buf) - - mockMinifluxClient := &testutil.MockMinifluxClient{ - MarkAsReadFunc: func(categoryID int64) error { - return errors.New("mark as read failed") - }, - } - mockApp := app.NewApp( - app.WithConfig(&config.Config{}), - app.WithMinifluxClientService(mockMinifluxClient), - app.WithDigestService(&testutil.MockDigestService{ - BuildDigestDataFunc: func(category *miniflux.Category, entries *miniflux.Entries, icons map[int64]*models.FeedIcon, groupBy digest.GroupingType, minifluxHost string) *models.HTMLTemplateData { - return &models.HTMLTemplateData{Entries: &miniflux.Entries{{ID: 1}}, Category: &miniflux.Category{Title: "title"}, FeedIcons: []*models.FeedIcon{}} - }, - }), - app.WithArchiveService(&testutil.MockArchiveService{ - MakeArchiveHTMLFunc: func(data *models.HTMLTemplateData, compress bool) (*os.File, error) { - return os.CreateTemp("", "test-archive-*.html") - }, - }), - app.WithEmailService(&testutil.MockEmailService{ - SendFunc: func(cfg *config.Config, file *os.File, data *models.HTMLTemplateData) error { - return nil - }, - }), - ) - data := &app.RawCategoryData{Entries: &miniflux.Entries{{ID: 1}}} - - CategoryDigestJob(mockApp, data, true) - - if !bytes.Contains(buf.Bytes(), []byte("mark as read failed")) { - t.Error("Expected error log for marking as read, but not found") - } - }) -} \ No newline at end of file + _ = overviewFile.Close() + _ = os.Remove(overviewFile.Name()) + for _, f := range groupedEntryFiles { + _ = f.Close() + _ = os.Remove(f.Name()) + } +} diff --git a/internal/templates/email.gotxt b/internal/templates/email.gotxt index 008ea25..9d14e89 100644 --- a/internal/templates/email.gotxt +++ b/internal/templates/email.gotxt @@ -1,9 +1,19 @@ -{{ if .Summary }} -{{ .Summary }} -{{ end }} -{{ if .URL }}you can view them them at: +Hello, + +Your daily digest of {{ .Entries | len }} entries from {{ .EntryGroups | len }} feeds is ready. + +{{ if .OverviewSummary -}} +Overall Summary: +{{ .OverviewSummary }} +{{- end }} + +You have {{.TotalEntries}} entries from {{.TotalFeeds}} feeds. + +{{ if .URL -}} +You can view the full digest online: {{ .URL }} -or download the attachment. -{{ else }} -The entries are attached. -{{ end }} +{{- end }} + +{{ if .EntryGroups -}} +The individual digests are attached to this email for offline reading. +{{- end }} diff --git a/internal/templates/entries.gohtml b/internal/templates/entries.gohtml deleted file mode 100644 index 266ff33..0000000 --- a/internal/templates/entries.gohtml +++ /dev/null @@ -1,85 +0,0 @@ - - - - - - - {{.Category.Title}} - - - - -
-
-

{{.Category.Title}}

- category feed generated on {{.GeneratedDate.Format - "Jan 2, 2006"}} -
- -
-

Summary

-

{{textToHTML .Summary}}

-
- -
- {{if .EntryGroups}} - {{range .EntryGroups}} -

{{.Title}}

- {{range .Entries}} -
-
- - {{.Title}} - -
- {{ htmlEscape .Content}} -
-
-
-
-
-
{{.Feed.Title}}
-
-
{{.Date.Format - "Jan 2"}}
-
- permalink -
- {{if .CommentsURL}} -
- comments -
- {{end}} -
- source -
-
-
- {{end}} - {{end}} - {{else}} -
No unread entries in this category.
- {{end}} -
-
- - - - - diff --git a/internal/templates/funcs.go b/internal/templates/funcs.go index f461bd5..31a92b6 100644 --- a/internal/templates/funcs.go +++ b/internal/templates/funcs.go @@ -11,12 +11,10 @@ import ( //go:embed static var staticFS embed.FS -func FuncMap() template.FuncMap { - return template.FuncMap{ - "include": include, - "textToHTML": textToHTML, - "htmlEscape": htmlEscape, - } +var TemplateFuncs = template.FuncMap{ + "include": include, + "textToHTML": textToHTML, + "htmlEscape": htmlEscape, } func include(filename string) (any, error) { @@ -41,4 +39,4 @@ func textToHTML(s string) template.HTML { func htmlEscape(s string) template.HTML { return template.HTML(s) -} \ No newline at end of file +} diff --git a/internal/templates/grouped-entries.gohtml b/internal/templates/grouped-entries.gohtml new file mode 100644 index 0000000..0822209 --- /dev/null +++ b/internal/templates/grouped-entries.gohtml @@ -0,0 +1,85 @@ + + + + + + + {{.PrimaryGroup.Title}} + + + + +
+
+

{{.PrimaryGroup.Title}}

+ generated on {{.GeneratedDate.Format + "Jan 2, 2006"}} +
+ +
+

Summary

+ {{if .PrimaryGroup.Summary}} +

{{textToHTML .PrimaryGroup.Summary}}

+ {{end}} +

+ This digest has {{.PrimaryGroup.TotalEntries}} entries from {{.PrimaryGroup.TotalFeeds}} feeds. +

+
+ +
+ {{range .PrimaryGroup.SubGroups}} +

{{.Title}}

+ {{range .Entries}} +
+
+ + {{.Title}} + +
+ {{ htmlEscape .Content}} +
+
+
+
+
+
{{.FeedTitle}}
+
+
{{.Date.Format + "Jan 2"}}
+
+ permalink +
+ {{if .CommentsURL}} +
+ comments +
+ {{end}} +
+ source +
+
+
+ {{end}} + {{end}} +
+
+ + + + + diff --git a/internal/templates/overview.gohtml b/internal/templates/overview.gohtml new file mode 100644 index 0000000..770cce3 --- /dev/null +++ b/internal/templates/overview.gohtml @@ -0,0 +1,41 @@ + + + + Miniflux Digest Overview + + + +
+
+

Miniflux Digest Overview

+ generated on {{.GeneratedDate.Format "Jan 2, 2006"}} +
+ +
+

Summary

+ {{if .OverviewSummary}} +

{{.OverviewSummary}}

+ {{end}} +

You have {{.TotalEntries}} entries from {{.TotalFeeds}} feeds.

+
+ +
+

Grouped Digests

+ +
+
+ + diff --git a/internal/templates/static/style.css b/internal/templates/static/style.css index 1075f79..b0d31e8 100644 --- a/internal/templates/static/style.css +++ b/internal/templates/static/style.css @@ -89,7 +89,7 @@ section.header .date { color: var(--header-date-color); } -section.summary p { +section.summary p:first-of-type { margin: 1rem auto; padding: 0.5rem 1rem; background-color: var(--container-background); @@ -256,4 +256,4 @@ details summary { details summary > * { display: inline; -} \ No newline at end of file +} diff --git a/internal/templates/templates.go b/internal/templates/templates.go index 5ca0762..34565a9 100644 --- a/internal/templates/templates.go +++ b/internal/templates/templates.go @@ -9,8 +9,8 @@ import ( ) type EmailTemplateData struct { - models.HTMLTemplateData - URL string + models.OverviewTemplateData + URL string Summary string } @@ -18,24 +18,32 @@ type EmailTemplateData struct { var embedFS embed.FS var ( - ArchiveTemplate *htmlTemplate.Template - EmailTemplate *textTemplate.Template + ArchiveTemplate *htmlTemplate.Template + OverviewTemplate *htmlTemplate.Template + EmailTemplate *textTemplate.Template ) func init() { var err error - archiveTemplateName := "entries.gohtml" + archiveTemplateName := "grouped-entries.gohtml" + overviewTemplateName := "overview.gohtml" emailTemplateName := "email.gotxt" - ArchiveTemplate, err = htmlTemplate.New(archiveTemplateName).Funcs(FuncMap()).ParseFS(embedFS, archiveTemplateName) + ArchiveTemplate, err = htmlTemplate.New(archiveTemplateName).Funcs(TemplateFuncs).ParseFS(embedFS, archiveTemplateName) if err != nil { log.Fatalf("Error parsing archive template: %v", err) } - EmailTemplate, err = textTemplate.New(emailTemplateName).ParseFS(embedFS, emailTemplateName) + OverviewTemplate, err = htmlTemplate.New(overviewTemplateName).Funcs(TemplateFuncs).ParseFS(embedFS, overviewTemplateName) + + if err != nil { + log.Fatalf("Error parsing overview template: %v", err) + } + + EmailTemplate, err = textTemplate.New(emailTemplateName).Funcs(TemplateFuncs).ParseFS(embedFS, emailTemplateName) if err != nil { log.Fatalf("Error parsing email template: %v", err) } -} +} \ No newline at end of file diff --git a/internal/templates/templates_test.go b/internal/templates/templates_test.go index 74ec91e..906de7e 100644 --- a/internal/templates/templates_test.go +++ b/internal/templates/templates_test.go @@ -5,6 +5,7 @@ import ( "miniflux-digest/internal/models" "miniflux-digest/internal/testutil" "testing" + "time" ) func TestTemplates(t *testing.T) { @@ -18,11 +19,26 @@ func TestTemplates(t *testing.T) { } func TestArchiveTemplateExecution(t *testing.T) { - data := models.HTMLTemplateData{ - Category: testutil.NewMockCategory(), + mockSubGroup := &models.EntryGroup{ + Title: "Test SubGroup Title", + Summary: "Test SubGroup Summary", Entries: testutil.NewMockEntries(), - FeedIcons: testutil.NewMockFeedIcons(), + Slug: "test-subgroup-title", } + + mockPrimaryGroup := &models.PrimaryGroupDigestData{ + Title: "Test Primary Group Title", + Slug: "test-primary-group-title", + SubGroups: []*models.EntryGroup{mockSubGroup}, + Summary: "Test Primary Group Summary", + } + + data := models.GroupedDigestPageData{ + PrimaryGroup: mockPrimaryGroup, + FeedIcons: testutil.NewMockFeedIcons(), + MinifluxHost: "http://localhost:8080", + } + var buf bytes.Buffer err := ArchiveTemplate.Execute(&buf, data) if err != nil { @@ -34,13 +50,12 @@ func TestArchiveTemplateExecution(t *testing.T) { } func TestEmailTemplateExecution(t *testing.T) { - data := models.HTMLTemplateData{ - Category: testutil.NewMockCategory(), + data := models.OverviewTemplateData{ Entries: testutil.NewMockEntries(), FeedIcons: testutil.NewMockFeedIcons(), } textData := &EmailTemplateData{ - HTMLTemplateData: data, + OverviewTemplateData: data, URL: "https://example.com", } var buf bytes.Buffer @@ -52,3 +67,86 @@ func TestEmailTemplateExecution(t *testing.T) { t.Error("EmailTemplate execution resulted in empty output") } } + +func TestOverviewTemplateExecution(t *testing.T) { + data := models.OverviewTemplateData{ + Entries: testutil.NewMockEntries(), + GeneratedDate: time.Now(), + FeedIcons: testutil.NewMockFeedIcons(), + OverviewSummary: "This is a test summary.", + TotalEntries: 5, + TotalFeeds: 2, + MinifluxHost: "http://localhost:8080", + PrimaryGroups: []*models.PrimaryGroupDigestData{ + { + Title: "Category A", + Slug: "category-a", + Summary: "Summary for Category A", + SubGroups: []*models.EntryGroup{ + { + Title: "Feed 1", + Slug: "feed-1", + Entries: testutil.NewMockEntries(), + }, + }, + }, + }, + } + + var buf bytes.Buffer + err := OverviewTemplate.Execute(&buf, data) + if err != nil { + t.Errorf("Failed to execute OverviewTemplate: %v", err) + } + if buf.Len() == 0 { + t.Error("OverviewTemplate execution resulted in empty output") + } +} + +func TestTextToHTML(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "simple text", + input: "Hello World", + expected: "Hello World", + }, + { + name: "text with newline", + input: "Line 1\nLine 2", + expected: "Line 1
Line 2", + }, + { + name: "text with multiple newlines", + input: "Line 1\n\nLine 3", + expected: "Line 1

Line 3", + }, + { + name: "text with HTML special characters", + input: "", + expected: "<script>alert('hi')</script>", + }, + { + name: "text with newlines and HTML special characters", + input: "Line 1\n

Line 2

", + expected: "Line 1
<p>Line 2</p>", + }, + { + name: "empty string", + input: "", + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := textToHTML(tt.input) + if string(result) != tt.expected { + t.Errorf("textToHTML(%q): got %q, want %q", tt.input, result, tt.expected) + } + }) + } +} diff --git a/internal/testutil/mock_data.go b/internal/testutil/mock_data.go index 4be1246..44ed10a 100644 --- a/internal/testutil/mock_data.go +++ b/internal/testutil/mock_data.go @@ -10,7 +10,6 @@ import ( "time" "miniflux-digest/internal/models" - miniflux "miniflux.app/v2/client" ) func loadImageAsBase64(path string) string { @@ -26,72 +25,82 @@ func loadImageAsBase64(path string) string { return base64.StdEncoding.EncodeToString(file) } -func NewMockCategory() *miniflux.Category { - return &miniflux.Category{ - ID: 1, - Title: "Test Category", - } -} - -func NewMockFeedRed() *miniflux.Feed { - return &miniflux.Feed{ - ID: 1, - Title: "Tech News", - } -} - -func NewMockFeedYellow() *miniflux.Feed { - return &miniflux.Feed{ - ID: 2, - Title: "The Daily Bugle - A Very Long Feed Name to Test Overflow", - } -} - -func NewMockFeedGreen() *miniflux.Feed { - return &miniflux.Feed{ - ID: 3, - Title: "Comics", - } -} - -func NewMockEntry() *miniflux.Entry { - return &miniflux.Entry{ - ID: 1, - Title: "Test Entry", - Content: "

Test Content

", - Feed: NewMockFeedRed(), - } -} - func NewMockFeedIconRed() *models.FeedIcon { - feed := NewMockFeedRed() icon := loadImageAsBase64("internal/testutil/images/red.png") return &models.FeedIcon{ - FeedID: feed.ID, + FeedID: 1, Data: "image/png;base64," + icon, } } func NewMockFeedIconYellow() *models.FeedIcon { - feed := NewMockFeedYellow() icon := loadImageAsBase64("internal/testutil/images/yellow.png") return &models.FeedIcon{ - FeedID: feed.ID, + FeedID: 2, Data: "image/png;base64," + icon, } } func NewMockFeedIconGreen() *models.FeedIcon { - feed := NewMockFeedGreen() icon := loadImageAsBase64("internal/testutil/images/green.png") return &models.FeedIcon{ - FeedID: feed.ID, + FeedID: 3, Data: "image/png;base64," + icon, } } -func MockNumEntries(n int) *miniflux.Entries { - entries := make(miniflux.Entries, 0, n) +func MockNumEntries(n int) []*models.Entry { + return CreateMockEntries(n) +} + +func NewMockEntries() []*models.Entry { + return CreateMockEntries(20) +} + +func NewMockFeedIcons() []*models.FeedIcon { + return []*models.FeedIcon{ + NewMockFeedIconRed(), + NewMockFeedIconYellow(), + NewMockFeedIconGreen(), + } +} + +func NewMockEntryGroup() *models.EntryGroup { + return &models.EntryGroup{ + Title: "Test Group", + Summary: "This is a test group summary.", + Entries: CreateMockEntries(5), + Slug: "test-group", + } +} + +type feedTemplate struct { + FeedID int64 + FeedTitle string + GroupID int64 + GroupTitle string +} + +func CreateMockEntries(n int) []*models.Entry { + if n == 0 { + return []*models.Entry{ + // Category A + {ID: 1, Title: "Entry 1", Date: time.Date(2024, 1, 2, 10, 0, 0, 0, time.UTC), FeedID: 100, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A"}, + {ID: 3, Title: "Entry 3", Date: time.Date(2024, 1, 2, 11, 0, 0, 0, time.UTC), FeedID: 100, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A"}, + {ID: 5, Title: "Entry 5", Date: time.Date(2024, 1, 2, 12, 0, 0, 0, time.UTC), FeedID: 300, FeedTitle: "Feed C", GroupID: 1, GroupTitle: "Category A"}, + {ID: 6, Title: "Entry 6", Date: time.Date(2024, 1, 2, 13, 0, 0, 0, time.UTC), FeedID: 400, FeedTitle: "Feed D", GroupID: 1, GroupTitle: "Category A"}, + + // Category B + {ID: 2, Title: "Entry 2", Date: time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC), FeedID: 200, FeedTitle: "Feed B", GroupID: 2, GroupTitle: "Category B"}, + {ID: 4, Title: "Entry 4", Content: "Content of entry 4", Date: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC), FeedID: 200, FeedTitle: "Feed B", GroupID: 2, GroupTitle: "Category B"}, + {ID: 7, Title: "Entry 7", Date: time.Date(2024, 1, 1, 14, 0, 0, 0, time.UTC), FeedID: 500, FeedTitle: "Feed E", GroupID: 2, GroupTitle: "Category B"}, + + // Category C + {ID: 8, Title: "Entry 8", Date: time.Date(2024, 1, 3, 10, 0, 0, 0, time.UTC), FeedID: 600, FeedTitle: "Feed F", GroupID: 3, GroupTitle: "Category C"}, + {ID: 9, Title: "Entry 9", Date: time.Date(2024, 1, 3, 11, 0, 0, 0, time.UTC), FeedID: 700, FeedTitle: "Feed G", GroupID: 3, GroupTitle: "Category C"}, + } + } + titleTemplates := []string{ "Short title", "This is a much longer title to test how the UI handles overflow", @@ -105,46 +114,39 @@ func MockNumEntries(n int) *miniflux.Entries { "

Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo.

Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt.

Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem.

This is a single, very long line of text inside a pre block that should cause the container to overflow horizontally if the CSS is not correctly handling it. This_is_a_very_long_word_that_will_not_break_and_should_force_a_horizontal_scrollbar. a_b_c_d_e_f_g_h_i_j_k_l_m_n_o_p_q_r_s_t_u_v_w_x_y_z_a_b_c_d_e_f_g_h_i_j_k_l_m_n_o_p_q_r_s_t_u_v_w_x_y_z_a_b_c_d_e_f_g_h_i_j_k_l_m_n_o_p_q_r_s_t_u_v_w_x_y_z
", } - for i := 1; i <= n; i++ { - var feed *miniflux.Feed - switch (i - 1) % 3 { - case 0: - feed = NewMockFeedRed() - case 1: - feed = NewMockFeedYellow() - case 2: - feed = NewMockFeedGreen() - } + templates := []feedTemplate{ + {FeedID: 1, FeedTitle: "Feed A", GroupID: 1, GroupTitle: "Category A"}, + {FeedID: 2, FeedTitle: "Feed B", GroupID: 1, GroupTitle: "Category A"}, + {FeedID: 3, FeedTitle: "Feed C", GroupID: 2, GroupTitle: "Category B"}, + {FeedID: 4, FeedTitle: "Feed D", GroupID: 2, GroupTitle: "Category B"}, + {FeedID: 5, FeedTitle: "Feed E", GroupID: 2, GroupTitle: "Category B"}, + {FeedID: 6, FeedTitle: "Feed F", GroupID: 3, GroupTitle: "Category C"}, + {FeedID: 7, FeedTitle: "Feed G", GroupID: 3, GroupTitle: "Category C"}, + } + entries := make([]*models.Entry, 0, n) + for i := 1; i <= n; i++ { + template := templates[(i-1)%len(templates)] title := titleTemplates[(i-1)%len(titleTemplates)] content := contentTemplates[(i-1)%len(contentTemplates)] + var commentsURL string + if i%2 == 0 { + commentsURL = fmt.Sprintf("https://example.com/comments/%d", i) + } - entry := &miniflux.Entry{ + entry := &models.Entry{ ID: int64(i), - UserID: 1, - FeedID: feed.ID, - Status: miniflux.EntryStatusUnread, Title: fmt.Sprintf("Entry %d - %s", i, title), URL: fmt.Sprintf("https://example.com/%d", i), + CommentsURL: commentsURL, Date: time.Now().Add(time.Duration(-i) * time.Hour), Content: content, - Author: fmt.Sprintf("Test Author %d", i), - Feed: feed, + FeedID: template.FeedID, + FeedTitle: template.FeedTitle, + GroupID: template.GroupID, + GroupTitle: template.GroupTitle, } entries = append(entries, entry) } - return &entries -} - -func NewMockEntries() *miniflux.Entries { - return MockNumEntries(20) -} - - -func NewMockFeedIcons() []*models.FeedIcon { - return []*models.FeedIcon{ - NewMockFeedIconRed(), - NewMockFeedIconYellow(), - NewMockFeedIconGreen(), - } + return entries } diff --git a/internal/testutil/mocks.go b/internal/testutil/mocks.go index 3f3bc4a..7fe7b32 100644 --- a/internal/testutil/mocks.go +++ b/internal/testutil/mocks.go @@ -1,9 +1,8 @@ package testutil import ( - "miniflux-digest/internal/app" + "miniflux-digest/internal/app/services" "miniflux-digest/internal/config" - "miniflux-digest/internal/digest" "miniflux-digest/internal/models" "os" "time" @@ -12,42 +11,24 @@ import ( ) type MockMinifluxClient struct { - app.MinifluxClientService - MarkAsReadFunc func(categoryID int64) error - CategoriesFunc func() ([]*miniflux.Category, error) - CategoryEntriesFunc func(categoryID int64, filter *miniflux.Filter) (*miniflux.Entries, error) - CategoryFeedsFunc func(categoryID int64) ([]*miniflux.Feed, error) + services.MinifluxClientService + + + + FeedIconFunc func(feedID int64) (*miniflux.FeedIcon, error) - StreamAllCategoryDataFunc func() <-chan *app.RawCategoryData + + GetAllUnreadEntriesFunc func() (*miniflux.Entries, error) + UpdateEntriesFunc func(entryIDs []int64, status string) error } -func (m *MockMinifluxClient) MarkCategoryAsRead(categoryID int64) error { - if m.MarkAsReadFunc != nil { - return m.MarkAsReadFunc(categoryID) - } - return nil -} -func (m *MockMinifluxClient) Categories() ([]*miniflux.Category, error) { - if m.CategoriesFunc != nil { - return m.CategoriesFunc() - } - return nil, nil -} -func (m *MockMinifluxClient) CategoryEntries(categoryID int64, filter *miniflux.Filter) (*miniflux.Entries, error) { - if m.CategoryEntriesFunc != nil { - return m.CategoryEntriesFunc(categoryID, filter) - } - return &miniflux.Entries{}, nil -} -func (m *MockMinifluxClient) CategoryFeeds(categoryID int64) ([]*miniflux.Feed, error) { - if m.CategoryFeedsFunc != nil { - return m.CategoryFeedsFunc(categoryID) - } - return nil, nil -} + + + + func (m *MockMinifluxClient) FeedIcon(feedID int64) (*miniflux.FeedIcon, error) { if m.FeedIconFunc != nil { @@ -58,43 +39,50 @@ func (m *MockMinifluxClient) FeedIcon(feedID int64) (*miniflux.FeedIcon, error) -func (m *MockMinifluxClient) StreamAllCategoryData() <-chan *app.RawCategoryData { - if m.StreamAllCategoryDataFunc != nil { - return m.StreamAllCategoryDataFunc() + + +func (m *MockMinifluxClient) GetAllUnreadEntries() (*miniflux.Entries, error) { + if m.GetAllUnreadEntriesFunc != nil { + return m.GetAllUnreadEntriesFunc() } - out := make(chan *app.RawCategoryData) - close(out) - return out + return &miniflux.Entries{}, nil +} + +func (m *MockMinifluxClient) UpdateEntries(entryIDs []int64, status string) error { + if m.UpdateEntriesFunc != nil { + return m.UpdateEntriesFunc(entryIDs, status) + } + return nil } type MockArchiveService struct { - app.ArchiveService - MakeArchiveHTMLFunc func(data *models.HTMLTemplateData, minify bool) (*os.File, error) + services.ArchiveService + MakeArchiveHTMLFunc func(data *models.OverviewTemplateData, minify bool) (*os.File, error) } -func (m *MockArchiveService) MakeArchiveHTML(data *models.HTMLTemplateData, minify bool) (*os.File, error) { +func (m *MockArchiveService) MakeArchiveHTML(data *models.OverviewTemplateData, minify bool) (*os.File, error) { return m.MakeArchiveHTMLFunc(data, minify) } func (m *MockArchiveService) CleanArchive(maxAge time.Duration) {} type MockEmailService struct { - app.EmailService - SendFunc func(cfg *config.Config, file *os.File, data *models.HTMLTemplateData) error + services.EmailService + SendFunc func(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error } -func (m *MockEmailService) Send(cfg *config.Config, file *os.File, data *models.HTMLTemplateData) error { - return m.SendFunc(cfg, file, data) +func (m *MockEmailService) Send(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) error { + return m.SendFunc(cfg, overviewFile, groupedEntryFiles, data) } type MockDigestService struct { - app.DigestService - BuildDigestDataFunc func(category *miniflux.Category, entries *miniflux.Entries, icons map[int64]*models.FeedIcon, groupBy digest.GroupingType, minifluxHost string) *models.HTMLTemplateData + services.DigestService + BuildDigestDataFunc func(category *miniflux.Category, entries *miniflux.Entries, icons map[int64]*models.FeedIcon, subGroupBy string, sortBy string, minifluxHost string) *models.OverviewTemplateData } -func (m *MockDigestService) BuildDigestData(category *miniflux.Category, entries *miniflux.Entries, icons map[int64]*models.FeedIcon, groupBy digest.GroupingType, minifluxHost string) *models.HTMLTemplateData { +func (m *MockDigestService) BuildDigestData(category *miniflux.Category, entries *miniflux.Entries, icons map[int64]*models.FeedIcon, subGroupBy string, sortBy string, minifluxHost string) *models.OverviewTemplateData { if m.BuildDigestDataFunc != nil { - return m.BuildDigestDataFunc(category, entries, icons, groupBy, minifluxHost) + return m.BuildDigestDataFunc(category, entries, icons, subGroupBy, sortBy, minifluxHost) } return nil } diff --git a/internal/webserver/webserver.go b/internal/webserver/webserver.go new file mode 100644 index 0000000..e383f31 --- /dev/null +++ b/internal/webserver/webserver.go @@ -0,0 +1,81 @@ +package webserver + +import ( + "fmt" + "log" + "net/http" + "os" + "path" + "path/filepath" + "strings" +) + +const ( + ArchiveBasePath = "web/miniflux-archive" + Port = ":8080" +) + +type noDirListingFileSystem struct { + fs http.FileSystem +} + +func (fs noDirListingFileSystem) Open(name string) (http.File, error) { + f, err := fs.fs.Open(name) + if err != nil { + return nil, err + } + stat, err := f.Stat() + if err != nil { + return nil, err + } + if stat.IsDir() { + _ = f.Close() // Ignore error on close, as we are returning an error anyway + return nil, os.ErrNotExist + } + return f, nil +} + +func SetupServer(archiveBasePath string) *http.ServeMux { + mux := http.NewServeMux() + + mux.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + if _, err := fmt.Fprintf(w, "OK"); err != nil { + log.Printf("Error writing healthcheck response: %v", err) + } + }) + + fs := http.FileServer(noDirListingFileSystem{http.Dir(archiveBasePath)}) + + mux.Handle("/archive/", http.StripPrefix("/archive/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/") { + strippedPath := strings.TrimPrefix(r.URL.Path, "/") + indexPath := path.Join(strippedPath, "index.html") + + if _, err := (noDirListingFileSystem{http.Dir(archiveBasePath)}).Open(indexPath); err == nil { + http.ServeFile(w, r, filepath.Join(archiveBasePath, indexPath)) + return + } + } + fs.ServeHTTP(w, r) + }))) + + return mux +} + +func StartServer(port string, handler http.Handler) { + log.Printf("Internal web server starting on port %s", port) + if err := http.ListenAndServe(port, handler); err != nil { + log.Fatalf("Internal web server failed to start: %v", err) + } +} + +func ListenAndServe(archiveBasePath string, port string) *http.ServeMux { + mux := SetupServer(archiveBasePath) + log.Printf("Internal web server starting on port %s", port) + if err := http.ListenAndServe(port, mux); err != nil { + log.Fatalf("Internal web server failed to start: %v", err) + } + + return mux +} \ No newline at end of file diff --git a/scripts/preview/main.go b/scripts/preview/main.go index 9defd4e..a1b83a5 100644 --- a/scripts/preview/main.go +++ b/scripts/preview/main.go @@ -1,16 +1,19 @@ package main import ( - "bytes" "flag" "fmt" "log" "os" "os/exec" + "os/signal" "path/filepath" "runtime" + "syscall" "miniflux-digest/internal/app" + "miniflux-digest/internal/app/services" + "miniflux-digest/internal/archive" "miniflux-digest/internal/config" "miniflux-digest/internal/digest" "miniflux-digest/internal/email" @@ -18,6 +21,8 @@ import ( "miniflux-digest/internal/models" "miniflux-digest/internal/templates" "miniflux-digest/internal/testutil" + "miniflux-digest/internal/webserver" + miniflux "miniflux.app/v2/client" ) @@ -28,6 +33,7 @@ func openBrowser(url string) error { switch runtime.GOOS { case "darwin": cmd = "open" + args = []string{url} case "windows": cmd = "cmd" args = []string{"/c", "start"} @@ -41,9 +47,8 @@ func openBrowser(url string) error { return exec.Command(cmd, args...).Start() } -func generateDigestData(cfg *config.Config, minifluxID int64) *models.HTMLTemplateData { +func generateMockDigest(cfg *config.Config) *models.OverviewTemplateData { log.Println("generateDigestData: Starting...") - var llmService llm.LLMService llmService, err := llm.NewGeminiService(cfg.AI.ApiKey) if err != nil { @@ -53,115 +58,181 @@ func generateDigestData(cfg *config.Config, minifluxID int64) *models.HTMLTempla digestSvc := digest.NewDigestService(llmService) log.Println("generateDigestData: DigestService initialized.") - if minifluxID != 0 { - log.Println("generateDigestData: Fetching real Miniflux data...") - minifluxClient := miniflux.NewClient(cfg.Miniflux.Host, cfg.Miniflux.ApiToken) - clientWrapper := app.NewMinifluxClientWrapper(minifluxClient) + var entries []*models.Entry + log.Println("generateDigestData: Using mock data...") + entries = testutil.CreateMockEntries(200) - rawData, err := clientWrapper.FetchRawCategoryData(minifluxID) - if err != nil { - log.Fatalf("Failed to fetch category data: %v", err) + icons := make(map[int64]*models.FeedIcon) + for _, entry := range entries { + if _, ok := icons[entry.FeedID]; !ok { + switch entry.FeedID { + case 1: + icons[entry.FeedID] = testutil.NewMockFeedIconRed() + case 2: + icons[entry.FeedID] = testutil.NewMockFeedIconYellow() + case 3: + icons[entry.FeedID] = testutil.NewMockFeedIconGreen() + default: + icons[entry.FeedID] = testutil.NewMockFeedIconGreen() + } } - log.Println("generateDigestData: Building digest data with real Miniflux data...") - return digestSvc.BuildDigestData(rawData.Category, rawData.Entries, rawData.Icons, cfg.Digest.GroupBy, cfg.Miniflux.Host) - } else { - log.Println("generateDigestData: Building digest data with mock data...") - return digestSvc.BuildDigestData( - testutil.NewMockCategory(), - testutil.MockNumEntries(200), - map[int64]*models.FeedIcon{ - 1: testutil.NewMockFeedIconRed(), - 2: testutil.NewMockFeedIconYellow(), - 3: testutil.NewMockFeedIconGreen(), - }, - cfg.Digest.GroupBy, - cfg.Miniflux.Host, - ) } + + log.Println("generateDigestData: Building digest data...") + return digestSvc.BuildDigestData( + entries, + icons, + cfg.Digest.GroupBy, + cfg.Digest.SubGroupBy, + cfg.Digest.SortBy, + cfg.Miniflux.Host, + ) } -func generateHTML(data *models.HTMLTemplateData, compress bool) ([]byte, error) { - log.Println("generateHTML: Starting...") - var buf bytes.Buffer - if err := templates.ArchiveTemplate.Execute(&buf, data); err != nil { - return nil, fmt.Errorf("failed to execute template: %w", err) - } +func generateMinifluxDigest(cfg *config.Config, minifluxClientService services.MinifluxClientService) *models.OverviewTemplateData { + log.Println("generateDigestData: Starting...") - html, err := digest.MinifyHTML(buf.Bytes(), compress) + llmService, err := llm.NewGeminiService(cfg.AI.ApiKey) if err != nil { - return nil, fmt.Errorf("failed to minify HTML: %w", err) + log.Fatalf("Failed to create LLM service: %v", err) } - log.Println("generateHTML: Finished.") - return html, nil -} -func writeHTMLToFile(html []byte) (string, error) { - log.Println("writeHTMLToFile: Starting...") - tmpDir, err := os.MkdirTemp("", "miniflux-digest-preview") + digestSvc := digest.NewDigestService(llmService) + log.Println("generateDigestData: DigestService initialized.") + + var entries []*models.Entry + log.Println("generateDigestData: Fetching real Miniflux data...") + + entries, err = minifluxClientService.GetAllUnreadEntries() if err != nil { - return "", fmt.Errorf("failed to create temporary directory: %w", err) + log.Fatalf("Failed to fetch entries: %v", err) } - filePath := filepath.Join(tmpDir, "preview.html") - if err := os.WriteFile(filePath, html, 0644); err != nil { - return "", fmt.Errorf("failed to write HTML to file: %w", err) + icons := make(map[int64]*models.FeedIcon) + for _, entry := range entries { + if _, ok := icons[entry.FeedID]; !ok { + icon, err := minifluxClientService.FeedIcon(entry.FeedID) + if err != nil { + log.Printf("Warning: failed to fetch icon for feed %d: %v", entry.FeedID, err) + continue + } + icons[entry.FeedID] = icon + } } - log.Println("writeHTMLToFile: Finished.") - return filePath, nil + + log.Println("generateDigestData: Building digest data...") + return digestSvc.BuildDigestData( + entries, + icons, + cfg.Digest.GroupBy, + cfg.Digest.SubGroupBy, + cfg.Digest.SortBy, + cfg.Miniflux.Host, + ) } -func main() { - log.Println("main: Starting preview script...") - emailFlag := flag.Bool("email", false, "Send the generated HTML as an email") - minifluxID := flag.Int64("miniflux", 0, "Miniflux category ID to fetch entries") +func setupAndParseFlags() (emailFlag, minifluxFlag, serveOnlyFlag bool) { + email := flag.Bool("email", false, "send mock data derived digest as an email") + miniflux := flag.Bool("miniflux", false, "use live data from miniflux to generate digest") + html := flag.Bool("html", false, "use mock data to generate digest") flag.Parse() + return *email, *miniflux, *html +} +func loadConfig() *config.Config { cfg, err := config.Load("./config.yaml") if err != nil { log.Fatalf("Failed to load config: %v", err) } log.Println("main: Config loaded.") + return cfg +} + +func generateAndArchiveHTML(cfg *config.Config, minifluxFlag bool) (*os.File, []*os.File, string, *models.OverviewTemplateData) { + var data *models.OverviewTemplateData + + if minifluxFlag { + minifluxClient := miniflux.NewClient(cfg.Miniflux.Host, cfg.Miniflux.ApiToken) + clientWrapper := app.NewMinifluxClientWrapper(minifluxClient) + data = generateMinifluxDigest(cfg, clientWrapper) + } else { + data = generateMockDigest(cfg) + } - data := generateDigestData(cfg, *minifluxID) log.Println("main: Digest data generated.") - html, err := generateHTML(data, cfg.Digest.Compress) + + archiveSvc := archive.NewArchiveService(webserver.ArchiveBasePath, templates.ArchiveTemplate, templates.OverviewTemplate) + overviewFile, groupedEntryFiles, err := archiveSvc.MakeArchiveHTML(data, cfg.Digest.Compress) if err != nil { log.Fatalf("Failed to generate HTML: %v", err) } log.Println("main: HTML generated.") - filePath, err := writeHTMLToFile(html) + log.Printf("overviewFile.Name(): %s", overviewFile.Name()) + log.Printf("webserver.ArchiveBasePath: %s", webserver.ArchiveBasePath) + + relativePath, err := filepath.Rel(webserver.ArchiveBasePath, overviewFile.Name()) if err != nil { - log.Fatalf("Failed to write HTML to file: %v", err) + log.Fatalf("Failed to get relative path: %v", err) } - log.Println("main: HTML written to file.") + log.Printf("relativePath: %s", relativePath) + overviewURL := fmt.Sprintf("http://localhost%s/archive/%s", webserver.Port, relativePath) + log.Printf("overviewURL: %s", overviewURL) - if *emailFlag { - log.Println("main: Email flag is true, sending email...") - emailSvc := &email.EmailServiceImpl{} - file, err := os.Open(filePath) - if err != nil { - log.Fatalf("Failed to open HTML file for email: %v", err) - } - defer func() { - if err := file.Close(); err != nil { - log.Printf("Error closing file: %v", err) - } - }() - if err := emailSvc.Send(cfg, file, data); err != nil { - log.Fatalf("Failed to send email: %v", err) - } - log.Printf("Successfully generated %s and sent email.", filePath) - } else { - log.Printf("Successfully generated %s.", filePath) + return overviewFile, groupedEntryFiles, overviewURL, data +} + +func handleEmail(cfg *config.Config, overviewFile *os.File, groupedEntryFiles []*os.File, data *models.OverviewTemplateData) { + log.Println("main: Email flag is true, sending email...") + emailSvc := &email.EmailServiceImpl{ + EmailTemplate: templates.EmailTemplate, } - log.Printf("Preview available at: file://%s", filePath) + if err := emailSvc.Send(cfg, overviewFile, groupedEntryFiles, data); err != nil { + log.Fatalf("Failed to send email: %v", err) + } + log.Printf("Successfully generated and sent email.") +} - log.Println("main: Attempting to open browser...") - if err := openBrowser(fmt.Sprintf("file://%s", filePath)); err != nil { +func handleWebServer() { + log.Printf("Successfully generated.") + + go func() { + webserver.ListenAndServe(webserver.ArchiveBasePath, webserver.Port) + }() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + <-sigChan + + log.Println("Shutting down web server...") +} + +func handleOpenUrl(url string) { + log.Printf("Attempting to open %s with the browser...", url) + if err := openBrowser(url); err != nil { log.Printf("Failed to open browser: %v", err) } - log.Println("main: Browser open attempt finished.") } + +func main() { + log.Println("main: Starting preview script...") + emailFlag, minifluxFlag, htmlFlag := setupAndParseFlags() + + cfg := loadConfig() + + if emailFlag { + overviewFile, groupedEntryFiles, _, data := generateAndArchiveHTML(cfg, minifluxFlag) + handleEmail(cfg, overviewFile, groupedEntryFiles, data) + } else if htmlFlag { + _, _, overviewURL, _ := generateAndArchiveHTML(cfg, minifluxFlag) + handleOpenUrl(overviewURL) + } else if minifluxFlag { + _, _, overviewURL, _ := generateAndArchiveHTML(cfg, minifluxFlag) + handleOpenUrl(overviewURL) + } else { + log.Println("main: Starting web server") + handleWebServer() + } +} \ No newline at end of file