diff --git a/cmd/sync.go b/cmd/sync.go new file mode 100644 index 0000000..4125f9e --- /dev/null +++ b/cmd/sync.go @@ -0,0 +1,54 @@ +/* +Copyright © 2025 Dennis Schoepf + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ +package cmd + +import ( + "freed/internal" + + "github.com/pterm/pterm" + "github.com/spf13/cobra" +) + +// syncCmd represents the sync command +var syncCmd = &cobra.Command{ + Use: "sync", + Short: "Syncs feeds to get the latest feed items", + Long: `Syncs feeds to get the latest feed items. Stores references to all articles since the +last sync.`, + Run: func(cmd *cobra.Command, args []string) { + if err := internal.SyncFeeds(); err != nil { + pterm.Error.Printf("Error syncing feeds: %v\n", err) + return + } + + pterm.Success.Printf("Feeds are in sync again") + }, +} + +func init() { + rootCmd.AddCommand(syncCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // syncCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // syncCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/internal/database/article.go b/internal/database/article.go index 4e3f9e2..f3044df 100644 --- a/internal/database/article.go +++ b/internal/database/article.go @@ -12,6 +12,10 @@ type Article struct { FeedId int64 } +type InsertMultipleOptions struct { + IgnoreDuplicates bool +} + func (a Article) Insert() (int64, error) { result, err := db.Exec("INSERT INTO article (name, url, readAt, feedId) VALUES (?, ?, ?, ?)", a.Name, a.Url, a.ReadAt, a.FeedId) @@ -43,6 +47,14 @@ func (a Article) MarkAsRead() error { } func InsertMultipleArticles(articles []Article) error { + return insertMultipleArticlesWithOpts(articles, InsertMultipleOptions{IgnoreDuplicates: false}) +} + +func InsertIgnoreMultipleArticles(articles []Article) error { + return insertMultipleArticlesWithOpts(articles, InsertMultipleOptions{IgnoreDuplicates: true}) +} + +func insertMultipleArticlesWithOpts(articles []Article, opts InsertMultipleOptions) error { if len(articles) == 0 { return nil } @@ -55,7 +67,15 @@ func InsertMultipleArticles(articles []Article) error { defer tx.Rollback() - stmt, err := tx.Prepare("INSERT INTO article (name, url, readAt, feedId) VALUES (?, ?, ?, ?)") + insertStmt := "INSERT " + + if opts.IgnoreDuplicates { + insertStmt = insertStmt + "OR IGNORE " + } + + finalStmt := insertStmt + "INTO article (name, url, readAt, feedId) VALUES (?, ?, ?, ?)" + + stmt, err := tx.Prepare(finalStmt) if err != nil { return err diff --git a/internal/database/feed.go b/internal/database/feed.go index eaf766f..0039a18 100644 --- a/internal/database/feed.go +++ b/internal/database/feed.go @@ -35,8 +35,36 @@ func (f Feed) Insert() (int64, error) { return id, nil } +// This might be a problem if we have a LOT of feeds configured +// as of now this works, so it is not at the top of the priority +// list +func FindAllFeeds() (*[]Feed, error) { + rows, err := db.Query("SELECT id, name, url, createdAt, lastSyncedAt FROM feed") + if err != nil { + return nil, err + } + defer rows.Close() + + var feeds []Feed + + for rows.Next() { + var feed Feed + err := rows.Scan(&feed.ID, &feed.Name, &feed.Url, &feed.CreatedAt, &feed.LastSyncedAt) + if err != nil { + return nil, err + } + feeds = append(feeds, feed) + } + + if err = rows.Err(); err != nil { + return nil, err + } + + return &feeds, nil +} + func FindAllFeedsWithArticleCount() (*[]FeedWithArticleCount, error) { - rows, err := db.Query("SELECT f.id, f.name, f.url, f.createdAt, f.lastSyncedAt, COUNT(a.id) FROM feed as f LEFT JOIN article as a ON a.feedId = f.id;") + rows, err := db.Query("SELECT f.id, f.name, f.url, f.createdAt, f.lastSyncedAt, COUNT(a.id) FROM feed as f LEFT JOIN article as a ON a.feedId = f.id") if err != nil { return nil, err } diff --git a/internal/feed.go b/internal/feed.go index 4d28cf8..e06884c 100644 --- a/internal/feed.go +++ b/internal/feed.go @@ -5,6 +5,8 @@ import ( "freed/internal/database" "net/url" "strconv" + "sync" + "time" "github.com/mmcdole/gofeed" "github.com/pterm/pterm" @@ -31,18 +33,21 @@ func AddFeed(feedUrl string) (string, int, error) { return "", 0, err } + // TODO: Make the amount of articles configurable articles := make([]database.Article, 0, 10) for i, v := range feed.Items { - if i < 10 { - a := database.Article{ - Name: v.Title, - Url: v.Link, - FeedId: feedId, - } - - articles = append(articles, a) + if i > 10 { + continue } + + a := database.Article{ + Name: v.Title, + Url: v.Link, + FeedId: feedId, + } + + articles = append(articles, a) } if err := database.InsertMultipleArticles(articles); err != nil { @@ -80,6 +85,86 @@ func GetAllFeedsAsTable() (pterm.TableData, error) { return tableData, nil } +func SyncFeeds() error { + now := time.Now() + startOfToday := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + feeds, err := database.FindAllFeeds() + + if err != nil { + return err + } + + var syncWg sync.WaitGroup + errChannel := make(chan error, len(*feeds)) + + for _, feed := range *feeds { + syncWg.Add(1) + go syncFeed(feed, startOfToday, &syncWg, errChannel) + } + + syncWg.Wait() + close(errChannel) + + var errors []error + + for err := range errChannel { + errors = append(errors, err) + } + + if len(errors) > 0 { + compositeErrMsg := fmt.Sprintf("Encountered %d errors:\n", len(errors)) + + for _, err := range errors { + compositeErrMsg = compositeErrMsg + " " + fmt.Sprintf("- %v\n", err) + } + + return fmt.Errorf(compositeErrMsg) + } + + return nil +} + +func syncFeed( + feed database.Feed, + syncBefore time.Time, + wg *sync.WaitGroup, + errorChannel chan<- error, +) { + defer wg.Done() + + if feed.LastSyncedAt.After(syncBefore) { + return + } + + gFeed, err := parseByUrl(feed.Url) + + if err != nil { + errorChannel <- err + return + } + + var articles []database.Article + + for _, item := range gFeed.Items { + if item.PublishedParsed.Before(*feed.LastSyncedAt) { + continue + } + + article := database.Article{ + Name: item.Title, + Url: item.Link, + FeedId: feed.ID, + } + + articles = append(articles, article) + } + + if err := database.InsertIgnoreMultipleArticles(articles); err != nil { + errorChannel <- err + return + } +} + func parseByUrl(u string) (*gofeed.Feed, error) { fp := gofeed.NewParser()