diff --git a/README.md b/README.md index 23c5b6d..eb8adb9 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,41 @@ cleed config --color-range cleed config --summary=1 ``` +#### Explore feeds + +```bash +# Explore feeds from the default repository (https://github.com/radulucut/cleed-explore) +cleed explore + +# Fetch the latest changes and explore feeds from the default repository +cleed explore --update + +# Explore feeds from a repository +cleed explore https://github.com/radulucut/cleed-explore.git + +# Limit the number of items to display from each list +cleed explore --limit 5 + +# Search for items (title, description) +cleed explore --search "news" + +# Import all feeds into my feeds +cleed explore --import --limit 0 + +# Remove a repository +cleed explore https://github.com/radulucut/cleed-explore.git --remove +``` + +> **Note** +> +> The explore command expects git to be installed in order to fetch the repository, and it will only look at `.opml` files when exploring a repository. + +#### Help + +```bash +cleed --help +``` + > **Color mapping** > > You can map the colors used in the feed reader to any color you want. This is useful if certain colors are not visible in your terminal based on the color scheme that you are using. diff --git a/cmd/cleed/explore.go b/cmd/cleed/explore.go new file mode 100644 index 0000000..bbde406 --- /dev/null +++ b/cmd/cleed/explore.go @@ -0,0 +1,80 @@ +package cleed + +import ( + "github.com/radulucut/cleed/internal" + "github.com/spf13/cobra" +) + +func (r *Root) initExplore() { + cmd := &cobra.Command{ + Use: "explore", + Short: "Explore feeds", + Long: `Explore feeds from a repository + +Examples: + # Explore feeds from the default repository + cleed explore + + # Fetch the latest changes and explore feeds from the default repository + cleed explore --update + + # Explore feeds from a repository + cleed explore https://github.com/radulucut/cleed-explore.git + + # Limit the number of items to display from each list + cleed explore --limit 5 + + # Search for items (title, description) + cleed explore --search "news" + + # Import all feeds into my feeds + cleed explore --import --limit 0 + + # Remove a repository + cleed explore https://github.com/radulucut/cleed-explore.git --remove +`, + + RunE: r.RunExplore, + } + + flags := cmd.Flags() + flags.Bool("import", false, "import feeds") + flags.Uint("limit", 10, "limit the number of items to display from each list") + flags.String("search", "", "search for items (title, description)") + flags.BoolP("update", "u", false, "fetch the latest changes") + flags.Bool("remove", false, "remove the repository") + + r.Cmd.AddCommand(cmd) +} + +func (r *Root) RunExplore(cmd *cobra.Command, args []string) error { + if cmd.Flag("remove").Changed { + url := "" + if len(args) > 0 { + url = args[0] + } + return r.feed.ExploreRemove(url) + } + limit, err := cmd.Flags().GetUint("limit") + if err != nil { + return err + } + opts := &internal.ExploreOptions{ + Limit: int(limit), + Update: cmd.Flag("update").Changed, + Query: cmd.Flag("search").Value.String(), + } + if len(args) > 0 { + opts.Url = args[0] + } + if opts.Query != "" { + if !cmd.Flag("limit").Changed { + opts.Limit = 25 + } + return r.feed.ExploreSearch(opts) + } + if cmd.Flag("import").Changed { + return r.feed.ExploreImport(opts) + } + return r.feed.Explore(opts) +} diff --git a/cmd/cleed/explore_test.go b/cmd/cleed/explore_test.go new file mode 100644 index 0000000..568fad4 --- /dev/null +++ b/cmd/cleed/explore_test.go @@ -0,0 +1,512 @@ +package cleed + +import ( + "bytes" + "os" + "path" + "testing" + "time" + + "github.com/radulucut/cleed/internal" + _storage "github.com/radulucut/cleed/internal/storage" + "github.com/radulucut/cleed/mocks" + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" +) + +func Test_Explore(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + timeMock := mocks.NewMockTime(ctrl) + timeMock.EXPECT().Now().Return(defaultCurrentTime).AnyTimes() + + out := new(bytes.Buffer) + printer := internal.NewPrinter(nil, out, out) + storage := _storage.NewLocalStorage("cleed_test", timeMock) + defer localStorageCleanup(t, storage) + + tempRepository, err := storage.JoinExploreDir("repo") + if err != nil { + t.Fatal(err) + } + + os.MkdirAll(tempRepository, 0755) + + err = os.WriteFile(path.Join(tempRepository, "feeds.opml"), []byte(` + + + Export from cleed/test + Mon, 01 Jan 2024 00:00:00 +0000 + + + + + + + + + + + + +`), 0644) + if err != nil { + t.Fatal(err) + } + + feed := internal.NewTerminalFeed(timeMock, printer, storage) + feed.SetAgent("cleed/test") + feed.SetDefaultExploreRepository("repo") + + root, err := NewRoot("0.1.0", timeMock, printer, storage, feed) + assert.NoError(t, err) + + os.Args = []string{"cleed", "explore"} + err = root.Cmd.Execute() + assert.NoError(t, err) + assert.Equal(t, `RSS Feed +RSS Feed description +https://rss-feed.com/rss + +Atom Feed +Atom Feed description +https://atom-feed.com/atom + +test 2 (2/2) +-------------------------------- +RSS Feed +RSS Feed description +https://rss-feed.com/rss + +Atom Feed +Atom Feed description +https://atom-feed.com/atom + +https://test.com + +test (3/3) +-------------------------------- +Displayed 5 out of 5 feeds from 2 lists +`, out.String()) +} + +func Test_Explore_Custom(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + timeMock := mocks.NewMockTime(ctrl) + timeMock.EXPECT().Now().Return(defaultCurrentTime).AnyTimes() + + out := new(bytes.Buffer) + printer := internal.NewPrinter(nil, out, out) + storage := _storage.NewLocalStorage("cleed_test", timeMock) + defer localStorageCleanup(t, storage) + + tempRepository, err := storage.JoinExploreDir("repo") + if err != nil { + t.Fatal(err) + } + os.MkdirAll(tempRepository, 0755) + + customRepository, err := storage.JoinExploreDir("custom") + if err != nil { + t.Fatal(err) + } + os.MkdirAll(customRepository, 0755) + + err = os.WriteFile(path.Join(customRepository, "feeds.opml"), []byte(` + + + Export from cleed/test + Mon, 01 Jan 2024 00:00:00 +0000 + + + + + + + + + + + + +`), 0644) + if err != nil { + t.Fatal(err) + } + + feed := internal.NewTerminalFeed(timeMock, printer, storage) + feed.SetAgent("cleed/test") + feed.SetDefaultExploreRepository("repo") + + root, err := NewRoot("0.1.0", timeMock, printer, storage, feed) + assert.NoError(t, err) + + os.Args = []string{"cleed", "explore", "custom"} + err = root.Cmd.Execute() + assert.NoError(t, err) + assert.Equal(t, `RSS Feed +RSS Feed description +https://rss-feed.com/rss + +Atom Feed +Atom Feed description +https://atom-feed.com/atom + +test 2 (2/2) +-------------------------------- +RSS Feed +RSS Feed description +https://rss-feed.com/rss + +Atom Feed +Atom Feed description +https://atom-feed.com/atom + +https://test.com + +test (3/3) +-------------------------------- +Displayed 5 out of 5 feeds from 2 lists +`, out.String()) +} + +func Test_Explore_Limit(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + timeMock := mocks.NewMockTime(ctrl) + timeMock.EXPECT().Now().Return(defaultCurrentTime).AnyTimes() + + out := new(bytes.Buffer) + printer := internal.NewPrinter(nil, out, out) + storage := _storage.NewLocalStorage("cleed_test", timeMock) + defer localStorageCleanup(t, storage) + + tempRepository, err := storage.JoinExploreDir("repo") + if err != nil { + t.Fatal(err) + } + + os.MkdirAll(tempRepository, 0755) + + err = os.WriteFile(path.Join(tempRepository, "feeds.opml"), []byte(` + + + Export from cleed/test + Mon, 01 Jan 2024 00:00:00 +0000 + + + + + + + + + + + + +`), 0644) + if err != nil { + t.Fatal(err) + } + + feed := internal.NewTerminalFeed(timeMock, printer, storage) + feed.SetAgent("cleed/test") + feed.SetDefaultExploreRepository("repo") + + root, err := NewRoot("0.1.0", timeMock, printer, storage, feed) + assert.NoError(t, err) + + os.Args = []string{"cleed", "explore", "--limit", "1"} + err = root.Cmd.Execute() + assert.NoError(t, err) + assert.Equal(t, `RSS Feed +RSS Feed description +https://rss-feed.com/rss + +test 2 (1/2) +-------------------------------- +RSS Feed +RSS Feed description +https://rss-feed.com/rss + +test (1/3) +-------------------------------- +Displayed 2 out of 5 feeds from 2 lists +`, out.String()) +} + +func Test_Explore_Search(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + timeMock := mocks.NewMockTime(ctrl) + timeMock.EXPECT().Now().Return(defaultCurrentTime).AnyTimes() + + out := new(bytes.Buffer) + printer := internal.NewPrinter(nil, out, out) + storage := _storage.NewLocalStorage("cleed_test", timeMock) + defer localStorageCleanup(t, storage) + + tempRepository, err := storage.JoinExploreDir("repo") + if err != nil { + t.Fatal(err) + } + + os.MkdirAll(tempRepository, 0755) + + err = os.WriteFile(path.Join(tempRepository, "feeds.opml"), []byte(` + + + Export from cleed/test + Mon, 01 Jan 2024 00:00:00 +0000 + + + + + + + + + + + + +`), 0644) + if err != nil { + t.Fatal(err) + } + + feed := internal.NewTerminalFeed(timeMock, printer, storage) + feed.SetAgent("cleed/test") + feed.SetDefaultExploreRepository("repo") + + root, err := NewRoot("0.1.0", timeMock, printer, storage, feed) + assert.NoError(t, err) + + os.Args = []string{"cleed", "explore", "--search", "Atom"} + err = root.Cmd.Execute() + assert.NoError(t, err) + assert.Equal(t, `Atom 2 Feed +Atom Feed description +https://atom-feed.com/atom + +Atom 1 Feed +Atom Feed description +https://atom-feed.com/atom + +Displayed 2 out of 2 items +`, out.String()) +} + +func Test_Explore_Import(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + timeMock := mocks.NewMockTime(ctrl) + timeMock.EXPECT().Now().Return(defaultCurrentTime).AnyTimes() + + out := new(bytes.Buffer) + printer := internal.NewPrinter(nil, out, out) + storage := _storage.NewLocalStorage("cleed_test", timeMock) + defer localStorageCleanup(t, storage) + + tempRepository, err := storage.JoinExploreDir("repo") + if err != nil { + t.Fatal(err) + } + + os.MkdirAll(tempRepository, 0755) + + err = os.WriteFile(path.Join(tempRepository, "feeds.opml"), []byte(` + + + Export from cleed/test + Mon, 01 Jan 2024 00:00:00 +0000 + + + + + + + + + + + + +`), 0644) + if err != nil { + t.Fatal(err) + } + + feed := internal.NewTerminalFeed(timeMock, printer, storage) + feed.SetAgent("cleed/test") + feed.SetDefaultExploreRepository("repo") + + root, err := NewRoot("0.1.0", timeMock, printer, storage, feed) + assert.NoError(t, err) + + os.Args = []string{"cleed", "explore", "--import"} + err = root.Cmd.Execute() + assert.NoError(t, err) + assert.Equal(t, "Imported 5 feeds from 2 lists\n", out.String()) + + lists, err := storage.LoadLists() + assert.NoError(t, err) + + expectedLists := []string{"test", "test 2"} + assert.Equal(t, expectedLists, lists) + + feeds, err := storage.GetFeedsFromList("test") + assert.NoError(t, err) + expectedFeeds := []*_storage.ListItem{ + {AddedAt: time.Unix(defaultCurrentTime.Unix(), 0), Address: "https://rss-feed.com/rss"}, + {AddedAt: time.Unix(defaultCurrentTime.Unix(), 0), Address: "https://atom-feed.com/atom"}, + {AddedAt: time.Unix(defaultCurrentTime.Unix(), 0), Address: "https://test.com"}, + } + assert.Equal(t, expectedFeeds, feeds) + + feeds, err = storage.GetFeedsFromList("test 2") + assert.NoError(t, err) + expectedFeeds = []*_storage.ListItem{ + {AddedAt: time.Unix(defaultCurrentTime.Unix(), 0), Address: "https://rss-feed.com/rss"}, + {AddedAt: time.Unix(defaultCurrentTime.Unix(), 0), Address: "https://atom-feed.com/atom"}, + } + assert.Equal(t, expectedFeeds, feeds) +} + +func Test_Explore_Import_With_Limit(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + timeMock := mocks.NewMockTime(ctrl) + timeMock.EXPECT().Now().Return(defaultCurrentTime).AnyTimes() + + out := new(bytes.Buffer) + printer := internal.NewPrinter(nil, out, out) + storage := _storage.NewLocalStorage("cleed_test", timeMock) + defer localStorageCleanup(t, storage) + + tempRepository, err := storage.JoinExploreDir("repo") + if err != nil { + t.Fatal(err) + } + + os.MkdirAll(tempRepository, 0755) + + err = os.WriteFile(path.Join(tempRepository, "feeds.opml"), []byte(` + + + Export from cleed/test + Mon, 01 Jan 2024 00:00:00 +0000 + + + + + + + + + + + + +`), 0644) + if err != nil { + t.Fatal(err) + } + + feed := internal.NewTerminalFeed(timeMock, printer, storage) + feed.SetAgent("cleed/test") + feed.SetDefaultExploreRepository("repo") + + root, err := NewRoot("0.1.0", timeMock, printer, storage, feed) + assert.NoError(t, err) + + os.Args = []string{"cleed", "explore", "--import", "--limit", "1"} + err = root.Cmd.Execute() + assert.NoError(t, err) + assert.Equal(t, "Imported 2 feeds from 2 lists\n", out.String()) + + lists, err := storage.LoadLists() + assert.NoError(t, err) + + expectedLists := []string{"test", "test 2"} + assert.Equal(t, expectedLists, lists) + + feeds, err := storage.GetFeedsFromList("test") + assert.NoError(t, err) + expectedFeeds := []*_storage.ListItem{ + {AddedAt: time.Unix(defaultCurrentTime.Unix(), 0), Address: "https://rss-feed.com/rss"}, + } + assert.Equal(t, expectedFeeds, feeds) + + feeds, err = storage.GetFeedsFromList("test 2") + assert.NoError(t, err) + expectedFeeds = []*_storage.ListItem{ + {AddedAt: time.Unix(defaultCurrentTime.Unix(), 0), Address: "https://rss-feed.com/rss"}, + } + assert.Equal(t, expectedFeeds, feeds) +} + +func Test_Explore_Remove(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + timeMock := mocks.NewMockTime(ctrl) + timeMock.EXPECT().Now().Return(defaultCurrentTime).AnyTimes() + + out := new(bytes.Buffer) + printer := internal.NewPrinter(nil, out, out) + storage := _storage.NewLocalStorage("cleed_test", timeMock) + defer localStorageCleanup(t, storage) + + tempRepository, err := storage.JoinExploreDir("repo") + if err != nil { + t.Fatal(err) + } + + os.MkdirAll(tempRepository, 0755) + + err = os.WriteFile(path.Join(tempRepository, "feeds.opml"), []byte(` + + + Export from cleed/test + Mon, 01 Jan 2024 00:00:00 +0000 + + + + + + + + + + + + +`), 0644) + if err != nil { + t.Fatal(err) + } + + feed := internal.NewTerminalFeed(timeMock, printer, storage) + feed.SetAgent("cleed/test") + feed.SetDefaultExploreRepository("repo") + + root, err := NewRoot("0.1.0", timeMock, printer, storage, feed) + assert.NoError(t, err) + + os.Args = []string{"cleed", "explore", "repo", "--remove"} + err = root.Cmd.Execute() + assert.NoError(t, err) + assert.Equal(t, "repo was removed\n", out.String()) + + _, err = os.Stat(tempRepository) + assert.True(t, os.IsNotExist(err)) +} diff --git a/cmd/cleed/list_test.go b/cmd/cleed/list_test.go index c1ce157..9ff73dc 100644 --- a/cmd/cleed/list_test.go +++ b/cmd/cleed/list_test.go @@ -813,7 +813,7 @@ func Test_List_ExportToOPML_Multiple_Lists(t *testing.T) { t.Fatal(err) } - err = os.WriteFile(path.Join(listsDir, "test 2"), + err = os.WriteFile(path.Join(listsDir, url.QueryEscape("test 2")), []byte(fmt.Sprintf("%d %s\n%d %s\n%d %s\n", defaultCurrentTime.Unix(), "https://rss-feed.com/rss", defaultCurrentTime.Unix()+300, "https://atom-feed.com/atom", diff --git a/cmd/cleed/root.go b/cmd/cleed/root.go index b58e90e..7251b46 100644 --- a/cmd/cleed/root.go +++ b/cmd/cleed/root.go @@ -19,6 +19,7 @@ func Execute() { storage := storage.NewLocalStorage("cleed", time) feed := internal.NewTerminalFeed(time, printer, storage) feed.SetAgent(fmt.Sprintf("cleed/v%s (github.com/radulucut/cleed)", Version)) + feed.SetDefaultExploreRepository("https://github.com/radulucut/cleed-explore.git") root, err := NewRoot(Version, time, printer, storage, feed) if err != nil { printer.ErrPrintf("Error: %v\n", err) @@ -117,6 +118,7 @@ Examples: root.initUnfollow() root.initList() root.initConfig() + root.initExplore() return root, nil } diff --git a/internal/explore.go b/internal/explore.go new file mode 100644 index 0000000..fa0b9c9 --- /dev/null +++ b/internal/explore.go @@ -0,0 +1,214 @@ +package internal + +import ( + "os" + "path/filepath" + "slices" + "strings" + + "github.com/radulucut/cleed/internal/utils" +) + +type ExploreOptions struct { + Url string + Update bool + Limit int + Query string +} + +func (f *TerminalFeed) ExploreRemove(url string) error { + if url == "" { + url = f.defaultExploreRepository + } + err := f.storage.RemoveExploreRepository(url) + if err != nil { + return utils.NewInternalError("failed to remove repository: " + err.Error()) + } + f.printer.Printf("%s was removed\n", url) + return nil +} + +type ExploreSearchItem struct { + ListColor uint8 + Outline *utils.OPMLOutline + Score int +} + +func (f *TerminalFeed) ExploreSearch(opts *ExploreOptions) error { + queryTokens := utils.Tokenize(opts.Query, nil) + if len(queryTokens) == 0 { + return utils.NewInternalError("query is empty") + } + lists, err := f.getExploreFeeds(opts.Url, opts.Update) + if err != nil { + return err + } + config, err := f.storage.LoadConfig() + if err != nil { + return utils.NewInternalError("failed to load config: " + err.Error()) + } + items := make([]*ExploreSearchItem, 0) + for i, list := range lists { + for _, outline := range list.Outlines { + if outline.XMLURL == "" { + continue + } + itemTokens := utils.Tokenize(outline.Text, nil) + itemTokens = utils.Tokenize(outline.Description, itemTokens) + score := utils.Score(queryTokens, itemTokens) + if score == -1 { + continue + } + items = append(items, &ExploreSearchItem{ + ListColor: mapColor(uint8(i+1%256), config), + Outline: outline, + Score: score, + }) + } + } + if len(items) == 0 { + f.printer.Printf("no results found for %s\n", opts.Query) + } + slices.SortFunc(items, func(a, b *ExploreSearchItem) int { + if a.Score == b.Score { + return strings.Compare(a.Outline.Text, b.Outline.Text) + } else if a.Score < b.Score { + return -1 + } + return 1 + }) + secondaryTextColor := mapColor(7, config) + totalDisplayed := 0 + l := min(opts.Limit, len(items)) + if l <= 0 { + l = len(items) + } + for i := l - 1; i >= 0; i-- { + item := items[i] + if item.Outline.XMLURL == "" { + continue + } + if item.Outline.Text != "" { + f.printer.Print(f.printer.ColorForeground(item.Outline.Text, item.ListColor), "\n") + } + if item.Outline.Description != "" { + f.printer.Print(item.Outline.Description, "\n") + } + f.printer.Print(f.printer.ColorForeground(item.Outline.XMLURL, secondaryTextColor), "\n\n") + totalDisplayed++ + } + f.printer.Printf("Displayed %d out of %s\n", totalDisplayed, utils.Pluralize(int64(len(items)), "item")) + return nil +} + +func (f *TerminalFeed) Explore(opts *ExploreOptions) error { + lists, err := f.getExploreFeeds(opts.Url, opts.Update) + if err != nil { + return err + } + if len(lists) == 0 { + f.printer.Printf("No feeds found\n") + return nil + } + slices.SortFunc(lists, func(a, b *utils.OPMLOutline) int { + return strings.Compare(a.Text, b.Text) + }) + config, err := f.storage.LoadConfig() + if err != nil { + return utils.NewInternalError("failed to load config: " + err.Error()) + } + secondaryTextColor := mapColor(7, config) + total := 0 + totalDisplayed := 0 + for i := len(lists) - 1; i >= 0; i-- { + list := lists[i] + listColor := mapColor(uint8(i+1%256), config) + for j, outline := range list.Outlines { + if outline.XMLURL == "" { + continue + } + if opts.Limit > 0 && j >= opts.Limit { + break + } + if outline.Text != "" { + f.printer.Print(f.printer.ColorForeground(outline.Text, listColor), "\n") + } + if outline.Description != "" { + f.printer.Print(outline.Description, "\n") + } + f.printer.Print(f.printer.ColorForeground(outline.XMLURL, secondaryTextColor), "\n\n") + } + displayed := len(list.Outlines) + if opts.Limit > 0 && opts.Limit < displayed { + displayed = opts.Limit + } + totalDisplayed += displayed + total += len(list.Outlines) + f.printer.Printf("%s (%d/%d)\n--------------------------------\n", + list.Text, + displayed, + len(list.Outlines), + ) + } + f.printer.Printf("Displayed %d out of %s from %s\n", totalDisplayed, utils.Pluralize(int64(total), "feed"), utils.Pluralize(int64(len(lists)), "list")) + return nil +} + +func (f *TerminalFeed) ExploreImport(opts *ExploreOptions) error { + lists, err := f.getExploreFeeds(opts.Url, opts.Update) + if err != nil { + return err + } + urls := make([]string, 0) + totalImported := 0 + for _, list := range lists { + for _, outline := range list.Outlines { + if outline.XMLURL == "" { + continue + } + if opts.Limit > 0 && len(urls) >= opts.Limit { + break + } + urls = append(urls, outline.XMLURL) + } + totalImported += len(urls) + err = f.storage.AddToList(urls, list.Text) + if err != nil { + f.printer.Printf("failed to add feeds to list %s: %v\n", list.Text, err) + } + urls = urls[:0] + } + f.printer.Printf("Imported %s from %s\n", utils.Pluralize(int64(totalImported), "feed"), utils.Pluralize(int64(len(lists)), "list")) + return nil +} + +func (f *TerminalFeed) getExploreFeeds(url string, update bool) ([]*utils.OPMLOutline, error) { + if url == "" { + url = f.defaultExploreRepository + } + root, err := f.storage.GetExploreRepositoryPath(url, update) + if err != nil { + return nil, err + } + lists := make([]*utils.OPMLOutline, 0) + filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() && strings.HasPrefix(filepath.Base(path), ".") { + return filepath.SkipDir + } + if filepath.Ext(path) == ".opml" { + opml, err := utils.ParseOPMLFile(path) + if err != nil { + p := strings.TrimPrefix(path, root) + f.printer.Printf("failed to parse OPML file %s: %v\n", p, err) + return nil + } + lists = append(lists, opml.Body.Outltines...) + return nil + } + return nil + }) + return lists, err +} diff --git a/internal/feed.go b/internal/feed.go index e93047f..aa10b36 100644 --- a/internal/feed.go +++ b/internal/feed.go @@ -30,7 +30,8 @@ type TerminalFeed struct { http *http.Client parser *gofeed.Parser - agent string + agent string + defaultExploreRepository string } func NewTerminalFeed( @@ -52,6 +53,10 @@ func (f *TerminalFeed) SetAgent(agent string) { f.agent = agent } +func (f *TerminalFeed) SetDefaultExploreRepository(repository string) { + f.defaultExploreRepository = repository +} + func (f *TerminalFeed) DisplayConfig() error { config, err := f.storage.LoadConfig() if err != nil { @@ -298,19 +303,14 @@ func (f *TerminalFeed) ExportToFile(path, list string) error { } func (f *TerminalFeed) ImportFromOPML(path, list string) error { - b, err := os.ReadFile(path) - if err != nil { - return utils.NewInternalError("failed to read file: " + err.Error()) - } - opml := &utils.OPML{} - err = xml.Unmarshal(b, opml) + opml, err := utils.ParseOPMLFile(path) if err != nil { return utils.NewInternalError("failed to parse OPML: " + err.Error()) } - if len(opml.Body.Oultines) == 0 { + if len(opml.Body.Outltines) == 0 { return utils.NewInternalError("no feeds found in OPML") } - for _, listOutline := range opml.Body.Oultines { + for _, listOutline := range opml.Body.Outltines { urls := make([]string, 0, len(listOutline.Outlines)) for _, feedOutline := range listOutline.Outlines { urls = append(urls, feedOutline.XMLURL) @@ -380,10 +380,14 @@ func (f *TerminalFeed) ExportToOPML(path, list string) error { if err == nil { if feed.Title != "" { fmt.Fprint(fo, " text=\"") - xml.EscapeText(fo, []byte(feed.Title)) + xml.EscapeText(fo, []byte(strings.TrimSpace(feed.Title))) fmt.Fprint(fo, "\"") } if feed.Description != "" { + feed.Description = strings.TrimSpace(feed.Description) + if len(feed.Description) > 200 { + feed.Description = feed.Description[:200] + "..." + } fmt.Fprint(fo, " description=\"") xml.EscapeText(fo, []byte(feed.Description)) fmt.Fprint(fo, "\"") @@ -433,13 +437,7 @@ func (f *TerminalFeed) ShowCacheInfo() error { f.printer.Print(runewidth.FillRight("URL", cellMax[0])) f.printer.Println(" Last fetch Fetch after") slices.SortFunc(items, func(a, b *storage.CacheInfoItem) int { - if a.URL < b.URL { - return -1 - } - if a.URL > b.URL { - return 1 - } - return 0 + return strings.Compare(a.URL, b.URL) }) for i := range items { f.printer.Print(runewidth.FillRight(items[i].URL, cellMax[0])) diff --git a/internal/storage/explore.go b/internal/storage/explore.go new file mode 100644 index 0000000..f5fa291 --- /dev/null +++ b/internal/storage/explore.go @@ -0,0 +1,63 @@ +package storage + +import ( + "os" + "os/exec" + + "github.com/radulucut/cleed/internal/utils" +) + +func (s *LocalStorage) GetExploreRepositoryPath(name string, update bool) (string, error) { + path, err := s.JoinExploreDir(name) + if err != nil { + return "", err + } + _, err = os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return path, cloneRepository(name, path) + } + return "", err + } + if !update { + return path, nil + } + return path, updateRepository(path) +} + +func (s *LocalStorage) RemoveExploreRepository(name string) error { + path, err := s.JoinExploreDir(name) + if err != nil { + return err + } + return os.RemoveAll(path) +} + +func cloneRepository(url, path string) error { + out, err := exec.Command("git", "clone", "--depth", "1", "--single-branch", "--no-tags", url, path).CombinedOutput() + if err != nil { + return utils.NewInternalError(string(out) + "(" + err.Error() + ")") + } + return nil +} + +func updateRepository(path string) error { + currentDir, err := os.Getwd() + if err != nil { + return err + } + defer os.Chdir(currentDir) + // change the working directory to the repository + err = os.Chdir(path) + if err != nil { + return err + } + // fetch the latest changes + out, err := exec.Command("git", "pull", "--depth", "1", "--no-tags", "origin").CombinedOutput() + if err != nil { + return utils.NewInternalError(string(out) + "(" + err.Error() + ")") + } + exec.Command("git", "reflog", "expire", "--expire=now", "--all").Run() + exec.Command("git", "gc", "--prune=now").Run() + return nil +} diff --git a/internal/storage/list.go b/internal/storage/list.go index 679314f..2b10715 100644 --- a/internal/storage/list.go +++ b/internal/storage/list.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "fmt" + "net/url" "os" "slices" "strconv" @@ -147,7 +148,11 @@ func (s *LocalStorage) LoadLists() ([]string, error) { if file.IsDir() { continue } - lists = append(lists, file.Name()) + name, err := url.QueryUnescape(file.Name()) + if err != nil { + name = file.Name() + } + lists = append(lists, name) } return lists, nil } diff --git a/internal/storage/storage.go b/internal/storage/storage.go index c8759e4..0807341 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -1,6 +1,7 @@ package storage import ( + "net/url" "os" "path" @@ -8,7 +9,8 @@ import ( ) const ( - listsDir = "lists" + listsDir = "lists" + exploreDir = "explore" ) type LocalStorage struct { @@ -109,5 +111,13 @@ func (s *LocalStorage) joinListsDir(file string) (string, error) { if err != nil { return "", err } - return path.Join(base, file), nil + return path.Join(base, url.QueryEscape(file)), nil +} + +func (s *LocalStorage) JoinExploreDir(file string) (string, error) { + base, err := s.JoinCacheDir(exploreDir) + if err != nil { + return "", err + } + return path.Join(base, url.QueryEscape(file)), nil } diff --git a/internal/utils/opml.go b/internal/utils/opml.go index c606215..f9155a1 100644 --- a/internal/utils/opml.go +++ b/internal/utils/opml.go @@ -2,6 +2,7 @@ package utils import ( "encoding/xml" + "os" ) type OPML struct { @@ -12,12 +13,13 @@ type OPML struct { } type OPMLHead struct { - Title string `xml:"title,omitempty"` + Title string `xml:"title,omitempty"` + DateCreated string `xml:"dateCreated,omitempty"` } type OPMLBody struct { - Text string `xml:"text,attr,omitempty"` - Oultines []*OPMLOutline `xml:"outline"` + Text string `xml:"text,attr,omitempty"` + Outltines []*OPMLOutline `xml:"outline"` } type OPMLOutline struct { @@ -26,3 +28,16 @@ type OPMLOutline struct { XMLURL string `xml:"xmlUrl,attr,omitempty"` Outlines []*OPMLOutline `xml:"outline"` } + +func ParseOPMLFile(path string) (*OPML, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + opml := &OPML{} + err = xml.Unmarshal(b, opml) + if err != nil { + return nil, err + } + return opml, nil +}