Integrate Google Spreadsheet Habits (#16)

* initial PR commit

* add boilerplate source & configuration

* simplify source config access

* don't require sources to save the whole config in a struct

* add telegram dependency to readme

* reorder dependencies in readme

* add a context argument to the source interface method FetchNewCards

* refactor main package

* implement auth & basic reads from sheets

* simplify method signature

* fix bug

* polish

* allow no-description cards in trello package

* fully implement FetchNewCards method for habits

* add comments

* update readme

* polish & add some unit tests

* get rid of problematic test case

.gitignore

@@ -1 +1,2 @@
 config.yml
+*.json

README.md

@@ -9,38 +9,52 @@ Run this as a cron job to periodically check custom data sources and automatical
 An example use case (which is already implemented) could be to create a Trello card for each GitHub issue that's been assigned to you.
 
 ## Currently Available Sources
- * Github Issues - https://github.com/issues/assigned
- * TodoDock Tasks - https://tododock.com
+| | |
+|:-|:-|
+| Assigned Github Issues                |   https://github.com/issues/assigned      |
+| TodoDock Tasks                        |   https://tododock.com                    |
+| Daily Habits (Google Spreadsheets)    |   https://www.google.com/sheets/about/    |
 
 Feel free to add new sources or improve the implementations of the existing ones. Contributions are always welcome!
 
 ## Configuration
-Copy and rename `config.example.yml` as `config.yml`, then set your own values in `config.yml` according to the following:
+Copy and rename `config.example.yml` as `config.yml`, then set your own values in `config.yml`. Most of the configuration parameters are self explanatory, so the following only covers some of them:
 
-#### Global Timeout
+### Global Timeout
 You can edit the `timeout_secs` config value in order to update global timeout (in seconds) for a single execution. 
 
 The execution will not terminate until the timeout is reached, so it's important that the timeout is shorter than the cron job period.
 
-#### Trello
+### Trello
 You need to set your [Trello API key & token](https://trello.com/app-key) in the configuraiton file, as well as the Trello board & list IDs.
 
 The given list will be the one that new cards is going to be inserted, and it has to be in the given board.
 
-#### Enabling/Disbling Individual Data Sources
+### Telegram
+You need a Telegram token & a chat ID in order to enable the integration if you want to receive messages on card updates & possible errors.
+
+### Data Sources
+Every data source must have the following configuration parameters under the `source_config` key:
+ * `name`
+ * `enabled`
+ * `strict`
+ * `label_id`
+ * `period`
+
+#### `enabled`
 In order to disable a source, just update the `enabled` flag to `false`. There's no need to remove/edit the other parameters for that source.
 
-#### Strict Mode
-Strict mode can be enabled for individual data sources by setting the `strict` flag to `true`. When strict mode is enabled, all the existing Trello cards in the board with the label for the corresponding data source will be deleted, unless the card also exists in the fresh data. 
+#### `strict`
+Strict mode, which is recommended for most cases, can be enabled for individual data sources by setting the `strict` flag to `true`.
 
-For instance, strict mode can be used to automatically remove resolved GitHub issues from the board. Every time the source is queried, it will return an up-to-date set of open issues. If the board contains any cards that doesn't exist in that set, they will be automatically deleted.
+When strict mode is enabled, all the existing Trello cards in the board with the label for the corresponding data source will be deleted, unless the card also exists in the fresh data.
 
-#### Telegram
-Since it's not very practical to manually check a log file for a cron job, entrello has an optional Telegram integration that you can use if you want to receive messages on card updates & possible errors.
+For instance, strict mode can be used to automatically remove resolved GitHub issues from the board. Every time the source is queried, it will return an up-to-date set of open issues. If the board contains any cards that doesn't exist in that set, they will be automatically deleted.
 
-You need a Telegram token & a chat ID in order to enable the integration.
+#### `label_id`
+Each data source must have a distinct Trello label associated with it.
 
-#### Custom Periods
+#### `period`
 You can define a custom query period for each source, by populating the `type` and `interval` fields under the `period` for a source.
 
 Example:
@@ -83,10 +97,12 @@ Both of the following jobs run every hour and both assume that `config.yml` is l
 ```
 
 ## 3rd Party Dependencies
-| Dependency | Purpose |
+| | |
 |:-|:-|
 | [adlio/trello](https://github.com/adlio/trello)           | Trello API Client |
+| [go-telegram-bot-api/telegram-bot-api](https://github.com/go-telegram-bot-api/telegram-bot-api) | Telegram Bot API |
 | [google/go-cmp](https://github.com/google/go-cmp)         | Equality Comparisons in Tests |
 | [go-github/github](https://github.com/google/go-github)   | GitHub API Client |
 | [golang/oauth2](https://github.com/golang/oauth2)         | OAuth 2.0 Client |
+| [googleapis/google-api-go-client](https://github.com/googleapis/google-api-go-client) | Google API Client |
 | [go-yaml/yaml](https://github.com/go-yaml/yaml)           | Decoding YAML Configuration |

cmd/entrello/main.go

@@ -6,14 +6,23 @@ import (
 	"time"
 
 	"github.com/utkuufuk/entrello/internal/config"
+	"github.com/utkuufuk/entrello/internal/github"
+	"github.com/utkuufuk/entrello/internal/habits"
 	"github.com/utkuufuk/entrello/internal/syslog"
+	"github.com/utkuufuk/entrello/internal/tododock"
 	"github.com/utkuufuk/entrello/internal/trello"
 )
 
 var (
 	logger syslog.Logger
 )
 
+type CardQueue struct {
+	add chan trello.Card
+	del chan trello.Card
+	err chan error
+}
+
 func main() {
 	// read config params
 	cfg, err := config.ReadConfig("config.yml")
@@ -30,7 +39,7 @@ func main() {
 	defer cancel()
 
 	// get a list of enabled sources and the corresponding labels for each source
-	sources, labels := getEnabledSourcesAndLabels(ctx, cfg.Sources)
+	sources, labels := getEnabledSourcesAndLabels(cfg.Sources)
 	if len(sources) == 0 {
 		return
 	}
@@ -49,7 +58,55 @@ func main() {
 	// concurrently fetch new cards from sources and start processing cards to be created & deleted
 	q := CardQueue{make(chan trello.Card), make(chan trello.Card), make(chan error)}
 	for _, src := range sources {
-		go queueActionables(src, client, q)
+		go src.queueActionables(ctx, client, q)
 	}
 	processActionables(ctx, client, q)
 }
+
+// getEnabledSourcesAndLabels returns a slice of enabled sources & their labels as a separate slice
+func getEnabledSourcesAndLabels(cfg config.Sources) (sources []source, labels []string) {
+	arr := []source{
+		{cfg.GithubIssues.SourceConfig, github.GetSource(cfg.GithubIssues)},
+		{cfg.TodoDock.SourceConfig, tododock.GetSource(cfg.TodoDock)},
+		{cfg.Habits.SourceConfig, habits.GetSource(cfg.Habits)},
+	}
+
+	now := time.Now()
+
+	for _, src := range arr {
+		if ok, err := src.shouldQuery(now); !ok {
+			if err != nil {
+				logger.Errorf("could not check if '%s' should be queried or not, skipping", src.cfg.Name)
+			}
+			continue
+		}
+		sources = append(sources, src)
+		labels = append(labels, src.cfg.Label)
+	}
+	return sources, labels
+}
+
+// processActionables listens to the card queue in an infinite loop and creates/deletes Trello cards
+// depending on which channel the cards come from. Terminates whenever the global timeout is reached.
+func processActionables(ctx context.Context, client trello.Client, q CardQueue) {
+	for {
+		select {
+		case c := <-q.add:
+			if err := client.CreateCard(c); err != nil {
+				logger.Errorf("could not create Trello card: %v", err)
+				break
+			}
+			logger.Printf("created new card: %s", c.Name)
+		case c := <-q.del:
+			if err := client.ArchiveCard(c); err != nil {
+				logger.Errorf("could not archive card card: %v", err)
+				break
+			}
+			logger.Printf("archived stale card: %s", c.Name)
+		case err := <-q.err:
+			logger.Errorf("%v", err)
+		case <-ctx.Done():
+			return
+		}
+	}
+}

cmd/entrello/source.go

@@ -6,65 +6,28 @@ import (
 	"time"
 
 	"github.com/utkuufuk/entrello/internal/config"
-	"github.com/utkuufuk/entrello/internal/github"
-	"github.com/utkuufuk/entrello/internal/tododock"
 	"github.com/utkuufuk/entrello/internal/trello"
 )
 
-// source defines an interface for a Trello card source
-type source interface {
-	// IsEnabled returns true if the source is enabled.
-	IsEnabled() bool
-
-	// IsStrict returns true if "strict" mode is enabled for the source
-	IsStrict() bool
-
-	// GetName returns a human-readable name of the source
-	GetName() string
-
-	// GetLabel returns the corresponding card label ID for the source
-	GetLabel() string
-
-	// GetPeriod returns the period in minutes that the source should be checked
-	GetPeriod() config.Period
-
-	// FetchNewCards returns a list of Trello cards to be inserted into the board from the source
-	FetchNewCards() ([]trello.Card, error)
-}
-
-// getEnabledSourcesAndLabels returns a list of enabled sources & all relevant label IDs
-func getEnabledSourcesAndLabels(ctx context.Context, cfg config.Sources) (sources []source, labels []string) {
-	arr := []source{
-		github.GetSource(ctx, cfg.GithubIssues),
-		tododock.GetSource(cfg.TodoDock),
-	}
-	now := time.Now()
-
-	for _, src := range arr {
-		if ok, err := shouldQuery(src, now); !ok {
-			if err != nil {
-				logger.Errorf("could not check if '%s' should be queried or not, skipping", src.GetName())
-			}
-			continue
-		}
-		sources = append(sources, src)
-		labels = append(labels, src.GetLabel())
+type source struct {
+	cfg config.SourceConfig
+	api interface {
+		FetchNewCards(ctx context.Context, cfg config.SourceConfig) ([]trello.Card, error)
 	}
-	return sources, labels
 }
 
-// shouldQuery checks if the given source should be queried at the given time
-func shouldQuery(src source, now time.Time) (bool, error) {
-	if !src.IsEnabled() {
+// shouldQuery checks if a the source should be queried at the given time
+func (s source) shouldQuery(now time.Time) (bool, error) {
+	if !s.cfg.Enabled {
 		return false, nil
 	}
 
-	interval := src.GetPeriod().Interval
+	interval := s.cfg.Period.Interval
 	if interval < 0 {
 		return false, fmt.Errorf("period interval must be a positive integer, got: '%d'", interval)
 	}
 
-	switch src.GetPeriod().Type {
+	switch s.cfg.Period.Type {
 	case config.PERIOD_TYPE_DEFAULT:
 		return true, nil
 	case config.PERIOD_TYPE_DAY:
@@ -84,5 +47,29 @@ func shouldQuery(src source, now time.Time) (bool, error) {
 		return now.Minute()%interval == 0, nil
 	}
 
-	return false, fmt.Errorf("unrecognized source period type: '%s'", src.GetPeriod().Type)
+	return false, fmt.Errorf("unrecognized source period type: '%s'", s.cfg.Period.Type)
+}
+
+// queueActionables fetches new cards from the source, then pushes those to be created and
+// to be deleted into the corresponding channels, as well as any errors encountered.
+func (s source) queueActionables(ctx context.Context, client trello.Client, q CardQueue) {
+	cards, err := s.api.FetchNewCards(ctx, s.cfg)
+	if err != nil {
+		q.err <- fmt.Errorf("could not fetch cards for source '%s': %v", s.cfg.Name, err)
+		return
+	}
+
+	new, stale := client.CompareWithExisting(cards, s.cfg.Label)
+
+	for _, c := range new {
+		q.add <- c
+	}
+
+	if !s.cfg.Strict {
+		return
+	}
+
+	for _, c := range stale {
+		q.del <- c
+	}
 }