go: par3 of new fetch, fixed search population, more testing needed?

backend/src/db.go

@@ -61,41 +61,6 @@ func initTables(db *sql.DB) error {
 	return nil
 }
 
-// insertSearch populates `search` table.
-// Assumes there are already `entries_content` on the db.
-func insertSearch(db *sql.DB, lastentryid int) error {
-	tx, err := db.Begin()
-	if err != nil {
-		return err
-	}
-	stmt, err := tx.Prepare(`
-    insert into search
-    select entriesid,content
-      from entries_content
-     where entriedid > ?;
-    `)
-	if err != nil {
-		return err
-	}
-	defer stmt.Close()
-	_, err = stmt.Exec(lastentryid)
-	if err != nil {
-		return err
-	}
-	err = tx.Commit()
-	if err != nil {
-		return err
-	}
-	sqlStmt := `
-    insert into search(search) values('optimize');
-    vacuum;`
-	_, err = db.Exec(sqlStmt)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
 // LoadDb loads bare minum data from a sqlite db, if exits, into Feeds
 func LoadDb(db *sql.DB) (feeds Feeds, err error) {
 	rows, err := db.Query(`
@@ -154,6 +119,7 @@ func InitDB(dbname string) (db *sql.DB, err error) {
 }
 
 func (feeds Feeds) Save(db *sql.DB) error {
+	fmt.Println("starting db save...")
 	tx, err := db.Begin()
 	if err != nil {
 		return err
@@ -182,6 +148,11 @@ func (feeds Feeds) Save(db *sql.DB) error {
 		return err
 	}
 	defer stmt_entry_content.Close()
+	stmt_entry_search, err := tx.Prepare("INSERT INTO search(entriesid,content) VALUES (?,?)")
+	if err != nil {
+		return err
+	}
+	defer stmt_entry_search.Close()
 	stmt_feeds_meta_update, err := tx.Prepare(`
     UPDATE feeds_metadata
        SET lastfetch = strftime('%s'), lastmodified = ?, etag = ?
@@ -192,7 +163,6 @@ func (feeds Feeds) Save(db *sql.DB) error {
 	}
 	defer stmt_feeds_meta_update.Close()
 
-	lastEntryId := 0
 	for _, feed := range feeds {
 		effectiveFeedId := feed.RawId
 		if feed.RawLastFetch.IsZero() { // first time seen
@@ -239,6 +209,13 @@ func (feeds Feeds) Save(db *sql.DB) error {
 			if err != nil {
 				return err
 			}
+			_, err = stmt_entry_search.Exec(
+				lastEntryId,
+				entry.Content,
+			)
+			if err != nil {
+				return err
+			}
 		}
 	}
 
@@ -247,12 +224,13 @@ func (feeds Feeds) Save(db *sql.DB) error {
 		return err
 	}
 
-	if lastEntryId > 0 {
-		err = insertSearch(db, lastEntryId)
-		if err != nil {
-			return err
-		}
+	sqlStmt := `
+    insert into search(search) values('optimize');
+    vacuum;`
+	_, err = db.Exec(sqlStmt)
+	if err != nil {
+		return err
 	}
-
+	fmt.Println("...save done!")
 	return nil
 }

backend/src/feed.go

@@ -39,8 +39,12 @@ func (feed *Feed) FetchMetadata() (err error) {
 
 	etags, ok := res.Header["Etag"]
 	if ok && len(etags) > 0 {
-		fmt.Printf("found an etag (%s) for url (%s)\n", etags[0], feed.Url)
-		fmt.Println("old etag: ", feed.RawEtag)
+		fmt.Printf(
+			"found an etag (%s) for url (%s), old value (%s)\n",
+			etags[0],
+			feed.Url,
+			feed.RawEtag,
+		)
 		if feed.RawEtag == etags[0] {
 			return fmt.Errorf("same etag (%s), skipping feed (%s)", feed.RawEtag, feed.Url)
 		}
@@ -49,8 +53,12 @@ func (feed *Feed) FetchMetadata() (err error) {
 
 	lastmodified, ok := res.Header["Last-Modified"]
 	if ok && len(lastmodified) > 0 {
-		fmt.Printf("found an last-modified (%s) for url (%s)\n", lastmodified[0], feed.Url)
-		fmt.Println("old last-modified: ", feed.RawLastModified)
+		fmt.Printf(
+			"found an last-modified (%s) for url (%s), old value (%s)\n",
+			lastmodified[0],
+			feed.Url,
+			feed.RawLastModified,
+		)
 		if feed.RawLastModified == lastmodified[0] {
 			return fmt.Errorf(
 				"same last-modified (%s), skipping feed (%s)",
@@ -68,7 +76,7 @@ func (feed *Feed) Fetch() error {
 
 	err := feed.FetchMetadata()
 	if err != nil {
-		fmt.Printf("dropping feed with error (%v)\n", err)
+		fmt.Printf("skipping feed fetch with reason (%v)\n", err)
 		return nil
 	}