Merge pull request #58 from rigon/40-performance-degradation-in-cache-db

40 performance degradation in cache db

.github/workflows/docker-image.yml

@@ -41,6 +41,12 @@ jobs:
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - name: Add to version
+      # Run only when it is not a release build
+      if: ${{ github.ref_type != 'tag' }}
+      run: |
+        echo "Adding to version: ${{ steps.meta.outputs.version }}"
+        sed -i 's/"version": "\(.*\)"/"version": "\1-${{ steps.meta.outputs.version }}"/' package.json
     - name: Build and push Docker image
       uses: docker/build-push-action@v4
       with:
@@ -57,7 +63,7 @@ jobs:
     needs: build
     runs-on: ubuntu-latest
     # Run only when it is a release build
-    if: github.event_name == 'create' && startsWith(github.ref, 'refs/tags/v')
+    if: github.ref_type == 'tag' && github.event_name == 'create' && startsWith(github.ref, 'refs/tags/v')
 
     steps:
     - name: Checkout

package.json

@@ -1,7 +1,7 @@
 {
   "name": "photo-gallery",
   "private": true,
-  "version": "1.5.2",
+  "version": "1.6.0",
   "type": "module",
   "description": "Photo Gallery is a self-hosted performant application to organize your photos. Built for speed with React and Go.",
   "scripts": {

server/album.go

@@ -131,7 +131,7 @@ func (album *Album) GetPhotos(collection *Collection, runningInBackground bool,
 			if updatedPhotos[photoId] <= 0 { // Files for this photo were processed
 				photo := album.photosMap[photoId]
 				// Fill photo info
-				photo.FillInfo()
+				photo.FillInfo(collection)
 				// Update cache
 				collection.cache.AddPhotoInfo(photo)
 			}

server/cache.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"errors"
 	"log"
+	"os"
 	"path/filepath"
 	"sync"
 
@@ -11,7 +13,7 @@ import (
 )
 
 var dbInfo = DbInfo{
-	Version: 9,
+	Version: 10,
 }
 
 type DbInfo struct {
@@ -29,10 +31,22 @@ type Cache struct {
 	wgFlush   sync.WaitGroup
 }
 
+// Flag album as loaded
+type AlbumSaved struct{}
+
+// Flag album with photos missing thumbnails
+type AlbumThumbs struct {
+	Name string
+}
+
 // Init cache: boltdb and gcache
-func (c *Cache) Init(collection *Collection, rebuildCache bool) error {
-	// Disk cache
-	var err error
+func (c *Cache) Init(collection *Collection, rebuildCache bool) (err error) {
+	// Ensure the thumbnail folder exist
+	thumbsDir := filepath.Join(collection.ThumbsPath, collection.Name+"-thumbs")
+	err = os.MkdirAll(thumbsDir, os.ModePerm)
+	if err != nil {
+		return
+	}
 
 	// Find filename for cache
 	// if collection.DbPath is a filename it will be located in thumbnails directory
@@ -48,7 +62,7 @@ func (c *Cache) Init(collection *Collection, rebuildCache bool) error {
 
 	c.store, err = bolthold.Open(filename, 0600, &bolthold.Options{Options: &bolt.Options{Timeout: 1}})
 	if err != nil {
-		log.Fatal(err)
+		return
 	}
 	// Check DB version
 	var current DbInfo
@@ -57,21 +71,22 @@ func (c *Cache) Init(collection *Collection, rebuildCache bool) error {
 		log.Printf("Current DB version v%d is different than required v%d\n", current.Version, dbInfo.Version)
 		if rebuildCache || err == bolthold.ErrNotFound {
 			log.Printf("Recreating cache DB for collection %s at %s", collection.Name, filename)
-			err := c.store.Bolt().Update(func(tx *bolt.Tx) error {
+			err = c.store.Bolt().Update(func(tx *bolt.Tx) error {
 				tx.DeleteBucket([]byte("DbInfo"))
 				tx.DeleteBucket([]byte("Photo"))
 				tx.DeleteBucket([]byte("AlbumSaved"))
+				tx.DeleteBucket([]byte("ThumbQueue"))
 				tx.DeleteBucket([]byte("_index:Photo:Date"))
-				tx.DeleteBucket([]byte("_index:Photo:HasThumb"))
 				tx.DeleteBucket([]byte("_index:Photo:Location"))
 				tx.DeleteBucket([]byte("_index:Photo:Size"))
 				return c.store.TxInsert(tx, "DbInfo", dbInfo)
 			})
 			if err != nil {
-				log.Fatal(err)
+				return
 			}
 		} else {
-			log.Fatal("Run command with option -r enabled to recreate cache DB")
+			log.Println("Run command with option -r enabled to recreate cache DB")
+			return errors.New("can't use current cache DB")
 		}
 	}
 
@@ -142,13 +157,10 @@ func (c *Cache) SaveAlbum(album *Album) error {
 }
 
 func (c *Cache) SetAlbumFullyScanned(album *Album) error {
-	// Flag this album as loaded
-	type AlbumSaved struct{}
 	return c.store.Upsert(album.Name, AlbumSaved{})
 }
 
 func (c *Cache) IsAlbumFullyScanned(album *Album) bool {
-	type AlbumSaved struct{}
 	var a AlbumSaved
 	return c.store.Get(album.Name, &a) == nil
 }
@@ -182,6 +194,15 @@ func (c *Cache) addInfoBatcher() {
 					if err != nil {
 						log.Println(err)
 					}
+
+					// Add album from the thumbnail queue
+					if !photo.HasThumb {
+						err = c.store.TxUpsert(tx, photo.Album, AlbumThumbs{photo.Album})
+						if err != nil {
+							log.Println(err)
+						}
+					}
+
 					c.wgFlush.Done()
 				}
 				return nil

server/cmdargs.go

@@ -17,16 +17,19 @@ type CmdArgs struct {
 	fullScan        bool
 	recreateCacheDB bool
 	webdavDisabled  bool
+	debug           bool
+	thumbsPath      string
 	collections     map[string]*Collection
 	port            int
 	host            string
 	nWorkersInfo    int
 	nWorkersThumb   int
 }
 
-func parseCollectionOptions(collectionOption string, defaultIndex int) (collection *Collection, err error) {
+func parseCollectionOptions(collectionOption string, index int, defaultThumbsPath string) (collection *Collection, err error) {
 	collection = NewCollection()
-	collection.Index = defaultIndex
+	collection.Index = index
+	collection.ThumbsPath = defaultThumbsPath
 	collection.Hide = false
 	collection.ReadOnly = false
 	collection.RenameOnReplace = true
@@ -42,11 +45,6 @@ func parseCollectionOptions(collectionOption string, defaultIndex int) (collecti
 			log.Printf("%s must be formatted as key=value\n", pair)
 		}
 		switch kv[0] {
-		case "index":
-			collection.Index, err = strconv.Atoi(kv[1])
-			if err != nil {
-				return
-			}
 		case "name":
 			collection.Name = kv[1]
 		case "path":
@@ -85,30 +83,32 @@ func parseCollectionOptions(collectionOption string, defaultIndex int) (collecti
 
 func ParseCmdArgs() (cmdArgs CmdArgs) {
 	var collectionArgs []string
-	zflag.StringSliceVar(&collectionArgs, "collection", collectionArgs, `Specify a new collection. Example name=Photos,path=/photos,thumbs=/tmp
+	zflag.StringSliceVar(&collectionArgs, "collection", collectionArgs, `Define a new collection. The order used will will be the same used in the interface.
+Example: -c name=Photos,path=/photos,thumbs=/tmp
 List of possible options:
-  index          Position in the collection list
   name           Name of the collection
-  path           Path to load the albums from
-  thumbs         Path to store the thumbnails
+  path           Location of the collection, i.e. path where the photos are stored
+  thumbs         Path to store the thumbnails (by default is the path set with --thumbs)
   db             Path to cache DB, if a filename is provided it will be located in thumbnails directory
   hide=false     Hide the collection from the list (does not affect webdav)
   rename=true    Rename files instead of overwriting them
   readonly=false`, zflag.OptShorthand('c'))
 	zflag.BoolVar(&cmdArgs.cacheThumbnails, "cache-thumbnails", true, "Generate missing thumbnails while scanning", zflag.OptAddNegative(), zflag.OptShorthand('b'))
-	zflag.BoolVar(&cmdArgs.disableScan, "disable-scan", false, "Disable scans on start, by default will cache photo info of new albums")
-	//zflag.BoolVar(&cmdArgs.fullScan, "full-scan", false, "Perform a full scan on start (validates if cached data is up to date)")
+	zflag.BoolVar(&cmdArgs.disableScan, "disable-scan", false, "Disable scans on start, by default will run a quick scan (cache info of new albums)")
+	zflag.BoolVar(&cmdArgs.fullScan, "full-scan", false, "Perform a full scan on start (validates if all cached data is up to date)")
 	zflag.BoolVar(&cmdArgs.recreateCacheDB, "recreate-cache", false, "Recreate cache DB, required after DB version upgrade", zflag.OptShorthand('r'))
 	zflag.BoolVar(&cmdArgs.webdavDisabled, "disable-webdav", false, "Disable WebDAV")
-	zflag.StringVar(&cmdArgs.host, "host", "localhost", "Specify a host", zflag.OptShorthand('h'))
+	zflag.BoolVar(&cmdArgs.debug, "debug", false, "Enable debug")
+	zflag.StringVar(&cmdArgs.thumbsPath, "thumbs", "", "Default path to store thumbnails", zflag.OptShorthand('t'))
+	zflag.StringVar(&cmdArgs.host, "host", "localhost", "Specify a host", zflag.OptShorthand('H'))
 	zflag.IntVar(&cmdArgs.port, "port", 3080, "Specify a port", zflag.OptShorthand('p'))
 	zflag.IntVar(&cmdArgs.nWorkersInfo, "workers-info", 2, "Number of concurrent workers to extract photos info")
 	zflag.IntVar(&cmdArgs.nWorkersThumb, "workers-thumb", runtime.NumCPU(), "Number of concurrent workers to generate thumbnails, by default number of CPUs")
 	zflag.Parse()
 
 	cmdArgs.collections = make(map[string]*Collection)
 	for i, c := range collectionArgs {
-		collection, err := parseCollectionOptions(c, i)
+		collection, err := parseCollectionOptions(c, i, cmdArgs.thumbsPath)
 		if err != nil {
 			log.Fatal(err)
 		}

server/file.go

@@ -237,9 +237,17 @@ func (file File) CreateThumbnail(thumbpath string, w io.Writer) (err error) {
 	}
 
 	// Error decoding image from source
+	if img == nil {
+		return errors.New("invalid image")
+	}
 	if err != nil {
 		return
 	}
 
+	// Ensure the directories exist
+	if err = os.MkdirAll(filepath.Dir(thumbpath), os.ModePerm); err != nil {
+		return
+	}
+
 	return CreateThumbnailFromImage(img, thumbpath, w)
 }

server/go.sum

@@ -59,8 +59,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
 golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
-golang.org/x/exp v0.0.0-20230807204917-050eac23e9de h1:l5Za6utMv/HsBWWqzt4S8X17j+kt1uVETUX5UFhn2rE=
-golang.org/x/exp v0.0.0-20230807204917-050eac23e9de/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 h1:EDuYyU/MkFXllv9QF9819VlI9a4tzGuCbhG0ExK9o1U=
 golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=

server/image.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"errors"
 	"fmt"
 	"image"
 	_ "image/gif"
@@ -114,6 +113,7 @@ func DecodeImage(filepath string, orientation Orientation) (image.Image, error)
 	// From: https://github.com/disintegration/imaging/blob/d40f48ce0f098c53ab1fcd6e0e402da682262da5/io.go#L424
 	switch orientation {
 	case orientationNormal:
+		// Nothing to do
 	case orientationFlipH:
 		img = imaging.FlipH(img)
 	case orientationFlipV:
@@ -167,10 +167,6 @@ func ExtractImageInfoOpened(fin *os.File) (format string, config image.Config, e
 }
 
 func CreateThumbnailFromImage(img image.Image, thumbpath string, w io.Writer) error {
-	if img == nil {
-		return errors.New("invalid image")
-	}
-
 	// Open output file thumbnail
 	fout, err := os.OpenFile(thumbpath, os.O_RDWR|os.O_CREATE, 0644)
 	if err != nil {

server/main.go

@@ -232,7 +232,10 @@ func main() {
 
 	InitWorkers(config)
 	for _, collection := range config.collections {
-		collection.cache.Init(collection, config.recreateCacheDB)
+		err := collection.cache.Init(collection, config.recreateCacheDB)
+		if err != nil {
+			log.Fatal(err)
+		}
 		defer collection.cache.End()
 	}
 
@@ -246,15 +249,16 @@ func main() {
 			}
 			// Clean thumbnails of deleted photos
 			if config.fullScan {
-				CleanupThumbnails(config.collections)
+				for _, collection := range config.collections {
+					collection.CleanupThumbnails()
+				}
 			}
 			// Then create thumbnails
 			if config.cacheThumbnails {
 				for _, collection := range config.collections {
 					collection.CreateThumbnails()
 				}
 			}
-
 			log.Println("Background scan complete!")
 		}()
 	}
@@ -286,8 +290,11 @@ func main() {
 	}))
 	e.Use(func(next echo.HandlerFunc) echo.HandlerFunc {
 		return func(c echo.Context) error {
-			defer ResumeBackgroundWork()
-			SuspendBackgroundWork()
+			// Suspend background work only when requests to the API are received
+			if matched, _ := path.Match("/api/*", c.Path()); matched {
+				defer ResumeBackgroundWork()
+				SuspendBackgroundWork()
+			}
 			return next(c)
 		}
 	})
@@ -309,6 +316,11 @@ func main() {
 		}
 	})
 
+	// Enable View Status interface
+	if config.debug {
+		ViewStatusInit(e.Group("/status"))
+	}
+
 	// API
 	api := e.Group("/api")
 	api.GET("/pseudos", pseudos)