add database refresh, backup, move when finished

cmd/chkbit/main.go

@@ -209,7 +209,7 @@ func (m *Main) process() (bool, error) {
 		defer wg.Done()
 		m.showStatus()
 	}()
-	m.context.Start(pathList)
+	m.context.Process(pathList)
 	wg.Wait()
 
 	return true, nil
@@ -346,7 +346,7 @@ func (m *Main) run() int {
 			fmt.Println("error: specify a path")
 			return 1
 		}
-		if err := chkbit.InitializeIndexDb(cli.Paths[0], cli.Force); err != nil {
+		if err := chkbit.InitializeIndexDb(cli.Paths[0], cli.IndexName, cli.Force); err != nil {
 			fmt.Println("error: " + err.Error())
 			return 1
 		}

context.go

@@ -46,7 +46,10 @@ func NewContext(numWorkers int, hashAlgo string, indexFilename string, ignoreFil
 		return nil, errors.New("the ignore filename must start with a dot")
 	}
 	if hashAlgo != "md5" && hashAlgo != "sha512" && hashAlgo != "blake3" {
-		return nil, errors.New(hashAlgo + " is unknown.")
+		return nil, errors.New(hashAlgo + " is unknown")
+	}
+	if numWorkers < 1 {
+		return nil, errors.New("expected numWorkers >= 1")
 	}
 	return &Context{
 		NumWorkers:     numWorkers,
@@ -112,10 +115,11 @@ func (context *Context) endWork() {
 }
 
 func (context *Context) isChkbitFile(name string) bool {
-	return name == context.IndexFilename || name == context.IgnoreFilename
+	// any file with the index prefix is ignored (to allow for .bak and db files)
+	return strings.HasPrefix(name, context.IndexFilename) || name == context.IgnoreFilename
 }
 
-func (context *Context) Start(pathList []string) {
+func (context *Context) Process(pathList []string) {
 	context.NumTotal = 0
 	context.NumIdxUpd = 0
 	context.NumNew = 0
@@ -124,7 +128,7 @@ func (context *Context) Start(pathList []string) {
 
 	err := context.store.Open(!context.UpdateIndex)
 	if err != nil {
-		context.logErr(indexDbName, err)
+		context.logErr("index", err)
 		context.LogQueue <- nil
 		return
 	}
@@ -147,6 +151,11 @@ func (context *Context) Start(pathList []string) {
 		}
 	}()
 	wg.Wait()
+
+	err = context.store.Finish()
+	if err != nil {
+		context.logErr("index", err)
+	}
 	context.LogQueue <- nil
 }
 
@@ -207,11 +216,9 @@ func (context *Context) UseIndexDb(pathList []string) (root string, relativePath
 	if len(pathList) == 0 {
 		return "", nil, errors.New("missing path(s)")
 	}
-	err = context.store.UseDb(pathList[0])
+	root, err = LocateIndexDb(pathList[0], context.IndexFilename)
 	if err == nil {
 
-		root = context.store.dbPath
-
 		for _, path := range pathList {
 			path, err = filepath.Abs(path)
 			if err != nil {
@@ -229,6 +236,8 @@ func (context *Context) UseIndexDb(pathList []string) (root string, relativePath
 			}
 			relativePathList = append(relativePathList, relativePath)
 		}
+
+		context.store.UseDb(root, context.IndexFilename, len(relativePathList) == 1 && relativePathList[0] == ".")
 	}
 
 	return

ignore_test.go

@@ -5,9 +5,11 @@ import (
 )
 
 func TestShouldIgnore(t *testing.T) {
-	context := &Context{
-		IncludeDot: true,
+	context, err := NewContext(1, "blake3", ".chkbit", ".chkbitignore")
+	if err != nil {
+		t.Error(err)
 	}
+	context.IncludeDot = true
 
 	ignore1 := &Ignore{
 		parentIgnore: nil,
@@ -39,6 +41,7 @@ func TestShouldIgnore(t *testing.T) {
 		expected bool
 	}{
 		// test root
+		{ignore1, ".chkbit-db", true},
 		{ignore1, "all.txt", true},
 		{ignore1, "readme.md", false},
 		{ignore1, "photo.jpg", true},
@@ -47,6 +50,7 @@ func TestShouldIgnore(t *testing.T) {
 		{ignore1, "tokyo", true},
 		{ignore1, "sydney", true},
 		// test in berlin
+		{ignore2, ".chkbit", true},
 		{ignore2, "all.txt", true},
 		{ignore2, "readme.md", true},
 		{ignore2, "photo.jpg", false},

index.go

@@ -223,7 +223,7 @@ func (i *Index) calcFile(name string, a string) (*idxInfo, error) {
 }
 
 func (i *Index) save() (bool, error) {
-	if i.modified {
+	if i.modified || !i.readonly && i.context.store.refreshDb {
 		if i.readonly {
 			return false, errors.New("error trying to save a readonly index")
 		}

store.go

@@ -2,51 +2,70 @@ package chkbit
 
 import (
 	"errors"
+	"io"
 	"os"
 	"path/filepath"
 
 	bolt "go.etcd.io/bbolt"
 )
 
 type store struct {
-	useDb  bool
-	dbPath string
-	conn   *bolt.DB
+	readOnly  bool
+	useDb     bool
+	refreshDb bool
+	dbPath    string
+	indexName string
+	dbFile    string
+	newFile   string
+	connR     *bolt.DB
+	connW     *bolt.DB
 }
 
 const (
-	indexDbName = ".chkbitdb"
+	dbSuffix    = "-db"
+	bakDbSuffix = ".bak"
+	newDbSuffix = ".new"
 )
 
-func (s *store) UseDb(path string) error {
-	var err error
-	s.dbPath, err = LocateIndexDb(path)
-	if err == nil {
-		s.useDb = true
-	}
-	return err
-}
-
-func (s *store) GetDbFile() string {
-	return filepath.Join(s.dbPath, indexDbName)
+func (s *store) UseDb(path string, indexName string, refresh bool) {
+	s.dbPath = path
+	s.indexName = indexName
+	s.useDb = true
+	s.refreshDb = refresh
 }
 
 func (s *store) Open(readOnly bool) error {
 	var err error
+	s.readOnly = readOnly
 	if s.useDb {
-		opt := &bolt.Options{
-			ReadOnly:     readOnly,
+		optR := &bolt.Options{
+			ReadOnly:     true,
 			Timeout:      0,
 			NoGrowSync:   false,
 			FreelistType: bolt.FreelistArrayType,
 		}
-		_, err = os.Stat(s.GetDbFile())
-		if os.IsNotExist(err) {
-			return nil
+		optW := &bolt.Options{
+			Timeout:      0,
+			NoGrowSync:   false,
+			FreelistType: bolt.FreelistArrayType,
 		}
-		s.conn, err = bolt.Open(s.GetDbFile(), 0600, opt)
-		if err == nil && !readOnly {
-			err = s.conn.Update(func(tx *bolt.Tx) error {
+		s.dbFile = getDbFile(s.dbPath, s.indexName, "")
+		s.connR, err = bolt.Open(s.dbFile, 0600, optR)
+
+		if !readOnly {
+			s.newFile = getDbFile(s.dbPath, s.indexName, newDbSuffix)
+
+			if s.refreshDb {
+				err = clearFile(s.newFile)
+				if err != nil {
+					return err
+				}
+			} else {
+				err = copyFile(s.dbFile, s.newFile)
+			}
+
+			s.connW, err = bolt.Open(s.newFile, 0600, optW)
+			err = s.connW.Update(func(tx *bolt.Tx) error {
 				_, err := tx.CreateBucketIfNotExists([]byte("data"))
 				return err
 			})
@@ -56,20 +75,39 @@ func (s *store) Open(readOnly bool) error {
 }
 
 func (s *store) Close() {
-	if s.useDb && s.conn != nil {
-		s.conn.Close()
+	if s.useDb {
+		if s.connW != nil {
+			s.connW.Close()
+		}
+		if s.connR != nil {
+			s.connR.Close()
+		}
 	}
 }
 
+func (s *store) Finish() error {
+	if s.useDb && !s.readOnly {
+		bakFile := getDbFile(s.dbPath, s.indexName, bakDbSuffix)
+		err := os.Rename(s.dbFile, bakFile)
+		if err != nil {
+			return err
+		}
+		err = os.Rename(s.newFile, s.dbFile)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (s *store) Load(indexPath string) ([]byte, error) {
 	var err error
 	var value []byte
 	if s.useDb {
-		if s.conn == nil {
-			// readOnly without db
-			return nil, nil
+		if s.connR == nil {
+			return nil, errors.New("db not loaded")
 		}
-		err = s.conn.View(func(tx *bolt.Tx) error {
+		err = s.connR.View(func(tx *bolt.Tx) error {
 			b := tx.Bucket([]byte("data"))
 			value = b.Get([]byte(indexPath))
 			return nil
@@ -89,7 +127,7 @@ func (s *store) Load(indexPath string) ([]byte, error) {
 func (s *store) Save(indexPath string, value []byte) error {
 	var err error
 	if s.useDb {
-		err = s.conn.Update(func(tx *bolt.Tx) error {
+		err = s.connW.Update(func(tx *bolt.Tx) error {
 			b := tx.Bucket([]byte("data"))
 			return b.Put([]byte(indexPath), value)
 		})
@@ -105,8 +143,8 @@ func (s *store) Save(indexPath string, value []byte) error {
 	return err
 }
 
-func InitializeIndexDb(path string, force bool) error {
-	file := filepath.Join(path, indexDbName)
+func InitializeIndexDb(path, indexName string, force bool) error {
+	file := getDbFile(path, indexName, "")
 	_, err := os.Stat(file)
 	if !os.IsNotExist(err) {
 		if force {
@@ -129,13 +167,13 @@ func InitializeIndexDb(path string, force bool) error {
 	})
 }
 
-func LocateIndexDb(path string) (string, error) {
+func LocateIndexDb(path, indexName string) (string, error) {
 	var err error
 	if path, err = filepath.Abs(path); err != nil {
 		return "", err
 	}
 	for {
-		file := filepath.Join(path, indexDbName)
+		file := getDbFile(path, indexName, "")
 		_, err = os.Stat(file)
 		if !os.IsNotExist(err) {
 			return path, nil
@@ -147,3 +185,33 @@ func LocateIndexDb(path string) (string, error) {
 		}
 	}
 }
+
+func getDbFile(path, indexFilename, suffix string) string {
+	return filepath.Join(path, indexFilename+dbSuffix+suffix)
+}
+
+func clearFile(file string) error {
+	_, err := os.Stat(file)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	return os.Remove(file)
+}
+
+func copyFile(src, dst string) error {
+
+	sf, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer sf.Close()
+
+	df, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer df.Close()
+
+	_, err = io.Copy(df, sf)
+	return err
+}