From 2755d0e1739eeab459ff47b0c742f349f879986d Mon Sep 17 00:00:00 2001 From: Christian Zangl Date: Fri, 29 Nov 2024 20:22:23 +0000 Subject: [PATCH 1/3] add json config support --- README.md | 44 +++++++++++++++++++++++++++++--------------- cmd/chkbit/help.go | 6 ++++++ cmd/chkbit/main.go | 36 ++++++++++++++++++++++-------------- scripts/tests | 13 +++++++++++-- 4 files changed, 68 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 33b5afa..62a07e6 100644 --- a/README.md +++ b/README.md @@ -83,27 +83,27 @@ Run `chkbit PATH` to verify only. ``` Usage: chkbit [ ...] [flags] +Ensures the safety of your files by verifying that their data integrity remains +intact over time, especially during transfers and backups. + + For help tips run "chkbit -H" or go to + https://github.com/laktak/chkbit + Arguments: [ ...] directories to check Flags: -h, --help Show context-sensitive help. -H, --tips Show tips. - -c, --check check mode: chkbit will verify files in readonly - mode (default mode) - -u, --update update mode: add and update indices - -a, --add-only add mode: only add new and modified files, - do not check existing (quicker) - -i, --show-ignored-only show-ignored mode: only show ignored files - -m, --show-missing show missing files/directories - -d, --include-dot include dot files + -m, --[no-]show-missing show missing files/directories + -d, --[no-]include-dot include dot files + -S, --[no-]skip-symlinks do not follow symlinks + -R, --[no-]no-recurse do not recurse into subdirectories + -D, --[no-]no-dir-in-index do not track directories in the index --force force update of damaged items (advanced usage only) - -S, --skip-symlinks do not follow symlinks - -R, --no-recurse do not recurse into subdirectories - -D, --no-dir-in-index do not track directories in the index -l, --log-file=STRING write to a logfile if specified - --log-verbose verbose logging + --[no-]log-verbose verbose logging --algo="blake3" hash algorithm: md5, sha512, blake3 (default: blake3) --index-name=".chkbit" filename where chkbit stores its hashes, @@ -112,10 +112,18 @@ Flags: filename that chkbit reads its ignore list from, needs to start with '.' (default: .chkbitignore) -w, --workers=5 number of workers to use (default: 5) - --plain show plain status instead of being fancy - -q, --quiet quiet, don't show progress/information - -v, --verbose verbose output + --[no-]plain show plain status instead of being fancy + -q, --[no-]quiet quiet, don't show progress/information + -v, --[no-]verbose verbose output -V, --version show version information + +mode + -c, --check check mode: chkbit will verify files in readonly + mode (default mode) + -u, --update update mode: add and update indices + -a, --add-only add mode: only add new and modified files, do not + check existing (quicker) + -i, --show-ignored-only show-ignored mode: only show ignored files ``` ``` @@ -142,6 +150,12 @@ Status codes: del: file/directory removed ign: ignored (see .chkbitignore) EXC: exception/panic + +Configuration file (json): +- location /home/spark/.config/chkbit/config.json +- key names are the option names with '-' replaced by '_' +- for example --include-dot is written as: + { "include_dot": true } ``` chkbit is set to use only 5 workers by default so it will not slow your system to a crawl. You can specify a higher number to make it a lot faster if the IO throughput can also keep up. diff --git a/cmd/chkbit/help.go b/cmd/chkbit/help.go index 348cec3..a1c3147 100644 --- a/cmd/chkbit/help.go +++ b/cmd/chkbit/help.go @@ -28,4 +28,10 @@ Status codes: del: file/directory removed ign: ignored (see .chkbitignore) EXC: exception/panic + +Configuration file (json): +- location +- key names are the option names with '-' replaced by '_' +- for example --include-dot is written as: + { "include_dot": true } ` diff --git a/cmd/chkbit/main.go b/cmd/chkbit/main.go index 671f1c6..49797c8 100644 --- a/cmd/chkbit/main.go +++ b/cmd/chkbit/main.go @@ -5,6 +5,7 @@ import ( "io" "log" "os" + "path/filepath" "strings" "sync" "time" @@ -44,25 +45,25 @@ var ( var cli struct { Paths []string `arg:"" optional:"" name:"paths" help:"directories to check"` Tips bool `short:"H" help:"Show tips."` - Check bool `short:"c" help:"check mode: chkbit will verify files in readonly mode (default mode)"` - Update bool `short:"u" help:"update mode: add and update indices"` - AddOnly bool `short:"a" help:"add mode: only add new and modified files, do not check existing (quicker)"` - ShowIgnoredOnly bool `short:"i" help:"show-ignored mode: only show ignored files"` - ShowMissing bool `short:"m" help:"show missing files/directories"` - IncludeDot bool `short:"d" help:"include dot files"` + Check bool `short:"c" help:"check mode: chkbit will verify files in readonly mode (default mode)" xor:"mode" group:"mode"` + Update bool `short:"u" help:"update mode: add and update indices" xor:"mode" group:"mode"` + AddOnly bool `short:"a" help:"add mode: only add new and modified files, do not check existing (quicker)" xor:"mode" group:"mode"` + ShowIgnoredOnly bool `short:"i" help:"show-ignored mode: only show ignored files" xor:"mode" group:"mode"` + ShowMissing bool `short:"m" help:"show missing files/directories" negatable:""` + IncludeDot bool `short:"d" help:"include dot files" negatable:""` + SkipSymlinks bool `short:"S" help:"do not follow symlinks" negatable:""` + NoRecurse bool `short:"R" help:"do not recurse into subdirectories" negatable:""` + NoDirInIndex bool `short:"D" help:"do not track directories in the index" negatable:""` Force bool `help:"force update of damaged items (advanced usage only)"` - SkipSymlinks bool `short:"S" help:"do not follow symlinks"` - NoRecurse bool `short:"R" help:"do not recurse into subdirectories"` - NoDirInIndex bool `short:"D" help:"do not track directories in the index"` LogFile string `short:"l" help:"write to a logfile if specified"` - LogVerbose bool `help:"verbose logging"` + LogVerbose bool `help:"verbose logging" negatable:""` Algo string `default:"blake3" help:"hash algorithm: md5, sha512, blake3 (default: blake3)"` IndexName string `default:".chkbit" help:"filename where chkbit stores its hashes, needs to start with '.' (default: .chkbit)"` IgnoreName string `default:".chkbitignore" help:"filename that chkbit reads its ignore list from, needs to start with '.' (default: .chkbitignore)"` Workers int `short:"w" default:"5" help:"number of workers to use (default: 5)"` - Plain bool `help:"show plain status instead of being fancy"` - Quiet bool `short:"q" help:"quiet, don't show progress/information"` - Verbose bool `short:"v" help:"verbose output"` + Plain bool `help:"show plain status instead of being fancy" negatable:""` + Quiet bool `short:"q" help:"quiet, don't show progress/information" negatable:""` + Verbose bool `short:"v" help:"verbose output" negatable:""` Version bool `short:"V" help:"show version information"` } @@ -285,14 +286,21 @@ func (m *Main) run() { os.Args = append(os.Args, "--help") } + var configPath = "chkbit-config.json" + configRoot, err := os.UserConfigDir() + if err == nil { + configPath = filepath.Join(configRoot, "chkbit/config.json") + } + kong.Parse(&cli, kong.Name("chkbit"), kong.Description(headerHelp), kong.UsageOnError(), + kong.Configuration(kong.JSON, configPath), ) if cli.Tips { - fmt.Println(helpTips) + fmt.Println(strings.ReplaceAll(helpTips, "", configPath)) os.Exit(0) } diff --git a/scripts/tests b/scripts/tests index 5a1b690..ad8c9ba 100755 --- a/scripts/tests +++ b/scripts/tests @@ -4,8 +4,17 @@ set -e script_dir=$(dirname "$(realpath "$0")") cd $script_dir/.. -# prep +echo "# test module" +go test -v . +echo "# test util" +go test -v ./cmd/chkbit/util -count=1 + +echo "# prep files" $script_dir/build -go test -v ./cmd/chkbit/util -count=1 +echo "# test files" +if [[ -f ~/.config/chkbit/config.json ]]; then + echo 'error: unable to test with config file preset' + exit 1 +fi go test -v ./scripts -count=1 From 8d519a2618ef85d58225d56aa2e3fedba6e6cb45 Mon Sep 17 00:00:00 2001 From: Christian Zangl Date: Fri, 29 Nov 2024 20:42:13 +0000 Subject: [PATCH 2/3] readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 62a07e6..6a25292 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,9 @@ For more information see the documentation on [pkg.go.dev](https://pkg.go.dev/gi You would typically run it only on *content* that you keep for a long time (e.g. your pictures, music, videos). -### Why is chkbit placing the index in `.chkbit` files (vs a database)? +### `.chkbit` files vs `.chkbitdb` database + +Note: a `.chkbitdb` database approach is being worked on in [#22](https://github.com/laktak/chkbit/issues/22) if you want to help with testing. The advantage of the .chkbit files is that From c0aee192359017ac7f0b6a4694d4cfbc47b098c6 Mon Sep 17 00:00:00 2001 From: Christian Zangl Date: Mon, 9 Dec 2024 19:58:16 +0000 Subject: [PATCH 3/3] add --no-config option --- README.md | 1 + cmd/chkbit/main.go | 14 +++++++++++++- scripts/run_test.go | 38 +++++++++++++++++++------------------- scripts/tests | 4 ---- 4 files changed, 33 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 6a25292..16a02a0 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ Flags: -S, --[no-]skip-symlinks do not follow symlinks -R, --[no-]no-recurse do not recurse into subdirectories -D, --[no-]no-dir-in-index do not track directories in the index + --no-config ignore the config file --force force update of damaged items (advanced usage only) -l, --log-file=STRING write to a logfile if specified diff --git a/cmd/chkbit/main.go b/cmd/chkbit/main.go index 49797c8..3adc68a 100644 --- a/cmd/chkbit/main.go +++ b/cmd/chkbit/main.go @@ -42,7 +42,7 @@ var ( termAlertFG = lterm.Fg4(1) ) -var cli struct { +type CLI struct { Paths []string `arg:"" optional:"" name:"paths" help:"directories to check"` Tips bool `short:"H" help:"Show tips."` Check bool `short:"c" help:"check mode: chkbit will verify files in readonly mode (default mode)" xor:"mode" group:"mode"` @@ -54,6 +54,7 @@ var cli struct { SkipSymlinks bool `short:"S" help:"do not follow symlinks" negatable:""` NoRecurse bool `short:"R" help:"do not recurse into subdirectories" negatable:""` NoDirInIndex bool `short:"D" help:"do not track directories in the index" negatable:""` + NoConfig bool `help:"ignore the config file"` Force bool `help:"force update of damaged items (advanced usage only)"` LogFile string `short:"l" help:"write to a logfile if specified"` LogVerbose bool `help:"verbose logging" negatable:""` @@ -67,6 +68,8 @@ var cli struct { Version bool `short:"V" help:"show version information"` } +var cli CLI + type Main struct { context *chkbit.Context dmgList []string @@ -299,6 +302,15 @@ func (m *Main) run() { kong.Configuration(kong.JSON, configPath), ) + if cli.NoConfig { + cli = CLI{} + kong.Parse(&cli, + kong.Name("chkbit"), + kong.Description(headerHelp), + kong.UsageOnError(), + ) + } + if cli.Tips { fmt.Println(strings.ReplaceAll(helpTips, "", configPath)) os.Exit(0) diff --git a/scripts/run_test.go b/scripts/run_test.go index 9cf0a51..46e912c 100644 --- a/scripts/run_test.go +++ b/scripts/run_test.go @@ -15,10 +15,12 @@ import ( var testDir = "/tmp/chkbit" -func getCmd() string { +func runCmd(args ...string) *exec.Cmd { _, filename, _, _ := runtime.Caller(0) prjRoot := filepath.Dir(filepath.Dir(filename)) - return filepath.Join(prjRoot, "chkbit") + tool := filepath.Join(prjRoot, "chkbit") + args = append([]string{"--no-config"}, args...) + return exec.Command(tool, args...) } func checkOut(t *testing.T, sout string, expected string) { @@ -139,12 +141,11 @@ func setupMiscFiles() { func TestRoot(t *testing.T) { setupMiscFiles() - tool := getCmd() root := filepath.Join(testDir, "root") // update index, no recourse t.Run("no-recourse", func(t *testing.T) { - cmd := exec.Command(tool, "-umR", filepath.Join(root, "day/office")) + cmd := runCmd("-umR", filepath.Join(root, "day/office")) out, err := cmd.Output() if err != nil { t.Fatalf("failed with '%s'\n", err) @@ -159,7 +160,7 @@ func TestRoot(t *testing.T) { // update remaining index from root t.Run("update-remaining", func(t *testing.T) { - cmd := exec.Command(tool, "-um", root) + cmd := runCmd("-um", root) out, err := cmd.Output() if err != nil { t.Fatalf("failed with '%s'\n", err) @@ -177,7 +178,7 @@ func TestRoot(t *testing.T) { os.RemoveAll(filepath.Join(root, "thing/change")) os.Remove(filepath.Join(root, "time/hour/minute/body-information.csv")) - cmd := exec.Command(tool, "-m", root) + cmd := runCmd("-m", root) out, err := cmd.Output() if err != nil { t.Fatalf("failed with '%s'\n", err) @@ -189,7 +190,7 @@ func TestRoot(t *testing.T) { // do not report missing without -m t.Run("no-missing", func(t *testing.T) { - cmd := exec.Command(tool, root) + cmd := runCmd(root) out, err := cmd.Output() if err != nil { t.Fatalf("failed with '%s'\n", err) @@ -201,7 +202,7 @@ func TestRoot(t *testing.T) { // check for missing and update t.Run("missing", func(t *testing.T) { - cmd := exec.Command(tool, "-um", root) + cmd := runCmd("-um", root) out, err := cmd.Output() if err != nil { t.Fatalf("failed with '%s'\n", err) @@ -214,7 +215,7 @@ func TestRoot(t *testing.T) { // check again t.Run("repeat", func(t *testing.T) { for i := 0; i < 10; i++ { - cmd := exec.Command(tool, "-uv", root) + cmd := runCmd("-uv", root) out, err := cmd.Output() if err != nil { t.Fatalf("failed with '%s'\n", err) @@ -233,7 +234,7 @@ func TestRoot(t *testing.T) { genFiles(filepath.Join(root, "way/add"), 99) genFile(filepath.Join(root, "time/add-file.txt"), 500) - cmd := exec.Command(tool, "-a", root) + cmd := runCmd("-a", root) out, err := cmd.Output() if err != nil { t.Fatalf("failed with '%s'\n", err) @@ -251,7 +252,7 @@ func TestRoot(t *testing.T) { // modify existing genFile(filepath.Join(root, "way/job/word-business.mp3"), 500) - cmd := exec.Command(tool, "-a", root) + cmd := runCmd("-a", root) out, err := cmd.Output() if err != nil { t.Fatalf("failed with '%s'\n", err) @@ -266,7 +267,7 @@ func TestRoot(t *testing.T) { // update remaining t.Run("update-remaining-add", func(t *testing.T) { - cmd := exec.Command(tool, "-u", root) + cmd := runCmd("-u", root) out, err := cmd.Output() if err != nil { t.Fatalf("failed with '%s'\n", err) @@ -281,7 +282,7 @@ func TestRoot(t *testing.T) { genFiles(filepath.Join(root, "way/.hidden"), 99) genFile(filepath.Join(root, "time/.ignored"), 999) - cmd := exec.Command(tool, "-u", root) + cmd := runCmd("-u", root) out, err := cmd.Output() if err != nil { t.Fatalf("failed with '%s'\n", err) @@ -293,7 +294,7 @@ func TestRoot(t *testing.T) { // include dot t.Run("include-dot", func(t *testing.T) { - cmd := exec.Command(tool, "-u", "-d", root) + cmd := runCmd("-u", "-d", root) out, err := cmd.Output() if err != nil { t.Fatalf("failed with '%s'\n", err) @@ -323,7 +324,6 @@ func TestDMG(t *testing.T) { panic(err) } - tool := getCmd() testFile := filepath.Join(testDmg, "test.txt") t1, _ := time.Parse(time.RFC3339, "2022-02-01T11:00:00Z") t2, _ := time.Parse(time.RFC3339, "2022-02-01T12:00:00Z") @@ -334,7 +334,7 @@ func TestDMG(t *testing.T) { os.WriteFile(testFile, []byte("foo1"), 0644) os.Chtimes(testFile, t2, t2) - cmd := exec.Command(tool, "-u", ".") + cmd := runCmd("-u", ".") if out, err := cmd.Output(); err != nil { t.Fatalf("failed with '%s'\n", err) } else { @@ -347,7 +347,7 @@ func TestDMG(t *testing.T) { os.WriteFile(testFile, []byte("foo2"), 0644) os.Chtimes(testFile, t1, t1) - cmd := exec.Command(tool, "-u", ".") + cmd := runCmd("-u", ".") if out, err := cmd.Output(); err != nil { t.Fatalf("failed with '%s'\n", err) } else { @@ -360,7 +360,7 @@ func TestDMG(t *testing.T) { os.WriteFile(testFile, []byte("foo3"), 0644) os.Chtimes(testFile, t3, t3) - cmd := exec.Command(tool, "-u", ".") + cmd := runCmd("-u", ".") if out, err := cmd.Output(); err != nil { t.Fatalf("failed with '%s'\n", err) } else { @@ -373,7 +373,7 @@ func TestDMG(t *testing.T) { os.WriteFile(testFile, []byte("foo4"), 0644) os.Chtimes(testFile, t3, t3) - cmd := exec.Command(tool, "-u", ".") + cmd := runCmd("-u", ".") if out, err := cmd.Output(); err != nil { if cmd.ProcessState.ExitCode() != 1 { t.Fatalf("expected to fail with exit code 1 vs %d!", cmd.ProcessState.ExitCode()) diff --git a/scripts/tests b/scripts/tests index ad8c9ba..736df2b 100755 --- a/scripts/tests +++ b/scripts/tests @@ -13,8 +13,4 @@ echo "# prep files" $script_dir/build echo "# test files" -if [[ -f ~/.config/chkbit/config.json ]]; then - echo 'error: unable to test with config file preset' - exit 1 -fi go test -v ./scripts -count=1