Merge pull request #29 from LeoVie/caching

Cache normalized files

.gitignore

@@ -5,4 +5,7 @@ build/*
 !build/.gitkeep
 _testdata/generated/reports/*
 !_testdata/generated/reports/.gitkeep
-_testdata/xdry.log
+_testdata/xdry.log
+xdry-cache_*.json
+/xdry.json
+/xdry.log

_testdata/xdry_1.json

@@ -4,7 +4,8 @@
       "level-1": 10,
       "level-2": 20
     },
-    "logPath": "%pwd%/test/_testdata/xdry.log"
+    "logPath": "%pwd%/test/_testdata/xdry.log",
+    "cacheDirectory": "%pwd%/test/_testdata/cache"
   },
   "directories": [
     "%pwd%/test/_testdata/php",

_testdata/xdry_minimal.json

@@ -0,0 +1,2 @@
+{
+}

src/internal/cache/cache.go

@@ -0,0 +1,86 @@
+package cache
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"os"
+	"x-dry-go/src/internal/structs"
+)
+
+type FileCache struct {
+	Path  string
+	Items map[string]structs.File `json:"items"`
+}
+
+func InitOrReadCache(path string) (*FileCache, error) {
+	if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
+		return initCache(path)
+	}
+
+	return readFileCache(path)
+}
+
+func initCache(path string) (*FileCache, error) {
+	cache := FileCache{
+		Path:  path,
+		Items: make(map[string]structs.File),
+	}
+
+	jsonFile, err := json.MarshalIndent(cache, "", " ")
+	if err != nil {
+		return nil, err
+	}
+
+	err = ioutil.WriteFile(cache.Path, jsonFile, 0644)
+	if err != nil {
+		return nil, err
+	}
+
+	return &cache, nil
+}
+
+func readFileCache(path string) (*FileCache, error) {
+	var cache FileCache
+
+	jsonFile, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+
+	byteValue, err := ioutil.ReadAll(jsonFile)
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(byteValue, &cache)
+	if err != nil {
+		return nil, err
+	}
+
+	return &cache, nil
+}
+
+func Store(cache FileCache, file structs.File) error {
+	cache.Items[file.Path] = file
+
+	jsonFile, err := json.MarshalIndent(cache, "", " ")
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(cache.Path, jsonFile, 0644)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func Get(cache FileCache, key string) (*structs.File, error) {
+	if file, ok := cache.Items[key]; ok {
+		return &file, nil
+	}
+
+	return nil, errors.New("cache item does not exist")
+}

src/internal/clone_detect/exact_match_clone_detect.go

@@ -6,7 +6,9 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"strconv"
 	"sync"
+	"x-dry-go/src/internal/cache"
 	"x-dry-go/src/internal/cli"
 	"x-dry-go/src/internal/compare"
 	"x-dry-go/src/internal/config"
@@ -158,10 +160,20 @@ func normalizeFiles(
 	levelNormalizers map[int][]config.Normalizer,
 	filepaths []string,
 ) map[string]structs.File {
+	fileCache, err := cache.InitOrReadCache("xdry-cache_level_" + strconv.Itoa(level) + ".json")
+
+	if err != nil {
+		fmt.Println("Error reading cache")
+	}
+
 	var (
 		normalizedFilesMutex sync.Mutex
 		normalizedFiles      = make(map[string]structs.File)
 	)
+	var (
+		cacheMutex       sync.Mutex
+		mutexedFileCache = fileCache
+	)
 
 	normalizers, ok := levelNormalizers[level]
 	if !ok {
@@ -181,6 +193,15 @@ func normalizeFiles(
 	wg := &sync.WaitGroup{}
 
 	for _, path := range filepaths {
+		normalizedFile, err := cache.Get(*fileCache, path)
+		if err == nil {
+			normalizedFilesMutex.Lock()
+			normalizedFiles[path] = *normalizedFile
+			normalizedFilesMutex.Unlock()
+			bar.Add(1)
+			continue
+		}
+
 		semaphore <- struct{}{}
 		wg.Add(1)
 
@@ -196,6 +217,9 @@ func normalizeFiles(
 			normalizedFilesMutex.Lock()
 			normalizedFiles[path] = normalizedFile
 			normalizedFilesMutex.Unlock()
+			cacheMutex.Lock()
+			cache.Store(*mutexedFileCache, normalizedFile)
+			cacheMutex.Unlock()
 
 			bar.Add(1)
 

src/internal/config/config.go

@@ -19,6 +19,7 @@ type Config struct {
 type Settings struct {
 	MinCloneLengths map[string]int `json:"minCloneLengths"`
 	LogPath         string         `json:"logPath"`
+	CacheDirectory  string         `json:"cacheDirectory"`
 }
 
 type Report struct {
@@ -91,9 +92,13 @@ func hydrateSettings(config *Config, configPath string, cwd string) {
 	if config.Settings.LogPath == "" {
 		config.Settings.LogPath = "xdry.log"
 	}
+	if config.Settings.CacheDirectory == "" {
+		config.Settings.CacheDirectory = "."
+	}
 
 	configDir := path.Dir(configPath)
 	config.Settings.LogPath = toAbsolutePath(config.Settings.LogPath, configDir, cwd)
+	config.Settings.CacheDirectory = toAbsolutePath(config.Settings.CacheDirectory, configDir, cwd)
 }
 
 func toAbsolutePath(directory string, configDir string, cwd string) string {

src/internal/config/config_test.go

@@ -24,7 +24,8 @@ func TestParseConfig(t *testing.T) {
 				"level-1": 10,
 				"level-2": 20,
 			},
-			LogPath: path.Join(cwd, "test", "_testdata", "xdry.log"),
+			LogPath:        path.Join(cwd, "test", "_testdata", "xdry.log"),
+			CacheDirectory: path.Join(cwd, "test", "_testdata", "cache"),
 		},
 		Directories: []string{
 			path.Join(cwd, "test", "_testdata", "php"),
@@ -47,3 +48,29 @@ func TestParseConfig(t *testing.T) {
 
 	g.Expect(actual).To(Equal(&want))
 }
+
+func TestParseConfigMinimal(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	configPath := path.Join(cwd, "..", "..", "..", "_testdata", "xdry_minimal.json")
+
+	want := Config{
+		Settings: Settings{
+			MinCloneLengths: nil,
+			LogPath:         path.Join(cwd, "..", "..", "..", "_testdata", "xdry.log"),
+			CacheDirectory:  path.Join(cwd, "..", "..", "..", "_testdata"),
+		},
+		Reports:     nil,
+		Directories: nil,
+		Normalizers: nil,
+	}
+
+	_, actual := ParseConfig(configPath, cwd)
+
+	g.Expect(actual).To(Equal(&want))
+}