diff --git a/pkg/sources/cache.go b/pkg/sources/cache.go new file mode 100644 index 0000000..0b3e21d --- /dev/null +++ b/pkg/sources/cache.go @@ -0,0 +1,135 @@ +// The cache file will be stored in: + +// Windows: %USERPROFILE%\.cache\depshub\ +// Linux/macOS: ~/.cache/depshub/ + +package sources + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + "time" +) + +type CacheItem struct { + Value []byte `json:"value"` // Store as raw JSON bytes + ExpiresAt time.Time `json:"expires_at"` + CreateTime time.Time `json:"create_time"` +} + +type FileCache struct { + filename string + mutex sync.RWMutex + data map[string]CacheItem +} + +// NewFileCache creates a new cache instance +func NewFileCache(cacheName string) (*FileCache, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return nil, err + } + + cacheDir := filepath.Join(homeDir, ".cache", "depshub") + if err := os.MkdirAll(cacheDir, 0755); err != nil { + return nil, err + } + + cacheFile := filepath.Join(cacheDir, cacheName+".json") + cache := &FileCache{ + filename: cacheFile, + data: make(map[string]CacheItem), + } + + // Load existing cache if it exists + if err := cache.load(); err != nil && !os.IsNotExist(err) { + return nil, err + } + + return cache, nil +} + +// Set adds or updates a cache entry with optional expiration duration +func (c *FileCache) Set(key string, value any, expiration time.Duration) error { + fmt.Println("Setting cache") + c.mutex.Lock() + defer c.mutex.Unlock() + + // Marshal the value to JSON bytes + jsonBytes, err := json.Marshal(value) + if err != nil { + return err + } + + c.data[key] = CacheItem{ + Value: jsonBytes, + CreateTime: time.Now(), + ExpiresAt: time.Now().Add(expiration), + } + + return c.save() +} + +// Get retrieves a value from the cache and unmarshals it into the provided destination +func (c *FileCache) Get(key string, dest any) (bool, error) { + c.mutex.RLock() + defer c.mutex.RUnlock() + + item, exists := c.data[key] + if !exists { + return false, nil + } + + // Check if item has expired + if !item.ExpiresAt.IsZero() && time.Now().After(item.ExpiresAt) { + delete(c.data, key) + c.save() + return false, nil + } + + // Unmarshal the JSON bytes into the destination + if err := json.Unmarshal(item.Value, dest); err != nil { + return true, err + } + + return true, nil +} + +// Delete removes an item from the cache +func (c *FileCache) Delete(key string) error { + c.mutex.Lock() + defer c.mutex.Unlock() + + delete(c.data, key) + return c.save() +} + +// Clear removes all items from the cache +func (c *FileCache) Clear() error { + c.mutex.Lock() + defer c.mutex.Unlock() + + c.data = make(map[string]CacheItem) + return c.save() +} + +// save writes the cache to disk +func (c *FileCache) save() error { + data, err := json.Marshal(c.data) + if err != nil { + return err + } + return os.WriteFile(c.filename, data, 0644) +} + +// load reads the cache from disk +func (c *FileCache) load() error { + data, err := os.ReadFile(c.filename) + if err != nil { + return err + } + return json.Unmarshal(data, &c.data) +} diff --git a/pkg/sources/cache_test.go b/pkg/sources/cache_test.go new file mode 100644 index 0000000..712dc6a --- /dev/null +++ b/pkg/sources/cache_test.go @@ -0,0 +1,282 @@ +package sources + +import ( + "fmt" + "os" + "testing" + "time" +) + +func TestNewFileCache(t *testing.T) { + tests := []struct { + name string + cacheName string + wantErr bool + }{ + { + name: "valid cache name", + cacheName: "test-cache", + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cache, err := NewFileCache(tt.cacheName) + if (err != nil) != tt.wantErr { + t.Errorf("NewFileCache() error = %v, wantErr %v", err, tt.wantErr) + return + } + if cache == nil { + t.Error("NewFileCache() returned nil cache") + } + + // Cleanup + if cache != nil { + os.Remove(cache.filename) + } + }) + } +} + +func TestFileCache_SetGet(t *testing.T) { + cache, err := NewFileCache("test-cache") + if err != nil { + t.Fatalf("Failed to create cache: %v", err) + } + defer os.Remove(cache.filename) + + tests := []struct { + name string + key string + value interface{} + expiration time.Duration + wantErr bool + wantExists bool + checkExpiry bool + }{ + { + name: "string value", + key: "string-key", + value: "test-value", + expiration: time.Hour, + wantExists: true, + }, + { + name: "map value", + key: "map-key", + value: map[string]string{ + "key": "value", + }, + expiration: time.Hour, + wantExists: true, + }, + { + name: "expired item", + key: "expired-key", + value: "expired-value", + expiration: -time.Hour, // Already expired + wantExists: false, + checkExpiry: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set value + err := cache.Set(tt.key, tt.value, tt.expiration) + if (err != nil) != tt.wantErr { + t.Errorf("FileCache.Set() error = %v, wantErr %v", err, tt.wantErr) + return + } + + // Get value + var got interface{} + exists, err := cache.Get(tt.key, &got) + if (err != nil) != tt.wantErr { + t.Errorf("FileCache.Get() error = %v, wantErr %v", err, tt.wantErr) + return + } + if exists != tt.wantExists { + t.Errorf("FileCache.Get() exists = %v, want %v", exists, tt.wantExists) + return + } + + if exists && !tt.checkExpiry { + // Check if retrieved value matches stored value + switch v := tt.value.(type) { + case string: + if got != v { + t.Errorf("FileCache.Get() = %v, want %v", got, v) + } + case map[string]string: + gotMap, ok := got.(map[string]interface{}) + if !ok { + t.Errorf("Failed to convert got to map[string]interface{}") + return + } + for k, want := range v { + if got := gotMap[k]; got != want { + t.Errorf("FileCache.Get() map[%s] = %v, want %v", k, got, want) + } + } + } + } + }) + } +} + +func TestFileCache_Delete(t *testing.T) { + cache, err := NewFileCache("test-cache") + if err != nil { + t.Fatalf("Failed to create cache: %v", err) + } + defer os.Remove(cache.filename) + + // Set a value + key := "test-key" + value := "test-value" + if err := cache.Set(key, value, time.Hour); err != nil { + t.Fatalf("Failed to set cache value: %v", err) + } + + // Delete the value + if err := cache.Delete(key); err != nil { + t.Errorf("FileCache.Delete() error = %v", err) + return + } + + // Verify it's gone + var got string + exists, err := cache.Get(key, &got) + if err != nil { + t.Errorf("FileCache.Get() error = %v", err) + return + } + if exists { + t.Error("FileCache.Get() returned exists = true after Delete") + } +} + +func TestFileCache_Clear(t *testing.T) { + cache, err := NewFileCache("test-cache") + if err != nil { + t.Fatalf("Failed to create cache: %v", err) + } + defer os.Remove(cache.filename) + + // Set multiple values + testData := map[string]string{ + "key1": "value1", + "key2": "value2", + } + for k, v := range testData { + if err := cache.Set(k, v, time.Hour); err != nil { + t.Fatalf("Failed to set cache value: %v", err) + } + } + + // Clear the cache + if err := cache.Clear(); err != nil { + t.Errorf("FileCache.Clear() error = %v", err) + return + } + + // Verify all values are gone + for k := range testData { + var got string + exists, err := cache.Get(k, &got) + if err != nil { + t.Errorf("FileCache.Get() error = %v", err) + return + } + if exists { + t.Errorf("FileCache.Get() returned exists = true after Clear for key %s", k) + } + } +} + +func TestFileCache_Persistence(t *testing.T) { + cacheName := "test-persistence-cache" + cache, err := NewFileCache(cacheName) + if err != nil { + t.Fatalf("Failed to create cache: %v", err) + } + defer os.Remove(cache.filename) + + // Set a value + key := "test-key" + value := "test-value" + if err := cache.Set(key, value, time.Hour); err != nil { + t.Fatalf("Failed to set cache value: %v", err) + } + + // Create a new cache instance with the same name + cache2, err := NewFileCache(cacheName) + if err != nil { + t.Fatalf("Failed to create second cache: %v", err) + } + + // Verify the value exists in the new instance + var got string + exists, err := cache2.Get(key, &got) + if err != nil { + t.Errorf("FileCache.Get() error = %v", err) + return + } + if !exists { + t.Error("FileCache.Get() returned exists = false for persisted value") + return + } + if got != value { + t.Errorf("FileCache.Get() = %v, want %v", got, value) + } +} + +func TestFileCache_Concurrent(t *testing.T) { + cache, err := NewFileCache("test-concurrent-cache") + if err != nil { + t.Fatalf("Failed to create cache: %v", err) + } + defer os.Remove(cache.filename) + + done := make(chan bool) + const goroutines = 10 + + // Concurrent writes + for i := 0; i < goroutines; i++ { + go func(id int) { + key := fmt.Sprintf("key-%d", id) + value := fmt.Sprintf("value-%d", id) + err := cache.Set(key, value, time.Hour) + if err != nil { + t.Errorf("Concurrent Set failed: %v", err) + } + done <- true + }(i) + } + + // Wait for all goroutines to finish + for i := 0; i < goroutines; i++ { + <-done + } + + // Verify all values were written correctly + for i := 0; i < goroutines; i++ { + key := fmt.Sprintf("key-%d", i) + expectedValue := fmt.Sprintf("value-%d", i) + var got string + exists, err := cache.Get(key, &got) + if err != nil { + t.Errorf("Get failed for key %s: %v", key, err) + continue + } + if !exists { + t.Errorf("Value not found for key %s", key) + continue + } + if got != expectedValue { + t.Errorf("Got %s, want %s for key %s", got, expectedValue, key) + } + } +} diff --git a/pkg/sources/crates/crates.go b/pkg/sources/crates/crates.go index 4e02432..190288a 100644 --- a/pkg/sources/crates/crates.go +++ b/pkg/sources/crates/crates.go @@ -34,8 +34,8 @@ type CratePackage struct { } func (s CratesSource) FetchPackageData(ctx context.Context, name string) (types.Package, error) { - var target CratePackage - var result types.Package + target := CratePackage{} + result := types.Package{} if err := s.fetchPackageInfo(ctx, name, &target); err != nil { return types.Package{}, err @@ -69,7 +69,14 @@ func (s CratesSource) FetchPackageData(ctx context.Context, name string) (types. Deprecated: deprecated, } + if result.Versions == nil { + result.Versions = make(map[string]types.PackageVersion) + } result.Versions[pv.Version] = pv + + if result.Time == nil { + result.Time = make(map[string]time.Time) + } result.Time[pv.Version] = version.CreatedAt } diff --git a/pkg/sources/fetch.go b/pkg/sources/fetch.go index e0cc3ac..a57dc98 100644 --- a/pkg/sources/fetch.go +++ b/pkg/sources/fetch.go @@ -3,7 +3,9 @@ package sources import ( "context" "fmt" + "time" + "github.com/depshubhq/depshub/pkg/sources/crates" "github.com/depshubhq/depshub/pkg/sources/go" "github.com/depshubhq/depshub/pkg/sources/npm" "github.com/depshubhq/depshub/pkg/types" @@ -28,11 +30,17 @@ func (f fetcher) Fetch(uniqueDependencies []types.Dependency) (types.PackagesInf // Launch goroutines for concurrent fetching npmSource := npm.NpmSource{} goSource := gosource.GoSource{} + cratesSource := crates.CratesSource{} background := context.Background() activeRequests := 0 // Use a semaphore to limit concurrent requests sem := make(chan struct{}, MaxConcurrent) + c, err := NewFileCache("dependencies") + + if err != nil { + return nil, err + } for _, dep := range uniqueDependencies { activeRequests++ @@ -46,11 +54,29 @@ func (f fetcher) Fetch(uniqueDependencies []types.Dependency) (types.PackagesInf var packageInfo types.Package var err error - switch dep.Manager { - case types.Npm: - packageInfo, err = npmSource.FetchPackageData(background, dep.Name) - case types.Go: - packageInfo, err = goSource.FetchPackageData(dep.Name, dep.Version) + key := fmt.Sprintf("%d-%s", dep.Manager, dep.Name) + + exists, err := c.Get(key, &packageInfo) + + if err != nil { + fmt.Printf("Error getting cache: %s\n", err) + } + + if !exists { + switch dep.Manager { + case types.Npm: + packageInfo, err = npmSource.FetchPackageData(background, dep.Name) + case types.Go: + packageInfo, err = goSource.FetchPackageData(dep.Name, dep.Version) + case types.Cargo: + packageInfo, err = cratesSource.FetchPackageData(background, dep.Name) + } + + if err != nil { + fmt.Printf("Error fetching package data: %s\n", err) + } else { + c.Set(key, packageInfo, 24*time.Hour) + } } resultChan <- packageResult{