Skip to content

Commit

Permalink
🚧 successfuly updated sync with sqlite
Browse files Browse the repository at this point in the history
  • Loading branch information
@acidjazz
acidjazz committed Feb 12, 2025
1 parent 5459d8f commit a2fb872
Show file tree
Hide file tree
Showing 6 changed files with 425 additions and 158 deletions.
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ require (
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mattn/go-sqlite3 v1.14.24 // indirect
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
github.com/mholt/archiver/v3 v3.5.1 // indirect
github.com/microsoft/go-rustaudit v0.0.0-20220730194248-4b17361d90a5 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -584,6 +584,8 @@ github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75/go.mod h1:
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
Expand Down
218 changes: 61 additions & 157 deletions pkg/cache/cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,10 @@ import (
"github.com/vulncheck-oss/cli/pkg/config"
"github.com/vulncheck-oss/cli/pkg/session"
"github.com/vulncheck-oss/cli/pkg/utils"
"golang.org/x/exp/slices"
"gopkg.in/yaml.v3"
"io"
"net/http"
"os"
"path/filepath"
"slices"
"time"
)

Expand Down Expand Up @@ -67,114 +65,66 @@ func (i *InfoFile) GetIndex(name string) *IndexInfo {
return nil
}

func syncSingleIndex(index string, configDir string, indexInfo *InfoFile) taskin.Task {
return taskin.Task{
Title: fmt.Sprintf("Syncing index: %s", index),
Task: func(t *taskin.Task) error {
response, err := session.Connect(config.Token()).GetIndexBackup(index)
if err != nil {
return err
}

if len(response.GetData()) == 0 {
return fmt.Errorf("no data received for index %s", index)
}

file, err := utils.ExtractFile(response.GetData()[0].URL)
if err != nil {
return err
}

filePath := filepath.Join(configDir, file)
lastUpdated := response.GetData()[0].DateAdded
date := utils.ParseDate(lastUpdated)

// if lastUpdated is equal to LastUpdated inside indexInfo, bypass the download
if indexInfo.IndexExists(index) {
if indexInfo.GetIndex(index).LastUpdated == lastUpdated {
t.Title = fmt.Sprintf("Index %s is already up to date", index)
return nil
}
}

// Downloading
t.Title = fmt.Sprintf("Downloading %s (last updated %s)", file, date)
if err := DownloadWithProgress(response.GetData()[0].URL, index, filePath, t); err != nil {
return err
}

// Extracting
t.Title = "Extracting"
indexDir := filepath.Join(configDir, index)
if _, err := os.Stat(indexDir); !os.IsNotExist(err) {
if err := os.RemoveAll(indexDir); err != nil {
return fmt.Errorf("failed to remove existing index directory: %w", err)
}
}
if err := os.MkdirAll(indexDir, 0755); err != nil {
return fmt.Errorf("failed to create index directory: %w", err)
}
if err := utils.Unzip(filePath, indexDir); err != nil {
return fmt.Errorf("failed to unzip index file: %w", err)
}

// Finalizing
t.Title = "Finalizing"
size, err := utils.GetDirectorySize(indexDir)
if err != nil {
return fmt.Errorf("failed to calculate size of index directory: %s", err)
}

updatedInfo := IndexInfo{
Name: index,
LastSync: time.Now(),
Size: size,
LastUpdated: lastUpdated,
}

found := false
for i, info := range indexInfo.Indices {
if info.Name == index {
indexInfo.Indices[i] = updatedInfo
found = true
break
}
}
if !found {
indexInfo.Indices = append(indexInfo.Indices, updatedInfo)
}

t.Title = fmt.Sprintf("Synced %s (Size: %s)", index, utils.GetSizeHuman(size))

if err := os.Remove(filePath); err != nil {
return fmt.Errorf("failed to remove downloaded zip file: %s", err)
}
func syncSingleIndex(index string, configDir string, indexInfo *InfoFile, force bool) taskin.Tasks {
response, err := session.Connect(config.Token()).GetIndexBackup(index)
if err != nil {
return taskin.Tasks{
{
Title: fmt.Sprintf("Error syncing index: %s", index),
Task: func(t *taskin.Task) error {
return err
},
},
}
}

return nil
},
if len(response.GetData()) == 0 {
return taskin.Tasks{
{
Title: fmt.Sprintf("No data received for index: %s", index),
Task: func(t *taskin.Task) error {
return fmt.Errorf("no data received for index %s", index)
},
},
}
}
}

func PurgeIndices() error {
configDir, err := config.IndicesDir()
file, err := utils.ExtractFile(response.GetData()[0].URL)
if err != nil {
return fmt.Errorf("failed to get indices directory: %w", err)
return taskin.Tasks{
{
Title: fmt.Sprintf("Error extracting file for index: %s", index),
Task: func(t *taskin.Task) error {
return err
},
},
}
}

// Remove the entire indices directory
if err := os.RemoveAll(configDir); err != nil {
return fmt.Errorf("failed to remove indices directory: %w", err)
filePath := filepath.Join(configDir, file)
lastUpdated := response.GetData()[0].DateAdded

if indexInfo.IndexExists(index) && indexInfo.GetIndex(index).LastUpdated == lastUpdated && !force {
return taskin.Tasks{
{
Title: fmt.Sprintf("Index %s is already up to date", index),
Task: func(t *taskin.Task) error {
return nil
},
},
}
}

// Recreate an empty indices directory
if err := os.MkdirAll(configDir, 0755); err != nil {
return fmt.Errorf("failed to recreate indices directory: %w", err)
childTasks := taskin.Tasks{
taskDownload(response.GetData()[0].URL, index, filePath),
taskExtract(index, configDir, filePath),
taskSqlite(index, configDir, filePath, lastUpdated, indexInfo),
}

return nil
return childTasks
}

func IndicesSync(indices []string) error {
func IndicesSync(indices []string, force bool) error {
configDir, err := config.IndicesDir()
if err != nil {
return err
Expand All @@ -186,36 +136,32 @@ func IndicesSync(indices []string) error {
return fmt.Errorf("failed to get cached indices: %w", err)
}

// Remove indices not in the provided list
for i := len(indexInfo.Indices) - 1; i >= 0; i-- {
if !slices.Contains(indices, indexInfo.Indices[i].Name) {
// Remove the index folder
indexDir := filepath.Join(configDir, indexInfo.Indices[i].Name)
if err := os.RemoveAll(indexDir); err != nil {
return fmt.Errorf("failed to remove index directory: %w", err)
}
// Remove the index from the list
indexInfo.Indices = append(indexInfo.Indices[:i], indexInfo.Indices[i+1:]...)
}
}

tasks := taskin.Tasks{
{
Title: fmt.Sprintf("Syncing %d indices", len(indices)),
Tasks: make(taskin.Tasks, len(indices)),
Tasks: make(taskin.Tasks, 0, len(indices)),
},
}

for i, index := range indices {
tasks[0].Tasks[i] = syncSingleIndex(index, configDir, &indexInfo)
for _, index := range indices {
tasks[0].Tasks = append(tasks[0].Tasks, syncSingleIndex(index, configDir, &indexInfo, force)...)
}

runner := taskin.New(tasks, taskin.Defaults)
if err := runner.Run(); err != nil {
return err
}

// Save updated sync info
data, err := yaml.Marshal(indexInfo)
if err != nil {
return fmt.Errorf("failed to marshal sync info: %w", err)
Expand All @@ -228,62 +174,20 @@ func IndicesSync(indices []string) error {
return nil
}

// DownloadWithProgress downloads a file from the given URL and saves it to the specified filename,
// updating the progress on the provided taskin.Task.
func DownloadWithProgress(url string, index string, filename string, task *taskin.Task) error {
resp, err := http.Get(url)
if err != nil {
return fmt.Errorf("failed to get URL: %w", err)
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
return fmt.Errorf("received non-200 status code: %d", resp.StatusCode)
}

file, err := os.Create(filename)
func PurgeIndices() error {
configDir, err := config.IndicesDir()
if err != nil {
return fmt.Errorf("failed to create file: %w", err)
}
defer file.Close()

size := resp.ContentLength
if size <= 0 {
return fmt.Errorf("unknown content length")
return fmt.Errorf("failed to get indices directory: %w", err)
}

written := int64(0)
buffer := make([]byte, 32*1024)
for {
nr, er := resp.Body.Read(buffer)
if nr > 0 {
nw, ew := file.Write(buffer[0:nr])
if nw > 0 {
written += int64(nw)
}
if ew != nil {
err = ew
break
}
if nr != nw {
err = io.ErrShortWrite
break
}
}
if er != nil {
if er != io.EOF {
err = er
}
break
}

progress := float64(written) / float64(size)
task.Progress(int(progress*100), 100)
task.Title = fmt.Sprintf("Downloading %s %.2f%%", index, progress*100)
// Remove the entire indices directory
if err := os.RemoveAll(configDir); err != nil {
return fmt.Errorf("failed to remove indices directory: %w", err)
}

if err != nil {
return fmt.Errorf("error during download: %w", err)
// Recreate an empty indices directory
if err := os.MkdirAll(configDir, 0755); err != nil {
return fmt.Errorf("failed to recreate indices directory: %w", err)
}

return nil
Expand Down
Loading

0 comments on commit a2fb872

Please sign in to comment.