Skip to content

Commit

Permalink
refactor(ubuntu): use iterator
Browse files Browse the repository at this point in the history
Signed-off-by: knqyf263 <knqyf263@gmail.com>
  • Loading branch information
knqyf263 committed Feb 6, 2025
1 parent 59266a4 commit b06d800
Show file tree
Hide file tree
Showing 7 changed files with 171 additions and 104 deletions.
9 changes: 5 additions & 4 deletions cmd/ubuntu.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package cmd

import (
"errors"
"time"

"github.com/inconshreveable/log15"
Expand Down Expand Up @@ -31,7 +32,7 @@ func fetchUbuntu(_ *cobra.Command, _ []string) (err error) {
return xerrors.Errorf("Failed to SetLogger. err: %w", err)
}

cveJSONs, err := fetcher.FetchUbuntuVulnList()
cveJSONs, count, err := fetcher.FetchUbuntuVulnList()
if err != nil {
return xerrors.Errorf("Failed to initialize vulnerability DB. err: %w", err)
}
Expand All @@ -40,7 +41,7 @@ func fetchUbuntu(_ *cobra.Command, _ []string) (err error) {
log15.Info("Initialize Database")
driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{})
if err != nil {
if xerrors.Is(err, db.ErrDBLocked) {
if errors.Is(err, db.ErrDBLocked) {
return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err)
}
return xerrors.Errorf("Failed to open DB. err: %w", err)
Expand All @@ -58,9 +59,9 @@ func fetchUbuntu(_ *cobra.Command, _ []string) (err error) {
return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
}

log15.Info("Fetched", "CVEs", len(cves))
log15.Info("Fetched", "CVEs", count)
log15.Info("Insert Ubuntu into DB", "db", driver.Name())
if err := driver.InsertUbuntu(cves); err != nil {
if err := driver.InsertUbuntu(cves, count); err != nil {
return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
}

Expand Down
3 changes: 2 additions & 1 deletion db/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package db

import (
"fmt"
"iter"
"time"

"golang.org/x/xerrors"
Expand Down Expand Up @@ -48,7 +49,7 @@ type DB interface {

InsertRedhat([]models.RedhatCVE) error
InsertDebian([]models.DebianCVE) error
InsertUbuntu([]models.UbuntuCVE) error
InsertUbuntu(iter.Seq2[models.UbuntuCVE, error], int) error
InsertMicrosoft([]models.MicrosoftCVE, []models.MicrosoftKBRelation) error
InsertArch([]models.ArchADV) error
}
Expand Down
35 changes: 19 additions & 16 deletions db/redis.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"io"
"iter"
"maps"
"os"
"slices"
Expand Down Expand Up @@ -1096,25 +1097,13 @@ func (r *RedisDriver) InsertDebian(cves []models.DebianCVE) error {
}

// InsertUbuntu :
func (r *RedisDriver) InsertUbuntu(cves []models.UbuntuCVE) (err error) {
func (r *RedisDriver) InsertUbuntu(cves iter.Seq2[models.UbuntuCVE, error], count int) (err error) {
ctx := context.Background()
batchSize := viper.GetInt("batch-size")
if batchSize < 1 {
return xerrors.Errorf("Failed to set batch-size. err: batch-size option is not set properly")
}

advs := map[string][]string{}
for _, c := range cves {
for _, r := range c.References {
if strings.HasPrefix(r.Reference, "https://ubuntu.com/security/notices/USN-") {
advs[strings.TrimPrefix(r.Reference, "https://ubuntu.com/security/notices/")] = append(advs[strings.TrimPrefix(r.Reference, "https://ubuntu.com/security/notices/")], c.Candidate)
}
}
}
for k := range advs {
advs[k] = util.Unique(advs[k])
}

// newDeps, oldDeps: {"CVEID": {"PKGNAME": {}}, "advisories": {"ADVISORYID": {}}}
newDeps := map[string]map[string]struct{}{"advisories": {}}
oldDepsStr, err := r.conn.HGet(ctx, depKey, ubuntuName).Result()
Expand All @@ -1129,14 +1118,27 @@ func (r *RedisDriver) InsertUbuntu(cves []models.UbuntuCVE) (err error) {
return xerrors.Errorf("Failed to unmarshal JSON. err: %w", err)
}

log15.Info("Insert CVEs", "cves", len(cves))
bar := pb.StartNew(len(cves)).SetWriter(func() io.Writer {
log15.Info("Insert CVEs", "cves", count)
bar := pb.StartNew(count).SetWriter(func() io.Writer {
if viper.GetBool("log-json") {
return io.Discard
}
return os.Stderr
}())
for chunk := range slices.Chunk(cves, batchSize) {
advs := map[string][]string{}

for chunk, err := range util.Chunk(cves, batchSize) {
if err != nil {
return xerrors.Errorf("Failed to chunk cves. err: %w", err)
}
for _, c := range chunk {
for _, r := range c.References {
if strings.HasPrefix(r.Reference, "https://ubuntu.com/security/notices/USN-") {
advs[strings.TrimPrefix(r.Reference, "https://ubuntu.com/security/notices/")] = slices.Compact(append(advs[strings.TrimPrefix(r.Reference, "https://ubuntu.com/security/notices/")], c.Candidate))
}
}
}

pipe := r.conn.Pipeline()
cvekey := fmt.Sprintf(cveKeyFormat, ubuntuName)
for _, cve := range chunk {
Expand Down Expand Up @@ -1168,6 +1170,7 @@ func (r *RedisDriver) InsertUbuntu(cves []models.UbuntuCVE) (err error) {
}
bar.Add(len(chunk))
}
bar.SetCurrent(int64(count)) // The file number doesn't always match the number of CVEs that are actually inserted.
bar.Finish()

log15.Info("Insert Advisories", "advisories", len(advs))
Expand Down
16 changes: 10 additions & 6 deletions db/ubuntu.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ import (
"errors"
"fmt"
"io"
"iter"
"os"
"slices"
"strings"

"github.com/cheggaaa/pb/v3"
Expand Down Expand Up @@ -78,16 +78,16 @@ func (r *RDBDriver) GetUbuntuMulti(cveIDs []string) (map[string]models.UbuntuCVE
}

// InsertUbuntu :
func (r *RDBDriver) InsertUbuntu(cves []models.UbuntuCVE) (err error) {
if err = r.deleteAndInsertUbuntu(cves); err != nil {
func (r *RDBDriver) InsertUbuntu(cves iter.Seq2[models.UbuntuCVE, error], count int) (err error) {
if err = r.deleteAndInsertUbuntu(cves, count); err != nil {
return xerrors.Errorf("Failed to insert Ubuntu CVE data. err: %s", err)
}

return nil
}

func (r *RDBDriver) deleteAndInsertUbuntu(cves []models.UbuntuCVE) (err error) {
bar := pb.StartNew(len(cves)).SetWriter(func() io.Writer {
func (r *RDBDriver) deleteAndInsertUbuntu(cves iter.Seq2[models.UbuntuCVE, error], count int) (err error) {
bar := pb.StartNew(count).SetWriter(func() io.Writer {
if viper.GetBool("log-json") {
return io.Discard
}
Expand Down Expand Up @@ -115,7 +115,11 @@ func (r *RDBDriver) deleteAndInsertUbuntu(cves []models.UbuntuCVE) (err error) {
return xerrors.New("Failed to set batch-size. err: batch-size option is not set properly")
}

for chunk := range slices.Chunk(cves, batchSize) {
for chunk, err := range util.Chunk(cves, batchSize) {
if err != nil {
return xerrors.Errorf("failed to insert Ubuntu CVE data. err: %w", err)
}

if err = tx.Create(chunk).Error; err != nil {
return xerrors.Errorf("Failed to insert. err: %w", err)
}
Expand Down
56 changes: 37 additions & 19 deletions fetcher/ubuntu.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"io"
"iter"
"path/filepath"

"github.com/inconshreveable/log15"
Expand All @@ -20,46 +21,63 @@ const (
)

// FetchUbuntuVulnList clones vuln-list and returns CVE JSONs
func FetchUbuntuVulnList() (entries []models.UbuntuCVEJSON, err error) {
func FetchUbuntuVulnList() (iter.Seq2[models.UbuntuCVEJSON, error], int, error) {
// Clone vuln-list repository
dir := filepath.Join(util.CacheDir(), "vuln-list")
updatedFiles, err := git.CloneOrPull(ubuntuRepoURL, dir, ubuntuDir)
if err != nil {
return nil, xerrors.Errorf("error in vulnsrc clone or pull: %w", err)
return nil, 0, xerrors.Errorf("error in vulnsrc clone or pull: %w", err)
}

// Only last_updated.json
if len(updatedFiles) <= 1 {
return nil, nil
return nil, 0, nil
}

rootDir := filepath.Join(dir, ubuntuDir)
targets, err := util.FilterTargets(ubuntuDir, updatedFiles)
if err != nil {
return nil, xerrors.Errorf("failed to filter target files: %w", err)
return nil, 0, xerrors.Errorf("failed to filter target files: %w", err)
} else if len(targets) == 0 {
log15.Debug("Ubuntu: no update file")
return nil, nil
return nil, 0, nil
}
log15.Debug(fmt.Sprintf("Ubuntu updated files: %d", len(targets)))

err = util.FileWalk(rootDir, targets, func(r io.Reader, _ string) error {
content, err := io.ReadAll(r)
if err != nil {
return err
}
count, err := countUbuntuCVEs(rootDir, targets)
if err != nil {
return nil, 0, xerrors.Errorf("failed to count Ubuntu CVEs: %w", err)
}

return func(yield func(models.UbuntuCVEJSON, error) bool) {

err = util.FileWalk(rootDir, targets, func(r io.Reader, _ string) error {
content, err := io.ReadAll(r)
if err != nil {
return err
}

cve := models.UbuntuCVEJSON{}
if err = json.Unmarshal(content, &cve); err != nil {
return xerrors.Errorf("failed to decode Ubuntu JSON: %w", err)
cve := models.UbuntuCVEJSON{}
if err = json.Unmarshal(content, &cve); err != nil {
return xerrors.Errorf("failed to decode Ubuntu JSON: %w", err)
}

if !yield(cve, nil) {
return err
}
return nil
})
if err != nil && !yield(models.UbuntuCVEJSON{}, xerrors.Errorf("error in Ubuntu walk: %w", err)) {
return
}
}, count, nil
}

entries = append(entries, cve)
func countUbuntuCVEs(rootDir string, targets map[string]struct{}) (int, error) {
count := 0
err := util.FileWalk(rootDir, targets, func(r io.Reader, _ string) error {

Check failure on line 78 in fetcher/ubuntu.go

View workflow job for this annotation

GitHub Actions / lint

unused-parameter: parameter 'r' seems to be unused, consider removing or renaming it as _ (revive)

Check failure on line 78 in fetcher/ubuntu.go

View workflow job for this annotation

GitHub Actions / Build

parameter 'r' seems to be unused, consider removing or renaming it as _ https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unused-parameter
count++
return nil
})
if err != nil {
return nil, xerrors.Errorf("error in Ubuntu walk: %w", err)
}

return entries, nil
return count, err
}
Loading

0 comments on commit b06d800

Please sign in to comment.