Skip to content

Commit

Permalink
✨ fully working CPE offline - now just needs tests
Browse files Browse the repository at this point in the history
  • Loading branch information
acidjazz committed Dec 3, 2024
1 parent 7b31bb9 commit 7e6189e
Show file tree
Hide file tree
Showing 9 changed files with 160 additions and 30 deletions.
2 changes: 0 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ require (
github.com/fumeapp/taskin v0.1.8
github.com/hashicorp/go-version v1.7.0
github.com/itchyny/gojq v0.12.16
github.com/octoper/go-ray v0.1.5
github.com/package-url/packageurl-go v0.1.3
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c
github.com/spf13/cobra v1.8.1
Expand Down Expand Up @@ -110,7 +109,6 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/go-containerregistry v0.19.1 // indirect
github.com/google/licensecheck v0.3.1 // indirect
Expand Down
8 changes: 0 additions & 8 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,6 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I=
github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/bradleyjkemp/cupaloy/v2 v2.6.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=
github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
Expand Down Expand Up @@ -384,9 +383,6 @@ github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomarkdown/markdown v0.0.0-20210208175418-bda154fe17d8/go.mod h1:aii0r/K0ZnHv7G0KF7xy1v0A7s2Ljrb5byB7MO5p6TU=
github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386 h1:EcQR3gusLHN46TAD+G+EbaaqJArt5vHhNpXAa12PQf4=
github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
Expand Down Expand Up @@ -433,7 +429,6 @@ github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwg
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
Expand Down Expand Up @@ -638,8 +633,6 @@ github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7
github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro=
github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg=
github.com/octoper/go-ray v0.1.5 h1:tTN+4HptuzSnw60E0wM71wegKsy8wYhnQ2THjMvXBGQ=
github.com/octoper/go-ray v0.1.5/go.mod h1:Y1I9cUEZ4oD94H0/M+xwHhvGVbFu1o/dW3fDrknELk8=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
Expand Down Expand Up @@ -886,7 +879,6 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
golang.org/dl v0.0.0-20190829154251-82a15e2f2ead/go.mod h1:IUMfjQLJQd4UTqG1Z90tenwKoCX93Gn3MAQJMOSBsDQ=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
Expand Down
15 changes: 10 additions & 5 deletions pkg/cmd/offline/cpe/cpe.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"github.com/vulncheck-oss/cli/pkg/cmd/offline/sync"
"github.com/vulncheck-oss/cli/pkg/cpe/cpeparse"
"github.com/vulncheck-oss/cli/pkg/cpe/cpeprocess"
"github.com/vulncheck-oss/cli/pkg/cpe/cpequery"
"github.com/vulncheck-oss/cli/pkg/search"
"github.com/vulncheck-oss/cli/pkg/ui"
)
Expand Down Expand Up @@ -34,18 +35,22 @@ func Command() *cobra.Command {
return err
}

indexAvailable, err := sync.EnsureIndexSync(indices, cpe.Vendor, false)
indexAvailable, err := sync.EnsureIndexSync(indices, cpe.Index, false)
if err != nil {
return err
}

if !indexAvailable {
return fmt.Errorf("index %s is required to proceed", cpe.Vendor)
return fmt.Errorf("index %s is required to proceed", cpe.Index)
}

query := search.QueryCPE(*cpe)
query, err := cpequery.Query(*cpe)

results, stats, err := search.IndexAdvisories(cpe.Vendor, query)
if err != nil {
return err
}

results, stats, err := search.IndexAdvisories(cpe.Index, query)

if err != nil {
return err
Expand All @@ -60,7 +65,7 @@ func Command() *cobra.Command {
ui.Stat("Files/Lines processed", fmt.Sprintf("%d/%d", stats.TotalFiles, stats.TotalLines))
ui.Stat("Search duration", stats.Duration.String())

ui.Json(cves)
// ui.Json(cves)

return nil
},
Expand Down
7 changes: 7 additions & 0 deletions pkg/cmd/offline/sync/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import (
"slices"
)

var specialIndices = []string{"cpecve"}

func Command() *cobra.Command {

var addIndices, removeIndices []string
Expand All @@ -37,6 +39,11 @@ func Command() *cobra.Command {
availableIndices[index.Name] = true
}

// Add special indices to availableIndices
for _, specialIndex := range specialIndices {
availableIndices[specialIndex] = true
}

// Handle purge flag
if purge {
if err := cache.PurgeIndices(); err != nil {
Expand Down
10 changes: 7 additions & 3 deletions pkg/cpe/cpeparse/cpeparse.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,19 @@ func Parse(s string) (*cpetypes.CPE, error) {
}

if cpe.IsMozilla() {
product, err := cpemozilla.Parse(cpe)
index, err := cpemozilla.Parse(cpe)
if err != nil {
return nil, err
}
cpe.Product = *product
cpe.Index = *index
}

if cpe.IsNginx() {
cpe.Vendor = "nginx"
cpe.Index = "nginx"
}

if cpe.Index == "*" {
cpe.Index = "cpecve"
}

return &cpe, nil
Expand Down
58 changes: 57 additions & 1 deletion pkg/cpe/cpeprocess/cpeprocess.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"github.com/vulncheck-oss/cli/pkg/cpe/cpemozilla"
"github.com/vulncheck-oss/cli/pkg/cpe/cpenginx"
"github.com/vulncheck-oss/cli/pkg/cpe/cpetypes"
"github.com/vulncheck-oss/cli/pkg/cpe/cpeutils"
)

func Process(cpe cpetypes.CPE, entries []interface{}) ([]string, error) {
Expand Down Expand Up @@ -44,5 +45,60 @@ func Process(cpe cpetypes.CPE, entries []interface{}) ([]string, error) {
return cpenginx.Process(cpe, nginxAdvisories)
}

return []string{}, nil
cveFindings := make([]string, 0)

if cpeutils.IsParseableVersion(cpetypes.Unquote(cpe.Version)) {
for _, entry := range entries {
jsonData, err := json.Marshal(entry)
if err != nil {
return nil, err
}

var vuln cpetypes.CPEVulnerabilities
if err := json.Unmarshal(jsonData, &vuln); err != nil {
return nil, err
}

cmpr, err := cpeutils.CompareVersions(cpetypes.Unquote(cpe.Version), cpetypes.Unquote(vuln.Version))
if err == nil {
if cmpr == 0 {
cveFindings = append(cveFindings, vuln.Cves...)
}
}
}
} else {

for _, entry := range entries {
jsonData, err := json.Marshal(entry)
if err != nil {
return nil, err
}

var vuln cpetypes.CPEVulnerabilities
if err := json.Unmarshal(jsonData, &vuln); err != nil {
return nil, err
}
cveFindings = append(cveFindings, vuln.Cves...)

}
}

cveFindings = RemoveDuplicatesUnordered(cveFindings)

return cveFindings, nil
}
func RemoveDuplicatesUnordered(elements []string) []string {
encountered := map[string]bool{}

func() {
for v := range elements {
encountered[elements[v]] = true
}
}()

result := []string{}
for key := range encountered {
result = append(result, key)
}
return result
}
72 changes: 72 additions & 0 deletions pkg/cpe/cpequery/cpequery.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package cpequery

import (
"fmt"
"github.com/vulncheck-oss/cli/pkg/cpe/cpetypes"
"github.com/vulncheck-oss/cli/pkg/cpe/cpeutils"
"strings"
)

func Query(cpe cpetypes.CPE) (string, error) {
if cpe.Product == "" {
return "true", nil
}
if cpe.IsMozilla() {
return fmt.Sprintf(".products | any(. == %q)", cpe.ProductUcFirst()), nil
}

if cpe.Vendor == "*" && cpe.Product == "*" {
return "", fmt.Errorf("need at least vendor or product specified")
}

// if IsParseableVersion - do NOT query for the version in the JSON
return BuildCPEQuery(cpe, !cpeutils.IsParseableVersion(cpetypes.Unquote(cpe.Version)))
}

func addCondition(field, value string) string {
if value != "*" {
return fmt.Sprintf(`(.%s == %q or .%s == "*")`, field, value, field)
}
return ""
}

func BuildCPEQuery(cpe cpetypes.CPE, queryVersion bool) (string, error) {
if cpe.Vendor == "*" && cpe.Product == "*" {
return "", fmt.Errorf("need at least vendor or product specified")
}

var conditions []string

fields := []struct {
name string
value string
}{
{"vendor", cpe.Vendor},
{"product", cpe.Product},
{"update", cpe.Update},
{"edition", cpe.Edition},
{"language", cpe.Language},
{"sw_edition", cpe.SoftwareEdition},
{"target_sw", cpe.TargetSoftware},
{"target_hw", cpe.TargetHardware},
{"other", cpe.Other},
}

// Add version field only if withVersion is true
if queryVersion {
fields = append(fields, struct{ name, value string }{"version", cpetypes.Unquote(cpe.Version)})
}

for _, field := range fields {
if condition := addCondition(field.name, field.value); condition != "" {
conditions = append(conditions, condition)
}
}

if len(conditions) == 0 {
return "true", nil
}

query := strings.Join(conditions, " and ")
return query, nil
}
7 changes: 7 additions & 0 deletions pkg/cpe/cpetypes/cpetypes.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ type MozillaComponent struct {
Bugzilla []string `json:"bugzilla"`
}

type CPEVulnerabilities struct {
CPE
CPE23URI string `json:"cpe23Uri"` // full cpe23uri string
Cves []string `json:"cves"` // associated CVEs
}

type AdvisoryCVES []string

func (ma MozillaAdvisories) CVES() AdvisoryCVES {
Expand Down Expand Up @@ -82,6 +88,7 @@ type CPE struct {
TargetSoftware string `json:"target_sw"`
TargetHardware string `json:"target_hw"`
Other string `json:"other"`
Index string `json:"index"`
}

func (c CPE) IsMozilla() bool {
Expand Down
11 changes: 0 additions & 11 deletions pkg/search/search.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"github.com/package-url/packageurl-go"
"github.com/tidwall/gjson"
"github.com/vulncheck-oss/cli/pkg/config"
"github.com/vulncheck-oss/cli/pkg/cpe/cpetypes"
"github.com/vulncheck-oss/cli/pkg/ui"
"github.com/vulncheck-oss/sdk-go"
"os"
Expand Down Expand Up @@ -255,16 +254,6 @@ func QueryPURL(instance packageurl.PackageURL) string {
}
return strings.Join(conditions, " and ")
}
func QueryCPE(cpe cpetypes.CPE) string {
if cpe.Product == "" {
return "true"
}
if cpe.IsMozilla() {
return fmt.Sprintf(".products | any(. == %q)", cpe.ProductUcFirst())
}

return "true"
}

func IPIndex(indexName, query string) ([]IPEntry, *Stats, error) {
startTime := time.Now()
Expand Down

0 comments on commit 7e6189e

Please sign in to comment.