feat!(fetcher/nvd): fetch from ghcr container (#130)

fetcher/nvd/nvd.go

@@ -2,86 +2,174 @@ package nvd
 
 import (
 	"archive/tar"
-	"bytes"
-	"compress/gzip"
+	"context"
 	"encoding/json"
 	"io"
+	"io/fs"
+	"os"
+	"os/exec"
 	"path/filepath"
 	"slices"
 	"strings"
 	"time"
 
 	"github.com/inconshreveable/log15"
+	"github.com/klauspost/compress/zstd"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"golang.org/x/xerrors"
-
-	"github.com/vulsio/go-cti/utils"
+	"oras.land/oras-go/v2"
+	"oras.land/oras-go/v2/registry/remote"
 )
 
-const repositoryURL = "https://github.com/vulsio/vuls-data-raw-nvd-api-cve/archive/refs/heads/main.tar.gz"
-
 // Fetch NVD CVE data
 func Fetch() (map[string][]string, error) {
 	log15.Info("Fetching NVD CVE...")
-	bs, err := utils.FetchURL(repositoryURL)
+
+	dir, err := os.MkdirTemp("", "go-cti")
 	if err != nil {
-		return nil, xerrors.Errorf("Failed to fetch NVD repository. err: %w", err)
+		return nil, xerrors.Errorf("Failed to create temp directory. err: %w", err)
 	}
-	return parse(bs)
+	defer os.RemoveAll(dir)
+
+	if err := fetch(dir); err != nil {
+		return nil, xerrors.Errorf("Failed to fetch vuls-data-raw-nvd-api-cve. err: %w", err)
+	}
+
+	return parse(dir)
 }
 
-func parse(bs []byte) (map[string][]string, error) {
-	cveToCwes := map[string][]string{}
+func fetch(dir string) error {
+	ctx := context.TODO()
+	repo, err := remote.NewRepository("ghcr.io/vulsio/vuls-data-db:vuls-data-raw-nvd-api-cve")
+	if err != nil {
+		return xerrors.Errorf("Failed to create client for ghcr.io/vulsio/vuls-data-db:vuls-data-raw-nvd-api-cve. err: %w", err)
+	}
 
-	gr, err := gzip.NewReader(bytes.NewReader(bs))
+	_, r, err := oras.Fetch(ctx, repo, repo.Reference.Reference, oras.DefaultFetchOptions)
 	if err != nil {
-		return nil, xerrors.Errorf("Failed to create gzip reader. err: %w", err)
+		return xerrors.Errorf("Failed to fetch manifest. err: %w", err)
+	}
+	defer r.Close()
+
+	var manifest ocispec.Manifest
+	if err := json.NewDecoder(r).Decode(&manifest); err != nil {
+		return xerrors.Errorf("Failed to decode manifest. err: %w", err)
 	}
-	defer gr.Close()
 
-	tr := tar.NewReader(gr)
+	l := func() *ocispec.Descriptor {
+		for _, l := range manifest.Layers {
+			if l.MediaType == "application/vnd.vulsio.vuls-data-db.dotgit.layer.v1.tar+zstd" {
+				return &l
+			}
+		}
+		return nil
+	}()
+	if l == nil {
+		return xerrors.Errorf("Failed to find digest and filename from layers, actual layers: %#v", manifest.Layers)
+	}
+
+	r, err = repo.Fetch(ctx, *l)
+	if err != nil {
+		return xerrors.Errorf("Failed to fetch content. err: %w", err)
+	}
+	defer r.Close()
+
+	zr, err := zstd.NewReader(r)
+	if err != nil {
+		return xerrors.Errorf("Failed to new zstd reader. err: %w", err)
+	}
+	defer zr.Close()
+
+	tr := tar.NewReader(zr)
 	for {
 		hdr, err := tr.Next()
 		if err == io.EOF {
 			break
 		}
 		if err != nil {
-			return nil, xerrors.Errorf("Failed to next tar reader. err: %w", err)
-		}
-
-		if hdr.FileInfo().IsDir() {
-			continue
+			return xerrors.Errorf("Failed to next tar reader. err: %w", err)
 		}
 
-		if !strings.HasPrefix(filepath.Base(hdr.Name), "CVE-") {
-			continue
-		}
+		p := filepath.Join(dir, hdr.Name)
 
-		if err := func() error {
-			ss := strings.Split(filepath.Base(hdr.Name), "-")
-			if len(ss) != 3 {
-				return xerrors.Errorf("Failed to parse year. err: invalid ID format. expected: %q, actual: %q", "CVE-yyyy-\\d{4,}.json", filepath.Base(hdr.Name))
+		switch hdr.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(p, 0755); err != nil {
+				return xerrors.Errorf("Failed to mkdir %s. err: %w", p, err)
 			}
-			if _, err := time.Parse("2006", ss[1]); err != nil {
-				return xerrors.Errorf("Failed to parse year. err: invalid ID format. expected: %q, actual: %q", "CVE-yyyy-\\d{4,}.json", filepath.Base(hdr.Name))
+		case tar.TypeReg:
+			if err := os.MkdirAll(filepath.Dir(p), 0755); err != nil {
+				return xerrors.Errorf("Failed to mkdir %s. err: %w", p, err)
 			}
 
-			var nvddata nvd
-			if err := json.NewDecoder(tr).Decode(&nvddata); err != nil {
-				return xerrors.Errorf("Failed to decode JSON. err: %w", err)
-			}
+			if err := func() error {
+				f, err := os.Create(p)
+				if err != nil {
+					return xerrors.Errorf("Failed to create %s. err: %w", p, err)
+				}
+				defer f.Close()
 
-			for _, w := range nvddata.Weaknesses {
-				for _, d := range w.Description {
-					if strings.HasPrefix(d.Value, "CWE-") && !slices.Contains(cveToCwes[nvddata.ID], d.Value) {
-						cveToCwes[nvddata.ID] = append(cveToCwes[nvddata.ID], d.Value)
-					}
+				if _, err := io.Copy(f, tr); err != nil {
+					return xerrors.Errorf("Failed to copy to %s. err: %w", p, err)
 				}
+
+				return nil
+			}(); err != nil {
+				return xerrors.Errorf("Failed to create %s. err: %w", p, err)
 			}
+		}
+	}
+
+	cmd := exec.Command("git", "-C", filepath.Join(dir, "vuls-data-raw-nvd-api-cve"), "restore", ".")
+	if err := cmd.Run(); err != nil {
+		return xerrors.Errorf("Failed to exec %q. err: %w", cmd.String(), err)
+	}
+
+	return nil
+}
+
+func parse(dir string) (map[string][]string, error) {
+	cveToCwes := make(map[string][]string)
+
+	if err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
 
+		if d.IsDir() || !(strings.HasPrefix(filepath.Base(path), "CVE-") && filepath.Ext(path) == ".json") {
 			return nil
-		}(); err != nil {
-			return nil, xerrors.Errorf("Failed to extract %s. err: %w", hdr.Name, err)
 		}
+
+		ss := strings.Split(filepath.Base(path), "-")
+		if len(ss) != 3 {
+			return xerrors.Errorf("Failed to parse year. err: invalid ID format. expected: %q, actual: %q", "CVE-yyyy-\\d{4,}.json", filepath.Base(path))
+		}
+		if _, err := time.Parse("2006", ss[1]); err != nil {
+			return xerrors.Errorf("Failed to parse year. err: invalid ID format. expected: %q, actual: %q", "CVE-yyyy-\\d{4,}.json", filepath.Base(path))
+		}
+
+		f, err := os.Open(path)
+		if err != nil {
+			return xerrors.Errorf("Failed to open %s. err: %w", path, err)
+		}
+		defer f.Close()
+
+		var nvddata nvd
+		if err := json.NewDecoder(f).Decode(&nvddata); err != nil {
+			return xerrors.Errorf("Failed to decode JSON. err: %w", err)
+		}
+
+		for _, w := range nvddata.Weaknesses {
+			for _, d := range w.Description {
+				if strings.HasPrefix(d.Value, "CWE-") && !slices.Contains(cveToCwes[nvddata.ID], d.Value) {
+					cveToCwes[nvddata.ID] = append(cveToCwes[nvddata.ID], d.Value)
+				}
+			}
+		}
+
+		return nil
+	}); err != nil {
+		return nil, xerrors.Errorf("Failed to walk %s. err: %w", dir, err)
 	}
 
 	return cveToCwes, nil

fetcher/nvd/nvd_test.go

@@ -2,7 +2,6 @@ package nvd
 
 import (
 	"maps"
-	"os"
 	"slices"
 	"testing"
 )
@@ -14,7 +13,7 @@ func TestParse(t *testing.T) {
 		expected map[string][]string
 	}{
 		{
-			in:   "testdata/main.tar.gz",
+			in:   "testdata/go-cti00001",
 			base: map[string][]string{},
 			expected: map[string][]string{
 				"CVE-2020-0002": {"CWE-787", "CWE-416"},
@@ -23,11 +22,7 @@ func TestParse(t *testing.T) {
 	}
 
 	for i, tt := range tests {
-		bs, err := os.ReadFile(tt.in)
-		if err != nil {
-			t.Fatalf("[%d] Failed to read file. err: %s", i, err)
-		}
-		actual, err := parse(bs)
+		actual, err := parse(tt.in)
 		if err != nil {
 			t.Fatalf("[%d] Failed to parse. err: %s", i, err)
 		}

fetcher/nvd/testdata/go-cti00001/vuls-data-raw-nvd-api-cve/2020/CVE-2020-0002.json

@@ -0,0 +1,119 @@
+{
+	"id": "CVE-2020-0002",
+	"sourceIdentifier": "security@android.com",
+	"vulnStatus": "Analyzed",
+	"published": "2020-01-08T19:15:12.923",
+	"lastModified": "2022-01-01T20:01:34.303",
+	"descriptions": [
+		{
+			"lang": "en",
+			"value": "In ih264d_init_decoder of ih264d_api.c, there is a possible out of bounds write due to a use after free. This could lead to remote code execution with no additional execution privileges needed. User interaction is needed for exploitation Product: Android Versions: Android-8.0, Android-8.1, Android-9, and Android-10 Android ID: A-142602711"
+		},
+		{
+			"lang": "es",
+			"value": "En la función ih264d_init_decoder del archivo ih264d_api.c, hay una posible escritura fuera de límites debido a un uso de la memoria previamente liberada. Esto podría conllevar a una ejecución de código remota sin ser necesarios privilegios de ejecución adicionales. Es requerida una interacción del usuario para su explotación Producto: Android, Versiones: Android-8.0, Android-8.1, Android-9 y Android-10, ID de Android: A-142602711."
+		}
+	],
+	"references": [
+		{
+			"source": "security@android.com",
+			"tags": [
+				"Patch",
+				"Vendor Advisory"
+			],
+			"url": "https://source.android.com/security/bulletin/2020-01-01"
+		}
+	],
+	"metrics": {
+		"cvssMetricV2": [
+			{
+				"source": "nvd@nist.gov",
+				"type": "Primary",
+				"cvssData": {
+					"version": "2.0",
+					"vectorString": "AV:N/AC:M/Au:N/C:C/I:C/A:C",
+					"accessVector": "NETWORK",
+					"accessComplexity": "MEDIUM",
+					"authentication": "NONE",
+					"confidentialityImpact": "COMPLETE",
+					"integrityImpact": "COMPLETE",
+					"availabilityImpact": "COMPLETE",
+					"baseScore": 9.3
+				},
+				"baseSeverity": "HIGH",
+				"exploitabilityScore": 8.6,
+				"impactScore": 10,
+				"userInteractionRequired": true
+			}
+		],
+		"cvssMetricV31": [
+			{
+				"source": "nvd@nist.gov",
+				"type": "Primary",
+				"cvssData": {
+					"version": "3.1",
+					"vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H",
+					"attackVector": "NETWORK",
+					"attackComplexity": "LOW",
+					"privilegesRequired": "NONE",
+					"userInteraction": "REQUIRED",
+					"scope": "UNCHANGED",
+					"confidentialityImpact": "HIGH",
+					"integrityImpact": "HIGH",
+					"availabilityImpact": "HIGH",
+					"baseScore": 8.8,
+					"baseSeverity": "HIGH"
+				},
+				"exploitabilityScore": 2.8,
+				"impactScore": 5.9
+			}
+		]
+	},
+	"weaknesses": [
+		{
+			"source": "nvd@nist.gov",
+			"type": "Primary",
+			"description": [
+				{
+					"lang": "en",
+					"value": "CWE-787"
+				},
+				{
+					"lang": "en",
+					"value": "CWE-416"
+				}
+			]
+		}
+	],
+	"configurations": [
+		{
+			"nodes": [
+				{
+					"operator": "OR",
+					"cpeMatch": [
+						{
+							"vulnerable": true,
+							"criteria": "cpe:2.3:o:google:android:8.0:*:*:*:*:*:*:*",
+							"matchCriteriaId": "B578E383-0D77-4AC7-9C81-3F0B8C18E033"
+						},
+						{
+							"vulnerable": true,
+							"criteria": "cpe:2.3:o:google:android:8.1:*:*:*:*:*:*:*",
+							"matchCriteriaId": "B06BE74B-83F4-41A3-8AD3-2E6248F7B0B2"
+						},
+						{
+							"vulnerable": true,
+							"criteria": "cpe:2.3:o:google:android:9.0:*:*:*:*:*:*:*",
+							"matchCriteriaId": "8DFAAD08-36DA-4C95-8200-C29FE5B6B854"
+						},
+						{
+							"vulnerable": true,
+							"criteria": "cpe:2.3:o:google:android:10.0:*:*:*:*:*:*:*",
+							"matchCriteriaId": "D558D965-FA70-4822-A770-419E73BA9ED3"
+						}
+					]
+				}
+			]
+		}
+	]
+}

go.mod

@@ -8,15 +8,18 @@ require (
 	github.com/go-redis/redis/v8 v8.11.5
 	github.com/google/go-cmp v0.6.0
 	github.com/inconshreveable/log15 v3.0.0-testing.5+incompatible
+	github.com/klauspost/compress v1.17.2
 	github.com/labstack/echo/v4 v4.13.3
 	github.com/mitchellh/go-homedir v1.1.0
+	github.com/opencontainers/image-spec v1.1.0
 	github.com/parnurzeal/gorequest v0.2.16
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/viper v1.19.0
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
 	gorm.io/driver/mysql v1.5.5
 	gorm.io/driver/postgres v1.5.7
 	gorm.io/gorm v1.25.7
+	oras.land/oras-go/v2 v2.5.0
 )
 
 require (
@@ -45,6 +48,7 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect