FWI-5594 - Some images have stale last scan date - pt2 (#867)

* normalize image ID and name to fix stale re-scan

* bump version and changelog

* minor fixes

plugins/trivy/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.28.8
+* normalize image ID and name to fix re-scan of stale images
+
 ## 0.28.7
 * update trivy to 0.48.1
 

plugins/trivy/cmd/trivy/main.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"strconv"
@@ -28,6 +27,7 @@ const outputFile = image.TempDir + "/final-report.json"
  * Images or images recommendations that no longer belongs to that cluster are filtered out
  */
 func main() {
+	ctx := context.TODO()
 	setLogLevel(os.Getenv("LOGRUS_LEVEL"))
 	setEnv()
 
@@ -37,45 +37,46 @@ func main() {
 	token := os.Getenv("FAIRWINDS_TOKEN")
 	noRecommendations := os.Getenv("NO_RECOMMENDATIONS")
 
-	lastReport, err := image.GetLastReport(host, org, cluster, token)
+	lastReport, err := image.FetchLastReport(ctx, host, org, cluster, token)
 	if err != nil {
 		logrus.Fatal(err)
 	}
 	logrus.Infof("Latest report has %d images", len(lastReport.Images))
 	for _, i := range lastReport.Images {
 		logrus.Debugf("%v - %v", i.Name, i.ID)
 	}
-	ctx := context.Background()
+
 	namespaceBlocklist, namespaceAllowlist := getNamespaceBlocklistAllowlistFromEnv()
 	logrus.Infof("%d namespaces allowed, %d namespaces blocked", len(namespaceAllowlist), len(namespaceBlocklist))
-	images, err := image.GetImages(ctx, namespaceBlocklist, namespaceAllowlist)
+
+	inClusterImages, err := image.GetImages(ctx, namespaceBlocklist, namespaceAllowlist)
 	if err != nil {
 		logrus.Fatal(err)
 	}
-	logrus.Infof("Found %d images in cluster", len(images))
-	for _, i := range images {
+	logrus.Infof("Found %d images in cluster", len(inClusterImages))
+	for _, i := range inClusterImages {
 		logrus.Debugf("%v - %v", i.Name, i.ID)
 	}
 
-	imagesToScan := image.GetUnscannedImagesToScan(images, lastReport.Images, numberToScan)
+	imagesToScan := image.GetUnscannedImagesToScan(inClusterImages, lastReport.Images, numberToScan)
 	unscannedCount := len(imagesToScan)
 	logrus.Infof("Found %d images that have never been scanned", unscannedCount)
-	imagesToScan = image.GetImagesToRescan(images, *lastReport, imagesToScan, numberToScan)
-	logrus.Infof("Will rescan %d additional images", len(imagesToScan)-unscannedCount)
+	imagesToScan = image.GetImagesToReScan(inClusterImages, *lastReport, imagesToScan, numberToScan)
+	logrus.Infof("Will re-scan %d additional images", len(imagesToScan)-unscannedCount)
 	for _, i := range imagesToScan {
 		logrus.Debugf("%v - %v", i.Name, i.ID)
 	}
 
 	// Owners info from latest report might be out-of-date, we need to update it using the cluster info
-	lastReport.Images = image.UpdateOwnersReferenceOnMatchingImages(lastReport.Images, images)
+	lastReport.Images = image.UpdateOwnersReferenceOnMatchingImages(lastReport.Images, inClusterImages)
 	// Remove any images from the report that are no longer in the cluster
-	lastReport.Images = image.GetMatchingImages(lastReport.Images, images, false)
+	lastReport.Images = image.GetMatchingImages(lastReport.Images, inClusterImages, false)
 	logrus.Infof("%d images after removing images no longer in cluster", len(lastReport.Images))
 	// Remove any images from the report that we're going to re-scan now
 	lastReport.Images = image.GetUnmatchingImages(lastReport.Images, imagesToScan, false)
 	logrus.Infof("%d images after removing images to be scanned", len(lastReport.Images))
 	// Remove any recommendations from the report that no longer have a corresponding image in the cluster
-	lastReport.Images = image.GetMatchingImages(lastReport.Images, images, true)
+	lastReport.Images = image.GetMatchingImages(lastReport.Images, inClusterImages, true)
 	logrus.Infof("%d images after removing recommendations that don't match", len(lastReport.Images))
 
 	logrus.Infof("Starting image scans")
@@ -99,7 +100,7 @@ func main() {
 		logrus.Fatalf("could not marshal report: %v", err)
 	}
 	logrus.Infof("Writing to file %s", outputFile)
-	err = ioutil.WriteFile(outputFile, data, 0644)
+	err = os.WriteFile(outputFile, data, 0644)
 	if err != nil {
 		logrus.Fatalf("could not write to output file: %v", err)
 	}

plugins/trivy/pkg/image/getimages.go

@@ -37,7 +37,7 @@ func GetImages(ctx context.Context, namespaceBlocklist, namespaceAllowlist []str
 	kubeClientResources := util.CreateKubeClientResources()
 
 	client := fwControllerUtils.Client{
-		Context:    context.TODO(),
+		Context:    ctx,
 		Dynamic:    kubeClientResources.DynamicClient,
 		RESTMapper: kubeClientResources.RESTMapper,
 	}
@@ -94,6 +94,10 @@ func GetImages(ctx context.Context, namespaceBlocklist, namespaceAllowlist []str
 				if _, found := keyToImage[imgKey]; found {
 					continue
 				}
+
+				imageID = strings.TrimPrefix(imageID, DockerIOprefix)
+				imageName = strings.TrimPrefix(imageName, DockerIOprefix)
+
 				keyToImage[imgKey] = models.Image{
 					ID:      imageID,
 					Name:    imageName,

plugins/trivy/pkg/image/insights.go

@@ -0,0 +1,38 @@
+package image
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"net/http"
+
+	"github.com/fairwindsops/insights-plugins/plugins/trivy/pkg/models"
+)
+
+// FetchLastReport returns the last report for Trivy from Fairwinds Insights
+func FetchLastReport(ctx context.Context, host, org, cluster, token string) (*models.MinimizedReport, error) {
+	url := fmt.Sprintf("%s/v0/organizations/%s/clusters/%s/data/trivy/latest.json", host, org, cluster)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusNotFound {
+		return &models.MinimizedReport{Images: make([]models.ImageDetailsWithRefs, 0), Vulnerabilities: map[string]models.VulnerabilityDetails{}}, nil
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("Bad Status code on get last report: %d", resp.StatusCode)
+	}
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	return unmarshalAndFixReport(body)
+}

plugins/trivy/pkg/image/scan.go

@@ -3,8 +3,6 @@ package image
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	"net/http"
 	"os"
 	"os/exec"
 	"regexp"
@@ -37,68 +35,6 @@ func init() {
 	}
 }
 
-// GetLastReport returns the last report for Trivy from Fairwinds Insights
-func GetLastReport(host, org, cluster, token string) (*models.MinimizedReport, error) {
-	url := fmt.Sprintf("%s/v0/organizations/%s/clusters/%s/data/trivy/latest.json", host, org, cluster)
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Set("Authorization", "Bearer "+token)
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode == http.StatusNotFound {
-		return &models.MinimizedReport{Images: make([]models.ImageDetailsWithRefs, 0), Vulnerabilities: map[string]models.VulnerabilityDetails{}}, nil
-	}
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("Bad Status code on get last report: %d", resp.StatusCode)
-	}
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-	return unmarshalBody(body)
-}
-
-func unmarshalBody(body []byte) (*models.MinimizedReport, error) {
-	var report models.MinimizedReport
-	err := json.Unmarshal(body, &report)
-	if err != nil {
-		return nil, err
-	}
-	fixOwners(&report)
-	return &report, nil
-}
-
-// fixOwners adapt older owners fields to the new ones
-func fixOwners(report *models.MinimizedReport) {
-	for i := range report.Images {
-		img := &report.Images[i]
-		if hasDeprecatedOwnerFields(*img) {
-			var container string
-			if img.OwnerContainer != nil {
-				container = *img.OwnerContainer
-			}
-			img.Owners = []models.Resource{
-				{
-					Name:      img.OwnerName,
-					Kind:      img.OwnerKind,
-					Namespace: img.Namespace,
-					Container: container,
-				},
-			}
-		}
-	}
-}
-
-func hasDeprecatedOwnerFields(img models.ImageDetailsWithRefs) bool {
-	return len(img.OwnerName) != 0 || len(img.OwnerKind) != 0 || len(img.Namespace) != 0
-}
-
 // ScanImages will download the set of images given and scan them with Trivy.
 func ScanImages(images []models.Image, maxConcurrentScans int, extraFlags string, ignoreErrors bool) []models.ImageReport {
 	logrus.Infof("Scanning %d images", len(images))
@@ -226,7 +162,7 @@ func ScanImage(extraFlags, pullRef string) (*models.TrivyResults, error) {
 	}()
 
 	report := models.TrivyResults{}
-	data, err := ioutil.ReadFile(reportFile)
+	data, err := os.ReadFile(reportFile)
 	if err != nil {
 		logrus.Errorf("Error reading report %s: %s", imageID, err)
 		return nil, err

plugins/trivy/pkg/image/util.go

@@ -1,12 +1,15 @@
 package image
 
 import (
+	"encoding/json"
 	"sort"
 	"strings"
 
 	"github.com/fairwindsops/insights-plugins/plugins/trivy/pkg/models"
 )
 
+const DockerIOprefix = "docker.io/"
+
 func GetMatchingImages(baseImages []models.ImageDetailsWithRefs, toMatch []models.Image, isRecommendation bool) []models.ImageDetailsWithRefs {
 	return getImages(baseImages, toMatch, isRecommendation, true)
 }
@@ -52,7 +55,7 @@ func GetUnscannedImagesToScan(imagesInCluster []models.Image, lastReportImages [
 	return imagesToScan
 }
 
-func GetImagesToRescan(images []models.Image, lastReport models.MinimizedReport, imagesToScan []models.Image, maxScans int) []models.Image {
+func GetImagesToReScan(images []models.Image, lastReport models.MinimizedReport, imagesToScan []models.Image, maxScans int) []models.Image {
 	sort.Slice(lastReport.Images, func(a, b int) bool {
 		return lastReport.Images[a].LastScan == nil || lastReport.Images[b].LastScan != nil && lastReport.Images[a].LastScan.Before(*lastReport.Images[b].LastScan)
 	})
@@ -125,3 +128,48 @@ func UpdateOwnersReferenceOnMatchingImages(baseImages []models.ImageDetailsWithR
 	}
 	return baseImages
 }
+
+func unmarshalAndFixReport(body []byte) (*models.MinimizedReport, error) {
+	var report models.MinimizedReport
+	err := json.Unmarshal(body, &report)
+	if err != nil {
+		return nil, err
+	}
+	fixOwners(&report)
+	normalizeDockerHubImages(&report)
+	return &report, nil
+}
+
+// fixOwners adapt older owners fields to the new ones
+func fixOwners(report *models.MinimizedReport) {
+	for i := range report.Images {
+		img := &report.Images[i]
+		if hasDeprecatedOwnerFields(*img) {
+			var container string
+			if img.OwnerContainer != nil {
+				container = *img.OwnerContainer
+			}
+			img.Owners = []models.Resource{
+				{
+					Name:      img.OwnerName,
+					Kind:      img.OwnerKind,
+					Namespace: img.Namespace,
+					Container: container,
+				},
+			}
+		}
+	}
+}
+
+func hasDeprecatedOwnerFields(img models.ImageDetailsWithRefs) bool {
+	return len(img.OwnerName) != 0 || len(img.OwnerKind) != 0 || len(img.Namespace) != 0
+}
+
+// normalizeDockerHubImages removes the docker.io/ prefix from the image names and IDs
+func normalizeDockerHubImages(report *models.MinimizedReport) {
+	for i := range report.Images {
+		img := &report.Images[i]
+		img.Name = strings.TrimPrefix(img.Name, DockerIOprefix)
+		img.ID = strings.TrimPrefix(img.ID, DockerIOprefix)
+	}
+}

plugins/trivy/pkg/image/util_test.go

@@ -1,6 +1,7 @@
 package image
 
 import (
+	"os"
 	"testing"
 
 	"github.com/fairwindsops/insights-plugins/plugins/trivy/pkg/models"
@@ -66,3 +67,35 @@ func TestToScanMatches(t *testing.T) {
 	assert.Equal(t, 3, len(matching))
 	assert.Equal(t, "quay.io/fairwinds/sample-1:1.2.3", matching[0].Name)
 }
+
+func TestShouldBeAbleToReadOldReports(t *testing.T) {
+	v1Body, err := os.ReadFile("testdata/v0.26/latest.json")
+	assert.NoError(t, err)
+
+	v2, err := unmarshalAndFixReport(v1Body)
+	assert.NoError(t, err)
+	assert.Equal(t, 28, len(v2.Images))
+	assert.Equal(t, 467, len(v2.Vulnerabilities))
+
+	for _, img := range v2.Images {
+		if img.RecommendationOnly {
+			assert.Len(t, img.Owners, 0)
+		} else {
+			assert.Len(t, img.Owners, 1)
+		}
+	}
+}
+
+func TestUnmarshalAndFixReport(t *testing.T) {
+	v2Body, err := os.ReadFile("testdata/v0.27/latest.json")
+	assert.NoError(t, err)
+
+	v2, err := unmarshalAndFixReport(v2Body)
+	assert.NoError(t, err)
+	assert.Equal(t, 3, len(v2.Images))
+	assert.Equal(t, 467, len(v2.Vulnerabilities))
+
+	assert.Len(t, v2.Images[0].Owners, 1)
+	assert.Len(t, v2.Images[1].Owners, 1)
+	assert.Len(t, v2.Images[2].Owners, 2)
+}

plugins/trivy/version.txt

@@ -1 +1 @@
-0.28.7
+0.28.8