Merge pull request #28 from flexera-public/FOPTS-2224

FOPTS-2224 The MD5 hash of the files is verified to ensure the integrity of the uploaded files

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## v1.7
+
+- The MD5 hash of the files is verified to ensure the integrity of the uploaded files.
+
 ## v1.6
 
 - Save two months of cvs files instead of only current month.

README.md

@@ -131,7 +131,7 @@ You should see 200/201s in the logs, which indicates that the exporter is workin
 | flexera.shard | string | `"NAM"` | Shard ("NAM", "EU", "AU") |
 | image.pullPolicy | string | `"Always"` |  |
 | image.repository | string | `"public.ecr.aws/flexera/cbi-oi-kubecost-exporter"` |  |
-| image.tag | string | `"1.6"` |  |
+| image.tag | string | `"1.7"` |  |
 | imagePullSecrets | list | `[]` |  |
 | includePreviousMonth | bool | `false` | Include data from previous month to the export process, only if we have files from every day of the previous month. |
 | kubecost.aggregation | string | `"pod"` | Aggregation Level ("namespace", "controller", "pod") |

helm-chart/Chart.yaml

@@ -6,10 +6,10 @@ description: Kubecost exporter helm chart for Kubernetes
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.6.0
+version: 1.7.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.6"
+appVersion: "1.7"

helm-chart/README.md

@@ -1,6 +1,6 @@
 # cbi-oi-kubecost-exporter
 
-![Version: 1.6.0](https://img.shields.io/badge/Version-1.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.6](https://img.shields.io/badge/AppVersion-1.6-informational?style=flat-square)
+![Version: 1.7.0](https://img.shields.io/badge/Version-1.7.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.7](https://img.shields.io/badge/AppVersion-1.7-informational?style=flat-square)
 
 ### Kubecost exporter helm chart for Kubernetes
 
@@ -94,7 +94,7 @@ You should see 200/201s in the logs, which indicates that the exporter is workin
 | flexera.shard | string | `"NAM"` | Shard ("NAM", "EU", "AU") |
 | image.pullPolicy | string | `"Always"` |  |
 | image.repository | string | `"public.ecr.aws/flexera/cbi-oi-kubecost-exporter"` |  |
-| image.tag | string | `"1.6"` |  |
+| image.tag | string | `"1.7"` |  |
 | imagePullSecrets | list | `[]` |  |
 | includePreviousMonth | bool | `false` | Include data from previous month to the export process, only if we have files from every day of the previous month. |
 | kubecost.aggregation | string | `"pod"` | Aggregation Level ("namespace", "controller", "pod") |

helm-chart/values.yaml

@@ -5,7 +5,7 @@
 image:
   repository: public.ecr.aws/flexera/cbi-oi-kubecost-exporter
   pullPolicy: Always
-  tag: "1.6"
+  tag: "1.7"
 
 imagePullSecrets: []
 

index.yaml

@@ -1,9 +1,19 @@
 apiVersion: v1
 entries:
   cbi-oi-kubecost-exporter:
+  - apiVersion: v2
+    appVersion: "1.7"
+    created: "2023-11-13T18:30:29.453053-06:00"
+    description: Kubecost exporter helm chart for Kubernetes
+    digest: 90fbd649c651fa2ad66d0705aced6cbecd908206c59c135ee585ce12188cd287
+    name: cbi-oi-kubecost-exporter
+    type: application
+    urls:
+    - helm-chart/cbi-oi-kubecost-exporter-1.7.0.tgz
+    version: 1.7.0
   - apiVersion: v2
     appVersion: "1.6"
-    created: "2023-11-10T11:47:21.844983-06:00"
+    created: "2023-11-13T18:30:29.45285-06:00"
     description: Kubecost exporter helm chart for Kubernetes
     digest: f998790d9229fce36398ef487edf6685a9a29e359dd2c78191c4d16344d47acd
     name: cbi-oi-kubecost-exporter
@@ -13,7 +23,7 @@ entries:
     version: 1.6.0
   - apiVersion: v2
     appVersion: "1.5"
-    created: "2023-11-10T11:47:21.844695-06:00"
+    created: "2023-11-13T18:30:29.452494-06:00"
     description: Kubecost exporter helm chart for Kubernetes
     digest: a0503a1af7b86a8f0769009de5a71a3613e8503a5f59cf86874d483f0f5f2abb
     name: cbi-oi-kubecost-exporter
@@ -23,7 +33,7 @@ entries:
     version: 1.5.0
   - apiVersion: v2
     appVersion: "1.4"
-    created: "2023-11-10T11:47:21.844329-06:00"
+    created: "2023-11-13T18:30:29.452172-06:00"
     description: Kubecost exporter helm chart for Kubernetes
     digest: 5b296e921178d84717345be19b92a4eea0c9ca1c65410128dec596692220c8e2
     name: cbi-oi-kubecost-exporter
@@ -33,7 +43,7 @@ entries:
     version: 1.4.2
   - apiVersion: v2
     appVersion: "1.4"
-    created: "2023-11-10T11:47:21.844085-06:00"
+    created: "2023-11-13T18:30:29.451641-06:00"
     description: Kubecost exporter helm chart for Kubernetes
     digest: ab6992a0af9f24d5c090db71679b7f274f3576b1cfbe91325e0d6e9af317657e
     name: cbi-oi-kubecost-exporter
@@ -43,7 +53,7 @@ entries:
     version: 1.4.1
   - apiVersion: v2
     appVersion: "1.4"
-    created: "2023-11-10T11:47:21.843733-06:00"
+    created: "2023-11-13T18:30:29.451346-06:00"
     description: Kubecost exporter helm chart for Kubernetes
     digest: 870030e989ba274f8b0f9d122a4ada24a4d29ef2c071408bf46d86e18942effd
     name: cbi-oi-kubecost-exporter
@@ -53,7 +63,7 @@ entries:
     version: 1.4.0
   - apiVersion: v2
     appVersion: "1.3"
-    created: "2023-11-10T11:47:21.843328-06:00"
+    created: "2023-11-13T18:30:29.451027-06:00"
     description: Kubecost exporter helm chart for Kubernetes
     digest: 0d967927bfab5f4bb6d40ce062710679e98b145c4b3f38709c0bf5d216ad8606
     name: cbi-oi-kubecost-exporter
@@ -63,7 +73,7 @@ entries:
     version: 1.3.0
   - apiVersion: v2
     appVersion: "1.2"
-    created: "2023-11-10T11:47:21.84241-06:00"
+    created: "2023-11-13T18:30:29.450165-06:00"
     description: Kubecost exporter helm chart for Kubernetes
     digest: d0c724c7bb085a1801e27f60ff748bc69aba14425cfd27fd132f99966692ed80
     name: cbi-oi-kubecost-exporter
@@ -73,12 +83,12 @@ entries:
     version: 1.2.0
   - apiVersion: v2
     appVersion: "1.1"
-    created: "2023-11-10T11:47:21.842028-06:00"
+    created: "2023-11-13T18:30:29.449849-06:00"
     description: Kubecost exporter helm chart for Kubernetes
     digest: c6f2681575b704b5934efea2357921896d55c4ad5e09692f31ab92b9e614cfea
     name: cbi-oi-kubecost-exporter
     type: application
     urls:
     - helm-chart/cbi-oi-kubecost-exporter-1.1.0.tgz
     version: 1.1.0
-generated: "2023-11-10T11:47:21.841228-06:00"
+generated: "2023-11-13T18:30:29.449326-06:00"

main.go

@@ -2,7 +2,9 @@ package main
 
 import (
 	"bytes"
+	"crypto/md5"
 	"encoding/csv"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -103,6 +105,13 @@ type (
 		}
 	}
 
+	OptimaFileUploadResponse struct {
+		ID           string `json:"id"`
+		Status       string `json:"status"`
+		BillUploadID string `json:"billUploadId"`
+		MD5          string `json:"md5"`
+	}
+
 	Config struct {
 		RefreshToken         string  `env:"REFRESH_TOKEN"`
 		OrgID                string  `env:"ORG_ID"`
@@ -309,6 +318,21 @@ func (a *App) uploadToFlexera() {
 			fileData, _ := os.ReadFile(fileName)
 			response = a.doPost(uploadFileURL, string(fileData), authHeaders)
 			checkForError(response)
+			bodyBytes, err := io.ReadAll(response.Body)
+			if err != nil {
+				log.Fatalf("Error reading response: %s", err.Error())
+			}
+			response.Body.Close()
+			var uploadResponse OptimaFileUploadResponse
+			if err = json.Unmarshal(bodyBytes, &uploadResponse); err != nil {
+				log.Fatalf("Error parsing response: %s", err.Error())
+			}
+
+			md5 := getMD5FromFileBytes(fileData)
+			if md5 != uploadResponse.MD5 {
+				log.Fatalf("MD5 of file %s does not match MD5 of uploaded file", fileName)
+			}
+			log.Printf("File %s uploaded and MD5 of file matches MD5 of uploaded file", fileName)
 		}
 
 		operationsURL := fmt.Sprintf("%s/%s/operations", billUploadURL, billUploadID)
@@ -618,3 +642,10 @@ func extractLabels(labels map[string]string, namespaceLabels map[string]string)
 	labelsJSON, _ := json.Marshal(mapLabels)
 	return string(labelsJSON)
 }
+
+func getMD5FromFileBytes(fileBytes []byte) string {
+	hash := md5.New()
+	hash.Write(fileBytes)
+
+	return hex.EncodeToString(hash.Sum(nil))
+}

main_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/hex"
 	"encoding/json"
 	"os"
 	"reflect"
@@ -488,3 +489,25 @@ func TestApp_DaysInMonth(t *testing.T) {
 		})
 	}
 }
+
+func TestGetMD5FromFileBytes(t *testing.T) {
+	// Define a test case with an input byte slice and the expected MD5 hash
+	testCases := []struct {
+		input    []byte
+		expected string
+	}{
+		{[]byte("Hello, World!"), "65a8e27d8879283831b664bd8b7f0ad4"},
+		{[]byte("12345"), "827ccb0eea8a706c4c34a16891f84e7b"},
+		{[]byte(""), "d41d8cd98f00b204e9800998ecf8427e"},
+		{[]byte("*/&!"), "e720300025e73ebfd5320f06e5e1919a"},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(hex.EncodeToString(testCase.input), func(t *testing.T) {
+			result := getMD5FromFileBytes(testCase.input)
+			if result != testCase.expected {
+				t.Errorf("Expected: %s, Got: %s", testCase.expected, result)
+			}
+		})
+	}
+}