Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add updateMap flag #1578

Merged
merged 7 commits into from
Oct 24, 2024
+613 −432
Merged

Add updateMap flag #1578

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
cc88a2e
add updateMap flag
fanny-jiang Oct 23, 2024
718cab8
fix mapping
fanny-jiang Oct 23, 2024
4287f7d
fix values parsing
fanny-jiang Oct 23, 2024
37777ae
remove invalid parent keys from mapping
fanny-jiang Oct 23, 2024
4c321bd
refactor
fanny-jiang Oct 24, 2024
41a7693
more refactor, add comments
fanny-jiang Oct 24, 2024
5a398a4
remove _
fanny-jiang Oct 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions tools/yaml-mapper/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ module github.com/DataDog/helm-charts

go 1.22.7

require helm.sh/helm/v3 v3.16.2

require (
github.com/Masterminds/semver/v3 v3.3.0 // indirect
github.com/cyphar/filepath-securejoin v0.3.1 // indirect
Expand All @@ -24,7 +26,6 @@ require (
golang.org/x/text v0.18.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
helm.sh/helm/v3 v3.16.2 // indirect
k8s.io/api v0.31.1 // indirect
k8s.io/apiextensions-apiserver v0.31.1 // indirect
k8s.io/apimachinery v0.31.1 // indirect
Expand All @@ -34,4 +35,4 @@ require (
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)
)
22 changes: 21 additions & 1 deletion tools/yaml-mapper/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,28 @@ github.com/cyphar/filepath-securejoin v0.3.1 h1:1V7cHiaW+C+39wEfpH6XlLBQo3j/PciW
github.com/cyphar/filepath-securejoin v0.3.1/go.mod h1:F7i41x/9cBF7lzCrVsYs9fuzwRZm4NQsGTBdpp6mETc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
Expand All @@ -30,8 +38,16 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
Expand Down Expand Up @@ -75,11 +91,15 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
helm.sh/helm/v3 v3.16.2 h1:Y9v7ry+ubQmi+cb5zw1Llx8OKHU9Hk9NQ/+P+LGBe2o=
helm.sh/helm/v3 v3.16.2/go.mod h1:SyTXgKBjNqi2NPsHCW5dDAsHqvGIu0kdNYNH9gQaw70=
k8s.io/api v0.31.1 h1:Xe1hX/fPW3PXYYv8BlozYqw63ytA92snr96zMW9gWTU=
Expand All @@ -99,4 +119,4 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h6
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
122 changes: 120 additions & 2 deletions tools/yaml-mapper/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ package main
import (
"flag"
"fmt"
"io"
"net/http"
"os"
"reflect"
"strings"
Expand All @@ -27,6 +29,7 @@ func main() {
fmt.Println(" -sourceFile (string)")
fmt.Println(" -destFile (string)")
fmt.Println(" -prefixFile (string)")
fmt.Println(" -updateMap (bool)")
return
}
}
Expand All @@ -37,21 +40,30 @@ func main() {
var sourceFile string
var destFile string
var prefixFile string
var updateMap bool
flag.StringVar(&mappingFile, "mappingFile", "mapping.yaml", "path to mapping YAML file")
flag.StringVar(&sourceFile, "sourceFile", "source.yaml", "path to source YAML file")
flag.StringVar(&destFile, "destFile", "destination.yaml", "path to destination YAML file")
flag.StringVar(&prefixFile, "prefixFile", "", "path to prefix YAML file. The content in this file will be prepended to the output")
flag.BoolVar(&updateMap, "updateMap", false, "Update default Mapping file with latest Datadog Helm Chart values")

flag.Parse()

fmt.Println("mappingFile:", mappingFile)
fmt.Println("sourceFile:", sourceFile)
fmt.Println("destFile:", destFile)
fmt.Println("prefixFile:", prefixFile)
fmt.Println("updateMap:", updateMap)
fmt.Println("printOutput:", *printPtr)
fmt.Println("")

// Read mapping file
tmpSourceFile := ""
if updateMap {
mappingFile = "mapping_datadog_helm_to_datadogagent_crd.yaml"
tmpSourceFile = getLatestValuesFile()
sourceFile = tmpSourceFile
}
mapping, err := os.ReadFile(mappingFile)
if err != nil {
fmt.Println(err)
Expand All @@ -65,6 +77,9 @@ func main() {

// Read source yaml file
source, err := os.ReadFile(sourceFile)
if tmpSourceFile != "" {
defer os.Remove(tmpSourceFile)
}
if err != nil {
fmt.Println(err)
return
Expand All @@ -81,7 +96,31 @@ func main() {
var destKey interface{}
var ok bool
interim := make(map[string]interface{})
for sourceKey := range mappingValues {
interimMap := make(map[string]interface{})

if updateMap {
interimMap = parseValues(sourceValues, make(map[string]interface{}), "")
updateMapping(mappingValues, interimMap)
newMapYaml, e := chartutil.Values(interimMap).YAML()
if e != nil {
fmt.Println(e)
return
}
newMapYaml = `# This file maps keys from the Datadog Helm chart (YAML) to the DatadogAgent CustomResource spec (YAML).

` + newMapYaml

e = os.WriteFile(mappingFile, []byte(newMapYaml), 0660)
if e != nil {
fmt.Printf("Error updating default mapping yaml. %v", e)
return
}

fmt.Printf("Default mapping file, %s, successfully updated", mappingFile)
return
}

for sourceKey, _ := range mappingValues {
pathVal, _ = sourceValues.PathValue(sourceKey)
// If there is no corresponding key in the destination, then the pathVal will be nil
if pathVal == nil {
Expand All @@ -91,8 +130,9 @@ func main() {
destKey, ok = mappingValues[sourceKey]
rt := reflect.TypeOf(destKey)
if !ok || destKey == "" || destKey == nil {
fmt.Printf("Warning: key not found: %s\n", sourceKey)
// Continue through loop
fmt.Printf("Warning: key not found: %s\n", sourceKey)
continue
} else if rt.Kind() == reflect.Slice {
// Provide support for the case where one source key may map to multiple destination keys
for _, v := range destKey.([]interface{}) {
Expand All @@ -101,9 +141,11 @@ func main() {
} else {
interim[destKey.(string)] = pathVal
}

}

// Create final mapping with properly nested map keys (converted from period-delimited keys)

result := make(map[string]interface{})
for k, v := range interim {
result = makeTable(k, v, result)
Expand Down Expand Up @@ -143,6 +185,17 @@ func main() {
fmt.Println("YAML file successfully written to", destFile)

return

}

func updateMapping(mappingValues chartutil.Values, interimMap map[string]interface{}) {
for sourceKey, sourceVal := range mappingValues {
if sourceVal == nil {
interimMap[sourceKey] = ""
} else {
interimMap[sourceKey] = sourceVal
}
}
}

func makeTable(path string, val interface{}, mapName map[string]interface{}) map[string]interface{} {
Expand Down Expand Up @@ -182,3 +235,68 @@ func mergeMaps(map1, map2 map[string]interface{}) map[string]interface{} {
}

func parsePath(key string) []string { return strings.Split(key, ".") }

func getLatestValuesFile() string {
chartVersion := getChartVersion()
chartValuesFile := downloadYaml(fmt.Sprintf("https://raw.githubusercontent.com/DataDog/helm-charts/refs/tags/datadog-%s/charts/datadog/values.yaml", chartVersion), "datadog-values")

return chartValuesFile
}

func getChartVersion() string {
chartYamlPath := downloadYaml("https://raw.githubusercontent.com/DataDog/helm-charts/main/charts/datadog/Chart.yaml", "datadog-Chart")

ddChart, err := chartutil.LoadChartfile(chartYamlPath)
defer os.Remove(chartYamlPath)
if err != nil {
fmt.Println(fmt.Printf("Error loading Chart.yaml: %s", err))
}
return ddChart.Version
}

func downloadYaml(url string, name string) string {
resp, err := http.Get(url)
if err != nil {
fmt.Printf("Error fetching yaml file: %v\n", err)
return ""
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
fmt.Printf("Failed to fetch yaml file %s: %v\n", url, resp.Status)
return ""
}

tmpFile, err := os.CreateTemp("", fmt.Sprintf("%s.yaml.*", name))
if err != nil {
fmt.Printf("Error creating temporary file: %v\n", err)
return ""
}
defer tmpFile.Close()

_, err = io.Copy(tmpFile, resp.Body)
if err != nil {
fmt.Printf("Error saving file: %v\n", err)
return ""
}

fmt.Printf("File downloaded and saved to temporary file: %s\n", tmpFile.Name())

return tmpFile.Name()
}

// TODO: fix handling deprecated helm values keys
// TODO: preserve comments

func parseValues(sourceValues chartutil.Values, valuesMap map[string]interface{}, prefix string) map[string]interface{} {
for key, value := range sourceValues {
currentKey := prefix + key
// If the value is a map, recursive call to get nested keys.
if nestedMap, ok := value.(map[string]interface{}); ok {
parseValues(nestedMap, valuesMap, currentKey+".")
} else {
valuesMap[currentKey] = ""
}
}
return valuesMap
}
Loading