diff --git a/kubernetes-addons/Observability/README.md b/kubernetes-addons/Observability/README.md index 7732b59c..4f53cfcc 100644 --- a/kubernetes-addons/Observability/README.md +++ b/kubernetes-addons/Observability/README.md @@ -122,7 +122,7 @@ You can either: - Import them manually to Grafana, - Use [`update-dashboards.sh`](./update-dashboards.sh) script to add them to Kubernetes as (more persistent) Grafana dashboard `configMap`s - - Script uses `$USER-` as dashboard `configMap` names, overwrites any pre-existing `configMap` with the same name, and _restarts Grafana_ so that it notices `configMap` updates + - Script uses `$USER-` as dashboard `configMap` names, and overwrites any pre-existing `configMap` with the same name - Or create your own dashboards based on them When dashboard is imported to Grafana, you can directly save changes to it, but such dashboards go away if Grafana is removed / re-installed. When dashboard is in `configMap`, Grafana saves its changes to a (selected) file, but you need to re-apply those files to Kubernetes with the script, for your changes to be there when that Grafana dashboard page is reloaded in browser. @@ -130,8 +130,8 @@ When dashboard is imported to Grafana, you can directly save changes to it, but Gotchas for dashboard `configMap` script usage: - If you change dashboard file name, you need to change also its 'uid' field (at end of the file), otherwise Grafana will see multiple `configMap`s for the same dashboard ID - - If there's no `uid` specified for the dashboard, Grafana will generate one on `configMap` load. Meaning that dashboard ID, and Grafana URL to it, will change on every reload -- Script assumes Prometheus / Grafana to be installed according to above instructions. If not, list of `labels` within script need to be updated to match Prometheus / Grafana installation +- If there's no `uid` specified for the dashboard, Grafana will generate one on `configMap` load. Meaning that dashboard ID, and Grafana URL to it, will change on every reload +- Script assumes default Prometheus / Grafana install (`monitoring` namespace, `grafana_dasboard=1` label identifying dashboard `configMap`s) NOTE: Services provide metrics only after they have processed at least one query, before that dashboards can be empty! diff --git a/kubernetes-addons/Observability/update-dashboards.sh b/kubernetes-addons/Observability/update-dashboards.sh index 848b59b1..23bafc8d 100755 --- a/kubernetes-addons/Observability/update-dashboards.sh +++ b/kubernetes-addons/Observability/update-dashboards.sh @@ -15,14 +15,15 @@ selector="app.kubernetes.io/name=grafana" grafana="kubectl -n $ns get pod --selector $selector --field-selector=status.phase=Running -o name" # Labels needed in configMap to get (Helm installed) Grafana to load it as dashboard -labels="grafana_dashboard=1 release=prometheus-stack app=kube-prometheus-stack-grafana" +labels="grafana_dashboard=1" error_exit () { name=${0##*/} echo - echo "Create/update dashboards specified in given JSON files to Grafana." - echo "Names for the created configMaps will be in '$USER-' format." + echo "Create/update dashboards specified in given JSON files for Grafana." + echo "Names for the created configMaps will be in '$USER-' format" + echo "and they go to (Grafana) '$ns' namespace." echo echo "usage: $name " echo @@ -44,11 +45,12 @@ for file in "$@"; do if [ ! -f "$file" ]; then error_exit "JSON file '$file' does not exist" fi - # Dashboard 'uid' is optional, but it should have a title... + # Dashboard 'uid' is optional as Grafana can generate one... uid=$(jq .uid "$file" | tail -1) if [ -z "$uid" ]; then error_exit "'$file' dashboard has invalid JSON" fi + # ...but it should have a title. title=$(jq .title "$file" | tail -1) if [ "$title" = "null" ]; then error_exit "'$file' dashboard has no 'title' field" @@ -103,22 +105,4 @@ done rm $tmp -echo -echo "Restarting Grafana so that it notices updated dashboards..." -pod=$($grafana) -echo "kubectl -n $ns delete $pod" -kubectl -n "$ns" delete "$pod" - -echo -echo "Waiting until new Grafana instance is running..." -while true; do - sleep 2 - pod=$($grafana) - if [ -n "$pod" ]; then - break - fi -done -echo "kubectl -n $ns wait $pod --for=condition=Ready" -kubectl -n "$ns" wait "$pod" --for=condition=Ready - -echo "DONE!" +echo "DONE! => Dashboard(s) should appear in Grafana after 1 min wait *and* Grafana page reload."