add sample code for local prometheus, improve grafana module

terraform/main.tf

@@ -136,6 +136,10 @@ module "runtime_secrets" {
     {
       name  = "kafka_consumer_api_secret"
       value = module.confluent.app_consumer_api_key.secret
+    },
+    {
+      name  = "grafana_viewer_key"
+      value = module.grafana.service_account_token_viewer_key
     }
   ]
 }
@@ -265,6 +269,12 @@ module "confluent" {
 module "grafana" {
   source = "./modules/grafana"
   # todo don't inherit prefix from cognito_auth_domain_prefix
-  url  = "https://${var.cognito_auth_domain_prefix}.grafana.net/"
-  auth = data.hcp_vault_secrets_app.rt_secrets_manual.secrets["GRAFANA_SA_TOKEN"]
+  slug          = var.cognito_auth_domain_prefix
+  url           = "https://${var.cognito_auth_domain_prefix}.grafana.net/"
+  auth          = data.hcp_vault_secrets_app.rt_secrets_manual.secrets["GRAFANA_SA_TOKEN"]
+  cloud_api_key = data.hcp_vault_secrets_app.rt_secrets_manual.secrets["GRAFANA_CLOUD_API_KEY"]
+}
+
+output "grafana_stack" {
+  value = module.grafana.cloud_stack
 }

terraform/modules/grafana/README.adoc

@@ -6,14 +6,19 @@
 * https://grafana.com/docs/grafana-cloud/developer-resources/infrastructure-as-code/terraform/[Grafana Cloud  Developer resources  Infrastructure as code  Terraform]
 * https://github.com/grafana/terraform-provider-grafana[Terraform Provider for Grafana GitHub Project]
 * https://grafana.com/docs/grafana-cloud/developer-resources/infrastructure-as-code/terraform/dashboards-github-action/#terraform-configuration-for-grafana-provider[Terraform configuration for Grafana provider]
+** first need to https://grafana.com/docs/grafana/latest/administration/service-accounts/#create-a-service-account-in-grafana[Create a service account in Grafana]
+** and https://grafana.com/docs/grafana/latest/administration/service-accounts/#add-a-token-to-a-service-account-in-grafana[Add a token to a service account], this will be used in the `auth` field of the grafana provider config block
+
 * https://newrelic.com/blog/how-to-relic/create-nr-dashboards-with-terraform-part-1[Creating dashboards with Terraform and JSON templates]
 
+
 == Direct Post via Influx Proxy
 
 ====
 HTTP POST metrics directly to Grafana Cloud from any application or service
 ====
 
+* Why: https://community.grafana.com/t/can-grafana-cloud-scrape-a-prometheus-endpoint-over-the-internet/83608/3[Can Grafana Cloud scrape a Prometheus endpoint over the Internet?] (No)
 * Source: https://grafana.com/blog/2022/07/25/new-in-grafana-mimir-ingest-graphite-datadog-influx-and-prometheus-metrics-into-a-single-storage-backend/[New in Grafana Mimir: Ingest Graphite, Datadog, Influx, and Prometheus metrics into a single storage backend]
 * Payload Format: https://github.com/grafana/influx2cortex?tab=readme-ov-file#influx-line-protocol-ingestion-and-translation[Influx Line Protocol ingestion and translation]
 
@@ -24,3 +29,28 @@ URL="https://influx-(...).grafana.net/api/v1/push/influx/write"
 curl -X POST -H "Authorization: Bearer $API_KEY" -H "Content-Type: text/plain" "$URL" -d "my_test_metric,room=kitchen,source=grafana_cloud_docs metric=35.2"
 ----
 
+
+needs API Key with the following scopes
+metrics:write logs:write traces:write profiles:write
+
+Anatomy of a metric
+
+.Influx metric format
+----
+my_test_metric,bar_label=abc,source=grafana_cloud_docs metric=35.2
+----
+
+.Prometheus metric
+----
+# HELP metric_name Description of the metric
+# TYPE metric_name type
+# Comment that's not parsed by prometheus
+my_test_metric{__proxy_source__="influx",bar_label="abc",source="grafana"} 35.2
+----
+
+
+* Name: Prometheus style name(required)
+* Label: Identifies a specific visualization of a metric
+* Source: The source of the metric, the instance job that is generating the metric
+* Metric: Metric to be pushed up, the specific value
+

terraform/modules/grafana/main.tf

@@ -1,15 +1,54 @@
 # url and credentials must be passed to the module
 provider "grafana" {
-  url  = var.url
+  url = var.url
+  # String, Sensitive) API token, basic auth in the username:password format or anonymous (string literal).
+  # May alternatively be set via the GRAFANA_AUTH environment variable.
+  # Why? the Grafana client is required for `grafana_service_account`. Set the auth and url provider attributes
   auth = var.auth
+  # String, Sensitive) Access Policy Token (or API key) for Grafana Cloud. May alternatively
+  # be set via the GRAFANA_CLOUD_API_KEY environment variable
+  cloud_api_key = var.cloud_api_key
 }
 
-// Create resources (optional: within the organization)
+# Create resources (optional: within the organization)
 resource "grafana_folder" "my_folder" {
   #org_id = grafana_organization.my_org.org_id # not for Grafana Cloud !
   title = "Terraform Folder"
 }
 
+# Creating a service account in Grafana instance to be used as auth and attach tokens
+# notice we can attach multiple tokens to one service account
+# permissions https://registry.terraform.io/providers/grafana/grafana/latest/docs/resources/service_account_permission
+resource "grafana_service_account" "viewer" {
+  name = "viewer-service-account"
+  role = "Viewer" # available in UI: Viewer, Editor, Admin, No Basic Role
+}
+
+# Creating a service account token in Grafana instance to be used for viewing resources in Grafana instance
+resource "grafana_service_account_token" "viewer" {
+  name               = "viewer"
+  service_account_id = grafana_service_account.viewer.id
+}
+
+
+# this required cloud_api_key on provider with at least stack:read privileges
+data "grafana_cloud_stack" "current" {
+  slug = var.slug
+}
+
+# │ Error: the Cloud API client is required for `grafana_cloud_access_policy`. Set the cloud_api_key provider attribute
+# https://grafana.com/docs/grafana-cloud/account-management/cloud-portal/:
+# Free accounts are not able to create or delete individual services from a Stack and are limited to one Stack.
+#resource "grafana_cloud_access_policy" "hase" {
+#  name   = "hase"
+#  region = "eu"
+#  scopes = ["logs:write"]
+#  realm {
+#    identifier = ""
+#    type       = "stack"
+#  }
+#}
+
 locals {
   dashboard_title = "Fancy generated dashboard 22"
 }

terraform/modules/grafana/output.tf

@@ -2,3 +2,13 @@
 #  value = resource.value
 #}
 
+# expose to caller
+output "service_account_token_viewer_key" {
+  value     = grafana_service_account_token.viewer.key
+  sensitive = true
+}
+
+# expose to caller
+output "cloud_stack" {
+  value = data.grafana_cloud_stack.current.url
+}

terraform/modules/grafana/variables.tf

@@ -6,6 +6,12 @@ variable "auth" {
   description = "<Grafana-Service-Account-token> or API token, basic auth in the username:password format or anonymous (string literal). May alternatively be set via the GRAFANA_AUTH environment variable."
 }
 
+variable "cloud_api_key" {
+  description = "Cloud Access Policy Token  for Grafana Cloud with at least stack read permissions"
+}
+
+variable "slug" {}
+
 ## more complex
 #variable "topics" {
 #  description = "List of Kafka Topics"

tools/Makefile

@@ -75,4 +75,17 @@ grafana: ## run grafana locally on port 3000 with docker
 		-e "GF_LOG_LEVEL=debug" \
 		--user "$(id -u)" \
   		--volume "$$PWD/grafana-data:/var/lib/grafana" \
- 	 grafana/grafana-oss 
+ 	 grafana/grafana-oss 
+
+# https://squaredup.com/blog/three-ways-to-run-prometheus/
+# https://prometheus.io/docs/prometheus/latest/getting_started/
+# Remote writing only Prometheus consumes more than 10Gb of RAM:
+# https://github.com/prometheus/prometheus/issues/5666
+.PHONY: prom
+prom: ## run prometheus locally on port 9090 with docker
+	@docker ps -a -q -f name=prometheus && docker rm -f prometheus
+	mkdir -p prometheus-data/
+	@docker run -d -p 9090:9090 --name prometheus \
+		--volume prometheus-data:/prometheus \
+		--volume "$$PWD"/prometheus.yml:/etc/prometheus/prometheus.yml \
+		prom/prometheus

tools/prometheus.yml

@@ -0,0 +1,19 @@
+global:
+  scrape_interval:     15s # By default, scrape targets every 15 seconds.
+
+  # Attach these labels to any time series or alerts when communicating with
+  # external systems (federation, remote storage, Alertmanager).
+  external_labels:
+    monitor: 'prom-monitor'
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: 'prometheus'
+
+    # Override the global default and scrape targets from this job every 5 seconds.
+    scrape_interval: 10s
+
+    static_configs:
+      - targets: ['localhost:9090']