Introduce remotecfg service (#6277)

Signed-off-by: Paschalis Tsilias <paschalis.tsilias@grafana.com>
Co-authored-by: Mischa Thompson <mischabear@gmail.com>
Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com>

CHANGELOG.md

@@ -27,6 +27,9 @@ Main (unreleased)
 
 - Expose track_timestamps_staleness on Prometheus scraping, to fix the issue where container metrics live for 5 minutes after the container disappears. (@ptodev)
 
+- Introduce the `remotecfg` service that enables loading configuration from a
+  remote endpoint. (@tpaschalis) 
+
 ### Enhancements
 
 - Include line numbers in profiles produced by `pyrsocope.java` component. (@korniltsev)

cmd/internal/flowmode/cmd_run.go

@@ -33,6 +33,7 @@ import (
 	httpservice "github.com/grafana/agent/service/http"
 	"github.com/grafana/agent/service/labelstore"
 	otel_service "github.com/grafana/agent/service/otel"
+	remotecfgservice "github.com/grafana/agent/service/remotecfg"
 	uiservice "github.com/grafana/agent/service/ui"
 	"github.com/grafana/ckit/advertise"
 	"github.com/grafana/ckit/peer"
@@ -243,6 +244,14 @@ func (fr *flowRun) Run(configPath string) error {
 		EnablePProf:      fr.enablePprof,
 	})
 
+	remoteCfgService, err := remotecfgservice.New(remotecfgservice.Options{
+		Logger:      log.With(l, "service", "remotecfg"),
+		StoragePath: fr.storagePath,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to create the remotecfg service: %w", err)
+	}
+
 	uiService := uiservice.New(uiservice.Options{
 		UIPrefix: fr.uiPrefix,
 		Cluster:  clusterService.Data().(cluster.Cluster),
@@ -267,6 +276,7 @@ func (fr *flowRun) Run(configPath string) error {
 			clusterService,
 			otelService,
 			labelService,
+			remoteCfgService,
 		},
 	})
 

docs/sources/flow/reference/config-blocks/remotecfg.md

@@ -0,0 +1,97 @@
+---
+aliases:
+- /docs/grafana-cloud/agent/flow/reference/config-blocks/remotecfg/
+- /docs/grafana-cloud/monitor-infrastructure/agent/flow/reference/config-blocks/remotecfg/
+- /docs/grafana-cloud/monitor-infrastructure/integrations/agent/flow/reference/config-blocks/remotecfg/
+- /docs/grafana-cloud/send-data/agent/flow/reference/config-blocks/remotecfg/
+canonical: remotecfgs://grafana.com/docs/agent/latest/flow/reference/config-blocks/remotecfg/
+description: Learn about the remotecfg configuration block
+menuTitle: remotecfg
+title: remotecfg block
+---
+
+# remotecfg block
+
+`remotecfg` is an optional configuration block that enables {{< param "PRODUCT_NAME" >}}
+to fetch and load the configuration from a remote endpoint.
+`remotecfg` is specified without a label and can only be provided once per
+configuration file.
+
+The [API definition][] for managing and fetching configuration that the
+`remotecfg` block uses is available under the Apache 2.0 license.
+
+[API definition]: https://github.com/grafana/agent-remote-config
+
+## Example
+
+```river
+remotecfg {
+	url = "SERVICE_URL"
+	basic_auth {
+		username      = "USERNAME"
+		password_file = "PASSWORD_FILE"
+	}
+
+	id             = constants.hostname
+	metadata       = {"cluster" = "dev", "namespace" = "otlp-dev"}
+	poll_frequency = "5m"
+}
+```
+
+## Arguments
+
+The following arguments are supported:
+
+Name             | Type                 | Description                                       | Default     | Required
+-----------------|----------------------|---------------------------------------------------|-------------|---------
+`url`            | `string`             | The address of the API to poll for configuration. | `""`        | no
+`id`             | `string`             | A self-reported ID.                               | `see below` | no
+`metadata`       | `map(string)`        | A set of self-reported metadata.                  | `{}`        | no
+`poll_frequency` | `duration`           | How often to poll the API for new configuration.  | `"1m"`      | no
+
+If the `url` is not set, then the service block is a no-op.
+
+If not set, the self-reported `id` that the Agent uses is a randomly generated,
+anonymous unique ID (UUID) that is stored as an `agent_seed.json` file in the
+Agent's storage path so that it can persist across restarts.
+
+The `id` and `metadata` fields are used in the periodic request sent to the
+remote endpoint so that the API can decide what configuration to serve.
+
+## Blocks
+
+The following blocks are supported inside the definition of `remotecfg`:
+
+Hierarchy | Block | Description | Required
+--------- | ----- | ----------- | --------
+basic_auth | [basic_auth][] | Configure basic_auth for authenticating to the endpoint. | no
+authorization | [authorization][] | Configure generic authorization to the endpoint. | no
+oauth2 | [oauth2][] | Configure OAuth2 for authenticating to the endpoint. | no
+oauth2 > tls_config | [tls_config][] | Configure TLS settings for connecting to the endpoint. | no
+tls_config | [tls_config][] | Configure TLS settings for connecting to the endpoint. | no
+
+The `>` symbol indicates deeper levels of nesting. For example,
+`oauth2 > tls_config` refers to a `tls_config` block defined inside
+an `oauth2` block.
+
+[basic_auth]: #basic_auth-block
+[authorization]: #authorization-block
+[oauth2]: #oauth2-block
+[tls_config]: #tls_config-block
+
+### basic_auth block
+
+{{< docs/shared lookup="flow/reference/components/basic-auth-block.md" source="agent" version="<AGENT_VERSION>" >}}
+
+### authorization block
+
+{{< docs/shared lookup="flow/reference/components/authorization-block.md" source="agent" version="<AGENT_VERSION>" >}}
+
+### oauth2 block
+
+{{< docs/shared lookup="flow/reference/components/oauth2-block.md" source="agent" version="<AGENT_VERSION>" >}}
+
+### tls_config block
+
+{{< docs/shared lookup="flow/reference/components/tls-config-block.md" source="agent" version="<AGENT_VERSION>" >}}
+

go.mod

@@ -608,6 +608,7 @@ require github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab
 require (
 	connectrpc.com/connect v1.14.0
 	github.com/githubexporter/github-exporter v0.0.0-20231025122338-656e7dc33fe7
+	github.com/grafana/agent-remote-config v0.0.2
 	github.com/grafana/jfr-parser/pprof v0.0.0-20240126072739-986e71dc0361
 	github.com/natefinch/atomic v1.0.1
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.87.0

go.sum

@@ -1044,6 +1044,8 @@ github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/gosnmp/gosnmp v1.36.0 h1:1Si+MImHcKIqFc3/kJEs2LOULP1nlFKlzPFyrMOk5Qk=
 github.com/gosnmp/gosnmp v1.36.0/go.mod h1:iLcZxN2MxKhH0jPQDVMZaSNypw1ykqVi27O79koQj6w=
 github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY=
+github.com/grafana/agent-remote-config v0.0.2 h1:s3FKgVzfY5Ij+xG0wVKgVvtDrh/Bz0ZvB3D5MM7LJxU=
+github.com/grafana/agent-remote-config v0.0.2/go.mod h1:amyG3pVNXKcMo+kNN46yhnAXAz/m/9Ew9MRf53n7XBg=
 github.com/grafana/cadvisor v0.0.0-20231110094609-5f7917925dea h1:Q5f5/nJJ0SbusZjA6F6XkJuHDbl2/PqdTGw6wHsuccA=
 github.com/grafana/cadvisor v0.0.0-20231110094609-5f7917925dea/go.mod h1:XjiOCFjmxXIWwauV5p39Mr2Yxlpyk72uKQH1UZvd4fQ=
 github.com/grafana/ckit v0.0.0-20230906125525-c046c99a5c04 h1:tG8Qxq4dN1WqakMmsPaxaH4+OQhYg5HVsarw5acLBX8=

pkg/flow/flow_services.go

@@ -1,8 +1,11 @@
 package flow
 
 import (
+	"context"
+
 	"github.com/grafana/agent/pkg/flow/internal/controller"
 	"github.com/grafana/agent/pkg/flow/internal/dag"
+	"github.com/grafana/agent/pkg/flow/internal/worker"
 	"github.com/grafana/agent/service"
 )
 
@@ -52,3 +55,38 @@ func serviceConsumersForGraph(graph *dag.Graph, serviceName string, includePeerS
 
 	return consumers
 }
+
+// NewController returns a new, unstarted, isolated Flow controller so that
+// services can instantiate their own components.
+func (f *Flow) NewController(id string) service.Controller {
+	return serviceController{
+		f: newController(controllerOptions{
+			Options: Options{
+				ControllerID:    id,
+				Logger:          f.opts.Logger,
+				Tracer:          f.opts.Tracer,
+				DataPath:        f.opts.DataPath,
+				Reg:             f.opts.Reg,
+				Services:        f.opts.Services,
+				OnExportsChange: nil, // NOTE(@tpaschalis, @wildum) The isolated controller shouldn't be able to export any values.
+			},
+			IsModule:       true,
+			ModuleRegistry: newModuleRegistry(),
+			WorkerPool:     worker.NewDefaultWorkerPool(),
+		}),
+	}
+}
+
+type serviceController struct {
+	f *Flow
+}
+
+func (sc serviceController) Run(ctx context.Context) { sc.f.Run(ctx) }
+func (sc serviceController) LoadSource(b []byte, args map[string]any) error {
+	source, err := ParseSource("", b)
+	if err != nil {
+		return err
+	}
+	return sc.f.LoadSource(source, args)
+}
+func (sc serviceController) Ready() bool { return sc.f.Ready() }

service/http/http_test.go

@@ -215,3 +215,5 @@ func (fakeHost) ListComponents(moduleID string, opts component.InfoOptions) ([]*
 }
 
 func (fakeHost) GetServiceConsumers(serviceName string) []service.Consumer { return nil }
+
+func (fakeHost) NewController(id string) service.Controller { return nil }

service/remotecfg/noop.go

@@ -0,0 +1,41 @@
+package remotecfg
+
+import (
+	"context"
+	"errors"
+
+	"connectrpc.com/connect"
+	agentv1 "github.com/grafana/agent-remote-config/api/gen/proto/go/agent/v1"
+)
+
+type noopClient struct{}
+
+// GetConfig returns the agent's configuration.
+func (c noopClient) GetConfig(context.Context, *connect.Request[agentv1.GetConfigRequest]) (*connect.Response[agentv1.GetConfigResponse], error) {
+	return nil, errors.New("noop client")
+}
+
+// GetAgent returns information about the agent.
+func (c noopClient) GetAgent(context.Context, *connect.Request[agentv1.GetAgentRequest]) (*connect.Response[agentv1.Agent], error) {
+	return nil, errors.New("noop client")
+}
+
+// ListAgents returns information about all agents.
+func (c noopClient) ListAgents(context.Context, *connect.Request[agentv1.ListAgentsRequest]) (*connect.Response[agentv1.Agents], error) {
+	return nil, errors.New("noop client")
+}
+
+// CreateAgent registers a new agent.
+func (c noopClient) CreateAgent(context.Context, *connect.Request[agentv1.CreateAgentRequest]) (*connect.Response[agentv1.Agent], error) {
+	return nil, errors.New("noop client")
+}
+
+// UpdateAgent updates an existing agent.
+func (c noopClient) UpdateAgent(context.Context, *connect.Request[agentv1.UpdateAgentRequest]) (*connect.Response[agentv1.Agent], error) {
+	return nil, errors.New("noop client")
+}
+
+// DeleteAgent deletes an existing agent.
+func (c noopClient) DeleteAgent(context.Context, *connect.Request[agentv1.DeleteAgentRequest]) (*connect.Response[agentv1.DeleteAgentResponse], error) {
+	return nil, errors.New("noop client")
+}