From 44b6985dbc49b09bc993bf4c98d593a74bd12bdd Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 24 Oct 2023 16:55:49 +0200
Subject: [PATCH 01/91] first draft of remote output

---
 internal/pkg/policy/parsed_policy.go |  5 ++
 internal/pkg/policy/policy_output.go | 71 +++++++++++++++++++++++++---
 2 files changed, 70 insertions(+), 6 deletions(-)

diff --git a/internal/pkg/policy/parsed_policy.go b/internal/pkg/policy/parsed_policy.go
index 87f8c5050..809d0dafd 100644
--- a/internal/pkg/policy/parsed_policy.go
+++ b/internal/pkg/policy/parsed_policy.go
@@ -129,6 +129,11 @@ func constructPolicyOutputs(outputsRaw json.RawMessage, roles map[string]RoleT)
 			p.Role = &role
 		}
 
+		if p.Type == OutputTypeRemoteElasticsearch {
+			p.ServiceToken = v.GetString(FieldOutputServiceToken)
+			p.Type = OutputTypeElasticsearch
+		}
+
 		result[k] = p
 	}
 
diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index f558ae0a4..f0065f5ab 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -17,16 +17,19 @@ import (
 
 	"github.com/elastic/fleet-server/v7/internal/pkg/apikey"
 	"github.com/elastic/fleet-server/v7/internal/pkg/bulk"
+	"github.com/elastic/fleet-server/v7/internal/pkg/config"
 	"github.com/elastic/fleet-server/v7/internal/pkg/dl"
+	"github.com/elastic/fleet-server/v7/internal/pkg/es"
 	"github.com/elastic/fleet-server/v7/internal/pkg/logger"
 	"github.com/elastic/fleet-server/v7/internal/pkg/model"
 	"github.com/elastic/fleet-server/v7/internal/pkg/smap"
 )
 
 const (
-	OutputTypeElasticsearch = "elasticsearch"
-	OutputTypeLogstash      = "logstash"
-	OutputTypeKafka         = "kafka"
+	OutputTypeElasticsearch       = "elasticsearch"
+	OutputTypeRemoteElasticsearch = "remote-elasticsearch"
+	OutputTypeLogstash            = "logstash"
+	OutputTypeKafka               = "kafka"
 )
 
 var (
@@ -35,9 +38,10 @@ var (
 )
 
 type Output struct {
-	Name string
-	Type string
-	Role *RoleT
+	Name         string
+	Type         string
+	ServiceToken string `json:"service_token,omitempty"`
+	Role         *RoleT
 }
 
 // Prepare prepares the output p to be sent to the elastic-agent
@@ -229,6 +233,35 @@ func (p *Output) prepareElasticsearch(
 		output.PermissionsHash = p.Role.Sha2 // for the sake of consistency
 	}
 
+	if outputMap.GetMap(p.Name).GetString("type") == OutputTypeRemoteElasticsearch {
+		om, _ := outputMap.GetMap(p.Name).Marshal()
+		var outputObj map[string]any
+		json.Unmarshal(om, &outputObj)
+		hosts := outputObj["hosts"].([]interface{})
+		outputAPIKey, err :=
+			generateRemoteOutputAPIKey(ctx, agent.Id, p, []string{hosts[0].(string)})
+		if err != nil {
+			return fmt.Errorf("failed generate output API key: %w", err)
+		}
+
+		// TODO update agent doc like above
+		// TODO what about update flow?
+
+		output.APIKey = outputAPIKey.Agent()
+		output.APIKeyID = outputAPIKey.ID
+		output.PermissionsHash = p.Role.Sha2
+
+		zlog.Debug().
+			Str("outputAPIKey.ID", outputAPIKey.ID).
+			Msg("generated remote output api key")
+
+			// replace type remote-elasticsearch with elasticsearch as agent doesn't recognize remote-elasticsearch
+		if err := setMapObj(outputMap, OutputTypeElasticsearch, p.Name, "type"); err != nil {
+			return err
+		}
+		// TODO remove the service token from the agent policy sent to the agent
+	}
+
 	// Always insert the `api_key` as part of the output block, this is required
 	// because only fleet server knows the api key for the specific agent, if we don't
 	// add it the agent will not receive the `api_key` and will not be able to connect
@@ -411,6 +444,32 @@ func generateOutputAPIKey(
 	)
 }
 
+func generateRemoteOutputAPIKey(ctx context.Context,
+	agentID string,
+	output *Output,
+	hosts []string,
+	// p.Name, p.Role.Raw, p.ServiceToken
+	// outputName string,
+	// roles []byte, serviceToken string
+) (*apikey.APIKey, error) {
+	name := fmt.Sprintf("%s:%s", agentID, output.Name)
+	cfg := config.Config{
+		Output: config.Output{
+			Elasticsearch: config.Elasticsearch{
+				Hosts:        hosts,
+				ServiceToken: output.ServiceToken,
+			},
+		},
+	}
+	// cfg.Output.Elasticsearch.Hosts
+	es, err := es.NewClient(ctx, &cfg, false)
+
+	if err != nil {
+		return nil, err
+	}
+	return apikey.Create(ctx, es, name, "", "false", output.Role.Raw, apikey.NewMetadata(agentID, output.Name, apikey.TypeOutput))
+}
+
 func setMapObj(obj map[string]interface{}, val interface{}, keys ...string) error {
 	if len(keys) == 0 {
 		return fmt.Errorf("no key to be updated: %w", ErrFailInjectAPIKey)

From f870e554486c48d690f15520320beec5cf352eb7 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 24 Oct 2023 17:34:36 +0200
Subject: [PATCH 02/91] remove commented code

---
 internal/pkg/policy/policy_output.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index f0065f5ab..06d9ea71f 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -448,9 +448,6 @@ func generateRemoteOutputAPIKey(ctx context.Context,
 	agentID string,
 	output *Output,
 	hosts []string,
-	// p.Name, p.Role.Raw, p.ServiceToken
-	// outputName string,
-	// roles []byte, serviceToken string
 ) (*apikey.APIKey, error) {
 	name := fmt.Sprintf("%s:%s", agentID, output.Name)
 	cfg := config.Config{
@@ -461,7 +458,6 @@ func generateRemoteOutputAPIKey(ctx context.Context,
 			},
 		},
 	}
-	// cfg.Output.Elasticsearch.Hosts
 	es, err := es.NewClient(ctx, &cfg, false)
 
 	if err != nil {

From 4605e04cfcdb7d19c63d4a0fbbd61631b58642e0 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 26 Oct 2023 13:09:56 +0200
Subject: [PATCH 03/91] storing remote es clients in bulker

---
 internal/pkg/bulk/engine.go          |  45 ++++++++---
 internal/pkg/policy/parsed_policy.go |   1 -
 internal/pkg/policy/policy_output.go | 111 ++++++++++++++-------------
 3 files changed, 92 insertions(+), 65 deletions(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 47dc5fbc1..fb043a2ba 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -65,18 +65,22 @@ type Bulk interface {
 	// Accessor used to talk to elastic search direcly bypassing bulk engine
 	Client() *elasticsearch.Client
 
+	GetRemoteClient(name string) *elasticsearch.Client
+	SetRemoteClient(name string, es *elasticsearch.Client)
+
 	ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error)
 }
 
 const kModBulk = "bulk"
 
 type Bulker struct {
-	es          esapi.Transport
-	ch          chan *bulkT
-	opts        bulkOptT
-	blkPool     sync.Pool
-	apikeyLimit *semaphore.Weighted
-	tracer      *apm.Tracer
+	es           esapi.Transport
+	remoteEsList map[string]esapi.Transport
+	ch           chan *bulkT
+	opts         bulkOptT
+	blkPool      sync.Pool
+	apikeyLimit  *semaphore.Weighted
+	tracer       *apm.Tracer
 }
 
 const (
@@ -98,12 +102,13 @@ func NewBulker(es esapi.Transport, tracer *apm.Tracer, opts ...BulkOpt) *Bulker
 	}
 
 	return &Bulker{
-		opts:        bopts,
-		es:          es,
-		ch:          make(chan *bulkT, bopts.blockQueueSz),
-		blkPool:     sync.Pool{New: poolFunc},
-		apikeyLimit: semaphore.NewWeighted(int64(bopts.apikeyMaxParallel)),
-		tracer:      tracer,
+		opts:         bopts,
+		es:           es,
+		remoteEsList: make(map[string]esapi.Transport),
+		ch:           make(chan *bulkT, bopts.blockQueueSz),
+		blkPool:      sync.Pool{New: poolFunc},
+		apikeyLimit:  semaphore.NewWeighted(int64(bopts.apikeyMaxParallel)),
+		tracer:       tracer,
 	}
 }
 
@@ -115,6 +120,22 @@ func (b *Bulker) Client() *elasticsearch.Client {
 	return client
 }
 
+func (b *Bulker) GetRemoteClient(name string) *elasticsearch.Client {
+	remoteEs := b.remoteEsList[name]
+	if remoteEs == nil {
+		return nil
+	}
+	client, ok := remoteEs.(*elasticsearch.Client)
+	if !ok {
+		panic("Client is not an elastic search pointer")
+	}
+	return client
+}
+
+func (b *Bulker) SetRemoteClient(name string, es *elasticsearch.Client) {
+	b.remoteEsList[name] = es
+}
+
 // read secrets one by one as there is no bulk API yet to read them in one request
 func (b *Bulker) ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error) {
 	result := make(map[string]string)
diff --git a/internal/pkg/policy/parsed_policy.go b/internal/pkg/policy/parsed_policy.go
index 809d0dafd..e8beabf20 100644
--- a/internal/pkg/policy/parsed_policy.go
+++ b/internal/pkg/policy/parsed_policy.go
@@ -131,7 +131,6 @@ func constructPolicyOutputs(outputsRaw json.RawMessage, roles map[string]RoleT)
 
 		if p.Type == OutputTypeRemoteElasticsearch {
 			p.ServiceToken = v.GetString(FieldOutputServiceToken)
-			p.Type = OutputTypeElasticsearch
 		}
 
 		result[k] = p
diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index 06d9ea71f..4535b89c0 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -60,6 +60,12 @@ func (p *Output) Prepare(ctx context.Context, zlog zerolog.Logger, bulker bulk.B
 		if err := p.prepareElasticsearch(ctx, zlog, bulker, agent, outputMap); err != nil {
 			return fmt.Errorf("failed to prepare elasticsearch output %q: %w", p.Name, err)
 		}
+	case OutputTypeRemoteElasticsearch:
+		zlog.Debug().Msg("preparing remote elasticsearch output")
+		p.createRemoteEsClientIfNotExists(ctx, bulker, outputMap)
+		if err := p.prepareElasticsearch(ctx, zlog, bulker, agent, outputMap); err != nil {
+			return fmt.Errorf("failed to prepare remote elasticsearch output %q: %w", p.Name, err)
+		}
 	case OutputTypeLogstash:
 		zlog.Debug().Msg("preparing logstash output")
 		zlog.Info().Msg("no actions required for logstash output preparation")
@@ -145,6 +151,7 @@ func (p *Output) prepareElasticsearch(
 			return err
 		}
 
+		// TODO use remote es for api key update
 		// hash provided is only for merging request together and not persisted
 		err = bulker.APIKeyUpdate(ctx, output.APIKeyID, newRoles.Sha2, newRoles.Raw)
 		if err != nil {
@@ -183,7 +190,7 @@ func (p *Output) prepareElasticsearch(
 
 		ctx := zlog.WithContext(ctx)
 		outputAPIKey, err :=
-			generateOutputAPIKey(ctx, bulker, agent.Id, p.Name, p.Role.Raw)
+			generateOutputAPIKey(ctx, bulker, agent.Id, p.Name, p.Role.Raw, p.Type)
 		if err != nil {
 			return fmt.Errorf("failed generate output API key: %w", err)
 		}
@@ -233,29 +240,9 @@ func (p *Output) prepareElasticsearch(
 		output.PermissionsHash = p.Role.Sha2 // for the sake of consistency
 	}
 
-	if outputMap.GetMap(p.Name).GetString("type") == OutputTypeRemoteElasticsearch {
-		om, _ := outputMap.GetMap(p.Name).Marshal()
-		var outputObj map[string]any
-		json.Unmarshal(om, &outputObj)
-		hosts := outputObj["hosts"].([]interface{})
-		outputAPIKey, err :=
-			generateRemoteOutputAPIKey(ctx, agent.Id, p, []string{hosts[0].(string)})
-		if err != nil {
-			return fmt.Errorf("failed generate output API key: %w", err)
-		}
-
-		// TODO update agent doc like above
-		// TODO what about update flow?
-
-		output.APIKey = outputAPIKey.Agent()
-		output.APIKeyID = outputAPIKey.ID
-		output.PermissionsHash = p.Role.Sha2
-
-		zlog.Debug().
-			Str("outputAPIKey.ID", outputAPIKey.ID).
-			Msg("generated remote output api key")
+	if p.Type == OutputTypeRemoteElasticsearch {
 
-			// replace type remote-elasticsearch with elasticsearch as agent doesn't recognize remote-elasticsearch
+		// replace type remote-elasticsearch with elasticsearch as agent doesn't recognize remote-elasticsearch
 		if err := setMapObj(outputMap, OutputTypeElasticsearch, p.Name, "type"); err != nil {
 			return err
 		}
@@ -278,6 +265,43 @@ func (p *Output) prepareElasticsearch(
 	return nil
 }
 
+func (p *Output) createRemoteEsClientIfNotExists(ctx context.Context, bulker bulk.Bulk, outputMap smap.Map) error {
+	remoteEs := bulker.GetRemoteClient(p.Name)
+	if remoteEs != nil {
+		// TODO replace if hosts or service token changed?
+		return nil
+	}
+	om, _ := outputMap.GetMap(p.Name).Marshal()
+	var outputObj map[string]any
+	err := json.Unmarshal(om, &outputObj)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal output: %w", err)
+	}
+	hosts, ok := outputObj["hosts"].([]interface{})
+	if !ok {
+		return fmt.Errorf("failed to get hsots from output: %w", err)
+	}
+	hostsStrings := make([]string, len(hosts))
+	for i, host := range hosts {
+		hostsStrings[i] = host.(string)
+	}
+
+	cfg := config.Config{
+		Output: config.Output{
+			Elasticsearch: config.Elasticsearch{
+				Hosts:        hostsStrings,
+				ServiceToken: p.ServiceToken,
+			},
+		},
+	}
+	es, err := es.NewClient(ctx, &cfg, false)
+	if err != nil {
+		return err
+	}
+	bulker.SetRemoteClient(p.Name, es)
+	return nil
+}
+
 func fetchAPIKeyRoles(ctx context.Context, b bulk.Bulk, apiKeyID string) (*RoleT, error) {
 	res, err := b.APIKeyRead(ctx, apiKeyID, true)
 	if err != nil {
@@ -431,39 +455,22 @@ func generateOutputAPIKey(
 	bulk bulk.Bulk,
 	agentID,
 	outputName string,
-	roles []byte) (*apikey.APIKey, error) {
+	roles []byte, outputType string) (*apikey.APIKey, error) {
 	name := fmt.Sprintf("%s:%s", agentID, outputName)
 	zerolog.Ctx(ctx).Info().Msgf("generating output API key %s for agent ID %s",
 		name, agentID)
-	return bulk.APIKeyCreate(
-		ctx,
-		name,
-		"",
-		roles,
-		apikey.NewMetadata(agentID, outputName, apikey.TypeOutput),
-	)
-}
-
-func generateRemoteOutputAPIKey(ctx context.Context,
-	agentID string,
-	output *Output,
-	hosts []string,
-) (*apikey.APIKey, error) {
-	name := fmt.Sprintf("%s:%s", agentID, output.Name)
-	cfg := config.Config{
-		Output: config.Output{
-			Elasticsearch: config.Elasticsearch{
-				Hosts:        hosts,
-				ServiceToken: output.ServiceToken,
-			},
-		},
-	}
-	es, err := es.NewClient(ctx, &cfg, false)
-
-	if err != nil {
-		return nil, err
+	if outputType == OutputTypeRemoteElasticsearch {
+		name := fmt.Sprintf("%s:%s", agentID, outputName)
+		return apikey.Create(ctx, bulk.GetRemoteClient(outputName), name, "", "false", roles, apikey.NewMetadata(agentID, outputName, apikey.TypeOutput))
+	} else {
+		return bulk.APIKeyCreate(
+			ctx,
+			name,
+			"",
+			roles,
+			apikey.NewMetadata(agentID, outputName, apikey.TypeOutput),
+		)
 	}
-	return apikey.Create(ctx, es, name, "", "false", output.Role.Raw, apikey.NewMetadata(agentID, output.Name, apikey.TypeOutput))
 }
 
 func setMapObj(obj map[string]interface{}, val interface{}, keys ...string) error {

From b6b47afd9ce71e0e349d6f82f26fba34e0d3991c Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 26 Oct 2023 13:51:48 +0200
Subject: [PATCH 04/91] remote service_token from policy sent to agent

---
 internal/pkg/policy/policy_output.go | 7 ++++---
 internal/pkg/testing/bulk.go         | 8 ++++++++
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index 4535b89c0..d38c0ca80 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -40,7 +40,7 @@ var (
 type Output struct {
 	Name         string
 	Type         string
-	ServiceToken string `json:"service_token,omitempty"`
+	ServiceToken string
 	Role         *RoleT
 }
 
@@ -243,10 +243,11 @@ func (p *Output) prepareElasticsearch(
 	if p.Type == OutputTypeRemoteElasticsearch {
 
 		// replace type remote-elasticsearch with elasticsearch as agent doesn't recognize remote-elasticsearch
-		if err := setMapObj(outputMap, OutputTypeElasticsearch, p.Name, "type"); err != nil {
+		if err := setMapObj(outputMap, OutputTypeElasticsearch, p.Name, FieldOutputType); err != nil {
 			return err
 		}
-		// TODO remove the service token from the agent policy sent to the agent
+		// remove the service token from the agent policy sent to the agent
+		delete(outputMap.GetMap(p.Name), FieldOutputServiceToken)
 	}
 
 	// Always insert the `api_key` as part of the output block, this is required
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index fa32f9da2..7b6272758 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -79,6 +79,14 @@ func (m *MockBulk) Client() *elasticsearch.Client {
 	return args.Get(0).(*elasticsearch.Client)
 }
 
+func (m *MockBulk) GetRemoteClient(name string) *elasticsearch.Client {
+	args := m.Called()
+	return args.Get(0).(*elasticsearch.Client)
+}
+
+func (m *MockBulk) SetRemoteClient(name string, es *elasticsearch.Client) {
+}
+
 func (m *MockBulk) ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error) {
 	result := make(map[string]string)
 	for _, id := range secretIds {

From d82a117354a5205ace85bf9ab87c246e2520b468 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 26 Oct 2023 14:17:13 +0200
Subject: [PATCH 05/91] fix lint

---
 internal/pkg/policy/policy_output.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index d38c0ca80..295de90bb 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -62,7 +62,10 @@ func (p *Output) Prepare(ctx context.Context, zlog zerolog.Logger, bulker bulk.B
 		}
 	case OutputTypeRemoteElasticsearch:
 		zlog.Debug().Msg("preparing remote elasticsearch output")
-		p.createRemoteEsClientIfNotExists(ctx, bulker, outputMap)
+		err := p.createRemoteEsClientIfNotExists(ctx, bulker, outputMap)
+		if err != nil {
+			return err
+		}
 		if err := p.prepareElasticsearch(ctx, zlog, bulker, agent, outputMap); err != nil {
 			return fmt.Errorf("failed to prepare remote elasticsearch output %q: %w", p.Name, err)
 		}

From 0ceb201110232ee1c75477a66a7a95dd5627179d Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 26 Oct 2023 14:23:58 +0200
Subject: [PATCH 06/91] rename remoteEsList and type

---
 internal/pkg/bulk/engine.go | 38 +++++++++++++++++--------------------
 1 file changed, 17 insertions(+), 21 deletions(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index fb043a2ba..62ba04c24 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -74,13 +74,13 @@ type Bulk interface {
 const kModBulk = "bulk"
 
 type Bulker struct {
-	es           esapi.Transport
-	remoteEsList map[string]esapi.Transport
-	ch           chan *bulkT
-	opts         bulkOptT
-	blkPool      sync.Pool
-	apikeyLimit  *semaphore.Weighted
-	tracer       *apm.Tracer
+	es              esapi.Transport
+	remoteEsClients map[string]*elasticsearch.Client
+	ch              chan *bulkT
+	opts            bulkOptT
+	blkPool         sync.Pool
+	apikeyLimit     *semaphore.Weighted
+	tracer          *apm.Tracer
 }
 
 const (
@@ -102,13 +102,13 @@ func NewBulker(es esapi.Transport, tracer *apm.Tracer, opts ...BulkOpt) *Bulker
 	}
 
 	return &Bulker{
-		opts:         bopts,
-		es:           es,
-		remoteEsList: make(map[string]esapi.Transport),
-		ch:           make(chan *bulkT, bopts.blockQueueSz),
-		blkPool:      sync.Pool{New: poolFunc},
-		apikeyLimit:  semaphore.NewWeighted(int64(bopts.apikeyMaxParallel)),
-		tracer:       tracer,
+		opts:            bopts,
+		es:              es,
+		remoteEsClients: make(map[string]*elasticsearch.Client),
+		ch:              make(chan *bulkT, bopts.blockQueueSz),
+		blkPool:         sync.Pool{New: poolFunc},
+		apikeyLimit:     semaphore.NewWeighted(int64(bopts.apikeyMaxParallel)),
+		tracer:          tracer,
 	}
 }
 
@@ -121,19 +121,15 @@ func (b *Bulker) Client() *elasticsearch.Client {
 }
 
 func (b *Bulker) GetRemoteClient(name string) *elasticsearch.Client {
-	remoteEs := b.remoteEsList[name]
+	remoteEs := b.remoteEsClients[name]
 	if remoteEs == nil {
 		return nil
 	}
-	client, ok := remoteEs.(*elasticsearch.Client)
-	if !ok {
-		panic("Client is not an elastic search pointer")
-	}
-	return client
+	return remoteEs
 }
 
 func (b *Bulker) SetRemoteClient(name string, es *elasticsearch.Client) {
-	b.remoteEsList[name] = es
+	b.remoteEsClients[name] = es
 }
 
 // read secrets one by one as there is no bulk API yet to read them in one request

From c63fb1e047dbec35fb85ede48941884a12a7d4c9 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 26 Oct 2023 14:37:05 +0200
Subject: [PATCH 07/91] added missing check

---
 internal/pkg/policy/policy_output.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index 295de90bb..5768aafb6 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -283,11 +283,14 @@ func (p *Output) createRemoteEsClientIfNotExists(ctx context.Context, bulker bul
 	}
 	hosts, ok := outputObj["hosts"].([]interface{})
 	if !ok {
-		return fmt.Errorf("failed to get hsots from output: %w", err)
+		return fmt.Errorf("failed to get hosts from output: %w", err)
 	}
 	hostsStrings := make([]string, len(hosts))
 	for i, host := range hosts {
-		hostsStrings[i] = host.(string)
+		hostsStrings[i], ok = host.(string)
+		if !ok {
+			return fmt.Errorf("failed to get hosts from output: %w", err)
+		}
 	}
 
 	cfg := config.Config{

From 6cd8491934ea463411b0707997b3877a37e70c82 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 30 Oct 2023 10:50:11 +0100
Subject: [PATCH 08/91] fixes after merge main

---
 internal/pkg/policy/parsed_policy.go |  6 +++++-
 internal/pkg/policy/policy_output.go | 21 +++++++--------------
 2 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/internal/pkg/policy/parsed_policy.go b/internal/pkg/policy/parsed_policy.go
index 1d46ad2d5..eb4076b14 100644
--- a/internal/pkg/policy/parsed_policy.go
+++ b/internal/pkg/policy/parsed_policy.go
@@ -112,7 +112,11 @@ func constructPolicyOutputs(outputs map[string]map[string]interface{}, roles map
 		}
 
 		if p.Type == OutputTypeRemoteElasticsearch {
-			p.ServiceToken = v.GetString(FieldOutputServiceToken)
+			serviceTokenStr, ok := v[FieldOutputServiceToken].(string)
+			if !ok {
+				return nil, fmt.Errorf("missing or invalid service token: %+v", v)
+			}
+			p.ServiceToken = serviceTokenStr
 		}
 
 		result[k] = p
diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index b6a4d39f1..c341d2764 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -250,11 +250,9 @@ func (p *Output) prepareElasticsearch(
 	if p.Type == OutputTypeRemoteElasticsearch {
 
 		// replace type remote-elasticsearch with elasticsearch as agent doesn't recognize remote-elasticsearch
-		if err := setMapObj(outputMap, OutputTypeElasticsearch, p.Name, FieldOutputType); err != nil {
-			return err
-		}
+		outputMap[p.Name][FieldOutputType] = OutputTypeElasticsearch
 		// remove the service token from the agent policy sent to the agent
-		delete(outputMap.GetMap(p.Name), FieldOutputServiceToken)
+		delete(outputMap[p.Name], FieldOutputServiceToken)
 	}
 
 	// Always insert the `api_key` as part of the output block, this is required
@@ -270,27 +268,22 @@ func (p *Output) prepareElasticsearch(
 	return nil
 }
 
-func (p *Output) createRemoteEsClientIfNotExists(ctx context.Context, bulker bulk.Bulk, outputMap smap.Map) error {
+func (p *Output) createRemoteEsClientIfNotExists(ctx context.Context, bulker bulk.Bulk, outputMap map[string]map[string]interface{}) error {
 	remoteEs := bulker.GetRemoteClient(p.Name)
 	if remoteEs != nil {
 		// TODO replace if hosts or service token changed?
 		return nil
 	}
-	om, _ := outputMap.GetMap(p.Name).Marshal()
-	var outputObj map[string]any
-	err := json.Unmarshal(om, &outputObj)
-	if err != nil {
-		return fmt.Errorf("failed to unmarshal output: %w", err)
-	}
-	hosts, ok := outputObj["hosts"].([]interface{})
+	hostsObj := outputMap[p.Name]["hosts"]
+	hosts, ok := hostsObj.([]interface{})
 	if !ok {
-		return fmt.Errorf("failed to get hosts from output: %w", err)
+		return fmt.Errorf("failed to get hosts from output: %w", hostsObj)
 	}
 	hostsStrings := make([]string, len(hosts))
 	for i, host := range hosts {
 		hostsStrings[i], ok = host.(string)
 		if !ok {
-			return fmt.Errorf("failed to get hosts from output: %w", err)
+			return fmt.Errorf("failed to get hosts from output: %w", host)
 		}
 	}
 

From 4b72d7152d021c2e0f02cf2b564fe50f7acfc3b1 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 30 Oct 2023 10:56:10 +0100
Subject: [PATCH 09/91] fix lint

---
 internal/pkg/policy/policy_output.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index c341d2764..6459de87a 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -277,13 +277,13 @@ func (p *Output) createRemoteEsClientIfNotExists(ctx context.Context, bulker bul
 	hostsObj := outputMap[p.Name]["hosts"]
 	hosts, ok := hostsObj.([]interface{})
 	if !ok {
-		return fmt.Errorf("failed to get hosts from output: %w", hostsObj)
+		return fmt.Errorf("failed to get hosts from output: %v", hostsObj)
 	}
 	hostsStrings := make([]string, len(hosts))
 	for i, host := range hosts {
 		hostsStrings[i], ok = host.(string)
 		if !ok {
-			return fmt.Errorf("failed to get hosts from output: %w", host)
+			return fmt.Errorf("failed to get hosts from output: %v", host)
 		}
 	}
 

From 1d6c31fb76506224ed62abcefbd74abe41da0de8 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 30 Oct 2023 11:01:48 +0100
Subject: [PATCH 10/91] added changelog

---
 .../1698660041-remote-es-output.yaml          | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 changelog/fragments/1698660041-remote-es-output.yaml

diff --git a/changelog/fragments/1698660041-remote-es-output.yaml b/changelog/fragments/1698660041-remote-es-output.yaml
new file mode 100644
index 000000000..72c9abc81
--- /dev/null
+++ b/changelog/fragments/1698660041-remote-es-output.yaml
@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Support for remote elasticsearch output
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component:
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

From 53de755842028e51e79ce98a84231bec96e9e462 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 30 Oct 2023 13:51:28 +0100
Subject: [PATCH 11/91] create bulker for remote es

---
 internal/pkg/bulk/engine.go          |  8 +++
 internal/pkg/policy/policy_output.go | 84 ++++++++++++++++++----------
 2 files changed, 64 insertions(+), 28 deletions(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index f412e8c16..b2fa802c1 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -68,6 +68,8 @@ type Bulk interface {
 	GetRemoteClient(name string) *elasticsearch.Client
 	SetRemoteClient(name string, es *elasticsearch.Client)
 
+	Opts() BulkOpt
+
 	ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error)
 }
 
@@ -120,6 +122,12 @@ func (b *Bulker) Client() *elasticsearch.Client {
 	return client
 }
 
+func (b *Bulker) Opts() BulkOpt {
+	return func(o *bulkOptT) {
+		o = &b.opts
+	}
+}
+
 func (b *Bulker) GetRemoteClient(name string) *elasticsearch.Client {
 	remoteEs := b.remoteEsClients[name]
 	if remoteEs == nil {
diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index 6459de87a..e9f12aa09 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -23,6 +23,7 @@ import (
 	"github.com/elastic/fleet-server/v7/internal/pkg/logger"
 	"github.com/elastic/fleet-server/v7/internal/pkg/model"
 	"github.com/elastic/fleet-server/v7/internal/pkg/smap"
+	"github.com/elastic/go-elasticsearch/v8"
 )
 
 const (
@@ -35,6 +36,8 @@ const (
 var (
 	ErrNoOutputPerms    = errors.New("output permission sections not found")
 	ErrFailInjectAPIKey = errors.New("fail inject api key")
+	// TODO clean up bulkers when a remote output is removed
+	bulkerMap = make(map[string]bulk.Bulk)
 )
 
 type Output struct {
@@ -57,16 +60,16 @@ func (p *Output) Prepare(ctx context.Context, zlog zerolog.Logger, bulker bulk.B
 	switch p.Type {
 	case OutputTypeElasticsearch:
 		zlog.Debug().Msg("preparing elasticsearch output")
-		if err := p.prepareElasticsearch(ctx, zlog, bulker, agent, outputMap); err != nil {
+		if err := p.prepareElasticsearch(ctx, zlog, bulker, bulker, agent, outputMap); err != nil {
 			return fmt.Errorf("failed to prepare elasticsearch output %q: %w", p.Name, err)
 		}
 	case OutputTypeRemoteElasticsearch:
 		zlog.Debug().Msg("preparing remote elasticsearch output")
-		err := p.createRemoteEsClientIfNotExists(ctx, bulker, outputMap)
+		newBulker, err := p.createAndGetBulker(ctx, bulker, outputMap)
 		if err != nil {
 			return err
 		}
-		if err := p.prepareElasticsearch(ctx, zlog, bulker, agent, outputMap); err != nil {
+		if err := p.prepareElasticsearch(ctx, zlog, bulker, newBulker, agent, outputMap); err != nil {
 			return fmt.Errorf("failed to prepare remote elasticsearch output %q: %w", p.Name, err)
 		}
 	case OutputTypeLogstash:
@@ -86,6 +89,7 @@ func (p *Output) prepareElasticsearch(
 	ctx context.Context,
 	zlog zerolog.Logger,
 	bulker bulk.Bulk,
+	outputBulker bulk.Bulk,
 	agent *model.Agent,
 	outputMap map[string]map[string]interface{}) error {
 	// The role is required to do api key management
@@ -141,7 +145,7 @@ func (p *Output) prepareElasticsearch(
 			Msg("Generating a new API key")
 
 		// query current api key for roles so we don't lose permissions in the meantime
-		currentRoles, err := fetchAPIKeyRoles(ctx, bulker, output.APIKeyID)
+		currentRoles, err := fetchAPIKeyRoles(ctx, outputBulker, output.APIKeyID)
 		if err != nil {
 			zlog.Error().
 				Str("apiKeyID", output.APIKeyID).
@@ -158,9 +162,8 @@ func (p *Output) prepareElasticsearch(
 			return err
 		}
 
-		// TODO use remote es for api key update
 		// hash provided is only for merging request together and not persisted
-		err = bulker.APIKeyUpdate(ctx, output.APIKeyID, newRoles.Sha2, newRoles.Raw)
+		err = outputBulker.APIKeyUpdate(ctx, output.APIKeyID, newRoles.Sha2, newRoles.Raw)
 		if err != nil {
 			zlog.Error().Err(err).Msg("fail generate output key")
 			zlog.Debug().RawJSON("roles", newRoles.Raw).Str("sha", newRoles.Sha2).Err(err).Msg("roles not updated")
@@ -197,7 +200,7 @@ func (p *Output) prepareElasticsearch(
 
 		ctx := zlog.WithContext(ctx)
 		outputAPIKey, err :=
-			generateOutputAPIKey(ctx, bulker, agent.Id, p.Name, p.Role.Raw, p.Type)
+			generateOutputAPIKey(ctx, outputBulker, agent.Id, p.Name, p.Role.Raw, p.Type)
 		if err != nil {
 			return fmt.Errorf("failed generate output API key: %w", err)
 		}
@@ -268,22 +271,53 @@ func (p *Output) prepareElasticsearch(
 	return nil
 }
 
-func (p *Output) createRemoteEsClientIfNotExists(ctx context.Context, bulker bulk.Bulk, outputMap map[string]map[string]interface{}) error {
-	remoteEs := bulker.GetRemoteClient(p.Name)
-	if remoteEs != nil {
+func (p *Output) createAndGetBulker(ctx context.Context, mainBulker bulk.Bulk, outputMap map[string]map[string]interface{}) (bulk.Bulk, error) {
+	bulker := bulkerMap[p.Name]
+	if bulker != nil {
 		// TODO replace if hosts or service token changed?
-		return nil
+		return bulker, nil
 	}
+	bulkCtx, bulkCancel := context.WithCancel(context.Background())
+	defer bulkCancel()
+	es, err := p.createRemoteEsClient(bulkCtx, outputMap)
+	if err != nil {
+		return nil, err
+	}
+	// starting a new bulker to create/update API keys for remote ES output
+	newBulker := bulk.NewBulker(es, nil, mainBulker.Opts())
+	bulkerMap[p.Name] = newBulker
+
+	errCh := make(chan error)
+	go func() {
+		runFunc := func() (err error) {
+			return newBulker.Run(bulkCtx)
+		}
+
+		errCh <- runFunc()
+	}()
+	go func() (err error) {
+		select {
+		case err = <-errCh:
+		case <-bulkCtx.Done():
+			err = bulkCtx.Err()
+		}
+		return
+	}()
+
+	return newBulker, nil
+}
+
+func (p *Output) createRemoteEsClient(ctx context.Context, outputMap map[string]map[string]interface{}) (*elasticsearch.Client, error) {
 	hostsObj := outputMap[p.Name]["hosts"]
 	hosts, ok := hostsObj.([]interface{})
 	if !ok {
-		return fmt.Errorf("failed to get hosts from output: %v", hostsObj)
+		return nil, fmt.Errorf("failed to get hosts from output: %v", hostsObj)
 	}
 	hostsStrings := make([]string, len(hosts))
 	for i, host := range hosts {
 		hostsStrings[i], ok = host.(string)
 		if !ok {
-			return fmt.Errorf("failed to get hosts from output: %v", host)
+			return nil, fmt.Errorf("failed to get hosts from output: %v", host)
 		}
 	}
 
@@ -297,10 +331,9 @@ func (p *Output) createRemoteEsClientIfNotExists(ctx context.Context, bulker bul
 	}
 	es, err := es.NewClient(ctx, &cfg, false)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	bulker.SetRemoteClient(p.Name, es)
-	return nil
+	return es, nil
 }
 
 func fetchAPIKeyRoles(ctx context.Context, b bulk.Bulk, apiKeyID string) (*RoleT, error) {
@@ -460,16 +493,11 @@ func generateOutputAPIKey(
 	name := fmt.Sprintf("%s:%s", agentID, outputName)
 	zerolog.Ctx(ctx).Info().Msgf("generating output API key %s for agent ID %s",
 		name, agentID)
-	if outputType == OutputTypeRemoteElasticsearch {
-		name := fmt.Sprintf("%s:%s", agentID, outputName)
-		return apikey.Create(ctx, bulk.GetRemoteClient(outputName), name, "", "false", roles, apikey.NewMetadata(agentID, outputName, apikey.TypeOutput))
-	} else {
-		return bulk.APIKeyCreate(
-			ctx,
-			name,
-			"",
-			roles,
-			apikey.NewMetadata(agentID, outputName, apikey.TypeOutput),
-		)
-	}
+	return bulk.APIKeyCreate(
+		ctx,
+		name,
+		"",
+		roles,
+		apikey.NewMetadata(agentID, outputName, apikey.TypeOutput),
+	)
 }

From 5bbc1f188fba72f8853b3ab0ae9df06c155a070e Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 30 Oct 2023 14:21:56 +0100
Subject: [PATCH 12/91] cleanup unused remoteEsClients and functions

---
 internal/pkg/bulk/engine.go          | 42 +++++++++-------------------
 internal/pkg/policy/policy_output.go |  4 +--
 internal/pkg/testing/bulk.go         |  7 ++---
 3 files changed, 17 insertions(+), 36 deletions(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index b2fa802c1..3c6cf7d14 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -65,9 +65,7 @@ type Bulk interface {
 	// Accessor used to talk to elastic search direcly bypassing bulk engine
 	Client() *elasticsearch.Client
 
-	GetRemoteClient(name string) *elasticsearch.Client
-	SetRemoteClient(name string, es *elasticsearch.Client)
-
+	// Reusing opts to create bulker for remote ES outputs
 	Opts() BulkOpt
 
 	ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error)
@@ -76,13 +74,12 @@ type Bulk interface {
 const kModBulk = "bulk"
 
 type Bulker struct {
-	es              esapi.Transport
-	remoteEsClients map[string]*elasticsearch.Client
-	ch              chan *bulkT
-	opts            bulkOptT
-	blkPool         sync.Pool
-	apikeyLimit     *semaphore.Weighted
-	tracer          *apm.Tracer
+	es          esapi.Transport
+	ch          chan *bulkT
+	opts        bulkOptT
+	blkPool     sync.Pool
+	apikeyLimit *semaphore.Weighted
+	tracer      *apm.Tracer
 }
 
 const (
@@ -104,13 +101,12 @@ func NewBulker(es esapi.Transport, tracer *apm.Tracer, opts ...BulkOpt) *Bulker
 	}
 
 	return &Bulker{
-		opts:            bopts,
-		es:              es,
-		remoteEsClients: make(map[string]*elasticsearch.Client),
-		ch:              make(chan *bulkT, bopts.blockQueueSz),
-		blkPool:         sync.Pool{New: poolFunc},
-		apikeyLimit:     semaphore.NewWeighted(int64(bopts.apikeyMaxParallel)),
-		tracer:          tracer,
+		opts:        bopts,
+		es:          es,
+		ch:          make(chan *bulkT, bopts.blockQueueSz),
+		blkPool:     sync.Pool{New: poolFunc},
+		apikeyLimit: semaphore.NewWeighted(int64(bopts.apikeyMaxParallel)),
+		tracer:      tracer,
 	}
 }
 
@@ -128,18 +124,6 @@ func (b *Bulker) Opts() BulkOpt {
 	}
 }
 
-func (b *Bulker) GetRemoteClient(name string) *elasticsearch.Client {
-	remoteEs := b.remoteEsClients[name]
-	if remoteEs == nil {
-		return nil
-	}
-	return remoteEs
-}
-
-func (b *Bulker) SetRemoteClient(name string, es *elasticsearch.Client) {
-	b.remoteEsClients[name] = es
-}
-
 // read secrets one by one as there is no bulk API yet to read them in one request
 func (b *Bulker) ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error) {
 	result := make(map[string]string)
diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index e9f12aa09..7fb83c3eb 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -200,7 +200,7 @@ func (p *Output) prepareElasticsearch(
 
 		ctx := zlog.WithContext(ctx)
 		outputAPIKey, err :=
-			generateOutputAPIKey(ctx, outputBulker, agent.Id, p.Name, p.Role.Raw, p.Type)
+			generateOutputAPIKey(ctx, outputBulker, agent.Id, p.Name, p.Role.Raw)
 		if err != nil {
 			return fmt.Errorf("failed generate output API key: %w", err)
 		}
@@ -489,7 +489,7 @@ func generateOutputAPIKey(
 	bulk bulk.Bulk,
 	agentID,
 	outputName string,
-	roles []byte, outputType string) (*apikey.APIKey, error) {
+	roles []byte) (*apikey.APIKey, error) {
 	name := fmt.Sprintf("%s:%s", agentID, outputName)
 	zerolog.Ctx(ctx).Info().Msgf("generating output API key %s for agent ID %s",
 		name, agentID)
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index 7b6272758..0b4a542aa 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -79,12 +79,9 @@ func (m *MockBulk) Client() *elasticsearch.Client {
 	return args.Get(0).(*elasticsearch.Client)
 }
 
-func (m *MockBulk) GetRemoteClient(name string) *elasticsearch.Client {
+func (m *MockBulk) Opts() bulk.BulkOpt {
 	args := m.Called()
-	return args.Get(0).(*elasticsearch.Client)
-}
-
-func (m *MockBulk) SetRemoteClient(name string, es *elasticsearch.Client) {
+	return args.Get(0).(bulk.BulkOpt)
 }
 
 func (m *MockBulk) ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error) {

From ab7487dfb85d8dcdc639df5322defe09f14b938a Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 30 Oct 2023 14:42:39 +0100
Subject: [PATCH 13/91] fix linter

---
 internal/pkg/bulk/engine.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 3c6cf7d14..c88d7b18a 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -120,7 +120,6 @@ func (b *Bulker) Client() *elasticsearch.Client {
 
 func (b *Bulker) Opts() BulkOpt {
 	return func(o *bulkOptT) {
-		o = &b.opts
 	}
 }
 

From e901a80e021aad672e716b14a59f1f9b39099bb0 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 30 Oct 2023 15:07:15 +0100
Subject: [PATCH 14/91] renamed to remote_elasticsearch

---
 internal/pkg/policy/policy_output.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index 7fb83c3eb..e4dd85346 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -28,7 +28,7 @@ import (
 
 const (
 	OutputTypeElasticsearch       = "elasticsearch"
-	OutputTypeRemoteElasticsearch = "remote-elasticsearch"
+	OutputTypeRemoteElasticsearch = "remote_elasticsearch"
 	OutputTypeLogstash            = "logstash"
 	OutputTypeKafka               = "kafka"
 )
@@ -252,7 +252,7 @@ func (p *Output) prepareElasticsearch(
 
 	if p.Type == OutputTypeRemoteElasticsearch {
 
-		// replace type remote-elasticsearch with elasticsearch as agent doesn't recognize remote-elasticsearch
+		// replace type remote_elasticsearch with elasticsearch as agent doesn't recognize remote_elasticsearch
 		outputMap[p.Name][FieldOutputType] = OutputTypeElasticsearch
 		// remove the service token from the agent policy sent to the agent
 		delete(outputMap[p.Name], FieldOutputServiceToken)

From 2465b2a1686e1d65752e0d36b7a5eb2f7447ddd6 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Wed, 1 Nov 2023 11:59:18 +0100
Subject: [PATCH 15/91] added tracer, removed opts

---
 internal/pkg/bulk/engine.go          | 9 ++++-----
 internal/pkg/policy/policy_output.go | 2 +-
 internal/pkg/testing/bulk.go         | 4 ++--
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index c88d7b18a..6fe6b17df 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -65,8 +65,8 @@ type Bulk interface {
 	// Accessor used to talk to elastic search direcly bypassing bulk engine
 	Client() *elasticsearch.Client
 
-	// Reusing opts to create bulker for remote ES outputs
-	Opts() BulkOpt
+	// Reusing tracer to create bulker for remote ES outputs
+	Tracer() *apm.Tracer
 
 	ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error)
 }
@@ -118,9 +118,8 @@ func (b *Bulker) Client() *elasticsearch.Client {
 	return client
 }
 
-func (b *Bulker) Opts() BulkOpt {
-	return func(o *bulkOptT) {
-	}
+func (b *Bulker) Tracer() *apm.Tracer {
+	return b.tracer
 }
 
 // read secrets one by one as there is no bulk API yet to read them in one request
diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index e4dd85346..34e0693fa 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -284,7 +284,7 @@ func (p *Output) createAndGetBulker(ctx context.Context, mainBulker bulk.Bulk, o
 		return nil, err
 	}
 	// starting a new bulker to create/update API keys for remote ES output
-	newBulker := bulk.NewBulker(es, nil, mainBulker.Opts())
+	newBulker := bulk.NewBulker(es, mainBulker.Tracer())
 	bulkerMap[p.Name] = newBulker
 
 	errCh := make(chan error)
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index 0b4a542aa..dcbfcd2e8 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -79,9 +79,9 @@ func (m *MockBulk) Client() *elasticsearch.Client {
 	return args.Get(0).(*elasticsearch.Client)
 }
 
-func (m *MockBulk) Opts() bulk.BulkOpt {
+func (m *MockBulk) Tracer() *apm.Tracer {
 	args := m.Called()
-	return args.Get(0).(bulk.BulkOpt)
+	return args.Get(0).(*apm.Tracer)
 }
 
 func (m *MockBulk) ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error) {

From 1fd90a880be3cc40704241e00fc097baccf1c28a Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Wed, 1 Nov 2023 14:45:17 +0100
Subject: [PATCH 16/91] enable intrumentation on remote es client

---
 internal/pkg/policy/policy_output.go | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index 34e0693fa..7815bfeff 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -16,6 +16,7 @@ import (
 	"go.elastic.co/apm/v2"
 
 	"github.com/elastic/fleet-server/v7/internal/pkg/apikey"
+	"github.com/elastic/fleet-server/v7/internal/pkg/build"
 	"github.com/elastic/fleet-server/v7/internal/pkg/bulk"
 	"github.com/elastic/fleet-server/v7/internal/pkg/config"
 	"github.com/elastic/fleet-server/v7/internal/pkg/dl"
@@ -329,13 +330,23 @@ func (p *Output) createRemoteEsClient(ctx context.Context, outputMap map[string]
 			},
 		},
 	}
-	es, err := es.NewClient(ctx, &cfg, false)
+	es, err := es.NewClient(ctx, &cfg, false, elasticsearchOptions(
+		true, build.Info{},
+	)...)
 	if err != nil {
 		return nil, err
 	}
 	return es, nil
 }
 
+func elasticsearchOptions(instumented bool, bi build.Info) []es.ConfigOption {
+	options := []es.ConfigOption{es.WithUserAgent("Remote-Fleet-Server", bi)}
+	if instumented {
+		options = append(options, es.InstrumentRoundTripper())
+	}
+	return options
+}
+
 func fetchAPIKeyRoles(ctx context.Context, b bulk.Bulk, apiKeyID string) (*RoleT, error) {
 	res, err := b.APIKeyRead(ctx, apiKeyID, true)
 	if err != nil {

From 7953456e90c3d69f2c1cef246bd50f67991df143 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Wed, 1 Nov 2023 15:01:25 +0100
Subject: [PATCH 17/91] updated changelog

---
 changelog/fragments/1698660041-remote-es-output.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/changelog/fragments/1698660041-remote-es-output.yaml b/changelog/fragments/1698660041-remote-es-output.yaml
index 72c9abc81..d8be9e1dd 100644
--- a/changelog/fragments/1698660041-remote-es-output.yaml
+++ b/changelog/fragments/1698660041-remote-es-output.yaml
@@ -25,7 +25,7 @@ component:
 # If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
 # NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
 # Please provide it if you are adding a fragment for a different PR.
-#pr: https://github.com/owner/repo/1234
+pr: 3051
 
 # Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
 # If not present is automatically filled by the tooling with the issue linked to the PR number.

From 11bc90ccba7180958ad4da2c77aac455c1170659 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 2 Nov 2023 09:36:22 +0100
Subject: [PATCH 18/91] try copy policyData

---
 internal/pkg/server/fleet_integration_test.go | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/internal/pkg/server/fleet_integration_test.go b/internal/pkg/server/fleet_integration_test.go
index b154b7e03..b2705b452 100644
--- a/internal/pkg/server/fleet_integration_test.go
+++ b/internal/pkg/server/fleet_integration_test.go
@@ -73,6 +73,18 @@ type tserver struct {
 	bulker    bulk.Bulk
 }
 
+var policyData = model.PolicyData{
+	Outputs: map[string]map[string]interface{}{
+		"default": {
+			"type": "elasticsearch",
+		},
+	},
+	OutputPermissions: json.RawMessage(`{"default": {}}`),
+	Inputs: []map[string]interface{}{{
+		"type": "fleet-server",
+	}},
+}
+
 func (s *tserver) baseURL() string {
 	input, _ := s.cfg.GetFleetInput()
 	tls := input.Server.TLS

From 8312727bd0cf1244344d04b59acbac80d3728c98 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 2 Nov 2023 09:58:21 +0100
Subject: [PATCH 19/91] revert policyData, printing out value of policy

---
 internal/pkg/policy/parsed_policy.go          |  1 +
 internal/pkg/server/fleet_integration_test.go | 12 ------------
 2 files changed, 1 insertion(+), 12 deletions(-)

diff --git a/internal/pkg/policy/parsed_policy.go b/internal/pkg/policy/parsed_policy.go
index 6a56d67e1..be6977342 100644
--- a/internal/pkg/policy/parsed_policy.go
+++ b/internal/pkg/policy/parsed_policy.go
@@ -53,6 +53,7 @@ type ParsedPolicy struct {
 }
 
 func NewParsedPolicy(ctx context.Context, bulker bulk.Bulk, p model.Policy) (*ParsedPolicy, error) {
+	fmt.Printf("%+v", p)
 	var err error
 	// Interpret the output permissions if available
 	var roles map[string]RoleT
diff --git a/internal/pkg/server/fleet_integration_test.go b/internal/pkg/server/fleet_integration_test.go
index b2705b452..b154b7e03 100644
--- a/internal/pkg/server/fleet_integration_test.go
+++ b/internal/pkg/server/fleet_integration_test.go
@@ -73,18 +73,6 @@ type tserver struct {
 	bulker    bulk.Bulk
 }
 
-var policyData = model.PolicyData{
-	Outputs: map[string]map[string]interface{}{
-		"default": {
-			"type": "elasticsearch",
-		},
-	},
-	OutputPermissions: json.RawMessage(`{"default": {}}`),
-	Inputs: []map[string]interface{}{{
-		"type": "fleet-server",
-	}},
-}
-
 func (s *tserver) baseURL() string {
 	input, _ := s.cfg.GetFleetInput()
 	tls := input.Server.TLS

From 8cae7a4ba37efa41b39b48b64d6790119235115d Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 2 Nov 2023 10:28:26 +0100
Subject: [PATCH 20/91] setting policyData in other integration tests

---
 internal/pkg/dl/migration_integration_test.go | 12 +++++++++++-
 internal/pkg/dl/policies_integration_test.go  | 12 +++++++++++-
 internal/pkg/policy/parsed_policy.go          |  1 -
 3 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/internal/pkg/dl/migration_integration_test.go b/internal/pkg/dl/migration_integration_test.go
index fdbfd8a7e..be63bcb84 100644
--- a/internal/pkg/dl/migration_integration_test.go
+++ b/internal/pkg/dl/migration_integration_test.go
@@ -75,6 +75,16 @@ func createSomePolicies(t *testing.T, n int, index string, bulker bulk.Bulk) []s
 
 	var created []string
 
+	var policyData = model.PolicyData{
+		Outputs: map[string]map[string]interface{}{
+			"default": {
+				"type": "elasticsearch",
+			},
+		},
+		OutputPermissions: json.RawMessage(`{"default": {}}`),
+		Inputs:            []map[string]interface{}{},
+	}
+
 	for i := 0; i < n; i++ {
 		now := time.Now().UTC()
 		nowStr := now.Format(time.RFC3339)
@@ -82,7 +92,7 @@ func createSomePolicies(t *testing.T, n int, index string, bulker bulk.Bulk) []s
 		policyModel := model.Policy{
 			ESDocument:         model.ESDocument{},
 			CoordinatorIdx:     int64(i),
-			Data:               nil,
+			Data:               &policyData,
 			DefaultFleetServer: false,
 			PolicyID:           fmt.Sprint(i),
 			RevisionIdx:        1,
diff --git a/internal/pkg/dl/policies_integration_test.go b/internal/pkg/dl/policies_integration_test.go
index 18cdf4011..f35de0065 100644
--- a/internal/pkg/dl/policies_integration_test.go
+++ b/internal/pkg/dl/policies_integration_test.go
@@ -8,6 +8,7 @@ package dl
 
 import (
 	"context"
+	"encoding/json"
 	"testing"
 	"time"
 
@@ -21,11 +22,20 @@ import (
 
 func createRandomPolicy(id string, revisionIdx int) model.Policy {
 	now := time.Now().UTC()
+	var policyData = model.PolicyData{
+		Outputs: map[string]map[string]interface{}{
+			"default": {
+				"type": "elasticsearch",
+			},
+		},
+		OutputPermissions: json.RawMessage(`{"default": {}}`),
+		Inputs:            []map[string]interface{}{},
+	}
 	return model.Policy{
 		PolicyID:           id,
 		RevisionIdx:        int64(revisionIdx),
 		CoordinatorIdx:     0,
-		Data:               nil,
+		Data:               &policyData,
 		DefaultFleetServer: false,
 		Timestamp:          now.Format(time.RFC3339),
 	}
diff --git a/internal/pkg/policy/parsed_policy.go b/internal/pkg/policy/parsed_policy.go
index be6977342..6a56d67e1 100644
--- a/internal/pkg/policy/parsed_policy.go
+++ b/internal/pkg/policy/parsed_policy.go
@@ -53,7 +53,6 @@ type ParsedPolicy struct {
 }
 
 func NewParsedPolicy(ctx context.Context, bulker bulk.Bulk, p model.Policy) (*ParsedPolicy, error) {
-	fmt.Printf("%+v", p)
 	var err error
 	// Interpret the output permissions if available
 	var roles map[string]RoleT

From bff9015d73beb519e2572c9e902a26ab4cf00e58 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 2 Nov 2023 10:47:09 +0100
Subject: [PATCH 21/91] added missing param

---
 internal/pkg/policy/policy_output_integration_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/pkg/policy/policy_output_integration_test.go b/internal/pkg/policy/policy_output_integration_test.go
index e1328cc8e..652a3a20e 100644
--- a/internal/pkg/policy/policy_output_integration_test.go
+++ b/internal/pkg/policy/policy_output_integration_test.go
@@ -152,7 +152,7 @@ func TestPolicyOutputESPrepareRealES(t *testing.T) {
 	}
 
 	err = output.prepareElasticsearch(
-		context.Background(), zerolog.Nop(), bulker, &agent, policyMap)
+		context.Background(), zerolog.Nop(), bulker, bulker, &agent, policyMap)
 	require.NoError(t, err)
 
 	// need to wait a bit before querying the agent again

From a52b32be3d15c8231bf9f92f917feaeb23344d6e Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 2 Nov 2023 14:10:59 +0100
Subject: [PATCH 22/91] watching for remote output cfg changes and restart

---
 internal/pkg/bulk/engine.go          | 48 +++++++++++++++++++++-------
 internal/pkg/policy/policy_output.go |  2 +-
 internal/pkg/server/fleet.go         | 15 ++++++++-
 internal/pkg/testing/bulk.go         |  8 +++++
 4 files changed, 59 insertions(+), 14 deletions(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 6fe6b17df..3aa5cf71d 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"reflect"
 	"strings"
 	"sync"
 	"time"
@@ -68,18 +69,24 @@ type Bulk interface {
 	// Reusing tracer to create bulker for remote ES outputs
 	Tracer() *apm.Tracer
 
+	CheckRemoteOutputChanged(name string, newCfg map[string]interface{})
+
+	RemoteOutputCh() chan bool
+
 	ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error)
 }
 
 const kModBulk = "bulk"
 
 type Bulker struct {
-	es          esapi.Transport
-	ch          chan *bulkT
-	opts        bulkOptT
-	blkPool     sync.Pool
-	apikeyLimit *semaphore.Weighted
-	tracer      *apm.Tracer
+	es                    esapi.Transport
+	ch                    chan *bulkT
+	opts                  bulkOptT
+	blkPool               sync.Pool
+	apikeyLimit           *semaphore.Weighted
+	tracer                *apm.Tracer
+	remoteOutputConfigMap map[string]map[string]interface{}
+	remoteOutputCh        chan bool
 }
 
 const (
@@ -101,12 +108,14 @@ func NewBulker(es esapi.Transport, tracer *apm.Tracer, opts ...BulkOpt) *Bulker
 	}
 
 	return &Bulker{
-		opts:        bopts,
-		es:          es,
-		ch:          make(chan *bulkT, bopts.blockQueueSz),
-		blkPool:     sync.Pool{New: poolFunc},
-		apikeyLimit: semaphore.NewWeighted(int64(bopts.apikeyMaxParallel)),
-		tracer:      tracer,
+		opts:                  bopts,
+		es:                    es,
+		ch:                    make(chan *bulkT, bopts.blockQueueSz),
+		blkPool:               sync.Pool{New: poolFunc},
+		apikeyLimit:           semaphore.NewWeighted(int64(bopts.apikeyMaxParallel)),
+		tracer:                tracer,
+		remoteOutputConfigMap: make(map[string]map[string]interface{}),
+		remoteOutputCh:        make(chan bool, 1),
 	}
 }
 
@@ -122,6 +131,21 @@ func (b *Bulker) Tracer() *apm.Tracer {
 	return b.tracer
 }
 
+// check if remote output cfg changed, and signal to remoteOutputCh channel if so
+func (b *Bulker) CheckRemoteOutputChanged(name string, newCfg map[string]interface{}) {
+	curCfg := b.remoteOutputConfigMap[name]
+
+	if !reflect.DeepEqual(curCfg, newCfg) {
+		log.Info().Str("name", name).Msg("remote output configuration has changed")
+		b.remoteOutputCh <- true
+	}
+	b.remoteOutputConfigMap[name] = curCfg
+}
+
+func (b *Bulker) RemoteOutputCh() chan bool {
+	return b.remoteOutputCh
+}
+
 // read secrets one by one as there is no bulk API yet to read them in one request
 func (b *Bulker) ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error) {
 	result := make(map[string]string)
diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index 7815bfeff..d66257d41 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -275,7 +275,7 @@ func (p *Output) prepareElasticsearch(
 func (p *Output) createAndGetBulker(ctx context.Context, mainBulker bulk.Bulk, outputMap map[string]map[string]interface{}) (bulk.Bulk, error) {
 	bulker := bulkerMap[p.Name]
 	if bulker != nil {
-		// TODO replace if hosts or service token changed?
+		mainBulker.CheckRemoteOutputChanged(p.Name, outputMap[p.Name])
 		return bulker, nil
 	}
 	bulkCtx, bulkCancel := context.WithCancel(context.Background())
diff --git a/internal/pkg/server/fleet.go b/internal/pkg/server/fleet.go
index fa489d8ee..8a4af86b6 100644
--- a/internal/pkg/server/fleet.go
+++ b/internal/pkg/server/fleet.go
@@ -53,6 +53,7 @@ type Fleet struct {
 	cfgCh    chan *config.Config
 	cache    cache.Cache
 	reporter state.Reporter
+	outputCh chan bool
 }
 
 // NewFleet creates the actual fleet server service.
@@ -68,6 +69,7 @@ func NewFleet(bi build.Info, reporter state.Reporter, standAlone bool) (*Fleet,
 		verCon:     verCon,
 		cfgCh:      make(chan *config.Config, 1),
 		reporter:   reporter,
+		outputCh:   make(chan bool, 1),
 	}, nil
 }
 
@@ -132,6 +134,8 @@ func (f *Fleet) Run(ctx context.Context, initCfg *config.Config) error {
 	started := false
 	ech := make(chan error, 2)
 
+	outputChanged := false
+
 LOOP:
 	for {
 		if started {
@@ -173,7 +177,7 @@ LOOP:
 		}
 
 		// Start or restart server
-		if configChangedServer(curCfg, newCfg) {
+		if configChangedServer(curCfg, newCfg) || outputChanged {
 			if srvCancel != nil {
 				log.Info().Msg("stopping server on configuration change")
 				stop(srvCancel, srvEg)
@@ -191,10 +195,13 @@ LOOP:
 		}
 
 		curCfg = newCfg
+		outputChanged = false
 
 		select {
 		case newCfg = <-f.cfgCh:
 			log.Info().Msg("Server configuration update")
+		case outputChanged = <-f.outputCh:
+			log.Info().Msg("Remote output configuration update")
 		case err := <-ech:
 			f.reporter.UpdateState(client.UnitStateFailed, fmt.Sprintf("Error - %s", err), nil) //nolint:errcheck // unclear on what should we do if updating the status fails?
 			log.Error().Err(err).Msg("Fleet Server failed")
@@ -414,6 +421,12 @@ func (f *Fleet) runServer(ctx context.Context, cfg *config.Config) (err error) {
 		return err
 	}
 
+	select {
+	case outputChanged := <-bulker.RemoteOutputCh():
+		f.outputCh <- outputChanged
+		log.Info().Msg("Remote output configuration update")
+	}
+
 	return g.Wait()
 }
 
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index dcbfcd2e8..a9197ebe8 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -84,6 +84,14 @@ func (m *MockBulk) Tracer() *apm.Tracer {
 	return args.Get(0).(*apm.Tracer)
 }
 
+func (m *MockBulk) CheckRemoteOutputChanged(name string, newCfg map[string]interface{}) {
+}
+
+func (m *MockBulk) RemoteOutputCh() chan bool {
+	args := m.Called()
+	return args.Get(0).(chan bool)
+}
+
 func (m *MockBulk) ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error) {
 	result := make(map[string]string)
 	for _, id := range secretIds {

From 41f4ced5e635fbcb8b1d8cd6e4e451a5b8f97553 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 2 Nov 2023 15:42:56 +0100
Subject: [PATCH 23/91] fixes

---
 internal/pkg/bulk/engine.go          | 11 +++++++++--
 internal/pkg/policy/policy_output.go |  8 ++++----
 internal/pkg/server/fleet.go         |  6 +++---
 3 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 3aa5cf71d..c6f7e80fb 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -135,11 +135,18 @@ func (b *Bulker) Tracer() *apm.Tracer {
 func (b *Bulker) CheckRemoteOutputChanged(name string, newCfg map[string]interface{}) {
 	curCfg := b.remoteOutputConfigMap[name]
 
+	// ignore output sent to agents where type is set to elasticsearch
+	if newCfg["type"] == "elasticsearch" {
+		return
+	}
+	b.remoteOutputConfigMap[name] = newCfg
+	if curCfg == nil {
+		return
+	}
 	if !reflect.DeepEqual(curCfg, newCfg) {
-		log.Info().Str("name", name).Msg("remote output configuration has changed")
+		log.Info().Str("name", name).Any("curCfg", curCfg).Any("newCfg", newCfg).Msg("remote output configuration has changed")
 		b.remoteOutputCh <- true
 	}
-	b.remoteOutputConfigMap[name] = curCfg
 }
 
 func (b *Bulker) RemoteOutputCh() chan bool {
diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index d66257d41..35bf009a4 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -66,7 +66,7 @@ func (p *Output) Prepare(ctx context.Context, zlog zerolog.Logger, bulker bulk.B
 		}
 	case OutputTypeRemoteElasticsearch:
 		zlog.Debug().Msg("preparing remote elasticsearch output")
-		newBulker, err := p.createAndGetBulker(ctx, bulker, outputMap)
+		newBulker, err := p.createAndGetBulker(bulker, outputMap)
 		if err != nil {
 			return err
 		}
@@ -272,10 +272,10 @@ func (p *Output) prepareElasticsearch(
 	return nil
 }
 
-func (p *Output) createAndGetBulker(ctx context.Context, mainBulker bulk.Bulk, outputMap map[string]map[string]interface{}) (bulk.Bulk, error) {
+func (p *Output) createAndGetBulker(mainBulker bulk.Bulk, outputMap map[string]map[string]interface{}) (bulk.Bulk, error) {
+	mainBulker.CheckRemoteOutputChanged(p.Name, outputMap[p.Name])
 	bulker := bulkerMap[p.Name]
 	if bulker != nil {
-		mainBulker.CheckRemoteOutputChanged(p.Name, outputMap[p.Name])
 		return bulker, nil
 	}
 	bulkCtx, bulkCancel := context.WithCancel(context.Background())
@@ -296,7 +296,7 @@ func (p *Output) createAndGetBulker(ctx context.Context, mainBulker bulk.Bulk, o
 
 		errCh <- runFunc()
 	}()
-	go func() (err error) {
+	go func() {
 		select {
 		case err = <-errCh:
 		case <-bulkCtx.Done():
diff --git a/internal/pkg/server/fleet.go b/internal/pkg/server/fleet.go
index 8a4af86b6..dc13af93d 100644
--- a/internal/pkg/server/fleet.go
+++ b/internal/pkg/server/fleet.go
@@ -421,11 +421,11 @@ func (f *Fleet) runServer(ctx context.Context, cfg *config.Config) (err error) {
 		return err
 	}
 
-	select {
-	case outputChanged := <-bulker.RemoteOutputCh():
+	go func() {
+		outputChanged := <-bulker.RemoteOutputCh()
 		f.outputCh <- outputChanged
 		log.Info().Msg("Remote output configuration update")
-	}
+	}()
 
 	return g.Wait()
 }

From 649bfdb4eb9669bb075d3bf8325b219ee67b8017 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 2 Nov 2023 15:50:00 +0100
Subject: [PATCH 24/91] fix linter

---
 internal/pkg/policy/policy_output.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index 35bf009a4..140d62978 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -302,7 +302,6 @@ func (p *Output) createAndGetBulker(mainBulker bulk.Bulk, outputMap map[string]m
 		case <-bulkCtx.Done():
 			err = bulkCtx.Err()
 		}
-		return
 	}()
 
 	return newBulker, nil

From f6994bdcdf940793fb39d55bf634bae109a6d919 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 2 Nov 2023 15:59:42 +0100
Subject: [PATCH 25/91] fix

---
 internal/pkg/bulk/engine.go | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index c6f7e80fb..0e10d4756 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -139,14 +139,11 @@ func (b *Bulker) CheckRemoteOutputChanged(name string, newCfg map[string]interfa
 	if newCfg["type"] == "elasticsearch" {
 		return
 	}
-	b.remoteOutputConfigMap[name] = newCfg
-	if curCfg == nil {
-		return
-	}
-	if !reflect.DeepEqual(curCfg, newCfg) {
-		log.Info().Str("name", name).Any("curCfg", curCfg).Any("newCfg", newCfg).Msg("remote output configuration has changed")
+	if curCfg != nil && !reflect.DeepEqual(curCfg, newCfg) {
+		log.Info().Str("name", name).Msg("remote output configuration has changed")
 		b.remoteOutputCh <- true
 	}
+	b.remoteOutputConfigMap[name] = newCfg
 }
 
 func (b *Bulker) RemoteOutputCh() chan bool {

From 84787bd904785a6eba5efa85902723d80ba5bed5 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Fri, 3 Nov 2023 09:33:52 +0100
Subject: [PATCH 26/91] copy map instead of reference

---
 internal/pkg/bulk/engine.go | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 0e10d4756..9fb870138 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -136,14 +136,15 @@ func (b *Bulker) CheckRemoteOutputChanged(name string, newCfg map[string]interfa
 	curCfg := b.remoteOutputConfigMap[name]
 
 	// ignore output sent to agents where type is set to elasticsearch
-	if newCfg["type"] == "elasticsearch" {
-		return
-	}
 	if curCfg != nil && !reflect.DeepEqual(curCfg, newCfg) {
 		log.Info().Str("name", name).Msg("remote output configuration has changed")
 		b.remoteOutputCh <- true
 	}
-	b.remoteOutputConfigMap[name] = newCfg
+	newCfgCopy := make(map[string]interface{})
+	for k, v := range newCfg {
+		newCfgCopy[k] = v
+	}
+	b.remoteOutputConfigMap[name] = newCfgCopy
 }
 
 func (b *Bulker) RemoteOutputCh() chan bool {

From d36f83a49d6831a7f083b073ff7e90e2fcab8776 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Fri, 3 Nov 2023 14:20:10 +0100
Subject: [PATCH 27/91] added integration test

---
 internal/pkg/server/fleet_integration_test.go |  39 +++--
 .../server/fleet_secrets_integration_test.go  |   2 +-
 .../remote_es_output_integration_test.go      | 161 ++++++++++++++++++
 3 files changed, 185 insertions(+), 17 deletions(-)
 create mode 100644 internal/pkg/server/remote_es_output_integration_test.go

diff --git a/internal/pkg/server/fleet_integration_test.go b/internal/pkg/server/fleet_integration_test.go
index b154b7e03..b465cd354 100644
--- a/internal/pkg/server/fleet_integration_test.go
+++ b/internal/pkg/server/fleet_integration_test.go
@@ -103,7 +103,7 @@ func WithAPM(url string, enabled bool) Option {
 	}
 }
 
-func startTestServer(t *testing.T, ctx context.Context, opts ...Option) (*tserver, error) {
+func startTestServer(t *testing.T, ctx context.Context, policyD model.PolicyData, opts ...Option) (*tserver, error) {
 	t.Helper()
 
 	cfg, err := config.LoadFile("../testing/fleet-server-testing.yml")
@@ -120,7 +120,7 @@ func startTestServer(t *testing.T, ctx context.Context, opts ...Option) (*tserve
 		PolicyID:           policyID,
 		RevisionIdx:        1,
 		DefaultFleetServer: true,
-		Data:               &policyData,
+		Data:               &policyD,
 	})
 	if err != nil {
 		return nil, err
@@ -392,7 +392,7 @@ func TestServerUnauthorized(t *testing.T) {
 	defer cancel()
 
 	// Start test server
-	srv, err := startTestServer(t, ctx)
+	srv, err := startTestServer(t, ctx, policyData)
 	require.NoError(t, err)
 
 	agentID := uuid.Must(uuid.NewV4()).String()
@@ -502,7 +502,7 @@ func TestServerInstrumentation(t *testing.T) {
 	defer server.Close()
 
 	// Start test server with instrumentation disabled
-	srv, err := startTestServer(t, ctx, WithAPM(server.URL, false))
+	srv, err := startTestServer(t, ctx, policyData, WithAPM(server.URL, false))
 	require.NoError(t, err)
 
 	agentID := "1e4954ce-af37-4731-9f4a-407b08e69e42"
@@ -586,7 +586,7 @@ func Test_SmokeTest_Agent_Calls(t *testing.T) {
 	defer cancel()
 
 	// Start test server
-	srv, err := startTestServer(t, ctx)
+	srv, err := startTestServer(t, ctx, policyData)
 	require.NoError(t, err)
 
 	cli := cleanhttp.DefaultClient()
@@ -696,7 +696,7 @@ func Test_SmokeTest_Agent_Calls(t *testing.T) {
 	require.Falsef(t, ok, "expected response to have no errors attribute, errors are present: %+v", ackObj)
 }
 
-func EnrollAgent(enrollBody string, t *testing.T, ctx context.Context, srv *tserver) string {
+func EnrollAgent(enrollBody string, t *testing.T, ctx context.Context, srv *tserver) (string, string) {
 	req, err := http.NewRequestWithContext(ctx, "POST", srv.baseURL()+"/api/fleet/agents/enroll", strings.NewReader(enrollBody))
 	require.NoError(t, err)
 	req.Header.Set("Authorization", "ApiKey "+srv.enrollKey)
@@ -721,7 +721,14 @@ func EnrollAgent(enrollBody string, t *testing.T, ctx context.Context, srv *tser
 	mm, ok := item.(map[string]interface{})
 	require.True(t, ok, "expected attribute item to be an object")
 	agentID := mm["id"]
-	return agentID.(string)
+
+	apiKey, ok := mm["access_api_key"]
+	require.True(t, ok, "expected attribute access_api_key is missing")
+	key, ok := apiKey.(string)
+	require.True(t, ok, "expected attribute access_api_key to be a string")
+	require.NotEmpty(t, key)
+
+	return agentID.(string), key
 }
 
 func Test_Agent_Enrollment_Id(t *testing.T) {
@@ -739,14 +746,14 @@ func Test_Agent_Enrollment_Id(t *testing.T) {
 	defer cancel()
 
 	// Start test server
-	srv, err := startTestServer(t, ctx)
+	srv, err := startTestServer(t, ctx, policyData)
 	require.NoError(t, err)
 
 	t.Log("Enroll the first agent with enrollment_id")
-	firstAgentID := EnrollAgent(enrollBodyWEnrollmentID, t, ctx, srv)
+	firstAgentID, _ := EnrollAgent(enrollBodyWEnrollmentID, t, ctx, srv)
 
 	t.Log("Enroll the second agent with the same enrollment_id")
-	secondAgentID := EnrollAgent(enrollBodyWEnrollmentID, t, ctx, srv)
+	secondAgentID, _ := EnrollAgent(enrollBodyWEnrollmentID, t, ctx, srv)
 
 	// cleanup
 	defer func() {
@@ -785,11 +792,11 @@ func Test_Agent_Enrollment_Id_Invalidated_API_key(t *testing.T) {
 	defer cancel()
 
 	// Start test server
-	srv, err := startTestServer(t, ctx)
+	srv, err := startTestServer(t, ctx, policyData)
 	require.NoError(t, err)
 
 	t.Log("Enroll the first agent with enrollment_id")
-	firstAgentID := EnrollAgent(enrollBodyWEnrollmentID, t, ctx, srv)
+	firstAgentID, _ := EnrollAgent(enrollBodyWEnrollmentID, t, ctx, srv)
 
 	agent, err := dl.FindAgent(ctx, srv.bulker, dl.QueryAgentByID, dl.FieldID, firstAgentID)
 	if err != nil {
@@ -804,7 +811,7 @@ func Test_Agent_Enrollment_Id_Invalidated_API_key(t *testing.T) {
 	}
 
 	t.Log("Enroll the second agent with the same enrollment_id")
-	secondAgentID := EnrollAgent(enrollBodyWEnrollmentID, t, ctx, srv)
+	secondAgentID, _ := EnrollAgent(enrollBodyWEnrollmentID, t, ctx, srv)
 
 	// cleanup
 	defer func() {
@@ -833,7 +840,7 @@ func Test_Agent_Auth_errors(t *testing.T) {
 	defer cancel()
 
 	// Start test server
-	srv, err := startTestServer(t, ctx)
+	srv, err := startTestServer(t, ctx, policyData)
 	require.NoError(t, err)
 
 	cli := cleanhttp.DefaultClient()
@@ -953,7 +960,7 @@ func Test_Agent_request_errors(t *testing.T) {
 	defer cancel()
 
 	// Start test server
-	srv, err := startTestServer(t, ctx)
+	srv, err := startTestServer(t, ctx, policyData)
 	require.NoError(t, err)
 
 	cli := cleanhttp.DefaultClient()
@@ -1028,7 +1035,7 @@ func Test_SmokeTest_CheckinPollTimeout(t *testing.T) {
 	defer cancel()
 
 	// Start test server
-	srv, err := startTestServer(t, ctx)
+	srv, err := startTestServer(t, ctx, policyData)
 	require.NoError(t, err)
 
 	cli := cleanhttp.DefaultClient()
diff --git a/internal/pkg/server/fleet_secrets_integration_test.go b/internal/pkg/server/fleet_secrets_integration_test.go
index 1fe27960c..8a40c4131 100644
--- a/internal/pkg/server/fleet_secrets_integration_test.go
+++ b/internal/pkg/server/fleet_secrets_integration_test.go
@@ -156,7 +156,7 @@ func Test_Agent_Policy_Secrets(t *testing.T) {
 	defer cancel()
 
 	// Start test server
-	srv, err := startTestServer(t, ctx)
+	srv, err := startTestServer(t, ctx, policyData)
 	require.NoError(t, err)
 
 	// create secret with kibana_system user
diff --git a/internal/pkg/server/remote_es_output_integration_test.go b/internal/pkg/server/remote_es_output_integration_test.go
new file mode 100644
index 000000000..dd9c22734
--- /dev/null
+++ b/internal/pkg/server/remote_es_output_integration_test.go
@@ -0,0 +1,161 @@
+//go:build integration
+
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/elastic/fleet-server/v7/internal/pkg/apikey"
+	"github.com/elastic/fleet-server/v7/internal/pkg/dl"
+	"github.com/elastic/fleet-server/v7/internal/pkg/model"
+	"github.com/hashicorp/go-cleanhttp"
+	"github.com/stretchr/testify/require"
+)
+
+func Checkin(t *testing.T, ctx context.Context, srv *tserver, agentID, key string) {
+	str := agentID
+	cli := cleanhttp.DefaultClient()
+	var obj map[string]interface{}
+
+	t.Logf("Fake a checkin for agent %s", str)
+	req, err := http.NewRequestWithContext(ctx, "POST", srv.baseURL()+"/api/fleet/agents/"+str+"/checkin", strings.NewReader(checkinBody))
+	require.NoError(t, err)
+	req.Header.Set("Authorization", "ApiKey "+key)
+	req.Header.Set("User-Agent", "elastic agent "+serverVersion)
+	req.Header.Set("Content-Type", "application/json")
+	res, err := cli.Do(req)
+	require.NoError(t, err)
+
+	require.Equal(t, http.StatusOK, res.StatusCode)
+	t.Log("Checkin successful, verify body")
+	p, _ := io.ReadAll(res.Body)
+	res.Body.Close()
+	err = json.Unmarshal(p, &obj)
+	require.NoError(t, err)
+
+	actionsRaw, ok := obj["actions"]
+	require.True(t, ok, "expected actions is missing")
+	actions, ok := actionsRaw.([]interface{})
+	require.True(t, ok, "expected actions to be an array")
+	require.Greater(t, len(actions), 0, "expected at least 1 action")
+	action, ok := actions[0].(map[string]interface{})
+	require.True(t, ok, "expected action to be an object")
+	typeRaw, _ := action["type"]
+	require.Equal(t, "POLICY_CHANGE", typeRaw)
+	dataRaw, _ := action["data"]
+	data, _ := dataRaw.(map[string]interface{})
+	policy, _ := data["policy"].(map[string]interface{})
+	outputs, _ := policy["outputs"].(map[string]interface{})
+	remoteES, _ := outputs["remoteES"].(map[string]interface{})
+	oType, _ := remoteES["type"].(string)
+	require.Equal(t, "elasticsearch", oType)
+	serviceToken, _ := remoteES["service_token"]
+	require.Equal(t, nil, serviceToken)
+	remoteAPIKey, _ := remoteES["api_key"]
+	defaultOutput, _ := outputs["default"].(map[string]interface{})
+	defaultAPIKey, _ := defaultOutput["api_key"]
+	require.False(t, remoteAPIKey == defaultAPIKey, "expected remote api key to be different than default")
+
+}
+
+func Test_Agent_Remote_ES_Output(t *testing.T) {
+	enrollBody := `{
+	    "type": "PERMANENT",
+	    "shared_id": "",
+	    "enrollment_id": "",
+	    "metadata": {
+		"user_provided": {},
+		"local": {},
+		"tags": []
+	    }
+	}`
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Start test server
+	srv, err := startTestServer(t, ctx, policyData)
+	require.NoError(t, err)
+
+	t.Log("Create policy with remote ES output")
+
+	var policyRemoteID = "policyRemoteID"
+	var policyDataRemoteES = model.PolicyData{
+		Outputs: map[string]map[string]interface{}{
+			"default": {
+				"type": "elasticsearch",
+			},
+			"remoteES": {
+				"type": "remote_elasticsearch",
+				// TODO start another fleet-server
+				"hosts": []string{"localhost:9200"},
+				// TODO create remote service token - superuser? manage_service_account
+				"service_token": srv.cfg.Output.Elasticsearch.ServiceToken,
+			},
+		},
+		OutputPermissions: json.RawMessage(`{"default": {}, "remoteES": {}}`),
+		Inputs:            []map[string]interface{}{},
+	}
+
+	_, err = dl.CreatePolicy(ctx, srv.bulker, model.Policy{
+		PolicyID:           policyRemoteID,
+		RevisionIdx:        1,
+		DefaultFleetServer: false,
+		Data:               &policyDataRemoteES,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Log("Create API key and enrollment key for new policy")
+
+	newKey, err := apikey.Create(ctx, srv.bulker.Client(), "default", "", "true", []byte(`{
+	    "fleet-apikey-enroll": {
+		"cluster": [],
+		"index": [],
+		"applications": [{
+		    "application": "fleet",
+		    "privileges": ["no-privileges"],
+		    "resources": ["*"]
+		}]
+	    }
+	}`), map[string]interface{}{
+		"managed_by": "fleet",
+		"managed":    true,
+		"type":       "enroll",
+		"policy_id":  policyRemoteID,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = dl.CreateEnrollmentAPIKey(ctx, srv.bulker, model.EnrollmentAPIKey{
+		Name:     "RemoteES",
+		APIKey:   newKey.Key,
+		APIKeyID: newKey.ID,
+		PolicyID: policyRemoteID,
+		Active:   true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Log("Enroll agent")
+	srvCopy := srv
+	srvCopy.enrollKey = newKey.Token()
+	agentID, key := EnrollAgent(enrollBody, t, ctx, srvCopy)
+
+	// cleanup
+	defer func() {
+		err2 := srv.bulker.Delete(ctx, dl.FleetAgents, agentID)
+		if err2 != nil {
+			t.Log("could not clean up agent")
+		}
+	}()
+
+	Checkin(t, ctx, srvCopy, agentID, key)
+}

From 255c28b1b5d7423a232a8d0b18c932226db97861 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Fri, 3 Nov 2023 15:26:46 +0100
Subject: [PATCH 28/91] unit test on CheckRemoteOutputChanged

---
 .../pkg/bulk/remote_output_changed_test.go    | 98 +++++++++++++++++++
 .../remote_es_output_integration_test.go      |  4 +
 2 files changed, 102 insertions(+)
 create mode 100644 internal/pkg/bulk/remote_output_changed_test.go

diff --git a/internal/pkg/bulk/remote_output_changed_test.go b/internal/pkg/bulk/remote_output_changed_test.go
new file mode 100644
index 000000000..f71205c94
--- /dev/null
+++ b/internal/pkg/bulk/remote_output_changed_test.go
@@ -0,0 +1,98 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+//go:build !integration
+
+package bulk
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_CheckRemoteOutputChanged(t *testing.T) {
+	testcases := []struct {
+		name    string
+		cfg     map[string]interface{}
+		newCfg  map[string]interface{}
+		changed bool
+	}{
+		{
+			name: "initial nil",
+			cfg:  nil,
+			newCfg: map[string]interface{}{
+				"type":          "remote_elasticsearch",
+				"hosts":         []string{"https://remote-es:443"},
+				"service_token": "token1",
+			},
+			changed: false,
+		},
+		{
+			name: "no changes",
+			cfg: map[string]interface{}{
+				"type":          "remote_elasticsearch",
+				"hosts":         []string{"https://remote-es:443"},
+				"service_token": "token1",
+			},
+			newCfg: map[string]interface{}{
+				"type":          "remote_elasticsearch",
+				"hosts":         []string{"https://remote-es:443"},
+				"service_token": "token1",
+			},
+			changed: false,
+		},
+		{
+			name: "change to service token",
+			cfg: map[string]interface{}{
+				"type":          "remote_elasticsearch",
+				"hosts":         []string{"https://remote-es:443"},
+				"service_token": "token1",
+			},
+			newCfg: map[string]interface{}{
+				"type":          "remote_elasticsearch",
+				"hosts":         []string{"https://remote-es:443"},
+				"service_token": "token2",
+			},
+			changed: true,
+		},
+		{
+			name: "change to advanced config",
+			cfg: map[string]interface{}{
+				"type":                "remote_elasticsearch",
+				"hosts":               []string{"https://remote-es:443"},
+				"service_token":       "token1",
+				"server.memory_limit": "4",
+			},
+			newCfg: map[string]interface{}{
+				"type":                "remote_elasticsearch",
+				"hosts":               []string{"https://remote-es:443"},
+				"service_token":       "token1",
+				"server.memory_limit": "5",
+			},
+			changed: true,
+		}}
+
+	expectedCount := 0
+	channelCount := 0
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			bulker := NewBulker(nil, nil)
+			bulker.remoteOutputConfigMap["remote1"] = tc.cfg
+			bulker.CheckRemoteOutputChanged("remote1", tc.newCfg)
+
+			if tc.changed {
+				expectedCount++
+			}
+
+			close(bulker.remoteOutputCh)
+			for _ = range bulker.remoteOutputCh {
+				channelCount++
+			}
+		})
+	}
+
+	assert.Equal(t, expectedCount, channelCount)
+}
diff --git a/internal/pkg/server/remote_es_output_integration_test.go b/internal/pkg/server/remote_es_output_integration_test.go
index dd9c22734..7c8cfb0fb 100644
--- a/internal/pkg/server/remote_es_output_integration_test.go
+++ b/internal/pkg/server/remote_es_output_integration_test.go
@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 //go:build integration
 
 package server

From 108fe479688471ab8af7169be404562700388264 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Fri, 3 Nov 2023 15:59:07 +0100
Subject: [PATCH 29/91] fix lint

---
 .../remote_es_output_integration_test.go      | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/internal/pkg/server/remote_es_output_integration_test.go b/internal/pkg/server/remote_es_output_integration_test.go
index 7c8cfb0fb..cfc5bd706 100644
--- a/internal/pkg/server/remote_es_output_integration_test.go
+++ b/internal/pkg/server/remote_es_output_integration_test.go
@@ -49,20 +49,20 @@ func Checkin(t *testing.T, ctx context.Context, srv *tserver, agentID, key strin
 	require.Greater(t, len(actions), 0, "expected at least 1 action")
 	action, ok := actions[0].(map[string]interface{})
 	require.True(t, ok, "expected action to be an object")
-	typeRaw, _ := action["type"]
+	typeRaw := action["type"]
 	require.Equal(t, "POLICY_CHANGE", typeRaw)
-	dataRaw, _ := action["data"]
-	data, _ := dataRaw.(map[string]interface{})
-	policy, _ := data["policy"].(map[string]interface{})
-	outputs, _ := policy["outputs"].(map[string]interface{})
-	remoteES, _ := outputs["remoteES"].(map[string]interface{})
-	oType, _ := remoteES["type"].(string)
+	dataRaw := action["data"]
+	data := dataRaw.(map[string]interface{})
+	policy := data["policy"].(map[string]interface{})
+	outputs := policy["outputs"].(map[string]interface{})
+	remoteES := outputs["remoteES"].(map[string]interface{})
+	oType := remoteES["type"].(string)
 	require.Equal(t, "elasticsearch", oType)
-	serviceToken, _ := remoteES["service_token"]
+	serviceToken := remoteES["service_token"]
 	require.Equal(t, nil, serviceToken)
-	remoteAPIKey, _ := remoteES["api_key"]
-	defaultOutput, _ := outputs["default"].(map[string]interface{})
-	defaultAPIKey, _ := defaultOutput["api_key"]
+	remoteAPIKey := remoteES["api_key"]
+	defaultOutput := outputs["default"].(map[string]interface{})
+	defaultAPIKey := defaultOutput["api_key"]
 	require.False(t, remoteAPIKey == defaultAPIKey, "expected remote api key to be different than default")
 
 }

From fd7e406b57461caabc68153941c38437945cd5fb Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Fri, 3 Nov 2023 16:05:15 +0100
Subject: [PATCH 30/91] fix lint

---
 .../server/remote_es_output_integration_test.go  | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/internal/pkg/server/remote_es_output_integration_test.go b/internal/pkg/server/remote_es_output_integration_test.go
index cfc5bd706..f8922d732 100644
--- a/internal/pkg/server/remote_es_output_integration_test.go
+++ b/internal/pkg/server/remote_es_output_integration_test.go
@@ -52,16 +52,22 @@ func Checkin(t *testing.T, ctx context.Context, srv *tserver, agentID, key strin
 	typeRaw := action["type"]
 	require.Equal(t, "POLICY_CHANGE", typeRaw)
 	dataRaw := action["data"]
-	data := dataRaw.(map[string]interface{})
-	policy := data["policy"].(map[string]interface{})
-	outputs := policy["outputs"].(map[string]interface{})
-	remoteES := outputs["remoteES"].(map[string]interface{})
+	data, ok := dataRaw.(map[string]interface{})
+	require.True(t, ok, "expected data to be map")
+	policy, ok := data["policy"].(map[string]interface{})
+	require.True(t, ok, "expected policy to be map")
+	outputs, ok := policy["outputs"].(map[string]interface{})
+	require.True(t, ok, "expected outputs to be map")
+	remoteES, ok := outputs["remoteES"].(map[string]interface{})
+	require.True(t, ok, "expected remoteES to be map")
 	oType := remoteES["type"].(string)
+	require.True(t, ok, "expected type to be string")
 	require.Equal(t, "elasticsearch", oType)
 	serviceToken := remoteES["service_token"]
 	require.Equal(t, nil, serviceToken)
 	remoteAPIKey := remoteES["api_key"]
-	defaultOutput := outputs["default"].(map[string]interface{})
+	defaultOutput, ok := outputs["default"].(map[string]interface{})
+	require.True(t, ok, "expected default to be map")
 	defaultAPIKey := defaultOutput["api_key"]
 	require.False(t, remoteAPIKey == defaultAPIKey, "expected remote api key to be different than default")
 

From 4f99d622b08a829e34b68beee0cdf50ecf350be3 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Fri, 3 Nov 2023 16:54:43 +0100
Subject: [PATCH 31/91] added tests on policy_output remote ES

---
 internal/pkg/policy/policy_output_test.go | 201 ++++++++++++++++++++++
 internal/pkg/testing/bulk.go              |   3 +-
 2 files changed, 202 insertions(+), 2 deletions(-)

diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go
index 8791bbc37..0c4a83445 100644
--- a/internal/pkg/policy/policy_output_test.go
+++ b/internal/pkg/policy/policy_output_test.go
@@ -295,3 +295,204 @@ func TestPolicyOutputESPrepare(t *testing.T) {
 		bulker.AssertExpectations(t)
 	})
 }
+
+func TestPolicyRemoteESOutputPrepareNoRole(t *testing.T) {
+	logger := testlog.SetLogger(t)
+	bulker := ftesting.NewMockBulk()
+	po := Output{
+		Type: OutputTypeRemoteElasticsearch,
+		Name: "test output",
+		Role: nil,
+	}
+
+	err := po.Prepare(context.Background(), logger, bulker, &model.Agent{}, map[string]map[string]interface{}{})
+	require.NotNil(t, err, "expected prepare to error")
+	bulker.AssertExpectations(t)
+}
+
+func TestPolicyRemoteESOutputPrepare(t *testing.T) {
+	t.Run("Permission hash == Agent Permission Hash no need to regenerate the key", func(t *testing.T) {
+		logger := testlog.SetLogger(t)
+		bulker := ftesting.NewMockBulk()
+
+		apiKey := bulk.APIKey{ID: "test_id_existing", Key: "existing-key"}
+
+		hashPerm := "abc123"
+		output := Output{
+			Type: OutputTypeRemoteElasticsearch,
+			Name: "test output",
+			Role: &RoleT{
+				Sha2: hashPerm,
+				Raw:  TestPayload,
+			},
+		}
+
+		policyMap := map[string]map[string]interface{}{
+			"test output": map[string]interface{}{
+				"hosts":         []interface{}{"http://localhost"},
+				"service_token": "serviceToken1",
+				"type":          OutputTypeRemoteElasticsearch,
+			},
+		}
+
+		testAgent := &model.Agent{
+			Outputs: map[string]*model.PolicyOutput{
+				output.Name: {
+					ESDocument:        model.ESDocument{},
+					APIKey:            apiKey.Agent(),
+					ToRetireAPIKeyIds: nil,
+					APIKeyID:          apiKey.ID,
+					PermissionsHash:   hashPerm,
+					Type:              OutputTypeRemoteElasticsearch,
+				},
+			},
+		}
+
+		err := output.Prepare(context.Background(), logger, bulker, testAgent, policyMap)
+		require.NoError(t, err, "expected prepare to pass")
+
+		key, ok := policyMap[output.Name]["api_key"].(string)
+		gotOutput := testAgent.Outputs[output.Name]
+
+		require.True(t, ok, "api key not present on policy map")
+		assert.Equal(t, apiKey.Agent(), key)
+
+		assert.Equal(t, apiKey.Agent(), gotOutput.APIKey)
+		assert.Equal(t, apiKey.ID, gotOutput.APIKeyID)
+		assert.Equal(t, output.Role.Sha2, gotOutput.PermissionsHash)
+		assert.Equal(t, output.Type, gotOutput.Type)
+		assert.Empty(t, gotOutput.ToRetireAPIKeyIds)
+
+		assert.Equal(t, OutputTypeElasticsearch, policyMap["test output"]["type"])
+		assert.Empty(t, policyMap["test output"]["service_token"])
+
+		bulker.AssertNotCalled(t, "Update",
+			mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything)
+		bulker.AssertNotCalled(t, "APIKeyCreate",
+			mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything)
+		bulker.AssertExpectations(t)
+	})
+
+	t.Run("Permission hash != Agent Permission Hash need to regenerate permissions", func(t *testing.T) {
+		logger := testlog.SetLogger(t)
+		bulker := ftesting.NewMockBulk()
+
+		oldAPIKey := bulk.APIKey{ID: "test_id", Key: "EXISTING-KEY"}
+		wantAPIKey := bulk.APIKey{ID: "test_id", Key: "EXISTING-KEY"}
+		hashPerm := "old-HASH"
+
+		bulker.On("Update",
+			mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
+			Return(nil).Once()
+
+		outputBulker := ftesting.NewMockBulk()
+		bulkerMap["test output"] = outputBulker
+
+		outputBulker.
+			On("APIKeyRead", mock.Anything, mock.Anything, mock.Anything).
+			Return(&bulk.APIKeyMetadata{ID: "test_id", RoleDescriptors: TestPayload}, nil).
+			Once()
+		outputBulker.On("APIKeyUpdate", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
+
+		output := Output{
+			Type: OutputTypeRemoteElasticsearch,
+			Name: "test output",
+			Role: &RoleT{
+				Sha2: "new-hash",
+				Raw:  TestPayload,
+			},
+		}
+
+		policyMap := map[string]map[string]interface{}{
+			"test output": map[string]interface{}{
+				"hosts":         []interface{}{"http://localhost"},
+				"service_token": "serviceToken1",
+				"type":          OutputTypeRemoteElasticsearch,
+			},
+		}
+
+		testAgent := &model.Agent{
+			Outputs: map[string]*model.PolicyOutput{
+				output.Name: {
+					ESDocument:        model.ESDocument{},
+					APIKey:            oldAPIKey.Agent(),
+					ToRetireAPIKeyIds: nil,
+					APIKeyID:          oldAPIKey.ID,
+					PermissionsHash:   hashPerm,
+					Type:              OutputTypeRemoteElasticsearch,
+				},
+			},
+		}
+
+		err := output.Prepare(context.Background(), logger, bulker, testAgent, policyMap)
+		require.NoError(t, err, "expected prepare to pass")
+
+		key, ok := policyMap[output.Name]["api_key"].(string)
+		gotOutput := testAgent.Outputs[output.Name]
+
+		require.True(t, ok, "unable to case api key")
+		require.Equal(t, wantAPIKey.Agent(), key)
+
+		assert.Equal(t, wantAPIKey.Agent(), gotOutput.APIKey)
+		assert.Equal(t, wantAPIKey.ID, gotOutput.APIKeyID)
+		assert.Equal(t, output.Role.Sha2, gotOutput.PermissionsHash)
+		assert.Equal(t, output.Type, gotOutput.Type)
+
+		assert.Equal(t, OutputTypeElasticsearch, policyMap["test output"]["type"])
+		assert.Empty(t, policyMap["test output"]["service_token"])
+
+		bulker.AssertExpectations(t)
+	})
+
+	t.Run("Generate API Key on new Agent", func(t *testing.T) {
+		logger := testlog.SetLogger(t)
+		bulker := ftesting.NewMockBulk()
+		bulker.On("Update",
+			mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
+			Return(nil).Once()
+		apiKey := bulk.APIKey{ID: "abc", Key: "new-key"}
+
+		outputBulker := ftesting.NewMockBulk()
+		outputBulker.On("APIKeyCreate",
+			mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
+			Return(&apiKey, nil).Once()
+		bulkerMap["test output"] = outputBulker
+
+		output := Output{
+			Type: OutputTypeRemoteElasticsearch,
+			Name: "test output",
+			Role: &RoleT{
+				Sha2: "new-hash",
+				Raw:  TestPayload,
+			},
+		}
+
+		policyMap := map[string]map[string]interface{}{
+			"test output": map[string]interface{}{
+				"hosts":         []interface{}{"http://localhost"},
+				"service_token": "serviceToken1",
+				"type":          OutputTypeRemoteElasticsearch,
+			},
+		}
+		testAgent := &model.Agent{Outputs: map[string]*model.PolicyOutput{}}
+
+		err := output.Prepare(context.Background(), logger, bulker, testAgent, policyMap)
+		require.NoError(t, err, "expected prepare to pass")
+
+		key, ok := policyMap[output.Name]["api_key"].(string)
+		gotOutput := testAgent.Outputs[output.Name]
+
+		require.True(t, ok, "unable to case api key")
+		assert.Equal(t, apiKey.Agent(), key)
+
+		assert.Equal(t, apiKey.Agent(), gotOutput.APIKey)
+		assert.Equal(t, apiKey.ID, gotOutput.APIKeyID)
+		assert.Equal(t, output.Role.Sha2, gotOutput.PermissionsHash)
+		assert.Empty(t, gotOutput.ToRetireAPIKeyIds)
+
+		assert.Equal(t, OutputTypeElasticsearch, policyMap["test output"]["type"])
+		assert.Empty(t, policyMap["test output"]["service_token"])
+
+		bulker.AssertExpectations(t)
+	})
+}
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index a9197ebe8..afe91099c 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -80,8 +80,7 @@ func (m *MockBulk) Client() *elasticsearch.Client {
 }
 
 func (m *MockBulk) Tracer() *apm.Tracer {
-	args := m.Called()
-	return args.Get(0).(*apm.Tracer)
+	return nil
 }
 
 func (m *MockBulk) CheckRemoteOutputChanged(name string, newCfg map[string]interface{}) {

From cc6a02ed58e22bea03c37c768a9a5af6f163310d Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Fri, 3 Nov 2023 17:08:31 +0100
Subject: [PATCH 32/91] added test on parsed_policy change

---
 .../pkg/policy/parsed_policy_data_test.go     | 54 +++++++++++++++++++
 internal/pkg/policy/parsed_policy_test.go     | 17 ++++++
 2 files changed, 71 insertions(+)

diff --git a/internal/pkg/policy/parsed_policy_data_test.go b/internal/pkg/policy/parsed_policy_data_test.go
index 31ef2ead1..cee65b996 100644
--- a/internal/pkg/policy/parsed_policy_data_test.go
+++ b/internal/pkg/policy/parsed_policy_data_test.go
@@ -644,3 +644,57 @@ const logstashOutputPolicy = `
    }
 }
 `
+
+const testPolicyRemoteES = `
+{
+   "id": "remote1",
+   "revision": 33,
+   "outputs": {
+      "remote": {
+         "type": "remote_elasticsearch",
+         "hosts": [
+            "https://5a8bb94bfbe0401a909e1496a9e884c2.us-central1.gcp.foundit.no:443"
+         ],
+         "service_token": "token1"
+      }
+   },
+   "output_permissions": {
+      "remote": {
+         "_fallback": {
+            "cluster": [
+               "monitor"
+            ],
+            "indices": [
+               {
+                  "names": [
+                     "logs-*",
+                     "metrics-*",
+                     "traces-*",
+                     ".logs-endpoint.diagnostic.collection-*"
+                  ],
+                  "privileges": [
+                     "auto_configure",
+                     "create_doc"
+                  ]
+               }
+            ]
+         }
+      }
+   },
+   "agent": {
+      "monitoring": {
+         "enabled": true,
+         "use_output": "remote",
+         "logs": true,
+         "metrics": true
+      }
+   },
+   "inputs": [
+   ],
+   "fleet": {
+      "hosts": [
+         "http://10.128.0.4:8220"
+      ]
+   }
+}
+`
diff --git a/internal/pkg/policy/parsed_policy_test.go b/internal/pkg/policy/parsed_policy_test.go
index 72f7aec6f..a135170b2 100644
--- a/internal/pkg/policy/parsed_policy_test.go
+++ b/internal/pkg/policy/parsed_policy_test.go
@@ -73,3 +73,20 @@ func TestNewParsedPolicyNoES(t *testing.T) {
 	// Validate that default was found
 	require.Equal(t, "remote_not_es", pp.Default.Name)
 }
+
+func TestNewParsedPolicyRemoteES(t *testing.T) {
+	// Load the model into the policy object
+	var m model.Policy
+	var d model.PolicyData
+	err := json.Unmarshal([]byte(testPolicyRemoteES), &d)
+	require.NoError(t, err)
+
+	m.Data = &d
+
+	pp, err := NewParsedPolicy(context.TODO(), nil, m)
+	require.NoError(t, err)
+
+	// Validate that default was found
+	require.Equal(t, "remote", pp.Default.Name)
+	require.Equal(t, "token1", pp.Outputs["remote"].ServiceToken)
+}

From 6dc08cd0785010b6b6a7a978e703754d5913396b Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 6 Nov 2023 09:05:52 +0100
Subject: [PATCH 33/91] fix lint

---
 internal/pkg/server/remote_es_output_integration_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/pkg/server/remote_es_output_integration_test.go b/internal/pkg/server/remote_es_output_integration_test.go
index f8922d732..7edb29f19 100644
--- a/internal/pkg/server/remote_es_output_integration_test.go
+++ b/internal/pkg/server/remote_es_output_integration_test.go
@@ -60,7 +60,7 @@ func Checkin(t *testing.T, ctx context.Context, srv *tserver, agentID, key strin
 	require.True(t, ok, "expected outputs to be map")
 	remoteES, ok := outputs["remoteES"].(map[string]interface{})
 	require.True(t, ok, "expected remoteES to be map")
-	oType := remoteES["type"].(string)
+	oType, ok := remoteES["type"].(string)
 	require.True(t, ok, "expected type to be string")
 	require.Equal(t, "elasticsearch", oType)
 	serviceToken := remoteES["service_token"]

From 9a0e8b17841760af27c6f7a2da1a5d528d7f581b Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 7 Nov 2023 16:10:04 +0100
Subject: [PATCH 34/91] invalidate api key if remote output is removed

---
 internal/pkg/api/handleAck.go        |   9 +-
 internal/pkg/bulk/engine.go          |  88 ++++++++++++++++-
 internal/pkg/model/schema.go         |   2 +
 internal/pkg/policy/policy_output.go | 138 +++++++++++----------------
 internal/pkg/policy/self.go          |   1 +
 internal/pkg/testing/bulk.go         |  10 +-
 6 files changed, 161 insertions(+), 87 deletions(-)

diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go
index b4c15b822..c08094b2e 100644
--- a/internal/pkg/api/handleAck.go
+++ b/internal/pkg/api/handleAck.go
@@ -556,16 +556,23 @@ func cleanRoles(roles json.RawMessage) (json.RawMessage, int, error) {
 
 func (ack *AckT) invalidateAPIKeys(ctx context.Context, zlog zerolog.Logger, toRetireAPIKeyIDs []model.ToRetireAPIKeyIdsItems, skip string) {
 	ids := make([]string, 0, len(toRetireAPIKeyIDs))
+	output := ""
 	for _, k := range toRetireAPIKeyIDs {
 		if k.ID == skip || k.ID == "" {
 			continue
 		}
 		ids = append(ids, k.ID)
+		output = k.Output
+	}
+	// using remote es bulker to invalidate api key - supposing all retire api keky ids belong to the same remote es
+	bulk := ack.bulk
+	if output != "" {
+		bulk = ack.bulk.GetBulker(output)
 	}
 
 	if len(ids) > 0 {
 		zlog.Info().Strs("fleet.policy.apiKeyIDsToRetire", ids).Msg("Invalidate old API keys")
-		if err := ack.bulk.APIKeyInvalidate(ctx, ids...); err != nil {
+		if err := bulk.APIKeyInvalidate(ctx, ids...); err != nil {
 			zlog.Info().Err(err).Strs("ids", ids).Msg("Failed to invalidate API keys")
 		}
 	}
diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 9fb870138..38fdfc0db 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -15,6 +15,8 @@ import (
 	"time"
 
 	"github.com/elastic/fleet-server/v7/internal/pkg/apikey"
+	"github.com/elastic/fleet-server/v7/internal/pkg/build"
+	"github.com/elastic/fleet-server/v7/internal/pkg/config"
 	"github.com/elastic/fleet-server/v7/internal/pkg/es"
 
 	"github.com/elastic/go-elasticsearch/v8"
@@ -66,13 +68,13 @@ type Bulk interface {
 	// Accessor used to talk to elastic search direcly bypassing bulk engine
 	Client() *elasticsearch.Client
 
-	// Reusing tracer to create bulker for remote ES outputs
-	Tracer() *apm.Tracer
-
 	CheckRemoteOutputChanged(name string, newCfg map[string]interface{})
 
 	RemoteOutputCh() chan bool
 
+	CreateAndGetBulker(outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, error)
+	GetBulker(outputName string) Bulk
+
 	ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error)
 }
 
@@ -87,6 +89,7 @@ type Bulker struct {
 	tracer                *apm.Tracer
 	remoteOutputConfigMap map[string]map[string]interface{}
 	remoteOutputCh        chan bool
+	bulkerMap             map[string]Bulk
 }
 
 const (
@@ -116,7 +119,86 @@ func NewBulker(es esapi.Transport, tracer *apm.Tracer, opts ...BulkOpt) *Bulker
 		tracer:                tracer,
 		remoteOutputConfigMap: make(map[string]map[string]interface{}),
 		remoteOutputCh:        make(chan bool, 1),
+		bulkerMap:             make(map[string]Bulk),
+	}
+}
+
+func (b *Bulker) GetBulker(outputName string) Bulk {
+	return b.bulkerMap[outputName]
+}
+
+func (b *Bulker) CreateAndGetBulker(outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, error) {
+	b.CheckRemoteOutputChanged(outputName, outputMap[outputName])
+	bulker := b.bulkerMap[outputName]
+	if bulker != nil {
+		return bulker, nil
+	}
+	bulkCtx, bulkCancel := context.WithCancel(context.Background())
+	defer bulkCancel()
+	es, err := b.createRemoteEsClient(bulkCtx, outputName, serviceToken, outputMap)
+	if err != nil {
+		return nil, err
+	}
+	// starting a new bulker to create/update API keys for remote ES output
+	newBulker := NewBulker(es, b.tracer)
+	b.bulkerMap[outputName] = newBulker
+
+	errCh := make(chan error)
+	go func() {
+		runFunc := func() (err error) {
+			return newBulker.Run(bulkCtx)
+		}
+
+		errCh <- runFunc()
+	}()
+	go func() {
+		select {
+		case err = <-errCh:
+		case <-bulkCtx.Done():
+			err = bulkCtx.Err()
+		}
+	}()
+
+	return newBulker, nil
+}
+
+func (b *Bulker) createRemoteEsClient(ctx context.Context, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (*elasticsearch.Client, error) {
+	hostsObj := outputMap[outputName]["hosts"]
+	hosts, ok := hostsObj.([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("failed to get hosts from output: %v", hostsObj)
+	}
+	hostsStrings := make([]string, len(hosts))
+	for i, host := range hosts {
+		hostsStrings[i], ok = host.(string)
+		if !ok {
+			return nil, fmt.Errorf("failed to get hosts from output: %v", host)
+		}
+	}
+
+	cfg := config.Config{
+		Output: config.Output{
+			Elasticsearch: config.Elasticsearch{
+				Hosts:        hostsStrings,
+				ServiceToken: serviceToken,
+			},
+		},
+	}
+	es, err := es.NewClient(ctx, &cfg, false, elasticsearchOptions(
+		true, build.Info{},
+	)...)
+	if err != nil {
+		return nil, err
+	}
+	return es, nil
+}
+
+func elasticsearchOptions(instumented bool, bi build.Info) []es.ConfigOption {
+	options := []es.ConfigOption{es.WithUserAgent("Remote-Fleet-Server", bi)}
+	if instumented {
+		options = append(options, es.InstrumentRoundTripper())
 	}
+	return options
 }
 
 func (b *Bulker) Client() *elasticsearch.Client {
diff --git a/internal/pkg/model/schema.go b/internal/pkg/model/schema.go
index edbf33e06..dfbcc808e 100644
--- a/internal/pkg/model/schema.go
+++ b/internal/pkg/model/schema.go
@@ -468,4 +468,6 @@ type ToRetireAPIKeyIdsItems struct {
 
 	// Date/time the API key was retired
 	RetiredAt string `json:"retired_at,omitempty"`
+
+	Output string `json:"output,omitempty"`
 }
diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index 140d62978..134baad44 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -16,15 +16,11 @@ import (
 	"go.elastic.co/apm/v2"
 
 	"github.com/elastic/fleet-server/v7/internal/pkg/apikey"
-	"github.com/elastic/fleet-server/v7/internal/pkg/build"
 	"github.com/elastic/fleet-server/v7/internal/pkg/bulk"
-	"github.com/elastic/fleet-server/v7/internal/pkg/config"
 	"github.com/elastic/fleet-server/v7/internal/pkg/dl"
-	"github.com/elastic/fleet-server/v7/internal/pkg/es"
 	"github.com/elastic/fleet-server/v7/internal/pkg/logger"
 	"github.com/elastic/fleet-server/v7/internal/pkg/model"
 	"github.com/elastic/fleet-server/v7/internal/pkg/smap"
-	"github.com/elastic/go-elasticsearch/v8"
 )
 
 const (
@@ -37,8 +33,6 @@ const (
 var (
 	ErrNoOutputPerms    = errors.New("output permission sections not found")
 	ErrFailInjectAPIKey = errors.New("fail inject api key")
-	// TODO clean up bulkers when a remote output is removed
-	bulkerMap = make(map[string]bulk.Bulk)
 )
 
 type Output struct {
@@ -66,7 +60,7 @@ func (p *Output) Prepare(ctx context.Context, zlog zerolog.Logger, bulker bulk.B
 		}
 	case OutputTypeRemoteElasticsearch:
 		zlog.Debug().Msg("preparing remote elasticsearch output")
-		newBulker, err := p.createAndGetBulker(bulker, outputMap)
+		newBulker, err := bulker.CreateAndGetBulker(p.Name, p.ServiceToken, outputMap)
 		if err != nil {
 			return err
 		}
@@ -115,6 +109,61 @@ func (p *Output) prepareElasticsearch(
 		agent.Outputs[p.Name] = output
 	}
 
+	// retire api key of removed remote output
+	var toRetireAPIKeys *model.ToRetireAPIKeyIdsItems
+	var removedOutputKey string
+	// find the first output that is removed - supposing one output can be removed at a time
+	for agentOutputKey, agentOutput := range agent.Outputs {
+		found := false
+		for outputMapKey := range outputMap {
+			if agentOutputKey == outputMapKey {
+				found = true
+			}
+		}
+		if !found {
+			zlog.Info().Str("APIKeyID", agentOutput.APIKeyID).Str("output", agentOutputKey).Msg("Output removed, will retire API key")
+			toRetireAPIKeys = &model.ToRetireAPIKeyIdsItems{
+				ID:        agentOutput.APIKeyID,
+				RetiredAt: time.Now().UTC().Format(time.RFC3339),
+				Output:    agentOutputKey,
+			}
+			removedOutputKey = agentOutputKey
+			break
+		}
+	}
+
+	if toRetireAPIKeys != nil {
+
+		// adding remote API key to new output toRetireAPIKeys
+		fields := map[string]interface{}{
+			dl.FieldPolicyOutputToRetireAPIKeyIDs: *toRetireAPIKeys,
+		}
+
+		// Using painless script to append the old keys to the history
+		body, err := renderUpdatePainlessScript(p.Name, fields)
+		if err != nil {
+			return fmt.Errorf("could not update painless script: %w", err)
+		}
+
+		if err = bulker.Update(ctx, dl.FleetAgents, agent.Id, body, bulk.WithRefresh(), bulk.WithRetryOnConflict(3)); err != nil {
+			zlog.Error().Err(err).Msg("fail update agent record")
+			return fmt.Errorf("fail update agent record: %w", err)
+		}
+
+		// remove output from agent doc
+		body, err = json.Marshal(map[string]interface{}{
+			"script": map[string]interface{}{
+				"lang":   "painless",
+				"source": fmt.Sprintf("ctx._source['outputs'].remove(\"%s\")", removedOutputKey),
+			},
+		})
+
+		if err = bulker.Update(ctx, dl.FleetAgents, agent.Id, body, bulk.WithRefresh(), bulk.WithRetryOnConflict(3)); err != nil {
+			zlog.Error().Err(err).Msg("fail update agent record")
+			return fmt.Errorf("fail update agent record: %w", err)
+		}
+	}
+
 	// Determine whether we need to generate an output ApiKey.
 	// This is accomplished by comparing the sha2 hash stored in the corresponding
 	// output in the agent record with the precalculated sha2 hash of the role.
@@ -227,6 +276,7 @@ func (p *Output) prepareElasticsearch(
 			fields[dl.FieldPolicyOutputToRetireAPIKeyIDs] = model.ToRetireAPIKeyIdsItems{
 				ID:        output.APIKeyID,
 				RetiredAt: time.Now().UTC().Format(time.RFC3339),
+				Output:    p.Name,
 			}
 		}
 
@@ -272,80 +322,6 @@ func (p *Output) prepareElasticsearch(
 	return nil
 }
 
-func (p *Output) createAndGetBulker(mainBulker bulk.Bulk, outputMap map[string]map[string]interface{}) (bulk.Bulk, error) {
-	mainBulker.CheckRemoteOutputChanged(p.Name, outputMap[p.Name])
-	bulker := bulkerMap[p.Name]
-	if bulker != nil {
-		return bulker, nil
-	}
-	bulkCtx, bulkCancel := context.WithCancel(context.Background())
-	defer bulkCancel()
-	es, err := p.createRemoteEsClient(bulkCtx, outputMap)
-	if err != nil {
-		return nil, err
-	}
-	// starting a new bulker to create/update API keys for remote ES output
-	newBulker := bulk.NewBulker(es, mainBulker.Tracer())
-	bulkerMap[p.Name] = newBulker
-
-	errCh := make(chan error)
-	go func() {
-		runFunc := func() (err error) {
-			return newBulker.Run(bulkCtx)
-		}
-
-		errCh <- runFunc()
-	}()
-	go func() {
-		select {
-		case err = <-errCh:
-		case <-bulkCtx.Done():
-			err = bulkCtx.Err()
-		}
-	}()
-
-	return newBulker, nil
-}
-
-func (p *Output) createRemoteEsClient(ctx context.Context, outputMap map[string]map[string]interface{}) (*elasticsearch.Client, error) {
-	hostsObj := outputMap[p.Name]["hosts"]
-	hosts, ok := hostsObj.([]interface{})
-	if !ok {
-		return nil, fmt.Errorf("failed to get hosts from output: %v", hostsObj)
-	}
-	hostsStrings := make([]string, len(hosts))
-	for i, host := range hosts {
-		hostsStrings[i], ok = host.(string)
-		if !ok {
-			return nil, fmt.Errorf("failed to get hosts from output: %v", host)
-		}
-	}
-
-	cfg := config.Config{
-		Output: config.Output{
-			Elasticsearch: config.Elasticsearch{
-				Hosts:        hostsStrings,
-				ServiceToken: p.ServiceToken,
-			},
-		},
-	}
-	es, err := es.NewClient(ctx, &cfg, false, elasticsearchOptions(
-		true, build.Info{},
-	)...)
-	if err != nil {
-		return nil, err
-	}
-	return es, nil
-}
-
-func elasticsearchOptions(instumented bool, bi build.Info) []es.ConfigOption {
-	options := []es.ConfigOption{es.WithUserAgent("Remote-Fleet-Server", bi)}
-	if instumented {
-		options = append(options, es.InstrumentRoundTripper())
-	}
-	return options
-}
-
 func fetchAPIKeyRoles(ctx context.Context, b bulk.Bulk, apiKeyID string) (*RoleT, error) {
 	res, err := b.APIKeyRead(ctx, apiKeyID, true)
 	if err != nil {
diff --git a/internal/pkg/policy/self.go b/internal/pkg/policy/self.go
index 8fc9b28e7..ed22568a0 100644
--- a/internal/pkg/policy/self.go
+++ b/internal/pkg/policy/self.go
@@ -193,6 +193,7 @@ func (m *selfMonitorT) groupByLatest(policies []model.Policy) map[string]model.P
 	return groupByLatest(policies)
 }
 
+// UnitStateDegraded / Failed
 func (m *selfMonitorT) updateState(ctx context.Context) (client.UnitState, error) {
 	m.mut.Lock()
 	defer m.mut.Unlock()
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index afe91099c..37fe69bfa 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -79,8 +79,14 @@ func (m *MockBulk) Client() *elasticsearch.Client {
 	return args.Get(0).(*elasticsearch.Client)
 }
 
-func (m *MockBulk) Tracer() *apm.Tracer {
-	return nil
+func (m *MockBulk) GetBulker(outputName string) bulk.Bulk {
+	args := m.Called()
+	return args.Get(0).(bulk.Bulk)
+}
+
+func (m *MockBulk) CreateAndGetBulker(outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (bulk.Bulk, error) {
+	args := m.Called(outputName, serviceToken, outputMap)
+	return args.Get(0).(bulk.Bulk), args.Error(1)
 }
 
 func (m *MockBulk) CheckRemoteOutputChanged(name string, newCfg map[string]interface{}) {

From 830b401085afae1bfc8ce5dd2c9efcadd53b8d31 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 7 Nov 2023 16:23:29 +0100
Subject: [PATCH 35/91] fix lint

---
 internal/pkg/policy/policy_output.go | 4 ++++
 internal/pkg/policy/self.go          | 1 -
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index 134baad44..0b0185b17 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -64,6 +64,7 @@ func (p *Output) Prepare(ctx context.Context, zlog zerolog.Logger, bulker bulk.B
 		if err != nil {
 			return err
 		}
+		// the outputBulker is different for remote ES, it is used to create/update Api keys in the remote ES client
 		if err := p.prepareElasticsearch(ctx, zlog, bulker, newBulker, agent, outputMap); err != nil {
 			return fmt.Errorf("failed to prepare remote elasticsearch output %q: %w", p.Name, err)
 		}
@@ -157,6 +158,9 @@ func (p *Output) prepareElasticsearch(
 				"source": fmt.Sprintf("ctx._source['outputs'].remove(\"%s\")", removedOutputKey),
 			},
 		})
+		if err != nil {
+			return fmt.Errorf("could not create request body to update agent: %w", err)
+		}
 
 		if err = bulker.Update(ctx, dl.FleetAgents, agent.Id, body, bulk.WithRefresh(), bulk.WithRetryOnConflict(3)); err != nil {
 			zlog.Error().Err(err).Msg("fail update agent record")
diff --git a/internal/pkg/policy/self.go b/internal/pkg/policy/self.go
index ed22568a0..8fc9b28e7 100644
--- a/internal/pkg/policy/self.go
+++ b/internal/pkg/policy/self.go
@@ -193,7 +193,6 @@ func (m *selfMonitorT) groupByLatest(policies []model.Policy) map[string]model.P
 	return groupByLatest(policies)
 }
 
-// UnitStateDegraded / Failed
 func (m *selfMonitorT) updateState(ctx context.Context) (client.UnitState, error) {
 	m.mut.Lock()
 	defer m.mut.Unlock()

From 62a49c6ecaec3b9a20794521e2dfdcb5a6b01788 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Wed, 8 Nov 2023 10:12:20 +0100
Subject: [PATCH 36/91] fix after conflicts

---
 internal/pkg/bulk/engine.go                     | 12 ++++++------
 internal/pkg/bulk/remote_output_changed_test.go |  4 +++-
 internal/pkg/model/schema.go                    |  5 +++--
 internal/pkg/policy/policy_output.go            |  2 +-
 internal/pkg/server/fleet.go                    |  2 ++
 internal/pkg/testing/bulk.go                    |  7 ++++---
 model/schema.json                               |  8 ++++----
 7 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 6f417bf99..e4d4ce650 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -68,11 +68,11 @@ type Bulk interface {
 	// Accessor used to talk to elastic search direcly bypassing bulk engine
 	Client() *elasticsearch.Client
 
-	CheckRemoteOutputChanged(name string, newCfg map[string]interface{})
+	CheckRemoteOutputChanged(zlog zerolog.Logger, name string, newCfg map[string]interface{})
 
 	RemoteOutputCh() chan bool
 
-	CreateAndGetBulker(outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, error)
+	CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, error)
 	GetBulker(outputName string) Bulk
 
 	ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error)
@@ -127,8 +127,8 @@ func (b *Bulker) GetBulker(outputName string) Bulk {
 	return b.bulkerMap[outputName]
 }
 
-func (b *Bulker) CreateAndGetBulker(outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, error) {
-	b.CheckRemoteOutputChanged(outputName, outputMap[outputName])
+func (b *Bulker) CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, error) {
+	b.CheckRemoteOutputChanged(zlog, outputName, outputMap[outputName])
 	bulker := b.bulkerMap[outputName]
 	if bulker != nil {
 		return bulker, nil
@@ -214,12 +214,12 @@ func (b *Bulker) Tracer() *apm.Tracer {
 }
 
 // check if remote output cfg changed, and signal to remoteOutputCh channel if so
-func (b *Bulker) CheckRemoteOutputChanged(name string, newCfg map[string]interface{}) {
+func (b *Bulker) CheckRemoteOutputChanged(zlog zerolog.Logger, name string, newCfg map[string]interface{}) {
 	curCfg := b.remoteOutputConfigMap[name]
 
 	// ignore output sent to agents where type is set to elasticsearch
 	if curCfg != nil && !reflect.DeepEqual(curCfg, newCfg) {
-		log.Info().Str("name", name).Msg("remote output configuration has changed")
+		zlog.Info().Str("name", name).Msg("remote output configuration has changed")
 		b.remoteOutputCh <- true
 	}
 	newCfgCopy := make(map[string]interface{})
diff --git a/internal/pkg/bulk/remote_output_changed_test.go b/internal/pkg/bulk/remote_output_changed_test.go
index f71205c94..d1837bd1e 100644
--- a/internal/pkg/bulk/remote_output_changed_test.go
+++ b/internal/pkg/bulk/remote_output_changed_test.go
@@ -9,6 +9,7 @@ package bulk
 import (
 	"testing"
 
+	testlog "github.com/elastic/fleet-server/v7/internal/pkg/testing/log"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -79,9 +80,10 @@ func Test_CheckRemoteOutputChanged(t *testing.T) {
 
 	for _, tc := range testcases {
 		t.Run(tc.name, func(t *testing.T) {
+			log := testlog.SetLogger(t)
 			bulker := NewBulker(nil, nil)
 			bulker.remoteOutputConfigMap["remote1"] = tc.cfg
-			bulker.CheckRemoteOutputChanged("remote1", tc.newCfg)
+			bulker.CheckRemoteOutputChanged(log, "remote1", tc.newCfg)
 
 			if tc.changed {
 				expectedCount++
diff --git a/internal/pkg/model/schema.go b/internal/pkg/model/schema.go
index dfbcc808e..ecb71f4aa 100644
--- a/internal/pkg/model/schema.go
+++ b/internal/pkg/model/schema.go
@@ -466,8 +466,9 @@ type ToRetireAPIKeyIdsItems struct {
 	// API Key identifier
 	ID string `json:"id,omitempty"`
 
+	// Output name where the API Key belongs
+	Output string `json:"output,omitempty"`
+
 	// Date/time the API key was retired
 	RetiredAt string `json:"retired_at,omitempty"`
-
-	Output string `json:"output,omitempty"`
 }
diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index 0b0185b17..20900b462 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -60,7 +60,7 @@ func (p *Output) Prepare(ctx context.Context, zlog zerolog.Logger, bulker bulk.B
 		}
 	case OutputTypeRemoteElasticsearch:
 		zlog.Debug().Msg("preparing remote elasticsearch output")
-		newBulker, err := bulker.CreateAndGetBulker(p.Name, p.ServiceToken, outputMap)
+		newBulker, err := bulker.CreateAndGetBulker(zlog, p.Name, p.ServiceToken, outputMap)
 		if err != nil {
 			return err
 		}
diff --git a/internal/pkg/server/fleet.go b/internal/pkg/server/fleet.go
index 7afaeeac4..95e0da548 100644
--- a/internal/pkg/server/fleet.go
+++ b/internal/pkg/server/fleet.go
@@ -421,6 +421,8 @@ func (f *Fleet) runServer(ctx context.Context, cfg *config.Config) (err error) {
 		return err
 	}
 
+	log := zerolog.Ctx(ctx)
+
 	go func() {
 		outputChanged := <-bulker.RemoteOutputCh()
 		f.outputCh <- outputChanged
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index 37fe69bfa..4b453d061 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -8,6 +8,7 @@ import (
 	"context"
 
 	"github.com/elastic/go-elasticsearch/v8"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/mock"
 	"go.elastic.co/apm/v2"
 
@@ -84,12 +85,12 @@ func (m *MockBulk) GetBulker(outputName string) bulk.Bulk {
 	return args.Get(0).(bulk.Bulk)
 }
 
-func (m *MockBulk) CreateAndGetBulker(outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (bulk.Bulk, error) {
-	args := m.Called(outputName, serviceToken, outputMap)
+func (m *MockBulk) CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (bulk.Bulk, error) {
+	args := m.Called(zlog, outputName, serviceToken, outputMap)
 	return args.Get(0).(bulk.Bulk), args.Error(1)
 }
 
-func (m *MockBulk) CheckRemoteOutputChanged(name string, newCfg map[string]interface{}) {
+func (m *MockBulk) CheckRemoteOutputChanged(zlog zerolog.Logger, name string, newCfg map[string]interface{}) {
 }
 
 func (m *MockBulk) RemoteOutputCh() chan bool {
diff --git a/model/schema.json b/model/schema.json
index 7d8880983..9857da512 100644
--- a/model/schema.json
+++ b/model/schema.json
@@ -387,6 +387,10 @@
             "description": "Date/time the API key was retired",
             "type": "string",
             "format": "date-time"
+          },
+          "output": {
+            "description": "Output name where the API Key belongs",
+            "type": "string"
           }
         }
       }
@@ -545,10 +549,6 @@
           "description": "Last checkin message",
           "type": "string"
         },
-        "last_checkin_message": {
-          "description": "Last checkin message",
-          "type": "string"
-        },
         "components": {
           "description": "Elastic Agent components detailed status information",
           "format": "raw"

From 0f1ed68c284267ee3d007482f0c9fdd98eb4518a Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Wed, 8 Nov 2023 16:00:30 +0100
Subject: [PATCH 37/91] reporting remote es error in fleet-server state

---
 internal/pkg/bulk/engine.go          | 12 ++++++++++++
 internal/pkg/policy/policy_output.go | 15 +++++++++++++++
 internal/pkg/policy/self.go          | 15 +++++++++++++++
 internal/pkg/policy/standalone.go    | 14 ++++++++++++++
 internal/pkg/testing/bulk.go         |  7 +++++++
 5 files changed, 63 insertions(+)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index e4d4ce650..e5c462555 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -74,6 +74,8 @@ type Bulk interface {
 
 	CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, error)
 	GetBulker(outputName string) Bulk
+	GetRemoteOutputErrorMap() map[string]string
+	SetRemoteOutputError(name string, status string)
 
 	ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error)
 }
@@ -90,6 +92,7 @@ type Bulker struct {
 	remoteOutputConfigMap map[string]map[string]interface{}
 	remoteOutputCh        chan bool
 	bulkerMap             map[string]Bulk
+	remoteOutputErrorMap  map[string]string
 }
 
 const (
@@ -120,9 +123,18 @@ func NewBulker(es esapi.Transport, tracer *apm.Tracer, opts ...BulkOpt) *Bulker
 		remoteOutputConfigMap: make(map[string]map[string]interface{}),
 		remoteOutputCh:        make(chan bool, 1),
 		bulkerMap:             make(map[string]Bulk),
+		remoteOutputErrorMap:  make(map[string]string),
 	}
 }
 
+func (b *Bulker) GetRemoteOutputErrorMap() map[string]string {
+	return b.remoteOutputErrorMap
+}
+
+func (b *Bulker) SetRemoteOutputError(name string, status string) {
+	b.remoteOutputErrorMap[name] = status
+}
+
 func (b *Bulker) GetBulker(outputName string) Bulk {
 	return b.bulkerMap[outputName]
 }
diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index 20900b462..da472c546 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -255,6 +255,21 @@ func (p *Output) prepareElasticsearch(
 		ctx := zlog.WithContext(ctx)
 		outputAPIKey, err :=
 			generateOutputAPIKey(ctx, outputBulker, agent.Id, p.Name, p.Role.Raw)
+			// reporting output error status to self monitor and not returning the error to keep fleet-server running
+		if outputAPIKey == nil && p.Type == OutputTypeRemoteElasticsearch {
+			if err != nil {
+				zerolog.Ctx(ctx).Warn().Msg("Could not create API key in remote ES")
+				bulker.SetRemoteOutputError(p.Name, err.Error())
+			} else {
+				bulker.SetRemoteOutputError(p.Name, "")
+			}
+
+			// replace type remote_elasticsearch with elasticsearch as agent doesn't recognize remote_elasticsearch
+			outputMap[p.Name][FieldOutputType] = OutputTypeElasticsearch
+			// remove the service token from the agent policy sent to the agent
+			delete(outputMap[p.Name], FieldOutputServiceToken)
+			return nil
+		}
 		if err != nil {
 			return fmt.Errorf("failed generate output API key: %w", err)
 		}
diff --git a/internal/pkg/policy/self.go b/internal/pkg/policy/self.go
index b9c8d7bd0..ff43ea78e 100644
--- a/internal/pkg/policy/self.go
+++ b/internal/pkg/policy/self.go
@@ -218,6 +218,21 @@ func (m *selfMonitorT) updateState(ctx context.Context) (client.UnitState, error
 		return client.UnitStateStarting, nil
 	}
 
+	remoteOutputErrorMap := m.bulker.GetRemoteOutputErrorMap()
+	hasError := false
+	remoteESPayload := make(map[string]interface{})
+	for key, value := range remoteOutputErrorMap {
+		if value != "" {
+			hasError = true
+			remoteESPayload[key] = value
+		}
+	}
+	if hasError {
+		m.state = client.UnitStateDegraded
+		m.reporter.UpdateState(client.UnitStateDegraded, "Could not connect to remote ES output", remoteESPayload)
+		return client.UnitStateDegraded, nil
+	}
+
 	state := client.UnitStateHealthy
 	extendMsg := ""
 	var payload map[string]interface{}
diff --git a/internal/pkg/policy/standalone.go b/internal/pkg/policy/standalone.go
index 597d267bd..cca5898c7 100644
--- a/internal/pkg/policy/standalone.go
+++ b/internal/pkg/policy/standalone.go
@@ -112,6 +112,20 @@ func (m *standAloneSelfMonitorT) check(ctx context.Context) {
 		message = fmt.Sprintf("Failed to request policies: %s", err)
 	}
 
+	remoteOutputErrorMap := m.bulker.GetRemoteOutputErrorMap()
+	hasError := false
+	remoteESPayload := make(map[string]interface{})
+	for key, value := range remoteOutputErrorMap {
+		if value != "" {
+			hasError = true
+			remoteESPayload[key] = value
+		}
+	}
+	if hasError {
+		state = client.UnitStateDegraded
+		message = fmt.Sprintf("Could not connect to remote ES output: %+v", remoteESPayload)
+	}
+
 	if current != state {
 		m.updateState(state, message)
 	}
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index 4b453d061..1826ac4f4 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -93,6 +93,13 @@ func (m *MockBulk) CreateAndGetBulker(zlog zerolog.Logger, outputName string, se
 func (m *MockBulk) CheckRemoteOutputChanged(zlog zerolog.Logger, name string, newCfg map[string]interface{}) {
 }
 
+func (m *MockBulk) GetRemoteOutputErrorMap() map[string]string {
+	args := m.Called()
+	return args.Get(0).(map[string]string)
+}
+
+func (m *MockBulk) SetRemoteOutputError(name string, status string) {}
+
 func (m *MockBulk) RemoteOutputCh() chan bool {
 	args := m.Called()
 	return args.Get(0).(chan bool)

From 1d865d8e92efea73520a1a6112c72cea22c2cde3 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Wed, 8 Nov 2023 16:19:04 +0100
Subject: [PATCH 38/91] ignore lint error

---
 internal/pkg/policy/self.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/pkg/policy/self.go b/internal/pkg/policy/self.go
index ff43ea78e..5a14374a6 100644
--- a/internal/pkg/policy/self.go
+++ b/internal/pkg/policy/self.go
@@ -229,7 +229,7 @@ func (m *selfMonitorT) updateState(ctx context.Context) (client.UnitState, error
 	}
 	if hasError {
 		m.state = client.UnitStateDegraded
-		m.reporter.UpdateState(client.UnitStateDegraded, "Could not connect to remote ES output", remoteESPayload)
+		m.reporter.UpdateState(client.UnitStateDegraded, "Could not connect to remote ES output", remoteESPayload) //nolint:errcheck // not clear what to do in failure cases
 		return client.UnitStateDegraded, nil
 	}
 

From 72a7249788907b15c372b74dda12e330dd46d4e7 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 9 Nov 2023 11:30:30 +0100
Subject: [PATCH 39/91] fix test

---
 internal/pkg/api/handleAck.go             | 2 +-
 internal/pkg/policy/policy_output_test.go | 9 +++++++--
 internal/pkg/testing/bulk.go              | 2 +-
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go
index c08094b2e..7bd5d7fb4 100644
--- a/internal/pkg/api/handleAck.go
+++ b/internal/pkg/api/handleAck.go
@@ -564,7 +564,7 @@ func (ack *AckT) invalidateAPIKeys(ctx context.Context, zlog zerolog.Logger, toR
 		ids = append(ids, k.ID)
 		output = k.Output
 	}
-	// using remote es bulker to invalidate api key - supposing all retire api keky ids belong to the same remote es
+	// using remote es bulker to invalidate api key - supposing all retire api key ids belong to the same remote es
 	bulk := ack.bulk
 	if output != "" {
 		bulk = ack.bulk.GetBulker(output)
diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go
index 0c4a83445..6de5fa439 100644
--- a/internal/pkg/policy/policy_output_test.go
+++ b/internal/pkg/policy/policy_output_test.go
@@ -304,6 +304,8 @@ func TestPolicyRemoteESOutputPrepareNoRole(t *testing.T) {
 		Name: "test output",
 		Role: nil,
 	}
+	outputBulker := ftesting.NewMockBulk()
+	bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker).Once()
 
 	err := po.Prepare(context.Background(), logger, bulker, &model.Agent{}, map[string]map[string]interface{}{})
 	require.NotNil(t, err, "expected prepare to error")
@@ -327,6 +329,9 @@ func TestPolicyRemoteESOutputPrepare(t *testing.T) {
 			},
 		}
 
+		outputBulker := ftesting.NewMockBulk()
+		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker).Once()
+
 		policyMap := map[string]map[string]interface{}{
 			"test output": map[string]interface{}{
 				"hosts":         []interface{}{"http://localhost"},
@@ -386,7 +391,7 @@ func TestPolicyRemoteESOutputPrepare(t *testing.T) {
 			Return(nil).Once()
 
 		outputBulker := ftesting.NewMockBulk()
-		bulkerMap["test output"] = outputBulker
+		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker).Once()
 
 		outputBulker.
 			On("APIKeyRead", mock.Anything, mock.Anything, mock.Anything).
@@ -456,7 +461,7 @@ func TestPolicyRemoteESOutputPrepare(t *testing.T) {
 		outputBulker.On("APIKeyCreate",
 			mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
 			Return(&apiKey, nil).Once()
-		bulkerMap["test output"] = outputBulker
+		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker).Once()
 
 		output := Output{
 			Type: OutputTypeRemoteElasticsearch,
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index 1826ac4f4..4cf3d9671 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -87,7 +87,7 @@ func (m *MockBulk) GetBulker(outputName string) bulk.Bulk {
 
 func (m *MockBulk) CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (bulk.Bulk, error) {
 	args := m.Called(zlog, outputName, serviceToken, outputMap)
-	return args.Get(0).(bulk.Bulk), args.Error(1)
+	return args.Get(0).(bulk.Bulk), nil
 }
 
 func (m *MockBulk) CheckRemoteOutputChanged(zlog zerolog.Logger, name string, newCfg map[string]interface{}) {

From 6ee9086fcd89b00fad26ec6673d08c4565ba9869 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 9 Nov 2023 11:53:39 +0100
Subject: [PATCH 40/91] fix test

---
 internal/pkg/policy/self_test.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/internal/pkg/policy/self_test.go b/internal/pkg/policy/self_test.go
index bfd314a79..b96fd6a5e 100644
--- a/internal/pkg/policy/self_test.go
+++ b/internal/pkg/policy/self_test.go
@@ -47,6 +47,8 @@ func TestSelfMonitor_DefaultPolicy(t *testing.T) {
 	mm.On("Subscribe").Return(ms).Once()
 	mm.On("Unsubscribe", mock.Anything).Return().Once()
 	bulker := ftesting.NewMockBulk()
+	emptyMap := make(map[string]string)
+	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
 
 	monitor := NewSelfMonitor(cfg, bulker, mm, "", reporter)
 	sm := monitor.(*selfMonitorT)
@@ -184,6 +186,9 @@ func TestSelfMonitor_DefaultPolicy_Degraded(t *testing.T) {
 	mm.On("Unsubscribe", mock.Anything).Return().Once()
 	bulker := ftesting.NewMockBulk()
 
+	emptyMap := make(map[string]string)
+	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
+
 	monitor := NewSelfMonitor(cfg, bulker, mm, "", reporter)
 	sm := monitor.(*selfMonitorT)
 	sm.checkTime = 100 * time.Millisecond

From 374b1ae3aea80780867d2c2ade7b3be9490a5e1f Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 9 Nov 2023 13:51:08 +0100
Subject: [PATCH 41/91] fix test

---
 internal/pkg/policy/self_test.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/internal/pkg/policy/self_test.go b/internal/pkg/policy/self_test.go
index b96fd6a5e..937d65eb6 100644
--- a/internal/pkg/policy/self_test.go
+++ b/internal/pkg/policy/self_test.go
@@ -345,6 +345,8 @@ func TestSelfMonitor_SpecificPolicy(t *testing.T) {
 	mm.On("Subscribe").Return(ms).Once()
 	mm.On("Unsubscribe", mock.Anything).Return().Once()
 	bulker := ftesting.NewMockBulk()
+	emptyMap := make(map[string]string)
+	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
 
 	monitor := NewSelfMonitor(cfg, bulker, mm, policyID, reporter)
 	sm := monitor.(*selfMonitorT)
@@ -481,6 +483,8 @@ func TestSelfMonitor_SpecificPolicy_Degraded(t *testing.T) {
 	mm.On("Subscribe").Return(ms).Once()
 	mm.On("Unsubscribe", mock.Anything).Return().Once()
 	bulker := ftesting.NewMockBulk()
+	emptyMap := make(map[string]string)
+	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
 
 	monitor := NewSelfMonitor(cfg, bulker, mm, policyID, reporter)
 	sm := monitor.(*selfMonitorT)

From dad52b060c29c63af3109b8004997fdeaa0d30dd Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 9 Nov 2023 14:07:40 +0100
Subject: [PATCH 42/91] fixed test

---
 internal/pkg/policy/standalone_test.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/internal/pkg/policy/standalone_test.go b/internal/pkg/policy/standalone_test.go
index d0cb67807..984183056 100644
--- a/internal/pkg/policy/standalone_test.go
+++ b/internal/pkg/policy/standalone_test.go
@@ -80,7 +80,8 @@ func TestStandAloneSelfMonitor(t *testing.T) {
 		t.Run(c.title, func(t *testing.T) {
 			bulker := ftesting.NewMockBulk()
 			bulker.On("Search", searchArguments...).Return(c.searchResult, c.searchErr)
-
+			emptyMap := make(map[string]string)
+			bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
 			reporter := &FakeReporter{}
 
 			sm := NewStandAloneSelfMonitor(bulker, reporter)

From 8d4dbbc363a2ab2d7d6e694090cfc1c39f4045cf Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Fri, 10 Nov 2023 13:40:09 +0100
Subject: [PATCH 43/91] start new bulker if remote config changed

---
 internal/pkg/bulk/engine.go          | 45 ++++++++++++++++------------
 internal/pkg/policy/policy_output.go | 14 ++++++---
 internal/pkg/server/fleet.go         | 12 --------
 internal/pkg/testing/bulk.go         | 12 ++++----
 4 files changed, 43 insertions(+), 40 deletions(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index e5c462555..cc77ce9ae 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -68,14 +68,13 @@ type Bulk interface {
 	// Accessor used to talk to elastic search direcly bypassing bulk engine
 	Client() *elasticsearch.Client
 
-	CheckRemoteOutputChanged(zlog zerolog.Logger, name string, newCfg map[string]interface{})
+	CheckRemoteOutputChanged(zlog zerolog.Logger, name string, newCfg map[string]interface{}) bool
 
-	RemoteOutputCh() chan bool
-
-	CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, error)
+	CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, bool, error)
 	GetBulker(outputName string) Bulk
 	GetRemoteOutputErrorMap() map[string]string
 	SetRemoteOutputError(name string, status string)
+	CancelFn() context.CancelFunc
 
 	ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error)
 }
@@ -90,9 +89,9 @@ type Bulker struct {
 	apikeyLimit           *semaphore.Weighted
 	tracer                *apm.Tracer
 	remoteOutputConfigMap map[string]map[string]interface{}
-	remoteOutputCh        chan bool
 	bulkerMap             map[string]Bulk
 	remoteOutputErrorMap  map[string]string
+	cancelFn              context.CancelFunc
 }
 
 const (
@@ -121,7 +120,6 @@ func NewBulker(es esapi.Transport, tracer *apm.Tracer, opts ...BulkOpt) *Bulker
 		apikeyLimit:           semaphore.NewWeighted(int64(bopts.apikeyMaxParallel)),
 		tracer:                tracer,
 		remoteOutputConfigMap: make(map[string]map[string]interface{}),
-		remoteOutputCh:        make(chan bool, 1),
 		bulkerMap:             make(map[string]Bulk),
 		remoteOutputErrorMap:  make(map[string]string),
 	}
@@ -139,25 +137,34 @@ func (b *Bulker) GetBulker(outputName string) Bulk {
 	return b.bulkerMap[outputName]
 }
 
-func (b *Bulker) CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, error) {
-	b.CheckRemoteOutputChanged(zlog, outputName, outputMap[outputName])
+func (b *Bulker) CancelFn() context.CancelFunc {
+	return b.cancelFn
+}
+
+func (b *Bulker) CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, bool, error) {
+	hasConfigChanged := b.CheckRemoteOutputChanged(zlog, outputName, outputMap[outputName])
 	bulker := b.bulkerMap[outputName]
-	if bulker != nil {
-		return bulker, nil
+	if bulker != nil && !hasConfigChanged {
+		return bulker, false, nil
+	}
+	if bulker != nil && hasConfigChanged {
+		bulker.CancelFn()
 	}
 	bulkCtx, bulkCancel := context.WithCancel(context.Background())
-	defer bulkCancel()
 	es, err := b.createRemoteEsClient(bulkCtx, outputName, serviceToken, outputMap)
 	if err != nil {
-		return nil, err
+		defer bulkCancel()
+		return nil, hasConfigChanged, err
 	}
 	// starting a new bulker to create/update API keys for remote ES output
 	newBulker := NewBulker(es, b.tracer)
+	newBulker.cancelFn = bulkCancel
 	b.bulkerMap[outputName] = newBulker
 
 	errCh := make(chan error)
 	go func() {
 		runFunc := func() (err error) {
+			zlog.Debug().Str("outputName", outputName).Msg("Bulker started")
 			return newBulker.Run(bulkCtx)
 		}
 
@@ -167,11 +174,12 @@ func (b *Bulker) CreateAndGetBulker(zlog zerolog.Logger, outputName string, serv
 		select {
 		case err = <-errCh:
 		case <-bulkCtx.Done():
+			zlog.Debug().Str("outputName", outputName).Msg("Bulk context done")
 			err = bulkCtx.Err()
 		}
 	}()
 
-	return newBulker, nil
+	return newBulker, hasConfigChanged, nil
 }
 
 func (b *Bulker) createRemoteEsClient(ctx context.Context, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (*elasticsearch.Client, error) {
@@ -226,23 +234,22 @@ func (b *Bulker) Tracer() *apm.Tracer {
 }
 
 // check if remote output cfg changed, and signal to remoteOutputCh channel if so
-func (b *Bulker) CheckRemoteOutputChanged(zlog zerolog.Logger, name string, newCfg map[string]interface{}) {
+func (b *Bulker) CheckRemoteOutputChanged(zlog zerolog.Logger, name string, newCfg map[string]interface{}) bool {
 	curCfg := b.remoteOutputConfigMap[name]
 
+	hasChanged := false
+
 	// ignore output sent to agents where type is set to elasticsearch
 	if curCfg != nil && !reflect.DeepEqual(curCfg, newCfg) {
 		zlog.Info().Str("name", name).Msg("remote output configuration has changed")
-		b.remoteOutputCh <- true
+		hasChanged = true
 	}
 	newCfgCopy := make(map[string]interface{})
 	for k, v := range newCfg {
 		newCfgCopy[k] = v
 	}
 	b.remoteOutputConfigMap[name] = newCfgCopy
-}
-
-func (b *Bulker) RemoteOutputCh() chan bool {
-	return b.remoteOutputCh
+	return hasChanged
 }
 
 // read secrets one by one as there is no bulk API yet to read them in one request
diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index da472c546..1b6e6a7b4 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -55,17 +55,17 @@ func (p *Output) Prepare(ctx context.Context, zlog zerolog.Logger, bulker bulk.B
 	switch p.Type {
 	case OutputTypeElasticsearch:
 		zlog.Debug().Msg("preparing elasticsearch output")
-		if err := p.prepareElasticsearch(ctx, zlog, bulker, bulker, agent, outputMap); err != nil {
+		if err := p.prepareElasticsearch(ctx, zlog, bulker, bulker, agent, outputMap, false); err != nil {
 			return fmt.Errorf("failed to prepare elasticsearch output %q: %w", p.Name, err)
 		}
 	case OutputTypeRemoteElasticsearch:
 		zlog.Debug().Msg("preparing remote elasticsearch output")
-		newBulker, err := bulker.CreateAndGetBulker(zlog, p.Name, p.ServiceToken, outputMap)
+		newBulker, hasConfigChanged, err := bulker.CreateAndGetBulker(zlog, p.Name, p.ServiceToken, outputMap)
 		if err != nil {
 			return err
 		}
 		// the outputBulker is different for remote ES, it is used to create/update Api keys in the remote ES client
-		if err := p.prepareElasticsearch(ctx, zlog, bulker, newBulker, agent, outputMap); err != nil {
+		if err := p.prepareElasticsearch(ctx, zlog, bulker, newBulker, agent, outputMap, hasConfigChanged); err != nil {
 			return fmt.Errorf("failed to prepare remote elasticsearch output %q: %w", p.Name, err)
 		}
 	case OutputTypeLogstash:
@@ -87,7 +87,8 @@ func (p *Output) prepareElasticsearch(
 	bulker bulk.Bulk,
 	outputBulker bulk.Bulk,
 	agent *model.Agent,
-	outputMap map[string]map[string]interface{}) error {
+	outputMap map[string]map[string]interface{},
+	hasConfigChanged bool) error {
 	// The role is required to do api key management
 	if p.Role == nil {
 		zlog.Error().
@@ -180,6 +181,9 @@ func (p *Output) prepareElasticsearch(
 	case output.APIKey == "":
 		zlog.Debug().Msg("must generate api key as default API key is not present")
 		needNewKey = true
+	case hasConfigChanged:
+		zlog.Debug().Msg("must generate api key as remote output config changed")
+		needNewKey = true
 	case p.Role.Sha2 != output.PermissionsHash:
 		// the is actually the OutputPermissionsHash for the default hash. The Agent
 		// document on ES does not have OutputPermissionsHash for any other output
@@ -269,6 +273,8 @@ func (p *Output) prepareElasticsearch(
 			// remove the service token from the agent policy sent to the agent
 			delete(outputMap[p.Name], FieldOutputServiceToken)
 			return nil
+		} else if p.Type == OutputTypeRemoteElasticsearch {
+			bulker.SetRemoteOutputError(p.Name, "")
 		}
 		if err != nil {
 			return fmt.Errorf("failed generate output API key: %w", err)
diff --git a/internal/pkg/server/fleet.go b/internal/pkg/server/fleet.go
index 95e0da548..4197962a2 100644
--- a/internal/pkg/server/fleet.go
+++ b/internal/pkg/server/fleet.go
@@ -52,7 +52,6 @@ type Fleet struct {
 	cfgCh    chan *config.Config
 	cache    cache.Cache
 	reporter state.Reporter
-	outputCh chan bool
 }
 
 // NewFleet creates the actual fleet server service.
@@ -68,7 +67,6 @@ func NewFleet(bi build.Info, reporter state.Reporter, standAlone bool) (*Fleet,
 		verCon:     verCon,
 		cfgCh:      make(chan *config.Config, 1),
 		reporter:   reporter,
-		outputCh:   make(chan bool, 1),
 	}, nil
 }
 
@@ -200,8 +198,6 @@ LOOP:
 		select {
 		case newCfg = <-f.cfgCh:
 			log.Info().Msg("Server configuration update")
-		case outputChanged = <-f.outputCh:
-			log.Info().Msg("Remote output configuration update")
 		case err := <-ech:
 			f.reporter.UpdateState(client.UnitStateFailed, fmt.Sprintf("Error - %s", err), nil) //nolint:errcheck // unclear on what should we do if updating the status fails?
 			log.Error().Err(err).Msg("Fleet Server failed")
@@ -421,14 +417,6 @@ func (f *Fleet) runServer(ctx context.Context, cfg *config.Config) (err error) {
 		return err
 	}
 
-	log := zerolog.Ctx(ctx)
-
-	go func() {
-		outputChanged := <-bulker.RemoteOutputCh()
-		f.outputCh <- outputChanged
-		log.Info().Msg("Remote output configuration update")
-	}()
-
 	return g.Wait()
 }
 
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index 4cf3d9671..b96b54a34 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -85,12 +85,14 @@ func (m *MockBulk) GetBulker(outputName string) bulk.Bulk {
 	return args.Get(0).(bulk.Bulk)
 }
 
-func (m *MockBulk) CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (bulk.Bulk, error) {
+func (m *MockBulk) CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (bulk.Bulk, bool, error) {
 	args := m.Called(zlog, outputName, serviceToken, outputMap)
-	return args.Get(0).(bulk.Bulk), nil
+	return args.Get(0).(bulk.Bulk), args.Get(1).(bool), nil
 }
 
-func (m *MockBulk) CheckRemoteOutputChanged(zlog zerolog.Logger, name string, newCfg map[string]interface{}) {
+func (m *MockBulk) CheckRemoteOutputChanged(zlog zerolog.Logger, name string, newCfg map[string]interface{}) bool {
+	args := m.Called()
+	return args.Get(0).(bool)
 }
 
 func (m *MockBulk) GetRemoteOutputErrorMap() map[string]string {
@@ -100,9 +102,9 @@ func (m *MockBulk) GetRemoteOutputErrorMap() map[string]string {
 
 func (m *MockBulk) SetRemoteOutputError(name string, status string) {}
 
-func (m *MockBulk) RemoteOutputCh() chan bool {
+func (m *MockBulk) CancelFn() context.CancelFunc {
 	args := m.Called()
-	return args.Get(0).(chan bool)
+	return args.Get(0).(context.CancelFunc)
 }
 
 func (m *MockBulk) ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error) {

From de6724ae3b4ea1570175a9139c6072243676ebe3 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Fri, 10 Nov 2023 14:10:42 +0100
Subject: [PATCH 44/91] fix tests

---
 internal/pkg/bulk/remote_output_changed_test.go | 17 ++---------------
 .../policy/policy_output_integration_test.go    |  2 +-
 internal/pkg/policy/policy_output_test.go       |  8 ++++----
 3 files changed, 7 insertions(+), 20 deletions(-)

diff --git a/internal/pkg/bulk/remote_output_changed_test.go b/internal/pkg/bulk/remote_output_changed_test.go
index d1837bd1e..4ea8e5bab 100644
--- a/internal/pkg/bulk/remote_output_changed_test.go
+++ b/internal/pkg/bulk/remote_output_changed_test.go
@@ -75,26 +75,13 @@ func Test_CheckRemoteOutputChanged(t *testing.T) {
 			changed: true,
 		}}
 
-	expectedCount := 0
-	channelCount := 0
-
 	for _, tc := range testcases {
 		t.Run(tc.name, func(t *testing.T) {
 			log := testlog.SetLogger(t)
 			bulker := NewBulker(nil, nil)
 			bulker.remoteOutputConfigMap["remote1"] = tc.cfg
-			bulker.CheckRemoteOutputChanged(log, "remote1", tc.newCfg)
-
-			if tc.changed {
-				expectedCount++
-			}
-
-			close(bulker.remoteOutputCh)
-			for _ = range bulker.remoteOutputCh {
-				channelCount++
-			}
+			hasChanged := bulker.CheckRemoteOutputChanged(log, "remote1", tc.newCfg)
+			assert.Equal(t, tc.changed, hasChanged)
 		})
 	}
-
-	assert.Equal(t, expectedCount, channelCount)
 }
diff --git a/internal/pkg/policy/policy_output_integration_test.go b/internal/pkg/policy/policy_output_integration_test.go
index db96c7d0a..36f950698 100644
--- a/internal/pkg/policy/policy_output_integration_test.go
+++ b/internal/pkg/policy/policy_output_integration_test.go
@@ -155,7 +155,7 @@ func TestPolicyOutputESPrepareRealES(t *testing.T) {
 	}
 
 	err = output.prepareElasticsearch(
-		ctx, zerolog.Nop(), bulker, bulker, &agent, policyMap)
+		ctx, zerolog.Nop(), bulker, bulker, &agent, policyMap, false)
 	require.NoError(t, err)
 
 	// need to wait a bit before querying the agent again
diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go
index 6de5fa439..39f08de02 100644
--- a/internal/pkg/policy/policy_output_test.go
+++ b/internal/pkg/policy/policy_output_test.go
@@ -305,7 +305,7 @@ func TestPolicyRemoteESOutputPrepareNoRole(t *testing.T) {
 		Role: nil,
 	}
 	outputBulker := ftesting.NewMockBulk()
-	bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker).Once()
+	bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
 
 	err := po.Prepare(context.Background(), logger, bulker, &model.Agent{}, map[string]map[string]interface{}{})
 	require.NotNil(t, err, "expected prepare to error")
@@ -330,7 +330,7 @@ func TestPolicyRemoteESOutputPrepare(t *testing.T) {
 		}
 
 		outputBulker := ftesting.NewMockBulk()
-		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker).Once()
+		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
 
 		policyMap := map[string]map[string]interface{}{
 			"test output": map[string]interface{}{
@@ -391,7 +391,7 @@ func TestPolicyRemoteESOutputPrepare(t *testing.T) {
 			Return(nil).Once()
 
 		outputBulker := ftesting.NewMockBulk()
-		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker).Once()
+		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
 
 		outputBulker.
 			On("APIKeyRead", mock.Anything, mock.Anything, mock.Anything).
@@ -461,7 +461,7 @@ func TestPolicyRemoteESOutputPrepare(t *testing.T) {
 		outputBulker.On("APIKeyCreate",
 			mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
 			Return(&apiKey, nil).Once()
-		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker).Once()
+		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
 
 		output := Output{
 			Type: OutputTypeRemoteElasticsearch,

From e465d3d0302e8ee65b736d0fa0673afd7abcc6d6 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Fri, 10 Nov 2023 15:10:23 +0100
Subject: [PATCH 45/91] added test for retire remote api key

---
 .../policy/policy_output_integration_test.go  | 112 +++++++++++++++++-
 1 file changed, 109 insertions(+), 3 deletions(-)

diff --git a/internal/pkg/policy/policy_output_integration_test.go b/internal/pkg/policy/policy_output_integration_test.go
index 36f950698..f52033fa1 100644
--- a/internal/pkg/policy/policy_output_integration_test.go
+++ b/internal/pkg/policy/policy_output_integration_test.go
@@ -135,7 +135,7 @@ func TestPolicyOutputESPrepareRealES(t *testing.T) {
 	ctx := testlog.SetLogger(t).WithContext(context.Background())
 	index, bulker := ftesting.SetupCleanIndex(ctx, t, dl.FleetAgents)
 
-	agentID := createAgent(ctx, t, index, bulker)
+	agentID := createAgent(ctx, t, index, bulker, map[string]*model.PolicyOutput{})
 	agent, err := dl.FindAgent(
 		ctx, bulker, dl.QueryAgentByID, dl.FieldID, agentID, dl.WithIndexName(index))
 	if err != nil {
@@ -169,7 +169,7 @@ func TestPolicyOutputESPrepareRealES(t *testing.T) {
 	}
 
 	gotOutput, ok := got.Outputs[output.Name]
-	require.True(t, ok, "no '%s' output fouled on agent document", output.Name)
+	require.True(t, ok, "no '%s' output found on agent document", output.Name)
 
 	assert.Empty(t, gotOutput.ToRetireAPIKeyIds)
 	assert.Equal(t, gotOutput.Type, OutputTypeElasticsearch)
@@ -178,7 +178,7 @@ func TestPolicyOutputESPrepareRealES(t *testing.T) {
 	assert.NotEmpty(t, gotOutput.APIKeyID)
 }
 
-func createAgent(ctx context.Context, t *testing.T, index string, bulker bulk.Bulk) string {
+func createAgent(ctx context.Context, t *testing.T, index string, bulker bulk.Bulk, outputs map[string]*model.PolicyOutput) string {
 	const nowStr = "2022-08-12T16:50:05Z"
 
 	agentID := uuid.Must(uuid.NewV4()).String()
@@ -191,6 +191,7 @@ func createAgent(ctx context.Context, t *testing.T, index string, bulker bulk.Bu
 		LastCheckinStatus: "",
 		UpdatedAt:         nowStr,
 		EnrolledAt:        nowStr,
+		Outputs:           outputs,
 	}
 
 	body, err := json.Marshal(agentModel)
@@ -202,3 +203,108 @@ func createAgent(ctx context.Context, t *testing.T, index string, bulker bulk.Bu
 
 	return agentID
 }
+
+func TestPolicyOutputESPrepareRemoteES(t *testing.T) {
+	ctx := testlog.SetLogger(t).WithContext(context.Background())
+	index, bulker := ftesting.SetupCleanIndex(ctx, t, dl.FleetAgents)
+
+	agentID := createAgent(ctx, t, index, bulker, map[string]*model.PolicyOutput{})
+	agent, err := dl.FindAgent(
+		ctx, bulker, dl.QueryAgentByID, dl.FieldID, agentID, dl.WithIndexName(index))
+	if err != nil {
+		require.NoError(t, err, "failed to find agent ID %q", agentID)
+	}
+
+	output := Output{
+		Type:         OutputTypeRemoteElasticsearch,
+		Name:         "test remote output",
+		ServiceToken: "token1",
+		Role: &RoleT{
+			Sha2: "new-hash",
+			Raw:  TestPayload,
+		},
+	}
+	policyMap := map[string]map[string]interface{}{
+		"test remote output": map[string]interface{}{
+			"hosts": []interface{}{"http://localhost:9200"},
+		},
+	}
+
+	err = output.prepareElasticsearch(
+		ctx, zerolog.Nop(), bulker, bulker, &agent, policyMap, false)
+	require.NoError(t, err)
+
+	// need to wait a bit before querying the agent again
+	// TODO: find a better way to query the updated agent
+	time.Sleep(time.Second)
+
+	got, err := dl.FindAgent(
+		ctx, bulker, dl.QueryAgentByID, dl.FieldID, agentID, dl.WithIndexName(index))
+	if err != nil {
+		require.NoError(t, err, "failed to find agent ID %q", agentID)
+	}
+
+	gotOutput, ok := got.Outputs[output.Name]
+	require.True(t, ok, "no '%s' output found on agent document", output.Name)
+
+	assert.Empty(t, gotOutput.ToRetireAPIKeyIds)
+	assert.Equal(t, gotOutput.Type, OutputTypeElasticsearch)
+	assert.Equal(t, gotOutput.PermissionsHash, output.Role.Sha2)
+	assert.NotEmpty(t, gotOutput.APIKey)
+	assert.NotEmpty(t, gotOutput.APIKeyID)
+}
+
+func TestPolicyOutputESPrepareESRetireRemoteAPIKeys(t *testing.T) {
+	ctx := testlog.SetLogger(t).WithContext(context.Background())
+	index, bulker := ftesting.SetupCleanIndex(ctx, t, dl.FleetAgents)
+
+	// simulate a previous remote output, that is removed from outputMap
+	agentID := createAgent(ctx, t, index, bulker, map[string]*model.PolicyOutput{
+		"remote output": &model.PolicyOutput{
+			APIKey:   "apiKey1:value",
+			APIKeyID: "apiKey1",
+		},
+	})
+	agent, err := dl.FindAgent(
+		ctx, bulker, dl.QueryAgentByID, dl.FieldID, agentID, dl.WithIndexName(index))
+	if err != nil {
+		require.NoError(t, err, "failed to find agent ID %q", agentID)
+	}
+
+	output := Output{
+		Type: OutputTypeElasticsearch,
+		Name: "test output",
+		Role: &RoleT{
+			Sha2: "new-hash",
+			Raw:  TestPayload,
+		},
+	}
+	policyMap := map[string]map[string]interface{}{
+		"test output": map[string]interface{}{},
+	}
+
+	err = output.prepareElasticsearch(
+		ctx, zerolog.Nop(), bulker, bulker, &agent, policyMap, false)
+	require.NoError(t, err)
+
+	// need to wait a bit before querying the agent again
+	// TODO: find a better way to query the updated agent
+	time.Sleep(time.Second)
+
+	got, err := dl.FindAgent(
+		ctx, bulker, dl.QueryAgentByID, dl.FieldID, agentID, dl.WithIndexName(index))
+	if err != nil {
+		require.NoError(t, err, "failed to find agent ID %q", agentID)
+	}
+
+	gotOutput, ok := got.Outputs[output.Name]
+	require.True(t, ok, "no '%s' output found on agent document", output.Name)
+
+	assert.Equal(t, len(gotOutput.ToRetireAPIKeyIds), 1)
+	assert.Equal(t, gotOutput.ToRetireAPIKeyIds[0].ID, "apiKey1")
+	assert.Equal(t, gotOutput.ToRetireAPIKeyIds[0].Output, "remote output")
+	assert.Equal(t, gotOutput.Type, OutputTypeElasticsearch)
+	assert.Equal(t, gotOutput.PermissionsHash, output.Role.Sha2)
+	assert.NotEmpty(t, gotOutput.APIKey)
+	assert.NotEmpty(t, gotOutput.APIKeyID)
+}

From bf6c9e31799273913d8a32fcf99bbec46786f4a5 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Fri, 10 Nov 2023 16:05:31 +0100
Subject: [PATCH 46/91] monitor tests

---
 internal/pkg/policy/self_test.go       | 249 +++++++++++++++++++++++++
 internal/pkg/policy/standalone_test.go |  39 +++-
 2 files changed, 287 insertions(+), 1 deletion(-)

diff --git a/internal/pkg/policy/self_test.go b/internal/pkg/policy/self_test.go
index 937d65eb6..ab347700b 100644
--- a/internal/pkg/policy/self_test.go
+++ b/internal/pkg/policy/self_test.go
@@ -642,3 +642,252 @@ func (r *FakeReporter) Current() (client.UnitState, string, map[string]interface
 	defer r.lock.Unlock()
 	return r.state, r.msg, r.payload
 }
+func TestSelfMonitor_RemoteOutput_Degraded(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	cfg := config.Fleet{
+		Agent: config.Agent{
+			ID: "agent-id",
+		},
+	}
+	reporter := &FakeReporter{}
+
+	chHitT := make(chan []es.HitT, 1)
+	defer close(chHitT)
+	ms := mmock.NewMockSubscription()
+	ms.On("Output").Return((<-chan []es.HitT)(chHitT))
+	mm := mmock.NewMockMonitor()
+	mm.On("Subscribe").Return(ms).Once()
+	mm.On("Unsubscribe", mock.Anything).Return().Once()
+	bulker := ftesting.NewMockBulk()
+
+	errorMap := make(map[string]string)
+	errorMap["remote output"] = "error connecting to remote output"
+	bulker.On("GetRemoteOutputErrorMap").Return(errorMap)
+
+	monitor := NewSelfMonitor(cfg, bulker, mm, "", reporter)
+	sm := monitor.(*selfMonitorT)
+	sm.checkTime = 100 * time.Millisecond
+
+	var policyLock sync.Mutex
+	var policyResult []model.Policy
+	sm.policyF = func(ctx context.Context, bulker bulk.Bulk, opt ...dl.Option) ([]model.Policy, error) {
+		policyLock.Lock()
+		defer policyLock.Unlock()
+		return policyResult, nil
+	}
+
+	var tokenLock sync.Mutex
+	var tokenResult []model.EnrollmentAPIKey
+	sm.enrollmentTokenF = func(ctx context.Context, bulker bulk.Bulk, policyID string) ([]model.EnrollmentAPIKey, error) {
+		tokenLock.Lock()
+		defer tokenLock.Unlock()
+		return tokenResult, nil
+	}
+
+	var merr error
+	var mwg sync.WaitGroup
+	mwg.Add(1)
+	go func() {
+		defer mwg.Done()
+		merr = monitor.Run(ctx)
+	}()
+
+	if err := monitor.(*selfMonitorT).waitStart(ctx); err != nil {
+		t.Fatal(err)
+	}
+
+	// should be set to starting
+	ftesting.Retry(t, ctx, func(ctx context.Context) error {
+		state, msg, _ := reporter.Current()
+		if state != client.UnitStateStarting {
+			return fmt.Errorf("should be reported as starting; instead its %s", state)
+		}
+		if msg != "Waiting on default policy with Fleet Server integration" {
+			return fmt.Errorf("should be matching with default policy")
+		}
+		return nil
+	}, ftesting.RetrySleep(1*time.Second))
+
+	policyID := uuid.Must(uuid.NewV4()).String()
+	rId := xid.New().String()
+	pData := model.PolicyData{Inputs: []map[string]interface{}{
+		{
+			"type": "fleet-server",
+		},
+	}}
+	policy := model.Policy{
+		ESDocument: model.ESDocument{
+			Id:      rId,
+			Version: 1,
+			SeqNo:   1,
+		},
+		PolicyID:           policyID,
+		CoordinatorIdx:     1,
+		Data:               &pData,
+		RevisionIdx:        1,
+		DefaultFleetServer: true,
+	}
+	policyData, err := json.Marshal(&policy)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	go func() {
+		chHitT <- []es.HitT{{
+			ID:      rId,
+			SeqNo:   1,
+			Version: 1,
+			Source:  policyData,
+		}}
+		policyLock.Lock()
+		defer policyLock.Unlock()
+		policyResult = append(policyResult, policy)
+	}()
+
+	// should be set to degraded because of remote output error
+	ftesting.Retry(t, ctx, func(ctx context.Context) error {
+		state, msg, _ := reporter.Current()
+		if state != client.UnitStateDegraded {
+			return fmt.Errorf("should be reported as degraded; instead its %s", state)
+		}
+		if msg != "Could not connect to remote ES output" {
+			return fmt.Errorf("expected remote ES connection error")
+		}
+		return nil
+	}, ftesting.RetrySleep(1*time.Second))
+
+	cancel()
+	mwg.Wait()
+	if merr != nil && merr != context.Canceled {
+		t.Fatal(merr)
+	}
+}
+
+func TestSelfMonitor_RemoteOutput_Back_To_Healthy(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	cfg := config.Fleet{
+		Agent: config.Agent{
+			ID: "agent-id",
+		},
+	}
+	reporter := &FakeReporter{}
+
+	chHitT := make(chan []es.HitT, 1)
+	defer close(chHitT)
+	ms := mmock.NewMockSubscription()
+	ms.On("Output").Return((<-chan []es.HitT)(chHitT))
+	mm := mmock.NewMockMonitor()
+	mm.On("Subscribe").Return(ms).Once()
+	mm.On("Unsubscribe", mock.Anything).Return().Once()
+	bulker := ftesting.NewMockBulk()
+
+	errorMap := make(map[string]string)
+	errorMap["remote output"] = "error connecting to remote output"
+	bulker.On("GetRemoteOutputErrorMap").Return(errorMap).Once()
+
+	// clear error
+	emptyMap := make(map[string]string)
+	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap).Once()
+
+	monitor := NewSelfMonitor(cfg, bulker, mm, "", reporter)
+	sm := monitor.(*selfMonitorT)
+	sm.checkTime = 100 * time.Millisecond
+
+	var policyLock sync.Mutex
+	var policyResult []model.Policy
+	sm.policyF = func(ctx context.Context, bulker bulk.Bulk, opt ...dl.Option) ([]model.Policy, error) {
+		policyLock.Lock()
+		defer policyLock.Unlock()
+		return policyResult, nil
+	}
+
+	var tokenLock sync.Mutex
+	var tokenResult []model.EnrollmentAPIKey
+	sm.enrollmentTokenF = func(ctx context.Context, bulker bulk.Bulk, policyID string) ([]model.EnrollmentAPIKey, error) {
+		tokenLock.Lock()
+		defer tokenLock.Unlock()
+		return tokenResult, nil
+	}
+
+	var merr error
+	var mwg sync.WaitGroup
+	mwg.Add(1)
+	go func() {
+		defer mwg.Done()
+		merr = monitor.Run(ctx)
+	}()
+
+	if err := monitor.(*selfMonitorT).waitStart(ctx); err != nil {
+		t.Fatal(err)
+	}
+
+	// should be set to starting
+	ftesting.Retry(t, ctx, func(ctx context.Context) error {
+		state, msg, _ := reporter.Current()
+		if state != client.UnitStateStarting {
+			return fmt.Errorf("should be reported as starting; instead its %s", state)
+		}
+		if msg != "Waiting on default policy with Fleet Server integration" {
+			return fmt.Errorf("should be matching with default policy")
+		}
+		return nil
+	}, ftesting.RetrySleep(1*time.Second))
+
+	policyID := uuid.Must(uuid.NewV4()).String()
+	rId := xid.New().String()
+	pData := model.PolicyData{Inputs: []map[string]interface{}{
+		{
+			"type": "fleet-server",
+		},
+	}}
+	policy := model.Policy{
+		ESDocument: model.ESDocument{
+			Id:      rId,
+			Version: 1,
+			SeqNo:   1,
+		},
+		PolicyID:           policyID,
+		CoordinatorIdx:     1,
+		Data:               &pData,
+		RevisionIdx:        1,
+		DefaultFleetServer: true,
+	}
+	policyData, err := json.Marshal(&policy)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	go func() {
+		chHitT <- []es.HitT{{
+			ID:      rId,
+			SeqNo:   1,
+			Version: 1,
+			Source:  policyData,
+		}}
+		policyLock.Lock()
+		defer policyLock.Unlock()
+		policyResult = append(policyResult, policy)
+	}()
+
+	// should now be set to healthy
+	ftesting.Retry(t, ctx, func(ctx context.Context) error {
+		state, msg, _ := reporter.Current()
+		if state != client.UnitStateHealthy {
+			return fmt.Errorf("should be reported as healthy; instead its %s", state)
+		}
+		if msg != "Running on default policy with Fleet Server integration" {
+			return fmt.Errorf("should be matching with default policy")
+		}
+		return nil
+	})
+
+	cancel()
+	mwg.Wait()
+	if merr != nil && merr != context.Canceled {
+		t.Fatal(merr)
+	}
+}
diff --git a/internal/pkg/policy/standalone_test.go b/internal/pkg/policy/standalone_test.go
index 984183056..386591262 100644
--- a/internal/pkg/policy/standalone_test.go
+++ b/internal/pkg/policy/standalone_test.go
@@ -81,7 +81,7 @@ func TestStandAloneSelfMonitor(t *testing.T) {
 			bulker := ftesting.NewMockBulk()
 			bulker.On("Search", searchArguments...).Return(c.searchResult, c.searchErr)
 			emptyMap := make(map[string]string)
-			bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
+			bulker.On("GetRemoteOutputErrorMap").Return(emptyMap).Once()
 			reporter := &FakeReporter{}
 
 			sm := NewStandAloneSelfMonitor(bulker, reporter)
@@ -95,3 +95,40 @@ func TestStandAloneSelfMonitor(t *testing.T) {
 		})
 	}
 }
+
+func TestStandAloneSelfMonitorRemoteOutput(t *testing.T) {
+
+	searchArguments := []any{mock.Anything, ".fleet-policies", mock.Anything, mock.Anything}
+
+	bulker := ftesting.NewMockBulk()
+	bulker.On("Search", searchArguments...).Return(&es.ResultT{
+		Aggregations: map[string]es.Aggregation{
+			dl.FieldPolicyID: es.Aggregation{},
+		},
+	}, nil)
+
+	errorMap := make(map[string]string)
+	errorMap["remote output"] = "error connecting to remote output"
+	bulker.On("GetRemoteOutputErrorMap").Return(errorMap).Once()
+
+	emptyMap := make(map[string]string)
+	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap).Once()
+
+	reporter := &FakeReporter{}
+
+	sm := NewStandAloneSelfMonitor(bulker, reporter)
+	sm.updateState(client.UnitStateStarting, "test")
+
+	sm.check(context.Background())
+	state := sm.State()
+
+	assert.Equal(t, client.UnitStateDegraded, state)
+	assert.Equal(t, state, reporter.state, "reported state should be the same")
+
+	// back to healthy
+	sm.check(context.Background())
+	state = sm.State()
+
+	assert.Equal(t, client.UnitStateHealthy, state)
+	assert.Equal(t, state, reporter.state, "reported state should be the same")
+}

From 608ba26ac5c7d7e7c19e186a2afa286330c14f4c Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 14 Nov 2023 15:40:49 +0100
Subject: [PATCH 47/91] remote es ping in standalone self monitor

---
 internal/pkg/bulk/engine.go            |  5 ++
 internal/pkg/policy/standalone.go      | 23 ++++++++++
 internal/pkg/policy/standalone_test.go | 63 ++++++++++++++++++++++++++
 internal/pkg/testing/bulk.go           |  5 ++
 internal/pkg/testing/esutil/client.go  | 55 ++++++++++++++++++++++
 5 files changed, 151 insertions(+)
 create mode 100644 internal/pkg/testing/esutil/client.go

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index cc77ce9ae..97df4064b 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -72,6 +72,7 @@ type Bulk interface {
 
 	CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, bool, error)
 	GetBulker(outputName string) Bulk
+	GetBulkerMap() map[string]Bulk 
 	GetRemoteOutputErrorMap() map[string]string
 	SetRemoteOutputError(name string, status string)
 	CancelFn() context.CancelFunc
@@ -137,6 +138,10 @@ func (b *Bulker) GetBulker(outputName string) Bulk {
 	return b.bulkerMap[outputName]
 }
 
+func (b *Bulker) GetBulkerMap() map[string]Bulk {
+	return b.bulkerMap
+} 
+
 func (b *Bulker) CancelFn() context.CancelFunc {
 	return b.cancelFn
 }
diff --git a/internal/pkg/policy/standalone.go b/internal/pkg/policy/standalone.go
index cca5898c7..228ea1186 100644
--- a/internal/pkg/policy/standalone.go
+++ b/internal/pkg/policy/standalone.go
@@ -16,6 +16,7 @@ import (
 	"github.com/elastic/fleet-server/v7/internal/pkg/dl"
 	"github.com/elastic/fleet-server/v7/internal/pkg/es"
 	"github.com/elastic/fleet-server/v7/internal/pkg/state"
+	"github.com/elastic/go-elasticsearch/v8/esapi"
 	"github.com/rs/zerolog"
 	"go.elastic.co/apm/v2"
 )
@@ -124,9 +125,31 @@ func (m *standAloneSelfMonitorT) check(ctx context.Context) {
 	if hasError {
 		state = client.UnitStateDegraded
 		message = fmt.Sprintf("Could not connect to remote ES output: %+v", remoteESPayload)
+		m.log.Debug().Msg(message)
+	} else {
+		bulkerMap := m.bulker.GetBulkerMap()
+		for outputName, outputBulker := range bulkerMap {
+			res, err := outputBulker.Client().Ping(outputBulker.Client().Ping.WithContext(ctx))
+			if err != nil {
+				m.log.Error().Err(err).Msg("error calling remote es ping")
+				state = client.UnitStateDegraded
+				message = fmt.Sprintf("Could not ping remote ES: %s, error: %s", outputName, err.Error())
+			} else if res.StatusCode != 200 {
+				state = client.UnitStateDegraded
+				message = fmt.Sprintf("Could not connect to remote ES output: %s, status code: %d", outputName, res.StatusCode)
+				m.log.Debug().Msg(message)
+			} else {
+				state = client.UnitStateHealthy
+				message = ""
+			}
+		}
 	}
 
 	if current != state {
 		m.updateState(state, message)
 	}
 }
+
+func GetRequest(req *esapi.CatHealthRequest) {
+	req.Format = "json"
+}
diff --git a/internal/pkg/policy/standalone_test.go b/internal/pkg/policy/standalone_test.go
index 386591262..bfd9e490e 100644
--- a/internal/pkg/policy/standalone_test.go
+++ b/internal/pkg/policy/standalone_test.go
@@ -9,15 +9,20 @@ package policy
 import (
 	"context"
 	"errors"
+	"io/ioutil"
+	"net/http"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 
 	"github.com/elastic/elastic-agent-client/v7/pkg/client"
+	"github.com/elastic/fleet-server/v7/internal/pkg/bulk"
 	"github.com/elastic/fleet-server/v7/internal/pkg/dl"
 	"github.com/elastic/fleet-server/v7/internal/pkg/es"
 	ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing"
+	"github.com/elastic/fleet-server/v7/internal/pkg/testing/esutil"
 )
 
 func TestStandAloneSelfMonitor(t *testing.T) {
@@ -82,6 +87,8 @@ func TestStandAloneSelfMonitor(t *testing.T) {
 			bulker.On("Search", searchArguments...).Return(c.searchResult, c.searchErr)
 			emptyMap := make(map[string]string)
 			bulker.On("GetRemoteOutputErrorMap").Return(emptyMap).Once()
+			emptyBulkerMap := make(map[string]bulk.Bulk)
+			bulker.On("GetBulkerMap").Return(emptyBulkerMap).Once()
 			reporter := &FakeReporter{}
 
 			sm := NewStandAloneSelfMonitor(bulker, reporter)
@@ -114,6 +121,9 @@ func TestStandAloneSelfMonitorRemoteOutput(t *testing.T) {
 	emptyMap := make(map[string]string)
 	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap).Once()
 
+	emptyBulkerMap := make(map[string]bulk.Bulk)
+	bulker.On("GetBulkerMap").Return(emptyBulkerMap).Once()
+
 	reporter := &FakeReporter{}
 
 	sm := NewStandAloneSelfMonitor(bulker, reporter)
@@ -132,3 +142,56 @@ func TestStandAloneSelfMonitorRemoteOutput(t *testing.T) {
 	assert.Equal(t, client.UnitStateHealthy, state)
 	assert.Equal(t, state, reporter.state, "reported state should be the same")
 }
+
+func TestStandAloneSelfMonitorRemoteOutputPing(t *testing.T) {
+
+	searchArguments := []any{mock.Anything, ".fleet-policies", mock.Anything, mock.Anything}
+
+	bulker := ftesting.NewMockBulk()
+	bulker.On("Search", searchArguments...).Return(&es.ResultT{
+		Aggregations: map[string]es.Aggregation{
+			dl.FieldPolicyID: es.Aggregation{},
+		},
+	}, nil)
+
+	emptyMap := make(map[string]string)
+	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
+
+	bulkerMap := make(map[string]bulk.Bulk)
+	outputBulker := ftesting.NewMockBulk()
+	mockES, mocktrans := esutil.MockESClient(t)
+
+	mocktrans.Response = &http.Response{
+		StatusCode: http.StatusInternalServerError,
+		Body:       nil,
+	}
+
+	outputBulker.On("Client").Return(mockES)
+	bulkerMap["output1"] = outputBulker
+	bulker.On("GetBulkerMap").Return(bulkerMap)
+
+	reporter := &FakeReporter{}
+
+	sm := NewStandAloneSelfMonitor(bulker, reporter)
+	sm.updateState(client.UnitStateStarting, "test")
+
+	sm.check(context.Background())
+	state := sm.State()
+
+	assert.Equal(t, client.UnitStateDegraded, state)
+	assert.Equal(t, state, reporter.state, "reported state should be the same")
+
+	// back to healthy
+	mocktrans.Response = &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       ioutil.NopCloser(strings.NewReader(`{}`)),
+	}
+
+	sm.check(context.Background())
+	state = sm.State()
+
+	// TODO does not work, gives error "the client noticed that the server is not Elasticsearch and we do not support this unknown product"
+	// assert.Equal(t, client.UnitStateHealthy, state)
+	// assert.Equal(t, state, reporter.state, "reported state should be the same")
+
+}
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index b96b54a34..0dc5e0543 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -85,6 +85,11 @@ func (m *MockBulk) GetBulker(outputName string) bulk.Bulk {
 	return args.Get(0).(bulk.Bulk)
 }
 
+func (m *MockBulk) GetBulkerMap() map[string]bulk.Bulk {
+	args := m.Called()
+	return args.Get(0).(map[string]bulk.Bulk)
+}
+
 func (m *MockBulk) CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (bulk.Bulk, bool, error) {
 	args := m.Called(zlog, outputName, serviceToken, outputMap)
 	return args.Get(0).(bulk.Bulk), args.Get(1).(bool), nil
diff --git a/internal/pkg/testing/esutil/client.go b/internal/pkg/testing/esutil/client.go
new file mode 100644
index 000000000..bd9cea49a
--- /dev/null
+++ b/internal/pkg/testing/esutil/client.go
@@ -0,0 +1,55 @@
+package esutil
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/elastic/go-elasticsearch/v8"
+	"github.com/stretchr/testify/require"
+)
+
+/*
+	Setup to convert a *elasticsearch.Client as a harmless mock
+	by replacing the Transport to nowhere
+*/
+
+type MockTransport struct {
+	Response    *http.Response
+	RoundTripFn func(req *http.Request) (*http.Response, error)
+}
+
+func (t *MockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	return t.RoundTripFn(req)
+}
+
+func MockESClient(t *testing.T) (*elasticsearch.Client, *MockTransport) {
+	mocktrans := MockTransport{
+		Response: sendBodyString("{}"), //nolint:bodyclose // nopcloser is used, linter does not see it
+	}
+
+	mocktrans.RoundTripFn = func(req *http.Request) (*http.Response, error) { return mocktrans.Response, nil }
+	client, err := elasticsearch.NewClient(elasticsearch.Config{
+		Transport: &mocktrans,
+	})
+	require.NoError(t, err)
+	return client, &mocktrans
+}
+
+func sendBodyString(body string) *http.Response {
+	return sendBody(strings.NewReader(body))
+}
+func sendBodyBytes(body []byte) *http.Response { return sendBody(bytes.NewReader(body)) }
+func sendBody(body io.Reader) *http.Response {
+	return &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       ioutil.NopCloser(body),
+		Header: http.Header{
+			"X-Elastic-Product": []string{"Elasticsearch"},
+			"Content-Type":      []string{"application/cbor"},
+		},
+	}
+}

From 699ac62634ea238fa9b643b02a5b3896c0772f07 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 14 Nov 2023 15:52:21 +0100
Subject: [PATCH 48/91] added break

---
 internal/pkg/policy/standalone.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/internal/pkg/policy/standalone.go b/internal/pkg/policy/standalone.go
index 228ea1186..386e8c3e6 100644
--- a/internal/pkg/policy/standalone.go
+++ b/internal/pkg/policy/standalone.go
@@ -120,6 +120,7 @@ func (m *standAloneSelfMonitorT) check(ctx context.Context) {
 		if value != "" {
 			hasError = true
 			remoteESPayload[key] = value
+			break
 		}
 	}
 	if hasError {
@@ -134,13 +135,12 @@ func (m *standAloneSelfMonitorT) check(ctx context.Context) {
 				m.log.Error().Err(err).Msg("error calling remote es ping")
 				state = client.UnitStateDegraded
 				message = fmt.Sprintf("Could not ping remote ES: %s, error: %s", outputName, err.Error())
+				break
 			} else if res.StatusCode != 200 {
 				state = client.UnitStateDegraded
 				message = fmt.Sprintf("Could not connect to remote ES output: %s, status code: %d", outputName, res.StatusCode)
 				m.log.Debug().Msg(message)
-			} else {
-				state = client.UnitStateHealthy
-				message = ""
+				break
 			}
 		}
 	}

From 81899b3c9dd4d676666145fab75749106ac813b0 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 14 Nov 2023 15:53:14 +0100
Subject: [PATCH 49/91] fix lint

---
 internal/pkg/testing/esutil/client.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/internal/pkg/testing/esutil/client.go b/internal/pkg/testing/esutil/client.go
index bd9cea49a..434892de0 100644
--- a/internal/pkg/testing/esutil/client.go
+++ b/internal/pkg/testing/esutil/client.go
@@ -1,7 +1,6 @@
 package esutil
 
 import (
-	"bytes"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -42,7 +41,6 @@ func MockESClient(t *testing.T) (*elasticsearch.Client, *MockTransport) {
 func sendBodyString(body string) *http.Response {
 	return sendBody(strings.NewReader(body))
 }
-func sendBodyBytes(body []byte) *http.Response { return sendBody(bytes.NewReader(body)) }
 func sendBody(body io.Reader) *http.Response {
 	return &http.Response{
 		StatusCode: http.StatusOK,

From 14f9507cc3c4eec08cac8d72cbfae77191320ec5 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 14 Nov 2023 16:15:53 +0100
Subject: [PATCH 50/91] fix issue

---
 internal/pkg/api/handleAck.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go
index 7bd5d7fb4..d1b491f51 100644
--- a/internal/pkg/api/handleAck.go
+++ b/internal/pkg/api/handleAck.go
@@ -567,7 +567,10 @@ func (ack *AckT) invalidateAPIKeys(ctx context.Context, zlog zerolog.Logger, toR
 	// using remote es bulker to invalidate api key - supposing all retire api key ids belong to the same remote es
 	bulk := ack.bulk
 	if output != "" {
-		bulk = ack.bulk.GetBulker(output)
+		outputBulk := ack.bulk.GetBulker(output)
+		if outputBulk != nil {
+			bulk = outputBulk
+		}
 	}
 
 	if len(ids) > 0 {

From 963b87abce885be61a6f47f369eb043bb5d07121 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 14 Nov 2023 16:21:46 +0100
Subject: [PATCH 51/91] license header

---
 internal/pkg/testing/esutil/client.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/internal/pkg/testing/esutil/client.go b/internal/pkg/testing/esutil/client.go
index 434892de0..d95ba31f3 100644
--- a/internal/pkg/testing/esutil/client.go
+++ b/internal/pkg/testing/esutil/client.go
@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package esutil
 
 import (

From 180cae897ebdfdfc17903eecd767667540c62d1f Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 14 Nov 2023 16:48:37 +0100
Subject: [PATCH 52/91] fix lint

---
 internal/pkg/bulk/engine.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 97df4064b..f22c011e8 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -72,7 +72,7 @@ type Bulk interface {
 
 	CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, bool, error)
 	GetBulker(outputName string) Bulk
-	GetBulkerMap() map[string]Bulk 
+	GetBulkerMap() map[string]Bulk
 	GetRemoteOutputErrorMap() map[string]string
 	SetRemoteOutputError(name string, status string)
 	CancelFn() context.CancelFunc
@@ -140,7 +140,7 @@ func (b *Bulker) GetBulker(outputName string) Bulk {
 
 func (b *Bulker) GetBulkerMap() map[string]Bulk {
 	return b.bulkerMap
-} 
+}
 
 func (b *Bulker) CancelFn() context.CancelFunc {
 	return b.cancelFn

From 3989856628a062cdd678374675c5a6d1af7906c2 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Wed, 15 Nov 2023 11:32:31 +0100
Subject: [PATCH 53/91] cleanup,  more test

---
 internal/pkg/api/handleAck_test.go | 40 ++++++++++++++++++++++++++++++
 internal/pkg/bulk/engine.go        |  5 ++--
 internal/pkg/server/fleet.go       |  5 +---
 3 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/internal/pkg/api/handleAck_test.go b/internal/pkg/api/handleAck_test.go
index 13b68d482..d6f5f0d01 100644
--- a/internal/pkg/api/handleAck_test.go
+++ b/internal/pkg/api/handleAck_test.go
@@ -571,6 +571,46 @@ func TestInvalidateAPIKeys(t *testing.T) {
 	}
 }
 
+func TestInvalidateAPIKeysRemoteOutput(t *testing.T) {
+	toRetire1 := []model.ToRetireAPIKeyIdsItems{{
+		ID:     "toRetire1",
+		Output: "remote1",
+	}}
+
+	wants := map[string][]string{
+		"1": {"toRetire1"},
+	}
+
+	agent := model.Agent{
+		Outputs: map[string]*model.PolicyOutput{
+			"1": {ToRetireAPIKeyIds: toRetire1},
+		},
+	}
+
+	for i, out := range agent.Outputs {
+		want := wants[i]
+
+		bulker := ftesting.NewMockBulk()
+		remoteBulker := ftesting.NewMockBulk()
+		bulker.On("GetBulker", mock.AnythingOfType("string")).Return(remoteBulker)
+		if len(want) > 0 {
+			remoteBulker.On("APIKeyInvalidate",
+				context.Background(), mock.MatchedBy(func(ids []string) bool {
+					// if A contains B and B contains A => A = B
+					return assert.Subset(t, ids, want) &&
+						assert.Subset(t, want, ids)
+				})).
+				Return(nil)
+		}
+
+		logger := testlog.SetLogger(t)
+		ack := &AckT{bulk: bulker}
+		ack.invalidateAPIKeys(context.Background(), logger, out.ToRetireAPIKeyIds, "")
+
+		bulker.AssertExpectations(t)
+	}
+}
+
 func TestAckHandleUpgrade(t *testing.T) {
 	tests := []struct {
 		name   string
diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index f22c011e8..aaf4681b4 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -238,13 +238,14 @@ func (b *Bulker) Tracer() *apm.Tracer {
 	return b.tracer
 }
 
-// check if remote output cfg changed, and signal to remoteOutputCh channel if so
+// check if remote output cfg changed
 func (b *Bulker) CheckRemoteOutputChanged(zlog zerolog.Logger, name string, newCfg map[string]interface{}) bool {
 	curCfg := b.remoteOutputConfigMap[name]
 
 	hasChanged := false
 
-	// ignore output sent to agents where type is set to elasticsearch
+	// TODO remoteOutputConfigMap empty when FS restarts - won't detect changes
+	// when output config first added, not reporting change
 	if curCfg != nil && !reflect.DeepEqual(curCfg, newCfg) {
 		zlog.Info().Str("name", name).Msg("remote output configuration has changed")
 		hasChanged = true
diff --git a/internal/pkg/server/fleet.go b/internal/pkg/server/fleet.go
index 4197962a2..be3c9410e 100644
--- a/internal/pkg/server/fleet.go
+++ b/internal/pkg/server/fleet.go
@@ -132,8 +132,6 @@ func (f *Fleet) Run(ctx context.Context, initCfg *config.Config) error {
 	started := false
 	ech := make(chan error, 2)
 
-	outputChanged := false
-
 LOOP:
 	for {
 		if started {
@@ -175,7 +173,7 @@ LOOP:
 		}
 
 		// Start or restart server
-		if configChangedServer(*log, curCfg, newCfg) || outputChanged {
+		if configChangedServer(*log, curCfg, newCfg) {
 			if srvCancel != nil {
 				log.Info().Msg("stopping server on configuration change")
 				stop(srvCancel, srvEg)
@@ -193,7 +191,6 @@ LOOP:
 		}
 
 		curCfg = newCfg
-		outputChanged = false
 
 		select {
 		case newCfg = <-f.cfgCh:

From 6b727b892aaf8a7d949bb279a43e52535e807d92 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Wed, 15 Nov 2023 14:28:43 +0100
Subject: [PATCH 54/91] added test on engine

---
 ...ged_test.go => bulk_remote_output_test.go} | 59 +++++++++++++++++++
 internal/pkg/bulk/engine.go                   |  5 +-
 2 files changed, 63 insertions(+), 1 deletion(-)
 rename internal/pkg/bulk/{remote_output_changed_test.go => bulk_remote_output_test.go} (53%)

diff --git a/internal/pkg/bulk/remote_output_changed_test.go b/internal/pkg/bulk/bulk_remote_output_test.go
similarity index 53%
rename from internal/pkg/bulk/remote_output_changed_test.go
rename to internal/pkg/bulk/bulk_remote_output_test.go
index 4ea8e5bab..488735215 100644
--- a/internal/pkg/bulk/remote_output_changed_test.go
+++ b/internal/pkg/bulk/bulk_remote_output_test.go
@@ -85,3 +85,62 @@ func Test_CheckRemoteOutputChanged(t *testing.T) {
 		})
 	}
 }
+
+func Test_CreateAndGetBulkerNew(t *testing.T) {
+	log := testlog.SetLogger(t)
+	bulker := NewBulker(nil, nil)
+	outputMap := make(map[string]map[string]interface{})
+	outputMap["remote1"] = map[string]interface{}{
+		"type":          "remote_elasticsearch",
+		"hosts":         []interface{}{"https://remote-es:443"},
+		"service_token": "token1",
+	}
+	newBulker, hasChanged, err := bulker.CreateAndGetBulker(log, "remote1", "token1", outputMap)
+	assert.NotNil(t, newBulker)
+	assert.Equal(t, false, hasChanged)
+	assert.Nil(t, err)
+}
+
+func Test_CreateAndGetBulkerExisting(t *testing.T) {
+	log := testlog.SetLogger(t)
+	bulker := NewBulker(nil, nil)
+	outputBulker := NewBulker(nil, nil)
+	bulker.bulkerMap["remote1"] = outputBulker
+	outputMap := make(map[string]map[string]interface{})
+	cfg := map[string]interface{}{
+		"type":          "remote_elasticsearch",
+		"hosts":         []interface{}{"https://remote-es:443"},
+		"service_token": "token1",
+	}
+	bulker.remoteOutputConfigMap["remote1"] = cfg
+	outputMap["remote1"] = cfg
+	newBulker, hasChanged, err := bulker.CreateAndGetBulker(log, "remote1", "token1", outputMap)
+	assert.Equal(t, outputBulker, newBulker)
+	assert.Equal(t, false, hasChanged)
+	assert.Nil(t, err)
+}
+
+func Test_CreateAndGetBulkerChanged(t *testing.T) {
+	log := testlog.SetLogger(t)
+	bulker := NewBulker(nil, nil)
+	outputBulker := NewBulker(nil, nil)
+	bulker.bulkerMap["remote1"] = outputBulker
+	outputMap := make(map[string]map[string]interface{})
+	bulker.remoteOutputConfigMap["remote1"] = map[string]interface{}{
+		"type":          "remote_elasticsearch",
+		"hosts":         []interface{}{"https://remote-es:443"},
+		"service_token": "token1",
+	}
+	outputMap["remote1"] = map[string]interface{}{
+		"type":          "remote_elasticsearch",
+		"hosts":         []interface{}{"https://remote-es:443"},
+		"service_token": "token2",
+	}
+	cancelFnCalled := false
+	outputBulker.cancelFn = func() { cancelFnCalled = true }
+	newBulker, hasChanged, err := bulker.CreateAndGetBulker(log, "remote1", "token2", outputMap)
+	assert.NotEqual(t, outputBulker, newBulker)
+	assert.Equal(t, true, hasChanged)
+	assert.Nil(t, err)
+	assert.Equal(t, true, cancelFnCalled)
+}
diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index aaf4681b4..13d163464 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -153,7 +153,10 @@ func (b *Bulker) CreateAndGetBulker(zlog zerolog.Logger, outputName string, serv
 		return bulker, false, nil
 	}
 	if bulker != nil && hasConfigChanged {
-		bulker.CancelFn()
+		cancelFn := bulker.CancelFn()
+		if cancelFn != nil {
+			cancelFn()
+		}
 	}
 	bulkCtx, bulkCancel := context.WithCancel(context.Background())
 	es, err := b.createRemoteEsClient(bulkCtx, outputName, serviceToken, outputMap)

From 60a3122d95fddfac673676e21b708981527a37a8 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Wed, 15 Nov 2023 15:27:39 +0100
Subject: [PATCH 55/91] use output bulker when read update api key

---
 internal/pkg/api/handleAck.go | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go
index d1b491f51..a1fd554f6 100644
--- a/internal/pkg/api/handleAck.go
+++ b/internal/pkg/api/handleAck.go
@@ -416,7 +416,7 @@ func (ack *AckT) handlePolicyChange(ctx context.Context, zlog zerolog.Logger, ag
 		return nil
 	}
 
-	for _, output := range agent.Outputs {
+	for outputName, output := range agent.Outputs {
 		if output.Type != policy.OutputTypeElasticsearch {
 			continue
 		}
@@ -424,7 +424,7 @@ func (ack *AckT) handlePolicyChange(ctx context.Context, zlog zerolog.Logger, ag
 		err := ack.updateAPIKey(ctx,
 			zlog,
 			agent.Id,
-			output.APIKeyID, output.PermissionsHash, output.ToRetireAPIKeyIds)
+			output.APIKeyID, output.PermissionsHash, output.ToRetireAPIKeyIds, outputName)
 		if err != nil {
 			return err
 		}
@@ -445,20 +445,31 @@ func (ack *AckT) updateAPIKey(ctx context.Context,
 	zlog zerolog.Logger,
 	agentID string,
 	apiKeyID, permissionHash string,
-	toRetireAPIKeyIDs []model.ToRetireAPIKeyIdsItems) error {
+	toRetireAPIKeyIDs []model.ToRetireAPIKeyIdsItems, outputName string) error {
+	bulk := ack.bulk
+	// use output bulker if exists
+	if outputName != "" {
+		outputBulk := ack.bulk.GetBulker(outputName)
+		if outputBulk != nil {
+			zlog.Debug().Str("outputName", outputName).Msg("Using output bulker in updateAPIKey")
+			bulk = outputBulk
+		}
+	}
 	if apiKeyID != "" {
-		res, err := ack.bulk.APIKeyRead(ctx, apiKeyID, true)
+		res, err := bulk.APIKeyRead(ctx, apiKeyID, true)
 		if err != nil {
 			if isAgentActive(ctx, zlog, ack.bulk, agentID) {
 				zlog.Error().
 					Err(err).
 					Str(LogAPIKeyID, apiKeyID).
+					Str("outputName", outputName).
 					Msg("Failed to read API Key roles")
 			} else {
 				// race when API key was invalidated before acking
 				zlog.Info().
 					Err(err).
 					Str(LogAPIKeyID, apiKeyID).
+					Str("outputName", outputName).
 					Msg("Failed to read invalidated API Key roles")
 
 				// prevents future checks
@@ -473,14 +484,15 @@ func (ack *AckT) updateAPIKey(ctx context.Context,
 					Str(LogAPIKeyID, apiKeyID).
 					Msg("Failed to cleanup roles")
 			} else if removedRolesCount > 0 {
-				if err := ack.bulk.APIKeyUpdate(ctx, apiKeyID, permissionHash, clean); err != nil {
-					zlog.Error().Err(err).RawJSON("roles", clean).Str(LogAPIKeyID, apiKeyID).Msg("Failed to update API Key")
+				if err := bulk.APIKeyUpdate(ctx, apiKeyID, permissionHash, clean); err != nil {
+					zlog.Error().Err(err).RawJSON("roles", clean).Str(LogAPIKeyID, apiKeyID).Str("outputName", outputName).Msg("Failed to update API Key")
 				} else {
 					zlog.Debug().
 						Str("hash.sha256", permissionHash).
 						Str(LogAPIKeyID, apiKeyID).
 						RawJSON("roles", clean).
 						Int("removedRoles", removedRolesCount).
+						Str("outputName", outputName).
 						Msg("Updating agent record to pick up reduced roles.")
 				}
 			}

From 8797823a149f773f73dae4ce4727168005a94a98 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Wed, 15 Nov 2023 16:20:43 +0100
Subject: [PATCH 56/91] added remote es ping to self.go

---
 internal/pkg/policy/policy_output.go |   2 +-
 internal/pkg/policy/self.go          |  24 +++++
 internal/pkg/policy/self_test.go     | 150 +++++++++++++++++++++++++++
 3 files changed, 175 insertions(+), 1 deletion(-)

diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index 1b6e6a7b4..4cb014116 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -189,7 +189,7 @@ func (p *Output) prepareElasticsearch(
 		// document on ES does not have OutputPermissionsHash for any other output
 		// besides the default one. It seems to me error-prone to rely on the default
 		// output permissions hash to generate new API keys for other outputs.
-		zlog.Debug().Msg("must generate api key as policy output permissions changed")
+		zlog.Debug().Msg("must update api key as policy output permissions changed")
 		needUpdateKey = true
 	default:
 		zlog.Debug().Msg("policy output permissions are the same")
diff --git a/internal/pkg/policy/self.go b/internal/pkg/policy/self.go
index 5a14374a6..e922df9dd 100644
--- a/internal/pkg/policy/self.go
+++ b/internal/pkg/policy/self.go
@@ -225,12 +225,36 @@ func (m *selfMonitorT) updateState(ctx context.Context) (client.UnitState, error
 		if value != "" {
 			hasError = true
 			remoteESPayload[key] = value
+			break
 		}
 	}
 	if hasError {
 		m.state = client.UnitStateDegraded
 		m.reporter.UpdateState(client.UnitStateDegraded, "Could not connect to remote ES output", remoteESPayload) //nolint:errcheck // not clear what to do in failure cases
 		return client.UnitStateDegraded, nil
+	} else {
+		bulkerMap := m.bulker.GetBulkerMap()
+		for outputName, outputBulker := range bulkerMap {
+			res, err := outputBulker.Client().Ping(outputBulker.Client().Ping.WithContext(ctx))
+			if err != nil {
+				m.log.Error().Err(err).Msg("error calling remote es ping")
+				m.state = client.UnitStateDegraded
+				message := fmt.Sprintf("Could not ping remote ES: %s, error: %s", outputName, err.Error())
+				m.reporter.UpdateState(m.state, message, nil) //nolint:errcheck // not clear what to do in failure cases
+				hasError = true
+				break
+			} else if res.StatusCode != 200 {
+				m.state = client.UnitStateDegraded
+				message := fmt.Sprintf("Could not connect to remote ES output: %s, status code: %d", outputName, res.StatusCode)
+				m.log.Debug().Msg(message)
+				m.reporter.UpdateState(m.state, message, nil) //nolint:errcheck // not clear what to do in failure cases
+				hasError = true
+				break
+			}
+		}
+		if hasError {
+			return m.state, nil
+		}
 	}
 
 	state := client.UnitStateHealthy
diff --git a/internal/pkg/policy/self_test.go b/internal/pkg/policy/self_test.go
index ab347700b..1b8431ee9 100644
--- a/internal/pkg/policy/self_test.go
+++ b/internal/pkg/policy/self_test.go
@@ -10,6 +10,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"sync"
 	"testing"
 	"time"
@@ -26,6 +27,7 @@ import (
 	"github.com/elastic/fleet-server/v7/internal/pkg/model"
 	mmock "github.com/elastic/fleet-server/v7/internal/pkg/monitor/mock"
 	ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing"
+	"github.com/elastic/fleet-server/v7/internal/pkg/testing/esutil"
 )
 
 func TestSelfMonitor_DefaultPolicy(t *testing.T) {
@@ -49,6 +51,8 @@ func TestSelfMonitor_DefaultPolicy(t *testing.T) {
 	bulker := ftesting.NewMockBulk()
 	emptyMap := make(map[string]string)
 	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
+	emptyBulkerMap := make(map[string]bulk.Bulk)
+	bulker.On("GetBulkerMap").Return(emptyBulkerMap)
 
 	monitor := NewSelfMonitor(cfg, bulker, mm, "", reporter)
 	sm := monitor.(*selfMonitorT)
@@ -188,6 +192,8 @@ func TestSelfMonitor_DefaultPolicy_Degraded(t *testing.T) {
 
 	emptyMap := make(map[string]string)
 	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
+	emptyBulkerMap := make(map[string]bulk.Bulk)
+	bulker.On("GetBulkerMap").Return(emptyBulkerMap)
 
 	monitor := NewSelfMonitor(cfg, bulker, mm, "", reporter)
 	sm := monitor.(*selfMonitorT)
@@ -347,6 +353,8 @@ func TestSelfMonitor_SpecificPolicy(t *testing.T) {
 	bulker := ftesting.NewMockBulk()
 	emptyMap := make(map[string]string)
 	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
+	emptyBulkerMap := make(map[string]bulk.Bulk)
+	bulker.On("GetBulkerMap").Return(emptyBulkerMap)
 
 	monitor := NewSelfMonitor(cfg, bulker, mm, policyID, reporter)
 	sm := monitor.(*selfMonitorT)
@@ -485,6 +493,8 @@ func TestSelfMonitor_SpecificPolicy_Degraded(t *testing.T) {
 	bulker := ftesting.NewMockBulk()
 	emptyMap := make(map[string]string)
 	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
+	emptyBulkerMap := make(map[string]bulk.Bulk)
+	bulker.On("GetBulkerMap").Return(emptyBulkerMap)
 
 	monitor := NewSelfMonitor(cfg, bulker, mm, policyID, reporter)
 	sm := monitor.(*selfMonitorT)
@@ -665,6 +675,8 @@ func TestSelfMonitor_RemoteOutput_Degraded(t *testing.T) {
 	errorMap := make(map[string]string)
 	errorMap["remote output"] = "error connecting to remote output"
 	bulker.On("GetRemoteOutputErrorMap").Return(errorMap)
+	emptyBulkerMap := make(map[string]bulk.Bulk)
+	bulker.On("GetBulkerMap").Return(emptyBulkerMap)
 
 	monitor := NewSelfMonitor(cfg, bulker, mm, "", reporter)
 	sm := monitor.(*selfMonitorT)
@@ -793,6 +805,9 @@ func TestSelfMonitor_RemoteOutput_Back_To_Healthy(t *testing.T) {
 	emptyMap := make(map[string]string)
 	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap).Once()
 
+	emptyBulkerMap := make(map[string]bulk.Bulk)
+	bulker.On("GetBulkerMap").Return(emptyBulkerMap)
+
 	monitor := NewSelfMonitor(cfg, bulker, mm, "", reporter)
 	sm := monitor.(*selfMonitorT)
 	sm.checkTime = 100 * time.Millisecond
@@ -891,3 +906,138 @@ func TestSelfMonitor_RemoteOutput_Back_To_Healthy(t *testing.T) {
 		t.Fatal(merr)
 	}
 }
+
+func TestSelfMonitor_RemoteOutput_Ping_Degraded(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	cfg := config.Fleet{
+		Agent: config.Agent{
+			ID: "agent-id",
+		},
+	}
+	reporter := &FakeReporter{}
+
+	chHitT := make(chan []es.HitT, 1)
+	defer close(chHitT)
+	ms := mmock.NewMockSubscription()
+	ms.On("Output").Return((<-chan []es.HitT)(chHitT))
+	mm := mmock.NewMockMonitor()
+	mm.On("Subscribe").Return(ms).Once()
+	mm.On("Unsubscribe", mock.Anything).Return().Once()
+	bulker := ftesting.NewMockBulk()
+
+	emptyMap := make(map[string]string)
+	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
+
+	bulkerMap := make(map[string]bulk.Bulk)
+	outputBulker := ftesting.NewMockBulk()
+	mockES, mocktrans := esutil.MockESClient(t)
+
+	mocktrans.Response = &http.Response{
+		StatusCode: http.StatusInternalServerError,
+		Body:       nil,
+	}
+
+	outputBulker.On("Client").Return(mockES)
+	bulkerMap["output1"] = outputBulker
+	bulker.On("GetBulkerMap").Return(bulkerMap)
+
+	monitor := NewSelfMonitor(cfg, bulker, mm, "", reporter)
+	sm := monitor.(*selfMonitorT)
+	sm.checkTime = 100 * time.Millisecond
+
+	var policyLock sync.Mutex
+	var policyResult []model.Policy
+	sm.policyF = func(ctx context.Context, bulker bulk.Bulk, opt ...dl.Option) ([]model.Policy, error) {
+		policyLock.Lock()
+		defer policyLock.Unlock()
+		return policyResult, nil
+	}
+
+	var tokenLock sync.Mutex
+	var tokenResult []model.EnrollmentAPIKey
+	sm.enrollmentTokenF = func(ctx context.Context, bulker bulk.Bulk, policyID string) ([]model.EnrollmentAPIKey, error) {
+		tokenLock.Lock()
+		defer tokenLock.Unlock()
+		return tokenResult, nil
+	}
+
+	var merr error
+	var mwg sync.WaitGroup
+	mwg.Add(1)
+	go func() {
+		defer mwg.Done()
+		merr = monitor.Run(ctx)
+	}()
+
+	if err := monitor.(*selfMonitorT).waitStart(ctx); err != nil {
+		t.Fatal(err)
+	}
+
+	// should be set to starting
+	ftesting.Retry(t, ctx, func(ctx context.Context) error {
+		state, msg, _ := reporter.Current()
+		if state != client.UnitStateStarting {
+			return fmt.Errorf("should be reported as starting; instead its %s", state)
+		}
+		if msg != "Waiting on default policy with Fleet Server integration" {
+			return fmt.Errorf("should be matching with default policy")
+		}
+		return nil
+	}, ftesting.RetrySleep(1*time.Second))
+
+	policyID := uuid.Must(uuid.NewV4()).String()
+	rId := xid.New().String()
+	pData := model.PolicyData{Inputs: []map[string]interface{}{
+		{
+			"type": "fleet-server",
+		},
+	}}
+	policy := model.Policy{
+		ESDocument: model.ESDocument{
+			Id:      rId,
+			Version: 1,
+			SeqNo:   1,
+		},
+		PolicyID:           policyID,
+		CoordinatorIdx:     1,
+		Data:               &pData,
+		RevisionIdx:        1,
+		DefaultFleetServer: true,
+	}
+	policyData, err := json.Marshal(&policy)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	go func() {
+		chHitT <- []es.HitT{{
+			ID:      rId,
+			SeqNo:   1,
+			Version: 1,
+			Source:  policyData,
+		}}
+		policyLock.Lock()
+		defer policyLock.Unlock()
+		policyResult = append(policyResult, policy)
+	}()
+
+	// should be set to degraded because of remote output error
+	ftesting.Retry(t, ctx, func(ctx context.Context) error {
+		state, msg, _ := reporter.Current()
+		if state != client.UnitStateDegraded {
+			return fmt.Errorf("should be reported as degraded; instead its %s", state)
+		}
+		if msg != "Could not connect to remote ES output: output1, status code: 500" {
+			return fmt.Errorf("expected remote ES error")
+		}
+		return nil
+	}, ftesting.RetrySleep(1*time.Second))
+
+	cancel()
+	mwg.Wait()
+	if merr != nil && merr != context.Canceled {
+		t.Fatal(merr)
+	}
+}

From 1be057590bc9ad13ab507297c3b0fff1a444325d Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 16 Nov 2023 11:17:58 +0100
Subject: [PATCH 57/91] review comments

---
 internal/pkg/api/handleAck_test.go           |  1 +
 internal/pkg/bulk/bulk_remote_output_test.go |  5 ++--
 internal/pkg/bulk/engine.go                  | 27 ++++++++++++--------
 internal/pkg/policy/policy_output.go         | 17 ++++++------
 internal/pkg/testing/bulk.go                 |  5 ----
 5 files changed, 30 insertions(+), 25 deletions(-)

diff --git a/internal/pkg/api/handleAck_test.go b/internal/pkg/api/handleAck_test.go
index d6f5f0d01..1cd0b9019 100644
--- a/internal/pkg/api/handleAck_test.go
+++ b/internal/pkg/api/handleAck_test.go
@@ -608,6 +608,7 @@ func TestInvalidateAPIKeysRemoteOutput(t *testing.T) {
 		ack.invalidateAPIKeys(context.Background(), logger, out.ToRetireAPIKeyIds, "")
 
 		bulker.AssertExpectations(t)
+		remoteBulker.AssertExpectations(t)
 	}
 }
 
diff --git a/internal/pkg/bulk/bulk_remote_output_test.go b/internal/pkg/bulk/bulk_remote_output_test.go
index 488735215..5b5dab693 100644
--- a/internal/pkg/bulk/bulk_remote_output_test.go
+++ b/internal/pkg/bulk/bulk_remote_output_test.go
@@ -13,7 +13,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func Test_CheckRemoteOutputChanged(t *testing.T) {
+func Test_hasChangedAndUpdateRemoteOutputConfig(t *testing.T) {
 	testcases := []struct {
 		name    string
 		cfg     map[string]interface{}
@@ -80,8 +80,9 @@ func Test_CheckRemoteOutputChanged(t *testing.T) {
 			log := testlog.SetLogger(t)
 			bulker := NewBulker(nil, nil)
 			bulker.remoteOutputConfigMap["remote1"] = tc.cfg
-			hasChanged := bulker.CheckRemoteOutputChanged(log, "remote1", tc.newCfg)
+			hasChanged := bulker.hasChangedAndUpdateRemoteOutputConfig(log, "remote1", tc.newCfg)
 			assert.Equal(t, tc.changed, hasChanged)
+			assert.Equal(t, tc.newCfg, bulker.remoteOutputConfigMap["remote1"])
 		})
 	}
 }
diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 13d163464..3efa08388 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -68,8 +68,6 @@ type Bulk interface {
 	// Accessor used to talk to elastic search direcly bypassing bulk engine
 	Client() *elasticsearch.Client
 
-	CheckRemoteOutputChanged(zlog zerolog.Logger, name string, newCfg map[string]interface{}) bool
-
 	CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, bool, error)
 	GetBulker(outputName string) Bulk
 	GetBulkerMap() map[string]Bulk
@@ -121,8 +119,9 @@ func NewBulker(es esapi.Transport, tracer *apm.Tracer, opts ...BulkOpt) *Bulker
 		apikeyLimit:           semaphore.NewWeighted(int64(bopts.apikeyMaxParallel)),
 		tracer:                tracer,
 		remoteOutputConfigMap: make(map[string]map[string]interface{}),
-		bulkerMap:             make(map[string]Bulk),
-		remoteOutputErrorMap:  make(map[string]string),
+		// remote ES bulkers
+		bulkerMap:            make(map[string]Bulk),
+		remoteOutputErrorMap: make(map[string]string),
 	}
 }
 
@@ -146,8 +145,12 @@ func (b *Bulker) CancelFn() context.CancelFunc {
 	return b.cancelFn
 }
 
+// for remote ES output, create a new bulker in bulkerMap if does not exist
+// if bulker exists for output, check if config changed
+// if not changed, return the existing bulker
+// if changed, stop the existing bulker and create a new one
 func (b *Bulker) CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, bool, error) {
-	hasConfigChanged := b.CheckRemoteOutputChanged(zlog, outputName, outputMap[outputName])
+	hasConfigChanged := b.hasChangedAndUpdateRemoteOutputConfig(zlog, outputName, outputMap[outputName])
 	bulker := b.bulkerMap[outputName]
 	if bulker != nil && !hasConfigChanged {
 		return bulker, false, nil
@@ -181,6 +184,7 @@ func (b *Bulker) CreateAndGetBulker(zlog zerolog.Logger, outputName string, serv
 	go func() {
 		select {
 		case err = <-errCh:
+			zlog.Error().Err(err).Str("outputName", outputName).Msg("Bulker error")
 		case <-bulkCtx.Done():
 			zlog.Debug().Str("outputName", outputName).Msg("Bulk context done")
 			err = bulkCtx.Err()
@@ -237,12 +241,8 @@ func (b *Bulker) Client() *elasticsearch.Client {
 	return client
 }
 
-func (b *Bulker) Tracer() *apm.Tracer {
-	return b.tracer
-}
-
 // check if remote output cfg changed
-func (b *Bulker) CheckRemoteOutputChanged(zlog zerolog.Logger, name string, newCfg map[string]interface{}) bool {
+func (b *Bulker) hasChangedAndUpdateRemoteOutputConfig(zlog zerolog.Logger, name string, newCfg map[string]interface{}) bool {
 	curCfg := b.remoteOutputConfigMap[name]
 
 	hasChanged := false
@@ -413,6 +413,13 @@ func (b *Bulker) Run(ctx context.Context) error {
 
 	}
 
+	// cancelling context of each remote bulker when Run exits
+	defer func() {
+		for _, bulker := range b.bulkerMap {
+			bulker.CancelFn()()
+		}
+	}()
+
 	return err
 }
 
diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index 4cb014116..cf29367ff 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -113,23 +113,24 @@ func (p *Output) prepareElasticsearch(
 
 	// retire api key of removed remote output
 	var toRetireAPIKeys *model.ToRetireAPIKeyIdsItems
-	var removedOutputKey string
+	var removedOutputName string
 	// find the first output that is removed - supposing one output can be removed at a time
-	for agentOutputKey, agentOutput := range agent.Outputs {
+	for agentOutputName, agentOutput := range agent.Outputs {
 		found := false
 		for outputMapKey := range outputMap {
-			if agentOutputKey == outputMapKey {
+			if agentOutputName == outputMapKey {
 				found = true
+				break
 			}
 		}
 		if !found {
-			zlog.Info().Str("APIKeyID", agentOutput.APIKeyID).Str("output", agentOutputKey).Msg("Output removed, will retire API key")
+			zlog.Info().Str(logger.APIKeyID, agentOutput.APIKeyID).Str("outputName", agentOutputName).Msg("Output removed, will retire API key")
 			toRetireAPIKeys = &model.ToRetireAPIKeyIdsItems{
 				ID:        agentOutput.APIKeyID,
 				RetiredAt: time.Now().UTC().Format(time.RFC3339),
-				Output:    agentOutputKey,
+				Output:    agentOutputName,
 			}
-			removedOutputKey = agentOutputKey
+			removedOutputName = agentOutputName
 			break
 		}
 	}
@@ -156,7 +157,7 @@ func (p *Output) prepareElasticsearch(
 		body, err = json.Marshal(map[string]interface{}{
 			"script": map[string]interface{}{
 				"lang":   "painless",
-				"source": fmt.Sprintf("ctx._source['outputs'].remove(\"%s\")", removedOutputKey),
+				"source": fmt.Sprintf("ctx._source['outputs'].remove(\"%s\")", removedOutputName),
 			},
 		})
 		if err != nil {
@@ -262,7 +263,7 @@ func (p *Output) prepareElasticsearch(
 			// reporting output error status to self monitor and not returning the error to keep fleet-server running
 		if outputAPIKey == nil && p.Type == OutputTypeRemoteElasticsearch {
 			if err != nil {
-				zerolog.Ctx(ctx).Warn().Msg("Could not create API key in remote ES")
+				zerolog.Ctx(ctx).Warn().Err(err).Msg("Could not create API key in remote ES")
 				bulker.SetRemoteOutputError(p.Name, err.Error())
 			} else {
 				bulker.SetRemoteOutputError(p.Name, "")
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index 0dc5e0543..8291dc2df 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -95,11 +95,6 @@ func (m *MockBulk) CreateAndGetBulker(zlog zerolog.Logger, outputName string, se
 	return args.Get(0).(bulk.Bulk), args.Get(1).(bool), nil
 }
 
-func (m *MockBulk) CheckRemoteOutputChanged(zlog zerolog.Logger, name string, newCfg map[string]interface{}) bool {
-	args := m.Called()
-	return args.Get(0).(bool)
-}
-
 func (m *MockBulk) GetRemoteOutputErrorMap() map[string]string {
 	args := m.Called()
 	return args.Get(0).(map[string]string)

From dcf39f932a6cddc2ac51be981fda01ad0a3218fe Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 16 Nov 2023 12:50:17 +0100
Subject: [PATCH 58/91] invalidate all ids with corresponding output

---
 internal/pkg/api/handleAck.go      | 30 +++++++++------
 internal/pkg/api/handleAck_test.go | 60 +++++++++++++-----------------
 internal/pkg/testing/bulk.go       |  2 +-
 3 files changed, 45 insertions(+), 47 deletions(-)

diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go
index a1fd554f6..79839d1f6 100644
--- a/internal/pkg/api/handleAck.go
+++ b/internal/pkg/api/handleAck.go
@@ -568,29 +568,35 @@ func cleanRoles(roles json.RawMessage) (json.RawMessage, int, error) {
 
 func (ack *AckT) invalidateAPIKeys(ctx context.Context, zlog zerolog.Logger, toRetireAPIKeyIDs []model.ToRetireAPIKeyIdsItems, skip string) {
 	ids := make([]string, 0, len(toRetireAPIKeyIDs))
-	output := ""
+	remoteIds := make(map[string][]string)
 	for _, k := range toRetireAPIKeyIDs {
 		if k.ID == skip || k.ID == "" {
 			continue
 		}
-		ids = append(ids, k.ID)
-		output = k.Output
-	}
-	// using remote es bulker to invalidate api key - supposing all retire api key ids belong to the same remote es
-	bulk := ack.bulk
-	if output != "" {
-		outputBulk := ack.bulk.GetBulker(output)
-		if outputBulk != nil {
-			bulk = outputBulk
+		if k.Output != "" {
+			if remoteIds[k.Output] == nil {
+				remoteIds[k.Output] = make([]string, 0)
+			}
+			remoteIds[k.Output] = append(remoteIds[k.Output], k.ID)
+		} else {
+			ids = append(ids, k.ID)
 		}
 	}
-
 	if len(ids) > 0 {
 		zlog.Info().Strs("fleet.policy.apiKeyIDsToRetire", ids).Msg("Invalidate old API keys")
-		if err := bulk.APIKeyInvalidate(ctx, ids...); err != nil {
+		if err := ack.bulk.APIKeyInvalidate(ctx, ids...); err != nil {
 			zlog.Info().Err(err).Strs("ids", ids).Msg("Failed to invalidate API keys")
 		}
 	}
+	// using remote es bulker to invalidate api key
+	for outputName, outputIds := range remoteIds {
+		outputBulk := ack.bulk.GetBulker(outputName)
+		if outputBulk != nil {
+			if err := outputBulk.APIKeyInvalidate(ctx, outputIds...); err != nil {
+				zlog.Info().Err(err).Strs("ids", outputIds).Str("outputName", outputName).Msg("Failed to invalidate API keys")
+			}
+		}
+	}
 }
 
 func (ack *AckT) handleUnenroll(ctx context.Context, zlog zerolog.Logger, agent *model.Agent) error {
diff --git a/internal/pkg/api/handleAck_test.go b/internal/pkg/api/handleAck_test.go
index 1cd0b9019..318c55695 100644
--- a/internal/pkg/api/handleAck_test.go
+++ b/internal/pkg/api/handleAck_test.go
@@ -572,44 +572,36 @@ func TestInvalidateAPIKeys(t *testing.T) {
 }
 
 func TestInvalidateAPIKeysRemoteOutput(t *testing.T) {
-	toRetire1 := []model.ToRetireAPIKeyIdsItems{{
+	toRetire := []model.ToRetireAPIKeyIdsItems{{
 		ID:     "toRetire1",
 		Output: "remote1",
+	}, {
+		ID:     "toRetire11",
+		Output: "remote1",
+	}, {
+		ID:     "toRetire2",
+		Output: "remote2",
 	}}
 
-	wants := map[string][]string{
-		"1": {"toRetire1"},
-	}
-
-	agent := model.Agent{
-		Outputs: map[string]*model.PolicyOutput{
-			"1": {ToRetireAPIKeyIds: toRetire1},
-		},
-	}
-
-	for i, out := range agent.Outputs {
-		want := wants[i]
-
-		bulker := ftesting.NewMockBulk()
-		remoteBulker := ftesting.NewMockBulk()
-		bulker.On("GetBulker", mock.AnythingOfType("string")).Return(remoteBulker)
-		if len(want) > 0 {
-			remoteBulker.On("APIKeyInvalidate",
-				context.Background(), mock.MatchedBy(func(ids []string) bool {
-					// if A contains B and B contains A => A = B
-					return assert.Subset(t, ids, want) &&
-						assert.Subset(t, want, ids)
-				})).
-				Return(nil)
-		}
-
-		logger := testlog.SetLogger(t)
-		ack := &AckT{bulk: bulker}
-		ack.invalidateAPIKeys(context.Background(), logger, out.ToRetireAPIKeyIds, "")
-
-		bulker.AssertExpectations(t)
-		remoteBulker.AssertExpectations(t)
-	}
+	bulker := ftesting.NewMockBulk()
+	remoteBulker := ftesting.NewMockBulk()
+	remoteBulker2 := ftesting.NewMockBulk()
+	bulker.On("GetBulker", "remote1").Return(remoteBulker)
+	bulker.On("GetBulker", "remote2").Return(remoteBulker2)
+
+	remoteBulker.On("APIKeyInvalidate",
+		context.Background(), []string{"toRetire1", "toRetire11"}).
+		Return(nil)
+	remoteBulker2.On("APIKeyInvalidate",
+		context.Background(), []string{"toRetire2"}).
+		Return(nil)
+
+	logger := testlog.SetLogger(t)
+	ack := &AckT{bulk: bulker}
+	ack.invalidateAPIKeys(context.Background(), logger, toRetire, "")
+
+	bulker.AssertExpectations(t)
+	remoteBulker.AssertExpectations(t)
 }
 
 func TestAckHandleUpgrade(t *testing.T) {
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index 8291dc2df..992735e97 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -81,7 +81,7 @@ func (m *MockBulk) Client() *elasticsearch.Client {
 }
 
 func (m *MockBulk) GetBulker(outputName string) bulk.Bulk {
-	args := m.Called()
+	args := m.Called(outputName)
 	return args.Get(0).(bulk.Bulk)
 }
 

From bb32a41b31fd60c5b213f481279505f223d41d0e Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 16 Nov 2023 13:45:32 +0100
Subject: [PATCH 59/91] only add toRetireAPIKeys if does not exist

---
 internal/pkg/policy/policy_output.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index cf29367ff..9d31aa330 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -475,8 +475,9 @@ if (ctx._source['outputs']['%s']==null)
 			source.WriteString(fmt.Sprintf(`
 if (ctx._source['outputs']['%s'].%s==null)
   {ctx._source['outputs']['%s'].%s=new ArrayList();}
-ctx._source['outputs']['%s'].%s.add(params.%s);
-`, outputName, field, outputName, field, outputName, field, field))
+if (!ctx._source['outputs']['%s'].%s.contains(params.%s))  
+  {ctx._source['outputs']['%s'].%s.add(params.%s);}
+`, outputName, field, outputName, field, outputName, field, field, outputName, field, field))
 		} else {
 			// Update the other fields
 			source.WriteString(fmt.Sprintf(`

From 3261472e16b0fd0f128b0debd49ae92d55440931 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 16 Nov 2023 14:28:59 +0100
Subject: [PATCH 60/91] added a retry loop to integration test

---
 .../policy/policy_output_integration_test.go  | 35 +++++++++----------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/internal/pkg/policy/policy_output_integration_test.go b/internal/pkg/policy/policy_output_integration_test.go
index f52033fa1..d77161216 100644
--- a/internal/pkg/policy/policy_output_integration_test.go
+++ b/internal/pkg/policy/policy_output_integration_test.go
@@ -234,24 +234,23 @@ func TestPolicyOutputESPrepareRemoteES(t *testing.T) {
 		ctx, zerolog.Nop(), bulker, bulker, &agent, policyMap, false)
 	require.NoError(t, err)
 
-	// need to wait a bit before querying the agent again
-	// TODO: find a better way to query the updated agent
-	time.Sleep(time.Second)
-
-	got, err := dl.FindAgent(
-		ctx, bulker, dl.QueryAgentByID, dl.FieldID, agentID, dl.WithIndexName(index))
-	if err != nil {
-		require.NoError(t, err, "failed to find agent ID %q", agentID)
-	}
-
-	gotOutput, ok := got.Outputs[output.Name]
-	require.True(t, ok, "no '%s' output found on agent document", output.Name)
-
-	assert.Empty(t, gotOutput.ToRetireAPIKeyIds)
-	assert.Equal(t, gotOutput.Type, OutputTypeElasticsearch)
-	assert.Equal(t, gotOutput.PermissionsHash, output.Role.Sha2)
-	assert.NotEmpty(t, gotOutput.APIKey)
-	assert.NotEmpty(t, gotOutput.APIKeyID)
+	ftesting.Retry(t, ctx, func(ctx context.Context) error {
+		got, err := dl.FindAgent(
+			ctx, bulker, dl.QueryAgentByID, dl.FieldID, agentID, dl.WithIndexName(index))
+		if err != nil {
+			require.NoError(t, err, "failed to find agent ID %q", agentID)
+		}
+
+		gotOutput, ok := got.Outputs[output.Name]
+		require.True(t, ok, "no '%s' output found on agent document", output.Name)
+
+		assert.Empty(t, gotOutput.ToRetireAPIKeyIds)
+		assert.Equal(t, gotOutput.Type, OutputTypeElasticsearch)
+		assert.Equal(t, gotOutput.PermissionsHash, output.Role.Sha2)
+		assert.NotEmpty(t, gotOutput.APIKey)
+		assert.NotEmpty(t, gotOutput.APIKeyID)
+		return nil
+	}, ftesting.RetrySleep(1*time.Second))
 }
 
 func TestPolicyOutputESPrepareESRetireRemoteAPIKeys(t *testing.T) {

From 991c6891d8c7615dfd8e210fb2a2d97e6667bb49 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 16 Nov 2023 15:32:42 +0100
Subject: [PATCH 61/91] fixed monitor tests

---
 internal/pkg/policy/self_test.go       | 24 +++++++++++++++++++++++-
 internal/pkg/policy/standalone_test.go |  8 +++++---
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/internal/pkg/policy/self_test.go b/internal/pkg/policy/self_test.go
index 1b8431ee9..6de24abaa 100644
--- a/internal/pkg/policy/self_test.go
+++ b/internal/pkg/policy/self_test.go
@@ -10,7 +10,9 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -907,7 +909,7 @@ func TestSelfMonitor_RemoteOutput_Back_To_Healthy(t *testing.T) {
 	}
 }
 
-func TestSelfMonitor_RemoteOutput_Ping_Degraded(t *testing.T) {
+func TestSelfMonitor_RemoteOutput_Ping_Degraded_Healthy(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -1035,6 +1037,26 @@ func TestSelfMonitor_RemoteOutput_Ping_Degraded(t *testing.T) {
 		return nil
 	}, ftesting.RetrySleep(1*time.Second))
 
+	// back to healthy
+	mocktrans.Response = &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       ioutil.NopCloser(strings.NewReader(`{}`)),
+		Header: http.Header{
+			"X-Elastic-Product": []string{"Elasticsearch"},
+		},
+	}
+
+	ftesting.Retry(t, ctx, func(ctx context.Context) error {
+		state, msg, _ := reporter.Current()
+		if state != client.UnitStateHealthy {
+			return fmt.Errorf("should be reported as healthy; instead its %s", state)
+		}
+		if msg != "Running on default policy with Fleet Server integration" {
+			return fmt.Errorf("should be matching with default policy")
+		}
+		return nil
+	}, ftesting.RetrySleep(1*time.Second))
+
 	cancel()
 	mwg.Wait()
 	if merr != nil && merr != context.Canceled {
diff --git a/internal/pkg/policy/standalone_test.go b/internal/pkg/policy/standalone_test.go
index bfd9e490e..903a72bc9 100644
--- a/internal/pkg/policy/standalone_test.go
+++ b/internal/pkg/policy/standalone_test.go
@@ -185,13 +185,15 @@ func TestStandAloneSelfMonitorRemoteOutputPing(t *testing.T) {
 	mocktrans.Response = &http.Response{
 		StatusCode: http.StatusOK,
 		Body:       ioutil.NopCloser(strings.NewReader(`{}`)),
+		Header: http.Header{
+			"X-Elastic-Product": []string{"Elasticsearch"},
+		},
 	}
 
 	sm.check(context.Background())
 	state = sm.State()
 
-	// TODO does not work, gives error "the client noticed that the server is not Elasticsearch and we do not support this unknown product"
-	// assert.Equal(t, client.UnitStateHealthy, state)
-	// assert.Equal(t, state, reporter.state, "reported state should be the same")
+	assert.Equal(t, client.UnitStateHealthy, state)
+	assert.Equal(t, state, reporter.state, "reported state should be the same")
 
 }

From 75fdb8f2378f15ab7f5f1b061824b28f0fd6412b Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 20 Nov 2023 13:41:07 +0100
Subject: [PATCH 62/91] remove break loop in self monitor

---
 internal/pkg/policy/self.go | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/internal/pkg/policy/self.go b/internal/pkg/policy/self.go
index e922df9dd..8eae7e53d 100644
--- a/internal/pkg/policy/self.go
+++ b/internal/pkg/policy/self.go
@@ -104,15 +104,11 @@ LOOP:
 		case <-ctx.Done():
 			break LOOP
 		case <-cT.C:
-			state, err := m.process(ctx)
+			_, err := m.process(ctx)
 			if err != nil {
 				return err
 			}
 			cT.Reset(m.checkTime)
-			if state == client.UnitStateHealthy {
-				// running; can stop
-				break LOOP
-			}
 		case hits := <-s.Output():
 			policies := make([]model.Policy, len(hits))
 			for i, hit := range hits {
@@ -121,14 +117,10 @@ LOOP:
 					return err
 				}
 			}
-			state, err := m.processPolicies(ctx, policies)
+			_, err := m.processPolicies(ctx, policies)
 			if err != nil {
 				return err
 			}
-			if state == client.UnitStateHealthy {
-				// running; can stop
-				break LOOP
-			}
 		}
 	}
 
@@ -231,7 +223,7 @@ func (m *selfMonitorT) updateState(ctx context.Context) (client.UnitState, error
 	if hasError {
 		m.state = client.UnitStateDegraded
 		m.reporter.UpdateState(client.UnitStateDegraded, "Could not connect to remote ES output", remoteESPayload) //nolint:errcheck // not clear what to do in failure cases
-		return client.UnitStateDegraded, nil
+		return m.state, nil
 	} else {
 		bulkerMap := m.bulker.GetBulkerMap()
 		for outputName, outputBulker := range bulkerMap {

From a8447b29f21286e43ec2fe0e49c6de7f15e11b3e Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 20 Nov 2023 14:06:24 +0100
Subject: [PATCH 63/91] fix lint

---
 internal/pkg/policy/self.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/internal/pkg/policy/self.go b/internal/pkg/policy/self.go
index 8eae7e53d..75312b869 100644
--- a/internal/pkg/policy/self.go
+++ b/internal/pkg/policy/self.go
@@ -104,7 +104,8 @@ LOOP:
 		case <-ctx.Done():
 			break LOOP
 		case <-cT.C:
-			_, err := m.process(ctx)
+			state, err := m.process(ctx)
+			m.log.Trace().Str("state", state.String()).Msg("self monitor state")
 			if err != nil {
 				return err
 			}
@@ -117,7 +118,8 @@ LOOP:
 					return err
 				}
 			}
-			_, err := m.processPolicies(ctx, policies)
+			state, err := m.processPolicies(ctx, policies)
+			m.log.Trace().Str("state", state.String()).Msg("self monitor state")
 			if err != nil {
 				return err
 			}
@@ -238,7 +240,6 @@ func (m *selfMonitorT) updateState(ctx context.Context) (client.UnitState, error
 			} else if res.StatusCode != 200 {
 				m.state = client.UnitStateDegraded
 				message := fmt.Sprintf("Could not connect to remote ES output: %s, status code: %d", outputName, res.StatusCode)
-				m.log.Debug().Msg(message)
 				m.reporter.UpdateState(m.state, message, nil) //nolint:errcheck // not clear what to do in failure cases
 				hasError = true
 				break

From 9c7d6fe3701e42d2b1188855b9f18b4cbe3beb94 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 20 Nov 2023 14:18:18 +0100
Subject: [PATCH 64/91] openapi spec: added degraded state to desc

---
 pkg/api/versions/2023_06_01/api/openapi.yml  | 1 +
 pkg/api/versions/2023_06_01/api/types.gen.go | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/pkg/api/versions/2023_06_01/api/openapi.yml b/pkg/api/versions/2023_06_01/api/openapi.yml
index fe663481f..00884d691 100644
--- a/pkg/api/versions/2023_06_01/api/openapi.yml
+++ b/pkg/api/versions/2023_06_01/api/openapi.yml
@@ -85,6 +85,7 @@ components:
           description: |
             A Unit state that fleet-server may report.
             Unit state is defined in the elastic-agent-client specification.
+            Fleet-server goes to degraded state if remote elasticsearch output is not reachable.
           enum:
             - starting
             - configuring
diff --git a/pkg/api/versions/2023_06_01/api/types.gen.go b/pkg/api/versions/2023_06_01/api/types.gen.go
index 7b412627d..1c025c03f 100644
--- a/pkg/api/versions/2023_06_01/api/types.gen.go
+++ b/pkg/api/versions/2023_06_01/api/types.gen.go
@@ -418,6 +418,7 @@ type StatusAPIResponse struct {
 
 	// Status A Unit state that fleet-server may report.
 	// Unit state is defined in the elastic-agent-client specification.
+	// Fleet-server goes to degraded state if remote elasticsearch output is not reachable.
 	Status StatusResponseStatus `json:"status"`
 
 	// Version Version information included in the response to an authorized status request.
@@ -426,6 +427,7 @@ type StatusAPIResponse struct {
 
 // StatusResponseStatus A Unit state that fleet-server may report.
 // Unit state is defined in the elastic-agent-client specification.
+// Fleet-server goes to degraded state if remote elasticsearch output is not reachable.
 type StatusResponseStatus string
 
 // StatusResponseVersion Version information included in the response to an authorized status request.

From 8df35d18f473ac911dfcc8802829e65e7324ee82 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 20 Nov 2023 15:38:52 +0100
Subject: [PATCH 65/91] added 2nd elasticsearch to integration test

---
 Makefile                                      |  8 ++++---
 dev-tools/integration/.env                    |  3 ++-
 dev-tools/integration/docker-compose.yml      | 24 +++++++++++++++++++
 .../get-elasticsearch-servicetoken.sh         |  7 +++---
 .../remote_es_output_integration_test.go      |  9 ++++---
 5 files changed, 39 insertions(+), 12 deletions(-)

diff --git a/Makefile b/Makefile
index 77697a501..f9b582f4e 100644
--- a/Makefile
+++ b/Makefile
@@ -297,12 +297,13 @@ export $(shell sed 's/=.*//' ./dev-tools/integration/.env)
 # Start ES with docker without waiting
 .PHONY: int-docker-start-async
 int-docker-start-async:
-	@docker compose -f ./dev-tools/integration/docker-compose.yml --env-file ./dev-tools/integration/.env up  -d --remove-orphans elasticsearch
+	@docker compose -f ./dev-tools/integration/docker-compose.yml --env-file ./dev-tools/integration/.env up  -d --remove-orphans elasticsearch elasticsearch-remote
 
 # Wait for ES to be ready
 .PHONY: int-docker-wait
 int-docker-wait:
 	@./dev-tools/integration/wait-for-elasticsearch.sh ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}@${TEST_ELASTICSEARCH_HOSTS}
+	@./dev-tools/integration/wait-for-elasticsearch.sh ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}@${TEST_REMOTE_ELASTICSEARCH_HOST}
 
 # Start integration docker setup with wait for when the ES is ready
 .PHONY: int-docker-start
@@ -331,7 +332,8 @@ test-int: prepare-test-context  ## - Run integration tests with full setup (slow
 .PHONY: test-int-set
 test-int-set: ## - Run integration tests without setup
 	# Initialize indices one before running all the tests
-	ELASTICSEARCH_SERVICE_TOKEN=$(shell ./dev-tools/integration/get-elasticsearch-servicetoken.sh ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}@${TEST_ELASTICSEARCH_HOSTS}) \
+	ELASTICSEARCH_SERVICE_TOKEN=$(shell ./dev-tools/integration/get-elasticsearch-servicetoken.sh ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}@${TEST_ELASTICSEARCH_HOSTS} "fleet-server") \
+	REMOTE_ELASTICSEARCH_SERVICE_TOKEN=$(shell ./dev-tools/integration/get-elasticsearch-servicetoken.sh ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}@${TEST_REMOTE_ELASTICSEARCH_HOST} "fleet-server-remote") \
 	ELASTICSEARCH_HOSTS=${TEST_ELASTICSEARCH_HOSTS} ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME} ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD} \
 	go test -v -tags=integration -count=1 -race -p 1 ./...
 
@@ -372,7 +374,7 @@ test-e2e: docker-cover-e2e-binaries build-e2e-agent-image e2e-certs build-docker
 .PHONY: test-e2e-set
 test-e2e-set: ## - Run the blackbox end to end tests without setup.
 	cd testing; \
-	ELASTICSEARCH_SERVICE_TOKEN=$(shell ./dev-tools/integration/get-elasticsearch-servicetoken.sh ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}@${TEST_ELASTICSEARCH_HOSTS}) \
+	ELASTICSEARCH_SERVICE_TOKEN=$(shell ./dev-tools/integration/get-elasticsearch-servicetoken.sh ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}@${TEST_ELASTICSEARCH_HOSTS} "fleet-server") \
 	ELASTICSEARCH_HOSTS=${TEST_ELASTICSEARCH_HOSTS} ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME} ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD} \
 	AGENT_E2E_IMAGE=$(shell cat "build/e2e-image") \
 	STANDALONE_E2E_IMAGE=$(DOCKER_IMAGE):$(DOCKER_IMAGE_TAG)$(if $(DEV),-dev,) \
diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env
index 17c5ef029..61b155cd7 100644
--- a/dev-tools/integration/.env
+++ b/dev-tools/integration/.env
@@ -1,4 +1,5 @@
-ELASTICSEARCH_VERSION=8.12.0-1a1295ff-SNAPSHOT
+ELASTICSEARCH_VERSION=8.12.0-7521d760-SNAPSHOT
 ELASTICSEARCH_USERNAME=elastic
 ELASTICSEARCH_PASSWORD=changeme
 TEST_ELASTICSEARCH_HOSTS=localhost:9200
+TEST_REMOTE_ELASTICSEARCH_HOST=localhost:9201
diff --git a/dev-tools/integration/docker-compose.yml b/dev-tools/integration/docker-compose.yml
index 537ca888e..de4979251 100644
--- a/dev-tools/integration/docker-compose.yml
+++ b/dev-tools/integration/docker-compose.yml
@@ -23,3 +23,27 @@ services:
       - ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
     ports:
       - 127.0.0.1:9200:9200
+      
+  elasticsearch-remote:
+    image: "docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION}-amd64"
+    container_name: elasticsearch-remote
+    environment:
+      - node.name=es02
+      - cluster.name=es-docker-cluster2
+      - discovery.seed_hosts=elasticsearch
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms1G -Xmx1G"
+      - "ELASTIC_USERNAME=${ELASTICSEARCH_USERNAME}"
+      - "ELASTIC_PASSWORD=${ELASTICSEARCH_PASSWORD}"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+          soft: 65536
+          hard: 65536
+    volumes:
+      - ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
+    ports:
+      - 127.0.0.1:9201:9200
+      
diff --git a/dev-tools/integration/get-elasticsearch-servicetoken.sh b/dev-tools/integration/get-elasticsearch-servicetoken.sh
index 639ffeb15..90d9c936b 100755
--- a/dev-tools/integration/get-elasticsearch-servicetoken.sh
+++ b/dev-tools/integration/get-elasticsearch-servicetoken.sh
@@ -1,8 +1,9 @@
 #!/bin/bash
 
 host="$1"
+account="$2"
 
-jsonBody="$(curl -sSL -XPOST "$host/_security/service/elastic/fleet-server/credential/token/token1")"
+jsonBody="$(curl -sSL -XPOST "$host/_security/service/elastic/$account/credential/token/token1")"
 
 # use grep and sed to get the service token value as we may not have jq or a similar tool on the instance
 token=$(echo ${jsonBody} |  grep -Eo '"value"[^}]*' | grep -Eo ':.*' | sed -r "s/://" | sed -r 's/"//g')
@@ -11,9 +12,9 @@ token=$(echo ${jsonBody} |  grep -Eo '"value"[^}]*' | grep -Eo ':.*' | sed -r "s
 # very useful during development, without recreating elasticsearch instance every time.
 if [ -z "$token" ]
 then
-    token=`cat .service_token` 
+    token=`cat .service_token_$account` 
 else
-    echo "$token" > .service_token
+    echo "$token" > .service_token_$account
 fi
 
 echo $token
diff --git a/internal/pkg/server/remote_es_output_integration_test.go b/internal/pkg/server/remote_es_output_integration_test.go
index 7edb29f19..2d2a04a67 100644
--- a/internal/pkg/server/remote_es_output_integration_test.go
+++ b/internal/pkg/server/remote_es_output_integration_test.go
@@ -11,6 +11,7 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
+	"os"
 	"strings"
 	"testing"
 
@@ -100,11 +101,9 @@ func Test_Agent_Remote_ES_Output(t *testing.T) {
 				"type": "elasticsearch",
 			},
 			"remoteES": {
-				"type": "remote_elasticsearch",
-				// TODO start another fleet-server
-				"hosts": []string{"localhost:9200"},
-				// TODO create remote service token - superuser? manage_service_account
-				"service_token": srv.cfg.Output.Elasticsearch.ServiceToken,
+				"type":          "remote_elasticsearch",
+				"hosts":         []string{"localhost:9201"},
+				"service_token": os.Getenv("REMOTE_ELASTICSEARCH_SERVICE_TOKEN"),
 			},
 		},
 		OutputPermissions: json.RawMessage(`{"default": {}, "remoteES": {}}`),

From 10021a89ccdffcdcb63bc8107b60815d7042cc7b Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 20 Nov 2023 16:10:23 +0100
Subject: [PATCH 66/91] added build.Info to remote bulkers from fleet

---
 internal/pkg/bulk/engine.go  | 2 +-
 internal/pkg/bulk/opt.go     | 8 ++++++++
 internal/pkg/server/fleet.go | 4 +++-
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 3efa08388..25d534024 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -217,7 +217,7 @@ func (b *Bulker) createRemoteEsClient(ctx context.Context, outputName string, se
 		},
 	}
 	es, err := es.NewClient(ctx, &cfg, false, elasticsearchOptions(
-		true, build.Info{},
+		true, b.opts.bi,
 	)...)
 	if err != nil {
 		return nil, err
diff --git a/internal/pkg/bulk/opt.go b/internal/pkg/bulk/opt.go
index e06c333f9..19001d1ba 100644
--- a/internal/pkg/bulk/opt.go
+++ b/internal/pkg/bulk/opt.go
@@ -12,6 +12,7 @@ import (
 	"github.com/rs/zerolog"
 	"go.elastic.co/apm/v2"
 
+	"github.com/elastic/fleet-server/v7/internal/pkg/build"
 	"github.com/elastic/fleet-server/v7/internal/pkg/config"
 )
 
@@ -81,6 +82,7 @@ type bulkOptT struct {
 	apikeyMaxParallel int
 	apikeyMaxReqSize  int
 	policyTokens      []config.PolicyToken
+	bi                build.Info
 }
 
 type BulkOpt func(*bulkOptT)
@@ -143,6 +145,12 @@ func WithPolicyTokens(tokens []config.PolicyToken) BulkOpt {
 	}
 }
 
+func WithBi(bi build.Info) BulkOpt {
+	return func(opt *bulkOptT) {
+		opt.bi = bi
+	}
+}
+
 func parseBulkOpts(opts ...BulkOpt) bulkOptT {
 	bopt := bulkOptT{
 		flushInterval:     defaultFlushInterval,
diff --git a/internal/pkg/server/fleet.go b/internal/pkg/server/fleet.go
index be3c9410e..95b08b9b7 100644
--- a/internal/pkg/server/fleet.go
+++ b/internal/pkg/server/fleet.go
@@ -333,7 +333,9 @@ func (f *Fleet) initBulker(ctx context.Context, tracer *apm.Tracer, cfg *config.
 		return nil, err
 	}
 
-	blk := bulk.NewBulker(es, tracer, bulk.BulkOptsFromCfg(cfg)...)
+	bulkOpts := bulk.BulkOptsFromCfg(cfg)
+	bulkOpts = append(bulkOpts, bulk.WithBi(f.bi))
+	blk := bulk.NewBulker(es, tracer, bulkOpts...)
 	return blk, nil
 }
 

From 4c99949e6a24ce94106db0a89ee300c424175c83 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 21 Nov 2023 13:28:09 +0100
Subject: [PATCH 67/91] added semaphore for updating bulkerMap

---
 internal/pkg/bulk/bulk_remote_output_test.go | 13 +++++++---
 internal/pkg/bulk/engine.go                  | 25 ++++++++++++++++----
 internal/pkg/policy/policy_output.go         |  6 ++---
 internal/pkg/testing/bulk.go                 |  4 ++--
 4 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/internal/pkg/bulk/bulk_remote_output_test.go b/internal/pkg/bulk/bulk_remote_output_test.go
index 5b5dab693..c08d48afb 100644
--- a/internal/pkg/bulk/bulk_remote_output_test.go
+++ b/internal/pkg/bulk/bulk_remote_output_test.go
@@ -7,6 +7,7 @@
 package bulk
 
 import (
+	"context"
 	"testing"
 
 	testlog "github.com/elastic/fleet-server/v7/internal/pkg/testing/log"
@@ -88,6 +89,8 @@ func Test_hasChangedAndUpdateRemoteOutputConfig(t *testing.T) {
 }
 
 func Test_CreateAndGetBulkerNew(t *testing.T) {
+	ctx, cn := context.WithCancel(context.Background())
+	defer cn()
 	log := testlog.SetLogger(t)
 	bulker := NewBulker(nil, nil)
 	outputMap := make(map[string]map[string]interface{})
@@ -96,13 +99,15 @@ func Test_CreateAndGetBulkerNew(t *testing.T) {
 		"hosts":         []interface{}{"https://remote-es:443"},
 		"service_token": "token1",
 	}
-	newBulker, hasChanged, err := bulker.CreateAndGetBulker(log, "remote1", "token1", outputMap)
+	newBulker, hasChanged, err := bulker.CreateAndGetBulker(ctx, log, "remote1", "token1", outputMap)
 	assert.NotNil(t, newBulker)
 	assert.Equal(t, false, hasChanged)
 	assert.Nil(t, err)
 }
 
 func Test_CreateAndGetBulkerExisting(t *testing.T) {
+	ctx, cn := context.WithCancel(context.Background())
+	defer cn()
 	log := testlog.SetLogger(t)
 	bulker := NewBulker(nil, nil)
 	outputBulker := NewBulker(nil, nil)
@@ -115,13 +120,15 @@ func Test_CreateAndGetBulkerExisting(t *testing.T) {
 	}
 	bulker.remoteOutputConfigMap["remote1"] = cfg
 	outputMap["remote1"] = cfg
-	newBulker, hasChanged, err := bulker.CreateAndGetBulker(log, "remote1", "token1", outputMap)
+	newBulker, hasChanged, err := bulker.CreateAndGetBulker(ctx, log, "remote1", "token1", outputMap)
 	assert.Equal(t, outputBulker, newBulker)
 	assert.Equal(t, false, hasChanged)
 	assert.Nil(t, err)
 }
 
 func Test_CreateAndGetBulkerChanged(t *testing.T) {
+	ctx, cn := context.WithCancel(context.Background())
+	defer cn()
 	log := testlog.SetLogger(t)
 	bulker := NewBulker(nil, nil)
 	outputBulker := NewBulker(nil, nil)
@@ -139,7 +146,7 @@ func Test_CreateAndGetBulkerChanged(t *testing.T) {
 	}
 	cancelFnCalled := false
 	outputBulker.cancelFn = func() { cancelFnCalled = true }
-	newBulker, hasChanged, err := bulker.CreateAndGetBulker(log, "remote1", "token2", outputMap)
+	newBulker, hasChanged, err := bulker.CreateAndGetBulker(ctx, log, "remote1", "token2", outputMap)
 	assert.NotEqual(t, outputBulker, newBulker)
 	assert.Equal(t, true, hasChanged)
 	assert.Nil(t, err)
diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 25d534024..34188a7c2 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -68,7 +68,7 @@ type Bulk interface {
 	// Accessor used to talk to elastic search direcly bypassing bulk engine
 	Client() *elasticsearch.Client
 
-	CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, bool, error)
+	CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, bool, error)
 	GetBulker(outputName string) Bulk
 	GetBulkerMap() map[string]Bulk
 	GetRemoteOutputErrorMap() map[string]string
@@ -91,6 +91,7 @@ type Bulker struct {
 	bulkerMap             map[string]Bulk
 	remoteOutputErrorMap  map[string]string
 	cancelFn              context.CancelFunc
+	remoteOutputLimit     *semaphore.Weighted
 }
 
 const (
@@ -122,6 +123,7 @@ func NewBulker(es esapi.Transport, tracer *apm.Tracer, opts ...BulkOpt) *Bulker
 		// remote ES bulkers
 		bulkerMap:            make(map[string]Bulk),
 		remoteOutputErrorMap: make(map[string]string),
+		remoteOutputLimit:    semaphore.NewWeighted(1),
 	}
 }
 
@@ -130,6 +132,7 @@ func (b *Bulker) GetRemoteOutputErrorMap() map[string]string {
 }
 
 func (b *Bulker) SetRemoteOutputError(name string, status string) {
+	// TODO concurrency control of updating map
 	b.remoteOutputErrorMap[name] = status
 }
 
@@ -145,11 +148,22 @@ func (b *Bulker) CancelFn() context.CancelFunc {
 	return b.cancelFn
 }
 
+func (b *Bulker) updateBulkerMap(ctx context.Context, outputName string, newBulker *Bulker) error {
+	// concurrency control of updating map
+	if err := b.remoteOutputLimit.Acquire(ctx, 1); err != nil {
+		return err
+	}
+	defer b.remoteOutputLimit.Release(1)
+
+	b.bulkerMap[outputName] = newBulker
+	return nil
+}
+
 // for remote ES output, create a new bulker in bulkerMap if does not exist
 // if bulker exists for output, check if config changed
 // if not changed, return the existing bulker
 // if changed, stop the existing bulker and create a new one
-func (b *Bulker) CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, bool, error) {
+func (b *Bulker) CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, bool, error) {
 	hasConfigChanged := b.hasChangedAndUpdateRemoteOutputConfig(zlog, outputName, outputMap[outputName])
 	bulker := b.bulkerMap[outputName]
 	if bulker != nil && !hasConfigChanged {
@@ -170,7 +184,11 @@ func (b *Bulker) CreateAndGetBulker(zlog zerolog.Logger, outputName string, serv
 	// starting a new bulker to create/update API keys for remote ES output
 	newBulker := NewBulker(es, b.tracer)
 	newBulker.cancelFn = bulkCancel
-	b.bulkerMap[outputName] = newBulker
+
+	err = b.updateBulkerMap(ctx, outputName, newBulker)
+	if err != nil {
+		return nil, hasConfigChanged, err
+	}
 
 	errCh := make(chan error)
 	go func() {
@@ -247,7 +265,6 @@ func (b *Bulker) hasChangedAndUpdateRemoteOutputConfig(zlog zerolog.Logger, name
 
 	hasChanged := false
 
-	// TODO remoteOutputConfigMap empty when FS restarts - won't detect changes
 	// when output config first added, not reporting change
 	if curCfg != nil && !reflect.DeepEqual(curCfg, newCfg) {
 		zlog.Info().Str("name", name).Msg("remote output configuration has changed")
diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index 9d31aa330..7a8b5bd85 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -60,7 +60,7 @@ func (p *Output) Prepare(ctx context.Context, zlog zerolog.Logger, bulker bulk.B
 		}
 	case OutputTypeRemoteElasticsearch:
 		zlog.Debug().Msg("preparing remote elasticsearch output")
-		newBulker, hasConfigChanged, err := bulker.CreateAndGetBulker(zlog, p.Name, p.ServiceToken, outputMap)
+		newBulker, hasConfigChanged, err := bulker.CreateAndGetBulker(ctx, zlog, p.Name, p.ServiceToken, outputMap)
 		if err != nil {
 			return err
 		}
@@ -265,7 +265,7 @@ func (p *Output) prepareElasticsearch(
 			if err != nil {
 				zerolog.Ctx(ctx).Warn().Err(err).Msg("Could not create API key in remote ES")
 				bulker.SetRemoteOutputError(p.Name, err.Error())
-			} else {
+			} else if bulker.GetRemoteOutputErrorMap()[p.Name] != "" {
 				bulker.SetRemoteOutputError(p.Name, "")
 			}
 
@@ -274,7 +274,7 @@ func (p *Output) prepareElasticsearch(
 			// remove the service token from the agent policy sent to the agent
 			delete(outputMap[p.Name], FieldOutputServiceToken)
 			return nil
-		} else if p.Type == OutputTypeRemoteElasticsearch {
+		} else if p.Type == OutputTypeRemoteElasticsearch && bulker.GetRemoteOutputErrorMap()[p.Name] != "" {
 			bulker.SetRemoteOutputError(p.Name, "")
 		}
 		if err != nil {
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index 992735e97..4b54a7f5f 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -90,8 +90,8 @@ func (m *MockBulk) GetBulkerMap() map[string]bulk.Bulk {
 	return args.Get(0).(map[string]bulk.Bulk)
 }
 
-func (m *MockBulk) CreateAndGetBulker(zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (bulk.Bulk, bool, error) {
-	args := m.Called(zlog, outputName, serviceToken, outputMap)
+func (m *MockBulk) CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (bulk.Bulk, bool, error) {
+	args := m.Called(ctx, zlog, outputName, serviceToken, outputMap)
 	return args.Get(0).(bulk.Bulk), args.Get(1).(bool), nil
 }
 

From 8182bab53b4b48b009807f058862ff9253b51958 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 21 Nov 2023 13:44:29 +0100
Subject: [PATCH 68/91] revert self monitor degraded on remote es error

---
 internal/pkg/policy/self.go            |  48 +--
 internal/pkg/policy/self_test.go       | 413 -------------------------
 internal/pkg/policy/standalone.go      |  37 ---
 internal/pkg/policy/standalone_test.go |  99 ------
 4 files changed, 8 insertions(+), 589 deletions(-)

diff --git a/internal/pkg/policy/self.go b/internal/pkg/policy/self.go
index 75312b869..b9c8d7bd0 100644
--- a/internal/pkg/policy/self.go
+++ b/internal/pkg/policy/self.go
@@ -105,11 +105,14 @@ LOOP:
 			break LOOP
 		case <-cT.C:
 			state, err := m.process(ctx)
-			m.log.Trace().Str("state", state.String()).Msg("self monitor state")
 			if err != nil {
 				return err
 			}
 			cT.Reset(m.checkTime)
+			if state == client.UnitStateHealthy {
+				// running; can stop
+				break LOOP
+			}
 		case hits := <-s.Output():
 			policies := make([]model.Policy, len(hits))
 			for i, hit := range hits {
@@ -119,10 +122,13 @@ LOOP:
 				}
 			}
 			state, err := m.processPolicies(ctx, policies)
-			m.log.Trace().Str("state", state.String()).Msg("self monitor state")
 			if err != nil {
 				return err
 			}
+			if state == client.UnitStateHealthy {
+				// running; can stop
+				break LOOP
+			}
 		}
 	}
 
@@ -212,44 +218,6 @@ func (m *selfMonitorT) updateState(ctx context.Context) (client.UnitState, error
 		return client.UnitStateStarting, nil
 	}
 
-	remoteOutputErrorMap := m.bulker.GetRemoteOutputErrorMap()
-	hasError := false
-	remoteESPayload := make(map[string]interface{})
-	for key, value := range remoteOutputErrorMap {
-		if value != "" {
-			hasError = true
-			remoteESPayload[key] = value
-			break
-		}
-	}
-	if hasError {
-		m.state = client.UnitStateDegraded
-		m.reporter.UpdateState(client.UnitStateDegraded, "Could not connect to remote ES output", remoteESPayload) //nolint:errcheck // not clear what to do in failure cases
-		return m.state, nil
-	} else {
-		bulkerMap := m.bulker.GetBulkerMap()
-		for outputName, outputBulker := range bulkerMap {
-			res, err := outputBulker.Client().Ping(outputBulker.Client().Ping.WithContext(ctx))
-			if err != nil {
-				m.log.Error().Err(err).Msg("error calling remote es ping")
-				m.state = client.UnitStateDegraded
-				message := fmt.Sprintf("Could not ping remote ES: %s, error: %s", outputName, err.Error())
-				m.reporter.UpdateState(m.state, message, nil) //nolint:errcheck // not clear what to do in failure cases
-				hasError = true
-				break
-			} else if res.StatusCode != 200 {
-				m.state = client.UnitStateDegraded
-				message := fmt.Sprintf("Could not connect to remote ES output: %s, status code: %d", outputName, res.StatusCode)
-				m.reporter.UpdateState(m.state, message, nil) //nolint:errcheck // not clear what to do in failure cases
-				hasError = true
-				break
-			}
-		}
-		if hasError {
-			return m.state, nil
-		}
-	}
-
 	state := client.UnitStateHealthy
 	extendMsg := ""
 	var payload map[string]interface{}
diff --git a/internal/pkg/policy/self_test.go b/internal/pkg/policy/self_test.go
index 6de24abaa..850fe381e 100644
--- a/internal/pkg/policy/self_test.go
+++ b/internal/pkg/policy/self_test.go
@@ -10,9 +10,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	"net/http"
-	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -29,7 +26,6 @@ import (
 	"github.com/elastic/fleet-server/v7/internal/pkg/model"
 	mmock "github.com/elastic/fleet-server/v7/internal/pkg/monitor/mock"
 	ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing"
-	"github.com/elastic/fleet-server/v7/internal/pkg/testing/esutil"
 )
 
 func TestSelfMonitor_DefaultPolicy(t *testing.T) {
@@ -654,412 +650,3 @@ func (r *FakeReporter) Current() (client.UnitState, string, map[string]interface
 	defer r.lock.Unlock()
 	return r.state, r.msg, r.payload
 }
-func TestSelfMonitor_RemoteOutput_Degraded(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	cfg := config.Fleet{
-		Agent: config.Agent{
-			ID: "agent-id",
-		},
-	}
-	reporter := &FakeReporter{}
-
-	chHitT := make(chan []es.HitT, 1)
-	defer close(chHitT)
-	ms := mmock.NewMockSubscription()
-	ms.On("Output").Return((<-chan []es.HitT)(chHitT))
-	mm := mmock.NewMockMonitor()
-	mm.On("Subscribe").Return(ms).Once()
-	mm.On("Unsubscribe", mock.Anything).Return().Once()
-	bulker := ftesting.NewMockBulk()
-
-	errorMap := make(map[string]string)
-	errorMap["remote output"] = "error connecting to remote output"
-	bulker.On("GetRemoteOutputErrorMap").Return(errorMap)
-	emptyBulkerMap := make(map[string]bulk.Bulk)
-	bulker.On("GetBulkerMap").Return(emptyBulkerMap)
-
-	monitor := NewSelfMonitor(cfg, bulker, mm, "", reporter)
-	sm := monitor.(*selfMonitorT)
-	sm.checkTime = 100 * time.Millisecond
-
-	var policyLock sync.Mutex
-	var policyResult []model.Policy
-	sm.policyF = func(ctx context.Context, bulker bulk.Bulk, opt ...dl.Option) ([]model.Policy, error) {
-		policyLock.Lock()
-		defer policyLock.Unlock()
-		return policyResult, nil
-	}
-
-	var tokenLock sync.Mutex
-	var tokenResult []model.EnrollmentAPIKey
-	sm.enrollmentTokenF = func(ctx context.Context, bulker bulk.Bulk, policyID string) ([]model.EnrollmentAPIKey, error) {
-		tokenLock.Lock()
-		defer tokenLock.Unlock()
-		return tokenResult, nil
-	}
-
-	var merr error
-	var mwg sync.WaitGroup
-	mwg.Add(1)
-	go func() {
-		defer mwg.Done()
-		merr = monitor.Run(ctx)
-	}()
-
-	if err := monitor.(*selfMonitorT).waitStart(ctx); err != nil {
-		t.Fatal(err)
-	}
-
-	// should be set to starting
-	ftesting.Retry(t, ctx, func(ctx context.Context) error {
-		state, msg, _ := reporter.Current()
-		if state != client.UnitStateStarting {
-			return fmt.Errorf("should be reported as starting; instead its %s", state)
-		}
-		if msg != "Waiting on default policy with Fleet Server integration" {
-			return fmt.Errorf("should be matching with default policy")
-		}
-		return nil
-	}, ftesting.RetrySleep(1*time.Second))
-
-	policyID := uuid.Must(uuid.NewV4()).String()
-	rId := xid.New().String()
-	pData := model.PolicyData{Inputs: []map[string]interface{}{
-		{
-			"type": "fleet-server",
-		},
-	}}
-	policy := model.Policy{
-		ESDocument: model.ESDocument{
-			Id:      rId,
-			Version: 1,
-			SeqNo:   1,
-		},
-		PolicyID:           policyID,
-		CoordinatorIdx:     1,
-		Data:               &pData,
-		RevisionIdx:        1,
-		DefaultFleetServer: true,
-	}
-	policyData, err := json.Marshal(&policy)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	go func() {
-		chHitT <- []es.HitT{{
-			ID:      rId,
-			SeqNo:   1,
-			Version: 1,
-			Source:  policyData,
-		}}
-		policyLock.Lock()
-		defer policyLock.Unlock()
-		policyResult = append(policyResult, policy)
-	}()
-
-	// should be set to degraded because of remote output error
-	ftesting.Retry(t, ctx, func(ctx context.Context) error {
-		state, msg, _ := reporter.Current()
-		if state != client.UnitStateDegraded {
-			return fmt.Errorf("should be reported as degraded; instead its %s", state)
-		}
-		if msg != "Could not connect to remote ES output" {
-			return fmt.Errorf("expected remote ES connection error")
-		}
-		return nil
-	}, ftesting.RetrySleep(1*time.Second))
-
-	cancel()
-	mwg.Wait()
-	if merr != nil && merr != context.Canceled {
-		t.Fatal(merr)
-	}
-}
-
-func TestSelfMonitor_RemoteOutput_Back_To_Healthy(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	cfg := config.Fleet{
-		Agent: config.Agent{
-			ID: "agent-id",
-		},
-	}
-	reporter := &FakeReporter{}
-
-	chHitT := make(chan []es.HitT, 1)
-	defer close(chHitT)
-	ms := mmock.NewMockSubscription()
-	ms.On("Output").Return((<-chan []es.HitT)(chHitT))
-	mm := mmock.NewMockMonitor()
-	mm.On("Subscribe").Return(ms).Once()
-	mm.On("Unsubscribe", mock.Anything).Return().Once()
-	bulker := ftesting.NewMockBulk()
-
-	errorMap := make(map[string]string)
-	errorMap["remote output"] = "error connecting to remote output"
-	bulker.On("GetRemoteOutputErrorMap").Return(errorMap).Once()
-
-	// clear error
-	emptyMap := make(map[string]string)
-	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap).Once()
-
-	emptyBulkerMap := make(map[string]bulk.Bulk)
-	bulker.On("GetBulkerMap").Return(emptyBulkerMap)
-
-	monitor := NewSelfMonitor(cfg, bulker, mm, "", reporter)
-	sm := monitor.(*selfMonitorT)
-	sm.checkTime = 100 * time.Millisecond
-
-	var policyLock sync.Mutex
-	var policyResult []model.Policy
-	sm.policyF = func(ctx context.Context, bulker bulk.Bulk, opt ...dl.Option) ([]model.Policy, error) {
-		policyLock.Lock()
-		defer policyLock.Unlock()
-		return policyResult, nil
-	}
-
-	var tokenLock sync.Mutex
-	var tokenResult []model.EnrollmentAPIKey
-	sm.enrollmentTokenF = func(ctx context.Context, bulker bulk.Bulk, policyID string) ([]model.EnrollmentAPIKey, error) {
-		tokenLock.Lock()
-		defer tokenLock.Unlock()
-		return tokenResult, nil
-	}
-
-	var merr error
-	var mwg sync.WaitGroup
-	mwg.Add(1)
-	go func() {
-		defer mwg.Done()
-		merr = monitor.Run(ctx)
-	}()
-
-	if err := monitor.(*selfMonitorT).waitStart(ctx); err != nil {
-		t.Fatal(err)
-	}
-
-	// should be set to starting
-	ftesting.Retry(t, ctx, func(ctx context.Context) error {
-		state, msg, _ := reporter.Current()
-		if state != client.UnitStateStarting {
-			return fmt.Errorf("should be reported as starting; instead its %s", state)
-		}
-		if msg != "Waiting on default policy with Fleet Server integration" {
-			return fmt.Errorf("should be matching with default policy")
-		}
-		return nil
-	}, ftesting.RetrySleep(1*time.Second))
-
-	policyID := uuid.Must(uuid.NewV4()).String()
-	rId := xid.New().String()
-	pData := model.PolicyData{Inputs: []map[string]interface{}{
-		{
-			"type": "fleet-server",
-		},
-	}}
-	policy := model.Policy{
-		ESDocument: model.ESDocument{
-			Id:      rId,
-			Version: 1,
-			SeqNo:   1,
-		},
-		PolicyID:           policyID,
-		CoordinatorIdx:     1,
-		Data:               &pData,
-		RevisionIdx:        1,
-		DefaultFleetServer: true,
-	}
-	policyData, err := json.Marshal(&policy)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	go func() {
-		chHitT <- []es.HitT{{
-			ID:      rId,
-			SeqNo:   1,
-			Version: 1,
-			Source:  policyData,
-		}}
-		policyLock.Lock()
-		defer policyLock.Unlock()
-		policyResult = append(policyResult, policy)
-	}()
-
-	// should now be set to healthy
-	ftesting.Retry(t, ctx, func(ctx context.Context) error {
-		state, msg, _ := reporter.Current()
-		if state != client.UnitStateHealthy {
-			return fmt.Errorf("should be reported as healthy; instead its %s", state)
-		}
-		if msg != "Running on default policy with Fleet Server integration" {
-			return fmt.Errorf("should be matching with default policy")
-		}
-		return nil
-	})
-
-	cancel()
-	mwg.Wait()
-	if merr != nil && merr != context.Canceled {
-		t.Fatal(merr)
-	}
-}
-
-func TestSelfMonitor_RemoteOutput_Ping_Degraded_Healthy(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	cfg := config.Fleet{
-		Agent: config.Agent{
-			ID: "agent-id",
-		},
-	}
-	reporter := &FakeReporter{}
-
-	chHitT := make(chan []es.HitT, 1)
-	defer close(chHitT)
-	ms := mmock.NewMockSubscription()
-	ms.On("Output").Return((<-chan []es.HitT)(chHitT))
-	mm := mmock.NewMockMonitor()
-	mm.On("Subscribe").Return(ms).Once()
-	mm.On("Unsubscribe", mock.Anything).Return().Once()
-	bulker := ftesting.NewMockBulk()
-
-	emptyMap := make(map[string]string)
-	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
-
-	bulkerMap := make(map[string]bulk.Bulk)
-	outputBulker := ftesting.NewMockBulk()
-	mockES, mocktrans := esutil.MockESClient(t)
-
-	mocktrans.Response = &http.Response{
-		StatusCode: http.StatusInternalServerError,
-		Body:       nil,
-	}
-
-	outputBulker.On("Client").Return(mockES)
-	bulkerMap["output1"] = outputBulker
-	bulker.On("GetBulkerMap").Return(bulkerMap)
-
-	monitor := NewSelfMonitor(cfg, bulker, mm, "", reporter)
-	sm := monitor.(*selfMonitorT)
-	sm.checkTime = 100 * time.Millisecond
-
-	var policyLock sync.Mutex
-	var policyResult []model.Policy
-	sm.policyF = func(ctx context.Context, bulker bulk.Bulk, opt ...dl.Option) ([]model.Policy, error) {
-		policyLock.Lock()
-		defer policyLock.Unlock()
-		return policyResult, nil
-	}
-
-	var tokenLock sync.Mutex
-	var tokenResult []model.EnrollmentAPIKey
-	sm.enrollmentTokenF = func(ctx context.Context, bulker bulk.Bulk, policyID string) ([]model.EnrollmentAPIKey, error) {
-		tokenLock.Lock()
-		defer tokenLock.Unlock()
-		return tokenResult, nil
-	}
-
-	var merr error
-	var mwg sync.WaitGroup
-	mwg.Add(1)
-	go func() {
-		defer mwg.Done()
-		merr = monitor.Run(ctx)
-	}()
-
-	if err := monitor.(*selfMonitorT).waitStart(ctx); err != nil {
-		t.Fatal(err)
-	}
-
-	// should be set to starting
-	ftesting.Retry(t, ctx, func(ctx context.Context) error {
-		state, msg, _ := reporter.Current()
-		if state != client.UnitStateStarting {
-			return fmt.Errorf("should be reported as starting; instead its %s", state)
-		}
-		if msg != "Waiting on default policy with Fleet Server integration" {
-			return fmt.Errorf("should be matching with default policy")
-		}
-		return nil
-	}, ftesting.RetrySleep(1*time.Second))
-
-	policyID := uuid.Must(uuid.NewV4()).String()
-	rId := xid.New().String()
-	pData := model.PolicyData{Inputs: []map[string]interface{}{
-		{
-			"type": "fleet-server",
-		},
-	}}
-	policy := model.Policy{
-		ESDocument: model.ESDocument{
-			Id:      rId,
-			Version: 1,
-			SeqNo:   1,
-		},
-		PolicyID:           policyID,
-		CoordinatorIdx:     1,
-		Data:               &pData,
-		RevisionIdx:        1,
-		DefaultFleetServer: true,
-	}
-	policyData, err := json.Marshal(&policy)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	go func() {
-		chHitT <- []es.HitT{{
-			ID:      rId,
-			SeqNo:   1,
-			Version: 1,
-			Source:  policyData,
-		}}
-		policyLock.Lock()
-		defer policyLock.Unlock()
-		policyResult = append(policyResult, policy)
-	}()
-
-	// should be set to degraded because of remote output error
-	ftesting.Retry(t, ctx, func(ctx context.Context) error {
-		state, msg, _ := reporter.Current()
-		if state != client.UnitStateDegraded {
-			return fmt.Errorf("should be reported as degraded; instead its %s", state)
-		}
-		if msg != "Could not connect to remote ES output: output1, status code: 500" {
-			return fmt.Errorf("expected remote ES error")
-		}
-		return nil
-	}, ftesting.RetrySleep(1*time.Second))
-
-	// back to healthy
-	mocktrans.Response = &http.Response{
-		StatusCode: http.StatusOK,
-		Body:       ioutil.NopCloser(strings.NewReader(`{}`)),
-		Header: http.Header{
-			"X-Elastic-Product": []string{"Elasticsearch"},
-		},
-	}
-
-	ftesting.Retry(t, ctx, func(ctx context.Context) error {
-		state, msg, _ := reporter.Current()
-		if state != client.UnitStateHealthy {
-			return fmt.Errorf("should be reported as healthy; instead its %s", state)
-		}
-		if msg != "Running on default policy with Fleet Server integration" {
-			return fmt.Errorf("should be matching with default policy")
-		}
-		return nil
-	}, ftesting.RetrySleep(1*time.Second))
-
-	cancel()
-	mwg.Wait()
-	if merr != nil && merr != context.Canceled {
-		t.Fatal(merr)
-	}
-}
diff --git a/internal/pkg/policy/standalone.go b/internal/pkg/policy/standalone.go
index 386e8c3e6..597d267bd 100644
--- a/internal/pkg/policy/standalone.go
+++ b/internal/pkg/policy/standalone.go
@@ -16,7 +16,6 @@ import (
 	"github.com/elastic/fleet-server/v7/internal/pkg/dl"
 	"github.com/elastic/fleet-server/v7/internal/pkg/es"
 	"github.com/elastic/fleet-server/v7/internal/pkg/state"
-	"github.com/elastic/go-elasticsearch/v8/esapi"
 	"github.com/rs/zerolog"
 	"go.elastic.co/apm/v2"
 )
@@ -113,43 +112,7 @@ func (m *standAloneSelfMonitorT) check(ctx context.Context) {
 		message = fmt.Sprintf("Failed to request policies: %s", err)
 	}
 
-	remoteOutputErrorMap := m.bulker.GetRemoteOutputErrorMap()
-	hasError := false
-	remoteESPayload := make(map[string]interface{})
-	for key, value := range remoteOutputErrorMap {
-		if value != "" {
-			hasError = true
-			remoteESPayload[key] = value
-			break
-		}
-	}
-	if hasError {
-		state = client.UnitStateDegraded
-		message = fmt.Sprintf("Could not connect to remote ES output: %+v", remoteESPayload)
-		m.log.Debug().Msg(message)
-	} else {
-		bulkerMap := m.bulker.GetBulkerMap()
-		for outputName, outputBulker := range bulkerMap {
-			res, err := outputBulker.Client().Ping(outputBulker.Client().Ping.WithContext(ctx))
-			if err != nil {
-				m.log.Error().Err(err).Msg("error calling remote es ping")
-				state = client.UnitStateDegraded
-				message = fmt.Sprintf("Could not ping remote ES: %s, error: %s", outputName, err.Error())
-				break
-			} else if res.StatusCode != 200 {
-				state = client.UnitStateDegraded
-				message = fmt.Sprintf("Could not connect to remote ES output: %s, status code: %d", outputName, res.StatusCode)
-				m.log.Debug().Msg(message)
-				break
-			}
-		}
-	}
-
 	if current != state {
 		m.updateState(state, message)
 	}
 }
-
-func GetRequest(req *esapi.CatHealthRequest) {
-	req.Format = "json"
-}
diff --git a/internal/pkg/policy/standalone_test.go b/internal/pkg/policy/standalone_test.go
index 903a72bc9..d1ebd77b6 100644
--- a/internal/pkg/policy/standalone_test.go
+++ b/internal/pkg/policy/standalone_test.go
@@ -9,9 +9,6 @@ package policy
 import (
 	"context"
 	"errors"
-	"io/ioutil"
-	"net/http"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -22,7 +19,6 @@ import (
 	"github.com/elastic/fleet-server/v7/internal/pkg/dl"
 	"github.com/elastic/fleet-server/v7/internal/pkg/es"
 	ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing"
-	"github.com/elastic/fleet-server/v7/internal/pkg/testing/esutil"
 )
 
 func TestStandAloneSelfMonitor(t *testing.T) {
@@ -102,98 +98,3 @@ func TestStandAloneSelfMonitor(t *testing.T) {
 		})
 	}
 }
-
-func TestStandAloneSelfMonitorRemoteOutput(t *testing.T) {
-
-	searchArguments := []any{mock.Anything, ".fleet-policies", mock.Anything, mock.Anything}
-
-	bulker := ftesting.NewMockBulk()
-	bulker.On("Search", searchArguments...).Return(&es.ResultT{
-		Aggregations: map[string]es.Aggregation{
-			dl.FieldPolicyID: es.Aggregation{},
-		},
-	}, nil)
-
-	errorMap := make(map[string]string)
-	errorMap["remote output"] = "error connecting to remote output"
-	bulker.On("GetRemoteOutputErrorMap").Return(errorMap).Once()
-
-	emptyMap := make(map[string]string)
-	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap).Once()
-
-	emptyBulkerMap := make(map[string]bulk.Bulk)
-	bulker.On("GetBulkerMap").Return(emptyBulkerMap).Once()
-
-	reporter := &FakeReporter{}
-
-	sm := NewStandAloneSelfMonitor(bulker, reporter)
-	sm.updateState(client.UnitStateStarting, "test")
-
-	sm.check(context.Background())
-	state := sm.State()
-
-	assert.Equal(t, client.UnitStateDegraded, state)
-	assert.Equal(t, state, reporter.state, "reported state should be the same")
-
-	// back to healthy
-	sm.check(context.Background())
-	state = sm.State()
-
-	assert.Equal(t, client.UnitStateHealthy, state)
-	assert.Equal(t, state, reporter.state, "reported state should be the same")
-}
-
-func TestStandAloneSelfMonitorRemoteOutputPing(t *testing.T) {
-
-	searchArguments := []any{mock.Anything, ".fleet-policies", mock.Anything, mock.Anything}
-
-	bulker := ftesting.NewMockBulk()
-	bulker.On("Search", searchArguments...).Return(&es.ResultT{
-		Aggregations: map[string]es.Aggregation{
-			dl.FieldPolicyID: es.Aggregation{},
-		},
-	}, nil)
-
-	emptyMap := make(map[string]string)
-	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
-
-	bulkerMap := make(map[string]bulk.Bulk)
-	outputBulker := ftesting.NewMockBulk()
-	mockES, mocktrans := esutil.MockESClient(t)
-
-	mocktrans.Response = &http.Response{
-		StatusCode: http.StatusInternalServerError,
-		Body:       nil,
-	}
-
-	outputBulker.On("Client").Return(mockES)
-	bulkerMap["output1"] = outputBulker
-	bulker.On("GetBulkerMap").Return(bulkerMap)
-
-	reporter := &FakeReporter{}
-
-	sm := NewStandAloneSelfMonitor(bulker, reporter)
-	sm.updateState(client.UnitStateStarting, "test")
-
-	sm.check(context.Background())
-	state := sm.State()
-
-	assert.Equal(t, client.UnitStateDegraded, state)
-	assert.Equal(t, state, reporter.state, "reported state should be the same")
-
-	// back to healthy
-	mocktrans.Response = &http.Response{
-		StatusCode: http.StatusOK,
-		Body:       ioutil.NopCloser(strings.NewReader(`{}`)),
-		Header: http.Header{
-			"X-Elastic-Product": []string{"Elasticsearch"},
-		},
-	}
-
-	sm.check(context.Background())
-	state = sm.State()
-
-	assert.Equal(t, client.UnitStateHealthy, state)
-	assert.Equal(t, state, reporter.state, "reported state should be the same")
-
-}

From 37106b0d518f5985962d79990b93f60ba1989e1e Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 21 Nov 2023 13:56:35 +0100
Subject: [PATCH 69/91] fix tests

---
 internal/pkg/api/handleAck_test.go                    |  1 +
 internal/pkg/policy/policy_output_test.go             | 11 ++++++-----
 .../pkg/server/remote_es_output_integration_test.go   |  2 +-
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/internal/pkg/api/handleAck_test.go b/internal/pkg/api/handleAck_test.go
index 318c55695..c863973bd 100644
--- a/internal/pkg/api/handleAck_test.go
+++ b/internal/pkg/api/handleAck_test.go
@@ -602,6 +602,7 @@ func TestInvalidateAPIKeysRemoteOutput(t *testing.T) {
 
 	bulker.AssertExpectations(t)
 	remoteBulker.AssertExpectations(t)
+	remoteBulker2.AssertExpectations(t)
 }
 
 func TestAckHandleUpgrade(t *testing.T) {
diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go
index 39f08de02..ba2f52722 100644
--- a/internal/pkg/policy/policy_output_test.go
+++ b/internal/pkg/policy/policy_output_test.go
@@ -305,10 +305,10 @@ func TestPolicyRemoteESOutputPrepareNoRole(t *testing.T) {
 		Role: nil,
 	}
 	outputBulker := ftesting.NewMockBulk()
-	bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
+	bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
 
 	err := po.Prepare(context.Background(), logger, bulker, &model.Agent{}, map[string]map[string]interface{}{})
-	require.NotNil(t, err, "expected prepare to error")
+	require.Error(t, err, "expected prepare to error")
 	bulker.AssertExpectations(t)
 }
 
@@ -330,7 +330,7 @@ func TestPolicyRemoteESOutputPrepare(t *testing.T) {
 		}
 
 		outputBulker := ftesting.NewMockBulk()
-		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
+		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
 
 		policyMap := map[string]map[string]interface{}{
 			"test output": map[string]interface{}{
@@ -391,7 +391,7 @@ func TestPolicyRemoteESOutputPrepare(t *testing.T) {
 			Return(nil).Once()
 
 		outputBulker := ftesting.NewMockBulk()
-		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
+		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
 
 		outputBulker.
 			On("APIKeyRead", mock.Anything, mock.Anything, mock.Anything).
@@ -461,7 +461,8 @@ func TestPolicyRemoteESOutputPrepare(t *testing.T) {
 		outputBulker.On("APIKeyCreate",
 			mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
 			Return(&apiKey, nil).Once()
-		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
+		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
+		bulker.On("GetRemoteOutputErrorMap").Return(make(map[string]string))
 
 		output := Output{
 			Type: OutputTypeRemoteElasticsearch,
diff --git a/internal/pkg/server/remote_es_output_integration_test.go b/internal/pkg/server/remote_es_output_integration_test.go
index 2d2a04a67..be25f3ff8 100644
--- a/internal/pkg/server/remote_es_output_integration_test.go
+++ b/internal/pkg/server/remote_es_output_integration_test.go
@@ -70,7 +70,7 @@ func Checkin(t *testing.T, ctx context.Context, srv *tserver, agentID, key strin
 	defaultOutput, ok := outputs["default"].(map[string]interface{})
 	require.True(t, ok, "expected default to be map")
 	defaultAPIKey := defaultOutput["api_key"]
-	require.False(t, remoteAPIKey == defaultAPIKey, "expected remote api key to be different than default")
+	require.NotEqual(t, remoteAPIKey, defaultAPIKey, "expected remote api key to be different than default")
 
 }
 

From 64277ffa87a4d8a1262102dda37c82776e849ff6 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 21 Nov 2023 15:34:50 +0100
Subject: [PATCH 70/91] verify api key exists in remote es

---
 .../remote_es_output_integration_test.go      | 37 ++++++++++++++++---
 1 file changed, 31 insertions(+), 6 deletions(-)

diff --git a/internal/pkg/server/remote_es_output_integration_test.go b/internal/pkg/server/remote_es_output_integration_test.go
index be25f3ff8..252cd2cd8 100644
--- a/internal/pkg/server/remote_es_output_integration_test.go
+++ b/internal/pkg/server/remote_es_output_integration_test.go
@@ -9,20 +9,23 @@ package server
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/elastic/fleet-server/v7/internal/pkg/apikey"
 	"github.com/elastic/fleet-server/v7/internal/pkg/dl"
 	"github.com/elastic/fleet-server/v7/internal/pkg/model"
+	ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing"
 	"github.com/hashicorp/go-cleanhttp"
 	"github.com/stretchr/testify/require"
 )
 
-func Checkin(t *testing.T, ctx context.Context, srv *tserver, agentID, key string) {
+func Checkin(t *testing.T, ctx context.Context, srv *tserver, agentID, key string) string {
 	str := agentID
 	cli := cleanhttp.DefaultClient()
 	var obj map[string]interface{}
@@ -66,12 +69,14 @@ func Checkin(t *testing.T, ctx context.Context, srv *tserver, agentID, key strin
 	require.Equal(t, "elasticsearch", oType)
 	serviceToken := remoteES["service_token"]
 	require.Equal(t, nil, serviceToken)
-	remoteAPIKey := remoteES["api_key"]
+	remoteAPIKey, ok := remoteES["api_key"].(string)
+	require.True(t, ok, "expected remoteAPIKey to be string")
 	defaultOutput, ok := outputs["default"].(map[string]interface{})
 	require.True(t, ok, "expected default to be map")
-	defaultAPIKey := defaultOutput["api_key"]
+	defaultAPIKey, ok := defaultOutput["api_key"].(string)
+	require.True(t, ok, "expected defaultAPIKey to be string")
 	require.NotEqual(t, remoteAPIKey, defaultAPIKey, "expected remote api key to be different than default")
-
+	return remoteAPIKey
 }
 
 func Test_Agent_Remote_ES_Output(t *testing.T) {
@@ -95,6 +100,7 @@ func Test_Agent_Remote_ES_Output(t *testing.T) {
 	t.Log("Create policy with remote ES output")
 
 	var policyRemoteID = "policyRemoteID"
+	remoteESHost := "localhost:9201"
 	var policyDataRemoteES = model.PolicyData{
 		Outputs: map[string]map[string]interface{}{
 			"default": {
@@ -102,7 +108,7 @@ func Test_Agent_Remote_ES_Output(t *testing.T) {
 			},
 			"remoteES": {
 				"type":          "remote_elasticsearch",
-				"hosts":         []string{"localhost:9201"},
+				"hosts":         []string{remoteESHost},
 				"service_token": os.Getenv("REMOTE_ELASTICSEARCH_SERVICE_TOKEN"),
 			},
 		},
@@ -166,5 +172,24 @@ func Test_Agent_Remote_ES_Output(t *testing.T) {
 		}
 	}()
 
-	Checkin(t, ctx, srvCopy, agentID, key)
+	remoteAPIKey := Checkin(t, ctx, srvCopy, agentID, key)
+	apiKeyID := strings.Split(remoteAPIKey, ":")[0]
+
+	ftesting.Retry(t, ctx, func(ctx context.Context) error {
+		requestURL := fmt.Sprintf("http://elastic:changeme@%s/_security/api_key?id=%s", remoteESHost, apiKeyID)
+		res, err := http.Get(requestURL)
+		if err != nil {
+			fmt.Printf("error making http request: %s\n", err)
+			t.Fatal("error querying remote api key")
+		}
+
+		require.Equal(t, 200, res.StatusCode)
+
+		defer res.Body.Close()
+		respString, err := io.ReadAll(res.Body)
+		require.NoError(t, err, "did not expect error when parsing api key response")
+
+		require.Contains(t, string(respString), "\"invalidated\":false")
+		return nil
+	}, ftesting.RetrySleep(1*time.Second))
 }

From ea1d3bd49a3a0611dbee16fd7ef4aeeeb0ad2024 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 21 Nov 2023 15:37:06 +0100
Subject: [PATCH 71/91] fix lint

---
 internal/pkg/server/remote_es_output_integration_test.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/internal/pkg/server/remote_es_output_integration_test.go b/internal/pkg/server/remote_es_output_integration_test.go
index 252cd2cd8..616379699 100644
--- a/internal/pkg/server/remote_es_output_integration_test.go
+++ b/internal/pkg/server/remote_es_output_integration_test.go
@@ -179,7 +179,6 @@ func Test_Agent_Remote_ES_Output(t *testing.T) {
 		requestURL := fmt.Sprintf("http://elastic:changeme@%s/_security/api_key?id=%s", remoteESHost, apiKeyID)
 		res, err := http.Get(requestURL)
 		if err != nil {
-			fmt.Printf("error making http request: %s\n", err)
 			t.Fatal("error querying remote api key")
 		}
 

From e15f9951cabfa9cac0620cdbfa87d1f397639fec Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 21 Nov 2023 15:42:05 +0100
Subject: [PATCH 72/91] remote remoteOutputErrorMap as not used

---
 internal/pkg/bulk/engine.go               | 17 ++---------------
 internal/pkg/policy/policy_output.go      |  5 -----
 internal/pkg/policy/policy_output_test.go |  1 -
 internal/pkg/policy/self_test.go          |  8 --------
 internal/pkg/policy/standalone_test.go    |  2 --
 internal/pkg/testing/bulk.go              |  7 -------
 6 files changed, 2 insertions(+), 38 deletions(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 34188a7c2..4014be157 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -71,8 +71,6 @@ type Bulk interface {
 	CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, bool, error)
 	GetBulker(outputName string) Bulk
 	GetBulkerMap() map[string]Bulk
-	GetRemoteOutputErrorMap() map[string]string
-	SetRemoteOutputError(name string, status string)
 	CancelFn() context.CancelFunc
 
 	ReadSecrets(ctx context.Context, secretIds []string) (map[string]string, error)
@@ -89,7 +87,6 @@ type Bulker struct {
 	tracer                *apm.Tracer
 	remoteOutputConfigMap map[string]map[string]interface{}
 	bulkerMap             map[string]Bulk
-	remoteOutputErrorMap  map[string]string
 	cancelFn              context.CancelFunc
 	remoteOutputLimit     *semaphore.Weighted
 }
@@ -121,21 +118,11 @@ func NewBulker(es esapi.Transport, tracer *apm.Tracer, opts ...BulkOpt) *Bulker
 		tracer:                tracer,
 		remoteOutputConfigMap: make(map[string]map[string]interface{}),
 		// remote ES bulkers
-		bulkerMap:            make(map[string]Bulk),
-		remoteOutputErrorMap: make(map[string]string),
-		remoteOutputLimit:    semaphore.NewWeighted(1),
+		bulkerMap:         make(map[string]Bulk),
+		remoteOutputLimit: semaphore.NewWeighted(1),
 	}
 }
 
-func (b *Bulker) GetRemoteOutputErrorMap() map[string]string {
-	return b.remoteOutputErrorMap
-}
-
-func (b *Bulker) SetRemoteOutputError(name string, status string) {
-	// TODO concurrency control of updating map
-	b.remoteOutputErrorMap[name] = status
-}
-
 func (b *Bulker) GetBulker(outputName string) Bulk {
 	return b.bulkerMap[outputName]
 }
diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index 7a8b5bd85..0b0d0b0f1 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -264,9 +264,6 @@ func (p *Output) prepareElasticsearch(
 		if outputAPIKey == nil && p.Type == OutputTypeRemoteElasticsearch {
 			if err != nil {
 				zerolog.Ctx(ctx).Warn().Err(err).Msg("Could not create API key in remote ES")
-				bulker.SetRemoteOutputError(p.Name, err.Error())
-			} else if bulker.GetRemoteOutputErrorMap()[p.Name] != "" {
-				bulker.SetRemoteOutputError(p.Name, "")
 			}
 
 			// replace type remote_elasticsearch with elasticsearch as agent doesn't recognize remote_elasticsearch
@@ -274,8 +271,6 @@ func (p *Output) prepareElasticsearch(
 			// remove the service token from the agent policy sent to the agent
 			delete(outputMap[p.Name], FieldOutputServiceToken)
 			return nil
-		} else if p.Type == OutputTypeRemoteElasticsearch && bulker.GetRemoteOutputErrorMap()[p.Name] != "" {
-			bulker.SetRemoteOutputError(p.Name, "")
 		}
 		if err != nil {
 			return fmt.Errorf("failed generate output API key: %w", err)
diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go
index ba2f52722..f41560dc4 100644
--- a/internal/pkg/policy/policy_output_test.go
+++ b/internal/pkg/policy/policy_output_test.go
@@ -462,7 +462,6 @@ func TestPolicyRemoteESOutputPrepare(t *testing.T) {
 			mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
 			Return(&apiKey, nil).Once()
 		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
-		bulker.On("GetRemoteOutputErrorMap").Return(make(map[string]string))
 
 		output := Output{
 			Type: OutputTypeRemoteElasticsearch,
diff --git a/internal/pkg/policy/self_test.go b/internal/pkg/policy/self_test.go
index 850fe381e..dd1f4a865 100644
--- a/internal/pkg/policy/self_test.go
+++ b/internal/pkg/policy/self_test.go
@@ -47,8 +47,6 @@ func TestSelfMonitor_DefaultPolicy(t *testing.T) {
 	mm.On("Subscribe").Return(ms).Once()
 	mm.On("Unsubscribe", mock.Anything).Return().Once()
 	bulker := ftesting.NewMockBulk()
-	emptyMap := make(map[string]string)
-	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
 	emptyBulkerMap := make(map[string]bulk.Bulk)
 	bulker.On("GetBulkerMap").Return(emptyBulkerMap)
 
@@ -188,8 +186,6 @@ func TestSelfMonitor_DefaultPolicy_Degraded(t *testing.T) {
 	mm.On("Unsubscribe", mock.Anything).Return().Once()
 	bulker := ftesting.NewMockBulk()
 
-	emptyMap := make(map[string]string)
-	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
 	emptyBulkerMap := make(map[string]bulk.Bulk)
 	bulker.On("GetBulkerMap").Return(emptyBulkerMap)
 
@@ -349,8 +345,6 @@ func TestSelfMonitor_SpecificPolicy(t *testing.T) {
 	mm.On("Subscribe").Return(ms).Once()
 	mm.On("Unsubscribe", mock.Anything).Return().Once()
 	bulker := ftesting.NewMockBulk()
-	emptyMap := make(map[string]string)
-	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
 	emptyBulkerMap := make(map[string]bulk.Bulk)
 	bulker.On("GetBulkerMap").Return(emptyBulkerMap)
 
@@ -489,8 +483,6 @@ func TestSelfMonitor_SpecificPolicy_Degraded(t *testing.T) {
 	mm.On("Subscribe").Return(ms).Once()
 	mm.On("Unsubscribe", mock.Anything).Return().Once()
 	bulker := ftesting.NewMockBulk()
-	emptyMap := make(map[string]string)
-	bulker.On("GetRemoteOutputErrorMap").Return(emptyMap)
 	emptyBulkerMap := make(map[string]bulk.Bulk)
 	bulker.On("GetBulkerMap").Return(emptyBulkerMap)
 
diff --git a/internal/pkg/policy/standalone_test.go b/internal/pkg/policy/standalone_test.go
index d1ebd77b6..117afe1a9 100644
--- a/internal/pkg/policy/standalone_test.go
+++ b/internal/pkg/policy/standalone_test.go
@@ -81,8 +81,6 @@ func TestStandAloneSelfMonitor(t *testing.T) {
 		t.Run(c.title, func(t *testing.T) {
 			bulker := ftesting.NewMockBulk()
 			bulker.On("Search", searchArguments...).Return(c.searchResult, c.searchErr)
-			emptyMap := make(map[string]string)
-			bulker.On("GetRemoteOutputErrorMap").Return(emptyMap).Once()
 			emptyBulkerMap := make(map[string]bulk.Bulk)
 			bulker.On("GetBulkerMap").Return(emptyBulkerMap).Once()
 			reporter := &FakeReporter{}
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index 4b54a7f5f..92915cc7f 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -95,13 +95,6 @@ func (m *MockBulk) CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger,
 	return args.Get(0).(bulk.Bulk), args.Get(1).(bool), nil
 }
 
-func (m *MockBulk) GetRemoteOutputErrorMap() map[string]string {
-	args := m.Called()
-	return args.Get(0).(map[string]string)
-}
-
-func (m *MockBulk) SetRemoteOutputError(name string, status string) {}
-
 func (m *MockBulk) CancelFn() context.CancelFunc {
 	args := m.Called()
 	return args.Get(0).(context.CancelFunc)

From 6d4f87e111cd8e261662d86efeea6c51d4e92083 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 21 Nov 2023 15:55:42 +0100
Subject: [PATCH 73/91] fix lint

---
 internal/pkg/server/remote_es_output_integration_test.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/internal/pkg/server/remote_es_output_integration_test.go b/internal/pkg/server/remote_es_output_integration_test.go
index 616379699..55b204382 100644
--- a/internal/pkg/server/remote_es_output_integration_test.go
+++ b/internal/pkg/server/remote_es_output_integration_test.go
@@ -177,7 +177,12 @@ func Test_Agent_Remote_ES_Output(t *testing.T) {
 
 	ftesting.Retry(t, ctx, func(ctx context.Context) error {
 		requestURL := fmt.Sprintf("http://elastic:changeme@%s/_security/api_key?id=%s", remoteESHost, apiKeyID)
-		res, err := http.Get(requestURL)
+
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil)
+		if err != nil {
+			t.Fatal("error creating request for remote api key")
+		}
+		res, err := http.DefaultClient.Do(req)
 		if err != nil {
 			t.Fatal("error querying remote api key")
 		}

From 85142e2e03d3a984b8d03dd2c56b976796c8cc01 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Wed, 22 Nov 2023 11:37:27 +0100
Subject: [PATCH 74/91] revert openapi description about degraded state

---
 pkg/api/versions/2023_06_01/api/openapi.yml  | 1 -
 pkg/api/versions/2023_06_01/api/types.gen.go | 2 --
 2 files changed, 3 deletions(-)

diff --git a/pkg/api/versions/2023_06_01/api/openapi.yml b/pkg/api/versions/2023_06_01/api/openapi.yml
index 00884d691..fe663481f 100644
--- a/pkg/api/versions/2023_06_01/api/openapi.yml
+++ b/pkg/api/versions/2023_06_01/api/openapi.yml
@@ -85,7 +85,6 @@ components:
           description: |
             A Unit state that fleet-server may report.
             Unit state is defined in the elastic-agent-client specification.
-            Fleet-server goes to degraded state if remote elasticsearch output is not reachable.
           enum:
             - starting
             - configuring
diff --git a/pkg/api/versions/2023_06_01/api/types.gen.go b/pkg/api/versions/2023_06_01/api/types.gen.go
index 1c025c03f..7b412627d 100644
--- a/pkg/api/versions/2023_06_01/api/types.gen.go
+++ b/pkg/api/versions/2023_06_01/api/types.gen.go
@@ -418,7 +418,6 @@ type StatusAPIResponse struct {
 
 	// Status A Unit state that fleet-server may report.
 	// Unit state is defined in the elastic-agent-client specification.
-	// Fleet-server goes to degraded state if remote elasticsearch output is not reachable.
 	Status StatusResponseStatus `json:"status"`
 
 	// Version Version information included in the response to an authorized status request.
@@ -427,7 +426,6 @@ type StatusAPIResponse struct {
 
 // StatusResponseStatus A Unit state that fleet-server may report.
 // Unit state is defined in the elastic-agent-client specification.
-// Fleet-server goes to degraded state if remote elasticsearch output is not reachable.
 type StatusResponseStatus string
 
 // StatusResponseVersion Version information included in the response to an authorized status request.

From 727daa4e0fe1c4d62bb32286f8b7080efac2aeda Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Wed, 22 Nov 2023 16:27:12 +0100
Subject: [PATCH 75/91] reading output from policies index if bulker not found

---
 internal/pkg/api/handleAck.go                | 13 +++++++
 internal/pkg/bulk/bulk_remote_output_test.go |  6 +--
 internal/pkg/bulk/engine.go                  | 12 ++++--
 internal/pkg/dl/policies.go                  | 40 ++++++++++++++++++++
 internal/pkg/policy/policy_output.go         |  2 +-
 internal/pkg/policy/policy_output_test.go    |  8 ++--
 internal/pkg/testing/bulk.go                 |  4 +-
 7 files changed, 71 insertions(+), 14 deletions(-)

diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go
index 79839d1f6..567ff29b0 100644
--- a/internal/pkg/api/handleAck.go
+++ b/internal/pkg/api/handleAck.go
@@ -591,6 +591,19 @@ func (ack *AckT) invalidateAPIKeys(ctx context.Context, zlog zerolog.Logger, toR
 	// using remote es bulker to invalidate api key
 	for outputName, outputIds := range remoteIds {
 		outputBulk := ack.bulk.GetBulker(outputName)
+
+		if outputBulk == nil {
+			// read output config from .fleet-policies, not filtering by policy id as agent could be reassigned
+			policy, err := dl.QueryOutputFromPolicy(ctx, ack.bulk, outputName)
+			if err != nil || policy == nil {
+				zlog.Debug().Str("outputName", outputName).Msg("Output policy not found")
+			} else {
+				outputBulk, _, err = ack.bulk.CreateAndGetBulker(ctx, zlog, outputName, policy.Data.Outputs)
+				if err != nil {
+					zlog.Debug().Str("outputName", outputName).Msg("Failed to recreate output bulker")
+				}
+			}
+		}
 		if outputBulk != nil {
 			if err := outputBulk.APIKeyInvalidate(ctx, outputIds...); err != nil {
 				zlog.Info().Err(err).Strs("ids", outputIds).Str("outputName", outputName).Msg("Failed to invalidate API keys")
diff --git a/internal/pkg/bulk/bulk_remote_output_test.go b/internal/pkg/bulk/bulk_remote_output_test.go
index c08d48afb..c777d22c0 100644
--- a/internal/pkg/bulk/bulk_remote_output_test.go
+++ b/internal/pkg/bulk/bulk_remote_output_test.go
@@ -99,7 +99,7 @@ func Test_CreateAndGetBulkerNew(t *testing.T) {
 		"hosts":         []interface{}{"https://remote-es:443"},
 		"service_token": "token1",
 	}
-	newBulker, hasChanged, err := bulker.CreateAndGetBulker(ctx, log, "remote1", "token1", outputMap)
+	newBulker, hasChanged, err := bulker.CreateAndGetBulker(ctx, log, "remote1", outputMap)
 	assert.NotNil(t, newBulker)
 	assert.Equal(t, false, hasChanged)
 	assert.Nil(t, err)
@@ -120,7 +120,7 @@ func Test_CreateAndGetBulkerExisting(t *testing.T) {
 	}
 	bulker.remoteOutputConfigMap["remote1"] = cfg
 	outputMap["remote1"] = cfg
-	newBulker, hasChanged, err := bulker.CreateAndGetBulker(ctx, log, "remote1", "token1", outputMap)
+	newBulker, hasChanged, err := bulker.CreateAndGetBulker(ctx, log, "remote1", outputMap)
 	assert.Equal(t, outputBulker, newBulker)
 	assert.Equal(t, false, hasChanged)
 	assert.Nil(t, err)
@@ -146,7 +146,7 @@ func Test_CreateAndGetBulkerChanged(t *testing.T) {
 	}
 	cancelFnCalled := false
 	outputBulker.cancelFn = func() { cancelFnCalled = true }
-	newBulker, hasChanged, err := bulker.CreateAndGetBulker(ctx, log, "remote1", "token2", outputMap)
+	newBulker, hasChanged, err := bulker.CreateAndGetBulker(ctx, log, "remote1", outputMap)
 	assert.NotEqual(t, outputBulker, newBulker)
 	assert.Equal(t, true, hasChanged)
 	assert.Nil(t, err)
diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 4014be157..6bcc96909 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -68,7 +68,7 @@ type Bulk interface {
 	// Accessor used to talk to elastic search direcly bypassing bulk engine
 	Client() *elasticsearch.Client
 
-	CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, bool, error)
+	CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, outputName string, outputMap map[string]map[string]interface{}) (Bulk, bool, error)
 	GetBulker(outputName string) Bulk
 	GetBulkerMap() map[string]Bulk
 	CancelFn() context.CancelFunc
@@ -150,7 +150,7 @@ func (b *Bulker) updateBulkerMap(ctx context.Context, outputName string, newBulk
 // if bulker exists for output, check if config changed
 // if not changed, return the existing bulker
 // if changed, stop the existing bulker and create a new one
-func (b *Bulker) CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (Bulk, bool, error) {
+func (b *Bulker) CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, outputName string, outputMap map[string]map[string]interface{}) (Bulk, bool, error) {
 	hasConfigChanged := b.hasChangedAndUpdateRemoteOutputConfig(zlog, outputName, outputMap[outputName])
 	bulker := b.bulkerMap[outputName]
 	if bulker != nil && !hasConfigChanged {
@@ -163,7 +163,7 @@ func (b *Bulker) CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, ou
 		}
 	}
 	bulkCtx, bulkCancel := context.WithCancel(context.Background())
-	es, err := b.createRemoteEsClient(bulkCtx, outputName, serviceToken, outputMap)
+	es, err := b.createRemoteEsClient(bulkCtx, outputName, outputMap)
 	if err != nil {
 		defer bulkCancel()
 		return nil, hasConfigChanged, err
@@ -199,7 +199,7 @@ func (b *Bulker) CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, ou
 	return newBulker, hasConfigChanged, nil
 }
 
-func (b *Bulker) createRemoteEsClient(ctx context.Context, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (*elasticsearch.Client, error) {
+func (b *Bulker) createRemoteEsClient(ctx context.Context, outputName string, outputMap map[string]map[string]interface{}) (*elasticsearch.Client, error) {
 	hostsObj := outputMap[outputName]["hosts"]
 	hosts, ok := hostsObj.([]interface{})
 	if !ok {
@@ -212,6 +212,10 @@ func (b *Bulker) createRemoteEsClient(ctx context.Context, outputName string, se
 			return nil, fmt.Errorf("failed to get hosts from output: %v", host)
 		}
 	}
+	serviceToken, ok := outputMap[outputName]["service_token"].(string)
+	if !ok {
+		return nil, fmt.Errorf("failed to get service token from output: %v", outputName)
+	}
 
 	cfg := config.Config{
 		Output: config.Output{
diff --git a/internal/pkg/dl/policies.go b/internal/pkg/dl/policies.go
index c3cf457bf..258a0e733 100644
--- a/internal/pkg/dl/policies.go
+++ b/internal/pkg/dl/policies.go
@@ -10,7 +10,9 @@ import (
 	"errors"
 
 	"github.com/elastic/fleet-server/v7/internal/pkg/bulk"
+	"github.com/elastic/fleet-server/v7/internal/pkg/es"
 	"github.com/elastic/fleet-server/v7/internal/pkg/model"
+	"github.com/rs/zerolog"
 
 	"github.com/elastic/fleet-server/v7/internal/pkg/dsl"
 )
@@ -18,6 +20,7 @@ import (
 var (
 	tmplQueryLatestPolicies = prepareQueryLatestPolicies()
 	ErrMissingAggregations  = errors.New("missing expected aggregation result")
+	tmplQueryPolicies       = prepareQueryPolicies()
 )
 
 func prepareQueryLatestPolicies() []byte {
@@ -72,3 +75,40 @@ func CreatePolicy(ctx context.Context, bulker bulk.Bulk, policy model.Policy, op
 	}
 	return bulker.Create(ctx, o.indexName, "", data, bulk.WithRefresh())
 }
+
+func prepareQueryPolicies() *dsl.Tmpl {
+	tmpl := dsl.NewTmpl()
+	root := dsl.NewRoot()
+	root.Size(100)
+	root.Sort().SortOrder("@timestamp", "desc")
+	root.Source().Includes("data.outputs")
+	tmpl.MustResolve(root)
+	return tmpl
+}
+
+// query policies last updated, find the one with matching output
+// can't filter on output in ES as the field is not mapped
+func QueryOutputFromPolicy(ctx context.Context, bulker bulk.Bulk, outputName string, opt ...Option) (*model.Policy, error) {
+	o := newOption(FleetPolicies, opt...)
+	params := map[string]interface{}{}
+	res, err := Search(ctx, bulker, tmplQueryPolicies, o.indexName, params)
+	if err != nil {
+		if errors.Is(err, es.ErrIndexNotFound) {
+			zerolog.Ctx(ctx).Debug().Str("index", o.indexName).Msg(es.ErrIndexNotFound.Error())
+			err = nil
+		}
+		return nil, err
+	}
+	var policy model.Policy
+	for _, hit := range res.Hits {
+		err = hit.Unmarshal(&policy)
+		if err != nil {
+			return nil, err
+		}
+		if policy.Data.Outputs[outputName] != nil {
+			return &policy, nil
+		}
+	}
+	zerolog.Ctx(ctx).Debug().Str("outputName", outputName).Msg("policy with output not found")
+	return nil, nil
+}
diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go
index 0b0d0b0f1..dfc51c21f 100644
--- a/internal/pkg/policy/policy_output.go
+++ b/internal/pkg/policy/policy_output.go
@@ -60,7 +60,7 @@ func (p *Output) Prepare(ctx context.Context, zlog zerolog.Logger, bulker bulk.B
 		}
 	case OutputTypeRemoteElasticsearch:
 		zlog.Debug().Msg("preparing remote elasticsearch output")
-		newBulker, hasConfigChanged, err := bulker.CreateAndGetBulker(ctx, zlog, p.Name, p.ServiceToken, outputMap)
+		newBulker, hasConfigChanged, err := bulker.CreateAndGetBulker(ctx, zlog, p.Name, outputMap)
 		if err != nil {
 			return err
 		}
diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go
index f41560dc4..d6480460b 100644
--- a/internal/pkg/policy/policy_output_test.go
+++ b/internal/pkg/policy/policy_output_test.go
@@ -305,7 +305,7 @@ func TestPolicyRemoteESOutputPrepareNoRole(t *testing.T) {
 		Role: nil,
 	}
 	outputBulker := ftesting.NewMockBulk()
-	bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
+	bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
 
 	err := po.Prepare(context.Background(), logger, bulker, &model.Agent{}, map[string]map[string]interface{}{})
 	require.Error(t, err, "expected prepare to error")
@@ -330,7 +330,7 @@ func TestPolicyRemoteESOutputPrepare(t *testing.T) {
 		}
 
 		outputBulker := ftesting.NewMockBulk()
-		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
+		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
 
 		policyMap := map[string]map[string]interface{}{
 			"test output": map[string]interface{}{
@@ -391,7 +391,7 @@ func TestPolicyRemoteESOutputPrepare(t *testing.T) {
 			Return(nil).Once()
 
 		outputBulker := ftesting.NewMockBulk()
-		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
+		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
 
 		outputBulker.
 			On("APIKeyRead", mock.Anything, mock.Anything, mock.Anything).
@@ -461,7 +461,7 @@ func TestPolicyRemoteESOutputPrepare(t *testing.T) {
 		outputBulker.On("APIKeyCreate",
 			mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
 			Return(&apiKey, nil).Once()
-		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
+		bulker.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(outputBulker, false).Once()
 
 		output := Output{
 			Type: OutputTypeRemoteElasticsearch,
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index 92915cc7f..c51000d5a 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -90,8 +90,8 @@ func (m *MockBulk) GetBulkerMap() map[string]bulk.Bulk {
 	return args.Get(0).(map[string]bulk.Bulk)
 }
 
-func (m *MockBulk) CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, outputName string, serviceToken string, outputMap map[string]map[string]interface{}) (bulk.Bulk, bool, error) {
-	args := m.Called(ctx, zlog, outputName, serviceToken, outputMap)
+func (m *MockBulk) CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, outputName string, outputMap map[string]map[string]interface{}) (bulk.Bulk, bool, error) {
+	args := m.Called(ctx, zlog, outputName, outputMap)
 	return args.Get(0).(bulk.Bulk), args.Get(1).(bool), nil
 }
 

From 462c0633dd410f24f72fe09181cae0b4129c3f4c Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Wed, 22 Nov 2023 16:50:48 +0100
Subject: [PATCH 76/91] unit test on handleAck

---
 internal/pkg/api/handleAck.go      |  7 +++---
 internal/pkg/api/handleAck_test.go | 34 ++++++++++++++++++++++++++++++
 internal/pkg/bulk/engine.go        |  7 +++---
 internal/pkg/testing/bulk.go       |  7 ++++--
 4 files changed, 47 insertions(+), 8 deletions(-)

diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go
index 567ff29b0..f92387ebe 100644
--- a/internal/pkg/api/handleAck.go
+++ b/internal/pkg/api/handleAck.go
@@ -452,7 +452,7 @@ func (ack *AckT) updateAPIKey(ctx context.Context,
 		outputBulk := ack.bulk.GetBulker(outputName)
 		if outputBulk != nil {
 			zlog.Debug().Str("outputName", outputName).Msg("Using output bulker in updateAPIKey")
-			bulk = outputBulk
+			bulk = *outputBulk
 		}
 	}
 	if apiKeyID != "" {
@@ -598,14 +598,15 @@ func (ack *AckT) invalidateAPIKeys(ctx context.Context, zlog zerolog.Logger, toR
 			if err != nil || policy == nil {
 				zlog.Debug().Str("outputName", outputName).Msg("Output policy not found")
 			} else {
-				outputBulk, _, err = ack.bulk.CreateAndGetBulker(ctx, zlog, outputName, policy.Data.Outputs)
+				blk, _, err := ack.bulk.CreateAndGetBulker(ctx, zlog, outputName, policy.Data.Outputs)
+				outputBulk = &blk
 				if err != nil {
 					zlog.Debug().Str("outputName", outputName).Msg("Failed to recreate output bulker")
 				}
 			}
 		}
 		if outputBulk != nil {
-			if err := outputBulk.APIKeyInvalidate(ctx, outputIds...); err != nil {
+			if err := (*outputBulk).APIKeyInvalidate(ctx, outputIds...); err != nil {
 				zlog.Info().Err(err).Strs("ids", outputIds).Str("outputName", outputName).Msg("Failed to invalidate API keys")
 			}
 		}
diff --git a/internal/pkg/api/handleAck_test.go b/internal/pkg/api/handleAck_test.go
index c863973bd..45839a916 100644
--- a/internal/pkg/api/handleAck_test.go
+++ b/internal/pkg/api/handleAck_test.go
@@ -605,6 +605,40 @@ func TestInvalidateAPIKeysRemoteOutput(t *testing.T) {
 	remoteBulker2.AssertExpectations(t)
 }
 
+func TestInvalidateAPIKeysRemoteOutputReadFromPolicies(t *testing.T) {
+	toRetire := []model.ToRetireAPIKeyIdsItems{{
+		ID:     "toRetire1",
+		Output: "remote1",
+	}}
+
+	remoteBulker := ftesting.NewMockBulk()
+	remoteBulker.On("APIKeyInvalidate",
+		context.Background(), []string{"toRetire1"}).
+		Return(nil)
+
+	bulkerFn := func(t *testing.T) *ftesting.MockBulk {
+		m := ftesting.NewMockBulk()
+		m.On("Search", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&es.ResultT{HitsT: es.HitsT{
+			Hits: []es.HitT{{
+				Source: []byte(`{"data":{"outputs":{"remote1":{}}}}`),
+			}},
+		}}, nil).Once()
+
+		m.On("CreateAndGetBulker", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(remoteBulker, false, nil)
+		m.On("GetBulker", "remote1").Return(nil)
+		return m
+	}
+
+	bulker := bulkerFn(t)
+
+	logger := testlog.SetLogger(t)
+	ack := &AckT{bulk: bulker}
+	ack.invalidateAPIKeys(context.Background(), logger, toRetire, "")
+
+	bulker.AssertExpectations(t)
+	remoteBulker.AssertExpectations(t)
+}
+
 func TestAckHandleUpgrade(t *testing.T) {
 	tests := []struct {
 		name   string
diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 6bcc96909..36e22d48b 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -69,7 +69,7 @@ type Bulk interface {
 	Client() *elasticsearch.Client
 
 	CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, outputName string, outputMap map[string]map[string]interface{}) (Bulk, bool, error)
-	GetBulker(outputName string) Bulk
+	GetBulker(outputName string) *Bulk
 	GetBulkerMap() map[string]Bulk
 	CancelFn() context.CancelFunc
 
@@ -123,8 +123,9 @@ func NewBulker(es esapi.Transport, tracer *apm.Tracer, opts ...BulkOpt) *Bulker
 	}
 }
 
-func (b *Bulker) GetBulker(outputName string) Bulk {
-	return b.bulkerMap[outputName]
+func (b *Bulker) GetBulker(outputName string) *Bulk {
+	blk := b.bulkerMap[outputName]
+	return &blk
 }
 
 func (b *Bulker) GetBulkerMap() map[string]Bulk {
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index c51000d5a..0f74205d8 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -80,9 +80,12 @@ func (m *MockBulk) Client() *elasticsearch.Client {
 	return args.Get(0).(*elasticsearch.Client)
 }
 
-func (m *MockBulk) GetBulker(outputName string) bulk.Bulk {
+func (m *MockBulk) GetBulker(outputName string) *bulk.Bulk {
 	args := m.Called(outputName)
-	return args.Get(0).(bulk.Bulk)
+	if args.Get(0) == nil {
+		return nil
+	}
+	return args.Get(0).(*bulk.Bulk)
 }
 
 func (m *MockBulk) GetBulkerMap() map[string]bulk.Bulk {

From 96bdb42b29faf38ea6cae68958487cec75b08cff Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Wed, 22 Nov 2023 17:10:56 +0100
Subject: [PATCH 77/91] added test to query policies

---
 internal/pkg/dl/policies_integration_test.go | 38 ++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/internal/pkg/dl/policies_integration_test.go b/internal/pkg/dl/policies_integration_test.go
index 0b917126b..ceb4fa9ee 100644
--- a/internal/pkg/dl/policies_integration_test.go
+++ b/internal/pkg/dl/policies_integration_test.go
@@ -14,6 +14,7 @@ import (
 
 	"github.com/gofrs/uuid"
 	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/fleet-server/v7/internal/pkg/bulk"
 	"github.com/elastic/fleet-server/v7/internal/pkg/model"
@@ -105,3 +106,40 @@ func TestCreatePolicy(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestQueryOutputFromPolicy(t *testing.T) {
+	ctx, cn := context.WithCancel(context.Background())
+	defer cn()
+	ctx = testlog.SetLogger(t).WithContext(ctx)
+
+	index, bulker := ftesting.SetupCleanIndex(ctx, t, FleetPolicies)
+
+	now := time.Now().UTC()
+	var policyData = model.PolicyData{
+		Outputs: map[string]map[string]interface{}{
+			"remote": {
+				"type": "remote_elasticsearch",
+			},
+		},
+		OutputPermissions: json.RawMessage(`{"default": {}}`),
+		Inputs:            []map[string]interface{}{},
+	}
+	rec := model.Policy{
+		PolicyID:           "policy1",
+		RevisionIdx:        1,
+		CoordinatorIdx:     0,
+		Data:               &policyData,
+		DefaultFleetServer: false,
+		Timestamp:          now.Format(time.RFC3339),
+	}
+	_, err := CreatePolicy(ctx, bulker, rec, WithIndexName(index))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	policy, err := QueryOutputFromPolicy(ctx, bulker, "remote", WithIndexName(index))
+	if err != nil {
+		t.Fatal(err)
+	}
+	require.Equal(t, map[string]interface{}{"type": "remote_elasticsearch"}, policy.Data.Outputs["remote"])
+}

From 053fbedc2b6725d4f84d1129c7beba38863f48b3 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 23 Nov 2023 09:51:49 +0100
Subject: [PATCH 78/91] fix test

---
 internal/pkg/api/handleAck.go | 7 +++----
 internal/pkg/bulk/engine.go   | 7 +++----
 internal/pkg/testing/bulk.go  | 4 ++--
 3 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go
index f92387ebe..567ff29b0 100644
--- a/internal/pkg/api/handleAck.go
+++ b/internal/pkg/api/handleAck.go
@@ -452,7 +452,7 @@ func (ack *AckT) updateAPIKey(ctx context.Context,
 		outputBulk := ack.bulk.GetBulker(outputName)
 		if outputBulk != nil {
 			zlog.Debug().Str("outputName", outputName).Msg("Using output bulker in updateAPIKey")
-			bulk = *outputBulk
+			bulk = outputBulk
 		}
 	}
 	if apiKeyID != "" {
@@ -598,15 +598,14 @@ func (ack *AckT) invalidateAPIKeys(ctx context.Context, zlog zerolog.Logger, toR
 			if err != nil || policy == nil {
 				zlog.Debug().Str("outputName", outputName).Msg("Output policy not found")
 			} else {
-				blk, _, err := ack.bulk.CreateAndGetBulker(ctx, zlog, outputName, policy.Data.Outputs)
-				outputBulk = &blk
+				outputBulk, _, err = ack.bulk.CreateAndGetBulker(ctx, zlog, outputName, policy.Data.Outputs)
 				if err != nil {
 					zlog.Debug().Str("outputName", outputName).Msg("Failed to recreate output bulker")
 				}
 			}
 		}
 		if outputBulk != nil {
-			if err := (*outputBulk).APIKeyInvalidate(ctx, outputIds...); err != nil {
+			if err := outputBulk.APIKeyInvalidate(ctx, outputIds...); err != nil {
 				zlog.Info().Err(err).Strs("ids", outputIds).Str("outputName", outputName).Msg("Failed to invalidate API keys")
 			}
 		}
diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 36e22d48b..6bcc96909 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -69,7 +69,7 @@ type Bulk interface {
 	Client() *elasticsearch.Client
 
 	CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, outputName string, outputMap map[string]map[string]interface{}) (Bulk, bool, error)
-	GetBulker(outputName string) *Bulk
+	GetBulker(outputName string) Bulk
 	GetBulkerMap() map[string]Bulk
 	CancelFn() context.CancelFunc
 
@@ -123,9 +123,8 @@ func NewBulker(es esapi.Transport, tracer *apm.Tracer, opts ...BulkOpt) *Bulker
 	}
 }
 
-func (b *Bulker) GetBulker(outputName string) *Bulk {
-	blk := b.bulkerMap[outputName]
-	return &blk
+func (b *Bulker) GetBulker(outputName string) Bulk {
+	return b.bulkerMap[outputName]
 }
 
 func (b *Bulker) GetBulkerMap() map[string]Bulk {
diff --git a/internal/pkg/testing/bulk.go b/internal/pkg/testing/bulk.go
index 0f74205d8..02cf32b0f 100644
--- a/internal/pkg/testing/bulk.go
+++ b/internal/pkg/testing/bulk.go
@@ -80,12 +80,12 @@ func (m *MockBulk) Client() *elasticsearch.Client {
 	return args.Get(0).(*elasticsearch.Client)
 }
 
-func (m *MockBulk) GetBulker(outputName string) *bulk.Bulk {
+func (m *MockBulk) GetBulker(outputName string) bulk.Bulk {
 	args := m.Called(outputName)
 	if args.Get(0) == nil {
 		return nil
 	}
-	return args.Get(0).(*bulk.Bulk)
+	return args.Get(0).(bulk.Bulk)
 }
 
 func (m *MockBulk) GetBulkerMap() map[string]bulk.Bulk {

From 60cd6ee8c3cf2f121bb37fb83e3515c202d567c7 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 23 Nov 2023 10:49:32 +0100
Subject: [PATCH 79/91] fix integration test

---
 .../remote_es_output_integration_test.go      | 35 +++++++++----------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/internal/pkg/server/remote_es_output_integration_test.go b/internal/pkg/server/remote_es_output_integration_test.go
index 55b204382..4c408f3d8 100644
--- a/internal/pkg/server/remote_es_output_integration_test.go
+++ b/internal/pkg/server/remote_es_output_integration_test.go
@@ -20,7 +20,6 @@ import (
 	"github.com/elastic/fleet-server/v7/internal/pkg/apikey"
 	"github.com/elastic/fleet-server/v7/internal/pkg/dl"
 	"github.com/elastic/fleet-server/v7/internal/pkg/model"
-	ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing"
 	"github.com/hashicorp/go-cleanhttp"
 	"github.com/stretchr/testify/require"
 )
@@ -175,25 +174,25 @@ func Test_Agent_Remote_ES_Output(t *testing.T) {
 	remoteAPIKey := Checkin(t, ctx, srvCopy, agentID, key)
 	apiKeyID := strings.Split(remoteAPIKey, ":")[0]
 
-	ftesting.Retry(t, ctx, func(ctx context.Context) error {
-		requestURL := fmt.Sprintf("http://elastic:changeme@%s/_security/api_key?id=%s", remoteESHost, apiKeyID)
+	// need to wait a bit before querying the api key
+	time.Sleep(time.Second)
 
-		req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil)
-		if err != nil {
-			t.Fatal("error creating request for remote api key")
-		}
-		res, err := http.DefaultClient.Do(req)
-		if err != nil {
-			t.Fatal("error querying remote api key")
-		}
+	requestURL := fmt.Sprintf("http://elastic:changeme@%s/_security/api_key?id=%s", remoteESHost, apiKeyID)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil)
+	if err != nil {
+		t.Fatal("error creating request for remote api key")
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal("error querying remote api key")
+	}
 
-		require.Equal(t, 200, res.StatusCode)
+	require.Equal(t, 200, res.StatusCode)
 
-		defer res.Body.Close()
-		respString, err := io.ReadAll(res.Body)
-		require.NoError(t, err, "did not expect error when parsing api key response")
+	defer res.Body.Close()
+	respString, err := io.ReadAll(res.Body)
+	require.NoError(t, err, "did not expect error when parsing api key response")
 
-		require.Contains(t, string(respString), "\"invalidated\":false")
-		return nil
-	}, ftesting.RetrySleep(1*time.Second))
+	require.Contains(t, string(respString), "\"invalidated\":false")
 }

From 760ea2f85671a7e9963fbee1659d0ce7ebca4d28 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 23 Nov 2023 13:09:06 +0100
Subject: [PATCH 80/91] added integration test for invalidate api key

---
 .../remote_es_output_integration_test.go      | 120 +++++++++++++++---
 1 file changed, 100 insertions(+), 20 deletions(-)

diff --git a/internal/pkg/server/remote_es_output_integration_test.go b/internal/pkg/server/remote_es_output_integration_test.go
index 4c408f3d8..9b78eb293 100644
--- a/internal/pkg/server/remote_es_output_integration_test.go
+++ b/internal/pkg/server/remote_es_output_integration_test.go
@@ -20,17 +20,17 @@ import (
 	"github.com/elastic/fleet-server/v7/internal/pkg/apikey"
 	"github.com/elastic/fleet-server/v7/internal/pkg/dl"
 	"github.com/elastic/fleet-server/v7/internal/pkg/model"
+	"github.com/gofrs/uuid"
 	"github.com/hashicorp/go-cleanhttp"
 	"github.com/stretchr/testify/require"
 )
 
-func Checkin(t *testing.T, ctx context.Context, srv *tserver, agentID, key string) string {
-	str := agentID
+func Checkin(t *testing.T, ctx context.Context, srv *tserver, agentID, key string, shouldHaveRemoveES bool) (string, string) {
 	cli := cleanhttp.DefaultClient()
 	var obj map[string]interface{}
 
-	t.Logf("Fake a checkin for agent %s", str)
-	req, err := http.NewRequestWithContext(ctx, "POST", srv.baseURL()+"/api/fleet/agents/"+str+"/checkin", strings.NewReader(checkinBody))
+	t.Logf("Fake a checkin for agent %s", agentID)
+	req, err := http.NewRequestWithContext(ctx, "POST", srv.baseURL()+"/api/fleet/agents/"+agentID+"/checkin", strings.NewReader(checkinBody))
 	require.NoError(t, err)
 	req.Header.Set("Authorization", "ApiKey "+key)
 	req.Header.Set("User-Agent", "elastic agent "+serverVersion)
@@ -49,9 +49,15 @@ func Checkin(t *testing.T, ctx context.Context, srv *tserver, agentID, key strin
 	require.True(t, ok, "expected actions is missing")
 	actions, ok := actionsRaw.([]interface{})
 	require.True(t, ok, "expected actions to be an array")
-	require.Greater(t, len(actions), 0, "expected at least 1 action")
+	require.Equal(t, len(actions), 1, "expected 1 action")
 	action, ok := actions[0].(map[string]interface{})
 	require.True(t, ok, "expected action to be an object")
+
+	aIDRaw, ok := action["id"]
+	require.True(t, ok, "expected action id attribute missing")
+	actionID, ok := aIDRaw.(string)
+	require.True(t, ok, "expected action id to be string")
+
 	typeRaw := action["type"]
 	require.Equal(t, "POLICY_CHANGE", typeRaw)
 	dataRaw := action["data"]
@@ -61,21 +67,57 @@ func Checkin(t *testing.T, ctx context.Context, srv *tserver, agentID, key strin
 	require.True(t, ok, "expected policy to be map")
 	outputs, ok := policy["outputs"].(map[string]interface{})
 	require.True(t, ok, "expected outputs to be map")
-	remoteES, ok := outputs["remoteES"].(map[string]interface{})
-	require.True(t, ok, "expected remoteES to be map")
-	oType, ok := remoteES["type"].(string)
-	require.True(t, ok, "expected type to be string")
-	require.Equal(t, "elasticsearch", oType)
-	serviceToken := remoteES["service_token"]
-	require.Equal(t, nil, serviceToken)
-	remoteAPIKey, ok := remoteES["api_key"].(string)
-	require.True(t, ok, "expected remoteAPIKey to be string")
+	var remoteAPIKey string
+	if shouldHaveRemoveES {
+		remoteES, ok := outputs["remoteES"].(map[string]interface{})
+		require.True(t, ok, "expected remoteES to be map")
+		oType, ok := remoteES["type"].(string)
+		require.True(t, ok, "expected type to be string")
+		require.Equal(t, "elasticsearch", oType)
+		serviceToken := remoteES["service_token"]
+		require.Equal(t, nil, serviceToken)
+		remoteAPIKey, ok = remoteES["api_key"].(string)
+		require.True(t, ok, "expected remoteAPIKey to be string")
+	}
 	defaultOutput, ok := outputs["default"].(map[string]interface{})
 	require.True(t, ok, "expected default to be map")
 	defaultAPIKey, ok := defaultOutput["api_key"].(string)
 	require.True(t, ok, "expected defaultAPIKey to be string")
 	require.NotEqual(t, remoteAPIKey, defaultAPIKey, "expected remote api key to be different than default")
-	return remoteAPIKey
+	return remoteAPIKey, actionID
+}
+
+func Ack(t *testing.T, ctx context.Context, srv *tserver, actionID, agentID, key string) {
+	t.Logf("Fake an ack for action %s for agent %s", actionID, agentID)
+	body := fmt.Sprintf(`{
+	    "events": [{
+		"action_id": "%s",
+		"agent_id": "%s",
+		"message": "test-message",
+		"type": "ACTION_RESULT",
+		"subtype": "ACKNOWLEDGED"
+	    }]
+	}`, actionID, agentID)
+	req, err := http.NewRequestWithContext(ctx, "POST", srv.baseURL()+"/api/fleet/agents/"+agentID+"/acks", strings.NewReader(body))
+	require.NoError(t, err)
+	req.Header.Set("Authorization", "ApiKey "+key)
+	req.Header.Set("Content-Type", "application/json")
+	cli := cleanhttp.DefaultClient()
+	res, err := cli.Do(req)
+	require.NoError(t, err)
+
+	require.Equal(t, http.StatusOK, res.StatusCode)
+	t.Log("Ack successful, verify body")
+	p, _ := io.ReadAll(res.Body)
+	res.Body.Close()
+	var ackObj map[string]interface{}
+	err = json.Unmarshal(p, &ackObj)
+	require.NoError(t, err)
+
+	// NOTE the checkin response will only have the errors attribute if it's set to true in the response.
+	// When decoding to a (typed) struct, the default will implicitly be false if it's missing
+	_, ok := ackObj["errors"]
+	require.Falsef(t, ok, "expected response to have no errors attribute, errors are present: %+v", ackObj)
 }
 
 func Test_Agent_Remote_ES_Output(t *testing.T) {
@@ -98,7 +140,7 @@ func Test_Agent_Remote_ES_Output(t *testing.T) {
 
 	t.Log("Create policy with remote ES output")
 
-	var policyRemoteID = "policyRemoteID"
+	var policyRemoteID = uuid.Must(uuid.NewV4()).String()
 	remoteESHost := "localhost:9201"
 	var policyDataRemoteES = model.PolicyData{
 		Outputs: map[string]map[string]interface{}{
@@ -113,6 +155,7 @@ func Test_Agent_Remote_ES_Output(t *testing.T) {
 		},
 		OutputPermissions: json.RawMessage(`{"default": {}, "remoteES": {}}`),
 		Inputs:            []map[string]interface{}{},
+		Agent:             json.RawMessage(`{"monitoring": {"use_output":"remoteES"}}`),
 	}
 
 	_, err = dl.CreatePolicy(ctx, srv.bulker, model.Policy{
@@ -165,15 +208,52 @@ func Test_Agent_Remote_ES_Output(t *testing.T) {
 
 	// cleanup
 	defer func() {
-		err2 := srv.bulker.Delete(ctx, dl.FleetAgents, agentID)
-		if err2 != nil {
+		err = srv.bulker.Delete(ctx, dl.FleetAgents, agentID)
+		if err != nil {
 			t.Log("could not clean up agent")
 		}
 	}()
 
-	remoteAPIKey := Checkin(t, ctx, srvCopy, agentID, key)
+	remoteAPIKey, actionID := Checkin(t, ctx, srvCopy, agentID, key, true)
 	apiKeyID := strings.Split(remoteAPIKey, ":")[0]
 
+	verifyRemoteAPIKey(t, ctx, remoteESHost, apiKeyID, false)
+
+	Ack(t, ctx, srvCopy, actionID, agentID, key)
+
+	t.Log("Update policy to remove remote ES output")
+
+	var policyData = model.PolicyData{
+		Outputs: map[string]map[string]interface{}{
+			"default": {
+				"type": "elasticsearch",
+			},
+		},
+		OutputPermissions: json.RawMessage(`{"default": {}}`),
+		Inputs:            []map[string]interface{}{},
+	}
+
+	_, err = dl.CreatePolicy(ctx, srv.bulker, model.Policy{
+		PolicyID:           policyRemoteID,
+		RevisionIdx:        2,
+		DefaultFleetServer: false,
+		Data:               &policyData,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Log("Checkin so that agent gets new policy revision")
+	_, actionID = Checkin(t, ctx, srvCopy, agentID, key, false)
+
+	t.Log("Ack so that fleet triggers remote api key invalidate")
+	Ack(t, ctx, srvCopy, actionID, agentID, key)
+
+	verifyRemoteAPIKey(t, ctx, remoteESHost, apiKeyID, true)
+
+}
+
+func verifyRemoteAPIKey(t *testing.T, ctx context.Context, remoteESHost, apiKeyID string, invalidated bool) {
 	// need to wait a bit before querying the api key
 	time.Sleep(time.Second)
 
@@ -194,5 +274,5 @@ func Test_Agent_Remote_ES_Output(t *testing.T) {
 	respString, err := io.ReadAll(res.Body)
 	require.NoError(t, err, "did not expect error when parsing api key response")
 
-	require.Contains(t, string(respString), "\"invalidated\":false")
+	require.Contains(t, string(respString), fmt.Sprintf("\"invalidated\":%t", invalidated))
 }

From 9382975df36401c4b859516a363de746a52e344f Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Thu, 23 Nov 2023 13:51:23 +0100
Subject: [PATCH 81/91] test for child bulker cancel

---
 internal/pkg/bulk/bulk_test.go | 85 ++++++++++++++++++++++++++++++++++
 1 file changed, 85 insertions(+)

diff --git a/internal/pkg/bulk/bulk_test.go b/internal/pkg/bulk/bulk_test.go
index 7c493f3d6..851f78f43 100644
--- a/internal/pkg/bulk/bulk_test.go
+++ b/internal/pkg/bulk/bulk_test.go
@@ -16,6 +16,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/elastic/fleet-server/v7/internal/pkg/apikey"
 	testlog "github.com/elastic/fleet-server/v7/internal/pkg/testing/log"
 )
 
@@ -272,6 +273,90 @@ func TestCancelCtx(t *testing.T) {
 	}
 }
 
+// verify that child bulker stops when bulker ctx cancelled
+func TestCancelCtxChildBulker(t *testing.T) {
+	bulker := NewBulker(nil, nil)
+
+	ctx, cancelF := context.WithCancel(context.Background())
+	ctx = testlog.SetLogger(t).WithContext(ctx)
+
+	logger := testlog.SetLogger(t)
+	outputMap := make(map[string]map[string]interface{})
+	outputMap["remote"] = map[string]interface{}{
+		"type":          "remote_elasticsearch",
+		"hosts":         []interface{}{"https://remote-es:443"},
+		"service_token": "token1",
+	}
+	childBulker, _, err := bulker.CreateAndGetBulker(ctx, logger, "remote", outputMap)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		_, err := childBulker.APIKeyAuth(ctx, apikey.APIKey{})
+
+		if !errors.Is(err, context.Canceled) {
+			t.Error("Expected context cancel err: ", err)
+		}
+	}()
+
+	time.Sleep(time.Millisecond)
+	cancelF()
+
+	wg.Wait()
+}
+
+// verify that old bulker ctx is canceled if cfg changed and a new one is started
+func TestCancelCtxChildBulkerReplaced(t *testing.T) {
+	bulker := NewBulker(nil, nil)
+
+	ctx, _ := context.WithCancel(context.Background())
+	ctx = testlog.SetLogger(t).WithContext(ctx)
+
+	logger := testlog.SetLogger(t)
+	outputMap := make(map[string]map[string]interface{})
+	outputMap["remote"] = map[string]interface{}{
+		"type":          "remote_elasticsearch",
+		"hosts":         []interface{}{"https://remote-es:443"},
+		"service_token": "token1",
+	}
+	childBulker, _, err := bulker.CreateAndGetBulker(ctx, logger, "remote", outputMap)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// output cfg changed
+	outputMap["remote"] = map[string]interface{}{
+		"type":          "remote_elasticsearch",
+		"hosts":         []interface{}{"https://remote-es:443"},
+		"service_token": "token2",
+	}
+	_, _, err = bulker.CreateAndGetBulker(ctx, logger, "remote", outputMap)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		_, err := childBulker.APIKeyAuth(ctx, apikey.APIKey{})
+
+		t.Log(err)
+		// TODO not context canceled error
+		// if !errors.Is(err, context.Canceled) {
+		// 	t.Error("Expected context cancel err: ", err)
+		// }
+	}()
+
+	wg.Wait()
+}
+
 func benchmarkMockBulk(b *testing.B, samples [][]byte) {
 	b.ReportAllocs()
 	mock := &mockBulkTransport{}

From 0006533d0d30fc11023e08094d8c1fa026aeb761 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 27 Nov 2023 10:09:40 +0100
Subject: [PATCH 82/91] fixed test, replace semaphore with mutex

---
 internal/pkg/bulk/bulk_test.go | 12 ++++++------
 internal/pkg/bulk/engine.go    |  9 +++------
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/internal/pkg/bulk/bulk_test.go b/internal/pkg/bulk/bulk_test.go
index 851f78f43..82dc6b7fc 100644
--- a/internal/pkg/bulk/bulk_test.go
+++ b/internal/pkg/bulk/bulk_test.go
@@ -314,7 +314,8 @@ func TestCancelCtxChildBulker(t *testing.T) {
 func TestCancelCtxChildBulkerReplaced(t *testing.T) {
 	bulker := NewBulker(nil, nil)
 
-	ctx, _ := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 	ctx = testlog.SetLogger(t).WithContext(ctx)
 
 	logger := testlog.SetLogger(t)
@@ -345,13 +346,12 @@ func TestCancelCtxChildBulkerReplaced(t *testing.T) {
 	go func() {
 		defer wg.Done()
 
-		_, err := childBulker.APIKeyAuth(ctx, apikey.APIKey{})
+		err := childBulker.APIKeyUpdate(ctx, "", "", make([]byte, 0))
 
 		t.Log(err)
-		// TODO not context canceled error
-		// if !errors.Is(err, context.Canceled) {
-		// 	t.Error("Expected context cancel err: ", err)
-		// }
+		if !errors.Is(err, context.Canceled) {
+			t.Error("Expected context cancel err: ", err)
+		}
 	}()
 
 	wg.Wait()
diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 6bcc96909..ad738421e 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -88,7 +88,7 @@ type Bulker struct {
 	remoteOutputConfigMap map[string]map[string]interface{}
 	bulkerMap             map[string]Bulk
 	cancelFn              context.CancelFunc
-	remoteOutputLimit     *semaphore.Weighted
+	remoteOutputMutex   sync.RWMutex
 }
 
 const (
@@ -119,7 +119,6 @@ func NewBulker(es esapi.Transport, tracer *apm.Tracer, opts ...BulkOpt) *Bulker
 		remoteOutputConfigMap: make(map[string]map[string]interface{}),
 		// remote ES bulkers
 		bulkerMap:         make(map[string]Bulk),
-		remoteOutputLimit: semaphore.NewWeighted(1),
 	}
 }
 
@@ -137,10 +136,8 @@ func (b *Bulker) CancelFn() context.CancelFunc {
 
 func (b *Bulker) updateBulkerMap(ctx context.Context, outputName string, newBulker *Bulker) error {
 	// concurrency control of updating map
-	if err := b.remoteOutputLimit.Acquire(ctx, 1); err != nil {
-		return err
-	}
-	defer b.remoteOutputLimit.Release(1)
+	b.remoteOutputMutex.Lock()
+	defer b.remoteOutputMutex.Unlock()
 
 	b.bulkerMap[outputName] = newBulker
 	return nil

From 41219f01762ee1beca92d9c0947799a1bf0d090e Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 27 Nov 2023 10:41:38 +0100
Subject: [PATCH 83/91] removed unused arg

---
 internal/pkg/bulk/engine.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index ad738421e..1e52e49cc 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -134,7 +134,7 @@ func (b *Bulker) CancelFn() context.CancelFunc {
 	return b.cancelFn
 }
 
-func (b *Bulker) updateBulkerMap(ctx context.Context, outputName string, newBulker *Bulker) error {
+func (b *Bulker) updateBulkerMap(outputName string, newBulker *Bulker) error {
 	// concurrency control of updating map
 	b.remoteOutputMutex.Lock()
 	defer b.remoteOutputMutex.Unlock()
@@ -169,7 +169,7 @@ func (b *Bulker) CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, ou
 	newBulker := NewBulker(es, b.tracer)
 	newBulker.cancelFn = bulkCancel
 
-	err = b.updateBulkerMap(ctx, outputName, newBulker)
+	err = b.updateBulkerMap(outputName, newBulker)
 	if err != nil {
 		return nil, hasConfigChanged, err
 	}

From 75c890b5c14b568193b7ecc5574e2d6281de6efa Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 27 Nov 2023 10:54:56 +0100
Subject: [PATCH 84/91] added warning log if api keys orphaned

---
 internal/pkg/api/handleAck.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go
index 567ff29b0..489aefba5 100644
--- a/internal/pkg/api/handleAck.go
+++ b/internal/pkg/api/handleAck.go
@@ -596,11 +596,11 @@ func (ack *AckT) invalidateAPIKeys(ctx context.Context, zlog zerolog.Logger, toR
 			// read output config from .fleet-policies, not filtering by policy id as agent could be reassigned
 			policy, err := dl.QueryOutputFromPolicy(ctx, ack.bulk, outputName)
 			if err != nil || policy == nil {
-				zlog.Debug().Str("outputName", outputName).Msg("Output policy not found")
+				zlog.Warn().Str("outputName", outputName).Any("ids", outputIds).Msg("Output policy not found, API keys will be orphaned")
 			} else {
 				outputBulk, _, err = ack.bulk.CreateAndGetBulker(ctx, zlog, outputName, policy.Data.Outputs)
 				if err != nil {
-					zlog.Debug().Str("outputName", outputName).Msg("Failed to recreate output bulker")
+					zlog.Warn().Str("outputName", outputName).Any("ids", outputIds).Msg("Failed to recreate output bulker, API keys will be orphaned")
 				}
 			}
 		}

From 4d2d67d1c00bee834b0fa30c39e28bdefabe0512 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 27 Nov 2023 11:02:10 +0100
Subject: [PATCH 85/91] removed unused error

---
 internal/pkg/bulk/engine.go | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 1e52e49cc..1e93e7a7e 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -134,13 +134,12 @@ func (b *Bulker) CancelFn() context.CancelFunc {
 	return b.cancelFn
 }
 
-func (b *Bulker) updateBulkerMap(outputName string, newBulker *Bulker) error {
+func (b *Bulker) updateBulkerMap(outputName string, newBulker *Bulker) {
 	// concurrency control of updating map
 	b.remoteOutputMutex.Lock()
 	defer b.remoteOutputMutex.Unlock()
 
 	b.bulkerMap[outputName] = newBulker
-	return nil
 }
 
 // for remote ES output, create a new bulker in bulkerMap if does not exist
@@ -169,10 +168,7 @@ func (b *Bulker) CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, ou
 	newBulker := NewBulker(es, b.tracer)
 	newBulker.cancelFn = bulkCancel
 
-	err = b.updateBulkerMap(outputName, newBulker)
-	if err != nil {
-		return nil, hasConfigChanged, err
-	}
+	b.updateBulkerMap(outputName, newBulker)
 
 	errCh := make(chan error)
 	go func() {

From cd877f1a18e984f1e0d2c4ac245369cfa55a95f4 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 27 Nov 2023 11:35:37 +0100
Subject: [PATCH 86/91] fix lint

---
 internal/pkg/bulk/engine.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 1e93e7a7e..3b0d3f464 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -88,7 +88,7 @@ type Bulker struct {
 	remoteOutputConfigMap map[string]map[string]interface{}
 	bulkerMap             map[string]Bulk
 	cancelFn              context.CancelFunc
-	remoteOutputMutex   sync.RWMutex
+	remoteOutputMutex     sync.RWMutex
 }
 
 const (
@@ -118,7 +118,7 @@ func NewBulker(es esapi.Transport, tracer *apm.Tracer, opts ...BulkOpt) *Bulker
 		tracer:                tracer,
 		remoteOutputConfigMap: make(map[string]map[string]interface{}),
 		// remote ES bulkers
-		bulkerMap:         make(map[string]Bulk),
+		bulkerMap: make(map[string]Bulk),
 	}
 }
 

From d1ffe3fc2f4a62ab13135f1eb63d5da3a36dfdbc Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 27 Nov 2023 13:49:01 +0100
Subject: [PATCH 87/91] try to fix test

---
 internal/pkg/bulk/bulk_test.go | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/internal/pkg/bulk/bulk_test.go b/internal/pkg/bulk/bulk_test.go
index 82dc6b7fc..4d7f85d6d 100644
--- a/internal/pkg/bulk/bulk_test.go
+++ b/internal/pkg/bulk/bulk_test.go
@@ -330,6 +330,15 @@ func TestCancelCtxChildBulkerReplaced(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	var waitBulker sync.WaitGroup
+	waitBulker.Add(1)
+	go func() {
+		defer waitBulker.Done()
+		if err := (childBulker.(*Bulker)).Run(ctx); !errors.Is(err, context.Canceled) {
+			t.Fatal(err)
+		}
+	}()
+
 	// output cfg changed
 	outputMap["remote"] = map[string]interface{}{
 		"type":          "remote_elasticsearch",
@@ -349,12 +358,16 @@ func TestCancelCtxChildBulkerReplaced(t *testing.T) {
 		err := childBulker.APIKeyUpdate(ctx, "", "", make([]byte, 0))
 
 		t.Log(err)
+		// TODO dial tcp: lookup remote-es: no such host
 		if !errors.Is(err, context.Canceled) {
-			t.Error("Expected context cancel err: ", err)
+			t.Fatal("Expected context cancel err: ", err)
 		}
+		ctx.Done()
 	}()
 
 	wg.Wait()
+	cancel()
+	waitBulker.Wait()
 }
 
 func benchmarkMockBulk(b *testing.B, samples [][]byte) {

From 58850689e8621cb4165b233952c78ac24401026e Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 27 Nov 2023 14:43:36 +0100
Subject: [PATCH 88/91] read output secret before prepare remote es

---
 internal/pkg/api/handleCheckin.go         | 7 +++++++
 internal/pkg/policy/parsed_policy.go      | 9 +--------
 internal/pkg/policy/parsed_policy_test.go | 1 -
 internal/pkg/policy/secret.go             | 2 +-
 internal/pkg/policy/secret_test.go        | 2 +-
 5 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/internal/pkg/api/handleCheckin.go b/internal/pkg/api/handleCheckin.go
index bef7f1de7..8d5f6645f 100644
--- a/internal/pkg/api/handleCheckin.go
+++ b/internal/pkg/api/handleCheckin.go
@@ -748,6 +748,13 @@ func processPolicy(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, a
 	}
 
 	data := model.ClonePolicyData(pp.Policy.Data)
+	for policyName, policyOutput := range data.Outputs {
+		err := policy.ProcessOutputSecret(ctx, policyOutput, bulker)
+		if err != nil {
+			return nil, fmt.Errorf("failed to process output secrets %q: %w",
+			policyName, err)
+		}
+	}
 	// Iterate through the policy outputs and prepare them
 	for _, policyOutput := range pp.Outputs {
 		err = policyOutput.Prepare(ctx, zlog, bulker, &agent, data.Outputs)
diff --git a/internal/pkg/policy/parsed_policy.go b/internal/pkg/policy/parsed_policy.go
index 6a56d67e1..1ffdb115c 100644
--- a/internal/pkg/policy/parsed_policy.go
+++ b/internal/pkg/policy/parsed_policy.go
@@ -65,7 +65,7 @@ func NewParsedPolicy(ctx context.Context, bulker bulk.Bulk, p model.Policy) (*Pa
 		return nil, err
 	}
 	for _, policyOutput := range p.Data.Outputs {
-		err := processOutputSecret(ctx, policyOutput, bulker)
+		err := ProcessOutputSecret(ctx, policyOutput, bulker)
 		if err != nil {
 			return nil, err
 		}
@@ -118,13 +118,6 @@ func constructPolicyOutputs(outputs map[string]map[string]interface{}, roles map
 			p.Role = &role
 		}
 
-		if p.Type == OutputTypeRemoteElasticsearch {
-			serviceTokenStr, ok := v[FieldOutputServiceToken].(string)
-			if !ok {
-				return nil, fmt.Errorf("missing or invalid service token: %+v", v)
-			}
-			p.ServiceToken = serviceTokenStr
-		}
 
 		result[k] = p
 	}
diff --git a/internal/pkg/policy/parsed_policy_test.go b/internal/pkg/policy/parsed_policy_test.go
index a135170b2..7fcfd7900 100644
--- a/internal/pkg/policy/parsed_policy_test.go
+++ b/internal/pkg/policy/parsed_policy_test.go
@@ -88,5 +88,4 @@ func TestNewParsedPolicyRemoteES(t *testing.T) {
 
 	// Validate that default was found
 	require.Equal(t, "remote", pp.Default.Name)
-	require.Equal(t, "token1", pp.Outputs["remote"].ServiceToken)
 }
diff --git a/internal/pkg/policy/secret.go b/internal/pkg/policy/secret.go
index d164536ff..fccd31ff0 100644
--- a/internal/pkg/policy/secret.go
+++ b/internal/pkg/policy/secret.go
@@ -145,7 +145,7 @@ func setSecretPath(output smap.Map, secretValue string, secretPaths []string) er
 }
 
 // Read secret from output and mutate output with secret value
-func processOutputSecret(ctx context.Context, output smap.Map, bulker bulk.Bulk) error {
+func ProcessOutputSecret(ctx context.Context, output smap.Map, bulker bulk.Bulk) error {
 	secrets := output.GetMap(FieldOutputSecrets)
 
 	delete(output, FieldOutputSecrets)
diff --git a/internal/pkg/policy/secret_test.go b/internal/pkg/policy/secret_test.go
index d1937ee06..9222795d2 100644
--- a/internal/pkg/policy/secret_test.go
+++ b/internal/pkg/policy/secret_test.go
@@ -236,7 +236,7 @@ func TestProcessOutputSecret(t *testing.T) {
 			expectOutput, err := smap.Parse([]byte(tc.expectOutputJSON))
 			assert.NoError(t, err)
 
-			err = processOutputSecret(context.Background(), output, bulker)
+			err = ProcessOutputSecret(context.Background(), output, bulker)
 			assert.NoError(t, err)
 
 			assert.Equal(t, expectOutput, output)

From 4d5b906910dd44dca639f4f68988d5e46fb71103 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Mon, 27 Nov 2023 15:39:18 +0100
Subject: [PATCH 89/91] try to fix test

---
 internal/pkg/api/handleCheckin.go    |  2 +-
 internal/pkg/bulk/bulk_test.go       | 18 ++++++++++++++----
 internal/pkg/bulk/engine.go          |  4 +++-
 internal/pkg/policy/parsed_policy.go |  1 -
 4 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/internal/pkg/api/handleCheckin.go b/internal/pkg/api/handleCheckin.go
index 8d5f6645f..8512a637c 100644
--- a/internal/pkg/api/handleCheckin.go
+++ b/internal/pkg/api/handleCheckin.go
@@ -752,7 +752,7 @@ func processPolicy(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, a
 		err := policy.ProcessOutputSecret(ctx, policyOutput, bulker)
 		if err != nil {
 			return nil, fmt.Errorf("failed to process output secrets %q: %w",
-			policyName, err)
+				policyName, err)
 		}
 	}
 	// Iterate through the policy outputs and prepare them
diff --git a/internal/pkg/bulk/bulk_test.go b/internal/pkg/bulk/bulk_test.go
index 4d7f85d6d..36fd2c9e6 100644
--- a/internal/pkg/bulk/bulk_test.go
+++ b/internal/pkg/bulk/bulk_test.go
@@ -17,7 +17,11 @@ import (
 	"time"
 
 	"github.com/elastic/fleet-server/v7/internal/pkg/apikey"
+	"github.com/elastic/fleet-server/v7/internal/pkg/config"
+	"github.com/elastic/fleet-server/v7/internal/pkg/es"
+	"github.com/elastic/fleet-server/v7/internal/pkg/testing/esutil"
 	testlog "github.com/elastic/fleet-server/v7/internal/pkg/testing/log"
+	"github.com/elastic/go-elasticsearch/v8"
 )
 
 // TODO:
@@ -325,6 +329,14 @@ func TestCancelCtxChildBulkerReplaced(t *testing.T) {
 		"hosts":         []interface{}{"https://remote-es:443"},
 		"service_token": "token1",
 	}
+	old := newESClient
+	defer func() { newESClient = old }()
+
+	newESClient = func(ctx context.Context, cfg *config.Config, longPoll bool, opts ...es.ConfigOption) (*elasticsearch.Client, error) {
+		mockClient, _ := esutil.MockESClient(t)
+		return mockClient, nil
+	}
+
 	childBulker, _, err := bulker.CreateAndGetBulker(ctx, logger, "remote", outputMap)
 	if err != nil {
 		t.Fatal(err)
@@ -335,7 +347,7 @@ func TestCancelCtxChildBulkerReplaced(t *testing.T) {
 	go func() {
 		defer waitBulker.Done()
 		if err := (childBulker.(*Bulker)).Run(ctx); !errors.Is(err, context.Canceled) {
-			t.Fatal(err)
+			t.Error(err)
 		}
 	}()
 
@@ -357,10 +369,8 @@ func TestCancelCtxChildBulkerReplaced(t *testing.T) {
 
 		err := childBulker.APIKeyUpdate(ctx, "", "", make([]byte, 0))
 
-		t.Log(err)
-		// TODO dial tcp: lookup remote-es: no such host
 		if !errors.Is(err, context.Canceled) {
-			t.Fatal("Expected context cancel err: ", err)
+			t.Error("Expected context cancel err: ", err)
 		}
 		ctx.Done()
 	}()
diff --git a/internal/pkg/bulk/engine.go b/internal/pkg/bulk/engine.go
index 3b0d3f464..b16d41ae2 100644
--- a/internal/pkg/bulk/engine.go
+++ b/internal/pkg/bulk/engine.go
@@ -192,6 +192,8 @@ func (b *Bulker) CreateAndGetBulker(ctx context.Context, zlog zerolog.Logger, ou
 	return newBulker, hasConfigChanged, nil
 }
 
+var newESClient = es.NewClient
+
 func (b *Bulker) createRemoteEsClient(ctx context.Context, outputName string, outputMap map[string]map[string]interface{}) (*elasticsearch.Client, error) {
 	hostsObj := outputMap[outputName]["hosts"]
 	hosts, ok := hostsObj.([]interface{})
@@ -218,7 +220,7 @@ func (b *Bulker) createRemoteEsClient(ctx context.Context, outputName string, ou
 			},
 		},
 	}
-	es, err := es.NewClient(ctx, &cfg, false, elasticsearchOptions(
+	es, err := newESClient(ctx, &cfg, false, elasticsearchOptions(
 		true, b.opts.bi,
 	)...)
 	if err != nil {
diff --git a/internal/pkg/policy/parsed_policy.go b/internal/pkg/policy/parsed_policy.go
index 1ffdb115c..8d0948079 100644
--- a/internal/pkg/policy/parsed_policy.go
+++ b/internal/pkg/policy/parsed_policy.go
@@ -118,7 +118,6 @@ func constructPolicyOutputs(outputs map[string]map[string]interface{}, roles map
 			p.Role = &role
 		}
 
-
 		result[k] = p
 	}
 

From d39141a3c08b1026b7087e95af2bedcf6d81ce6b Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 28 Nov 2023 09:25:53 +0100
Subject: [PATCH 90/91] removed test

---
 internal/pkg/bulk/bulk_test.go | 66 ----------------------------------
 1 file changed, 66 deletions(-)

diff --git a/internal/pkg/bulk/bulk_test.go b/internal/pkg/bulk/bulk_test.go
index 36fd2c9e6..d8f6c936a 100644
--- a/internal/pkg/bulk/bulk_test.go
+++ b/internal/pkg/bulk/bulk_test.go
@@ -314,72 +314,6 @@ func TestCancelCtxChildBulker(t *testing.T) {
 	wg.Wait()
 }
 
-// verify that old bulker ctx is canceled if cfg changed and a new one is started
-func TestCancelCtxChildBulkerReplaced(t *testing.T) {
-	bulker := NewBulker(nil, nil)
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	ctx = testlog.SetLogger(t).WithContext(ctx)
-
-	logger := testlog.SetLogger(t)
-	outputMap := make(map[string]map[string]interface{})
-	outputMap["remote"] = map[string]interface{}{
-		"type":          "remote_elasticsearch",
-		"hosts":         []interface{}{"https://remote-es:443"},
-		"service_token": "token1",
-	}
-	old := newESClient
-	defer func() { newESClient = old }()
-
-	newESClient = func(ctx context.Context, cfg *config.Config, longPoll bool, opts ...es.ConfigOption) (*elasticsearch.Client, error) {
-		mockClient, _ := esutil.MockESClient(t)
-		return mockClient, nil
-	}
-
-	childBulker, _, err := bulker.CreateAndGetBulker(ctx, logger, "remote", outputMap)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var waitBulker sync.WaitGroup
-	waitBulker.Add(1)
-	go func() {
-		defer waitBulker.Done()
-		if err := (childBulker.(*Bulker)).Run(ctx); !errors.Is(err, context.Canceled) {
-			t.Error(err)
-		}
-	}()
-
-	// output cfg changed
-	outputMap["remote"] = map[string]interface{}{
-		"type":          "remote_elasticsearch",
-		"hosts":         []interface{}{"https://remote-es:443"},
-		"service_token": "token2",
-	}
-	_, _, err = bulker.CreateAndGetBulker(ctx, logger, "remote", outputMap)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-
-		err := childBulker.APIKeyUpdate(ctx, "", "", make([]byte, 0))
-
-		if !errors.Is(err, context.Canceled) {
-			t.Error("Expected context cancel err: ", err)
-		}
-		ctx.Done()
-	}()
-
-	wg.Wait()
-	cancel()
-	waitBulker.Wait()
-}
-
 func benchmarkMockBulk(b *testing.B, samples [][]byte) {
 	b.ReportAllocs()
 	mock := &mockBulkTransport{}

From 299cdaeba8ec62890e1a01280e794407458dbe99 Mon Sep 17 00:00:00 2001
From: Julia Bardi <julia.bardi@elastic.co>
Date: Tue, 28 Nov 2023 10:10:35 +0100
Subject: [PATCH 91/91] removed unused imports

---
 internal/pkg/bulk/bulk_test.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/internal/pkg/bulk/bulk_test.go b/internal/pkg/bulk/bulk_test.go
index d8f6c936a..90a60790b 100644
--- a/internal/pkg/bulk/bulk_test.go
+++ b/internal/pkg/bulk/bulk_test.go
@@ -17,11 +17,7 @@ import (
 	"time"
 
 	"github.com/elastic/fleet-server/v7/internal/pkg/apikey"
-	"github.com/elastic/fleet-server/v7/internal/pkg/config"
-	"github.com/elastic/fleet-server/v7/internal/pkg/es"
-	"github.com/elastic/fleet-server/v7/internal/pkg/testing/esutil"
 	testlog "github.com/elastic/fleet-server/v7/internal/pkg/testing/log"
-	"github.com/elastic/go-elasticsearch/v8"
 )
 
 // TODO: