Truncate DiscoveryConfig.Status ErrorMessage (#52347) (#52844)

Some fetchers might generate a lot of errors which are aggregated and
sent to the `DiscoveryConfig.Status.ErrorMessage`.

This might cause gRPC clients to fail when listing DiscoveryConfigs with
a `grpc: received message larger than max (xyz vs. 4194304)` error.

Allowing 100KB for the error message should give the user enough context
on what's causing the errors.

Author: marcoandredinis

api/defaults/defaults.go

@@ -148,6 +148,12 @@ const (
 	DefaultChunkSize = 1000
 )
 
+const (
+	// DefaultMaxErrorMessageSize is the default maximum size of an error message.
+	// This can be used to truncate large error messages, which might cause gRPC messages to exceed the maximum allowed size.
+	DefaultMaxErrorMessageSize = 1024 * 100 // 100KB
+)
+
 const (
 	// When running in "SSH Proxy" role this port will be used for incoming
 	// connections from SSH nodes who wish to use "reverse tunnell" (when they

lib/srv/discovery/access_graph.go

@@ -33,6 +33,7 @@ import (
 	"google.golang.org/grpc/credentials"
 
 	"github.com/gravitational/teleport/api/client/proto"
+	"github.com/gravitational/teleport/api/defaults"
 	discoveryconfigv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/discoveryconfig/v1"
 	usageeventsv1 "github.com/gravitational/teleport/api/gen/proto/go/usageevents/v1"
 	"github.com/gravitational/teleport/api/metadata"
@@ -490,6 +491,11 @@ func (s *Server) updateDiscoveryConfigStatus(fetchers []aws_sync.AWSSync, pushEr
 			// If this is a pre-run, the status is syncing.
 			status.State = discoveryconfigv1.DiscoveryConfigState_DISCOVERY_CONFIG_STATE_SYNCING.String()
 		}
+
+		// Ensure the error message is truncated to the maximum allowed size.
+		// Too large error messages will cause failures when clients (which use the default MaxCallRecvMsgSize of 4MB) try to read DiscoveryConfigs.
+		status.ErrorMessage = truncateErrorMessage(status)
+
 		ctx, cancel := context.WithTimeout(s.ctx, 5*time.Second)
 		defer cancel()
 		_, err := s.AccessPoint.UpdateDiscoveryConfigStatus(ctx, fetcher.DiscoveryConfigName(), status)
@@ -502,6 +508,20 @@ func (s *Server) updateDiscoveryConfigStatus(fetchers []aws_sync.AWSSync, pushEr
 	}
 }
 
+func truncateErrorMessage(discoveryConfigStatus discoveryconfig.Status) *string {
+	if discoveryConfigStatus.ErrorMessage == nil {
+		return nil
+	}
+
+	if len(*discoveryConfigStatus.ErrorMessage) <= defaults.DefaultMaxErrorMessageSize {
+		return discoveryConfigStatus.ErrorMessage
+	}
+
+	newErrorMessage := (*discoveryConfigStatus.ErrorMessage)[:defaults.DefaultMaxErrorMessageSize]
+
+	return &newErrorMessage
+}
+
 func buildFetcherStatus(fetcher aws_sync.AWSSync, pushErr error, lastUpdate time.Time) discoveryconfig.Status {
 	count, err := fetcher.Status()
 	err = trace.NewAggregate(err, pushErr)

lib/srv/discovery/status_test.go

@@ -0,0 +1,61 @@
+/*
+ * Teleport
+ * Copyright (C) 2025  Gravitational, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package discovery
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/gravitational/teleport/api/types/discoveryconfig"
+)
+
+func stringPointer(s string) *string {
+	return &s
+}
+
+func TestTruncateErrorMessage(t *testing.T) {
+	for _, tt := range []struct {
+		name     string
+		in       discoveryconfig.Status
+		expected *string
+	}{
+		{
+			name:     "nil error message",
+			in:       discoveryconfig.Status{},
+			expected: nil,
+		},
+		{
+			name:     "small error messages are not changed",
+			in:       discoveryconfig.Status{ErrorMessage: stringPointer("small error message")},
+			expected: stringPointer("small error message"),
+		},
+		{
+			name:     "large error messages are truncated",
+			in:       discoveryconfig.Status{ErrorMessage: stringPointer(strings.Repeat("A", 1024*100+1))},
+			expected: stringPointer(strings.Repeat("A", 1024*100)),
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			got := truncateErrorMessage(tt.in)
+			require.Equal(t, tt.expected, got)
+		})
+	}
+}