Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Machine ID: Warn when returned cert TTL is less than expected #52833

Merged
merged 3 commits into from
Mar 12, 2025
Merged

Machine ID: Warn when returned cert TTL is less than expected #52833

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2ddb748
Machine ID: Warn when returned cert TTL is less than expected
timothyb89 Mar 6, 2025
ec2bd58
Add warning on most outputs; add TODO comment for future improvements
timothyb89 Mar 11, 2025
db69ac3
Mute slonglint false positivess
timothyb89 Mar 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions lib/tbot/output_utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -396,6 +396,54 @@ func generateIdentity(
return newIdentity, nil
}

// warnOnEarlyExpiration logs a warning if the given identity is likely to
// expire problematically early. This can happen if either the configured TTL is
// less than the renewal interval, or if the server returns certs valid for a
// shorter-than-expected period of time.
// This assumes the identity was just renewed, for the purposes of calculating
// TTLs, and may log false positive warnings if the time delta is large; the
// time calculations include a 1m buffer to mitigate this.
func warnOnEarlyExpiration(
ctx context.Context,
log *slog.Logger,
ident *identity.Identity,
lifetime config.CredentialLifetime,
) {
// Calculate a rough TTL, assuming this was called shortly after the
// identity was returned. We'll add a minute buffer to compensate and avoid
// superfluous warning messages.
effectiveTTL := time.Until(ident.TLSIdentity.Expires) + time.Minute

if effectiveTTL < lifetime.TTL {
l := log.With(
"requested_ttl", lifetime.TTL,
"renewal_interval", lifetime.RenewalInterval,
"effective_ttl", effectiveTTL,
"expires", ident.TLSIdentity.Expires,
"roles", ident.TLSIdentity.Groups,
)

// TODO(timothyb89): we can technically fetch our individual roles
// without explicit permission, and could determine which role in
// particular limited the TTL.

if effectiveTTL < lifetime.RenewalInterval {
//nolint:sloglint // multiline string is actually constant
l.WarnContext(ctx, "The server returned an identity shorter than "+
"expected and below the configured renewal interval, probably "+
"due to a `max_session_ttl` configured on a server-side role. "+
"Unless corrected, the credentials will be invalid for some "+
"period until renewal.")
} else {
//nolint:sloglint // multiline string is actually constant
l.WarnContext(ctx, "The server returned an identity shorter than "+
"the requested TTL, probably due to a `max_session_ttl` "+
"configured on a server-side role. It may not remain valid as "+
"long as expected.")
Comment on lines +439 to +442
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did some experimenting and found that bots can always fetch their own roles, so technically we could make some queries and log exactly which role caused the shortened TTL. Any thoughts on whether or not that would be worth the effort?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That'd be a really nice touch, but I think it depends on how many roles bots have in the common case. My hunch is that it's low enough that it wouldn't take users long to figure out which is the problematic role.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, and adds network calls on what's already a semi-error path, which isn't ideal. I think I'll stick with the simpler implementation for now, and maybe add a note to indicate we could improve this in the future if we see users having trouble.

}
}
}

// fetchDefaultRoles requests the bot's own role from the auth server and
// extracts its full list of allowed roles.
func fetchDefaultRoles(ctx context.Context, roleGetter services.RoleGetter, identity *identity.Identity) ([]string, error) {
Expand Down
5 changes: 4 additions & 1 deletion lib/tbot/service_application_output.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,12 +131,13 @@ func (s *ApplicationOutputService) generate(ctx context.Context) error {
return trace.Wrap(err)
}

effectiveLifetime := cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime)
routedIdentity, err := generateIdentity(
ctx,
s.botAuthClient,
id,
roles,
cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime).TTL,
effectiveLifetime.TTL,
func(req *proto.UserCertsRequest) {
req.RouteToApp = routeToApp
},
Expand All @@ -145,6 +146,8 @@ func (s *ApplicationOutputService) generate(ctx context.Context) error {
return trace.Wrap(err)
}

warnOnEarlyExpiration(ctx, s.log.With("output", s), id, effectiveLifetime)

s.log.InfoContext(
ctx,
"Generated identity for app",
Expand Down
5 changes: 4 additions & 1 deletion lib/tbot/service_database_output.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,12 +133,13 @@ func (s *DatabaseOutputService) generate(ctx context.Context) error {
return trace.Wrap(err)
}

effectiveLifetime := cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime)
routedIdentity, err := generateIdentity(
ctx,
s.botAuthClient,
id,
roles,
cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime).TTL,
effectiveLifetime.TTL,
func(req *proto.UserCertsRequest) {
req.RouteToDatabase = route
},
Expand All @@ -147,6 +148,8 @@ func (s *DatabaseOutputService) generate(ctx context.Context) error {
return trace.Wrap(err)
}

warnOnEarlyExpiration(ctx, s.log.With("output", s), id, effectiveLifetime)

s.log.InfoContext(
ctx,
"Generated identity for database",
Expand Down
5 changes: 4 additions & 1 deletion lib/tbot/service_identity_output.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,12 +113,13 @@ func (s *IdentityOutputService) generate(ctx context.Context) error {
}
}

effectiveLifetime := cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime)
id, err := generateIdentity(
ctx,
s.botAuthClient,
s.getBotIdentity(),
roles,
cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime).TTL,
effectiveLifetime.TTL,
func(req *proto.UserCertsRequest) {
req.ReissuableRoleImpersonation = s.cfg.AllowReissue
},
Expand Down Expand Up @@ -152,6 +153,8 @@ func (s *IdentityOutputService) generate(ctx context.Context) error {
}
}

warnOnEarlyExpiration(ctx, s.log.With("output", s), id, effectiveLifetime)

hostCAs, err := s.botAuthClient.GetCertAuthorities(ctx, types.HostCA, false)
if err != nil {
return trace.Wrap(err)
Expand Down
5 changes: 4 additions & 1 deletion lib/tbot/service_kubernetes_output.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,12 +149,13 @@ func (s *KubernetesOutputService) generate(ctx context.Context) error {
// and will fail to generate certs if the cluster doesn't exist or is
// offline.

effectiveLifetime := cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime)
routedIdentity, err := generateIdentity(
ctx,
s.botAuthClient,
id,
roles,
cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime).TTL,
effectiveLifetime.TTL,
func(req *proto.UserCertsRequest) {
req.KubernetesCluster = kubeClusterName
},
Expand All @@ -163,6 +164,8 @@ func (s *KubernetesOutputService) generate(ctx context.Context) error {
return trace.Wrap(err)
}

warnOnEarlyExpiration(ctx, s.log.With("output", s), id, effectiveLifetime)

s.log.InfoContext(
ctx,
"Generated identity for Kubernetes cluster",
Expand Down
6 changes: 5 additions & 1 deletion lib/tbot/service_kubernetes_v2_output.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,17 +112,21 @@ func (s *KubernetesV2OutputService) generate(ctx context.Context) error {
return trace.Wrap(err, "fetching default roles")
}

effectiveLifetime := cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime)
id, err := generateIdentity(
ctx,
s.botAuthClient,
s.getBotIdentity(),
roles,
cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime).TTL,
effectiveLifetime.TTL,
nil,
)
if err != nil {
return trace.Wrap(err, "generating identity")
}

warnOnEarlyExpiration(ctx, s.log.With("output", s), id, effectiveLifetime)

// create a client that uses the impersonated identity, so that when we
// fetch information, we can ensure access rights are enforced.
facade := identity.NewFacade(s.botCfg.FIPS, s.botCfg.Insecure, id)
Expand Down
6 changes: 5 additions & 1 deletion lib/tbot/service_spiffe_svid_output.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,17 +179,21 @@ func (s *SPIFFESVIDOutputService) requestSVID(
return nil, nil, nil, trace.Wrap(err, "fetching roles")
}

effectiveLifetime := cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime)
id, err := generateIdentity(
ctx,
s.botAuthClient,
s.getBotIdentity(),
roles,
cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime).TTL,
effectiveLifetime.TTL,
nil,
)
if err != nil {
return nil, nil, nil, trace.Wrap(err, "generating identity")
}

warnOnEarlyExpiration(ctx, s.log.With("output", s), id, effectiveLifetime)

// create a client that uses the impersonated identity, so that when we
// fetch information, we can ensure access rights are enforced.
facade := identity.NewFacade(s.botCfg.FIPS, s.botCfg.Insecure, id)
Expand Down
6 changes: 5 additions & 1 deletion lib/tbot/service_ssh_host_output.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,17 +104,21 @@ func (s *SSHHostOutputService) generate(ctx context.Context) error {
}
}

effectiveLifetime := cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime)
id, err := generateIdentity(
ctx,
s.botAuthClient,
s.getBotIdentity(),
roles,
cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime).TTL,
effectiveLifetime.TTL,
nil,
)
if err != nil {
return trace.Wrap(err, "generating identity")
}

warnOnEarlyExpiration(ctx, s.log.With("output", s), id, effectiveLifetime)

// create a client that uses the impersonated identity, so that when we
// fetch information, we can ensure access rights are enforced.
facade := identity.NewFacade(s.botCfg.FIPS, s.botCfg.Insecure, id)
Expand Down
6 changes: 5 additions & 1 deletion lib/tbot/service_workload_identity_jwt.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,18 +173,22 @@ func (s *WorkloadIdentityJWTService) requestJWTSVID(
}
defer impersonatedClient.Close()

effectiveLifetime := cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime)
credentials, err := workloadidentity.IssueJWTWorkloadIdentity(
ctx,
s.log,
impersonatedClient,
s.cfg.Selector,
s.cfg.Audiences,
cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime).TTL,
effectiveLifetime.TTL,
nil,
)
if err != nil {
return nil, trace.Wrap(err, "generating JWT SVID")
}

warnOnEarlyExpiration(ctx, s.log.With("output", s), id, effectiveLifetime)

var credential *workloadidentityv1pb.Credential
switch len(credentials) {
case 0:
Expand Down
6 changes: 5 additions & 1 deletion lib/tbot/service_workload_identity_x509.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,17 +191,21 @@ func (s *WorkloadIdentityX509Service) requestSVID(
return nil, nil, trace.Wrap(err, "fetching roles")
}

effectiveLifetime := cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime)
id, err := generateIdentity(
ctx,
s.botAuthClient,
s.getBotIdentity(),
roles,
cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime).TTL,
effectiveLifetime.TTL,
nil,
)
if err != nil {
return nil, nil, trace.Wrap(err, "generating identity")
}

warnOnEarlyExpiration(ctx, s.log.With("output", s), id, effectiveLifetime)

// create a client that uses the impersonated identity, so that when we
// fetch information, we can ensure access rights are enforced.
facade := identity.NewFacade(s.botCfg.FIPS, s.botCfg.Insecure, id)
Expand Down
Loading