-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathmain.go
274 lines (233 loc) · 8.87 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
package main
import (
"bytes"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"os"
"strings"
"time"
)
// Validator represents the structure of validator data from the API response
type Validator struct {
Validator string `json:"validator"` // Address of the validator
Name string `json:"name"` // Name of the validator
IsJailed bool `json:"isJailed"` // Indicates if validator is jailed
IsActive bool `json:"isActive"` // Indicates if validator is active
Commission string `json:"commission"` // Commission rate charged by validator
UnjailableAfter *int64 `json:"unjailableAfter"` // Timestamp when validator can be unjailed (null if not jailed)
}
// NotificationBackoff handles exponential backoff for alerts to prevent notification spam
type NotificationBackoff struct {
LastSent time.Time // When the last notification was sent
BackoffFactor int // Current backoff multiplier
}
// ValidatorState tracks validator status between checks for state change detection
type ValidatorState struct {
IsJailed bool // Current jailed status
IsActive bool // Current active status
FirstRun bool // Indicates first check to prevent false recovery alerts
}
// Global variables for tracking notification state and validator status
var (
jailedBackoff = &NotificationBackoff{}
inactiveBackoff = &NotificationBackoff{}
recoveryBackoff = &NotificationBackoff{}
validatorState = &ValidatorState{FirstRun: true}
)
// Constants for backoff timing
const (
initialBackoff = time.Minute // Base backoff interval
maxBackoff = 15 * time.Minute // Maximum backoff interval to prevent excessive delays
)
// getEnv retrieves environment variable with fallback for CRON_INTERVAL
func getEnv(key string) string {
val := os.Getenv(key)
if val == "" && key == "CRON_INTERVAL" {
return "1m" // Default check interval
}
if val == "" {
log.Fatalf("ENV variable %s is required", key)
}
return val
}
// notifyDiscord sends alerts to Discord webhook
func notifyDiscord(webhook, message string) {
payload := map[string]string{"content": message}
data, _ := json.Marshal(payload)
resp, err := http.Post(webhook, "application/json", bytes.NewBuffer(data))
if err != nil {
log.Printf("Failed to send Discord notification: %v", err)
return
}
defer resp.Body.Close()
if resp.StatusCode != 204 && resp.StatusCode != 200 {
log.Printf("Discord webhook returned status: %s", resp.Status)
} else {
log.Println("Sent Discord alert:", message)
}
}
// shouldNotify determines if enough time has passed since last notification
// based on the current backoff factor
func shouldNotify(state *NotificationBackoff) bool {
if time.Since(state.LastSent) >= time.Duration(state.BackoffFactor)*initialBackoff {
return true
}
return false
}
// formatValidatorIdentifier creates a consistent identifier string with name and address
// to uniquely identify validators in logs and notifications
func formatValidatorIdentifier(name, address string) string {
if name == "" {
return fmt.Sprintf("%s", address)
}
return fmt.Sprintf("%s (%s)", name, address)
}
// updateBackoff increases the backoff factor exponentially up to a maximum value
func updateBackoff(state *NotificationBackoff) {
state.LastSent = time.Now()
if state.BackoffFactor == 0 {
state.BackoffFactor = 1
} else {
state.BackoffFactor *= 2
if time.Duration(state.BackoffFactor)*initialBackoff > maxBackoff {
state.BackoffFactor = int(maxBackoff / initialBackoff)
}
}
}
// resetBackoff resets backoff state when conditions return to normal
func resetBackoff(state *NotificationBackoff) {
state.BackoffFactor = 0
state.LastSent = time.Time{}
}
// runCheck performs a single validation check cycle
func runCheck(apiEndpoint, validatorAddress, discordWebhook string) {
startTime := time.Now()
log.Printf("Fetching validator status for address: %s", validatorAddress)
validator, err := fetchValidatorData(apiEndpoint, validatorAddress)
if err != nil {
log.Printf("Error fetching validator data: %v", err)
return
}
validatorName := validator.Name
if validatorName == "" {
validatorName = validatorAddress[:10] + "..." // Use truncated address if name is not available
}
validatorIdentifier := formatValidatorIdentifier(validatorName, validatorAddress)
// Log detailed validator information
log.Printf("Validator %s status: active=%v, jailed=%v, commission=%s",
validatorIdentifier, validator.IsActive, validator.IsJailed, validator.Commission)
// Recovery detection - only after first run completed
if !validatorState.FirstRun {
// Check for jailed -> not jailed transition
if validatorState.IsJailed && !validator.IsJailed {
message := fmt.Sprintf("✅ Validator %s has RECOVERED from jailed state", validatorIdentifier)
notifyDiscord(discordWebhook, message)
log.Printf("Recovery detected: %s", message)
resetBackoff(recoveryBackoff)
}
// Check for inactive -> active transition
if !validatorState.IsActive && validator.IsActive {
message := fmt.Sprintf("✅ Validator %s is now ACTIVE", validatorIdentifier)
notifyDiscord(discordWebhook, message)
log.Printf("Recovery detected: %s", message)
resetBackoff(recoveryBackoff)
}
}
// Update state for next comparison
validatorState.IsJailed = validator.IsJailed
validatorState.IsActive = validator.IsActive
validatorState.FirstRun = false
// Handle jailed status alerts with backoff
if validator.IsJailed {
if shouldNotify(jailedBackoff) {
unjailMsg := ""
if validator.UnjailableAfter != nil {
unjailTime := time.Unix(*validator.UnjailableAfter/1000, 0)
unjailMsg = fmt.Sprintf(" (unjailable after %s)", unjailTime.Format(time.RFC3339))
}
message := fmt.Sprintf("🚨 Validator %s is JAILED%s", validatorIdentifier, unjailMsg)
notifyDiscord(discordWebhook, message)
log.Printf("Alert: %s", message)
updateBackoff(jailedBackoff)
}
} else {
resetBackoff(jailedBackoff)
}
// Handle inactive status alerts with backoff
if !validator.IsActive {
if shouldNotify(inactiveBackoff) {
message := fmt.Sprintf("🚨 Validator %s is INACTIVE", validatorIdentifier)
notifyDiscord(discordWebhook, message)
log.Printf("Alert: %s", message)
updateBackoff(inactiveBackoff)
}
} else {
resetBackoff(inactiveBackoff)
}
elapsed := time.Since(startTime)
log.Printf("Validator %s monitor check complete (took %dms)", validatorIdentifier, elapsed.Milliseconds())
}
// fetchValidatorData retrieves validator data from the API and finds the requested validator
// Case-insensitive comparison is used for addresses to prevent configuration errors
func fetchValidatorData(apiEndpoint string, validatorAddress string) (*Validator, error) {
startTime := time.Now()
payload := []byte(`{"type":"validatorSummaries"}`)
req, err := http.NewRequest("POST", apiEndpoint, bytes.NewBuffer(payload))
if err != nil {
return nil, fmt.Errorf("error creating request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
client := http.Client{Timeout: 10 * time.Second}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("API request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("API returned non-200 status: %s", resp.Status)
}
var allValidators []Validator
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("error reading response body: %w", err)
}
if err := json.Unmarshal(body, &allValidators); err != nil {
return nil, fmt.Errorf("error parsing API response: %w", err)
}
log.Printf("API returned data for %d validators (took %dms)", len(allValidators), time.Since(startTime).Milliseconds())
lowercaseInputAddress := strings.ToLower(validatorAddress)
for _, val := range allValidators {
if strings.ToLower(val.Validator) == lowercaseInputAddress {
return &val, nil
}
}
return nil, fmt.Errorf("validator with address '%s' not found among %d validators",
validatorAddress, len(allValidators))
}
// main initializes the application and starts the monitoring loop
func main() {
log.SetFlags(log.LstdFlags | log.Lmicroseconds | log.LUTC)
log.Printf("Validator Monitor starting up...")
apiEndpoint := getEnv("API_ENDPOINT")
validatorAddress := getEnv("VALIDATOR_ADDRESS")
discordWebhook := getEnv("DISCORD_WEBHOOK")
cronInterval := getEnv("CRON_INTERVAL")
duration, err := time.ParseDuration(cronInterval)
if err != nil {
log.Fatalf("Invalid CRON_INTERVAL '%s': %v", cronInterval, err)
}
log.Printf("Configuration loaded - API: %s, Address: %s, Interval: %s",
apiEndpoint, validatorAddress, duration)
// Initial notification to confirm monitoring has started
notifyDiscord(discordWebhook, fmt.Sprintf("🔄 Validator monitoring started for %s (checking every %s)",
validatorAddress, duration))
// Main monitoring loop
for {
runCheck(apiEndpoint, validatorAddress, discordWebhook)
log.Printf("Sleeping for %s before next check", duration)
time.Sleep(duration)
}
}