From 9c7358f5ea557654d946626d96f23a6411e4c5dd Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Mon, 7 Mar 2022 11:04:22 +0100 Subject: [PATCH 01/19] PMM-9630 Use status, status_code only for some versions. --- .../postgres/pgstatmonitor/pgstatmonitor.go | 29 ++++++++++--------- .../pgstatmonitor/pgstatmonitor_models.go | 17 ++--------- .../pgstatmonitor/pgstatmonitor_test.go | 4 +-- .../pgstatmonitor/stat_monitor_cache.go | 8 +++-- 4 files changed, 26 insertions(+), 32 deletions(-) diff --git a/agents/postgres/pgstatmonitor/pgstatmonitor.go b/agents/postgres/pgstatmonitor/pgstatmonitor.go index cbf8a6e03..01211cd8e 100644 --- a/agents/postgres/pgstatmonitor/pgstatmonitor.go +++ b/agents/postgres/pgstatmonitor/pgstatmonitor.go @@ -88,7 +88,7 @@ const ( commandTypeUpdate = "UPDATE" commandTypeInsert = "INSERT" commandTypeDelete = "DELETE" - commandTypeUtiity = "UTILITY" + commandTypeUtility = "UTILITY" ) var commandTypeToText = []string{ @@ -97,7 +97,7 @@ var commandTypeToText = []string{ commandTypeUpdate, commandTypeInsert, commandTypeDelete, - commandTypeUtiity, + commandTypeUtility, commandTextNotAvailable, } @@ -160,38 +160,41 @@ func getPGVersion(q *reform.Querier) (pgVersion float64, err error) { return strconv.ParseFloat(v, 64) } -func getPGMonitorVersion(q *reform.Querier) (pgStatMonitorVersion, error) { +func getPGMonitorVersion(q *reform.Querier) (pgStatMonitorVersion, string, error) { var result string err := q.QueryRow(fmt.Sprintf("SELECT /* %s */ pg_stat_monitor_version()", queryTag)).Scan(&result) if err != nil { - return pgStatMonitorVersion06, errors.Wrap(err, "failed to get pg_stat_monitor version from DB") + return pgStatMonitorVersion06, "", errors.Wrap(err, "failed to get pg_stat_monitor version from DB") } pgsmVersion, err := ver.NewVersion(result) if err != nil { - return pgStatMonitorVersion06, errors.Wrap(err, "failed to parse pg_stat_monitor version") + return pgStatMonitorVersion06, "", errors.Wrap(err, "failed to parse pg_stat_monitor version") } pgVersion, err := getPGVersion(q) if err != nil { - return pgStatMonitorVersion06, err + return pgStatMonitorVersion06, "", err } + version := pgStatMonitorVersion06 switch { case pgsmVersion.Core().GreaterThanOrEqual(v10): if pgVersion >= 14 { - return pgStatMonitorVersion10PG14, nil + version = pgStatMonitorVersion10PG14 + break } if pgVersion >= 13 { - return pgStatMonitorVersion10PG13, nil + version = pgStatMonitorVersion10PG13 + break } - return pgStatMonitorVersion10PG12, nil + version = pgStatMonitorVersion10PG12 case pgsmVersion.GreaterThanOrEqual(v09): - return pgStatMonitorVersion09, nil + version = pgStatMonitorVersion09 case pgsmVersion.GreaterThanOrEqual(v08): - return pgStatMonitorVersion08, nil - default: - return pgStatMonitorVersion06, nil + version = pgStatMonitorVersion08 } + + return version, pgsmVersion.Prerelease(), nil } // Run extracts stats data and sends it to the channel until ctx is canceled. diff --git a/agents/postgres/pgstatmonitor/pgstatmonitor_models.go b/agents/postgres/pgstatmonitor/pgstatmonitor_models.go index f34b1922b..4776df7bd 100644 --- a/agents/postgres/pgstatmonitor/pgstatmonitor_models.go +++ b/agents/postgres/pgstatmonitor/pgstatmonitor_models.go @@ -28,7 +28,7 @@ import ( ) var ( - v10 = version.Must(version.NewVersion("1.0.0-beta-2")) + v10 = version.Must(version.NewVersion("1.0.0")) v09 = version.Must(version.NewVersion("0.9")) v08 = version.Must(version.NewVersion("0.8")) ) @@ -83,13 +83,6 @@ type pgStatMonitor struct { WalRecords int64 WalFpi int64 WalBytes int64 - // state_code = 0 state 'PARSING' - // state_code = 1 state 'PLANNING' - // state_code = 2 state 'ACTIVE' - // state_code = 3 state 'FINISHED' - // state_code = 4 state 'FINISHED WITH ERROR' - StateCode int64 - State string // < pg0.6 @@ -180,9 +173,7 @@ func NewPgStatMonitorStructs(v pgStatMonitorVersion) (*pgStatMonitor, reform.Vie field{info: parse.FieldInfo{Name: "Message", Type: "*string", Column: "message"}, pointer: &s.Message}, field{info: parse.FieldInfo{Name: "WalRecords", Type: "int64", Column: "wal_records"}, pointer: &s.WalRecords}, field{info: parse.FieldInfo{Name: "WalFpi", Type: "int64", Column: "wal_fpi"}, pointer: &s.WalFpi}, - field{info: parse.FieldInfo{Name: "WalBytes", Type: "int64", Column: "wal_bytes"}, pointer: &s.WalBytes}, - field{info: parse.FieldInfo{Name: "StateCode", Type: "int64", Column: "state_code"}, pointer: &s.StateCode}, - field{info: parse.FieldInfo{Name: "State", Type: "string", Column: "state"}, pointer: &s.State}) + field{info: parse.FieldInfo{Name: "WalBytes", Type: "int64", Column: "wal_bytes"}, pointer: &s.WalBytes}) } if v <= pgStatMonitorVersion10PG12 { @@ -259,7 +250,7 @@ func (v *pgStatMonitorAllViewType) NewStruct() reform.Struct { // String returns a string representation of this struct or record. func (s pgStatMonitor) String() string { - res := make([]string, 51) + res := make([]string, 49) res[0] = "Bucket: " + reform.Inspect(s.Bucket, true) res[1] = "BucketStartTime: " + reform.Inspect(s.BucketStartTime, true) res[2] = "UserID: " + reform.Inspect(s.UserID, true) @@ -309,8 +300,6 @@ func (s pgStatMonitor) String() string { res[46] = "WalRecords: " + reform.Inspect(s.WalRecords, true) res[47] = "WalFpi: " + reform.Inspect(s.WalFpi, true) res[48] = "WalBytes: " + reform.Inspect(s.WalBytes, true) - res[49] = "StateCode: " + reform.Inspect(s.StateCode, true) - res[50] = "State: " + reform.Inspect(s.State, true) return strings.Join(res, ", ") } diff --git a/agents/postgres/pgstatmonitor/pgstatmonitor_test.go b/agents/postgres/pgstatmonitor/pgstatmonitor_test.go index 0eeee92ef..2003dd97e 100644 --- a/agents/postgres/pgstatmonitor/pgstatmonitor_test.go +++ b/agents/postgres/pgstatmonitor/pgstatmonitor_test.go @@ -114,7 +114,7 @@ func TestPGStatMonitorSchema(t *testing.T) { assert.NoError(t, err) }() - version, err := getPGMonitorVersion(db.Querier) + version, _, err := getPGMonitorVersion(db.Querier) assert.NoError(t, err) _, view := NewPgStatMonitorStructs(version) @@ -177,7 +177,7 @@ func TestPGStatMonitorSchema(t *testing.T) { var selectCMDType, insertCMDType string var mPlansCallsCnt, mPlansTimeCnt float32 - pgsmVersion, err := getPGMonitorVersion(db.Querier) + pgsmVersion, _, err := getPGMonitorVersion(db.Querier) assert.NoError(t, err) switch pgsmVersion { case pgStatMonitorVersion06: diff --git a/agents/postgres/pgstatmonitor/stat_monitor_cache.go b/agents/postgres/pgstatmonitor/stat_monitor_cache.go index bd5b770d5..66a752280 100644 --- a/agents/postgres/pgstatmonitor/stat_monitor_cache.go +++ b/agents/postgres/pgstatmonitor/stat_monitor_cache.go @@ -83,7 +83,7 @@ func (ssc *statMonitorCache) getStatMonitorExtended(ctx context.Context, q *refo databases := queryDatabases(q) usernames := queryUsernames(q) - pgMonitorVersion, err := getPGMonitorVersion(q) + pgMonitorVersion, prerelease, err := getPGMonitorVersion(q) if err != nil { err = errors.Wrap(err, "failed to get row and view for pg_stat_monitor version") return @@ -92,10 +92,12 @@ func (ssc *statMonitorCache) getStatMonitorExtended(ctx context.Context, q *refo row, view := NewPgStatMonitorStructs(pgMonitorVersion) conditions := "WHERE queryid IS NOT NULL AND query IS NOT NULL" - if pgMonitorVersion >= pgStatMonitorVersion09 { - // only pg_stat_monitor 0.9.0 and above supports state_code. It tells what is the query's current state. + if (pgMonitorVersion == pgStatMonitorVersion09) || + (pgMonitorVersion >= pgStatMonitorVersion10PG12 && pgMonitorVersion <= pgStatMonitorVersion10PG14 && prerelease != "") { + // only pg_stat_monitor 0.9.0, 1.0.0-beta-2, 1.0.0-rc.1 supports state_code. It tells what is the query's current state. // To have correct data in QAN, we have to get only queries that are either 'FINISHED' or 'FINISHED WITH ERROR'. conditions += " AND (state_code = 3 OR state_code = 4)" + ssc.l.Debug("PGSM version with state and state_code") } rows, e := q.SelectRows(view, conditions) if e != nil { From f27e24956dba34c55213445d42557f4bd7e79990 Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Mon, 7 Mar 2022 11:22:20 +0100 Subject: [PATCH 02/19] PMM-9630 Remove unnecessary line. --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8411f118e..2f6beb176 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -85,7 +85,6 @@ services: -c shared_preload_libraries='${PG_PRELOADED_LIBS:-pg_stat_statements}' -c track_activity_query_size=2048 -c pg_stat_statements.max=10000 - -c pg_stat_monitor.pgsm_query_max_len=10000 -c pg_stat_statements.track=all -c pg_stat_statements.save=off -c track_io_timing=on From 4de69d664655337755616a155af9fccd9993c312 Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Mon, 7 Mar 2022 11:22:39 +0100 Subject: [PATCH 03/19] PMM-9630 Fix test. --- agents/postgres/pgstatmonitor/pgstatmonitor_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/postgres/pgstatmonitor/pgstatmonitor_test.go b/agents/postgres/pgstatmonitor/pgstatmonitor_test.go index 2003dd97e..9827be1e0 100644 --- a/agents/postgres/pgstatmonitor/pgstatmonitor_test.go +++ b/agents/postgres/pgstatmonitor/pgstatmonitor_test.go @@ -92,7 +92,7 @@ func filter(mb []*agentpb.MetricsBucket) []*agentpb.MetricsBucket { func TestVersion(t *testing.T) { pgsmVersion, err := ver.NewVersion("1.0.0-beta-2") require.NoError(t, err) - require.True(t, pgsmVersion.GreaterThanOrEqual(v10)) + require.True(t, pgsmVersion.LessThan(v10)) } func TestPGStatMonitorSchema(t *testing.T) { From 22ece8fd3577937ee59418d5c9efdd65d954212b Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Tue, 8 Mar 2022 09:57:47 +0100 Subject: [PATCH 04/19] PMM-9630 Handle RC 2. --- agents/postgres/pgstatmonitor/stat_monitor_cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/postgres/pgstatmonitor/stat_monitor_cache.go b/agents/postgres/pgstatmonitor/stat_monitor_cache.go index 66a752280..2301e6722 100644 --- a/agents/postgres/pgstatmonitor/stat_monitor_cache.go +++ b/agents/postgres/pgstatmonitor/stat_monitor_cache.go @@ -93,7 +93,7 @@ func (ssc *statMonitorCache) getStatMonitorExtended(ctx context.Context, q *refo row, view := NewPgStatMonitorStructs(pgMonitorVersion) conditions := "WHERE queryid IS NOT NULL AND query IS NOT NULL" if (pgMonitorVersion == pgStatMonitorVersion09) || - (pgMonitorVersion >= pgStatMonitorVersion10PG12 && pgMonitorVersion <= pgStatMonitorVersion10PG14 && prerelease != "") { + (pgMonitorVersion >= pgStatMonitorVersion10PG12 && pgMonitorVersion <= pgStatMonitorVersion10PG14 && prerelease != "" && prerelease != "rc.2") { // only pg_stat_monitor 0.9.0, 1.0.0-beta-2, 1.0.0-rc.1 supports state_code. It tells what is the query's current state. // To have correct data in QAN, we have to get only queries that are either 'FINISHED' or 'FINISHED WITH ERROR'. conditions += " AND (state_code = 3 OR state_code = 4)" From 3e99acb75e2c6cc672334ce8ac230f0a42f4a110 Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Tue, 8 Mar 2022 09:57:47 +0100 Subject: [PATCH 05/19] Revert "PMM-9630 Handle RC 2." This reverts commit 22ece8fd3577937ee59418d5c9efdd65d954212b. --- agents/postgres/pgstatmonitor/stat_monitor_cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/postgres/pgstatmonitor/stat_monitor_cache.go b/agents/postgres/pgstatmonitor/stat_monitor_cache.go index 2301e6722..66a752280 100644 --- a/agents/postgres/pgstatmonitor/stat_monitor_cache.go +++ b/agents/postgres/pgstatmonitor/stat_monitor_cache.go @@ -93,7 +93,7 @@ func (ssc *statMonitorCache) getStatMonitorExtended(ctx context.Context, q *refo row, view := NewPgStatMonitorStructs(pgMonitorVersion) conditions := "WHERE queryid IS NOT NULL AND query IS NOT NULL" if (pgMonitorVersion == pgStatMonitorVersion09) || - (pgMonitorVersion >= pgStatMonitorVersion10PG12 && pgMonitorVersion <= pgStatMonitorVersion10PG14 && prerelease != "" && prerelease != "rc.2") { + (pgMonitorVersion >= pgStatMonitorVersion10PG12 && pgMonitorVersion <= pgStatMonitorVersion10PG14 && prerelease != "") { // only pg_stat_monitor 0.9.0, 1.0.0-beta-2, 1.0.0-rc.1 supports state_code. It tells what is the query's current state. // To have correct data in QAN, we have to get only queries that are either 'FINISHED' or 'FINISHED WITH ERROR'. conditions += " AND (state_code = 3 OR state_code = 4)" From 9006b7c057ad8826f684d3a41be3b73e342d906f Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Wed, 9 Mar 2022 09:15:53 +0100 Subject: [PATCH 06/19] PMM-9630 Bump PSGM framework version and defaults. --- docker-compose-pg-load.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker-compose-pg-load.yml b/docker-compose-pg-load.yml index 76519d86d..5ff2a4b57 100644 --- a/docker-compose-pg-load.yml +++ b/docker-compose-pg-load.yml @@ -4,13 +4,14 @@ version: '3.7' services: postgres-pgmonitor: - image: ${POSTGRES_IMAGE:-perconalab/percona-distribution-postgresql:13.3} + image: ${POSTGRES_IMAGE:-perconalab/percona-distribution-postgresql:14.1} container_name: pmm-agent-postgres-pgmonitor command: > -c shared_preload_libraries=pg_stat_monitor -c track_activity_query_size=2048 -c pg_stat_monitor.pgsm_query_max_len=10000 -c pg_stat_monitor.pgsm_normalized_query=0 + -c pg_stat_monitor.pgsm_enable_query_plan=1 -c track_io_timing=on ports: - 127.0.0.1:5432:5432 @@ -21,7 +22,7 @@ services: - test_db_postgres:/docker-entrypoint-initdb.d/ postgres-load: - image: ${POSTGRES_IMAGE:-perconalab/percona-distribution-postgresql:13.3} + image: ${POSTGRES_IMAGE:-perconalab/percona-distribution-postgresql:14.1} container_name: pmm-agent-postgres-load depends_on: - postgres-pgmonitor From dfe4d3eda6cfda99d8dc346bcd9258fb01fd8dde Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Thu, 10 Mar 2022 11:54:14 +0100 Subject: [PATCH 07/19] PMM-9630 If refactor. --- agents/postgres/pgstatmonitor/stat_monitor_cache.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/agents/postgres/pgstatmonitor/stat_monitor_cache.go b/agents/postgres/pgstatmonitor/stat_monitor_cache.go index 66a752280..bc926a1e4 100644 --- a/agents/postgres/pgstatmonitor/stat_monitor_cache.go +++ b/agents/postgres/pgstatmonitor/stat_monitor_cache.go @@ -92,9 +92,8 @@ func (ssc *statMonitorCache) getStatMonitorExtended(ctx context.Context, q *refo row, view := NewPgStatMonitorStructs(pgMonitorVersion) conditions := "WHERE queryid IS NOT NULL AND query IS NOT NULL" - if (pgMonitorVersion == pgStatMonitorVersion09) || - (pgMonitorVersion >= pgStatMonitorVersion10PG12 && pgMonitorVersion <= pgStatMonitorVersion10PG14 && prerelease != "") { - // only pg_stat_monitor 0.9.0, 1.0.0-beta-2, 1.0.0-rc.1 supports state_code. It tells what is the query's current state. + if pgMonitorVersion >= pgStatMonitorVersion09 && pgMonitorVersion <= pgStatMonitorVersion10PG14 && prerelease != "" { + // only pg_stat_monitor 0.9.0, 1.0.0-beta-2, 1.0.0-rc.1, 1.0.0-rc.2 supports state_code. It tells what is the query's current state. // To have correct data in QAN, we have to get only queries that are either 'FINISHED' or 'FINISHED WITH ERROR'. conditions += " AND (state_code = 3 OR state_code = 4)" ssc.l.Debug("PGSM version with state and state_code") From eb4edf6f73c4f1a91c5b8fd2dbfc1d02b0162ad0 Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Tue, 26 Apr 2022 11:44:51 +0200 Subject: [PATCH 08/19] PMM-9875 Errors view draft. --- agents/postgres/pgstatmonitor/models.go | 9 ++ .../postgres/pgstatmonitor/models_reform.go | 91 +++++++++++++++++++ .../postgres/pgstatmonitor/pgstatmonitor.go | 4 + 3 files changed, 104 insertions(+) diff --git a/agents/postgres/pgstatmonitor/models.go b/agents/postgres/pgstatmonitor/models.go index 82b589d2e..679234a3f 100644 --- a/agents/postgres/pgstatmonitor/models.go +++ b/agents/postgres/pgstatmonitor/models.go @@ -49,6 +49,15 @@ type pgStatMonitorSettingsTextValue struct { Value string `reform:"value"` } +// pgStatMonitorErrors represents a row in pg_stat_monitor_errors view. +//reform:pg_stat_monitor_settings +type pgStatMonitorErrors struct { + Severity string `reform:"severity"` + Message string `reform:"message"` + MessageTime string `reform:"msgtime"` + Calls int8 `reform:"calls"` +} + // pgStatMonitorExtended contains pgStatMonitor data and extends it with database, username and tables data. // It's made for performance reason. type pgStatMonitorExtended struct { diff --git a/agents/postgres/pgstatmonitor/models_reform.go b/agents/postgres/pgstatmonitor/models_reform.go index 38344853a..a3bbef036 100644 --- a/agents/postgres/pgstatmonitor/models_reform.go +++ b/agents/postgres/pgstatmonitor/models_reform.go @@ -332,9 +332,100 @@ var ( _ fmt.Stringer = (*pgStatMonitorSettingsTextValue)(nil) ) +type pgStatMonitorErrorsViewType struct { + s parse.StructInfo + z []interface{} +} + +// Schema returns a schema name in SQL database (""). +func (v *pgStatMonitorErrorsViewType) Schema() string { + return v.s.SQLSchema +} + +// Name returns a view or table name in SQL database ("pg_stat_monitor_settings"). +func (v *pgStatMonitorErrorsViewType) Name() string { + return v.s.SQLName +} + +// Columns returns a new slice of column names for that view or table in SQL database. +func (v *pgStatMonitorErrorsViewType) Columns() []string { + return []string{ + "severity", + "message", + "msgtime", + "calls", + } +} + +// NewStruct makes a new struct for that view or table. +func (v *pgStatMonitorErrorsViewType) NewStruct() reform.Struct { + return new(pgStatMonitorErrors) +} + +// pgStatMonitorErrorsView represents pg_stat_monitor_settings view or table in SQL database. +var pgStatMonitorErrorsView = &pgStatMonitorErrorsViewType{ + s: parse.StructInfo{ + Type: "pgStatMonitorErrors", + SQLName: "pg_stat_monitor_settings", + Fields: []parse.FieldInfo{ + {Name: "Severity", Type: "string", Column: "severity"}, + {Name: "Message", Type: "string", Column: "message"}, + {Name: "MessageTime", Type: "string", Column: "msgtime"}, + {Name: "Calls", Type: "int8", Column: "calls"}, + }, + PKFieldIndex: -1, + }, + z: new(pgStatMonitorErrors).Values(), +} + +// String returns a string representation of this struct or record. +func (s pgStatMonitorErrors) String() string { + res := make([]string, 4) + res[0] = "Severity: " + reform.Inspect(s.Severity, true) + res[1] = "Message: " + reform.Inspect(s.Message, true) + res[2] = "MessageTime: " + reform.Inspect(s.MessageTime, true) + res[3] = "Calls: " + reform.Inspect(s.Calls, true) + return strings.Join(res, ", ") +} + +// Values returns a slice of struct or record field values. +// Returned interface{} values are never untyped nils. +func (s *pgStatMonitorErrors) Values() []interface{} { + return []interface{}{ + s.Severity, + s.Message, + s.MessageTime, + s.Calls, + } +} + +// Pointers returns a slice of pointers to struct or record fields. +// Returned interface{} values are never untyped nils. +func (s *pgStatMonitorErrors) Pointers() []interface{} { + return []interface{}{ + &s.Severity, + &s.Message, + &s.MessageTime, + &s.Calls, + } +} + +// View returns View object for that struct. +func (s *pgStatMonitorErrors) View() reform.View { + return pgStatMonitorErrorsView +} + +// check interfaces +var ( + _ reform.View = pgStatMonitorErrorsView + _ reform.Struct = (*pgStatMonitorErrors)(nil) + _ fmt.Stringer = (*pgStatMonitorErrors)(nil) +) + func init() { parse.AssertUpToDate(&pgStatDatabaseView.s, new(pgStatDatabase)) parse.AssertUpToDate(&pgUserView.s, new(pgUser)) parse.AssertUpToDate(&pgStatMonitorSettingsView.s, new(pgStatMonitorSettings)) parse.AssertUpToDate(&pgStatMonitorSettingsTextValueView.s, new(pgStatMonitorSettingsTextValue)) + parse.AssertUpToDate(&pgStatMonitorErrorsView.s, new(pgStatMonitorErrors)) } diff --git a/agents/postgres/pgstatmonitor/pgstatmonitor.go b/agents/postgres/pgstatmonitor/pgstatmonitor.go index ae6c01e34..d87fa3882 100644 --- a/agents/postgres/pgstatmonitor/pgstatmonitor.go +++ b/agents/postgres/pgstatmonitor/pgstatmonitor.go @@ -170,6 +170,10 @@ func newPgStatMonitorQAN(q *reform.Querier, dbCloser io.Closer, agentID string, return nil, errors.Wrap(err, "failed to get settings") } + //var errors []reform.Struct + errs, err := q.SelectAllFrom(pgStatMonitorErrorsView, "") + fmt.Printf("\n\n\n\n\n\n\n\n err:%+v \n\n\n\n\n\n\n\n", errs) + var normalizedQuery bool waitTime := defaultWaitTime for _, row := range settings { From cb2e585cb9caf17057dad20a2cfe950711b50318 Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Wed, 27 Apr 2022 12:51:20 +0200 Subject: [PATCH 09/19] PMM-9630 Code. --- agents/postgres/pgstatmonitor/stat_monitor_cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/postgres/pgstatmonitor/stat_monitor_cache.go b/agents/postgres/pgstatmonitor/stat_monitor_cache.go index 632f37cce..52084e416 100644 --- a/agents/postgres/pgstatmonitor/stat_monitor_cache.go +++ b/agents/postgres/pgstatmonitor/stat_monitor_cache.go @@ -83,7 +83,7 @@ func (ssc *statMonitorCache) getStatMonitorExtended(ctx context.Context, q *refo databases := queryDatabases(q) usernames := queryUsernames(q) - pgMonitorVersion, _, err := getPGMonitorVersion(q) + pgMonitorVersion, prerelease, err := getPGMonitorVersion(q) if err != nil { err = errors.Wrap(err, "failed to get row and view for pg_stat_monitor version") return From 02b11167e38c9f56aa24b58b10ecdf983977740d Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Fri, 29 Apr 2022 09:42:36 +0200 Subject: [PATCH 10/19] PMM-9875 Changes. --- agents/postgres/pgstatmonitor/models.go | 4 +- .../postgres/pgstatmonitor/models_reform.go | 8 ++-- .../postgres/pgstatmonitor/pgstatmonitor.go | 42 +++++++++++++++++-- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/agents/postgres/pgstatmonitor/models.go b/agents/postgres/pgstatmonitor/models.go index 679234a3f..694aeb0ac 100644 --- a/agents/postgres/pgstatmonitor/models.go +++ b/agents/postgres/pgstatmonitor/models.go @@ -50,12 +50,12 @@ type pgStatMonitorSettingsTextValue struct { } // pgStatMonitorErrors represents a row in pg_stat_monitor_errors view. -//reform:pg_stat_monitor_settings +//reform:pg_stat_monitor_errors type pgStatMonitorErrors struct { Severity string `reform:"severity"` Message string `reform:"message"` MessageTime string `reform:"msgtime"` - Calls int8 `reform:"calls"` + Calls int64 `reform:"calls"` } // pgStatMonitorExtended contains pgStatMonitor data and extends it with database, username and tables data. diff --git a/agents/postgres/pgstatmonitor/models_reform.go b/agents/postgres/pgstatmonitor/models_reform.go index a3bbef036..310c1fd99 100644 --- a/agents/postgres/pgstatmonitor/models_reform.go +++ b/agents/postgres/pgstatmonitor/models_reform.go @@ -342,7 +342,7 @@ func (v *pgStatMonitorErrorsViewType) Schema() string { return v.s.SQLSchema } -// Name returns a view or table name in SQL database ("pg_stat_monitor_settings"). +// Name returns a view or table name in SQL database ("pg_stat_monitor_errors"). func (v *pgStatMonitorErrorsViewType) Name() string { return v.s.SQLName } @@ -362,16 +362,16 @@ func (v *pgStatMonitorErrorsViewType) NewStruct() reform.Struct { return new(pgStatMonitorErrors) } -// pgStatMonitorErrorsView represents pg_stat_monitor_settings view or table in SQL database. +// pgStatMonitorErrorsView represents pg_stat_monitor_errors view or table in SQL database. var pgStatMonitorErrorsView = &pgStatMonitorErrorsViewType{ s: parse.StructInfo{ Type: "pgStatMonitorErrors", - SQLName: "pg_stat_monitor_settings", + SQLName: "pg_stat_monitor_errors", Fields: []parse.FieldInfo{ {Name: "Severity", Type: "string", Column: "severity"}, {Name: "Message", Type: "string", Column: "message"}, {Name: "MessageTime", Type: "string", Column: "msgtime"}, - {Name: "Calls", Type: "int8", Column: "calls"}, + {Name: "Calls", Type: "int64", Column: "calls"}, }, PKFieldIndex: -1, }, diff --git a/agents/postgres/pgstatmonitor/pgstatmonitor.go b/agents/postgres/pgstatmonitor/pgstatmonitor.go index 91175e7fd..826a151df 100644 --- a/agents/postgres/pgstatmonitor/pgstatmonitor.go +++ b/agents/postgres/pgstatmonitor/pgstatmonitor.go @@ -59,6 +59,7 @@ type PGStatMonitorQAN struct { // options. pgsmNormalizedQuery bool waitTime time.Duration + timeDiff time.Duration disableQueryExamples bool } @@ -170,10 +171,6 @@ func newPgStatMonitorQAN(q *reform.Querier, dbCloser io.Closer, agentID string, return nil, errors.Wrap(err, "failed to get settings") } - //var errors []reform.Struct - errs, err := q.SelectAllFrom(pgStatMonitorErrorsView, "") - fmt.Printf("\n\n\n\n\n\n\n\n err:%+v \n\n\n\n\n\n\n\n", errs) - var normalizedQuery bool waitTime := defaultWaitTime for _, row := range settings { @@ -210,6 +207,20 @@ func newPgStatMonitorQAN(q *reform.Querier, dbCloser io.Closer, agentID string, } } + // var pgTime string + // err = q.QueryRow(fmt.Sprintf("SELECT /* %s */ NOW()", queryTag)).Scan(&pgTime) + // if err != nil { + // return nil, errors.Wrap(err, "postgres time parsing failed") + // } + // parsedPGTime, err := time.Parse("2006-01-02T15:04:05.000000Z07:00", pgTime) + // if err != nil { + // return nil, errors.Wrap(err, "postgres time parsing failed") + // } + + // fmt.Printf("\n\n\n\n\n %d\n", parsedPGTime.UnixNano()/int64(time.Millisecond)) + // time.Sleep(2 * time.Second) + // fmt.Printf("%d \n\n\n\n\n", time.Now().UnixNano()/int64(time.Millisecond)) + return &PGStatMonitorQAN{ q: q, dbCloser: dbCloser, @@ -341,6 +352,28 @@ func (m *PGStatMonitorQAN) getNewBuckets(ctx context.Context, periodLengthSecs u return nil, err } + row := &pgStatMonitorErrors{} + rows, err := m.q.SelectRows(pgStatMonitorErrorsView, "") + //errors, err := m.q.SelectAllFrom(pgStatMonitorErrorsView, "") + //fmt.Printf("\n\n\n\n\n\n\n\n err:%+v \n\n\n\n\n\n\n\n", errors) + if err != nil { + return nil, err + } + + // for _, error := range errors { + // fmt.Println(error) + // } + + for ctx.Err() == nil { + if err = m.q.NextRow(row, rows); err != nil { + if errors.Is(err, reform.ErrNoRows) { + err = nil + } + break + } + fmt.Println(row.MessageTime) + } + buckets := m.makeBuckets(current, prev) m.l.Debugf("Made %d buckets out of %d stat monitor in %d interval.", len(buckets), len(current), periodLengthSecs) @@ -365,6 +398,7 @@ func (m *PGStatMonitorQAN) getNewBuckets(ctx context.Context, periodLengthSecs u func (m *PGStatMonitorQAN) makeBuckets(current, cache map[time.Time]map[string]*pgStatMonitorExtended) []*agentpb.MetricsBucket { res := make([]*agentpb.MetricsBucket, 0, len(current)) + //var errors []reform.Struct for bucketStartTime, bucket := range current { prev := cache[bucketStartTime] for queryID, currentPSM := range bucket { From 702f78da30604057725387c878a927b279ffac01 Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Fri, 29 Apr 2022 12:55:08 +0200 Subject: [PATCH 11/19] PMM-9875 More severity levels. --- .../postgres/pgstatmonitor/pgstatmonitor.go | 45 +++++++++---------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/agents/postgres/pgstatmonitor/pgstatmonitor.go b/agents/postgres/pgstatmonitor/pgstatmonitor.go index 826a151df..7beffc493 100644 --- a/agents/postgres/pgstatmonitor/pgstatmonitor.go +++ b/agents/postgres/pgstatmonitor/pgstatmonitor.go @@ -207,20 +207,6 @@ func newPgStatMonitorQAN(q *reform.Querier, dbCloser io.Closer, agentID string, } } - // var pgTime string - // err = q.QueryRow(fmt.Sprintf("SELECT /* %s */ NOW()", queryTag)).Scan(&pgTime) - // if err != nil { - // return nil, errors.Wrap(err, "postgres time parsing failed") - // } - // parsedPGTime, err := time.Parse("2006-01-02T15:04:05.000000Z07:00", pgTime) - // if err != nil { - // return nil, errors.Wrap(err, "postgres time parsing failed") - // } - - // fmt.Printf("\n\n\n\n\n %d\n", parsedPGTime.UnixNano()/int64(time.Millisecond)) - // time.Sleep(2 * time.Second) - // fmt.Printf("%d \n\n\n\n\n", time.Now().UnixNano()/int64(time.Millisecond)) - return &PGStatMonitorQAN{ q: q, dbCloser: dbCloser, @@ -354,24 +340,37 @@ func (m *PGStatMonitorQAN) getNewBuckets(ctx context.Context, periodLengthSecs u row := &pgStatMonitorErrors{} rows, err := m.q.SelectRows(pgStatMonitorErrorsView, "") - //errors, err := m.q.SelectAllFrom(pgStatMonitorErrorsView, "") - //fmt.Printf("\n\n\n\n\n\n\n\n err:%+v \n\n\n\n\n\n\n\n", errors) if err != nil { return nil, err } - // for _, error := range errors { - // fmt.Println(error) - // } - + now := time.Now().UTC() for ctx.Err() == nil { if err = m.q.NextRow(row, rows); err != nil { if errors.Is(err, reform.ErrNoRows) { - err = nil + break } - break + + return nil, err + } + + messageTime, err := time.Parse("2006-01-02 15:04:05", row.MessageTime) + if err != nil { + return nil, err + } + if now.After(messageTime) { + continue + } + + template := "Message: %s, Calls: %d" + switch row.Severity { + case "INFO": + m.l.Infof(template, row.Message, row.Calls) + case "WARNING": + m.l.Warningf(template, row.Message, row.Calls) + case "ERROR": + m.l.Errorf(template, row.Message, row.Calls) } - fmt.Println(row.MessageTime) } buckets := m.makeBuckets(current, prev) From 88e51fa93e726c990e13fdb74882f277e51227d7 Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Fri, 29 Apr 2022 13:04:35 +0200 Subject: [PATCH 12/19] PMM-9875 Remove UTC. --- agents/postgres/pgstatmonitor/pgstatmonitor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/postgres/pgstatmonitor/pgstatmonitor.go b/agents/postgres/pgstatmonitor/pgstatmonitor.go index 7beffc493..d53474d46 100644 --- a/agents/postgres/pgstatmonitor/pgstatmonitor.go +++ b/agents/postgres/pgstatmonitor/pgstatmonitor.go @@ -344,7 +344,7 @@ func (m *PGStatMonitorQAN) getNewBuckets(ctx context.Context, periodLengthSecs u return nil, err } - now := time.Now().UTC() + now := time.Now() for ctx.Err() == nil { if err = m.q.NextRow(row, rows); err != nil { if errors.Is(err, reform.ErrNoRows) { From f6619355d628f371e30fe0c09efbaeb8d6160a9a Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Wed, 27 Apr 2022 14:48:34 +0200 Subject: [PATCH 13/19] Revert "Merge branch 'PMM-9630-status-code-usage' into PMM-9875-errors-view-usage" This reverts commit 1a4397528b57be80f821526a0d74d35fbdc8f3bb. --- agents/postgres/pgstatmonitor/pgstatmonitor.go | 4 ++-- .../pgstatmonitor/pgstatmonitor_models.go | 17 ++++++++++++++--- .../pgstatmonitor/pgstatmonitor_test.go | 2 +- .../pgstatmonitor/stat_monitor_cache.go | 7 +++---- docker-compose-pg-load.yml | 5 ++--- docker-compose.yml | 1 + 6 files changed, 23 insertions(+), 13 deletions(-) diff --git a/agents/postgres/pgstatmonitor/pgstatmonitor.go b/agents/postgres/pgstatmonitor/pgstatmonitor.go index d53474d46..51240b4ed 100644 --- a/agents/postgres/pgstatmonitor/pgstatmonitor.go +++ b/agents/postgres/pgstatmonitor/pgstatmonitor.go @@ -93,7 +93,7 @@ const ( commandTypeUpdate = "UPDATE" commandTypeInsert = "INSERT" commandTypeDelete = "DELETE" - commandTypeUtility = "UTILITY" + commandTypeUtiity = "UTILITY" ) var commandTypeToText = []string{ @@ -102,7 +102,7 @@ var commandTypeToText = []string{ commandTypeUpdate, commandTypeInsert, commandTypeDelete, - commandTypeUtility, + commandTypeUtiity, commandTextNotAvailable, } diff --git a/agents/postgres/pgstatmonitor/pgstatmonitor_models.go b/agents/postgres/pgstatmonitor/pgstatmonitor_models.go index 4776df7bd..f34b1922b 100644 --- a/agents/postgres/pgstatmonitor/pgstatmonitor_models.go +++ b/agents/postgres/pgstatmonitor/pgstatmonitor_models.go @@ -28,7 +28,7 @@ import ( ) var ( - v10 = version.Must(version.NewVersion("1.0.0")) + v10 = version.Must(version.NewVersion("1.0.0-beta-2")) v09 = version.Must(version.NewVersion("0.9")) v08 = version.Must(version.NewVersion("0.8")) ) @@ -83,6 +83,13 @@ type pgStatMonitor struct { WalRecords int64 WalFpi int64 WalBytes int64 + // state_code = 0 state 'PARSING' + // state_code = 1 state 'PLANNING' + // state_code = 2 state 'ACTIVE' + // state_code = 3 state 'FINISHED' + // state_code = 4 state 'FINISHED WITH ERROR' + StateCode int64 + State string // < pg0.6 @@ -173,7 +180,9 @@ func NewPgStatMonitorStructs(v pgStatMonitorVersion) (*pgStatMonitor, reform.Vie field{info: parse.FieldInfo{Name: "Message", Type: "*string", Column: "message"}, pointer: &s.Message}, field{info: parse.FieldInfo{Name: "WalRecords", Type: "int64", Column: "wal_records"}, pointer: &s.WalRecords}, field{info: parse.FieldInfo{Name: "WalFpi", Type: "int64", Column: "wal_fpi"}, pointer: &s.WalFpi}, - field{info: parse.FieldInfo{Name: "WalBytes", Type: "int64", Column: "wal_bytes"}, pointer: &s.WalBytes}) + field{info: parse.FieldInfo{Name: "WalBytes", Type: "int64", Column: "wal_bytes"}, pointer: &s.WalBytes}, + field{info: parse.FieldInfo{Name: "StateCode", Type: "int64", Column: "state_code"}, pointer: &s.StateCode}, + field{info: parse.FieldInfo{Name: "State", Type: "string", Column: "state"}, pointer: &s.State}) } if v <= pgStatMonitorVersion10PG12 { @@ -250,7 +259,7 @@ func (v *pgStatMonitorAllViewType) NewStruct() reform.Struct { // String returns a string representation of this struct or record. func (s pgStatMonitor) String() string { - res := make([]string, 49) + res := make([]string, 51) res[0] = "Bucket: " + reform.Inspect(s.Bucket, true) res[1] = "BucketStartTime: " + reform.Inspect(s.BucketStartTime, true) res[2] = "UserID: " + reform.Inspect(s.UserID, true) @@ -300,6 +309,8 @@ func (s pgStatMonitor) String() string { res[46] = "WalRecords: " + reform.Inspect(s.WalRecords, true) res[47] = "WalFpi: " + reform.Inspect(s.WalFpi, true) res[48] = "WalBytes: " + reform.Inspect(s.WalBytes, true) + res[49] = "StateCode: " + reform.Inspect(s.StateCode, true) + res[50] = "State: " + reform.Inspect(s.State, true) return strings.Join(res, ", ") } diff --git a/agents/postgres/pgstatmonitor/pgstatmonitor_test.go b/agents/postgres/pgstatmonitor/pgstatmonitor_test.go index 9827be1e0..2003dd97e 100644 --- a/agents/postgres/pgstatmonitor/pgstatmonitor_test.go +++ b/agents/postgres/pgstatmonitor/pgstatmonitor_test.go @@ -92,7 +92,7 @@ func filter(mb []*agentpb.MetricsBucket) []*agentpb.MetricsBucket { func TestVersion(t *testing.T) { pgsmVersion, err := ver.NewVersion("1.0.0-beta-2") require.NoError(t, err) - require.True(t, pgsmVersion.LessThan(v10)) + require.True(t, pgsmVersion.GreaterThanOrEqual(v10)) } func TestPGStatMonitorSchema(t *testing.T) { diff --git a/agents/postgres/pgstatmonitor/stat_monitor_cache.go b/agents/postgres/pgstatmonitor/stat_monitor_cache.go index 52084e416..aa63d0ec1 100644 --- a/agents/postgres/pgstatmonitor/stat_monitor_cache.go +++ b/agents/postgres/pgstatmonitor/stat_monitor_cache.go @@ -83,7 +83,7 @@ func (ssc *statMonitorCache) getStatMonitorExtended(ctx context.Context, q *refo databases := queryDatabases(q) usernames := queryUsernames(q) - pgMonitorVersion, prerelease, err := getPGMonitorVersion(q) + pgMonitorVersion, _, err := getPGMonitorVersion(q) if err != nil { err = errors.Wrap(err, "failed to get row and view for pg_stat_monitor version") return @@ -92,11 +92,10 @@ func (ssc *statMonitorCache) getStatMonitorExtended(ctx context.Context, q *refo row, view := NewPgStatMonitorStructs(pgMonitorVersion) conditions := "WHERE queryid IS NOT NULL AND query IS NOT NULL" - if pgMonitorVersion >= pgStatMonitorVersion09 && pgMonitorVersion <= pgStatMonitorVersion10PG14 && prerelease != "" { - // only pg_stat_monitor 0.9.0, 1.0.0-beta-2, 1.0.0-rc.1, 1.0.0-rc.2 supports state_code. It tells what is the query's current state. + if pgMonitorVersion >= pgStatMonitorVersion09 { + // only pg_stat_monitor 0.9.0 and above supports state_code. It tells what is the query's current state. // To have correct data in QAN, we have to get only queries that are either 'FINISHED' or 'FINISHED WITH ERROR'. conditions += " AND (state_code = 3 OR state_code = 4)" - ssc.l.Debug("PGSM version with state and state_code") } rows, e := q.SelectRows(view, conditions) if e != nil { diff --git a/docker-compose-pg-load.yml b/docker-compose-pg-load.yml index 5ff2a4b57..76519d86d 100644 --- a/docker-compose-pg-load.yml +++ b/docker-compose-pg-load.yml @@ -4,14 +4,13 @@ version: '3.7' services: postgres-pgmonitor: - image: ${POSTGRES_IMAGE:-perconalab/percona-distribution-postgresql:14.1} + image: ${POSTGRES_IMAGE:-perconalab/percona-distribution-postgresql:13.3} container_name: pmm-agent-postgres-pgmonitor command: > -c shared_preload_libraries=pg_stat_monitor -c track_activity_query_size=2048 -c pg_stat_monitor.pgsm_query_max_len=10000 -c pg_stat_monitor.pgsm_normalized_query=0 - -c pg_stat_monitor.pgsm_enable_query_plan=1 -c track_io_timing=on ports: - 127.0.0.1:5432:5432 @@ -22,7 +21,7 @@ services: - test_db_postgres:/docker-entrypoint-initdb.d/ postgres-load: - image: ${POSTGRES_IMAGE:-perconalab/percona-distribution-postgresql:14.1} + image: ${POSTGRES_IMAGE:-perconalab/percona-distribution-postgresql:13.3} container_name: pmm-agent-postgres-load depends_on: - postgres-pgmonitor diff --git a/docker-compose.yml b/docker-compose.yml index a0cd4d02a..6477bd61c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -118,6 +118,7 @@ services: -c shared_preload_libraries='${PG_PRELOADED_LIBS:-pg_stat_statements}' -c track_activity_query_size=2048 -c pg_stat_statements.max=10000 + -c pg_stat_monitor.pgsm_query_max_len=10000 -c pg_stat_statements.track=all -c pg_stat_statements.save=off -c track_io_timing=on From 063a7bfacf2dc47cf7dd796e017c0f7b4b4e2a0f Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Fri, 29 Apr 2022 13:06:55 +0200 Subject: [PATCH 14/19] PMM-9875 Remove comment. --- agents/postgres/pgstatmonitor/pgstatmonitor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/agents/postgres/pgstatmonitor/pgstatmonitor.go b/agents/postgres/pgstatmonitor/pgstatmonitor.go index 51240b4ed..b7547cfbc 100644 --- a/agents/postgres/pgstatmonitor/pgstatmonitor.go +++ b/agents/postgres/pgstatmonitor/pgstatmonitor.go @@ -397,7 +397,6 @@ func (m *PGStatMonitorQAN) getNewBuckets(ctx context.Context, periodLengthSecs u func (m *PGStatMonitorQAN) makeBuckets(current, cache map[time.Time]map[string]*pgStatMonitorExtended) []*agentpb.MetricsBucket { res := make([]*agentpb.MetricsBucket, 0, len(current)) - //var errors []reform.Struct for bucketStartTime, bucket := range current { prev := cache[bucketStartTime] for queryID, currentPSM := range bucket { From a1f6426c885593ca8870eb1f7c442e2244f5bdf8 Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Fri, 29 Apr 2022 13:07:54 +0200 Subject: [PATCH 15/19] PMM-9875 Typo. --- agents/postgres/pgstatmonitor/pgstatmonitor.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/agents/postgres/pgstatmonitor/pgstatmonitor.go b/agents/postgres/pgstatmonitor/pgstatmonitor.go index b7547cfbc..1423bd17a 100644 --- a/agents/postgres/pgstatmonitor/pgstatmonitor.go +++ b/agents/postgres/pgstatmonitor/pgstatmonitor.go @@ -59,7 +59,6 @@ type PGStatMonitorQAN struct { // options. pgsmNormalizedQuery bool waitTime time.Duration - timeDiff time.Duration disableQueryExamples bool } @@ -93,7 +92,7 @@ const ( commandTypeUpdate = "UPDATE" commandTypeInsert = "INSERT" commandTypeDelete = "DELETE" - commandTypeUtiity = "UTILITY" + commandTypeUtility = "UTILITY" ) var commandTypeToText = []string{ @@ -102,7 +101,7 @@ var commandTypeToText = []string{ commandTypeUpdate, commandTypeInsert, commandTypeDelete, - commandTypeUtiity, + commandTypeUtility, commandTextNotAvailable, } From 669d8cacee9af57eddbbbe10da9de6c28223ef84 Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Fri, 29 Apr 2022 13:17:23 +0200 Subject: [PATCH 16/19] PMM-9875 Lint. --- agents/postgres/pgstatmonitor/pgstatmonitor.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/agents/postgres/pgstatmonitor/pgstatmonitor.go b/agents/postgres/pgstatmonitor/pgstatmonitor.go index 1423bd17a..bc2a3e728 100644 --- a/agents/postgres/pgstatmonitor/pgstatmonitor.go +++ b/agents/postgres/pgstatmonitor/pgstatmonitor.go @@ -340,7 +340,7 @@ func (m *PGStatMonitorQAN) getNewBuckets(ctx context.Context, periodLengthSecs u row := &pgStatMonitorErrors{} rows, err := m.q.SelectRows(pgStatMonitorErrorsView, "") if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to query pg_stat_monitor_errors view") } now := time.Now() @@ -350,12 +350,12 @@ func (m *PGStatMonitorQAN) getNewBuckets(ctx context.Context, periodLengthSecs u break } - return nil, err + return nil, errors.Wrap(err, "cannot read row from errors view") } messageTime, err := time.Parse("2006-01-02 15:04:05", row.MessageTime) if err != nil { - return nil, err + return nil, errors.Wrap(err, "cannot parse messageTime") } if now.After(messageTime) { continue From ff56485c359ab5b28da4ec94fb0d05fb7ed7ec3a Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Fri, 29 Apr 2022 15:00:59 +0200 Subject: [PATCH 17/19] PMM-9875 Move block into method. --- .../postgres/pgstatmonitor/pgstatmonitor.go | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/agents/postgres/pgstatmonitor/pgstatmonitor.go b/agents/postgres/pgstatmonitor/pgstatmonitor.go index bc2a3e728..041c49168 100644 --- a/agents/postgres/pgstatmonitor/pgstatmonitor.go +++ b/agents/postgres/pgstatmonitor/pgstatmonitor.go @@ -337,10 +337,32 @@ func (m *PGStatMonitorQAN) getNewBuckets(ctx context.Context, periodLengthSecs u return nil, err } + m.checkErrorsView(ctx) + + buckets := m.makeBuckets(current, prev) + m.l.Debugf("Made %d buckets out of %d stat monitor in %d interval.", + len(buckets), len(current), periodLengthSecs) + + // merge prev and current in cache + m.monitorCache.refresh(current) + m.l.Debugf("statMonitorCache: %s", m.monitorCache.stats()) + + // add agent_id and timestamps + for i, b := range buckets { + b.Common.AgentId = m.agentID + b.Common.PeriodLengthSecs = periodLengthSecs + + buckets[i] = b + } + + return buckets, nil +} + +func (m *PGStatMonitorQAN) checkErrorsView(ctx context.Context) error { row := &pgStatMonitorErrors{} rows, err := m.q.SelectRows(pgStatMonitorErrorsView, "") if err != nil { - return nil, errors.Wrap(err, "failed to query pg_stat_monitor_errors view") + return errors.Wrap(err, "failed to query pg_stat_monitor_errors view") } now := time.Now() @@ -350,12 +372,12 @@ func (m *PGStatMonitorQAN) getNewBuckets(ctx context.Context, periodLengthSecs u break } - return nil, errors.Wrap(err, "cannot read row from errors view") + return errors.Wrap(err, "cannot read row from errors view") } messageTime, err := time.Parse("2006-01-02 15:04:05", row.MessageTime) if err != nil { - return nil, errors.Wrap(err, "cannot parse messageTime") + return errors.Wrap(err, "cannot parse messageTime") } if now.After(messageTime) { continue @@ -371,24 +393,6 @@ func (m *PGStatMonitorQAN) getNewBuckets(ctx context.Context, periodLengthSecs u m.l.Errorf(template, row.Message, row.Calls) } } - - buckets := m.makeBuckets(current, prev) - m.l.Debugf("Made %d buckets out of %d stat monitor in %d interval.", - len(buckets), len(current), periodLengthSecs) - - // merge prev and current in cache - m.monitorCache.refresh(current) - m.l.Debugf("statMonitorCache: %s", m.monitorCache.stats()) - - // add agent_id and timestamps - for i, b := range buckets { - b.Common.AgentId = m.agentID - b.Common.PeriodLengthSecs = periodLengthSecs - - buckets[i] = b - } - - return buckets, nil } // makeBuckets uses current state of pg_stat_monitor table and accumulated previous state From 1f9aa6c64dc958c9993a577f780c9fa6e81da449 Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Fri, 29 Apr 2022 15:02:23 +0200 Subject: [PATCH 18/19] PMM-9875 Add missing return. --- agents/postgres/pgstatmonitor/pgstatmonitor.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agents/postgres/pgstatmonitor/pgstatmonitor.go b/agents/postgres/pgstatmonitor/pgstatmonitor.go index 041c49168..77494c49c 100644 --- a/agents/postgres/pgstatmonitor/pgstatmonitor.go +++ b/agents/postgres/pgstatmonitor/pgstatmonitor.go @@ -393,6 +393,8 @@ func (m *PGStatMonitorQAN) checkErrorsView(ctx context.Context) error { m.l.Errorf(template, row.Message, row.Calls) } } + + return nil } // makeBuckets uses current state of pg_stat_monitor table and accumulated previous state From ba4a2a08da49e55ef8897cfabb07b484524fdf2f Mon Sep 17 00:00:00 2001 From: Jiri Ctvrtka Date: Fri, 29 Apr 2022 15:49:38 +0200 Subject: [PATCH 19/19] PMM-9875 Lint. --- agents/postgres/pgstatmonitor/pgstatmonitor.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/agents/postgres/pgstatmonitor/pgstatmonitor.go b/agents/postgres/pgstatmonitor/pgstatmonitor.go index 77494c49c..e4d2bb1a7 100644 --- a/agents/postgres/pgstatmonitor/pgstatmonitor.go +++ b/agents/postgres/pgstatmonitor/pgstatmonitor.go @@ -337,7 +337,10 @@ func (m *PGStatMonitorQAN) getNewBuckets(ctx context.Context, periodLengthSecs u return nil, err } - m.checkErrorsView(ctx) + err = m.checkErrorsView(ctx) + if err != nil { + return nil, err + } buckets := m.makeBuckets(current, prev) m.l.Debugf("Made %d buckets out of %d stat monitor in %d interval.",