From c5834604ef1ef28251e35bf570418d60995a303e Mon Sep 17 00:00:00 2001 From: lambeb <141648830+lambeb@users.noreply.github.com> Date: Mon, 2 Dec 2024 12:52:37 +0000 Subject: [PATCH 1/4] Added filter for generic_not_found error messages --- lib/filters/generic_not_found_filter.py | 23 +++ lib/send_alerts.py | 2 + .../filters/test_generic_not_found_filter.py | 154 +++++++++++++++++ tests/test_main.py | 162 ++++++++++++++++++ 4 files changed, 341 insertions(+) create mode 100644 lib/filters/generic_not_found_filter.py create mode 100644 tests/lib/filters/test_generic_not_found_filter.py diff --git a/lib/filters/generic_not_found_filter.py b/lib/filters/generic_not_found_filter.py new file mode 100644 index 0000000..9d1f1b1 --- /dev/null +++ b/lib/filters/generic_not_found_filter.py @@ -0,0 +1,23 @@ +import logging +from lib.log_processor import ProcessedLogEntry + + +def generic_not_found_filter(log_entry: ProcessedLogEntry) -> bool: + + if not isinstance(log_entry.severity, str): + return False + + if not isinstance(log_entry.message, str): + return False + + if log_entry.severity != "ERROR": + return False + + if ( + 'generic::not_found: Failed to fetch "latest' not in log_entry.message + and 'generic::not_found: Failed to fetch "version_' not in log_entry.message + ): + return False + + logging.info(f"Skipping generic not found alert") + return True diff --git a/lib/send_alerts.py b/lib/send_alerts.py index 380c853..264c0d4 100644 --- a/lib/send_alerts.py +++ b/lib/send_alerts.py @@ -32,6 +32,7 @@ from lib.filters.execute_sql_filter import execute_sql_filter from lib.filters.paramiko_filter import paramiko_filter from lib.filters.bootstrapper_filter import bootstrapper_filter +from lib.filters.generic_not_found_filter import generic_not_found_filter def log_entry_skipped(log_entry: ProcessedLogEntry): @@ -50,6 +51,7 @@ def log_entry_skipped(log_entry: ProcessedLogEntry): execute_sql_filter, paramiko_filter, bootstrapper_filter, + generic_not_found_filter, ] for filter in filters: diff --git a/tests/lib/filters/test_generic_not_found_filter.py b/tests/lib/filters/test_generic_not_found_filter.py new file mode 100644 index 0000000..615011a --- /dev/null +++ b/tests/lib/filters/test_generic_not_found_filter.py @@ -0,0 +1,154 @@ +import pytest +import datetime +import dataclasses + +from lib.log_processor import ProcessedLogEntry +from lib.filters.generic_not_found_filter import generic_not_found_filter + + +@pytest.fixture() +def processed_log_entry_generic_not_found_error_latest() -> ProcessedLogEntry: + return ProcessedLogEntry( + message='alert: ERROR: [AuditLog] generic::not_found: Failed to fetch "latest"', + data=dict(description="dummy"), + severity="ERROR", + platform="cloud_run_revision", + application="slack-alerts", + log_name="/logs/cloudfunctions", + timestamp=datetime.datetime(2024, 5, 20, 10, 23, 56, 32425), + log_query={ + "resource.type": "cloud_run_revision", + "resource.labels.instance_id": "00f46b928521d49fcdbf455e4592829a1631850562c1b37283d70572deaca72b851130f7fbca367bbb5a75b386efa9832f3d974f1a5a463b2fb9af0fb2a9c2fb4e57", + }, + ) + + +@pytest.fixture() +def processed_log_entry_generic_not_found_error_version() -> ProcessedLogEntry: + return ProcessedLogEntry( + message='alert: ERROR: [AuditLog] generic::not_found: Failed to fetch "version_255"', + data=dict(description="dummy"), + severity="ERROR", + platform="cloud_run_revision", + application="slack-alerts", + log_name="/logs/cloudfunctions", + timestamp=datetime.datetime(2024, 5, 20, 10, 23, 56, 32425), + log_query={ + "resource.type": "cloud_run_revision", + "resource.labels.instance_id": "00f46b928521d49fcdbf455e4592829a1631850562c1b37283d70572df455e4592829a1631850562c1b3725a75b386efa9832f3d974f1a5a463b2fb9af0fb2a9b4e57", + }, + ) + + +def test_log_is_not_skipped_when_its_first_run_generic_not_found_error_latest( + processed_log_entry_generic_not_found_error_latest: ProcessedLogEntry, +): + log_is_skipped = generic_not_found_filter( + processed_log_entry_generic_not_found_error_latest + ) + assert log_is_skipped is True + + +def test_log_is_not_skipped_when_its_first_run_generic_not_found_error_version( + processed_log_entry_generic_not_found_error_version: ProcessedLogEntry, +): + log_is_skipped = generic_not_found_filter( + processed_log_entry_generic_not_found_error_version + ) + assert log_is_skipped is True + + +def test_log_is_skipped_when_its_from_cloud_run_revision_when_generic_not_found_error_latest( + processed_log_entry_generic_not_found_error_latest: ProcessedLogEntry, +): + log_is_skipped = generic_not_found_filter( + processed_log_entry_generic_not_found_error_latest + ) + assert log_is_skipped is True + + +def test_log_is_skipped_when_its_from_cloud_run_revision_when_generic_not_found_error_version( + processed_log_entry_generic_not_found_error_version: ProcessedLogEntry, +): + log_is_skipped = generic_not_found_filter( + processed_log_entry_generic_not_found_error_version + ) + assert log_is_skipped is True + + +def test_log_message_is_not_a_string_when_generic_not_found_error_latest( + processed_log_entry_generic_not_found_error_latest: ProcessedLogEntry, +): + processed_log_entry_generic_not_found_error_latest = dataclasses.replace( + processed_log_entry_generic_not_found_error_latest, message=1234 + ) + log_is_skipped = generic_not_found_filter( + processed_log_entry_generic_not_found_error_latest + ) + + assert log_is_skipped is False + + +def test_log_message_is_not_a_string_when_generic_not_found_error_version( + processed_log_entry_generic_not_found_error_version: ProcessedLogEntry, +): + processed_log_entry_generic_not_found_error_version = dataclasses.replace( + processed_log_entry_generic_not_found_error_version, message=1234 + ) + log_is_skipped = generic_not_found_filter( + processed_log_entry_generic_not_found_error_version + ) + + assert log_is_skipped is False + + +def test_log_message_is_not_skipped_when_it_does_not_contain_generic_not_found_error_latest( + processed_log_entry_generic_not_found_error_latest: ProcessedLogEntry, +): + processed_log_entry_generic_not_found_error_latest = dataclasses.replace( + processed_log_entry_generic_not_found_error_latest, message="foo" + ) + log_is_skipped = generic_not_found_filter( + processed_log_entry_generic_not_found_error_latest + ) + + assert log_is_skipped is False + + +def test_log_message_is_not_skipped_when_it_does_not_contain_generic_not_found_error_version( + processed_log_entry_generic_not_found_error_version: ProcessedLogEntry, +): + processed_log_entry_generic_not_found_error_version = dataclasses.replace( + processed_log_entry_generic_not_found_error_version, message="foo" + ) + log_is_skipped = generic_not_found_filter( + processed_log_entry_generic_not_found_error_version + ) + + assert log_is_skipped is False + + +def test_log_message_is_not_skipped_when_it_contains_severity_info_latest( + processed_log_entry_generic_not_found_error_latest: ProcessedLogEntry, +): + processed_log_entry_generic_not_found_error_latest = dataclasses.replace( + processed_log_entry_generic_not_found_error_latest, severity="INFO" + ) + log_is_skipped = generic_not_found_filter( + processed_log_entry_generic_not_found_error_latest + ) + + assert log_is_skipped is False + + +def test_log_message_is_not_skipped_when_it_contains_severity_info_version( + processed_log_entry_generic_not_found_error_version: ProcessedLogEntry, +): + processed_log_entry_generic_not_found_error_version = dataclasses.replace( + processed_log_entry_generic_not_found_error_version, severity="INFO" + ) + log_is_skipped = generic_not_found_filter( + processed_log_entry_generic_not_found_error_version + ) + + assert log_is_skipped is False diff --git a/tests/test_main.py b/tests/test_main.py index 6f13d91..db753a0 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1480,3 +1480,165 @@ def test_skip_bootstrapper_alerts(run_slack_alerter, number_of_http_calls, caplo logging.INFO, "Skipping bootstrapper alert", ) in caplog.record_tuples + + +def test_skip_generic_not_found_alerts_latest( + run_slack_alerter, number_of_http_calls, caplog +): + # arrange + example_log_entry = { + "protoPayload": { + "@type": "type.googleapis.com/google.cloud.audit.AuditLog", + "status": { + "code": 5, + "message": 'generic::not_found: Failed to fetch "latest"', + }, + "authenticationInfo": { + "principalEmail": "628324858917@cloudbuild.gserviceaccount.com", + "serviceAccountDelegationInfo": [ + { + "firstPartyPrincipal": { + "principalEmail": "cloud-build-argo-foreman@prod.google.com" + } + } + ], + "principalSubject": "serviceAccount:628324858917@cloudbuild.gserviceaccount.com", + }, + "requestMetadata": { + "callerIp": "34.89.11.189", + "callerSuppliedUserAgent": "go-containerregistry,gzip(gfe)", + "requestAttributes": {}, + "destinationAttributes": {}, + }, + "serviceName": "artifactregistry.googleapis.com", + "methodName": "Docker-HeadManifest", + "authorizationInfo": [ + { + "resource": "projects/ons-blaise-v2-prod/locations/europe-west2/repositories/gcf-artifacts", + "permission": "artifactregistry.repositories.downloadArtifacts", + "granted": True, + "resourceAttributes": {}, + "permissionType": "DATA_READ", + } + ], + "resourceName": "projects/ons-blaise-v2-prod/locations/europe-west2/repositories/gcf-artifacts/dockerImages/ons--blaise--v2--prod__europe--west2__nifi--receipt%2Fcache", + "request": { + "@type": "type.googleapis.com/google.logging.type.HttpRequest", + "requestUrl": "/v2/ons-blaise-v2-prod/gcf-artifacts/ons--blaise--v2--prod__europe--west2__nifi--receipt/cache/manifests/latest", + "requestMethod": "HEAD", + }, + "resourceLocation": { + "currentLocations": ["europe-west2"], + "originalLocations": ["europe-west2"], + }, + }, + "insertId": "1h15w4udgp0q", + "resource": { + "type": "audited_resource", + "labels": { + "project_id": "ons-blaise-v2-prod", + "service": "artifactregistry.googleapis.com", + "method": "Docker-HeadManifest", + }, + }, + "timestamp": "2024-12-02T12:01:35.011476126Z", + "severity": "ERROR", + "logName": "projects/ons-blaise-v2-prod/logs/cloudaudit.googleapis.com%2Fdata_access", + "receiveTimestamp": "2024-12-02T12:01:35.148295977Z", + } + + event = create_event(example_log_entry) + + # act + with caplog.at_level(logging.INFO): + response = run_slack_alerter(event) + + # assert + assert response == "Alert skipped" + assert number_of_http_calls() == 0 + assert ( + "root", + logging.INFO, + "Skipping generic not found alert", + ) in caplog.record_tuples + + +def test_skip_generic_not_found_alerts_version( + run_slack_alerter, number_of_http_calls, caplog +): + # arrange + example_log_entry = { + "protoPayload": { + "@type": "type.googleapis.com/google.cloud.audit.AuditLog", + "status": { + "code": 5, + "message": 'generic::not_found: Failed to fetch "version_1"', + }, + "authenticationInfo": { + "principalEmail": "628324858917@cloudbuild.gserviceaccount.com", + "serviceAccountDelegationInfo": [ + { + "firstPartyPrincipal": { + "principalEmail": "cloud-build-argo-foreman@prod.google.com" + } + } + ], + "principalSubject": "serviceAccount:628324858917@cloudbuild.gserviceaccount.com", + }, + "requestMetadata": { + "callerIp": "34.89.11.189", + "callerSuppliedUserAgent": "go-containerregistry/v0.19.1,gzip(gfe)", + "requestAttributes": {}, + "destinationAttributes": {}, + }, + "serviceName": "artifactregistry.googleapis.com", + "methodName": "Docker-HeadManifest", + "authorizationInfo": [ + { + "resource": "projects/ons-blaise-v2-prod/locations/europe-west2/repositories/gcf-artifacts", + "permission": "artifactregistry.repositories.downloadArtifacts", + "granted": True, + "resourceAttributes": {}, + "permissionType": "DATA_READ", + } + ], + "resourceName": "projects/ons-blaise-v2-prod/locations/europe-west2/repositories/gcf-artifacts/dockerImages/ons--blaise--v2--prod__europe--west2__nifi--receipt", + "request": { + "requestMethod": "HEAD", + "@type": "type.googleapis.com/google.logging.type.HttpRequest", + "requestUrl": "/v2/ons-blaise-v2-prod/gcf-artifacts/ons--blaise--v2--prod__europe--west2__nifi--receipt/manifests/version_1", + }, + "resourceLocation": { + "currentLocations": ["europe-west2"], + "originalLocations": ["europe-west2"], + }, + }, + "insertId": "1snjiu3dbkh4", + "resource": { + "type": "audited_resource", + "labels": { + "method": "Docker-HeadManifest", + "service": "artifactregistry.googleapis.com", + "project_id": "ons-blaise-v2-prod", + }, + }, + "timestamp": "2024-12-02T12:01:36.494768789Z", + "severity": "ERROR", + "logName": "projects/ons-blaise-v2-prod/logs/cloudaudit.googleapis.com%2Fdata_access", + "receiveTimestamp": "2024-12-02T12:01:36.786461532Z", + } + + event = create_event(example_log_entry) + + # act + with caplog.at_level(logging.INFO): + response = run_slack_alerter(event) + + # assert + assert response == "Alert skipped" + assert number_of_http_calls() == 0 + assert ( + "root", + logging.INFO, + "Skipping generic not found alert", + ) in caplog.record_tuples From 42ecaca7c8559752be81d72a7288ffc81640bb31 Mon Sep 17 00:00:00 2001 From: lambeb <141648830+lambeb@users.noreply.github.com> Date: Mon, 2 Dec 2024 14:10:31 +0000 Subject: [PATCH 2/4] Want to test sandbox alerts in sandbox, will put back in after --- lib/send_alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/send_alerts.py b/lib/send_alerts.py index 264c0d4..220eee7 100644 --- a/lib/send_alerts.py +++ b/lib/send_alerts.py @@ -37,7 +37,7 @@ def log_entry_skipped(log_entry: ProcessedLogEntry): filters = [ - sandbox_filter, + # sandbox_filter, all_preprod_and_training_alerts_except_erroneous_questionnaire_filter, osconfig_agent_filter, auditlog_filter, From b815735c64086ff97af01321253071bb403cc9b1 Mon Sep 17 00:00:00 2001 From: lambeb <141648830+lambeb@users.noreply.github.com> Date: Mon, 2 Dec 2024 14:13:20 +0000 Subject: [PATCH 3/4] Want to test sandbox alerts in sandbox, will put back in after --- tests/test_main.py | 94 +++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index db753a0..f0664d5 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -941,53 +941,53 @@ def test_skip_watching_ip_space_exhausted_error( "Skipping ip space exhausted alert", ) in caplog.record_tuples - -def test_skip_sandbox_alerts_skips_alerts_for_sandboxes( - run_slack_alerter, number_of_http_calls, caplog -): - # arrange - example_log_entry = { - "insertId": "65675a1e000906c02cfcdb54", - "jsonPayload": { - "logName": "projects/ons-blaise-v2-dev-jw09/logs/%40google-cloud%2Fprofiler", - "message": "Successfully collected profile HEAP.", - "resource": { - "type": "gae_app", - "labels": { - "version_id": "20231129t144628", - "module_id": "dqs-ui", - "zone": "europe-west2-1", - }, - }, - "timestamp": "2023-11-29T15:34:54.591Z", - }, - "resource": { - "type": "gae_app", - "labels": { - "module_id": "dqs-ui", - "zone": "europe-west2-1", - "project_id": "ons-blaise-v2-dev-jw09", - "version_id": "20231129t144628", - }, - }, - "timestamp": "2023-11-29T15:34:54.591552Z", - "severity": "DEBUG", - "labels": { - "clone_id": "0087599d4250c01bc120294e520c07b780b217e53173b5358cac87748d40d22082f17f1f7fa68823b4a41fe1f57308d3702a9095b6b347e32e672d8952a88afb65" - }, - "logName": "projects/ons-blaise-v2-dev-jw09/logs/stdout", - "receiveTimestamp": "2023-11-29T15:34:54.921342975Z", - } - event = create_event(example_log_entry) - - # act - with caplog.at_level(logging.INFO): - response = run_slack_alerter(event) - - # assert - assert response == "Alert skipped" - assert number_of_http_calls() == 0 - assert ("root", logging.INFO, "Skipping sandbox alert") in caplog.record_tuples +# +# def test_skip_sandbox_alerts_skips_alerts_for_sandboxes( +# run_slack_alerter, number_of_http_calls, caplog +# ): +# # arrange +# example_log_entry = { +# "insertId": "65675a1e000906c02cfcdb54", +# "jsonPayload": { +# "logName": "projects/ons-blaise-v2-dev-jw09/logs/%40google-cloud%2Fprofiler", +# "message": "Successfully collected profile HEAP.", +# "resource": { +# "type": "gae_app", +# "labels": { +# "version_id": "20231129t144628", +# "module_id": "dqs-ui", +# "zone": "europe-west2-1", +# }, +# }, +# "timestamp": "2023-11-29T15:34:54.591Z", +# }, +# "resource": { +# "type": "gae_app", +# "labels": { +# "module_id": "dqs-ui", +# "zone": "europe-west2-1", +# "project_id": "ons-blaise-v2-dev-jw09", +# "version_id": "20231129t144628", +# }, +# }, +# "timestamp": "2023-11-29T15:34:54.591552Z", +# "severity": "DEBUG", +# "labels": { +# "clone_id": "0087599d4250c01bc120294e520c07b780b217e53173b5358cac87748d40d22082f17f1f7fa68823b4a41fe1f57308d3702a9095b6b347e32e672d8952a88afb65" +# }, +# "logName": "projects/ons-blaise-v2-dev-jw09/logs/stdout", +# "receiveTimestamp": "2023-11-29T15:34:54.921342975Z", +# } +# event = create_event(example_log_entry) +# +# # act +# with caplog.at_level(logging.INFO): +# response = run_slack_alerter(event) +# +# # assert +# assert response == "Alert skipped" +# assert number_of_http_calls() == 0 +# assert ("root", logging.INFO, "Skipping sandbox alert") in caplog.record_tuples def test_skip_sandbox_alerts_does_not_skip_alerts_for_formal_environments( From eb8316c78ee0e69561cf13dd002a99d0347b570b Mon Sep 17 00:00:00 2001 From: lambeb <141648830+lambeb@users.noreply.github.com> Date: Mon, 2 Dec 2024 16:18:43 +0000 Subject: [PATCH 4/4] Putting back in code to prevent alerts on sandboxes --- lib/send_alerts.py | 2 +- tests/test_main.py | 94 +++++++++++++++++++++++----------------------- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/lib/send_alerts.py b/lib/send_alerts.py index 220eee7..264c0d4 100644 --- a/lib/send_alerts.py +++ b/lib/send_alerts.py @@ -37,7 +37,7 @@ def log_entry_skipped(log_entry: ProcessedLogEntry): filters = [ - # sandbox_filter, + sandbox_filter, all_preprod_and_training_alerts_except_erroneous_questionnaire_filter, osconfig_agent_filter, auditlog_filter, diff --git a/tests/test_main.py b/tests/test_main.py index f0664d5..db753a0 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -941,53 +941,53 @@ def test_skip_watching_ip_space_exhausted_error( "Skipping ip space exhausted alert", ) in caplog.record_tuples -# -# def test_skip_sandbox_alerts_skips_alerts_for_sandboxes( -# run_slack_alerter, number_of_http_calls, caplog -# ): -# # arrange -# example_log_entry = { -# "insertId": "65675a1e000906c02cfcdb54", -# "jsonPayload": { -# "logName": "projects/ons-blaise-v2-dev-jw09/logs/%40google-cloud%2Fprofiler", -# "message": "Successfully collected profile HEAP.", -# "resource": { -# "type": "gae_app", -# "labels": { -# "version_id": "20231129t144628", -# "module_id": "dqs-ui", -# "zone": "europe-west2-1", -# }, -# }, -# "timestamp": "2023-11-29T15:34:54.591Z", -# }, -# "resource": { -# "type": "gae_app", -# "labels": { -# "module_id": "dqs-ui", -# "zone": "europe-west2-1", -# "project_id": "ons-blaise-v2-dev-jw09", -# "version_id": "20231129t144628", -# }, -# }, -# "timestamp": "2023-11-29T15:34:54.591552Z", -# "severity": "DEBUG", -# "labels": { -# "clone_id": "0087599d4250c01bc120294e520c07b780b217e53173b5358cac87748d40d22082f17f1f7fa68823b4a41fe1f57308d3702a9095b6b347e32e672d8952a88afb65" -# }, -# "logName": "projects/ons-blaise-v2-dev-jw09/logs/stdout", -# "receiveTimestamp": "2023-11-29T15:34:54.921342975Z", -# } -# event = create_event(example_log_entry) -# -# # act -# with caplog.at_level(logging.INFO): -# response = run_slack_alerter(event) -# -# # assert -# assert response == "Alert skipped" -# assert number_of_http_calls() == 0 -# assert ("root", logging.INFO, "Skipping sandbox alert") in caplog.record_tuples + +def test_skip_sandbox_alerts_skips_alerts_for_sandboxes( + run_slack_alerter, number_of_http_calls, caplog +): + # arrange + example_log_entry = { + "insertId": "65675a1e000906c02cfcdb54", + "jsonPayload": { + "logName": "projects/ons-blaise-v2-dev-jw09/logs/%40google-cloud%2Fprofiler", + "message": "Successfully collected profile HEAP.", + "resource": { + "type": "gae_app", + "labels": { + "version_id": "20231129t144628", + "module_id": "dqs-ui", + "zone": "europe-west2-1", + }, + }, + "timestamp": "2023-11-29T15:34:54.591Z", + }, + "resource": { + "type": "gae_app", + "labels": { + "module_id": "dqs-ui", + "zone": "europe-west2-1", + "project_id": "ons-blaise-v2-dev-jw09", + "version_id": "20231129t144628", + }, + }, + "timestamp": "2023-11-29T15:34:54.591552Z", + "severity": "DEBUG", + "labels": { + "clone_id": "0087599d4250c01bc120294e520c07b780b217e53173b5358cac87748d40d22082f17f1f7fa68823b4a41fe1f57308d3702a9095b6b347e32e672d8952a88afb65" + }, + "logName": "projects/ons-blaise-v2-dev-jw09/logs/stdout", + "receiveTimestamp": "2023-11-29T15:34:54.921342975Z", + } + event = create_event(example_log_entry) + + # act + with caplog.at_level(logging.INFO): + response = run_slack_alerter(event) + + # assert + assert response == "Alert skipped" + assert number_of_http_calls() == 0 + assert ("root", logging.INFO, "Skipping sandbox alert") in caplog.record_tuples def test_skip_sandbox_alerts_does_not_skip_alerts_for_formal_environments(