Cherry-pick logging changes into release/v0.17.x branch (#703)

* Add terraform cleanup (#675) * Add logging to structured logs validator (#677) * Add logging around log query * add loging around json validation * Print entire processing report instead of messages (#679) * print entire report * add newline * Add visbility into query (#681) * add visbility into query * fix formatting * Log check result fails and improve failed schema output (#683) * add logging * Add more logging, failed results * Update validator/src/main/java/com/amazon/aoc/validators/AbstractStructuredLogValidator.java Co-authored-by: Anthony Mirabella <a9@aneurysm9.com> Co-authored-by: Anthony Mirabella <a9@aneurysm9.com> Co-authored-by: Anthony Mirabella <a9@aneurysm9.com>
aws-observability · Jun 8, 2022 · d874e67 · d874e67
1 parent c243f7a
commit d874e67
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 15 deletions.
diff --git a/validator/src/main/java/com/amazon/aoc/validators/AbstractStructuredLogValidator.java b/validator/src/main/java/com/amazon/aoc/validators/AbstractStructuredLogValidator.java
@@ -38,11 +38,7 @@
 
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 
 @Log4j2
 public abstract class AbstractStructuredLogValidator implements IValidator {
@@ -91,7 +87,6 @@ public void validate() throws Exception {
     RetryHelper.retry(getMaxRetryCount(), CHECK_INTERVAL_IN_MILLI, true, () -> {
       Instant startTime = Instant.now().minusSeconds(CHECK_DURATION_IN_SECONDS)
               .truncatedTo(ChronoUnit.MINUTES);
-
       fetchAndValidateLogs(startTime);
       checkResult();
     });
@@ -124,18 +119,23 @@ protected void fetchAndValidateLogs(Instant startTime) throws Exception {
 
   private void checkResult() throws BaseException {
     if (schemasToValidate.isEmpty() || schemasToValidate.keySet().equals(validatedSchema)) {
+      log.info(String.format("[StructuredLogValidator] log structure validation successful"
+              + "isEmpty: %b isEqual: %b",
+              schemasToValidate.isEmpty(), schemasToValidate.keySet().equals(validatedSchema)));
       return;
     }
-    String[] failedTargets = new String[schemasToValidate.size()];
-    int i = 0;
+    ArrayList<String> failedTargets = new ArrayList<>();
     for (String key : schemasToValidate.keySet()) {
-      failedTargets[i] = key;
-      i++;
+      if (!validatedSchema.contains(key)) {
+        failedTargets.add(key);
+      }
     }
+    log.info(String.format("[StructuredLogValidator] log structure validation failed for %s",
+            failedTargets));
     throw new BaseException(
             ExceptionCode.LOG_FORMAT_NOT_MATCHED,
             String.format("[StructuredLogValidator] log structure validation failed for %s",
-                    StringUtils.join(",", failedTargets)));
+                    failedTargets));
   }
 
   protected void validateJsonSchema(String logEventMsg) throws Exception {
@@ -150,6 +150,11 @@ protected void validateJsonSchema(String logEventMsg) throws Exception {
               JsonLoader.fromString(logEventNode.toString()));
       if (report.isSuccess()) {
         validatedSchema.add(key);
+      } else {
+        // This will probably generate a lot of extra logs
+        // may want to log this to a different level in the future.
+        log.info("[StructuredLogValidator] failed to validate schema \n");
+        log.info(report.toString() + "\n");
       }
     }
   }

diff --git a/...dator/src/main/java/com/amazon/aoc/validators/ContainerInsightStructuredLogValidator.java b/...dator/src/main/java/com/amazon/aoc/validators/ContainerInsightStructuredLogValidator.java
@@ -4,6 +4,7 @@
 import com.amazon.aoc.fileconfigs.LocalPathExpectedTemplate;
 import com.amazon.aoc.helpers.MustacheHelper;
 import com.amazon.aoc.models.Context;
+import com.amazonaws.AmazonClientException;
 import com.amazonaws.services.logs.model.FilteredLogEvent;
 import com.fasterxml.jackson.databind.JsonNode;
 import lombok.extern.log4j.Log4j2;
@@ -68,12 +69,25 @@ protected int getMaxRetryCount() {
   protected void fetchAndValidateLogs(Instant startTime) throws Exception {
     log.info("Fetch and validate logs with types: "
             + String.join(", ", schemasToValidate.keySet()));
+
     for (String logType : schemasToValidate.keySet()) {
       String filterPattern = String.format("{ $.Type = \"%s\"}", logType);
-      List<FilteredLogEvent> logEvents = cloudWatchService.filterLogs(logGroupName, filterPattern,
-              startTime.toEpochMilli(), QUERY_LIMIT);
-      for (FilteredLogEvent logEvent : logEvents) {
-        validateJsonSchema(logEvent.getMessage());
+      try {
+        log.info(String.format("[StructuredLogValidator] Filtering logs in log group %s"
+                + "with filter pattern %s", logGroupName, filterPattern));
+        List<FilteredLogEvent> logEvents = cloudWatchService.filterLogs(logGroupName, filterPattern,
+                startTime.toEpochMilli(), QUERY_LIMIT);
+        for (FilteredLogEvent logEvent : logEvents) {
+          validateJsonSchema(logEvent.getMessage());
+        }
+      } catch (AmazonClientException e) {
+        log.info(String.format("[StructuredLogValidator] failed to retrieve filtered logs "
+                + "in log group %s with filter pattern %s", logGroupName, filterPattern));
+        throw e;
+      } catch (Exception e) {
+        log.info(String.format("[StructuredLogValidator] error in fetch and validate logs: %s",
+                e));
+        throw e;
       }
     }
   }