From b8e3992244f3ee7d368d77c8d75264d110a3c064 Mon Sep 17 00:00:00 2001 From: gsidhwani_nr Date: Mon, 18 Nov 2024 15:59:37 +0530 Subject: [PATCH 1/4] feat: configurable custom fields as a main attributes or seprated custom subfields attributes --- custom-log4j2-appender/build-jar.gradle | 4 +- custom-log4j2-appender/build-shadowJar.gradle | 4 +- custom-log4j2-appender/build.gradle | 4 +- .../publications/mavenJava/pom-default.xml | 60 ---------------- custom-log4j2-appender/publish-jar-legacy.sh | 2 +- custom-log4j2-appender/publish-jar.sh | 2 +- custom-log4j2-appender/publish-shadowJar.sh | 2 +- .../main/java/com/newrelic/labs/LogEntry.java | 10 +-- .../java/com/newrelic/labs/LogForwarder.java | 69 +++++++++++++++---- .../labs/NewRelicBatchingAppender.java | 47 +++++++------ 10 files changed, 94 insertions(+), 110 deletions(-) delete mode 100644 custom-log4j2-appender/build/publications/mavenJava/pom-default.xml diff --git a/custom-log4j2-appender/build-jar.gradle b/custom-log4j2-appender/build-jar.gradle index 897e6b2..ca0591a 100644 --- a/custom-log4j2-appender/build-jar.gradle +++ b/custom-log4j2-appender/build-jar.gradle @@ -20,7 +20,7 @@ jar { 'Implementation-Title': 'Custom Log4j2 Appender', 'Implementation-Vendor': 'New Relic Labs', 'Implementation-Vendor-Id': 'com.newrelic.labs', - 'Implementation-Version': '1.0.2' + 'Implementation-Version': '1.0.3' ) } } @@ -53,7 +53,7 @@ publishing { groupId = 'com.newrelic.labs' artifactId = 'custom-log4j2-appender' - version = '1.0.2' + version = '1.0.3' pom { name = 'Custom Log4j2 Appender' diff --git a/custom-log4j2-appender/build-shadowJar.gradle b/custom-log4j2-appender/build-shadowJar.gradle index d956b0d..b3a0e89 100644 --- a/custom-log4j2-appender/build-shadowJar.gradle +++ b/custom-log4j2-appender/build-shadowJar.gradle @@ -27,7 +27,7 @@ shadowJar { 'Implementation-Title': 'Custom Log4j2 Appender', 'Implementation-Vendor': 'New Relic Labs', 'Implementation-Vendor-Id': 'com.newrelic.labs', - 'Implementation-Version': '1.0.2' + 'Implementation-Version': '1.0.3' ) } } @@ -55,7 +55,7 @@ publishing { groupId = 'com.newrelic.labs' artifactId = 'custom-log4j2-appender' - version = '1.0.2' + version = '1.0.3' pom { name = 'Custom Log4j2 Appender' diff --git a/custom-log4j2-appender/build.gradle b/custom-log4j2-appender/build.gradle index d956b0d..b3a0e89 100644 --- a/custom-log4j2-appender/build.gradle +++ b/custom-log4j2-appender/build.gradle @@ -27,7 +27,7 @@ shadowJar { 'Implementation-Title': 'Custom Log4j2 Appender', 'Implementation-Vendor': 'New Relic Labs', 'Implementation-Vendor-Id': 'com.newrelic.labs', - 'Implementation-Version': '1.0.2' + 'Implementation-Version': '1.0.3' ) } } @@ -55,7 +55,7 @@ publishing { groupId = 'com.newrelic.labs' artifactId = 'custom-log4j2-appender' - version = '1.0.2' + version = '1.0.3' pom { name = 'Custom Log4j2 Appender' diff --git a/custom-log4j2-appender/build/publications/mavenJava/pom-default.xml b/custom-log4j2-appender/build/publications/mavenJava/pom-default.xml deleted file mode 100644 index 0a71468..0000000 --- a/custom-log4j2-appender/build/publications/mavenJava/pom-default.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - - - - - 4.0.0 - com.newrelic.labs - custom-log4j2-appender - 0.0.9 - Custom Log4j2 Appender - A custom Log4j2 appender for New Relic. - https://github.com/newrelic/newrelic-java-log4j-appender - - - The Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - - - - - newrelic - New Relic - gsidhwani@newrelic.com - - - - scm:git:git://github.com/newrelic/newrelic-java-log4j-appender.git - scm:git:ssh://github.com/newrelic/newrelic-java-log4j-appender.git - https://github.com/newrelic/newrelic-java-log4j-appender - - - - com.squareup.okhttp3 - okhttp - 4.9.3 - runtime - - - com.fasterxml.jackson.core - jackson-databind - 2.13.1 - runtime - - - org.apache.logging.log4j - log4j-core - 2.14.1 - runtime - - - org.apache.logging.log4j - log4j-api - 2.14.1 - runtime - - - diff --git a/custom-log4j2-appender/publish-jar-legacy.sh b/custom-log4j2-appender/publish-jar-legacy.sh index 4aeb966..f9583a3 100755 --- a/custom-log4j2-appender/publish-jar-legacy.sh +++ b/custom-log4j2-appender/publish-jar-legacy.sh @@ -7,7 +7,7 @@ cp build-jar.gradle build.gradle # Set variables GROUP_ID="com.newrelic.labs" ARTIFACT_ID="custom-log4j2-appender" -VERSION="1.0.2" +VERSION="1.0.3" KEY_ID="0ED9FD74E81E6D83FAE25F235640EA0B1C631C6F" # Replace with your actual key ID # Get the current directory (assuming the script is run from the custom-log4j2-appender directory) diff --git a/custom-log4j2-appender/publish-jar.sh b/custom-log4j2-appender/publish-jar.sh index a2cc941..3df1a83 100755 --- a/custom-log4j2-appender/publish-jar.sh +++ b/custom-log4j2-appender/publish-jar.sh @@ -7,7 +7,7 @@ cp build-jar.gradle build.gradle # Set variables GROUP_ID="io.github.newrelic-experimental" ARTIFACT_ID="custom-log4j2-appender" -VERSION="1.0.2" +VERSION="1.0.3" KEY_ID="0ED9FD74E81E6D83FAE25F235640EA0B1C631C6F" # Replace with your actual key ID # Get the current directory (assuming the script is run from the custom-log4j2-appender directory) diff --git a/custom-log4j2-appender/publish-shadowJar.sh b/custom-log4j2-appender/publish-shadowJar.sh index b5e3117..1ae5dbd 100755 --- a/custom-log4j2-appender/publish-shadowJar.sh +++ b/custom-log4j2-appender/publish-shadowJar.sh @@ -6,7 +6,7 @@ cp build-shadowJar.gradle build.gradle # Set variables GROUP_ID="io.github.newrelic-experimental" ARTIFACT_ID="custom-log4j2-appender" -VERSION="1.0.2" +VERSION="1.0.3" KEY_ID="0ED9FD74E81E6D83FAE25F235640EA0B1C631C6F" # Replace with your actual key ID # Get the current directory (assuming the script is run from the custom-log4j2-appender directory) diff --git a/custom-log4j2-appender/src/main/java/com/newrelic/labs/LogEntry.java b/custom-log4j2-appender/src/main/java/com/newrelic/labs/LogEntry.java index 6c20592..d398076 100644 --- a/custom-log4j2-appender/src/main/java/com/newrelic/labs/LogEntry.java +++ b/custom-log4j2-appender/src/main/java/com/newrelic/labs/LogEntry.java @@ -8,16 +8,16 @@ public class LogEntry { private final String name; private final String logtype; private final long timestamp; - private final Map custom; // Add custom fields + public LogEntry(String message, String applicationName, String name, String logtype, long timestamp, - Map custom) { + Map custom, boolean mergeCustomFields) { this.message = message; this.applicationName = applicationName; this.name = name; this.logtype = logtype; this.timestamp = timestamp; - this.custom = custom; // Initialize custom fields + } public String getMessage() { @@ -39,8 +39,4 @@ public String getLogType() { public long getTimestamp() { return timestamp; } - - public Map getcustom() { - return custom; - } } \ No newline at end of file diff --git a/custom-log4j2-appender/src/main/java/com/newrelic/labs/LogForwarder.java b/custom-log4j2-appender/src/main/java/com/newrelic/labs/LogForwarder.java index e402c33..7be7189 100644 --- a/custom-log4j2-appender/src/main/java/com/newrelic/labs/LogForwarder.java +++ b/custom-log4j2-appender/src/main/java/com/newrelic/labs/LogForwarder.java @@ -37,7 +37,7 @@ public boolean isInitialized() { return apiKey != null && apiURL != null; } - public void flush(List logEntries) { + public void flush(List logEntries, boolean mergeCustomFields, Map customFields) { InetAddress localhost = null; try { localhost = InetAddress.getLocalHost(); @@ -59,8 +59,17 @@ public void flush(List logEntries) { logEvent.put("source", "NRBatchingAppender"); // Add custom fields - if (entry.getcustom() != null) { - logEvent.put("custom", entry.getcustom()); + if (customFields != null) { + if (mergeCustomFields) { + // Traverse all keys and add each field separately + Map customFields1 = customFields; + for (Map.Entry field : customFields1.entrySet()) { + logEvent.put(field.getKey(), field.getValue()); + } + } else { + // Directly add the custom fields as a single entry + logEvent.put("custom", customFields); + } } logEvents.add(logEvent); @@ -70,7 +79,7 @@ public void flush(List logEntries) { byte[] compressedPayload = gzipCompress(jsonPayload); if (compressedPayload.length > maxMessageSize) { - splitAndSendLogs(logEntries); + splitAndSendLogs(logEntries, mergeCustomFields, customFields); } else { sendLogs(logEvents); } @@ -79,14 +88,37 @@ public void flush(List logEntries) { } } - private void splitAndSendLogs(List logEntries) throws IOException { + private void splitAndSendLogs(List logEntries, boolean mergeCustomFields, + Map customFields) throws IOException { List subBatch = new ArrayList<>(); int currentSize = 0; for (LogEntry entry : logEntries) { - String entryJson = objectMapper.writeValueAsString(entry); + Map logEvent = objectMapper.convertValue(entry, LowercaseKeyMap.class); + logEvent.put("hostname", InetAddress.getLocalHost().getHostName()); + logEvent.put("logtype", entry.getLogType()); + logEvent.put("timestamp", entry.getTimestamp()); + logEvent.put("applicationName", entry.getApplicationName()); + logEvent.put("name", entry.getName()); + logEvent.put("source", "NRBatchingAppender"); + + // Add custom fields + if (customFields != null) { + if (mergeCustomFields) { + // Traverse all keys and add each field separately + Map customFields1 = customFields; + for (Map.Entry field : customFields1.entrySet()) { + logEvent.put(field.getKey(), field.getValue()); + } + } else { + // Directly add the custom fields as a single entry + logEvent.put("custom", customFields); + } + } + + String entryJson = objectMapper.writeValueAsString(logEvent); int entrySize = gzipCompress(entryJson).length; if (currentSize + entrySize > maxMessageSize) { - sendLogs(convertToLogEvents(subBatch)); + sendLogs(convertToLogEvents(subBatch, mergeCustomFields, customFields)); subBatch.clear(); currentSize = 0; } @@ -94,11 +126,12 @@ private void splitAndSendLogs(List logEntries) throws IOException { currentSize += entrySize; } if (!subBatch.isEmpty()) { - sendLogs(convertToLogEvents(subBatch)); + sendLogs(convertToLogEvents(subBatch, mergeCustomFields, customFields)); } } - private List> convertToLogEvents(List logEntries) { + private List> convertToLogEvents(List logEntries, boolean mergeCustomFields, + Map customFields) { List> logEvents = new ArrayList<>(); try { InetAddress localhost = InetAddress.getLocalHost(); @@ -114,9 +147,17 @@ private List> convertToLogEvents(List logEntries) logEvent.put("source", "NRBatchingAppender"); // Add custom fields - // Add custom fields - if (entry.getcustom() != null) { - logEvent.put("custom", entry.getcustom()); + if (customFields != null) { + if (mergeCustomFields) { + // Traverse all keys and add each field separately + Map customFields1 = customFields; + for (Map.Entry field : customFields1.entrySet()) { + logEvent.put(field.getKey(), field.getValue()); + } + } else { + // Directly add the custom fields as a single entry + logEvent.put("custom", customFields); + } } logEvents.add(logEvent); @@ -176,12 +217,12 @@ private byte[] gzipCompress(String input) throws IOException { return bos.toByteArray(); } - public void close() { + public void close(boolean mergeCustomFields, Map customFields) { List remainingLogs = new ArrayList<>(); logQueue.drainTo(remainingLogs); if (!remainingLogs.isEmpty()) { System.out.println("Flushing remaining " + remainingLogs.size() + " log events to New Relic..."); - flush(remainingLogs); + flush(remainingLogs, mergeCustomFields, customFields); } } } diff --git a/custom-log4j2-appender/src/main/java/com/newrelic/labs/NewRelicBatchingAppender.java b/custom-log4j2-appender/src/main/java/com/newrelic/labs/NewRelicBatchingAppender.java index 6bad31b..444c098 100644 --- a/custom-log4j2-appender/src/main/java/com/newrelic/labs/NewRelicBatchingAppender.java +++ b/custom-log4j2-appender/src/main/java/com/newrelic/labs/NewRelicBatchingAppender.java @@ -5,6 +5,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -33,6 +34,7 @@ public class NewRelicBatchingAppender extends AbstractAppender { private final String apiUrl; private final String applicationName; private final String logType; + private final boolean mergeCustomFields; private final String name; private final LogForwarder logForwarder; private static final Logger logger = StatusLogger.getLogger(); @@ -40,16 +42,18 @@ public class NewRelicBatchingAppender extends AbstractAppender { private final int batchSize; private final long maxMessageSize; private final long flushInterval; - private final Map customFields; + private final Map customFields; private static final int DEFAULT_BATCH_SIZE = 5000; private static final long DEFAULT_MAX_MESSAGE_SIZE = 1048576; // 1 MB private static final long DEFAULT_FLUSH_INTERVAL = 120000; // 2 minutes private static final String LOG_TYPE = "muleLog"; // defaultType + private static final boolean MERGE_CUSTOM_FIELDS = false; // by default there will be a separate field custom block + // for custom fields i.e. custom.attribute1 protected NewRelicBatchingAppender(String name, Filter filter, Layout layout, final boolean ignoreExceptions, String apiKey, String apiUrl, String applicationName, Integer batchSize, - Long maxMessageSize, Long flushInterval, String logType, String customFields) { + Long maxMessageSize, Long flushInterval, String logType, String customFields, Boolean mergeCustomFields) { super(name, filter, layout, ignoreExceptions, Property.EMPTY_ARRAY); this.queue = new LinkedBlockingQueue<>(); this.apiKey = apiKey; @@ -61,12 +65,13 @@ protected NewRelicBatchingAppender(String name, Filter filter, Layout 0)) ? logType : LOG_TYPE; this.customFields = parsecustomFields(customFields); + this.mergeCustomFields = mergeCustomFields != null ? mergeCustomFields : MERGE_CUSTOM_FIELDS; this.logForwarder = new LogForwarder(apiKey, apiUrl, this.maxMessageSize, this.queue); startFlushingTask(); } - private Map parsecustomFields(String customFields) { - Map custom = new HashMap<>(); + private Map parsecustomFields(String customFields) { + Map custom = new HashMap<>(); if (customFields != null && !customFields.isEmpty()) { String[] pairs = customFields.split(","); for (String pair : pairs) { @@ -87,7 +92,8 @@ public static NewRelicBatchingAppender createAppender(@PluginAttribute("name") S @PluginAttribute(value = "batchSize") Integer batchSize, @PluginAttribute(value = "maxMessageSize") Long maxMessageSize, @PluginAttribute("logType") String logType, @PluginAttribute(value = "flushInterval") Long flushInterval, - @PluginAttribute("customFields") String customFields) { + @PluginAttribute("customFields") String customFields, + @PluginAttribute(value = "mergeCustomFields") Boolean mergeCustomFields) { if (name == null) { logger.error("No name provided for NewRelicBatchingAppender"); @@ -104,7 +110,7 @@ public static NewRelicBatchingAppender createAppender(@PluginAttribute("name") S } return new NewRelicBatchingAppender(name, filter, layout, true, apiKey, apiUrl, applicationName, batchSize, - maxMessageSize, flushInterval, logType, customFields); + maxMessageSize, flushInterval, logType, customFields, mergeCustomFields); } @Override @@ -127,28 +133,29 @@ public void append(LogEvent event) { // Extract custom fields from the event context Map custom = new HashMap<>(extractcustom(event)); // Add static custom fields from configuration without a prefix - for (Map.Entry entry : this.customFields.entrySet()) { + for (Entry entry : this.customFields.entrySet()) { custom.putIfAbsent(entry.getKey(), entry.getValue()); } // Directly add to the queue - queue.add(new LogEntry(message, applicationName, muleAppName, logType, timestamp, custom)); + queue.add( + new LogEntry(message, applicationName, muleAppName, logType, timestamp, custom, mergeCustomFields)); // Check if the batch size is reached and flush immediately - if (queue.size() >= batchSize) { - flushQueue(); - } + if (queue.size() >= batchSize) { + flushQueue(); + } } catch (Exception e) { logger.error("Unable to insert log entry into log queue. ", e); } } + private void flushQueue() { - List batch = new ArrayList<>(); - queue.drainTo(batch, batchSize); - if (!batch.isEmpty()) { - logger.debug("Flushing {} log entries to New Relic", batch.size()); - logForwarder.flush(batch); - } + List batch = new ArrayList<>(); + queue.drainTo(batch, batchSize); + if (!batch.isEmpty()) { + logger.debug("Flushing {} log entries to New Relic", batch.size()); + logForwarder.flush(batch, mergeCustomFields, customFields); + } } - private Map extractcustom(LogEvent event) { Map custom = new HashMap<>(); @@ -182,7 +189,7 @@ public void run() { queue.drainTo(batch, batchSize); if (!batch.isEmpty()) { logger.debug("Flushing {} log entries to New Relic", batch.size()); - logForwarder.flush(batch); + logForwarder.flush(batch, mergeCustomFields, customFields); } Thread.sleep(flushInterval); } catch (InterruptedException e) { @@ -209,7 +216,7 @@ public boolean stop(final long timeout, final TimeUnit timeUnit) { setStopping(); final boolean stopped = super.stop(timeout, timeUnit, false); try { - logForwarder.close(); + logForwarder.close(mergeCustomFields, customFields); } catch (Exception e) { logger.error("Unable to close appender", e); } From ae3ba4d13dbd7eefaa6e3f58afb434e31baf53dc Mon Sep 17 00:00:00 2001 From: Gulab Sidhwani <113113837+gsidhwani-nr@users.noreply.github.com> Date: Mon, 18 Nov 2024 17:34:37 +0530 Subject: [PATCH 2/4] Updated ReadMe for latest update. --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c7ae8bb..026bcaf 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,8 @@ Replace `[your-api-key]` with the ingest key obtained from the New Relic platfor batchSize="5000" maxMessageSize="1048576" flushInterval="120000" - customFields="businessGroup=exampleGroup,environment=production"> + customFields="businessGroup=exampleGroup,environment=production" + mergeCustomFields="true"> @@ -93,11 +94,18 @@ Replace `[your-api-key]` with the ingest key obtained from the New Relic platfor | maxMessageSize | No | 1048576 | Maximum size (in bytes) of the payload to be sent in a single HTTP request | | flushInterval | No | 120000 | Interval (in milliseconds) at which the log entries are flushed to New Relic| | customFields | No | | Add extra context to your logs with custom fields, represented as comma-separated name-value pairs.| +| mergeCustomFields | No | false | (Default: false) All custom fields will be available as `custom.feild1`, `custom.field2` else `feild1` , `feild2` will be available as the main attributes | + ## Custom Fields [ v1.0.1 + ] Custom fields provide a way to include additional custom data in your logs. They are represented as comma-separated name-value pairs. This feature allows you to add more context to your logs, making them more meaningful and easier to analyze. +## Configuring Custom Fields as Subfields of Custom Fields [v1.0.3+] +Starting from version 1.0.3, a new configuration parameter `mergeCustomFields` has been added. By default, all custom fields will be available as subfields under the `custom` field (e.g., `custom.field1`, `custom.field2`). If `mergeCustomFields` is set to `true`, custom fields will be available as main attributes (e.g., `field1`, `field2`). + + + ### TLS 1.2 Requirement New Relic only accepts connections from clients using TLS version 1.2 or greater. Ensure that your execution environment is configured to use TLS 1.2 or greater. From e1e6d002da3e158a3a628fcd6b3847c492d7410d Mon Sep 17 00:00:00 2001 From: Gulab Sidhwani <113113837+gsidhwani-nr@users.noreply.github.com> Date: Mon, 18 Nov 2024 17:41:07 +0530 Subject: [PATCH 3/4] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 026bcaf..ddf7323 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ # New Relic Log4j2 Appender -A custom Log4j2 appender that sends logs to New Relic. +A custom Log4j2 appender that sends logs to New Relic. This appender supports both plain text log messages and JSON log objects. ## Installation @@ -28,7 +28,7 @@ Add the library to your project using Maven Central: com.newrelic.labs custom-log4j2-appender - 1.0.2 + 1.0.3 ``` @@ -38,7 +38,7 @@ Or, if using a locally built JAR file: com.newrelic.labs custom-log4j2-appender - 1.0.2 + 1.0.3 system ${project.basedir}/src/main/resources/custom-log4j2-appender.jar From 5f06e2e6e2ee8c56897d8d9a08a9701c37512bb2 Mon Sep 17 00:00:00 2001 From: Gulab Sidhwani <113113837+gsidhwani-nr@users.noreply.github.com> Date: Mon, 18 Nov 2024 17:42:51 +0530 Subject: [PATCH 4/4] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ddf7323..bdd1c1e 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Replace `[your-api-key]` with the ingest key obtained from the New Relic platfor | maxMessageSize | No | 1048576 | Maximum size (in bytes) of the payload to be sent in a single HTTP request | | flushInterval | No | 120000 | Interval (in milliseconds) at which the log entries are flushed to New Relic| | customFields | No | | Add extra context to your logs with custom fields, represented as comma-separated name-value pairs.| -| mergeCustomFields | No | false | (Default: false) All custom fields will be available as `custom.feild1`, `custom.field2` else `feild1` , `feild2` will be available as the main attributes | +| mergeCustomFields | No | false | (Default: false) All custom fields will be available as `custom.field1`, `custom.field2` else `field1` , `field2` will be available as the main attributes |