From a7fa8f4f83d89e131fd2bd413f8ed8eecbcc96b6 Mon Sep 17 00:00:00 2001 From: Konstantinos Kagkelidis Date: Mon, 11 Jul 2022 18:53:34 +0300 Subject: [PATCH 01/16] ARGO-3921 Import fix in argo-streaming multijob --- .../batch_multi/src/main/java/argo/batch/MongoStatusOutput.java | 1 - 1 file changed, 1 deletion(-) diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoStatusOutput.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoStatusOutput.java index 5b33e4f9..0f543f81 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoStatusOutput.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoStatusOutput.java @@ -18,7 +18,6 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; -import jdk.nashorn.internal.parser.JSONParser; import org.apache.avro.generic.GenericData; import org.bson.types.ObjectId; import org.joda.time.DateTime; From 6f238121842cea61b4f116a988ac057fb1ac9ae8 Mon Sep 17 00:00:00 2001 From: Konstantinos Kagkelidis Date: Mon, 11 Jul 2022 19:00:48 +0300 Subject: [PATCH 02/16] ARGO-3922 Fix streaming job exception when handling ui urls with spaces --- .../main/java/argo/streaming/TrimEvent.java | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/TrimEvent.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/TrimEvent.java index e95368a6..f60c075b 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/TrimEvent.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/TrimEvent.java @@ -3,10 +3,8 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; +import java.net.URL; import java.text.ParseException; import java.util.ArrayList; import org.apache.flink.util.Collector; @@ -71,12 +69,12 @@ public void flatMap(String in, Collector out) throws Exception { json = trimEvent(json, removedFields); if (this.helpUrl != null) { String helpUrlPath = "https://" + this.helpUrl + "/" + json.get("metric").getAsString(); //the help url is in the form of helpUrl/metric e.g https://poem.egi.eu/ui/public_metrics/metricA - json.addProperty("url.help", uriString(helpUrlPath)); + json.addProperty("url.help", urlString(helpUrlPath)); } if (this.historyUrl != null) { String finalHistUrlPath = initialHistPath + json.get("endpoint_group").getAsString() + "/" + json.get("service").getAsString() + "/" + json.get("hostname").getAsString() + dateParams; - json.addProperty("url.history", uriString(finalHistUrlPath)); + json.addProperty("url.history", urlString(finalHistUrlPath)); } break; case "endpoint": @@ -87,7 +85,7 @@ public void flatMap(String in, Collector out) throws Exception { json = trimEvent(json, removedFields); if (this.historyUrl != null) { String finalHistUrlPath = initialHistPath + json.get("endpoint_group").getAsString() + "/" + json.get("service").getAsString() + "/" + json.get("hostname").getAsString() + dateParams; - json.addProperty("url.history", uriString(finalHistUrlPath)); + json.addProperty("url.history", urlString(finalHistUrlPath)); } break; @@ -98,7 +96,7 @@ public void flatMap(String in, Collector out) throws Exception { json = trimEvent(json, removedFields); if (this.historyUrl != null) { String finalHistUrlPath = initialHistPath + json.get("endpoint_group").getAsString() + "/" + json.get("service").getAsString() + dateParams; - json.addProperty("url.history", uriString(finalHistUrlPath)); + json.addProperty("url.history", urlString(finalHistUrlPath)); } break; @@ -113,7 +111,7 @@ public void flatMap(String in, Collector out) throws Exception { json = trimEvent(json, removedFields); if (this.historyUrl != null) { String finalHistUrlPath = initialHistPath + json.get("endpoint_group").getAsString() + dateParams; - json.addProperty("url.history", uriString(finalHistUrlPath)); + json.addProperty("url.history", urlString(finalHistUrlPath)); } break; @@ -151,9 +149,9 @@ private String buildDateParams(String dateStr) throws ParseException { return param; } - private String uriString(String urlpath) throws UnsupportedEncodingException, MalformedURLException, URISyntaxException { - URI uri = new URI(urlpath); - return uri.toString(); + private String urlString(String urlpath) throws MalformedURLException { + URL url = new URL(urlpath); + return url.toString(); } From bf468f55ad4831d0efe7e1cbc8cc5ddc3e79c196 Mon Sep 17 00:00:00 2001 From: cthermolia-grnet Date: Wed, 27 Jul 2022 12:37:55 +0300 Subject: [PATCH 03/16] ARGO-3943 Status extending current time appears in the timeline of EGI ALL report --- .../src/main/java/timelines/Timeline.java | 18 +++++++----------- .../src/main/java/timelines/Utils.java | 1 + .../src/test/java/timelines/TimelineTest.java | 12 ++++++------ .../src/main/java/argo/ar/CalcGroupAR.java | 8 ++++---- .../src/main/java/argo/batch/ArgoMultiJob.java | 8 ++++---- .../java/argo/batch/CalcEndpointTimeline.java | 8 ++++---- .../main/java/argo/batch/CalcPrevStatus.java | 2 +- .../test/java/argo/batch/ArgoMultiJobTest.java | 14 +++++++------- .../src/test/java/argo/batch/TestUtils.java | 8 ++++---- 9 files changed, 38 insertions(+), 41 deletions(-) diff --git a/flink_jobs_v2/Timelines/src/main/java/timelines/Timeline.java b/flink_jobs_v2/Timelines/src/main/java/timelines/Timeline.java index 7371a4d5..fb0e7e46 100644 --- a/flink_jobs_v2/Timelines/src/main/java/timelines/Timeline.java +++ b/flink_jobs_v2/Timelines/src/main/java/timelines/Timeline.java @@ -10,13 +10,9 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; import java.util.Set; import java.util.TreeMap; - import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import org.joda.time.LocalDate; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; @@ -474,14 +470,13 @@ public int opInt(int[][][] truthTable, int op, int a, int b) { * periods * */ - public void fillWithStatus(String start, String end, Integer intStatus, DateTime dtUtc) throws ParseException { + public void fillWithStatus(String start, String end, Integer intStatus, DateTime now) throws ParseException { DateTime startDay = this.date.toDateTimeAtStartOfDay(); DateTime endDay = startDay.withTime(23, 59, 59, 0); - - //for (String[] period : periods) { + DateTime startDt = Utils.convertStringtoDate("yyyy-MM-dd'T'HH:mm:ss'Z'", start); DateTime endDt = Utils.convertStringtoDate("yyyy-MM-dd'T'HH:mm:ss'Z'", end); - + DateTime floor = samples.floorKey(startDt); DateTime ceiling = samples.ceilingKey(endDt); @@ -505,17 +500,18 @@ public void fillWithStatus(String start, String end, Integer intStatus, DateTime endFloorStatus = this.samples.get(endFloor); } - if ((floor == null || !floor.equals(startDt)) && !startDt.isAfter(dtUtc)) { // if start period does not match with the initial timeline's timestamp then need to add the start to the timeline with the specified status + if ((floor == null || !floor.equals(startDt)) && !startDt.isAfter(now)) { // if start period does not match with the initial timeline's timestamp then need to add the start to the timeline with the specified status this.samples.put(startDt, intStatus); //else the timestamp already contained and in next step its status will be replaced } //if a timestamp exists after a period then this timestamp should be added with the initial status of that period taken from the initial timeline //this timestamp should not extend todays current time (in the case that the computations occur during today and before the end of the day) - if (addCeiling && !endDt.isAfter(dtUtc) && endDt.plusMinutes(1).isBefore(endDay)) { + + if (addCeiling && !endDt.isAfter(now) && endDt.plusMinutes(1).isBefore(endDay)) { this.samples.put(endDt.plusMinutes(1), endFloorStatus); } for (DateTime dt : samples.keySet()) { - if (!dt.isAfter(dtUtc) && !dt.isBefore(startDt) && !dt.isAfter(endDt)) { //if timestamps exist between the period then replace the timestamps with the specified status + if (!dt.isAfter(now) && !dt.isBefore(startDt) && !dt.isAfter(endDt)) { //if timestamps exist between the period then replace the timestamps with the specified status this.samples.replace(dt, intStatus); } } diff --git a/flink_jobs_v2/Timelines/src/main/java/timelines/Utils.java b/flink_jobs_v2/Timelines/src/main/java/timelines/Utils.java index 5b6a5d46..a766f3f0 100644 --- a/flink_jobs_v2/Timelines/src/main/java/timelines/Utils.java +++ b/flink_jobs_v2/Timelines/src/main/java/timelines/Utils.java @@ -5,6 +5,7 @@ */ package timelines; +import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; diff --git a/flink_jobs_v2/Timelines/src/test/java/timelines/TimelineTest.java b/flink_jobs_v2/Timelines/src/test/java/timelines/TimelineTest.java index afee79df..bd310108 100644 --- a/flink_jobs_v2/Timelines/src/test/java/timelines/TimelineTest.java +++ b/flink_jobs_v2/Timelines/src/test/java/timelines/TimelineTest.java @@ -361,7 +361,7 @@ public void testOpInt() throws ParseException { @Test public void testFillStatus() throws ParseException { DateTimeFormatter dtf = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); - DateTime dtUtc = new DateTime(DateTimeZone.UTC); + DateTime now = new DateTime(); Timeline instance = new Timeline(); instance.insertDateTimeStamps(createTimestampList2(), true); @@ -399,7 +399,7 @@ public void testFillStatus() throws ParseException { periods.add(period4); periods.add(period5); for (String[] period : periods) { - instance.fillWithStatus(period[0], period[1], 2, dtUtc); + instance.fillWithStatus(period[0], period[1], 2, now); instance.optimize(); } TreeMap expResult = createUnknownTimeline(); @@ -441,7 +441,7 @@ public void testFillStatus() throws ParseException { instance = new Timeline(); instance.insertDateTimeStamps(createTimestampList2(), true); for (String[] period : periods) { - instance.fillWithStatus(period[0], period[1], 2, dtUtc); + instance.fillWithStatus(period[0], period[1], 2, now); instance.optimize(); } @@ -484,7 +484,7 @@ public void testFillStatus() throws ParseException { instance = new Timeline(); instance.insertDateTimeStamps(createTimestampList2(), true); for (String[] period : periods) { - instance.fillWithStatus(period[0], period[1], 2, dtUtc); + instance.fillWithStatus(period[0], period[1], 2, now); instance.optimize(); } expResult = createUnknownTimeline3(); @@ -502,7 +502,7 @@ public void testFillStatus() throws ParseException { instance.insertDateTimeStamps(createTimestampList2(), true); for (String[] period : periods) { - instance.fillWithStatus(period[0], period[1], 2, dtUtc); + instance.fillWithStatus(period[0], period[1], 2, now); instance.optimize(); } expResult = createUnknownTimeline4(); @@ -545,7 +545,7 @@ public void testFillStatus() throws ParseException { instance.insertDateTimeStamps(createTimestampList4(), true); for (String[] period : periods) { - instance.fillWithStatus(period[0], period[1], 2, dtUtc); + instance.fillWithStatus(period[0], period[1], 2, now); instance.optimize(); } expResult = createUnknownTimeline5(); diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/ar/CalcGroupAR.java b/flink_jobs_v2/batch_multi/src/main/java/argo/ar/CalcGroupAR.java index 66d15ca3..0bdd4e06 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/ar/CalcGroupAR.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/ar/CalcGroupAR.java @@ -47,8 +47,8 @@ public class CalcGroupAR extends RichFlatMapFunction rec; - private DateTime dtUtc; + private DateTime now; /** * Initialization method of the RichGroupReduceFunction operator @@ -175,7 +175,7 @@ public void flatMap(StatusTimeline in, Collector out) throws Ex if (this.recMgr.isExcluded(in.getGroup())) { ArrayList> periods = this.recMgr.getPeriods(in.getGroup(), this.runDate); for (Map interval : periods) { - timeline.fillWithStatus(interval.get("start"), interval.get("end"), this.opsMgr.getDefaultUnknownInt(), dtUtc); + timeline.fillWithStatus(interval.get("start"), interval.get("end"), this.opsMgr.getDefaultUnknownInt(), now); timeline.optimize(); } } diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/ArgoMultiJob.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/ArgoMultiJob.java index 36aa600f..efa4d8f4 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/ArgoMultiJob.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/ArgoMultiJob.java @@ -108,8 +108,8 @@ public static void main(String[] args) throws Exception { env.getConfig().setGlobalJobParameters(params); env.setParallelism(1); - DateTime dtUtc = new DateTime(DateTimeZone.UTC); - + DateTime now = new DateTime(); + if (params.get("N") != null) { rankNum = params.getInt("N"); } @@ -235,7 +235,7 @@ public static void main(String[] args) throws Exception { //Create StatusMetricTimeline dataset for endpoints DataSet statusEndpointTimeline = statusMetricTimeline.groupBy("group", "service", "hostname") - .reduceGroup(new CalcEndpointTimeline(params, dtUtc)).withBroadcastSet(mpsDS, "mps").withBroadcastSet(opsDS, "ops") + .reduceGroup(new CalcEndpointTimeline(params, now)).withBroadcastSet(mpsDS, "mps").withBroadcastSet(opsDS, "ops") .withBroadcastSet(egpDS, "egp").withBroadcastSet(ggpDS, "ggp") .withBroadcastSet(apsDS, "aps").withBroadcastSet(downDS, "down"); @@ -298,7 +298,7 @@ public static void main(String[] args) throws Exception { withBroadcastSet(ggpDS, "ggp").withBroadcastSet(confDS, "conf"); // DataSet statusGroupTimelineAR = statusGroupTimeline.flatMap(new ExcludeGroupMetrics(params)).withBroadcastSet(recDS, "rec").withBroadcastSet(opsDS, "ops"); - DataSet endpointGroupArDS = statusGroupTimeline.flatMap(new CalcGroupAR(params, dtUtc)).withBroadcastSet(mpsDS, "mps") + DataSet endpointGroupArDS = statusGroupTimeline.flatMap(new CalcGroupAR(params, now)).withBroadcastSet(mpsDS, "mps") .withBroadcastSet(apsDS, "aps").withBroadcastSet(opsDS, "ops").withBroadcastSet(egpDS, "egp"). withBroadcastSet(ggpDS, "ggp").withBroadcastSet(confDS, "conf").withBroadcastSet(weightDS, "weight").withBroadcastSet(recDS, "rec"); diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcEndpointTimeline.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcEndpointTimeline.java index 14b24ce3..16ba78db 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcEndpointTimeline.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcEndpointTimeline.java @@ -38,9 +38,9 @@ public class CalcEndpointTimeline extends RichGroupReduceFunction downtime; - private DateTime dtUtc; + private DateTime now; @Override public void open(Configuration parameters) throws IOException { @@ -121,7 +121,7 @@ public void reduce(Iterable in, Collector out) t if (downPeriod != null && !downPeriod.isEmpty()) { - mergedTimeline.fillWithStatus(downPeriod.get(0), downPeriod.get(1), this.opsMgr.getDefaultDownInt(), dtUtc); + mergedTimeline.fillWithStatus(downPeriod.get(0), downPeriod.get(1), this.opsMgr.getDefaultDownInt(), now); } ArrayList timestatuCol = new ArrayList(); for (Map.Entry entry : mergedTimeline.getSamples()) { diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcPrevStatus.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcPrevStatus.java index 35590d03..3233aad2 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcPrevStatus.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcPrevStatus.java @@ -129,7 +129,7 @@ public void reduce(Iterable in, Collector out) throw out.collect(item); } } - + prevStatus = item.getStatus(); prevTimestamp = item.getTimestamp(); } diff --git a/flink_jobs_v2/batch_multi/src/test/java/argo/batch/ArgoMultiJobTest.java b/flink_jobs_v2/batch_multi/src/test/java/argo/batch/ArgoMultiJobTest.java index af976772..83afa5e3 100644 --- a/flink_jobs_v2/batch_multi/src/test/java/argo/batch/ArgoMultiJobTest.java +++ b/flink_jobs_v2/batch_multi/src/test/java/argo/batch/ArgoMultiJobTest.java @@ -93,7 +93,7 @@ public void testMain() throws Exception { System.setProperty("run.date", "2022-01-14"); final ParameterTool params = ParameterTool.fromSystemProperties(); - DateTime dtUtc=new DateTime(DateTimeZone.UTC); + DateTime now=new DateTime(); ApiResourceManager amr = mockAmr(); @@ -265,13 +265,13 @@ public void testMain() throws Exception { //*************** Test Endpoint Timeline //Create StatusMetricTimeline dataset for endpoints DataSet statusEndpointTimeline = statusMetricTimeline.groupBy("group", "service", "hostname") - .reduceGroup(new CalcEndpointTimeline(params, dtUtc)).withBroadcastSet(mpsDS, "mps").withBroadcastSet(opsDS, "ops") + .reduceGroup(new CalcEndpointTimeline(params, now)).withBroadcastSet(mpsDS, "mps").withBroadcastSet(opsDS, "ops") .withBroadcastSet(egpDS, "egp").withBroadcastSet(ggpDS, "ggp") .withBroadcastSet(apsDS, "aps").withBroadcastSet(downDS, "down"); ArrayList aggrJson = new ArrayList(); aggrJson.add(amr.getResourceJSON(ApiResource.AGGREGATION)); - ArrayList expEndpTimelines = TestUtils.prepareLevelTimeline(expMetricTimelines, opsJson, aggrJson, downDS.collect(), params.get("run.date"), LEVEL.HOSTNAME, dtUtc); + ArrayList expEndpTimelines = TestUtils.prepareLevelTimeline(expMetricTimelines, opsJson, aggrJson, downDS.collect(), params.get("run.date"), LEVEL.HOSTNAME, now); Assert.assertEquals(TestUtils.compareLists(expEndpTimelines, statusEndpointTimeline.collect()), true); //*************** Test Service Timeline @@ -279,7 +279,7 @@ public void testMain() throws Exception { .reduceGroup(new CalcServiceTimeline(params)).withBroadcastSet(mpsDS, "mps").withBroadcastSet(opsDS, "ops") .withBroadcastSet(egpDS, "egp").withBroadcastSet(ggpDS, "ggp") .withBroadcastSet(apsDS, "aps"); - ArrayList expServTimelines = TestUtils.prepareLevelTimeline(expEndpTimelines, opsJson, aggrJson, downDS.collect(), params.get("run.date"), LEVEL.SERVICE, dtUtc); + ArrayList expServTimelines = TestUtils.prepareLevelTimeline(expEndpTimelines, opsJson, aggrJson, downDS.collect(), params.get("run.date"), LEVEL.SERVICE, now); Assert.assertEquals(TestUtils.compareLists(expServTimelines, statusServiceTimeline.collect()), true); //*************** Test Function Timeline @@ -288,7 +288,7 @@ public void testMain() throws Exception { .withBroadcastSet(egpDS, "egp").withBroadcastSet(ggpDS, "ggp") .withBroadcastSet(apsDS, "aps"); - ArrayList expFunctionTimelines = TestUtils.prepareLevelTimeline(expServTimelines, opsJson, aggrJson, downDS.collect(), params.get("run.date"), LEVEL.FUNCTION, dtUtc); + ArrayList expFunctionTimelines = TestUtils.prepareLevelTimeline(expServTimelines, opsJson, aggrJson, downDS.collect(), params.get("run.date"), LEVEL.FUNCTION, now); Assert.assertEquals(TestUtils.compareLists(expFunctionTimelines, statusEndGroupFunctionTimeline.collect()), true); //*************** Test Group Timeline @@ -296,7 +296,7 @@ public void testMain() throws Exception { .reduceGroup(new CalcGroupTimeline(params)).withBroadcastSet(mpsDS, "mps").withBroadcastSet(opsDS, "ops") .withBroadcastSet(egpDS, "egp").withBroadcastSet(ggpDS, "ggp") .withBroadcastSet(apsDS, "aps"); - ArrayList expGroupTimelines = TestUtils.prepareLevelTimeline(expFunctionTimelines, opsJson, aggrJson, downDS.collect(), params.get("run.date"), LEVEL.GROUP, dtUtc); + ArrayList expGroupTimelines = TestUtils.prepareLevelTimeline(expFunctionTimelines, opsJson, aggrJson, downDS.collect(), params.get("run.date"), LEVEL.GROUP, now); Assert.assertEquals(TestUtils.compareLists(expGroupTimelines, statusGroupTimeline.collect()), true); if (calcStatus) { @@ -469,7 +469,7 @@ public void testMain() throws Exception { List expectedServAr = TestUtils.prepareServiceAR(servArData, params.get("run.date")); Assert.assertEquals(TestUtils.compareLists(expectedServAr, serviceArDS.collect()), true); - DataSet endpointGroupArDS = statusGroupTimeline.flatMap(new CalcGroupAR(params,dtUtc)).withBroadcastSet(mpsDS, "mps") + DataSet endpointGroupArDS = statusGroupTimeline.flatMap(new CalcGroupAR(params,now)).withBroadcastSet(mpsDS, "mps") .withBroadcastSet(apsDS, "aps").withBroadcastSet(opsDS, "ops").withBroadcastSet(egpDS, "egp"). withBroadcastSet(ggpDS, "ggp").withBroadcastSet(downDS, "down").withBroadcastSet(cfgDS, "conf").withBroadcastSet(weightDS, "weight").withBroadcastSet(recDS, "rec"); List groupArData = loadExpectedArData(groupExpectedData, LEVEL.GROUP, env); diff --git a/flink_jobs_v2/batch_multi/src/test/java/argo/batch/TestUtils.java b/flink_jobs_v2/batch_multi/src/test/java/argo/batch/TestUtils.java index 5916ff33..8ea5398d 100644 --- a/flink_jobs_v2/batch_multi/src/test/java/argo/batch/TestUtils.java +++ b/flink_jobs_v2/batch_multi/src/test/java/argo/batch/TestUtils.java @@ -61,7 +61,7 @@ public static enum LEVEL { // an enum class to define the level of the calculati * @throws ParseException * @throws IOException */ - public static ArrayList prepareLevelTimeline(List list, List opsDS, List aggrDS, List downDS, String runDate, LEVEL level, DateTime dtUtc) throws ParseException, IOException { + public static ArrayList prepareLevelTimeline(List list, List opsDS, List aggrDS, List downDS, String runDate, LEVEL level, DateTime now) throws ParseException, IOException { String group = "", function = "", service = "", hostname = "", metric = ""; @@ -162,12 +162,12 @@ public static ArrayList prepareLevelTimeline(List timelines = parseEndpTimelines(alltimelines, opsDS, aggrDS, downDS, runDate, level, dtUtc); + ArrayList timelines = parseEndpTimelines(alltimelines, opsDS, aggrDS, downDS, runDate, level, now); return timelines; } - private static ArrayList parseEndpTimelines(ArrayList> timelist, List opsDS, List aggrDS, List downDS, String runDate, LEVEL level, DateTime dtUtc) throws ParseException, IOException { + private static ArrayList parseEndpTimelines(ArrayList> timelist, List opsDS, List aggrDS, List downDS, String runDate, LEVEL level, DateTime now) throws ParseException, IOException { OperationsManager opsMgr = new OperationsManager(); opsMgr.loadJsonString(opsDS); @@ -224,7 +224,7 @@ private static ArrayList parseEndpTimelines(ArrayList downPeriod = downMgr.getPeriod(hostname, service); if (downPeriod != null && !downPeriod.isEmpty()) { - initialTimeline.fillWithStatus(downPeriod.get(0), downPeriod.get(1), opsMgr.getDefaultDownInt(), dtUtc); + initialTimeline.fillWithStatus(downPeriod.get(0), downPeriod.get(1), opsMgr.getDefaultDownInt(), now); } } From 294a89323a2afe6ce58cca2d08cf1c9ea2c53d10 Mon Sep 17 00:00:00 2001 From: Konstantinos Kagkelidis Date: Tue, 2 Aug 2022 21:26:01 +0300 Subject: [PATCH 04/16] Produce streaming events with friendly urls instead of encoded hostnames --- .../profilesmanager/EndpointGroupManager.java | 20 +++++++++++++ .../java/argo/streaming/AmsStreamStatus.java | 1 + .../src/main/java/status/StatusEvent.java | 5 ++++ .../src/main/java/status/StatusManager.java | 29 +++++++++++++++++-- 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/flink_jobs_v2/ProfilesManager/src/main/java/profilesmanager/EndpointGroupManager.java b/flink_jobs_v2/ProfilesManager/src/main/java/profilesmanager/EndpointGroupManager.java index f9c57fde..0bb49e0a 100644 --- a/flink_jobs_v2/ProfilesManager/src/main/java/profilesmanager/EndpointGroupManager.java +++ b/flink_jobs_v2/ProfilesManager/src/main/java/profilesmanager/EndpointGroupManager.java @@ -302,6 +302,26 @@ public HashMap getGroupTags(String group, String type, String ho return null; } + + /** + * Finds in the filtered EndpointItem fList a EndpointItem with specific + * group, type. service, hostname and tag and returns it's value + * + * @param group a group + * @param type a type + * @param hostname a hostname + * @param service a service + * @param tag this is the tag name + * @return return tag value + **/ + public String getTag(String group, String type, String hostname, String service, String tag) { + HashMap results = this.getGroupTags(group, type, hostname, service); + if (results != null){ + return results.get(tag); + } + + return null; + } /** * counts the items in the filtered list diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java index 5bf3f9bc..0916a1f7 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java @@ -514,6 +514,7 @@ public void open(Configuration parameters) throws IOException, ParseException, U sm.setStrictInterval(strictInterval); // sm.setTimeout(config.timeout); sm.setReport(config.report); + sm.setGroupType(amr.getEgroup()); // load all the connector data sm.loadAll(config.runDate, downList, egpListFull, mpsList, apsList, opsList); diff --git a/flink_jobs_v2/stream_status/src/main/java/status/StatusEvent.java b/flink_jobs_v2/stream_status/src/main/java/status/StatusEvent.java index aec06845..837e11cd 100644 --- a/flink_jobs_v2/stream_status/src/main/java/status/StatusEvent.java +++ b/flink_jobs_v2/stream_status/src/main/java/status/StatusEvent.java @@ -36,6 +36,7 @@ public class StatusEvent{ // Record all statuses of endpoint's metrics private @SerializedName("metric_statuses") String metricStatuses[]; private @SerializedName("metric_names") String metricNames[]; + private @SerializedName("hostname_url") String hostnameURL; public StatusEvent() { @@ -64,6 +65,7 @@ public StatusEvent() { this.groupStatuses = new String[0]; this.metricStatuses = new String[0]; this.metricNames = new String[0]; + this.hostnameURL = ""; } @@ -94,6 +96,7 @@ public StatusEvent (String report, String type, String dt, String group,String s this.groupStatuses = null; this.metricStatuses = null; this.metricNames = null; + this.hostnameURL = ""; } @@ -186,6 +189,7 @@ public void setMetricStatuses(String[] metricStatuses) { public String getRepeat() {return repeat;} public String getSummary() {return this.summary;} public String getMessage() {return this.message;} + public String getHostnameURL() { return this.hostnameURL;} public void setReport(String report) {this.report = report;} public void setType(String type) {this.type = type;} @@ -207,6 +211,7 @@ public void setMetricStatuses(String[] metricStatuses) { public void setStatus(String status) {this.status = status;} public void setSummary(String summary) {this.summary = summary;} public void setMessage (String message) {this.message = message;} + public void setHostnameURL (String hostnameURL) {this.hostnameURL = hostnameURL;} public int getDateInt() { return Integer.parseInt(this.dt); diff --git a/flink_jobs_v2/stream_status/src/main/java/status/StatusManager.java b/flink_jobs_v2/stream_status/src/main/java/status/StatusManager.java index 69e8047c..170fcdf8 100644 --- a/flink_jobs_v2/stream_status/src/main/java/status/StatusManager.java +++ b/flink_jobs_v2/stream_status/src/main/java/status/StatusManager.java @@ -44,6 +44,8 @@ public class StatusManager { // Name of the report used private String report; + + private String groupType; // Sync file structures necessary for status computation public EndpointGroupManager egp = new EndpointGroupManager(); @@ -78,6 +80,14 @@ public void setReport(String report) { public String getReport() { return this.report; } + + public String getGroupType() { + return this.groupType; + } + + public void setGroupType(String groupType) { + this.groupType = groupType; + } // Get Operation Manager public OperationsManager getOps() { @@ -708,7 +718,7 @@ public Map> getMetricStatuses(StatusNode endpoint, Ope * arraylists keyed: "endpoints", "services", "statuses" * */ - public Map> getGroupEndpointStatuses(StatusNode egroup, OperationsManager ops) { + public Map> getGroupEndpointStatuses(StatusNode egroup, String groupName, OperationsManager ops, EndpointGroupManager egp) { Map> results = new HashMap>(); ArrayList endpoints = new ArrayList(); ArrayList services = new ArrayList(); @@ -726,6 +736,13 @@ public Map> getGroupEndpointStatuses(StatusNode egroup StatusNode service = serviceEntry.getValue(); for (Entry endpointEntry : service.children.entrySet()) { String endpointName = endpointEntry.getKey(); + // check if endpoint need to be in url friendly format + String endpointURL = egp.getTag(groupName, this.groupType, endpointName, serviceName,"info_URL"); + if (endpointURL != null) { + endpointName = endpointURL; + } + + StatusNode endpoint = endpointEntry.getValue(); // Add endpoint information to results results.get("endpoints").add(endpointName); @@ -888,6 +905,14 @@ public ArrayList setStatus(String group, String service, String hostname evtEndpoint = genEvent("endpoint", group, service, hostname, metric, ops.getStrStatus(endpNewStatus), monHost, ts, ops.getStrStatus(oldEndpointStatus), oldEndpointTS, repeat, summary, message); + + // check if endpoind should get a friendly url hostname + String hostnameURL = this.egp.getTag(group, this.groupType, hostname, service, "info_URL"); + if (hostnameURL != null) { + evtEndpoint.setHostnameURL(hostnameURL); + } else { + evtEndpoint.setHostnameURL(hostname); + } // Create metric,endpoint status level object statusEndpoint = new String[]{evtEndpoint.getStatus(), evtEndpoint.getPrevStatus(), evtEndpoint.getTsMonitored(), evtEndpoint.getPrevTs()}; @@ -966,7 +991,7 @@ public ArrayList setStatus(String group, String service, String hostname statusEgroup = new String[]{evtEgroup.getStatus(), evtEgroup.getPrevStatus(), evtEgroup.getTsMonitored(), evtEgroup.getPrevTs()}; // generate group endpoint information - Map> groupStatuses = getGroupEndpointStatuses(groupNode, ops); + Map> groupStatuses = getGroupEndpointStatuses(groupNode,group, ops, egp); evtEgroup.setGroupEndpoints(groupStatuses.get("endpoints").toArray(new String[0])); evtEgroup.setGroupServices(groupStatuses.get("services").toArray(new String[0])); evtEgroup.setGroupStatuses(groupStatuses.get("statuses").toArray(new String[0])); From 08772a7ba4e8156703e7c9763e9d858bfcd17a5c Mon Sep 17 00:00:00 2001 From: cthermolia-grnet Date: Tue, 30 Aug 2022 13:16:41 +0300 Subject: [PATCH 05/16] ARGO-3731 Investigate upgrading mongodb connector to v3.12 --- flink_jobs_v2/batch_multi/pom.xml | 4 ++-- flink_jobs_v2/stream_status/pom.xml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flink_jobs_v2/batch_multi/pom.xml b/flink_jobs_v2/batch_multi/pom.xml index 5000bc0f..6f333abb 100644 --- a/flink_jobs_v2/batch_multi/pom.xml +++ b/flink_jobs_v2/batch_multi/pom.xml @@ -130,7 +130,7 @@ language governing permissions and limitations under the License. --> org.mongodb mongo-java-driver - 3.2.2 + 3.12.11 @@ -253,7 +253,7 @@ language governing permissions and limitations under the License. --> org.mongodb mongo-java-driver - 3.2.2 + 3.12.11 diff --git a/flink_jobs_v2/stream_status/pom.xml b/flink_jobs_v2/stream_status/pom.xml index 6f33f39e..af95d34c 100644 --- a/flink_jobs_v2/stream_status/pom.xml +++ b/flink_jobs_v2/stream_status/pom.xml @@ -153,7 +153,7 @@ language governing permissions and limitations under the License. --> org.mongodb mongo-java-driver - 3.2.2 + 3.12.11 compile @@ -218,7 +218,7 @@ language governing permissions and limitations under the License. --> org.mongodb mongo-java-driver - 3.2.2 + 3.12.11 compile From f895e3e3f871407e6305a6ef2c953d48a72516be Mon Sep 17 00:00:00 2001 From: cthermolia-grnet Date: Wed, 24 Aug 2022 11:23:10 +0300 Subject: [PATCH 06/16] ARGO-3961 Change DateTime timezone to utc during handling timestamps in Timelines and multi_job --- .../src/main/java/timelines/Timeline.java | 79 ++++++++++--------- .../java/timelines/TimelineAggregator.java | 13 +-- .../src/main/java/timelines/Utils.java | 32 ++++---- .../timelines/TimelineAggregatorTest.java | 4 +- .../src/test/java/timelines/TimelineTest.java | 2 +- .../src/main/java/argo/ar/CalcEndpointAR.java | 3 +- .../src/main/java/argo/ar/CalcGroupAR.java | 3 +- .../src/main/java/argo/ar/CalcServiceAR.java | 3 +- .../main/java/argo/batch/ArgoMultiJob.java | 3 +- .../java/argo/batch/CalcEndpointTimeline.java | 3 +- .../argo/batch/CalcGroupFunctionTimeline.java | 3 +- .../java/argo/batch/CalcGroupTimeline.java | 3 +- .../java/argo/batch/CalcServiceTimeline.java | 3 +- .../java/argo/batch/CalcStatusEndGroup.java | 3 +- .../java/argo/batch/CalcStatusEndpoint.java | 3 +- .../java/argo/batch/CalcStatusService.java | 3 +- .../argo/batch/MongoEndGroupArOutput.java | 3 +- .../argo/batch/MongoEndpointArOutput.java | 3 +- .../java/argo/batch/MongoServiceArOutput.java | 3 +- .../java/argo/batch/MongoStatusOutput.java | 3 +- .../CalcEndpointFlipFlopTrends.java | 3 +- .../calculations/CalcGroupFlipFlopTrends.java | 3 +- .../CalcMetricFlipFlopTrends.java | 3 +- .../CalcServiceFlipFlopTrends.java | 3 +- .../calculations/MongoTrendsOutput.java | 3 +- .../src/main/java/utils/Utils.java | 35 ++++---- .../java/argo/batch/ArgoMultiJobTest.java | 4 +- .../src/test/java/argo/batch/TestUtils.java | 2 +- 28 files changed, 128 insertions(+), 103 deletions(-) diff --git a/flink_jobs_v2/Timelines/src/main/java/timelines/Timeline.java b/flink_jobs_v2/Timelines/src/main/java/timelines/Timeline.java index fb0e7e46..1990047e 100644 --- a/flink_jobs_v2/Timelines/src/main/java/timelines/Timeline.java +++ b/flink_jobs_v2/Timelines/src/main/java/timelines/Timeline.java @@ -6,14 +6,19 @@ * and open the template in the editor. */ import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Set; +import java.util.TimeZone; import java.util.TreeMap; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.joda.time.LocalDate; +import org.joda.time.LocalTime; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; import org.slf4j.Logger; @@ -61,13 +66,13 @@ public Timeline() { * would define the date of the timeline * * */ - public Timeline(String timestamp) { - DateTime tmp_date = new DateTime(); - DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); - tmp_date = fmt.parseDateTime(timestamp); + public Timeline(String timestamp) throws ParseException { + DateTime tmp_date = Utils.convertStringtoDate("yyyy-MM-dd'T'HH:mm:ss'Z'", timestamp); tmp_date.withTime(0, 0, 0, 0); + this.date = tmp_date.toLocalDate(); this.samples = new TreeMap(); + } /** @@ -78,20 +83,18 @@ public Timeline(String timestamp) { * at a timestamp , describing midnight (00:00:00) * */ - Timeline(String timestamp, int state) { - DateTime tmp_date = new DateTime(); - DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); - tmp_date = fmt.parseDateTime(timestamp); - tmp_date = tmp_date.withTime(0, 0, 0, 0); + Timeline(String timestamp, int state) throws ParseException { + DateTime tmp_date = Utils.convertStringtoDate("yyyy-MM-dd'T'HH:mm:ss'Z'", timestamp); + tmp_date.withTime(0, 0, 0, 0); + this.date = tmp_date.toLocalDate(); this.samples = new TreeMap(); this.samples.put(tmp_date, state); } - public int get(String timestamp) { - DateTime tmp_date = new DateTime(); - DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); - tmp_date = fmt.parseDateTime(timestamp); + public int get(String timestamp) throws ParseException { + DateTime tmp_date = Utils.convertStringtoDate("yyyy-MM-dd'T'HH:mm:ss'Z'", timestamp); + if (this.samples.floorEntry(tmp_date) == null) { return -1; } @@ -113,11 +116,10 @@ public int get(DateTime point) { * * inserts a pair of timestamp, status in the map. */ - public void insert(String timestamp, int status) { + public void insert(String timestamp, int status) throws ParseException { + + DateTime tmp_date = Utils.convertStringtoDate("yyyy-MM-dd'T'HH:mm:ss'Z'", timestamp); - DateTime tmp_date = new DateTime(); - DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); - tmp_date = fmt.parseDateTime(timestamp); this.samples.put(tmp_date, status); } @@ -133,7 +135,7 @@ public void insert(DateTime timestamp, int status) { samples.put(timestamp, status); } - public void insertStringTimeStamps(TreeMap timestamps, boolean optimize) { + public void insertStringTimeStamps(TreeMap timestamps, boolean optimize) throws ParseException { DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); for (String dt : timestamps.keySet()) { int status = timestamps.get(dt); @@ -144,9 +146,9 @@ public void insertStringTimeStamps(TreeMap timestamps, boolean this.optimize(); } if (this.date == null) { - DateTime tmp_date = new DateTime(); - tmp_date = this.samples.firstKey(); - tmp_date = tmp_date.withTime(0, 0, 0, 0); + DateTime tmp_date=this.samples.firstKey(); + tmp_date.withTime(0, 0, 0, 0); + this.date = tmp_date.toLocalDate(); } @@ -156,7 +158,7 @@ public void insertStringTimeStamps(TreeMap timestamps, boolean * @param timestamps a map of timestamp, status to be stored in the * timeline. the timestamps are in the form of datetime */ - public void insertDateTimeStamps(TreeMap timestamps, boolean optimize) { + public void insertDateTimeStamps(TreeMap timestamps, boolean optimize) throws ParseException { for (DateTime dt : timestamps.keySet()) { int status = timestamps.get(dt); this.insert(dt, status); @@ -165,8 +167,8 @@ public void insertDateTimeStamps(TreeMap timestamps, boolean this.optimize(); } if (this.date == null) { - DateTime tmp_date = new DateTime(); - tmp_date = this.samples.firstKey(); + DateTime tmp_date=this.samples.firstKey(); + tmp_date = tmp_date.withTime(0, 0, 0, 0); this.date = tmp_date.toLocalDate(); } @@ -181,12 +183,13 @@ public void insertDateTimeStamps(TreeMap timestamps, boolean * timestamp is the midnight (00:00:00) of the date of the given timestamp * and the status is the given state */ - public void setFirst(String timestamp, int state) { - DateTime tmp_date = new DateTime(); - DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); - tmp_date = fmt.parseDateTime(timestamp); - this.samples = new TreeMap(); + public void setFirst(String timestamp, int state) throws ParseException { + DateTime tmp_date = Utils.convertStringtoDate("yyyy-MM-dd'T'HH:mm:ss'Z'", timestamp); + tmp_date = tmp_date.withTime(0, 0, 0, 0); + + + this.samples = new TreeMap(); this.samples.put(tmp_date, state); this.optimize(); } @@ -371,15 +374,12 @@ public TreeMap buildStringTimeStampMap(ArrayList time * in the map * */ - public TreeMap buildDateTimeStampMap(ArrayList timestampList, ArrayList states) { + public TreeMap buildDateTimeStampMap(ArrayList timestampList, ArrayList states) throws ParseException { TreeMap timestampMap = new TreeMap(); for (String[] timestamp : timestampList) { - - DateTime tmp_date = new DateTime(); - DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); - tmp_date = fmt.parseDateTime(timestamp[0]); + DateTime tmp_date=Utils.convertStringtoDate("yyyy-MM-dd'T'HH:mm:ss'Z'", timestamp[0]); int status = states.indexOf(timestamp[1]); timestampMap.put(tmp_date, status); } @@ -471,12 +471,15 @@ public int opInt(int[][][] truthTable, int op, int a, int b) { * */ public void fillWithStatus(String start, String end, Integer intStatus, DateTime now) throws ParseException { - DateTime startDay = this.date.toDateTimeAtStartOfDay(); + + DateTime startDay = this.date.toDateTimeAtStartOfDay(DateTimeZone.UTC); + DateTime endDay = startDay.withTime(23, 59, 59, 0); - + DateTime startDt = Utils.convertStringtoDate("yyyy-MM-dd'T'HH:mm:ss'Z'", start); + DateTime endDt = Utils.convertStringtoDate("yyyy-MM-dd'T'HH:mm:ss'Z'", end); - + DateTime floor = samples.floorKey(startDt); DateTime ceiling = samples.ceilingKey(endDt); @@ -505,7 +508,7 @@ public void fillWithStatus(String start, String end, Integer intStatus, DateTime } //if a timestamp exists after a period then this timestamp should be added with the initial status of that period taken from the initial timeline //this timestamp should not extend todays current time (in the case that the computations occur during today and before the end of the day) - + if (addCeiling && !endDt.isAfter(now) && endDt.plusMinutes(1).isBefore(endDay)) { this.samples.put(endDt.plusMinutes(1), endFloorStatus); } diff --git a/flink_jobs_v2/Timelines/src/main/java/timelines/TimelineAggregator.java b/flink_jobs_v2/Timelines/src/main/java/timelines/TimelineAggregator.java index 794ba20f..4a0103f4 100644 --- a/flink_jobs_v2/Timelines/src/main/java/timelines/TimelineAggregator.java +++ b/flink_jobs_v2/Timelines/src/main/java/timelines/TimelineAggregator.java @@ -7,6 +7,7 @@ import java.util.Set; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.joda.time.LocalDate; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; @@ -35,7 +36,7 @@ public TimelineAggregator(String timestamp) throws ParseException { } private LocalDate getLocalDate(String date) { - DateTime tmp_date = new DateTime(); + DateTime tmp_date = new DateTime(DateTimeZone.UTC); DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd"); tmp_date = fmt.parseDateTime(date); tmp_date.withTime(0, 0, 0, 0); @@ -97,7 +98,7 @@ public void clear() { * */ public String tsFromDate(String date) { - DateTime tmp_date = new DateTime(); + DateTime tmp_date = new DateTime(DateTimeZone.UTC); DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd"); tmp_date = fmt.parseDateTime(date); tmp_date = tmp_date.withTime(0, 0, 0, 0); @@ -113,7 +114,7 @@ public String tsFromDate(String date) { * Creates a timeline with the given status set at midnight of the date * defined from timestamp and add this timeline to the input timelines */ - public void createTimeline(String name, String timestamp, int prevState) { + public void createTimeline(String name, String timestamp, int prevState) throws ParseException { Timeline temp = new Timeline(timestamp, prevState); this.inputs.put(name, temp); @@ -127,7 +128,7 @@ public void createTimeline(String name, String timestamp, int prevState) { * have an existing timeline add a new timeline to the inputs * */ - public void insert(String name, String timestamp, int status) { + public void insert(String name, String timestamp, int status) throws ParseException { // Check if timeline exists, if not create it if (this.inputs.containsKey(name) == false) { Timeline temp = new Timeline(timestamp, status); @@ -147,7 +148,7 @@ public void insert(String name, String timestamp, int status) { * timeline contains the given status for the midnight (00:00:00) of the * timestamp */ - public void setFirst(String name, String timestamp, int status) { + public void setFirst(String name, String timestamp, int status) throws ParseException { // Check if timeline exists, if not create it if (this.inputs.containsKey(name) == false) { Timeline temp = new Timeline(timestamp, status); @@ -170,7 +171,7 @@ public Set> getSamples() { return this.output.getSamples(); } - public void clearAndSetDate(String timestamp) { + public void clearAndSetDate(String timestamp) throws ParseException { this.output = new Timeline(timestamp); this.inputs.clear(); diff --git a/flink_jobs_v2/Timelines/src/main/java/timelines/Utils.java b/flink_jobs_v2/Timelines/src/main/java/timelines/Utils.java index a766f3f0..4a611684 100644 --- a/flink_jobs_v2/Timelines/src/main/java/timelines/Utils.java +++ b/flink_jobs_v2/Timelines/src/main/java/timelines/Utils.java @@ -14,6 +14,7 @@ import org.slf4j.LoggerFactory; import java.util.TimeZone; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.joda.time.Minutes; import org.joda.time.Seconds; import org.joda.time.format.DateTimeFormat; @@ -36,37 +37,37 @@ public static String convertDateToString(String format, DateTime date) throws Pa public static DateTime convertStringtoDate(String format, String dateStr) throws ParseException { - DateTimeFormatter formatter = DateTimeFormat.forPattern(format); - - DateTime dt = formatter.parseDateTime(dateStr); + SimpleDateFormat sdf = new SimpleDateFormat(format); + sdf.setTimeZone(TimeZone.getTimeZone("UTC")); + Date date = sdf. parse(dateStr); + return new DateTime(date.getTime(),DateTimeZone.UTC); - return dt; } public static DateTime createDate(String format, Date dateStr, int hour, int min, int sec) throws ParseException { //String format = "yyyy-MM-dd'T'HH:mm:ss'Z'"; - SimpleDateFormat sdf = new SimpleDateFormat(format); - sdf.setTimeZone(TimeZone.getDefault()); Calendar newCalendar = Calendar.getInstance(); + newCalendar.setTimeZone(TimeZone.getTimeZone("UTC")); + newCalendar.setTime(dateStr); newCalendar.set(Calendar.HOUR_OF_DAY, hour); newCalendar.set(Calendar.MINUTE, min); newCalendar.set(Calendar.SECOND, sec); newCalendar.set(Calendar.MILLISECOND, 0); - return new DateTime(newCalendar.getTime()); + return new DateTime(newCalendar.getTime(), DateTimeZone.UTC); } public static boolean isPreviousDate(String format, Date nowDate, Date firstDate) throws ParseException { // String format = "yyyy-MM-dd'T'HH:mm:ss'Z'"; Calendar cal = Calendar.getInstance(); - SimpleDateFormat sdf = new SimpleDateFormat(format); - sdf.setTimeZone(TimeZone.getDefault()); + cal.setTimeZone(TimeZone.getTimeZone("UTC")); cal.setTime(nowDate); Calendar calFirst = Calendar.getInstance(); + calFirst.setTimeZone(TimeZone.getTimeZone("UTC")); calFirst.setTime(firstDate); if (firstDate.before(nowDate)) { @@ -77,10 +78,8 @@ public static boolean isPreviousDate(String format, Date nowDate, Date firstDate } public static DateTime createDate(String format, int year, int month, int day, int hour, int min, int sec) throws ParseException { - // String format = "yyyy-MM-dd'T'HH:mm:ss'Z'"; - SimpleDateFormat sdf = new SimpleDateFormat(format); - sdf.setTimeZone(TimeZone.getDefault()); Calendar newCalendar = Calendar.getInstance(); + newCalendar.setTimeZone(TimeZone.getTimeZone("UTC")); newCalendar.set(Calendar.YEAR, year); newCalendar.set(Calendar.MONTH, month); newCalendar.set(Calendar.DAY_OF_MONTH, day); @@ -89,22 +88,21 @@ public static DateTime createDate(String format, int year, int month, int day, i newCalendar.set(Calendar.MINUTE, min); newCalendar.set(Calendar.SECOND, sec); newCalendar.set(Calendar.MILLISECOND, 0); - return new DateTime(newCalendar.getTime()); + + return new DateTime(newCalendar.getTime(),DateTimeZone.UTC); } public static DateTime setTime(String format, DateTime dateStr, int hour, int min, int sec, int mill) throws ParseException { - //String format = "yyyy-MM-dd'T'HH:mm:ss'Z'"; - SimpleDateFormat sdf = new SimpleDateFormat(format); - sdf.setTimeZone(TimeZone.getDefault()); Calendar newCalendar = Calendar.getInstance(); + newCalendar.setTimeZone(TimeZone.getTimeZone("UTC")); newCalendar.setTime(dateStr.toDate()); newCalendar.set(Calendar.HOUR_OF_DAY, hour); newCalendar.set(Calendar.MINUTE, min); newCalendar.set(Calendar.SECOND, sec); newCalendar.set(Calendar.MILLISECOND, mill); - return new DateTime(newCalendar.getTime()); + return new DateTime(newCalendar.getTime(), DateTimeZone.UTC); } public static int calcDayMinutes(DateTime startDay, DateTime endDay) throws ParseException { diff --git a/flink_jobs_v2/Timelines/src/test/java/timelines/TimelineAggregatorTest.java b/flink_jobs_v2/Timelines/src/test/java/timelines/TimelineAggregatorTest.java index 01e87f71..5b188f91 100644 --- a/flink_jobs_v2/Timelines/src/test/java/timelines/TimelineAggregatorTest.java +++ b/flink_jobs_v2/Timelines/src/test/java/timelines/TimelineAggregatorTest.java @@ -81,7 +81,7 @@ public void testCreateTimeline() throws ParseException { instance.createTimeline(name, timestamp, prevState); HashMap expRes = new HashMap<>(); Timeline exptimeline = new Timeline(timestamp); - exptimeline.insert(Utils.createDate("yyyy-MM-dd'T'HH:mm:ss'Z'", 2021, 0, 31, 0, 0, 0), 0); + exptimeline.insert(Utils.createDate("yyyy-MM-dd'T'HH:mm:ss'Z'", 2021, 0, 31, 12, 50, 0), 0); expRes.put(name, exptimeline); assertEquals(expRes.toString(), instance.getInputs().toString()); @@ -103,7 +103,7 @@ public void testInsert() throws ParseException { instance.insert(name, timestamp, status); HashMap expRes = new HashMap<>(); Timeline exptimeline = new Timeline(timestamp); - exptimeline.insert(Utils.createDate("yyyy-MM-dd'T'HH:mm:ss'Z'", 2021, 0, 31, 0, 0, 0), 0); + exptimeline.insert(Utils.createDate("yyyy-MM-dd'T'HH:mm:ss'Z'", 2021, 0, 31, 12, 50, 0), 0); expRes.put(name, exptimeline); assertEquals(expRes.toString(), instance.getInputs().toString()); diff --git a/flink_jobs_v2/Timelines/src/test/java/timelines/TimelineTest.java b/flink_jobs_v2/Timelines/src/test/java/timelines/TimelineTest.java index bd310108..5d668057 100644 --- a/flink_jobs_v2/Timelines/src/test/java/timelines/TimelineTest.java +++ b/flink_jobs_v2/Timelines/src/test/java/timelines/TimelineTest.java @@ -362,7 +362,7 @@ public void testOpInt() throws ParseException { public void testFillStatus() throws ParseException { DateTimeFormatter dtf = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); DateTime now = new DateTime(); - + now= now.withZone(DateTimeZone.UTC); Timeline instance = new Timeline(); instance.insertDateTimeStamps(createTimestampList2(), true); diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/ar/CalcEndpointAR.java b/flink_jobs_v2/batch_multi/src/main/java/argo/ar/CalcEndpointAR.java index f038bcd9..8cb5ba78 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/ar/CalcEndpointAR.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/ar/CalcEndpointAR.java @@ -22,6 +22,7 @@ import java.util.TreeMap; import org.apache.flink.api.common.functions.RichFlatMapFunction; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import profilesmanager.AggregationProfileManager; import profilesmanager.EndpointGroupManager; import profilesmanager.GroupGroupManager; @@ -135,7 +136,7 @@ public void flatMap(StatusTimeline in, Collector out) throws Excepti ArrayList timestatusList = in.getTimestamps(); TreeMap timestampMap = new TreeMap(); for (TimeStatus ts : timestatusList) { - timestampMap.put(new DateTime(ts.getTimestamp()), ts.getStatus()); + timestampMap.put(new DateTime(ts.getTimestamp(), DateTimeZone.UTC), ts.getStatus()); } Timeline timeline = new Timeline(); diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/ar/CalcGroupAR.java b/flink_jobs_v2/batch_multi/src/main/java/argo/ar/CalcGroupAR.java index 0bdd4e06..d05f8f42 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/ar/CalcGroupAR.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/ar/CalcGroupAR.java @@ -26,6 +26,7 @@ import java.util.TreeMap; import org.apache.flink.api.common.functions.RichFlatMapFunction; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import profilesmanager.AggregationProfileManager; import profilesmanager.EndpointGroupManager; import profilesmanager.GroupGroupManager; @@ -164,7 +165,7 @@ public void flatMap(StatusTimeline in, Collector out) throws Ex TreeMap timestampMap = new TreeMap(); for (TimeStatus ts : timestatusList) { - timestampMap.put(new DateTime(ts.getTimestamp()), ts.getStatus()); + timestampMap.put(new DateTime(ts.getTimestamp(), DateTimeZone.UTC), ts.getStatus()); } Timeline timeline = new Timeline(); diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/ar/CalcServiceAR.java b/flink_jobs_v2/batch_multi/src/main/java/argo/ar/CalcServiceAR.java index e4de2101..e21add07 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/ar/CalcServiceAR.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/ar/CalcServiceAR.java @@ -23,6 +23,7 @@ import java.util.TreeMap; import org.apache.flink.api.common.functions.RichFlatMapFunction; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import profilesmanager.AggregationProfileManager; import profilesmanager.EndpointGroupManager; import profilesmanager.GroupGroupManager; @@ -143,7 +144,7 @@ public void flatMap(StatusTimeline in, Collector out) throws Exceptio TreeMap timestampMap = new TreeMap(); for (TimeStatus ts : timestatusList) { - timestampMap.put(new DateTime(ts.getTimestamp()), ts.getStatus()); + timestampMap.put(new DateTime(ts.getTimestamp(), DateTimeZone.UTC), ts.getStatus()); } Timeline timeline = new Timeline(); diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/ArgoMultiJob.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/ArgoMultiJob.java index efa4d8f4..60bf79e6 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/ArgoMultiJob.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/ArgoMultiJob.java @@ -108,8 +108,7 @@ public static void main(String[] args) throws Exception { env.getConfig().setGlobalJobParameters(params); env.setParallelism(1); - DateTime now = new DateTime(); - + DateTime now = new DateTime(DateTimeZone.UTC); if (params.get("N") != null) { rankNum = params.getInt("N"); } diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcEndpointTimeline.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcEndpointTimeline.java index 16ba78db..1fbbf41e 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcEndpointTimeline.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcEndpointTimeline.java @@ -18,6 +18,7 @@ import java.util.HashMap; import java.util.Map; import java.util.TreeMap; +import org.joda.time.DateTimeZone; import profilesmanager.AggregationProfileManager; import profilesmanager.DowntimeManager; import profilesmanager.MetricProfileManager; @@ -102,7 +103,7 @@ public void reduce(Iterable in, Collector out) t TreeMap samples = new TreeMap<>(); for (TimeStatus timestatus : timestatusList) { - samples.put(new DateTime(timestatus.getTimestamp()), timestatus.getStatus()); + samples.put(new DateTime(timestatus.getTimestamp(),DateTimeZone.UTC), timestatus.getStatus()); } Timeline timeline = new Timeline(); timeline.insertDateTimeStamps(samples, true); diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcGroupFunctionTimeline.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcGroupFunctionTimeline.java index 2d0acd93..16da5e00 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcGroupFunctionTimeline.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcGroupFunctionTimeline.java @@ -16,6 +16,7 @@ import java.util.HashMap; import java.util.Map; import java.util.TreeMap; +import org.joda.time.DateTimeZone; import profilesmanager.AggregationProfileManager; import profilesmanager.OperationsManager; @@ -79,7 +80,7 @@ public void reduce(Iterable in, Collector out) t ArrayList timestatusList = item.getTimestamps(); TreeMap samples = new TreeMap<>(); for (TimeStatus timestatus : timestatusList) { - DateTime dt = new DateTime(timestatus.getTimestamp()); + DateTime dt = new DateTime(timestatus.getTimestamp(), DateTimeZone.UTC); samples.put(dt, timestatus.getStatus()); } Timeline timeline = new Timeline(); diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcGroupTimeline.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcGroupTimeline.java index 0f428062..6b95abfe 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcGroupTimeline.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcGroupTimeline.java @@ -16,6 +16,7 @@ import java.util.HashMap; import java.util.Map; import java.util.TreeMap; +import org.joda.time.DateTimeZone; import profilesmanager.AggregationProfileManager; import profilesmanager.OperationsManager; @@ -77,7 +78,7 @@ public void reduce(Iterable in, Collector out) t ArrayList timestatusList = item.getTimestamps(); TreeMap samples = new TreeMap<>(); for (TimeStatus timestatus : timestatusList) { - DateTime dt = new DateTime(timestatus.getTimestamp()); + DateTime dt = new DateTime(timestatus.getTimestamp(), DateTimeZone.UTC); samples.put(dt, timestatus.getStatus()); } Timeline timeline = new Timeline(); diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcServiceTimeline.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcServiceTimeline.java index 2aba23e6..4c0f0f24 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcServiceTimeline.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcServiceTimeline.java @@ -15,6 +15,7 @@ import java.util.HashMap; import java.util.Map; import java.util.TreeMap; +import org.joda.time.DateTimeZone; import profilesmanager.AggregationProfileManager; import profilesmanager.OperationsManager; @@ -80,7 +81,7 @@ public void reduce(Iterable in, Collector out) t ArrayList timestatusList = item.getTimestamps(); TreeMap samples = new TreeMap(); for (TimeStatus timestatus : timestatusList) { - DateTime dt = new DateTime(timestatus.getTimestamp()); + DateTime dt = new DateTime(timestatus.getTimestamp(), DateTimeZone.UTC); samples.put(dt, timestatus.getStatus()); } Timeline timeline = new Timeline(); diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcStatusEndGroup.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcStatusEndGroup.java index f69da237..d352b517 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcStatusEndGroup.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcStatusEndGroup.java @@ -10,6 +10,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.util.Collector; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import profilesmanager.AggregationProfileManager; @@ -85,7 +86,7 @@ public void flatMap(StatusTimeline in, Collector out) throws Excep StatusMetric cur = new StatusMetric(); cur.setDateInt(dateInt); cur.setGroup(endpointGroup); - cur.setTimestamp(utils.Utils.convertDateToString("yyyy-MM-dd'T'HH:mm:ss'Z'", new DateTime(item.getTimestamp()))); + cur.setTimestamp(utils.Utils.convertDateToString("yyyy-MM-dd'T'HH:mm:ss'Z'", new DateTime(item.getTimestamp(), DateTimeZone.UTC))); cur.setStatus(opsMgr.getStrStatus(item.getStatus())); cur.setHasThr(hasThr); out.collect(cur); diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcStatusEndpoint.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcStatusEndpoint.java index 40936087..d4d2d699 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcStatusEndpoint.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcStatusEndpoint.java @@ -14,6 +14,7 @@ import argo.avro.MetricProfile; import java.util.ArrayList; import org.apache.flink.api.common.functions.RichFlatMapFunction; +import org.joda.time.DateTimeZone; import profilesmanager.AggregationProfileManager; import profilesmanager.EndpointGroupManager; import profilesmanager.MetricProfileManager; @@ -105,7 +106,7 @@ public void flatMap(StatusTimeline in, Collector out) throws Excep cur.setFunction(function); cur.setInfo(info); cur.setHasThr(hasThr); - cur.setTimestamp(Utils.convertDateToString("yyyy-MM-dd'T'HH:mm:ss'Z'", new DateTime(item.getTimestamp()))); + cur.setTimestamp(Utils.convertDateToString("yyyy-MM-dd'T'HH:mm:ss'Z'", new DateTime(item.getTimestamp(),DateTimeZone.UTC))); cur.setStatus(opsMgr.getStrStatus(item.getStatus())); out.collect(cur); } diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcStatusService.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcStatusService.java index f831efd8..8eb192e4 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcStatusService.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/CalcStatusService.java @@ -8,6 +8,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.util.Collector; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import profilesmanager.AggregationProfileManager; @@ -84,7 +85,7 @@ public void flatMap(StatusTimeline in, Collector out) throws Excep cur.setGroup(endpointGroup); cur.setFunction(function); cur.setService(service); - cur.setTimestamp(Utils.convertDateToString("yyyy-MM-dd'T'HH:mm:ss'Z'", new DateTime(item.getTimestamp()))); + cur.setTimestamp(Utils.convertDateToString("yyyy-MM-dd'T'HH:mm:ss'Z'", new DateTime(item.getTimestamp(), DateTimeZone.UTC))); cur.setStatus(opsMgr.getStrStatus(item.getStatus())); cur.setHasThr(hasThr); out.collect(cur); diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoEndGroupArOutput.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoEndGroupArOutput.java index 22ce09c5..2e2b5e8c 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoEndGroupArOutput.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoEndGroupArOutput.java @@ -16,6 +16,7 @@ import com.mongodb.client.model.UpdateOptions; import org.bson.types.ObjectId; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; /** @@ -87,7 +88,7 @@ private void initMongo() { this.mDB = mClient.getDatabase(dbName); this.mCol = mDB.getCollection(colName); if (this.clearMongo) { - String oidString = Long.toHexString(new DateTime().getMillis() / 1000L) + "0000000000000000"; + String oidString = Long.toHexString(new DateTime(DateTimeZone.UTC).getMillis() / 1000L) + "0000000000000000"; this.nowId = new ObjectId(oidString); } diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoEndpointArOutput.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoEndpointArOutput.java index 57b12cee..a7fa8cdc 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoEndpointArOutput.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoEndpointArOutput.java @@ -21,6 +21,7 @@ import java.util.Iterator; import org.bson.types.ObjectId; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.json.simple.JSONObject; /** @@ -90,7 +91,7 @@ private void initMongo() { this.mDB = mClient.getDatabase(dbName); this.mCol = mDB.getCollection(colName); if (this.clearMongo) { - String oidString = Long.toHexString(new DateTime().getMillis() / 1000L) + "0000000000000000"; + String oidString = Long.toHexString(new DateTime(DateTimeZone.UTC).getMillis() / 1000L) + "0000000000000000"; this.nowId = new ObjectId(oidString); } diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoServiceArOutput.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoServiceArOutput.java index ecd3a20b..9266c1af 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoServiceArOutput.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoServiceArOutput.java @@ -16,6 +16,7 @@ import com.mongodb.client.model.UpdateOptions; import org.bson.types.ObjectId; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; /** @@ -87,7 +88,7 @@ private void initMongo() { this.mDB = mClient.getDatabase(dbName); this.mCol = mDB.getCollection(colName); if (this.clearMongo) { - String oidString = Long.toHexString(new DateTime().getMillis() / 1000L) + "0000000000000000"; + String oidString = Long.toHexString(new DateTime(DateTimeZone.UTC).getMillis() / 1000L) + "0000000000000000"; this.nowId = new ObjectId(oidString); } diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoStatusOutput.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoStatusOutput.java index 0f543f81..23ded98a 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoStatusOutput.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/MongoStatusOutput.java @@ -21,6 +21,7 @@ import org.apache.avro.generic.GenericData; import org.bson.types.ObjectId; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.json.simple.JSONArray; import org.json.simple.JSONObject; @@ -105,7 +106,7 @@ private void initMongo() { this.mDB = mClient.getDatabase(dbName); this.mCol = mDB.getCollection(colName); if (this.clearMongo) { - String oidString = Long.toHexString(new DateTime().getMillis() / 1000L) + "0000000000000000"; + String oidString = Long.toHexString(new DateTime(DateTimeZone.UTC).getMillis() / 1000L) + "0000000000000000"; this.nowId = new ObjectId(oidString); } diff --git a/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/CalcEndpointFlipFlopTrends.java b/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/CalcEndpointFlipFlopTrends.java index f94311ea..9d020225 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/CalcEndpointFlipFlopTrends.java +++ b/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/CalcEndpointFlipFlopTrends.java @@ -13,6 +13,7 @@ import org.apache.flink.api.common.functions.FlatMapFunction; import org.apache.flink.util.Collector; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import timelines.Timeline; @@ -45,7 +46,7 @@ public void flatMap(StatusTimeline in, Collector out) throws Exc ArrayList timestatusList = in.getTimestamps(); TreeMap timestampMap = new TreeMap(); for (TimeStatus ts : timestatusList) { - timestampMap.put(new DateTime(ts.getTimestamp()), ts.getStatus()); + timestampMap.put(new DateTime(ts.getTimestamp(),DateTimeZone.UTC), ts.getStatus()); } Timeline timeline = new Timeline(); diff --git a/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/CalcGroupFlipFlopTrends.java b/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/CalcGroupFlipFlopTrends.java index 25a47b26..d48076cc 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/CalcGroupFlipFlopTrends.java +++ b/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/CalcGroupFlipFlopTrends.java @@ -14,6 +14,7 @@ import org.apache.flink.api.common.functions.FlatMapFunction; import org.apache.flink.util.Collector; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import timelines.Timeline; @@ -37,7 +38,7 @@ public void flatMap(StatusTimeline in, Collector out) throws Except TreeMap timestampMap = new TreeMap(); for (TimeStatus ts : timestatusList) { - timestampMap.put(new DateTime(ts.getTimestamp()), ts.getStatus()); + timestampMap.put(new DateTime(ts.getTimestamp(), DateTimeZone.UTC), ts.getStatus()); } Timeline timeline = new Timeline(); diff --git a/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/CalcMetricFlipFlopTrends.java b/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/CalcMetricFlipFlopTrends.java index 28dfacec..2fba8ad0 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/CalcMetricFlipFlopTrends.java +++ b/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/CalcMetricFlipFlopTrends.java @@ -13,6 +13,7 @@ import org.apache.flink.api.common.functions.FlatMapFunction; import org.apache.flink.util.Collector; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import timelines.Timeline; @@ -61,7 +62,7 @@ public void flatMap(StatusTimeline in, Collector out) throws Excep TreeMap timestampMap = new TreeMap(); for (TimeStatus ts : timestatusList) { - timestampMap.put(new DateTime(ts.getTimestamp()), ts.getStatus()); + timestampMap.put(new DateTime(ts.getTimestamp(), DateTimeZone.UTC), ts.getStatus()); } Timeline timeline = new Timeline(); diff --git a/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/CalcServiceFlipFlopTrends.java b/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/CalcServiceFlipFlopTrends.java index bea07b1d..0f11c439 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/CalcServiceFlipFlopTrends.java +++ b/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/CalcServiceFlipFlopTrends.java @@ -14,6 +14,7 @@ import org.apache.flink.api.common.functions.FlatMapFunction; import org.apache.flink.util.Collector; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import timelines.Timeline; @@ -39,7 +40,7 @@ public void flatMap(StatusTimeline in, Collector out) throws Exce TreeMap timestampMap = new TreeMap(); for (TimeStatus ts : timestatusList) { - timestampMap.put(new DateTime(ts.getTimestamp()), ts.getStatus()); + timestampMap.put(new DateTime(ts.getTimestamp(), DateTimeZone.UTC), ts.getStatus()); } Timeline timeline = new Timeline(); timeline.insertDateTimeStamps(timestampMap, true); diff --git a/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/MongoTrendsOutput.java b/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/MongoTrendsOutput.java index bd12179c..7a2b8f03 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/MongoTrendsOutput.java +++ b/flink_jobs_v2/batch_multi/src/main/java/trends/calculations/MongoTrendsOutput.java @@ -16,6 +16,7 @@ import org.bson.conversions.Bson; import org.bson.types.ObjectId; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; /** * MongoTrendsOutput for storing trends data to mongodb @@ -80,7 +81,7 @@ private void initMongo() { this.mDB = mClient.getDatabase(dbName); this.mCol = mDB.getCollection(colName); if (this.clearMongo) { - String oidString = Long.toHexString(new DateTime().getMillis() / 1000L) + "0000000000000000"; + String oidString = Long.toHexString(new DateTime( DateTimeZone.UTC).getMillis() / 1000L) + "0000000000000000"; this.nowId = new ObjectId(oidString); } diff --git a/flink_jobs_v2/batch_multi/src/main/java/utils/Utils.java b/flink_jobs_v2/batch_multi/src/main/java/utils/Utils.java index 4fa10486..5416e167 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/utils/Utils.java +++ b/flink_jobs_v2/batch_multi/src/main/java/utils/Utils.java @@ -4,6 +4,7 @@ * and open the template in the editor. */ package utils; + import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; @@ -13,6 +14,7 @@ import org.slf4j.LoggerFactory; import java.util.TimeZone; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.joda.time.Minutes; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; @@ -30,35 +32,39 @@ public static String convertDateToString(String format, DateTime date) throws Pa } public static DateTime convertStringtoDate(String format, String dateStr) throws ParseException { - DateTimeFormatter formatter = DateTimeFormat.forPattern(format); - DateTime dt = formatter.parseDateTime(dateStr); - return dt; + + SimpleDateFormat sdf = new SimpleDateFormat(format); + sdf.setTimeZone(TimeZone.getTimeZone("UTC")); + Date date = sdf.parse(dateStr); + return new DateTime(date.getTime(), DateTimeZone.UTC); + } public static DateTime createDate(String format, Date dateStr, int hour, int min, int sec) throws ParseException { //String format = "yyyy-MM-dd'T'HH:mm:ss'Z'"; - SimpleDateFormat sdf = new SimpleDateFormat(format); - sdf.setTimeZone(TimeZone.getDefault()); Calendar newCalendar = Calendar.getInstance(); + newCalendar.setTimeZone(TimeZone.getTimeZone("UTC")); + newCalendar.setTime(dateStr); newCalendar.set(Calendar.HOUR_OF_DAY, hour); newCalendar.set(Calendar.MINUTE, min); newCalendar.set(Calendar.SECOND, sec); newCalendar.set(Calendar.MILLISECOND, 0); - return new DateTime(newCalendar.getTime()); + return new DateTime(newCalendar.getTime(), DateTimeZone.UTC); } public static boolean isPreviousDate(String format, Date nowDate, Date firstDate) throws ParseException { // String format = "yyyy-MM-dd'T'HH:mm:ss'Z'"; Calendar cal = Calendar.getInstance(); - SimpleDateFormat sdf = new SimpleDateFormat(format); - sdf.setTimeZone(TimeZone.getDefault()); + cal.setTimeZone(TimeZone.getTimeZone("UTC")); cal.setTime(nowDate); Calendar calFirst = Calendar.getInstance(); + calFirst.setTimeZone(TimeZone.getTimeZone("UTC")); + calFirst.setTime(firstDate); if (firstDate.before(nowDate)) { @@ -84,10 +90,10 @@ public static boolean checkParameters(ParameterTool params, String... vars) { public static DateTime createDate(String format, int year, int month, int day, int hour, int min, int sec) throws ParseException { // String format = "yyyy-MM-dd'T'HH:mm:ss'Z'"; - SimpleDateFormat sdf = new SimpleDateFormat(format); - sdf.setTimeZone(TimeZone.getDefault()); Calendar newCalendar = Calendar.getInstance(); + newCalendar.setTimeZone(TimeZone.getTimeZone("UTC")); newCalendar.set(Calendar.YEAR, year); + newCalendar.set(Calendar.MONTH, month); newCalendar.set(Calendar.DAY_OF_MONTH, day); @@ -96,22 +102,21 @@ public static DateTime createDate(String format, int year, int month, int day, i newCalendar.set(Calendar.SECOND, sec); newCalendar.set(Calendar.MILLISECOND, 0); - return new DateTime(newCalendar.getTime()); + return new DateTime(newCalendar.getTime(), DateTimeZone.UTC); } public static DateTime setTime(String format, DateTime dateStr, int hour, int min, int sec, int mill) throws ParseException { //String format = "yyyy-MM-dd'T'HH:mm:ss'Z'"; - SimpleDateFormat sdf = new SimpleDateFormat(format); - sdf.setTimeZone(TimeZone.getDefault()); Calendar newCalendar = Calendar.getInstance(); + newCalendar.setTimeZone(TimeZone.getTimeZone("UTC")); newCalendar.setTime(dateStr.toDate()); newCalendar.set(Calendar.HOUR_OF_DAY, hour); newCalendar.set(Calendar.MINUTE, min); newCalendar.set(Calendar.SECOND, sec); newCalendar.set(Calendar.MILLISECOND, mill); - return new DateTime(newCalendar.getTime()); + return new DateTime(newCalendar.getTime(), DateTimeZone.UTC); } public static int calcDayMinutes(DateTime startDay, DateTime endDay) throws ParseException { @@ -124,4 +129,4 @@ public static int calcDayMinutes(DateTime startDay, DateTime endDay) throws Pars return minutesInt; } -} \ No newline at end of file +} diff --git a/flink_jobs_v2/batch_multi/src/test/java/argo/batch/ArgoMultiJobTest.java b/flink_jobs_v2/batch_multi/src/test/java/argo/batch/ArgoMultiJobTest.java index 83afa5e3..ee5446a5 100644 --- a/flink_jobs_v2/batch_multi/src/test/java/argo/batch/ArgoMultiJobTest.java +++ b/flink_jobs_v2/batch_multi/src/test/java/argo/batch/ArgoMultiJobTest.java @@ -93,8 +93,8 @@ public void testMain() throws Exception { System.setProperty("run.date", "2022-01-14"); final ParameterTool params = ParameterTool.fromSystemProperties(); - DateTime now=new DateTime(); - + DateTime now=new DateTime(DateTimeZone.UTC); + ApiResourceManager amr = mockAmr(); DataSource cfgDS = env.fromElements(amr.getResourceJSON(ApiResource.CONFIG)); diff --git a/flink_jobs_v2/batch_multi/src/test/java/argo/batch/TestUtils.java b/flink_jobs_v2/batch_multi/src/test/java/argo/batch/TestUtils.java index 8ea5398d..76e12a71 100644 --- a/flink_jobs_v2/batch_multi/src/test/java/argo/batch/TestUtils.java +++ b/flink_jobs_v2/batch_multi/src/test/java/argo/batch/TestUtils.java @@ -191,7 +191,7 @@ private static ArrayList parseEndpTimelines(ArrayList Date: Wed, 31 Aug 2022 12:51:05 +0300 Subject: [PATCH 07/16] Streaming job doPull, doAck requests to AMS should use the header ams credential method --- .../argo/streaming/ArgoMessagingClient.java | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java index ff17a84c..ddc6179d 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java @@ -181,12 +181,11 @@ public void logIssue(CloseableHttpResponse resp) throws UnsupportedOperationExce * Properly compose url for each AMS request */ public String composeURL(String method) { + if (method.equals("publish")) { return proto + "://" + endpoint + "/v1/projects/" + project + "/topics/" + topic + ":" + method; - } else { - - return proto + "://" + endpoint + "/v1/projects/" + project + "/subscriptions/" + sub + ":" + method + "?key=" - + token; + }else{ + return proto + "://" + endpoint + "/v1/projects/" + project + "/subscriptions/" + sub + ":" + method; } } @@ -201,11 +200,17 @@ public MsgAck doPull() throws IOException, KeyManagementException, NoSuchAlgorit // Create the http post to pull HttpPost postPull = new HttpPost(this.composeURL("pull")); - StringEntity postBody = new StringEntity( - "{\"maxMessages\":\"" + this.maxMessages + "\",\"returnImmediately\":\"true\"}"); + String body = "{\"maxMessages\":\"" + this.maxMessages + "\",\"returnImmediately\":\"true\"}"; + + postPull.addHeader("Accept", "application/json"); + postPull.addHeader("x-api-key", this.token); + postPull.addHeader("Content-type", "application/json"); + + StringEntity postBody = new StringEntity(body); postBody.setContentType("application/json"); + postPull.setEntity(postBody); - + if (this.httpClient == null) { this.httpClient = buildHttpClient(); } @@ -311,7 +316,13 @@ public String doAck(String ackId) throws IOException { // Create the http post to ack HttpPost postAck = new HttpPost(this.composeURL("acknowledge")); - StringEntity postBody = new StringEntity("{\"ackIds\":[" + ackId + "]}"); + String body = "{\"ackIds\":[" + ackId + "]}"; + postAck.addHeader("Accept", "application/json"); + postAck.addHeader("x-api-key", this.token); + postAck.addHeader("Content-type", "application/json"); + + StringEntity postBody = new StringEntity(body); + postBody.setContentType("application/json"); postAck.setEntity(postBody); @@ -361,7 +372,6 @@ public void publish(String in) throws IOException, NoSuchAlgorithmException, Key HttpPost postPublish = new HttpPost(this.composeURL("publish")); String body = "{\"messages\": [ {\"data\":\"" + encodedString + "\"}]}"; - postPublish.addHeader("Accept", "application/json"); postPublish.addHeader("x-api-key", this.token); postPublish.addHeader("Content-type", "application/json"); From 78f906657abdf66eb54eaab661e2a4cd0e544268 Mon Sep 17 00:00:00 2001 From: cthermolia-grnet Date: Wed, 31 Aug 2022 12:51:05 +0300 Subject: [PATCH 08/16] ARGO-3953 Ingestion jobs should use the header ams credential method --- flink_jobs_v2/ams_ingest_metric/.gitignore | 8 + .../ams_ingest_metric/metric_data.avsc | 18 + flink_jobs_v2/ams_ingest_metric/pom.xml | 378 ++++++++ .../src/main/java/argo/avro/MetricData.java | 811 ++++++++++++++++++ .../main/java/argo/avro/MetricDataOld.java | 536 ++++++++++++ .../java/argo/streaming/AmsIngestMetric.java | 263 ++++++ .../argo/streaming/ArgoMessagingClient.java | 351 ++++++++ .../argo/streaming/ArgoMessagingSource.java | 135 +++ .../streaming/HBaseMetricOutputFormat.java | 137 +++ .../main/java/argo/streaming/MetricParse.java | 65 ++ .../argo/streaming/SpecificAvroWriter.java | 93 ++ .../main/java/argo/streaming/TSBucketer.java | 25 + .../src/main/resources/log4j.properties | 23 + .../main/resources/schemas/metric_data.avsc | 18 + flink_jobs_v2/ams_ingest_sync/.gitignore | 8 + flink_jobs_v2/ams_ingest_sync/pom.xml | 346 ++++++++ .../src/main/java/argo/avro/Downtime.java | 286 ++++++ .../main/java/argo/avro/GroupEndpoint.java | 336 ++++++++ .../src/main/java/argo/avro/GroupGroup.java | 286 ++++++ .../main/java/argo/avro/MetricProfile.java | 286 ++++++ .../src/main/java/argo/avro/Weight.java | 236 +++++ .../java/argo/streaming/AmsIngestSync.java | 116 +++ .../argo/streaming/ArgoMessagingClient.java | 350 ++++++++ .../argo/streaming/ArgoMessagingSource.java | 135 +++ .../argo/streaming/SyncHDFSOutputFormat.java | 262 ++++++ .../main/java/argo/streaming/SyncParse.java | 81 ++ .../java/argo/streaming/TestSyncDecoding.java | 80 ++ .../src/test/resources/metric_profile.json | 34 + .../resources/request_metric_profile.json | 12 + flink_jobs_v2/pom.xml | 5 +- .../argo/streaming/ArgoMessagingClient.java | 2 +- 31 files changed, 5720 insertions(+), 2 deletions(-) create mode 100644 flink_jobs_v2/ams_ingest_metric/.gitignore create mode 100644 flink_jobs_v2/ams_ingest_metric/metric_data.avsc create mode 100644 flink_jobs_v2/ams_ingest_metric/pom.xml create mode 100644 flink_jobs_v2/ams_ingest_metric/src/main/java/argo/avro/MetricData.java create mode 100644 flink_jobs_v2/ams_ingest_metric/src/main/java/argo/avro/MetricDataOld.java create mode 100644 flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/AmsIngestMetric.java create mode 100644 flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/ArgoMessagingClient.java create mode 100644 flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/ArgoMessagingSource.java create mode 100644 flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/HBaseMetricOutputFormat.java create mode 100644 flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/MetricParse.java create mode 100644 flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/SpecificAvroWriter.java create mode 100644 flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/TSBucketer.java create mode 100644 flink_jobs_v2/ams_ingest_metric/src/main/resources/log4j.properties create mode 100644 flink_jobs_v2/ams_ingest_metric/src/main/resources/schemas/metric_data.avsc create mode 100644 flink_jobs_v2/ams_ingest_sync/.gitignore create mode 100644 flink_jobs_v2/ams_ingest_sync/pom.xml create mode 100644 flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/Downtime.java create mode 100644 flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/GroupEndpoint.java create mode 100644 flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/GroupGroup.java create mode 100644 flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/MetricProfile.java create mode 100644 flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/Weight.java create mode 100644 flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/AmsIngestSync.java create mode 100644 flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/ArgoMessagingClient.java create mode 100644 flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/ArgoMessagingSource.java create mode 100644 flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/SyncHDFSOutputFormat.java create mode 100644 flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/SyncParse.java create mode 100644 flink_jobs_v2/ams_ingest_sync/src/test/java/argo/streaming/TestSyncDecoding.java create mode 100644 flink_jobs_v2/ams_ingest_sync/src/test/resources/metric_profile.json create mode 100644 flink_jobs_v2/ams_ingest_sync/src/test/resources/request_metric_profile.json diff --git a/flink_jobs_v2/ams_ingest_metric/.gitignore b/flink_jobs_v2/ams_ingest_metric/.gitignore new file mode 100644 index 00000000..6c4e323f --- /dev/null +++ b/flink_jobs_v2/ams_ingest_metric/.gitignore @@ -0,0 +1,8 @@ +/target/ +.project +.settings/ +.classpath/ +.classpath +/nbproject +nbactions.xml + diff --git a/flink_jobs_v2/ams_ingest_metric/metric_data.avsc b/flink_jobs_v2/ams_ingest_metric/metric_data.avsc new file mode 100644 index 00000000..c35bbe38 --- /dev/null +++ b/flink_jobs_v2/ams_ingest_metric/metric_data.avsc @@ -0,0 +1,18 @@ +{"namespace": "argo.avro", + "type": "record", + "name": "metric_data", + "fields": [ + {"name": "timestamp", "type": "string"}, + {"name": "service", "type": "string"}, + {"name": "hostname", "type": "string"}, + {"name": "metric", "type": "string"}, + {"name": "status", "type": "string"}, + {"name": "monitoring_host", "type": ["null", "string"]}, + {"name": "summary", "type": ["null", "string"]}, + {"name": "message", "type": ["null", "string"]}, + {"name": "tags", "type" : ["null", {"name" : "Tags", + "type" : "map", + "values" : ["null", "string"] + }] + }] +} diff --git a/flink_jobs_v2/ams_ingest_metric/pom.xml b/flink_jobs_v2/ams_ingest_metric/pom.xml new file mode 100644 index 00000000..e2fb50ce --- /dev/null +++ b/flink_jobs_v2/ams_ingest_metric/pom.xml @@ -0,0 +1,378 @@ + + + 4.0.0 + + argo.streaming + ams-ingest-metric + 0.1 + jar + + ARGO AMS Ingest Metric Data job + + + + + UTF-8 + 1.3.2 + + + + + + cloudera + https://repository.cloudera.com/artifactory/cloudera-repos/ + + + + apache.snapshots + Apache Development Snapshot Repository + https://repository.apache.org/content/repositories/snapshots/ + + false + + + true + + + + + + + + + org.apache.avro + avro + 1.7.7 + + + org.apache.flink + flink-connector-filesystem_2.10 + ${flink.version} + + + org.apache.flink + flink-java + ${flink.version} + + + org.apache.flink + flink-streaming-java_2.10 + ${flink.version} + + + org.apache.flink + flink-clients_2.10 + ${flink.version} + + + org.apache.flink + flink-connector-kafka-0.9_2.10 + ${flink.version} + + + commons-codec + commons-codec + 20041127.091804 + + + com.google.code.gson + gson + 2.8.9 + + + + org.apache.hbase + hbase-client + 1.2.0-cdh5.7.4 + + + + org.apache.httpcomponents + httpclient + 4.5.13 + + + org.apache.httpcomponents + fluent-hc + 4.5.13 + + + + + + + + + + + build-jar + + false + + + + org.apache.avro + avro + 1.7.7 + + + org.apache.flink + flink-java + ${flink.version} + provided + + + org.apache.flink + flink-connector-filesystem_2.10 + ${flink.version} + + + org.apache.flink + flink-streaming-java_2.10 + ${flink.version} + provided + + + org.apache.flink + flink-clients_2.10 + ${flink.version} + provided + + + org.apache.flink + flink-connector-kafka-0.9_2.10 + ${flink.version} + provided + + + commons-codec + commons-codec + 20041127.091804 + provided + + + + com.google.code.gson + gson + 2.8.9 + + + + org.apache.hbase + hbase-client + 1.2.0-cdh5.7.4 + + + org.apache.httpcomponents + httpclient + 4.5.13 + + + org.apache.httpcomponents + fluent-hc + 4.5.13 + + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.1 + + + package + + shade + + + + + + + + + + + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.1 + + + + package + + shade + + + + + + org.apache.flink:flink-annotations + org.apache.flink:flink-shaded-hadoop1 + org.apache.flink:flink-shaded-hadoop2 + org.apache.flink:flink-shaded-curator-recipes + org.apache.flink:flink-core + org.apache.flink:flink-java + org.apache.flink:flink-scala_2.10 + org.apache.flink:flink-runtime_2.10 + org.apache.flink:flink-optimizer_2.10 + org.apache.flink:flink-clients_2.10 + org.apache.flink:flink-avro_2.10 + org.apache.flink:flink-examples-batch_2.10 + org.apache.flink:flink-examples-streaming_2.10 + org.apache.flink:flink-streaming-java_2.10 + + + org.scala-lang:scala-library + org.scala-lang:scala-compiler + org.scala-lang:scala-reflect + com.amazonaws:aws-java-sdk + com.typesafe.akka:akka-actor_* + com.typesafe.akka:akka-remote_* + com.typesafe.akka:akka-slf4j_* + io.netty:netty-all + io.netty:netty + commons-fileupload:commons-fileupload + org.apache.avro:avro + commons-collections:commons-collections + org.codehaus.jackson:jackson-core-asl + org.codehaus.jackson:jackson-mapper-asl + com.thoughtworks.paranamer:paranamer + org.xerial.snappy:snappy-java + org.apache.commons:commons-compress + org.tukaani:xz + com.esotericsoftware.kryo:kryo + com.esotericsoftware.minlog:minlog + org.objenesis:objenesis + com.twitter:chill_* + com.twitter:chill-java + com.twitter:chill-avro_* + com.twitter:chill-bijection_* + com.twitter:bijection-core_* + com.twitter:bijection-avro_* + commons-lang:commons-lang + junit:junit + de.javakaffee:kryo-serializers + joda-time:joda-time + org.apache.commons:commons-lang3 + org.slf4j:slf4j-api + org.slf4j:slf4j-log4j12 + log4j:log4j + org.apache.commons:commons-math + org.apache.sling:org.apache.sling.commons.json + commons-logging:commons-logging + commons-codec:commons-codec + com.fasterxml.jackson.core:jackson-core + com.fasterxml.jackson.core:jackson-databind + com.fasterxml.jackson.core:jackson-annotations + stax:stax-api + com.typesafe:config + org.uncommons.maths:uncommons-maths + com.github.scopt:scopt_* + commons-io:commons-io + commons-cli:commons-cli + + + + + org.apache.flink:* + + + org/apache/flink/shaded/com/** + web-docs/** + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + argo.streaming.AmsIngestMetric + + + false + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.7 + 1.7 + + + + + + + + + + diff --git a/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/avro/MetricData.java b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/avro/MetricData.java new file mode 100644 index 00000000..77800770 --- /dev/null +++ b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/avro/MetricData.java @@ -0,0 +1,811 @@ +/** + * Autogenerated by Avro + * + * DO NOT EDIT DIRECTLY + */ +package argo.avro; + +import org.apache.avro.specific.SpecificData; + +@SuppressWarnings("all") +@org.apache.avro.specific.AvroGenerated +public class MetricData extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { + private static final long serialVersionUID = 3861438289744595870L; + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"MetricData\",\"namespace\":\"argo.avro\",\"fields\":[{\"name\":\"timestamp\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"service\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"hostname\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"metric\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"status\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"monitoring_host\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}]},{\"name\":\"actual_data\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"summary\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}]},{\"name\":\"message\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}]},{\"name\":\"tags\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"avro.java.string\":\"String\"}]}]}"); + public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } + @Deprecated public java.lang.String timestamp; + @Deprecated public java.lang.String service; + @Deprecated public java.lang.String hostname; + @Deprecated public java.lang.String metric; + @Deprecated public java.lang.String status; + @Deprecated public java.lang.String monitoring_host; + @Deprecated public java.lang.String actual_data; + @Deprecated public java.lang.String summary; + @Deprecated public java.lang.String message; + @Deprecated public java.util.Map tags; + + /** + * Default constructor. Note that this does not initialize fields + * to their default values from the schema. If that is desired then + * one should use newBuilder(). + */ + public MetricData() {} + + /** + * All-args constructor. + * @param timestamp The new value for timestamp + * @param service The new value for service + * @param hostname The new value for hostname + * @param metric The new value for metric + * @param status The new value for status + * @param monitoring_host The new value for monitoring_host + * @param actual_data The new value for actual_data + * @param summary The new value for summary + * @param message The new value for message + * @param tags The new value for tags + */ + public MetricData(java.lang.String timestamp, java.lang.String service, java.lang.String hostname, java.lang.String metric, java.lang.String status, java.lang.String monitoring_host, java.lang.String actual_data, java.lang.String summary, java.lang.String message, java.util.Map tags) { + this.timestamp = timestamp; + this.service = service; + this.hostname = hostname; + this.metric = metric; + this.status = status; + this.monitoring_host = monitoring_host; + this.actual_data = actual_data; + this.summary = summary; + this.message = message; + this.tags = tags; + } + + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + // Used by DatumWriter. Applications should not call. + public java.lang.Object get(int field$) { + switch (field$) { + case 0: return timestamp; + case 1: return service; + case 2: return hostname; + case 3: return metric; + case 4: return status; + case 5: return monitoring_host; + case 6: return actual_data; + case 7: return summary; + case 8: return message; + case 9: return tags; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + // Used by DatumReader. Applications should not call. + @SuppressWarnings(value="unchecked") + public void put(int field$, java.lang.Object value$) { + switch (field$) { + case 0: timestamp = (java.lang.String)value$; break; + case 1: service = (java.lang.String)value$; break; + case 2: hostname = (java.lang.String)value$; break; + case 3: metric = (java.lang.String)value$; break; + case 4: status = (java.lang.String)value$; break; + case 5: monitoring_host = (java.lang.String)value$; break; + case 6: actual_data = (java.lang.String)value$; break; + case 7: summary = (java.lang.String)value$; break; + case 8: message = (java.lang.String)value$; break; + case 9: tags = (java.util.Map)value$; break; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + /** + * Gets the value of the 'timestamp' field. + * @return The value of the 'timestamp' field. + */ + public java.lang.String getTimestamp() { + return timestamp; + } + + /** + * Sets the value of the 'timestamp' field. + * @param value the value to set. + */ + public void setTimestamp(java.lang.String value) { + this.timestamp = value; + } + + /** + * Gets the value of the 'service' field. + * @return The value of the 'service' field. + */ + public java.lang.String getService() { + return service; + } + + /** + * Sets the value of the 'service' field. + * @param value the value to set. + */ + public void setService(java.lang.String value) { + this.service = value; + } + + /** + * Gets the value of the 'hostname' field. + * @return The value of the 'hostname' field. + */ + public java.lang.String getHostname() { + return hostname; + } + + /** + * Sets the value of the 'hostname' field. + * @param value the value to set. + */ + public void setHostname(java.lang.String value) { + this.hostname = value; + } + + /** + * Gets the value of the 'metric' field. + * @return The value of the 'metric' field. + */ + public java.lang.String getMetric() { + return metric; + } + + /** + * Sets the value of the 'metric' field. + * @param value the value to set. + */ + public void setMetric(java.lang.String value) { + this.metric = value; + } + + /** + * Gets the value of the 'status' field. + * @return The value of the 'status' field. + */ + public java.lang.String getStatus() { + return status; + } + + /** + * Sets the value of the 'status' field. + * @param value the value to set. + */ + public void setStatus(java.lang.String value) { + this.status = value; + } + + /** + * Gets the value of the 'monitoring_host' field. + * @return The value of the 'monitoring_host' field. + */ + public java.lang.String getMonitoringHost() { + return monitoring_host; + } + + /** + * Sets the value of the 'monitoring_host' field. + * @param value the value to set. + */ + public void setMonitoringHost(java.lang.String value) { + this.monitoring_host = value; + } + + /** + * Gets the value of the 'actual_data' field. + * @return The value of the 'actual_data' field. + */ + public java.lang.String getActualData() { + return actual_data; + } + + /** + * Sets the value of the 'actual_data' field. + * @param value the value to set. + */ + public void setActualData(java.lang.String value) { + this.actual_data = value; + } + + /** + * Gets the value of the 'summary' field. + * @return The value of the 'summary' field. + */ + public java.lang.String getSummary() { + return summary; + } + + /** + * Sets the value of the 'summary' field. + * @param value the value to set. + */ + public void setSummary(java.lang.String value) { + this.summary = value; + } + + /** + * Gets the value of the 'message' field. + * @return The value of the 'message' field. + */ + public java.lang.String getMessage() { + return message; + } + + /** + * Sets the value of the 'message' field. + * @param value the value to set. + */ + public void setMessage(java.lang.String value) { + this.message = value; + } + + /** + * Gets the value of the 'tags' field. + * @return The value of the 'tags' field. + */ + public java.util.Map getTags() { + return tags; + } + + /** + * Sets the value of the 'tags' field. + * @param value the value to set. + */ + public void setTags(java.util.Map value) { + this.tags = value; + } + + /** + * Creates a new MetricData RecordBuilder. + * @return A new MetricData RecordBuilder + */ + public static argo.avro.MetricData.Builder newBuilder() { + return new argo.avro.MetricData.Builder(); + } + + /** + * Creates a new MetricData RecordBuilder by copying an existing Builder. + * @param other The existing builder to copy. + * @return A new MetricData RecordBuilder + */ + public static argo.avro.MetricData.Builder newBuilder(argo.avro.MetricData.Builder other) { + return new argo.avro.MetricData.Builder(other); + } + + /** + * Creates a new MetricData RecordBuilder by copying an existing MetricData instance. + * @param other The existing instance to copy. + * @return A new MetricData RecordBuilder + */ + public static argo.avro.MetricData.Builder newBuilder(argo.avro.MetricData other) { + return new argo.avro.MetricData.Builder(other); + } + + /** + * RecordBuilder for MetricData instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private java.lang.String timestamp; + private java.lang.String service; + private java.lang.String hostname; + private java.lang.String metric; + private java.lang.String status; + private java.lang.String monitoring_host; + private java.lang.String actual_data; + private java.lang.String summary; + private java.lang.String message; + private java.util.Map tags; + + /** Creates a new Builder */ + private Builder() { + super(SCHEMA$); + } + + /** + * Creates a Builder by copying an existing Builder. + * @param other The existing Builder to copy. + */ + private Builder(argo.avro.MetricData.Builder other) { + super(other); + if (isValidValue(fields()[0], other.timestamp)) { + this.timestamp = data().deepCopy(fields()[0].schema(), other.timestamp); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.service)) { + this.service = data().deepCopy(fields()[1].schema(), other.service); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.hostname)) { + this.hostname = data().deepCopy(fields()[2].schema(), other.hostname); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.metric)) { + this.metric = data().deepCopy(fields()[3].schema(), other.metric); + fieldSetFlags()[3] = true; + } + if (isValidValue(fields()[4], other.status)) { + this.status = data().deepCopy(fields()[4].schema(), other.status); + fieldSetFlags()[4] = true; + } + if (isValidValue(fields()[5], other.monitoring_host)) { + this.monitoring_host = data().deepCopy(fields()[5].schema(), other.monitoring_host); + fieldSetFlags()[5] = true; + } + if (isValidValue(fields()[6], other.actual_data)) { + this.actual_data = data().deepCopy(fields()[6].schema(), other.actual_data); + fieldSetFlags()[6] = true; + } + if (isValidValue(fields()[7], other.summary)) { + this.summary = data().deepCopy(fields()[7].schema(), other.summary); + fieldSetFlags()[7] = true; + } + if (isValidValue(fields()[8], other.message)) { + this.message = data().deepCopy(fields()[8].schema(), other.message); + fieldSetFlags()[8] = true; + } + if (isValidValue(fields()[9], other.tags)) { + this.tags = data().deepCopy(fields()[9].schema(), other.tags); + fieldSetFlags()[9] = true; + } + } + + /** + * Creates a Builder by copying an existing MetricData instance + * @param other The existing instance to copy. + */ + private Builder(argo.avro.MetricData other) { + super(SCHEMA$); + if (isValidValue(fields()[0], other.timestamp)) { + this.timestamp = data().deepCopy(fields()[0].schema(), other.timestamp); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.service)) { + this.service = data().deepCopy(fields()[1].schema(), other.service); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.hostname)) { + this.hostname = data().deepCopy(fields()[2].schema(), other.hostname); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.metric)) { + this.metric = data().deepCopy(fields()[3].schema(), other.metric); + fieldSetFlags()[3] = true; + } + if (isValidValue(fields()[4], other.status)) { + this.status = data().deepCopy(fields()[4].schema(), other.status); + fieldSetFlags()[4] = true; + } + if (isValidValue(fields()[5], other.monitoring_host)) { + this.monitoring_host = data().deepCopy(fields()[5].schema(), other.monitoring_host); + fieldSetFlags()[5] = true; + } + if (isValidValue(fields()[6], other.actual_data)) { + this.actual_data = data().deepCopy(fields()[6].schema(), other.actual_data); + fieldSetFlags()[6] = true; + } + if (isValidValue(fields()[7], other.summary)) { + this.summary = data().deepCopy(fields()[7].schema(), other.summary); + fieldSetFlags()[7] = true; + } + if (isValidValue(fields()[8], other.message)) { + this.message = data().deepCopy(fields()[8].schema(), other.message); + fieldSetFlags()[8] = true; + } + if (isValidValue(fields()[9], other.tags)) { + this.tags = data().deepCopy(fields()[9].schema(), other.tags); + fieldSetFlags()[9] = true; + } + } + + /** + * Gets the value of the 'timestamp' field. + * @return The value. + */ + public java.lang.String getTimestamp() { + return timestamp; + } + + /** + * Sets the value of the 'timestamp' field. + * @param value The value of 'timestamp'. + * @return This builder. + */ + public argo.avro.MetricData.Builder setTimestamp(java.lang.String value) { + validate(fields()[0], value); + this.timestamp = value; + fieldSetFlags()[0] = true; + return this; + } + + /** + * Checks whether the 'timestamp' field has been set. + * @return True if the 'timestamp' field has been set, false otherwise. + */ + public boolean hasTimestamp() { + return fieldSetFlags()[0]; + } + + + /** + * Clears the value of the 'timestamp' field. + * @return This builder. + */ + public argo.avro.MetricData.Builder clearTimestamp() { + timestamp = null; + fieldSetFlags()[0] = false; + return this; + } + + /** + * Gets the value of the 'service' field. + * @return The value. + */ + public java.lang.String getService() { + return service; + } + + /** + * Sets the value of the 'service' field. + * @param value The value of 'service'. + * @return This builder. + */ + public argo.avro.MetricData.Builder setService(java.lang.String value) { + validate(fields()[1], value); + this.service = value; + fieldSetFlags()[1] = true; + return this; + } + + /** + * Checks whether the 'service' field has been set. + * @return True if the 'service' field has been set, false otherwise. + */ + public boolean hasService() { + return fieldSetFlags()[1]; + } + + + /** + * Clears the value of the 'service' field. + * @return This builder. + */ + public argo.avro.MetricData.Builder clearService() { + service = null; + fieldSetFlags()[1] = false; + return this; + } + + /** + * Gets the value of the 'hostname' field. + * @return The value. + */ + public java.lang.String getHostname() { + return hostname; + } + + /** + * Sets the value of the 'hostname' field. + * @param value The value of 'hostname'. + * @return This builder. + */ + public argo.avro.MetricData.Builder setHostname(java.lang.String value) { + validate(fields()[2], value); + this.hostname = value; + fieldSetFlags()[2] = true; + return this; + } + + /** + * Checks whether the 'hostname' field has been set. + * @return True if the 'hostname' field has been set, false otherwise. + */ + public boolean hasHostname() { + return fieldSetFlags()[2]; + } + + + /** + * Clears the value of the 'hostname' field. + * @return This builder. + */ + public argo.avro.MetricData.Builder clearHostname() { + hostname = null; + fieldSetFlags()[2] = false; + return this; + } + + /** + * Gets the value of the 'metric' field. + * @return The value. + */ + public java.lang.String getMetric() { + return metric; + } + + /** + * Sets the value of the 'metric' field. + * @param value The value of 'metric'. + * @return This builder. + */ + public argo.avro.MetricData.Builder setMetric(java.lang.String value) { + validate(fields()[3], value); + this.metric = value; + fieldSetFlags()[3] = true; + return this; + } + + /** + * Checks whether the 'metric' field has been set. + * @return True if the 'metric' field has been set, false otherwise. + */ + public boolean hasMetric() { + return fieldSetFlags()[3]; + } + + + /** + * Clears the value of the 'metric' field. + * @return This builder. + */ + public argo.avro.MetricData.Builder clearMetric() { + metric = null; + fieldSetFlags()[3] = false; + return this; + } + + /** + * Gets the value of the 'status' field. + * @return The value. + */ + public java.lang.String getStatus() { + return status; + } + + /** + * Sets the value of the 'status' field. + * @param value The value of 'status'. + * @return This builder. + */ + public argo.avro.MetricData.Builder setStatus(java.lang.String value) { + validate(fields()[4], value); + this.status = value; + fieldSetFlags()[4] = true; + return this; + } + + /** + * Checks whether the 'status' field has been set. + * @return True if the 'status' field has been set, false otherwise. + */ + public boolean hasStatus() { + return fieldSetFlags()[4]; + } + + + /** + * Clears the value of the 'status' field. + * @return This builder. + */ + public argo.avro.MetricData.Builder clearStatus() { + status = null; + fieldSetFlags()[4] = false; + return this; + } + + /** + * Gets the value of the 'monitoring_host' field. + * @return The value. + */ + public java.lang.String getMonitoringHost() { + return monitoring_host; + } + + /** + * Sets the value of the 'monitoring_host' field. + * @param value The value of 'monitoring_host'. + * @return This builder. + */ + public argo.avro.MetricData.Builder setMonitoringHost(java.lang.String value) { + validate(fields()[5], value); + this.monitoring_host = value; + fieldSetFlags()[5] = true; + return this; + } + + /** + * Checks whether the 'monitoring_host' field has been set. + * @return True if the 'monitoring_host' field has been set, false otherwise. + */ + public boolean hasMonitoringHost() { + return fieldSetFlags()[5]; + } + + + /** + * Clears the value of the 'monitoring_host' field. + * @return This builder. + */ + public argo.avro.MetricData.Builder clearMonitoringHost() { + monitoring_host = null; + fieldSetFlags()[5] = false; + return this; + } + + /** + * Gets the value of the 'actual_data' field. + * @return The value. + */ + public java.lang.String getActualData() { + return actual_data; + } + + /** + * Sets the value of the 'actual_data' field. + * @param value The value of 'actual_data'. + * @return This builder. + */ + public argo.avro.MetricData.Builder setActualData(java.lang.String value) { + validate(fields()[6], value); + this.actual_data = value; + fieldSetFlags()[6] = true; + return this; + } + + /** + * Checks whether the 'actual_data' field has been set. + * @return True if the 'actual_data' field has been set, false otherwise. + */ + public boolean hasActualData() { + return fieldSetFlags()[6]; + } + + + /** + * Clears the value of the 'actual_data' field. + * @return This builder. + */ + public argo.avro.MetricData.Builder clearActualData() { + actual_data = null; + fieldSetFlags()[6] = false; + return this; + } + + /** + * Gets the value of the 'summary' field. + * @return The value. + */ + public java.lang.String getSummary() { + return summary; + } + + /** + * Sets the value of the 'summary' field. + * @param value The value of 'summary'. + * @return This builder. + */ + public argo.avro.MetricData.Builder setSummary(java.lang.String value) { + validate(fields()[7], value); + this.summary = value; + fieldSetFlags()[7] = true; + return this; + } + + /** + * Checks whether the 'summary' field has been set. + * @return True if the 'summary' field has been set, false otherwise. + */ + public boolean hasSummary() { + return fieldSetFlags()[7]; + } + + + /** + * Clears the value of the 'summary' field. + * @return This builder. + */ + public argo.avro.MetricData.Builder clearSummary() { + summary = null; + fieldSetFlags()[7] = false; + return this; + } + + /** + * Gets the value of the 'message' field. + * @return The value. + */ + public java.lang.String getMessage() { + return message; + } + + /** + * Sets the value of the 'message' field. + * @param value The value of 'message'. + * @return This builder. + */ + public argo.avro.MetricData.Builder setMessage(java.lang.String value) { + validate(fields()[8], value); + this.message = value; + fieldSetFlags()[8] = true; + return this; + } + + /** + * Checks whether the 'message' field has been set. + * @return True if the 'message' field has been set, false otherwise. + */ + public boolean hasMessage() { + return fieldSetFlags()[8]; + } + + + /** + * Clears the value of the 'message' field. + * @return This builder. + */ + public argo.avro.MetricData.Builder clearMessage() { + message = null; + fieldSetFlags()[8] = false; + return this; + } + + /** + * Gets the value of the 'tags' field. + * @return The value. + */ + public java.util.Map getTags() { + return tags; + } + + /** + * Sets the value of the 'tags' field. + * @param value The value of 'tags'. + * @return This builder. + */ + public argo.avro.MetricData.Builder setTags(java.util.Map value) { + validate(fields()[9], value); + this.tags = value; + fieldSetFlags()[9] = true; + return this; + } + + /** + * Checks whether the 'tags' field has been set. + * @return True if the 'tags' field has been set, false otherwise. + */ + public boolean hasTags() { + return fieldSetFlags()[9]; + } + + + /** + * Clears the value of the 'tags' field. + * @return This builder. + */ + public argo.avro.MetricData.Builder clearTags() { + tags = null; + fieldSetFlags()[9] = false; + return this; + } + + @Override + public MetricData build() { + try { + MetricData record = new MetricData(); + record.timestamp = fieldSetFlags()[0] ? this.timestamp : (java.lang.String) defaultValue(fields()[0]); + record.service = fieldSetFlags()[1] ? this.service : (java.lang.String) defaultValue(fields()[1]); + record.hostname = fieldSetFlags()[2] ? this.hostname : (java.lang.String) defaultValue(fields()[2]); + record.metric = fieldSetFlags()[3] ? this.metric : (java.lang.String) defaultValue(fields()[3]); + record.status = fieldSetFlags()[4] ? this.status : (java.lang.String) defaultValue(fields()[4]); + record.monitoring_host = fieldSetFlags()[5] ? this.monitoring_host : (java.lang.String) defaultValue(fields()[5]); + record.actual_data = fieldSetFlags()[6] ? this.actual_data : (java.lang.String) defaultValue(fields()[6]); + record.summary = fieldSetFlags()[7] ? this.summary : (java.lang.String) defaultValue(fields()[7]); + record.message = fieldSetFlags()[8] ? this.message : (java.lang.String) defaultValue(fields()[8]); + record.tags = fieldSetFlags()[9] ? this.tags : (java.util.Map) defaultValue(fields()[9]); + return record; + } catch (Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } + +} diff --git a/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/avro/MetricDataOld.java b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/avro/MetricDataOld.java new file mode 100644 index 00000000..dafb8d40 --- /dev/null +++ b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/avro/MetricDataOld.java @@ -0,0 +1,536 @@ +/** + * Autogenerated by Avro + * + * DO NOT EDIT DIRECTLY + */ +package argo.avro; +@SuppressWarnings("all") +@org.apache.avro.specific.AvroGenerated +public class MetricDataOld extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"MetricData\",\"namespace\":\"argo.avro\",\"fields\":[{\"name\":\"timestamp\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"service\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"hostname\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"metric\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"status\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"monitoring_host\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}]},{\"name\":\"summary\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}]},{\"name\":\"message\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}]},{\"name\":\"tags\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"avro.java.string\":\"String\"}]}]}"); + public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } + @Deprecated public java.lang.String timestamp; + @Deprecated public java.lang.String service; + @Deprecated public java.lang.String hostname; + @Deprecated public java.lang.String metric; + @Deprecated public java.lang.String status; + @Deprecated public java.lang.String monitoring_host; + @Deprecated public java.lang.String summary; + @Deprecated public java.lang.String message; + @Deprecated public java.util.Map tags; + + /** + * Default constructor. + */ + public MetricDataOld() {} + + /** + * All-args constructor. + */ + public MetricDataOld(java.lang.String timestamp, java.lang.String service, java.lang.String hostname, java.lang.String metric, java.lang.String status, java.lang.String monitoring_host, java.lang.String summary, java.lang.String message, java.util.Map tags) { + this.timestamp = timestamp; + this.service = service; + this.hostname = hostname; + this.metric = metric; + this.status = status; + this.monitoring_host = monitoring_host; + this.summary = summary; + this.message = message; + this.tags = tags; + } + + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + // Used by DatumWriter. Applications should not call. + public java.lang.Object get(int field$) { + switch (field$) { + case 0: return timestamp; + case 1: return service; + case 2: return hostname; + case 3: return metric; + case 4: return status; + case 5: return monitoring_host; + case 6: return summary; + case 7: return message; + case 8: return tags; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + // Used by DatumReader. Applications should not call. + @SuppressWarnings(value="unchecked") + public void put(int field$, java.lang.Object value$) { + switch (field$) { + case 0: timestamp = (java.lang.String)value$; break; + case 1: service = (java.lang.String)value$; break; + case 2: hostname = (java.lang.String)value$; break; + case 3: metric = (java.lang.String)value$; break; + case 4: status = (java.lang.String)value$; break; + case 5: monitoring_host = (java.lang.String)value$; break; + case 6: summary = (java.lang.String)value$; break; + case 7: message = (java.lang.String)value$; break; + case 8: tags = (java.util.Map)value$; break; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + /** + * Gets the value of the 'timestamp' field. + */ + public java.lang.String getTimestamp() { + return timestamp; + } + + /** + * Sets the value of the 'timestamp' field. + * @param value the value to set. + */ + public void setTimestamp(java.lang.String value) { + this.timestamp = value; + } + + /** + * Gets the value of the 'service' field. + */ + public java.lang.String getService() { + return service; + } + + /** + * Sets the value of the 'service' field. + * @param value the value to set. + */ + public void setService(java.lang.String value) { + this.service = value; + } + + /** + * Gets the value of the 'hostname' field. + */ + public java.lang.String getHostname() { + return hostname; + } + + /** + * Sets the value of the 'hostname' field. + * @param value the value to set. + */ + public void setHostname(java.lang.String value) { + this.hostname = value; + } + + /** + * Gets the value of the 'metric' field. + */ + public java.lang.String getMetric() { + return metric; + } + + /** + * Sets the value of the 'metric' field. + * @param value the value to set. + */ + public void setMetric(java.lang.String value) { + this.metric = value; + } + + /** + * Gets the value of the 'status' field. + */ + public java.lang.String getStatus() { + return status; + } + + /** + * Sets the value of the 'status' field. + * @param value the value to set. + */ + public void setStatus(java.lang.String value) { + this.status = value; + } + + /** + * Gets the value of the 'monitoring_host' field. + */ + public java.lang.String getMonitoringHost() { + return monitoring_host; + } + + /** + * Sets the value of the 'monitoring_host' field. + * @param value the value to set. + */ + public void setMonitoringHost(java.lang.String value) { + this.monitoring_host = value; + } + + /** + * Gets the value of the 'summary' field. + */ + public java.lang.String getSummary() { + return summary; + } + + /** + * Sets the value of the 'summary' field. + * @param value the value to set. + */ + public void setSummary(java.lang.String value) { + this.summary = value; + } + + /** + * Gets the value of the 'message' field. + */ + public java.lang.String getMessage() { + return message; + } + + /** + * Sets the value of the 'message' field. + * @param value the value to set. + */ + public void setMessage(java.lang.String value) { + this.message = value; + } + + /** + * Gets the value of the 'tags' field. + */ + public java.util.Map getTags() { + return tags; + } + + /** + * Sets the value of the 'tags' field. + * @param value the value to set. + */ + public void setTags(java.util.Map value) { + this.tags = value; + } + + /** Creates a new MetricData RecordBuilder */ + public static argo.avro.MetricDataOld.Builder newBuilder() { + return new argo.avro.MetricDataOld.Builder(); + } + + /** Creates a new MetricData RecordBuilder by copying an existing Builder */ + public static argo.avro.MetricDataOld.Builder newBuilder(argo.avro.MetricDataOld.Builder other) { + return new argo.avro.MetricDataOld.Builder(other); + } + + /** Creates a new MetricData RecordBuilder by copying an existing MetricData instance */ + public static argo.avro.MetricDataOld.Builder newBuilder(argo.avro.MetricDataOld other) { + return new argo.avro.MetricDataOld.Builder(other); + } + + /** + * RecordBuilder for MetricData instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private java.lang.String timestamp; + private java.lang.String service; + private java.lang.String hostname; + private java.lang.String metric; + private java.lang.String status; + private java.lang.String monitoring_host; + private java.lang.String summary; + private java.lang.String message; + private java.util.Map tags; + + /** Creates a new Builder */ + private Builder() { + super(argo.avro.MetricDataOld.SCHEMA$); + } + + /** Creates a Builder by copying an existing Builder */ + private Builder(argo.avro.MetricDataOld.Builder other) { + super(other); + } + + /** Creates a Builder by copying an existing MetricData instance */ + private Builder(argo.avro.MetricDataOld other) { + super(argo.avro.MetricDataOld.SCHEMA$); + if (isValidValue(fields()[0], other.timestamp)) { + this.timestamp = data().deepCopy(fields()[0].schema(), other.timestamp); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.service)) { + this.service = data().deepCopy(fields()[1].schema(), other.service); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.hostname)) { + this.hostname = data().deepCopy(fields()[2].schema(), other.hostname); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.metric)) { + this.metric = data().deepCopy(fields()[3].schema(), other.metric); + fieldSetFlags()[3] = true; + } + if (isValidValue(fields()[4], other.status)) { + this.status = data().deepCopy(fields()[4].schema(), other.status); + fieldSetFlags()[4] = true; + } + if (isValidValue(fields()[5], other.monitoring_host)) { + this.monitoring_host = data().deepCopy(fields()[5].schema(), other.monitoring_host); + fieldSetFlags()[5] = true; + } + if (isValidValue(fields()[6], other.summary)) { + this.summary = data().deepCopy(fields()[6].schema(), other.summary); + fieldSetFlags()[6] = true; + } + if (isValidValue(fields()[7], other.message)) { + this.message = data().deepCopy(fields()[7].schema(), other.message); + fieldSetFlags()[7] = true; + } + if (isValidValue(fields()[8], other.tags)) { + this.tags = data().deepCopy(fields()[8].schema(), other.tags); + fieldSetFlags()[8] = true; + } + } + + /** Gets the value of the 'timestamp' field */ + public java.lang.String getTimestamp() { + return timestamp; + } + + /** Sets the value of the 'timestamp' field */ + public argo.avro.MetricDataOld.Builder setTimestamp(java.lang.String value) { + validate(fields()[0], value); + this.timestamp = value; + fieldSetFlags()[0] = true; + return this; + } + + /** Checks whether the 'timestamp' field has been set */ + public boolean hasTimestamp() { + return fieldSetFlags()[0]; + } + + /** Clears the value of the 'timestamp' field */ + public argo.avro.MetricDataOld.Builder clearTimestamp() { + timestamp = null; + fieldSetFlags()[0] = false; + return this; + } + + /** Gets the value of the 'service' field */ + public java.lang.String getService() { + return service; + } + + /** Sets the value of the 'service' field */ + public argo.avro.MetricDataOld.Builder setService(java.lang.String value) { + validate(fields()[1], value); + this.service = value; + fieldSetFlags()[1] = true; + return this; + } + + /** Checks whether the 'service' field has been set */ + public boolean hasService() { + return fieldSetFlags()[1]; + } + + /** Clears the value of the 'service' field */ + public argo.avro.MetricDataOld.Builder clearService() { + service = null; + fieldSetFlags()[1] = false; + return this; + } + + /** Gets the value of the 'hostname' field */ + public java.lang.String getHostname() { + return hostname; + } + + /** Sets the value of the 'hostname' field */ + public argo.avro.MetricDataOld.Builder setHostname(java.lang.String value) { + validate(fields()[2], value); + this.hostname = value; + fieldSetFlags()[2] = true; + return this; + } + + /** Checks whether the 'hostname' field has been set */ + public boolean hasHostname() { + return fieldSetFlags()[2]; + } + + /** Clears the value of the 'hostname' field */ + public argo.avro.MetricDataOld.Builder clearHostname() { + hostname = null; + fieldSetFlags()[2] = false; + return this; + } + + /** Gets the value of the 'metric' field */ + public java.lang.String getMetric() { + return metric; + } + + /** Sets the value of the 'metric' field */ + public argo.avro.MetricDataOld.Builder setMetric(java.lang.String value) { + validate(fields()[3], value); + this.metric = value; + fieldSetFlags()[3] = true; + return this; + } + + /** Checks whether the 'metric' field has been set */ + public boolean hasMetric() { + return fieldSetFlags()[3]; + } + + /** Clears the value of the 'metric' field */ + public argo.avro.MetricDataOld.Builder clearMetric() { + metric = null; + fieldSetFlags()[3] = false; + return this; + } + + /** Gets the value of the 'status' field */ + public java.lang.String getStatus() { + return status; + } + + /** Sets the value of the 'status' field */ + public argo.avro.MetricDataOld.Builder setStatus(java.lang.String value) { + validate(fields()[4], value); + this.status = value; + fieldSetFlags()[4] = true; + return this; + } + + /** Checks whether the 'status' field has been set */ + public boolean hasStatus() { + return fieldSetFlags()[4]; + } + + /** Clears the value of the 'status' field */ + public argo.avro.MetricDataOld.Builder clearStatus() { + status = null; + fieldSetFlags()[4] = false; + return this; + } + + /** Gets the value of the 'monitoring_host' field */ + public java.lang.String getMonitoringHost() { + return monitoring_host; + } + + /** Sets the value of the 'monitoring_host' field */ + public argo.avro.MetricDataOld.Builder setMonitoringHost(java.lang.String value) { + validate(fields()[5], value); + this.monitoring_host = value; + fieldSetFlags()[5] = true; + return this; + } + + /** Checks whether the 'monitoring_host' field has been set */ + public boolean hasMonitoringHost() { + return fieldSetFlags()[5]; + } + + /** Clears the value of the 'monitoring_host' field */ + public argo.avro.MetricDataOld.Builder clearMonitoringHost() { + monitoring_host = null; + fieldSetFlags()[5] = false; + return this; + } + + /** Gets the value of the 'summary' field */ + public java.lang.String getSummary() { + return summary; + } + + /** Sets the value of the 'summary' field */ + public argo.avro.MetricDataOld.Builder setSummary(java.lang.String value) { + validate(fields()[6], value); + this.summary = value; + fieldSetFlags()[6] = true; + return this; + } + + /** Checks whether the 'summary' field has been set */ + public boolean hasSummary() { + return fieldSetFlags()[6]; + } + + /** Clears the value of the 'summary' field */ + public argo.avro.MetricDataOld.Builder clearSummary() { + summary = null; + fieldSetFlags()[6] = false; + return this; + } + + /** Gets the value of the 'message' field */ + public java.lang.String getMessage() { + return message; + } + + /** Sets the value of the 'message' field */ + public argo.avro.MetricDataOld.Builder setMessage(java.lang.String value) { + validate(fields()[7], value); + this.message = value; + fieldSetFlags()[7] = true; + return this; + } + + /** Checks whether the 'message' field has been set */ + public boolean hasMessage() { + return fieldSetFlags()[7]; + } + + /** Clears the value of the 'message' field */ + public argo.avro.MetricDataOld.Builder clearMessage() { + message = null; + fieldSetFlags()[7] = false; + return this; + } + + /** Gets the value of the 'tags' field */ + public java.util.Map getTags() { + return tags; + } + + /** Sets the value of the 'tags' field */ + public argo.avro.MetricDataOld.Builder setTags(java.util.Map value) { + validate(fields()[8], value); + this.tags = value; + fieldSetFlags()[8] = true; + return this; + } + + /** Checks whether the 'tags' field has been set */ + public boolean hasTags() { + return fieldSetFlags()[8]; + } + + /** Clears the value of the 'tags' field */ + public argo.avro.MetricDataOld.Builder clearTags() { + tags = null; + fieldSetFlags()[8] = false; + return this; + } + + @Override + public MetricDataOld build() { + try { + MetricDataOld record = new MetricDataOld(); + record.timestamp = fieldSetFlags()[0] ? this.timestamp : (java.lang.String) defaultValue(fields()[0]); + record.service = fieldSetFlags()[1] ? this.service : (java.lang.String) defaultValue(fields()[1]); + record.hostname = fieldSetFlags()[2] ? this.hostname : (java.lang.String) defaultValue(fields()[2]); + record.metric = fieldSetFlags()[3] ? this.metric : (java.lang.String) defaultValue(fields()[3]); + record.status = fieldSetFlags()[4] ? this.status : (java.lang.String) defaultValue(fields()[4]); + record.monitoring_host = fieldSetFlags()[5] ? this.monitoring_host : (java.lang.String) defaultValue(fields()[5]); + record.summary = fieldSetFlags()[6] ? this.summary : (java.lang.String) defaultValue(fields()[6]); + record.message = fieldSetFlags()[7] ? this.message : (java.lang.String) defaultValue(fields()[7]); + record.tags = fieldSetFlags()[8] ? this.tags : (java.util.Map) defaultValue(fields()[8]); + return record; + } catch (Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } +} diff --git a/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/AmsIngestMetric.java b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/AmsIngestMetric.java new file mode 100644 index 00000000..78a0fac3 --- /dev/null +++ b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/AmsIngestMetric.java @@ -0,0 +1,263 @@ +package argo.streaming; + + +import java.util.Arrays; +import java.util.concurrent.TimeUnit; + +import org.apache.avro.AvroRuntimeException; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.Decoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.commons.codec.binary.Base64; +import org.apache.flink.api.common.functions.FlatMapFunction; +import org.apache.flink.api.common.io.OutputFormat; +import org.apache.flink.api.common.restartstrategy.RestartStrategies; +import org.apache.flink.api.common.time.Time; +import org.apache.flink.api.java.utils.ParameterTool; + +import org.apache.flink.runtime.state.filesystem.FsStateBackend; +import org.apache.flink.streaming.api.datastream.DataStream; + +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; + +import org.apache.flink.streaming.connectors.fs.bucketing.Bucketer; +import org.apache.flink.streaming.connectors.fs.bucketing.BucketingSink; +import org.apache.flink.util.Collector; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +import com.google.gson.JsonElement; + +import com.google.gson.JsonParser; + +import argo.avro.MetricData; +import argo.avro.MetricDataOld; + + +/** + * Flink Job : Stream metric data from ARGO messaging to Hbase + * job required cli parameters: + * + * --ams.endpoint : ARGO messaging api endoint to connect to msg.example.com + * --ams.port : ARGO messaging api port + * --ams.token : ARGO messaging api token + * --ams.project : ARGO messaging api project to connect to + * --ams.sub : ARGO messaging subscription to pull from + * --hbase.master : hbase endpoint + * --hbase.master.port : hbase master port + * --hbase.zk.quorum : comma separated list of hbase zookeeper servers + * --hbase.zk.port : port used by hbase zookeeper servers + * --hbase.namespace : table namespace used (usually tenant name) + * --hbase.table : table name (usually metric_data) + * --check.path : checkpoint path + * --check.interval : checkpoint interval + * --hdfs.path : hdfs destination to write the data + * --ams.batch : num of messages to be retrieved per request to AMS service + * --ams.interval : interval (in ms) between AMS service requests + * --ams.proxy : optional http proxy url + * --ams.verify : optional turn on/off ssl verify + */ +public class AmsIngestMetric { + // setup logger + static Logger LOG = LoggerFactory.getLogger(AmsIngestMetric.class); + + /** + * Check if flink job has been called with ams rate params + */ + public static boolean hasAmsRateArgs(ParameterTool paramTool) { + String args[] = { "ams.batch", "ams.interval" }; + return hasArgs(args, paramTool); + } + + + /** + * Check if flink job has been called with checkpoint cli arguments + */ + public static boolean hasCheckArgs(ParameterTool paramTool) { + String args[] = { "check.path", "check.interval" }; + return hasArgs(args, paramTool); + } + + /** + * Check if flink job has been called with hdfs cli arguments + */ + public static boolean hasHdfsArgs(ParameterTool paramTool) { + String args[] = { "hdfs.path" }; + return hasArgs(args, paramTool); + } + + /** + * Check if flink job has been called with hbase cli arguments + */ + public static boolean hasHbaseArgs(ParameterTool paramTool) { + String args[] = { "hbase.master", "hbase.master.port", "hbase.zk.quorum", "hbase.zk.port", "hbase.namespace", + "hbase.table" }; + return hasArgs(args, paramTool); + } + + /** + * Check if a list of expected cli arguments have been provided to this flink job + */ + public static boolean hasArgs(String[] reqArgs, ParameterTool paramTool) { + + for (String reqArg : reqArgs) { + if (!paramTool.has(reqArg)) + return false; + } + + return true; + } + + public static void main(String[] args) throws Exception { + + // Create flink execution environment + StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment(); + see.setParallelism(1); + // On failure attempt max 10 times to restart with a retry interval of 2 minutes + see.setRestartStrategy(RestartStrategies.fixedDelayRestart(10, Time.of(2, TimeUnit.MINUTES))); + + // Initialize cli parameter tool + final ParameterTool parameterTool = ParameterTool.fromArgs(args); + + // set ams client batch and interval to default values + int batch = 1; + long interval = 100L; + long inactivityThresh = 1800000L; // default inactivity threshold value ~ 30mins + + if (hasAmsRateArgs(parameterTool)) { + batch = parameterTool.getInt("ams.batch"); + interval = parameterTool.getLong("ams.interval"); + } + + + // Initialize Input Source : ARGO Messaging Source + String endpoint = parameterTool.getRequired("ams.endpoint"); + String port = parameterTool.getRequired("ams.port"); + String token = parameterTool.getRequired("ams.token"); + String project = parameterTool.getRequired("ams.project"); + String sub = parameterTool.getRequired("ams.sub"); + + + // Check if checkpointing is desired + if (hasCheckArgs(parameterTool)) { + String checkPath = parameterTool.get("check.path"); + String checkInterval = parameterTool.get("check.interval"); + // Establish check-pointing mechanism using the cli-parameter check.path + see.setStateBackend(new FsStateBackend(checkPath)); + // Establish the check-pointing interval + long checkInt = Long.parseLong(checkInterval); + see.enableCheckpointing(checkInt); + } + + // Ingest sync avro encoded data from AMS endpoint + ArgoMessagingSource ams = new ArgoMessagingSource(endpoint, port, token, project, sub, batch, interval); + + if (parameterTool.has("ams.verify")) { + ams.setVerify(parameterTool.getBoolean("ams.verify")); + } + + if (parameterTool.has("ams.proxy")) { + ams.setProxy(parameterTool.get("ams.proxy")); + } + + DataStream metricDataJSON = see.addSource(ams); + DataStream metricDataPOJO = metricDataJSON.flatMap(new FlatMapFunction() { + + /** + * Flat Map Function that accepts AMS message and exports the metric data object (encoded in the payload) + */ + private static final long serialVersionUID = 1L; + + @Override + public void flatMap(String value, Collector out) throws Exception { + + JsonParser jsonParser = new JsonParser(); + // parse the json root object + JsonElement jRoot = jsonParser.parse(value); + // parse the json field "data" and read it as string + // this is the base64 string payload + String data = jRoot.getAsJsonObject().get("data").getAsString(); + // Decode from base64 + byte[] decoded64 = Base64.decodeBase64(data.getBytes("UTF-8")); + // Decode from avro + + DatumReader avroReader = new SpecificDatumReader(MetricData.getClassSchema()); + Decoder decoder = DecoderFactory.get().binaryDecoder(decoded64, null); + + + MetricData item; + try { + item = avroReader.read(null, decoder); + } catch (java.io.EOFException ex) + { + //convert from old to new + avroReader = new SpecificDatumReader(MetricDataOld.getClassSchema(),MetricData.getClassSchema()); + decoder = DecoderFactory.get().binaryDecoder(decoded64, null); + item = avroReader.read(null, decoder); + } + if (item != null) { + LOG.info("Captured data -- {}", item.toString()); + out.collect(item); + } + + + } + }); + + // Check if saving to HDFS is desired + if (hasHdfsArgs(parameterTool)) { + String basePath = parameterTool.getRequired("hdfs.path"); + // Establish a bucketing sink to be able to store events with different daily + // timestamp parts (YYYY-MM-DD) + // in different daily files + BucketingSink bs = new BucketingSink(basePath); + bs.setInactiveBucketThreshold(inactivityThresh); + Bucketer tsBuck = new TSBucketer(); + bs.setBucketer(tsBuck); + bs.setPartPrefix("mdata"); + // Change default in progress prefix: _ to allow loading the file in + // AvroInputFormat + bs.setInProgressPrefix(""); + // Add .prog extension when a file is in progress mode + bs.setInProgressSuffix(".prog"); + // Add .pend extension when a file is in pending mode + bs.setPendingSuffix(".pend"); + + bs.setWriter(new SpecificAvroWriter()); + metricDataPOJO.addSink(bs); + } + + // Check if saving to Hbase is desired + if (hasHbaseArgs(parameterTool)) { + // Initialize Output : Hbase Output Format + HBaseMetricOutputFormat hbf = new HBaseMetricOutputFormat(); + hbf.setMaster(parameterTool.getRequired("hbase.master")); + hbf.setMasterPort(parameterTool.getRequired("hbase.master-port")); + hbf.setZkQuorum(parameterTool.getRequired("hbase.zk.quorum")); + hbf.setZkPort(parameterTool.getRequired("hbase.zk.port")); + hbf.setNamespace(parameterTool.getRequired("hbase.namespace")); + hbf.setTableName(parameterTool.getRequired("hbase.table")); + + metricDataPOJO.writeUsingOutputFormat(hbf); + } + + // Create a job title message to discern job in flink dashboard/cli + StringBuilder jobTitleSB = new StringBuilder(); + jobTitleSB.append("Ingesting metric data from "); + jobTitleSB.append(endpoint); + jobTitleSB.append(":"); + jobTitleSB.append(port); + jobTitleSB.append("/v1/projects/"); + jobTitleSB.append(project); + jobTitleSB.append("/subscriptions/"); + jobTitleSB.append(sub); + + see.execute(jobTitleSB.toString()); + } + +} \ No newline at end of file diff --git a/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/ArgoMessagingClient.java b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/ArgoMessagingClient.java new file mode 100644 index 00000000..2bd0bb7c --- /dev/null +++ b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/ArgoMessagingClient.java @@ -0,0 +1,351 @@ +package argo.streaming; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.ssl.SSLContextBuilder; +import org.mortbay.log.Log; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; + +/** + * Simple http client for pulling and acknowledging messages from AMS service + * http API + */ +public class ArgoMessagingClient { + + static Logger LOG = LoggerFactory.getLogger(ArgoMessagingClient.class); + // Http Client for contanting AMS service + private CloseableHttpClient httpClient = null; + // AMS endpoint (hostname:port or hostname) + private String endpoint = null; + // AMS project (/v1/projects/{project}) + private String project = null; + // AMS token (?key={token}) + private String token = null; + // AMS subscription (/v1/projects/{project}/subscriptions/{sub}) + private String sub = null; + // protocol (https,http) + private String proto = null; + // numer of message to be pulled; + private String maxMessages = ""; + // ssl verify or not + private boolean verify = true; + // proxy + private URI proxy = null; + + // Utility inner class for holding list of messages and acknowledgements + private class MsgAck { + + String[] msgs; + String[] ackIds; + + private MsgAck(String[] msgs, String[] ackIds) { + this.msgs = msgs; + this.ackIds = ackIds; + } + + } + + public ArgoMessagingClient() { + this.httpClient = HttpClients.createDefault(); + this.proto = "https"; + this.token = "token"; + this.endpoint = "localhost"; + this.project = "test_project"; + this.sub = "test_sub"; + this.maxMessages = "100"; + this.proxy = null; + } + + public ArgoMessagingClient(String method, String token, String endpoint, String project, String sub, int batch, + boolean verify) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { + + this.proto = method; + this.token = token; + this.endpoint = endpoint; + this.project = project; + this.sub = sub; + this.maxMessages = String.valueOf(batch); + this.verify = verify; + + this.httpClient = buildHttpClient(); + + } + + /** + * Initializes Http Client (if not initialized during constructor) + * + * @return + */ + private CloseableHttpClient buildHttpClient() + throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { + if (this.verify) { + return this.httpClient = HttpClients.createDefault(); + } else { + return this.httpClient = HttpClients.custom().setSSLSocketFactory(selfSignedSSLF()).build(); + } + } + + /** + * Create an SSL Connection Socket Factory with a strategy to trust self + * signed certificates + */ + private SSLConnectionSocketFactory selfSignedSSLF() + throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { + SSLContextBuilder sslBuild = new SSLContextBuilder(); + sslBuild.loadTrustMaterial(null, new TrustSelfSignedStrategy()); + return new SSLConnectionSocketFactory(sslBuild.build(), NoopHostnameVerifier.INSTANCE); + } + + /** + * Set AMS http client to use http proxy + */ + public void setProxy(String proxyURL) throws URISyntaxException { + // parse proxy url + this.proxy = URI.create(proxyURL); + } + + /** + * Set AMS http client to NOT use an http proxy + */ + public void unsetProxy() { + this.proxy = null; + } + + /** + * Create a configuration for using http proxy on each request + */ + private RequestConfig createProxyCfg() { + HttpHost proxy = new HttpHost(this.proxy.getHost(), this.proxy.getPort(), this.proxy.getScheme()); + RequestConfig config = RequestConfig.custom().setProxy(proxy).build(); + return config; + } + + public void logIssue(CloseableHttpResponse resp) throws UnsupportedOperationException, IOException { + InputStreamReader isRdr = new InputStreamReader(resp.getEntity().getContent()); + BufferedReader bRdr = new BufferedReader(isRdr); + int statusCode = resp.getStatusLine().getStatusCode(); + + // Parse error content from api response + StringBuilder result = new StringBuilder(); + String rLine; + while ((rLine = bRdr.readLine()) != null) { + result.append(rLine); + } + isRdr.close(); + Log.warn("ApiStatusCode={}, ApiErrorMessage={}", statusCode, result); + + } + + /** + * Properly compose url for each AMS request + */ + public String composeURL(String method) { + + return proto + "://" + endpoint + "/v1/projects/" + project + "/subscriptions/" + sub + ":" + method; + } + + /** + * Executes a pull request against AMS api + */ + public MsgAck doPull() throws IOException, KeyManagementException, NoSuchAlgorithmException, KeyStoreException { + + ArrayList msgList = new ArrayList(); + ArrayList ackIdList = new ArrayList(); + + // Create the http post to pull + HttpPost postPull = new HttpPost(this.composeURL("pull")); + String body = "{\"maxMessages\":\"" + this.maxMessages + "\",\"returnImmediately\":\"true\"}"; + + postPull.addHeader("Accept", "application/json"); + postPull.addHeader("x-api-key", this.token); + postPull.addHeader("Content-type", "application/json"); + + StringEntity postBody = new StringEntity(body); + postBody.setContentType("application/json"); + + postPull.setEntity(postBody); + + if (this.httpClient == null) { + this.httpClient = buildHttpClient(); + } + + // check for proxy + if (this.proxy != null) { + postPull.setConfig(createProxyCfg()); + } + + CloseableHttpResponse response = this.httpClient.execute(postPull); + String msg = ""; + String ackId = ""; + StringBuilder result = new StringBuilder(); + + HttpEntity entity = response.getEntity(); + + int statusCode = response.getStatusLine().getStatusCode(); + + if (entity != null && statusCode == 200) { + + InputStreamReader isRdr = new InputStreamReader(entity.getContent()); + BufferedReader bRdr = new BufferedReader(isRdr); + + String rLine; + + while ((rLine = bRdr.readLine()) != null) { + result.append(rLine); + } + + // Gather message from json + JsonParser jsonParser = new JsonParser(); + // parse the json root object + Log.info("response: {}", result.toString()); + JsonElement jRoot = jsonParser.parse(result.toString()); + + JsonArray jRec = jRoot.getAsJsonObject().get("receivedMessages").getAsJsonArray(); + + // if has elements + for (JsonElement jMsgItem : jRec) { + JsonElement jMsg = jMsgItem.getAsJsonObject().get("message"); + JsonElement jAckId = jMsgItem.getAsJsonObject().get("ackId"); + msg = jMsg.toString(); + ackId = jAckId.toString(); + msgList.add(msg); + ackIdList.add(ackId); + } + + isRdr.close(); + + } else { + + logIssue(response); + + } + + response.close(); + + String[] msgArr = msgList.toArray(new String[0]); + String[] ackIdArr = ackIdList.toArray(new String[0]); + + // Return a Message array + return new MsgAck(msgArr, ackIdArr); + + } + + /** + * Executes a combination of Pull & Ack requests against AMS api + */ + public String[] consume() throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException { + String[] msgs = new String[0]; + // Try first to pull a message + try { + + MsgAck msgAck = doPull(); + // get last ackid + String ackId = ""; + if (msgAck.ackIds.length > 0) { + ackId = msgAck.ackIds[msgAck.ackIds.length - 1]; + } + + if (ackId != "") { + // Do an ack for the received message + String ackRes = doAck(ackId); + if (ackRes == "") { + Log.info("Message Acknowledged ackid:" + ackId); + msgs = msgAck.msgs; + + } else { + Log.warn("No acknowledment for ackid:" + ackId + "-" + ackRes); + } + } + } catch (IOException e) { + LOG.error(e.getMessage()); + } + return msgs; + + } + + /** + * Executes an Acknowledge request against AMS api + */ + public String doAck(String ackId) throws IOException { + + // Create the http post to ack + HttpPost postAck = new HttpPost(this.composeURL("acknowledge")); + String body = "{\"ackIds\":[" + ackId + "]}"; + postAck.addHeader("Accept", "application/json"); + postAck.addHeader("x-api-key", this.token); + postAck.addHeader("Content-type", "application/json"); + + StringEntity postBody = new StringEntity(body); + + postBody.setContentType("application/json"); + postAck.setEntity(postBody); + + // check for proxy + if (this.proxy != null) { + postAck.setConfig(createProxyCfg()); + } + + CloseableHttpResponse response = httpClient.execute(postAck); + String resMsg = ""; + StringBuilder result = new StringBuilder(); + + HttpEntity entity = response.getEntity(); + int status = response.getStatusLine().getStatusCode(); + + if (status != 200) { + + InputStreamReader isRdr = new InputStreamReader(entity.getContent()); + BufferedReader bRdr = new BufferedReader(isRdr); + + String rLine; + + while ((rLine = bRdr.readLine()) != null) { + result.append(rLine); + } + + resMsg = result.toString(); + isRdr.close(); + + } else { + // Log any api errors + logIssue(response); + } + response.close(); + // Return a resposeMessage + return resMsg; + + } + + /** + * Close AMS http client + */ + public void close() throws IOException { + this.httpClient.close(); + } +} diff --git a/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/ArgoMessagingSource.java b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/ArgoMessagingSource.java new file mode 100644 index 00000000..30997677 --- /dev/null +++ b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/ArgoMessagingSource.java @@ -0,0 +1,135 @@ +package argo.streaming; + +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.source.RichSourceFunction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Custom source to connect to AMS service. Uses ArgoMessaging client + */ +public class ArgoMessagingSource extends RichSourceFunction { + + private static final long serialVersionUID = 1L; + + // setup logger + static Logger LOG = LoggerFactory.getLogger(ArgoMessagingSource.class); + + private String endpoint = null; + private String port = null; + private String token = null; + private String project = null; + private String sub = null; + private int batch = 1; + private long interval = 100L; + private boolean verify = true; + private boolean useProxy = false; + private String proxyURL = ""; + private transient Object rateLck; // lock for waiting to establish rate + + private volatile boolean isRunning = true; + + private ArgoMessagingClient client = null; + + + public ArgoMessagingSource(String endpoint, String port, String token, String project, String sub, int batch, Long interval) { + this.endpoint = endpoint; + this.port = port; + this.token = token; + this.project = project; + this.sub = sub; + this.interval = interval; + this.batch = batch; + this.verify = true; + + } + + /** + * Set verify to true or false. If set to false AMS client will be able to contact AMS endpoints that use self-signed certificates + */ + public void setVerify(boolean verify) { + this.verify=verify; + } + /** + * Set proxy details for AMS client + */ + public void setProxy(String proxyURL) { + this.useProxy = true; + this.proxyURL = proxyURL; + } + + /** + * Unset proxy details for AMS client + */ + public void unsetProxy(String proxyURL) { + this.useProxy = false; + this.proxyURL = ""; + } + + + @Override + public void cancel() { + isRunning = false; + + } + + @Override + public void run(SourceContext ctx) throws Exception { + // This is the main run logic + while (isRunning) { + String[] res = this.client.consume(); + if (res.length > 0) { + for (String msg : res) { + ctx.collect(msg); + } + + } + synchronized (rateLck) { + rateLck.wait(this.interval); + } + + } + + } + + /** + * AMS Source initialization + */ + @Override + public void open(Configuration parameters) throws Exception { + // init rate lock + rateLck = new Object(); + // init client + String fendpoint = this.endpoint; + if (this.port != null && !this.port.isEmpty()) { + fendpoint = this.endpoint + ":" + port; + } + try { + client = new ArgoMessagingClient("https", this.token, fendpoint, this.project, this.sub, this.batch, this.verify); + if (this.useProxy) { + client.setProxy(this.proxyURL); + } + } catch (KeyManagementException e) { + e.printStackTrace(); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + } catch (KeyStoreException e) { + e.printStackTrace(); + } + } + + @Override + public void close() throws Exception { + if (this.client != null) { + client.close(); + } + synchronized (rateLck) { + rateLck.notify(); + } + } + +} diff --git a/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/HBaseMetricOutputFormat.java b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/HBaseMetricOutputFormat.java new file mode 100644 index 00000000..5a4b084e --- /dev/null +++ b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/HBaseMetricOutputFormat.java @@ -0,0 +1,137 @@ +package argo.streaming; + +import java.io.IOException; + +import org.apache.flink.api.common.io.OutputFormat; +import org.apache.flink.configuration.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.Bytes; + +import argo.avro.MetricData; +import argo.avro.MetricDataOld; + +/** + * Hbase Output Format for storing Metric Data to an hbase destination + */ +public class HBaseMetricOutputFormat implements OutputFormat { + + private String master = null; + private String masterPort = null; + private String zkQuorum = null; + private String zkPort = null; + private String namespace = null; + private String tname = null; + private Connection connection = null; + private Table ht = null; + + + private static final long serialVersionUID = 1L; + + // Setters + public void setMasterPort(String masterPort) { + this.masterPort = masterPort; + } + + public void setMaster(String master) { + this.master = master; + } + + public void setZkQuorum(String zkQuorum) { + this.zkQuorum = zkQuorum; + } + + public void setZkPort(String zkPort) { + this.zkPort = zkPort; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + public void setTableName(String tname) { + this.tname = tname; + } + + @Override + public void configure(Configuration parameters) { + + } + + /** + * Initialize Hbase remote connection + */ + @Override + public void open(int taskNumber, int numTasks) throws IOException { + // Create hadoop based configuration for hclient to use + org.apache.hadoop.conf.Configuration config = HBaseConfiguration.create(); + // Modify configuration to job needs + config.setInt("timeout", 120000); + if (masterPort != null && !masterPort.isEmpty()){ + config.set("hbase.master", master + ":" + masterPort); + }else { + config.set("hbase.master", master + ":60000"); + } + + config.set("hbase.zookeeper.quorum", zkQuorum); + config.set("hbase.zookeeper.property.clientPort", (zkPort)); + // Create the connection + connection = ConnectionFactory.createConnection(config); + if (namespace != null) { + ht = connection.getTable(TableName.valueOf(namespace + ":" + tname)); + } else { + ht = connection.getTable(TableName.valueOf(tname)); + } + + } + + /** + * Store a Metric Data object as an Hbase Record + */ + @Override + public void writeRecord(MetricData record) throws IOException { + + String ts = record.getTimestamp(); + String host = record.getHostname(); + String service = record.getService(); + String metric = record.getMetric(); + String mHost = record.getMonitoringHost(); + String status = record.getStatus(); + String summary = record.getSummary(); + String msg = record.getMessage(); + String tags = record.getTags().toString(); + + // Compile key + String key = host + "|" + service + "|" + metric + "|" +ts+ "|" + mHost; + + // Prepare columns + Put put = new Put(Bytes.toBytes(key)); + put.addColumn(Bytes.toBytes("data"), Bytes.toBytes("timestamp"), Bytes.toBytes(ts)); + put.addColumn(Bytes.toBytes("data"), Bytes.toBytes("host"), Bytes.toBytes(host)); + put.addColumn(Bytes.toBytes("data"), Bytes.toBytes("service"), Bytes.toBytes(service)); + put.addColumn(Bytes.toBytes("data"), Bytes.toBytes("metric"), Bytes.toBytes(metric)); + put.addColumn(Bytes.toBytes("data"), Bytes.toBytes("monitoring_host"), Bytes.toBytes(mHost)); + put.addColumn(Bytes.toBytes("data"), Bytes.toBytes("status"), Bytes.toBytes(status)); + put.addColumn(Bytes.toBytes("data"), Bytes.toBytes("summary"), Bytes.toBytes(summary)); + put.addColumn(Bytes.toBytes("data"), Bytes.toBytes("msg"), Bytes.toBytes(msg)); + put.addColumn(Bytes.toBytes("data"), Bytes.toBytes("tags"), Bytes.toBytes(tags)); + + // Insert row in hbase + ht.put(put); + + } + + /** + * Close Hbase Connection + */ + @Override + public void close() throws IOException { + ht.close(); + connection.close(); + } + +} \ No newline at end of file diff --git a/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/MetricParse.java b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/MetricParse.java new file mode 100644 index 00000000..40873bfb --- /dev/null +++ b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/MetricParse.java @@ -0,0 +1,65 @@ +package argo.streaming; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.Map.Entry; + +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificDatumReader; + +import com.google.gson.JsonElement; + +import argo.avro.MetricData; +import argo.avro.MetricDataOld; + + + +/** + * SyncParse is a utility class providing methods to parse specific connector data in avro format + */ +public class MetricParse { + + /** + * Parses a byte array and decodes avro MetricData objects + */ + public static ArrayList parseGroupEndpoint(byte[] avroBytes) throws IOException{ + + ArrayList result = new ArrayList(); + + DatumReader avroReader = new SpecificDatumReader(MetricDataOld.getClassSchema(),MetricData.getClassSchema(),new SpecificData()); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(avroBytes, null); + + while (!decoder.isEnd()){ + MetricData cur = avroReader.read(null, decoder); + result.add(cur); + } + + return result; + } + + + /** + * Parses attributes from a json attribute element + */ + public static Map parseAttributes(JsonElement jAttr) throws IOException{ + + Map result = new HashMap(); + if (jAttr!=null){ + Set> jItems = jAttr.getAsJsonObject().entrySet(); + + for (Entry jItem : jItems){ + result.put(jItem.getKey(), jItem.getValue().getAsString()); + } + } + + return result; + } + + +} diff --git a/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/SpecificAvroWriter.java b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/SpecificAvroWriter.java new file mode 100644 index 00000000..5ea8c472 --- /dev/null +++ b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/SpecificAvroWriter.java @@ -0,0 +1,93 @@ +package argo.streaming; + +import java.io.IOException; + + +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.flink.streaming.connectors.fs.StreamWriterBase; +import org.apache.flink.streaming.connectors.fs.Writer; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + + +/** + * Implements a specific AvroWriter For MetricData Avro Objects + */ +public class SpecificAvroWriter extends StreamWriterBase { + + + private static final long serialVersionUID = 1L; + + private transient FSDataOutputStream outputStream = null; + private transient DataFileWriter outputWriter = null; + + + public SpecificAvroWriter(){ + + } + + + + + + @Override + public void close() throws IOException { + if(outputWriter != null) { + outputWriter.sync(); + outputWriter.close(); + } + outputWriter = null; + outputStream = null; + + } + + + + /** + * Write the avro element to the output writer + */ + @Override + public void write(T item) throws IOException { + if (outputStream == null) { + throw new IllegalStateException("AvroWriter has not been opened."); + } + outputWriter.append(item); + + + } + + @Override + public void open(FileSystem fs, Path path) throws IOException { + super.open(fs, path); + if (outputStream != null) { + throw new IllegalStateException("AvroWriter has already been opened."); + } + outputStream = getStream(); + + if(outputWriter == null) { + DatumWriter writer = new SpecificDatumWriter(argo.avro.MetricData.getClassSchema()); + outputWriter = new DataFileWriter(writer); + + outputWriter.create(argo.avro.MetricData.getClassSchema(),outputStream); + } + + } + + @Override + public long flush() throws IOException { + if (outputWriter != null) { + outputWriter.sync(); + } + return super.flush(); + } + + @Override + public Writer duplicate() { + return new SpecificAvroWriter(); + } + + +} diff --git a/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/TSBucketer.java b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/TSBucketer.java new file mode 100644 index 00000000..bae5cedf --- /dev/null +++ b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/TSBucketer.java @@ -0,0 +1,25 @@ +package argo.streaming; + +import org.apache.flink.streaming.connectors.fs.Clock; +import org.apache.flink.streaming.connectors.fs.bucketing.Bucketer; +import org.apache.hadoop.fs.Path; + +import com.esotericsoftware.minlog.Log; + +import argo.avro.MetricData; + + +public class TSBucketer implements Bucketer { + + private static final long serialVersionUID = 1L; + + /** + * Create a specific bucket based on the timestamp field of element + */ + @Override + public Path getBucketPath(final Clock clock, final Path basePath, final MetricData element) { + + String dailyPart = element.getTimestamp().split("T")[0]; + return new Path(basePath + "/" + dailyPart); + } +} \ No newline at end of file diff --git a/flink_jobs_v2/ams_ingest_metric/src/main/resources/log4j.properties b/flink_jobs_v2/ams_ingest_metric/src/main/resources/log4j.properties new file mode 100644 index 00000000..da32ea0f --- /dev/null +++ b/flink_jobs_v2/ams_ingest_metric/src/main/resources/log4j.properties @@ -0,0 +1,23 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +log4j.rootLogger=INFO, console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n diff --git a/flink_jobs_v2/ams_ingest_metric/src/main/resources/schemas/metric_data.avsc b/flink_jobs_v2/ams_ingest_metric/src/main/resources/schemas/metric_data.avsc new file mode 100644 index 00000000..737e0ead --- /dev/null +++ b/flink_jobs_v2/ams_ingest_metric/src/main/resources/schemas/metric_data.avsc @@ -0,0 +1,18 @@ +{"namespace": "argo.avro", + "type": "record", + "name": "metric_data", + "fields": [ + {"name": "timestamp", "type": "string"}, + {"name": "service", "type": "string"}, + {"name": "hostname", "type": "string"}, + {"name": "metric", "type": "string"}, + {"name": "status", "type": "string"}, + {"name": "monitoring_host", "type": ["null", "string"]}, + {"name": "summary", "type": ["null", "string"]}, + {"name": "message", "type": ["null", "string"]}, + {"name": "tags", "type" : ["null", {"name" : "Tags", + "type" : "map", + "values" : ["null", "string"] + }] + }] +} \ No newline at end of file diff --git a/flink_jobs_v2/ams_ingest_sync/.gitignore b/flink_jobs_v2/ams_ingest_sync/.gitignore new file mode 100644 index 00000000..6c4e323f --- /dev/null +++ b/flink_jobs_v2/ams_ingest_sync/.gitignore @@ -0,0 +1,8 @@ +/target/ +.project +.settings/ +.classpath/ +.classpath +/nbproject +nbactions.xml + diff --git a/flink_jobs_v2/ams_ingest_sync/pom.xml b/flink_jobs_v2/ams_ingest_sync/pom.xml new file mode 100644 index 00000000..b0f1a1ee --- /dev/null +++ b/flink_jobs_v2/ams_ingest_sync/pom.xml @@ -0,0 +1,346 @@ + + + + + 4.0.0 + argo.streaming + 0.0.1-SNAPSHOT + ams-ingest-sync + Stream sync data from AMS to HDFS + + + UTF-8 + 1.3.1 + 1.7.7 + 1.2.17 + 2.10 + + + + + apache.snapshots + Apache Development Snapshot Repository + https://repository.apache.org/content/repositories/snapshots/ + + false + + + true + + + + + + + + + + org.apache.flink + flink-java + ${flink.version} + + + org.apache.flink + flink-streaming-java_2.10 + ${flink.version} + + + org.apache.flink + flink-clients_2.10 + ${flink.version} + + + org.apache.flink + flink-connector-filesystem_2.10 + 1.2.1 + + + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + + + log4j + log4j + ${log4j.version} + + + org.apache.httpcomponents + httpclient + 4.5.13 + + + com.google.code.gson + gson + 2.8.9 + + + junit + junit + 4.13.1 + test + + + junit-addons + junit-addons + 1.4 + test + + + + + + + build-jar + + + false + + + + + org.apache.flink + flink-java + ${flink.version} + provided + + + org.apache.flink + flink-streaming-java_2.10 + ${flink.version} + provided + + + org.apache.flink + flink-clients_2.10 + ${flink.version} + provided + + + org.apache.flink + flink-connector-filesystem_2.10 + 1.2.1 + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + provided + + + log4j + log4j + ${log4j.version} + provided + + + org.apache.httpcomponents + httpclient + 4.5.13 + + + com.google.code.gson + gson + 2.8.9 + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.1 + + + package + + shade + + + + + + + + + + + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.1 + + + + package + + shade + + + + + + org.apache.flink:flink-annotations + org.apache.flink:flink-shaded-hadoop2 + org.apache.flink:flink-shaded-curator-recipes + org.apache.flink:flink-core + org.apache.flink:flink-java + org.apache.flink:flink-scala_2.10 + org.apache.flink:flink-runtime_2.10 + org.apache.flink:flink-optimizer_2.10 + org.apache.flink:flink-clients_2.10 + org.apache.flink:flink-avro_2.10 + org.apache.flink:flink-examples-batch_2.10 + org.apache.flink:flink-examples-streaming_2.10 + org.apache.flink:flink-streaming-java_2.10 + org.apache.flink:flink-streaming-scala_2.10 + org.apache.flink:flink-scala-shell_2.10 + org.apache.flink:flink-python + org.apache.flink:flink-metrics-core + org.apache.flink:flink-metrics-jmx + org.apache.flink:flink-statebackend-rocksdb_2.10 + + + + log4j:log4j + org.scala-lang:scala-library + org.scala-lang:scala-compiler + org.scala-lang:scala-reflect + com.data-artisans:flakka-actor_* + com.data-artisans:flakka-remote_* + com.data-artisans:flakka-slf4j_* + io.netty:netty-all + io.netty:netty + commons-fileupload:commons-fileupload + org.apache.avro:avro + commons-collections:commons-collections + org.codehaus.jackson:jackson-core-asl + org.codehaus.jackson:jackson-mapper-asl + com.thoughtworks.paranamer:paranamer + org.xerial.snappy:snappy-java + org.apache.commons:commons-compress + org.tukaani:xz + com.esotericsoftware.kryo:kryo + com.esotericsoftware.minlog:minlog + org.objenesis:objenesis + com.twitter:chill_* + com.twitter:chill-java + commons-lang:commons-lang + junit:junit + org.apache.commons:commons-lang3 + org.slf4j:slf4j-api + org.slf4j:slf4j-log4j12 + log4j:log4j + org.apache.commons:commons-math + org.apache.sling:org.apache.sling.commons.json + commons-logging:commons-logging + commons-codec:commons-codec + com.fasterxml.jackson.core:jackson-core + com.fasterxml.jackson.core:jackson-databind + com.fasterxml.jackson.core:jackson-annotations + stax:stax-api + com.typesafe:config + org.uncommons.maths:uncommons-maths + com.github.scopt:scopt_* + commons-io:commons-io + commons-cli:commons-cli + + + + + org.apache.flink:* + + + org/apache/flink/shaded/com/** + web-docs/** + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + false + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.7 + 1.7 + + + + + + + + ams-ingest-sync + diff --git a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/Downtime.java b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/Downtime.java new file mode 100644 index 00000000..b73e100d --- /dev/null +++ b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/Downtime.java @@ -0,0 +1,286 @@ +/** + * Autogenerated by Avro + * + * DO NOT EDIT DIRECTLY + */ +package argo.avro; +@SuppressWarnings("all") +@org.apache.avro.specific.AvroGenerated +public class Downtime extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"Downtime\",\"namespace\":\"argo.avro\",\"fields\":[{\"name\":\"hostname\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"service\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"start_time\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"end_time\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}]}"); + public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } + @Deprecated public java.lang.String hostname; + @Deprecated public java.lang.String service; + @Deprecated public java.lang.String start_time; + @Deprecated public java.lang.String end_time; + + /** + * Default constructor. + */ + public Downtime() {} + + /** + * All-args constructor. + */ + public Downtime(java.lang.String hostname, java.lang.String service, java.lang.String start_time, java.lang.String end_time) { + this.hostname = hostname; + this.service = service; + this.start_time = start_time; + this.end_time = end_time; + } + + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + // Used by DatumWriter. Applications should not call. + public java.lang.Object get(int field$) { + switch (field$) { + case 0: return hostname; + case 1: return service; + case 2: return start_time; + case 3: return end_time; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + // Used by DatumReader. Applications should not call. + @SuppressWarnings(value="unchecked") + public void put(int field$, java.lang.Object value$) { + switch (field$) { + case 0: hostname = (java.lang.String)value$; break; + case 1: service = (java.lang.String)value$; break; + case 2: start_time = (java.lang.String)value$; break; + case 3: end_time = (java.lang.String)value$; break; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + /** + * Gets the value of the 'hostname' field. + */ + public java.lang.String getHostname() { + return hostname; + } + + /** + * Sets the value of the 'hostname' field. + * @param value the value to set. + */ + public void setHostname(java.lang.String value) { + this.hostname = value; + } + + /** + * Gets the value of the 'service' field. + */ + public java.lang.String getService() { + return service; + } + + /** + * Sets the value of the 'service' field. + * @param value the value to set. + */ + public void setService(java.lang.String value) { + this.service = value; + } + + /** + * Gets the value of the 'start_time' field. + */ + public java.lang.String getStartTime() { + return start_time; + } + + /** + * Sets the value of the 'start_time' field. + * @param value the value to set. + */ + public void setStartTime(java.lang.String value) { + this.start_time = value; + } + + /** + * Gets the value of the 'end_time' field. + */ + public java.lang.String getEndTime() { + return end_time; + } + + /** + * Sets the value of the 'end_time' field. + * @param value the value to set. + */ + public void setEndTime(java.lang.String value) { + this.end_time = value; + } + + /** Creates a new Downtime RecordBuilder */ + public static argo.avro.Downtime.Builder newBuilder() { + return new argo.avro.Downtime.Builder(); + } + + /** Creates a new Downtime RecordBuilder by copying an existing Builder */ + public static argo.avro.Downtime.Builder newBuilder(argo.avro.Downtime.Builder other) { + return new argo.avro.Downtime.Builder(other); + } + + /** Creates a new Downtime RecordBuilder by copying an existing Downtime instance */ + public static argo.avro.Downtime.Builder newBuilder(argo.avro.Downtime other) { + return new argo.avro.Downtime.Builder(other); + } + + /** + * RecordBuilder for Downtime instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private java.lang.String hostname; + private java.lang.String service; + private java.lang.String start_time; + private java.lang.String end_time; + + /** Creates a new Builder */ + private Builder() { + super(argo.avro.Downtime.SCHEMA$); + } + + /** Creates a Builder by copying an existing Builder */ + private Builder(argo.avro.Downtime.Builder other) { + super(other); + } + + /** Creates a Builder by copying an existing Downtime instance */ + private Builder(argo.avro.Downtime other) { + super(argo.avro.Downtime.SCHEMA$); + if (isValidValue(fields()[0], other.hostname)) { + this.hostname = data().deepCopy(fields()[0].schema(), other.hostname); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.service)) { + this.service = data().deepCopy(fields()[1].schema(), other.service); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.start_time)) { + this.start_time = data().deepCopy(fields()[2].schema(), other.start_time); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.end_time)) { + this.end_time = data().deepCopy(fields()[3].schema(), other.end_time); + fieldSetFlags()[3] = true; + } + } + + /** Gets the value of the 'hostname' field */ + public java.lang.String getHostname() { + return hostname; + } + + /** Sets the value of the 'hostname' field */ + public argo.avro.Downtime.Builder setHostname(java.lang.String value) { + validate(fields()[0], value); + this.hostname = value; + fieldSetFlags()[0] = true; + return this; + } + + /** Checks whether the 'hostname' field has been set */ + public boolean hasHostname() { + return fieldSetFlags()[0]; + } + + /** Clears the value of the 'hostname' field */ + public argo.avro.Downtime.Builder clearHostname() { + hostname = null; + fieldSetFlags()[0] = false; + return this; + } + + /** Gets the value of the 'service' field */ + public java.lang.String getService() { + return service; + } + + /** Sets the value of the 'service' field */ + public argo.avro.Downtime.Builder setService(java.lang.String value) { + validate(fields()[1], value); + this.service = value; + fieldSetFlags()[1] = true; + return this; + } + + /** Checks whether the 'service' field has been set */ + public boolean hasService() { + return fieldSetFlags()[1]; + } + + /** Clears the value of the 'service' field */ + public argo.avro.Downtime.Builder clearService() { + service = null; + fieldSetFlags()[1] = false; + return this; + } + + /** Gets the value of the 'start_time' field */ + public java.lang.String getStartTime() { + return start_time; + } + + /** Sets the value of the 'start_time' field */ + public argo.avro.Downtime.Builder setStartTime(java.lang.String value) { + validate(fields()[2], value); + this.start_time = value; + fieldSetFlags()[2] = true; + return this; + } + + /** Checks whether the 'start_time' field has been set */ + public boolean hasStartTime() { + return fieldSetFlags()[2]; + } + + /** Clears the value of the 'start_time' field */ + public argo.avro.Downtime.Builder clearStartTime() { + start_time = null; + fieldSetFlags()[2] = false; + return this; + } + + /** Gets the value of the 'end_time' field */ + public java.lang.String getEndTime() { + return end_time; + } + + /** Sets the value of the 'end_time' field */ + public argo.avro.Downtime.Builder setEndTime(java.lang.String value) { + validate(fields()[3], value); + this.end_time = value; + fieldSetFlags()[3] = true; + return this; + } + + /** Checks whether the 'end_time' field has been set */ + public boolean hasEndTime() { + return fieldSetFlags()[3]; + } + + /** Clears the value of the 'end_time' field */ + public argo.avro.Downtime.Builder clearEndTime() { + end_time = null; + fieldSetFlags()[3] = false; + return this; + } + + @Override + public Downtime build() { + try { + Downtime record = new Downtime(); + record.hostname = fieldSetFlags()[0] ? this.hostname : (java.lang.String) defaultValue(fields()[0]); + record.service = fieldSetFlags()[1] ? this.service : (java.lang.String) defaultValue(fields()[1]); + record.start_time = fieldSetFlags()[2] ? this.start_time : (java.lang.String) defaultValue(fields()[2]); + record.end_time = fieldSetFlags()[3] ? this.end_time : (java.lang.String) defaultValue(fields()[3]); + return record; + } catch (Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } +} diff --git a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/GroupEndpoint.java b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/GroupEndpoint.java new file mode 100644 index 00000000..2386b1d2 --- /dev/null +++ b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/GroupEndpoint.java @@ -0,0 +1,336 @@ +/** + * Autogenerated by Avro + * + * DO NOT EDIT DIRECTLY + */ +package argo.avro; +@SuppressWarnings("all") +@org.apache.avro.specific.AvroGenerated +public class GroupEndpoint extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"GroupEndpoint\",\"namespace\":\"argo.avro\",\"fields\":[{\"name\":\"type\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"group\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"service\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"hostname\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"tags\",\"type\":[\"null\",{\"type\":\"map\",\"values\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"avro.java.string\":\"String\"}]}]}"); + public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } + @Deprecated public java.lang.String type; + @Deprecated public java.lang.String group; + @Deprecated public java.lang.String service; + @Deprecated public java.lang.String hostname; + @Deprecated public java.util.Map tags; + + /** + * Default constructor. + */ + public GroupEndpoint() {} + + /** + * All-args constructor. + */ + public GroupEndpoint(java.lang.String type, java.lang.String group, java.lang.String service, java.lang.String hostname, java.util.Map tags) { + this.type = type; + this.group = group; + this.service = service; + this.hostname = hostname; + this.tags = tags; + } + + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + // Used by DatumWriter. Applications should not call. + public java.lang.Object get(int field$) { + switch (field$) { + case 0: return type; + case 1: return group; + case 2: return service; + case 3: return hostname; + case 4: return tags; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + // Used by DatumReader. Applications should not call. + @SuppressWarnings(value="unchecked") + public void put(int field$, java.lang.Object value$) { + switch (field$) { + case 0: type = (java.lang.String)value$; break; + case 1: group = (java.lang.String)value$; break; + case 2: service = (java.lang.String)value$; break; + case 3: hostname = (java.lang.String)value$; break; + case 4: tags = (java.util.Map)value$; break; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + /** + * Gets the value of the 'type' field. + */ + public java.lang.String getType() { + return type; + } + + /** + * Sets the value of the 'type' field. + * @param value the value to set. + */ + public void setType(java.lang.String value) { + this.type = value; + } + + /** + * Gets the value of the 'group' field. + */ + public java.lang.String getGroup() { + return group; + } + + /** + * Sets the value of the 'group' field. + * @param value the value to set. + */ + public void setGroup(java.lang.String value) { + this.group = value; + } + + /** + * Gets the value of the 'service' field. + */ + public java.lang.String getService() { + return service; + } + + /** + * Sets the value of the 'service' field. + * @param value the value to set. + */ + public void setService(java.lang.String value) { + this.service = value; + } + + /** + * Gets the value of the 'hostname' field. + */ + public java.lang.String getHostname() { + return hostname; + } + + /** + * Sets the value of the 'hostname' field. + * @param value the value to set. + */ + public void setHostname(java.lang.String value) { + this.hostname = value; + } + + /** + * Gets the value of the 'tags' field. + */ + public java.util.Map getTags() { + return tags; + } + + /** + * Sets the value of the 'tags' field. + * @param value the value to set. + */ + public void setTags(java.util.Map value) { + this.tags = value; + } + + /** Creates a new GroupEndpoint RecordBuilder */ + public static argo.avro.GroupEndpoint.Builder newBuilder() { + return new argo.avro.GroupEndpoint.Builder(); + } + + /** Creates a new GroupEndpoint RecordBuilder by copying an existing Builder */ + public static argo.avro.GroupEndpoint.Builder newBuilder(argo.avro.GroupEndpoint.Builder other) { + return new argo.avro.GroupEndpoint.Builder(other); + } + + /** Creates a new GroupEndpoint RecordBuilder by copying an existing GroupEndpoint instance */ + public static argo.avro.GroupEndpoint.Builder newBuilder(argo.avro.GroupEndpoint other) { + return new argo.avro.GroupEndpoint.Builder(other); + } + + /** + * RecordBuilder for GroupEndpoint instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private java.lang.String type; + private java.lang.String group; + private java.lang.String service; + private java.lang.String hostname; + private java.util.Map tags; + + /** Creates a new Builder */ + private Builder() { + super(argo.avro.GroupEndpoint.SCHEMA$); + } + + /** Creates a Builder by copying an existing Builder */ + private Builder(argo.avro.GroupEndpoint.Builder other) { + super(other); + } + + /** Creates a Builder by copying an existing GroupEndpoint instance */ + private Builder(argo.avro.GroupEndpoint other) { + super(argo.avro.GroupEndpoint.SCHEMA$); + if (isValidValue(fields()[0], other.type)) { + this.type = data().deepCopy(fields()[0].schema(), other.type); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.group)) { + this.group = data().deepCopy(fields()[1].schema(), other.group); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.service)) { + this.service = data().deepCopy(fields()[2].schema(), other.service); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.hostname)) { + this.hostname = data().deepCopy(fields()[3].schema(), other.hostname); + fieldSetFlags()[3] = true; + } + if (isValidValue(fields()[4], other.tags)) { + this.tags = data().deepCopy(fields()[4].schema(), other.tags); + fieldSetFlags()[4] = true; + } + } + + /** Gets the value of the 'type' field */ + public java.lang.String getType() { + return type; + } + + /** Sets the value of the 'type' field */ + public argo.avro.GroupEndpoint.Builder setType(java.lang.String value) { + validate(fields()[0], value); + this.type = value; + fieldSetFlags()[0] = true; + return this; + } + + /** Checks whether the 'type' field has been set */ + public boolean hasType() { + return fieldSetFlags()[0]; + } + + /** Clears the value of the 'type' field */ + public argo.avro.GroupEndpoint.Builder clearType() { + type = null; + fieldSetFlags()[0] = false; + return this; + } + + /** Gets the value of the 'group' field */ + public java.lang.String getGroup() { + return group; + } + + /** Sets the value of the 'group' field */ + public argo.avro.GroupEndpoint.Builder setGroup(java.lang.String value) { + validate(fields()[1], value); + this.group = value; + fieldSetFlags()[1] = true; + return this; + } + + /** Checks whether the 'group' field has been set */ + public boolean hasGroup() { + return fieldSetFlags()[1]; + } + + /** Clears the value of the 'group' field */ + public argo.avro.GroupEndpoint.Builder clearGroup() { + group = null; + fieldSetFlags()[1] = false; + return this; + } + + /** Gets the value of the 'service' field */ + public java.lang.String getService() { + return service; + } + + /** Sets the value of the 'service' field */ + public argo.avro.GroupEndpoint.Builder setService(java.lang.String value) { + validate(fields()[2], value); + this.service = value; + fieldSetFlags()[2] = true; + return this; + } + + /** Checks whether the 'service' field has been set */ + public boolean hasService() { + return fieldSetFlags()[2]; + } + + /** Clears the value of the 'service' field */ + public argo.avro.GroupEndpoint.Builder clearService() { + service = null; + fieldSetFlags()[2] = false; + return this; + } + + /** Gets the value of the 'hostname' field */ + public java.lang.String getHostname() { + return hostname; + } + + /** Sets the value of the 'hostname' field */ + public argo.avro.GroupEndpoint.Builder setHostname(java.lang.String value) { + validate(fields()[3], value); + this.hostname = value; + fieldSetFlags()[3] = true; + return this; + } + + /** Checks whether the 'hostname' field has been set */ + public boolean hasHostname() { + return fieldSetFlags()[3]; + } + + /** Clears the value of the 'hostname' field */ + public argo.avro.GroupEndpoint.Builder clearHostname() { + hostname = null; + fieldSetFlags()[3] = false; + return this; + } + + /** Gets the value of the 'tags' field */ + public java.util.Map getTags() { + return tags; + } + + /** Sets the value of the 'tags' field */ + public argo.avro.GroupEndpoint.Builder setTags(java.util.Map value) { + validate(fields()[4], value); + this.tags = value; + fieldSetFlags()[4] = true; + return this; + } + + /** Checks whether the 'tags' field has been set */ + public boolean hasTags() { + return fieldSetFlags()[4]; + } + + /** Clears the value of the 'tags' field */ + public argo.avro.GroupEndpoint.Builder clearTags() { + tags = null; + fieldSetFlags()[4] = false; + return this; + } + + @Override + public GroupEndpoint build() { + try { + GroupEndpoint record = new GroupEndpoint(); + record.type = fieldSetFlags()[0] ? this.type : (java.lang.String) defaultValue(fields()[0]); + record.group = fieldSetFlags()[1] ? this.group : (java.lang.String) defaultValue(fields()[1]); + record.service = fieldSetFlags()[2] ? this.service : (java.lang.String) defaultValue(fields()[2]); + record.hostname = fieldSetFlags()[3] ? this.hostname : (java.lang.String) defaultValue(fields()[3]); + record.tags = fieldSetFlags()[4] ? this.tags : (java.util.Map) defaultValue(fields()[4]); + return record; + } catch (Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } +} diff --git a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/GroupGroup.java b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/GroupGroup.java new file mode 100644 index 00000000..a7712d67 --- /dev/null +++ b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/GroupGroup.java @@ -0,0 +1,286 @@ +/** + * Autogenerated by Avro + * + * DO NOT EDIT DIRECTLY + */ +package argo.avro; +@SuppressWarnings("all") +@org.apache.avro.specific.AvroGenerated +public class GroupGroup extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"GroupGroup\",\"namespace\":\"argo.avro\",\"fields\":[{\"name\":\"type\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"group\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"subgroup\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"tags\",\"type\":[\"null\",{\"type\":\"map\",\"values\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"avro.java.string\":\"String\"}]}]}"); + public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } + @Deprecated public java.lang.String type; + @Deprecated public java.lang.String group; + @Deprecated public java.lang.String subgroup; + @Deprecated public java.util.Map tags; + + /** + * Default constructor. + */ + public GroupGroup() {} + + /** + * All-args constructor. + */ + public GroupGroup(java.lang.String type, java.lang.String group, java.lang.String subgroup, java.util.Map tags) { + this.type = type; + this.group = group; + this.subgroup = subgroup; + this.tags = tags; + } + + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + // Used by DatumWriter. Applications should not call. + public java.lang.Object get(int field$) { + switch (field$) { + case 0: return type; + case 1: return group; + case 2: return subgroup; + case 3: return tags; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + // Used by DatumReader. Applications should not call. + @SuppressWarnings(value="unchecked") + public void put(int field$, java.lang.Object value$) { + switch (field$) { + case 0: type = (java.lang.String)value$; break; + case 1: group = (java.lang.String)value$; break; + case 2: subgroup = (java.lang.String)value$; break; + case 3: tags = (java.util.Map)value$; break; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + /** + * Gets the value of the 'type' field. + */ + public java.lang.String getType() { + return type; + } + + /** + * Sets the value of the 'type' field. + * @param value the value to set. + */ + public void setType(java.lang.String value) { + this.type = value; + } + + /** + * Gets the value of the 'group' field. + */ + public java.lang.String getGroup() { + return group; + } + + /** + * Sets the value of the 'group' field. + * @param value the value to set. + */ + public void setGroup(java.lang.String value) { + this.group = value; + } + + /** + * Gets the value of the 'subgroup' field. + */ + public java.lang.String getSubgroup() { + return subgroup; + } + + /** + * Sets the value of the 'subgroup' field. + * @param value the value to set. + */ + public void setSubgroup(java.lang.String value) { + this.subgroup = value; + } + + /** + * Gets the value of the 'tags' field. + */ + public java.util.Map getTags() { + return tags; + } + + /** + * Sets the value of the 'tags' field. + * @param value the value to set. + */ + public void setTags(java.util.Map value) { + this.tags = value; + } + + /** Creates a new GroupGroup RecordBuilder */ + public static argo.avro.GroupGroup.Builder newBuilder() { + return new argo.avro.GroupGroup.Builder(); + } + + /** Creates a new GroupGroup RecordBuilder by copying an existing Builder */ + public static argo.avro.GroupGroup.Builder newBuilder(argo.avro.GroupGroup.Builder other) { + return new argo.avro.GroupGroup.Builder(other); + } + + /** Creates a new GroupGroup RecordBuilder by copying an existing GroupGroup instance */ + public static argo.avro.GroupGroup.Builder newBuilder(argo.avro.GroupGroup other) { + return new argo.avro.GroupGroup.Builder(other); + } + + /** + * RecordBuilder for GroupGroup instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private java.lang.String type; + private java.lang.String group; + private java.lang.String subgroup; + private java.util.Map tags; + + /** Creates a new Builder */ + private Builder() { + super(argo.avro.GroupGroup.SCHEMA$); + } + + /** Creates a Builder by copying an existing Builder */ + private Builder(argo.avro.GroupGroup.Builder other) { + super(other); + } + + /** Creates a Builder by copying an existing GroupGroup instance */ + private Builder(argo.avro.GroupGroup other) { + super(argo.avro.GroupGroup.SCHEMA$); + if (isValidValue(fields()[0], other.type)) { + this.type = data().deepCopy(fields()[0].schema(), other.type); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.group)) { + this.group = data().deepCopy(fields()[1].schema(), other.group); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.subgroup)) { + this.subgroup = data().deepCopy(fields()[2].schema(), other.subgroup); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.tags)) { + this.tags = data().deepCopy(fields()[3].schema(), other.tags); + fieldSetFlags()[3] = true; + } + } + + /** Gets the value of the 'type' field */ + public java.lang.String getType() { + return type; + } + + /** Sets the value of the 'type' field */ + public argo.avro.GroupGroup.Builder setType(java.lang.String value) { + validate(fields()[0], value); + this.type = value; + fieldSetFlags()[0] = true; + return this; + } + + /** Checks whether the 'type' field has been set */ + public boolean hasType() { + return fieldSetFlags()[0]; + } + + /** Clears the value of the 'type' field */ + public argo.avro.GroupGroup.Builder clearType() { + type = null; + fieldSetFlags()[0] = false; + return this; + } + + /** Gets the value of the 'group' field */ + public java.lang.String getGroup() { + return group; + } + + /** Sets the value of the 'group' field */ + public argo.avro.GroupGroup.Builder setGroup(java.lang.String value) { + validate(fields()[1], value); + this.group = value; + fieldSetFlags()[1] = true; + return this; + } + + /** Checks whether the 'group' field has been set */ + public boolean hasGroup() { + return fieldSetFlags()[1]; + } + + /** Clears the value of the 'group' field */ + public argo.avro.GroupGroup.Builder clearGroup() { + group = null; + fieldSetFlags()[1] = false; + return this; + } + + /** Gets the value of the 'subgroup' field */ + public java.lang.String getSubgroup() { + return subgroup; + } + + /** Sets the value of the 'subgroup' field */ + public argo.avro.GroupGroup.Builder setSubgroup(java.lang.String value) { + validate(fields()[2], value); + this.subgroup = value; + fieldSetFlags()[2] = true; + return this; + } + + /** Checks whether the 'subgroup' field has been set */ + public boolean hasSubgroup() { + return fieldSetFlags()[2]; + } + + /** Clears the value of the 'subgroup' field */ + public argo.avro.GroupGroup.Builder clearSubgroup() { + subgroup = null; + fieldSetFlags()[2] = false; + return this; + } + + /** Gets the value of the 'tags' field */ + public java.util.Map getTags() { + return tags; + } + + /** Sets the value of the 'tags' field */ + public argo.avro.GroupGroup.Builder setTags(java.util.Map value) { + validate(fields()[3], value); + this.tags = value; + fieldSetFlags()[3] = true; + return this; + } + + /** Checks whether the 'tags' field has been set */ + public boolean hasTags() { + return fieldSetFlags()[3]; + } + + /** Clears the value of the 'tags' field */ + public argo.avro.GroupGroup.Builder clearTags() { + tags = null; + fieldSetFlags()[3] = false; + return this; + } + + @Override + public GroupGroup build() { + try { + GroupGroup record = new GroupGroup(); + record.type = fieldSetFlags()[0] ? this.type : (java.lang.String) defaultValue(fields()[0]); + record.group = fieldSetFlags()[1] ? this.group : (java.lang.String) defaultValue(fields()[1]); + record.subgroup = fieldSetFlags()[2] ? this.subgroup : (java.lang.String) defaultValue(fields()[2]); + record.tags = fieldSetFlags()[3] ? this.tags : (java.util.Map) defaultValue(fields()[3]); + return record; + } catch (Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } +} diff --git a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/MetricProfile.java b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/MetricProfile.java new file mode 100644 index 00000000..1fe15e09 --- /dev/null +++ b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/MetricProfile.java @@ -0,0 +1,286 @@ +/** + * Autogenerated by Avro + * + * DO NOT EDIT DIRECTLY + */ +package argo.avro; +@SuppressWarnings("all") +@org.apache.avro.specific.AvroGenerated +public class MetricProfile extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"MetricProfile\",\"namespace\":\"argo.avro\",\"fields\":[{\"name\":\"profile\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"service\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"metric\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"tags\",\"type\":[\"null\",{\"type\":\"map\",\"values\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"avro.java.string\":\"String\"}]}]}"); + public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } + @Deprecated public java.lang.String profile; + @Deprecated public java.lang.String service; + @Deprecated public java.lang.String metric; + @Deprecated public java.util.Map tags; + + /** + * Default constructor. + */ + public MetricProfile() {} + + /** + * All-args constructor. + */ + public MetricProfile(java.lang.String profile, java.lang.String service, java.lang.String metric, java.util.Map tags) { + this.profile = profile; + this.service = service; + this.metric = metric; + this.tags = tags; + } + + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + // Used by DatumWriter. Applications should not call. + public java.lang.Object get(int field$) { + switch (field$) { + case 0: return profile; + case 1: return service; + case 2: return metric; + case 3: return tags; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + // Used by DatumReader. Applications should not call. + @SuppressWarnings(value="unchecked") + public void put(int field$, java.lang.Object value$) { + switch (field$) { + case 0: profile = (java.lang.String)value$; break; + case 1: service = (java.lang.String)value$; break; + case 2: metric = (java.lang.String)value$; break; + case 3: tags = (java.util.Map)value$; break; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + /** + * Gets the value of the 'profile' field. + */ + public java.lang.String getProfile() { + return profile; + } + + /** + * Sets the value of the 'profile' field. + * @param value the value to set. + */ + public void setProfile(java.lang.String value) { + this.profile = value; + } + + /** + * Gets the value of the 'service' field. + */ + public java.lang.String getService() { + return service; + } + + /** + * Sets the value of the 'service' field. + * @param value the value to set. + */ + public void setService(java.lang.String value) { + this.service = value; + } + + /** + * Gets the value of the 'metric' field. + */ + public java.lang.String getMetric() { + return metric; + } + + /** + * Sets the value of the 'metric' field. + * @param value the value to set. + */ + public void setMetric(java.lang.String value) { + this.metric = value; + } + + /** + * Gets the value of the 'tags' field. + */ + public java.util.Map getTags() { + return tags; + } + + /** + * Sets the value of the 'tags' field. + * @param value the value to set. + */ + public void setTags(java.util.Map value) { + this.tags = value; + } + + /** Creates a new MetricProfile RecordBuilder */ + public static argo.avro.MetricProfile.Builder newBuilder() { + return new argo.avro.MetricProfile.Builder(); + } + + /** Creates a new MetricProfile RecordBuilder by copying an existing Builder */ + public static argo.avro.MetricProfile.Builder newBuilder(argo.avro.MetricProfile.Builder other) { + return new argo.avro.MetricProfile.Builder(other); + } + + /** Creates a new MetricProfile RecordBuilder by copying an existing MetricProfile instance */ + public static argo.avro.MetricProfile.Builder newBuilder(argo.avro.MetricProfile other) { + return new argo.avro.MetricProfile.Builder(other); + } + + /** + * RecordBuilder for MetricProfile instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private java.lang.String profile; + private java.lang.String service; + private java.lang.String metric; + private java.util.Map tags; + + /** Creates a new Builder */ + private Builder() { + super(argo.avro.MetricProfile.SCHEMA$); + } + + /** Creates a Builder by copying an existing Builder */ + private Builder(argo.avro.MetricProfile.Builder other) { + super(other); + } + + /** Creates a Builder by copying an existing MetricProfile instance */ + private Builder(argo.avro.MetricProfile other) { + super(argo.avro.MetricProfile.SCHEMA$); + if (isValidValue(fields()[0], other.profile)) { + this.profile = data().deepCopy(fields()[0].schema(), other.profile); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.service)) { + this.service = data().deepCopy(fields()[1].schema(), other.service); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.metric)) { + this.metric = data().deepCopy(fields()[2].schema(), other.metric); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.tags)) { + this.tags = data().deepCopy(fields()[3].schema(), other.tags); + fieldSetFlags()[3] = true; + } + } + + /** Gets the value of the 'profile' field */ + public java.lang.String getProfile() { + return profile; + } + + /** Sets the value of the 'profile' field */ + public argo.avro.MetricProfile.Builder setProfile(java.lang.String value) { + validate(fields()[0], value); + this.profile = value; + fieldSetFlags()[0] = true; + return this; + } + + /** Checks whether the 'profile' field has been set */ + public boolean hasProfile() { + return fieldSetFlags()[0]; + } + + /** Clears the value of the 'profile' field */ + public argo.avro.MetricProfile.Builder clearProfile() { + profile = null; + fieldSetFlags()[0] = false; + return this; + } + + /** Gets the value of the 'service' field */ + public java.lang.String getService() { + return service; + } + + /** Sets the value of the 'service' field */ + public argo.avro.MetricProfile.Builder setService(java.lang.String value) { + validate(fields()[1], value); + this.service = value; + fieldSetFlags()[1] = true; + return this; + } + + /** Checks whether the 'service' field has been set */ + public boolean hasService() { + return fieldSetFlags()[1]; + } + + /** Clears the value of the 'service' field */ + public argo.avro.MetricProfile.Builder clearService() { + service = null; + fieldSetFlags()[1] = false; + return this; + } + + /** Gets the value of the 'metric' field */ + public java.lang.String getMetric() { + return metric; + } + + /** Sets the value of the 'metric' field */ + public argo.avro.MetricProfile.Builder setMetric(java.lang.String value) { + validate(fields()[2], value); + this.metric = value; + fieldSetFlags()[2] = true; + return this; + } + + /** Checks whether the 'metric' field has been set */ + public boolean hasMetric() { + return fieldSetFlags()[2]; + } + + /** Clears the value of the 'metric' field */ + public argo.avro.MetricProfile.Builder clearMetric() { + metric = null; + fieldSetFlags()[2] = false; + return this; + } + + /** Gets the value of the 'tags' field */ + public java.util.Map getTags() { + return tags; + } + + /** Sets the value of the 'tags' field */ + public argo.avro.MetricProfile.Builder setTags(java.util.Map value) { + validate(fields()[3], value); + this.tags = value; + fieldSetFlags()[3] = true; + return this; + } + + /** Checks whether the 'tags' field has been set */ + public boolean hasTags() { + return fieldSetFlags()[3]; + } + + /** Clears the value of the 'tags' field */ + public argo.avro.MetricProfile.Builder clearTags() { + tags = null; + fieldSetFlags()[3] = false; + return this; + } + + @Override + public MetricProfile build() { + try { + MetricProfile record = new MetricProfile(); + record.profile = fieldSetFlags()[0] ? this.profile : (java.lang.String) defaultValue(fields()[0]); + record.service = fieldSetFlags()[1] ? this.service : (java.lang.String) defaultValue(fields()[1]); + record.metric = fieldSetFlags()[2] ? this.metric : (java.lang.String) defaultValue(fields()[2]); + record.tags = fieldSetFlags()[3] ? this.tags : (java.util.Map) defaultValue(fields()[3]); + return record; + } catch (Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } +} diff --git a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/Weight.java b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/Weight.java new file mode 100644 index 00000000..0238d7cf --- /dev/null +++ b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/avro/Weight.java @@ -0,0 +1,236 @@ +/** + * Autogenerated by Avro + * + * DO NOT EDIT DIRECTLY + */ +package argo.avro; +@SuppressWarnings("all") +@org.apache.avro.specific.AvroGenerated +public class Weight extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"Weight\",\"namespace\":\"argo.avro\",\"fields\":[{\"name\":\"type\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"site\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"weight\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}]}"); + public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } + @Deprecated public java.lang.String type; + @Deprecated public java.lang.String site; + @Deprecated public java.lang.String weight; + + /** + * Default constructor. + */ + public Weight() {} + + /** + * All-args constructor. + */ + public Weight(java.lang.String type, java.lang.String site, java.lang.String weight) { + this.type = type; + this.site = site; + this.weight = weight; + } + + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + // Used by DatumWriter. Applications should not call. + public java.lang.Object get(int field$) { + switch (field$) { + case 0: return type; + case 1: return site; + case 2: return weight; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + // Used by DatumReader. Applications should not call. + @SuppressWarnings(value="unchecked") + public void put(int field$, java.lang.Object value$) { + switch (field$) { + case 0: type = (java.lang.String)value$; break; + case 1: site = (java.lang.String)value$; break; + case 2: weight = (java.lang.String)value$; break; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + /** + * Gets the value of the 'type' field. + */ + public java.lang.String getType() { + return type; + } + + /** + * Sets the value of the 'type' field. + * @param value the value to set. + */ + public void setType(java.lang.String value) { + this.type = value; + } + + /** + * Gets the value of the 'site' field. + */ + public java.lang.String getSite() { + return site; + } + + /** + * Sets the value of the 'site' field. + * @param value the value to set. + */ + public void setSite(java.lang.String value) { + this.site = value; + } + + /** + * Gets the value of the 'weight' field. + */ + public java.lang.String getWeight() { + return weight; + } + + /** + * Sets the value of the 'weight' field. + * @param value the value to set. + */ + public void setWeight(java.lang.String value) { + this.weight = value; + } + + /** Creates a new Weight RecordBuilder */ + public static argo.avro.Weight.Builder newBuilder() { + return new argo.avro.Weight.Builder(); + } + + /** Creates a new Weight RecordBuilder by copying an existing Builder */ + public static argo.avro.Weight.Builder newBuilder(argo.avro.Weight.Builder other) { + return new argo.avro.Weight.Builder(other); + } + + /** Creates a new Weight RecordBuilder by copying an existing Weight instance */ + public static argo.avro.Weight.Builder newBuilder(argo.avro.Weight other) { + return new argo.avro.Weight.Builder(other); + } + + /** + * RecordBuilder for Weight instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private java.lang.String type; + private java.lang.String site; + private java.lang.String weight; + + /** Creates a new Builder */ + private Builder() { + super(argo.avro.Weight.SCHEMA$); + } + + /** Creates a Builder by copying an existing Builder */ + private Builder(argo.avro.Weight.Builder other) { + super(other); + } + + /** Creates a Builder by copying an existing Weight instance */ + private Builder(argo.avro.Weight other) { + super(argo.avro.Weight.SCHEMA$); + if (isValidValue(fields()[0], other.type)) { + this.type = data().deepCopy(fields()[0].schema(), other.type); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.site)) { + this.site = data().deepCopy(fields()[1].schema(), other.site); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.weight)) { + this.weight = data().deepCopy(fields()[2].schema(), other.weight); + fieldSetFlags()[2] = true; + } + } + + /** Gets the value of the 'type' field */ + public java.lang.String getType() { + return type; + } + + /** Sets the value of the 'type' field */ + public argo.avro.Weight.Builder setType(java.lang.String value) { + validate(fields()[0], value); + this.type = value; + fieldSetFlags()[0] = true; + return this; + } + + /** Checks whether the 'type' field has been set */ + public boolean hasType() { + return fieldSetFlags()[0]; + } + + /** Clears the value of the 'type' field */ + public argo.avro.Weight.Builder clearType() { + type = null; + fieldSetFlags()[0] = false; + return this; + } + + /** Gets the value of the 'site' field */ + public java.lang.String getSite() { + return site; + } + + /** Sets the value of the 'site' field */ + public argo.avro.Weight.Builder setSite(java.lang.String value) { + validate(fields()[1], value); + this.site = value; + fieldSetFlags()[1] = true; + return this; + } + + /** Checks whether the 'site' field has been set */ + public boolean hasSite() { + return fieldSetFlags()[1]; + } + + /** Clears the value of the 'site' field */ + public argo.avro.Weight.Builder clearSite() { + site = null; + fieldSetFlags()[1] = false; + return this; + } + + /** Gets the value of the 'weight' field */ + public java.lang.String getWeight() { + return weight; + } + + /** Sets the value of the 'weight' field */ + public argo.avro.Weight.Builder setWeight(java.lang.String value) { + validate(fields()[2], value); + this.weight = value; + fieldSetFlags()[2] = true; + return this; + } + + /** Checks whether the 'weight' field has been set */ + public boolean hasWeight() { + return fieldSetFlags()[2]; + } + + /** Clears the value of the 'weight' field */ + public argo.avro.Weight.Builder clearWeight() { + weight = null; + fieldSetFlags()[2] = false; + return this; + } + + @Override + public Weight build() { + try { + Weight record = new Weight(); + record.type = fieldSetFlags()[0] ? this.type : (java.lang.String) defaultValue(fields()[0]); + record.site = fieldSetFlags()[1] ? this.site : (java.lang.String) defaultValue(fields()[1]); + record.weight = fieldSetFlags()[2] ? this.weight : (java.lang.String) defaultValue(fields()[2]); + return record; + } catch (Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } +} diff --git a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/AmsIngestSync.java b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/AmsIngestSync.java new file mode 100644 index 00000000..43815d66 --- /dev/null +++ b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/AmsIngestSync.java @@ -0,0 +1,116 @@ +package argo.streaming; + +import java.util.concurrent.TimeUnit; + +import org.apache.flink.api.common.restartstrategy.RestartStrategies; +import org.apache.flink.api.common.time.Time; +import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Flink Streaming JOB for Ingesting Sync Data to HDFS + * job required cli parameters: + * --ams.endpoint : ARGO messaging api endoint to connect to msg.example.com + * --ams.port : ARGO messaging api port + * --ams.token : ARGO messaging api token + * --ams.project : ARGO messaging api project to connect to + * --ams.sub.metric : ARGO messaging subscription to pull metric data from + * --ams.sub.sync : ARGO messaging subscription to pull sync data from + * --hdfs.path : Hdfs destination path to store the data + * --ams.batch : num of messages to be retrieved per request to AMS service + * --ams.interval : interval (in ms) between AMS service requests + * --ams.proxy : optional http proxy url + * --ams.verify : optional turn on/off ssl verify + */ +public class AmsIngestSync { + + // setup logger + static Logger LOG = LoggerFactory.getLogger(AmsIngestSync.class); + + /** + * Check if a list of expected cli arguments have been provided to this flink job + */ + public static boolean hasArgs(String[] reqArgs, ParameterTool paramTool) { + + for (String reqArg : reqArgs) { + if (!paramTool.has(reqArg)) + return false; + } + + return true; + } + + /** + * Check if flink job has been called with ams rate params + */ + public static boolean hasAmsRateArgs(ParameterTool paramTool) { + String args[] = { "ams.batch", "ams.interval" }; + return hasArgs(args, paramTool); + } + + // main job function + public static void main(String[] args) throws Exception { + + // Create flink execution enviroment + StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment(); + see.setParallelism(1); + // Fixed restart strategy: on failure attempt max 10 times to restart with a retry interval of 2 minutes + see.setRestartStrategy(RestartStrategies.fixedDelayRestart(10, Time.of(2, TimeUnit.MINUTES))); + // Initialize cli parameter tool + final ParameterTool parameterTool = ParameterTool.fromArgs(args); + + // Initialize Input Source : ARGO Messaging Source + String endpoint = parameterTool.getRequired("ams.endpoint"); + String port = parameterTool.getRequired("ams.port"); + String token = parameterTool.getRequired("ams.token"); + String project = parameterTool.getRequired("ams.project"); + String sub = parameterTool.getRequired("ams.sub"); + String basePath = parameterTool.getRequired("hdfs.path"); + + // set ams client batch and interval to default values + int batch = 1; + long interval = 100L; + + if (hasAmsRateArgs(parameterTool)) { + batch = parameterTool.getInt("ams.batch"); + interval = parameterTool.getLong("ams.interval"); + } + + + //Ingest sync avro encoded data from AMS endpoint + ArgoMessagingSource ams = new ArgoMessagingSource(endpoint, port, token, project, sub, batch, interval); + + if (parameterTool.has("ams.verify")){ + ams.setVerify(parameterTool.getBoolean("ams.verify")); + } + + if (parameterTool.has("ams.proxy")) { + ams.setProxy(parameterTool.get("ams.proxy")); + } + DataStream syncDataStream = see + .addSource(ams); + + SyncHDFSOutputFormat hdfsOut = new SyncHDFSOutputFormat(); + hdfsOut.setBasePath(basePath); + + syncDataStream.writeUsingOutputFormat(hdfsOut); + + // Create a job title message to discern job in flink dashboard/cli + StringBuilder jobTitleSB = new StringBuilder(); + jobTitleSB.append("Ingesting sync data from "); + jobTitleSB.append(endpoint); + jobTitleSB.append(":"); + jobTitleSB.append(port); + jobTitleSB.append("/v1/projects/"); + jobTitleSB.append(project); + jobTitleSB.append("/subscriptions/"); + jobTitleSB.append(sub); + + see.execute(jobTitleSB.toString()); + + } + +} diff --git a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/ArgoMessagingClient.java b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/ArgoMessagingClient.java new file mode 100644 index 00000000..f7402f8a --- /dev/null +++ b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/ArgoMessagingClient.java @@ -0,0 +1,350 @@ +package argo.streaming; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.ssl.SSLContextBuilder; +import org.mortbay.log.Log; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; + +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; + +/** + * Simple http client for pulling and acknowledging messages from AMS service + * http API + */ +public class ArgoMessagingClient { + + static Logger LOG = LoggerFactory.getLogger(ArgoMessagingClient.class); + // Http Client for contanting AMS service + private CloseableHttpClient httpClient = null; + // AMS endpoint (hostname:port or hostname) + private String endpoint = null; + // AMS project (/v1/projects/{project}) + private String project = null; + // AMS token (?key={token}) + private String token = null; + // AMS subscription (/v1/projects/{project}/subscriptions/{sub}) + private String sub = null; + // protocol (https,http) + private String proto = null; + // numer of message to be pulled; + private String maxMessages = ""; + // ssl verify or not + private boolean verify = true; + // proxy + private URI proxy = null; + + // Utility inner class for holding list of messages and acknowledgements + private class MsgAck { + + String[] msgs; + String[] ackIds; + + private MsgAck(String[] msgs, String[] ackIds) { + this.msgs = msgs; + this.ackIds = ackIds; + } + + } + + public ArgoMessagingClient() { + this.httpClient = HttpClients.createDefault(); + this.proto = "https"; + this.token = "token"; + this.endpoint = "localhost"; + this.project = "test_project"; + this.sub = "test_sub"; + this.maxMessages = "100"; + this.proxy = null; + } + + public ArgoMessagingClient(String method, String token, String endpoint, String project, String sub, int batch, + boolean verify) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { + + this.proto = method; + this.token = token; + this.endpoint = endpoint; + this.project = project; + this.sub = sub; + this.maxMessages = String.valueOf(batch); + this.verify = verify; + + this.httpClient = buildHttpClient(); + + } + + /** + * Initializes Http Client (if not initialized during constructor) + * + * @return + */ + private CloseableHttpClient buildHttpClient() + throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { + if (this.verify) { + return this.httpClient = HttpClients.createDefault(); + } else { + return this.httpClient = HttpClients.custom().setSSLSocketFactory(selfSignedSSLF()).build(); + } + } + + /** + * Create an SSL Connection Socket Factory with a strategy to trust self + * signed certificates + */ + private SSLConnectionSocketFactory selfSignedSSLF() + throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { + SSLContextBuilder sslBuild = new SSLContextBuilder(); + sslBuild.loadTrustMaterial(null, new TrustSelfSignedStrategy()); + return new SSLConnectionSocketFactory(sslBuild.build(), NoopHostnameVerifier.INSTANCE); + } + + /** + * Set AMS http client to use http proxy + */ + public void setProxy(String proxyURL) throws URISyntaxException { + // parse proxy url + this.proxy = URI.create(proxyURL); + } + + /** + * Set AMS http client to NOT use an http proxy + */ + public void unsetProxy() { + this.proxy = null; + } + + /** + * Create a configuration for using http proxy on each request + */ + private RequestConfig createProxyCfg() { + HttpHost proxy = new HttpHost(this.proxy.getHost(), this.proxy.getPort(), this.proxy.getScheme()); + RequestConfig config = RequestConfig.custom().setProxy(proxy).build(); + return config; + } + + public void logIssue(CloseableHttpResponse resp) throws UnsupportedOperationException, IOException { + InputStreamReader isRdr = new InputStreamReader(resp.getEntity().getContent()); + BufferedReader bRdr = new BufferedReader(isRdr); + int statusCode = resp.getStatusLine().getStatusCode(); + + // Parse error content from api response + StringBuilder result = new StringBuilder(); + String rLine; + while ((rLine = bRdr.readLine()) != null) { + result.append(rLine); + } + isRdr.close(); + Log.warn("ApiStatusCode={}, ApiErrorMessage={}", statusCode, result); + + } + + /** + * Properly compose url for each AMS request + */ + public String composeURL(String method) { + return proto + "://" + endpoint + "/v1/projects/" + project + "/subscriptions/" + sub + ":" + method; + } + + /** + * Executes a pull request against AMS api + */ + public MsgAck doPull() throws IOException, KeyManagementException, NoSuchAlgorithmException, KeyStoreException { + + ArrayList msgList = new ArrayList(); + ArrayList ackIdList = new ArrayList(); + + HttpPost postPull = new HttpPost(this.composeURL("pull")); + String body = "{\"maxMessages\":\"" + this.maxMessages + "\",\"returnImmediately\":\"true\"}"; + + postPull.addHeader("Accept", "application/json"); + postPull.addHeader("x-api-key", this.token); + postPull.addHeader("Content-type", "application/json"); + + StringEntity postBody = new StringEntity(body); + postBody.setContentType("application/json"); + + postPull.setEntity(postBody); + + if (this.httpClient == null) { + this.httpClient = buildHttpClient(); + } + + // check for proxy + if (this.proxy != null) { + postPull.setConfig(createProxyCfg()); + } + + CloseableHttpResponse response = this.httpClient.execute(postPull); + String msg = ""; + String ackId = ""; + StringBuilder result = new StringBuilder(); + + HttpEntity entity = response.getEntity(); + + int statusCode = response.getStatusLine().getStatusCode(); + + if (entity != null && statusCode == 200) { + + InputStreamReader isRdr = new InputStreamReader(entity.getContent()); + BufferedReader bRdr = new BufferedReader(isRdr); + + String rLine; + + while ((rLine = bRdr.readLine()) != null) { + result.append(rLine); + } + + // Gather message from json + JsonParser jsonParser = new JsonParser(); + // parse the json root object + Log.info("response: {}", result.toString()); + JsonElement jRoot = jsonParser.parse(result.toString()); + + JsonArray jRec = jRoot.getAsJsonObject().get("receivedMessages").getAsJsonArray(); + + // if has elements + for (JsonElement jMsgItem : jRec) { + JsonElement jMsg = jMsgItem.getAsJsonObject().get("message"); + JsonElement jAckId = jMsgItem.getAsJsonObject().get("ackId"); + msg = jMsg.toString(); + ackId = jAckId.toString(); + msgList.add(msg); + ackIdList.add(ackId); + } + + isRdr.close(); + + } else { + + logIssue(response); + + } + + response.close(); + + String[] msgArr = msgList.toArray(new String[0]); + String[] ackIdArr = ackIdList.toArray(new String[0]); + + // Return a Message array + return new MsgAck(msgArr, ackIdArr); + + } + + /** + * Executes a combination of Pull & Ack requests against AMS api + */ + public String[] consume() throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException { + String[] msgs = new String[0]; + // Try first to pull a message + try { + + MsgAck msgAck = doPull(); + // get last ackid + String ackId = ""; + if (msgAck.ackIds.length > 0) { + ackId = msgAck.ackIds[msgAck.ackIds.length - 1]; + } + + if (ackId != "") { + // Do an ack for the received message + String ackRes = doAck(ackId); + if (ackRes == "") { + Log.info("Message Acknowledged ackid:" + ackId); + msgs = msgAck.msgs; + + } else { + Log.warn("No acknowledment for ackid:" + ackId + "-" + ackRes); + } + } + } catch (IOException e) { + LOG.error(e.getMessage()); + } + return msgs; + + } + + /** + * Executes an Acknowledge request against AMS api + */ + public String doAck(String ackId) throws IOException { + + // Create the http post to ack + + + HttpPost postAck = new HttpPost(this.composeURL("acknowledge")); + String body = "{\"ackIds\":[" + ackId + "]}"; + postAck.addHeader("Accept", "application/json"); + postAck.addHeader("x-api-key", this.token); + postAck.addHeader("Content-type", "application/json"); + + StringEntity postBody = new StringEntity(body); + + postBody.setContentType("application/json"); + postAck.setEntity(postBody); + + // check for proxy + if (this.proxy != null) { + postAck.setConfig(createProxyCfg()); + } + + CloseableHttpResponse response = httpClient.execute(postAck); + String resMsg = ""; + StringBuilder result = new StringBuilder(); + + HttpEntity entity = response.getEntity(); + int status = response.getStatusLine().getStatusCode(); + + if (status != 200) { + + InputStreamReader isRdr = new InputStreamReader(entity.getContent()); + BufferedReader bRdr = new BufferedReader(isRdr); + + String rLine; + + while ((rLine = bRdr.readLine()) != null) { + result.append(rLine); + } + + resMsg = result.toString(); + isRdr.close(); + + } else { + // Log any api errors + logIssue(response); + } + response.close(); + // Return a resposeMessage + return resMsg; + + } + + /** + * Close AMS http client + */ + public void close() throws IOException { + this.httpClient.close(); + } +} diff --git a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/ArgoMessagingSource.java b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/ArgoMessagingSource.java new file mode 100644 index 00000000..30997677 --- /dev/null +++ b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/ArgoMessagingSource.java @@ -0,0 +1,135 @@ +package argo.streaming; + +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.source.RichSourceFunction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Custom source to connect to AMS service. Uses ArgoMessaging client + */ +public class ArgoMessagingSource extends RichSourceFunction { + + private static final long serialVersionUID = 1L; + + // setup logger + static Logger LOG = LoggerFactory.getLogger(ArgoMessagingSource.class); + + private String endpoint = null; + private String port = null; + private String token = null; + private String project = null; + private String sub = null; + private int batch = 1; + private long interval = 100L; + private boolean verify = true; + private boolean useProxy = false; + private String proxyURL = ""; + private transient Object rateLck; // lock for waiting to establish rate + + private volatile boolean isRunning = true; + + private ArgoMessagingClient client = null; + + + public ArgoMessagingSource(String endpoint, String port, String token, String project, String sub, int batch, Long interval) { + this.endpoint = endpoint; + this.port = port; + this.token = token; + this.project = project; + this.sub = sub; + this.interval = interval; + this.batch = batch; + this.verify = true; + + } + + /** + * Set verify to true or false. If set to false AMS client will be able to contact AMS endpoints that use self-signed certificates + */ + public void setVerify(boolean verify) { + this.verify=verify; + } + /** + * Set proxy details for AMS client + */ + public void setProxy(String proxyURL) { + this.useProxy = true; + this.proxyURL = proxyURL; + } + + /** + * Unset proxy details for AMS client + */ + public void unsetProxy(String proxyURL) { + this.useProxy = false; + this.proxyURL = ""; + } + + + @Override + public void cancel() { + isRunning = false; + + } + + @Override + public void run(SourceContext ctx) throws Exception { + // This is the main run logic + while (isRunning) { + String[] res = this.client.consume(); + if (res.length > 0) { + for (String msg : res) { + ctx.collect(msg); + } + + } + synchronized (rateLck) { + rateLck.wait(this.interval); + } + + } + + } + + /** + * AMS Source initialization + */ + @Override + public void open(Configuration parameters) throws Exception { + // init rate lock + rateLck = new Object(); + // init client + String fendpoint = this.endpoint; + if (this.port != null && !this.port.isEmpty()) { + fendpoint = this.endpoint + ":" + port; + } + try { + client = new ArgoMessagingClient("https", this.token, fendpoint, this.project, this.sub, this.batch, this.verify); + if (this.useProxy) { + client.setProxy(this.proxyURL); + } + } catch (KeyManagementException e) { + e.printStackTrace(); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + } catch (KeyStoreException e) { + e.printStackTrace(); + } + } + + @Override + public void close() throws Exception { + if (this.client != null) { + client.close(); + } + synchronized (rateLck) { + rateLck.notify(); + } + } + +} diff --git a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/SyncHDFSOutputFormat.java b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/SyncHDFSOutputFormat.java new file mode 100644 index 00000000..dec9b7f6 --- /dev/null +++ b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/SyncHDFSOutputFormat.java @@ -0,0 +1,262 @@ +package argo.streaming; + +import java.io.BufferedOutputStream; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; + +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.commons.codec.binary.Base64; +import org.apache.flink.api.common.io.OutputFormat; +import org.apache.flink.configuration.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.mortbay.log.Log; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; + +import argo.avro.Downtime; +import argo.avro.GroupEndpoint; +import argo.avro.GroupGroup; +import argo.avro.MetricProfile; +import argo.avro.Weight; + +/** + * Custom Output format for storing Sync Data to HDFS + */ +class SyncHDFSOutputFormat implements OutputFormat { + + // setup logger + static Logger LOG = LoggerFactory.getLogger(SyncHDFSOutputFormat.class); + + private static final long serialVersionUID = 1L; + + private URI basePath; + private org.apache.hadoop.conf.Configuration hadoopConf; + private FileSystem hdfs; + + public void setBasePath(String url) throws URISyntaxException { + this.basePath = new URI(url); + } + + @Override + public void configure(Configuration parameters) { + + } + + /** + * Initialize hadoop configuration and hdfs object + */ + @Override + public void open(int taskNumber, int numTasks) throws IOException { + // create hdfs configuration + hadoopConf = new org.apache.hadoop.conf.Configuration(); + hadoopConf.set("fs.defaultFS", basePath.getScheme() + "://" + basePath.getHost() + ":" + basePath.getPort()); + hdfs = FileSystem.newInstance(hadoopConf); + + } + + /** + * Accepts a binary payload of avro group endpoint records and creates an avro file in designated hdfs path + */ + private void writeGroupEndpoint(byte[] payload, Path file) throws IllegalArgumentException, IOException { + if (hdfs == null) { + return; + } + FSDataOutputStream os = hdfs.create(file); + DatumReader avroReader = new SpecificDatumReader(GroupEndpoint.getClassSchema(), + GroupEndpoint.getClassSchema(), new SpecificData()); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(payload, null); + + DatumWriter avroWriter = new SpecificDatumWriter(GroupEndpoint.getClassSchema(), + new SpecificData()); + DataFileWriter dfw = new DataFileWriter(avroWriter); + dfw.create(GroupEndpoint.getClassSchema(), os); + + while (!decoder.isEnd()) { + GroupEndpoint cur = avroReader.read(null, decoder); + dfw.append(cur); + + } + + dfw.close(); + os.close(); + } + + /** + * Accepts a binary payload of avro group of groups records and creates an avro file in designated hdfs path + */ + private void writeGroupGroup(byte[] payload, Path file) throws IllegalArgumentException, IOException { + if (hdfs == null) { + return; + } + FSDataOutputStream os = hdfs.create(file); + DatumReader avroReader = new SpecificDatumReader(GroupGroup.getClassSchema(), + GroupGroup.getClassSchema(), new SpecificData()); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(payload, null); + + DatumWriter avroWriter = new SpecificDatumWriter(GroupGroup.getClassSchema(), + new SpecificData()); + DataFileWriter dfw = new DataFileWriter(avroWriter); + dfw.create(GroupGroup.getClassSchema(), os); + + while (!decoder.isEnd()) { + GroupGroup cur = avroReader.read(null, decoder); + dfw.append(cur); + + } + + dfw.close(); + os.close(); + } + + /** + * Accepts a binary payload of weight records and creates an avro file in designated hdfs path + */ + private void writeWeight(byte[] payload, Path file) throws IllegalArgumentException, IOException { + if (hdfs == null) { + return; + } + FSDataOutputStream os = hdfs.create(file); + DatumReader avroReader = new SpecificDatumReader(Weight.getClassSchema(), + Weight.getClassSchema(), new SpecificData()); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(payload, null); + + DatumWriter avroWriter = new SpecificDatumWriter(Weight.getClassSchema(), new SpecificData()); + DataFileWriter dfw = new DataFileWriter(avroWriter); + dfw.create(Weight.getClassSchema(), os); + + while (!decoder.isEnd()) { + Weight cur = avroReader.read(null, decoder); + dfw.append(cur); + + } + + dfw.close(); + os.close(); + } + + /** + * Accepts a binary payload of avro downtime records and creates an avro file in designated hdfs path + */ + private void writeDowntime(byte[] payload, Path file) throws IllegalArgumentException, IOException { + if (hdfs == null) { + return; + } + FSDataOutputStream os = hdfs.create(file); + DatumReader avroReader = new SpecificDatumReader(Downtime.getClassSchema(), + Downtime.getClassSchema(), new SpecificData()); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(payload, null); + + DatumWriter avroWriter = new SpecificDatumWriter(Downtime.getClassSchema(), + new SpecificData()); + DataFileWriter dfw = new DataFileWriter(avroWriter); + dfw.create(Downtime.getClassSchema(), os); + + while (!decoder.isEnd()) { + Downtime cur = avroReader.read(null, decoder); + dfw.append(cur); + + } + + dfw.close(); + os.close(); + } + + /** + * Accepts a binary payload of metric profile records and creates an avro file in designated hdfs path + */ + private void writeMetricProfile(byte[] payload, Path file) throws IllegalArgumentException, IOException { + if (hdfs == null) { + return; + } + FSDataOutputStream os = hdfs.create(file); + DatumReader avroReader = new SpecificDatumReader(MetricProfile.getClassSchema(), + MetricProfile.getClassSchema(), new SpecificData()); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(payload, null); + + DatumWriter avroWriter = new SpecificDatumWriter(MetricProfile.getClassSchema(), + new SpecificData()); + DataFileWriter dfw = new DataFileWriter(avroWriter); + dfw.create(MetricProfile.getClassSchema(), os); + + while (!decoder.isEnd()) { + MetricProfile cur = avroReader.read(null, decoder); + dfw.append(cur); + + } + + dfw.close(); + os.close(); + } + + /** + * Accepts an AMS json message, parses it's attributes and decodes the data payload. + * Then according to the attributes select an appropriate sync writing method to + * store the data as an hdfs avro file + */ + @Override + public void writeRecord(String record) throws IOException { + if (hdfs == null) { + return; + } + + JsonParser jsonParser = new JsonParser(); + // parse the json root object + JsonElement jRoot = jsonParser.parse(record); + // parse the json field "data" and read it as string + // this is the base64 string payload + String data = jRoot.getAsJsonObject().get("data").getAsString(); + // Decode from base64 + byte[] decoded64 = Base64.decodeBase64(data.getBytes("UTF-8")); + JsonElement jAttr = jRoot.getAsJsonObject().get("attributes"); + Map attr = SyncParse.parseAttributes(jAttr); + + if (attr.containsKey("type") && attr.containsKey("report") && attr.containsKey("partition_date")) { + + String sType = attr.get("type"); + String sReport = attr.get("report"); + String sPdate = attr.get("partition_date"); + + Path path = new Path(basePath.toString() + "/" + sReport + "/" + sType + "_" + sPdate + ".avro"); + LOG.info("Saving to:" + path.toString()); + + if (sType.equalsIgnoreCase("metric_profile")) { + writeMetricProfile(decoded64, path); + } else if (sType.equalsIgnoreCase("group_endpoints")) { + writeGroupEndpoint(decoded64, path); + } else if (sType.equalsIgnoreCase("group_groups")) { + writeGroupGroup(decoded64, path); + } else if (sType.equalsIgnoreCase("downtimes")) { + writeDowntime(decoded64, path); + } else if (sType.equalsIgnoreCase("weights")) { + writeWeight(decoded64, path); + } + } + + } + + @Override + public void close() throws IOException { + if (hdfs != null) { + hdfs.close(); + } + } + +} \ No newline at end of file diff --git a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/SyncParse.java b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/SyncParse.java new file mode 100644 index 00000000..29dc89a3 --- /dev/null +++ b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/SyncParse.java @@ -0,0 +1,81 @@ +package argo.streaming; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.Map.Entry; + +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificDatumReader; + +import com.google.gson.JsonElement; + +import argo.avro.GroupEndpoint; +import argo.avro.MetricProfile; + + +/** + * SyncParse is a utility class providing methods to parse specific connector data in avro format + */ +public class SyncParse { + + /** + * Parses a byte arrray and decodes avro GroupEndpoint objects + */ + public static ArrayList parseGroupEndpoint(byte[] avroBytes) throws IOException{ + + ArrayList result = new ArrayList(); + + DatumReader avroReader = new SpecificDatumReader(GroupEndpoint.getClassSchema(),GroupEndpoint.getClassSchema(),new SpecificData()); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(avroBytes, null); + + while (!decoder.isEnd()){ + GroupEndpoint cur = avroReader.read(null, decoder); + result.add(cur); + } + + return result; + } + + /** + * Parses a byte arrray and decodes avro MetricProfile objects + */ + public static ArrayList parseMetricProfile(byte[] avroBytes) throws IOException{ + + ArrayList result = new ArrayList(); + + DatumReader avroReader = new SpecificDatumReader(MetricProfile.getClassSchema(),MetricProfile.getClassSchema(),new SpecificData()); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(avroBytes, null); + + while (!decoder.isEnd()){ + MetricProfile cur = avroReader.read(null, decoder); + result.add(cur); + } + + return result; + } + + /** + * Parses attributes from a json attribute element + */ + public static Map parseAttributes(JsonElement jAttr) throws IOException{ + + Map result = new HashMap(); + if (jAttr!=null){ + Set> jItems = jAttr.getAsJsonObject().entrySet(); + + for (Entry jItem : jItems){ + result.put(jItem.getKey(), jItem.getValue().getAsString()); + } + } + + return result; + } + + +} diff --git a/flink_jobs_v2/ams_ingest_sync/src/test/java/argo/streaming/TestSyncDecoding.java b/flink_jobs_v2/ams_ingest_sync/src/test/java/argo/streaming/TestSyncDecoding.java new file mode 100644 index 00000000..4b380048 --- /dev/null +++ b/flink_jobs_v2/ams_ingest_sync/src/test/java/argo/streaming/TestSyncDecoding.java @@ -0,0 +1,80 @@ +package argo.streaming; + +import static org.junit.Assert.assertEquals; + +import java.io.BufferedReader; +import java.io.File; + +import java.io.FileReader; +import java.io.IOException; + +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Map; + +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.commons.codec.binary.Base64; +import org.junit.Test; + +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; + +import argo.avro.MetricProfile; + +public class TestSyncDecoding { + + @Test + public void test() throws URISyntaxException, IOException { + URL resJsonFile = TestSyncDecoding.class.getResource("/request_metric_profile.json"); + File jsonFile = new File(resJsonFile.toURI()); + + BufferedReader br = new BufferedReader(new FileReader(jsonFile)); + + JsonParser jsonParser = new JsonParser(); + JsonElement jRoot = jsonParser.parse(br); + + JsonElement jMsg = jRoot.getAsJsonObject().get("messages").getAsJsonArray().get(0); + + String data = jMsg.getAsJsonObject().get("data").getAsString(); + // Decode from base64 + byte[] decoded64 = Base64.decodeBase64(data.getBytes("UTF-8")); + JsonElement jAttr = jMsg.getAsJsonObject().get("attributes"); + + Map attr = SyncParse.parseAttributes(jAttr); + + assertEquals("check report attr", attr.get("report"), "ops-mon"); + assertEquals("check type attr", attr.get("type"), "metric_profile"); + assertEquals("check partition date attr", attr.get("partition_date"), "2017-11-12"); + + URL resMetricProfile = TestSyncDecoding.class.getResource("/metric_profile.json"); + File jsonMetricProfile = new File(resMetricProfile.toURI()); + + BufferedReader br2 = new BufferedReader(new FileReader(jsonMetricProfile)); + + ArrayList mpContents = new ArrayList(); + + String line = ""; + while ((line = br2.readLine()) != null && line.length() != 0) { + mpContents.add(line); + } + + // Check decoding of MetricProfile avro objects + DatumReader avroReader = new SpecificDatumReader(MetricProfile.getClassSchema(), + MetricProfile.getClassSchema(), new SpecificData()); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(decoded64, null); + + int i=0; + while (!decoder.isEnd()) { + MetricProfile cur = avroReader.read(null, decoder); + assertEquals("check decoded avro records",cur.toString(),mpContents.get(i)); + i++; + } + + } + +} diff --git a/flink_jobs_v2/ams_ingest_sync/src/test/resources/metric_profile.json b/flink_jobs_v2/ams_ingest_sync/src/test/resources/metric_profile.json new file mode 100644 index 00000000..30596c89 --- /dev/null +++ b/flink_jobs_v2/ams_ingest_sync/src/test/resources/metric_profile.json @@ -0,0 +1,34 @@ +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "ARC-CE", "metric": "org.nordugrid.ARC-CE-ARIS", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "ARC-CE", "metric": "org.nordugrid.ARC-CE-IGTF", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "ARC-CE", "metric": "org.nordugrid.ARC-CE-result", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "ARC-CE", "metric": "org.nordugrid.ARC-CE-srm", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "ARC-CE", "metric": "org.nordugrid.ARC-CE-sw-csh", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "CREAM-CE", "metric": "emi.cream.CREAMCE-AllowedSubmission", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "CREAM-CE", "metric": "emi.cream.CREAMCE-JobPurge", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "CREAM-CE", "metric": "emi.cream.CREAMCE-ServiceInfo", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "CREAM-CE", "metric": "eu.egi.CREAM-IGTF", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "CREAM-CE", "metric": "hr.srce.CREAMCE-CertLifetime", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "GRAM5", "metric": "hr.srce.GRAM-Auth", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "GRAM5", "metric": "hr.srce.GRAM-CertLifetime", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "GRAM5", "metric": "hr.srce.GRAM-Command", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "QCG.Computing", "metric": "hr.srce.QCG-Computing-CertLifetime", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "QCG.Computing", "metric": "pl.plgrid.QCG-Computing", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "SRM", "metric": "hr.srce.SRM2-CertLifetime", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "SRM", "metric": "org.sam.SRM-Del", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "SRM", "metric": "org.sam.SRM-Get", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "SRM", "metric": "org.sam.SRM-GetSURLs", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "SRM", "metric": "org.sam.SRM-GetTURLs", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "SRM", "metric": "org.sam.SRM-Ls", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "SRM", "metric": "org.sam.SRM-LsDir", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "SRM", "metric": "org.sam.SRM-Put", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "Site-BDII", "metric": "org.bdii.Entries", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "Site-BDII", "metric": "org.bdii.Freshness", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "eu.egi.cloud.vm-management.occi", "metric": "eu.egi.OCCI-IGTF", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "eu.egi.cloud.vm-management.occi", "metric": "eu.egi.cloud.OCCI-Context", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "eu.egi.cloud.vm-management.occi", "metric": "eu.egi.cloud.OCCI-VM", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "eu.egi.cloud.vm-management.occi", "metric": "org.nagios.OCCI-TCP", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "org.openstack.nova", "metric": "eu.egi.Keystone-IGTF", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "org.openstack.nova", "metric": "eu.egi.cloud.OpenStack-VM", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "org.openstack.nova", "metric": "org.nagios.Keystone-TCP", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "unicore6.TargetSystemFactory", "metric": "emi.unicore.TargetSystemFactory", "tags": {"fqan": "", "vo": "ops"}} +{"profile": "ch.cern.SAM.ARGO_MON_CRITICAL", "service": "unicore6.TargetSystemFactory", "metric": "emi.unicore.UNICORE-Job", "tags": {"fqan": "", "vo": "ops"}} diff --git a/flink_jobs_v2/ams_ingest_sync/src/test/resources/request_metric_profile.json b/flink_jobs_v2/ams_ingest_sync/src/test/resources/request_metric_profile.json new file mode 100644 index 00000000..4b57faa5 --- /dev/null +++ b/flink_jobs_v2/ams_ingest_sync/src/test/resources/request_metric_profile.json @@ -0,0 +1,12 @@ +{ + "messages": [ + { + "attributes": { + "report": "ops-mon", + "type": "metric_profile", + "partition_date": "2017-11-12" + }, + "data": "OmNoLmNlcm4uU0FNLkFSR09fTU9OX0NSSVRJQ0FMDEFSQy1DRTJvcmcubm9yZHVncmlkLkFSQy1DRS1BUklTAgQIZnFhbgAEdm8Gb3BzADpjaC5jZXJuLlNBTS5BUkdPX01PTl9DUklUSUNBTAxBUkMtQ0Uyb3JnLm5vcmR1Z3JpZC5BUkMtQ0UtSUdURgIECGZxYW4ABHZvBm9wcwA6Y2guY2Vybi5TQU0uQVJHT19NT05fQ1JJVElDQUwMQVJDLUNFNm9yZy5ub3JkdWdyaWQuQVJDLUNFLXJlc3VsdAIECGZxYW4ABHZvBm9wcwA6Y2guY2Vybi5TQU0uQVJHT19NT05fQ1JJVElDQUwMQVJDLUNFMG9yZy5ub3JkdWdyaWQuQVJDLUNFLXNybQIECGZxYW4ABHZvBm9wcwA6Y2guY2Vybi5TQU0uQVJHT19NT05fQ1JJVElDQUwMQVJDLUNFNm9yZy5ub3JkdWdyaWQuQVJDLUNFLXN3LWNzaAIECGZxYW4ABHZvBm9wcwA6Y2guY2Vybi5TQU0uQVJHT19NT05fQ1JJVElDQUwQQ1JFQU0tQ0VGZW1pLmNyZWFtLkNSRUFNQ0UtQWxsb3dlZFN1Ym1pc3Npb24CBAhmcWFuAAR2bwZvcHMAOmNoLmNlcm4uU0FNLkFSR09fTU9OX0NSSVRJQ0FMEENSRUFNLUNFNGVtaS5jcmVhbS5DUkVBTUNFLUpvYlB1cmdlAgQIZnFhbgAEdm8Gb3BzADpjaC5jZXJuLlNBTS5BUkdPX01PTl9DUklUSUNBTBBDUkVBTS1DRTplbWkuY3JlYW0uQ1JFQU1DRS1TZXJ2aWNlSW5mbwIECGZxYW4ABHZvBm9wcwA6Y2guY2Vybi5TQU0uQVJHT19NT05fQ1JJVElDQUwQQ1JFQU0tQ0UiZXUuZWdpLkNSRUFNLUlHVEYCBAhmcWFuAAR2bwZvcHMAOmNoLmNlcm4uU0FNLkFSR09fTU9OX0NSSVRJQ0FMEENSRUFNLUNFOGhyLnNyY2UuQ1JFQU1DRS1DZXJ0TGlmZXRpbWUCBAhmcWFuAAR2bwZvcHMAOmNoLmNlcm4uU0FNLkFSR09fTU9OX0NSSVRJQ0FMCkdSQU01ImhyLnNyY2UuR1JBTS1BdXRoAgQIZnFhbgAEdm8Gb3BzADpjaC5jZXJuLlNBTS5BUkdPX01PTl9DUklUSUNBTApHUkFNNTJoci5zcmNlLkdSQU0tQ2VydExpZmV0aW1lAgQIZnFhbgAEdm8Gb3BzADpjaC5jZXJuLlNBTS5BUkdPX01PTl9DUklUSUNBTApHUkFNNShoci5zcmNlLkdSQU0tQ29tbWFuZAIECGZxYW4ABHZvBm9wcwA6Y2guY2Vybi5TQU0uQVJHT19NT05fQ1JJVElDQUwaUUNHLkNvbXB1dGluZ0Roci5zcmNlLlFDRy1Db21wdXRpbmctQ2VydExpZmV0aW1lAgQIZnFhbgAEdm8Gb3BzADpjaC5jZXJuLlNBTS5BUkdPX01PTl9DUklUSUNBTBpRQ0cuQ29tcHV0aW5nLnBsLnBsZ3JpZC5RQ0ctQ29tcHV0aW5nAgQIZnFhbgAEdm8Gb3BzADpjaC5jZXJuLlNBTS5BUkdPX01PTl9DUklUSUNBTAZTUk0yaHIuc3JjZS5TUk0yLUNlcnRMaWZldGltZQIECGZxYW4ABHZvBm9wcwA6Y2guY2Vybi5TQU0uQVJHT19NT05fQ1JJVElDQUwGU1JNHm9yZy5zYW0uU1JNLURlbAIECGZxYW4ABHZvBm9wcwA6Y2guY2Vybi5TQU0uQVJHT19NT05fQ1JJVElDQUwGU1JNHm9yZy5zYW0uU1JNLUdldAIECGZxYW4ABHZvBm9wcwA6Y2guY2Vybi5TQU0uQVJHT19NT05fQ1JJVElDQUwGU1JNKG9yZy5zYW0uU1JNLUdldFNVUkxzAgQIZnFhbgAEdm8Gb3BzADpjaC5jZXJuLlNBTS5BUkdPX01PTl9DUklUSUNBTAZTUk0ob3JnLnNhbS5TUk0tR2V0VFVSTHMCBAhmcWFuAAR2bwZvcHMAOmNoLmNlcm4uU0FNLkFSR09fTU9OX0NSSVRJQ0FMBlNSTRxvcmcuc2FtLlNSTS1McwIECGZxYW4ABHZvBm9wcwA6Y2guY2Vybi5TQU0uQVJHT19NT05fQ1JJVElDQUwGU1JNIm9yZy5zYW0uU1JNLUxzRGlyAgQIZnFhbgAEdm8Gb3BzADpjaC5jZXJuLlNBTS5BUkdPX01PTl9DUklUSUNBTAZTUk0eb3JnLnNhbS5TUk0tUHV0AgQIZnFhbgAEdm8Gb3BzADpjaC5jZXJuLlNBTS5BUkdPX01PTl9DUklUSUNBTBJTaXRlLUJESUkgb3JnLmJkaWkuRW50cmllcwIECGZxYW4ABHZvBm9wcwA6Y2guY2Vybi5TQU0uQVJHT19NT05fQ1JJVElDQUwSU2l0ZS1CRElJJG9yZy5iZGlpLkZyZXNobmVzcwIECGZxYW4ABHZvBm9wcwA6Y2guY2Vybi5TQU0uQVJHT19NT05fQ1JJVElDQUw+ZXUuZWdpLmNsb3VkLnZtLW1hbmFnZW1lbnQub2NjaSBldS5lZ2kuT0NDSS1JR1RGAgQIZnFhbgAEdm8Gb3BzADpjaC5jZXJuLlNBTS5BUkdPX01PTl9DUklUSUNBTD5ldS5lZ2kuY2xvdWQudm0tbWFuYWdlbWVudC5vY2NpMmV1LmVnaS5jbG91ZC5PQ0NJLUNvbnRleHQCBAhmcWFuAAR2bwZvcHMAOmNoLmNlcm4uU0FNLkFSR09fTU9OX0NSSVRJQ0FMPmV1LmVnaS5jbG91ZC52bS1tYW5hZ2VtZW50Lm9jY2koZXUuZWdpLmNsb3VkLk9DQ0ktVk0CBAhmcWFuAAR2bwZvcHMAOmNoLmNlcm4uU0FNLkFSR09fTU9OX0NSSVRJQ0FMPmV1LmVnaS5jbG91ZC52bS1tYW5hZ2VtZW50Lm9jY2kmb3JnLm5hZ2lvcy5PQ0NJLVRDUAIECGZxYW4ABHZvBm9wcwA6Y2guY2Vybi5TQU0uQVJHT19NT05fQ1JJVElDQUwkb3JnLm9wZW5zdGFjay5ub3ZhKGV1LmVnaS5LZXlzdG9uZS1JR1RGAgQIZnFhbgAEdm8Gb3BzADpjaC5jZXJuLlNBTS5BUkdPX01PTl9DUklUSUNBTCRvcmcub3BlbnN0YWNrLm5vdmEyZXUuZWdpLmNsb3VkLk9wZW5TdGFjay1WTQIECGZxYW4ABHZvBm9wcwA6Y2guY2Vybi5TQU0uQVJHT19NT05fQ1JJVElDQUwkb3JnLm9wZW5zdGFjay5ub3ZhLm9yZy5uYWdpb3MuS2V5c3RvbmUtVENQAgQIZnFhbgAEdm8Gb3BzADpjaC5jZXJuLlNBTS5BUkdPX01PTl9DUklUSUNBTDh1bmljb3JlNi5UYXJnZXRTeXN0ZW1GYWN0b3J5PmVtaS51bmljb3JlLlRhcmdldFN5c3RlbUZhY3RvcnkCBAhmcWFuAAR2bwZvcHMAOmNoLmNlcm4uU0FNLkFSR09fTU9OX0NSSVRJQ0FMOHVuaWNvcmU2LlRhcmdldFN5c3RlbUZhY3RvcnkuZW1pLnVuaWNvcmUuVU5JQ09SRS1Kb2ICBAhmcWFuAAR2bwZvcHMA" + } + ] +} diff --git a/flink_jobs_v2/pom.xml b/flink_jobs_v2/pom.xml index 05ad02e2..3690b2c0 100644 --- a/flink_jobs_v2/pom.xml +++ b/flink_jobs_v2/pom.xml @@ -17,7 +17,10 @@ ProfilesManager batch_multi stream_status - AmsPublisher + AmsPublisher + ams_ingest_metric + ams_ingest_sync + diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java index ff17a84c..4c25f9a1 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java @@ -417,4 +417,4 @@ public void publish(String in) throws IOException, NoSuchAlgorithmException, Key public void close() throws IOException { this.httpClient.close(); } -} +} \ No newline at end of file From adf8c81b71c03b878810ed9602020e8d8631536e Mon Sep 17 00:00:00 2001 From: cthermolia-grnet Date: Thu, 1 Sep 2022 15:08:54 +0300 Subject: [PATCH 09/16] ARGO-3973 Change dependencies versions in flink_jobs_v2 as suggested from github --- flink_jobs_v2/batch_multi/pom.xml | 2 +- flink_jobs_v2/stream_status/pom.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flink_jobs_v2/batch_multi/pom.xml b/flink_jobs_v2/batch_multi/pom.xml index 6f333abb..bb2d67cc 100644 --- a/flink_jobs_v2/batch_multi/pom.xml +++ b/flink_jobs_v2/batch_multi/pom.xml @@ -111,7 +111,7 @@ language governing permissions and limitations under the License. --> com.google.code.gson gson - 2.2.4 + 2.8.9 diff --git a/flink_jobs_v2/stream_status/pom.xml b/flink_jobs_v2/stream_status/pom.xml index af95d34c..9be9795e 100644 --- a/flink_jobs_v2/stream_status/pom.xml +++ b/flink_jobs_v2/stream_status/pom.xml @@ -113,7 +113,7 @@ language governing permissions and limitations under the License. --> com.google.code.gson gson - 2.7 + 2.8.9 @@ -148,7 +148,7 @@ language governing permissions and limitations under the License. --> xerces xercesImpl - 2.12.0 + 2.12.2 org.mongodb From ce417b7b28aa75a09c9a230a440b65ed3d6046a6 Mon Sep 17 00:00:00 2001 From: cthermolia-grnet Date: Tue, 6 Sep 2022 13:37:00 +0300 Subject: [PATCH 10/16] ARGO-3904 Handle repeated events such as reminders different from normal ones --- .../argo/streaming/MongoStatusOutput.java | 5 +- .../main/java/argo/streaming/TrimEvent.java | 22 +++--- .../src/main/java/status/StatusEvent.java | 7 +- .../src/main/java/status/StatusManager.java | 74 +++++++++++-------- .../test/java/status/StatusManagerTest.java | 2 +- 5 files changed, 67 insertions(+), 43 deletions(-) diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/MongoStatusOutput.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/MongoStatusOutput.java index 45725293..0394b581 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/MongoStatusOutput.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/MongoStatusOutput.java @@ -214,8 +214,9 @@ public StatusEvent jsonToStatusEvent(JsonObject jRoot) { String monHost = extractJson("monitor_host",jRoot); String repeat = extractJson("repeat",jRoot); String message = extractJson("message",jRoot); - String summary = extractJson("summary",jRoot); - return new StatusEvent(rep,tp,dt,eGroup,service,hostname,metric,status,monHost,tsm,tsp,prevStatus,prevTs,repeat,summary,message); + String summary = extractJson("summary",jRoot); + String reminder = extractJson("reminder",jRoot); + return new StatusEvent(rep,tp,dt,eGroup,service,hostname,metric,status,monHost,tsm,tsp,prevStatus,prevTs,repeat,summary,message, reminder); } /** diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/TrimEvent.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/TrimEvent.java index f60c075b..9e7d8b62 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/TrimEvent.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/TrimEvent.java @@ -3,8 +3,10 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import java.text.ParseException; import java.util.ArrayList; import org.apache.flink.util.Collector; @@ -69,12 +71,12 @@ public void flatMap(String in, Collector out) throws Exception { json = trimEvent(json, removedFields); if (this.helpUrl != null) { String helpUrlPath = "https://" + this.helpUrl + "/" + json.get("metric").getAsString(); //the help url is in the form of helpUrl/metric e.g https://poem.egi.eu/ui/public_metrics/metricA - json.addProperty("url.help", urlString(helpUrlPath)); + json.addProperty("url.help", uriString(helpUrlPath)); } if (this.historyUrl != null) { String finalHistUrlPath = initialHistPath + json.get("endpoint_group").getAsString() + "/" + json.get("service").getAsString() + "/" + json.get("hostname").getAsString() + dateParams; - json.addProperty("url.history", urlString(finalHistUrlPath)); + json.addProperty("url.history", uriString(finalHistUrlPath)); } break; case "endpoint": @@ -85,7 +87,7 @@ public void flatMap(String in, Collector out) throws Exception { json = trimEvent(json, removedFields); if (this.historyUrl != null) { String finalHistUrlPath = initialHistPath + json.get("endpoint_group").getAsString() + "/" + json.get("service").getAsString() + "/" + json.get("hostname").getAsString() + dateParams; - json.addProperty("url.history", urlString(finalHistUrlPath)); + json.addProperty("url.history", uriString(finalHistUrlPath)); } break; @@ -96,7 +98,7 @@ public void flatMap(String in, Collector out) throws Exception { json = trimEvent(json, removedFields); if (this.historyUrl != null) { String finalHistUrlPath = initialHistPath + json.get("endpoint_group").getAsString() + "/" + json.get("service").getAsString() + dateParams; - json.addProperty("url.history", urlString(finalHistUrlPath)); + json.addProperty("url.history", uriString(finalHistUrlPath)); } break; @@ -111,7 +113,7 @@ public void flatMap(String in, Collector out) throws Exception { json = trimEvent(json, removedFields); if (this.historyUrl != null) { String finalHistUrlPath = initialHistPath + json.get("endpoint_group").getAsString() + dateParams; - json.addProperty("url.history", urlString(finalHistUrlPath)); + json.addProperty("url.history", uriString(finalHistUrlPath)); } break; @@ -149,10 +151,10 @@ private String buildDateParams(String dateStr) throws ParseException { return param; } - private String urlString(String urlpath) throws MalformedURLException { - URL url = new URL(urlpath); - return url.toString(); + private String uriString(String urlpath) throws UnsupportedEncodingException, MalformedURLException, URISyntaxException { + URI uri = new URI(urlpath); + return uri.toString(); } -} +} \ No newline at end of file diff --git a/flink_jobs_v2/stream_status/src/main/java/status/StatusEvent.java b/flink_jobs_v2/stream_status/src/main/java/status/StatusEvent.java index 837e11cd..14d2c947 100644 --- a/flink_jobs_v2/stream_status/src/main/java/status/StatusEvent.java +++ b/flink_jobs_v2/stream_status/src/main/java/status/StatusEvent.java @@ -19,6 +19,8 @@ public class StatusEvent{ private String message; private String status; + + private String reminder; private @SerializedName("prev_status") String prevStatus; private @SerializedName("prev_ts") String prevTs; @@ -56,6 +58,7 @@ public StatusEvent() { this.repeat = ""; this.summary = ""; this.message = ""; + this.reminder = ""; this.statusEgroup = new String[0]; this.statusService = new String[0]; this.statusEndpoint = new String[0]; @@ -70,7 +73,7 @@ public StatusEvent() { } - public StatusEvent (String report, String type, String dt, String group,String service,String hostname,String metric,String status,String monHost, String tsMonitored, String tsProcessed, String prevStatus, String prevTs, String repeat, String summary, String message){ + public StatusEvent (String report, String type, String dt, String group,String service,String hostname,String metric,String status,String monHost, String tsMonitored, String tsProcessed, String prevStatus, String prevTs, String repeat, String summary, String message,String reminder){ this.report = report; this.type =type; this.group = group; @@ -87,6 +90,8 @@ public StatusEvent (String report, String type, String dt, String group,String s this.repeat = repeat; this.summary = summary; this.message = message; + + this.reminder = reminder; this.statusEgroup = null; this.statusService = null; this.statusEndpoint = null; diff --git a/flink_jobs_v2/stream_status/src/main/java/status/StatusManager.java b/flink_jobs_v2/stream_status/src/main/java/status/StatusManager.java index 170fcdf8..edc55490 100644 --- a/flink_jobs_v2/stream_status/src/main/java/status/StatusManager.java +++ b/flink_jobs_v2/stream_status/src/main/java/status/StatusManager.java @@ -44,7 +44,7 @@ public class StatusManager { // Name of the report used private String report; - + private String groupType; // Sync file structures necessary for status computation @@ -80,13 +80,13 @@ public void setReport(String report) { public String getReport() { return this.report; } - + public String getGroupType() { - return this.groupType; + return this.groupType; } - + public void setGroupType(String groupType) { - this.groupType = groupType; + this.groupType = groupType; } // Get Operation Manager @@ -590,7 +590,7 @@ public ArrayList dumpStatus(String tsStr) throws ParseException { Date metricTs = metricNode.item.timestamp; // Generate metric status event evtMetric = genEvent("metric", groupName, serviceName, endpointName, metricName, metricStatus, - "", metricTs, metricStatus, metricTs, true, "", ""); + "", metricTs, metricStatus, metricTs, true, "", "", false); statusMetric = new String[]{evtMetric.getStatus(), evtMetric.getPrevStatus(), evtMetric.getTsProcessed(), evtMetric.getPrevTs()}; evtMetric.setStatusMetric(statusMetric); @@ -599,7 +599,7 @@ public ArrayList dumpStatus(String tsStr) throws ParseException { } // Generate endpoint status event evtEndpoint = genEvent("endpoint", groupName, serviceName, endpointName, "", endpointStatus, "", ts, - endpointStatus, endpointTs, true, "", ""); + endpointStatus, endpointTs, true, "", "", false); statusEndpoint = new String[]{evtEndpoint.getStatus(), evtEndpoint.getPrevStatus(), evtEndpoint.getTsMonitored(), evtEndpoint.getPrevTs()}; evtEndpoint.setStatusMetric(statusMetric); @@ -609,7 +609,7 @@ public ArrayList dumpStatus(String tsStr) throws ParseException { } // Generate service status event evtService = genEvent("service", groupName, serviceName, "", "", serviceStatus, "", ts, serviceStatus, - serviceTs, true, "", ""); + serviceTs, true, "", "", false); statusService = new String[]{evtService.getStatus(), evtService.getPrevStatus(), evtService.getTsMonitored(), evtService.getPrevTs()}; evtService.setStatusMetric(statusMetric); @@ -619,7 +619,7 @@ public ArrayList dumpStatus(String tsStr) throws ParseException { results.add(eventToString(evtService)); } // Generate endpoint group status event - evtEgroup = genEvent("grpoup", groupName, "", "", "", groupStatus, "", ts, groupStatus, groupTs, true, "", ""); + evtEgroup = genEvent("grpoup", groupName, "", "", "", groupStatus, "", ts, groupStatus, groupTs, true, "", "", false); statusEgroup = new String[]{evtEgroup.getStatus(), evtEgroup.getPrevStatus(), evtEgroup.getTsMonitored(), evtEgroup.getPrevTs()}; evtEgroup.setStatusMetric(statusMetric); evtEgroup.setStatusEndpoint(statusEndpoint); @@ -737,12 +737,11 @@ public Map> getGroupEndpointStatuses(StatusNode egroup for (Entry endpointEntry : service.children.entrySet()) { String endpointName = endpointEntry.getKey(); // check if endpoint need to be in url friendly format - String endpointURL = egp.getTag(groupName, this.groupType, endpointName, serviceName,"info_URL"); + String endpointURL = egp.getTag(groupName, this.groupType, endpointName, serviceName, "info_URL"); if (endpointURL != null) { - endpointName = endpointURL; + endpointName = endpointURL; } - - + StatusNode endpoint = endpointEntry.getValue(); // Add endpoint information to results results.get("endpoints").add(endpointName); @@ -858,17 +857,21 @@ public ArrayList setStatus(String group, String service, String hostname } boolean repeat = false; - - if ( isStatusToRepeat(metricNode.item.status)) { + + if (isStatusToRepeat(metricNode.item.status)) { repeat = hasTimeDiff(ts, metricNode.item.genTs, statusInterval, metricNode.item.status); } oldMetricTS = metricNode.item.timestamp; oldMetricStatus = metricNode.item.status; + boolean isReminder = false; if (metricNode.item.status != status || repeat) { + if (repeat && metricNode.item.status == status) { + isReminder = true; + } // generate event evtMetric = genEvent("metric", group, service, hostname, metric, ops.getStrStatus(status), - monHost, ts, ops.getStrStatus(oldMetricStatus), oldMetricTS, repeat, summary, message); + monHost, ts, ops.getStrStatus(oldMetricStatus), oldMetricTS, repeat, summary, message, isReminder); // Create metric status level object statusMetric = new String[]{evtMetric.getStatus(), evtMetric.getPrevStatus(), evtMetric.getTsMonitored(), evtMetric.getPrevTs()}; @@ -897,19 +900,24 @@ public ArrayList setStatus(String group, String service, String hostname } boolean repeat = false; - if ( isStatusToRepeat(endpointNode.item.status)) { + if (isStatusToRepeat(endpointNode.item.status)) { repeat = hasTimeDiff(ts, endpointNode.item.genTs, statusInterval, endpointNode.item.status); } // generate event + boolean isReminder = false; + if (repeat && endpointNode.item.status == endpNewStatus) { + isReminder = true; + } + evtEndpoint = genEvent("endpoint", group, service, hostname, metric, ops.getStrStatus(endpNewStatus), monHost, ts, - ops.getStrStatus(oldEndpointStatus), oldEndpointTS, repeat, summary, message); - + ops.getStrStatus(oldEndpointStatus), oldEndpointTS, repeat, summary, message, isReminder); + // check if endpoind should get a friendly url hostname String hostnameURL = this.egp.getTag(group, this.groupType, hostname, service, "info_URL"); if (hostnameURL != null) { - evtEndpoint.setHostnameURL(hostnameURL); + evtEndpoint.setHostnameURL(hostnameURL); } else { evtEndpoint.setHostnameURL(hostname); } @@ -945,13 +953,17 @@ public ArrayList setStatus(String group, String service, String hostname } boolean repeat = false; - if ( isStatusToRepeat(serviceNode.item.status)) { - repeat = hasTimeDiff(ts, serviceNode.item.genTs, statusInterval, serviceNode.item.status); + if (isStatusToRepeat(serviceNode.item.status)) { + repeat = hasTimeDiff(ts, serviceNode.item.genTs, statusInterval, serviceNode.item.status); } // generate event + boolean isReminder = false; + if (repeat && serviceNode.item.status == servNewStatus) { + isReminder = true; + } evtService = genEvent("service", group, service, hostname, metric, ops.getStrStatus(servNewStatus), - monHost, ts, ops.getStrStatus(oldServiceStatus), oldServiceTS, repeat, summary, message); + monHost, ts, ops.getStrStatus(oldServiceStatus), oldServiceTS, repeat, summary, message, isReminder); // Create metric, endpoint, service status metric objects statusService = new String[]{evtService.getStatus(), evtService.getPrevStatus(), evtService.getTsMonitored(), evtService.getPrevTs()}; @@ -979,19 +991,23 @@ public ArrayList setStatus(String group, String service, String hostname } boolean repeat = false; - if ( isStatusToRepeat(groupNode.item.status)) { - repeat = hasTimeDiff(ts, groupNode.item.genTs, statusInterval, groupNode.item.status); + if (isStatusToRepeat(groupNode.item.status)) { + repeat = hasTimeDiff(ts, groupNode.item.genTs, statusInterval, groupNode.item.status); } + boolean isReminder = false; + if (repeat && groupNode.item.status == groupNewStatus) { + isReminder = true; + } // generate event evtEgroup = genEvent("endpoint_group", group, service, hostname, metric, ops.getStrStatus(groupNewStatus), - monHost, ts, ops.getStrStatus(oldGroupStatus), oldGroupTS, repeat, summary, message); + monHost, ts, ops.getStrStatus(oldGroupStatus), oldGroupTS, repeat, summary, message, isReminder); // Create metric, endpoint, service, egroup status metric objects statusEgroup = new String[]{evtEgroup.getStatus(), evtEgroup.getPrevStatus(), evtEgroup.getTsMonitored(), evtEgroup.getPrevTs()}; // generate group endpoint information - Map> groupStatuses = getGroupEndpointStatuses(groupNode,group, ops, egp); + Map> groupStatuses = getGroupEndpointStatuses(groupNode, group, ops, egp); evtEgroup.setGroupEndpoints(groupStatuses.get("endpoints").toArray(new String[0])); evtEgroup.setGroupServices(groupStatuses.get("services").toArray(new String[0])); evtEgroup.setGroupStatuses(groupStatuses.get("statuses").toArray(new String[0])); @@ -1032,7 +1048,7 @@ public ArrayList setStatus(String group, String service, String hostname * @return A string containing the event in json format */ private StatusEvent genEvent(String type, String group, String service, String hostname, String metric, String status, - String monHost, Date ts, String prevStatus, Date prevTs, boolean repeat, String summary, String message) throws ParseException { + String monHost, Date ts, String prevStatus, Date prevTs, boolean repeat, String summary, String message, boolean reminder) throws ParseException { String tsStr = toZulu(ts); String dt = tsStr.split("T")[0].replaceAll("-", ""); String tsProc = toZulu(new Date()); @@ -1045,7 +1061,7 @@ private StatusEvent genEvent(String type, String group, String service, String h } StatusEvent evnt = new StatusEvent(this.report, type, dt, group, service, hostname, metric, status, monHost, - toZulu(ts), tsProc, prevStatus, toZulu(prevTs), new Boolean(repeat).toString(), summary, message); + toZulu(ts), tsProc, prevStatus, toZulu(prevTs), new Boolean(repeat).toString(), summary, message, new Boolean(reminder).toString()); return evnt; } diff --git a/flink_jobs_v2/stream_status/src/test/java/status/StatusManagerTest.java b/flink_jobs_v2/stream_status/src/test/java/status/StatusManagerTest.java index 7982766d..98614e02 100644 --- a/flink_jobs_v2/stream_status/src/test/java/status/StatusManagerTest.java +++ b/flink_jobs_v2/stream_status/src/test/java/status/StatusManagerTest.java @@ -90,7 +90,7 @@ public void test() throws URISyntaxException, IOException, ParseException { String jproc = jRoot.getAsJsonObject().get("ts_processed").getAsString(); StatusEvent evnt = new StatusEvent("Critical","metric","20170303","GR-01-AUTH", "CREAM-CE", "cream01.grid.auth.gr", - "eu.egi.CREAM-IGTF", "OK", "mon01.argo.eu", "2017-03-03T15:00:00Z", jproc,"WARNING","2017-03-03T05:00:00Z", "false","sum4","msg4"); + "eu.egi.CREAM-IGTF", "OK", "mon01.argo.eu", "2017-03-03T15:00:00Z", jproc,"WARNING","2017-03-03T05:00:00Z", "false","sum4","msg4","false"); evnt.setStatusMetric(new String[] {"OK","WARNING","2017-03-03T15:00:00Z","2017-03-03T05:00:00Z"}); From 3f1fcc0e057b27c641e2615e64cf281645059dc3 Mon Sep 17 00:00:00 2001 From: cthermolia-grnet Date: Thu, 8 Sep 2022 10:02:42 +0300 Subject: [PATCH 11/16] Write generated alerts to Ams instead of Kafka in stream_status job --- .../java/argo/streaming/AmsStreamStatus.java | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java index 0916a1f7..ebd06bf0 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java @@ -152,9 +152,14 @@ public static boolean hasArgs(String[] reqArgs, ParameterTool paramTool) { } public static boolean hasAmsPubArgs(ParameterTool paramTool) { - String amsPubArgs[] = {"ams.project.publish", "ams.token.publish", "ams.topic"}; + String amsPubArgs[] = {"ams.project.publish", "ams.token.publish", "ams.notification.topic"}; return hasArgs(amsPubArgs, paramTool); } + public static boolean hasAmsArgs(ParameterTool paramTool) { + String amsPubArgs[] = {"ams.project.publish", "ams.token.publish", "ams.alert.topic"}; + return hasArgs(amsPubArgs, paramTool); + } + /** * Main dataflow of flink job @@ -249,7 +254,6 @@ public static void main(String[] args) throws Exception { .flatMap(new MetricDataWithGroup(conf)).setParallelism(1); DataStream events = groupMdata.connect(syncB).flatMap(new StatusMap(conf, looseInterval, strictInterval)); - DataStream eventsClone = events; if (hasKafkaArgs(parameterTool)) { // Initialize kafka parameters String kafkaServers = parameterTool.get("kafka.servers"); @@ -260,6 +264,17 @@ public static void main(String[] args) throws Exception { kafkaProps); events.addSink(kSink); + }else if(hasAmsArgs(parameterTool)){ + String topic = parameterTool.get("ams.alert.topic"); + String tokenpub = parameterTool.get("ams.token.publish"); + String projectpub = parameterTool.get("ams.project.publish"); + + ArgoMessagingSink ams = new ArgoMessagingSink(endpoint, port, tokenpub, projectpub, topic, interval); + if (parameterTool.has("proxy")) { + String proxyURL = parameterTool.get("proxy"); + ams.setProxy(proxyURL); + } + events.addSink(ams); } if (hasHbaseArgs(parameterTool)) { @@ -288,7 +303,7 @@ public static void main(String[] args) throws Exception { //events.print(); } if (hasAmsPubArgs(parameterTool)) { - String topic = parameterTool.get("ams.topic"); + String topic = parameterTool.get("ams.notification.topic"); String tokenpub = parameterTool.get("ams.token.publish"); String projectpub = parameterTool.get("ams.project.publish"); From e8794ab4ba06b7b8a6c8e7bc7122af3926ce087f Mon Sep 17 00:00:00 2001 From: cthermolia-grnet Date: Mon, 12 Sep 2022 15:13:44 +0300 Subject: [PATCH 12/16] ARGO-3984 Make streaming job initialise by default with no run.date argument and advance the subscription to the lastest message --- .../java/argo/streaming/AmsStreamStatus.java | 55 ++--- .../argo/streaming/ArgoMessagingClient.java | 198 ++++++++++++++++-- .../argo/streaming/ArgoMessagingSink.java | 6 +- .../argo/streaming/ArgoMessagingSource.java | 8 +- .../java/argo/streaming/StatusConfig.java | 8 +- 5 files changed, 233 insertions(+), 42 deletions(-) diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java index 0916a1f7..d47aa601 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java @@ -47,8 +47,14 @@ import argo.avro.GroupEndpoint; import argo.avro.MetricData; import argo.avro.MetricProfile; +import java.util.Calendar; +import java.util.TimeZone; import org.apache.commons.lang.StringUtils; import org.apache.flink.core.fs.FileSystem; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; import profilesmanager.EndpointGroupManager; import profilesmanager.MetricProfileManager; import status.StatusManager; @@ -80,22 +86,23 @@ * if it is not defined url history wont be constructed --url.help (optional) * the url to be used as a basis to create a help url , eg. * poem.egi.eu/ui/public_metrics it can be optional , meaning if it is not - * defined url help wont be constructed - * --interval.loose(Optional)interval to repeat events for WARNING, CRITICAL, UNKNOWN . - * it can be in the format of DAYS, HOURS, MINUTES eg. 1h, 2d, 30m to define the period . Any of these formats is - * transformed to minutes in the computations - * if not defined the default value is 1440m - * - * --interval.strict(Optional)interval to repeat events for CRITICAL . - * it can be in the format of DAYS, HOURS, MINUTES eg. 1h, 2d, 30m to define the period . Any of these formats is - * transformed to minutes in the computations - * if not defined the default value is 1440m + * defined url help wont be constructed --interval.loose(Optional)interval to + * repeat events for WARNING, CRITICAL, UNKNOWN . it can be in the format of + * DAYS, HOURS, MINUTES eg. 1h, 2d, 30m to define the period . Any of these + * formats is transformed to minutes in the computations if not defined the + * default value is 1440m + * + * --interval.strict(Optional)interval to repeat events for CRITICAL . it can be + * in the format of DAYS, HOURS, MINUTES eg. 1h, 2d, 30m to define the period . + * Any of these formats is transformed to minutes in the computations if not + * defined the default value is 1440m * */ public class AmsStreamStatus { // setup logger static Logger LOG = LoggerFactory.getLogger(AmsStreamStatus.class); + private static String runDate; /** * Sets configuration parameters to streaming enviroment @@ -180,7 +187,10 @@ public static void main(String[] args) throws Exception { String apiToken = parameterTool.getRequired("api.token"); String reportID = parameterTool.getRequired("report.uuid"); int apiInterval = parameterTool.getInt("api.interval"); - + runDate = parameterTool.get("run.date"); + if (runDate != null) { + runDate = runDate + "T00:00:00.000Z"; + } int looseInterval = 1440; int strictInterval = 1440; @@ -219,7 +229,7 @@ public static void main(String[] args) throws Exception { // Establish the metric data AMS stream // Ingest sync avro encoded data from AMS endpoint - ArgoMessagingSource amsMetric = new ArgoMessagingSource(endpoint, port, token, project, subMetric, batch, interval); + ArgoMessagingSource amsMetric = new ArgoMessagingSource(endpoint, port, token, project, subMetric, batch, interval, runDate); ArgoApiSource apiSync = new ArgoApiSource(apiEndpoint, apiToken, reportID, apiInterval, interval); if (parameterTool.has("ams.verify")) { @@ -249,7 +259,6 @@ public static void main(String[] args) throws Exception { .flatMap(new MetricDataWithGroup(conf)).setParallelism(1); DataStream events = groupMdata.connect(syncB).flatMap(new StatusMap(conf, looseInterval, strictInterval)); - DataStream eventsClone = events; if (hasKafkaArgs(parameterTool)) { // Initialize kafka parameters String kafkaServers = parameterTool.get("kafka.servers"); @@ -292,7 +301,7 @@ public static void main(String[] args) throws Exception { String tokenpub = parameterTool.get("ams.token.publish"); String projectpub = parameterTool.get("ams.project.publish"); - ArgoMessagingSink ams = new ArgoMessagingSink(endpoint, port, tokenpub, projectpub, topic, interval); + ArgoMessagingSink ams = new ArgoMessagingSink(endpoint, port, tokenpub, projectpub, topic, interval, runDate); if (parameterTool.has("proxy")) { String proxyURL = parameterTool.get("proxy"); ams.setProxy(proxyURL); @@ -777,21 +786,19 @@ private static int getInterval(String intervalParam) { if (matches) { String intervals[] = new String[]{}; - IntervalType intervalType=null; + IntervalType intervalType = null; if (intervalParam.contains("h")) { - intervalType=IntervalType.HOURS; + intervalType = IntervalType.HOURS; intervals = intervalParam.split("h"); - } - - else if (intervalParam.contains("d")) { - intervalType=IntervalType.DAY; + } else if (intervalParam.contains("d")) { + intervalType = IntervalType.DAY; intervals = intervalParam.split("d"); } else if (intervalParam.contains("m")) { - intervalType=IntervalType.MINUTES; + intervalType = IntervalType.MINUTES; intervals = intervalParam.split("m"); } - if (intervalType!=null && StringUtils.isNumeric(intervals[0])) { + if (intervalType != null && StringUtils.isNumeric(intervals[0])) { int interval = Integer.parseInt(intervals[0]); switch (intervalType) { case DAY: @@ -799,7 +806,7 @@ else if (intervalParam.contains("d")) { case HOURS: return interval * 60; case MINUTES: - return interval ; + return interval; default: return 1440; } @@ -810,5 +817,5 @@ else if (intervalParam.contains("d")) { } return 1440; } - + } diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java index cfe11fb5..a6184354 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java @@ -26,11 +26,13 @@ import com.google.gson.JsonArray; import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import com.google.gson.JsonParser; import java.util.Base64; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; /** * Simple http client for pulling and acknowledging messages from AMS service @@ -58,6 +60,7 @@ public class ArgoMessagingClient { // proxy private URI proxy = null; private String topic; + private String runDate; // Utility inner class for holding list of messages and acknowledgements private class MsgAck { @@ -81,10 +84,11 @@ public ArgoMessagingClient() { this.sub = "test_sub"; this.maxMessages = "100"; this.proxy = null; + this.runDate = null; } public ArgoMessagingClient(String method, String token, String endpoint, String project, String sub, int batch, - boolean verify) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { + boolean verify, String runDate) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { this.proto = method; this.token = token; @@ -95,11 +99,12 @@ public ArgoMessagingClient(String method, String token, String endpoint, String this.verify = verify; this.httpClient = buildHttpClient(); + this.runDate = runDate; } public ArgoMessagingClient(String method, String token, String endpoint, String project, String topic, - boolean verify) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { + boolean verify, String runDate) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { this.proto = method; this.token = token; @@ -107,7 +112,7 @@ public ArgoMessagingClient(String method, String token, String endpoint, String this.project = project; this.topic = topic; this.verify = verify; - + this.runDate = runDate; this.httpClient = buildHttpClient(); } @@ -182,10 +187,15 @@ public void logIssue(CloseableHttpResponse resp) throws UnsupportedOperationExce */ public String composeURL(String method) { - if (method.equals("publish")) { - return proto + "://" + endpoint + "/v1/projects/" + project + "/topics/" + topic + ":" + method; - }else{ - return proto + "://" + endpoint + "/v1/projects/" + project + "/subscriptions/" + sub + ":" + method; + switch (method) { + case "publish": + return proto + "://" + endpoint + "/v1/projects/" + project + "/topics/" + topic + ":" + method; + case "offsets": + return proto + "://" + endpoint + "/v1/projects/" + project + "/subscriptions/" + this.sub + ":" + method; + case "timeToOffset": + return proto + "://" + endpoint + "/v1/projects/" + project + "/subscriptions/" + this.sub + ":" + method+"?time=" + this.runDate; + default: + return proto + "://" + endpoint + "/v1/projects/" + project + "/subscriptions/" + sub + ":" + method; } } @@ -200,17 +210,17 @@ public MsgAck doPull() throws IOException, KeyManagementException, NoSuchAlgorit // Create the http post to pull HttpPost postPull = new HttpPost(this.composeURL("pull")); - String body = "{\"maxMessages\":\"" + this.maxMessages + "\",\"returnImmediately\":\"true\"}"; - + String body = "{\"maxMessages\":\"" + this.maxMessages + "\",\"returnImmediately\":\"true\"}"; + postPull.addHeader("Accept", "application/json"); postPull.addHeader("x-api-key", this.token); postPull.addHeader("Content-type", "application/json"); StringEntity postBody = new StringEntity(body); postBody.setContentType("application/json"); - + postPull.setEntity(postBody); - + if (this.httpClient == null) { this.httpClient = buildHttpClient(); } @@ -309,6 +319,170 @@ public String[] consume() throws KeyManagementException, NoSuchAlgorithmExceptio } + /** + * Executes an offsetByTimestamp request against AMS api + */ + public int offsetByTimestamp() throws IOException, KeyManagementException, NoSuchAlgorithmException, KeyStoreException { + + + String method = "timeToOffset"; + if (this.runDate == null) { + method = "offsets"; + } + HttpGet getOffset = new HttpGet((this.composeURL(method))); + getOffset.addHeader("Accept", "application/json"); + getOffset.addHeader("x-api-key", this.token); + getOffset.addHeader("Content-type", "application/json"); + + if (this.httpClient == null) { + this.httpClient = buildHttpClient(); + } + + // check for proxy + if (this.proxy != null) { + getOffset.setConfig(createProxyCfg()); + } + + CloseableHttpResponse response = this.httpClient.execute(getOffset); + int offset = 0; + StringBuilder result = new StringBuilder(); + + HttpEntity entity = response.getEntity(); + + int statusCode = response.getStatusLine().getStatusCode(); + + if (entity != null && statusCode == 200) { + + InputStreamReader isRdr = new InputStreamReader(entity.getContent()); + BufferedReader bRdr = new BufferedReader(isRdr); + + String rLine; + + while ((rLine = bRdr.readLine()) != null) { + result.append(rLine); + } + + // Gather message from json + JsonParser jsonParser = new JsonParser(); + // parse the json root object + Log.info("response: {}", result.toString()); + JsonElement jRoot = jsonParser.parse(result.toString()); + + if (this.runDate != null) { + offset = jRoot.getAsJsonObject().get("offset").getAsInt(); + } else { + offset = jRoot.getAsJsonObject().get("max").getAsInt(); + + } + // if has elements + + isRdr.close(); + + } else { + + logIssue(response); + + } + + response.close(); + + // Return a Message array + return offset; + + } + + /** + * Executes a get offset requests against AMS api + */ + public int offset() throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException { + // Try first to pull a message + int offset = 0; + try { + + offset = offsetByTimestamp(); + // get last ackid + + if (offset != 0) { + // Do an ack for the received message + Log.info("Current offset is :" + offset); + + } else { + Log.warn("Not updated offset:"); + } + + } catch (IOException e) { + LOG.error(e.getMessage()); + } + return offset; + + } + + /** + * Executes a modify offset request against AMS api + * @offset , the offset number to modify + */ + public int modifyOffset(int offset) throws IOException, KeyManagementException, NoSuchAlgorithmException, KeyStoreException { + + // Create the http post to pull + String url = proto + "://" + endpoint + "/v1/projects/" + project + "/subscriptions/" + this.sub + ":" + "modifyOffset"; + + HttpPost postModifyOffset = new HttpPost(url); + + postModifyOffset.addHeader("Accept", "application/json"); + postModifyOffset.addHeader("x-api-key", this.token); + postModifyOffset.addHeader("Content-type", "application/json"); + System.out.println("modify offset is :" + offset); + String body = "{\"offset\":" + offset + "}"; + + StringEntity postBody = new StringEntity(body); + postBody.setContentType("application/json"); + + postModifyOffset.setEntity(postBody); + + if (this.httpClient == null) { + this.httpClient = buildHttpClient(); + } + + // check for proxy + if (this.proxy != null) { + postModifyOffset.setConfig(createProxyCfg()); + } + + CloseableHttpResponse response = this.httpClient.execute(postModifyOffset); + StringBuilder result = new StringBuilder(); + + HttpEntity entity = response.getEntity(); + + int statusCode = response.getStatusLine().getStatusCode(); + + if (entity != null && statusCode == 200) { + + try ( InputStreamReader isRdr = new InputStreamReader(entity.getContent())) { + BufferedReader bRdr = new BufferedReader(isRdr); + + String rLine; + while ((rLine = bRdr.readLine()) != null) { + result.append(rLine); + } + isRdr.close(); + // Gather message from json + // JsonParser jsonParser = new JsonParser(); + // parse the json root object + Log.info("modify offset response: {}", result.toString()); + } + } else { + + logIssue(response); + + } + + response.close(); + + // Return a Message array + return offset; + + } + /** * Executes an Acknowledge request against AMS api */ @@ -427,4 +601,4 @@ public void publish(String in) throws IOException, NoSuchAlgorithmException, Key public void close() throws IOException { this.httpClient.close(); } -} \ No newline at end of file +} diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSink.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSink.java index bfdcba3f..56970122 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSink.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSink.java @@ -23,12 +23,13 @@ public class ArgoMessagingSink extends RichSinkFunction { private boolean verify = true; private boolean useProxy = false; private String proxyURL = ""; + private String date; // private transient Object rateLck; // lock for waiting to establish rate // private volatile boolean isRunning = true; private ArgoMessagingClient client = null; - public ArgoMessagingSink(String endpoint, String port, String token, String project, String topic, Long interval) { + public ArgoMessagingSink(String endpoint, String port, String token, String project, String topic, Long interval, String date) { this.endpoint = endpoint; this.port = port; this.token = token; @@ -36,6 +37,7 @@ public ArgoMessagingSink(String endpoint, String port, String token, String proj this.topic = topic; this.interval = interval; this.verify = true; + this.date=date; } @@ -46,7 +48,7 @@ public void open(Configuration parameters) throws Exception { if (this.port != null && !this.port.isEmpty()) { fendpoint = this.endpoint + ":" + port; } - client = new ArgoMessagingClient("https", this.token, fendpoint, this.project, this.topic, this.verify); + client = new ArgoMessagingClient("https", this.token, fendpoint, this.project, this.topic, this.verify, this.date); if (this.useProxy) { client.setProxy(this.proxyURL); } diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSource.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSource.java index 30997677..dc7b9df8 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSource.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSource.java @@ -34,9 +34,10 @@ public class ArgoMessagingSource extends RichSourceFunction { private volatile boolean isRunning = true; private ArgoMessagingClient client = null; + private String runDate; - public ArgoMessagingSource(String endpoint, String port, String token, String project, String sub, int batch, Long interval) { + public ArgoMessagingSource(String endpoint, String port, String token, String project, String sub, int batch, Long interval, String runDate) { this.endpoint = endpoint; this.port = port; this.token = token; @@ -45,6 +46,7 @@ public ArgoMessagingSource(String endpoint, String port, String token, String pr this.interval = interval; this.batch = batch; this.verify = true; + this.runDate=runDate; } @@ -109,7 +111,7 @@ public void open(Configuration parameters) throws Exception { fendpoint = this.endpoint + ":" + port; } try { - client = new ArgoMessagingClient("https", this.token, fendpoint, this.project, this.sub, this.batch, this.verify); + client = new ArgoMessagingClient("https", this.token, fendpoint, this.project, this.sub, this.batch, this.verify, this.runDate); if (this.useProxy) { client.setProxy(this.proxyURL); } @@ -132,4 +134,4 @@ public void close() throws Exception { } } -} +} \ No newline at end of file diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/StatusConfig.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/StatusConfig.java index 91e49567..3246f734 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/StatusConfig.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/StatusConfig.java @@ -1,8 +1,14 @@ package argo.streaming; import java.io.Serializable; +import java.util.Calendar; +import java.util.TimeZone; import org.apache.flink.api.java.utils.ParameterTool; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; public class StatusConfig implements Serializable { @@ -57,7 +63,7 @@ public StatusConfig(ParameterTool pt) { this.amsProject = pt.getRequired("ams.project"); this.apiEndpoint = pt.getRequired("api.endpoint"); - this.runDate = pt.getRequired("run.date"); + this.runDate = pt.get("run.date"); this.report = pt.getRequired("report"); // Optional timeout parameter if (pt.has("timeout")) { From 227f1cb0d8515bc49f2c2207e4352730d326d641 Mon Sep 17 00:00:00 2001 From: kaggis Date: Tue, 13 Sep 2022 16:35:39 +0300 Subject: [PATCH 13/16] Revert "ARGO-3984 Make streaming job initialise by default with no run.date argument and advance the subscription to the lastest message" --- .../java/argo/streaming/AmsStreamStatus.java | 54 ++--- .../argo/streaming/ArgoMessagingClient.java | 198 ++---------------- .../argo/streaming/ArgoMessagingSink.java | 6 +- .../argo/streaming/ArgoMessagingSource.java | 8 +- .../java/argo/streaming/StatusConfig.java | 8 +- 5 files changed, 41 insertions(+), 233 deletions(-) diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java index f1bec87d..ebd06bf0 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java @@ -47,14 +47,8 @@ import argo.avro.GroupEndpoint; import argo.avro.MetricData; import argo.avro.MetricProfile; -import java.util.Calendar; -import java.util.TimeZone; import org.apache.commons.lang.StringUtils; import org.apache.flink.core.fs.FileSystem; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; import profilesmanager.EndpointGroupManager; import profilesmanager.MetricProfileManager; import status.StatusManager; @@ -86,23 +80,22 @@ * if it is not defined url history wont be constructed --url.help (optional) * the url to be used as a basis to create a help url , eg. * poem.egi.eu/ui/public_metrics it can be optional , meaning if it is not - * defined url help wont be constructed --interval.loose(Optional)interval to - * repeat events for WARNING, CRITICAL, UNKNOWN . it can be in the format of - * DAYS, HOURS, MINUTES eg. 1h, 2d, 30m to define the period . Any of these - * formats is transformed to minutes in the computations if not defined the - * default value is 1440m - * - * --interval.strict(Optional)interval to repeat events for CRITICAL . it can be - * in the format of DAYS, HOURS, MINUTES eg. 1h, 2d, 30m to define the period . - * Any of these formats is transformed to minutes in the computations if not - * defined the default value is 1440m + * defined url help wont be constructed + * --interval.loose(Optional)interval to repeat events for WARNING, CRITICAL, UNKNOWN . + * it can be in the format of DAYS, HOURS, MINUTES eg. 1h, 2d, 30m to define the period . Any of these formats is + * transformed to minutes in the computations + * if not defined the default value is 1440m + * + * --interval.strict(Optional)interval to repeat events for CRITICAL . + * it can be in the format of DAYS, HOURS, MINUTES eg. 1h, 2d, 30m to define the period . Any of these formats is + * transformed to minutes in the computations + * if not defined the default value is 1440m * */ public class AmsStreamStatus { // setup logger static Logger LOG = LoggerFactory.getLogger(AmsStreamStatus.class); - private static String runDate; /** * Sets configuration parameters to streaming enviroment @@ -192,10 +185,7 @@ public static void main(String[] args) throws Exception { String apiToken = parameterTool.getRequired("api.token"); String reportID = parameterTool.getRequired("report.uuid"); int apiInterval = parameterTool.getInt("api.interval"); - runDate = parameterTool.get("run.date"); - if (runDate != null) { - runDate = runDate + "T00:00:00.000Z"; - } + int looseInterval = 1440; int strictInterval = 1440; @@ -234,7 +224,7 @@ public static void main(String[] args) throws Exception { // Establish the metric data AMS stream // Ingest sync avro encoded data from AMS endpoint - ArgoMessagingSource amsMetric = new ArgoMessagingSource(endpoint, port, token, project, subMetric, batch, interval, runDate); + ArgoMessagingSource amsMetric = new ArgoMessagingSource(endpoint, port, token, project, subMetric, batch, interval); ArgoApiSource apiSync = new ArgoApiSource(apiEndpoint, apiToken, reportID, apiInterval, interval); if (parameterTool.has("ams.verify")) { @@ -317,7 +307,7 @@ public static void main(String[] args) throws Exception { String tokenpub = parameterTool.get("ams.token.publish"); String projectpub = parameterTool.get("ams.project.publish"); - ArgoMessagingSink ams = new ArgoMessagingSink(endpoint, port, tokenpub, projectpub, topic, interval, runDate); + ArgoMessagingSink ams = new ArgoMessagingSink(endpoint, port, tokenpub, projectpub, topic, interval); if (parameterTool.has("proxy")) { String proxyURL = parameterTool.get("proxy"); ams.setProxy(proxyURL); @@ -802,19 +792,21 @@ private static int getInterval(String intervalParam) { if (matches) { String intervals[] = new String[]{}; - IntervalType intervalType = null; + IntervalType intervalType=null; if (intervalParam.contains("h")) { - intervalType = IntervalType.HOURS; + intervalType=IntervalType.HOURS; intervals = intervalParam.split("h"); - } else if (intervalParam.contains("d")) { - intervalType = IntervalType.DAY; + } + + else if (intervalParam.contains("d")) { + intervalType=IntervalType.DAY; intervals = intervalParam.split("d"); } else if (intervalParam.contains("m")) { - intervalType = IntervalType.MINUTES; + intervalType=IntervalType.MINUTES; intervals = intervalParam.split("m"); } - if (intervalType != null && StringUtils.isNumeric(intervals[0])) { + if (intervalType!=null && StringUtils.isNumeric(intervals[0])) { int interval = Integer.parseInt(intervals[0]); switch (intervalType) { case DAY: @@ -822,7 +814,7 @@ private static int getInterval(String intervalParam) { case HOURS: return interval * 60; case MINUTES: - return interval; + return interval ; default: return 1440; } @@ -833,5 +825,5 @@ private static int getInterval(String intervalParam) { } return 1440; } - + } diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java index a6184354..cfe11fb5 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java @@ -26,13 +26,11 @@ import com.google.gson.JsonArray; import com.google.gson.JsonElement; -import com.google.gson.JsonObject; import com.google.gson.JsonParser; import java.util.Base64; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; /** * Simple http client for pulling and acknowledging messages from AMS service @@ -60,7 +58,6 @@ public class ArgoMessagingClient { // proxy private URI proxy = null; private String topic; - private String runDate; // Utility inner class for holding list of messages and acknowledgements private class MsgAck { @@ -84,11 +81,10 @@ public ArgoMessagingClient() { this.sub = "test_sub"; this.maxMessages = "100"; this.proxy = null; - this.runDate = null; } public ArgoMessagingClient(String method, String token, String endpoint, String project, String sub, int batch, - boolean verify, String runDate) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { + boolean verify) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { this.proto = method; this.token = token; @@ -99,12 +95,11 @@ public ArgoMessagingClient(String method, String token, String endpoint, String this.verify = verify; this.httpClient = buildHttpClient(); - this.runDate = runDate; } public ArgoMessagingClient(String method, String token, String endpoint, String project, String topic, - boolean verify, String runDate) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { + boolean verify) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { this.proto = method; this.token = token; @@ -112,7 +107,7 @@ public ArgoMessagingClient(String method, String token, String endpoint, String this.project = project; this.topic = topic; this.verify = verify; - this.runDate = runDate; + this.httpClient = buildHttpClient(); } @@ -187,15 +182,10 @@ public void logIssue(CloseableHttpResponse resp) throws UnsupportedOperationExce */ public String composeURL(String method) { - switch (method) { - case "publish": - return proto + "://" + endpoint + "/v1/projects/" + project + "/topics/" + topic + ":" + method; - case "offsets": - return proto + "://" + endpoint + "/v1/projects/" + project + "/subscriptions/" + this.sub + ":" + method; - case "timeToOffset": - return proto + "://" + endpoint + "/v1/projects/" + project + "/subscriptions/" + this.sub + ":" + method+"?time=" + this.runDate; - default: - return proto + "://" + endpoint + "/v1/projects/" + project + "/subscriptions/" + sub + ":" + method; + if (method.equals("publish")) { + return proto + "://" + endpoint + "/v1/projects/" + project + "/topics/" + topic + ":" + method; + }else{ + return proto + "://" + endpoint + "/v1/projects/" + project + "/subscriptions/" + sub + ":" + method; } } @@ -210,17 +200,17 @@ public MsgAck doPull() throws IOException, KeyManagementException, NoSuchAlgorit // Create the http post to pull HttpPost postPull = new HttpPost(this.composeURL("pull")); - String body = "{\"maxMessages\":\"" + this.maxMessages + "\",\"returnImmediately\":\"true\"}"; - + String body = "{\"maxMessages\":\"" + this.maxMessages + "\",\"returnImmediately\":\"true\"}"; + postPull.addHeader("Accept", "application/json"); postPull.addHeader("x-api-key", this.token); postPull.addHeader("Content-type", "application/json"); StringEntity postBody = new StringEntity(body); postBody.setContentType("application/json"); - + postPull.setEntity(postBody); - + if (this.httpClient == null) { this.httpClient = buildHttpClient(); } @@ -319,170 +309,6 @@ public String[] consume() throws KeyManagementException, NoSuchAlgorithmExceptio } - /** - * Executes an offsetByTimestamp request against AMS api - */ - public int offsetByTimestamp() throws IOException, KeyManagementException, NoSuchAlgorithmException, KeyStoreException { - - - String method = "timeToOffset"; - if (this.runDate == null) { - method = "offsets"; - } - HttpGet getOffset = new HttpGet((this.composeURL(method))); - getOffset.addHeader("Accept", "application/json"); - getOffset.addHeader("x-api-key", this.token); - getOffset.addHeader("Content-type", "application/json"); - - if (this.httpClient == null) { - this.httpClient = buildHttpClient(); - } - - // check for proxy - if (this.proxy != null) { - getOffset.setConfig(createProxyCfg()); - } - - CloseableHttpResponse response = this.httpClient.execute(getOffset); - int offset = 0; - StringBuilder result = new StringBuilder(); - - HttpEntity entity = response.getEntity(); - - int statusCode = response.getStatusLine().getStatusCode(); - - if (entity != null && statusCode == 200) { - - InputStreamReader isRdr = new InputStreamReader(entity.getContent()); - BufferedReader bRdr = new BufferedReader(isRdr); - - String rLine; - - while ((rLine = bRdr.readLine()) != null) { - result.append(rLine); - } - - // Gather message from json - JsonParser jsonParser = new JsonParser(); - // parse the json root object - Log.info("response: {}", result.toString()); - JsonElement jRoot = jsonParser.parse(result.toString()); - - if (this.runDate != null) { - offset = jRoot.getAsJsonObject().get("offset").getAsInt(); - } else { - offset = jRoot.getAsJsonObject().get("max").getAsInt(); - - } - // if has elements - - isRdr.close(); - - } else { - - logIssue(response); - - } - - response.close(); - - // Return a Message array - return offset; - - } - - /** - * Executes a get offset requests against AMS api - */ - public int offset() throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException { - // Try first to pull a message - int offset = 0; - try { - - offset = offsetByTimestamp(); - // get last ackid - - if (offset != 0) { - // Do an ack for the received message - Log.info("Current offset is :" + offset); - - } else { - Log.warn("Not updated offset:"); - } - - } catch (IOException e) { - LOG.error(e.getMessage()); - } - return offset; - - } - - /** - * Executes a modify offset request against AMS api - * @offset , the offset number to modify - */ - public int modifyOffset(int offset) throws IOException, KeyManagementException, NoSuchAlgorithmException, KeyStoreException { - - // Create the http post to pull - String url = proto + "://" + endpoint + "/v1/projects/" + project + "/subscriptions/" + this.sub + ":" + "modifyOffset"; - - HttpPost postModifyOffset = new HttpPost(url); - - postModifyOffset.addHeader("Accept", "application/json"); - postModifyOffset.addHeader("x-api-key", this.token); - postModifyOffset.addHeader("Content-type", "application/json"); - System.out.println("modify offset is :" + offset); - String body = "{\"offset\":" + offset + "}"; - - StringEntity postBody = new StringEntity(body); - postBody.setContentType("application/json"); - - postModifyOffset.setEntity(postBody); - - if (this.httpClient == null) { - this.httpClient = buildHttpClient(); - } - - // check for proxy - if (this.proxy != null) { - postModifyOffset.setConfig(createProxyCfg()); - } - - CloseableHttpResponse response = this.httpClient.execute(postModifyOffset); - StringBuilder result = new StringBuilder(); - - HttpEntity entity = response.getEntity(); - - int statusCode = response.getStatusLine().getStatusCode(); - - if (entity != null && statusCode == 200) { - - try ( InputStreamReader isRdr = new InputStreamReader(entity.getContent())) { - BufferedReader bRdr = new BufferedReader(isRdr); - - String rLine; - while ((rLine = bRdr.readLine()) != null) { - result.append(rLine); - } - isRdr.close(); - // Gather message from json - // JsonParser jsonParser = new JsonParser(); - // parse the json root object - Log.info("modify offset response: {}", result.toString()); - } - } else { - - logIssue(response); - - } - - response.close(); - - // Return a Message array - return offset; - - } - /** * Executes an Acknowledge request against AMS api */ @@ -601,4 +427,4 @@ public void publish(String in) throws IOException, NoSuchAlgorithmException, Key public void close() throws IOException { this.httpClient.close(); } -} +} \ No newline at end of file diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSink.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSink.java index 56970122..bfdcba3f 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSink.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSink.java @@ -23,13 +23,12 @@ public class ArgoMessagingSink extends RichSinkFunction { private boolean verify = true; private boolean useProxy = false; private String proxyURL = ""; - private String date; // private transient Object rateLck; // lock for waiting to establish rate // private volatile boolean isRunning = true; private ArgoMessagingClient client = null; - public ArgoMessagingSink(String endpoint, String port, String token, String project, String topic, Long interval, String date) { + public ArgoMessagingSink(String endpoint, String port, String token, String project, String topic, Long interval) { this.endpoint = endpoint; this.port = port; this.token = token; @@ -37,7 +36,6 @@ public ArgoMessagingSink(String endpoint, String port, String token, String proj this.topic = topic; this.interval = interval; this.verify = true; - this.date=date; } @@ -48,7 +46,7 @@ public void open(Configuration parameters) throws Exception { if (this.port != null && !this.port.isEmpty()) { fendpoint = this.endpoint + ":" + port; } - client = new ArgoMessagingClient("https", this.token, fendpoint, this.project, this.topic, this.verify, this.date); + client = new ArgoMessagingClient("https", this.token, fendpoint, this.project, this.topic, this.verify); if (this.useProxy) { client.setProxy(this.proxyURL); } diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSource.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSource.java index dc7b9df8..30997677 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSource.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSource.java @@ -34,10 +34,9 @@ public class ArgoMessagingSource extends RichSourceFunction { private volatile boolean isRunning = true; private ArgoMessagingClient client = null; - private String runDate; - public ArgoMessagingSource(String endpoint, String port, String token, String project, String sub, int batch, Long interval, String runDate) { + public ArgoMessagingSource(String endpoint, String port, String token, String project, String sub, int batch, Long interval) { this.endpoint = endpoint; this.port = port; this.token = token; @@ -46,7 +45,6 @@ public ArgoMessagingSource(String endpoint, String port, String token, String pr this.interval = interval; this.batch = batch; this.verify = true; - this.runDate=runDate; } @@ -111,7 +109,7 @@ public void open(Configuration parameters) throws Exception { fendpoint = this.endpoint + ":" + port; } try { - client = new ArgoMessagingClient("https", this.token, fendpoint, this.project, this.sub, this.batch, this.verify, this.runDate); + client = new ArgoMessagingClient("https", this.token, fendpoint, this.project, this.sub, this.batch, this.verify); if (this.useProxy) { client.setProxy(this.proxyURL); } @@ -134,4 +132,4 @@ public void close() throws Exception { } } -} \ No newline at end of file +} diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/StatusConfig.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/StatusConfig.java index 3246f734..91e49567 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/StatusConfig.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/StatusConfig.java @@ -1,14 +1,8 @@ package argo.streaming; import java.io.Serializable; -import java.util.Calendar; -import java.util.TimeZone; import org.apache.flink.api.java.utils.ParameterTool; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; public class StatusConfig implements Serializable { @@ -63,7 +57,7 @@ public StatusConfig(ParameterTool pt) { this.amsProject = pt.getRequired("ams.project"); this.apiEndpoint = pt.getRequired("api.endpoint"); - this.runDate = pt.get("run.date"); + this.runDate = pt.getRequired("run.date"); this.report = pt.getRequired("report"); // Optional timeout parameter if (pt.has("timeout")) { From bd769fe4c29d44be764e3d4841de5d27455a9b9b Mon Sep 17 00:00:00 2001 From: cthermolia-grnet Date: Thu, 22 Sep 2022 12:28:21 +0300 Subject: [PATCH 14/16] ARGO-3987 Make AMS related code common in flink jobs #357 --- .../main/java/ams/publisher/AmsPublisher.java | 116 --- .../ams/publisher/ArgoMessagingClient.java | 212 ------ .../java/ams/publisher/ArgoMessagingSink.java | 90 --- .../.gitignore | 4 +- .../{AmsPublisher => ams-connector}/pom.xml | 10 +- .../ams/connector}/ArgoMessagingClient.java | 4 +- .../ams/connector}/ArgoMessagingSink.java | 2 +- .../ams/connector}/ArgoMessagingSource.java | 4 +- flink_jobs_v2/ams_ingest_metric/pom.xml | 709 +++++++++--------- .../java/argo/streaming/AmsIngestMetric.java | 440 ++++++----- .../argo/streaming/ArgoMessagingClient.java | 351 --------- .../argo/streaming/ArgoMessagingSource.java | 135 ---- flink_jobs_v2/ams_ingest_sync/pom.xml | 661 ++++++++-------- .../java/argo/streaming/AmsIngestSync.java | 202 ++--- .../argo/streaming/ArgoMessagingClient.java | 350 --------- .../argo/streaming/ArgoMessagingSource.java | 135 ---- flink_jobs_v2/pom.xml | 5 +- flink_jobs_v2/stream_status/pom.xml | 6 + .../java/argo/streaming/AmsStreamStatus.java | 2 + .../java/argo/streaming/StatusConfig.java | 2 +- 20 files changed, 1037 insertions(+), 2403 deletions(-) delete mode 100644 flink_jobs_v2/AmsPublisher/src/main/java/ams/publisher/AmsPublisher.java delete mode 100644 flink_jobs_v2/AmsPublisher/src/main/java/ams/publisher/ArgoMessagingClient.java delete mode 100644 flink_jobs_v2/AmsPublisher/src/main/java/ams/publisher/ArgoMessagingSink.java rename flink_jobs_v2/{AmsPublisher => ams-connector}/.gitignore (70%) rename flink_jobs_v2/{AmsPublisher => ams-connector}/pom.xml (98%) rename flink_jobs_v2/{stream_status/src/main/java/argo/streaming => ams-connector/src/main/java/ams/connector}/ArgoMessagingClient.java (99%) rename flink_jobs_v2/{stream_status/src/main/java/argo/streaming => ams-connector/src/main/java/ams/connector}/ArgoMessagingSink.java (99%) rename flink_jobs_v2/{stream_status/src/main/java/argo/streaming => ams-connector/src/main/java/ams/connector}/ArgoMessagingSource.java (90%) delete mode 100644 flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/ArgoMessagingClient.java delete mode 100644 flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/ArgoMessagingSource.java delete mode 100644 flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/ArgoMessagingClient.java delete mode 100644 flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/ArgoMessagingSource.java diff --git a/flink_jobs_v2/AmsPublisher/src/main/java/ams/publisher/AmsPublisher.java b/flink_jobs_v2/AmsPublisher/src/main/java/ams/publisher/AmsPublisher.java deleted file mode 100644 index bc163b7a..00000000 --- a/flink_jobs_v2/AmsPublisher/src/main/java/ams/publisher/AmsPublisher.java +++ /dev/null @@ -1,116 +0,0 @@ -package ams.publisher; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import org.apache.flink.api.common.restartstrategy.RestartStrategies; -import org.apache.flink.api.common.time.Time; -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * AmsPublisher implements a flink connector to messaging system - * parameters: - * --ams.endpoint : ARGO messaging api endoint to connect to - * msg.example.com - * --ams.port : ARGO messaging api port - * --ams.token : ARGO messaging api token of publisher - * --ams.project : ARGO messaging api project to connect to - * --ams.topic : ARGO messaging topic to publish messages - * --ams.verify : optional turn on/off ssl verify - * --ams.interval : optional interval to timeout connection - - */ -public class AmsPublisher { - - // setup logger - static Logger LOG = LoggerFactory.getLogger(AmsPublisher.class); - - /** - * Check if a list of expected cli arguments have been provided to this - * flink job - */ - public static boolean hasArgs(String[] reqArgs, ParameterTool paramTool) { - - for (String reqArg : reqArgs) { - if (!paramTool.has(reqArg)) { - return false; - } - } - - return true; - } - - /** - * Check if flink job has been called with ams rate params - */ - public static boolean hasAmsRateArgs(ParameterTool paramTool) { - String args[] = {"ams.interval"}; - return hasArgs(args, paramTool); - } - - // main job function - public static void main(String[] args) throws Exception { - - // Create flink execution enviroment - StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment(); - see.setParallelism(1); - // Fixed restart strategy: on failure attempt max 10 times to restart with a retry interval of 2 minutes - see.setRestartStrategy(RestartStrategies.fixedDelayRestart(10, Time.of(2, TimeUnit.MINUTES))); - // Initialize cli parameter tool - final ParameterTool parameterTool = ParameterTool.fromArgs(args); - - // Initialize Input Source : ARGO Messaging Source - String endpoint = parameterTool.getRequired("ams.endpoint"); - String port = parameterTool.get("ams.port"); - String token = parameterTool.getRequired("ams.token"); - String project = parameterTool.getRequired("ams.project"); - String topic = parameterTool.getRequired("ams.topic"); - - // set ams client batch and interval to default values - //int batch = 1; - long interval = 100L; - - if (hasAmsRateArgs(parameterTool)) { - interval = parameterTool.getLong("ams.interval"); - } - - //Ingest sync avro encoded data from AMS endpoint - ArgoMessagingSink ams = new ArgoMessagingSink(endpoint, port, token, project, topic, interval); - - if (parameterTool.has("ams.verify")) { - ams.setVerify(parameterTool.getBoolean("ams.verify")); - } - - if (parameterTool.has("ams.proxy")) { - ams.setProxy(parameterTool.get("ams.proxy")); - } - - List events = new ArrayList<>(); - int i; - for (i = 0; i < 10; i++) { - events.add("hello world! _ "+i); - } - DataStream eventstreams = see.fromCollection(events); - eventstreams.addSink(ams); - - // Create a job title message to discern job in flink dashboard/cli - StringBuilder jobTitleSB = new StringBuilder(); - jobTitleSB.append("Publish data to AMS "); - jobTitleSB.append(endpoint); - jobTitleSB.append(":"); - jobTitleSB.append(port); - jobTitleSB.append("/v1/projects/"); - jobTitleSB.append(project); - jobTitleSB.append("/topic/"); - jobTitleSB.append(topic); - - see.execute(jobTitleSB.toString()); - - } - -} diff --git a/flink_jobs_v2/AmsPublisher/src/main/java/ams/publisher/ArgoMessagingClient.java b/flink_jobs_v2/AmsPublisher/src/main/java/ams/publisher/ArgoMessagingClient.java deleted file mode 100644 index 30838f19..00000000 --- a/flink_jobs_v2/AmsPublisher/src/main/java/ams/publisher/ArgoMessagingClient.java +++ /dev/null @@ -1,212 +0,0 @@ -package ams.publisher; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.URI; -import java.net.URISyntaxException; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.util.Base64; -import javax.ws.rs.core.Request; -import org.apache.http.HttpEntity; -import org.apache.http.HttpHost; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.TrustSelfSignedStrategy; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.ssl.SSLContextBuilder; -import org.mortbay.log.Log; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; - -/** - * Simple http client for pulling and acknowledging messages from AMS service - * http API - */ -public class ArgoMessagingClient { - - static Logger LOG = LoggerFactory.getLogger(ArgoMessagingClient.class); - // Http Client for contanting AMS service - private CloseableHttpClient httpClient = null; - // AMS endpoint (hostname:port or hostname) - private String endpoint = null; - // AMS project (/v1/projects/{project}) - private String project = null; - // AMS token (?key={token}) - private String token = null; - // AMS subscription (/v1/projects/{project}/subscriptions/{sub}) - private String topic = null; - // protocol (https,http) - private String proto = null; - // numer of message to be pulled; - private String maxMessages = ""; - // ssl verify or not - private boolean verify = true; - // proxy - private URI proxy = null; - - public ArgoMessagingClient() { - this.httpClient = HttpClients.createDefault(); - this.proto = "https"; - this.token = "token"; - this.endpoint = "localhost"; - this.project = "test_project"; - this.topic = "test_topic"; - this.maxMessages = "100"; - this.proxy = null; - } - - public ArgoMessagingClient(String method, String token, String endpoint, String project, String topic, - boolean verify) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { - - this.proto = method; - this.token = token; - this.endpoint = endpoint; - this.project = project; - this.topic = topic; - this.verify = verify; - - this.httpClient = buildHttpClient(); - - } - - /** - * Initializes Http Client (if not initialized during constructor) - * - * @return - */ - private CloseableHttpClient buildHttpClient() - throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { - if (this.verify) { - return this.httpClient = HttpClients.createDefault(); - } else { - return this.httpClient = HttpClients.custom().setSSLSocketFactory(selfSignedSSLF()).build(); - } - } - - /** - * Create an SSL Connection Socket Factory with a strategy to trust self - * signed certificates - */ - private SSLConnectionSocketFactory selfSignedSSLF() - throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { - SSLContextBuilder sslBuild = new SSLContextBuilder(); - sslBuild.loadTrustMaterial(null, new TrustSelfSignedStrategy()); - return new SSLConnectionSocketFactory(sslBuild.build(), NoopHostnameVerifier.INSTANCE); - } - - /** - * Set AMS http client to use http proxy - */ - public void setProxy(String proxyURL) throws URISyntaxException { - // parse proxy url - this.proxy = URI.create(proxyURL); - } - - /** - * Set AMS http client to NOT use an http proxy - */ - public void unsetProxy() { - this.proxy = null; - } - - /** - * Create a configuration for using http proxy on each request - */ - private RequestConfig createProxyCfg() { - HttpHost proxy = new HttpHost(this.proxy.getHost(), this.proxy.getPort(), this.proxy.getScheme()); - RequestConfig config = RequestConfig.custom().setProxy(proxy).build(); - return config; - } - - public void logIssue(CloseableHttpResponse resp) throws UnsupportedOperationException, IOException { - int statusCode; - StringBuilder result; - try ( InputStreamReader isRdr = new InputStreamReader(resp.getEntity().getContent())) { - BufferedReader bRdr = new BufferedReader(isRdr); - statusCode = resp.getStatusLine().getStatusCode(); - // Parse error content from api response - result = new StringBuilder(); - String rLine; - while ((rLine = bRdr.readLine()) != null) { - result.append(rLine); - } - } - Log.warn("ApiStatusCode={}, ApiErrorMessage={}", statusCode, result); - - } - - /** - * Properly compose url for each AMS request - */ - public String composeURL(String method) { - return proto + "://" + endpoint + "/v1/projects/" + project + "/topics/" + topic + ":" + method; - } - - /** - * Executes a combination of Pull & Ack requests against AMS api - */ - public void publish(String in) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException { - String encodedString = Base64.getEncoder().encodeToString(in.getBytes()); - // Create the http post to pull - HttpPost postPublish = new HttpPost(this.composeURL("publish")); - String body = "{\"messages\": [ {\"data\":\"" + encodedString + "\"}]}"; - - postPublish.addHeader("Accept", "application/json"); - postPublish.addHeader("x-api-key", this.token); - postPublish.addHeader("Content-type", "application/json"); - - StringEntity postBody = new StringEntity(body); - postPublish.setEntity(postBody); - - if (this.httpClient == null) { - this.httpClient = buildHttpClient(); - } - - // check for proxy - if (this.proxy != null) { - postPublish.setConfig(createProxyCfg()); - } - - try ( CloseableHttpResponse response = this.httpClient.execute(postPublish)) { - StringBuilder result = new StringBuilder(); - - HttpEntity entity = response.getEntity(); - - int statusCode = response.getStatusLine().getStatusCode(); - - if (entity != null && statusCode == 200) { - - try ( InputStreamReader isRdr = new InputStreamReader(entity.getContent())) { - BufferedReader bRdr = new BufferedReader(isRdr); - String rLine; - while ((rLine = bRdr.readLine()) != null) { - result.append(rLine); - } - Log.info("response: {}", result.toString()); - } - } else { - - logIssue(response); - - } - response.close(); - } - - } - - /** - * Close AMS http client - */ - public void close() throws IOException { - this.httpClient.close(); - } -} diff --git a/flink_jobs_v2/AmsPublisher/src/main/java/ams/publisher/ArgoMessagingSink.java b/flink_jobs_v2/AmsPublisher/src/main/java/ams/publisher/ArgoMessagingSink.java deleted file mode 100644 index 5ef865bf..00000000 --- a/flink_jobs_v2/AmsPublisher/src/main/java/ams/publisher/ArgoMessagingSink.java +++ /dev/null @@ -1,90 +0,0 @@ -package ams.publisher; - -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ArgoMessagingSink extends RichSinkFunction { - - private static final long serialVersionUID = 1L; - - // setup logger - static Logger LOG = LoggerFactory.getLogger(ArgoMessagingSink.class); - - private String endpoint = null; - private String port = null; - private String token = null; - private String project = null; - private String topic = null; - private int batch = 1; - private long interval = 100L; - private boolean verify = true; - private boolean useProxy = false; - private String proxyURL = ""; -// private transient Object rateLck; // lock for waiting to establish rate - - // private volatile boolean isRunning = true; - private ArgoMessagingClient client = null; - - public ArgoMessagingSink(String endpoint, String port, String token, String project, String topic, Long interval) { - this.endpoint = endpoint; - this.port = port; - this.token = token; - this.project = project; - this.topic = topic; - this.interval = interval; - this.verify = true; - - } - - @Override - public void open(Configuration parameters) throws Exception { - - String fendpoint = this.endpoint; - if (this.port != null && !this.port.isEmpty()) { - fendpoint = this.endpoint + ":" + port; - } - client = new ArgoMessagingClient("https", this.token, fendpoint, this.project, this.topic, this.verify); - if (this.useProxy) { - client.setProxy(this.proxyURL); - } - } - - @Override - public void invoke(String in) throws Exception { - this.client.publish(in); - } - - @Override - public void close() throws Exception { - - this.client.close(); - - } - - /** - * Set verify to true or false. If set to false AMS client will be able to - * contact AMS endpoints that use self-signed certificates - */ - public void setVerify(boolean verify) { - this.verify = verify; - } - - /** - * Set proxy details for AMS client - */ - public void setProxy(String proxyURL) { - this.useProxy = true; - this.proxyURL = proxyURL; - } - - /** - * Unset proxy details for AMS client - */ - public void unsetProxy(String proxyURL) { - this.useProxy = false; - this.proxyURL = ""; - } - -} diff --git a/flink_jobs_v2/AmsPublisher/.gitignore b/flink_jobs_v2/ams-connector/.gitignore similarity index 70% rename from flink_jobs_v2/AmsPublisher/.gitignore rename to flink_jobs_v2/ams-connector/.gitignore index 402ca49c..78fae2e1 100644 --- a/flink_jobs_v2/AmsPublisher/.gitignore +++ b/flink_jobs_v2/ams-connector/.gitignore @@ -4,6 +4,4 @@ .classpath/ .classpath /nbproject -nbactions.xml -.idea/ -*.iml +/nbactions.xml diff --git a/flink_jobs_v2/AmsPublisher/pom.xml b/flink_jobs_v2/ams-connector/pom.xml similarity index 98% rename from flink_jobs_v2/AmsPublisher/pom.xml rename to flink_jobs_v2/ams-connector/pom.xml index 676ae168..4cb5cdb4 100644 --- a/flink_jobs_v2/AmsPublisher/pom.xml +++ b/flink_jobs_v2/ams-connector/pom.xml @@ -19,10 +19,10 @@ 2.0.0 - argo.streaming + ams.connector 2.0.0 - AmsPublisher - Stream data to AMS + ams.connector + Connect to AMS UTF-8 @@ -338,5 +338,5 @@ --> - AmsPublisher - + ams-connector + \ No newline at end of file diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java b/flink_jobs_v2/ams-connector/src/main/java/ams/connector/ArgoMessagingClient.java similarity index 99% rename from flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java rename to flink_jobs_v2/ams-connector/src/main/java/ams/connector/ArgoMessagingClient.java index 4a1bae5f..653740a3 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingClient.java +++ b/flink_jobs_v2/ams-connector/src/main/java/ams/connector/ArgoMessagingClient.java @@ -1,4 +1,6 @@ -package argo.streaming; +package ams.connector; + + import java.io.BufferedReader; import java.io.IOException; diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSink.java b/flink_jobs_v2/ams-connector/src/main/java/ams/connector/ArgoMessagingSink.java similarity index 99% rename from flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSink.java rename to flink_jobs_v2/ams-connector/src/main/java/ams/connector/ArgoMessagingSink.java index 79873242..e2205f92 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSink.java +++ b/flink_jobs_v2/ams-connector/src/main/java/ams/connector/ArgoMessagingSink.java @@ -1,4 +1,4 @@ -package argo.streaming; +package ams.connector; import org.apache.flink.configuration.Configuration; diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSource.java b/flink_jobs_v2/ams-connector/src/main/java/ams/connector/ArgoMessagingSource.java similarity index 90% rename from flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSource.java rename to flink_jobs_v2/ams-connector/src/main/java/ams/connector/ArgoMessagingSource.java index e938335d..806d1e68 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/ArgoMessagingSource.java +++ b/flink_jobs_v2/ams-connector/src/main/java/ams/connector/ArgoMessagingSource.java @@ -1,4 +1,4 @@ -package argo.streaming; +package ams.connector; import java.security.KeyManagementException; import java.security.KeyStoreException; @@ -115,6 +115,8 @@ public void open(Configuration parameters) throws Exception { if (this.useProxy) { client.setProxy(this.proxyURL); } + int offset=client.offset(); //get the offset of the subscription, that corresponds to the date + client.modifyOffset(offset); //mofify the offset of the subscription to point to the offset index of the date. if date is null then the index points to the latest offset (max) } catch (KeyManagementException e) { e.printStackTrace(); } catch (NoSuchAlgorithmException e) { diff --git a/flink_jobs_v2/ams_ingest_metric/pom.xml b/flink_jobs_v2/ams_ingest_metric/pom.xml index e2fb50ce..312d16a3 100644 --- a/flink_jobs_v2/ams_ingest_metric/pom.xml +++ b/flink_jobs_v2/ams_ingest_metric/pom.xml @@ -1,378 +1,385 @@ +license agreements. See the NOTICE file distributed with this work for additional +information regarding copyright ownership. The ASF licenses this file to +you under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +by applicable law or agreed to in writing, software distributed under the +License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +OF ANY KIND, either express or implied. See the License for the specific +language governing permissions and limitations under the License. --> - 4.0.0 + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 - argo.streaming - ams-ingest-metric - 0.1 - jar + argo.streaming + ams-ingest-metric + 0.1 + jar - ARGO AMS Ingest Metric Data job + ARGO AMS Ingest Metric Data job - - UTF-8 - 1.3.2 - + + UTF-8 + 1.3.2 + - + - - cloudera - https://repository.cloudera.com/artifactory/cloudera-repos/ - + + cloudera + https://repository.cloudera.com/artifactory/cloudera-repos/ + - - apache.snapshots - Apache Development Snapshot Repository - https://repository.apache.org/content/repositories/snapshots/ - - false - - - true - - - + + apache.snapshots + Apache Development Snapshot Repository + https://repository.apache.org/content/repositories/snapshots/ + + false + + + true + + + - + - - - org.apache.avro - avro - 1.7.7 - - - org.apache.flink - flink-connector-filesystem_2.10 - ${flink.version} - - - org.apache.flink - flink-java - ${flink.version} - - - org.apache.flink - flink-streaming-java_2.10 - ${flink.version} - - - org.apache.flink - flink-clients_2.10 - ${flink.version} - - - org.apache.flink - flink-connector-kafka-0.9_2.10 - ${flink.version} - - - commons-codec - commons-codec - 20041127.091804 - - - com.google.code.gson - gson - 2.8.9 - - - - org.apache.hbase - hbase-client - 1.2.0-cdh5.7.4 - - - - org.apache.httpcomponents - httpclient - 4.5.13 - - - org.apache.httpcomponents - fluent-hc - 4.5.13 - + + + + ams.connector + ams-connector + 2.0.0 + jar + + + org.apache.avro + avro + 1.7.7 + + + org.apache.flink + flink-connector-filesystem_2.10 + ${flink.version} + + + org.apache.flink + flink-java + ${flink.version} + + + org.apache.flink + flink-streaming-java_2.10 + ${flink.version} + + + org.apache.flink + flink-clients_2.10 + ${flink.version} + + + org.apache.flink + flink-connector-kafka-0.9_2.10 + ${flink.version} + + + commons-codec + commons-codec + 20041127.091804 + + + com.google.code.gson + gson + 2.8.9 + + + + org.apache.hbase + hbase-client + 1.2.0-cdh5.7.4 + + + + org.apache.httpcomponents + httpclient + 4.5.13 + + + org.apache.httpcomponents + fluent-hc + 4.5.13 + - + - - - - build-jar - - false - - - - org.apache.avro - avro - 1.7.7 - - - org.apache.flink - flink-java - ${flink.version} - provided - - - org.apache.flink - flink-connector-filesystem_2.10 - ${flink.version} - - - org.apache.flink - flink-streaming-java_2.10 - ${flink.version} - provided - - - org.apache.flink - flink-clients_2.10 - ${flink.version} - provided - - - org.apache.flink - flink-connector-kafka-0.9_2.10 - ${flink.version} - provided - - - commons-codec - commons-codec - 20041127.091804 - provided - - - - com.google.code.gson - gson - 2.8.9 - - - - org.apache.hbase - hbase-client - 1.2.0-cdh5.7.4 - - - org.apache.httpcomponents - httpclient - 4.5.13 - - - org.apache.httpcomponents - fluent-hc - 4.5.13 - + + + + build-jar + + false + + + + org.apache.avro + avro + 1.7.7 + + + org.apache.flink + flink-java + ${flink.version} + provided + + + org.apache.flink + flink-connector-filesystem_2.10 + ${flink.version} + + + org.apache.flink + flink-streaming-java_2.10 + ${flink.version} + provided + + + org.apache.flink + flink-clients_2.10 + ${flink.version} + provided + + + org.apache.flink + flink-connector-kafka-0.9_2.10 + ${flink.version} + provided + + + commons-codec + commons-codec + 20041127.091804 + provided + + + + com.google.code.gson + gson + 2.8.9 + + + + org.apache.hbase + hbase-client + 1.2.0-cdh5.7.4 + + + org.apache.httpcomponents + httpclient + 4.5.13 + + + org.apache.httpcomponents + fluent-hc + 4.5.13 + - + - - - - - org.apache.maven.plugins - maven-shade-plugin - 2.4.1 - - - package - - shade - - - - - - - - - - - - - + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.1 + + + package + + shade + + + + + + + + + + + + + - - - - - org.apache.maven.plugins - maven-shade-plugin - 2.4.1 - - - - package - - shade - - - - - - org.apache.flink:flink-annotations - org.apache.flink:flink-shaded-hadoop1 - org.apache.flink:flink-shaded-hadoop2 - org.apache.flink:flink-shaded-curator-recipes - org.apache.flink:flink-core - org.apache.flink:flink-java - org.apache.flink:flink-scala_2.10 - org.apache.flink:flink-runtime_2.10 - org.apache.flink:flink-optimizer_2.10 - org.apache.flink:flink-clients_2.10 - org.apache.flink:flink-avro_2.10 - org.apache.flink:flink-examples-batch_2.10 - org.apache.flink:flink-examples-streaming_2.10 - org.apache.flink:flink-streaming-java_2.10 + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.1 + + + + package + + shade + + + + + + org.apache.flink:flink-annotations + org.apache.flink:flink-shaded-hadoop1 + org.apache.flink:flink-shaded-hadoop2 + org.apache.flink:flink-shaded-curator-recipes + org.apache.flink:flink-core + org.apache.flink:flink-java + org.apache.flink:flink-scala_2.10 + org.apache.flink:flink-runtime_2.10 + org.apache.flink:flink-optimizer_2.10 + org.apache.flink:flink-clients_2.10 + org.apache.flink:flink-avro_2.10 + org.apache.flink:flink-examples-batch_2.10 + org.apache.flink:flink-examples-streaming_2.10 + org.apache.flink:flink-streaming-java_2.10 - - org.scala-lang:scala-library - org.scala-lang:scala-compiler - org.scala-lang:scala-reflect - com.amazonaws:aws-java-sdk - com.typesafe.akka:akka-actor_* - com.typesafe.akka:akka-remote_* - com.typesafe.akka:akka-slf4j_* - io.netty:netty-all - io.netty:netty - commons-fileupload:commons-fileupload - org.apache.avro:avro - commons-collections:commons-collections - org.codehaus.jackson:jackson-core-asl - org.codehaus.jackson:jackson-mapper-asl - com.thoughtworks.paranamer:paranamer - org.xerial.snappy:snappy-java - org.apache.commons:commons-compress - org.tukaani:xz - com.esotericsoftware.kryo:kryo - com.esotericsoftware.minlog:minlog - org.objenesis:objenesis - com.twitter:chill_* - com.twitter:chill-java - com.twitter:chill-avro_* - com.twitter:chill-bijection_* - com.twitter:bijection-core_* - com.twitter:bijection-avro_* - commons-lang:commons-lang - junit:junit - de.javakaffee:kryo-serializers - joda-time:joda-time - org.apache.commons:commons-lang3 - org.slf4j:slf4j-api - org.slf4j:slf4j-log4j12 - log4j:log4j - org.apache.commons:commons-math - org.apache.sling:org.apache.sling.commons.json - commons-logging:commons-logging - commons-codec:commons-codec - com.fasterxml.jackson.core:jackson-core - com.fasterxml.jackson.core:jackson-databind - com.fasterxml.jackson.core:jackson-annotations - stax:stax-api - com.typesafe:config - org.uncommons.maths:uncommons-maths - com.github.scopt:scopt_* - commons-io:commons-io - commons-cli:commons-cli - - - - - org.apache.flink:* - - - org/apache/flink/shaded/com/** - web-docs/** - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - argo.streaming.AmsIngestMetric - - - false - - - - + + org.scala-lang:scala-library + org.scala-lang:scala-compiler + org.scala-lang:scala-reflect + com.amazonaws:aws-java-sdk + com.typesafe.akka:akka-actor_* + com.typesafe.akka:akka-remote_* + com.typesafe.akka:akka-slf4j_* + io.netty:netty-all + io.netty:netty + commons-fileupload:commons-fileupload + org.apache.avro:avro + commons-collections:commons-collections + org.codehaus.jackson:jackson-core-asl + org.codehaus.jackson:jackson-mapper-asl + com.thoughtworks.paranamer:paranamer + org.xerial.snappy:snappy-java + org.apache.commons:commons-compress + org.tukaani:xz + com.esotericsoftware.kryo:kryo + com.esotericsoftware.minlog:minlog + org.objenesis:objenesis + com.twitter:chill_* + com.twitter:chill-java + com.twitter:chill-avro_* + com.twitter:chill-bijection_* + com.twitter:bijection-core_* + com.twitter:bijection-avro_* + commons-lang:commons-lang + junit:junit + de.javakaffee:kryo-serializers + joda-time:joda-time + org.apache.commons:commons-lang3 + org.slf4j:slf4j-api + org.slf4j:slf4j-log4j12 + log4j:log4j + org.apache.commons:commons-math + org.apache.sling:org.apache.sling.commons.json + commons-logging:commons-logging + commons-codec:commons-codec + com.fasterxml.jackson.core:jackson-core + com.fasterxml.jackson.core:jackson-databind + com.fasterxml.jackson.core:jackson-annotations + stax:stax-api + com.typesafe:config + org.uncommons.maths:uncommons-maths + com.github.scopt:scopt_* + commons-io:commons-io + commons-cli:commons-cli + + + + + org.apache.flink:* + + + org/apache/flink/shaded/com/** + web-docs/** + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + argo.streaming.AmsIngestMetric + + + false + + + + - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - 1.7 - 1.7 - - - + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.7 + 1.7 + + + - - + + - + diff --git a/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/AmsIngestMetric.java b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/AmsIngestMetric.java index 78a0fac3..fd329d20 100644 --- a/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/AmsIngestMetric.java +++ b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/AmsIngestMetric.java @@ -1,6 +1,6 @@ package argo.streaming; - +import ams.connector.ArgoMessagingSource; import java.util.Arrays; import java.util.concurrent.TimeUnit; @@ -26,11 +26,9 @@ import org.apache.flink.streaming.connectors.fs.bucketing.BucketingSink; import org.apache.flink.util.Collector; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import com.google.gson.JsonElement; import com.google.gson.JsonParser; @@ -38,226 +36,222 @@ import argo.avro.MetricData; import argo.avro.MetricDataOld; - /** - * Flink Job : Stream metric data from ARGO messaging to Hbase - * job required cli parameters: - * - * --ams.endpoint : ARGO messaging api endoint to connect to msg.example.com - * --ams.port : ARGO messaging api port - * --ams.token : ARGO messaging api token - * --ams.project : ARGO messaging api project to connect to - * --ams.sub : ARGO messaging subscription to pull from - * --hbase.master : hbase endpoint - * --hbase.master.port : hbase master port - * --hbase.zk.quorum : comma separated list of hbase zookeeper servers - * --hbase.zk.port : port used by hbase zookeeper servers - * --hbase.namespace : table namespace used (usually tenant name) - * --hbase.table : table name (usually metric_data) - * --check.path : checkpoint path - * --check.interval : checkpoint interval - * --hdfs.path : hdfs destination to write the data - * --ams.batch : num of messages to be retrieved per request to AMS service - * --ams.interval : interval (in ms) between AMS service requests - * --ams.proxy : optional http proxy url - * --ams.verify : optional turn on/off ssl verify + * Flink Job : Stream metric data from ARGO messaging to Hbase job required cli + * parameters: + * + * --ams.endpoint : ARGO messaging api endoint to connect to msg.example.com + * --ams.port : ARGO messaging api port --ams.token : ARGO messaging api token + * --ams.project : ARGO messaging api project to connect to --ams.sub : ARGO + * messaging subscription to pull from --hbase.master : hbase endpoint + * --hbase.master.port : hbase master port --hbase.zk.quorum : comma separated + * list of hbase zookeeper servers --hbase.zk.port : port used by hbase + * zookeeper servers --hbase.namespace : table namespace used (usually tenant + * name) --hbase.table : table name (usually metric_data) --check.path : + * checkpoint path --check.interval : checkpoint interval --hdfs.path : hdfs + * destination to write the data --ams.batch : num of messages to be retrieved + * per request to AMS service --ams.interval : interval (in ms) between AMS + * service requests --ams.proxy : optional http proxy url --ams.verify : + * optional turn on/off ssl verify */ public class AmsIngestMetric { - // setup logger - static Logger LOG = LoggerFactory.getLogger(AmsIngestMetric.class); - - /** - * Check if flink job has been called with ams rate params - */ - public static boolean hasAmsRateArgs(ParameterTool paramTool) { - String args[] = { "ams.batch", "ams.interval" }; - return hasArgs(args, paramTool); - } - - - /** - * Check if flink job has been called with checkpoint cli arguments - */ - public static boolean hasCheckArgs(ParameterTool paramTool) { - String args[] = { "check.path", "check.interval" }; - return hasArgs(args, paramTool); - } - - /** - * Check if flink job has been called with hdfs cli arguments - */ - public static boolean hasHdfsArgs(ParameterTool paramTool) { - String args[] = { "hdfs.path" }; - return hasArgs(args, paramTool); - } - - /** - * Check if flink job has been called with hbase cli arguments - */ - public static boolean hasHbaseArgs(ParameterTool paramTool) { - String args[] = { "hbase.master", "hbase.master.port", "hbase.zk.quorum", "hbase.zk.port", "hbase.namespace", - "hbase.table" }; - return hasArgs(args, paramTool); - } - - /** - * Check if a list of expected cli arguments have been provided to this flink job - */ - public static boolean hasArgs(String[] reqArgs, ParameterTool paramTool) { - - for (String reqArg : reqArgs) { - if (!paramTool.has(reqArg)) - return false; - } - - return true; - } - - public static void main(String[] args) throws Exception { - - // Create flink execution environment - StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment(); - see.setParallelism(1); - // On failure attempt max 10 times to restart with a retry interval of 2 minutes - see.setRestartStrategy(RestartStrategies.fixedDelayRestart(10, Time.of(2, TimeUnit.MINUTES))); - - // Initialize cli parameter tool - final ParameterTool parameterTool = ParameterTool.fromArgs(args); - - // set ams client batch and interval to default values - int batch = 1; - long interval = 100L; - long inactivityThresh = 1800000L; // default inactivity threshold value ~ 30mins - - if (hasAmsRateArgs(parameterTool)) { - batch = parameterTool.getInt("ams.batch"); - interval = parameterTool.getLong("ams.interval"); - } - - - // Initialize Input Source : ARGO Messaging Source - String endpoint = parameterTool.getRequired("ams.endpoint"); - String port = parameterTool.getRequired("ams.port"); - String token = parameterTool.getRequired("ams.token"); - String project = parameterTool.getRequired("ams.project"); - String sub = parameterTool.getRequired("ams.sub"); - - - // Check if checkpointing is desired - if (hasCheckArgs(parameterTool)) { - String checkPath = parameterTool.get("check.path"); - String checkInterval = parameterTool.get("check.interval"); - // Establish check-pointing mechanism using the cli-parameter check.path - see.setStateBackend(new FsStateBackend(checkPath)); - // Establish the check-pointing interval - long checkInt = Long.parseLong(checkInterval); - see.enableCheckpointing(checkInt); - } - - // Ingest sync avro encoded data from AMS endpoint - ArgoMessagingSource ams = new ArgoMessagingSource(endpoint, port, token, project, sub, batch, interval); - - if (parameterTool.has("ams.verify")) { - ams.setVerify(parameterTool.getBoolean("ams.verify")); - } - - if (parameterTool.has("ams.proxy")) { - ams.setProxy(parameterTool.get("ams.proxy")); - } - - DataStream metricDataJSON = see.addSource(ams); - DataStream metricDataPOJO = metricDataJSON.flatMap(new FlatMapFunction() { - - /** - * Flat Map Function that accepts AMS message and exports the metric data object (encoded in the payload) - */ - private static final long serialVersionUID = 1L; - - @Override - public void flatMap(String value, Collector out) throws Exception { - - JsonParser jsonParser = new JsonParser(); - // parse the json root object - JsonElement jRoot = jsonParser.parse(value); - // parse the json field "data" and read it as string - // this is the base64 string payload - String data = jRoot.getAsJsonObject().get("data").getAsString(); - // Decode from base64 - byte[] decoded64 = Base64.decodeBase64(data.getBytes("UTF-8")); - // Decode from avro - - DatumReader avroReader = new SpecificDatumReader(MetricData.getClassSchema()); - Decoder decoder = DecoderFactory.get().binaryDecoder(decoded64, null); - - - MetricData item; - try { - item = avroReader.read(null, decoder); - } catch (java.io.EOFException ex) - { - //convert from old to new - avroReader = new SpecificDatumReader(MetricDataOld.getClassSchema(),MetricData.getClassSchema()); - decoder = DecoderFactory.get().binaryDecoder(decoded64, null); - item = avroReader.read(null, decoder); - } - if (item != null) { - LOG.info("Captured data -- {}", item.toString()); - out.collect(item); - } - - - } - }); - - // Check if saving to HDFS is desired - if (hasHdfsArgs(parameterTool)) { - String basePath = parameterTool.getRequired("hdfs.path"); - // Establish a bucketing sink to be able to store events with different daily - // timestamp parts (YYYY-MM-DD) - // in different daily files - BucketingSink bs = new BucketingSink(basePath); - bs.setInactiveBucketThreshold(inactivityThresh); - Bucketer tsBuck = new TSBucketer(); - bs.setBucketer(tsBuck); - bs.setPartPrefix("mdata"); - // Change default in progress prefix: _ to allow loading the file in - // AvroInputFormat - bs.setInProgressPrefix(""); - // Add .prog extension when a file is in progress mode - bs.setInProgressSuffix(".prog"); - // Add .pend extension when a file is in pending mode - bs.setPendingSuffix(".pend"); - - bs.setWriter(new SpecificAvroWriter()); - metricDataPOJO.addSink(bs); - } - - // Check if saving to Hbase is desired - if (hasHbaseArgs(parameterTool)) { - // Initialize Output : Hbase Output Format - HBaseMetricOutputFormat hbf = new HBaseMetricOutputFormat(); - hbf.setMaster(parameterTool.getRequired("hbase.master")); - hbf.setMasterPort(parameterTool.getRequired("hbase.master-port")); - hbf.setZkQuorum(parameterTool.getRequired("hbase.zk.quorum")); - hbf.setZkPort(parameterTool.getRequired("hbase.zk.port")); - hbf.setNamespace(parameterTool.getRequired("hbase.namespace")); - hbf.setTableName(parameterTool.getRequired("hbase.table")); - - metricDataPOJO.writeUsingOutputFormat(hbf); - } - - // Create a job title message to discern job in flink dashboard/cli - StringBuilder jobTitleSB = new StringBuilder(); - jobTitleSB.append("Ingesting metric data from "); - jobTitleSB.append(endpoint); - jobTitleSB.append(":"); - jobTitleSB.append(port); - jobTitleSB.append("/v1/projects/"); - jobTitleSB.append(project); - jobTitleSB.append("/subscriptions/"); - jobTitleSB.append(sub); - - see.execute(jobTitleSB.toString()); - } - -} \ No newline at end of file + // setup logger + + static Logger LOG = LoggerFactory.getLogger(AmsIngestMetric.class); + private static String runDate; + /** + * Check if flink job has been called with ams rate params + */ + public static boolean hasAmsRateArgs(ParameterTool paramTool) { + String args[] = {"ams.batch", "ams.interval"}; + return hasArgs(args, paramTool); + } + + /** + * Check if flink job has been called with checkpoint cli arguments + */ + public static boolean hasCheckArgs(ParameterTool paramTool) { + String args[] = {"check.path", "check.interval"}; + return hasArgs(args, paramTool); + } + + /** + * Check if flink job has been called with hdfs cli arguments + */ + public static boolean hasHdfsArgs(ParameterTool paramTool) { + String args[] = {"hdfs.path"}; + return hasArgs(args, paramTool); + } + + /** + * Check if flink job has been called with hbase cli arguments + */ + public static boolean hasHbaseArgs(ParameterTool paramTool) { + String args[] = {"hbase.master", "hbase.master.port", "hbase.zk.quorum", "hbase.zk.port", "hbase.namespace", + "hbase.table"}; + return hasArgs(args, paramTool); + } + + /** + * Check if a list of expected cli arguments have been provided to this + * flink job + */ + public static boolean hasArgs(String[] reqArgs, ParameterTool paramTool) { + + for (String reqArg : reqArgs) { + if (!paramTool.has(reqArg)) { + return false; + } + } + + return true; + } + + public static void main(String[] args) throws Exception { + + // Create flink execution environment + StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment(); + see.setParallelism(1); + // On failure attempt max 10 times to restart with a retry interval of 2 minutes + see.setRestartStrategy(RestartStrategies.fixedDelayRestart(10, Time.of(2, TimeUnit.MINUTES))); + + // Initialize cli parameter tool + final ParameterTool parameterTool = ParameterTool.fromArgs(args); + + // set ams client batch and interval to default values + int batch = 1; + long interval = 100L; + long inactivityThresh = 1800000L; // default inactivity threshold value ~ 30mins + + if (hasAmsRateArgs(parameterTool)) { + batch = parameterTool.getInt("ams.batch"); + interval = parameterTool.getLong("ams.interval"); + } + + // Initialize Input Source : ARGO Messaging Source + String endpoint = parameterTool.getRequired("ams.endpoint"); + String port = parameterTool.getRequired("ams.port"); + String token = parameterTool.getRequired("ams.token"); + String project = parameterTool.getRequired("ams.project"); + String sub = parameterTool.getRequired("ams.sub"); + runDate = parameterTool.get("run.date"); + if (runDate != null) { + runDate = runDate + "T00:00:00.000Z"; + } + + // Check if checkpointing is desired + if (hasCheckArgs(parameterTool)) { + String checkPath = parameterTool.get("check.path"); + String checkInterval = parameterTool.get("check.interval"); + // Establish check-pointing mechanism using the cli-parameter check.path + see.setStateBackend(new FsStateBackend(checkPath)); + // Establish the check-pointing interval + long checkInt = Long.parseLong(checkInterval); + see.enableCheckpointing(checkInt); + } + + // Ingest sync avro encoded data from AMS endpoint + ArgoMessagingSource ams = new ArgoMessagingSource(endpoint, port, token, project, sub, batch, interval, runDate); + + if (parameterTool.has("ams.verify")) { + ams.setVerify(parameterTool.getBoolean("ams.verify")); + } + + if (parameterTool.has("ams.proxy")) { + ams.setProxy(parameterTool.get("ams.proxy")); + } + + DataStream metricDataJSON = see.addSource(ams); + DataStream metricDataPOJO = metricDataJSON.flatMap(new FlatMapFunction() { + + /** + * Flat Map Function that accepts AMS message and exports the metric + * data object (encoded in the payload) + */ + private static final long serialVersionUID = 1L; + + @Override + public void flatMap(String value, Collector out) throws Exception { + + JsonParser jsonParser = new JsonParser(); + // parse the json root object + JsonElement jRoot = jsonParser.parse(value); + // parse the json field "data" and read it as string + // this is the base64 string payload + String data = jRoot.getAsJsonObject().get("data").getAsString(); + // Decode from base64 + byte[] decoded64 = Base64.decodeBase64(data.getBytes("UTF-8")); + // Decode from avro + + DatumReader avroReader = new SpecificDatumReader(MetricData.getClassSchema()); + Decoder decoder = DecoderFactory.get().binaryDecoder(decoded64, null); + + MetricData item; + try { + item = avroReader.read(null, decoder); + } catch (java.io.EOFException ex) { + //convert from old to new + avroReader = new SpecificDatumReader(MetricDataOld.getClassSchema(), MetricData.getClassSchema()); + decoder = DecoderFactory.get().binaryDecoder(decoded64, null); + item = avroReader.read(null, decoder); + } + if (item != null) { + LOG.info("Captured data -- {}", item.toString()); + out.collect(item); + } + + } + }); + + // Check if saving to HDFS is desired + if (hasHdfsArgs(parameterTool)) { + String basePath = parameterTool.getRequired("hdfs.path"); + // Establish a bucketing sink to be able to store events with different daily + // timestamp parts (YYYY-MM-DD) + // in different daily files + BucketingSink bs = new BucketingSink(basePath); + bs.setInactiveBucketThreshold(inactivityThresh); + Bucketer tsBuck = new TSBucketer(); + bs.setBucketer(tsBuck); + bs.setPartPrefix("mdata"); + // Change default in progress prefix: _ to allow loading the file in + // AvroInputFormat + bs.setInProgressPrefix(""); + // Add .prog extension when a file is in progress mode + bs.setInProgressSuffix(".prog"); + // Add .pend extension when a file is in pending mode + bs.setPendingSuffix(".pend"); + + bs.setWriter(new SpecificAvroWriter()); + metricDataPOJO.addSink(bs); + } + + // Check if saving to Hbase is desired + if (hasHbaseArgs(parameterTool)) { + // Initialize Output : Hbase Output Format + HBaseMetricOutputFormat hbf = new HBaseMetricOutputFormat(); + hbf.setMaster(parameterTool.getRequired("hbase.master")); + hbf.setMasterPort(parameterTool.getRequired("hbase.master-port")); + hbf.setZkQuorum(parameterTool.getRequired("hbase.zk.quorum")); + hbf.setZkPort(parameterTool.getRequired("hbase.zk.port")); + hbf.setNamespace(parameterTool.getRequired("hbase.namespace")); + hbf.setTableName(parameterTool.getRequired("hbase.table")); + + metricDataPOJO.writeUsingOutputFormat(hbf); + } + + // Create a job title message to discern job in flink dashboard/cli + StringBuilder jobTitleSB = new StringBuilder(); + jobTitleSB.append("Ingesting metric data from "); + jobTitleSB.append(endpoint); + jobTitleSB.append(":"); + jobTitleSB.append(port); + jobTitleSB.append("/v1/projects/"); + jobTitleSB.append(project); + jobTitleSB.append("/subscriptions/"); + jobTitleSB.append(sub); + + see.execute(jobTitleSB.toString()); + } + +} diff --git a/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/ArgoMessagingClient.java b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/ArgoMessagingClient.java deleted file mode 100644 index 2bd0bb7c..00000000 --- a/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/ArgoMessagingClient.java +++ /dev/null @@ -1,351 +0,0 @@ -package argo.streaming; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.URI; -import java.net.URISyntaxException; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; - -import org.apache.http.HttpEntity; -import org.apache.http.HttpHost; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.TrustSelfSignedStrategy; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.ssl.SSLContextBuilder; -import org.mortbay.log.Log; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; - -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.CloseableHttpResponse; - -/** - * Simple http client for pulling and acknowledging messages from AMS service - * http API - */ -public class ArgoMessagingClient { - - static Logger LOG = LoggerFactory.getLogger(ArgoMessagingClient.class); - // Http Client for contanting AMS service - private CloseableHttpClient httpClient = null; - // AMS endpoint (hostname:port or hostname) - private String endpoint = null; - // AMS project (/v1/projects/{project}) - private String project = null; - // AMS token (?key={token}) - private String token = null; - // AMS subscription (/v1/projects/{project}/subscriptions/{sub}) - private String sub = null; - // protocol (https,http) - private String proto = null; - // numer of message to be pulled; - private String maxMessages = ""; - // ssl verify or not - private boolean verify = true; - // proxy - private URI proxy = null; - - // Utility inner class for holding list of messages and acknowledgements - private class MsgAck { - - String[] msgs; - String[] ackIds; - - private MsgAck(String[] msgs, String[] ackIds) { - this.msgs = msgs; - this.ackIds = ackIds; - } - - } - - public ArgoMessagingClient() { - this.httpClient = HttpClients.createDefault(); - this.proto = "https"; - this.token = "token"; - this.endpoint = "localhost"; - this.project = "test_project"; - this.sub = "test_sub"; - this.maxMessages = "100"; - this.proxy = null; - } - - public ArgoMessagingClient(String method, String token, String endpoint, String project, String sub, int batch, - boolean verify) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { - - this.proto = method; - this.token = token; - this.endpoint = endpoint; - this.project = project; - this.sub = sub; - this.maxMessages = String.valueOf(batch); - this.verify = verify; - - this.httpClient = buildHttpClient(); - - } - - /** - * Initializes Http Client (if not initialized during constructor) - * - * @return - */ - private CloseableHttpClient buildHttpClient() - throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { - if (this.verify) { - return this.httpClient = HttpClients.createDefault(); - } else { - return this.httpClient = HttpClients.custom().setSSLSocketFactory(selfSignedSSLF()).build(); - } - } - - /** - * Create an SSL Connection Socket Factory with a strategy to trust self - * signed certificates - */ - private SSLConnectionSocketFactory selfSignedSSLF() - throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { - SSLContextBuilder sslBuild = new SSLContextBuilder(); - sslBuild.loadTrustMaterial(null, new TrustSelfSignedStrategy()); - return new SSLConnectionSocketFactory(sslBuild.build(), NoopHostnameVerifier.INSTANCE); - } - - /** - * Set AMS http client to use http proxy - */ - public void setProxy(String proxyURL) throws URISyntaxException { - // parse proxy url - this.proxy = URI.create(proxyURL); - } - - /** - * Set AMS http client to NOT use an http proxy - */ - public void unsetProxy() { - this.proxy = null; - } - - /** - * Create a configuration for using http proxy on each request - */ - private RequestConfig createProxyCfg() { - HttpHost proxy = new HttpHost(this.proxy.getHost(), this.proxy.getPort(), this.proxy.getScheme()); - RequestConfig config = RequestConfig.custom().setProxy(proxy).build(); - return config; - } - - public void logIssue(CloseableHttpResponse resp) throws UnsupportedOperationException, IOException { - InputStreamReader isRdr = new InputStreamReader(resp.getEntity().getContent()); - BufferedReader bRdr = new BufferedReader(isRdr); - int statusCode = resp.getStatusLine().getStatusCode(); - - // Parse error content from api response - StringBuilder result = new StringBuilder(); - String rLine; - while ((rLine = bRdr.readLine()) != null) { - result.append(rLine); - } - isRdr.close(); - Log.warn("ApiStatusCode={}, ApiErrorMessage={}", statusCode, result); - - } - - /** - * Properly compose url for each AMS request - */ - public String composeURL(String method) { - - return proto + "://" + endpoint + "/v1/projects/" + project + "/subscriptions/" + sub + ":" + method; - } - - /** - * Executes a pull request against AMS api - */ - public MsgAck doPull() throws IOException, KeyManagementException, NoSuchAlgorithmException, KeyStoreException { - - ArrayList msgList = new ArrayList(); - ArrayList ackIdList = new ArrayList(); - - // Create the http post to pull - HttpPost postPull = new HttpPost(this.composeURL("pull")); - String body = "{\"maxMessages\":\"" + this.maxMessages + "\",\"returnImmediately\":\"true\"}"; - - postPull.addHeader("Accept", "application/json"); - postPull.addHeader("x-api-key", this.token); - postPull.addHeader("Content-type", "application/json"); - - StringEntity postBody = new StringEntity(body); - postBody.setContentType("application/json"); - - postPull.setEntity(postBody); - - if (this.httpClient == null) { - this.httpClient = buildHttpClient(); - } - - // check for proxy - if (this.proxy != null) { - postPull.setConfig(createProxyCfg()); - } - - CloseableHttpResponse response = this.httpClient.execute(postPull); - String msg = ""; - String ackId = ""; - StringBuilder result = new StringBuilder(); - - HttpEntity entity = response.getEntity(); - - int statusCode = response.getStatusLine().getStatusCode(); - - if (entity != null && statusCode == 200) { - - InputStreamReader isRdr = new InputStreamReader(entity.getContent()); - BufferedReader bRdr = new BufferedReader(isRdr); - - String rLine; - - while ((rLine = bRdr.readLine()) != null) { - result.append(rLine); - } - - // Gather message from json - JsonParser jsonParser = new JsonParser(); - // parse the json root object - Log.info("response: {}", result.toString()); - JsonElement jRoot = jsonParser.parse(result.toString()); - - JsonArray jRec = jRoot.getAsJsonObject().get("receivedMessages").getAsJsonArray(); - - // if has elements - for (JsonElement jMsgItem : jRec) { - JsonElement jMsg = jMsgItem.getAsJsonObject().get("message"); - JsonElement jAckId = jMsgItem.getAsJsonObject().get("ackId"); - msg = jMsg.toString(); - ackId = jAckId.toString(); - msgList.add(msg); - ackIdList.add(ackId); - } - - isRdr.close(); - - } else { - - logIssue(response); - - } - - response.close(); - - String[] msgArr = msgList.toArray(new String[0]); - String[] ackIdArr = ackIdList.toArray(new String[0]); - - // Return a Message array - return new MsgAck(msgArr, ackIdArr); - - } - - /** - * Executes a combination of Pull & Ack requests against AMS api - */ - public String[] consume() throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException { - String[] msgs = new String[0]; - // Try first to pull a message - try { - - MsgAck msgAck = doPull(); - // get last ackid - String ackId = ""; - if (msgAck.ackIds.length > 0) { - ackId = msgAck.ackIds[msgAck.ackIds.length - 1]; - } - - if (ackId != "") { - // Do an ack for the received message - String ackRes = doAck(ackId); - if (ackRes == "") { - Log.info("Message Acknowledged ackid:" + ackId); - msgs = msgAck.msgs; - - } else { - Log.warn("No acknowledment for ackid:" + ackId + "-" + ackRes); - } - } - } catch (IOException e) { - LOG.error(e.getMessage()); - } - return msgs; - - } - - /** - * Executes an Acknowledge request against AMS api - */ - public String doAck(String ackId) throws IOException { - - // Create the http post to ack - HttpPost postAck = new HttpPost(this.composeURL("acknowledge")); - String body = "{\"ackIds\":[" + ackId + "]}"; - postAck.addHeader("Accept", "application/json"); - postAck.addHeader("x-api-key", this.token); - postAck.addHeader("Content-type", "application/json"); - - StringEntity postBody = new StringEntity(body); - - postBody.setContentType("application/json"); - postAck.setEntity(postBody); - - // check for proxy - if (this.proxy != null) { - postAck.setConfig(createProxyCfg()); - } - - CloseableHttpResponse response = httpClient.execute(postAck); - String resMsg = ""; - StringBuilder result = new StringBuilder(); - - HttpEntity entity = response.getEntity(); - int status = response.getStatusLine().getStatusCode(); - - if (status != 200) { - - InputStreamReader isRdr = new InputStreamReader(entity.getContent()); - BufferedReader bRdr = new BufferedReader(isRdr); - - String rLine; - - while ((rLine = bRdr.readLine()) != null) { - result.append(rLine); - } - - resMsg = result.toString(); - isRdr.close(); - - } else { - // Log any api errors - logIssue(response); - } - response.close(); - // Return a resposeMessage - return resMsg; - - } - - /** - * Close AMS http client - */ - public void close() throws IOException { - this.httpClient.close(); - } -} diff --git a/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/ArgoMessagingSource.java b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/ArgoMessagingSource.java deleted file mode 100644 index 30997677..00000000 --- a/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/ArgoMessagingSource.java +++ /dev/null @@ -1,135 +0,0 @@ -package argo.streaming; - -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; - -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.source.RichSourceFunction; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Custom source to connect to AMS service. Uses ArgoMessaging client - */ -public class ArgoMessagingSource extends RichSourceFunction { - - private static final long serialVersionUID = 1L; - - // setup logger - static Logger LOG = LoggerFactory.getLogger(ArgoMessagingSource.class); - - private String endpoint = null; - private String port = null; - private String token = null; - private String project = null; - private String sub = null; - private int batch = 1; - private long interval = 100L; - private boolean verify = true; - private boolean useProxy = false; - private String proxyURL = ""; - private transient Object rateLck; // lock for waiting to establish rate - - private volatile boolean isRunning = true; - - private ArgoMessagingClient client = null; - - - public ArgoMessagingSource(String endpoint, String port, String token, String project, String sub, int batch, Long interval) { - this.endpoint = endpoint; - this.port = port; - this.token = token; - this.project = project; - this.sub = sub; - this.interval = interval; - this.batch = batch; - this.verify = true; - - } - - /** - * Set verify to true or false. If set to false AMS client will be able to contact AMS endpoints that use self-signed certificates - */ - public void setVerify(boolean verify) { - this.verify=verify; - } - /** - * Set proxy details for AMS client - */ - public void setProxy(String proxyURL) { - this.useProxy = true; - this.proxyURL = proxyURL; - } - - /** - * Unset proxy details for AMS client - */ - public void unsetProxy(String proxyURL) { - this.useProxy = false; - this.proxyURL = ""; - } - - - @Override - public void cancel() { - isRunning = false; - - } - - @Override - public void run(SourceContext ctx) throws Exception { - // This is the main run logic - while (isRunning) { - String[] res = this.client.consume(); - if (res.length > 0) { - for (String msg : res) { - ctx.collect(msg); - } - - } - synchronized (rateLck) { - rateLck.wait(this.interval); - } - - } - - } - - /** - * AMS Source initialization - */ - @Override - public void open(Configuration parameters) throws Exception { - // init rate lock - rateLck = new Object(); - // init client - String fendpoint = this.endpoint; - if (this.port != null && !this.port.isEmpty()) { - fendpoint = this.endpoint + ":" + port; - } - try { - client = new ArgoMessagingClient("https", this.token, fendpoint, this.project, this.sub, this.batch, this.verify); - if (this.useProxy) { - client.setProxy(this.proxyURL); - } - } catch (KeyManagementException e) { - e.printStackTrace(); - } catch (NoSuchAlgorithmException e) { - e.printStackTrace(); - } catch (KeyStoreException e) { - e.printStackTrace(); - } - } - - @Override - public void close() throws Exception { - if (this.client != null) { - client.close(); - } - synchronized (rateLck) { - rateLck.notify(); - } - } - -} diff --git a/flink_jobs_v2/ams_ingest_sync/pom.xml b/flink_jobs_v2/ams_ingest_sync/pom.xml index b0f1a1ee..79fff9ba 100644 --- a/flink_jobs_v2/ams_ingest_sync/pom.xml +++ b/flink_jobs_v2/ams_ingest_sync/pom.xml @@ -1,346 +1,353 @@ +license agreements. See the NOTICE file distributed with this work for additional +information regarding copyright ownership. The ASF licenses this file to +you under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +by applicable law or agreed to in writing, software distributed under the +License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +OF ANY KIND, either express or implied. See the License for the specific +language governing permissions and limitations under the License. --> + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - 4.0.0 - argo.streaming - 0.0.1-SNAPSHOT - ams-ingest-sync - Stream sync data from AMS to HDFS + 4.0.0 + argo.streaming + 0.0.1-SNAPSHOT + ams-ingest-sync + Stream sync data from AMS to HDFS - - UTF-8 - 1.3.1 - 1.7.7 - 1.2.17 - 2.10 - + + UTF-8 + 1.3.1 + 1.7.7 + 1.2.17 + 2.10 + - - - apache.snapshots - Apache Development Snapshot Repository - https://repository.apache.org/content/repositories/snapshots/ - - false - - - true - - - + + + apache.snapshots + Apache Development Snapshot Repository + https://repository.apache.org/content/repositories/snapshots/ + + false + + + true + + + - + - - - - org.apache.flink - flink-java - ${flink.version} - - - org.apache.flink - flink-streaming-java_2.10 - ${flink.version} - - - org.apache.flink - flink-clients_2.10 - ${flink.version} - - - org.apache.flink - flink-connector-filesystem_2.10 - 1.2.1 - + + + + ams.connector + ams-connector + 2.0.0 + jar + + + + org.apache.flink + flink-java + ${flink.version} + + + org.apache.flink + flink-streaming-java_2.10 + ${flink.version} + + + org.apache.flink + flink-clients_2.10 + ${flink.version} + + + org.apache.flink + flink-connector-filesystem_2.10 + 1.2.1 + - - - org.slf4j - slf4j-log4j12 - ${slf4j.version} - - - log4j - log4j - ${log4j.version} - - - org.apache.httpcomponents - httpclient - 4.5.13 - - - com.google.code.gson - gson - 2.8.9 - - - junit - junit - 4.13.1 - test - - - junit-addons - junit-addons - 1.4 - test - - + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + + + log4j + log4j + ${log4j.version} + + + org.apache.httpcomponents + httpclient + 4.5.13 + + + com.google.code.gson + gson + 2.8.9 + + + junit + junit + 4.13.1 + test + + + junit-addons + junit-addons + 1.4 + test + + - - - - build-jar + + + + build-jar - - false - + + false + - - - org.apache.flink - flink-java - ${flink.version} - provided - - - org.apache.flink - flink-streaming-java_2.10 - ${flink.version} - provided - - - org.apache.flink - flink-clients_2.10 - ${flink.version} - provided - - - org.apache.flink - flink-connector-filesystem_2.10 - 1.2.1 - - - org.slf4j - slf4j-log4j12 - ${slf4j.version} - provided - - - log4j - log4j - ${log4j.version} - provided - - - org.apache.httpcomponents - httpclient - 4.5.13 - - - com.google.code.gson - gson - 2.8.9 - - + + + org.apache.flink + flink-java + ${flink.version} + provided + + + org.apache.flink + flink-streaming-java_2.10 + ${flink.version} + provided + + + org.apache.flink + flink-clients_2.10 + ${flink.version} + provided + + + org.apache.flink + flink-connector-filesystem_2.10 + 1.2.1 + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + provided + + + log4j + log4j + ${log4j.version} + provided + + + org.apache.httpcomponents + httpclient + 4.5.13 + + + com.google.code.gson + gson + 2.8.9 + + - - - - - org.apache.maven.plugins - maven-shade-plugin - 2.4.1 - - - package - - shade - - - - - - - - - - - - - + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.1 + + + package + + shade + + + + + + + + + + + + + - - - - - org.apache.maven.plugins - maven-shade-plugin - 2.4.1 - - - - package - - shade - - - - - - org.apache.flink:flink-annotations - org.apache.flink:flink-shaded-hadoop2 - org.apache.flink:flink-shaded-curator-recipes - org.apache.flink:flink-core - org.apache.flink:flink-java - org.apache.flink:flink-scala_2.10 - org.apache.flink:flink-runtime_2.10 - org.apache.flink:flink-optimizer_2.10 - org.apache.flink:flink-clients_2.10 - org.apache.flink:flink-avro_2.10 - org.apache.flink:flink-examples-batch_2.10 - org.apache.flink:flink-examples-streaming_2.10 - org.apache.flink:flink-streaming-java_2.10 - org.apache.flink:flink-streaming-scala_2.10 - org.apache.flink:flink-scala-shell_2.10 - org.apache.flink:flink-python - org.apache.flink:flink-metrics-core - org.apache.flink:flink-metrics-jmx - org.apache.flink:flink-statebackend-rocksdb_2.10 + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.1 + + + + package + + shade + + + + + + org.apache.flink:flink-annotations + org.apache.flink:flink-shaded-hadoop2 + org.apache.flink:flink-shaded-curator-recipes + org.apache.flink:flink-core + org.apache.flink:flink-java + org.apache.flink:flink-scala_2.10 + org.apache.flink:flink-runtime_2.10 + org.apache.flink:flink-optimizer_2.10 + org.apache.flink:flink-clients_2.10 + org.apache.flink:flink-avro_2.10 + org.apache.flink:flink-examples-batch_2.10 + org.apache.flink:flink-examples-streaming_2.10 + org.apache.flink:flink-streaming-java_2.10 + org.apache.flink:flink-streaming-scala_2.10 + org.apache.flink:flink-scala-shell_2.10 + org.apache.flink:flink-python + org.apache.flink:flink-metrics-core + org.apache.flink:flink-metrics-jmx + org.apache.flink:flink-statebackend-rocksdb_2.10 - + - log4j:log4j - org.scala-lang:scala-library - org.scala-lang:scala-compiler - org.scala-lang:scala-reflect - com.data-artisans:flakka-actor_* - com.data-artisans:flakka-remote_* - com.data-artisans:flakka-slf4j_* - io.netty:netty-all - io.netty:netty - commons-fileupload:commons-fileupload - org.apache.avro:avro - commons-collections:commons-collections - org.codehaus.jackson:jackson-core-asl - org.codehaus.jackson:jackson-mapper-asl - com.thoughtworks.paranamer:paranamer - org.xerial.snappy:snappy-java - org.apache.commons:commons-compress - org.tukaani:xz - com.esotericsoftware.kryo:kryo - com.esotericsoftware.minlog:minlog - org.objenesis:objenesis - com.twitter:chill_* - com.twitter:chill-java - commons-lang:commons-lang - junit:junit - org.apache.commons:commons-lang3 - org.slf4j:slf4j-api - org.slf4j:slf4j-log4j12 - log4j:log4j - org.apache.commons:commons-math - org.apache.sling:org.apache.sling.commons.json - commons-logging:commons-logging - commons-codec:commons-codec - com.fasterxml.jackson.core:jackson-core - com.fasterxml.jackson.core:jackson-databind - com.fasterxml.jackson.core:jackson-annotations - stax:stax-api - com.typesafe:config - org.uncommons.maths:uncommons-maths - com.github.scopt:scopt_* - commons-io:commons-io - commons-cli:commons-cli - - - - - org.apache.flink:* - - - org/apache/flink/shaded/com/** - web-docs/** - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - false - - - - + log4j:log4j + org.scala-lang:scala-library + org.scala-lang:scala-compiler + org.scala-lang:scala-reflect + com.data-artisans:flakka-actor_* + com.data-artisans:flakka-remote_* + com.data-artisans:flakka-slf4j_* + io.netty:netty-all + io.netty:netty + commons-fileupload:commons-fileupload + org.apache.avro:avro + commons-collections:commons-collections + org.codehaus.jackson:jackson-core-asl + org.codehaus.jackson:jackson-mapper-asl + com.thoughtworks.paranamer:paranamer + org.xerial.snappy:snappy-java + org.apache.commons:commons-compress + org.tukaani:xz + com.esotericsoftware.kryo:kryo + com.esotericsoftware.minlog:minlog + org.objenesis:objenesis + com.twitter:chill_* + com.twitter:chill-java + commons-lang:commons-lang + junit:junit + org.apache.commons:commons-lang3 + org.slf4j:slf4j-api + org.slf4j:slf4j-log4j12 + log4j:log4j + org.apache.commons:commons-math + org.apache.sling:org.apache.sling.commons.json + commons-logging:commons-logging + commons-codec:commons-codec + com.fasterxml.jackson.core:jackson-core + com.fasterxml.jackson.core:jackson-databind + com.fasterxml.jackson.core:jackson-annotations + stax:stax-api + com.typesafe:config + org.uncommons.maths:uncommons-maths + com.github.scopt:scopt_* + commons-io:commons-io + commons-cli:commons-cli + + + + + org.apache.flink:* + + + org/apache/flink/shaded/com/** + web-docs/** + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + false + + + + - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - 1.7 - 1.7 - - - + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.7 + 1.7 + + + - - - - ams-ingest-sync + + + + ams-ingest-sync diff --git a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/AmsIngestSync.java b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/AmsIngestSync.java index 43815d66..f87cefb4 100644 --- a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/AmsIngestSync.java +++ b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/AmsIngestSync.java @@ -1,5 +1,6 @@ package argo.streaming; +import ams.connector.ArgoMessagingSource; import java.util.concurrent.TimeUnit; import org.apache.flink.api.common.restartstrategy.RestartStrategies; @@ -11,106 +12,111 @@ import org.slf4j.LoggerFactory; /** - * Flink Streaming JOB for Ingesting Sync Data to HDFS - * job required cli parameters: - * --ams.endpoint : ARGO messaging api endoint to connect to msg.example.com - * --ams.port : ARGO messaging api port - * --ams.token : ARGO messaging api token - * --ams.project : ARGO messaging api project to connect to - * --ams.sub.metric : ARGO messaging subscription to pull metric data from - * --ams.sub.sync : ARGO messaging subscription to pull sync data from - * --hdfs.path : Hdfs destination path to store the data - * --ams.batch : num of messages to be retrieved per request to AMS service - * --ams.interval : interval (in ms) between AMS service requests - * --ams.proxy : optional http proxy url - * --ams.verify : optional turn on/off ssl verify + * Flink Streaming JOB for Ingesting Sync Data to HDFS job required cli + * parameters: --ams.endpoint : ARGO messaging api endoint to connect to + * msg.example.com --ams.port : ARGO messaging api port --ams.token : ARGO + * messaging api token --ams.project : ARGO messaging api project to connect to + * --ams.sub.metric : ARGO messaging subscription to pull metric data from + * --ams.sub.sync : ARGO messaging subscription to pull sync data from + * --hdfs.path : Hdfs destination path to store the data --ams.batch : num of + * messages to be retrieved per request to AMS service --ams.interval : interval + * (in ms) between AMS service requests --ams.proxy : optional http proxy url + * --ams.verify : optional turn on/off ssl verify + * + * --run.date : optional date to run ingestion */ public class AmsIngestSync { - // setup logger - static Logger LOG = LoggerFactory.getLogger(AmsIngestSync.class); - - /** - * Check if a list of expected cli arguments have been provided to this flink job - */ - public static boolean hasArgs(String[] reqArgs, ParameterTool paramTool) { - - for (String reqArg : reqArgs) { - if (!paramTool.has(reqArg)) - return false; - } - - return true; - } - - /** - * Check if flink job has been called with ams rate params - */ - public static boolean hasAmsRateArgs(ParameterTool paramTool) { - String args[] = { "ams.batch", "ams.interval" }; - return hasArgs(args, paramTool); - } - - // main job function - public static void main(String[] args) throws Exception { - - // Create flink execution enviroment - StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment(); - see.setParallelism(1); - // Fixed restart strategy: on failure attempt max 10 times to restart with a retry interval of 2 minutes - see.setRestartStrategy(RestartStrategies.fixedDelayRestart(10, Time.of(2, TimeUnit.MINUTES))); - // Initialize cli parameter tool - final ParameterTool parameterTool = ParameterTool.fromArgs(args); - - // Initialize Input Source : ARGO Messaging Source - String endpoint = parameterTool.getRequired("ams.endpoint"); - String port = parameterTool.getRequired("ams.port"); - String token = parameterTool.getRequired("ams.token"); - String project = parameterTool.getRequired("ams.project"); - String sub = parameterTool.getRequired("ams.sub"); - String basePath = parameterTool.getRequired("hdfs.path"); - - // set ams client batch and interval to default values - int batch = 1; - long interval = 100L; - - if (hasAmsRateArgs(parameterTool)) { - batch = parameterTool.getInt("ams.batch"); - interval = parameterTool.getLong("ams.interval"); - } - - - //Ingest sync avro encoded data from AMS endpoint - ArgoMessagingSource ams = new ArgoMessagingSource(endpoint, port, token, project, sub, batch, interval); - - if (parameterTool.has("ams.verify")){ - ams.setVerify(parameterTool.getBoolean("ams.verify")); - } - - if (parameterTool.has("ams.proxy")) { - ams.setProxy(parameterTool.get("ams.proxy")); - } - DataStream syncDataStream = see - .addSource(ams); - - SyncHDFSOutputFormat hdfsOut = new SyncHDFSOutputFormat(); - hdfsOut.setBasePath(basePath); - - syncDataStream.writeUsingOutputFormat(hdfsOut); - - // Create a job title message to discern job in flink dashboard/cli - StringBuilder jobTitleSB = new StringBuilder(); - jobTitleSB.append("Ingesting sync data from "); - jobTitleSB.append(endpoint); - jobTitleSB.append(":"); - jobTitleSB.append(port); - jobTitleSB.append("/v1/projects/"); - jobTitleSB.append(project); - jobTitleSB.append("/subscriptions/"); - jobTitleSB.append(sub); - - see.execute(jobTitleSB.toString()); - - } + // setup logger + static Logger LOG = LoggerFactory.getLogger(AmsIngestSync.class); + private static String runDate; + + /** + * Check if a list of expected cli arguments have been provided to this + * flink job + */ + public static boolean hasArgs(String[] reqArgs, ParameterTool paramTool) { + + for (String reqArg : reqArgs) { + if (!paramTool.has(reqArg)) { + return false; + } + } + + return true; + } + + /** + * Check if flink job has been called with ams rate params + */ + public static boolean hasAmsRateArgs(ParameterTool paramTool) { + String args[] = {"ams.batch", "ams.interval"}; + return hasArgs(args, paramTool); + } + + // main job function + public static void main(String[] args) throws Exception { + + // Create flink execution enviroment + StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment(); + see.setParallelism(1); + // Fixed restart strategy: on failure attempt max 10 times to restart with a retry interval of 2 minutes + see.setRestartStrategy(RestartStrategies.fixedDelayRestart(10, Time.of(2, TimeUnit.MINUTES))); + // Initialize cli parameter tool + final ParameterTool parameterTool = ParameterTool.fromArgs(args); + + // Initialize Input Source : ARGO Messaging Source + String endpoint = parameterTool.getRequired("ams.endpoint"); + String port = parameterTool.getRequired("ams.port"); + String token = parameterTool.getRequired("ams.token"); + String project = parameterTool.getRequired("ams.project"); + String sub = parameterTool.getRequired("ams.sub"); + String basePath = parameterTool.getRequired("hdfs.path"); + runDate = parameterTool.get("run.date"); + if (runDate != null) { + runDate = runDate + "T00:00:00.000Z"; + } + + // set ams client batch and interval to default values + int batch = 1; + long interval = 100L; + + if (hasAmsRateArgs(parameterTool)) { + batch = parameterTool.getInt("ams.batch"); + interval = parameterTool.getLong("ams.interval"); + } + + //Ingest sync avro encoded data from AMS endpoint + ArgoMessagingSource ams = new ArgoMessagingSource(endpoint, port, token, project, sub, batch, interval, runDate); + + if (parameterTool.has("ams.verify")) { + ams.setVerify(parameterTool.getBoolean("ams.verify")); + } + + if (parameterTool.has("ams.proxy")) { + ams.setProxy(parameterTool.get("ams.proxy")); + } + DataStream syncDataStream = see + .addSource(ams); + + SyncHDFSOutputFormat hdfsOut = new SyncHDFSOutputFormat(); + hdfsOut.setBasePath(basePath); + + syncDataStream.writeUsingOutputFormat(hdfsOut); + + // Create a job title message to discern job in flink dashboard/cli + StringBuilder jobTitleSB = new StringBuilder(); + jobTitleSB.append("Ingesting sync data from "); + jobTitleSB.append(endpoint); + jobTitleSB.append(":"); + jobTitleSB.append(port); + jobTitleSB.append("/v1/projects/"); + jobTitleSB.append(project); + jobTitleSB.append("/subscriptions/"); + jobTitleSB.append(sub); + + see.execute(jobTitleSB.toString()); + + } } diff --git a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/ArgoMessagingClient.java b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/ArgoMessagingClient.java deleted file mode 100644 index f7402f8a..00000000 --- a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/ArgoMessagingClient.java +++ /dev/null @@ -1,350 +0,0 @@ -package argo.streaming; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.URI; -import java.net.URISyntaxException; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; - -import org.apache.http.HttpEntity; -import org.apache.http.HttpHost; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.TrustSelfSignedStrategy; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.ssl.SSLContextBuilder; -import org.mortbay.log.Log; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonParser; - -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.CloseableHttpResponse; - -/** - * Simple http client for pulling and acknowledging messages from AMS service - * http API - */ -public class ArgoMessagingClient { - - static Logger LOG = LoggerFactory.getLogger(ArgoMessagingClient.class); - // Http Client for contanting AMS service - private CloseableHttpClient httpClient = null; - // AMS endpoint (hostname:port or hostname) - private String endpoint = null; - // AMS project (/v1/projects/{project}) - private String project = null; - // AMS token (?key={token}) - private String token = null; - // AMS subscription (/v1/projects/{project}/subscriptions/{sub}) - private String sub = null; - // protocol (https,http) - private String proto = null; - // numer of message to be pulled; - private String maxMessages = ""; - // ssl verify or not - private boolean verify = true; - // proxy - private URI proxy = null; - - // Utility inner class for holding list of messages and acknowledgements - private class MsgAck { - - String[] msgs; - String[] ackIds; - - private MsgAck(String[] msgs, String[] ackIds) { - this.msgs = msgs; - this.ackIds = ackIds; - } - - } - - public ArgoMessagingClient() { - this.httpClient = HttpClients.createDefault(); - this.proto = "https"; - this.token = "token"; - this.endpoint = "localhost"; - this.project = "test_project"; - this.sub = "test_sub"; - this.maxMessages = "100"; - this.proxy = null; - } - - public ArgoMessagingClient(String method, String token, String endpoint, String project, String sub, int batch, - boolean verify) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { - - this.proto = method; - this.token = token; - this.endpoint = endpoint; - this.project = project; - this.sub = sub; - this.maxMessages = String.valueOf(batch); - this.verify = verify; - - this.httpClient = buildHttpClient(); - - } - - /** - * Initializes Http Client (if not initialized during constructor) - * - * @return - */ - private CloseableHttpClient buildHttpClient() - throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { - if (this.verify) { - return this.httpClient = HttpClients.createDefault(); - } else { - return this.httpClient = HttpClients.custom().setSSLSocketFactory(selfSignedSSLF()).build(); - } - } - - /** - * Create an SSL Connection Socket Factory with a strategy to trust self - * signed certificates - */ - private SSLConnectionSocketFactory selfSignedSSLF() - throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { - SSLContextBuilder sslBuild = new SSLContextBuilder(); - sslBuild.loadTrustMaterial(null, new TrustSelfSignedStrategy()); - return new SSLConnectionSocketFactory(sslBuild.build(), NoopHostnameVerifier.INSTANCE); - } - - /** - * Set AMS http client to use http proxy - */ - public void setProxy(String proxyURL) throws URISyntaxException { - // parse proxy url - this.proxy = URI.create(proxyURL); - } - - /** - * Set AMS http client to NOT use an http proxy - */ - public void unsetProxy() { - this.proxy = null; - } - - /** - * Create a configuration for using http proxy on each request - */ - private RequestConfig createProxyCfg() { - HttpHost proxy = new HttpHost(this.proxy.getHost(), this.proxy.getPort(), this.proxy.getScheme()); - RequestConfig config = RequestConfig.custom().setProxy(proxy).build(); - return config; - } - - public void logIssue(CloseableHttpResponse resp) throws UnsupportedOperationException, IOException { - InputStreamReader isRdr = new InputStreamReader(resp.getEntity().getContent()); - BufferedReader bRdr = new BufferedReader(isRdr); - int statusCode = resp.getStatusLine().getStatusCode(); - - // Parse error content from api response - StringBuilder result = new StringBuilder(); - String rLine; - while ((rLine = bRdr.readLine()) != null) { - result.append(rLine); - } - isRdr.close(); - Log.warn("ApiStatusCode={}, ApiErrorMessage={}", statusCode, result); - - } - - /** - * Properly compose url for each AMS request - */ - public String composeURL(String method) { - return proto + "://" + endpoint + "/v1/projects/" + project + "/subscriptions/" + sub + ":" + method; - } - - /** - * Executes a pull request against AMS api - */ - public MsgAck doPull() throws IOException, KeyManagementException, NoSuchAlgorithmException, KeyStoreException { - - ArrayList msgList = new ArrayList(); - ArrayList ackIdList = new ArrayList(); - - HttpPost postPull = new HttpPost(this.composeURL("pull")); - String body = "{\"maxMessages\":\"" + this.maxMessages + "\",\"returnImmediately\":\"true\"}"; - - postPull.addHeader("Accept", "application/json"); - postPull.addHeader("x-api-key", this.token); - postPull.addHeader("Content-type", "application/json"); - - StringEntity postBody = new StringEntity(body); - postBody.setContentType("application/json"); - - postPull.setEntity(postBody); - - if (this.httpClient == null) { - this.httpClient = buildHttpClient(); - } - - // check for proxy - if (this.proxy != null) { - postPull.setConfig(createProxyCfg()); - } - - CloseableHttpResponse response = this.httpClient.execute(postPull); - String msg = ""; - String ackId = ""; - StringBuilder result = new StringBuilder(); - - HttpEntity entity = response.getEntity(); - - int statusCode = response.getStatusLine().getStatusCode(); - - if (entity != null && statusCode == 200) { - - InputStreamReader isRdr = new InputStreamReader(entity.getContent()); - BufferedReader bRdr = new BufferedReader(isRdr); - - String rLine; - - while ((rLine = bRdr.readLine()) != null) { - result.append(rLine); - } - - // Gather message from json - JsonParser jsonParser = new JsonParser(); - // parse the json root object - Log.info("response: {}", result.toString()); - JsonElement jRoot = jsonParser.parse(result.toString()); - - JsonArray jRec = jRoot.getAsJsonObject().get("receivedMessages").getAsJsonArray(); - - // if has elements - for (JsonElement jMsgItem : jRec) { - JsonElement jMsg = jMsgItem.getAsJsonObject().get("message"); - JsonElement jAckId = jMsgItem.getAsJsonObject().get("ackId"); - msg = jMsg.toString(); - ackId = jAckId.toString(); - msgList.add(msg); - ackIdList.add(ackId); - } - - isRdr.close(); - - } else { - - logIssue(response); - - } - - response.close(); - - String[] msgArr = msgList.toArray(new String[0]); - String[] ackIdArr = ackIdList.toArray(new String[0]); - - // Return a Message array - return new MsgAck(msgArr, ackIdArr); - - } - - /** - * Executes a combination of Pull & Ack requests against AMS api - */ - public String[] consume() throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException { - String[] msgs = new String[0]; - // Try first to pull a message - try { - - MsgAck msgAck = doPull(); - // get last ackid - String ackId = ""; - if (msgAck.ackIds.length > 0) { - ackId = msgAck.ackIds[msgAck.ackIds.length - 1]; - } - - if (ackId != "") { - // Do an ack for the received message - String ackRes = doAck(ackId); - if (ackRes == "") { - Log.info("Message Acknowledged ackid:" + ackId); - msgs = msgAck.msgs; - - } else { - Log.warn("No acknowledment for ackid:" + ackId + "-" + ackRes); - } - } - } catch (IOException e) { - LOG.error(e.getMessage()); - } - return msgs; - - } - - /** - * Executes an Acknowledge request against AMS api - */ - public String doAck(String ackId) throws IOException { - - // Create the http post to ack - - - HttpPost postAck = new HttpPost(this.composeURL("acknowledge")); - String body = "{\"ackIds\":[" + ackId + "]}"; - postAck.addHeader("Accept", "application/json"); - postAck.addHeader("x-api-key", this.token); - postAck.addHeader("Content-type", "application/json"); - - StringEntity postBody = new StringEntity(body); - - postBody.setContentType("application/json"); - postAck.setEntity(postBody); - - // check for proxy - if (this.proxy != null) { - postAck.setConfig(createProxyCfg()); - } - - CloseableHttpResponse response = httpClient.execute(postAck); - String resMsg = ""; - StringBuilder result = new StringBuilder(); - - HttpEntity entity = response.getEntity(); - int status = response.getStatusLine().getStatusCode(); - - if (status != 200) { - - InputStreamReader isRdr = new InputStreamReader(entity.getContent()); - BufferedReader bRdr = new BufferedReader(isRdr); - - String rLine; - - while ((rLine = bRdr.readLine()) != null) { - result.append(rLine); - } - - resMsg = result.toString(); - isRdr.close(); - - } else { - // Log any api errors - logIssue(response); - } - response.close(); - // Return a resposeMessage - return resMsg; - - } - - /** - * Close AMS http client - */ - public void close() throws IOException { - this.httpClient.close(); - } -} diff --git a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/ArgoMessagingSource.java b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/ArgoMessagingSource.java deleted file mode 100644 index 30997677..00000000 --- a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/ArgoMessagingSource.java +++ /dev/null @@ -1,135 +0,0 @@ -package argo.streaming; - -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; - -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.source.RichSourceFunction; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Custom source to connect to AMS service. Uses ArgoMessaging client - */ -public class ArgoMessagingSource extends RichSourceFunction { - - private static final long serialVersionUID = 1L; - - // setup logger - static Logger LOG = LoggerFactory.getLogger(ArgoMessagingSource.class); - - private String endpoint = null; - private String port = null; - private String token = null; - private String project = null; - private String sub = null; - private int batch = 1; - private long interval = 100L; - private boolean verify = true; - private boolean useProxy = false; - private String proxyURL = ""; - private transient Object rateLck; // lock for waiting to establish rate - - private volatile boolean isRunning = true; - - private ArgoMessagingClient client = null; - - - public ArgoMessagingSource(String endpoint, String port, String token, String project, String sub, int batch, Long interval) { - this.endpoint = endpoint; - this.port = port; - this.token = token; - this.project = project; - this.sub = sub; - this.interval = interval; - this.batch = batch; - this.verify = true; - - } - - /** - * Set verify to true or false. If set to false AMS client will be able to contact AMS endpoints that use self-signed certificates - */ - public void setVerify(boolean verify) { - this.verify=verify; - } - /** - * Set proxy details for AMS client - */ - public void setProxy(String proxyURL) { - this.useProxy = true; - this.proxyURL = proxyURL; - } - - /** - * Unset proxy details for AMS client - */ - public void unsetProxy(String proxyURL) { - this.useProxy = false; - this.proxyURL = ""; - } - - - @Override - public void cancel() { - isRunning = false; - - } - - @Override - public void run(SourceContext ctx) throws Exception { - // This is the main run logic - while (isRunning) { - String[] res = this.client.consume(); - if (res.length > 0) { - for (String msg : res) { - ctx.collect(msg); - } - - } - synchronized (rateLck) { - rateLck.wait(this.interval); - } - - } - - } - - /** - * AMS Source initialization - */ - @Override - public void open(Configuration parameters) throws Exception { - // init rate lock - rateLck = new Object(); - // init client - String fendpoint = this.endpoint; - if (this.port != null && !this.port.isEmpty()) { - fendpoint = this.endpoint + ":" + port; - } - try { - client = new ArgoMessagingClient("https", this.token, fendpoint, this.project, this.sub, this.batch, this.verify); - if (this.useProxy) { - client.setProxy(this.proxyURL); - } - } catch (KeyManagementException e) { - e.printStackTrace(); - } catch (NoSuchAlgorithmException e) { - e.printStackTrace(); - } catch (KeyStoreException e) { - e.printStackTrace(); - } - } - - @Override - public void close() throws Exception { - if (this.client != null) { - client.close(); - } - synchronized (rateLck) { - rateLck.notify(); - } - } - -} diff --git a/flink_jobs_v2/pom.xml b/flink_jobs_v2/pom.xml index 3690b2c0..af10410c 100644 --- a/flink_jobs_v2/pom.xml +++ b/flink_jobs_v2/pom.xml @@ -16,11 +16,10 @@ Timelines ProfilesManager batch_multi - stream_status - AmsPublisher + stream_status ams_ingest_metric ams_ingest_sync - + ams-connector diff --git a/flink_jobs_v2/stream_status/pom.xml b/flink_jobs_v2/stream_status/pom.xml index 9be9795e..7f52a2d5 100644 --- a/flink_jobs_v2/stream_status/pom.xml +++ b/flink_jobs_v2/stream_status/pom.xml @@ -63,6 +63,12 @@ language governing permissions and limitations under the License. --> handling. This approach is preferred and leads to much cleaner jar files. --> + + ams.connector + ams-connector + 2.0.0 + jar + api.resource.manager ApiResourceManager diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java index 03d1e5fd..dfd8836d 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java @@ -1,6 +1,8 @@ package argo.streaming; import Utils.IntervalType; +import ams.connector.ArgoMessagingSink; +import ams.connector.ArgoMessagingSource; import java.io.IOException; import java.net.URISyntaxException; import java.text.ParseException; diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/StatusConfig.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/StatusConfig.java index 91e49567..4054563c 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/StatusConfig.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/StatusConfig.java @@ -57,7 +57,7 @@ public StatusConfig(ParameterTool pt) { this.amsProject = pt.getRequired("ams.project"); this.apiEndpoint = pt.getRequired("api.endpoint"); - this.runDate = pt.getRequired("run.date"); + this.runDate = pt.get("run.date"); this.report = pt.getRequired("report"); // Optional timeout parameter if (pt.has("timeout")) { From fdc9e176de5464bf2ae34eee49d2c7df16b40337 Mon Sep 17 00:00:00 2001 From: cthermolia-grnet Date: Thu, 29 Sep 2022 13:24:30 +0300 Subject: [PATCH 15/16] ARGO-3977 Improve logs generated by the flink jobs --- .../java/argo/streaming/AmsIngestMetric.java | 12 ++++ .../src/main/resources/log4j.properties | 2 +- .../java/argo/streaming/AmsIngestSync.java | 13 +++- .../argo/streaming/SyncHDFSOutputFormat.java | 3 +- .../src/main/resources/log4j.properties | 23 +++++++ .../main/java/argo/batch/ArgoMultiJob.java | 22 +++++-- .../main/java/argo/batch/PickEndpoints.java | 2 + .../src/main/java/utils/Utils.java | 2 +- .../src/main/resources/log4j.properties | 3 +- .../java/argo/batch/ArgoMultiJobTest.java | 5 +- .../java/argo/streaming/AmsStreamStatus.java | 60 +++++++++++-------- .../src/main/java/status/StatusManager.java | 4 +- .../src/main/resources/log4j.properties | 2 +- 13 files changed, 115 insertions(+), 38 deletions(-) create mode 100644 flink_jobs_v2/ams_ingest_sync/src/main/resources/log4j.properties diff --git a/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/AmsIngestMetric.java b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/AmsIngestMetric.java index fd329d20..283832f4 100644 --- a/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/AmsIngestMetric.java +++ b/flink_jobs_v2/ams_ingest_metric/src/main/java/argo/streaming/AmsIngestMetric.java @@ -35,6 +35,8 @@ import argo.avro.MetricData; import argo.avro.MetricDataOld; +import org.apache.flink.api.common.JobID; +import org.slf4j.MDC; /** * Flink Job : Stream metric data from ARGO messaging to Hbase job required cli @@ -59,6 +61,7 @@ public class AmsIngestMetric { static Logger LOG = LoggerFactory.getLogger(AmsIngestMetric.class); private static String runDate; + /** * Check if flink job has been called with ams rate params */ @@ -109,6 +112,7 @@ public static boolean hasArgs(String[] reqArgs, ParameterTool paramTool) { public static void main(String[] args) throws Exception { + configJID(); // Create flink execution environment StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment(); see.setParallelism(1); @@ -138,6 +142,7 @@ public static void main(String[] args) throws Exception { if (runDate != null) { runDate = runDate + "T00:00:00.000Z"; } + // Check if checkpointing is desired if (hasCheckArgs(parameterTool)) { @@ -253,5 +258,12 @@ public void flatMap(String value, Collector out) throws Exception { see.execute(jobTitleSB.toString()); } + private static String getJID() { + return JobID.generate().toString(); + } + private static void configJID() { //config the JID in the log4j.properties + String jobId = getJID(); + MDC.put("JID", jobId); + } } diff --git a/flink_jobs_v2/ams_ingest_metric/src/main/resources/log4j.properties b/flink_jobs_v2/ams_ingest_metric/src/main/resources/log4j.properties index da32ea0f..a9bc74e2 100644 --- a/flink_jobs_v2/ams_ingest_metric/src/main/resources/log4j.properties +++ b/flink_jobs_v2/ams_ingest_metric/src/main/resources/log4j.properties @@ -20,4 +20,4 @@ log4j.rootLogger=INFO, console log4j.appender.console=org.apache.log4j.ConsoleAppender log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n +log4j.appender.console.layout.ConversionPattern= %d{HH:mm:ss,SSS} %X{JID} %-5p %-60c %x - %m%n diff --git a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/AmsIngestSync.java b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/AmsIngestSync.java index f87cefb4..26e46a11 100644 --- a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/AmsIngestSync.java +++ b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/AmsIngestSync.java @@ -2,6 +2,7 @@ import ams.connector.ArgoMessagingSource; import java.util.concurrent.TimeUnit; +import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.api.common.time.Time; @@ -10,6 +11,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; /** * Flink Streaming JOB for Ingesting Sync Data to HDFS job required cli @@ -30,7 +32,6 @@ public class AmsIngestSync { // setup logger static Logger LOG = LoggerFactory.getLogger(AmsIngestSync.class); private static String runDate; - /** * Check if a list of expected cli arguments have been provided to this * flink job @@ -56,6 +57,7 @@ public static boolean hasAmsRateArgs(ParameterTool paramTool) { // main job function public static void main(String[] args) throws Exception { + configJID(); // Create flink execution enviroment StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment(); @@ -76,6 +78,8 @@ public static void main(String[] args) throws Exception { if (runDate != null) { runDate = runDate + "T00:00:00.000Z"; } + + // set ams client batch and interval to default values int batch = 1; @@ -118,5 +122,12 @@ public static void main(String[] args) throws Exception { see.execute(jobTitleSB.toString()); } + private static String getJID() { + return JobID.generate().toString(); + } + private static void configJID() { + String jobId = getJID(); + MDC.put("JID", jobId); + } } diff --git a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/SyncHDFSOutputFormat.java b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/SyncHDFSOutputFormat.java index dec9b7f6..9c009088 100644 --- a/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/SyncHDFSOutputFormat.java +++ b/flink_jobs_v2/ams_ingest_sync/src/main/java/argo/streaming/SyncHDFSOutputFormat.java @@ -44,12 +44,13 @@ class SyncHDFSOutputFormat implements OutputFormat { // setup logger static Logger LOG = LoggerFactory.getLogger(SyncHDFSOutputFormat.class); - + private static final long serialVersionUID = 1L; private URI basePath; private org.apache.hadoop.conf.Configuration hadoopConf; private FileSystem hdfs; + public void setBasePath(String url) throws URISyntaxException { this.basePath = new URI(url); diff --git a/flink_jobs_v2/ams_ingest_sync/src/main/resources/log4j.properties b/flink_jobs_v2/ams_ingest_sync/src/main/resources/log4j.properties new file mode 100644 index 00000000..a9bc74e2 --- /dev/null +++ b/flink_jobs_v2/ams_ingest_sync/src/main/resources/log4j.properties @@ -0,0 +1,23 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +log4j.rootLogger=INFO, console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern= %d{HH:mm:ss,SSS} %X{JID} %-5p %-60c %x - %m%n diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/ArgoMultiJob.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/ArgoMultiJob.java index 60bf79e6..022b63df 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/ArgoMultiJob.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/ArgoMultiJob.java @@ -51,8 +51,10 @@ import trends.status.ServiceTrendsCounter; import java.util.List; +import org.apache.flink.api.common.JobID; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import org.slf4j.MDC; /** * Implements an ARGO Status Batch Job in flink @@ -84,7 +86,8 @@ */ public class ArgoMultiJob { - static Logger LOG = LoggerFactory.getLogger(ArgoMultiJob.class); + static Logger LOG = LoggerFactory.getLogger( ArgoMultiJob.class); + private static String dbURI; private static String reportID; private static Integer rankNum; @@ -97,9 +100,10 @@ public class ArgoMultiJob { private static boolean calcStatusTrends = false; private static boolean calcFlipFlops = false; private static boolean calcTagTrends = false; - + public static void main(String[] args) throws Exception { + configJID(); final ParameterTool params = ParameterTool.fromArgs(args); // set up the execution environment @@ -142,6 +146,7 @@ public static void main(String[] args) throws Exception { List confData = confDS.collect(); ReportManager cfgMgr = new ReportManager(); cfgMgr.loadJsonString(confData); + enableComputations(cfgMgr.activeComputations, params); DataSource opsDS = env.fromElements(amr.getResourceJSON(ApiResource.OPS)); @@ -226,7 +231,7 @@ public static void main(String[] args) throws Exception { DataSet stDetailDS = mdataTotalDS.groupBy("group", "service", "hostname", "metric") .sortGroup("timestamp", Order.ASCENDING).reduceGroup(new CalcPrevStatus(params)) .withBroadcastSet(mpsDS, "mps").withBroadcastSet(recDS, "rec").withBroadcastSet(opsDS, "ops"); - + //Create StatusMetricTimeline dataset for endpoints DataSet statusMetricTimeline = stDetailDS.groupBy("group", "service", "hostname", "metric").sortGroup("timestamp", Order.ASCENDING) .reduceGroup(new CalcMetricTimeline(params)).withBroadcastSet(mpsDS, "mps").withBroadcastSet(opsDS, "ops") @@ -466,7 +471,6 @@ private static void enableComputations(ReportManager.ActiveComputations activeCo calcStatusTrends = isOFF(params, "calcStatusTrends", activeComputations); calcFlipFlops = isOFF(params, "calcFlipFlops", activeComputations); calcTagTrends = isOFF(params, "calcTagTrends", activeComputations); - System.out.println("calcTagTrends--- "+calcTagTrends); if (!calcStatus && !calcAR && !calcStatusTrends && !calcFlipFlops && !calcStatusTrends) { System.exit(0); @@ -496,4 +500,14 @@ public static boolean isOFF(ParameterTool params, String paramName, ReportManage } } } + + private static String getJID() { + return JobID.generate().toString(); + } + + private static void configJID() {//config the JID in the log4j.properties + String jobId=getJID(); + MDC.put("JID", jobId); + + } } diff --git a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/PickEndpoints.java b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/PickEndpoints.java index 7ff4de6d..6e8c3ba7 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/argo/batch/PickEndpoints.java +++ b/flink_jobs_v2/batch_multi/src/main/java/argo/batch/PickEndpoints.java @@ -37,6 +37,7 @@ public class PickEndpoints extends RichFlatMapFunction final ParameterTool params; + public PickEndpoints(ParameterTool params) { this.params = params; } @@ -108,6 +109,7 @@ public void open(Configuration parameters) throws IOException, ParseException { if (!this.thr.get(0).isEmpty()) { this.thrMgr.parseJSON(this.thr.get(0)); } + } @Override diff --git a/flink_jobs_v2/batch_multi/src/main/java/utils/Utils.java b/flink_jobs_v2/batch_multi/src/main/java/utils/Utils.java index 5416e167..93bc0ea9 100644 --- a/flink_jobs_v2/batch_multi/src/main/java/utils/Utils.java +++ b/flink_jobs_v2/batch_multi/src/main/java/utils/Utils.java @@ -74,7 +74,7 @@ public static boolean isPreviousDate(String format, Date nowDate, Date firstDate } } - public static boolean checkParameters(ParameterTool params, String... vars) { + public static boolean checkParameters(ParameterTool params,String... vars) { for (String var : vars) { diff --git a/flink_jobs_v2/batch_multi/src/main/resources/log4j.properties b/flink_jobs_v2/batch_multi/src/main/resources/log4j.properties index 65bd0b8c..c7aaed4c 100644 --- a/flink_jobs_v2/batch_multi/src/main/resources/log4j.properties +++ b/flink_jobs_v2/batch_multi/src/main/resources/log4j.properties @@ -20,4 +20,5 @@ log4j.rootLogger=INFO, console log4j.appender.console=org.apache.log4j.ConsoleAppender log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n \ No newline at end of file +log4j.appender.console.layout.ConversionPattern= %d{HH:mm:ss,SSS} %X{JID} %-5p %-60c %x - %m%n + diff --git a/flink_jobs_v2/batch_multi/src/test/java/argo/batch/ArgoMultiJobTest.java b/flink_jobs_v2/batch_multi/src/test/java/argo/batch/ArgoMultiJobTest.java index ee5446a5..853d5651 100644 --- a/flink_jobs_v2/batch_multi/src/test/java/argo/batch/ArgoMultiJobTest.java +++ b/flink_jobs_v2/batch_multi/src/test/java/argo/batch/ArgoMultiJobTest.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.operators.Order; import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.ExecutionEnvironment; @@ -77,7 +78,7 @@ public class ArgoMultiJobTest { private static boolean calcStatusTrends = false; private static boolean calcFlipFlops = false; private static boolean calcTagTrends = false; - + public ArgoMultiJobTest() { } @@ -104,7 +105,7 @@ public void testMain() throws Exception { List confData = confDS.collect(); ReportManager cfgMgr = new ReportManager(); cfgMgr.loadJsonString(confData); - + enableComputations(cfgMgr.activeComputations, params); DataSource apsDS = env.fromElements(amr.getResourceJSON(ApiResource.AGGREGATION)); DataSource opsDS = env.fromElements(amr.getResourceJSON(ApiResource.OPS)); diff --git a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java index dfd8836d..17070db4 100644 --- a/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java +++ b/flink_jobs_v2/stream_status/src/main/java/argo/streaming/AmsStreamStatus.java @@ -50,7 +50,9 @@ import argo.avro.MetricData; import argo.avro.MetricProfile; import org.apache.commons.lang.StringUtils; +import org.apache.flink.api.common.JobID; import org.apache.flink.core.fs.FileSystem; +import org.slf4j.MDC; import profilesmanager.EndpointGroupManager; import profilesmanager.MetricProfileManager; import status.StatusManager; @@ -82,16 +84,16 @@ * if it is not defined url history wont be constructed --url.help (optional) * the url to be used as a basis to create a help url , eg. * poem.egi.eu/ui/public_metrics it can be optional , meaning if it is not - * defined url help wont be constructed - * --interval.loose(Optional)interval to repeat events for WARNING, CRITICAL, UNKNOWN . - * it can be in the format of DAYS, HOURS, MINUTES eg. 1h, 2d, 30m to define the period . Any of these formats is - * transformed to minutes in the computations - * if not defined the default value is 1440m - * - * --interval.strict(Optional)interval to repeat events for CRITICAL . - * it can be in the format of DAYS, HOURS, MINUTES eg. 1h, 2d, 30m to define the period . Any of these formats is - * transformed to minutes in the computations - * if not defined the default value is 1440m + * defined url help wont be constructed --interval.loose(Optional)interval to + * repeat events for WARNING, CRITICAL, UNKNOWN . it can be in the format of + * DAYS, HOURS, MINUTES eg. 1h, 2d, 30m to define the period . Any of these + * formats is transformed to minutes in the computations if not defined the + * default value is 1440m + * + * --interval.strict(Optional)interval to repeat events for CRITICAL . it can be + * in the format of DAYS, HOURS, MINUTES eg. 1h, 2d, 30m to define the period . + * Any of these formats is transformed to minutes in the computations if not + * defined the default value is 1440m * */ public class AmsStreamStatus { @@ -99,7 +101,7 @@ public class AmsStreamStatus { static Logger LOG = LoggerFactory.getLogger(AmsStreamStatus.class); private static String runDate; - + /** * Sets configuration parameters to streaming enviroment * @@ -158,17 +160,18 @@ public static boolean hasAmsPubArgs(ParameterTool paramTool) { String amsPubArgs[] = {"ams.project.publish", "ams.token.publish", "ams.notification.topic"}; return hasArgs(amsPubArgs, paramTool); } - public static boolean hasAmsArgs(ParameterTool paramTool) { + + public static boolean hasAmsArgs(ParameterTool paramTool) { String amsPubArgs[] = {"ams.project.publish", "ams.token.publish", "ams.alert.topic"}; return hasArgs(amsPubArgs, paramTool); } - /** * Main dataflow of flink job */ public static void main(String[] args) throws Exception { + configJID(); // Initialize cli parameter tool final ParameterTool parameterTool = ParameterTool.fromArgs(args); @@ -184,6 +187,7 @@ public static void main(String[] args) throws Exception { String project = parameterTool.getRequired("ams.project"); String subMetric = parameterTool.getRequired("ams.sub.metric"); + String apiEndpoint = parameterTool.getRequired("api.endpoint"); String apiToken = parameterTool.getRequired("api.token"); String reportID = parameterTool.getRequired("report.uuid"); @@ -193,7 +197,6 @@ public static void main(String[] args) throws Exception { runDate = runDate + "T00:00:00.000Z"; } - int looseInterval = 1440; int strictInterval = 1440; @@ -272,8 +275,8 @@ public static void main(String[] args) throws Exception { kafkaProps); events.addSink(kSink); - }else if(hasAmsArgs(parameterTool)){ - String topic = parameterTool.get("ams.alert.topic"); + } else if (hasAmsArgs(parameterTool)) { + String topic = parameterTool.get("ams.alert.topic"); String tokenpub = parameterTool.get("ams.token.publish"); String projectpub = parameterTool.get("ams.project.publish"); @@ -800,21 +803,19 @@ private static int getInterval(String intervalParam) { if (matches) { String intervals[] = new String[]{}; - IntervalType intervalType=null; + IntervalType intervalType = null; if (intervalParam.contains("h")) { - intervalType=IntervalType.HOURS; + intervalType = IntervalType.HOURS; intervals = intervalParam.split("h"); - } - - else if (intervalParam.contains("d")) { - intervalType=IntervalType.DAY; + } else if (intervalParam.contains("d")) { + intervalType = IntervalType.DAY; intervals = intervalParam.split("d"); } else if (intervalParam.contains("m")) { - intervalType=IntervalType.MINUTES; + intervalType = IntervalType.MINUTES; intervals = intervalParam.split("m"); } - if (intervalType!=null && StringUtils.isNumeric(intervals[0])) { + if (intervalType != null && StringUtils.isNumeric(intervals[0])) { int interval = Integer.parseInt(intervals[0]); switch (intervalType) { case DAY: @@ -822,7 +823,7 @@ else if (intervalParam.contains("d")) { case HOURS: return interval * 60; case MINUTES: - return interval ; + return interval; default: return 1440; } @@ -834,4 +835,13 @@ else if (intervalParam.contains("d")) { return 1440; } + private static String getJID() { + return JobID.generate().toString(); + } + + private static void configJID() { //config the JID in the log4j.properties + String jobId = getJID(); + MDC.put("JID", jobId); + } + } diff --git a/flink_jobs_v2/stream_status/src/main/java/status/StatusManager.java b/flink_jobs_v2/stream_status/src/main/java/status/StatusManager.java index edc55490..be02c439 100644 --- a/flink_jobs_v2/stream_status/src/main/java/status/StatusManager.java +++ b/flink_jobs_v2/stream_status/src/main/java/status/StatusManager.java @@ -41,7 +41,7 @@ public class StatusManager { // Initialize logger static Logger LOG = LoggerFactory.getLogger(StatusManager.class); - + // Name of the report used private String report; @@ -73,6 +73,8 @@ public class StatusManager { int looseInterval = 1440; int strictInterval = 1440; + + public void setReport(String report) { this.report = report; } diff --git a/flink_jobs_v2/stream_status/src/main/resources/log4j.properties b/flink_jobs_v2/stream_status/src/main/resources/log4j.properties index da32ea0f..a9bc74e2 100644 --- a/flink_jobs_v2/stream_status/src/main/resources/log4j.properties +++ b/flink_jobs_v2/stream_status/src/main/resources/log4j.properties @@ -20,4 +20,4 @@ log4j.rootLogger=INFO, console log4j.appender.console=org.apache.log4j.ConsoleAppender log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n +log4j.appender.console.layout.ConversionPattern= %d{HH:mm:ss,SSS} %X{JID} %-5p %-60c %x - %m%n From 8049784263264f0831b4fef86902ef15b3fc06b9 Mon Sep 17 00:00:00 2001 From: Konstantinos Kagkelidis Date: Mon, 5 Dec 2022 16:57:26 +0200 Subject: [PATCH 16/16] ARGO-4131 Prepare v2.1.0 release --- flink_jobs_v2/ApiResourceManager/pom.xml | 4 ++-- flink_jobs_v2/ProfilesManager/pom.xml | 4 ++-- flink_jobs_v2/Timelines/pom.xml | 4 ++-- flink_jobs_v2/ams-connector/pom.xml | 6 +++--- flink_jobs_v2/ams_ingest_metric/pom.xml | 4 ++-- flink_jobs_v2/ams_ingest_sync/pom.xml | 6 +++--- flink_jobs_v2/batch_multi/pom.xml | 8 ++++---- flink_jobs_v2/pom.xml | 2 +- flink_jobs_v2/stream_status/pom.xml | 10 +++++----- 9 files changed, 24 insertions(+), 24 deletions(-) diff --git a/flink_jobs_v2/ApiResourceManager/pom.xml b/flink_jobs_v2/ApiResourceManager/pom.xml index bdd2dfcc..ad52bead 100644 --- a/flink_jobs_v2/ApiResourceManager/pom.xml +++ b/flink_jobs_v2/ApiResourceManager/pom.xml @@ -4,9 +4,9 @@ flink.jobs.v2 flink_jobs_v2 - 2.0.0 + 2.1.0 - 2.0.0 + 2.1.0 api.resource.manager ApiResourceManager jar diff --git a/flink_jobs_v2/ProfilesManager/pom.xml b/flink_jobs_v2/ProfilesManager/pom.xml index e19d65ac..33daffb2 100644 --- a/flink_jobs_v2/ProfilesManager/pom.xml +++ b/flink_jobs_v2/ProfilesManager/pom.xml @@ -4,11 +4,11 @@ flink.jobs.v2 flink_jobs_v2 - 2.0.0 + 2.1.0 profiles.manager ProfilesManager - 2.0.0 + 2.1.0 jar diff --git a/flink_jobs_v2/Timelines/pom.xml b/flink_jobs_v2/Timelines/pom.xml index 9c070420..daed1286 100644 --- a/flink_jobs_v2/Timelines/pom.xml +++ b/flink_jobs_v2/Timelines/pom.xml @@ -4,11 +4,11 @@ flink.jobs.v2 flink_jobs_v2 - 2.0.0 + 2.1.0 timeline.manager Timelines - 2.0.0 + 2.1.0 jar diff --git a/flink_jobs_v2/ams-connector/pom.xml b/flink_jobs_v2/ams-connector/pom.xml index 4cb5cdb4..17013fe5 100644 --- a/flink_jobs_v2/ams-connector/pom.xml +++ b/flink_jobs_v2/ams-connector/pom.xml @@ -16,17 +16,17 @@ flink.jobs.v2 flink_jobs_v2 - 2.0.0 + 2.1.0 ams.connector - 2.0.0 + 2.1.0 ams.connector Connect to AMS UTF-8 - 1.3.1 + 1.3.2 1.7.7 1.2.17 2.10 diff --git a/flink_jobs_v2/ams_ingest_metric/pom.xml b/flink_jobs_v2/ams_ingest_metric/pom.xml index 312d16a3..6ece83cb 100644 --- a/flink_jobs_v2/ams_ingest_metric/pom.xml +++ b/flink_jobs_v2/ams_ingest_metric/pom.xml @@ -14,7 +14,7 @@ language governing permissions and limitations under the License. --> argo.streaming ams-ingest-metric - 0.1 + 2.1.0 jar ARGO AMS Ingest Metric Data job @@ -63,7 +63,7 @@ language governing permissions and limitations under the License. --> ams.connector ams-connector - 2.0.0 + 2.1.0 jar diff --git a/flink_jobs_v2/ams_ingest_sync/pom.xml b/flink_jobs_v2/ams_ingest_sync/pom.xml index 79fff9ba..e7fe8b30 100644 --- a/flink_jobs_v2/ams_ingest_sync/pom.xml +++ b/flink_jobs_v2/ams_ingest_sync/pom.xml @@ -14,13 +14,13 @@ language governing permissions and limitations under the License. --> 4.0.0 argo.streaming - 0.0.1-SNAPSHOT + 2.1.0 ams-ingest-sync Stream sync data from AMS to HDFS UTF-8 - 1.3.1 + 1.3.2 1.7.7 1.2.17 2.10 @@ -57,7 +57,7 @@ language governing permissions and limitations under the License. --> ams.connector ams-connector - 2.0.0 + 2.1.0 jar diff --git a/flink_jobs_v2/batch_multi/pom.xml b/flink_jobs_v2/batch_multi/pom.xml index bb2d67cc..93ca60c5 100644 --- a/flink_jobs_v2/batch_multi/pom.xml +++ b/flink_jobs_v2/batch_multi/pom.xml @@ -15,7 +15,7 @@ language governing permissions and limitations under the License. --> flink.jobs.v2 flink_jobs_v2 - 2.0.0 + 2.1.0 argo.batch ArgoMultiJob @@ -175,19 +175,19 @@ language governing permissions and limitations under the License. --> timeline.manager Timelines - 2.0.0 + 2.1.0 jar profiles.manager ProfilesManager - 2.0.0 + 2.1.0 jar api.resource.manager ApiResourceManager - 2.0.0 + 2.1.0 jar diff --git a/flink_jobs_v2/pom.xml b/flink_jobs_v2/pom.xml index af10410c..d8ebce62 100644 --- a/flink_jobs_v2/pom.xml +++ b/flink_jobs_v2/pom.xml @@ -3,7 +3,7 @@ 4.0.0 flink.jobs.v2 flink_jobs_v2 - 2.0.0 + 2.1.0 pom UTF-8 diff --git a/flink_jobs_v2/stream_status/pom.xml b/flink_jobs_v2/stream_status/pom.xml index 7f52a2d5..86c136d5 100644 --- a/flink_jobs_v2/stream_status/pom.xml +++ b/flink_jobs_v2/stream_status/pom.xml @@ -14,11 +14,11 @@ language governing permissions and limitations under the License. --> flink.jobs.v2 flink_jobs_v2 - 2.0.0 + 2.1.0 argo.streaming streaming-status - 2.0.0 + 2.1.0 jar ARGO Streaming status job @@ -66,19 +66,19 @@ language governing permissions and limitations under the License. --> ams.connector ams-connector - 2.0.0 + 2.1.0 jar api.resource.manager ApiResourceManager - 2.0.0 + 2.1.0 jar profiles.manager ProfilesManager - 2.0.0 + 2.1.0 jar