From 4521822bcbdf599745f0da9fbc4db8080c6b05af Mon Sep 17 00:00:00 2001 From: Tamara Hills Date: Tue, 17 Nov 2015 15:54:21 -0500 Subject: [PATCH 1/4] Bug 1225522 - Packs the AdvancedTelemetry Histograms into dict format. r=marshall. --- apps/system/js/advanced_telemetry.js | 71 ++++++++++++++++++- .../test/unit/advanced_telemetry_test.js | 54 ++++++++++++-- 2 files changed, 120 insertions(+), 5 deletions(-) diff --git a/apps/system/js/advanced_telemetry.js b/apps/system/js/advanced_telemetry.js index 81110a08bc99..7423772e64b0 100644 --- a/apps/system/js/advanced_telemetry.js +++ b/apps/system/js/advanced_telemetry.js @@ -383,7 +383,7 @@ // to accumulate until the retry interval expires. self.startRetryBatch(); } - + self.request.packHistograms(); self.request.send({ timeout: AT.REPORT_TIMEOUT, onload: onload, @@ -512,6 +512,75 @@ return JSON.stringify(packet); }; + AdvancedTelemetryPing.prototype.packHistogram = + function packHistogram(hgram) { + if (typeof hgram.ranges === 'undefined' || + typeof hgram.counts === 'undefined') { + return hgram; + } + let r = hgram.ranges; + let c = hgram.counts; + let retgram = { + range: [r[1], r[r.length - 1]], + bucket_count: r.length, + histogram_type: hgram.histogram_type, + values: {}, + sum: hgram.sum + }; + + if (hgram.histogram_type === 0) { + retgram.log_sum = hgram.log_sum; + retgram.log_sum_squares = hgram.log_sum_squares; + } else { + retgram.sum_squares_lo = hgram.sum_squares_lo; + retgram.sum_squares_hi = hgram.sum_squares_hi; + } + + let first = true; + let last = 0; + + for (let i = 0; i < c.length; i++) { + let value = c[i]; + if (!value) { + continue; + } + + // add a lower bound + if (i && first) { + retgram.values[r[i - 1]] = 0; + } + first = false; + last = i + 1; + retgram.values[r[i]] = value; + } + + // add an upper bound + if (last && last < c.length) { + retgram.values[r[last]] = 0; + } + return retgram; + }, + + AdvancedTelemetryPing.prototype.packHistograms = function() { + var nameAddonHist = this.packet.payload.addonHistograms; + var nameKeyHist = this.packet.payload.keyedHistograms; + + // Pack the Addon Histograms. + for(var addon in nameAddonHist) { + for(var addonHistName in nameAddonHist[addon]) { + nameAddonHist[addon][addonHistName] = + this.packHistogram(nameAddonHist[addon][addonHistName]); + } + } + + // Pack the Keyed Histograms. + for(var key in nameKeyHist) { + for(var hist2 in nameKeyHist[key]) { + nameKeyHist[key][hist2] = this.packHistogram(nameKeyHist[key][hist2]); + } + } + }; + AdvancedTelemetryPing.prototype.send = function(xhrAttrs) { return new Promise((resolve, reject) => { this.data = this.getData(this.packet); diff --git a/apps/system/test/unit/advanced_telemetry_test.js b/apps/system/test/unit/advanced_telemetry_test.js index 294a9024fe82..8929ff68f883 100644 --- a/apps/system/test/unit/advanced_telemetry_test.js +++ b/apps/system/test/unit/advanced_telemetry_test.js @@ -306,7 +306,7 @@ suite('AdvancedTelemetry:', function() { suite('Sending the Metrics:', function() { var at, clock, XHR, xhr, mockSettings; var transmitSpy; - var wrapper; + var wrapper, packedWrapper; var gzipCompressedData = new Uint8Array([1,2,3,4,5]); setup(function(done) { @@ -358,6 +358,52 @@ suite('AdvancedTelemetry:', function() { } }; + packedWrapper = { + type: AdvancedTelemetry.REASON, + id: 'uuid', + creationDate: 'testDate', + version: AdvancedTelemetry.TELEMETRY_VERSION, + application: { + architecture: 'arm', + buildId: 'build', + name: AdvancedTelemetry.TELEMETRY_APP_NAME, + version: '43', + vendor: 'Mozilla', + platformVersion: '43', + xpcomAbi: 'arm-gcc3', + channel: 'default' + }, + clientId: 'uuid', + payload: { + keyedHistograms: { + DEVTOOLS_HUD_REFLOW_DURATION: { + verticalhome: { + range: [1, 1000], + bucket_count:10, + histogram_type: 0, + values: {'5':0, '12':2, '29':1, '70':1, '170':0}, + sum: 189, + log_sum:14.329224586486816, + log_sum_squares: 53.6346640586853 + } + } + }, + addonHistograms: { + communications: { + rn_metric: { + range: [1, 10000], + bucket_count:10, + histogram_type:1, + values:{'0':0, '1':1, '1251':0}, + sum: 99, + sum_squares_lo:9801, + sum_squares_hi:0 + } + } + } + } + }; + transmitSpy = this.sinon.spy(AdvancedTelemetry.prototype, 'transmit'); clock = this.sinon.useFakeTimers(); XHR = sinon.useFakeXMLHttpRequest(); @@ -478,9 +524,9 @@ suite('AdvancedTelemetry:', function() { assert.equal(req.clientId, wrapper.clientId); // Verify the application object picked up the settings correctly. - assert.deepEqual(req.application, wrapper.application); - // Verify that the Histograms are intact. - assert.deepEqual(req.payload, wrapper.payload); + assert.deepEqual(req.application, packedWrapper.application); + // Verify that the Histograms are intact and are properly packed. + assert.deepEqual(req.payload, packedWrapper.payload); done(); }); From f98bb6e835f0a7d5d6bd9980f8645c497a8bf5a1 Mon Sep 17 00:00:00 2001 From: Tamara Hills Date: Wed, 18 Nov 2015 09:30:34 -0500 Subject: [PATCH 2/4] fix nits from feedback. --- apps/system/js/advanced_telemetry.js | 9 ++++++++- apps/system/test/unit/advanced_telemetry_test.js | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/system/js/advanced_telemetry.js b/apps/system/js/advanced_telemetry.js index 7423772e64b0..2e95886c2ba7 100644 --- a/apps/system/js/advanced_telemetry.js +++ b/apps/system/js/advanced_telemetry.js @@ -512,8 +512,13 @@ return JSON.stringify(packet); }; + // This method converts a raw histogram with the .counts and .ranges arrays + // in it to one with just .values using the dict format. AdvancedTelemetryPing.prototype.packHistogram = function packHistogram(hgram) { + //Constant for Histogram types + const HISTOGRAM_EXP = 0; + if (typeof hgram.ranges === 'undefined' || typeof hgram.counts === 'undefined') { return hgram; @@ -528,7 +533,7 @@ sum: hgram.sum }; - if (hgram.histogram_type === 0) { + if (hgram.histogram_type === HISTOGRAM_EXP) { retgram.log_sum = hgram.log_sum; retgram.log_sum_squares = hgram.log_sum_squares; } else { @@ -561,6 +566,8 @@ return retgram; }, + // This method loops through the payload of histograms and converts each + // histogram payload to the 'dict' format required by the server for parsing. AdvancedTelemetryPing.prototype.packHistograms = function() { var nameAddonHist = this.packet.payload.addonHistograms; var nameKeyHist = this.packet.payload.keyedHistograms; diff --git a/apps/system/test/unit/advanced_telemetry_test.js b/apps/system/test/unit/advanced_telemetry_test.js index 8929ff68f883..cade73255275 100644 --- a/apps/system/test/unit/advanced_telemetry_test.js +++ b/apps/system/test/unit/advanced_telemetry_test.js @@ -364,7 +364,7 @@ suite('AdvancedTelemetry:', function() { creationDate: 'testDate', version: AdvancedTelemetry.TELEMETRY_VERSION, application: { - architecture: 'arm', + architecture: 'arm', buildId: 'build', name: AdvancedTelemetry.TELEMETRY_APP_NAME, version: '43', From 8821da19b3decc014a1ac50c494d7ddde9f585f9 Mon Sep 17 00:00:00 2001 From: Tamara Hills Date: Mon, 30 Nov 2015 12:43:13 -0500 Subject: [PATCH 3/4] Fix nits and correct level error in addons. --- apps/system/js/advanced_telemetry.js | 21 +++++----- .../test/unit/advanced_telemetry_test.js | 38 +++++++++---------- 2 files changed, 26 insertions(+), 33 deletions(-) diff --git a/apps/system/js/advanced_telemetry.js b/apps/system/js/advanced_telemetry.js index 2e95886c2ba7..a2aa7728d6aa 100644 --- a/apps/system/js/advanced_telemetry.js +++ b/apps/system/js/advanced_telemetry.js @@ -443,7 +443,7 @@ oldHist.log_sum += newHist.log_sum; oldHist.log_sum_squares += newHist.log_sum_squares; - for(var i = 0; i < oldHist.counts.length; i++) { + for (var i = 0; i < oldHist.counts.length; i++) { oldHist.counts[i] += newHist.counts[i]; } } @@ -451,11 +451,11 @@ function mergeLists(oldList, newList) { var histMap = new Map(); - for(var keyOld in oldList) { + for (var keyOld in oldList) { histMap.set(keyOld, oldList[keyOld]); } - for(var keyNew in newList) { + for (var keyNew in newList) { if (histMap.has(keyNew)) { mergeHistogram(oldList[keyNew], newList[keyNew]); } else { @@ -467,13 +467,13 @@ function mergeKeyed(keyedOldHist, payloadOld, payloadNew) { var keyedOldMapKeys = new Map(); - for(var keyOld in keyedOldHist) { + for (var keyOld in keyedOldHist) { keyedOldMapKeys.set(keyOld, keyedOldHist[keyOld]); } var keyedNew = payloadNew.keyedHistograms; var keyedOld = payloadOld.keyedHistograms; - for(var keyNew in keyedNew) { + for (var keyNew in keyedNew) { if (keyedOldMapKeys.has(keyNew)) { mergeLists(keyedOld[keyNew], keyedNew[keyNew]); } else { @@ -573,16 +573,13 @@ var nameKeyHist = this.packet.payload.keyedHistograms; // Pack the Addon Histograms. - for(var addon in nameAddonHist) { - for(var addonHistName in nameAddonHist[addon]) { - nameAddonHist[addon][addonHistName] = - this.packHistogram(nameAddonHist[addon][addonHistName]); - } + for (var addon in nameAddonHist) { + nameAddonHist[addon] = this.packHistogram(nameAddonHist[addon]); } // Pack the Keyed Histograms. - for(var key in nameKeyHist) { - for(var hist2 in nameKeyHist[key]) { + for (var key in nameKeyHist) { + for (var hist2 in nameKeyHist[key]) { nameKeyHist[key][hist2] = this.packHistogram(nameKeyHist[key][hist2]); } } diff --git a/apps/system/test/unit/advanced_telemetry_test.js b/apps/system/test/unit/advanced_telemetry_test.js index cade73255275..7b26d9776e68 100644 --- a/apps/system/test/unit/advanced_telemetry_test.js +++ b/apps/system/test/unit/advanced_telemetry_test.js @@ -342,17 +342,15 @@ suite('AdvancedTelemetry:', function() { } }, addonHistograms: { - communications: { - rn_metric: { - min: 1, - max: 10000, - histogram_type: 1, - sum: 99, - sum_squares_lo: 9801, - sum_squares_hi: 0, - ranges: [0, 1, 1251, 2501, 3751, 5001, 6250, 7500, 8750, 10000], - counts: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0] - } + rn_metric: { + min: 1, + max: 10000, + histogram_type: 1, + sum: 99, + sum_squares_lo: 9801, + sum_squares_hi: 0, + ranges: [0, 1, 1251, 2501, 3751, 5001, 6250, 7500, 8750, 10000], + counts: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0] } } } @@ -389,16 +387,14 @@ suite('AdvancedTelemetry:', function() { } }, addonHistograms: { - communications: { - rn_metric: { - range: [1, 10000], - bucket_count:10, - histogram_type:1, - values:{'0':0, '1':1, '1251':0}, - sum: 99, - sum_squares_lo:9801, - sum_squares_hi:0 - } + rn_metric: { + range: [1, 10000], + bucket_count:10, + histogram_type:1, + values:{'0':0, '1':1, '1251':0}, + sum: 99, + sum_squares_lo:9801, + sum_squares_hi:0 } } } From 9cd97b6835fcaa6e2d6e82e0c5b767f2ee26db56 Mon Sep 17 00:00:00 2001 From: Tamara Hills Date: Tue, 1 Dec 2015 12:55:44 -0500 Subject: [PATCH 4/4] Bug 1229083 - Adds the addon level to the addonHistograms node. r=marshall. --- apps/system/js/advanced_telemetry.js | 24 ++-- .../test/unit/advanced_telemetry_test.js | 130 ++++++++++-------- 2 files changed, 86 insertions(+), 68 deletions(-) diff --git a/apps/system/js/advanced_telemetry.js b/apps/system/js/advanced_telemetry.js index a2aa7728d6aa..f9277adb36c5 100644 --- a/apps/system/js/advanced_telemetry.js +++ b/apps/system/js/advanced_telemetry.js @@ -96,7 +96,7 @@ var array = longLine.match(/.{1,1000}/g); // Output each chunk to the console array.forEach(function (value) { - console.log(AT_DEBUG_PREFIX + value); + console.log(value); }); } else { // Its < what ADB can handle so just output it normally. @@ -430,9 +430,10 @@ // Merge the new metrics into the old metrics and then transmit the // result to the server. function mergePayloads(payloadOld, payloadNew) { - mergeKeyed(payloadOld.keyedHistograms, payloadOld, payloadNew); - mergeLists(payloadOld.addonHistograms, - payloadNew.addonHistograms); + mergeHistogramType(payloadOld.keyedHistograms, payloadOld, payloadNew, + 'keyedHistograms'); + mergeHistogramType(payloadOld.addonHistograms, payloadOld, payloadNew, + 'addonHistograms'); return payloadOld; } }); @@ -464,15 +465,15 @@ } } - function mergeKeyed(keyedOldHist, payloadOld, payloadNew) { + function mergeHistogramType(oldHist, payloadOld, payloadNew, histType) { var keyedOldMapKeys = new Map(); - for (var keyOld in keyedOldHist) { - keyedOldMapKeys.set(keyOld, keyedOldHist[keyOld]); + for (var keyOld in oldHist) { + keyedOldMapKeys.set(keyOld, oldHist[keyOld]); } - var keyedNew = payloadNew.keyedHistograms; - var keyedOld = payloadOld.keyedHistograms; + var keyedNew = payloadNew[histType]; + var keyedOld = payloadOld[histType]; for (var keyNew in keyedNew) { if (keyedOldMapKeys.has(keyNew)) { mergeLists(keyedOld[keyNew], keyedNew[keyNew]); @@ -574,7 +575,10 @@ // Pack the Addon Histograms. for (var addon in nameAddonHist) { - nameAddonHist[addon] = this.packHistogram(nameAddonHist[addon]); + for (var hist in nameAddonHist[addon]) { + nameAddonHist[addon][hist] = + this.packHistogram(nameAddonHist[addon][hist]); + } } // Pack the Keyed Histograms. diff --git a/apps/system/test/unit/advanced_telemetry_test.js b/apps/system/test/unit/advanced_telemetry_test.js index 7b26d9776e68..755a5259fc6f 100644 --- a/apps/system/test/unit/advanced_telemetry_test.js +++ b/apps/system/test/unit/advanced_telemetry_test.js @@ -65,15 +65,17 @@ suite('AdvancedTelemetry:', function() { } }, addonHistograms: { - COMM_SMS: { - min: 1, - max: 1000, - histogram_type:0, - sum:58, - log_sum: 10.935960054397583, - log_sum_squares: 29.95399808883667, - ranges: [0,1,2,5,12,29,70,170,412,1000], - counts:[0,0,0,0,4,0,0,0,0,0] + addon1: { + COMM_SMS: { + min: 1, + max: 1000, + histogram_type: 0, + sum: 58, + log_sum: 10.935960054397583, + log_sum_squares: 29.95399808883667, + ranges: [0, 1, 2, 5, 12, 29, 70, 170, 412, 1000], + counts: [0, 0, 0, 0, 4, 0, 0, 0, 0, 0] + } } } }; @@ -158,25 +160,29 @@ suite('AdvancedTelemetry:', function() { } }, addonHistograms: { - COMM_SMS: { - min: 1, - max: 1000, - histogram_type:0, - sum:58, - log_sum: 10.935960054397583, - log_sum_squares: 29.95399808883667, - ranges: [0,1,2,5,12,29,70,170,412,1000], - counts:[0,0,0,0,4,0,0,0,0,0] + addon1: { + COMM_SMS: { + min: 1, + max: 1000, + histogram_type: 0, + sum: 58, + log_sum: 10.935960054397583, + log_sum_squares: 29.95399808883667, + ranges: [0, 1, 2, 5, 12, 29, 70, 170, 412, 1000], + counts: [0, 0, 0, 0, 4, 0, 0, 0, 0, 0] + } }, - COMM_DIALER: { - min: 1, - max: 1000, - histogram_type:0, - sum:58, - log_sum: 10.935960054397583, - log_sum_squares: 29.95399808883667, - ranges: [0,1,2,5,12,29,70,170,412,1000], - counts:[0,0,0,0,4,0,0,0,0,0] + addon2: { + COMM_DIALER: { + min: 1, + max: 1000, + histogram_type: 0, + sum: 58, + log_sum: 10.935960054397583, + log_sum_squares: 29.95399808883667, + ranges: [0, 1, 2, 5, 12, 29, 70, 170, 412, 1000], + counts: [0, 0, 0, 0, 4, 0, 0, 0, 0, 0] + } } } }; @@ -241,20 +247,24 @@ suite('AdvancedTelemetry:', function() { } }, addonHistograms:{ - COMM_SMS:{ - min:1,max:1000,histogram_type:0, - sum:116, - log_sum:21.871920108795166, - log_sum_squares:59.90799617767334, - ranges:[0,1,2,5,12,29,70,170,412,1000], - counts:[0,0,0,0,8,0,0,0,0,0]}, - COMM_DIALER:{ - min:1,max:1000,histogram_type:0, - sum:58, - log_sum:10.935960054397583, - log_sum_squares:29.95399808883667, - ranges:[0,1,2,5,12,29,70,170,412,1000], - counts:[0,0,0,0,4,0,0,0,0,0]} + addon1: { + COMM_SMS: { + min: 1, max: 1000, histogram_type: 0, + sum: 116, + log_sum: 21.871920108795166, + log_sum_squares: 59.90799617767334, + ranges: [0, 1, 2, 5, 12, 29, 70, 170, 412, 1000], + counts: [0, 0, 0, 0, 8, 0, 0, 0, 0, 0]} + }, + addon2: { + COMM_DIALER: { + min: 1, max: 1000, histogram_type: 0, + sum: 58, + log_sum: 10.935960054397583, + log_sum_squares: 29.95399808883667, + ranges: [0, 1, 2, 5, 12, 29, 70, 170, 412, 1000], + counts: [0, 0, 0, 0, 4, 0, 0, 0, 0, 0]} + } } }; @@ -342,15 +352,17 @@ suite('AdvancedTelemetry:', function() { } }, addonHistograms: { - rn_metric: { - min: 1, - max: 10000, - histogram_type: 1, - sum: 99, - sum_squares_lo: 9801, - sum_squares_hi: 0, - ranges: [0, 1, 1251, 2501, 3751, 5001, 6250, 7500, 8750, 10000], - counts: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0] + addon3: { + rn_metric: { + min: 1, + max: 10000, + histogram_type: 1, + sum: 99, + sum_squares_lo: 9801, + sum_squares_hi: 0, + ranges: [0, 1, 1251, 2501, 3751, 5001, 6250, 7500, 8750, 10000], + counts: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0] + } } } } @@ -387,14 +399,16 @@ suite('AdvancedTelemetry:', function() { } }, addonHistograms: { - rn_metric: { - range: [1, 10000], - bucket_count:10, - histogram_type:1, - values:{'0':0, '1':1, '1251':0}, - sum: 99, - sum_squares_lo:9801, - sum_squares_hi:0 + addon3: { + rn_metric: { + range: [1, 10000], + bucket_count: 10, + histogram_type: 1, + values: {'0': 0, '1': 1, '1251': 0}, + sum: 99, + sum_squares_lo: 9801, + sum_squares_hi: 0 + } } } }