Skip to content

Commit

Permalink
Implement lazy totals calculation in the DataView.
Browse files Browse the repository at this point in the history 
Add a ‘lazyTotalsCalculation’ grouping info option (default = false)
that delays the totals calculation until the item is accessed via
getItem(index).  This can dramatically improve the perceived
performance of the DataView operations on very large datasets.
  • Loading branch information
@mleibman
mleibman committed Dec 4, 2013
1 parent e0fa6a8 commit dc5f4ce
Show file tree
Hide file tree
Showing 3 changed files with 78 additions and 43 deletions.
23 changes: 15 additions & 8 deletions examples/example-grouping.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,8 @@ <h2>View Source:</h2>
new Slick.Data.Aggregators.Avg("percentComplete"),
new Slick.Data.Aggregators.Sum("cost")
],
aggregateCollapsed: false
aggregateCollapsed: false,
lazyTotalsCalculation: true
});
}

Expand All @@ -187,7 +188,8 @@ <h2>View Source:</h2>
new Slick.Data.Aggregators.Avg("percentComplete"),
new Slick.Data.Aggregators.Sum("cost")
],
aggregateCollapsed: aggregateCollapsed
aggregateCollapsed: aggregateCollapsed,
lazyTotalsCalculation: true
});
}

Expand All @@ -202,7 +204,8 @@ <h2>View Source:</h2>
new Slick.Data.Aggregators.Sum("duration"),
new Slick.Data.Aggregators.Sum("cost")
],
aggregateCollapsed: true
aggregateCollapsed: true,
lazyTotalsCalculation: true
},
{
getter: "effortDriven",
Expand All @@ -213,7 +216,8 @@ <h2>View Source:</h2>
new Slick.Data.Aggregators.Avg("percentComplete"),
new Slick.Data.Aggregators.Sum("cost")
],
collapsed: true
collapsed: true,
lazyTotalsCalculation: true
}
]);
}
Expand All @@ -229,7 +233,8 @@ <h2>View Source:</h2>
new Slick.Data.Aggregators.Sum("duration"),
new Slick.Data.Aggregators.Sum("cost")
],
aggregateCollapsed: true
aggregateCollapsed: true,
lazyTotalsCalculation: true
},
{
getter: "effortDriven",
Expand All @@ -239,7 +244,8 @@ <h2>View Source:</h2>
aggregators :[
new Slick.Data.Aggregators.Sum("duration"),
new Slick.Data.Aggregators.Sum("cost")
]
],
lazyTotalsCalculation: true
},
{
getter: "percentComplete",
Expand All @@ -250,7 +256,8 @@ <h2>View Source:</h2>
new Slick.Data.Aggregators.Avg("percentComplete")
],
aggregateCollapsed: true,
collapsed: true
collapsed: true,
lazyTotalsCalculation: true
}
]);
}
Expand All @@ -265,7 +272,7 @@ <h2>View Source:</h2>
d["id"] = "id_" + i;
d["num"] = i;
d["title"] = "Task " + i;
d["duration"] = Math.round(Math.random() * 14);
d["duration"] = Math.round(Math.random() * 30);
d["percentComplete"] = Math.round(Math.random() * 100);
d["start"] = someDates[ Math.floor((Math.random()*2)) ];
d["finish"] = someDates[ Math.floor((Math.random()*2)) ];
Expand Down
8 changes: 8 additions & 0 deletions slick.core.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,14 @@
* @type {Group}
*/
this.group = null;

/***
* Whether the totals have been fully initialized / calculated.
* Will be set to false for lazy-calculated group totals.
* @param initialized
* @type {Boolean}
*/
this.initialized = false;
}

GroupTotals.prototype = new NonDataItem();
Expand Down
90 changes: 55 additions & 35 deletions slick.dataview.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@
aggregateCollapsed: false,
aggregateChildGroups: false,
collapsed: false,
displayTotalsRow: true
displayTotalsRow: true,
lazyTotalsCalculation: false
};
var groupingInfos = [];
var groups = [];
Expand Down Expand Up @@ -363,7 +364,22 @@
}

function getItem(i) {
return rows[i];
var item = rows[i];

// if this is a grou row, make sure totals are calculated and update the title
if (item && item.__group && item.totals && !item.totals.initialized) {
var gi = groupingInfos[item.level];
if (!gi.displayTotalsRow) {
calculateTotals(item.totals);
item.title = gi.formatter ? gi.formatter(item) : item.value;
}
}
// if this is a totals row, make sure it's calculated
else if (item && item.__groupTotals && !item.initialized) {
calculateTotals(item);
}

return item;
}

function getItemMetadata(i) {
Expand Down Expand Up @@ -421,7 +437,7 @@
* @param varArgs Either a Slick.Group's "groupingKey" property, or a
* variable argument list of grouping values denoting a unique path to the row. For
* example, calling collapseGroup('high', '10%') will collapse the '10%' subgroup of
* the 'high' setGrouping.
* the 'high' group.
*/
function collapseGroup(varArgs) {
var args = Array.prototype.slice.call(arguments);
Expand All @@ -437,7 +453,7 @@
* @param varArgs Either a Slick.Group's "groupingKey" property, or a
* variable argument list of grouping values denoting a unique path to the row. For
* example, calling expandGroup('high', '10%') will expand the '10%' subgroup of
* the 'high' setGrouping.
* the 'high' group.
*/
function expandGroup(varArgs) {
var args = Array.prototype.slice.call(arguments);
Expand Down Expand Up @@ -503,27 +519,50 @@
return groups;
}

// TODO: lazy totals calculation
function calculateGroupTotals(group) {
// TODO: try moving iterating over groups into compiled accumulator
function calculateTotals(totals) {
var group = totals.group;
var gi = groupingInfos[group.level];
var isLeafLevel = (group.level == groupingInfos.length);
var totals = new Slick.GroupTotals();
var agg, idx = gi.aggregators.length;

if (!isLeafLevel && gi.aggregateChildGroups) {
// make sure all the subgroups are calculated
var i = group.groups.length;
while (i--) {
if (!group.groups[i].initialized) {
calculateTotals(group.groups[i]);
}
}
}

while (idx--) {
agg = gi.aggregators[idx];
agg.init();
gi.compiledAccumulators[idx].call(agg,
(!isLeafLevel && gi.aggregateChildGroups) ? group.groups : group.rows);
if (!isLeafLevel && gi.aggregateChildGroups) {
gi.compiledAccumulators[idx].call(agg, group.groups);
} else {
gi.compiledAccumulators[idx].call(agg, group.rows);
}
agg.storeResult(totals);
}
totals.initialized = true;
}

function addGroupTotals(group) {
var gi = groupingInfos[group.level];
var totals = new Slick.GroupTotals();
totals.group = group;
group.totals = totals;
if (!gi.lazyTotalsCalculation) {
calculateTotals(totals);
}
}

function calculateTotals(groups, level) {
function addTotals(groups, level) {
level = level || 0;
var gi = groupingInfos[level];
var groupCollapsed = gi.collapsed;
var toggledGroups = toggledGroupsByLevel[level];
var idx = groups.length, g;
while (idx--) {
g = groups[idx];
Expand All @@ -532,38 +571,20 @@
continue;
}

// Do a depth-first aggregation so that parent setGrouping aggregators can access subgroup totals.
// Do a depth-first aggregation so that parent group aggregators can access subgroup totals.
if (g.groups) {
calculateTotals(g.groups, level + 1);
addTotals(g.groups, level + 1);
}

if (gi.aggregators.length && (
gi.aggregateEmpty || g.rows.length || (g.groups && g.groups.length))) {
calculateGroupTotals(g);
addGroupTotals(g);
}
}
}

function finalizeGroups(groups, level) {
level = level || 0;
var gi = groupingInfos[level];
var groupCollapsed = gi.collapsed;
var toggledGroups = toggledGroupsByLevel[level];
var idx = groups.length, g;
while (idx--) {
g = groups[idx];
g.collapsed = groupCollapsed ^ toggledGroups[g.groupingKey];
g.title = gi.formatter ? gi.formatter(g) : g.value;

if (g.groups) {
finalizeGroups(g.groups, level + 1);
// Let the non-leaf setGrouping rows get garbage-collected.
// They may have been used by aggregates that go over all of the descendants,
// but at this point they are no longer needed.
g.rows = [];
}
}
}
}

function flattenGroupedRows(groups, level) {
level = level || 0;
Expand Down Expand Up @@ -793,8 +814,7 @@
if (groupingInfos.length) {
groups = extractGroups(newRows);
if (groups.length) {
calculateTotals(groups);
finalizeGroups(groups);
addTotals(groups);
newRows = flattenGroupedRows(groups);
}
}
Expand Down

0 comments on commit dc5f4ce

Please sign in to comment.