Skip to content

Commit ffb46c4

Browse files
texodustexodus
authored
Merge pull request #317 from jpmorganchase/context-sort
Context sort
2 parents e725ea2 + 72427cb commit ffb46c4

File tree

6 files changed

+132
-79
lines changed

6 files changed

+132
-79
lines changed

‎packages/perspective/bench/js/report.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
*
88
*/
99

10-
import "../less/benchmark.less";
10+
import "!!style-loader!css-loader!less-loader!../less/benchmark.less";
1111

1212
import CodeMirror from 'codemirror';
1313
import '!!style-loader!css-loader!codemirror/lib/codemirror.css';

‎packages/perspective/bench/results/results.json‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎packages/perspective/src/js/perspective.js‎

Lines changed: 34 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,7 @@ export default function(Module) {
281281
return this.nsides;
282282
};
283283

284-
view.prototype._column_names = function() {
284+
view.prototype._column_names = function(skip_depth = false) {
285285
let col_names = [];
286286
let aggs = this.ctx.get_column_names();
287287
for (let key = 0; key < this.ctx.unity_get_column_count(); key++) {
@@ -297,6 +297,10 @@ export default function(Module) {
297297
continue;
298298
}
299299
let col_path = this.ctx.unity_get_column_path(key + 1);
300+
if (skip_depth && col_path.size() < skip_depth) {
301+
col_path.delete();
302+
continue;
303+
}
300304
col_name = [];
301305
for (let cnix = 0; cnix < col_path.size(); cnix++) {
302306
col_name.push(__MODULE__.scalar_vec_to_val(col_path, cnix));
@@ -386,26 +390,29 @@ export default function(Module) {
386390
let start_row = options.start_row || (viewport.top ? viewport.top : 0);
387391
let end_row = options.end_row || (viewport.height ? start_row + viewport.height : this.ctx.get_row_count());
388392
let start_col = options.start_col || (viewport.left ? viewport.left : 0);
389-
let end_col = options.end_col || (viewport.width ? start_row + viewport.width : this.ctx.unity_get_column_count() + (this.sides() === 0 ? 0 : 1));
393+
let end_col = options.end_col || (viewport.width ? start_col + viewport.width : this.ctx.unity_get_column_count() + (this.sides() === 0 ? 0 : 1));
390394
let slice;
395+
const sorted = typeof this.config.sort !== "undefined" && this.config.sort.length > 0;
391396
if (this.config.row_pivot[0] === "psp_okey") {
392397
end_row += this.config.column_pivot.length;
393398
}
394399
if (this.sides() === 0) {
395400
slice = __MODULE__.get_data_zero(this.ctx, start_row, end_row, start_col, end_col);
396401
} else if (this.sides() === 1) {
397402
slice = __MODULE__.get_data_one(this.ctx, start_row, end_row, start_col, end_col);
398-
} else {
403+
} else if (!sorted) {
399404
slice = __MODULE__.get_data_two(this.ctx, start_row, end_row, start_col, end_col);
405+
} else {
406+
slice = __MODULE__.get_data_two_skip_headers(this.ctx, this.config.column_pivot.length, start_row, end_row, start_col, end_col);
400407
}
401408

402409
let data = formatter.initDataValue();
403410

404-
let col_names = [[]].concat(this._column_names());
411+
let col_names = [[]].concat(this._column_names(this.sides() === 2 && sorted ? this.config.column_pivot.length : false));
405412
let row;
406413
let ridx = -1;
407414
for (let idx = 0; idx < slice.length; idx++) {
408-
let cidx = idx % (end_col - start_col);
415+
let cidx = idx % Math.min(end_col - start_col, col_names.slice(start_col, end_col - start_col + 1).length);
409416
if (cidx === 0) {
410417
if (row) {
411418
formatter.addRow(data, row);
@@ -967,21 +974,6 @@ export default function(Module) {
967974
}
968975
}
969976

970-
// Sort
971-
let sort = [];
972-
if (config.sort) {
973-
sort = config.sort.map(x => {
974-
if (!Array.isArray(x)) {
975-
return [config.aggregate.map(agg => agg.column).indexOf(x), 1];
976-
} else {
977-
return [config.aggregate.map(agg => agg.column).indexOf(x[0]), defaults.SORT_ORDERS.indexOf(x[1])];
978-
}
979-
});
980-
if (config.column_pivot.length > 0 && config.row_pivot.length > 0) {
981-
config.sort = config.sort.filter(x => config.row_pivot.indexOf(x[0]) === -1);
982-
}
983-
}
984-
985977
let schema = this.gnode.get_tblschema();
986978

987979
// Row Pivots
@@ -1005,26 +997,42 @@ export default function(Module) {
1005997
aggregates.push([agg.name || agg.column.join(defaults.COLUMN_SEPARATOR_STRING), agg_op, agg.column]);
1006998
}
1007999
} else {
1008-
let agg_op = __MODULE__.t_aggtype.AGGTYPE_DISTINCT_COUNT;
1009-
if (config.column_only) {
1010-
agg_op = __MODULE__.t_aggtype.AGGTYPE_ANY;
1011-
}
10121000
let t_aggs = schema.columns();
1001+
let t_aggtypes = schema.types();
10131002
for (let aidx = 0; aidx < t_aggs.size(); aidx++) {
10141003
let column = t_aggs.get(aidx);
1004+
let agg_op = __MODULE__.t_aggtype.AGGTYPE_ANY;
1005+
if (!config.column_only) {
1006+
agg_op = _string_to_aggtype[defaults.AGGREGATE_DEFAULTS[get_column_type(t_aggtypes.get(aidx).value)]];
1007+
}
10151008
if (column !== "psp_okey") {
10161009
aggregates.push([column, agg_op, [column]]);
10171010
}
10181011
}
10191012
t_aggs.delete();
10201013
}
10211014

1015+
// Sort
1016+
let sort = [];
1017+
if (config.sort) {
1018+
sort = config.sort.map(x => {
1019+
if (!Array.isArray(x)) {
1020+
return [aggregates.map(agg => agg[0]).indexOf(x), 1];
1021+
} else {
1022+
return [aggregates.map(agg => agg[0]).indexOf(x[0]), defaults.SORT_ORDERS.indexOf(x[1])];
1023+
}
1024+
});
1025+
if (config.column_pivot.length > 0 && config.row_pivot.length > 0) {
1026+
config.sort = config.sort.filter(x => config.row_pivot.indexOf(x[0]) === -1);
1027+
}
1028+
}
1029+
10221030
let context;
10231031
let sides = 0;
10241032
if (config.row_pivot.length > 0 || config.column_pivot.length > 0) {
10251033
if (config.column_pivot && config.column_pivot.length > 0) {
10261034
config.row_pivot = config.row_pivot || [];
1027-
context = __MODULE__.make_context_two(schema, config.row_pivot, config.column_pivot, filter_op, filters, aggregates, []);
1035+
context = __MODULE__.make_context_two(schema, config.row_pivot, config.column_pivot, filter_op, filters, aggregates, [], sort.length > 0);
10281036
sides = 2;
10291037
this.pool.register_context(this.gnode.get_id(), name, __MODULE__.t_ctx_type.TWO_SIDED_CONTEXT, context.$$.ptr);
10301038

@@ -1040,17 +1048,8 @@ export default function(Module) {
10401048
context.set_depth(__MODULE__.t_header.HEADER_COLUMN, config.column_pivot.length);
10411049
}
10421050

1043-
const groups = context.unity_get_column_count() / aggregates.length;
1044-
const new_sort = [];
1045-
1046-
for (let z = 0; z < groups; z++) {
1047-
for (let s of sort) {
1048-
new_sort.push([s[0] + z * aggregates.length, s[1]]);
1049-
}
1050-
}
1051-
10521051
if (sort.length > 0) {
1053-
__MODULE__.sort(context, new_sort);
1052+
__MODULE__.sort(context, sort);
10541053
}
10551054
} else {
10561055
context = __MODULE__.make_context_one(schema, config.row_pivot, filter_op, filters, aggregates, sort);

‎packages/perspective/test/js/pivots.js‎

Lines changed: 64 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -219,14 +219,14 @@ module.exports = perspective => {
219219
row_pivot: ["x"]
220220
});
221221
var answer = [
222-
{__ROW_PATH__: [], x: 4, y: 4, z: 2},
222+
{__ROW_PATH__: [], x: 10, y: 4, z: 2},
223223
{__ROW_PATH__: [1], x: 1, y: 1, z: 1},
224-
{__ROW_PATH__: [2], x: 1, y: 1, z: 1},
225-
{__ROW_PATH__: [3], x: 1, y: 1, z: 1},
226-
{__ROW_PATH__: [4], x: 1, y: 1, z: 1}
224+
{__ROW_PATH__: [2], x: 2, y: 1, z: 1},
225+
{__ROW_PATH__: [3], x: 3, y: 1, z: 1},
226+
{__ROW_PATH__: [4], x: 4, y: 1, z: 1}
227227
];
228228
let result2 = await view.to_json();
229-
expect(answer).toEqual(result2);
229+
expect(result2).toEqual(answer);
230230
});
231231

232232
it("['x'] test update pkey column", async function() {
@@ -312,51 +312,47 @@ module.exports = perspective => {
312312
var view = table.view({
313313
row_pivot: ["z"]
314314
});
315-
var answer = [{__ROW_PATH__: [], x: 4, y: 4, z: 2}, {__ROW_PATH__: [false], x: 2, y: 2, z: 1}, {__ROW_PATH__: [true], x: 2, y: 2, z: 1}];
315+
var answer = [{__ROW_PATH__: [], x: 10, y: 4, z: 2}, {__ROW_PATH__: [false], x: 6, y: 2, z: 1}, {__ROW_PATH__: [true], x: 4, y: 2, z: 1}];
316316
let result2 = await view.to_json();
317-
expect(answer).toEqual(result2);
317+
expect(result2).toEqual(answer);
318318
});
319319

320320
it("['x', 'z']", async function() {
321321
var table = perspective.table(data);
322322
var view = table.view({
323323
row_pivot: ["x", "z"]
324324
});
325-
326325
var answer = [
327-
{__ROW_PATH__: [], x: 4, y: 4, z: 2},
326+
{__ROW_PATH__: [], x: 10, y: 4, z: 2},
328327
{__ROW_PATH__: [1], x: 1, y: 1, z: 1},
329328
{__ROW_PATH__: [1, true], x: 1, y: 1, z: 1},
330-
{__ROW_PATH__: [2], x: 1, y: 1, z: 1},
331-
{__ROW_PATH__: [2, false], x: 1, y: 1, z: 1},
332-
{__ROW_PATH__: [3], x: 1, y: 1, z: 1},
333-
{__ROW_PATH__: [3, true], x: 1, y: 1, z: 1},
334-
{__ROW_PATH__: [4], x: 1, y: 1, z: 1},
335-
{__ROW_PATH__: [4, false], x: 1, y: 1, z: 1}
329+
{__ROW_PATH__: [2], x: 2, y: 1, z: 1},
330+
{__ROW_PATH__: [2, false], x: 2, y: 1, z: 1},
331+
{__ROW_PATH__: [3], x: 3, y: 1, z: 1},
332+
{__ROW_PATH__: [3, true], x: 3, y: 1, z: 1},
333+
{__ROW_PATH__: [4], x: 4, y: 1, z: 1},
334+
{__ROW_PATH__: [4, false], x: 4, y: 1, z: 1}
336335
];
337-
338336
let result2 = await view.to_json();
339-
expect(answer).toEqual(result2);
337+
expect(result2).toEqual(answer);
340338
});
341339

342340
it("['x', 'z'] windowed", async function() {
343341
var table = perspective.table(data);
344342
var view = table.view({
345343
row_pivot: ["x", "z"]
346344
});
347-
348345
var answer = [
349346
{__ROW_PATH__: [1, true], x: 1, y: 1, z: 1},
350-
{__ROW_PATH__: [2], x: 1, y: 1, z: 1},
351-
{__ROW_PATH__: [2, false], x: 1, y: 1, z: 1},
352-
{__ROW_PATH__: [3], x: 1, y: 1, z: 1},
353-
{__ROW_PATH__: [3, true], x: 1, y: 1, z: 1},
354-
{__ROW_PATH__: [4], x: 1, y: 1, z: 1},
355-
{__ROW_PATH__: [4, false], x: 1, y: 1, z: 1}
347+
{__ROW_PATH__: [2], x: 2, y: 1, z: 1},
348+
{__ROW_PATH__: [2, false], x: 2, y: 1, z: 1},
349+
{__ROW_PATH__: [3], x: 3, y: 1, z: 1},
350+
{__ROW_PATH__: [3, true], x: 3, y: 1, z: 1},
351+
{__ROW_PATH__: [4], x: 4, y: 1, z: 1},
352+
{__ROW_PATH__: [4, false], x: 4, y: 1, z: 1}
356353
];
357-
358354
let result2 = await view.to_json({start_row: 2});
359-
expect(answer).toEqual(result2);
355+
expect(result2).toEqual(answer);
360356
});
361357

362358
it("['x', 'z'], pivot_depth = 1", async function() {
@@ -365,17 +361,15 @@ module.exports = perspective => {
365361
row_pivot: ["x", "z"],
366362
row_pivot_depth: 1
367363
});
368-
369364
var answer = [
370-
{__ROW_PATH__: [], x: 4, y: 4, z: 2},
365+
{__ROW_PATH__: [], x: 10, y: 4, z: 2},
371366
{__ROW_PATH__: [1], x: 1, y: 1, z: 1},
372-
{__ROW_PATH__: [2], x: 1, y: 1, z: 1},
373-
{__ROW_PATH__: [3], x: 1, y: 1, z: 1},
374-
{__ROW_PATH__: [4], x: 1, y: 1, z: 1}
367+
{__ROW_PATH__: [2], x: 2, y: 1, z: 1},
368+
{__ROW_PATH__: [3], x: 3, y: 1, z: 1},
369+
{__ROW_PATH__: [4], x: 4, y: 1, z: 1}
375370
];
376-
377371
let result2 = await view.to_json();
378-
expect(answer).toEqual(result2);
372+
expect(result2).toEqual(answer);
379373
});
380374
});
381375

@@ -436,21 +430,52 @@ module.exports = perspective => {
436430
expect(answer).toEqual(result2);
437431
});
438432

439-
it("['x']", async function() {
433+
it("['x'] by ['y']", async function() {
440434
var table = perspective.table(data);
441435
var view = table.view({
442436
column_pivot: ["y"],
443437
row_pivot: ["x"]
444438
});
445439
var answer = [
446-
{__ROW_PATH__: [], "a|x": 1, "a|y": 1, "a|z": 1, "b|x": 1, "b|y": 1, "b|z": 1, "c|x": 1, "c|y": 1, "c|z": 1, "d|x": 1, "d|y": 1, "d|z": 1},
440+
{__ROW_PATH__: [], "a|x": 1, "a|y": 1, "a|z": 1, "b|x": 2, "b|y": 1, "b|z": 1, "c|x": 3, "c|y": 1, "c|z": 1, "d|x": 4, "d|y": 1, "d|z": 1},
447441
{__ROW_PATH__: [1], "a|x": 1, "a|y": 1, "a|z": 1, "b|x": null, "b|y": null, "b|z": null, "c|x": null, "c|y": null, "c|z": null, "d|x": null, "d|y": null, "d|z": null},
448-
{__ROW_PATH__: [2], "a|x": null, "a|y": null, "a|z": null, "b|x": 1, "b|y": 1, "b|z": 1, "c|x": null, "c|y": null, "c|z": null, "d|x": null, "d|y": null, "d|z": null},
449-
{__ROW_PATH__: [3], "a|x": null, "a|y": null, "a|z": null, "b|x": null, "b|y": null, "b|z": null, "c|x": 1, "c|y": 1, "c|z": 1, "d|x": null, "d|y": null, "d|z": null},
450-
{__ROW_PATH__: [4], "a|x": null, "a|y": null, "a|z": null, "b|x": null, "b|y": null, "b|z": null, "c|x": null, "c|y": null, "c|z": null, "d|x": 1, "d|y": 1, "d|z": 1}
442+
{__ROW_PATH__: [2], "a|x": null, "a|y": null, "a|z": null, "b|x": 2, "b|y": 1, "b|z": 1, "c|x": null, "c|y": null, "c|z": null, "d|x": null, "d|y": null, "d|z": null},
443+
{__ROW_PATH__: [3], "a|x": null, "a|y": null, "a|z": null, "b|x": null, "b|y": null, "b|z": null, "c|x": 3, "c|y": 1, "c|z": 1, "d|x": null, "d|y": null, "d|z": null},
444+
{__ROW_PATH__: [4], "a|x": null, "a|y": null, "a|z": null, "b|x": null, "b|y": null, "b|z": null, "c|x": null, "c|y": null, "c|z": null, "d|x": 4, "d|y": 1, "d|z": 1}
451445
];
452446
let result2 = await view.to_json();
453-
expect(answer).toEqual(result2);
447+
expect(result2).toEqual(answer);
448+
});
449+
450+
it("['y'] by ['z'], sorted by 'x'", async function() {
451+
var table = perspective.table([
452+
{x: 7, y: "A", z: true},
453+
{x: 2, y: "A", z: false},
454+
{x: 5, y: "A", z: true},
455+
{x: 4, y: "A", z: false},
456+
{x: 1, y: "B", z: true},
457+
{x: 8, y: "B", z: false},
458+
{x: 3, y: "B", z: true},
459+
{x: 6, y: "B", z: false},
460+
{x: 9, y: "C", z: true},
461+
{x: 10, y: "C", z: false},
462+
{x: 11, y: "C", z: true},
463+
{x: 12, y: "C", z: false}
464+
]);
465+
var view = table.view({
466+
column_pivot: ["z"],
467+
row_pivot: ["y"],
468+
sort: [["x", "desc"]]
469+
});
470+
471+
let answer = [
472+
{__ROW_PATH__: [], "false|x": 42, "false|y": 3, "false|z": 1, "true|x": 36, "true|y": 3, "true|z": 1},
473+
{__ROW_PATH__: ["C"], "false|x": 22, "false|y": 1, "false|z": 1, "true|x": 20, "true|y": 1, "true|z": 1},
474+
{__ROW_PATH__: ["A"], "false|x": 6, "false|y": 1, "false|z": 1, "true|x": 12, "true|y": 1, "true|z": 1},
475+
{__ROW_PATH__: ["B"], "false|x": 14, "false|y": 1, "false|z": 1, "true|x": 4, "true|y": 1, "true|z": 1}
476+
];
477+
let result2 = await view.to_json();
478+
expect(result2).toEqual(answer);
454479
});
455480
});
456481
};

‎packages/perspective/test/js/updates.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -564,7 +564,7 @@ module.exports = perspective => {
564564
return _.pick(x, "y");
565565
});
566566
let result2 = await view.to_json();
567-
expect(result.slice(1, 3)).toEqual(result2);
567+
expect(result2).toEqual(result.slice(1, 3));
568568
});
569569
});
570570
};

‎src/cpp/main.cpp‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1066,14 +1066,15 @@ make_context_one(t_schema schema, val j_pivots, t_filter_op combiner, val j_filt
10661066
*/
10671067
t_ctx2_sptr
10681068
make_context_two(t_schema schema, val j_rpivots, val j_cpivots, t_filter_op combiner,
1069-
val j_filters, val j_aggs, val j_sortby) {
1069+
val j_filters, val j_aggs, val j_sortby, bool show_totals) {
10701070
auto fvec = _get_fterms(schema, j_filters);
10711071
auto aggspecs = _get_aggspecs(j_aggs);
10721072
auto rpivots = vecFromJSArray<std::string>(j_rpivots);
10731073
auto cpivots = vecFromJSArray<std::string>(j_cpivots);
10741074
auto svec = _get_sort(j_sortby);
1075+
t_totals total = show_totals ? TOTALS_BEFORE : TOTALS_HIDDEN;
10751076

1076-
auto cfg = t_config(rpivots, cpivots, aggspecs, TOTALS_HIDDEN, combiner, fvec);
1077+
auto cfg = t_config(rpivots, cpivots, aggspecs, total, combiner, fvec);
10771078
auto ctx2 = std::make_shared<t_ctx2>(schema, cfg);
10781079

10791080
ctx2->init();
@@ -1124,6 +1125,33 @@ get_data(T ctx, t_uint32 start_row, t_uint32 end_row, t_uint32 start_col, t_uint
11241125
return arr;
11251126
}
11261127

1128+
val
1129+
get_data_two_skip_headers(t_ctx2_sptr ctx, t_uint32 depth, t_uint32 start_row, t_uint32 end_row, t_uint32 start_col, t_uint32 end_col) {
1130+
auto col_length = ctx->unity_get_column_count();
1131+
std::vector<t_uindex> col_nums;
1132+
col_nums.push_back(0);
1133+
for (t_uindex i = 0; i < col_length; ++i) {
1134+
if (ctx->unity_get_column_path(i + 1).size() == depth) {
1135+
col_nums.push_back(i + 1);
1136+
}
1137+
}
1138+
col_nums = std::vector<t_uindex>(col_nums.begin() + start_col, col_nums.begin() + std::min(end_col, (t_uint32)col_nums.size()));
1139+
auto slice = ctx->get_data(start_row, end_row, col_nums.front(), col_nums.back() + 1);
1140+
val arr = val::array();
1141+
t_uindex i = 0;
1142+
auto iter = slice.begin();
1143+
while (iter != slice.end()) {
1144+
t_uindex prev = col_nums.front();
1145+
for (auto idx = col_nums.begin(); idx != col_nums.end(); idx++, i++) {
1146+
t_uindex col_num = *idx;
1147+
iter += col_num - prev;
1148+
prev = col_num;
1149+
arr.set(i, scalar_to_val(*iter));
1150+
}
1151+
if (iter != slice.end()) iter++;
1152+
}
1153+
return arr;
1154+
}
11271155
/**
11281156
* Main
11291157
*/
@@ -1433,6 +1461,7 @@ EMSCRIPTEN_BINDINGS(perspective) {
14331461
function("get_data_zero", &get_data<t_ctx0_sptr>);
14341462
function("get_data_one", &get_data<t_ctx1_sptr>);
14351463
function("get_data_two", &get_data<t_ctx2_sptr>);
1464+
function("get_data_two_skip_headers", &get_data_two_skip_headers);
14361465
function("col_to_js_typed_array_zero", &col_to_js_typed_array<t_ctx0_sptr>);
14371466
function("col_to_js_typed_array_one", &col_to_js_typed_array<t_ctx1_sptr>);
14381467
function("col_to_js_typed_array_two", &col_to_js_typed_array<t_ctx2_sptr>);

0 commit comments

Comments
 (0)