Merge pull request #413 from jpmorganchase/port-js

Port View to C++

Author: texodus

API.md

@@ -0,0 +1,46 @@
+# Perspective API design notes
+
+### Goals
+- Make the core C++ library an extensible base for developers to build on using different languages.
+- Port features previously segregated to Javascript into C++, thereby removing dependency on JS for the core library.
+- Move business logic into C++, thus allowing the purpose of the JS library to act as a bound wrapper rather than a dependent extension.
+
+This document aims to elaborate on the design of the core library API, as well as information designed to aid the core developer in creating new features/extending the library into new language bindings.
+
+### Components
+- Table: C++ class that will implement the features previously found in the `perspective.js` table, including all prototype methods and class members.
+- t_table: C++ class that manages the underlying operations for `Table`.
+- View: C++ class that will implement all features and members previously found in the `perspective.js` view. 
+- pool: A collection of `gnode`(s) managed in C++.
+- gnode: A "graph node" that contains any number of `t_table`s, although in implementation we only assign one `t_table` to one `gnode`.
+
+### Steps for porting
+1. Transliterate JS to C++: liberal use of `embind` and `val` in order to start porting features previously segregated to Javascript into C++.
+2. Make C++ functions portable: start removing the use of all JS-dependent data structures, members, and methods in order to allow for C++ portability.
+3. Add the translation layer: now that C++ functions expect generic C++ parameters (and not a `val`, on which any number of arbitrary JS-only operations may be called), add the translator layer that converts JS input into what the portable C++ API expects, as well as converts the output into something suitable for JS. 
+4. Convert the JS library into simple function calls: though the `perspective.js` API will not change, the underlying implementation for methods will simply return the output of the C++ method, with translation where necessary.
+
+### C++
+`View`: class which will implement all methods from the `perspective.js` version of view. 
+- Expects C++ data structure input and returns native data structures. 
+- Should be as portable and extensible as possible. 
+- Should abstract away concepts like pool and gnode.
+
+`Table`: class which implements all methods from the `perspective.js` table. 
+- Expects C++ data structure input and returns native data structures. 
+- Should be as portable and extensible as possible.
+
+### Translation/Helper layer
+`make_view`: helper function in `emscripten.cpp` that returns a `std::shared_ptr<View<CONTEXT_T>>`.
+- Should construct the underlying `View` object.
+- Should expect native C++ data structures as input.
+
+*TBD: how should we implement the translation layer?*
+- As separate methods that are called, which makes `make_view` extremely dependent on side-effects?
+- As a layer in `perspective.js`, which makes it harder to create C++ native data structures?
+- As a new binding in the C++ `src` folder, allowing for a place for developers to add their own language bindings? 
+
+### Javascript
+`View`: the view class on the Javascript side should be a wrapper around calls to the Emscripten `__MODULE__`.
+- Should maintain an internal reference to the C++ `View` object created on instantiation.
+- Methods should call the corresponding method on the C++ `View`, and return output that is ready for JS consumption.

CMakeLists.txt

@@ -325,6 +325,7 @@ set (SOURCE_FILES
 	src/cpp/tree_context_common.cpp
 	src/cpp/utils.cpp
 	src/cpp/update_task.cpp
+	src/cpp/view.cpp
 	src/cpp/vocab.cpp
 	)
 

packages/perspective/src/js/perspective.js

@@ -10,6 +10,7 @@
 import * as defaults from "./defaults.js";
 import {DataAccessor, clean_data} from "./DataAccessor/DataAccessor.js";
 import {DateParser} from "./DataAccessor/DateParser.js";
+import {extract_map, extract_vector} from "./translator.js";
 import {bindall, get_column_type} from "./utils.js";
 
 import {Precision} from "@apache-arrow/es5-esm/type";
@@ -228,6 +229,16 @@ export default function(Module) {
         this.callbacks = callbacks;
         this.name = name;
         this.table = table;
+
+        this._View = undefined;
+        if (sides === 0) {
+            this._View = __MODULE__.make_view_zero(pool, ctx, sides, gnode, name, defaults.COLUMN_SEPARATOR_STRING);
+        } else if (sides === 1) {
+            this._View = __MODULE__.make_view_one(pool, ctx, sides, gnode, name, defaults.COLUMN_SEPARATOR_STRING);
+        } else if (sides === 2) {
+            this._View = __MODULE__.make_view_two(pool, ctx, sides, gnode, name, defaults.COLUMN_SEPARATOR_STRING);
+        }
+
         bindall(this);
     }
 
@@ -237,8 +248,10 @@ export default function(Module) {
      * they are garbage collected - you must call this method to reclaim these.
      */
     view.prototype.delete = async function() {
-        this.pool.unregister_context(this.gnode.get_id(), this.name);
+        this._View.delete_view();
+        this._View.delete();
         this.ctx.delete();
+
         this.table.views.splice(this.table.views.indexOf(this), 1);
         this.table = undefined;
         let i = 0,
@@ -266,38 +279,15 @@ export default function(Module) {
     };
 
     view.prototype._column_names = function(skip_depth = false) {
-        let col_names = [];
-        let aggs = this.ctx.get_column_names();
-        for (let key = 0; key < this.ctx.unity_get_column_count(); key++) {
-            let col_name;
-            if (this.sides() === 0) {
-                col_name = aggs.get(key);
-                if (col_name === "psp_okey") {
-                    continue;
-                }
-            } else {
-                let name = aggs.get(key % aggs.size()).name();
-                if (name === "psp_okey") {
-                    continue;
-                }
-                let col_path = this.ctx.unity_get_column_path(key + 1);
-                if (skip_depth && col_path.size() < skip_depth) {
-                    col_path.delete();
-                    continue;
-                }
-                col_name = [];
-                for (let cnix = 0; cnix < col_path.size(); cnix++) {
-                    col_name.push(__MODULE__.scalar_vec_to_val(col_path, cnix));
-                }
-                col_name = col_name.reverse();
-                col_name.push(name);
-                col_name = col_name.join(defaults.COLUMN_SEPARATOR_STRING);
-                col_path.delete();
-            }
-            col_names.push(col_name);
+        let skip = false,
+            depth = 0;
+
+        if (skip_depth !== false) {
+            skip = true;
+            depth = Number(skip_depth);
         }
-        aggs.delete();
-        return col_names;
+
+        return extract_vector(this._View._column_names(skip, depth));
     };
 
     /**
@@ -312,42 +302,14 @@ export default function(Module) {
      * @returns {Promise<Object>} A Promise of this {@link view}'s schema.
      */
     view.prototype.schema = async function() {
-        // get type mapping
-        let schema = this.gnode.get_tblschema();
-        let _types = schema.types();
-        let names = schema.columns();
-        schema.delete();
+        let new_schema = extract_map(this._View.schema());
 
-        let types = {};
-        for (let i = 0; i < names.size(); i++) {
-            types[names.get(i)] = _types.get(i).value;
-        }
-        let new_schema = {};
-        let col_names = this._column_names();
-        for (let col_name of col_names) {
-            col_name = col_name.split(defaults.COLUMN_SEPARATOR_STRING);
-            col_name = col_name[col_name.length - 1];
-            if (types[col_name] === 1 || types[col_name] === 2) {
-                new_schema[col_name] = "integer";
-            } else if (types[col_name] === 19) {
-                new_schema[col_name] = "string";
-            } else if (types[col_name] === 9 || types[col_name] === 10) {
-                new_schema[col_name] = "float";
-            } else if (types[col_name] === 11) {
-                new_schema[col_name] = "boolean";
-            } else if (types[col_name] === 12) {
-                new_schema[col_name] = "datetime";
-            } else if (types[col_name] === 13) {
-                new_schema[col_name] = "date";
-            }
+        for (let name in new_schema) {
             if (this.sides() > 0 && this.config.row_pivot.length > 0) {
-                new_schema[col_name] = map_aggregate_types(col_name, new_schema[col_name], this.config.aggregate);
+                new_schema[name] = map_aggregate_types(name, new_schema[name], this.config.aggregate);
             }
         }
 
-        _types.delete();
-        names.delete();
-
         return new_schema;
     };
 
@@ -571,7 +533,7 @@ export default function(Module) {
      * @returns {Promise<number>} The number of aggregated rows.
      */
     view.prototype.num_rows = async function() {
-        return this.ctx.get_row_count();
+        return this._View.num_rows();
     };
 
     /**
@@ -584,7 +546,7 @@ export default function(Module) {
      * @returns {Promise<number>} The number of aggregated columns.
      */
     view.prototype.num_columns = async function() {
-        return this.ctx.unity_get_column_count();
+        return this._View.num_columns();
     };
 
     /**
@@ -595,7 +557,7 @@ export default function(Module) {
      * @returns {Promise<bool>} Whether this row is expanded.
      */
     view.prototype.get_row_expanded = async function(idx) {
-        return this.ctx.unity_get_row_expanded(idx);
+        return this._View.get_row_expanded(idx);
     };
 
     /**
@@ -606,11 +568,7 @@ export default function(Module) {
      * @returns {Promise<void>}
      */
     view.prototype.expand = async function(idx) {
-        if (this.nsides === 2 && this.ctx.unity_get_row_depth(idx) < this.config.row_pivot.length) {
-            return this.ctx.open(__MODULE__.t_header.HEADER_ROW, idx);
-        } else if (this.nsides < 2) {
-            return this.ctx.open(idx);
-        }
+        return this._View.expand(idx);
     };
 
     /**
@@ -621,27 +579,15 @@ export default function(Module) {
      * @returns {Promise<void>}
      */
     view.prototype.collapse = async function(idx) {
-        if (this.nsides === 2) {
-            return this.ctx.close(__MODULE__.t_header.HEADER_ROW, idx);
-        } else {
-            return this.ctx.close(idx);
-        }
+        return this._View.collapse(idx);
     };
 
     /**
      * Set expansion `depth` pf the pivot tree.
      *
      */
     view.prototype.set_depth = async function(depth) {
-        if (this.config.row_pivot.length >= depth) {
-            if (this.nsides === 2) {
-                return this.ctx.set_depth(__MODULE__.t_header.HEADER_ROW, depth);
-            } else {
-                return this.ctx.set_depth(depth);
-            }
-        } else {
-            console.warn(`Cannot expand past ${this.config.row_pivot.length}`);
-        }
+        return this._View.set_depth(depth, this.config.row_pivot.length);
     };
 
     /**

packages/perspective/src/js/translator.js

@@ -0,0 +1,34 @@
+/******************************************************************************
+ *
+ * Copyright (c) 2019, the Perspective Authors.
+ *
+ * This file is part of the Perspective library, distributed under the terms of
+ * the Apache License 2.0.  The full license can be found in the LICENSE file.
+ *
+ */
+
+/** Translation layer
+ * Interface between C++ and JS to handle conversions/data structures that
+ * were previously handled in non-portable perspective.js
+ */
+export const extract_vector = function(vector) {
+    let extracted = [];
+    for (let i = 0; i < vector.size(); i++) {
+        let item = vector.get(i);
+        extracted.push(item);
+    }
+    vector.delete();
+    return extracted;
+};
+
+export const extract_map = function(map) {
+    let extracted = {};
+    let keys = map.keys();
+    for (let i = 0; i < keys.size(); i++) {
+        let key = keys.get(i);
+        extracted[key] = map.get(key);
+    }
+    map.delete();
+    keys.delete();
+    return extracted;
+};

packages/perspective/test/js/constructors.js

@@ -679,6 +679,15 @@ module.exports = perspective => {
             table.delete();
         });
 
+        it("Handles utf16 column names", async function() {
+            var table = perspective.table({š: [1, 2, 3]});
+            let view = table.view({});
+            let result = await view.schema();
+            expect(result).toEqual({š: "integer"});
+            view.delete();
+            table.delete();
+        });
+
         it("Handles utf16", async function() {
             var table = perspective.table(data_6);
             let view = table.view({});

packages/perspective/test/js/pivots.js

@@ -322,7 +322,7 @@ module.exports = perspective => {
         it("['x'] does not translate type when only pivoted by column", async function() {
             var table = perspective.table(data);
             var view = table.view({
-                col_pivot: ["y"],
+                column_pivot: ["y"],
                 aggregate: [{column: "x", op: "avg"}]
             });
             let result2 = await view.schema();