add incremental data loading in table view with 1k add data rows buffer (#41)

Author: RandomFractals

src/views/tableView.ts

@@ -26,9 +26,11 @@ const d3 = require('d3-dsv');
  * Defines Table view class for managing state and behaviour of Table webview panels.
  */
 export class TableView {
+  // table view tracking vars
   public static currentView: TableView | undefined;
   private static _views: Map<string, TableView> = new Map<string, TableView>();
 
+  // table view instance vars
   private readonly _webviewPanel: WebviewPanel;
   private readonly _extensionUri: Uri;
   private readonly _documentUri: Uri;
@@ -37,6 +39,15 @@ export class TableView {
   private readonly _fileExtension: string;
   private _disposables: Disposable[] = [];
 
+  // tabular data vars
+  private _tableData: [] = [];
+  private _currentDataPage: number = 0;
+  private _totalRows: number = 0;
+
+  // default page data size for data streaming setup
+  // TODO: move this to tabular data viewer config options later
+  private readonly _pageDataSize: number = 1000;
+
   /**
    * Reveals current table view or creates new table webview panel for tabular data display.
    * 
@@ -154,12 +165,13 @@ export class TableView {
     // process webview messages
     this.webviewPanel.webview.onDidReceiveMessage((message: any) => {
       const command: string = message.command;
-      const text = message.text;
       switch (command) {
         case 'refresh':
           // reload data view and config
           this.refresh();
           break;
+        case 'addData':
+          this.addData(message.dataPage);
         case 'saveData':
           this.saveData(message.data, message.dataFileName, message.dataFileType);
           break;
@@ -171,6 +183,11 @@ export class TableView {
    * Reloads table view on data save changes or vscode IDE realod.
    */
   public async refresh(): Promise<void> {
+    // clear loaded data info
+    this._totalRows = 0;
+    this._currentDataPage = 0;
+    this._tableData = [];
+
     // load data
     workspace.fs.readFile(this._documentUri).then((binaryData: Uint8Array) => {
       const textData: string = new TextDecoder().decode(binaryData);
@@ -181,18 +198,44 @@ export class TableView {
       let tableData: any = dsvParser.parse(textData, d3.autoType);
       this.logTableData(tableData);
 
-      // update webview
+      // save table data for data streaming later
+      this._tableData = tableData;
+      this._totalRows = this._tableData.length;
+
+      // send initial data rows to table view for display
+      const initialDataRows: Array<any> = tableData.slice(0, Math.min(this._pageDataSize, this._totalRows));
       this.webviewPanel.webview.postMessage({
         command: 'refresh',
         fileName: this._fileName,
         documentUrl: this._documentUri.toString(),
-        tableData: tableData
-      });      
+        tableData: initialDataRows,
+        totalRows: this._totalRows
+      });
     }, reason => {
       window.showErrorMessage(`Could not load \`${this._documentUri}\` content. Reason: \n ${reason}`);
     });
   }
 
+  /**
+   * Sends more data rows to the table view for incremental data loading and display.
+   * 
+   * @param dataPage Requested data page index for loading the next set of data rows.
+   */
+  public async addData(dataPage: number): Promise<void> {
+    const nextRows: number = dataPage * this._pageDataSize;
+    if (nextRows < this._totalRows) {
+      // get the next set of data rows to load in table view
+      const dataRows: Array<any> =
+        this._tableData.slice(nextRows, Math.min(nextRows + this._pageDataSize, this._totalRows));
+
+      // send the next set of data rows to display
+      this.webviewPanel.webview.postMessage({
+        command: 'addData',
+        dataRows: dataRows,
+      });
+    }
+  }
+
   /**
    * Saves table data in a new data file.
    */

web/scripts/tableView.js

@@ -12,6 +12,9 @@ let saveDataFileName = '';
 let tableContainer, table, progressRing, saveFileTypeSelector;
 let tableColumns = [];
 let tableData = [];
+let loadedRows = 0;
+let totalRows = 0;
+let dataPage = 0;
 
 // table view settings
 const toolbarHeight = 40; // table view toolbar height offset
@@ -89,8 +92,12 @@ window.addEventListener('message', event => {
       fileName = event.data.fileName;
       vscode.setState({ documentUrl: documentUrl });
       tableData = event.data.tableData;
+      totalRows = event.data.totalRows;
       loadData(tableData, fileName);
       break;
+    case 'addData':
+      addData(table, event.data.dataRows);
+      break;
   }
 });
 
@@ -140,10 +147,26 @@ function reloadData() {
  * Loads and displays table data.
  * 
  * @param {*} tableData Data array to display in tabulator table.
- * @param {*} fileName Data file name for table config persistence and reload.
  */
-function loadData(tableData, documentUrl) {
+function loadData(tableData) {
   logTableData(tableData);
+  if (table === undefined) {
+    createTable(tableData);
+  }
+  else {
+    // reload table data
+    clearTable(table);
+    addData(table, tableData);
+    progressRing.style.visibility = 'hidden';
+  }
+}
+
+/**
+ * Creates new Tabulator table with initial set of data to display.
+ * 
+ * @param {*} tableData Data array to display in tabulator table.
+ */
+function createTable(tableData) {
   if (table === undefined) {
     table = new Tabulator('#table-container', {
       height: window.innerHeight - toolbarHeight,
@@ -211,63 +234,64 @@ function loadData(tableData, documentUrl) {
       }
     });
 
-    table.on('tableBuilt', function () {
-      // hide data loading progress ring
-      progressRing.style.visibility = 'hidden';
-
-      // get table columns for debug
-      const columns = table.getColumns();
-      console.log('tableView.columns:', columns);
-
-      // add row selection column
-      table.addColumn({
-        formatter: 'rowSelection',
-        titleFormatter: 'rowSelection',
-        headerMenu: [], // don't show header context menu
-        headerSort: false,
-        download: false // don't include it in data save
-      }, true) // add as 1st column
-      .then(function (column) {
-        // column - the component for the newly created column
-        // run code after column has been added
-      })
-      .catch(function (error) {
-        // log adding row selection column error for now
-        console.error('tableView.addRowSelectionCollumn: Error\n', error);
-      });
-    });
-  }
-  else {
-    // reload table data
-    clearTable(table);
-    addData(table, tableData);
-    progressRing.style.visibility = 'hidden';
+    // update table settings after initial data rows load
+    table.on('tableBuilt', onTableBuilt);
   }
 }
 
 /**
- * Removes all table data.
+ * Updates Tabulator table after initial set of data rows is loaded.
  */
-function clearTable(table) {
-  if (table) {
-    table.clearData();  
+function onTableBuilt () {
+  // hide data loading progress ring
+  progressRing.style.visibility = 'hidden';
+
+  // get table columns for debug
+  const columns = table.getColumns();
+  console.log('tableView.columns:', columns);
+
+  // add row selection column
+  table.addColumn({
+    formatter: 'rowSelection',
+    titleFormatter: 'rowSelection',
+    headerMenu: [], // don't show header context menu
+    headerSort: false,
+    download: false // don't include it in data save
+  }, true) // add as 1st column
+    .then(function (column) {
+      // column - the component for the newly created column
+      // run code after column has been added
+    })
+    .catch(function (error) {
+      // log adding row selection column error for now
+      console.error('tableView.addRowSelectionCollumn: Error\n', error);
+    });
+
+  // request more data for incremental data loading
+  loadedRows = table.getRows().length;
+  if (loadedRows < totalRows) {
+    dataPage++;
+    progressRing.style.visibility = 'visible';
+    vscode.postMessage({
+      command: 'addData',
+      dataPage: dataPage
+    });
   }
 }
 
 /**
- * Scrolls table data display to the first table row.
+ * Clears displayed table data.
  */
-function scrollToFirstRow() {
-  const rows = table.getRows();
-  table.scrollToRow(rows[0], 'top', true);
-}
+function clearTable(table) {
+  if (table) {
+    // clear displayed table view
+    table.clearData();
 
-/**
- * Scrolls table data display to the last table row.
- */
-function scrollToLastRow() {
-  const rows = table.getRows();
-  table.scrollToRow(rows[rows.length-1], 'top', true);
+    // reset rows and data page counters
+    loadedRows = 0;
+    totalRows = 0;
+    dataPage = 0;
+  }
 }
 
 /**
@@ -279,7 +303,23 @@ function scrollToLastRow() {
 function addData(table, tableData) {
   if (table && tableData) {
     table.addData(tableData, true)
-      .then(function (rows) { //rows - array of the row components for the rows updated or added
+      .then(function (rows) { // rows - array of the row components for the rows updated or added
+        // update loaded rows counter
+        loadedRows += rows.length;
+        if (loadedRows < totalRows) {
+          // request more data to load and display
+          dataPage++;
+          progressRing.style.visibility = 'visible';
+          vscode.postMessage({
+            command: 'addData',
+            dataPage: dataPage
+          });
+        }
+        else {
+          // hide data loading progress ring
+          progressRing.style.visibility = 'hidden';
+          console.log('tableView:rowCount:', loadedRows);
+        }
       })
       .catch(function (error) {
         // handle error updating data
@@ -288,6 +328,22 @@ function addData(table, tableData) {
   }
 }
 
+/**
+ * Scrolls table data display to the first table row.
+ */
+function scrollToFirstRow() {
+  const rows = table.getRows();
+  table.scrollToRow(rows[0], 'top', true);
+}
+
+/**
+ * Scrolls table data display to the last table row.
+ */
+function scrollToLastRow() {
+  const rows = table.getRows();
+  table.scrollToRow(rows[rows.length-1], 'top', true);
+}
+
 /**
  * Saves table data as CSV, TSV, or JSON data array document.
  */
@@ -335,6 +391,6 @@ function saveData() {
  * @param tableData Loaded able data.
  */
 function logTableData(tableData) {
-  console.log('tabular.data.view:rowCount:', tableData.length);
+  console.log('tableView:rowCount:', tableData.length);
   console.log('1st 10 rows:', tableData.slice(0, 10));
 }