Update gather functionality to support 0.4.1 of python-program-analysis

build/ci/postInstall.js

@@ -14,7 +14,7 @@ var constants_1 = require("../constants");
  * The solution is to modify the type definition file after `npm install`.
  */
 function fixJupyterLabDTSFiles() {
-    var relativePath = path.join('node_modules', '@jupyterlab', 'coreutils', 'lib', 'settingregistry.d.ts');
+    var relativePath = path.join('node_modules', '@jupyterlab', 'services', 'node_modules', '@jupyterlab', 'coreutils', 'lib', 'settingregistry.d.ts');
     var filePath = path.join(constants_1.ExtensionRootDir, relativePath);
     if (!fs.existsSync(filePath)) {
         throw new Error("Type Definition file from JupyterLab not found '" + filePath + "' (pvsc post install script)");

package.json

@@ -1694,8 +1694,8 @@
                 },
                 "python.dataScience.enableGather": {
                     "type": "boolean",
-                    "default": false,
-                    "description": "Enable code gathering for single cells in the interactive window (experimental).",
+                    "default": true,
+                    "description": "Enable code gather. For a gathered cell, it and all the code cells it depends on will be exported to a new notebook.",
                     "scope": "resource"
                 },
                 "python.dataScience.codeLenses": {
@@ -2679,7 +2679,8 @@
     "dependencies": {
         "@blueprintjs/select": "3.10.0",
         "@jupyterlab/services": "^3.2.1",
-        "@msrvida/python-program-analysis": "^0.2.6",
+        "@jupyterlab/coreutils": "^3.1.0",
+        "@msrvida/python-program-analysis": "^0.4.1",
         "ansi-regex": "^4.1.0",
         "arch": "^2.1.0",
         "azure-storage": "^2.10.3",

package.nls.json

@@ -378,5 +378,7 @@
     "DataScience.notebookNotFound" : "python -m jupyter notebook --version is not running",
     "DataScience.findJupyterCommandProgress" : "Active interpreter does not support {0}. Searching for the best available interpreter.",
     "DataScience.findJupyterCommandProgressCheckInterpreter": "Checking {0}.",
-    "DataScience.findJupyterCommandProgressSearchCurrentPath": "Searching current path."
+    "DataScience.findJupyterCommandProgressSearchCurrentPath": "Searching current path.",
+    "DataScience.gatheredScriptDescription": "# This file contains the minimal amount of code required to produce the code cell gathered.\n",
+    "DataScience.gatheredNotebookDescriptionInMarkdown": "## This notebook was generated for a gathered cell."
 }

src/client/common/utils/localize.ts

@@ -294,6 +294,8 @@ export namespace DataScience {
     export const findJupyterCommandProgress = localize('DataScience.findJupyterCommandProgress', 'Active interpreter does not support {0}. Searching for the best available interpreter.');
     export const findJupyterCommandProgressCheckInterpreter = localize('DataScience.findJupyterCommandProgressCheckInterpreter', 'Checking {0}.');
     export const findJupyterCommandProgressSearchCurrentPath = localize('DataScience.findJupyterCommandProgressSearchCurrentPath', 'Searching current path.');
+    export const gatheredScriptDescription = localize('DataScience.gatheredScriptDescription', '# This file contains the minimal amount of code required to produce the code cell gathered.\n');
+    export const gatheredNotebookDescriptionInMarkdown = localize('DataScience.gatheredNotebookDescriptionInMarkdown', '## This notebook was generated for a gathered cell.');
 }
 
 export namespace DebugConfigStrings {

src/client/datascience/cellFactory.ts

@@ -107,45 +107,54 @@ export function hasCells(document: TextDocument, settings?: IDataScienceSettings
     return false;
 }
 
-export function generateCellRanges(document: TextDocument, settings?: IDataScienceSettings): { range: Range; title: string; cell_type: string }[] {
-    // Implmentation of getCells here based on Don's Jupyter extension work
+function generateCellRangesFromStringArray(lines: string[], settings?: IDataScienceSettings): { range: Range; title: string; cell_type: string }[] {
     const matcher = new CellMatcher(settings);
-    const cells : { range: Range; title: string; cell_type: string }[] = [];
-    for (let index = 0; index < document.lineCount; index += 1) {
-        const line = document.lineAt(index);
-        if (matcher.isCell(line.text)) {
-            if (cells.length > 0) {
-                const previousCell = cells[cells.length - 1];
-                previousCell.range = new Range(previousCell.range.start, document.lineAt(index - 1).range.end);
+    const cells: { range: Range; title: string; cell_type: string }[] = [];
+
+    for (let index = 0; index < lines.length; index += 1) {
+        if (matcher.isCell(lines[index])) {
+
+            // We have a cell, find the next cell
+            let endCellIndex = index + 1;
+            while (endCellIndex < lines.length && !matcher.isCell(lines[endCellIndex])) {
+                endCellIndex += 1;
             }
+            endCellIndex -= 1; // Get last line of cell.
 
-            const results = matcher.exec(line.text);
-            if (results !== undefined) {
+            const result = matcher.exec(lines[index]);
+            if (result !== undefined) {
                 cells.push({
-                    range: line.range,
-                    title: results,
-                    cell_type: matcher.getCellType(line.text)
+                    range: new Range(index + 1, 0, endCellIndex, lines[endCellIndex].length),
+                    title: result,
+                    cell_type: matcher.getCellType(lines[index])
                 });
             }
+            index = endCellIndex;
         }
     }
 
-    if (cells.length >= 1) {
-        const line = document.lineAt(document.lineCount - 1);
-        const previousCell = cells[cells.length - 1];
-        previousCell.range = new Range(previousCell.range.start, line.range.end);
-    }
-
     return cells;
 }
 
-export function generateCellsFromDocument(document: TextDocument, settings?: IDataScienceSettings): ICell[] {
+export function generateCellsFromString(source: string, settings?: IDataScienceSettings): ICell[] {
+    const lines: string[] = source.splitLines({ trim: false, removeEmptyEntries: false });
+
     // Get our ranges. They'll determine our cells
-    const ranges = generateCellRanges(document, settings);
+    const ranges = generateCellRangesFromStringArray(lines, settings);
 
     // For each one, get its text and turn it into a cell
     return Array.prototype.concat(...ranges.map(r => {
-        const code = document.getText(r.range);
-        return generateCells(settings, code, document.fileName, r.range.start.line, false, uuid());
+        const code = lines.slice(r.range.start.line, r.range.end.line).join('\n');
+        return generateCells(settings, code, '', r.range.start.line, false, uuid());
     }));
 }
+
+export function generateCellRangesFromDocument(document: TextDocument, settings?: IDataScienceSettings): { range: Range; title: string; cell_type: string }[] {
+    const lines: string[] = document.getText().splitLines({ trim: false, removeEmptyEntries: false });
+
+    return generateCellRangesFromStringArray(lines, settings);
+}
+
+export function generateCellsFromDocument(document: TextDocument, settings?: IDataScienceSettings): ICell[] {
+    return generateCellsFromString(document.getText(), settings);
+}

src/client/datascience/constants.ts

@@ -203,6 +203,7 @@ export enum Telemetry {
     NotebookOpenCount = 'DATASCIENCE.NATIVE.NOTEBOOK_OPEN_COUNT',
     NotebookOpenTime = 'DS_INTERNAL.NATIVE.NOTEBOOK_OPEN_TIME',
     SessionIdleTimeout = 'DATASCIENCE.JUPYTER_IDLE_TIMEOUT',
+    ConnectedToNotebook = 'DATASCIENCE.JUPYTER.CONNECTED',
     JupyterNotInstalledErrorShown = 'DATASCIENCE.JUPYTER_NOT_INSTALLED_ERROR_SHOWN',
     UserInstalledJupyter = 'DATASCIENCE.USER_INSTALLED_JUPYTER',
     UserDidNotInstallJupyter = 'DATASCIENCE.USER_DID_NOT_INSTALL_JUPYTER'

src/client/datascience/editor-integration/codeLensFactory.ts

@@ -9,7 +9,7 @@ import { IFileSystem } from '../../common/platform/types';
 import { IConfigurationService } from '../../common/types';
 import * as localize from '../../common/utils/localize';
 import { noop } from '../../common/utils/misc';
-import { generateCellRanges } from '../cellFactory';
+import { generateCellRangesFromDocument } from '../cellFactory';
 import { CodeLensCommands, Commands } from '../constants';
 import { InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes';
 import { ICell, ICellHashProvider, ICodeLensFactory, IFileHashes, IInteractiveWindowListener } from '../types';
@@ -63,7 +63,7 @@ export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowList
     }
 
     public createCodeLenses(document: TextDocument): CodeLens[] {
-        const ranges = generateCellRanges(document, this.configService.getSettings().datascience);
+        const ranges = generateCellRangesFromDocument(document, this.configService.getSettings().datascience);
         const commands = this.enumerateCommands();
         const hashes = this.configService.getSettings().datascience.addGotoCodeLenses ? this.hashProvider.getHashes() : [];
         const codeLenses: CodeLens[] = [];

src/client/datascience/editor-integration/decorator.ts

@@ -8,7 +8,7 @@ import { IExtensionSingleActivationService } from '../../activation/types';
 import { IDocumentManager } from '../../common/application/types';
 import { PYTHON_LANGUAGE } from '../../common/constants';
 import { IConfigurationService, IDisposable, IDisposableRegistry } from '../../common/types';
-import { generateCellRanges } from '../cellFactory';
+import { generateCellRangesFromDocument } from '../cellFactory';
 
 @injectable()
 export class Decorator implements IExtensionSingleActivationService, IDisposable {
@@ -100,7 +100,7 @@ export class Decorator implements IExtensionSingleActivationService, IDisposable
             const settings = this.configuration.getSettings().datascience;
             if (settings.decorateCells && settings.enabled) {
                 // Find all of the cells
-                const cells = generateCellRanges(editor.document, this.configuration.getSettings().datascience);
+                const cells = generateCellRangesFromDocument(editor.document, this.configuration.getSettings().datascience);
 
                 // Find the range for our active cell.
                 const currentRange = cells.map(c => c.range).filter(r => r.contains(editor.selection.anchor));

src/client/datascience/gather/gather.ts

@@ -1,7 +1,5 @@
-import { DataflowAnalyzer } from '@msrvida/python-program-analysis';
-import { Cell as ICell, LogCell } from '@msrvida/python-program-analysis/dist/es5/cell';
-import { CellSlice } from '@msrvida/python-program-analysis/dist/es5/cellslice';
-import { ExecutionLogSlicer } from '@msrvida/python-program-analysis/dist/es5/log-slicer';
+import { CellSlice, DataflowAnalyzer, ExecutionLogSlicer } from '@msrvida/python-program-analysis';
+import { Cell as IGatherCell } from '@msrvida/python-program-analysis/dist/es5/cell';
 
 import { inject, injectable } from 'inversify';
 // tslint:disable-next-line: no-require-imports
@@ -63,33 +61,36 @@ export class GatherExecution implements IGatherExecution, INotebookExecutionLogg
                 cloneCell.data.source = cellMatcher.stripFirstMarker(concatMultilineStringInput(vscCell.data.source));
 
                 // Convert IVscCell to IGatherCell
-                const cell = convertVscToGatherCell(cloneCell) as LogCell;
+                const cell = convertVscToGatherCell(cloneCell) as IGatherCell;
 
                 // Call internal logging method
                 this._executionSlicer.logExecution(cell);
             }
         }
     }
 
+    public async resetLog(): Promise<void> {
+        this._executionSlicer.reset();
+    }
+
     /**
      * For a given code cell, returns a string representing a program containing all the code it depends on.
      */
     public gatherCode(vscCell: IVscCell): string {
-        // sliceAllExecutions does a lookup based on executionEventId
-        const cell = convertVscToGatherCell(vscCell);
-        if (cell === undefined) {
+        const gatherCell = convertVscToGatherCell(vscCell);
+        if (!gatherCell) {
             return '';
         }
 
         // Get the default cell marker as we need to replace #%% with it.
         const defaultCellMarker = this.configService.getSettings().datascience.defaultCellMarker || Identifiers.DefaultCodeCellMarker;
 
         // Call internal slice method
-        const slices = this._executionSlicer.sliceAllExecutions(cell);
+        const slices = this._executionSlicer.sliceAllExecutions(gatherCell);
         const program = slices.length > 0 ? slices[0].cellSlices.reduce(concat, '').replace(/#%%/g, defaultCellMarker) : '';
 
         // Add a comment at the top of the file explaining what gather does
-        const descriptor = '# This file contains the minimal amount of code required to produce the code cell you gathered.\n';
+        const descriptor = localize.DataScience.gatheredScriptDescription();
         return descriptor.concat(program);
     }
 
@@ -122,7 +123,7 @@ export class GatherExecution implements IGatherExecution, INotebookExecutionLogg
 /**
  * Accumulator to concatenate cell slices for a sliced program, preserving cell structures.
  */
-function concat(existingText: string, newText: CellSlice) {
+function concat(existingText: string, newText: CellSlice): string {
     // Include our cell marker so that cell slices are preserved
     return `${existingText}#%%\n${newText.textSliceLines}\n\n`;
 }
@@ -131,14 +132,11 @@ function concat(existingText: string, newText: CellSlice) {
  * This is called to convert VS Code ICells to Gather ICells for logging.
  * @param cell A cell object conforming to the VS Code cell interface
  */
-function convertVscToGatherCell(cell: IVscCell): ICell | undefined {
+function convertVscToGatherCell(cell: IVscCell): IGatherCell | undefined {
     // This should always be true since we only want to log code cells. Putting this here so types match for outputs property
     if (cell.data.cell_type === 'code') {
-        const result: ICell = {
+        const result: IGatherCell = {
             // tslint:disable-next-line no-unnecessary-local-variable
-            id: cell.id,
-            gathered: false,
-            dirty: false,
             text: cell.data.source,
 
             // This may need to change for native notebook support since in the original Gather code this refers to the number of times that this same cell was executed
@@ -147,9 +145,7 @@ function convertVscToGatherCell(cell: IVscCell): ICell | undefined {
 
             // This may need to change for native notebook support, since this is intended to persist in the metadata for a notebook that is saved and then re-loaded
             persistentId: cell.id,
-            outputs: cell.data.outputs,
-            hasError: cell.state === CellState.error,
-            is_cell: true
+            hasError: cell.state === CellState.error
             // tslint:disable-next-line: no-any
         } as any;
         return result;

src/client/datascience/gather/gatherListener.ts

@@ -1,23 +1,16 @@
 import { inject, injectable } from 'inversify';
-import { Event, EventEmitter, Position, ViewColumn } from 'vscode';
-import { IApplicationShell, IDocumentManager } from '../../common/application/types';
-import { PYTHON_LANGUAGE } from '../../common/constants';
-import { IFileSystem } from '../../common/platform/types';
+import { Event, EventEmitter } from 'vscode';
 import { noop } from '../../common/utils/misc';
 import { InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes';
-import { ICell, IGatherExecution, IInteractiveWindowListener } from '../types';
+import { IGatherExecution, IInteractiveWindowListener, INotebook } from '../types';
 
 @injectable()
 export class GatherListener implements IInteractiveWindowListener {
     // tslint:disable-next-line: no-any
     private postEmitter: EventEmitter<{ message: string; payload: any }> = new EventEmitter<{ message: string; payload: any }>();
 
     constructor(
-        @inject(IDocumentManager) private documentManager: IDocumentManager,
-        @inject(IApplicationShell) private applicationShell: IApplicationShell,
-        @inject(IGatherExecution) private gatherExecution: IGatherExecution,
-        @inject(IFileSystem) private fileSystem: IFileSystem
-    ) { }
+        @inject(IGatherExecution) private gatherExecution: IGatherExecution) { }
 
     public dispose() {
         noop();
@@ -31,52 +24,15 @@ export class GatherListener implements IInteractiveWindowListener {
     // tslint:disable-next-line: no-any
     public onMessage(message: string, payload?: any): void {
         switch (message) {
-            case InteractiveWindowMessages.GatherCode:
-                if (payload) {
-                    const cell = payload as ICell;
-                    this.gatherCode(cell);
+            case InteractiveWindowMessages.ConnectedToNotebook:
+                const nb = payload as INotebook;
+                if (nb) {
+                    void nb.addGatherSupport(this.gatherExecution);
                 }
                 break;
 
             default:
                 break;
         }
     }
-
-    public gatherCode(payload: ICell) {
-        this.gatherCodeInternal(payload).catch(err => {
-            this.applicationShell.showErrorMessage(err);
-        });
-    }
-
-    private gatherCodeInternal = async (cell: ICell) => {
-        const slicedProgram = this.gatherExecution.gatherCode(cell);
-
-        // Don't want to open the gathered code on top of the interactive window
-        let viewColumn: ViewColumn | undefined;
-        const fileNameMatch = this.documentManager.visibleTextEditors.filter(textEditor => this.fileSystem.arePathsSame(textEditor.document.fileName, cell.file));
-        const definedVisibleEditors = this.documentManager.visibleTextEditors.filter(textEditor => textEditor.viewColumn !== undefined);
-        if (this.documentManager.visibleTextEditors.length > 0 && fileNameMatch.length > 0) {
-            // Original file is visible
-            viewColumn = fileNameMatch[0].viewColumn;
-        } else if (this.documentManager.visibleTextEditors.length > 0 && definedVisibleEditors.length > 0) {
-            // There is a visible text editor, just not the original file. Make sure viewColumn isn't undefined
-            viewColumn = definedVisibleEditors[0].viewColumn;
-        } else {
-            // Only one panel open and interactive window is occupying it, or original file is open but hidden
-            viewColumn = ViewColumn.Beside;
-        }
-
-        // Create a new open editor with the returned program in the right panel
-        const doc = await this.documentManager.openTextDocument({
-            content: slicedProgram,
-            language: PYTHON_LANGUAGE
-        });
-        const editor = await this.documentManager.showTextDocument(doc, viewColumn);
-
-        // Edit the document so that it is dirty (add a space at the end)
-        editor.edit((editBuilder) => {
-            editBuilder.insert(new Position(editor.document.lineCount, 0), '\n');
-        });
-    }
 }