Skip to content

Update gather functionality to support 0.4.1 of python-program-analysis #8219

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 46 commits into from
Nov 5, 2019
Merged
Show file tree
Hide file tree
Changes from 25 commits
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
704135a
Main changes for update
greazer Oct 25, 2019
d803bdb
Gather working (using 0.4.1 of python analysis)
greazer Oct 25, 2019
ed0ebd2
Prep for PR
greazer Oct 25, 2019
dfc4319
Implement gather for native editor (first pass)
greazer Oct 26, 2019
a807f70
Merge remote-tracking branch 'origin/master' into dev/jimgries/update…
greazer Oct 26, 2019
6b4e1ea
Fix cell rangification
greazer Oct 26, 2019
b827c1a
Cleanup
greazer Oct 26, 2019
30f1fcc
Fix error
greazer Oct 27, 2019
338361f
Support knowng cell has been exec'd
greazer Oct 28, 2019
af781c1
Bug fix and show gathered notebook
greazer Oct 28, 2019
4e2d305
Make strings localizable
greazer Oct 28, 2019
4a01d0a
executedInCurrentKernal => executeKernalId
greazer Oct 29, 2019
7b13a51
Add ConnectedToNotebook
greazer Oct 29, 2019
839a75d
Simplify logic to make use of cellVM
greazer Oct 29, 2019
0e1b8fc
Attempt to fix multi-nb gather (failed)
greazer Oct 30, 2019
a0094e5
Support separate notebook gathers
greazer Oct 31, 2019
03df1b0
Merge master
greazer Oct 31, 2019
b58fe0e
PR adjustments
greazer Oct 31, 2019
c96acd6
More PR updates
greazer Oct 31, 2019
643e17c
More PR updates
greazer Oct 31, 2019
6a3ca5a
cleanup
greazer Oct 31, 2019
5039dee
Yet another PR update
greazer Oct 31, 2019
17001d6
Fixups
greazer Nov 1, 2019
9282b31
Default gather support to true
greazer Nov 1, 2019
2f0b7a7
Update
greazer Nov 1, 2019
92ac7f9
Gather cleaned up and working
greazer Nov 1, 2019
dbfacb3
String updates
greazer Nov 1, 2019
6cedfde
Update strings, make gatherToScript
greazer Nov 1, 2019
d9696d6
merge master
greazer Nov 1, 2019
b2ee0ca
Merge remote-tracking branch 'origin/master' into dev/jimgries/update…
greazer Nov 1, 2019
d11f679
fix compile errors (why don't I see these?)
greazer Nov 1, 2019
db2600f
Fix executionslicer
greazer Nov 2, 2019
d33ae55
Fix unit test
greazer Nov 2, 2019
ff1a43d
Fix use of gather
greazer Nov 2, 2019
0dc45b2
Fix most of the unit tests
greazer Nov 3, 2019
b380304
Disabling failing tests (for now)
greazer Nov 3, 2019
549d4d1
Fix remaining Unit Tests
greazer Nov 3, 2019
7bc674c
Fix functional tests
greazer Nov 4, 2019
c5eb676
Show file in gathered nb
greazer Nov 4, 2019
19fe23e
Add a note about gather being conservative.
greazer Nov 4, 2019
6d92142
Don't use the ICell.file property to get nb name
greazer Nov 4, 2019
0bfb438
Remove unecessary gather APIs
greazer Nov 4, 2019
361bd23
Put back old algorithm for generating code lenses
rchiodo Nov 4, 2019
27c79c2
Fixup type
rchiodo Nov 4, 2019
21eed9d
Gather doesn't work on markdown
rchiodo Nov 4, 2019
38aea3c
Turn off gather in markdown for interactive too
rchiodo Nov 4, 2019
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build/ci/postInstall.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ var constants_1 = require("../constants");
* The solution is to modify the type definition file after `npm install`.
*/
function fixJupyterLabDTSFiles() {
var relativePath = path.join('node_modules', '@jupyterlab', 'coreutils', 'lib', 'settingregistry.d.ts');
var relativePath = path.join('node_modules', '@jupyterlab', 'services', 'node_modules', '@jupyterlab', 'coreutils', 'lib', 'settingregistry.d.ts');
var filePath = path.join(constants_1.ExtensionRootDir, relativePath);
if (!fs.existsSync(filePath)) {
throw new Error("Type Definition file from JupyterLab not found '" + filePath + "' (pvsc post install script)");
Expand Down
511 changes: 58 additions & 453 deletions package-lock.json

Large diffs are not rendered by default.

7 changes: 4 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -1694,8 +1694,8 @@
},
"python.dataScience.enableGather": {
"type": "boolean",
"default": false,
"description": "Enable code gathering for single cells in the interactive window (experimental).",
"default": true,
"description": "Enable code gather. For a gathered cell, it and all the code cells it depends on will be exported to a new notebook.",
"scope": "resource"
},
"python.dataScience.codeLenses": {
Expand Down Expand Up @@ -2679,7 +2679,8 @@
"dependencies": {
"@blueprintjs/select": "3.10.0",
"@jupyterlab/services": "^3.2.1",
"@msrvida/python-program-analysis": "^0.2.6",
"@jupyterlab/coreutils": "^3.1.0",
"@msrvida/python-program-analysis": "^0.4.1",
"ansi-regex": "^4.1.0",
"arch": "^2.1.0",
"azure-storage": "^2.10.3",
Expand Down
4 changes: 3 additions & 1 deletion package.nls.json
Original file line number Diff line number Diff line change
Expand Up @@ -378,5 +378,7 @@
"DataScience.notebookNotFound" : "python -m jupyter notebook --version is not running",
"DataScience.findJupyterCommandProgress" : "Active interpreter does not support {0}. Searching for the best available interpreter.",
"DataScience.findJupyterCommandProgressCheckInterpreter": "Checking {0}.",
"DataScience.findJupyterCommandProgressSearchCurrentPath": "Searching current path."
"DataScience.findJupyterCommandProgressSearchCurrentPath": "Searching current path.",
"DataScience.gatheredScriptDescription": "# This file contains the minimal amount of code required to produce the code cell gathered.\n",
"DataScience.gatheredNotebookDescriptionInMarkdown": "## This notebook was generated for a gathered cell."
}
2 changes: 2 additions & 0 deletions src/client/common/utils/localize.ts
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,8 @@ export namespace DataScience {
export const findJupyterCommandProgress = localize('DataScience.findJupyterCommandProgress', 'Active interpreter does not support {0}. Searching for the best available interpreter.');
export const findJupyterCommandProgressCheckInterpreter = localize('DataScience.findJupyterCommandProgressCheckInterpreter', 'Checking {0}.');
export const findJupyterCommandProgressSearchCurrentPath = localize('DataScience.findJupyterCommandProgressSearchCurrentPath', 'Searching current path.');
export const gatheredScriptDescription = localize('DataScience.gatheredScriptDescription', '# This file contains the minimal amount of code required to produce the code cell gathered.\n');
export const gatheredNotebookDescriptionInMarkdown = localize('DataScience.gatheredNotebookDescriptionInMarkdown', '## This notebook was generated for a gathered cell.');
}

export namespace DebugConfigStrings {
Expand Down
57 changes: 33 additions & 24 deletions src/client/datascience/cellFactory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,45 +107,54 @@ export function hasCells(document: TextDocument, settings?: IDataScienceSettings
return false;
}

export function generateCellRanges(document: TextDocument, settings?: IDataScienceSettings): { range: Range; title: string; cell_type: string }[] {
// Implmentation of getCells here based on Don's Jupyter extension work
function generateCellRangesFromStringArray(lines: string[], settings?: IDataScienceSettings): { range: Range; title: string; cell_type: string }[] {
const matcher = new CellMatcher(settings);
const cells : { range: Range; title: string; cell_type: string }[] = [];
for (let index = 0; index < document.lineCount; index += 1) {
const line = document.lineAt(index);
if (matcher.isCell(line.text)) {
if (cells.length > 0) {
const previousCell = cells[cells.length - 1];
previousCell.range = new Range(previousCell.range.start, document.lineAt(index - 1).range.end);
const cells: { range: Range; title: string; cell_type: string }[] = [];

for (let index = 0; index < lines.length; index += 1) {
if (matcher.isCell(lines[index])) {

// We have a cell, find the next cell
let endCellIndex = index + 1;
while (endCellIndex < lines.length && !matcher.isCell(lines[endCellIndex])) {
endCellIndex += 1;
}
endCellIndex -= 1; // Get last line of cell.

const results = matcher.exec(line.text);
if (results !== undefined) {
const result = matcher.exec(lines[index]);
if (result !== undefined) {
cells.push({
range: line.range,
title: results,
cell_type: matcher.getCellType(line.text)
range: new Range(index + 1, 0, endCellIndex, lines[endCellIndex].length),
title: result,
cell_type: matcher.getCellType(lines[index])
});
}
index = endCellIndex;
}
}

if (cells.length >= 1) {
const line = document.lineAt(document.lineCount - 1);
const previousCell = cells[cells.length - 1];
previousCell.range = new Range(previousCell.range.start, line.range.end);
}

return cells;
}

export function generateCellsFromDocument(document: TextDocument, settings?: IDataScienceSettings): ICell[] {
export function generateCellsFromString(source: string, settings?: IDataScienceSettings): ICell[] {
const lines: string[] = source.splitLines({ trim: false, removeEmptyEntries: false });

// Get our ranges. They'll determine our cells
const ranges = generateCellRanges(document, settings);
const ranges = generateCellRangesFromStringArray(lines, settings);

// For each one, get its text and turn it into a cell
return Array.prototype.concat(...ranges.map(r => {
const code = document.getText(r.range);
return generateCells(settings, code, document.fileName, r.range.start.line, false, uuid());
const code = lines.slice(r.range.start.line, r.range.end.line).join('\n');
return generateCells(settings, code, '', r.range.start.line, false, uuid());
}));
}

export function generateCellRangesFromDocument(document: TextDocument, settings?: IDataScienceSettings): { range: Range; title: string; cell_type: string }[] {
const lines: string[] = document.getText().splitLines({ trim: false, removeEmptyEntries: false });

return generateCellRangesFromStringArray(lines, settings);
}

export function generateCellsFromDocument(document: TextDocument, settings?: IDataScienceSettings): ICell[] {
return generateCellsFromString(document.getText(), settings);
}
1 change: 1 addition & 0 deletions src/client/datascience/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@ export enum Telemetry {
NotebookOpenCount = 'DATASCIENCE.NATIVE.NOTEBOOK_OPEN_COUNT',
NotebookOpenTime = 'DS_INTERNAL.NATIVE.NOTEBOOK_OPEN_TIME',
SessionIdleTimeout = 'DATASCIENCE.JUPYTER_IDLE_TIMEOUT',
ConnectedToNotebook = 'DATASCIENCE.JUPYTER.CONNECTED',
JupyterNotInstalledErrorShown = 'DATASCIENCE.JUPYTER_NOT_INSTALLED_ERROR_SHOWN',
UserInstalledJupyter = 'DATASCIENCE.USER_INSTALLED_JUPYTER',
UserDidNotInstallJupyter = 'DATASCIENCE.USER_DID_NOT_INSTALL_JUPYTER'
Expand Down
4 changes: 2 additions & 2 deletions src/client/datascience/editor-integration/codeLensFactory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { IFileSystem } from '../../common/platform/types';
import { IConfigurationService } from '../../common/types';
import * as localize from '../../common/utils/localize';
import { noop } from '../../common/utils/misc';
import { generateCellRanges } from '../cellFactory';
import { generateCellRangesFromDocument } from '../cellFactory';
import { CodeLensCommands, Commands } from '../constants';
import { InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes';
import { ICell, ICellHashProvider, ICodeLensFactory, IFileHashes, IInteractiveWindowListener } from '../types';
Expand Down Expand Up @@ -63,7 +63,7 @@ export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowList
}

public createCodeLenses(document: TextDocument): CodeLens[] {
const ranges = generateCellRanges(document, this.configService.getSettings().datascience);
const ranges = generateCellRangesFromDocument(document, this.configService.getSettings().datascience);
const commands = this.enumerateCommands();
const hashes = this.configService.getSettings().datascience.addGotoCodeLenses ? this.hashProvider.getHashes() : [];
const codeLenses: CodeLens[] = [];
Expand Down
4 changes: 2 additions & 2 deletions src/client/datascience/editor-integration/decorator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { IExtensionSingleActivationService } from '../../activation/types';
import { IDocumentManager } from '../../common/application/types';
import { PYTHON_LANGUAGE } from '../../common/constants';
import { IConfigurationService, IDisposable, IDisposableRegistry } from '../../common/types';
import { generateCellRanges } from '../cellFactory';
import { generateCellRangesFromDocument } from '../cellFactory';

@injectable()
export class Decorator implements IExtensionSingleActivationService, IDisposable {
Expand Down Expand Up @@ -100,7 +100,7 @@ export class Decorator implements IExtensionSingleActivationService, IDisposable
const settings = this.configuration.getSettings().datascience;
if (settings.decorateCells && settings.enabled) {
// Find all of the cells
const cells = generateCellRanges(editor.document, this.configuration.getSettings().datascience);
const cells = generateCellRangesFromDocument(editor.document, this.configuration.getSettings().datascience);

// Find the range for our active cell.
const currentRange = cells.map(c => c.range).filter(r => r.contains(editor.selection.anchor));
Expand Down
34 changes: 15 additions & 19 deletions src/client/datascience/gather/gather.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
import { DataflowAnalyzer } from '@msrvida/python-program-analysis';
import { Cell as ICell, LogCell } from '@msrvida/python-program-analysis/dist/es5/cell';
import { CellSlice } from '@msrvida/python-program-analysis/dist/es5/cellslice';
import { ExecutionLogSlicer } from '@msrvida/python-program-analysis/dist/es5/log-slicer';
import { CellSlice, DataflowAnalyzer, ExecutionLogSlicer } from '@msrvida/python-program-analysis';
import { Cell as IGatherCell } from '@msrvida/python-program-analysis/dist/es5/cell';

import { inject, injectable } from 'inversify';
// tslint:disable-next-line: no-require-imports
Expand Down Expand Up @@ -63,33 +61,36 @@ export class GatherExecution implements IGatherExecution, INotebookExecutionLogg
cloneCell.data.source = cellMatcher.stripFirstMarker(concatMultilineStringInput(vscCell.data.source));

// Convert IVscCell to IGatherCell
const cell = convertVscToGatherCell(cloneCell) as LogCell;
const cell = convertVscToGatherCell(cloneCell) as IGatherCell;

// Call internal logging method
this._executionSlicer.logExecution(cell);
}
}
}

public async resetLog(): Promise<void> {
this._executionSlicer.reset();
}

/**
* For a given code cell, returns a string representing a program containing all the code it depends on.
*/
public gatherCode(vscCell: IVscCell): string {
// sliceAllExecutions does a lookup based on executionEventId
const cell = convertVscToGatherCell(vscCell);
if (cell === undefined) {
const gatherCell = convertVscToGatherCell(vscCell);
if (!gatherCell) {
return '';
}

// Get the default cell marker as we need to replace #%% with it.
const defaultCellMarker = this.configService.getSettings().datascience.defaultCellMarker || Identifiers.DefaultCodeCellMarker;

// Call internal slice method
const slices = this._executionSlicer.sliceAllExecutions(cell);
const slices = this._executionSlicer.sliceAllExecutions(gatherCell);
const program = slices.length > 0 ? slices[0].cellSlices.reduce(concat, '').replace(/#%%/g, defaultCellMarker) : '';

// Add a comment at the top of the file explaining what gather does
const descriptor = '# This file contains the minimal amount of code required to produce the code cell you gathered.\n';
const descriptor = localize.DataScience.gatheredScriptDescription();
return descriptor.concat(program);
}

Expand Down Expand Up @@ -122,7 +123,7 @@ export class GatherExecution implements IGatherExecution, INotebookExecutionLogg
/**
* Accumulator to concatenate cell slices for a sliced program, preserving cell structures.
*/
function concat(existingText: string, newText: CellSlice) {
function concat(existingText: string, newText: CellSlice): string {
// Include our cell marker so that cell slices are preserved
return `${existingText}#%%\n${newText.textSliceLines}\n\n`;
}
Expand All @@ -131,14 +132,11 @@ function concat(existingText: string, newText: CellSlice) {
* This is called to convert VS Code ICells to Gather ICells for logging.
* @param cell A cell object conforming to the VS Code cell interface
*/
function convertVscToGatherCell(cell: IVscCell): ICell | undefined {
function convertVscToGatherCell(cell: IVscCell): IGatherCell | undefined {
// This should always be true since we only want to log code cells. Putting this here so types match for outputs property
if (cell.data.cell_type === 'code') {
const result: ICell = {
const result: IGatherCell = {
// tslint:disable-next-line no-unnecessary-local-variable
id: cell.id,
gathered: false,
dirty: false,
text: cell.data.source,

// This may need to change for native notebook support since in the original Gather code this refers to the number of times that this same cell was executed
Expand All @@ -147,9 +145,7 @@ function convertVscToGatherCell(cell: IVscCell): ICell | undefined {

// This may need to change for native notebook support, since this is intended to persist in the metadata for a notebook that is saved and then re-loaded
persistentId: cell.id,
outputs: cell.data.outputs,
hasError: cell.state === CellState.error,
is_cell: true
hasError: cell.state === CellState.error
// tslint:disable-next-line: no-any
} as any;
return result;
Expand Down
58 changes: 7 additions & 51 deletions src/client/datascience/gather/gatherListener.ts
Original file line number Diff line number Diff line change
@@ -1,23 +1,16 @@
import { inject, injectable } from 'inversify';
import { Event, EventEmitter, Position, ViewColumn } from 'vscode';
import { IApplicationShell, IDocumentManager } from '../../common/application/types';
import { PYTHON_LANGUAGE } from '../../common/constants';
import { IFileSystem } from '../../common/platform/types';
import { Event, EventEmitter } from 'vscode';
import { noop } from '../../common/utils/misc';
import { InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes';
import { ICell, IGatherExecution, IInteractiveWindowListener } from '../types';
import { IGatherExecution, IInteractiveWindowListener, INotebook } from '../types';

@injectable()
export class GatherListener implements IInteractiveWindowListener {
// tslint:disable-next-line: no-any
private postEmitter: EventEmitter<{ message: string; payload: any }> = new EventEmitter<{ message: string; payload: any }>();

constructor(
@inject(IDocumentManager) private documentManager: IDocumentManager,
@inject(IApplicationShell) private applicationShell: IApplicationShell,
@inject(IGatherExecution) private gatherExecution: IGatherExecution,
@inject(IFileSystem) private fileSystem: IFileSystem
) { }
@inject(IGatherExecution) private gatherExecution: IGatherExecution) { }

public dispose() {
noop();
Expand All @@ -31,52 +24,15 @@ export class GatherListener implements IInteractiveWindowListener {
// tslint:disable-next-line: no-any
public onMessage(message: string, payload?: any): void {
switch (message) {
case InteractiveWindowMessages.GatherCode:
if (payload) {
const cell = payload as ICell;
this.gatherCode(cell);
case InteractiveWindowMessages.ConnectedToNotebook:
const nb = payload as INotebook;
if (nb) {
void nb.addGatherSupport(this.gatherExecution);
}
break;

default:
break;
}
}

public gatherCode(payload: ICell) {
this.gatherCodeInternal(payload).catch(err => {
this.applicationShell.showErrorMessage(err);
});
}

private gatherCodeInternal = async (cell: ICell) => {
const slicedProgram = this.gatherExecution.gatherCode(cell);

// Don't want to open the gathered code on top of the interactive window
let viewColumn: ViewColumn | undefined;
const fileNameMatch = this.documentManager.visibleTextEditors.filter(textEditor => this.fileSystem.arePathsSame(textEditor.document.fileName, cell.file));
const definedVisibleEditors = this.documentManager.visibleTextEditors.filter(textEditor => textEditor.viewColumn !== undefined);
if (this.documentManager.visibleTextEditors.length > 0 && fileNameMatch.length > 0) {
// Original file is visible
viewColumn = fileNameMatch[0].viewColumn;
} else if (this.documentManager.visibleTextEditors.length > 0 && definedVisibleEditors.length > 0) {
// There is a visible text editor, just not the original file. Make sure viewColumn isn't undefined
viewColumn = definedVisibleEditors[0].viewColumn;
} else {
// Only one panel open and interactive window is occupying it, or original file is open but hidden
viewColumn = ViewColumn.Beside;
}

// Create a new open editor with the returned program in the right panel
const doc = await this.documentManager.openTextDocument({
content: slicedProgram,
language: PYTHON_LANGUAGE
});
const editor = await this.documentManager.showTextDocument(doc, viewColumn);

// Edit the document so that it is dirty (add a space at the end)
editor.edit((editBuilder) => {
editBuilder.insert(new Position(editor.document.lineCount, 0), '\n');
});
}
}
Loading