Add blank lines to separate blocks of indented code (#1515)

Fixes #259

Author: DonJayamanne

news/2 Fixes/259.md

@@ -0,0 +1 @@
+Add blank lines to separate blocks of indented code (function defs, classes, and the like) so as to ensure the code can be run within a Python interactive prompt.

pythonFiles/normalizeForInterpreter.py

@@ -0,0 +1,116 @@
+import ast
+import io
+import operator
+import os
+import sys
+import token
+import tokenize
+
+
+class Visitor(ast.NodeVisitor):
+    def __init__(self, lines):
+        self._lines = lines
+        self.line_numbers_with_nodes = set()
+        self.line_numbers_with_statements = []
+
+    def generic_visit(self, node):
+        if hasattr(node, 'col_offset') and hasattr(node, 'lineno') and node.col_offset == 0:
+            self.line_numbers_with_nodes.add(node.lineno)
+            if isinstance(node, ast.stmt):
+                self.line_numbers_with_statements.append(node.lineno)
+
+        ast.NodeVisitor.generic_visit(self, node)
+
+
+def _tokenize(source):
+    """Tokenize Python source code."""
+    # Using an undocumented API as the documented one in Python 2.7 does not work as needed
+    # cross-version.
+    return tokenize.generate_tokens(io.StringIO(source).readline)
+
+
+def _indent_size(line):
+    for index, char in enumerate(line):
+        if not char.isspace():
+            return index
+
+
+def _get_global_statement_blocks(source, lines):
+    """Return a list of all global statement blocks.
+
+    The list comprises of 3-item tuples that contain the starting line number,
+    ending line number and whether the statement is a single line.
+
+    """
+    tree = ast.parse(source)
+    visitor = Visitor(lines)
+    visitor.visit(tree)
+
+    statement_ranges = []
+    for index, line_number in enumerate(visitor.line_numbers_with_statements):
+        remaining_line_numbers = visitor.line_numbers_with_statements[index+1:]
+        end_line_number = len(lines) if len(remaining_line_numbers) == 0 else min(remaining_line_numbers) - 1
+        current_statement_is_oneline = line_number == end_line_number
+
+        if len(statement_ranges) == 0:
+            statement_ranges.append((line_number, end_line_number, current_statement_is_oneline))
+            continue
+
+        previous_statement = statement_ranges[-1]
+        previous_statement_is_oneline = previous_statement[2]
+        if previous_statement_is_oneline and current_statement_is_oneline:
+            statement_ranges[-1] = previous_statement[0], end_line_number, True
+        else:
+            statement_ranges.append((line_number, end_line_number, current_statement_is_oneline))
+
+    return statement_ranges
+
+
+def normalize_lines(source):
+    """Normalize blank lines for sending to the terminal.
+
+    Blank lines within a statement block are removed to prevent the REPL
+    from thinking the block is finished. Newlines are added to separate
+    top-level statements so that the REPL does not think there is a syntax
+    error.
+
+    """
+    lines = source.splitlines(False)
+    # Find out if we have any trailing blank lines
+    has_blank_lines = len(lines[-1].strip()) == 0 or source.endswith(os.linesep)
+
+    # Step 1: Remove empty lines.
+    tokens = _tokenize(source)
+    newlines_indexes_to_remove = (spos[0] for (toknum, tokval, spos, epos, line) in tokens
+                                    if len(line.strip()) == 0 and token.tok_name[toknum] == 'NL' and spos[0] == epos[0])
+
+    for line_number in reversed(list(newlines_indexes_to_remove)):
+        del lines[line_number-1]
+
+    # Step 2: Add blank lines between each global statement block.
+    # A consequtive single lines blocks of code will be treated as a single statement,
+    # just to ensure we do not unnecessarily add too many blank lines.
+    source = os.linesep.join(lines)
+    tokens = _tokenize(source)
+    dedent_indexes = (spos[0] for (toknum, tokval, spos, epos, line) in tokens
+                                if toknum == token.DEDENT and _indent_size(line) == 0)
+
+    global_statement_ranges = _get_global_statement_blocks(source, lines)
+
+    for line_number in filter(lambda x: x > 1, map(operator.itemgetter(0), reversed(global_statement_ranges))):
+        lines.insert(line_number-1, '')
+
+    sys.stdout.write(os.linesep.join(lines) + (os.linesep if has_blank_lines else ''))
+    sys.stdout.flush()
+
+
+if __name__ == '__main__':
+    contents = sys.argv[1]
+    try:
+        default_encoding = sys.getdefaultencoding()
+        contents = contents.encode(default_encoding, 'surrogateescape').decode(default_encoding, 'replace')
+    except (UnicodeError, LookupError):
+        pass
+    if isinstance(contents, bytes):
+        contents = contents.decode('utf8')
+    normalize_lines(contents)

src/client/terminals/codeExecution/helper.ts

@@ -2,30 +2,41 @@
 // Licensed under the MIT License.
 
 import { inject, injectable } from 'inversify';
+import * as path from 'path';
 import { Range, TextEditor, Uri } from 'vscode';
 import { IApplicationShell, IDocumentManager } from '../../common/application/types';
-import { PYTHON_LANGUAGE } from '../../common/constants';
+import { EXTENSION_ROOT_DIR, PYTHON_LANGUAGE } from '../../common/constants';
 import '../../common/extensions';
+import { IProcessService } from '../../common/process/types';
+import { IConfigurationService } from '../../common/types';
+import { IEnvironmentVariablesProvider } from '../../common/variables/types';
 import { IServiceContainer } from '../../ioc/types';
 import { ICodeExecutionHelper } from '../types';
 
 @injectable()
 export class CodeExecutionHelper implements ICodeExecutionHelper {
     private readonly documentManager: IDocumentManager;
     private readonly applicationShell: IApplicationShell;
+    private readonly envVariablesProvider: IEnvironmentVariablesProvider;
+    private readonly processService: IProcessService;
+    private readonly configurationService: IConfigurationService;
     constructor(@inject(IServiceContainer) serviceContainer: IServiceContainer) {
         this.documentManager = serviceContainer.get<IDocumentManager>(IDocumentManager);
         this.applicationShell = serviceContainer.get<IApplicationShell>(IApplicationShell);
+        this.envVariablesProvider = serviceContainer.get<IEnvironmentVariablesProvider>(IEnvironmentVariablesProvider);
+        this.processService = serviceContainer.get<IProcessService>(IProcessService);
+        this.configurationService = serviceContainer.get<IConfigurationService>(IConfigurationService);
     }
     public async normalizeLines(code: string, resource?: Uri): Promise<string> {
         try {
             if (code.trim().length === 0) {
                 return '';
             }
-            const regex = /(\n)([ \t]*\r?\n)([ \t]+\S+)/gm;
-            return code.replace(regex, (_, a, b, c) => {
-                return `${a}${c}`;
-            });
+            const env = await this.envVariablesProvider.getEnvironmentVariables(resource);
+            const pythonPath = this.configurationService.getSettings(resource).pythonPath;
+            const args = [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'normalizeForInterpreter.py'), code];
+            const proc = await this.processService.exec(pythonPath, args, { env, throwOnStdErr: true });
+            return proc.stdout;
         } catch (ex) {
             console.error(ex, 'Python: Failed to normalize code for execution in terminal');
             return code;

src/test/pythonFiles/terminalExec/sample1_normalized.py

@@ -1,18 +1,21 @@
 # Sample block 1
+
 def square(x):
     return x**2
 
 print('hello')
 # Sample block 2
+
 a = 2
+
 if a < 2:
     print('less than 2')
 else:
     print('more than 2')
 
 print('hello')
-
 # Sample block 3
+
 for i in range(5):
     print(i)
     print(i)

src/test/pythonFiles/terminalExec/sample3_normalized.py

@@ -1,4 +1,5 @@
 if True:
     print(1)
     print(2)
+
 print(3)

src/test/pythonFiles/terminalExec/sample3_raw.py

@@ -2,4 +2,5 @@
     print(1)
 
     print(2)
+
 print(3)

src/test/pythonFiles/terminalExec/sample6_normalized.py

@@ -0,0 +1,15 @@
+if True:
+    print(1)
+else: print(2)
+
+print('🔨')
+print(3)
+print(3)
+
+if True:
+    print(1)
+else: print(2)
+
+if True:
+    print(1)
+else: print(2)

src/test/pythonFiles/terminalExec/sample6_raw.py

@@ -0,0 +1,12 @@
+if True:
+    print(1)
+else: print(2)
+print('🔨')
+print(3)
+print(3)
+if True:
+    print(1)
+else: print(2)
+if True:
+    print(1)
+else: print(2)

src/test/pythonFiles/terminalExec/sample7_normalized.py

@@ -0,0 +1,8 @@
+if True:
+    print(1)
+    print(1)
+else:
+    print(2)
+    print(2)
+
+print(3)

src/test/pythonFiles/terminalExec/sample7_raw.py

@@ -0,0 +1,9 @@
+if True:
+    print(1)
+
+    print(1)
+else:
+    print(2)
+
+    print(2)
+print(3)

src/test/pythonFiles/terminalExec/sample_normalized.py

@@ -0,0 +1,5 @@
+import sys
+print(sys.executable)
+print("1234")
+print(1)
+print(2)

src/test/pythonFiles/terminalExec/sample_raw.py

@@ -0,0 +1,8 @@
+import sys
+
+print(sys.executable)
+
+print("1234")
+
+print(1)
+print(2)

src/test/terminals/codeExecution/helper.test.ts

@@ -11,9 +11,15 @@ import * as TypeMoq from 'typemoq';
 import { Range, Selection, TextDocument, TextEditor, TextLine, Uri } from 'vscode';
 import { IApplicationShell, IDocumentManager } from '../../../client/common/application/types';
 import { EXTENSION_ROOT_DIR, PYTHON_LANGUAGE } from '../../../client/common/constants';
+import { BufferDecoder } from '../../../client/common/process/decoder';
+import { ProcessService } from '../../../client/common/process/proc';
+import { IProcessService } from '../../../client/common/process/types';
+import { IConfigurationService, IPythonSettings } from '../../../client/common/types';
+import { IEnvironmentVariablesProvider } from '../../../client/common/variables/types';
 import { IServiceContainer } from '../../../client/ioc/types';
 import { CodeExecutionHelper } from '../../../client/terminals/codeExecution/helper';
 import { ICodeExecutionHelper } from '../../../client/terminals/types';
+import { PYTHON_PATH } from '../../common';
 
 const TEST_FILES_PATH = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'terminalExec');
 
@@ -24,12 +30,24 @@ suite('Terminal - Code Execution Helper', () => {
     let helper: ICodeExecutionHelper;
     let document: TypeMoq.IMock<TextDocument>;
     let editor: TypeMoq.IMock<TextEditor>;
+    let processService: TypeMoq.IMock<IProcessService>;
+    let configService: TypeMoq.IMock<IConfigurationService>;
     setup(() => {
         const serviceContainer = TypeMoq.Mock.ofType<IServiceContainer>();
         documentManager = TypeMoq.Mock.ofType<IDocumentManager>();
         applicationShell = TypeMoq.Mock.ofType<IApplicationShell>();
+        const envVariablesProvider = TypeMoq.Mock.ofType<IEnvironmentVariablesProvider>();
+        processService = TypeMoq.Mock.ofType<IProcessService>();
+        configService = TypeMoq.Mock.ofType<IConfigurationService>();
+        const pythonSettings = TypeMoq.Mock.ofType<IPythonSettings>();
+        pythonSettings.setup(p => p.pythonPath).returns(() => PYTHON_PATH);
+        configService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(() => pythonSettings.object);
+        envVariablesProvider.setup(e => e.getEnvironmentVariables(TypeMoq.It.isAny())).returns(() => Promise.resolve({}));
         serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IDocumentManager), TypeMoq.It.isAny())).returns(() => documentManager.object);
         serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IApplicationShell), TypeMoq.It.isAny())).returns(() => applicationShell.object);
+        serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IEnvironmentVariablesProvider), TypeMoq.It.isAny())).returns(() => envVariablesProvider.object);
+        serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IProcessService), TypeMoq.It.isAny())).returns(() => processService.object);
+        serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IConfigurationService), TypeMoq.It.isAny())).returns(() => configService.object);
         helper = new CodeExecutionHelper(serviceContainer.object);
 
         document = TypeMoq.Mock.ofType<TextDocument>();
@@ -38,18 +56,23 @@ suite('Terminal - Code Execution Helper', () => {
     });
 
     async function ensureBlankLinesAreRemoved(source: string, expectedSource: string) {
+        const actualProcessService = new ProcessService(new BufferDecoder());
+        processService.setup(p => p.exec(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny()))
+            .returns((file, args, options) => {
+                return actualProcessService.exec.apply(actualProcessService, [file, args, options]);
+            });
         const normalizedZCode = await helper.normalizeLines(source);
         expect(normalizedZCode).to.be.equal(expectedSource);
     }
     test('Ensure blank lines are NOT removed when code is not indented (simple)', async () => {
-        const code = ['import sys', '', 'print(sys.executable)', '', 'print("1234")', '', 'print(1)', 'print(2)'];
-        const expectedCode = code.join(EOL);
+        const code = ['import sys', '', '', '', 'print(sys.executable)', '', 'print("1234")', '', '', 'print(1)', 'print(2)'];
+        const expectedCode = code.filter(line => line.trim().length > 0).join(EOL);
         await ensureBlankLinesAreRemoved(code.join(EOL), expectedCode);
     });
-    ['sample1', 'sample2', 'sample3', 'sample4', 'sample5'].forEach(fileName => {
-        test(`Ensure blank lines are removed (${fileName})`, async () => {
-            const code = await fs.readFile(path.join(TEST_FILES_PATH, `${fileName}_raw.py`), 'utf8');
-            const expectedCode = await fs.readFile(path.join(TEST_FILES_PATH, `${fileName}_normalized.py`), 'utf8');
+    ['', '1', '2', '3', '4', '5', '6', '7'].forEach(fileNameSuffix => {
+        test(`Ensure blank lines are removed (Sample${fileNameSuffix})`, async () => {
+            const code = await fs.readFile(path.join(TEST_FILES_PATH, `sample${fileNameSuffix}_raw.py`), 'utf8');
+            const expectedCode = await fs.readFile(path.join(TEST_FILES_PATH, `sample${fileNameSuffix}_normalized.py`), 'utf8');
             await ensureBlankLinesAreRemoved(code, expectedCode);
         });
         // test(`Ensure blank lines are removed, including leading empty lines (${fileName})`, async () => {
@@ -58,11 +81,6 @@ suite('Terminal - Code Execution Helper', () => {
         //     await ensureBlankLinesAreRemoved(['', '', ''].join(EOL) + EOL + code, expectedCode);
         // });
     });
-    test('Ensure blank lines are removed (sample2)', async () => {
-        const code = await fs.readFile(path.join(TEST_FILES_PATH, 'sample2_raw.py'), 'utf8');
-        const expectedCode = await fs.readFile(path.join(TEST_FILES_PATH, 'sample2_normalized.py'), 'utf8');
-        await ensureBlankLinesAreRemoved(code, expectedCode);
-    });
     test('Display message if there\s no active file', async () => {
         documentManager.setup(doc => doc.activeTextEditor).returns(() => undefined);
 

src/test/terminals/codeExecution/terminalCodeExec.test.ts

@@ -18,7 +18,7 @@ import { ICodeExecutionService } from '../../../client/terminals/types';
 import { PYTHON_PATH } from '../../common';
 
 // tslint:disable-next-line:max-func-body-length
-suite('Terminal Code Execution', () => {
+suite('Terminal - Code Execution', () => {
     // tslint:disable-next-line:max-func-body-length
     ['Terminal Execution', 'Repl Execution', 'Django Execution'].forEach(testSuiteName => {
         let terminalSettings: TypeMoq.IMock<ITerminalSettings>;