Fix indentation on single-liners and f-strings in type formatting (#1312)

* Undo changes

* Test fixes

* Increase timeout

* Remove double event listening

* Remove test

* Revert "Remove test"

This reverts commit e240c3f.

* Revert "Remove double event listening"

This reverts commit af573be.

* #1096 The if statement is automatically formatted incorrectly

* Merge fix

* Add more tests

* More tests

* Typo

* Test

* Also better handle multiline arguments

* Add a couple missing periods

[skip ci]

* Undo changes

* Test fixes

* Increase timeout

* Remove double event listening

* Remove test

* Revert "Remove test"

This reverts commit e240c3f.

* Revert "Remove double event listening"

This reverts commit af573be.

* Merge fix

* #1257 On type formatting errors for args and kwargs

* Handle f-strings

* Stop importing from test code

* #1308 Single line statements leading to an indentation on the next line

* #726 editing python after inline if statement invalid indent

* Undo change

* Move constant

Author: Mikhail Arkhipov

src/client/common/constants.ts

@@ -69,5 +69,8 @@ export function isTestExecution(): boolean {
     // tslint:disable-next-line:interface-name no-string-literal
     return process.env['VSC_PYTHON_CI_TEST'] === '1';
 }
+export function isPythonAnalysisEngineTest(): boolean {
+    return process.env.VSC_PYTHON_ANALYSIS === '1';
+}
 
 export const EXTENSION_ROOT_DIR = path.join(__dirname, '..', '..', '..');

src/client/extension.ts

@@ -11,12 +11,11 @@ import {
     extensions, IndentAction, languages, Memento,
     OutputChannel, window
 } from 'vscode';
-import { IS_ANALYSIS_ENGINE_TEST } from '../test/constants';
 import { AnalysisExtensionActivator } from './activation/analysis';
 import { ClassicExtensionActivator } from './activation/classic';
 import { IExtensionActivator } from './activation/types';
 import { PythonSettings } from './common/configSettings';
-import { STANDARD_OUTPUT_CHANNEL } from './common/constants';
+import { isPythonAnalysisEngineTest, STANDARD_OUTPUT_CHANNEL } from './common/constants';
 import { FeatureDeprecationManager } from './common/featureDeprecationManager';
 import { createDeferred } from './common/helpers';
 import { PythonInstaller } from './common/installer/pythonInstallation';
@@ -75,7 +74,7 @@ export async function activate(context: ExtensionContext) {
     const configuration = serviceManager.get<IConfigurationService>(IConfigurationService);
     const pythonSettings = configuration.getSettings();
 
-    const activator: IExtensionActivator = IS_ANALYSIS_ENGINE_TEST || !pythonSettings.jediEnabled
+    const activator: IExtensionActivator = isPythonAnalysisEngineTest() || !pythonSettings.jediEnabled
         ? new AnalysisExtensionActivator(serviceManager, pythonSettings)
         : new ClassicExtensionActivator(serviceManager, pythonSettings);
 
@@ -108,7 +107,11 @@ export async function activate(context: ExtensionContext) {
     languages.setLanguageConfiguration(PYTHON.language!, {
         onEnterRules: [
             {
-                beforeText: /^\s*(?:def|class|for|if|elif|else|while|try|with|finally|except|async)\b.*/,
+                beforeText: /^\s*(?:def|class|for|if|elif|else|while|try|with|finally|except)\b.*:\s*\S+/,
+                action: { indentAction: IndentAction.None }
+            },
+            {
+                beforeText: /^\s*(?:def|class|for|if|elif|else|while|try|with|finally|except|async)\b.*:\s*/,
                 action: { indentAction: IndentAction.Indent }
             },
             {

src/client/formatters/lineFormatter.ts

@@ -95,15 +95,22 @@ export class LineFormatter {
                     }
                     break;
                 case Char.Period:
-                    this.builder.append('.');
-                    return;
                 case Char.At:
-                    this.builder.append('@');
+                case Char.ExclamationMark:
+                    this.builder.append(this.text[t.start]);
                     return;
                 default:
                     break;
             }
         }
+        // Do not append space if operator is preceded by '(' or ',' as in foo(**kwarg)
+        if (index > 0) {
+            const prev = this.tokens.getItemAt(index - 1);
+            if (this.isOpenBraceType(prev.type) || prev.type === TokenType.Comma) {
+                this.builder.append(this.text.substring(t.start, t.end));
+                return;
+            }
+        }
         this.builder.softAppendSpace();
         this.builder.append(this.text.substring(t.start, t.end));
         this.builder.softAppendSpace();

src/client/language/tokenizer.ts

@@ -85,10 +85,16 @@ export class Tokenizer implements ITokenizer {
         }
     }
 
+    // tslint:disable-next-line:cyclomatic-complexity
     private handleCharacter(): boolean {
+        // f-strings
+        const fString = this.cs.currentChar === Char.f && (this.cs.nextChar === Char.SingleQuote || this.cs.nextChar === Char.DoubleQuote);
+        if (fString) {
+            this.cs.moveNext();
+        }
         const quoteType = this.getQuoteType();
         if (quoteType !== QuoteType.None) {
-            this.handleString(quoteType);
+            this.handleString(quoteType, fString);
             return true;
         }
         if (this.cs.currentChar === Char.Hash) {
@@ -342,8 +348,8 @@ export class Tokenizer implements ITokenizer {
         return QuoteType.None;
     }
 
-    private handleString(quoteType: QuoteType): void {
-        const start = this.cs.position;
+    private handleString(quoteType: QuoteType, fString: boolean): void {
+        const start = fString ? this.cs.position - 1 : this.cs.position;
         if (quoteType === QuoteType.Single || quoteType === QuoteType.Double) {
             this.cs.moveNext();
             this.skipToSingleEndQuote(quoteType === QuoteType.Single

src/test/format/extension.lineFormatter.test.ts

@@ -73,12 +73,16 @@ suite('Formatting - line formatter', () => {
         const actual = formatter.formatLine('foo(x,y= \"a\",');
         assert.equal(actual, 'foo(x, y=\"a\",');
     });
-   test('Equals in multiline arguments', () => {
+    test('Equals in multiline arguments', () => {
         const actual = formatter.formatLine('x = 1,y =-2)');
         assert.equal(actual, 'x=1, y=-2)');
     });
     test('Equals in multiline arguments starting comma', () => {
         const actual = formatter.formatLine(',x = 1,y =m)');
         assert.equal(actual, ', x=1, y=m)');
     });
+    test('Operators without following space', () => {
+        const actual = formatter.formatLine('foo( *a, ** b, ! c)');
+        assert.equal(actual, 'foo(*a, **b, !c)');
+    });
 });

src/test/language/tokenizer.test.ts

@@ -79,6 +79,43 @@ suite('Language.Tokenizer', () => {
         assert.equal(tokens.getItemAt(0).type, TokenType.String);
         assert.equal(tokens.getItemAt(0).length, 12);
     });
+    test('Strings: single quoted f-string ', async () => {
+        const t = new Tokenizer();
+        // tslint:disable-next-line:quotemark
+        const tokens = t.tokenize("a+f'quoted'");
+        assert.equal(tokens.count, 3);
+        assert.equal(tokens.getItemAt(0).type, TokenType.Identifier);
+        assert.equal(tokens.getItemAt(1).type, TokenType.Operator);
+        assert.equal(tokens.getItemAt(2).type, TokenType.String);
+        assert.equal(tokens.getItemAt(2).length, 9);
+    });
+    test('Strings: double quoted f-string ', async () => {
+        const t = new Tokenizer();
+        const tokens = t.tokenize('x(1,f"quoted")');
+        assert.equal(tokens.count, 6);
+        assert.equal(tokens.getItemAt(0).type, TokenType.Identifier);
+        assert.equal(tokens.getItemAt(1).type, TokenType.OpenBrace);
+        assert.equal(tokens.getItemAt(2).type, TokenType.Number);
+        assert.equal(tokens.getItemAt(3).type, TokenType.Comma);
+        assert.equal(tokens.getItemAt(4).type, TokenType.String);
+        assert.equal(tokens.getItemAt(4).length, 9);
+        assert.equal(tokens.getItemAt(5).type, TokenType.CloseBrace);
+    });
+    test('Strings: single quoted multiline f-string ', async () => {
+        const t = new Tokenizer();
+        // tslint:disable-next-line:quotemark
+        const tokens = t.tokenize("f'''quoted'''");
+        assert.equal(tokens.count, 1);
+        assert.equal(tokens.getItemAt(0).type, TokenType.String);
+        assert.equal(tokens.getItemAt(0).length, 13);
+    });
+    test('Strings: double quoted multiline f-string ', async () => {
+        const t = new Tokenizer();
+        const tokens = t.tokenize('f"""quoted """');
+        assert.equal(tokens.count, 1);
+        assert.equal(tokens.getItemAt(0).type, TokenType.String);
+        assert.equal(tokens.getItemAt(0).length, 14);
+    });
     test('Comments', async () => {
         const t = new Tokenizer();
         const tokens = t.tokenize(' #co"""mment1\n\t\n#comm\'ent2 ');