Parsing f-strings

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs

@@ -144,6 +144,12 @@ public IMember GetValueFromExpression(Expression expr, LookupOptions options) {
                 case LambdaExpression lambda:
                     m = GetValueFromLambda(lambda);
                     break;
+                case FString fString:
+                    m = GetValueFromFString(fString);
+                    break;
+                case FormatSpecifier formatSpecifier:
+                    m = GetValueFromFormatSpecifier(formatSpecifier);
+                    break;
                 default:
                     m = GetValueFromBinaryOp(expr) ?? GetConstantFromLiteral(expr, options);
                     break;
@@ -154,6 +160,14 @@ public IMember GetValueFromExpression(Expression expr, LookupOptions options) {
             return m;
         }
 
+        private IMember GetValueFromFormatSpecifier(FormatSpecifier formatSpecifier) {
+            return new PythonFString(formatSpecifier.Unparsed, Interpreter, GetLoc(formatSpecifier));
+        }
+
+        private IMember GetValueFromFString(FString fString) {
+            return new PythonFString(fString.Unparsed, Interpreter, GetLoc(fString));
+        }
+
         private IMember GetValueFromName(NameExpression expr, LookupOptions options) {
             if (expr == null || string.IsNullOrEmpty(expr.Name)) {
                 return null;

src/Analysis/Ast/Impl/Values/PythonString.cs

@@ -13,6 +13,7 @@
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
 
+using System;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Parsing;
 
@@ -32,4 +33,21 @@ internal sealed class PythonUnicodeString : PythonConstant {
         public PythonUnicodeString(string s, IPythonInterpreter interpreter, LocationInfo location = null)
             : base(s, interpreter.GetBuiltinType(interpreter.GetUnicodeTypeId()), location) { }
     }
+
+    internal sealed class PythonFString : PythonInstance, IEquatable<PythonFString> {
+        public readonly string UnparsedFString;
+
+        public PythonFString(string unparsedFString, IPythonInterpreter interpreter, LocationInfo location = null)
+            : base(interpreter.GetBuiltinType(interpreter.GetUnicodeTypeId()), location) {
+            UnparsedFString = unparsedFString;
+        }
+
+
+        public bool Equals(PythonFString other) {
+            if (!base.Equals(other)) {
+                return false;
+            }
+            return UnparsedFString?.Equals(other?.UnparsedFString) == true;
+        }
+    }
 }

src/Analysis/Ast/Test/TypingTests.cs

@@ -225,6 +225,15 @@ from typing import AnyStr
                 .And.HaveVariable("y").OfType("AnyStr");
         }
 
+        [TestMethod, Priority(0)]
+        public async Task FStringIsStringType() {
+            const string code = @"
+x = f'{1}'
+";
+            var analysis = await GetAnalysisAsync(code);
+            analysis.Should().HaveVariable("x").OfType(BuiltinTypeId.Str);
+        }
+
         [TestMethod, Priority(0)]
         public async Task OptionalNone() {
             const string code = @"

src/LanguageServer/Impl/Sources/HoverSource.cs

@@ -40,7 +40,8 @@ public Hover GetHover(IDocumentAnalysis analysis, SourceLocation location) {
             ExpressionLocator.FindExpression(analysis.Ast, location,
                 FindExpressionOptions.Hover, out var node, out var statement, out var scope);
 
-            if (node is ConstantExpression || !(node is Expression expr)) {
+            if (node is ConstantExpression || node is FString || !(node is Expression expr)) {
+                // node is FString only if it didn't save an f-string subexpression
                 return null; // No hover for literals.
             }
 

src/LanguageServer/Test/CompletionTests.cs

@@ -864,6 +864,17 @@ public async Task NoCompletionInOpenString() {
             result.Should().HaveNoCompletion();
         }
 
+        [DataRow("f'.")]
+        [DataRow("f'a.")]
+        [DataRow("f'a.'")]
+        [DataTestMethod, Priority(0)]
+        public async Task NoCompletionInFStringConstant(string openFString) {
+            var analysis = await GetAnalysisAsync(openFString);
+            var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion);
+            var result = cs.GetCompletions(analysis, new SourceLocation(1, 5));
+            result.Should().HaveNoCompletion();
+        }
+
         [TestMethod, Priority(0)]
         public async Task NoCompletionBadImportExpression() {
             var analysis = await GetAnalysisAsync("import os,.");

src/LanguageServer/Test/HoverTests.cs

@@ -206,6 +206,24 @@ import os.path as PATH
             AssertHover(hs, analysis, new SourceLocation(2, 20), $"module {name}*", new SourceSpan(2, 19, 2, 23));
         }
 
+        [TestMethod, Priority(0)]
+        public async Task FStringExpressions() {
+            const string code = @"
+some = ''
+f'{some'
+Fr'{some'
+f'{some}'
+f'hey {some}'
+";
+            var analysis = await GetAnalysisAsync(code);
+            var hs = new HoverSource(new PlainTextDocumentationSource());
+            AssertHover(hs, analysis, new SourceLocation(3, 4), @"some: str", new SourceSpan(3, 4, 3, 8));
+            AssertHover(hs, analysis, new SourceLocation(4, 5), @"some: str", new SourceSpan(4, 5, 4, 9));
+            AssertHover(hs, analysis, new SourceLocation(5, 4), @"some: str", new SourceSpan(5, 4, 5, 8));
+            hs.GetHover(analysis, new SourceLocation(6, 3)).Should().BeNull();
+            AssertHover(hs, analysis, new SourceLocation(6, 8), @"some: str", new SourceSpan(6, 8, 6, 12));
+        }
+
         private static void AssertHover(HoverSource hs, IDocumentAnalysis analysis, SourceLocation position, string hoverText, SourceSpan? span = null) {
             var hover = hs.GetHover(analysis, position);
 

src/Parsing/Impl/Ast/FString.cs

@@ -0,0 +1,84 @@
+using System.Collections.Generic;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.Python.Parsing.Ast {
+    public class FString : Expression {
+        private readonly Node[] _children;
+        private readonly string _openQuotes;
+
+        public FString(Node[] children, string openQuotes, string unparsed) {
+            _children = children;
+            _openQuotes = openQuotes;
+            Unparsed = unparsed;
+        }
+
+        public override IEnumerable<Node> GetChildNodes() {
+            return _children;
+        }
+
+        public readonly string Unparsed;
+
+        public override void Walk(PythonWalker walker) {
+            if (walker.Walk(this)) {
+                foreach (var child in _children) {
+                    child.Walk(walker);
+                }
+            }
+            walker.PostWalk(this);
+        }
+
+        public async override Task WalkAsync(PythonWalkerAsync walker, CancellationToken cancellationToken = default) {
+            if (await walker.WalkAsync(this, cancellationToken)) {
+                foreach (var child in _children) {
+                    await child.WalkAsync(walker, cancellationToken);
+                }
+            }
+            await walker.PostWalkAsync(this, cancellationToken);
+        }
+
+        internal override void AppendCodeString(StringBuilder res, PythonAst ast, CodeFormattingOptions format) {
+            var verbatimPieces = this.GetVerbatimNames(ast);
+            var verbatimComments = this.GetListWhiteSpace(ast);
+            if (verbatimPieces != null) {
+                // string+ / bytes+, such as "abc" "abc", which can spawn multiple lines, and 
+                // have comments in between the peices.
+                for (var i = 0; i < verbatimPieces.Length; i++) {
+                    if (verbatimComments != null && i < verbatimComments.Length) {
+                        format.ReflowComment(res, verbatimComments[i]);
+                    }
+                    res.Append(verbatimPieces[i]);
+                }
+            } else {
+                format.ReflowComment(res, this.GetPreceedingWhiteSpaceDefaultNull(ast));
+                if (this.GetExtraVerbatimText(ast) != null) {
+                    res.Append(this.GetExtraVerbatimText(ast));
+                } else {
+                    RecursiveAppendRepr(res, ast, format);
+                }
+            }
+        }
+
+        private void RecursiveAppendRepr(StringBuilder res, PythonAst ast, CodeFormattingOptions format) {
+            res.Append('f');
+            res.Append(_openQuotes);
+            foreach (var child in _children) {
+                AppendChild(res, ast, format, child);
+            }
+            res.Append(_openQuotes);
+        }
+
+        private static void AppendChild(StringBuilder res, PythonAst ast, CodeFormattingOptions format, Node child) {
+            if (child is ConstantExpression expr) {
+                // Non-Verbatim AppendCodeString for ConstantExpression adds quotes around string
+                // Remove those quotes
+                var childStrBuilder = new StringBuilder();
+                child.AppendCodeString(childStrBuilder, ast, format);
+                res.Append(childStrBuilder.ToString().Trim('\''));
+            } else {
+                child.AppendCodeString(res, ast, format);
+            }
+        }
+    }
+}

src/Parsing/Impl/Ast/FormatSpecifier.cs

@@ -0,0 +1,58 @@
+using System.Collections.Generic;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.Python.Parsing.Ast {
+    public class FormatSpecifier : Expression {
+        private readonly Node[] _children;
+
+        public FormatSpecifier(Node[] children, string unparsed) {
+            _children = children;
+            Unparsed = unparsed;
+        }
+
+        public override IEnumerable<Node> GetChildNodes() {
+            return _children;
+        }
+
+        public readonly string Unparsed;
+
+        public override void Walk(PythonWalker walker) {
+            if (walker.Walk(this)) {
+                foreach (var child in _children) {
+                    child.Walk(walker);
+                }
+            }
+            walker.PostWalk(this);
+        }
+
+        public async override Task WalkAsync(PythonWalkerAsync walker, CancellationToken cancellationToken = default) {
+            if (await walker.WalkAsync(this, cancellationToken)) {
+                foreach (var child in _children) {
+                    await child.WalkAsync(walker, cancellationToken);
+                }
+            }
+            await walker.PostWalkAsync(this, cancellationToken);
+        }
+
+        internal override void AppendCodeString(StringBuilder res, PythonAst ast, CodeFormattingOptions format) {
+            // There is no leading f
+            foreach (var child in _children) {
+                AppendChild(res, ast, format, child);
+            }
+        }
+
+        private static void AppendChild(StringBuilder res, PythonAst ast, CodeFormattingOptions format, Node child) {
+            if (child is ConstantExpression expr) {
+                // Non-Verbatim AppendCodeString for ConstantExpression adds quotes around string
+                // Remove those quotes
+                var childStrBuilder = new StringBuilder();
+                child.AppendCodeString(childStrBuilder, ast, format);
+                res.Append(childStrBuilder.ToString().Trim('\''));
+            } else {
+                child.AppendCodeString(res, ast, format);
+            }
+        }
+    }
+}

src/Parsing/Impl/Ast/FormattedValue.cs

@@ -0,0 +1,56 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.Python.Parsing.Ast {
+    public class FormattedValue : Node {
+        public FormattedValue(Expression value, char? conversion, Expression formatSpecifier) {
+            Value = value;
+            FormatSpecifier = formatSpecifier;
+            Conversion = conversion;
+        }
+
+        public Expression Value { get; }
+        public Expression FormatSpecifier { get; }
+        public char? Conversion { get; }
+
+        public override IEnumerable<Node> GetChildNodes() {
+            yield return Value;
+            if (FormatSpecifier != null) {
+                yield return FormatSpecifier;
+            }
+        }
+
+        public override void Walk(PythonWalker walker) {
+            if (walker.Walk(this)) {
+                Value.Walk(walker);
+                FormatSpecifier?.Walk(walker);
+            }
+            walker.PostWalk(this);
+        }
+
+        public async override Task WalkAsync(PythonWalkerAsync walker, CancellationToken cancellationToken = default) {
+            if (await walker.WalkAsync(this, cancellationToken)) {
+                await Value.WalkAsync(walker, cancellationToken);
+                await FormatSpecifier?.WalkAsync(walker, cancellationToken);
+            }
+            await walker.PostWalkAsync(this, cancellationToken);
+        }
+
+        internal override void AppendCodeString(StringBuilder res, PythonAst ast, CodeFormattingOptions format) {
+            res.Append('{');
+            Value.AppendCodeString(res, ast, format);
+            if (Conversion.HasValue) {
+                res.Append('!');
+                res.Append(Conversion.Value);
+            }
+            if (FormatSpecifier != null) {
+                res.Append(':');
+                FormatSpecifier.AppendCodeString(res, ast, format);
+            }
+            res.Append('}');
+        }
+    }
+}

src/Parsing/Impl/Ast/PythonWalker.Generated.cs

@@ -283,6 +283,18 @@ public virtual void PostWalk(ErrorStatement node) { }
         // DecoratorStatement
         public virtual bool Walk(DecoratorStatement node) { return true; }
         public virtual void PostWalk(DecoratorStatement node) { }
+
+        // FString
+        public virtual bool Walk(FString node) { return true; }
+        public virtual void PostWalk(FString node) { }
+
+        // FormatSpecifier
+        public virtual bool Walk(FormatSpecifier node) { return true; }
+        public virtual void PostWalk(FormatSpecifier node) { }
+
+        // FormattedValue
+        public virtual bool Walk(FormattedValue node) { return true; }
+        public virtual void PostWalk(FormattedValue node) { }
     }
 
 
@@ -794,6 +806,15 @@ private bool Contains(Statement stmt) {
 
         // DecoratorStatement
         public override bool Walk(DecoratorStatement node) { return Contains(node); }
+
+        // FString
+        public override bool Walk(FString node) { return Location >= node.StartIndex && Location <= node.EndIndex; }
+
+        // FormatSpecifier
+        public override bool Walk(FormatSpecifier node) { return Location >= node.StartIndex && Location <= node.EndIndex; }
+
+        // FormattedValue
+        public override bool Walk(FormattedValue node) { return Location >= node.StartIndex && Location <= node.EndIndex; }
     }
 
 }

src/Parsing/Impl/Ast/PythonWalkerAsync.Generated.cs

@@ -285,6 +285,18 @@ public class PythonWalkerAsync {
         // DecoratorStatement
         public virtual Task<bool> WalkAsync(DecoratorStatement node, CancellationToken cancellationToken = default) => Task.FromResult(true);
         public virtual Task PostWalkAsync(DecoratorStatement node, CancellationToken cancellationToken = default) => Task.CompletedTask;
+
+        // FString
+        public virtual Task<bool> WalkAsync(FString node, CancellationToken cancellationToken = default) => Task.FromResult(true);
+        public virtual Task PostWalkAsync(FString node, CancellationToken cancellationToken = default) => Task.CompletedTask;
+
+        // FormatSpecifier
+        public virtual Task<bool> WalkAsync(FormatSpecifier node, CancellationToken cancellationToken = default) => Task.FromResult(true);
+        public virtual Task PostWalkAsync(FormatSpecifier node, CancellationToken cancellationToken = default) => Task.CompletedTask;
+
+        // FormattedValue
+        public virtual Task<bool> WalkAsync(FormattedValue node, CancellationToken cancellationToken = default) => Task.FromResult(true);
+        public virtual Task PostWalkAsync(FormattedValue node, CancellationToken cancellationToken = default) => Task.CompletedTask;
     }
 
 
@@ -862,5 +874,17 @@ public override Task<bool> WalkAsync(ErrorStatement node, CancellationToken canc
         // DecoratorStatement
         public override Task<bool> WalkAsync(DecoratorStatement node, CancellationToken cancellationToken = default)
             => Task.FromResult(Contains(node));
+
+        // FString
+        public override Task<bool> WalkAsync(FString node, CancellationToken cancellationToken = default)
+            => Task.FromResult(Location >= node.StartIndex && Location <= node.EndIndex);
+
+        // FormatSpecifier
+        public override Task<bool> WalkAsync(FormatSpecifier node, CancellationToken cancellationToken = default)
+            => Task.FromResult(Location >= node.StartIndex && Location <= node.EndIndex);
+
+        // FormattedValue
+        public override Task<bool> WalkAsync(FormattedValue node, CancellationToken cancellationToken = default)
+            => Task.FromResult(Location >= node.StartIndex && Location <= node.EndIndex);
     }
 }