Special handling of variables representing class members (#1424)

* Initial

* Update test

* Tell between class members with/without self better

* Move interface

* Fix reference search

Author: Mikhail Arkhipov

src/Analysis/Ast/Impl/Analyzer/Definitions/LookupOptions.cs

@@ -23,6 +23,8 @@ public enum LookupOptions {
         Nonlocal = 2,
         Global = 4,
         Builtins = 8,
-        Normal = Local | Nonlocal | Global | Builtins
+        ClassMembers = 16,
+        Normal = Local | Nonlocal | Global | Builtins,
+        All = Normal | ClassMembers
     }
 }

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs

@@ -63,12 +63,14 @@ public void DeclareVariable(string name, IMember value, VariableSource source, L
 
         public IMember LookupNameInScopes(string name, out IScope scope, out IVariable v, LookupOptions options) {
             scope = null;
+            var classMembers = (options & LookupOptions.ClassMembers) == LookupOptions.ClassMembers;
 
             switch (options) {
+                case LookupOptions.All:
                 case LookupOptions.Normal:
                     // Regular lookup: all scopes and builtins.
                     for (var s = CurrentScope; s != null; s = (Scope)s.OuterScope) {
-                        if (s.Variables.Contains(name)) {
+                        if (s.Variables.TryGetVariable(name, out var v1) && (!v1.IsClassMember || classMembers)) {
                             scope = s;
                             break;
                         }

src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs

@@ -93,7 +93,7 @@ public void HandleAnnotatedExpression(ExpressionWithAnnotation expr, IMember val
         private void TryHandleClassVariable(AssignmentStatement node, IMember value) {
             var mex = node.Left.OfType<MemberExpression>().FirstOrDefault();
             if (!string.IsNullOrEmpty(mex?.Name) && mex.Target is NameExpression nex && nex.Name.EqualsOrdinal("self")) {
-                var m = Eval.LookupNameInScopes(nex.Name, out var scope, LookupOptions.Local);
+                var m = Eval.LookupNameInScopes(nex.Name, out _, LookupOptions.Local);
                 var cls = m.GetPythonType<IPythonClassType>();
                 if (cls != null) {
                     using (Eval.OpenScope(Eval.Module, cls.ClassDefinition, out _)) {

src/Analysis/Ast/Impl/Linting/UndefinedVariables/ExpressionWalker.cs

@@ -78,7 +78,7 @@ public override bool Walk(NameExpression node) {
             }
 
             var analysis = _walker.Analysis;
-            var m = analysis.ExpressionEvaluator.LookupNameInScopes(node.Name, out var variableDefinitionScope, out var v);
+            var m = analysis.ExpressionEvaluator.LookupNameInScopes(node.Name, out var variableDefinitionScope, out var v, LookupOptions.All);
             if (m == null) {
                 _walker.ReportUndefinedVariable(node);
             }

src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs

@@ -1,4 +1,19 @@
-using System;
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using Microsoft.Python.Analysis.Specializations.Typing;
@@ -7,7 +22,7 @@
 using Microsoft.Python.Core;
 
 namespace Microsoft.Python.Analysis.Types {
-    internal partial class PythonClassType : PythonType, IPythonClassType, IGenericType, IEquatable<IPythonClassType> {
+    internal partial class PythonClassType {
         private bool _isGeneric;
         private object _genericParameterLock = new object();
         private Dictionary<string, PythonClassType> _specificTypeCache;

src/Analysis/Ast/Impl/Types/PythonClassType.cs

@@ -33,7 +33,7 @@ namespace Microsoft.Python.Analysis.Types {
     internal partial class PythonClassType : PythonType, IPythonClassType, IGenericType, IEquatable<IPythonClassType> {
         private static readonly string[] _classMethods = { "mro", "__dict__", @"__weakref__" };
 
-        private ReentrancyGuard<IPythonClassType> _memberGuard = new ReentrancyGuard<IPythonClassType>();
+        private readonly ReentrancyGuard<IPythonClassType> _memberGuard = new ReentrancyGuard<IPythonClassType>();
         private string _genericName;
         private List<IPythonType> _bases;
         private IReadOnlyList<IPythonType> _mro;

src/Analysis/Ast/Impl/Types/PythonType.cs

@@ -115,10 +115,16 @@ internal bool TrySetTypeId(BuiltinTypeId typeId) {
         internal void AddMembers(IEnumerable<IVariable> variables, bool overwrite) {
             lock (_lock) {
                 if (!_readonly) {
-                    foreach (var v in variables.Where(m => overwrite || !Members.ContainsKey(m.Name))) {
-                        // If variable holds function or a class, use value as member. 
-                        // If it holds an instance, use the variable itself (i.e. it is a data member).
-                        WritableMembers[v.Name] = v.Value;
+                    foreach (var v in variables.OfType<Variable>()) {
+                        var hasMember = Members.ContainsKey(v.Name);
+                        if (overwrite || !hasMember) {
+                            // If variable holds function or a class, use value as member. 
+                            // If it holds an instance, use the variable itself (i.e. it is a data member).
+                            WritableMembers[v.Name] = v.Value;
+                        }
+                        if (hasMember) {
+                            v.IsClassMember = true;
+                        }
                     }
                 }
             }

src/Analysis/Ast/Impl/Values/Definitions/IVariable.cs

@@ -35,6 +35,11 @@ public interface IVariable: ILocatedMember {
         /// </summary>
         IMember Value { get; }
 
+        /// <summary>
+        /// Variable represents class member.
+        /// </summary>
+        bool IsClassMember { get; }
+
         /// <summary>
         /// Assigns value to the variable.
         /// </summary>

src/Analysis/Ast/Impl/Values/Variable.cs

@@ -32,6 +32,7 @@ public Variable(string name, IMember value, VariableSource source, Location loca
         public string Name { get; }
         public VariableSource Source { get; }
         public IMember Value { get; private set; }
+        public bool IsClassMember { get; internal set; }
 
         public void Assign(IMember value, Location location) {
             if (value is IVariable v) {

src/Analysis/Ast/Test/ClassesTests.cs

@@ -632,7 +632,7 @@ def __init__(self):
             sw.Stop();
             // Desktop: product time is typically less few seconds second.
             // Test run time: typically ~ 20 sec.
-            sw.ElapsedMilliseconds.Should().BeLessThan(60000); 
+            sw.ElapsedMilliseconds.Should().BeLessThan(60000);
         }
 
         [TestMethod, Priority(0)]
@@ -651,6 +651,32 @@ def foo(*args, **kwargs):
             analysis.Should().HaveVariable("a").OfType(BuiltinTypeId.Int);
         }
 
+        [TestMethod, Priority(0)]
+        public async Task MemberCtorAssignment() {
+            const string code = @"
+class x:
+    y: int
+    z = y
+
+a = x().z
+";
+            var analysis = await GetAnalysisAsync(code);
+            analysis.Should().HaveVariable("a").OfType(BuiltinTypeId.Int);
+        }
+
+        [TestMethod, Priority(0)]
+        public async Task AmbiguousMemberAssignment() {
+            const string code = @"
+class x:
+    x: int
+    y = x
+
+a = x().y
+";
+            var analysis = await GetAnalysisAsync(code);
+            analysis.Should().HaveVariable("a").OfType(BuiltinTypeId.Int);
+        }
+
         [TestMethod, Priority(0)]
         public async Task IOErrorBase() {
             const string code = @"

src/Analysis/Ast/Test/FunctionTests.cs

@@ -606,5 +606,27 @@ def test(foo: Foo = func()):
             var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X);
             analysis.Should().HaveVariable("x").OfType("Foo");
         }
+
+        [TestMethod, Priority(0)]
+        public async Task AmbiguousOptionalParameterType() {
+            const string code = @"
+from typing import Optional
+class A: ...
+
+class B:
+    def __init__(self, A: Optional[A]):
+        self.name = name
+
+    @property
+    def A(self) -> int:
+";
+            var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X);
+            var a = analysis.Should().HaveClass("A").Which;
+            analysis.Should().HaveClass("B")
+                .Which.Should().HaveMethod("__init__")
+                .Which.Should().HaveParameterAt(1)
+                .Which.Should().HaveType("A")
+                .Which.Should().BeOfType(a.GetType());
+        }
     }
 }

src/Core/Impl/Text/IndexSpan.cs

@@ -23,18 +23,16 @@ namespace Microsoft.Python.Core.Text {
     /// It is closed on the left and open on the right: [Start .. End). 
     /// </summary>
     public struct IndexSpan : IEquatable<IndexSpan> {
-        private readonly int _start, _length;
-
         public IndexSpan(int start, int length) {
-            _start = start;
-            _length = length;
+            Start = start;
+            Length = length;
         }
 
-        public int Start => _start;
+        public int Start { get; }
 
-        public int End => _start + _length;
+        public int End => Start + Length;
 
-        public int Length => _length;
+        public int Length { get; }
 
         public override int GetHashCode() => Length.GetHashCode() ^ Start.GetHashCode();
 
@@ -49,7 +47,7 @@ public IndexSpan(int start, int length) {
         }
 
         #region IEquatable<IndexSpan> Members
-        public bool Equals(IndexSpan other) => _length == other._length && _start == other._start;
+        public bool Equals(IndexSpan other) => Length == other.Length && Start == other.Start;
         #endregion
 
         public static IndexSpan FromBounds(int start, int end) => new IndexSpan(start, end - start);

src/LanguageServer/Impl/Completion/TopLevelCompletion.cs

@@ -39,7 +39,7 @@ public static CompletionResult GetCompletions(Node statement, ScopeStatement sco
             IEnumerable<CompletionItem> items;
             using (eval.OpenScope(scope)) {
                 // Get variables declared in the module.
-                var variables = eval.CurrentScope.EnumerateTowardsGlobal.SelectMany(s => s.Variables).ToArray();
+                var variables = eval.CurrentScope.EnumerateTowardsGlobal.SelectMany(s => s.Variables).Where(v => !v.IsClassMember).ToArray();
                 items = variables.Select(v => context.ItemSource.CreateCompletionItem(v.Name, v)).ToArray();
             }
 

src/LanguageServer/Impl/Sources/DefinitionSource.cs

@@ -191,7 +191,7 @@ private Reference HandleImport(IDocumentAnalysis analysis, ImportStatement state
         private Reference TryFromVariable(string name, IDocumentAnalysis analysis, SourceLocation location, Node statement, out ILocatedMember definingMember) {
             definingMember = null;
 
-            var m = analysis.ExpressionEvaluator.LookupNameInScopes(name, out var scope);
+            var m = analysis.ExpressionEvaluator.LookupNameInScopes(name, out var scope, LookupOptions.All);
             if (m == null || !(scope.Variables[name] is IVariable v)) {
                 return null;
             }

src/LanguageServer/Impl/Sources/HoverSource.cs

@@ -13,10 +13,12 @@
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
 
+using System;
 using Microsoft.Python.Analysis;
 using Microsoft.Python.Analysis.Analyzer;
 using Microsoft.Python.Analysis.Analyzer.Expressions;
 using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Analysis.Values;
 using Microsoft.Python.Core;
 using Microsoft.Python.Core.Collections;
 using Microsoft.Python.Core.Text;
@@ -38,7 +40,7 @@ public Hover GetHover(IDocumentAnalysis analysis, SourceLocation location) {
             }
 
             ExpressionLocator.FindExpression(analysis.Ast, location,
-                FindExpressionOptions.Hover, out var node, out var statement, out var scope);
+                FindExpressionOptions.Hover, out var node, out var statement, out var hoverScopeStatement);
 
             if (!HasHover(node) || !(node is Expression expr)) {
                 return null;
@@ -52,7 +54,7 @@ public Hover GetHover(IDocumentAnalysis analysis, SourceLocation location) {
             var eval = analysis.ExpressionEvaluator;
             switch (statement) {
                 case FromImportStatement fi when node is NameExpression nex: {
-                        var contents = HandleFromImport(fi, location, scope, analysis);
+                        var contents = HandleFromImport(fi, location, hoverScopeStatement, analysis);
                         if (contents != null) {
                             return new Hover {
                                 contents = contents,
@@ -63,7 +65,7 @@ public Hover GetHover(IDocumentAnalysis analysis, SourceLocation location) {
                         break;
                     }
                 case ImportStatement imp: {
-                        var contents = HandleImport(imp, location, scope, analysis);
+                        var contents = HandleImport(imp, location, hoverScopeStatement, analysis);
                         if (contents != null) {
                             return new Hover {
                                 contents = contents,
@@ -77,8 +79,30 @@ public Hover GetHover(IDocumentAnalysis analysis, SourceLocation location) {
 
             IMember value;
             IPythonType type;
-            using (eval.OpenScope(analysis.Document, scope)) {
-                value = analysis.ExpressionEvaluator.GetValueFromExpression(expr);
+            using (eval.OpenScope(analysis.Document, hoverScopeStatement)) {
+                // Here we can be hovering over a class member. Class members are declared
+                // as members as well as special variables in the class scope. If this is
+                // a name expression (rather than a member expression) and it is a class
+                // variable NOT in the immediate class scope, filter it out. Consider:
+                //   class A:
+                //     x = 1
+                //     y = x
+                // hover over 'x' in 'y = x' should produce proper tooltip. However, in
+                //   class A:
+                //     x = 1
+                //     def func(self):
+                //       y = x
+                // hover over 'x' in 'y = x' should not produce tooltip.
+
+                IVariable variable = null;
+                if (expr is NameExpression nex) {
+                    analysis.ExpressionEvaluator.LookupNameInScopes(nex.Name, out _, out variable, LookupOptions.All);
+                    if (IsInvalidClassMember(variable, hoverScopeStatement, location.ToIndex(analysis.Ast))) {
+                        return null;
+                    }
+                }
+
+                value = variable?.Value ?? analysis.ExpressionEvaluator.GetValueFromExpression(expr, LookupOptions.All);
                 type = value?.GetPythonType();
                 if (type == null) {
                     return null;
@@ -122,19 +146,36 @@ public Hover GetHover(IDocumentAnalysis analysis, SourceLocation location) {
             };
         }
 
-        private bool HasHover(Node node) {
+        private static bool HasHover(Node node) {
             switch (node) {
                 // No hover for literals
-                case ConstantExpression constExpr:
+                case ConstantExpression _:
                 // node is FString only if it didn't save an f-string subexpression
-                case FString fStr:
-                case NamedExpression namedExpr:
+                case FString _:
+                case NamedExpression _:
                     return false;
                 default:
                     return true;
             }
         }
 
+        private bool IsInvalidClassMember(IVariable v, ScopeStatement scope, int hoverPosition) {
+            if (v == null || !v.IsClassMember) {
+                return false;
+            }
+            switch (v.Value) {
+                case IPythonClassType cls when cls.ClassDefinition == scope:
+                    return hoverPosition  > cls.ClassDefinition.HeaderIndex;
+                case IPythonFunctionType ft when ft.FunctionDefinition == scope:
+                    return hoverPosition > ft.FunctionDefinition.HeaderIndex;
+                case IPythonPropertyType prop when prop.FunctionDefinition == scope:
+                    return hoverPosition > prop.FunctionDefinition.HeaderIndex;
+                case IPythonInstance _:
+                    return !(scope is ClassDefinition);
+            }
+            return true;
+        }
+
         private MarkupContent HandleImport(ImportStatement imp, SourceLocation location, ScopeStatement scope, IDocumentAnalysis analysis) {
             // 'import A.B, B.C, D.E as F, G, H'
             var eval = analysis.ExpressionEvaluator;