Unpacking typing Tuples and Lists correctly (#1553)

Fixes #1457
Fixes #1494
Fixes #411
Fixes #1382
Fixes #1293

Author: Mikhail Arkhipov

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs

@@ -44,6 +44,7 @@ public IMember GetValueFromCallable(CallExpression expr) {
             // so we can invoke Call over the instance. Second, an type info
             // so we can create an instance of the type (as in C() where C is class).
             IMember value = null;
+            var args = ArgumentSet.Empty(expr, this);
             switch (target) {
                 case IPythonBoundType bt: // Bound property, method or an iterator.
                     value = GetValueFromBound(bt, expr);
@@ -52,7 +53,7 @@ public IMember GetValueFromCallable(CallExpression expr) {
                     value = GetValueFromInstanceCall(pi, expr);
                     break;
                 case IPythonFunctionType ft: // Standalone function or a class method call.
-                    var instance = ft.DeclaringType != null ? new PythonInstance(ft.DeclaringType) : null;
+                    var instance = ft.DeclaringType?.CreateInstance(args);
                     value = GetValueFromFunctionType(ft, instance, expr);
                     break;
                 case IPythonClassType cls:
@@ -61,7 +62,7 @@ public IMember GetValueFromCallable(CallExpression expr) {
                 case IPythonType t:
                     // Target is type (info), the call creates instance.
                     // For example, 'x = C; y = x()' or 'x = C()' where C is class
-                    value = new PythonInstance(t);
+                    value = t.CreateInstance(args);
                     break;
             }
 
@@ -93,14 +94,14 @@ public IMember GetValueFromClassCtor(IPythonClassType cls, CallExpression expr)
             var init = cls.GetMember<IPythonFunctionType>(@"__init__");
             if (init != null) {
                 using (OpenScope(cls.DeclaringModule, cls.ClassDefinition, out _)) {
-                    var a = new ArgumentSet(init, 0, new PythonInstance(cls), expr, this);
+                    var a = new ArgumentSet(init, 0, cls, expr, this);
                     if (a.Errors.Count > 0) {
                         // AddDiagnostics(Module.Uri, a.Errors);
                     }
                     args = a.Evaluate();
                 }
             }
-            return cls.CreateInstance(cls.Name, args);
+            return cls.CreateInstance(args);
         }
 
         private IMember GetValueFromBound(IPythonBoundType t, CallExpression expr) {
@@ -146,12 +147,14 @@ public IMember GetValueFromFunctionType(IPythonFunctionType fn, IPythonInstance
             // Pick the best overload.
             FunctionDefinition fd;
             ArgumentSet args;
+            var instanceType = instance?.GetPythonType();
+
             if (fn.Overloads.Count == 1) {
                 fd = fn.Overloads[0].FunctionDefinition;
-                args = new ArgumentSet(fn, 0, instance, expr, this);
+                args = new ArgumentSet(fn, 0, instanceType, expr, this);
                 args = args.Evaluate();
             } else {
-                args = FindOverload(fn, instance, expr);
+                args = FindOverload(fn, instanceType, expr);
                 if (args == null) {
                     return UnknownType;
                 }
@@ -167,21 +170,20 @@ public IMember GetValueFromFunctionType(IPythonFunctionType fn, IPythonInstance
                 }
             }
 
-            // If instance is not the same as the declaring type, then call most probably comes
+            // If instance type is not the same as the declaring type, then call most probably comes
             // from the derived class which means that the original 'self' and 'cls' variables
             // are no longer valid and function has to be re-evaluated with new arguments.
             // Note that there is nothing to re-evaluate in stubs.
-            var instanceType = instance?.GetPythonType();
             if (instanceType == null || fn.DeclaringType == null || fn.IsSpecialized ||
                 instanceType.IsSpecialized || fn.DeclaringType.IsSpecialized ||
                 instanceType.Equals(fn.DeclaringType) ||
                 fn.IsStub || !string.IsNullOrEmpty(fn.Overloads[args.OverloadIndex].GetReturnDocumentation())) {
 
                 LoadFunctionDependencyModules(fn);
 
-                var t = instance?.Call(fn.Name, args) ?? fn.Call(null, fn.Name, args);
-                if (!t.IsUnknown()) {
-                    return t;
+                var m = instance?.Call(fn.Name, args) ?? fn.Call(null, fn.Name, args);
+                if (!m.IsUnknown()) {
+                    return m;
                 }
             }
 
@@ -244,14 +246,14 @@ private IMember TryEvaluateWithArguments(IPythonFunctionType fn, IArgumentSet ar
             return result;
         }
 
-        private ArgumentSet FindOverload(IPythonFunctionType fn, IPythonInstance instance, CallExpression expr) {
+        private ArgumentSet FindOverload(IPythonFunctionType fn, IPythonType instanceType, CallExpression expr) {
             if (fn.Overloads.Count == 1) {
                 return null;
             }
 
             var sets = new List<ArgumentSet>();
             for (var i = 0; i < fn.Overloads.Count; i++) {
-                var a = new ArgumentSet(fn, i, instance, expr, this);
+                var a = new ArgumentSet(fn, i, instanceType, expr, this);
                 var args = a.Evaluate();
                 sets.Add(args);
             }
@@ -329,7 +331,8 @@ public IReadOnlyList<IParameterInfo> CreateFunctionParameters(IPythonClassType s
                     // The reason is that if method might be called on a derived class.
                     // Declare self or cls in this scope.
                     if (declareVariables) {
-                        DeclareVariable(p0.Name, new PythonInstance(self), VariableSource.Declaration, p0.NameExpression);
+                        DeclareVariable(p0.Name, self.CreateInstance(ArgumentSet.Empty(p0.NameExpression, this)),
+                            VariableSource.Declaration, p0.NameExpression);
                     }
                     // Set parameter info, declare type as annotation type for generic self 
                     // e.g def test(self: T)
@@ -377,7 +380,8 @@ private void DeclareParameter(Parameter p, ParameterInfo pi) {
             } else {
                 paramType = pi?.DefaultValue?.GetPythonType() ?? UnknownType;
             }
-            DeclareVariable(p.Name, new PythonInstance(paramType), VariableSource.Declaration, p.NameExpression);
+            DeclareVariable(p.Name, paramType.CreateInstance(ArgumentSet.Empty(p.NameExpression, this)),
+                VariableSource.Declaration, p.NameExpression);
         }
     }
 }

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs

@@ -45,11 +45,11 @@ public IMember GetValueFromIndex(IndexExpression expr) {
             var type = target.GetPythonType();
             if (type != null) {
                 if (!(target is IPythonInstance instance)) {
-                    instance = new PythonInstance(type);
+                    instance = type.CreateInstance(ArgumentSet.Empty(expr, this));
                 }
                 var index = GetValueFromExpression(expr.Index);
                 if (index != null) {
-                    return type.Index(instance, new ArgumentSet(new []{index}, expr, this));
+                    return type.Index(instance, new ArgumentSet(new[] { index }, expr, this));
                 }
             }
 

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Constants.cs

@@ -38,7 +38,7 @@ public IPythonInstance GetConstantFromLiteral(Expression expr) {
             }
 
             var t = SuppressBuiltinLookup ? UnknownType : (GetTypeFromLiteral(expr) ?? UnknownType);
-            return new PythonInstance(t);
+            return t.CreateInstance(ArgumentSet.Empty(expr, this));
         }
 
         public IPythonType GetTypeFromLiteral(Expression expr) {

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs

@@ -172,12 +172,12 @@ private IMember CreateClassInstance(PythonClassType cls, IReadOnlyList<IMember>
             var initOverload = initFunc?.DeclaringType == cls ? initFunc.Overloads.FirstOrDefault() : null;
 
             var argSet = initOverload != null
-                    ? new ArgumentSet(initFunc, 0, null, callExpr, this)
+                    ? new ArgumentSet(initFunc, 0, cls, callExpr, this)
                     : new ArgumentSet(constructorArguments, callExpr, this);
 
             argSet.Evaluate();
             var specificType = cls.CreateSpecificType(argSet);
-            return new PythonInstance(specificType);
+            return specificType.CreateInstance(argSet);
         }
 
         private ScopeStatement GetScope(IMember m) {

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs

@@ -258,7 +258,7 @@ private IMember GetValueFromMember(MemberExpression expr) {
                     f.AddReference(GetLocationOfName(expr));
                     return f.ToUnbound();
                 }
-                instance = new PythonInstance(type);
+                instance = type.CreateInstance(ArgumentSet.Empty(expr, this));
             }
 
             instance = instance ?? m as IPythonInstance;

src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs

@@ -13,12 +13,12 @@
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
 
-using System.Diagnostics;
-using System.Linq;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Values;
 using Microsoft.Python.Core;
 using Microsoft.Python.Parsing.Ast;
+using System.Diagnostics;
+using System.Linq;
 
 namespace Microsoft.Python.Analysis.Analyzer.Handlers {
     internal sealed class AssignmentHandler : StatementHandler {
@@ -30,10 +30,12 @@ public void HandleAssignment(AssignmentStatement node) {
             }
 
             var value = Eval.GetValueFromExpression(node.Right) ?? Eval.UnknownType;
+            // Filter out parenthesis expression in assignment because it makes no difference.
+            var lhs = node.Left.Select(s => s.RemoveParenthesis());
             // Check PEP hint first
             var valueType = Eval.GetTypeFromPepHint(node.Right);
             if (valueType != null) {
-                HandleTypedVariable(valueType, value, node.Left.FirstOrDefault());
+                HandleTypedVariable(valueType, value, lhs.FirstOrDefault());
                 return;
             }
 
@@ -44,44 +46,49 @@ public void HandleAssignment(AssignmentStatement node) {
                 value = Eval.UnknownType;
             }
 
-            if (node.Left.FirstOrDefault() is SequenceExpression seq) {
-                // Tuple = Tuple. Transfer values.
-                var seqHandler = new SequenceExpressionHandler(Walker);
-                seqHandler.HandleAssignment(seq.Items, node.Right, value);
-                return;
+            foreach (var expr in lhs) {
+                switch (expr) {
+                    case SequenceExpression seq:
+                        // Tuple = Tuple. Transfer values.
+                        var seqHandler = new SequenceExpressionHandler(Walker);
+                        seqHandler.HandleAssignment(seq, value);
+                        break;
+                    case ExpressionWithAnnotation annExpr:
+                        HandleAnnotatedExpression(annExpr, value);
+                        break;
+                    case NameExpression nameExpr:
+                        HandleNameExpression(nameExpr, value);
+                        break;
+                    case MemberExpression memberExpr:
+                        TryHandleClassVariable(memberExpr, value);
+                        break;
+                }
             }
+        }
 
-            // Process annotations, if any.
-            foreach (var expr in node.Left.OfType<ExpressionWithAnnotation>()) {
-                // x: List[str] = [...]
-                HandleAnnotatedExpression(expr, value);
-            }
+        private bool IsValidAssignment(string name, Location loc) => !Eval.GetInScope(name).IsDeclaredAfter(loc);
 
-            foreach (var ne in node.Left.OfType<NameExpression>()) {
-                IScope scope;
-                if (Eval.CurrentScope.NonLocals[ne.Name] != null) {
-                    Eval.LookupNameInScopes(ne.Name, out scope, LookupOptions.Nonlocal);
-                    scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne));
-                    continue;
-                }
-                if (Eval.CurrentScope.Globals[ne.Name] != null) {
-                    Eval.LookupNameInScopes(ne.Name, out scope, LookupOptions.Global);
-                    scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne));
-                    continue;
-                }
+        private void HandleNameExpression(NameExpression ne, IMember value) {
+            IScope scope;
+            if (Eval.CurrentScope.NonLocals[ne.Name] != null) {
+                Eval.LookupNameInScopes(ne.Name, out scope, LookupOptions.Nonlocal);
+                scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne));
+                return;
+            }
 
-                var source = value.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration;
-                var location = Eval.GetLocationOfName(ne);
-                if (IsValidAssignment(ne.Name, location)) {
-                    Eval.DeclareVariable(ne.Name, value ?? Module.Interpreter.UnknownType, source, location);
-                }
+            if (Eval.CurrentScope.Globals[ne.Name] != null) {
+                Eval.LookupNameInScopes(ne.Name, out scope, LookupOptions.Global);
+                scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne));
+                return;
             }
 
-            TryHandleClassVariable(node, value);
+            var source = value.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration;
+            var location = Eval.GetLocationOfName(ne);
+            if (IsValidAssignment(ne.Name, location)) {
+                Eval.DeclareVariable(ne.Name, value ?? Module.Interpreter.UnknownType, source, location);
+            }
         }
 
-        private bool IsValidAssignment(string name, Location loc) => !Eval.GetInScope(name).IsDeclaredAfter(loc);
-
         public void HandleAnnotatedExpression(ExpressionWithAnnotation expr, IMember value) {
             if (expr?.Annotation == null) {
                 return;
@@ -95,8 +102,7 @@ public void HandleAnnotatedExpression(ExpressionWithAnnotation expr, IMember val
             HandleTypedVariable(variableType, value, expr.Expression);
         }
 
-        private void TryHandleClassVariable(AssignmentStatement node, IMember value) {
-            var mex = node.Left.OfType<MemberExpression>().FirstOrDefault();
+        private void TryHandleClassVariable(MemberExpression mex, IMember value) {
             if (!string.IsNullOrEmpty(mex?.Name) && mex.Target is NameExpression nex && nex.Name.EqualsOrdinal("self")) {
                 var m = Eval.LookupNameInScopes(nex.Name, out _, LookupOptions.Local);
                 var cls = m.GetPythonType<IPythonClassType>();
@@ -121,7 +127,8 @@ private void HandleTypedVariable(IPythonType variableType, IMember value, Expres
                     instance = value;
                 }
             }
-            instance = instance ?? variableType?.CreateInstance(variableType.Name, ArgumentSet.Empty(expr, Eval)) ?? Eval.UnknownType;
+            var args = ArgumentSet.Empty(expr, Eval);
+            instance = instance ?? variableType?.CreateInstance(args) ?? Eval.UnknownType.CreateInstance(ArgumentSet.WithoutContext);
 
             if (expr is NameExpression ne) {
                 Eval.DeclareVariable(ne.Name, instance, VariableSource.Declaration, ne);

src/Analysis/Ast/Impl/Analyzer/Handlers/LoopHandler.cs

@@ -35,7 +35,7 @@ public bool HandleFor(ForStatement node) {
                     // x = [('abc', 42, True), ('abc', 23, False)]
                     // for some_str, (some_int, some_bool) in x:
                     var h = new SequenceExpressionHandler(Walker);
-                    h.HandleAssignment(seq.Items, node.List, value);
+                    h.HandleAssignment(seq, value);
                     break;
             }