Persistence of generics and named tuples

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs

@@ -102,17 +102,10 @@ private IMember CreateSpecificTypeFromIndex(IGenericType gt, IReadOnlyList<IMemb
                 if (!GenericClassParameterValid(genericTypeArgs, args, expr)) {
                     return UnknownType;
                 }
-
-                // Generic[T1, T2, ...] expression. Create generic base for the class.
-                return new GenericClassParameter(genericTypeArgs, Module);
             }
 
             // For other types just use supplied arguments
-            if (args.Count > 0) {
-                return gt.CreateSpecificType(new ArgumentSet(args, expr, this));
-            }
-
-            return UnknownType;
+            return args.Count > 0 ? gt.CreateSpecificType(new ArgumentSet(args, expr, this)) : UnknownType;
         }
 
         private IReadOnlyList<IMember> EvaluateIndex(IndexExpression expr) {

src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs

@@ -16,7 +16,6 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
-using System.Xml.Serialization;
 using Microsoft.Python.Analysis.Analyzer.Evaluation;
 using Microsoft.Python.Analysis.Documents;
 using Microsoft.Python.Analysis.Modules;
@@ -246,118 +245,134 @@ private void MergeStub() {
                 var sourceType = sourceVar?.Value.GetPythonType();
 
                 // If stub says 'Any' but we have better type, keep the current type.
-                if (!IsStubBetterType(sourceType, stubType)) {
+                if (stubType.IsUnknown() || (stubType.DeclaringModule is TypingModule && stubType.Name == "Any")) {
                     continue;
                 }
 
-                // If type does not exist in module, but exists in stub, declare it unless it is an import.
-                // If types are the classes, merge members. Otherwise, replace type from one from the stub.
-                switch (sourceType) {
-                    case null:
-                        // Nothing in sources, but there is type in the stub. Declare it.
-                        if (v.Source == VariableSource.Declaration || v.Source == VariableSource.Generic) {
-                            Eval.DeclareVariable(v.Name, v.Value, v.Source);
+                TryReplaceMember(v, sourceType, stubType);
+            }
+
+            UpdateVariables();
+        }
+
+        private void TryReplaceMember(IVariable v, IPythonType sourceType, IPythonType stubType) {
+            // If type does not exist in module, but exists in stub, declare it unless it is an import.
+            // If types are the classes, take class from the stub, then add missing members.
+            // Otherwise, replace type from one from the stub.
+            switch (sourceType) {
+                case null:
+                    // Nothing in sources, but there is type in the stub. Declare it.
+                    if (v.Source == VariableSource.Declaration || v.Source == VariableSource.Generic) {
+                        Eval.DeclareVariable(v.Name, v.Value, v.Source);
+                    }
+                    break;
+
+                case PythonClassType sourceClass when Module.Equals(sourceClass.DeclaringModule):
+                    // Transfer documentation first so we get class documentation
+                    // that came from class definition win over one that may
+                    // come from __init__ during the member merge below.
+                    TransferDocumentationAndLocation(sourceClass, stubType);
+
+                    // Replace the class entirely since stub members may use generic types
+                    // and the class definition is important. We transfer missing members
+                    // from the original class to the stub.
+                    Eval.DeclareVariable(v.Name, v.Value, v.Source);
+
+                    // Go through source class members and pick those that are
+                    // not present in the stub class.
+                    foreach (var name in sourceClass.GetMemberNames()) {
+
+                        var sourceMember = sourceClass.GetMember(name);
+                        if (sourceMember.IsUnknown()) {
+                            continue; // Anything is better than unknowns.
                         }
-                        break;
+                        var sourceMemberType = sourceMember?.GetPythonType();
 
-                    case PythonClassType cls when Module.Equals(cls.DeclaringModule):
-                        // Transfer documentation first so we get class documentation
-                        // that came from class definition win over one that may
-                        // come from __init__ during the member merge below.
-                        TransferDocumentationAndLocation(sourceType, stubType);
+                        var stubMember = stubType.GetMember(name);
+                        var stubMemberType = stubMember.GetPythonType();
 
-                        // If class exists and belongs to this module, add or replace
-                        // its members with ones from the stub, preserving documentation.
                         // Don't augment types that do not come from this module.
-                        // Do not replace __class__ since it has to match class type and we are not
-                        // replacing class type, we are only merging members.
-                        foreach (var name in stubType.GetMemberNames().Except(new[] { "__class__", "__base__", "__bases__", "__mro__", "mro" })) {
-                            var stubMember = stubType.GetMember(name);
-                            var member = cls.GetMember(name);
-
-                            var memberType = member?.GetPythonType();
-                            var stubMemberType = stubMember.GetPythonType();
-
-                            if (sourceType.IsBuiltin || stubType.IsBuiltin) {
-                                // If stub type does not have an immediate member such as __init__() and
-                                // rather have it inherited from object, we do not want to use the inherited
-                                // since current class may either have its own of inherits it from the object.
-                                continue;
-                            }
-
-                            if (stubMemberType?.MemberType == PythonMemberType.Method && stubMemberType?.DeclaringModule.ModuleType == ModuleType.Builtins) {
-                                // Leave methods coming from object at the object and don't copy them into the derived class.
-                            }
-
-                            if (!IsStubBetterType(memberType, stubMemberType)) {
-                                continue;
-                            }
-
-                            // Get documentation from the current type, if any, since stubs
-                            // typically do not contain documentation while scraped code does.
-                            TransferDocumentationAndLocation(memberType, stubMemberType);
-                            cls.AddMember(name, stubMember, overwrite: true);
+                        if (sourceType.IsBuiltin || stubType.IsBuiltin) {
+                            // If source type does not have an immediate member such as __init__() and
+                            // rather have it inherited from object, we do not want to use the inherited
+                            // since stub class may either have its own of inherits it from the object.
+                            continue;
                         }
-                        break;
-
-                    case IPythonModule _:
-                        // We do not re-declare modules.
-                        break;
 
-                    default:
-                        var stubModule = stubType.DeclaringModule;
-                        if (stubType is IPythonModule || stubModule.ModuleType == ModuleType.Builtins) {
-                            // Modules members that are modules should remain as they are, i.e. os.path
-                            // should remain library with its own stub attached.
-                            break;
+                        if (stubMemberType?.MemberType == PythonMemberType.Method && stubMemberType?.DeclaringModule.ModuleType == ModuleType.Builtins) {
+                            // Leave methods coming from object at the object and don't copy them into the derived class.
                         }
-                        // We do not re-declaring variables that are imported.
-                        if (v.Source == VariableSource.Declaration) {
-                            // Re-declare variable with the data from the stub.
-                            TransferDocumentationAndLocation(sourceType, stubType);
-                            // TODO: choose best type between the scrape and the stub. Stub probably should always win.
-                            var source = Eval.CurrentScope.Variables[v.Name]?.Source ?? v.Source;
-                            Eval.DeclareVariable(v.Name, v.Value, source);
+
+                        // Get documentation from the current type, if any, since stubs
+                        // typically do not contain documentation while scraped code does.
+                        TransferDocumentationAndLocation(sourceMemberType, stubMemberType);
+
+                        // If stub says 'Any' but we have better type, use member from the original class.
+                        if (stubMember != null && !(stubType.DeclaringModule is TypingModule && stubType.Name == "Any")) {
+                            continue; // Stub already have the member, don't replace.
                         }
 
+                        (stubType as PythonType)?.AddMember(name, stubMember, overwrite: true);
+                    }
+                    break;
+
+                case IPythonModule _:
+                    // We do not re-declare modules.
+                    break;
+
+                default:
+                    var stubModule = stubType.DeclaringModule;
+                    if (stubType is IPythonModule || stubModule.ModuleType == ModuleType.Builtins) {
+                        // Modules members that are modules should remain as they are, i.e. os.path
+                        // should remain library with its own stub attached.
                         break;
-                }
+                    }
+                    // We do not re-declaring variables that are imported.
+                    if (v.Source == VariableSource.Declaration) {
+                        // Re-declare variable with the data from the stub.
+                        TransferDocumentationAndLocation(sourceType, stubType);
+                        // TODO: choose best type between the scrape and the stub. Stub probably should always win.
+                        var source = Eval.CurrentScope.Variables[v.Name]?.Source ?? v.Source;
+                        Eval.DeclareVariable(v.Name, v.Value, source);
+                    }
+
+                    break;
             }
         }
 
-        private static bool IsStubBetterType(IPythonType sourceType, IPythonType stubType) {
-            if (stubType.IsUnknown()) {
-                // Do not use worse types than what is in the module.
-                return false;
-            }
-            if (sourceType.IsUnknown()) {
-                return true; // Anything is better than unknowns.
-            }
-            if (sourceType.DeclaringModule.ModuleType == ModuleType.Specialized) {
-                // Types in specialized modules should never be touched.
-                return false;
-            }
-            if (stubType.DeclaringModule is TypingModule && stubType.Name == "Any") {
-                // If stub says 'Any' but we have better type, keep the current type.
-                return false;
-            }
-            // Types should be compatible except it is allowed to replace function by a class.
-            // Consider stub of threading that replaces def RLock(): by class RLock().
-            // Similarly, in _json, make_scanner function is replaced by a class.
-            if (sourceType.MemberType == PythonMemberType.Function && stubType.MemberType == PythonMemberType.Class) {
-                return true;
-            }
-            // Random replaces method (variable) by a function.
-            if (sourceType.MemberType == PythonMemberType.Method && stubType.MemberType == PythonMemberType.Function) {
-                return true;
+        private void UpdateVariables() {
+            // Second pass: if we replaced any classes by new from the stub, we need to update 
+            // variables that may still be holding old content. For example, ctypes
+            // declares 'c_voidp = c_void_p' so when we replace 'class c_void_p'
+            // by class from the stub, we need to go and update 'c_voidp' variable.
+            foreach (var v in GlobalScope.Variables) {
+                var variableType = v.Value.GetPythonType();
+                if (!variableType.DeclaringModule.Equals(Module) && !variableType.DeclaringModule.Equals(Module.Stub)) {
+                    continue;
+                }
+                // Check if type that the variable references actually declared here.
+                var typeInScope = GlobalScope.Variables[variableType.Name].GetPythonType();
+                if (typeInScope == null || variableType == typeInScope) {
+                    continue;
+                }
+
+                if (v.Value == variableType) {
+                    Eval.DeclareVariable(v.Name, typeInScope, v.Source);
+                } else if (v.Value is IPythonInstance) {
+                    Eval.DeclareVariable(v.Name, new PythonInstance(typeInScope), v.Source);
+                }
             }
-            return sourceType.MemberType == stubType.MemberType;
         }
 
         private static void TransferDocumentationAndLocation(IPythonType s, IPythonType d) {
-            if (s.IsUnknown() || d.IsBuiltin || s.IsBuiltin) {
+            if (s.IsUnknown() || s.IsBuiltin || d == null || d.IsBuiltin) {
                 return; // Do not transfer location of unknowns or builtins
             }
+
+            if (d.DeclaringModule != s.DeclaringModule.Stub) {
+                return; // Do not change unrelated types.
+            }
+
             // Documentation and location are always get transferred from module type
             // to the stub type and never the other way around. This makes sure that
             // we show documentation from the original module and goto definition
@@ -368,7 +383,7 @@ private static void TransferDocumentationAndLocation(IPythonType s, IPythonType
                 var transferDoc = true;
                 if (src is PythonClassType srcClass && dst is PythonClassType dstClass) {
                     // Higher lever source wins
-                    if(srcClass.DocumentationSource == PythonClassType.ClassDocumentationSource.Class ||
+                    if (srcClass.DocumentationSource == PythonClassType.ClassDocumentationSource.Class ||
                        (srcClass.DocumentationSource == PythonClassType.ClassDocumentationSource.Init && dstClass.DocumentationSource == PythonClassType.ClassDocumentationSource.Base)) {
                         dstClass.SetDocumentation(srcClass.Documentation);
                         transferDoc = false;

src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs

@@ -54,7 +54,6 @@ public void EvaluateClass() {
 
                 var bases = ProcessBases();
                 _class.SetBases(bases);
-                _class.DecideGeneric();
                 // Declare __class__ variable in the scope.
                 Eval.DeclareVariable("__class__", _class, VariableSource.Declaration);
                 ProcessClassBody();

src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs

@@ -73,7 +73,7 @@ public override void Evaluate() {
                             v => v.GetPythonType<IPythonClassType>() == null &&
                                  v.GetPythonType<IPythonFunctionType>() == null)
                         ) {
-                        ((VariableCollection)Eval.CurrentScope.Variables).Clear();
+                            ((VariableCollection)Eval.CurrentScope.Variables).Clear();
                     }
                 }
             }
@@ -129,7 +129,7 @@ public override bool Walk(ReturnStatement node) {
                 if (FunctionDefinition.Name.EqualsOrdinal("__init__") && _function.DeclaringType.MemberType == PythonMemberType.Class
                     && !value.IsOfType(BuiltinTypeId.NoneType)) {
 
-                    Eval.ReportDiagnostics(Module.Uri, new Diagnostics.DiagnosticsEntry(
+                    Eval.ReportDiagnostics(Module.Uri, new DiagnosticsEntry(
                             Resources.ReturnInInit,
                             node.GetLocation(Eval).Span,
                             ErrorCodes.ReturnInInit,

src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs

@@ -13,9 +13,7 @@
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
 
-
 using System.Collections.Generic;
-using System.ComponentModel;
 using System.Linq;
 using System.Security.Cryptography.X509Certificates;
 using Microsoft.Python.Analysis.Analyzer.Evaluation;

src/Analysis/Ast/Impl/Extensions/PythonFunctionExtensions.cs

@@ -39,9 +39,9 @@ public static IScope GetScope(this IPythonFunctionType f) {
             return gs?.TraverseBreadthFirst(s => s.Children).FirstOrDefault(s => s.Node == f.FunctionDefinition);
         }
 
-        public static string GetQualifiedName(this IPythonClassMember cm) {
+        public static string GetQualifiedName(this IPythonClassMember cm, string baseName = null) {
             var s = new Stack<string>();
-            s.Push(cm.Name);
+            s.Push(baseName ?? cm.Name);
             for (var p = cm.DeclaringType as IPythonClassMember; p != null; p = p.DeclaringType as IPythonClassMember) {
                 s.Push(p.Name);
             }