Add persistence tests for 60+ system modules

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs

@@ -43,15 +43,18 @@ public void DeclareVariable(string name, IMember value, VariableSource source)
         public void DeclareVariable(string name, IMember value, VariableSource source, IPythonModule module)
             => DeclareVariable(name, value, source, new Location(module));
 
-        public void DeclareVariable(string name, IMember value, VariableSource source, Node location, bool overwrite = false)
+        public void DeclareVariable(string name, IMember value, VariableSource source, Node location, bool overwrite = true)
             => DeclareVariable(name, value, source, GetLocationOfName(location), overwrite);
 
-        public void DeclareVariable(string name, IMember value, VariableSource source, Location location, bool overwrite = false) {
+        public void DeclareVariable(string name, IMember value, VariableSource source, Location location, bool overwrite = true) {
+            var member = GetInScope(name);
+            if (member != null && !overwrite) {
+                return;
+            }
             if (source == VariableSource.Import && value is IVariable v) {
                 CurrentScope.LinkVariable(name, v, location);
                 return;
             }
-            var member = GetInScope(name);
             if (member != null) {
                 if (!value.IsUnknown()) {
                     CurrentScope.DeclareVariable(name, value, source, location);

src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs

@@ -58,18 +58,25 @@ public void HandleAssignment(AssignmentStatement node) {
             }
 
             foreach (var ne in node.Left.OfType<NameExpression>()) {
+                IScope scope;
                 if (Eval.CurrentScope.NonLocals[ne.Name] != null) {
-                    Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Nonlocal);
+                    Eval.LookupNameInScopes(ne.Name, out scope, LookupOptions.Nonlocal);
                     scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne));
                     continue;
                 }
-
                 if (Eval.CurrentScope.Globals[ne.Name] != null) {
-                    Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Global);
+                    Eval.LookupNameInScopes(ne.Name, out scope, LookupOptions.Global);
                     scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne));
                     continue;
                 }
 
+                var m = Eval.LookupNameInScopes(ne.Name, out scope);
+                if(m?.MemberType == PythonMemberType.Class) {
+                    // Ignore assignments to classes: enum.py does Enum = None
+                    // which prevents persistence from restoring proper type from enum:Enum.
+                    continue; 
+                }
+
                 var source = value.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration;
                 Eval.DeclareVariable(ne.Name, value ?? Module.Interpreter.UnknownType, source, Eval.GetLocationOfName(ne));
             }
@@ -97,7 +104,7 @@ private void TryHandleClassVariable(AssignmentStatement node, IMember value) {
                 var cls = m.GetPythonType<IPythonClassType>();
                 if (cls != null) {
                     using (Eval.OpenScope(Eval.Module, cls.ClassDefinition, out _)) {
-                        Eval.DeclareVariable(mex.Name, value, VariableSource.Declaration, Eval.GetLocationOfName(mex), true);
+                        Eval.DeclareVariable(mex.Name, value, VariableSource.Declaration, Eval.GetLocationOfName(mex));
                     }
                 }
             }

src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs

@@ -59,7 +59,7 @@ private void AssignVariables(FromImportStatement node, IImportSearchResult impor
                 // TODO: warn this is not a good style per
                 // TODO: https://docs.python.org/3/faq/programming.html#what-are-the-best-practices-for-using-import-in-a-module
                 // TODO: warn this is invalid if not in the global scope.
-                HandleModuleImportStar(variableModule, imports is ImplicitPackageImport);
+                HandleModuleImportStar(variableModule, imports is ImplicitPackageImport, node.StartIndex);
                 return;
             }
 
@@ -68,14 +68,16 @@ private void AssignVariables(FromImportStatement node, IImportSearchResult impor
                 if (!string.IsNullOrEmpty(memberName)) {
                     var nameExpression = asNames[i] ?? names[i];
                     var variableName = nameExpression?.Name ?? memberName;
-                    var exported = variableModule.Analysis?.GlobalScope.Variables[memberName] ?? variableModule.GetMember(memberName);
+                    var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName];
+                    var exported = variable ?? variableModule.GetMember(memberName);
                     var value = exported ?? GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName);
-                    Eval.DeclareVariable(variableName, value, VariableSource.Import, nameExpression);
+                    // Do not allow imported variables to override local declarations
+                    Eval.DeclareVariable(variableName, value, VariableSource.Import, nameExpression, CanOverwriteVariable(variableName, node.StartIndex));
                 }
             }
         }
 
-        private void HandleModuleImportStar(PythonVariableModule variableModule, bool isImplicitPackage) {
+        private void HandleModuleImportStar(PythonVariableModule variableModule, bool isImplicitPackage, int importPosition) {
             if (variableModule.Module == Module) {
                 // from self import * won't define any new members
                 return;
@@ -100,10 +102,31 @@ private void HandleModuleImportStar(PythonVariableModule variableModule, bool is
                 }
 
                 var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName];
-                Eval.DeclareVariable(memberName, variable ?? member, VariableSource.Import);
+                // Do not allow imported variables to override local declarations
+                Eval.DeclareVariable(memberName, variable ?? member, VariableSource.Import, Eval.DefaultLocation, CanOverwriteVariable(memberName, importPosition));
             }
         }
 
+        private bool CanOverwriteVariable(string name, int importPosition) {
+            var v = Eval.CurrentScope.Variables[name];
+            if(v == null) {
+                return true; // Variable does not exist
+            }
+            // Allow overwrite if import is below the variable. Consider
+            //   x = 1
+            //   x = 2
+            //   from A import * # brings another x
+            //   x = 3
+            var references = v.References.Where(r => r.DocumentUri == Module.Uri).ToArray();
+            if(references.Length == 0) {
+                // No references to the variable in this file - the variable 
+                // is imported from another module. OK to overwrite.
+                return true; 
+            }
+            var firstAssignmentPosition = references.Min(r => r.Span.ToIndexSpan(Ast).Start);
+            return firstAssignmentPosition < importPosition;
+        }
+
         private IMember GetValueFromImports(PythonVariableModule parentModule, IImportChildrenSource childrenSource, string memberName) {
             if (childrenSource == null || !childrenSource.TryGetChildImport(memberName, out var childImport)) {
                 return Interpreter.UnknownType;

src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs

@@ -255,7 +255,7 @@ private void MergeStub() {
                 switch (sourceType) {
                     case null:
                         // Nothing in sources, but there is type in the stub. Declare it.
-                        if (v.Source == VariableSource.Declaration) {
+                        if (v.Source == VariableSource.Declaration || v.Source == VariableSource.Generic) {
                             Eval.DeclareVariable(v.Name, v.Value, v.Source);
                         }
                         break;
@@ -347,6 +347,10 @@ private static bool IsStubBetterType(IPythonType sourceType, IPythonType stubTyp
             if (sourceType.MemberType == PythonMemberType.Function && stubType.MemberType == PythonMemberType.Class) {
                 return true;
             }
+            // Random replaces method (variable) by a function.
+            if (sourceType.MemberType == PythonMemberType.Method && stubType.MemberType == PythonMemberType.Function) {
+                return true;
+            }
             return sourceType.MemberType == stubType.MemberType;
         }
 

src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs

@@ -14,6 +14,25 @@
 // permissions and limitations under the License.
 
 namespace Microsoft.Python.Analysis {
+    public enum AnalysisCachingLevel {
+        /// <summary>
+        /// No caching of analysis.
+        /// </summary>
+        None,
+
+        /// <summary>
+        /// Cache analysis results of system (language) modules.
+        /// Do not cache user-installed modules or site-packages.
+        /// </summary>
+        System,
+
+        /// <summary>
+        /// Full caching, includes system and library modules.
+        /// Does not enable caching of user code analysis.
+        /// </summary>
+        Library
+    }
+
     public class AnalysisOptions {
         public bool LintingEnabled { get; set; }
 
@@ -29,5 +48,10 @@ public class AnalysisOptions {
         /// improve performance when library code has to be re-analyzed.
         /// </summary>
         public bool KeepLibraryAst { get; set; }
+
+        /// <summary>
+        /// Defines level of caching analysis engine will maintain.
+        /// </summary>
+        public AnalysisCachingLevel AnalysisCachingLevel { get; set; }
     }
 }

src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericTypeParameter.cs

@@ -27,5 +27,8 @@ public interface IGenericTypeParameter: IPythonType, IEquatable<IGenericTypePara
         /// </summary>
         /// <remarks>See 'https://docs.python.org/3/library/typing.html#typing.TypeVar'</remarks>
         IReadOnlyList<IPythonType> Constraints { get; }
+        IPythonType Bound { get; }
+        IPythonType Covariant { get; }
+        IPythonType Contravariant { get; }
     }
 }

src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs

@@ -25,15 +25,33 @@
 
 namespace Microsoft.Python.Analysis.Specializations.Typing.Types {
     internal sealed class GenericTypeParameter : PythonType, IGenericTypeParameter {
-        public GenericTypeParameter(string name, IPythonModule declaringModule, IReadOnlyList<IPythonType> constraints, string documentation, IndexSpan location)
-            : base(name, new Location(declaringModule), documentation) {
+        public GenericTypeParameter(
+            string name,
+            IPythonModule declaringModule,
+            IReadOnlyList<IPythonType> constraints,
+            IPythonType bound,
+            IPythonType covariant,
+            IPythonType contravariant,
+            Location location)
+            : base(name, location, GetDocumentation(name, constraints, bound, covariant, contravariant)) {
             Constraints = constraints ?? Array.Empty<IPythonType>();
+            Bound = bound;
+            Covariant = covariant;
+            Contravariant = contravariant;
         }
+
+        #region IGenericTypeParameter
         public IReadOnlyList<IPythonType> Constraints { get; }
+        public IPythonType Bound { get; }
+        public IPythonType Covariant { get; }
+        public IPythonType Contravariant { get; }
+        #endregion
 
+        #region IPythonType
         public override BuiltinTypeId TypeId => BuiltinTypeId.Type;
         public override PythonMemberType MemberType => PythonMemberType.Generic;
         public override bool IsSpecialized => true;
+        #endregion
 
         private static bool TypeVarArgumentsValid(IArgumentSet argSet) {
             var args = argSet.Arguments;
@@ -90,23 +108,29 @@ public static IPythonType FromTypeVar(IArgumentSet argSet, IPythonModule declari
                 // Type constraints may be specified as type name strings.
                 var typeString = (a as IPythonConstant)?.GetString();
                 return !string.IsNullOrEmpty(typeString) ? argSet.Eval.GetTypeFromString(typeString) : a.GetPythonType();
-            }).ToArray() ?? Array.Empty<IPythonType>();
+            }).ToArray();
+
+            var bound = args.Where(a => a.Name == "bound").Select(a => a.Value as IPythonType).FirstOrDefault();
+            var covariant = args.Where(a => a.Name == "covariant").Select(a => a.Value as IPythonType).FirstOrDefault();
+            var contravariant = args.Where(a => a.Name == "contravariant").Select(a => a.Value as IPythonType).FirstOrDefault();
 
-            var documentation = GetDocumentation(args, constraints);
-            return new GenericTypeParameter(name, declaringModule, constraints, documentation, location);
+            return new GenericTypeParameter(name, declaringModule, constraints, bound, covariant, contravariant, new Location(declaringModule, location));
         }
 
-        private static string GetDocumentation(IReadOnlyList<IArgument> args, IReadOnlyList<IPythonType> constraints) {
-            var name = (args[0].Value as IPythonConstant).GetString();
-            var keyWordArgs = args.Skip(1)
-                .Where(x => !x.ValueIsDefault)
-                .Select(x => $"{x.Name}={(x.Value as IPythonConstant)?.Value}");
+        private static string GetDocumentation(string name, IReadOnlyList<IPythonType> constraints, IPythonType bound, IPythonType covariant, IPythonType contravariant) {
+            var constaintStrings = constraints != null ? constraints.Select(c => c.IsUnknown() ? "?" : c.Name) : Enumerable.Empty<string>();
+            var boundStrings = bound.IsUnknown() ? Enumerable.Empty<string>() : Enumerable.Repeat($"bound={bound.Name}", 1);
+            var covariantStrings = covariant.IsUnknown() ? Enumerable.Empty<string>() : Enumerable.Repeat($"covariant={covariant.Name}", 1);
+            var contravariantStrings = contravariant.IsUnknown() ? Enumerable.Empty<string>() : Enumerable.Repeat($"contravariant={contravariant.Name}", 1);
+
+            var docArgs = Enumerable.Repeat($"'{name}'", 1)
+                .Concat(constaintStrings).Concat(boundStrings).Concat(covariantStrings).Concat(contravariantStrings);
 
-            var docArgs = constraints.Select(c => c.IsUnknown() ? "?" : c.Name).Concat(keyWordArgs).Prepend($"'{name}'");
             var documentation = CodeFormatter.FormatSequence("TypeVar", '(', docArgs);
             return documentation;
         }
 
-        public bool Equals(IGenericTypeParameter other) => Name.Equals(other.Name);
+        public bool Equals(IGenericTypeParameter other) => Name.Equals(other?.Name);
     }
+
 }

src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs

@@ -103,6 +103,8 @@ private void SpecializeMembers() {
             _members["ValuesView"] = new SpecializedGenericType("ValuesView", CreateValuesViewType, this);
             _members["ItemsView"] = new SpecializedGenericType("ItemsView", CreateItemsViewType, this);
 
+            _members["AbstractSet"] = new SpecializedGenericType("AbstractSet",
+                typeArgs => CreateListType("AbstractSet", BuiltinTypeId.Set, typeArgs, true), this);
             _members["Set"] = new SpecializedGenericType("Set",
                 typeArgs => CreateListType("Set", BuiltinTypeId.Set, typeArgs, true), this);
             _members["MutableSet"] = new SpecializedGenericType("MutableSet",

src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs

@@ -15,6 +15,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using Microsoft.Python.Analysis.Specializations.Typing;
 using Microsoft.Python.Analysis.Utilities;
@@ -23,13 +24,14 @@
 
 namespace Microsoft.Python.Analysis.Types {
     internal partial class PythonClassType {
+        private readonly object _genericParameterLock = new object();
+        private readonly ReentrancyGuard<IPythonClassType> _genericSpecializationGuard = new ReentrancyGuard<IPythonClassType>();
+        private readonly ReentrancyGuard<IPythonClassType> _genericResolutionGuard = new ReentrancyGuard<IPythonClassType>();
+
         private bool _isGeneric;
-        private object _genericParameterLock = new object();
         private Dictionary<string, PythonClassType> _specificTypeCache;
         private Dictionary<IGenericTypeParameter, IPythonType> _genericParameters;
         private IReadOnlyList<IGenericTypeParameter> _parameters = new List<IGenericTypeParameter>();
-        private ReentrancyGuard<IPythonClassType> _genericSpecializationGuard = new ReentrancyGuard<IPythonClassType>();
-        private ReentrancyGuard<IPythonClassType> _genericResolutionGuard = new ReentrancyGuard<IPythonClassType>();
 
         #region IGenericType
         /// <summary>
@@ -66,7 +68,7 @@ public IPythonType CreateSpecificType(IArgumentSet args) {
                 // type parameter T -> int, U -> str, etc.
                 var genericTypeToSpecificType = GetSpecificTypes(args, genericTypeParameters, newBases);
 
-                PythonClassType classType = new PythonClassType(BaseName, new Location(DeclaringModule));
+                var classType = new PythonClassType(BaseName, new Location(DeclaringModule));
                 // Storing generic parameters allows methods returning generic types 
                 // to know what type parameter returns what specific type
                 StoreGenericParameters(classType, genericTypeParameters, genericTypeToSpecificType);

src/Analysis/Ast/Impl/Types/PythonClassType.cs

@@ -40,7 +40,7 @@ internal enum ClassDocumentationSource {
 
         private readonly ReentrancyGuard<IPythonClassType> _memberGuard = new ReentrancyGuard<IPythonClassType>();
         private string _genericName;
-        private List<IPythonType> _bases;
+        private List<IPythonType> _bases = new List<IPythonType>();
         private IReadOnlyList<IPythonType> _mro;
         private string _documentation;
 
@@ -187,7 +187,7 @@ public IReadOnlyList<IPythonType> Mro {
                 if (_mro != null) {
                     return _mro;
                 }
-                if (_bases == null) {
+                if (_bases.Count == 0) {
                     return new IPythonType[] { this };
                 }
                 _mro = new IPythonType[] { this };
@@ -211,7 +211,7 @@ public IReadOnlyList<IPythonType> Mro {
         internal override void SetDocumentation(string documentation) => _documentation = documentation;
 
         internal void SetBases(IEnumerable<IPythonType> bases) {
-            if (_bases != null) {
+            if (_bases.Count > 0) {
                 return; // Already set
             }
 

src/Analysis/Ast/Impl/Types/PythonType.cs

@@ -64,13 +64,12 @@ public override void AddReference(Location location) {
 
         #region IPythonType
 
-        public virtual string Name => TypeId == BuiltinTypeId.Ellipsis ? "..." : BaseName;
-
         public virtual string QualifiedName
             => DeclaringModule.ModuleType == ModuleType.Builtins
                        ? TypeId == BuiltinTypeId.Ellipsis ? "ellipsis" : Name
                        : this.GetQualifiedName();
 
+        public virtual string Name => TypeId == BuiltinTypeId.Ellipsis ? "..." : BaseName;
         public virtual string Documentation { get; private set; }
         public virtual BuiltinTypeId TypeId => _typeId;
         public string BaseName { get; }

src/Analysis/Ast/Impl/Utilities/ReentrancyGuard.cs

@@ -40,7 +40,7 @@ public IDisposable Push(T t, out bool reentered) {
 
         public void Pop() {
             _stack.Value.Pop();
-            if(_stack.Value.Count == 0) {
+            if (_stack.Value.Count == 0) {
                 _stack.Value = null;
             }
         }

src/Caching/Impl/Extensions/IndexSpanExtensions.cs

@@ -14,11 +14,13 @@
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
 
+using System.Diagnostics;
 using Microsoft.Python.Analysis.Caching.Models;
 using Microsoft.Python.Core.Text;
 
 namespace Microsoft.Python.Analysis.Caching {
     internal static class IndexSpanExtensions {
+        [DebuggerStepThrough]
         public static IndexSpanModel ToModel(this IndexSpan span) => new IndexSpanModel {
             Start = span.Start,
             Length = span.Length