Treat persistent module as regular specialized (#1334)

* Remove old qualified name

* Node storage

* Class and scope to use AST map

* Library analysis

* Fix SO

* Keep small AST with imports

* AST reduction

* Final field

* Initial

* Reload

* Ignore post-final requests

* Drop AST

* Remove local variables

* Test fixes

* Fix overload match

* Tests

* Add locks

* Remove local variables

* Drop file content to save memory

* Cache PEP hints

* Recreate AST

* Fix specialization

* Fix locations

* usings

* Test fixes

* Add options to keep data in memory

* Fix test

* Fix lambda parameters

* Fix argument set
Fix global scope node

* Fix overload doc

* Fix stub merge errors

* Fix async issues

* Undo some changes

* Fix test

* Fix race condition

* Partial

* Models and views

* Restore log null checks

* Fix merge conflict

* Fix merge issue

* Null check

* Partial

* Partial

* Partial

* Fix test

* Partial

* Partial

* First test

* Baseline comparison

* Builtins

* Partial

* Type fixes

* Fix type names, part I

* Qualified name

* Properly write variables

* Partial

* Construct module from model

* Test

* Variable creations

* Factories

* Factories

* Split construction

* Restore

* Save builtins

* Test passes

* Qualified name

* Better export detection

* Test fixes

* More consistent qualified names

* Sys test

* Demo

* Complete sys write/read

* Partial

* Partial

* Test staility

* Perf bug

* Baseline, remove debug code, deactivate db

* Test fixes

* Test fix

* Simplify a bit

* Baselines and use : separator

* Baselines

* PR feedback

* Merge master

* Remove registry reference

* PR feedback

* PR feedback

* Restore persistence + update test

* Better handle persistent module in dependencies

* Undo

* Fix merge issue

* Remove debug code

Author: Mikhail Arkhipov

src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs

@@ -34,7 +34,7 @@ private enum KeyType { Default, Typeshed, LibraryAsDocument }
         public AnalysisModuleKey(IPythonModule module) {
             Name = module.Name;
             FilePath = module.ModuleType == ModuleType.CompiledBuiltin ? null : module.FilePath;
-            _type = module is StubPythonModule stub && stub.IsTypeshed
+            _type = module is StubPythonModule stub && stub.IsTypeshed              
                 ? KeyType.Typeshed
                 : module.ModuleType == ModuleType.Library && module is IDocument document && document.IsOpen
                     ? KeyType.LibraryAsDocument

src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs

@@ -20,7 +20,6 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Python.Analysis.Core.DependencyResolution;
-using Microsoft.Python.Analysis.Documents;
 using Microsoft.Python.Analysis.Modules;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Core;
@@ -264,8 +263,8 @@ private HashSet<AnalysisModuleKey> FindDependencies(IPythonModule module, Python
             return dependencies;
         }
 
-        private static bool Ignore(IModuleManagement moduleResolution, string name)
-            => moduleResolution.BuiltinModuleName.EqualsOrdinal(name) || moduleResolution.GetSpecializedModule(name) != null;
+        private static bool Ignore(IModuleManagement moduleResolution, string fullName, string modulePath)
+            => moduleResolution.BuiltinModuleName.EqualsOrdinal(fullName) || moduleResolution.GetSpecializedModule(fullName, modulePath) != null;
 
         private void UpdateAnalysisTcs(int analysisVersion) {
             _analysisVersion = analysisVersion;
@@ -332,10 +331,10 @@ public override bool Walk(FromImportStatement fromImport) {
 
             private void HandleSearchResults(IImportSearchResult searchResult) {
                 switch (searchResult) {
-                    case ModuleImport moduleImport when !Ignore(_moduleResolution, moduleImport.FullName):
+                    case ModuleImport moduleImport when !Ignore(_moduleResolution, moduleImport.FullName, moduleImport.ModulePath):
                         Dependencies.Add(new AnalysisModuleKey(moduleImport.FullName, moduleImport.ModulePath, _isTypeshed));
                         return;
-                    case PossibleModuleImport possibleModuleImport when !Ignore(_moduleResolution, possibleModuleImport.PrecedingModuleFullName):
+                    case PossibleModuleImport possibleModuleImport when !Ignore(_moduleResolution, possibleModuleImport.PrecedingModuleFullName, possibleModuleImport.PrecedingModulePath):
                         Dependencies.Add(new AnalysisModuleKey(possibleModuleImport.PrecedingModuleFullName, possibleModuleImport.PrecedingModulePath, _isTypeshed));
                         return;
                     default:

src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs

@@ -47,6 +47,7 @@ internal sealed class PythonAnalyzerSession {
         private readonly IPythonAnalyzer _analyzer;
         private readonly ILogger _log;
         private readonly ITelemetryService _telemetry;
+        private readonly IModuleDatabaseService _moduleDatabaseService;
 
         private State _state;
         private bool _isCanceled;
@@ -86,6 +87,7 @@ public PythonAnalyzerSession(IServiceManager services,
             _analyzer = _services.GetService<IPythonAnalyzer>();
             _log = _services.GetService<ILogger>();
             _telemetry = _services.GetService<ITelemetryService>();
+            _moduleDatabaseService = _services.GetService<IModuleDatabaseService>();
             _progress = progress;
         }
 
@@ -297,7 +299,6 @@ private void AnalyzeEntry() {
                 }
 
                 var startTime = stopWatch?.Elapsed ?? TimeSpan.Zero;
-
                 AnalyzeEntry(null, _entry, module, ast, Version);
 
                 LogCompleted(null, module, stopWatch, startTime);
@@ -383,8 +384,12 @@ private IDocumentAnalysis CreateAnalysis(IDependencyChainNode<PythonAnalyzerEntr
 
             ast.Reduce(x => x is ImportStatement || x is FromImportStatement);
             document.SetAst(ast);
+
             var eval = new ExpressionEval(walker.Eval.Services, document, ast);
-            return new LibraryAnalysis(document, version, walker.GlobalScope, eval, walker.StarImportMemberNames);
+            var analysis  = new LibraryAnalysis(document, version, walker.GlobalScope, eval, walker.StarImportMemberNames);
+
+            _moduleDatabaseService?.StoreModuleAnalysisAsync(analysis, CancellationToken.None).DoNotWait();
+            return analysis;
         }
 
         private enum State {

src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs

@@ -26,7 +26,6 @@ public static LocationInfo GetLocation(this Node node, IExpressionEvaluator eval
 
             return GetLocation(node, eval.Ast, eval.Module);
         }
-
         public static LocationInfo GetLocation(this Node node, IDocumentAnalysis analysis) {
             if (node == null || node.StartIndex >= node.EndIndex) {
                 return LocationInfo.Empty;

src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs

@@ -56,7 +56,7 @@ public interface IModuleManagement: IModuleResolution {
         /// <summary>
         /// Returns specialized module, if any.
         /// </summary>
-        IPythonModule GetSpecializedModule(string name);
+        IPythonModule GetSpecializedModule(string name, string modulePath = null);
 
         /// <summary>
         /// Root directory of the path resolver.

src/Analysis/Ast/Impl/Modules/PythonModule.cs

@@ -79,6 +79,7 @@ protected PythonModule(string name, ModuleType moduleType, IServiceContainer ser
             Log = services.GetService<ILogger>();
             Interpreter = services.GetService<IPythonInterpreter>();
             Analysis = new EmptyAnalysis(services, this);
+            GlobalScope = Analysis.GlobalScope;
 
             _diagnosticsService = services.GetService<IDiagnosticsService>();
             SetDeclaringModule(this);
@@ -150,10 +151,10 @@ public virtual string Documentation {
         #endregion
 
         #region IMemberContainer
-        public virtual IMember GetMember(string name) => Analysis.GlobalScope.Variables[name]?.Value;
+        public virtual IMember GetMember(string name) => GlobalScope.Variables[name]?.Value;
         public virtual IEnumerable<string> GetMemberNames() {
             // drop imported modules and typing.
-            return Analysis.GlobalScope.Variables
+            return GlobalScope.Variables
                 .Where(v => {
                     // Instances are always fine.
                     if (v.Value is IPythonInstance) {
@@ -209,6 +210,8 @@ protected virtual void Dispose(bool disposing) {
             _disposeToken.TryMarkDisposed();
             var analyzer = Services.GetService<IPythonAnalyzer>();
             analyzer.RemoveAnalysis(this);
+            _parseCts?.Dispose();
+            _linkedParseCts?.Dispose();
         }
         #endregion
 

src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs

@@ -36,6 +36,7 @@
 namespace Microsoft.Python.Analysis.Modules.Resolution {
     internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManagement {
         private readonly ConcurrentDictionary<string, IPythonModule> _specialized = new ConcurrentDictionary<string, IPythonModule>();
+        private IModuleDatabaseService _dbService;
         private IRunningDocumentTable _rdt;
 
         public MainModuleResolution(string root, IServiceContainer services)
@@ -76,11 +77,6 @@ protected override IPythonModule CreateModule(string name) {
                 }
             }
 
-            var dbs = _services.GetService<IModuleDatabaseService>();
-            if (dbs != null && dbs.TryCreateModule(name, moduleImport.ModulePath, out var m) != ModuleStorageState.DoesNotExist && m != null) {
-                return m;
-            }
-
             // If there is a stub, make sure it is loaded and attached
             // First check stub next to the module.
             if (!TryCreateModuleStub(name, moduleImport.ModulePath, out var stub)) {
@@ -156,8 +152,17 @@ public IPythonModule SpecializeModule(string name, Func<string, IPythonModule> s
         /// <summary>
         /// Returns specialized module, if any.
         /// </summary>
-        public IPythonModule GetSpecializedModule(string name)
-        => _specialized.TryGetValue(name, out var module) ? module : null;
+        public IPythonModule GetSpecializedModule(string fullName, string modulePath = null) {
+            if (_specialized.TryGetValue(fullName, out var module)) {
+                return module;
+            }
+            var dbs = GetDbService();
+            if (dbs != null && dbs.TryCreateModule(fullName, modulePath, out module) != ModuleStorageState.DoesNotExist && module != null) {
+                SpecializeModule(fullName, s => module);
+                return module;
+            }
+            return null;
+        }
 
         internal async Task LoadBuiltinTypesAsync(CancellationToken cancellationToken = default) {
             var analyzer = _services.GetService<IPythonAnalyzer>();
@@ -237,5 +242,8 @@ private bool TryCreateModuleStub(string name, string modulePath, out IPythonModu
 
         private IRunningDocumentTable GetRdt()
             => _rdt ?? (_rdt = _services.GetService<IRunningDocumentTable>());
+
+        private IModuleDatabaseService GetDbService()
+            => _dbService ?? (_dbService = _services.GetService<IModuleDatabaseService>());
     }
 }

src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs

@@ -21,7 +21,6 @@
 using Microsoft.Python.Analysis.Caching;
 using Microsoft.Python.Analysis.Core.DependencyResolution;
 using Microsoft.Python.Analysis.Core.Interpreter;
-using Microsoft.Python.Analysis.Documents;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Core;
 using Microsoft.Python.Core.IO;

src/Analysis/Ast/Impl/Types/PythonPropertyType.cs

@@ -51,9 +51,9 @@ public string Description {
         }
 
         public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args)
-            => _getter?.Call(args, instance?.GetPythonType() ?? DeclaringType);
+                => _getter.Call(args, instance?.GetPythonType() ?? DeclaringType);
 
-        public IMember ReturnType => _getter?.Call(ArgumentSet.WithoutContext, DeclaringType)?.GetPythonType();
+        public IMember ReturnType => _getter?.Call(ArgumentSet.WithoutContext, DeclaringType);
         #endregion
 
         internal void AddOverload(IPythonFunctionOverload overload) => _getter = _getter ?? overload;

src/Caching/Test/Files/Requests.json

@@ -208,7 +208,7 @@
       "Name": "Session"
     },
     {
-      "Value": "i:typing:Any",
+      "Value": "i:requests.structures:LookupDict",
       "Id": 753305199,
       "Name": "codes"
     },