Fix caching of compiled modules, analysis hang

src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs

@@ -212,7 +212,7 @@ public IReadOnlyList<IPythonModule> LoadedModules {
         public IDocumentAnalysis TryRestoreCachedAnalysis(IPythonModule module) {
             var moduleType = module.ModuleType;
             var moduleDatabaseService = _services.GetService<IModuleDatabaseService>();
-            if (!moduleType.CanBeCached() || moduleDatabaseService == null || !moduleDatabaseService.ModuleExistsInStorage(module.Name, module.FilePath)) {
+            if (!moduleType.CanBeCached() || moduleDatabaseService == null || !moduleDatabaseService.ModuleExistsInStorage(module.Name, module.FilePath, moduleType)) {
                 return null;
             }
 

src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs

@@ -16,6 +16,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Python.Analysis.Dependencies;
+using Microsoft.Python.Analysis.Modules;
 using Microsoft.Python.Analysis.Types;
 
 namespace Microsoft.Python.Analysis.Caching {
@@ -43,6 +44,6 @@ internal interface IModuleDatabaseService: IModuleDatabaseCache {
         /// <summary>
         /// Determines if module analysis exists in the storage.
         /// </summary>
-        bool ModuleExistsInStorage(string moduleName, string filePath);
+        bool ModuleExistsInStorage(string moduleName, string filePath, ModuleType moduleType);
     }
 }

src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs

@@ -84,8 +84,13 @@ protected override IPythonModule CreateModule(string name) {
                 }
             }
 
+            var moduleType = moduleImport.IsBuiltin ? ModuleType.CompiledBuiltin
+                : moduleImport.IsCompiled ? ModuleType.Compiled
+                : moduleImport.IsLibrary ? ModuleType.Library
+                : ModuleType.User;
+
             var dbs = GetDbService();
-            moduleImport.IsPersistent = dbs != null && dbs.ModuleExistsInStorage(name, moduleImport.ModulePath);
+            moduleImport.IsPersistent = dbs != null && dbs.ModuleExistsInStorage(name, moduleImport.ModulePath, moduleType);
 
             IPythonModule stub = null;
             if (!moduleImport.IsPersistent) {
@@ -111,15 +116,15 @@ protected override IPythonModule CreateModule(string name) {
 
             if (moduleImport.IsCompiled) {
                 Log?.Log(TraceEventType.Verbose, "Create compiled (scraped): ", moduleImport.FullName, moduleImport.ModulePath, moduleImport.RootPath);
-                return new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, moduleImport.IsPersistent, false, Services);
+                return new CompiledPythonModule(moduleImport.FullName, moduleType, moduleImport.ModulePath, stub, moduleImport.IsPersistent, false, Services);
             }
 
             Log?.Log(TraceEventType.Verbose, "Import: ", moduleImport.FullName, moduleImport.ModulePath);
             // Module inside workspace == user code.
 
             var mco = new ModuleCreationOptions {
                 ModuleName = moduleImport.FullName,
-                ModuleType = moduleImport.IsLibrary ? ModuleType.Library : ModuleType.User,
+                ModuleType = moduleType,
                 FilePath = moduleImport.ModulePath,
                 Stub = stub,
                 IsPersistent = moduleImport.IsPersistent

src/Caching/Impl/ModuleDatabase.cs

@@ -14,6 +14,7 @@
 // permissions and limitations under the License.
 
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
@@ -32,8 +33,7 @@
 
 namespace Microsoft.Python.Analysis.Caching {
     internal sealed class ModuleDatabase : IModuleDatabaseService {
-        private readonly Dictionary<string, IDependencyProvider> _dependencies = new Dictionary<string, IDependencyProvider>();
-        private readonly object _lock = new object();
+        private readonly ConcurrentDictionary<string, IDependencyProvider> _dependencies = new ConcurrentDictionary<string, IDependencyProvider>();
 
         private readonly IServiceContainer _services;
         private readonly ILogger _log;
@@ -49,7 +49,7 @@ public ModuleDatabase(IServiceContainer services) {
         }
 
         public string CacheFolderBaseName => "analysis.v";
-        public int DatabaseFormatVersion => 3;
+        public int DatabaseFormatVersion => 4;
         public string CacheFolder { get; }
 
         /// <summary>
@@ -64,15 +64,14 @@ public bool TryRestoreDependencies(IPythonModule module, out IDependencyProvider
                 return false;
             }
 
-            lock (_lock) {
-                if (_dependencies.TryGetValue(module.Name, out dp)) {
-                    return true;
-                }
-                if (FindModuleModel(module.Name, module.FilePath, out var model)) {
-                    dp = new DependencyProvider(module, model);
-                    _dependencies[module.Name] = dp;
-                    return true;
-                }
+            if (_dependencies.TryGetValue(module.Name, out dp)) {
+                return true;
+            }
+
+            if (FindModuleModel(module.Name, module.FilePath, module.ModuleType, out var model)) {
+                dp = new DependencyProvider(module, model);
+                _dependencies[module.Name] = dp;
+                return true;
             }
             return false;
         }
@@ -90,10 +89,8 @@ public bool TryRestoreGlobalScope(IPythonModule module, out IRestoredGlobalScope
                 return false;
             }
 
-            lock (_lock) {
-                if (FindModuleModel(module.Name, module.FilePath, out var model)) {
-                    gs = new RestoredGlobalScope(model, module);
-                }
+            if (FindModuleModel(module.Name, module.FilePath, module.ModuleType, out var model)) {
+                gs = new RestoredGlobalScope(model, module);
             }
 
             return gs != null;
@@ -108,14 +105,14 @@ public Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationTok
         /// <summary>
         /// Determines if module analysis exists in the storage.
         /// </summary>
-        public bool ModuleExistsInStorage(string moduleName, string filePath) {
+        public bool ModuleExistsInStorage(string moduleName, string filePath, ModuleType moduleType) {
             if (GetCachingLevel() == AnalysisCachingLevel.None) {
                 return false;
             }
 
             for (var retries = 50; retries > 0; --retries) {
                 try {
-                    var dbPath = FindDatabaseFile(moduleName, filePath);
+                    var dbPath = FindDatabaseFile(moduleName, filePath, moduleType);
                     return !string.IsNullOrEmpty(dbPath);
                 } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) {
                     Thread.Sleep(10);
@@ -172,9 +169,9 @@ private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken c
         /// by name, version, current Python interpreter version and/or hash of the
         /// module content (typically file sizes).
         /// </summary>
-        private string FindDatabaseFile(string moduleName, string filePath) {
+        private string FindDatabaseFile(string moduleName, string filePath, ModuleType moduleType) {
             var interpreter = _services.GetService<IPythonInterpreter>();
-            var uniqueId = ModuleUniqueId.GetUniqueId(moduleName, filePath, ModuleType.Specialized, _services, GetCachingLevel());
+            var uniqueId = ModuleUniqueId.GetUniqueId(moduleName, filePath, moduleType, _services, GetCachingLevel());
             if (string.IsNullOrEmpty(uniqueId)) {
                 return null;
             }
@@ -199,15 +196,15 @@ private string FindDatabaseFile(string moduleName, string filePath) {
             return _fs.FileExists(dbPath) ? dbPath : null;
         }
 
-        private bool FindModuleModel(string moduleName, string filePath, out ModuleModel model) {
+        private bool FindModuleModel(string moduleName, string filePath, ModuleType moduleType, out ModuleModel model) {
             model = null;
 
             // We don't cache results here. Module resolution service decides when to call in here
             // and it is responsible of overall management of the loaded Python modules.
             for (var retries = 50; retries > 0; --retries) {
                 try {
                     // TODO: make combined db rather than per module?
-                    var dbPath = FindDatabaseFile(moduleName, filePath);
+                    var dbPath = FindDatabaseFile(moduleName, filePath, moduleType);
                     if (string.IsNullOrEmpty(dbPath)) {
                         return false;
                     }

src/Caching/Impl/ModuleUniqueId.cs

@@ -78,7 +78,7 @@ public static string GetUniqueId(string moduleName, string filePath, ModuleType
             }
 
             var config = interpreter.Configuration;
-            if (moduleType.IsCompiled() || string.IsNullOrEmpty(filePath) || modulePathType == PythonLibraryPathType.StdLib) {
+            if (moduleType == ModuleType.CompiledBuiltin || string.IsNullOrEmpty(filePath) || modulePathType == PythonLibraryPathType.StdLib) {
                 // If module is a standard library, unique id is its name + interpreter version.
                 return $"{moduleName}({config.Version.Major}.{config.Version.Minor})";
             }