Persistent analysis, part I

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs

@@ -17,7 +17,6 @@
 using System.Diagnostics;
 using System.Linq;
 using Microsoft.Python.Analysis.Documents;
-using Microsoft.Python.Analysis.Extensions;
 using Microsoft.Python.Analysis.Modules;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Values;

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs

@@ -62,7 +62,7 @@ public IMember GetValueFromList(ListExpression expression) {
                 var value = GetValueFromExpression(item) ?? UnknownType;
                 contents.Add(value);
             }
-            return PythonCollectionType.CreateList(Module.Interpreter, contents, exact: expression.Items.Count <= MaxCollectionSize);
+            return PythonCollectionType.CreateList(Module, contents, exact: expression.Items.Count <= MaxCollectionSize);
         }
 
         public IMember GetValueFromDictionary(DictionaryExpression expression) {
@@ -72,7 +72,7 @@ public IMember GetValueFromDictionary(DictionaryExpression expression) {
                 var value = GetValueFromExpression(item.SliceStop) ?? UnknownType;
                 contents[key] = value;
             }
-            return new PythonDictionary(Interpreter, contents, exact: expression.Items.Count <= MaxCollectionSize);
+            return new PythonDictionary(Module, contents, exact: expression.Items.Count <= MaxCollectionSize);
         }
 
         private IMember GetValueFromTuple(TupleExpression expression) {
@@ -81,7 +81,7 @@ private IMember GetValueFromTuple(TupleExpression expression) {
                 var value = GetValueFromExpression(item) ?? UnknownType;
                 contents.Add(value);
             }
-            return PythonCollectionType.CreateTuple(Module.Interpreter, contents, exact: expression.Items.Count <= MaxCollectionSize);
+            return PythonCollectionType.CreateTuple(Module, contents, exact: expression.Items.Count <= MaxCollectionSize);
         }
 
         public IMember GetValueFromSet(SetExpression expression) {
@@ -90,7 +90,7 @@ public IMember GetValueFromSet(SetExpression expression) {
                 var value = GetValueFromExpression(item) ?? UnknownType;
                 contents.Add(value);
             }
-            return PythonCollectionType.CreateSet(Interpreter, contents, exact: expression.Items.Count <= MaxCollectionSize);
+            return PythonCollectionType.CreateSet(Module, contents, exact: expression.Items.Count <= MaxCollectionSize);
         }
 
         public IMember GetValueFromGenerator(GeneratorExpression expression) {
@@ -110,14 +110,14 @@ public IMember GetValueFromComprehension(Comprehension node) {
                 switch (node) {
                     case ListComprehension lc:
                         var v1 = GetValueFromExpression(lc.Item) ?? UnknownType;
-                        return PythonCollectionType.CreateList(Interpreter, new[] { v1 });
+                        return PythonCollectionType.CreateList(Module, new[] { v1 });
                     case SetComprehension sc:
                         var v2 = GetValueFromExpression(sc.Item) ?? UnknownType;
-                        return PythonCollectionType.CreateSet(Interpreter, new[] { v2 });
+                        return PythonCollectionType.CreateSet(Module, new[] { v2 });
                     case DictionaryComprehension dc:
                         var k = GetValueFromExpression(dc.Key) ?? UnknownType;
                         var v = GetValueFromExpression(dc.Value) ?? UnknownType;
-                        return new PythonDictionary(new PythonDictionaryType(Interpreter), new Dictionary<IMember, IMember> { { k, v } });
+                        return new PythonDictionary(new PythonDictionaryType(Interpreter.ModuleResolution.BuiltinsModule), new Dictionary<IMember, IMember> { { k, v } });
                 }
 
                 return UnknownType;

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs

@@ -106,9 +106,9 @@ private IMember GetValueFromBinaryOp(Expression expr) {
 
                 switch (leftTypeId) {
                     case BuiltinTypeId.List:
-                        return PythonCollectionType.CreateConcatenatedList(Module.Interpreter, lc, rc);
+                        return PythonCollectionType.CreateConcatenatedList(Module, lc, rc);
                     case BuiltinTypeId.Tuple:
-                        return PythonCollectionType.CreateConcatenatedTuple(Module.Interpreter, lc, rc);
+                        return PythonCollectionType.CreateConcatenatedTuple(Module, lc, rc);
                 }
             }
 

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs

@@ -59,6 +59,7 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, GlobalSc
         public ModuleSymbolTable SymbolTable { get; } = new ModuleSymbolTable();
         public IPythonType UnknownType => Interpreter.UnknownType;
         public Location DefaultLocation { get; }
+        public IPythonModule BuiltinsModule => Interpreter.ModuleResolution.BuiltinsModule;
 
         public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(Module) ?? LocationInfo.Empty;
 

src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs

@@ -174,7 +174,7 @@ private PythonVariableModule GetOrCreateVariableModule(in string fullName, in Py
                 return variableModule;
             }
 
-            variableModule = new PythonVariableModule(fullName, Interpreter);
+            variableModule = new PythonVariableModule(fullName, Eval.Interpreter);
             _variableModules[fullName] = variableModule;
             parentModule?.AddChildModule(memberName, variableModule);
             return variableModule;

src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs

@@ -107,7 +107,7 @@ private void HandleAllAppendExtend(CallExpression node) {
             IPythonCollection values = null;
             switch (me.Name) {
                 case "append":
-                    values = PythonCollectionType.CreateList(Module.Interpreter, new List<IMember> { v }, exact: true);
+                    values = PythonCollectionType.CreateList(Module, new List<IMember> { v }, exact: true);
                     break;
                 case "extend":
                     values = v as IPythonCollection;
@@ -129,7 +129,7 @@ private void ExtendAll(Node location, IPythonCollection values) {
             }
 
             var all = scope.Variables[AllVariableName]?.Value as IPythonCollection;
-            var list = PythonCollectionType.CreateConcatenatedList(Module.Interpreter, all, values);
+            var list = PythonCollectionType.CreateConcatenatedList(Module, all, values);
             var source = list.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration;
 
             Eval.DeclareVariable(AllVariableName, list, source, location);
@@ -228,6 +228,8 @@ private void MergeStub() {
                 return;
             }
 
+            var builtins = Module.Interpreter.ModuleResolution.BuiltinsModule;
+
             // Note that scrape can pick up more functions than the stub contains
             // Or the stub can have definitions that scraping had missed. Therefore
             // merge is the combination of the two with the documentation coming
@@ -258,6 +260,10 @@ private void MergeStub() {
 
                         var memberType = member?.GetPythonType();
                         var stubMemberType = stubMember.GetPythonType();
+
+                        if (builtins.Equals(memberType?.DeclaringModule) || builtins.Equals(stubMemberType.DeclaringModule)) {
+                            continue; // Leave builtins alone.
+                        }
                         if (!IsStubBetterType(memberType, stubMemberType)) {
                             continue;
                         }
@@ -271,7 +277,7 @@ private void MergeStub() {
                     // Re-declare variable with the data from the stub unless member is a module.
                     // Modules members that are modules should remain as they are, i.e. os.path
                     // should remain library with its own stub attached.
-                    if (!(stubType is IPythonModule)) {
+                    if (!(stubType is IPythonModule) && !builtins.Equals(stubType.DeclaringModule)) {
                         sourceType.TransferDocumentationAndLocation(stubType);
                         // TODO: choose best type between the scrape and the stub. Stub probably should always win.
                         var source = Eval.CurrentScope.Variables[v.Name]?.Source ?? VariableSource.Declaration;

src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs

@@ -56,7 +56,9 @@ public PythonAnalyzer(IServiceManager services, string cacheFolderPath = null) {
             _startNextSession = StartNextSession;
 
             _progress = new ProgressReporter(services.GetService<IProgressService>());
-            _services.AddService(new StubCache(_services, cacheFolderPath));
+
+            _services.AddService(new CacheFolderService(_services, cacheFolderPath));
+            _services.AddService(new StubCache(_services));
         }
 
         public void Dispose() {

src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs

@@ -19,6 +19,7 @@
 using System.Runtime;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.Python.Analysis.Caching;
 using Microsoft.Python.Analysis.Dependencies;
 using Microsoft.Python.Analysis.Diagnostics;
 using Microsoft.Python.Analysis.Modules;

src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs

@@ -18,7 +18,6 @@
 using System.Diagnostics;
 using System.Linq;
 using Microsoft.Python.Analysis.Analyzer.Evaluation;
-using Microsoft.Python.Analysis.Extensions;
 using Microsoft.Python.Analysis.Modules;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Values;

src/Analysis/Ast/Impl/Caching/CacheFolders.cs

@@ -19,26 +19,27 @@
 using System.Security.Cryptography;
 using System.Text;
 using Microsoft.Python.Core;
-using Microsoft.Python.Core.IO;
 using Microsoft.Python.Core.Logging;
 using Microsoft.Python.Core.OS;
 
 namespace Microsoft.Python.Analysis.Caching {
-    internal static class CacheFolders {
-        public static string GetCacheFilePath(string root, string moduleName, string content, IFileSystem fs) {
-            // Folder for all module versions and variants
-            // {root}/module_name/content_hash.pyi
-            var folder = Path.Combine(root, moduleName);
-
-            var filePath = Path.Combine(root, folder, $"{FileNameFromContent(content)}.pyi");
-            if (fs.StringComparison == StringComparison.Ordinal) {
-                filePath = filePath.ToLowerInvariant();
-            }
+    internal sealed class CacheFolderService: ICacheFolderService {
+        public CacheFolderService(IServiceContainer services, string cacheRootFolder) {
+            CacheFolder = cacheRootFolder ?? GetCacheFolder(services);
+        }
+
+        public string CacheFolder { get; }
 
-            return filePath;
+        public string GetFileNameFromContent(string content) {
+            // File name depends on the content so we can distinguish between different versions.
+            using (var hash = SHA256.Create()) {
+                return Convert
+                    .ToBase64String(hash.ComputeHash(new UTF8Encoding(false).GetBytes(content)))
+                    .Replace('/', '_').Replace('+', '-');
+            }
         }
 
-        public static string GetCacheFolder(IServiceContainer services) {
+        private static string GetCacheFolder(IServiceContainer services) {
             var platform = services.GetService<IOSPlatform>();
             var logger = services.GetService<ILogger>();
 
@@ -95,17 +96,6 @@ public static string GetCacheFolder(IServiceContainer services) {
             return cachePath;
         }
 
-        public static string FileNameFromContent(string content) {
-            // File name depends on the content so we can distinguish between different versions.
-            var hash = SHA256.Create();
-            return Convert
-                .ToBase64String(hash.ComputeHash(new UTF8Encoding(false).GetBytes(content)))
-                .Replace('/', '_').Replace('+', '-');
-        }
-
-        public static string GetAnalysisCacheFilePath(string analysisRootFolder, string moduleName, string content, IFileSystem fs)
-            => GetCacheFilePath(analysisRootFolder, moduleName, content, fs);
-
         private static bool CheckPathRooted(string varName, string path, ILogger logger) {
             if (!string.IsNullOrWhiteSpace(path) && Path.IsPathRooted(path)) {
                 return true;

src/Analysis/Ast/Impl/Caching/Definitions/ICacheFolderService.cs

@@ -0,0 +1,21 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+namespace Microsoft.Python.Analysis.Caching {
+    public interface ICacheFolderService {
+        string CacheFolder { get; }
+        string GetFileNameFromContent(string content);
+    }
+}

src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs

@@ -0,0 +1,37 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Python.Analysis.Types;
+
+namespace Microsoft.Python.Analysis.Caching {
+    internal interface IModuleDatabaseService {
+        /// <summary>
+        /// Creates module representation from module persistent state.
+        /// </summary>
+        /// <param name="moduleName">Module name. If the name is not qualified
+        /// the module will ge resolved against active Python version.</param>
+        /// <param name="filePath">Module file path.</param>
+        /// <param name="module">Python module.</param>
+        /// <returns>Module storage state</returns>
+        ModuleStorageState TryCreateModule(string moduleName, string filePath, out IPythonModule module);
+
+        /// <summary>
+        /// Writes module data to the database.
+        /// </summary>
+        Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationToken cancellationToken = default);
+    }
+}

src/Analysis/Ast/Impl/Caching/Definitions/ModuleStorageState.cs

@@ -0,0 +1,42 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+namespace Microsoft.Python.Analysis.Caching {
+    /// <summary>
+    /// Describes module data stored in a database.
+    /// </summary>
+    public enum ModuleStorageState {
+        /// <summary>
+        /// Module does not exist in the database.
+        /// </summary>
+        DoesNotExist,
+
+        /// <summary>
+        /// Partial data. This means module is still being analyzed
+        /// and the data on the module members is incomplete.
+        /// </summary>
+        Partial,
+
+        /// <summary>
+        /// Modules exist and the analysis is complete.
+        /// </summary>
+        Complete,
+
+        /// <summary>
+        /// Storage is corrupted or incompatible.
+        /// </summary>
+        Corrupted
+    }
+}

src/Analysis/Ast/Impl/Caching/StubCache.cs

@@ -27,13 +27,14 @@ internal sealed class StubCache : IStubCache {
 
         private readonly IFileSystem _fs;
         private readonly ILogger _log;
+        private readonly ICacheFolderService _cfs;
 
-        public StubCache(IServiceContainer services, string cacheRootFolder = null) {
+        public StubCache(IServiceContainer services) {
             _fs = services.GetService<IFileSystem>();
             _log = services.GetService<ILogger>();
 
-            cacheRootFolder = cacheRootFolder ?? CacheFolders.GetCacheFolder(services);
-            StubCacheFolder = Path.Combine(cacheRootFolder, $"stubs.v{_stubCacheFormatVersion}");
+            _cfs = services.GetService<ICacheFolderService>();
+            StubCacheFolder = Path.Combine(_cfs.CacheFolder, $"stubs.v{_stubCacheFormatVersion}");
         }
 
         public string StubCacheFolder { get; }
@@ -58,7 +59,7 @@ public string GetCacheFilePath(string filePath) {
                 dir = dir.ToLowerInvariant();
             }
 
-            var dirHash = CacheFolders.FileNameFromContent(dir);
+            var dirHash = _cfs.GetFileNameFromContent(dir);
             var stubFile = Path.Combine(StubCacheFolder, Path.Combine(dirHash, name));
             return Path.ChangeExtension(stubFile, ".pyi");
         }

src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs

@@ -15,6 +15,7 @@
 
 
 using System.Collections.Generic;
+using System.ComponentModel;
 using System.Linq;
 using Microsoft.Python.Analysis.Analyzer.Evaluation;
 using Microsoft.Python.Analysis.Types;
@@ -57,7 +58,7 @@ internal static void DeclareParametersInScope(this IArgumentSet args, Expression
             }
 
             if (args.ListArgument != null && !string.IsNullOrEmpty(args.ListArgument.Name)) {
-                var type = new PythonCollectionType(null, BuiltinTypeId.List, eval.Interpreter, false);
+                var type = new PythonCollectionType(null, BuiltinTypeId.List, eval.BuiltinsModule, false);
                 var list = new PythonCollection(type, args.ListArgument.Values);
                 eval.DeclareVariable(args.ListArgument.Name, list, VariableSource.Declaration, args.ListArgument.Location);
             }