Lazy types in caching

src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs

@@ -22,7 +22,7 @@
 
 namespace Microsoft.Python.Analysis.Analyzer {
     [DebuggerDisplay("{Name} : {FilePath}")]
-    internal readonly struct AnalysisModuleKey : IEquatable<AnalysisModuleKey> {
+    public readonly struct AnalysisModuleKey : IEquatable<AnalysisModuleKey> {
         public string Name { get; }
         public string FilePath { get; }
         public bool IsTypeshed { get; }
@@ -34,7 +34,7 @@ public AnalysisModuleKey(IPythonModule module) : this(
             module.IsTypeshed,
             IsNonUserAsDocumentModule(module)) { }
 
-        public AnalysisModuleKey(string name, string filePath, bool isTypeshed)
+        public AnalysisModuleKey(string name, string filePath, bool isTypeshed = false)
             : this(name, filePath, isTypeshed, false) { }
 
         private AnalysisModuleKey(string name, string filePath, bool isTypeshed, bool isNonUserAsDocument) {

src/Analysis/Ast/Impl/Analyzer/CachedAnalysis.cs

@@ -0,0 +1,80 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Python.Analysis.Analyzer.Evaluation;
+using Microsoft.Python.Analysis.Diagnostics;
+using Microsoft.Python.Analysis.Documents;
+using Microsoft.Python.Analysis.Utilities;
+using Microsoft.Python.Analysis.Values;
+using Microsoft.Python.Core;
+using Microsoft.Python.Core.Diagnostics;
+using Microsoft.Python.Parsing.Ast;
+
+namespace Microsoft.Python.Analysis.Analyzer {
+    /// <summary>
+    /// Analysis of a module restored from database.
+    /// </summary>
+    internal sealed class CachedAnalysis : IDocumentAnalysis {
+        public CachedAnalysis(IDocument document, IServiceContainer services) {
+            Check.ArgumentNotNull(nameof(document), document);
+            Document = document;
+            ExpressionEvaluator = new ExpressionEval(services, document, AstUtilities.MakeEmptyAst(document.Uri));
+        }
+
+        #region IDocumentAnalysis
+        /// <summary>
+        /// Analyzed document.
+        /// </summary>
+        public IDocument Document { get; }
+
+        /// <summary>
+        /// Version of the analysis. Usually matches document version,
+        /// but can be lower when document or its dependencies were
+        /// updated since.
+        /// </summary>
+        public int Version => 0;
+
+        /// <summary>
+        /// Empty AST.
+        /// </summary>
+        public PythonAst Ast => ExpressionEvaluator.Ast;
+
+        /// <summary>
+        /// Document/module global scope.
+        /// </summary>
+        public IGlobalScope GlobalScope => Document.GlobalScope;
+
+        /// <summary>
+        /// Expression evaluator used in the analysis.
+        /// Only supports scope operation since there is no AST
+        /// when library analysis is complete.
+        /// </summary>
+        public IExpressionEvaluator ExpressionEvaluator { get; }
+
+        /// <summary>
+        /// Members of the module which are transferred during a star import. null means __all__ was not defined.
+        /// </summary>
+        public IReadOnlyList<string> StarImportMemberNames => Array.Empty<string>();
+
+        /// <summary>
+        /// Analysis diagnostics.
+        /// </summary>
+        public IEnumerable<DiagnosticsEntry> Diagnostics => Enumerable.Empty<DiagnosticsEntry>();
+        #endregion
+    }
+}

src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs

@@ -21,11 +21,6 @@ namespace Microsoft.Python.Analysis.Analyzer {
     /// Represents document that can be analyzed asynchronously.
     /// </summary>
     internal interface IAnalyzable {
-        /// <summary>
-        /// Returns object that can calculate dependencies of this entry.
-        /// </summary>
-        IDependencyProvider DependencyProvider { get; }
-
         /// <summary>
         /// Notifies document that analysis is about to begin.
         /// </summary>

src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs

@@ -71,12 +71,5 @@ public interface IPythonAnalyzer {
         /// Fires when analysis is complete.
         /// </summary>
         event EventHandler<AnalysisCompleteEventArgs> AnalysisComplete;
-
-        /// <summary>
-        /// Attempts to restore modules analysis from database.
-        /// </summary>
-        /// <param name="module"></param>
-        /// <returns></returns>
-        IDocumentAnalysis TryRestoreCachedAnalysis(IPythonModule module);
     }
 }

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs

@@ -89,10 +89,13 @@ private IMember GetValueFromBinaryOp(Expression expr, LookupOptions lookupOption
                 if (op.IsComparison()) {
                     return Interpreter.GetBuiltinType(BuiltinTypeId.Bool);
                 }
-
                 return UnknownType;
             }
 
+            var leftValue = left is IVariable v1 ? v1.Value : left;
+            var rightValue = right is IVariable v2 ? v2.Value : right;
+            var isInstance = leftValue is IPythonInstance || rightValue is IPythonInstance;
+
             var leftType = left.GetPythonType();
             var rightType = right.GetPythonType();
 
@@ -121,7 +124,7 @@ private IMember GetValueFromBinaryOp(Expression expr, LookupOptions lookupOption
 
             if (leftIsSupported && rightIsSupported) {
                 if (TryGetValueFromBuiltinBinaryOp(op, leftTypeId, rightTypeId, Interpreter.LanguageVersion.Is3x(), out var member)) {
-                    return member;
+                    return isInstance ? new PythonInstance(member.GetPythonType()) : member;
                 }
             }
 
@@ -136,11 +139,10 @@ private IMember GetValueFromBinaryOp(Expression expr, LookupOptions lookupOption
                     ret = CallOperator(op, left, leftType, right, rightType, expr, tryLeft: false);
                 }
 
-                if (!ret.IsUnknown()) {
-                    return ret;
+                if (ret.IsUnknown()) {
+                    ret = op.IsComparison() ? Interpreter.GetBuiltinType(BuiltinTypeId.Bool) : left;
                 }
-
-                return op.IsComparison() ? Interpreter.GetBuiltinType(BuiltinTypeId.Bool) : left;
+                return isInstance ? new PythonInstance(ret.GetPythonType()) : ret;
             }
 
             if (rightIsSupported) {

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs

@@ -163,7 +163,7 @@ public IDisposable OpenScope(IPythonModule module, ScopeStatement node, out Scop
                     // node points to global scope, it is not a function or a class.
                     scope = gs;
                 } else {
-                    scope = outerScope.Children.OfType<Scope>().FirstOrDefault(s => s.Node == node);
+                    scope = outerScope.GetChildScope(node) as Scope;
                     if (scope == null) {
                         scope = new Scope(node, outerScope, Module);
                         outerScope.AddChildScope(scope);

src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs

@@ -15,6 +15,7 @@
 
 using System.Diagnostics;
 using System.Linq;
+using Microsoft.Python.Analysis.Specializations.Typing;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Values;
 using Microsoft.Python.Parsing.Ast;
@@ -29,14 +30,22 @@ public void HandleAssignment(AssignmentStatement node, LookupOptions lookupOptio
             }
 
             // Filter out parenthesis expression in assignment because it makes no difference.
-            var lhs = node.Left.Select(s => s.RemoveParenthesis());
+            var lhs = node.Left.Select(s => s.RemoveParenthesis()).ToArray();
 
             // Note that this is handling assignments of the same value to multiple variables,
             // i.e. with "x = y = z = value", x/y/z are the items in lhs. If an expression looks
             // like "x, y, z = value", then "x, y, z" is a *single* lhs value and its unpacking
             // will be handled by AssignToExpr.
             var value = ExtractRhs(node.Right, lhs.FirstOrDefault(), lookupOptions);
             if (value != null) {
+                // Named tuple may get assigned to variable that has name different from the tuple itself.
+                // Then the name may end up conflicting with other types in module when stub merges with
+                // module types. For example, 'tokenize' stub declares _TokenInfo = NamedTuple('TokenInfo', ...)
+                // but there is also 'class TokenInfo(_TokenInfo)' so we have to use the variable name
+                // in order to avoid type naming conflicts.
+                if (value is ITypingNamedTupleType nt && lhs.Length == 1 && lhs[0] is NameExpression nex) {
+                    nt.SetName(nex.Name);
+                }
                 foreach (var expr in lhs) {
                     AssignToExpr(expr, value);
                 }

src/Analysis/Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs

@@ -104,15 +104,9 @@ public void EnsureModule(in IPythonModule module) {
         }
 
         public ModuleWalker WalkModule(IPythonModule module, PythonAst ast) {
-            var analyzer = _services.GetService<IPythonAnalyzer>();
-            var analysis = analyzer.TryRestoreCachedAnalysis(module);
-            if (analysis != null) {
-                return null;
-            }
-
             // If module has stub, make sure it is processed too.
             if (module.Stub?.Analysis is EmptyAnalysis) {
-                WalkModule(module.Stub, module.GetAst());
+                WalkModule(module.Stub, module.Stub.GetAst());
             }
 
             var eval = new ExpressionEval(_services, module, ast);

src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs

@@ -20,8 +20,6 @@
 using System.Runtime.ExceptionServices;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.Python.Analysis.Analyzer.Evaluation;
-using Microsoft.Python.Analysis.Caching;
 using Microsoft.Python.Analysis.Dependencies;
 using Microsoft.Python.Analysis.Diagnostics;
 using Microsoft.Python.Analysis.Documents;
@@ -116,7 +114,6 @@ public void RemoveAnalysis(IPythonModule module) {
                 key = new AnalysisModuleKey(module);
                 _analysisEntries.Remove(key);
             }
-
             _dependencyResolver.Remove(key);
         }
 
@@ -131,7 +128,8 @@ public void EnqueueDocumentForAnalysis(IPythonModule module, ImmutableArray<IPyt
                 version = _version + 1;
             }
 
-            if (entry.Invalidate(analysisDependencies, version, out var dependencies)) {
+            var invalidate = entry.Invalidate(analysisDependencies, version, out var dependencies);
+            if (invalidate) {
                 AnalyzeDocument(key, entry, dependencies);
             }
         }
@@ -157,7 +155,8 @@ public void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int
                 }
             }
 
-            if (entry.Invalidate(module, ast, bufferVersion, version, out var dependencies)) {
+            var invalidate = entry.Invalidate(module, ast, bufferVersion, version, out var dependencies) || module.Analysis is EmptyAnalysis;
+            if (invalidate) {
                 AnalyzeDocument(key, entry, dependencies);
             }
         }
@@ -187,13 +186,13 @@ public async Task ResetAnalyzer() {
             lock (_syncObj) {
                 _forceGCOnNextSession = true;
 
-                _analysisEntries.Split(kvp => kvp.Value.Module is IBuiltinsPythonModule, out var entriesToPreserve, out var entriesToRemove);
+                _analysisEntries.Split(kvp => kvp.Value.Module is IBuiltinsPythonModule, out var entriesToPreserve, out _);
                 _analysisEntries.Clear();
                 foreach (var (key, entry) in entriesToPreserve) {
                     _analysisEntries.Add(key, entry);
                 }
 
-                _dependencyResolver.RemoveKeys(entriesToRemove.Select(e => e.Key));
+                _dependencyResolver.Reset();
             }
 
             _services.GetService<IRunningDocumentTable>().ReloadAll();
@@ -208,30 +207,20 @@ public IReadOnlyList<IPythonModule> LoadedModules {
         }
 
         public event EventHandler<AnalysisCompleteEventArgs> AnalysisComplete;
-
-        public IDocumentAnalysis TryRestoreCachedAnalysis(IPythonModule module) {
-            var moduleType = module.ModuleType;
-            var moduleDatabaseService = _services.GetService<IModuleDatabaseService>();
-            if (!moduleType.CanBeCached() || moduleDatabaseService == null || !moduleDatabaseService.ModuleExistsInStorage(module.Name, module.FilePath, moduleType)) {
-                return null;
-            }
-
-            if (moduleDatabaseService.TryRestoreGlobalScope(module, out var gs)) {
-                _log?.Log(TraceEventType.Verbose, "Restored from database: ", module.Name);
-                var analysis = new DocumentAnalysis((IDocument)module, 1, gs, new ExpressionEval(_services, module, module.GetAst()), Array.Empty<string>());
-                gs.ReconstructVariables();
-                return analysis;
-            }
-
-            _log?.Log(TraceEventType.Verbose, "Restore from database failed for module ", module.Name);
-
-            return null;
-        }
         #endregion
 
         internal void RaiseAnalysisComplete(int moduleCount, double msElapsed) {
-            _analysisCompleteEvent.Set();
-            AnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs(moduleCount, msElapsed));
+            var analisysComplete = false;
+            lock (_syncObj) {
+                var notAnalyzed = _analysisEntries.Values.ExcludeDefault().Where(e => e.NotAnalyzed).ToArray();
+                analisysComplete = notAnalyzed.Length == 0
+                                  && _nextSession == null
+                                  && (_currentSession == null || _currentSession.IsCompleted);
+            }
+            if (analisysComplete) {
+                _analysisCompleteEvent.Set();
+                AnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs(moduleCount, msElapsed));
+            }
         }
 
         private void AnalyzeDocument(in AnalysisModuleKey key, in PythonAnalyzerEntry entry, in ImmutableArray<AnalysisModuleKey> dependencies) {
@@ -242,7 +231,7 @@ private void AnalyzeDocument(in AnalysisModuleKey key, in PythonAnalyzerEntry en
             var graphVersion = _dependencyResolver.ChangeValue(key, entry, entry.IsUserOrBuiltin || key.IsNonUserAsDocument, dependencies);
 
             lock (_syncObj) {
-                if (_version > graphVersion) {
+                if (_version >= graphVersion) {
                     return;
                 }
 
@@ -277,7 +266,7 @@ private bool TryCreateSession(in int graphVersion, in PythonAnalyzerEntry entry,
                 return false;
             }
 
-            LoadMissingDocuments(entry.Module.Interpreter, walker.MissingKeys);
+            LoadMissingDocuments(entry.Module.Interpreter, walker);
 
             lock (_syncObj) {
                 if (_currentSession == null) {
@@ -329,25 +318,26 @@ private PythonAnalyzerSession CreateSession(in IDependencyChainWalker<AnalysisMo
             return new PythonAnalyzerSession(_services, _progress, _startNextSession, _disposeToken.CancellationToken, walker, _version, entry, forceGC: forceGC);
         }
 
-        private void LoadMissingDocuments(IPythonInterpreter interpreter, ImmutableArray<AnalysisModuleKey> missingKeys) {
-            if (missingKeys.Count == 0) {
-                return;
-            }
-
-            var foundKeys = ImmutableArray<AnalysisModuleKey>.Empty;
-            foreach (var key in missingKeys) {
+        private void LoadMissingDocuments(IPythonInterpreter interpreter, IDependencyChainWalker<AnalysisModuleKey, PythonAnalyzerEntry> walker) {
+            foreach (var key in walker.MissingKeys.ToArray()) {
                 lock (_syncObj) {
                     if (_analysisEntries.TryGetValue(key, out _)) {
+                        walker.MissingKeys = walker.MissingKeys.Remove(key);
                         continue;
                     }
                 }
 
                 var (moduleName, _, isTypeshed) = key;
                 var moduleResolution = isTypeshed ? interpreter.TypeshedResolution : interpreter.ModuleResolution;
+
                 var module = moduleResolution.GetOrLoadModule(moduleName);
                 if (module != null && module.ModuleType != ModuleType.Unresolved) {
                     var entry = GetOrCreateAnalysisEntry(module, out _);
-                    _dependencyResolver.TryAddValue(key, entry, entry.IsUserModule, ImmutableArray<AnalysisModuleKey>.Empty);
+                    if (module.ModuleType == ModuleType.Specialized) {
+                        walker.MissingKeys = walker.MissingKeys.Remove(key);
+                    } else {
+                        _dependencyResolver.TryAddValue(key, entry, entry.IsUserModule, ImmutableArray<AnalysisModuleKey>.Empty);
+                    }
                 }
             }
         }
@@ -359,12 +349,18 @@ private PythonAnalyzerEntry GetOrCreateAnalysisEntry(IPythonModule module, out A
 
         private PythonAnalyzerEntry GetOrCreateAnalysisEntry(IPythonModule module, AnalysisModuleKey key) {
             lock (_syncObj) {
-                if (!_analysisEntries.TryGetValue(key, out var entry)) {
-                    var emptyAnalysis = new EmptyAnalysis(_services, (IDocument)module);
-                    entry = new PythonAnalyzerEntry(emptyAnalysis);
-                    _analysisEntries[key] = entry;
+                if (_analysisEntries.TryGetValue(key, out var entry)) {
+                    return entry;
+                }
+
+                if (module.ModuleType == ModuleType.Specialized) {
+                    entry = new PythonAnalyzerEntry(new CachedAnalysis((IDocument)module, _services));
+                } else {
+                    entry = new PythonAnalyzerEntry(new EmptyAnalysis(_services, (IDocument)module));
                     _analysisCompleteEvent.Reset();
                 }
+
+                _analysisEntries[key] = entry;
                 return entry;
             }
         }