Stability improvements (#749)

- Reanalyze user module out of the order if required
- Skip walker if it is outdated and doesn't have a lot of unanalyzed modules.
- Initialize PriorityProducerConsumer<IDependencyChainNode<TValue>> only when GetNextAsync is called for the first time
- Fix ProcessServices so that error stream doesn't hang the process

Author: AlexanderSher

src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs

@@ -14,21 +14,29 @@
 // permissions and limitations under the License.
 
 using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.Python.Analysis.Core.DependencyResolution;
 using Microsoft.Python.Analysis.Modules;
 using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Core;
 using Microsoft.Python.Core.Collections;
 using Microsoft.Python.Parsing.Ast;
 
 namespace Microsoft.Python.Analysis.Analyzer {
+    [DebuggerDisplay("{_module.Name}({_module.ModuleType})")]
     internal sealed class PythonAnalyzerEntry {
         private readonly object _syncObj = new object();
         private TaskCompletionSource<IDocumentAnalysis> _analysisTcs;
         private IPythonModule _module;
+        private bool _isUserModule;
         private PythonAst _ast;
         private IDocumentAnalysis _previousAnalysis;
-        private ImmutableArray<IPythonModule> _analysisDependencies;
+        private HashSet<AnalysisModuleKey> _parserDependencies;
+        private HashSet<AnalysisModuleKey> _analysisDependencies;
         private int _bufferVersion;
         private int _analysisVersion;
 
@@ -40,29 +48,23 @@ public IPythonModule Module {
             }
         }
 
-        public IDocumentAnalysis PreviousAnalysis {
-            get {
-                lock (_syncObj) {
-                    return _previousAnalysis;
-                }
-            }
-        }
-
-        public ImmutableArray<IPythonModule> AnalysisDependencies {
+        public bool IsUserModule {
             get {
                 lock (_syncObj) {
-                    return _analysisDependencies;
+                    return _isUserModule;
                 }
             }
         }
 
-        public int BufferVersion {
+        public IDocumentAnalysis PreviousAnalysis {
             get {
                 lock (_syncObj) {
-                    return _bufferVersion;
+                    return _previousAnalysis;
                 }
             }
         }
+        
+        public int BufferVersion => _bufferVersion;
 
         public int AnalysisVersion {
             get {
@@ -73,15 +75,13 @@ public int AnalysisVersion {
         }
 
         public bool NotAnalyzed => PreviousAnalysis is EmptyAnalysis;
+        
+        public PythonAnalyzerEntry(EmptyAnalysis emptyAnalysis) {
+            _previousAnalysis = emptyAnalysis;
+            _isUserModule = emptyAnalysis.Document.ModuleType == ModuleType.User;
 
-        public PythonAnalyzerEntry(IPythonModule module, PythonAst ast, IDocumentAnalysis previousAnalysis, int bufferVersion, int analysisVersion) {
-            _module = module;
-            _ast = ast;
-            _previousAnalysis = previousAnalysis;
-            _analysisDependencies = ImmutableArray<IPythonModule>.Empty;
-
-            _bufferVersion = bufferVersion;
-            _analysisVersion = analysisVersion;
+            _bufferVersion = -1;
+            _analysisVersion = 0;
             _analysisTcs = new TaskCompletionSource<IDocumentAnalysis>(TaskCreationOptions.RunContinuationsAsynchronously);
         }
 
@@ -92,7 +92,7 @@ public bool IsValidVersion(int version, out IPythonModule module, out PythonAst
             lock (_syncObj) {
                 module = _module;
                 ast = _ast;
-                return _previousAnalysis is EmptyAnalysis || _analysisVersion <= version;
+                return _previousAnalysis is EmptyAnalysis || _isUserModule || _analysisVersion <= version;
             }
         }
 
@@ -106,7 +106,7 @@ public void TrySetAnalysis(IDocumentAnalysis analysis, int version) {
                     return;
                 }
 
-                _analysisDependencies = ImmutableArray<IPythonModule>.Empty;
+                _analysisDependencies = null;
                 UpdateAnalysisTcs(version);
             }
 
@@ -147,36 +147,142 @@ public void Invalidate(int analysisVersion) {
             }
         }
 
-        public void Invalidate(ImmutableArray<IPythonModule> dependencies, int analysisVersion) {
+        public bool Invalidate(ImmutableArray<IPythonModule> analysisDependencies, int analysisVersion, out ImmutableArray<AnalysisModuleKey> dependencies) {
+            dependencies = ImmutableArray<AnalysisModuleKey>.Empty;
+            IPythonModule module;
+            int version;
             lock (_syncObj) {
                 if (_analysisVersion >= analysisVersion) {
-                    return;
+                    return false;
+                }
+
+                version = _analysisVersion;
+                module = _module;
+            }
+
+            var dependenciesHashSet = new HashSet<AnalysisModuleKey>();
+            foreach (var dependency in analysisDependencies) {
+                if (dependency != module && (dependency.ModuleType == ModuleType.User && dependency.Analysis.Version < version || dependency.Analysis is EmptyAnalysis)) {
+                    dependenciesHashSet.Add(new AnalysisModuleKey(dependency));
+                }
+            }
+
+            if (dependenciesHashSet.Count == 0) {
+                return false;
+            }
+
+            lock (_syncObj) {
+                if (_analysisVersion >= analysisVersion) {
+                    return false;
                 }
 
                 UpdateAnalysisTcs(analysisVersion);
-                _analysisDependencies = _analysisDependencies.AddRange(dependencies);
+                if (_analysisDependencies == null) {
+                    _analysisDependencies = dependenciesHashSet;
+                } else {
+                    var countBefore = _analysisDependencies.Count;
+                    _analysisDependencies.UnionWith(dependenciesHashSet);
+                    if (countBefore == _analysisDependencies.Count) {
+                        return false;
+                    }
+                }
+
+                dependencies = _parserDependencies != null
+                    ? ImmutableArray<AnalysisModuleKey>.Create(_parserDependencies.Union(_analysisDependencies).ToArray())
+                    : ImmutableArray<AnalysisModuleKey>.Create(_analysisDependencies);
+                return true;
             }
         }
 
-        public void Invalidate(IPythonModule module, PythonAst ast, int bufferVersion, int analysisVersion) {
+        public bool Invalidate(IPythonModule module, PythonAst ast, int bufferVersion, int analysisVersion, out ImmutableArray<AnalysisModuleKey> dependencies) {
+            dependencies = ImmutableArray<AnalysisModuleKey>.Empty;
+            if (_bufferVersion >= bufferVersion) {
+                return false;
+            }
+
+            var dependenciesHashSet = FindDependencies(module, ast, bufferVersion);
             lock (_syncObj) {
                 if (_analysisVersion >= analysisVersion && _bufferVersion >= bufferVersion) {
-                    return;
+                    return false;
                 }
 
                 _ast = ast;
                 _module = module;
-                _bufferVersion = bufferVersion;
+                _isUserModule = module.ModuleType == ModuleType.User;
+                _parserDependencies = dependenciesHashSet;
 
+                Interlocked.Exchange(ref _bufferVersion, bufferVersion);
                 UpdateAnalysisTcs(analysisVersion);
+                dependencies = _analysisDependencies != null 
+                    ? ImmutableArray<AnalysisModuleKey>.Create(_parserDependencies.Union(_analysisDependencies).ToArray())
+                    : ImmutableArray<AnalysisModuleKey>.Create(_parserDependencies);
+                return true;
+            }
+        }
+
+        private HashSet<AnalysisModuleKey> FindDependencies(IPythonModule module, PythonAst ast, int bufferVersion) {
+            var isTypeshed = module is StubPythonModule stub && stub.IsTypeshed;
+            var moduleResolution = module.Interpreter.ModuleResolution;
+            var pathResolver = isTypeshed
+                ? module.Interpreter.TypeshedResolution.CurrentPathResolver
+                : moduleResolution.CurrentPathResolver;
+
+            var dependencies = new HashSet<AnalysisModuleKey>();
+
+            if (module.Stub != null) {
+                dependencies.Add(new AnalysisModuleKey(module.Stub));
+            }
+
+            foreach (var node in ast.TraverseDepthFirst<Node>(n => n.GetChildNodes())) {
+                if (_bufferVersion > bufferVersion) {
+                    return dependencies;
+                }
+
+                switch (node) {
+                    case ImportStatement import:
+                        foreach (var moduleName in import.Names) {
+                            HandleSearchResults(isTypeshed, dependencies, moduleResolution, pathResolver.FindImports(module.FilePath, moduleName, import.ForceAbsolute));
+                        }
+                        break;
+                    case FromImportStatement fromImport:
+                        var imports = pathResolver.FindImports(module.FilePath, fromImport);
+                        HandleSearchResults(isTypeshed, dependencies, moduleResolution, imports);
+                        if (imports is IImportChildrenSource childrenSource) {
+                            foreach (var name in fromImport.Names) {
+                                if (childrenSource.TryGetChildImport(name.Name, out var childImport)) {
+                                    HandleSearchResults(isTypeshed, dependencies, moduleResolution, childImport);
+                                }
+                            }
+                        }
+                        break;
+                }
+            }
+
+            dependencies.Remove(new AnalysisModuleKey(module));
+            return dependencies;
+        }
+
+        private static void HandleSearchResults(bool isTypeshed, HashSet<AnalysisModuleKey> dependencies, IModuleManagement moduleResolution, IImportSearchResult searchResult) {
+            switch (searchResult) {
+                case ModuleImport moduleImport when !Ignore(moduleResolution, moduleImport.FullName):
+                    dependencies.Add(new AnalysisModuleKey(moduleImport.FullName, moduleImport.ModulePath, isTypeshed));
+                    return;
+                case PossibleModuleImport possibleModuleImport when !Ignore(moduleResolution, possibleModuleImport.PrecedingModuleFullName):
+                    dependencies.Add(new AnalysisModuleKey(possibleModuleImport.PrecedingModuleFullName, possibleModuleImport.PrecedingModulePath, isTypeshed));
+                    return;
+                default:
+                    return;
             }
         }
 
+        private static bool Ignore(IModuleManagement moduleResolution, string name)
+            => moduleResolution.BuiltinModuleName.EqualsOrdinal(name) || moduleResolution.GetSpecializedModule(name) != null;
+
         private void UpdateAnalysisTcs(int analysisVersion) {
             _analysisVersion = analysisVersion;
             if (_analysisTcs.Task.Status == TaskStatus.RanToCompletion) {
                 _previousAnalysis = _analysisTcs.Task.Result;
-                _analysisDependencies = ImmutableArray<IPythonModule>.Empty;
+                _analysisDependencies = null;
             }
 
             if (_analysisTcs.Task.IsCompleted) {

src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs

@@ -217,9 +217,10 @@ private static void RemoveLoopEdges(WalkingVertex<TKey, TValue> vertex, ref int
 
         private sealed class DependencyChainWalker : IDependencyChainWalker<TKey, TValue> {
             private readonly DependencyResolver<TKey, TValue> _dependencyResolver;
-            private readonly PriorityProducerConsumer<IDependencyChainNode<TValue>> _ppc;
+            private readonly ImmutableArray<WalkingVertex<TKey, TValue>> _startingVertices;
             private readonly object _syncObj;
             private int _remaining;
+            private PriorityProducerConsumer<IDependencyChainNode<TValue>> _ppc;
 
             public ImmutableArray<TKey> MissingKeys { get; }
             public ImmutableArray<TValue> AffectedValues { get; }
@@ -243,19 +244,30 @@ public DependencyChainWalker(
 
                 _syncObj = new object();
                 _dependencyResolver = dependencyResolver;
-                _ppc = new PriorityProducerConsumer<IDependencyChainNode<TValue>>();
+                _startingVertices = startingVertices;
                 AffectedValues = affectedValues;
                 Version = version;
                 MissingKeys = missingKeys;
 
                 _remaining = totalNodesCount;
-                foreach (var vertex in startingVertices) {
-                    _ppc.Produce(new DependencyChainNode(this, vertex));
-                }
             }
 
-            public Task<IDependencyChainNode<TValue>> GetNextAsync(CancellationToken cancellationToken) =>
-                _ppc.ConsumeAsync(cancellationToken);
+            public Task<IDependencyChainNode<TValue>> GetNextAsync(CancellationToken cancellationToken) {
+                PriorityProducerConsumer<IDependencyChainNode<TValue>> ppc;
+                lock (_syncObj) {
+                    if (_ppc == null) {
+                        _ppc = new PriorityProducerConsumer<IDependencyChainNode<TValue>>();
+
+                        foreach (var vertex in _startingVertices) {
+                            _ppc.Produce(new DependencyChainNode(this, vertex));
+                        }
+                    }
+
+                    ppc = _ppc;
+                }
+
+                return ppc.ConsumeAsync(cancellationToken);
+            }
 
             public void MarkCompleted(WalkingVertex<TKey, TValue> vertex, bool commitChanges) {
                 var verticesToProduce = new List<WalkingVertex<TKey, TValue>>();

src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs

@@ -85,17 +85,15 @@ private string ScrapeModule() {
                 CreateNoWindow = true,
                 StandardOutputEncoding = Encoding.UTF8,
                 RedirectStandardInput = true,
-                RedirectStandardOutput = true,
-                RedirectStandardError = true
+                RedirectStandardOutput = true
             };
             var ps = Services.GetService<IProcessServices>();
 
             Log?.Log(TraceEventType.Verbose, "Scrape", startInfo.FileName, startInfo.Arguments);
 
             try {
                 var token = new CancellationTokenSource(30000).Token;
-                var lines = ps.ExecuteAndCaptureOutputAsync(startInfo, token).GetAwaiter().GetResult();
-                return string.Join(Environment.NewLine, lines);
+                return ps.ExecuteAndCaptureOutputAsync(startInfo, token).GetAwaiter().GetResult();
             } catch (Exception ex) when (!ex.IsCriticalException()) { }
 
             return string.Empty;

src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs

@@ -19,7 +19,6 @@
 
 namespace Microsoft.Python.Analysis.Modules {
     public interface IModuleCache {
-        Task<IDocument> ImportFromCacheAsync(string name, CancellationToken cancellationToken);
         string GetCacheFilePath(string filePath);
         string ReadCachedModule(string filePath);
         void WriteCachedModule(string filePath, string code);

src/Analysis/Ast/Impl/Modules/Definitions/IModuleCache.cs

@@ -43,35 +43,7 @@ public ModuleCache(IPythonInterpreter interpreter, IServiceContainer services) {
             _log = services.GetService<ILogger>();
             _skipCache = string.IsNullOrEmpty(_interpreter.Configuration.DatabasePath);
         }
-
-        public async Task<IDocument> ImportFromCacheAsync(string name, CancellationToken cancellationToken) {
-            if (string.IsNullOrEmpty(ModuleCachePath)) {
-                return null;
-            }
-
-            var cache = GetCacheFilePath("python.{0}.pyi".FormatInvariant(name));
-            if (!_fs.FileExists(cache)) {
-                cache = GetCacheFilePath("python._{0}.pyi".FormatInvariant(name));
-                if (!_fs.FileExists(cache)) {
-                    cache = GetCacheFilePath("{0}.pyi".FormatInvariant(name));
-                    if (!_fs.FileExists(cache)) {
-                        return null;
-                    }
-                }
-            }
-
-            var rdt = _services.GetService<IRunningDocumentTable>();
-            var mco = new ModuleCreationOptions {
-                ModuleName = name,
-                ModuleType = ModuleType.Compiled,
-                FilePath = cache
-            };
-            var module = rdt.AddModule(mco);
-
-            await module.LoadAndAnalyzeAsync(cancellationToken);
-            return module;
-        }
-
+        
         public string GetCacheFilePath(string filePath) {
             if (string.IsNullOrEmpty(filePath) || !PathEqualityComparer.IsValidPath(ModuleCachePath)) {
                 if (!_loggedBadDbPath) {

src/Analysis/Ast/Impl/Modules/ModuleCache.cs

@@ -61,7 +61,8 @@ public IReadOnlyList<IPythonCollection> Items
         public IMember this[IMember key] =>
             _contents.TryGetValue(key, out var value) ? value : UnknownType;
 
-        public override IPythonIterator GetIterator() => Call(@"iterkeys", ArgumentSet.Empty) as IPythonIterator;
+        public override IPythonIterator GetIterator() => 
+            Call(@"iterkeys", ArgumentSet.Empty) as IPythonIterator ?? new EmptyIterator(Type.DeclaringModule.Interpreter.UnknownType);
 
         public override IMember Index(object key) => key is IMember m ? this[m] : UnknownType;