Ensure builtins module is created synchronously (#1798)

* Remove stale reference

* Don't suppress LHS diagnostics on augmented assign

* Revert "Don't suppress LHS diagnostics on augmented assign"

This reverts commit 6109ac7.

* Sync builtins creation

* Sync builtins

* Usings

* Debug code

Author: Mikhail Arkhipov

src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs

@@ -14,6 +14,7 @@
 // permissions and limitations under the License.
 
 using Microsoft.Python.Analysis.Dependencies;
+using Microsoft.Python.Parsing.Ast;
 
 namespace Microsoft.Python.Analysis.Analyzer {
     /// <summary>
@@ -30,6 +31,11 @@ internal interface IAnalyzable {
         /// </summary>
         void NotifyAnalysisBegins();
 
+        /// <summary>
+        /// Performs standard analysis pass. Does not include any restoration from databases.
+        /// </summary>
+        ModuleWalker Analyze(PythonAst ast);
+
         /// <summary>
         /// Notifies document that its analysis is now complete.
         /// </summary>

src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs

@@ -352,10 +352,11 @@ private IDocumentAnalysis DoAnalyzeEntry(IDependencyChainNode<PythonAnalyzerEntr
                 return analysis;
             }
 
-            var walker = new ModuleWalker(_services, module, ast, _analyzerCancellationToken);
-            ast.Walk(walker);
-            walker.Complete();
-            return CreateAnalysis(node, (IDocument)module, ast, version, walker);
+            if (module is IAnalyzable analyzable) {
+                var walker = analyzable.Analyze(ast);
+                return CreateAnalysis(node, (IDocument)module, ast, version, walker);
+            }
+            return new EmptyAnalysis(_services, (IDocument)module);
         }
 
         private bool MarkNodeWalked(IDependencyChainNode<PythonAnalyzerEntry> node) {

src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs

@@ -22,7 +22,6 @@
 using Microsoft.Python.Analysis.Modules.Resolution;
 using Microsoft.Python.Analysis.Specializations.Typing;
 using Microsoft.Python.Analysis.Types;
-using Microsoft.Python.Analysis.Values;
 using Microsoft.Python.Core;
 using Microsoft.Python.Core.Collections;
 using Microsoft.Python.Core.Services;

src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs

@@ -17,8 +17,9 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
+using System.Threading;
+using Microsoft.Python.Analysis.Analyzer;
 using Microsoft.Python.Analysis.Specializations;
-using Microsoft.Python.Analysis.Specializations.Typing;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Values;
 using Microsoft.Python.Core;
@@ -41,15 +42,28 @@ internal sealed class BuiltinsPythonModule : CompiledPythonModule, IBuiltinsPyth
         public BuiltinsPythonModule(string moduleName, string filePath, IServiceContainer services)
             : base(moduleName, ModuleType.Builtins, filePath, null, false, false, services) { } // TODO: builtins stub & persistence
 
+        #region IMemberContainer
         public override IMember GetMember(string name) => _hiddenNames.Contains(name) ? null : base.GetMember(name);
 
         public IMember GetAnyMember(string name) => base.GetMember(name);
 
         public override IEnumerable<string> GetMemberNames() => base.GetMemberNames().Except(_hiddenNames).ToArray();
+        #endregion
+
+        public void Initialize() => ParseAndLogExceptions(CancellationToken.None);
 
         protected override string[] GetScrapeArguments(IPythonInterpreter interpreter)
             => !InstallPath.TryGetFile("scrape_module.py", out var sb) ? null : new[] { "-W", "ignore", "-B", "-E", sb };
 
+        protected override void Parse() { }
+
+        protected override void Analyze(PythonAst ast, int version) {
+            NotifyAnalysisBegins();
+            var walker = Analyze(ast);
+            var analysis = new DocumentAnalysis(this, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames);
+            NotifyAnalysisComplete(analysis);
+        }
+
         protected override void OnAnalysisComplete() {
             SpecializeTypes();
             SpecializeFunctions();

src/Analysis/Ast/Impl/Modules/PythonModule.cs

@@ -22,10 +22,11 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Python.Analysis.Analyzer;
+using Microsoft.Python.Analysis.Analyzer.Evaluation;
+using Microsoft.Python.Analysis.Analyzer.Handlers;
 using Microsoft.Python.Analysis.Dependencies;
 using Microsoft.Python.Analysis.Diagnostics;
 using Microsoft.Python.Analysis.Documents;
-using Microsoft.Python.Analysis.Specializations.Typing;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Values;
 using Microsoft.Python.Core;
@@ -332,7 +333,7 @@ public void Invalidate() {
             Services.GetService<IPythonAnalyzer>().InvalidateAnalysis(this);
         }
 
-        private void Parse() {
+        protected virtual void Parse() {
             _parseCts?.Cancel();
             _parseCts = new CancellationTokenSource();
 
@@ -343,7 +344,7 @@ private void Parse() {
             _parsingTask = Task.Run(() => ParseAndLogExceptions(_linkedParseCts.Token), _linkedParseCts.Token);
         }
 
-        private void ParseAndLogExceptions(CancellationToken cancellationToken) {
+        protected void ParseAndLogExceptions(CancellationToken cancellationToken) {
             try {
                 Parse(cancellationToken);
             } catch (Exception ex) when (!(ex is OperationCanceledException)) {
@@ -352,6 +353,15 @@ private void ParseAndLogExceptions(CancellationToken cancellationToken) {
             }
         }
 
+        protected virtual void Analyze(PythonAst ast, int version) {
+            if (ContentState < State.Analyzing) {
+                ContentState = State.Analyzing;
+
+                var analyzer = Services.GetService<IPythonAnalyzer>();
+                analyzer.EnqueueDocumentForAnalysis(this, ast, version);
+            }
+        }
+
         private void Parse(CancellationToken cancellationToken) {
             CollectingErrorSink sink = null;
             int version;
@@ -372,8 +382,7 @@ private void Parse(CancellationToken cancellationToken) {
             }
 
             var ast = parser.ParseFile(Uri);
-
-            //Log?.Log(TraceEventType.Verbose, $"Parse complete: {Name}");
+            // Log?.Log(TraceEventType.Verbose, $"Parse complete: {Name} ({ModuleType})");
 
             lock (_syncObj) {
                 cancellationToken.ThrowIfCancellationRequested();
@@ -397,13 +406,7 @@ private void Parse(CancellationToken cancellationToken) {
             }
 
             NewAst?.Invoke(this, EventArgs.Empty);
-
-            if (ContentState < State.Analyzing) {
-                ContentState = State.Analyzing;
-
-                var analyzer = Services.GetService<IPythonAnalyzer>();
-                analyzer.EnqueueDocumentForAnalysis(this, ast, version);
-            }
+            Analyze(ast, version);
 
             lock (_syncObj) {
                 _parsingTask = null;
@@ -450,6 +453,14 @@ public void NotifyAnalysisBegins() {
             }
         }
 
+        public ModuleWalker Analyze(PythonAst ast) {
+            var eval = new ExpressionEval(Services, this, ast);
+            var walker = new ModuleWalker(eval, SimpleImportedVariableHandler.Instance);
+            ast.Walk(walker);
+            walker.Complete();
+            return walker;
+        }
+
         public void NotifyAnalysisComplete(IDocumentAnalysis analysis) {
             lock (_syncObj) {
                 if (analysis.Version < Analysis.Version) {

src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs

@@ -39,6 +39,7 @@ namespace Microsoft.Python.Analysis.Modules.Resolution {
     internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManagement {
         private readonly ConcurrentDictionary<string, IPythonModule> _specialized = new ConcurrentDictionary<string, IPythonModule>();
         private readonly IUIService _ui;
+        private BuiltinsPythonModule _builtins;
         private IModuleDatabaseService _dbService;
         private IRunningDocumentTable _rdt;
 
@@ -53,7 +54,7 @@ public MainModuleResolution(string root, IServiceContainer services, ImmutableAr
         public string BuiltinModuleName => BuiltinTypeId.Unknown.GetModuleName(Interpreter.LanguageVersion);
         public ImmutableArray<PythonLibraryPath> LibraryPaths { get; private set; } = ImmutableArray<PythonLibraryPath>.Empty;
 
-        public IBuiltinsPythonModule BuiltinsModule { get; private set; }
+        public IBuiltinsPythonModule BuiltinsModule => _builtins;
 
         public IEnumerable<IPythonModule> GetImportedModules(CancellationToken cancellationToken) {
             foreach (var module in _specialized.Values) {
@@ -178,12 +179,8 @@ public IPythonModule GetSpecializedModule(string fullName, bool allowCreation =
         public bool IsSpecializedModule(string fullName, string modulePath = null)
             => _specialized.ContainsKey(fullName);
 
-        private async Task AddBuiltinTypesToPathResolverAsync(CancellationToken cancellationToken = default) {
-            var analyzer = Services.GetService<IPythonAnalyzer>();
-            await analyzer.GetAnalysisAsync(BuiltinsModule, Timeout.Infinite, cancellationToken);
-
+        private void AddBuiltinTypesToPathResolver() {
             Check.InvalidOperation(!(BuiltinsModule.Analysis is EmptyAnalysis), "Builtins analysis did not complete correctly.");
-
             // Add built-in module names
             var builtinModuleNamesMember = BuiltinsModule.GetAnyMember("__builtin_module_names__");
             var value = (builtinModuleNamesMember as IVariable)?.Value ?? builtinModuleNamesMember;
@@ -215,16 +212,16 @@ public async Task ReloadAsync(CancellationToken cancellationToken = default) {
             ReloadModulePaths(addedRoots, cancellationToken);
 
             if (!builtinsIsCreated) {
-                var builtinsModule = CreateBuiltinsModule(Services, Interpreter, StubCache);
-                BuiltinsModule = builtinsModule;
-                builtinsRef = new ModuleRef(builtinsModule);
+                _builtins = CreateBuiltinsModule(Services, Interpreter, StubCache);
+                builtinsRef = new ModuleRef(_builtins);
+                _builtins.Initialize();
             }
 
             Modules[BuiltinModuleName] = builtinsRef;
-            await AddBuiltinTypesToPathResolverAsync(cancellationToken);
+            AddBuiltinTypesToPathResolver();
         }
 
-        private static IBuiltinsPythonModule CreateBuiltinsModule(IServiceContainer services, IPythonInterpreter interpreter, IStubCache stubCache) {
+        private static BuiltinsPythonModule CreateBuiltinsModule(IServiceContainer services, IPythonInterpreter interpreter, IStubCache stubCache) {
             var moduleName = BuiltinTypeId.Unknown.GetModuleName(interpreter.LanguageVersion);
             var modulePath = stubCache.GetCacheFilePath(interpreter.Configuration.InterpreterPath);
             return new BuiltinsPythonModule(moduleName, modulePath, services);