Skip to content
This repository was archived by the owner on Apr 14, 2022. It is now read-only.

Commit c62c1d8

Browse files
author
Mikhail Arkhipov
authored
Ensure builtins module is created synchronously (#1798)
* Remove stale reference * Don't suppress LHS diagnostics on augmented assign * Revert "Don't suppress LHS diagnostics on augmented assign" This reverts commit 6109ac7. * Sync builtins creation * Sync builtins * Usings * Debug code
1 parent 0bd7737 commit c62c1d8

File tree

6 files changed

+56
-29
lines changed

6 files changed

+56
-29
lines changed

src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
// permissions and limitations under the License.
1515

1616
using Microsoft.Python.Analysis.Dependencies;
17+
using Microsoft.Python.Parsing.Ast;
1718

1819
namespace Microsoft.Python.Analysis.Analyzer {
1920
/// <summary>
@@ -30,6 +31,11 @@ internal interface IAnalyzable {
3031
/// </summary>
3132
void NotifyAnalysisBegins();
3233

34+
/// <summary>
35+
/// Performs standard analysis pass. Does not include any restoration from databases.
36+
/// </summary>
37+
ModuleWalker Analyze(PythonAst ast);
38+
3339
/// <summary>
3440
/// Notifies document that its analysis is now complete.
3541
/// </summary>

src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -527,11 +527,11 @@ private IDocumentAnalysis RestoreOrAnalyzeModule(IDependencyChainSingleNode<Pyth
527527
return analysis;
528528
}
529529

530-
var eval = new ExpressionEval(_services, module, ast);
531-
var walker = new ModuleWalker(eval, SimpleImportedVariableHandler.Instance);
532-
ast.Walk(walker);
533-
walker.Complete();
534-
return CreateAnalysis(node, (IDocument)module, ast, version, walker);
530+
if (module is IAnalyzable analyzable) {
531+
var walker = analyzable.Analyze(ast);
532+
return CreateAnalysis(node, (IDocument)module, ast, version, walker);
533+
}
534+
return new EmptyAnalysis(_services, (IDocument)module);
535535
}
536536

537537
private bool MarkNodeWalked(IDependencyChainNode node) {

src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
using Microsoft.Python.Analysis.Modules.Resolution;
2323
using Microsoft.Python.Analysis.Specializations.Typing;
2424
using Microsoft.Python.Analysis.Types;
25-
using Microsoft.Python.Analysis.Values;
2625
using Microsoft.Python.Core;
2726
using Microsoft.Python.Core.Collections;
2827
using Microsoft.Python.Core.Services;

src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,9 @@
1717
using System.Collections.Generic;
1818
using System.Diagnostics;
1919
using System.Linq;
20+
using System.Threading;
21+
using Microsoft.Python.Analysis.Analyzer;
2022
using Microsoft.Python.Analysis.Specializations;
21-
using Microsoft.Python.Analysis.Specializations.Typing;
2223
using Microsoft.Python.Analysis.Types;
2324
using Microsoft.Python.Analysis.Values;
2425
using Microsoft.Python.Core;
@@ -41,15 +42,28 @@ internal sealed class BuiltinsPythonModule : CompiledPythonModule, IBuiltinsPyth
4142
public BuiltinsPythonModule(string moduleName, string filePath, IServiceContainer services)
4243
: base(moduleName, ModuleType.Builtins, filePath, null, false, false, services) { } // TODO: builtins stub & persistence
4344

45+
#region IMemberContainer
4446
public override IMember GetMember(string name) => _hiddenNames.Contains(name) ? null : base.GetMember(name);
4547

4648
public IMember GetAnyMember(string name) => base.GetMember(name);
4749

4850
public override IEnumerable<string> GetMemberNames() => base.GetMemberNames().Except(_hiddenNames).ToArray();
51+
#endregion
52+
53+
public void Initialize() => ParseAndLogExceptions(CancellationToken.None);
4954

5055
protected override string[] GetScrapeArguments(IPythonInterpreter interpreter)
5156
=> !InstallPath.TryGetFile("scrape_module.py", out var sb) ? null : new[] { "-W", "ignore", "-B", "-E", sb };
5257

58+
protected override void Parse() { }
59+
60+
protected override void Analyze(PythonAst ast, int version) {
61+
NotifyAnalysisBegins();
62+
var walker = Analyze(ast);
63+
var analysis = new DocumentAnalysis(this, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames);
64+
NotifyAnalysisComplete(analysis);
65+
}
66+
5367
protected override void OnAnalysisComplete() {
5468
SpecializeTypes();
5569
SpecializeFunctions();

src/Analysis/Ast/Impl/Modules/PythonModule.cs

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,11 @@
2222
using System.Threading;
2323
using System.Threading.Tasks;
2424
using Microsoft.Python.Analysis.Analyzer;
25+
using Microsoft.Python.Analysis.Analyzer.Evaluation;
26+
using Microsoft.Python.Analysis.Analyzer.Handlers;
2527
using Microsoft.Python.Analysis.Dependencies;
2628
using Microsoft.Python.Analysis.Diagnostics;
2729
using Microsoft.Python.Analysis.Documents;
28-
using Microsoft.Python.Analysis.Specializations.Typing;
2930
using Microsoft.Python.Analysis.Types;
3031
using Microsoft.Python.Analysis.Values;
3132
using Microsoft.Python.Core;
@@ -305,7 +306,7 @@ public void Invalidate() {
305306
Services.GetService<IPythonAnalyzer>().InvalidateAnalysis(this);
306307
}
307308

308-
private void Parse() {
309+
protected virtual void Parse() {
309310
_parseCts?.Cancel();
310311
_parseCts = new CancellationTokenSource();
311312

@@ -316,7 +317,7 @@ private void Parse() {
316317
_parsingTask = Task.Run(() => ParseAndLogExceptions(_linkedParseCts.Token), _linkedParseCts.Token);
317318
}
318319

319-
private void ParseAndLogExceptions(CancellationToken cancellationToken) {
320+
protected void ParseAndLogExceptions(CancellationToken cancellationToken) {
320321
try {
321322
Parse(cancellationToken);
322323
} catch (Exception ex) when (!(ex is OperationCanceledException)) {
@@ -325,6 +326,15 @@ private void ParseAndLogExceptions(CancellationToken cancellationToken) {
325326
}
326327
}
327328

329+
protected virtual void Analyze(PythonAst ast, int version) {
330+
if (ContentState < State.Analyzing) {
331+
ContentState = State.Analyzing;
332+
333+
var analyzer = Services.GetService<IPythonAnalyzer>();
334+
analyzer.EnqueueDocumentForAnalysis(this, ast, version);
335+
}
336+
}
337+
328338
private void Parse(CancellationToken cancellationToken) {
329339
CollectingErrorSink sink = null;
330340
int version;
@@ -345,7 +355,6 @@ private void Parse(CancellationToken cancellationToken) {
345355
}
346356

347357
var ast = parser.ParseFile(Uri);
348-
349358
// Log?.Log(TraceEventType.Verbose, $"Parse complete: {Name} ({ModuleType})");
350359

351360
lock (_syncObj) {
@@ -370,13 +379,7 @@ private void Parse(CancellationToken cancellationToken) {
370379
}
371380

372381
NewAst?.Invoke(this, EventArgs.Empty);
373-
374-
if (ContentState < State.Analyzing) {
375-
ContentState = State.Analyzing;
376-
377-
var analyzer = Services.GetService<IPythonAnalyzer>();
378-
analyzer.EnqueueDocumentForAnalysis(this, ast, version);
379-
}
382+
Analyze(ast, version);
380383

381384
lock (_syncObj) {
382385
_parsingTask = null;
@@ -423,6 +426,14 @@ public void NotifyAnalysisBegins() {
423426
}
424427
}
425428

429+
public ModuleWalker Analyze(PythonAst ast) {
430+
var eval = new ExpressionEval(Services, this, ast);
431+
var walker = new ModuleWalker(eval, SimpleImportedVariableHandler.Instance);
432+
ast.Walk(walker);
433+
walker.Complete();
434+
return walker;
435+
}
436+
426437
public void NotifyAnalysisComplete(IDocumentAnalysis analysis) {
427438
lock (_syncObj) {
428439
if (analysis.Version < Analysis.Version) {

src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ namespace Microsoft.Python.Analysis.Modules.Resolution {
3939
internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManagement {
4040
private readonly ConcurrentDictionary<string, IPythonModule> _specialized = new ConcurrentDictionary<string, IPythonModule>();
4141
private readonly IUIService _ui;
42+
private BuiltinsPythonModule _builtins;
4243
private IModuleDatabaseService _dbService;
4344
private IRunningDocumentTable _rdt;
4445

@@ -53,7 +54,7 @@ public MainModuleResolution(string root, IServiceContainer services, ImmutableAr
5354
public string BuiltinModuleName => BuiltinTypeId.Unknown.GetModuleName(Interpreter.LanguageVersion);
5455
public ImmutableArray<PythonLibraryPath> LibraryPaths { get; private set; } = ImmutableArray<PythonLibraryPath>.Empty;
5556

56-
public IBuiltinsPythonModule BuiltinsModule { get; private set; }
57+
public IBuiltinsPythonModule BuiltinsModule => _builtins;
5758

5859
public IEnumerable<IPythonModule> GetImportedModules(CancellationToken cancellationToken) {
5960
foreach (var module in _specialized.Values) {
@@ -185,12 +186,8 @@ public IPythonModule GetSpecializedModule(string fullName, bool allowCreation =
185186
public bool IsSpecializedModule(string fullName, string modulePath = null)
186187
=> _specialized.ContainsKey(fullName);
187188

188-
private async Task AddBuiltinTypesToPathResolverAsync(CancellationToken cancellationToken = default) {
189-
var analyzer = Services.GetService<IPythonAnalyzer>();
190-
await analyzer.GetAnalysisAsync(BuiltinsModule, Timeout.Infinite, cancellationToken);
191-
189+
private void AddBuiltinTypesToPathResolver() {
192190
Check.InvalidOperation(!(BuiltinsModule.Analysis is EmptyAnalysis), "Builtins analysis did not complete correctly.");
193-
194191
// Add built-in module names
195192
var builtinModuleNamesMember = BuiltinsModule.GetAnyMember("__builtin_module_names__");
196193
var value = builtinModuleNamesMember is IVariable variable ? variable.Value : builtinModuleNamesMember;
@@ -222,16 +219,16 @@ public async Task ReloadAsync(CancellationToken cancellationToken = default) {
222219
ReloadModulePaths(addedRoots, cancellationToken);
223220

224221
if (!builtinsIsCreated) {
225-
var builtinsModule = CreateBuiltinsModule(Services, Interpreter, StubCache);
226-
BuiltinsModule = builtinsModule;
227-
builtinsRef = new ModuleRef(builtinsModule);
222+
_builtins = CreateBuiltinsModule(Services, Interpreter, StubCache);
223+
builtinsRef = new ModuleRef(_builtins);
224+
_builtins.Initialize();
228225
}
229226

230227
Modules[BuiltinModuleName] = builtinsRef;
231-
await AddBuiltinTypesToPathResolverAsync(cancellationToken);
228+
AddBuiltinTypesToPathResolver();
232229
}
233230

234-
private static IBuiltinsPythonModule CreateBuiltinsModule(IServiceContainer services, IPythonInterpreter interpreter, IStubCache stubCache) {
231+
private static BuiltinsPythonModule CreateBuiltinsModule(IServiceContainer services, IPythonInterpreter interpreter, IStubCache stubCache) {
235232
var moduleName = BuiltinTypeId.Unknown.GetModuleName(interpreter.LanguageVersion);
236233
var modulePath = stubCache.GetCacheFilePath(interpreter.Configuration.InterpreterPath);
237234
return new BuiltinsPythonModule(moduleName, modulePath, services);

0 commit comments

Comments
 (0)