Skip to content

Commit b22ec54

Browse files
AlexanderSherAlexanderSher
committed
- Address CR Comments
- Merge "Port import tests from old LS (microsoft#606)"
1 parent e2c12dc commit b22ec54

23 files changed

+570
-176
lines changed

‎src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ public override async Task<bool> WalkAsync(ForStatement node, CancellationToken
7575
}
7676

7777
public override async Task<bool> WalkAsync(FromImportStatement node, CancellationToken cancellationToken = default)
78-
=> ImportHandler.HandleFromImportAsync(node, cancellationToken);
78+
=> ImportHandler.HandleFromImport(node, cancellationToken);
7979

8080
public override Task<bool> WalkAsync(GlobalStatement node, CancellationToken cancellationToken = default)
8181
=> NonLocalHandler.HandleGlobalAsync(node, cancellationToken);
@@ -84,7 +84,7 @@ public override Task<bool> WalkAsync(IfStatement node, CancellationToken cancell
8484
=> ConditionalHandler.HandleIfAsync(node, cancellationToken);
8585

8686
public override async Task<bool> WalkAsync(ImportStatement node, CancellationToken cancellationToken = default)
87-
=> ImportHandler.HandleImportAsync(node, cancellationToken);
87+
=> ImportHandler.HandleImport(node, cancellationToken);
8888

8989
public override Task<bool> WalkAsync(NonlocalStatement node, CancellationToken cancellationToken = default)
9090
=> NonLocalHandler.HandleNonLocalAsync(node, cancellationToken);

‎src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525

2626
namespace Microsoft.Python.Analysis.Analyzer.Handlers {
2727
internal sealed partial class ImportHandler {
28-
public bool HandleFromImportAsync(FromImportStatement node, CancellationToken cancellationToken = default) {
28+
public bool HandleFromImport(FromImportStatement node, CancellationToken cancellationToken = default) {
2929
cancellationToken.ThrowIfCancellationRequested();
3030
if (Module.ModuleType == ModuleType.Specialized) {
3131
return false;

‎src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ namespace Microsoft.Python.Analysis.Analyzer.Handlers {
3030
internal sealed partial class ImportHandler : StatementHandler {
3131
public ImportHandler(AnalysisWalker walker) : base(walker) { }
3232

33-
public bool HandleImportAsync(ImportStatement node, CancellationToken cancellationToken = default) {
33+
public bool HandleImport(ImportStatement node, CancellationToken cancellationToken = default) {
3434
cancellationToken.ThrowIfCancellationRequested();
3535
if (Module.ModuleType == ModuleType.Specialized) {
3636
return false;

‎src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs

Lines changed: 48 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,8 @@
1414
// permissions and limitations under the License.
1515

1616
using System;
17-
using System.Collections;
1817
using System.Collections.Generic;
1918
using System.Diagnostics;
20-
using System.IO;
21-
using System.Linq;
2219
using System.Threading;
2320
using System.Threading.Tasks;
2421
using Microsoft.Python.Analysis.Core.DependencyResolution;
@@ -28,7 +25,6 @@
2825
using Microsoft.Python.Analysis.Types;
2926
using Microsoft.Python.Core;
3027
using Microsoft.Python.Core.Collections;
31-
using Microsoft.Python.Core.Diagnostics;
3228
using Microsoft.Python.Core.Disposables;
3329
using Microsoft.Python.Core.IO;
3430
using Microsoft.Python.Core.Logging;
@@ -38,8 +34,8 @@
3834
namespace Microsoft.Python.Analysis.Analyzer {
3935
public sealed class PythonAnalyzer : IPythonAnalyzer, IDisposable {
4036
private readonly IServiceManager _services;
41-
private readonly IDependencyResolver<ModuleKey, AnalysisEntry> _dependencyResolver;
42-
private readonly Dictionary<ModuleKey, AnalysisEntry> _analysisEntries = new Dictionary<ModuleKey, AnalysisEntry>();
37+
private readonly IDependencyResolver<ModuleKey, PythonAnalyzerEntry> _dependencyResolver;
38+
private readonly Dictionary<ModuleKey, PythonAnalyzerEntry> _analysisEntries = new Dictionary<ModuleKey, PythonAnalyzerEntry>();
4339
private readonly DisposeToken _disposeToken = DisposeToken.Create<PythonAnalyzer>();
4440
private readonly object _syncObj = new object();
4541
private readonly AsyncManualResetEvent _analysisCompleteEvent = new AsyncManualResetEvent();
@@ -51,7 +47,7 @@ public sealed class PythonAnalyzer : IPythonAnalyzer, IDisposable {
5147
public PythonAnalyzer(IServiceManager services) {
5248
_services = services;
5349
_log = services.GetService<ILogger>();
54-
_dependencyResolver = new DependencyResolver<ModuleKey, AnalysisEntry>(new ModuleDependencyFinder(services.GetService<IFileSystem>()));
50+
_dependencyResolver = new DependencyResolver<ModuleKey, PythonAnalyzerEntry>(new ModuleDependencyFinder(services.GetService<IFileSystem>()));
5551
_analysisCompleteEvent.Set();
5652
}
5753

@@ -65,50 +61,54 @@ public void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int
6561

6662
public async Task<IDocumentAnalysis> GetAnalysisAsync(IPythonModule module, int waitTime, CancellationToken cancellationToken) {
6763
var key = new ModuleKey(module);
68-
AnalysisEntry entry;
64+
PythonAnalyzerEntry entry;
6965
lock (_syncObj) {
7066
if (!_analysisEntries.TryGetValue(key, out entry)) {
71-
return new EmptyAnalysis(_services, (IDocument)module);
67+
var emptyAnalysis = new EmptyAnalysis(_services, (IDocument)module);
68+
entry = new PythonAnalyzerEntry(module, emptyAnalysis.Ast, emptyAnalysis, 0);
69+
_analysisEntries[key] = entry;
7270
}
7371
}
7472

75-
using (var timeoutCts = new CancellationTokenSource()) {
76-
if (waitTime >= 0 && !Debugger.IsAttached) {
77-
timeoutCts.CancelAfter(waitTime);
78-
}
7973

80-
using (var cts = CancellationTokenSource.CreateLinkedTokenSource(timeoutCts.Token, cancellationToken)) {
81-
var timeoutToken = timeoutCts.Token;
82-
while (!timeoutToken.IsCancellationRequested) {
83-
try {
84-
var analysis = await entry.GetAnalysisAsync(cts.Token);
85-
lock (_syncObj) {
86-
if (entry.Version == analysis.Version) {
87-
return analysis;
88-
}
89-
}
90-
} catch (OperationCanceledException) when (!timeoutToken.IsCancellationRequested && !cancellationToken.IsCancellationRequested) {
91-
lock (_syncObj) {
92-
if (!_analysisEntries.TryGetValue(key, out entry)) {
93-
return new EmptyAnalysis(_services, (IDocument)module);
94-
}
95-
}
74+
if (waitTime == 0 || Debugger.IsAttached) {
75+
return await GetAnalysisAsync(entry, default, cancellationToken);
76+
}
77+
78+
using (var timeoutCts = new CancellationTokenSource(waitTime))
79+
using (var cts = CancellationTokenSource.CreateLinkedTokenSource(timeoutCts.Token, cancellationToken)) {
80+
cts.CancelAfter(waitTime);
81+
var timeoutToken = timeoutCts.Token;
82+
return await GetAnalysisAsync(entry, timeoutToken, cts.Token);
83+
}
84+
}
85+
86+
private async Task<IDocumentAnalysis> GetAnalysisAsync(PythonAnalyzerEntry entry, CancellationToken timeoutCt, CancellationToken cancellationToken) {
87+
while (!timeoutCt.IsCancellationRequested) {
88+
try {
89+
var analysis = await entry.GetAnalysisAsync(cancellationToken);
90+
lock (_syncObj) {
91+
if (entry.Version == analysis.Version) {
92+
return analysis;
9693
}
9794
}
95+
} catch (OperationCanceledException) when (timeoutCt.IsCancellationRequested) {
96+
return entry.PreviousAnalysis;
9897
}
9998
}
10099

101100
return entry.PreviousAnalysis;
102101
}
103102

104-
private async Task AnalyzeDocumentAsync(IPythonModule module, PythonAst ast, int version, CancellationToken cancellationToken) {
103+
private async Task AnalyzeDocumentAsync(IPythonModule module, PythonAst ast, int bufferVersion, CancellationToken cancellationToken) {
105104
var key = new ModuleKey(module);
106-
AnalysisEntry entry;
105+
PythonAnalyzerEntry entry;
107106
lock (_syncObj) {
108107
if (_analysisEntries.TryGetValue(key, out entry)) {
109-
entry.Invalidate(_version);
108+
entry.Invalidate(_version + 1, ast);
110109
} else {
111-
_analysisEntries[key] = entry = new AnalysisEntry(module, ast, new EmptyAnalysis(_services, (IDocument)module), _version);
110+
entry = new PythonAnalyzerEntry(module, ast, new EmptyAnalysis(_services, (IDocument)module), _version);
111+
_analysisEntries[key] = entry;
112112
}
113113
}
114114

@@ -118,13 +118,13 @@ private async Task AnalyzeDocumentAsync(IPythonModule module, PythonAst ast, int
118118
using (var cts = CancellationTokenSource.CreateLinkedTokenSource(_disposeToken.CancellationToken, cancellationToken)) {
119119
var analysisToken = cts.Token;
120120

121-
var walker = await _dependencyResolver.AddChangesAsync(new ModuleKey(module), entry, version, cts.Token);
121+
var walker = await _dependencyResolver.AddChangesAsync(new ModuleKey(module), entry, bufferVersion, cts.Token);
122122
var abortAnalysisOnVersionChange = true;
123123
lock (_syncObj) {
124124
if (_version < walker.Version) {
125125
_version = walker.Version;
126126
foreach (var affectedEntry in walker.AffectedValues) {
127-
affectedEntry.Invalidate(_version);
127+
affectedEntry.Invalidate(_version, affectedEntry.Ast);
128128
if (affectedEntry.UserNotAnalyzed) {
129129
abortAnalysisOnVersionChange = false;
130130
}
@@ -137,12 +137,13 @@ private async Task AnalyzeDocumentAsync(IPythonModule module, PythonAst ast, int
137137
}
138138

139139
var stopWatch = Stopwatch.StartNew();
140-
IDependencyChainNode<AnalysisEntry> node;
140+
IDependencyChainNode<PythonAnalyzerEntry> node;
141141
while ((node = await walker.GetNextAsync(analysisToken)) != null) {
142142
lock (_syncObj) {
143143
if (_version > walker.Version) {
144144
if (abortAnalysisOnVersionChange) {
145-
break;
145+
stopWatch.Stop();
146+
return;
146147
}
147148

148149
if (!node.Value.UserNotAnalyzed) {
@@ -159,7 +160,6 @@ private async Task AnalyzeDocumentAsync(IPythonModule module, PythonAst ast, int
159160
}
160161
}
161162

162-
stopWatch.Stop();
163163

164164
if (walker.MissingKeys.Where(k => !k.IsTypeshed).Count == 0) {
165165
Interlocked.Exchange(ref _runningTasks, 0);
@@ -175,15 +175,15 @@ private static void LoadMissingDocuments(IPythonInterpreter interpreter, Immutab
175175
}
176176
}
177177

178-
private void StartAnalysis(IDependencyChainNode<AnalysisEntry> node, int version, Stopwatch stopWatch, CancellationToken cancellationToken)
178+
private void StartAnalysis(IDependencyChainNode<PythonAnalyzerEntry> node, int version, Stopwatch stopWatch, CancellationToken cancellationToken)
179179
=> Task.Run(() => AnalyzeAsync(node, version, stopWatch, cancellationToken), cancellationToken).DoNotWait();
180180

181181
/// <summary>
182182
/// Performs analysis of the document. Returns document global scope
183183
/// with declared variables and inner scopes. Does not analyze chain
184184
/// of dependencies, it is intended for the single file analysis.
185185
/// </summary>
186-
private async Task AnalyzeAsync(IDependencyChainNode<AnalysisEntry> node, int version, Stopwatch stopWatch, CancellationToken cancellationToken) {
186+
private async Task AnalyzeAsync(IDependencyChainNode<PythonAnalyzerEntry> node, int version, Stopwatch stopWatch, CancellationToken cancellationToken) {
187187
try {
188188
var startTime = stopWatch.ElapsedMilliseconds;
189189
var module = node.Value.Module;
@@ -202,22 +202,16 @@ private async Task AnalyzeAsync(IDependencyChainNode<AnalysisEntry> node, int ve
202202
var analysis = new DocumentAnalysis((IDocument)module, version, walker.GlobalScope, walker.Eval);
203203

204204
(module as IAnalyzable)?.NotifyAnalysisComplete(analysis);
205-
lock (_syncObj) {
206-
node.Value.TrySetAnalysis(analysis, version);
207-
}
205+
node.Value.TrySetAnalysis(analysis, version, _syncObj);
208206

209-
node.MarkCompleted();
210207
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) complete in {stopWatch.ElapsedMilliseconds - startTime} ms.");
211208
} catch (OperationCanceledException oce) {
212-
lock (_syncObj) {
213-
node.Value.TryCancel(oce, version);
214-
}
209+
node.Value.TryCancel(oce, version, _syncObj);
215210
} catch (Exception exception) {
216-
lock (_syncObj) {
217-
node.Value.TrySetException(exception, version);
218-
}
211+
node.Value.TrySetException(exception, version, _syncObj);
219212
} finally {
220213
Interlocked.Decrement(ref _runningTasks);
214+
node.MarkCompleted();
221215
}
222216
}
223217

@@ -230,7 +224,7 @@ private struct ModuleKey : IEquatable<ModuleKey> {
230224
public ModuleKey(IPythonModule module) {
231225
Name = module.Name;
232226
FilePath = module.ModuleType == ModuleType.CompiledBuiltin ? null : module.FilePath;
233-
IsTypeshed = module.ModuleType == ModuleType.Stub;
227+
IsTypeshed = module is StubPythonModule stub && stub.IsTypeshed;
234228
}
235229

236230
public ModuleKey(string name, string filePath, bool isTypeshed) {
@@ -266,69 +260,17 @@ public void Deconstruct(out string moduleName, out string filePath, out bool isT
266260
public override string ToString() => $"{Name}({FilePath})";
267261
}
268262

269-
private sealed class AnalysisEntry {
270-
private TaskCompletionSource<IDocumentAnalysis> _analysisTcs;
271-
272-
public IPythonModule Module { get; }
273-
public PythonAst Ast { get; }
274-
public IDocumentAnalysis PreviousAnalysis { get; private set; }
275-
public int Version { get; private set; }
276-
public bool UserNotAnalyzed => PreviousAnalysis is EmptyAnalysis && Module.ModuleType == ModuleType.User;
277-
278-
public AnalysisEntry(IPythonModule module, PythonAst ast, IDocumentAnalysis previousAnalysis, int version) {
279-
Module = module;
280-
Ast = ast;
281-
PreviousAnalysis = previousAnalysis;
282-
283-
Version = version;
284-
_analysisTcs = new TaskCompletionSource<IDocumentAnalysis>(TaskCreationOptions.RunContinuationsAsynchronously);
285-
}
286-
287-
public Task<IDocumentAnalysis> GetAnalysisAsync(CancellationToken cancellationToken)
288-
=> _analysisTcs.Task.ContinueWith(t => t.GetAwaiter().GetResult(), cancellationToken);
289-
290-
public void TrySetAnalysis(IDocumentAnalysis analysis, int version) {
291-
if (version >= Version) {
292-
_analysisTcs.TrySetResult(analysis);
293-
}
294-
}
295-
296-
public void TrySetException(Exception ex, int version) {
297-
if (version >= Version) {
298-
_analysisTcs.TrySetException(ex);
299-
}
300-
}
301-
302-
public void TryCancel(OperationCanceledException oce, int version) {
303-
if (version >= Version) {
304-
_analysisTcs.TrySetCanceled(oce.CancellationToken);
305-
}
306-
}
307-
308-
public void Invalidate(int version) {
309-
if (Version >= version) {
310-
return;
311-
}
312-
Version = version;
313-
314-
if (!_analysisTcs.TrySetCanceled() && _analysisTcs.Task.Status == TaskStatus.RanToCompletion) {
315-
PreviousAnalysis = _analysisTcs.Task.Result;
316-
}
317-
_analysisTcs = new TaskCompletionSource<IDocumentAnalysis>();
318-
}
319-
}
320-
321-
private sealed class ModuleDependencyFinder : IDependencyFinder<ModuleKey, AnalysisEntry> {
263+
private sealed class ModuleDependencyFinder : IDependencyFinder<ModuleKey, PythonAnalyzerEntry> {
322264
private readonly IFileSystem _fileSystem;
323265

324266
public ModuleDependencyFinder(IFileSystem fileSystem) {
325267
_fileSystem = fileSystem;
326268
}
327269

328-
public Task<ImmutableArray<ModuleKey>> FindDependenciesAsync(AnalysisEntry value, CancellationToken cancellationToken) {
270+
public Task<ImmutableArray<ModuleKey>> FindDependenciesAsync(PythonAnalyzerEntry value, CancellationToken cancellationToken) {
329271
var dependencies = new HashSet<ModuleKey>();
330272
var module = value.Module;
331-
var isTypeshed = module.ModuleType == ModuleType.Stub; // TODO: This is not the correct way to determine Typeshed.
273+
var isTypeshed = module is StubPythonModule stub && stub.IsTypeshed;
332274
var moduleResolution = module.Interpreter.ModuleResolution;
333275
var pathResolver = isTypeshed
334276
? module.Interpreter.TypeshedResolution.CurrentPathResolver

0 commit comments

Comments
 (0)