Skip to content
This repository was archived by the owner on Apr 14, 2022. It is now read-only.

Commit 1192f5d

Browse files
author
Mikhail Arkhipov
authored
Wire up events from RDT to module resolution (#703)
* Fix #668 (partial) * Tests * Revert "Tests" This reverts commit 7ffc9db. * Basic event wire up * Add test * Test * Formatting * Fix search paths mess * Handle deleted files better * PR feedback * MErge issues * Fix race condition at builtins load * PR feedback
1 parent ddc2060 commit 1192f5d

26 files changed

+277
-89
lines changed

‎src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,12 @@ public interface IPythonAnalyzer {
3737
/// Invalidates current analysis for the module, assuming that AST for the new analysis will be provided later.
3838
/// </summary>
3939
void InvalidateAnalysis(IPythonModule module);
40-
40+
41+
/// <summary>
42+
/// Removes modules from the analysis.
43+
/// </summary>
44+
void RemoveAnalysis(IPythonModule module);
45+
4146
/// <summary>
4247
/// Get most recent analysis for module. If after specified time analysis isn't available, returns previously calculated analysis.
4348
/// </summary>

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ public IDisposable OpenScope(IPythonModule module, ScopeStatement node, out Scop
175175
if (scope == null) {
176176
scope = new Scope(node, fromScope, true);
177177
fromScope.AddChildScope(scope);
178-
_scopeLookupCache[node] = fromScope;
178+
_scopeLookupCache[node] = scope;
179179
}
180180
}
181181

‎src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,12 @@ public void InvalidateAnalysis(IPythonModule module) {
112112
}
113113
}
114114

115+
public void RemoveAnalysis(IPythonModule module) {
116+
lock (_syncObj) {
117+
_analysisEntries.Remove(new AnalysisModuleKey(module));
118+
}
119+
}
120+
115121
public void EnqueueDocumentForAnalysis(IPythonModule module, ImmutableArray<IPythonModule> analysisDependencies) {
116122
var key = new AnalysisModuleKey(module);
117123
PythonAnalyzerEntry entry;
@@ -269,7 +275,7 @@ private async Task<int> AnalyzeAffectedEntriesAsync(IDependencyChainWalker<Analy
269275

270276
return remaining;
271277
}
272-
278+
273279
private void LoadMissingDocuments(IPythonInterpreter interpreter, ImmutableArray<AnalysisModuleKey> missingKeys) {
274280
foreach (var (moduleName, _, isTypeshed) in missingKeys) {
275281
var moduleResolution = isTypeshed ? interpreter.TypeshedResolution : interpreter.ModuleResolution;
@@ -356,7 +362,7 @@ private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, Pytho
356362
// Python analyzer to call NotifyAnalysisComplete.
357363
walker.Complete();
358364
cancellationToken.ThrowIfCancellationRequested();
359-
var analysis = new DocumentAnalysis((IDocument) module, version, walker.GlobalScope, walker.Eval);
365+
var analysis = new DocumentAnalysis((IDocument)module, version, walker.GlobalScope, walker.Eval);
360366

361367
(module as IAnalyzable)?.NotifyAnalysisComplete(analysis);
362368
entry.TrySetAnalysis(analysis, version);

‎src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,18 +47,19 @@ private async Task LoadBuiltinTypesAsync(string root, IServiceManager sm, Cancel
4747

4848
sm.AddService(this);
4949
_moduleResolution = new MainModuleResolution(root, sm);
50-
await _moduleResolution.InitializeAsync(cancellationToken);
5150

52-
_stubResolution = new TypeshedResolution(sm);
53-
await _stubResolution.InitializeAsync(cancellationToken);
54-
55-
var builtinModule = _moduleResolution.BuiltinsModule;
5651
lock (_lock) {
52+
var builtinModule = _moduleResolution.CreateBuiltinsModule();
5753
_builtinTypes[BuiltinTypeId.NoneType]
5854
= new PythonType("NoneType", builtinModule, string.Empty, LocationInfo.Empty, BuiltinTypeId.NoneType);
5955
_builtinTypes[BuiltinTypeId.Unknown]
6056
= UnknownType = new PythonType("Unknown", builtinModule, string.Empty, LocationInfo.Empty);
6157
}
58+
await _moduleResolution.InitializeAsync(cancellationToken);
59+
60+
_stubResolution = new TypeshedResolution(sm);
61+
await _stubResolution.InitializeAsync(cancellationToken);
62+
6263
await _moduleResolution.LoadBuiltinTypesAsync(cancellationToken);
6364
}
6465

‎src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121

2222
namespace Microsoft.Python.Analysis.Dependencies {
2323
internal sealed class DependencyResolver<TKey, TValue> : IDependencyResolver<TKey, TValue> {
24-
private readonly IDependencyFinder<TKey, TValue> _dependencyFinder;
2524
private readonly DependencyGraph<TKey, TValue> _vertices = new DependencyGraph<TKey, TValue>();
2625
private readonly Dictionary<TKey, DependencyVertex<TKey, TValue>> _changedVertices = new Dictionary<TKey, DependencyVertex<TKey, TValue>>();
2726
private readonly object _syncObj = new object();

‎src/Analysis/Ast/Impl/Documents/Definitions/IRunningDocumentTable.cs

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,20 @@ public interface IRunningDocumentTable: IEnumerable<IDocument> {
5555
/// </summary>
5656
IDocument GetDocument(string name);
5757

58+
/// <summary>
59+
/// Increase reference count of the document.
60+
/// </summary>
61+
/// <param name="uri"></param>
62+
/// <returns>New lock count or -1 if document was not found.</returns>
63+
int LockDocument(Uri uri);
64+
65+
/// <summary>
66+
/// Decrease reference count of the document.
67+
/// </summary>
68+
/// <param name="uri"></param>
69+
/// <returns>New lock count or -1 if document was not found.</returns>
70+
int UnlockDocument(Uri uri);
71+
5872
/// <summary>
5973
/// Fires when document is opened.
6074
/// </summary>
@@ -64,5 +78,10 @@ public interface IRunningDocumentTable: IEnumerable<IDocument> {
6478
/// Fires when document is closed.
6579
/// </summary>
6680
event EventHandler<DocumentEventArgs> Closed;
81+
82+
/// <summary>
83+
/// Fires when document is removed.
84+
/// </summary>
85+
event EventHandler<DocumentEventArgs> Removed;
6786
}
6887
}

‎src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@
1919
using System.Diagnostics;
2020
using System.IO;
2121
using System.Linq;
22+
using Microsoft.Python.Analysis.Analyzer;
2223
using Microsoft.Python.Analysis.Modules;
2324
using Microsoft.Python.Core;
24-
using Microsoft.Python.Core.Diagnostics;
2525

2626
namespace Microsoft.Python.Analysis.Documents {
2727
/// <summary>
@@ -52,6 +52,7 @@ public RunningDocumentTable(string workspaceRoot, IServiceContainer services) {
5252

5353
public event EventHandler<DocumentEventArgs> Opened;
5454
public event EventHandler<DocumentEventArgs> Closed;
55+
public event EventHandler<DocumentEventArgs> Removed;
5556

5657
/// <summary>
5758
/// Adds file to the list of available documents.
@@ -60,17 +61,19 @@ public RunningDocumentTable(string workspaceRoot, IServiceContainer services) {
6061
/// <param name="content">Document content</param>
6162
/// <param name="filePath">Optional file path, if different from the URI.</param>
6263
public IDocument OpenDocument(Uri uri, string content, string filePath = null) {
63-
var justOpened = false;
64+
bool justOpened;
6465
DocumentEntry entry;
6566
lock (_lock) {
6667
entry = FindDocument(null, uri);
6768
if (entry == null) {
69+
var resolver = _services.GetService<IPythonInterpreter>().ModuleResolution.CurrentPathResolver;
70+
var moduleType = resolver.IsLibraryFile(uri.ToAbsolutePath()) ? ModuleType.Library : ModuleType.User;
6871
var mco = new ModuleCreationOptions {
6972
ModuleName = Path.GetFileNameWithoutExtension(uri.LocalPath),
7073
Content = content,
7174
FilePath = filePath,
7275
Uri = uri,
73-
ModuleType = ModuleType.User
76+
ModuleType = moduleType
7477
};
7578
entry = CreateDocument(mco);
7679
}
@@ -114,39 +117,61 @@ public IDocument GetDocument(string name) {
114117
}
115118
}
116119

120+
public int LockDocument(Uri uri) {
121+
lock (_lock) {
122+
if (_documentsByUri.TryGetValue(uri, out var entry)) {
123+
return ++entry.LockCount;
124+
}
125+
return -1;
126+
}
127+
}
128+
129+
public int UnlockDocument(Uri uri) {
130+
lock (_lock) {
131+
if (_documentsByUri.TryGetValue(uri, out var entry)) {
132+
return --entry.LockCount;
133+
}
134+
return -1;
135+
}
136+
}
137+
117138
public IEnumerator<IDocument> GetEnumerator() => _documentsByUri.Values.Select(e => e.Document).GetEnumerator();
118139

119140
public void CloseDocument(Uri documentUri) {
120-
var justClosed = false;
141+
var closed = false;
142+
var removed = false;
121143
DocumentEntry entry;
122144
lock (_lock) {
123145
if (_documentsByUri.TryGetValue(documentUri, out entry)) {
124146
Debug.Assert(entry.LockCount >= 1);
125147

126148
if (entry.Document.IsOpen) {
127149
entry.Document.IsOpen = false;
128-
justClosed = true;
150+
closed = true;
129151
}
130152

131153
entry.LockCount--;
132154

133155
if (entry.LockCount == 0) {
134156
_documentsByUri.Remove(documentUri);
135157
_documentsByName.Remove(entry.Document.Name);
158+
removed = true;
136159
entry.Document.Dispose();
137160
}
138-
// TODO: Remove from module resolution?
139161
}
140162
}
141-
if(justClosed) {
163+
if (closed) {
142164
Closed?.Invoke(this, new DocumentEventArgs(entry.Document));
143165
}
166+
if (removed) {
167+
Removed?.Invoke(this, new DocumentEventArgs(entry.Document));
168+
}
144169
}
145170

146171
IEnumerator IEnumerable.GetEnumerator() => _documentsByUri.Values.GetEnumerator();
147172

148173
public void Dispose() {
149-
lock(_lock) {
174+
lock (_lock) {
150175
foreach (var d in _documentsByUri.Values.OfType<IDisposable>()) {
151176
d.Dispose();
152177
}

‎src/Analysis/Ast/Impl/Modules/Definitions/IModuleCache.cs

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,6 @@
1313
// See the Apache Version 2.0 License for specific language governing
1414
// permissions and limitations under the License.
1515

16-
using System.Threading;
17-
using System.Threading.Tasks;
18-
using Microsoft.Python.Analysis.Documents;
19-
2016
namespace Microsoft.Python.Analysis.Modules {
2117
public interface IModuleCache {
2218
string GetCacheFilePath(string filePath);

‎src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
// See the Apache Version 2.0 License for specific language governing
1414
// permissions and limitations under the License.
1515

16-
using System.Collections.Generic;
1716
using System.Threading;
1817
using System.Threading.Tasks;
1918
using Microsoft.Python.Analysis.Core.DependencyResolution;

‎src/Analysis/Ast/Impl/Modules/ModuleCache.cs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,7 @@
1818
using System.IO;
1919
using System.Security.Cryptography;
2020
using System.Text;
21-
using System.Threading;
2221
using System.Threading.Tasks;
23-
using Microsoft.Python.Analysis.Documents;
2422
using Microsoft.Python.Core;
2523
using Microsoft.Python.Core.IO;
2624
using Microsoft.Python.Core.Logging;
@@ -43,7 +41,7 @@ public ModuleCache(IPythonInterpreter interpreter, IServiceContainer services) {
4341
_log = services.GetService<ILogger>();
4442
_skipCache = string.IsNullOrEmpty(_interpreter.Configuration.DatabasePath);
4543
}
46-
44+
4745
public string GetCacheFilePath(string filePath) {
4846
if (string.IsNullOrEmpty(filePath) || !PathEqualityComparer.IsValidPath(ModuleCachePath)) {
4947
if (!_loggedBadDbPath) {

‎src/Analysis/Ast/Impl/Modules/PythonModule.cs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
using Microsoft.Python.Analysis.Values;
3030
using Microsoft.Python.Core;
3131
using Microsoft.Python.Core.Diagnostics;
32+
using Microsoft.Python.Core.Disposables;
3233
using Microsoft.Python.Core.IO;
3334
using Microsoft.Python.Core.Logging;
3435
using Microsoft.Python.Core.Text;
@@ -53,7 +54,7 @@ private enum State {
5354
}
5455

5556
private readonly DocumentBuffer _buffer = new DocumentBuffer();
56-
private readonly CancellationTokenSource _allProcessingCts = new CancellationTokenSource();
57+
private readonly DisposeToken _disposeToken = DisposeToken.Create< PythonModule>();
5758
private IReadOnlyList<DiagnosticsEntry> _parseErrors = Array.Empty<DiagnosticsEntry>();
5859
private readonly IDiagnosticsService _diagnosticsService;
5960

@@ -242,8 +243,7 @@ private void LoadContent(string content) {
242243

243244
protected virtual void Dispose(bool disposing) {
244245
_diagnosticsService?.Remove(Uri);
245-
_allProcessingCts.Cancel();
246-
_allProcessingCts.Dispose();
246+
_disposeToken.TryMarkDisposed();
247247
}
248248
#endregion
249249

@@ -310,7 +310,7 @@ public void Update(IEnumerable<DocumentChange> changes) {
310310
_parseCts = new CancellationTokenSource();
311311

312312
_linkedParseCts?.Dispose();
313-
_linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, _parseCts.Token);
313+
_linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_disposeToken.CancellationToken, _parseCts.Token);
314314

315315
_buffer.Update(changes);
316316
Parse();
@@ -332,7 +332,7 @@ private void Parse() {
332332
_parseCts = new CancellationTokenSource();
333333

334334
_linkedParseCts?.Dispose();
335-
_linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, _parseCts.Token);
335+
_linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_disposeToken.CancellationToken, _parseCts.Token);
336336

337337
ContentState = State.Parsing;
338338
_parsingTask = Task.Run(() => Parse(_linkedParseCts.Token), _linkedParseCts.Token);
@@ -383,7 +383,7 @@ private void Parse(CancellationToken cancellationToken) {
383383
ContentState = State.Analyzing;
384384

385385
var analyzer = Services.GetService<IPythonAnalyzer>();
386-
analyzer.EnqueueDocumentForAnalysis(this, ast, version, _allProcessingCts.Token);
386+
analyzer.EnqueueDocumentForAnalysis(this, ast, version, _disposeToken.CancellationToken);
387387
}
388388

389389
lock (AnalysisLock) {

0 commit comments

Comments
 (0)