Skip to content
This repository was archived by the owner on Apr 14, 2022. It is now read-only.

Commit efe4ca2

Browse files
AlexanderSherAlexanderSher
authored
Fix #923: Runaway memory usage when working with msticpy (#972)
* Fix #923: Runaway memory usage when working with msticpy * - Fix sorting - Mark session canceled if ast isn't set yet
1 parent b828161 commit efe4ca2

20 files changed

+437
-251
lines changed

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
using System;
1717
using System.Collections.Generic;
1818
using System.Diagnostics;
19-
using System.Reflection;
2019
using Microsoft.Python.Analysis.Analyzer.Symbols;
2120
using Microsoft.Python.Analysis.Diagnostics;
2221
using Microsoft.Python.Analysis.Modules;

‎src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs

Lines changed: 72 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -197,43 +197,73 @@ private void AnalyzeDocument(AnalysisModuleKey key, PythonAnalyzerEntry entry, I
197197
_analysisCompleteEvent.Reset();
198198
_log?.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name}({entry.Module.ModuleType}) queued");
199199

200-
var snapshot = _dependencyResolver.NotifyChanges(key, entry, dependencies);
201-
200+
var graphVersion = _dependencyResolver.ChangeValue(key, entry, dependencies);
201+
202202
lock (_syncObj) {
203-
if (_version > snapshot.Version) {
203+
if (_version > graphVersion) {
204204
return;
205205
}
206206

207-
_version = snapshot.Version;
207+
_version = graphVersion;
208+
_currentSession?.Cancel();
208209
}
209210

210-
if (snapshot.MissingKeys.Count > 0) {
211-
LoadMissingDocuments(entry.Module.Interpreter, snapshot.MissingKeys);
212-
}
211+
UpdateDependentEntriesDepth(entry, dependencies, graphVersion);
213212

214-
if (TryCreateSession(snapshot, entry, cancellationToken, out var session)) {
213+
if (TryCreateSession(graphVersion, entry, cancellationToken, out var session)) {
215214
session.Start(true);
216215
}
217216
}
218217

219-
private bool TryCreateSession(DependencyGraphSnapshot<AnalysisModuleKey, PythonAnalyzerEntry> snapshot, PythonAnalyzerEntry entry, CancellationToken cancellationToken, out PythonAnalyzerSession session) {
218+
private void UpdateDependentEntriesDepth(PythonAnalyzerEntry entry, ImmutableArray<AnalysisModuleKey> dependentKeys, int graphVersion) {
219+
if (dependentKeys.Count == 0) {
220+
return;
221+
}
222+
223+
var dependentEntries = new List<PythonAnalyzerEntry>();
224+
lock (_syncObj) {
225+
if (_version > graphVersion) {
226+
return;
227+
}
228+
229+
foreach (var key in dependentKeys) {
230+
if (_analysisEntries.TryGetValue(key, out var value)) {
231+
dependentEntries.Add(value);
232+
}
233+
}
234+
}
235+
236+
if (dependentEntries.Count == 0) {
237+
return;
238+
}
239+
240+
var depth = entry.Depth;
241+
foreach (var dependentEntry in dependentEntries) {
242+
dependentEntry.SetDepth(graphVersion, depth);
243+
}
244+
}
245+
246+
private bool TryCreateSession(int graphVersion, PythonAnalyzerEntry entry, CancellationToken cancellationToken, out PythonAnalyzerSession session) {
220247
var analyzeUserModuleOutOfOrder = false;
221248
lock (_syncObj) {
222249
if (_currentSession != null) {
223-
if (_currentSession.Version > snapshot.Version || _nextSession != null && _nextSession.Version > snapshot.Version) {
250+
if (_currentSession.Version > graphVersion || _nextSession != null && _nextSession.Version > graphVersion) {
224251
session = null;
225252
return false;
226253
}
227254

228255
analyzeUserModuleOutOfOrder = !_currentSession.IsCompleted && entry.IsUserModule && _currentSession.AffectedEntriesCount >= _maxTaskRunning;
229-
if (_version > snapshot.Version && analyzeUserModuleOutOfOrder) {
256+
if (_version > graphVersion && analyzeUserModuleOutOfOrder) {
230257
session = CreateSession(null, entry, cancellationToken);
231258
return true;
232259
}
233260
}
234261
}
235262

236-
if (!_dependencyResolver.TryCreateWalker(snapshot.Version, out var walker)) {
263+
var snapshot = _dependencyResolver.CurrentGraphSnapshot;
264+
LoadMissingDocuments(entry.Module.Interpreter, snapshot.MissingKeys);
265+
266+
if (!_dependencyResolver.TryCreateWalker(snapshot, out var walker)) {
237267
session = null;
238268
return false;
239269
}
@@ -254,7 +284,6 @@ private bool TryCreateSession(DependencyGraphSnapshot<AnalysisModuleKey, PythonA
254284
return true;
255285
}
256286

257-
_currentSession.Cancel();
258287
_nextSession = session = CreateSession(walker, analyzeUserModuleOutOfOrder ? entry : null, cancellationToken);
259288
return analyzeUserModuleOutOfOrder;
260289
}
@@ -278,9 +307,37 @@ private PythonAnalyzerSession CreateSession(IDependencyChainWalker<AnalysisModul
278307
=> new PythonAnalyzerSession(_services, _progress, _analysisCompleteEvent, _startNextSession, _disposeToken.CancellationToken, cancellationToken, walker, _version, entry);
279308

280309
private void LoadMissingDocuments(IPythonInterpreter interpreter, ImmutableArray<AnalysisModuleKey> missingKeys) {
281-
foreach (var (moduleName, _, isTypeshed) in missingKeys) {
310+
if (missingKeys.Count == 0) {
311+
return;
312+
}
313+
314+
var foundKeys = ImmutableArray<AnalysisModuleKey>.Empty;
315+
foreach (var missingKey in missingKeys) {
316+
lock (_syncObj) {
317+
if (_analysisEntries.TryGetValue(missingKey, out _)) {
318+
continue;
319+
}
320+
}
321+
322+
var (moduleName, _, isTypeshed) = missingKey;
282323
var moduleResolution = isTypeshed ? interpreter.TypeshedResolution : interpreter.ModuleResolution;
283-
moduleResolution.GetOrLoadModule(moduleName);
324+
var module = moduleResolution.GetOrLoadModule(moduleName);
325+
if (module != null && module.ModuleType != ModuleType.Unresolved) {
326+
foundKeys = foundKeys.Add(missingKey);
327+
}
328+
}
329+
330+
if (foundKeys.Count > 0) {
331+
foreach (var foundKey in foundKeys) {
332+
PythonAnalyzerEntry entry;
333+
lock (_syncObj) {
334+
if (!_analysisEntries.TryGetValue(foundKey, out entry)) {
335+
continue;
336+
}
337+
}
338+
339+
_dependencyResolver.TryAddValue(foundKey, entry, ImmutableArray<AnalysisModuleKey>.Empty);
340+
}
284341
}
285342
}
286343
}

‎src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ internal sealed class PythonAnalyzerEntry {
3939
private HashSet<AnalysisModuleKey> _analysisDependencies;
4040
private int _bufferVersion;
4141
private int _analysisVersion;
42+
private int _depth;
4243

4344
public IPythonModule Module {
4445
get {
@@ -74,11 +75,21 @@ public int AnalysisVersion {
7475
}
7576
}
7677

78+
public int Depth {
79+
get {
80+
lock (_syncObj) {
81+
return _depth;
82+
}
83+
}
84+
}
85+
7786
public bool NotAnalyzed => PreviousAnalysis is EmptyAnalysis;
7887

7988
public PythonAnalyzerEntry(EmptyAnalysis emptyAnalysis) {
8089
_previousAnalysis = emptyAnalysis;
90+
_module = emptyAnalysis.Document;
8191
_isUserModule = emptyAnalysis.Document.ModuleType == ModuleType.User;
92+
_depth = _isUserModule ? 0 : -1;
8293

8394
_bufferVersion = -1;
8495
_analysisVersion = 0;
@@ -92,10 +103,24 @@ public bool IsValidVersion(int version, out IPythonModule module, out PythonAst
92103
lock (_syncObj) {
93104
module = _module;
94105
ast = _ast;
106+
if (ast == null || module == null) {
107+
return false;
108+
}
109+
95110
return _previousAnalysis is EmptyAnalysis || _isUserModule || _analysisVersion <= version;
96111
}
97112
}
98113

114+
public void SetDepth(int version, int depth) {
115+
lock (_syncObj) {
116+
if (_analysisVersion > version) {
117+
return;
118+
}
119+
120+
_depth = _depth == -1 ? depth : Math.Min(_depth, depth);
121+
}
122+
}
123+
99124
public void TrySetAnalysis(IDocumentAnalysis analysis, int version) {
100125
lock (_syncObj) {
101126
if (_previousAnalysis is EmptyAnalysis) {
@@ -139,7 +164,7 @@ public void TryCancel(OperationCanceledException oce, int version) {
139164

140165
public void Invalidate(int analysisVersion) {
141166
lock (_syncObj) {
142-
if (_analysisVersion >= analysisVersion) {
167+
if (_analysisVersion >= analysisVersion || !_analysisTcs.Task.IsCompleted) {
143168
return;
144169
}
145170

@@ -176,7 +201,6 @@ public bool Invalidate(ImmutableArray<IPythonModule> analysisDependencies, int a
176201
return false;
177202
}
178203

179-
UpdateAnalysisTcs(analysisVersion);
180204
if (_analysisDependencies == null) {
181205
_analysisDependencies = dependenciesHashSet;
182206
} else {
@@ -187,6 +211,7 @@ public bool Invalidate(ImmutableArray<IPythonModule> analysisDependencies, int a
187211
}
188212
}
189213

214+
UpdateAnalysisTcs(analysisVersion);
190215
dependencies = _parserDependencies != null
191216
? ImmutableArray<AnalysisModuleKey>.Create(_parserDependencies.Union(_analysisDependencies).ToArray())
192217
: ImmutableArray<AnalysisModuleKey>.Create(_analysisDependencies);

‎src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ private void LogResults(double elapsed, int originalRemaining, int remaining, in
193193
if (_log == null) {
194194
return;
195195
}
196-
196+
197197
if (remaining == 0) {
198198
_log.Log(TraceEventType.Verbose, $"Analysis version {version} of {originalRemaining} entries has been completed in {elapsed} ms.");
199199
} else if (remaining < originalRemaining) {
@@ -208,8 +208,8 @@ private async Task<int> AnalyzeAffectedEntriesAsync(Stopwatch stopWatch, Cancell
208208
var remaining = 0;
209209
var ace = new AsyncCountdownEvent(0);
210210

211+
bool isCanceled;
211212
while ((node = await _walker.GetNextAsync(cancellationToken)) != null) {
212-
bool isCanceled;
213213
lock (_syncObj) {
214214
isCanceled = _isCanceled;
215215
}
@@ -229,16 +229,18 @@ private async Task<int> AnalyzeAffectedEntriesAsync(Stopwatch stopWatch, Cancell
229229

230230
await ace.WaitAsync(cancellationToken);
231231

232-
if (_walker.MissingKeys.All(k => k.IsTypeshed)) {
233-
Interlocked.Exchange(ref _runningTasks, 0);
234-
bool isCanceled;
235-
lock (_syncObj) {
236-
isCanceled = _isCanceled;
237-
}
232+
lock (_syncObj) {
233+
isCanceled = _isCanceled;
234+
}
238235

236+
if (_walker.MissingKeys.Count == 0 || _walker.MissingKeys.All(k => k.IsTypeshed)) {
237+
Interlocked.Exchange(ref _runningTasks, 0);
238+
239239
if (!isCanceled) {
240240
_analysisCompleteEvent.Set();
241241
}
242+
} else if (!isCanceled && _log != null && _log.LogLevel >= TraceEventType.Verbose) {
243+
_log?.Log(TraceEventType.Verbose, $"Missing keys: {string.Join(", ", _walker.MissingKeys)}");
242244
}
243245

244246
return remaining;
@@ -259,6 +261,11 @@ private void Analyze(IDependencyChainNode<PythonAnalyzerEntry> node, AsyncCountd
259261
ace?.AddOne();
260262
var entry = node.Value;
261263
if (!entry.IsValidVersion(_walker.Version, out module, out var ast)) {
264+
if (ast == null) {
265+
// Entry doesn't have ast yet. There should be at least one more session.
266+
Cancel();
267+
}
268+
262269
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled.");
263270
node.Skip();
264271
return;
@@ -296,6 +303,11 @@ private void Analyze(PythonAnalyzerEntry entry, int version, CancellationToken c
296303
var stopWatch = Stopwatch.StartNew();
297304
try {
298305
if (!entry.IsValidVersion(version, out var module, out var ast)) {
306+
if (ast == null) {
307+
// Entry doesn't have ast yet. There should be at least one more session.
308+
Cancel();
309+
}
310+
299311
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled.");
300312
return;
301313
}

0 commit comments

Comments
 (0)