Skip to content
This repository was archived by the owner on Nov 4, 2024. It is now read-only.

Commit 96be2d9

Browse files
AlexanderSherAlexanderSher
authored
Fix microsoft#825: Unresolved import not going away automatically after installing packages (microsoft#871)
1 parent e1b687d commit 96be2d9

File tree

17 files changed

+162
-60
lines changed

17 files changed

+162
-60
lines changed

‎src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,5 +54,10 @@ public interface IPythonAnalyzer {
5454
/// Runs linters on the modules
5555
/// </summary>
5656
IReadOnlyList<DiagnosticsEntry> LintModule(IPythonModule module);
57+
58+
/// <summary>
59+
/// Removes all the modules from the analysis, except Typeshed and builtin
60+
/// </summary>
61+
void ResetAnalyzer();
5762
}
5863
}

‎src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,17 @@ public IReadOnlyList<DiagnosticsEntry> LintModule(IPythonModule module) {
177177
return new LinterAggregator().Lint(module.Analysis, _services);
178178
}
179179

180+
public void ResetAnalyzer() {
181+
lock (_syncObj) {
182+
_analysisEntries.Split(kvp => kvp.Key.IsTypeshed || kvp.Value.Module is IBuiltinsPythonModule, out var entriesToPreserve, out var entriesToRemove);
183+
_analysisEntries.Clear();
184+
foreach (var (key, entry) in entriesToPreserve) {
185+
_analysisEntries.Add(key, entry);
186+
}
187+
188+
_dependencyResolver.RemoveKeys(entriesToRemove.Select(e => e.Key));
189+
}
190+
}
180191

181192
private void AnalyzeDocument(AnalysisModuleKey key, PythonAnalyzerEntry entry, ImmutableArray<AnalysisModuleKey> dependencies, CancellationToken cancellationToken) {
182193
_analysisCompleteEvent.Reset();

‎src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ private async Task StartAsync(IDependencyChainWalker<AnalysisModuleKey, PythonAn
142142
}
143143

144144
if (!isCanceled) {
145-
_progress.ReportRemaining(walker.Remaining);
145+
_progress.ReportRemaining(remaining);
146146
}
147147
}
148148

‎src/Analysis/Ast/Impl/Dependencies/DependencyGraph.cs

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
using System.Collections.Generic;
1717
using System.Linq;
18+
using Microsoft.Python.Core;
1819
using Microsoft.Python.Core.Collections;
1920

2021
namespace Microsoft.Python.Analysis.Dependencies {
@@ -64,7 +65,7 @@ public DependencyVertex<TKey, TValue> AddOrUpdate(TKey key, TValue value, Immuta
6465

6566
_verticesByKey[key] = changedVertex;
6667

67-
var missingKeysHashSet = new HashSet<TKey>();
68+
6869
var vertices = _verticesByIndex
6970
.Where(v => !v.IsSealed || v.HasMissingKeys)
7071
.Select(v => GetOrCreateNonSealedVertex(version, v.Index))
@@ -75,6 +76,33 @@ public DependencyVertex<TKey, TValue> AddOrUpdate(TKey key, TValue value, Immuta
7576
return changedVertex;
7677
}
7778

79+
CreateNewSnapshot(vertices, version);
80+
81+
return changedVertex;
82+
}
83+
84+
public void RemoveKeys(ImmutableArray<TKey> keys) {
85+
var version = Snapshot.Version + 1;
86+
87+
_verticesByIndex.Clear();
88+
foreach (var key in keys) {
89+
_verticesByKey.Remove(key);
90+
}
91+
92+
foreach (var (key, currentVertex) in _verticesByKey) {
93+
var changedVertex = new DependencyVertex<TKey, TValue>(key, currentVertex.Value, currentVertex.IncomingKeys, version, _verticesByIndex.Count);
94+
_verticesByIndex.Add(changedVertex);
95+
}
96+
97+
foreach (var vertex in _verticesByIndex) {
98+
_verticesByKey[vertex.Key] = vertex;
99+
}
100+
101+
CreateNewSnapshot(_verticesByIndex, version);
102+
}
103+
104+
private void CreateNewSnapshot(IEnumerable<DependencyVertex<TKey, TValue>> vertices, int version) {
105+
var missingKeysHashSet = new HashSet<TKey>();
78106
foreach (var vertex in vertices) {
79107
var newIncoming = ImmutableArray<int>.Empty;
80108
var oldIncoming = vertex.Incoming;
@@ -105,11 +133,9 @@ public DependencyVertex<TKey, TValue> AddOrUpdate(TKey key, TValue value, Immuta
105133
vertex.Seal();
106134
}
107135

108-
Snapshot = new DependencyGraphSnapshot<TKey, TValue>(version,
136+
Snapshot = new DependencyGraphSnapshot<TKey, TValue>(version,
109137
ImmutableArray<DependencyVertex<TKey, TValue>>.Create(_verticesByIndex),
110138
ImmutableArray<TKey>.Create(missingKeysHashSet));
111-
112-
return changedVertex;
113139
}
114140

115141
private DependencyVertex<TKey, TValue> GetOrCreateNonSealedVertex(int version, int index) {

‎src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,18 +29,49 @@ public IDependencyChainWalker<TKey, TValue> NotifyChanges(TKey key, TValue value
2929
=> NotifyChanges(key, value, ImmutableArray<TKey>.Create(incomingKeys));
3030

3131
public IDependencyChainWalker<TKey, TValue> NotifyChanges(TKey key, TValue value, ImmutableArray<TKey> incomingKeys) {
32-
int version;
33-
ImmutableArray<DependencyVertex<TKey, TValue>> changedVertices;
34-
DependencyGraphSnapshot<TKey, TValue> snapshot;
32+
AddOrUpdate(key, value, incomingKeys, out var snapshot, out var changedVertices);
33+
return CreateDependencyChainWalker(this, snapshot, changedVertices);
34+
}
35+
36+
public IDependencyChainWalker<TKey, TValue> RemoveKeys(params TKey[] keys)
37+
=> RemoveKeys(ImmutableArray<TKey>.Create(keys));
3538

39+
public IDependencyChainWalker<TKey, TValue> RemoveKeys(ImmutableArray<TKey> keys) {
40+
Remove(keys, out var snapshot, out var changedVertices);
41+
return CreateDependencyChainWalker(this, snapshot, changedVertices);
42+
}
43+
44+
private void AddOrUpdate(TKey key, TValue value, ImmutableArray<TKey> incomingKeys, out DependencyGraphSnapshot<TKey, TValue> snapshot, out ImmutableArray<DependencyVertex<TKey, TValue>> changedVertices) {
3645
lock (_syncObj) {
3746
var dependencyVertex = _vertices.AddOrUpdate(key, value, incomingKeys);
3847
snapshot = _vertices.Snapshot;
3948

40-
version = _vertices.Snapshot.Version;
4149
_changedVertices[key] = dependencyVertex;
4250
changedVertices = ImmutableArray<DependencyVertex<TKey, TValue>>.Create(_changedVertices.Values);
4351
}
52+
}
53+
54+
private void Remove(ImmutableArray<TKey> keys, out DependencyGraphSnapshot<TKey, TValue> snapshot, out ImmutableArray<DependencyVertex<TKey, TValue>> changedVertices) {
55+
lock (_syncObj) {
56+
_vertices.RemoveKeys(keys);
57+
snapshot = _vertices.Snapshot;
58+
59+
foreach (var key in keys) {
60+
_changedVertices.Remove(key);
61+
}
62+
63+
foreach (var vertex in snapshot.Vertices.Where(v => v.HasMissingKeys)) {
64+
_changedVertices[vertex.Key] = vertex;
65+
}
66+
67+
changedVertices = ImmutableArray<DependencyVertex<TKey, TValue>>.Create(_changedVertices.Values);
68+
}
69+
}
70+
71+
private static IDependencyChainWalker<TKey, TValue> CreateDependencyChainWalker(
72+
DependencyResolver<TKey, TValue> dependencyResolver,
73+
DependencyGraphSnapshot<TKey, TValue> snapshot,
74+
ImmutableArray<DependencyVertex<TKey, TValue>> changedVertices) {
4475

4576
var walkingGraph = CreateWalkingGraph(snapshot.Vertices, changedVertices);
4677
var affectedValues = walkingGraph.Select(v => v.DependencyVertex.Value);
@@ -52,7 +83,7 @@ public IDependencyChainWalker<TKey, TValue> NotifyChanges(TKey key, TValue value
5283
vertex.SecondPass?.Seal();
5384
}
5485

55-
return new DependencyChainWalker(this, startingVertices, affectedValues, snapshot.MissingKeys, totalNodesCount, version);
86+
return new DependencyChainWalker(dependencyResolver, startingVertices, affectedValues, snapshot.MissingKeys, totalNodesCount, snapshot.Version);
5687
}
5788

5889
private void CommitChanges(DependencyVertex<TKey, TValue> vertex) {
@@ -63,7 +94,7 @@ private void CommitChanges(DependencyVertex<TKey, TValue> vertex) {
6394
}
6495
}
6596

66-
private ImmutableArray<WalkingVertex<TKey, TValue>> CreateWalkingGraph(ImmutableArray<DependencyVertex<TKey, TValue>> snapshot, ImmutableArray<DependencyVertex<TKey, TValue>> changedVertices) {
97+
private static ImmutableArray<WalkingVertex<TKey, TValue>> CreateWalkingGraph(ImmutableArray<DependencyVertex<TKey, TValue>> snapshot, ImmutableArray<DependencyVertex<TKey, TValue>> changedVertices) {
6798
var analysisGraph = ImmutableArray<WalkingVertex<TKey, TValue>>.Empty;
6899
var nodesByVertexIndex = new Dictionary<int, WalkingVertex<TKey, TValue>>();
69100

@@ -92,7 +123,7 @@ private ImmutableArray<WalkingVertex<TKey, TValue>> CreateWalkingGraph(Immutable
92123
return analysisGraph;
93124
}
94125

95-
private int FindLoops(ImmutableArray<WalkingVertex<TKey, TValue>> graph) {
126+
private static int FindLoops(ImmutableArray<WalkingVertex<TKey, TValue>> graph) {
96127
var index = 0;
97128
var loopNumber = 0;
98129
var stackP = new Stack<WalkingVertex<TKey, TValue>>();
@@ -107,7 +138,7 @@ private int FindLoops(ImmutableArray<WalkingVertex<TKey, TValue>> graph) {
107138
return loopNumber;
108139
}
109140

110-
private void CheckForLoop(WalkingVertex<TKey, TValue> vertex, Stack<WalkingVertex<TKey, TValue>> stackP, Stack<WalkingVertex<TKey, TValue>> stackS, ref int counter, ref int loopNumber) {
141+
private static void CheckForLoop(WalkingVertex<TKey, TValue> vertex, Stack<WalkingVertex<TKey, TValue>> stackP, Stack<WalkingVertex<TKey, TValue>> stackS, ref int counter, ref int loopNumber) {
111142
vertex.Index = counter++;
112143
stackP.Push(vertex);
113144
stackS.Push(vertex);

‎src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,5 +27,6 @@ namespace Microsoft.Python.Analysis.Dependencies {
2727
/// </summary>
2828
internal interface IDependencyResolver<TKey, TValue> {
2929
IDependencyChainWalker<TKey, TValue> NotifyChanges(TKey key, TValue value, ImmutableArray<TKey> incomingKeys);
30+
IDependencyChainWalker<TKey, TValue> RemoveKeys(ImmutableArray<TKey> keys);
3031
}
3132
}

‎src/Analysis/Ast/Impl/Documents/Definitions/IDocument.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ public interface IDocument: IPythonModule, IDisposable {
6868
void Update(IEnumerable<DocumentChange> changes);
6969

7070
/// <summary>
71-
/// Resets document buffer to the provided content.
71+
/// Resets document buffer to the provided content or tries to load it if content is null, then parses and analyzes document.
7272
/// </summary>
7373
void Reset(string content);
7474

‎src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ internal sealed class DocumentBuffer {
2929

3030
public void Reset(int version, string content) {
3131
Version = version;
32+
_content = content ?? string.Empty;
3233
_sb = null;
33-
_content = content;
3434
}
3535

3636
public void Update(IEnumerable<DocumentChange> changes) {

‎src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,8 @@ private DocumentEntry CreateDocument(ModuleCreationOptions mco) {
225225
var entry = new DocumentEntry(document);
226226
_documentsByUri[document.Uri] = entry;
227227
_documentsByName[mco.ModuleName] = entry;
228+
229+
_services.GetService<IPythonAnalyzer>().InvalidateAnalysis(document);
228230
return entry;
229231
}
230232

‎src/Analysis/Ast/Impl/Modules/PythonModule.cs

Lines changed: 10 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s
111111
if (ModuleType == ModuleType.Specialized || ModuleType == ModuleType.Unresolved) {
112112
ContentState = State.Analyzed;
113113
}
114-
InitializeContent(creationOptions.Content);
114+
InitializeContent(creationOptions.Content, 0);
115115
}
116116

117117
#region IPythonType
@@ -184,18 +184,6 @@ public virtual IEnumerable<string> GetMemberNames() {
184184
/// </summary>
185185
public IGlobalScope GlobalScope { get; private set; }
186186

187-
/// <summary>
188-
/// Ensures that module content is loaded and analysis has started.
189-
/// Typically module content is loaded at the creation time, but delay
190-
/// loaded (lazy) modules may choose to defer content retrieval and
191-
/// analysis until later time, when module members are actually needed.
192-
/// </summary>
193-
public async Task LoadAndAnalyzeAsync(CancellationToken cancellationToken = default) {
194-
InitializeContent(null);
195-
await GetAstAsync(cancellationToken);
196-
await Services.GetService<IPythonAnalyzer>().GetAnalysisAsync(this, -1, cancellationToken);
197-
}
198-
199187
/// <summary>
200188
/// If module is a stub points to the primary module.
201189
/// Typically used in code navigation scenarios when user
@@ -215,27 +203,22 @@ protected virtual string LoadContent() {
215203
return null; // Keep content as null so module can be loaded later.
216204
}
217205

218-
private void InitializeContent(string content) {
219-
bool startParse;
206+
private void InitializeContent(string content, int version) {
220207
lock (AnalysisLock) {
221-
LoadContent(content);
208+
LoadContent(content, version);
222209

223-
startParse = ContentState < State.Parsing && _parsingTask == null;
210+
var startParse = ContentState < State.Parsing && _parsingTask == null;
224211
if (startParse) {
225212
Parse();
226213
}
227214
}
228-
229-
if (startParse) {
230-
Services.GetService<IPythonAnalyzer>().InvalidateAnalysis(this);
231-
}
232215
}
233216

234-
private void LoadContent(string content) {
217+
private void LoadContent(string content, int version) {
235218
if (ContentState < State.Loading) {
236219
try {
237220
content = content ?? LoadContent();
238-
_buffer.Reset(0, content);
221+
_buffer.Reset(version, content);
239222
ContentState = State.Loaded;
240223
} catch (IOException) { } catch (UnauthorizedAccessException) { }
241224
}
@@ -330,9 +313,12 @@ public void Update(IEnumerable<DocumentChange> changes) {
330313
public void Reset(string content) {
331314
lock (AnalysisLock) {
332315
if (content != Content) {
333-
InitializeContent(content);
316+
ContentState = State.None;
317+
InitializeContent(content, _buffer.Version + 1);
334318
}
335319
}
320+
321+
Services.GetService<IPythonAnalyzer>().InvalidateAnalysis(this);
336322
}
337323

338324
private void Parse() {

‎src/Analysis/Ast/Impl/Modules/PythonPackage.cs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,5 @@ public PythonVariableModule(IPythonModule module) {
7575
public IMember CreateInstance(string typeName = null, LocationInfo location = null, IArgumentSet args = null) => this;
7676

7777
public bool Equals(IPythonModule other) => other is PythonVariableModule module && Name.EqualsOrdinal(module.Name);
78-
79-
public Task LoadAndAnalyzeAsync(CancellationToken cancellationToken = default) => throw new NotSupportedException("Can't analyze analysis-only type");
8078
}
8179
}

‎src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -90,13 +90,10 @@ protected override IPythonModule CreateModule(string name) {
9090
return null;
9191
}
9292

93-
IPythonModule module;
94-
if (!string.IsNullOrEmpty(moduleImport.ModulePath) && Uri.TryCreate(moduleImport.ModulePath, UriKind.Absolute, out var uri)) {
95-
module = GetRdt().GetDocument(uri);
96-
if (module != null) {
97-
GetRdt().LockDocument(uri);
98-
return module;
99-
}
93+
var module = GetRdt().GetDocument(moduleImport.FullName);
94+
if (module != null) {
95+
GetRdt().LockDocument(module.Uri);
96+
return module;
10097
}
10198

10299
// If there is a stub, make sure it is loaded and attached
@@ -177,7 +174,8 @@ public IPythonModule GetSpecializedModule(string name)
177174
=> _specialized.TryGetValue(name, out var module) ? module : null;
178175

179176
internal async Task LoadBuiltinTypesAsync(CancellationToken cancellationToken = default) {
180-
await BuiltinsModule.LoadAndAnalyzeAsync(cancellationToken);
177+
var analyzer = _services.GetService<IPythonAnalyzer>();
178+
await analyzer.GetAnalysisAsync(BuiltinsModule, -1, cancellationToken);
181179

182180
Check.InvalidOperation(!(BuiltinsModule.Analysis is EmptyAnalysis), "After await");
183181

‎src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,8 +153,6 @@ public IPythonModule GetOrCreate(string name, ModuleResolutionBase mrb) {
153153
}
154154

155155
var module = mrb.CreateModule(name);
156-
((IDocument)module)?.Reset(null);
157-
158156
lock (_syncObj) {
159157
_creating = false;
160158
_module = module;

‎src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -47,15 +47,7 @@ public interface IPythonModule : IPythonType, IPythonFile, ILocatedMember {
4747
/// Global cope of the module.
4848
/// </summary>
4949
IGlobalScope GlobalScope { get; }
50-
51-
/// <summary>
52-
/// Ensures that module content is loaded and analysis has completed.
53-
/// Typically module content is loaded at the creation time, but delay
54-
/// loaded (lazy) modules may choose to defer content retrieval and
55-
/// analysis until later time, when module members are actually needed.
56-
/// </summary>
57-
Task LoadAndAnalyzeAsync(CancellationToken cancellationToken = default);
58-
50+
5951
/// <summary>
6052
/// If module is a stub points to the primary module.
6153
/// Typically used in code navigation scenarios when user

0 commit comments

Comments
 (0)