Skip to content
This repository was archived by the owner on Apr 14, 2022. It is now read-only.

Commit 3751b7a

Browse files
Mikhail ArkhipovAlexanderSherjakebailey
committed
Improve handling of circular dependencies (#1686)
* Refactoring * WIP * Bug fixes * Fix part of the test * Fully restore tests * Remove stale reference * Merge issues * Merge fixes * Fix null ref * Fix creation of entries for missing keys * Increase cancellation time in debug * Cache walker + add failing test * Fix GetMemberNames * Remove `InvalidateAnalysis` call from constructor * Build fix * Merge issues * Fix FindReferences * Tests * Tests * Add restoration from db in loop handling * Revert "Add restoration from db in loop handling" This reverts commit f1a2efd. * Make numpy worh with library caching * Fix saving of ALL import statements * Fix stub merge typo (members only in source were not transferred) * Don't re-analyze cached loop * Fix TypeVar bound to Unknown * Fix lambda restore * Walk loops fully * Fix diagnostics on reopen * Fix gotodef test * Fix key exception * Add original test * Handle annotations in loops * Null check * Call ActivityTracker on modules in loops Co-authored-by: Alexander Sher <[email protected]> Co-authored-by: Jake Bailey <[email protected]>
1 parent cf915c3 commit 3751b7a

File tree

60 files changed

+1873
-700
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+1873
-700
lines changed

‎src/Analysis/Ast/Impl/Analyzer/ActivityTracker.cs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515

1616
using System.Collections.Generic;
1717
using System.Diagnostics;
18-
using System.Linq;
1918

2019
namespace Microsoft.Python.Analysis.Analyzer {
2120
internal static class ActivityTracker {

‎src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,9 @@ internal abstract class AnalysisWalker : PythonWalker {
3939
public PythonAst Ast => Eval.Ast;
4040
protected ModuleSymbolTable SymbolTable => Eval.SymbolTable;
4141

42-
protected AnalysisWalker(ExpressionEval eval) {
42+
protected AnalysisWalker(ExpressionEval eval, IImportedVariableHandler importedVariableHandler) {
4343
Eval = eval;
44-
ImportHandler = new ImportHandler(this);
44+
ImportHandler = new ImportHandler(this, importedVariableHandler);
4545
AssignmentHandler = new AssignmentHandler(this);
4646
LoopHandler = new LoopHandler(this);
4747
ConditionalHandler = new ConditionalHandler(this);

‎src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,5 +71,12 @@ public interface IPythonAnalyzer {
7171
/// Fires when analysis is complete.
7272
/// </summary>
7373
event EventHandler<AnalysisCompleteEventArgs> AnalysisComplete;
74+
75+
/// <summary>
76+
/// Attempts to restore modules analysis from database.
77+
/// </summary>
78+
/// <param name="module"></param>
79+
/// <returns></returns>
80+
IDocumentAnalysis TryRestoreCachedAnalysis(IPythonModule module);
7481
}
7582
}

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,6 @@ public T GetInScope<T>(string name, IScope scope) where T : class, IMember
4040
public void DeclareVariable(string name, IMember value, VariableSource source)
4141
=> DeclareVariable(name, value, source, default(Location));
4242

43-
public void DeclareVariable(string name, IMember value, VariableSource source, IPythonModule module)
44-
=> DeclareVariable(name, value, source, new Location(module));
45-
4643
public void DeclareVariable(string name, IMember value, VariableSource source, Node location, bool overwrite = true)
4744
=> DeclareVariable(name, value, source, GetLocationOfName(location), overwrite);
4845

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs
6060
public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(this) ?? LocationInfo.Empty;
6161

6262
public Location GetLocationOfName(Node node) {
63-
if (node == null || (Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library)) {
63+
if (node == null || Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library) {
6464
return DefaultLocation;
6565
}
6666

@@ -233,7 +233,8 @@ private IMember GetValueFromName(NameExpression expr, LookupOptions options = Lo
233233
}
234234

235235
private IMember GetValueFromMember(MemberExpression expr, LookupOptions lookupOptions = LookupOptions.Normal) {
236-
if (expr?.Target == null || string.IsNullOrEmpty(expr.Name)) {
236+
var memberName = expr?.Name;
237+
if (expr?.Target == null || string.IsNullOrEmpty(memberName)) {
237238
return null;
238239
}
239240

@@ -243,8 +244,9 @@ private IMember GetValueFromMember(MemberExpression expr, LookupOptions lookupOp
243244
}
244245

245246
var type = m.GetPythonType();
246-
var value = type?.GetMember(expr.Name);
247-
type?.AddMemberReference(expr.Name, this, GetLocationOfName(expr));
247+
var value = type?.GetMember(memberName);
248+
var location = GetLocationOfName(expr);
249+
type?.AddMemberReference(memberName, this, location);
248250

249251
if (type is IPythonModule) {
250252
return value;

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/FunctionCallEvaluator.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
// permissions and limitations under the License.
1515

1616
using System;
17+
using Microsoft.Python.Analysis.Analyzer.Handlers;
1718
using Microsoft.Python.Analysis.Analyzer.Symbols;
1819
using Microsoft.Python.Analysis.Types;
1920
using Microsoft.Python.Parsing.Ast;
@@ -28,7 +29,7 @@ internal sealed class FunctionCallEvaluator: AnalysisWalker {
2829
private readonly FunctionDefinition _function;
2930
private IMember _result;
3031

31-
public FunctionCallEvaluator(IPythonModule declaringModule, FunctionDefinition fd, ExpressionEval eval): base(eval) {
32+
public FunctionCallEvaluator(IPythonModule declaringModule, FunctionDefinition fd, ExpressionEval eval): base(eval, SimpleImportedVariableHandler.Instance) {
3233
_declaringModule = declaringModule ?? throw new ArgumentNullException(nameof(declaringModule));
3334
_eval = eval ?? throw new ArgumentNullException(nameof(eval));
3435
_function = fd ?? throw new ArgumentNullException(nameof(fd));

‎src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs‎

Lines changed: 41 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -59,12 +59,21 @@ private void AssignVariables(FromImportStatement node, IImportSearchResult impor
5959

6060
for (var i = 0; i < names.Count; i++) {
6161
var memberName = names[i].Name;
62-
if (!string.IsNullOrEmpty(memberName)) {
63-
var nameExpression = asNames[i] ?? names[i];
64-
var variableName = nameExpression?.Name ?? memberName;
65-
if (!string.IsNullOrEmpty(variableName)) {
66-
DeclareVariable(variableModule, memberName, imports, variableName, node.StartIndex, nameExpression);
67-
}
62+
if (string.IsNullOrEmpty(memberName)) {
63+
continue;
64+
}
65+
66+
var nameExpression = asNames[i] ?? names[i];
67+
var variableName = nameExpression?.Name ?? memberName;
68+
if (!string.IsNullOrEmpty(variableName)) {
69+
DeclareVariable(variableModule, memberName, imports, variableName, node.StartIndex, nameExpression);
70+
}
71+
72+
if (imports is IImportChildrenSource cs
73+
&& cs.TryGetChildImport(memberName, out var csr)
74+
&& HandleImportSearchResult(csr, variableModule, null, names[i], out var childModule)) {
75+
76+
_importedVariableHandler.EnsureModule(childModule);
6877
}
6978
}
7079
}
@@ -74,10 +83,11 @@ private void HandleModuleImportStar(PythonVariableModule variableModule, IImport
7483
// from self import * won't define any new members
7584
return;
7685
}
86+
7787
// If __all__ is present, take it, otherwise declare all members from the module that do not begin with an underscore.
7888
var memberNames = imports is ImplicitPackageImport
7989
? variableModule.GetMemberNames()
80-
: variableModule.Analysis.StarImportMemberNames ?? variableModule.GetMemberNames().Where(s => !s.StartsWithOrdinal("_"));
90+
: _importedVariableHandler.GetMemberNames(variableModule).ToArray();
8191

8292
foreach (var memberName in memberNames) {
8393
DeclareVariable(variableModule, memberName, imports, memberName, importPosition, nameExpression);
@@ -104,23 +114,36 @@ private void DeclareVariable(PythonVariableModule variableModule, string memberN
104114
value = GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName);
105115

106116
// First try exported or child submodules.
107-
value = value ?? variableModule.GetMember(memberName);
117+
var member = variableModule.GetMember(memberName);
108118

109119
// Value may be variable or submodule. If it is variable, we need it in order to add reference.
110-
var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName];
111-
value = variable?.Value?.Equals(value) == true ? variable : value;
112-
113-
// If nothing is exported, variables are still accessible.
114-
value = value ?? variableModule.Analysis?.GlobalScope?.Variables[memberName]?.Value ?? Eval.UnknownType;
120+
var variable = _importedVariableHandler.GetVariable(variableModule, memberName);
121+
122+
if (member is PythonVariableModule vm && vm.Equals(variable?.Value)) {
123+
// If member is submodule, use actual variable so it can be linked through for goto definition.
124+
value = variable;
125+
} else if (value == null) {
126+
if (member is PythonVariableModule) {
127+
// If member is submodule, use it.
128+
value = member;
129+
} else if (variable?.Value != null) {
130+
// Otherwise use variable, if available so references can be linked.
131+
value = variable;
132+
} else if (member != null) {
133+
value = member;
134+
} else {
135+
value = Eval.UnknownType;
136+
}
137+
}
115138
}
116-
139+
117140
// Do not allow imported variables to override local declarations
118141
var canOverwrite = CanOverwriteVariable(variableName, importPosition, value);
119-
142+
120143
// Do not declare references to '*'
121144
var locationExpression = nameLocation is NameExpression nex && nex.Name == "*" ? null : nameLocation;
122145
Eval.DeclareVariable(variableName, value, VariableSource.Import, locationExpression, canOverwrite);
123-
146+
124147
// Make sure module is loaded and analyzed.
125148
if (value is IPythonModule m) {
126149
ModuleResolution.GetOrLoadModule(m.Name);
@@ -132,8 +155,8 @@ private bool CanOverwriteVariable(string name, int importPosition, IMember newVa
132155
if (v == null) {
133156
return true; // Variable does not exist
134157
}
135-
136-
if(newValue.IsUnknown()) {
158+
159+
if (newValue.IsUnknown()) {
137160
return false; // Do not overwrite potentially good value with unknowns.
138161
}
139162

‎src/Analysis/Ast/Impl/Analyzer/Handlers/IImportedVariableHandler.cs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// Copyright(c) Microsoft Corporation
2+
// All rights reserved.
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the License); you may not use
5+
// this file except in compliance with the License. You may obtain a copy of the
6+
// License at http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
9+
// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
10+
// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
11+
// MERCHANTABILITY OR NON-INFRINGEMENT.
12+
//
13+
// See the Apache Version 2.0 License for specific language governing
14+
// permissions and limitations under the License.
15+
16+
using System.Collections.Generic;
17+
using Microsoft.Python.Analysis.Modules;
18+
using Microsoft.Python.Analysis.Values;
19+
20+
namespace Microsoft.Python.Analysis.Analyzer.Handlers {
21+
internal interface IImportedVariableHandler {
22+
IEnumerable<string> GetMemberNames(PythonVariableModule variableModule);
23+
IVariable GetVariable(in PythonVariableModule module, in string name);
24+
void EnsureModule(in PythonVariableModule module);
25+
}
26+
}

‎src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs‎

Lines changed: 15 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,12 @@
2929

3030
namespace Microsoft.Python.Analysis.Analyzer.Handlers {
3131
internal sealed partial class ImportHandler : StatementHandler {
32+
private readonly IImportedVariableHandler _importedVariableHandler;
3233
private readonly Dictionary<string, PythonVariableModule> _variableModules = new Dictionary<string, PythonVariableModule>();
3334

34-
public ImportHandler(AnalysisWalker walker) : base(walker) { }
35+
public ImportHandler(AnalysisWalker walker, in IImportedVariableHandler importedVariableHandler) : base(walker) {
36+
_importedVariableHandler = importedVariableHandler;
37+
}
3538

3639
public bool HandleImport(ImportStatement node) {
3740
if (Module.ModuleType == ModuleType.Specialized) {
@@ -65,7 +68,9 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa
6568
lastModule = default;
6669
break;
6770
}
71+
6872
resolvedModules[i] = (nameExpression.Name, lastModule);
73+
_importedVariableHandler.EnsureModule(lastModule);
6974
}
7075

7176
// "import fob.oar.baz as baz" is handled as baz = import_module('fob.oar.baz')
@@ -89,15 +94,6 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa
8994
Eval.DeclareVariable(importNames[0], firstModule, VariableSource.Import, moduleImportExpression.Names[0]);
9095
}
9196
}
92-
93-
// import a.b.c.d => declares a, b in the current module, c in b, d in c.
94-
for (var i = 1; i < resolvedModules.Length - 1; i++) {
95-
var (childName, childModule) = resolvedModules[i + 1];
96-
if (!string.IsNullOrEmpty(childName) && childModule != null) {
97-
var parent = resolvedModules[i].module;
98-
parent?.AddChildModule(childName, childModule);
99-
}
100-
}
10197
}
10298

10399
private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonVariableModule parent, in NameExpression asNameExpression, in Node location, out PythonVariableModule variableModule) {
@@ -112,8 +108,7 @@ private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonV
112108
return TryGetPackageFromImport(packageImport, parent, out variableModule);
113109
case RelativeImportBeyondTopLevel importBeyondTopLevel:
114110
var message = Resources.ErrorRelativeImportBeyondTopLevel.FormatInvariant(importBeyondTopLevel.RelativeImportName);
115-
Eval.ReportDiagnostics(Eval.Module.Uri,
116-
new DiagnosticsEntry(message, location.GetLocation(Eval).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis));
111+
Eval.ReportDiagnostics(Eval.Module.Uri, new DiagnosticsEntry(message, location.GetLocation(Eval).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis));
117112
variableModule = default;
118113
return false;
119114
case ImportNotFound importNotFound:
@@ -177,7 +172,7 @@ private bool TryGetModulePossibleImport(PossibleModuleImport possibleModuleImpor
177172
return false;
178173
}
179174
}
180-
175+
181176
return true;
182177
}
183178

@@ -196,24 +191,22 @@ private void MakeUnresolvedImport(string variableName, string moduleName, Node l
196191
}
197192

198193
private PythonVariableModule GetOrCreateVariableModule(in string fullName, in PythonVariableModule parentModule, in string memberName) {
199-
if (_variableModules.TryGetValue(fullName, out var variableModule)) {
200-
return variableModule;
194+
if (!_variableModules.TryGetValue(fullName, out var variableModule)) {
195+
variableModule = new PythonVariableModule(fullName, Eval.Interpreter);
196+
_variableModules[fullName] = variableModule;
201197
}
202-
203-
variableModule = new PythonVariableModule(fullName, Eval.Interpreter);
204-
_variableModules[fullName] = variableModule;
198+
205199
parentModule?.AddChildModule(memberName, variableModule);
206200
return variableModule;
207201
}
208202

209203
private PythonVariableModule GetOrCreateVariableModule(in IPythonModule module, in PythonVariableModule parentModule, in string memberName) {
210204
var moduleFullName = module.Name;
211-
if (_variableModules.TryGetValue(moduleFullName, out var variableModule)) {
212-
return variableModule;
205+
if (!_variableModules.TryGetValue(moduleFullName, out var variableModule)) {
206+
variableModule = new PythonVariableModule(module);
207+
_variableModules[moduleFullName] = variableModule;
213208
}
214209

215-
variableModule = new PythonVariableModule(module);
216-
_variableModules[moduleFullName] = variableModule;
217210
parentModule?.AddChildModule(memberName, variableModule);
218211
return variableModule;
219212
}

0 commit comments

Comments
 (0)