Skip to content
This repository was archived by the owner on Apr 14, 2022. It is now read-only.

Commit 6a2bcac

Browse files
author
Mikhail Arkhipov
authored
Find all references and rename (#819)
* Fix #668 (partial) * Tests * Revert "Tests" This reverts commit 7ffc9db. * Partial * Part 2 * Part 3 * Part 4 * Part 4 * Part 5 * Analysis tests * Tests * Tests * Basic test * Tests * Tests * Tests * Tests * Tests * Linked variables * Tests * Tests * Tests * Merge issues * Fix merge issue * Basic single file * Handle closed files * Closed files * Tests * Tests * Fix 454 * Remove references on analysis change * Rename * Rename non-user code * Linter * Ref fix * Member expressions * Linter improvements * Tests * Definitions only for LHS * Add test * Add null checks and tests * Null checks * Fix variable leak in expressions with comprehensions * Temp * Fix scope lookup + test * Handle virtual space * Better handle nested sequence expression assignments * More extensive handling of RHS sequences * PR feedback * Enable test * Test stability * Don't collect references to library modules * Handle virtual envs * PR feedback * null check * Remove references in all modules * Add test * Perf improvement * Null check in tests * Store span rather than node * Fix AST associations with locations * Handle null nodes * Temp * Add AST to nodes * Variable refs * Null ref * Merge issues * Handle import * * Undo debug * Merge issues * Better handle variable * Null check * Better handle nested variables without location * Fix rename test * Usings * Fix merge conflict
1 parent 3833fb5 commit 6a2bcac

File tree

172 files changed

+3387
-1535
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

172 files changed

+3387
-1535
lines changed

‎TROUBLESHOOTING.md

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,6 @@ in [Filing an issue](#filing-an-issue).
99

1010
There are a few known issues in the current version of the language server:
1111

12-
- Find references and rename functionality is not yet implemented.
13-
- See [#699](https://github.com/Microsoft/python-language-server/issues/699),
14-
[#577](https://github.com/Microsoft/python-language-server/issues/577).
1512
- Not all `__all__` statements can be handled.
1613
- Some modules may have an incorrect list of exported names.
1714
See [#620](https://github.com/Microsoft/python-language-server/issues/620),

‎src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,13 @@ public override bool Walk(ExpressionStatement node) {
6464
case Comprehension comp:
6565
Eval.ProcessComprehension(comp);
6666
return false;
67+
case CallExpression callex when callex.Target is NameExpression nex && !string.IsNullOrEmpty(nex.Name):
68+
Eval.LookupNameInScopes(nex.Name)?.AddReference(Eval.GetLocationOfName(nex));
69+
return false;
70+
case CallExpression callex when callex.Target is MemberExpression mex && !string.IsNullOrEmpty(mex.Name):
71+
var t = Eval.GetValueFromExpression(mex.Target)?.GetPythonType();
72+
t?.GetMember(mex.Name).AddReference(Eval.GetLocationOfName(mex));
73+
return false;
6774
default:
6875
return base.Walk(node);
6976
}

‎src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,16 @@
1313
// See the Apache Version 2.0 License for specific language governing
1414
// permissions and limitations under the License.
1515

16-
using System;
17-
1816
namespace Microsoft.Python.Analysis.Analyzer {
1917
/// <summary>
2018
/// Represents document that can be analyzed asynchronously.
2119
/// </summary>
2220
internal interface IAnalyzable {
21+
/// <summary>
22+
/// Notifies document that analysis is about to begin.
23+
/// </summary>
24+
void NotifyAnalysisBegins();
25+
2326
/// <summary>
2427
/// Notifies document that its analysis is now complete.
2528
/// </summary>

‎src/Analysis/Ast/Impl/Analyzer/Definitions/IExpressionEvaluator.cs

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515

1616
using System;
1717
using System.Collections.Generic;
18-
using Microsoft.Python.Analysis.Analyzer.Evaluation;
1918
using Microsoft.Python.Analysis.Diagnostics;
2019
using Microsoft.Python.Analysis.Types;
2120
using Microsoft.Python.Analysis.Values;
@@ -56,18 +55,39 @@ public interface IExpressionEvaluator {
5655
/// <summary>
5756
/// Evaluates expression in the currently open scope.
5857
/// </summary>
59-
IMember GetValueFromExpression(Expression expr);
58+
IMember GetValueFromExpression(Expression expr, LookupOptions options = LookupOptions.Normal);
6059

61-
IMember LookupNameInScopes(string name, out IScope scope, LookupOptions options = LookupOptions.Normal);
60+
IMember LookupNameInScopes(string name, out IScope scope, out IVariable v, LookupOptions options = LookupOptions.Normal);
6261

6362
IPythonType GetTypeFromString(string typeString);
6463

64+
/// <summary>
65+
/// Module AST.
66+
/// </summary>
6567
PythonAst Ast { get; }
68+
69+
/// <summary>
70+
/// Associated module.
71+
/// </summary>
6672
IPythonModule Module { get; }
73+
74+
/// <summary>
75+
/// Interpreter used in the module analysis.
76+
/// </summary>
6777
IPythonInterpreter Interpreter { get; }
78+
79+
/// <summary>
80+
/// Application service container.
81+
/// </summary>
6882
IServiceContainer Services { get; }
6983

7084
void ReportDiagnostics(Uri documentUri, DiagnosticsEntry entry);
85+
7186
IEnumerable<DiagnosticsEntry> Diagnostics { get; }
87+
88+
/// <summary>
89+
/// Represents built-in 'unknown' type. <see cref="BuiltinTypeId.Unknown"/>.
90+
/// </summary>
91+
IPythonType UnknownType { get; }
7292
}
7393
}

‎src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,5 +59,10 @@ public interface IPythonAnalyzer {
5959
/// Removes all the modules from the analysis, except Typeshed and builtin
6060
/// </summary>
6161
void ResetAnalyzer();
62+
63+
/// <summary>
64+
/// Returns list of currently loaded modules.
65+
/// </summary>
66+
IReadOnlyList<IPythonModule> LoadedModules { get; }
6267
}
6368
}

‎src/Analysis/Ast/Impl/Analyzer/Definitions/LookupOptions.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,10 @@ namespace Microsoft.Python.Analysis.Analyzer {
1919
[Flags]
2020
public enum LookupOptions {
2121
None = 0,
22-
Local,
23-
Nonlocal,
24-
Global,
25-
Builtins,
22+
Local = 1,
23+
Nonlocal = 2,
24+
Global = 4,
25+
Builtins = 8,
2626
Normal = Local | Nonlocal | Global | Builtins
2727
}
2828
}

‎src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ public EmptyAnalysis(IServiceContainer services, IDocument document) {
8686
Document = document ?? throw new ArgumentNullException(nameof(document));
8787
GlobalScope = new EmptyGlobalScope(document);
8888

89-
_emptyAst = _emptyAst ?? (_emptyAst = Parser.CreateParser(new StringReader(string.Empty), PythonLanguageVersion.None).ParseFile());
89+
_emptyAst = _emptyAst ?? (_emptyAst = Parser.CreateParser(new StringReader(string.Empty), PythonLanguageVersion.None).ParseFile(document.Uri));
9090
ExpressionEvaluator = new ExpressionEval(services, document, Ast);
9191
}
9292

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ public IMember GetValueFromCallable(CallExpression expr) {
3232
}
3333

3434
var target = GetValueFromExpression(expr.Target);
35+
target?.AddReference(GetLocationOfName(expr.Target));
36+
3537
var result = GetValueFromGeneric(target, expr);
3638
if (result != null) {
3739
return result;
@@ -58,7 +60,7 @@ public IMember GetValueFromCallable(CallExpression expr) {
5860
case IPythonType t:
5961
// Target is type (info), the call creates instance.
6062
// For example, 'x = C; y = x()' or 'x = C()' where C is class
61-
value = new PythonInstance(t, GetLoc(expr));
63+
value = new PythonInstance(t);
6264
break;
6365
}
6466

@@ -74,9 +76,9 @@ public IMember GetValueFromLambda(LambdaExpression expr) {
7476
return null;
7577
}
7678

77-
var loc = GetLoc(expr);
78-
var ft = new PythonFunctionType(expr.Function, Module, null, loc);
79-
var overload = new PythonFunctionOverload(expr.Function, ft, Module, GetLoc(expr));
79+
var location = GetLocationOfName(expr.Function);
80+
var ft = new PythonFunctionType(expr.Function, null, location);
81+
var overload = new PythonFunctionOverload(expr.Function, ft, location);
8082
ft.AddOverload(overload);
8183
return ft;
8284
}
@@ -93,7 +95,7 @@ public IMember GetValueFromClassCtor(IPythonClassType cls, CallExpression expr)
9395
}
9496
args = a.Evaluate();
9597
}
96-
return cls.CreateInstance(cls.Name, GetLoc(expr), args);
98+
return cls.CreateInstance(cls.Name, args);
9799
}
98100

99101
public IMember GetValueFromBound(IPythonBoundType t, CallExpression expr) {
@@ -190,7 +192,7 @@ public IMember GetValueFromFunctionType(IPythonFunctionType fn, IPythonInstance
190192

191193
// Try and evaluate with specific arguments. Note that it does not
192194
// make sense to evaluate stubs since they already should be annotated.
193-
if (fn.DeclaringModule is IDocument doc && fd?.IsInAst(doc.GetAnyAst()) == true) {
195+
if (fn.DeclaringModule is IDocument doc && fd?.Ast == doc.GetAnyAst()) {
194196
// Stubs are coming from another module.
195197
return TryEvaluateWithArguments(fn.DeclaringModule, fd, args);
196198
}

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public IMember GetValueFromList(ListExpression expression) {
6060
var value = GetValueFromExpression(item) ?? UnknownType;
6161
contents.Add(value);
6262
}
63-
return PythonCollectionType.CreateList(Module.Interpreter, GetLoc(expression), contents, exact: true);
63+
return PythonCollectionType.CreateList(Module.Interpreter, contents);
6464
}
6565

6666
public IMember GetValueFromDictionary(DictionaryExpression expression) {
@@ -70,7 +70,7 @@ public IMember GetValueFromDictionary(DictionaryExpression expression) {
7070
var value = GetValueFromExpression(item.SliceStop) ?? UnknownType;
7171
contents[key] = value;
7272
}
73-
return new PythonDictionary(Interpreter, GetLoc(expression), contents, exact: true);
73+
return new PythonDictionary(Interpreter, contents);
7474
}
7575

7676
private IMember GetValueFromTuple(TupleExpression expression) {
@@ -79,7 +79,7 @@ private IMember GetValueFromTuple(TupleExpression expression) {
7979
var value = GetValueFromExpression(item) ?? UnknownType;
8080
contents.Add(value);
8181
}
82-
return PythonCollectionType.CreateTuple(Module.Interpreter, GetLoc(expression), contents, exact: true);
82+
return PythonCollectionType.CreateTuple(Module.Interpreter, contents);
8383
}
8484

8585
public IMember GetValueFromSet(SetExpression expression) {
@@ -88,7 +88,7 @@ public IMember GetValueFromSet(SetExpression expression) {
8888
var value = GetValueFromExpression(item) ?? UnknownType;
8989
contents.Add(value);
9090
}
91-
return PythonCollectionType.CreateSet(Interpreter, GetLoc(expression), contents, exact: true);
91+
return PythonCollectionType.CreateSet(Interpreter, contents);
9292
}
9393

9494
public IMember GetValueFromGenerator(GeneratorExpression expression) {
@@ -108,14 +108,14 @@ public IMember GetValueFromComprehension(Comprehension node) {
108108
switch (node) {
109109
case ListComprehension lc:
110110
var v1 = GetValueFromExpression(lc.Item) ?? UnknownType;
111-
return PythonCollectionType.CreateList(Interpreter, GetLoc(lc), new[] { v1 });
111+
return PythonCollectionType.CreateList(Interpreter, new[] { v1 });
112112
case SetComprehension sc:
113113
var v2 = GetValueFromExpression(sc.Item) ?? UnknownType;
114-
return PythonCollectionType.CreateSet(Interpreter, GetLoc(sc), new[] { v2 });
114+
return PythonCollectionType.CreateSet(Interpreter, new[] { v2 });
115115
case DictionaryComprehension dc:
116116
var k = GetValueFromExpression(dc.Key) ?? UnknownType;
117117
var v = GetValueFromExpression(dc.Value) ?? UnknownType;
118-
return new PythonDictionary(new PythonDictionaryType(Interpreter), GetLoc(dc), new Dictionary<IMember, IMember> { { k, v } });
118+
return new PythonDictionary(new PythonDictionaryType(Interpreter), new Dictionary<IMember, IMember> { { k, v } });
119119
}
120120

121121
return UnknownType;
@@ -138,20 +138,20 @@ internal void ProcessComprehension(Comprehension node) {
138138
if (value != null) {
139139
switch (cfor.Left) {
140140
case NameExpression nex when value is IPythonCollection coll:
141-
DeclareVariable(nex.Name, coll.GetIterator().Next, VariableSource.Declaration, GetLoc(nex));
141+
DeclareVariable(nex.Name, coll.GetIterator().Next, VariableSource.Declaration, nex);
142142
break;
143143
case NameExpression nex:
144-
DeclareVariable(nex.Name, UnknownType, VariableSource.Declaration, GetLoc(nex));
144+
DeclareVariable(nex.Name, UnknownType, VariableSource.Declaration, nex);
145145
break;
146146
case TupleExpression tex when value is IPythonDictionary dict && tex.Items.Count > 0:
147147
if (tex.Items[0] is NameExpression nx0 && !string.IsNullOrEmpty(nx0.Name)) {
148-
DeclareVariable(nx0.Name, dict.Keys.FirstOrDefault() ?? UnknownType, VariableSource.Declaration, GetLoc(nx0));
148+
DeclareVariable(nx0.Name, dict.Keys.FirstOrDefault() ?? UnknownType, VariableSource.Declaration, nx0);
149149
}
150150
if (tex.Items.Count > 1 && tex.Items[1] is NameExpression nx1 && !string.IsNullOrEmpty(nx1.Name)) {
151-
DeclareVariable(nx1.Name, dict.Values.FirstOrDefault() ?? UnknownType, VariableSource.Declaration, GetLoc(nx1));
151+
DeclareVariable(nx1.Name, dict.Values.FirstOrDefault() ?? UnknownType, VariableSource.Declaration, nx1);
152152
}
153153
foreach (var item in tex.Items.Skip(2).OfType<NameExpression>().Where(x => !string.IsNullOrEmpty(x.Name))) {
154-
DeclareVariable(item.Name, UnknownType, VariableSource.Declaration, GetLoc(item));
154+
DeclareVariable(item.Name, UnknownType, VariableSource.Declaration, item);
155155
}
156156
break;
157157
}

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Constants.cs

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,20 +24,19 @@
2424
namespace Microsoft.Python.Analysis.Analyzer.Evaluation {
2525
internal sealed partial class ExpressionEval {
2626
public IPythonInstance GetConstantFromLiteral(Expression expr, LookupOptions options) {
27-
var location = GetLoc(expr);
2827
if (expr is ConstantExpression ce) {
2928
switch (ce.Value) {
3029
case string s:
31-
return new PythonUnicodeString(s, Interpreter, location);
30+
return new PythonUnicodeString(s, Interpreter);
3231
case AsciiString b:
33-
return new PythonAsciiString(b, Interpreter, location);
32+
return new PythonAsciiString(b, Interpreter);
3433
case int integer:
35-
return new PythonConstant(integer, Interpreter.GetBuiltinType(BuiltinTypeId.Int), location);
34+
return new PythonConstant(integer, Interpreter.GetBuiltinType(BuiltinTypeId.Int));
3635
}
3736
}
3837

3938
var t = SuppressBuiltinLookup ? UnknownType : (GetTypeFromLiteral(expr) ?? UnknownType);
40-
return new PythonInstance(t, location);
39+
return new PythonInstance(t);
4140
}
4241

4342
public IPythonType GetTypeFromLiteral(Expression expr) {

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ private IMember CreateSpecificTypeFromIndex(IGenericType gt, IReadOnlyList<IMemb
7979
if (gt.Name.EqualsOrdinal("Generic")) {
8080
// Generic[T1, T2, ...] expression. Create generic base for the class.
8181
if (genericTypeArgs.Length > 0) {
82-
return new GenericClassParameter(genericTypeArgs, Module, GetLoc(expr));
82+
return new GenericClassParameter(genericTypeArgs, Module);
8383
} else {
8484
// TODO: report too few type arguments for Generic[].
8585
return UnknownType;
@@ -88,7 +88,7 @@ private IMember CreateSpecificTypeFromIndex(IGenericType gt, IReadOnlyList<IMemb
8888

8989
// For other types just use supplied arguments
9090
if (indices.Count > 0) {
91-
return gt.CreateSpecificType(new ArgumentSet(indices), Module, GetLoc(expr));
91+
return gt.CreateSpecificType(new ArgumentSet(indices));
9292
}
9393
// TODO: report too few type arguments for the generic expression.
9494
return UnknownType;
@@ -127,7 +127,6 @@ private IReadOnlyList<IMember> EvaluateCallArgs(CallExpression expr) {
127127
private IMember CreateClassInstance(PythonClassType cls, IReadOnlyList<IMember> constructorArguments, CallExpression callExpr) {
128128
// Look at the constructor arguments and create argument set
129129
// based on the __init__ definition.
130-
var location = GetLoc(callExpr);
131130
var initFunc = cls.GetMember(@"__init__") as IPythonFunctionType;
132131
var initOverload = initFunc?.DeclaringType == cls ? initFunc.Overloads.FirstOrDefault() : null;
133132

@@ -136,8 +135,8 @@ private IMember CreateClassInstance(PythonClassType cls, IReadOnlyList<IMember>
136135
: new ArgumentSet(constructorArguments);
137136

138137
argSet.Evaluate();
139-
var specificType = cls.CreateSpecificType(argSet, Module, location);
140-
return new PythonInstance(specificType, location);
138+
var specificType = cls.CreateSpecificType(argSet);
139+
return new PythonInstance(specificType);
141140
}
142141

143142
private ScopeStatement GetScope(IMember m) {

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Hints.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ namespace Microsoft.Python.Analysis.Analyzer.Evaluation {
2727
/// </summary>
2828
internal sealed partial class ExpressionEval {
2929
public IPythonType GetTypeFromPepHint(Node node) {
30-
var location = GetLoc(node);
30+
var location = GetLocationInfo(node);
3131
var content = (Module as IDocument)?.Content;
3232
if (string.IsNullOrEmpty(content) || !location.EndLine.HasValue) {
3333
return null;

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,6 @@
1313
// See the Apache Version 2.0 License for specific language governing
1414
// permissions and limitations under the License.
1515

16-
using System;
17-
using System.Collections.Generic;
1816
using Microsoft.Python.Analysis.Modules;
1917
using Microsoft.Python.Analysis.Types;
2018
using Microsoft.Python.Analysis.Types.Collections;
@@ -54,14 +52,16 @@ private IMember GetValueFromUnaryOp(UnaryExpression expr, string op) {
5452
}
5553
}
5654
return instance is IPythonConstant c && instance.TryGetConstant<int>(out var value)
57-
? new PythonConstant(-value, c.Type, GetLoc(expr))
55+
? new PythonConstant(-value, c.Type)
5856
: instance;
5957
}
6058
return UnknownType;
6159
}
6260

6361
private IMember GetValueFromBinaryOp(Expression expr) {
64-
if (expr is AndExpression) {
62+
if (expr is AndExpression a) {
63+
GetValueFromExpression(a.Left);
64+
GetValueFromExpression(a.Right);
6565
return Interpreter.GetBuiltinType(BuiltinTypeId.Bool);
6666
}
6767

@@ -108,7 +108,6 @@ private IMember GetValueFromBinaryOp(Expression expr) {
108108
var left = GetValueFromExpression(binop.Left) ?? UnknownType;
109109
var right = GetValueFromExpression(binop.Right) ?? UnknownType;
110110

111-
112111
var rightType = right.GetPythonType();
113112
if (rightType?.TypeId == BuiltinTypeId.Float) {
114113
return right;
@@ -131,7 +130,7 @@ private IMember GetValueFromBinaryOp(Expression expr) {
131130
&& leftType?.TypeId == BuiltinTypeId.List && rightType?.TypeId == BuiltinTypeId.List
132131
&& left is IPythonCollection lc && right is IPythonCollection rc) {
133132

134-
return PythonCollectionType.CreateConcatenatedList(Module.Interpreter, GetLoc(expr), lc, rc);
133+
return PythonCollectionType.CreateConcatenatedList(Module.Interpreter, lc, rc);
135134
}
136135

137136
return left.IsUnknown() ? right : left;

0 commit comments

Comments
 (0)