Skip to content
This repository was archived by the owner on Nov 4, 2024. It is now read-only.

Commit dfc3e09

Browse files
author
Mikhail Arkhipov
authored
Linter for undefined variables (microsoft#693)
* Fix microsoft#668 (partial) * Undef variables, first cut * Reorg * Tests * Tests * Tests * Revert "Tests" This reverts commit 7ffc9db. * Options and tests * Don't squiggle builtin-ins * Test for function arguments * Fix tuple assignment in analysis * Don't false positive on functions and classes forward references * Disable tracking assignment location Fix declaration of variables in for loop * Track multiple locations * Tests * using * Properly look at locations * Comprehensions etc * Test update * Add support for lambdas * Add lambda linting * Handle tuple assignment better * Test update * Correct comprehension iterator handlint * Fix race condition at builtins load * Merge master * Restore linter
1 parent bf69f85 commit dfc3e09

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+1521
-99
lines changed

‎src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -62,23 +62,24 @@ public override bool Walk(AssignmentStatement node) {
6262
}
6363

6464
public override bool Walk(ExpressionStatement node) {
65-
AssignmentHandler.HandleAnnotatedExpression(node.Expression as ExpressionWithAnnotation, null);
65+
switch (node.Expression) {
66+
case ExpressionWithAnnotation ea:
67+
AssignmentHandler.HandleAnnotatedExpression(ea, null);
68+
break;
69+
case Comprehension comp:
70+
Eval.ProcessComprehension(comp);
71+
break;
72+
}
6673
return false;
6774
}
6875

69-
public override bool Walk(ForStatement node) {
70-
LoopHandler.HandleFor(node);
71-
return base.Walk(node);
72-
}
73-
76+
public override bool Walk(ForStatement node) => LoopHandler.HandleFor(node);
7477
public override bool Walk(FromImportStatement node) => ImportHandler.HandleFromImport(node);
7578
public override bool Walk(GlobalStatement node) => NonLocalHandler.HandleGlobal(node);
7679
public override bool Walk(IfStatement node) => ConditionalHandler.HandleIf(node);
7780

7881
public override bool Walk(ImportStatement node) => ImportHandler.HandleImport(node);
79-
80-
public override bool Walk(NonlocalStatement node)
81-
=> NonLocalHandler.HandleNonLocal(node);
82+
public override bool Walk(NonlocalStatement node) => NonLocalHandler.HandleNonLocal(node);
8283

8384
public override bool Walk(TryStatement node) {
8485
TryExceptHandler.HandleTryExcept(node);

‎src/Analysis/Ast/Impl/Analyzer/Definitions/IExpressionEvaluator.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
using System;
1717
using System.Collections.Generic;
18+
using Microsoft.Python.Analysis.Analyzer.Evaluation;
1819
using Microsoft.Python.Analysis.Diagnostics;
1920
using Microsoft.Python.Analysis.Types;
2021
using Microsoft.Python.Analysis.Values;
@@ -57,7 +58,7 @@ public interface IExpressionEvaluator {
5758
/// </summary>
5859
IMember GetValueFromExpression(Expression expr);
5960

60-
IMember LookupNameInScopes(string name, out IScope scope);
61+
IMember LookupNameInScopes(string name, out IScope scope, LookupOptions options = LookupOptions.Normal);
6162

6263
IPythonType GetTypeFromString(string typeString);
6364

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/LookupOptions.cs renamed to ‎src/Analysis/Ast/Impl/Analyzer/Definitions/LookupOptions.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@
1515

1616
using System;
1717

18-
namespace Microsoft.Python.Analysis.Analyzer.Evaluation {
18+
namespace Microsoft.Python.Analysis.Analyzer {
1919
[Flags]
20-
internal enum LookupOptions {
20+
public enum LookupOptions {
2121
None = 0,
2222
Local,
2323
Nonlocal,

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,18 @@ public IMember GetValueFromCallable(CallExpression expr) {
6969
return value;
7070
}
7171

72+
public IMember GetValueFromLambda(LambdaExpression expr) {
73+
if (expr == null) {
74+
return null;
75+
}
76+
77+
var loc = GetLoc(expr);
78+
var ft = new PythonFunctionType(expr.Function, Module, null, loc);
79+
var overload = new PythonFunctionOverload(expr.Function, ft, Module, GetLoc(expr));
80+
ft.AddOverload(overload);
81+
return ft;
82+
}
83+
7284
public IMember GetValueFromClassCtor(IPythonClassType cls, CallExpression expr) {
7385
SymbolTable.Evaluate(cls.ClassDefinition);
7486
// Determine argument types

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,5 +98,65 @@ public IMember GetValueFromGenerator(GeneratorExpression expression) {
9898
}
9999
return UnknownType;
100100
}
101+
102+
public IMember GetValueFromComprehension(Comprehension node) {
103+
var oldVariables = CurrentScope.Variables.OfType<Variable>().ToDictionary(k => k.Name, v => v);
104+
try {
105+
ProcessComprehension(node);
106+
switch (node) {
107+
case ListComprehension lc:
108+
var v1 = GetValueFromExpression(lc.Item) ?? UnknownType;
109+
return PythonCollectionType.CreateList(Interpreter, GetLoc(lc), new[] { v1 });
110+
case SetComprehension sc:
111+
var v2 = GetValueFromExpression(sc.Item) ?? UnknownType;
112+
return PythonCollectionType.CreateSet(Interpreter, GetLoc(sc), new[] { v2 });
113+
case DictionaryComprehension dc:
114+
var k = GetValueFromExpression(dc.Key) ?? UnknownType;
115+
var v = GetValueFromExpression(dc.Value) ?? UnknownType;
116+
return new PythonDictionary(new PythonDictionaryType(Interpreter), GetLoc(dc), new Dictionary<IMember, IMember> { { k, v } });
117+
}
118+
119+
return UnknownType;
120+
} finally {
121+
// Remove temporary variables since this is assignment and the right hand
122+
// side comprehension does not leak internal variables into the scope.
123+
var newVariables = CurrentScope.Variables.ToDictionary(k => k.Name, v => v);
124+
var variables = (VariableCollection)CurrentScope.Variables;
125+
foreach (var kvp in newVariables) {
126+
if (!oldVariables.ContainsKey(kvp.Key)) {
127+
variables.RemoveVariable(kvp.Key);
128+
} else {
129+
variables.DeclareVariable(oldVariables[kvp.Key]);
130+
}
131+
}
132+
}
133+
}
134+
135+
internal void ProcessComprehension(Comprehension node) {
136+
foreach (var cfor in node.Iterators.OfType<ComprehensionFor>().Where(c => c.Left != null)) {
137+
var value = GetValueFromExpression(cfor.List);
138+
if (value != null) {
139+
switch (cfor.Left) {
140+
case NameExpression nex when value is IPythonCollection coll:
141+
DeclareVariable(nex.Name, coll.GetIterator().Next, VariableSource.Declaration, GetLoc(nex));
142+
break;
143+
case NameExpression nex:
144+
DeclareVariable(nex.Name, UnknownType, VariableSource.Declaration, GetLoc(nex));
145+
break;
146+
case TupleExpression tex when value is IPythonDictionary dict && tex.Items.Count > 0:
147+
if (tex.Items[0] is NameExpression nx0 && !string.IsNullOrEmpty(nx0.Name)) {
148+
DeclareVariable(nx0.Name, dict.Keys.FirstOrDefault() ?? UnknownType, VariableSource.Declaration, GetLoc(nx0));
149+
}
150+
if (tex.Items.Count > 1 && tex.Items[1] is NameExpression nx1 && !string.IsNullOrEmpty(nx1.Name)) {
151+
DeclareVariable(nx1.Name, dict.Values.FirstOrDefault() ?? UnknownType, VariableSource.Declaration, GetLoc(nx1));
152+
}
153+
foreach (var item in tex.Items.Skip(2).OfType<NameExpression>().Where(x => !string.IsNullOrEmpty(x.Name))) {
154+
DeclareVariable(item.Name, UnknownType, VariableSource.Declaration, GetLoc(item));
155+
}
156+
break;
157+
}
158+
}
159+
}
160+
}
101161
}
102162
}

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -141,8 +141,8 @@ public IPythonType GetTypeFromAnnotation(Expression expr, out bool isGeneric, Lo
141141
/// as a child of the specified scope. Scope is pushed on the stack
142142
/// and will be removed when returned the disposable is disposed.
143143
/// </summary>
144-
public IDisposable OpenScope(IPythonModule module, ScopeStatement node, out Scope fromScope) {
145-
fromScope = null;
144+
public IDisposable OpenScope(IPythonModule module, ScopeStatement node, out Scope outerScope) {
145+
outerScope = null;
146146
if (node == null) {
147147
return Disposable.Empty;
148148
}
@@ -156,25 +156,25 @@ public IDisposable OpenScope(IPythonModule module, ScopeStatement node, out Scop
156156
}
157157

158158
if (node.Parent != null) {
159-
if (!_scopeLookupCache.TryGetValue(node.Parent, out fromScope)) {
160-
fromScope = gs
159+
if (!_scopeLookupCache.TryGetValue(node.Parent, out outerScope)) {
160+
outerScope = gs
161161
.TraverseDepthFirst(s => s.Children.OfType<Scope>())
162162
.FirstOrDefault(s => s.Node == node.Parent);
163-
_scopeLookupCache[node.Parent] = fromScope;
163+
_scopeLookupCache[node.Parent] = outerScope;
164164
}
165165
}
166166

167-
fromScope = fromScope ?? gs;
168-
if (fromScope != null) {
167+
outerScope = outerScope ?? gs;
168+
if (outerScope != null) {
169169
Scope scope;
170170
if (node is PythonAst) {
171171
// node points to global scope, it is not a function or a class.
172172
scope = gs;
173173
} else {
174-
scope = fromScope.Children.OfType<Scope>().FirstOrDefault(s => s.Node == node);
174+
scope = outerScope.Children.OfType<Scope>().FirstOrDefault(s => s.Node == node);
175175
if (scope == null) {
176-
scope = new Scope(node, fromScope, true);
177-
fromScope.AddChildScope(scope);
176+
scope = new Scope(node, outerScope, true);
177+
outerScope.AddChildScope(scope);
178178
_scopeLookupCache[node] = scope;
179179
}
180180
}

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,15 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs
6969
public LocationInfo GetLocation(Node node) => node?.GetLocation(Module, Ast) ?? LocationInfo.Empty;
7070
public IEnumerable<DiagnosticsEntry> Diagnostics => _diagnostics;
7171

72+
public void ReportDiagnostics(Uri documentUri, DiagnosticsEntry entry) {
73+
// Do not add if module is library, etc. Only handle user code.
74+
if (Module.ModuleType == ModuleType.User) {
75+
lock (_lock) {
76+
_diagnostics.Add(entry);
77+
}
78+
}
79+
}
80+
7281
public IMember GetValueFromExpression(Expression expr)
7382
=> GetValueFromExpression(expr, DefaultLookupOptions);
7483

@@ -89,9 +98,7 @@ public IMember GetValueFromExpression(Expression expr, LookupOptions options) {
8998
return null;
9099
}
91100

92-
while (expr is ParenthesisExpression parExpr) {
93-
expr = parExpr.Expression;
94-
}
101+
expr = expr.RemoveParenthesis();
95102

96103
IMember m;
97104
switch (expr) {
@@ -131,6 +138,12 @@ public IMember GetValueFromExpression(Expression expr, LookupOptions options) {
131138
case GeneratorExpression genex:
132139
m = GetValueFromGenerator(genex);
133140
break;
141+
case Comprehension comp:
142+
m = GetValueFromComprehension(comp);
143+
break;
144+
case LambdaExpression lambda:
145+
m = GetValueFromLambda(lambda);
146+
break;
134147
default:
135148
m = GetValueFromBinaryOp(expr) ?? GetConstantFromLiteral(expr, options);
136149
break;
@@ -228,14 +241,5 @@ private IMember GetValueFromConditional(ConditionalExpression expr) {
228241

229242
return trueValue ?? falseValue ?? UnknownType;
230243
}
231-
232-
public void ReportDiagnostics(Uri documentUri, DiagnosticsEntry entry) {
233-
// Do not add if module is library, etc. Only handle user code.
234-
if (Module.ModuleType == ModuleType.User) {
235-
lock (_lock) {
236-
_diagnostics.Add(entry);
237-
}
238-
}
239-
}
240244
}
241245
}

‎src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515

1616
using System.Diagnostics;
1717
using System.Linq;
18-
using Microsoft.Python.Analysis.Analyzer.Evaluation;
1918
using Microsoft.Python.Analysis.Types;
2019
using Microsoft.Python.Analysis.Values;
2120
using Microsoft.Python.Core;
@@ -62,7 +61,7 @@ public void HandleAssignment(AssignmentStatement node) {
6261
if (Eval.CurrentScope.NonLocals[ne.Name] != null) {
6362
Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Nonlocal);
6463
if (scope != null) {
65-
scope.Variables[ne.Name].Value = value;
64+
scope.Variables[ne.Name].Assign(value, Eval.GetLoc(ne));
6665
} else {
6766
// TODO: report variable is not declared in outer scopes.
6867
}
@@ -72,7 +71,7 @@ public void HandleAssignment(AssignmentStatement node) {
7271
if (Eval.CurrentScope.Globals[ne.Name] != null) {
7372
Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Global);
7473
if (scope != null) {
75-
scope.Variables[ne.Name].Value = value;
74+
scope.Variables[ne.Name].Assign(value, Eval.GetLoc(ne));
7675
} else {
7776
// TODO: report variable is not declared in global scope.
7877
}

‎src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,6 @@
1515

1616
using System;
1717
using System.Linq;
18-
using Microsoft.Python.Analysis.Types;
19-
using Microsoft.Python.Analysis.Values;
2018
using Microsoft.Python.Core.OS;
2119
using Microsoft.Python.Parsing;
2220
using Microsoft.Python.Parsing.Ast;

‎src/Analysis/Ast/Impl/Analyzer/Handlers/LoopHandler.cs

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -21,35 +21,27 @@ namespace Microsoft.Python.Analysis.Analyzer.Handlers {
2121
internal sealed class LoopHandler : StatementHandler {
2222
public LoopHandler(AnalysisWalker walker) : base(walker) { }
2323

24-
public void HandleFor(ForStatement node) {
24+
public bool HandleFor(ForStatement node) {
2525
var iterable = Eval.GetValueFromExpression(node.List);
2626
var iterator = (iterable as IPythonIterable)?.GetIterator();
27-
if (iterator == null) {
28-
// TODO: report that expression does not evaluate to iterable.
29-
}
3027
var value = iterator?.Next ?? Eval.UnknownType;
31-
3228
switch (node.Left) {
3329
case NameExpression nex:
3430
// for x in y:
35-
Eval.DeclareVariable(nex.Name, value, VariableSource.Declaration, Eval.GetLoc(node.Left));
31+
if (!string.IsNullOrEmpty(nex.Name)) {
32+
Eval.DeclareVariable(nex.Name, value, VariableSource.Declaration, Eval.GetLoc(nex));
33+
}
3634
break;
3735
case TupleExpression tex:
3836
// x = [('abc', 42, True), ('abc', 23, False)]
39-
// for some_str, some_int, some_bool in x:
40-
var names = tex.Items.OfType<NameExpression>().Select(x => x.Name).ToArray();
41-
if (value is IPythonIterable valueIterable) {
42-
var valueIterator = valueIterable.GetIterator();
43-
foreach (var n in names) {
44-
Eval.DeclareVariable(n, valueIterator?.Next ?? Eval.UnknownType, VariableSource.Declaration, Eval.GetLoc(node.Left));
45-
}
46-
} else {
47-
// TODO: report that expression yields value that does not evaluate to iterable.
48-
}
37+
// for some_str, (some_int, some_bool) in x:
38+
var h = new TupleExpressionHandler(Walker);
39+
h.HandleTupleAssignment(tex, node.List, value);
4940
break;
5041
}
5142

5243
node.Body?.Walk(Walker);
44+
return false;
5345
}
5446

5547
public void HandleWhile(WhileStatement node) {

‎src/Analysis/Ast/Impl/Analyzer/Handlers/StatementHandler.cs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,8 @@
1414
// permissions and limitations under the License.
1515

1616
using Microsoft.Python.Analysis.Analyzer.Evaluation;
17-
using Microsoft.Python.Analysis.Analyzer.Symbols;
1817
using Microsoft.Python.Analysis.Modules;
1918
using Microsoft.Python.Analysis.Types;
20-
using Microsoft.Python.Analysis.Values;
2119
using Microsoft.Python.Core.Logging;
2220
using Microsoft.Python.Parsing.Ast;
2321

0 commit comments

Comments
 (0)