Skip to content
This repository was archived by the owner on Apr 14, 2022. It is now read-only.

Unpacking typing Tuples and Lists correctly #1553

Merged
merged 86 commits into from
Sep 16, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
86 commits
Select commit Hold shift + click to select a range
61d60c4
Persistent analysis, part I (#1224)
Jun 20, 2019
b2c9afb
Merge master into database branch (#1312)
Jul 11, 2019
ab8ecaf
Merge master
Jul 15, 2019
4a2fd79
Treat persistent module as regular specialized (#1334)
Jul 23, 2019
da3eb70
Store location of module members in the database (#1339)
Jul 26, 2019
a502cb5
Merge master
Jul 30, 2019
2143719
Multiple fixes and additional tests for persistence (#1372)
Aug 8, 2019
ff46089
Merge master
Aug 10, 2019
78b1bcf
Merge master
Aug 13, 2019
f267b80
Add persistence tests for 60+ system modules (#1429)
Aug 15, 2019
3c53083
Merge master
Aug 16, 2019
69806e0
Test update (we no longer reassign classes)
Aug 16, 2019
356ae4b
Fix typing tests
Aug 16, 2019
36a1d72
Update baseline
Aug 16, 2019
41c94e7
Fix missing keys issue (#1451)
AlexanderSher Aug 16, 2019
6284174
Typevar fix
Aug 17, 2019
28ec49b
Merge master
Aug 20, 2019
8b4c00a
Merge mast
Aug 21, 2019
86c8426
Persistence of generics and named tuples (#1459)
Aug 21, 2019
a82c356
- Fix #1455: AF: Library module of type Stub has been analyzed alread…
AlexanderSher Aug 22, 2019
ab93845
Merge master
Aug 23, 2019
79685eb
Update baselines since __spec__ member was added
Aug 26, 2019
809dc5c
Merge master (partial)
Aug 26, 2019
0f7f7ca
Undo some of changes to match master.
Aug 26, 2019
252d303
Fix null ref
Aug 26, 2019
a013b3a
Merge branch 'master' of https://github.com/Microsoft/python-language…
Aug 27, 2019
4de0812
Merge master
Aug 28, 2019
0913ea3
Merge master
Aug 30, 2019
8adb6cc
Merge master
Aug 30, 2019
c1883b5
Merge master
Sep 3, 2019
b923349
Resolve conflicts
Sep 3, 2019
28ce8e3
Fix null ref on bases due to new AnyStr behavior
Sep 3, 2019
e7f29d7
Handling dependencies in module persistence (#1471)
Sep 5, 2019
8cf831f
Upgrade to 3.0
Sep 5, 2019
958617b
Merge branch 'net3' of https://github.com/MikhailArkhipov/python-lang…
Sep 5, 2019
54d67cb
Undo change + .NET 3
Sep 5, 2019
26ea552
Get back to 2.0
Sep 5, 2019
fff4e0c
Merge branch 'net3' of https://github.com/MikhailArkhipov/python-lang…
Sep 5, 2019
227726c
Update components
Sep 5, 2019
2dd8fcf
Display caching level
Sep 5, 2019
c0f16e6
Fix null ref due to conflicting changes
Sep 5, 2019
987a837
Null ref fix
Sep 5, 2019
6d79b49
Simplify
Sep 5, 2019
28a4ce5
Pass AST whish is required during analysis when module does not have …
Sep 5, 2019
e24a4fd
tensorflow test
Sep 6, 2019
c4404cf
Fix from import model
Sep 6, 2019
c20485b
Undo debug
Sep 6, 2019
119e1b2
Improve goto def in import and fix class base processing (#1529)
Sep 9, 2019
2313d44
Usings
Sep 10, 2019
af2d7c7
Test fix
Sep 10, 2019
8e3289c
Remove CompletionItemKind.None
Sep 10, 2019
af70958
Add info to asserts
Sep 10, 2019
83ddd00
Merge master
Sep 10, 2019
967e741
PR feedback
Sep 10, 2019
653fd80
Undo test change, enable class reassignment
Sep 10, 2019
655818e
Prevent merge of unrelated types
Sep 11, 2019
48561b8
more tests
Sep 11, 2019
61014c1
Fix variable update
Sep 11, 2019
ab29574
Null check
Sep 11, 2019
81e36ce
Baselines
Sep 11, 2019
8323cd2
Move analysis complete event
Sep 11, 2019
f44c051
More stable enumeration
Sep 11, 2019
87093f3
Improve restoring imports in tests
Sep 11, 2019
a81874e
Make extension for tests
Sep 11, 2019
6a6fbb2
Add URI null check
Sep 12, 2019
8547a9b
Fix completion typo
Sep 12, 2019
8aa373c
Add assert on unresolved modules
Sep 12, 2019
820fdd4
Add more details to assert
Sep 12, 2019
db002c4
Using
Sep 12, 2019
56b8d88
To 3.7
Sep 12, 2019
68d38e7
Implement check for big/little endian
Sep 12, 2019
8dcd2bd
Merge master
Sep 12, 2019
93be7b1
Sync
Sep 12, 2019
2bdc314
Synchronization
Sep 13, 2019
3ee351c
Add hard assert
Sep 13, 2019
55b85e4
Fix exception
Sep 14, 2019
97f280c
Merge branch 'master' of https://github.com/Microsoft/python-language…
Sep 16, 2019
4108948
Fix typo
Sep 16, 2019
d6260fe
Add lock
Sep 16, 2019
c08dc38
Adding support for egg and zip files (#1477)
CTrando Sep 16, 2019
fbfafb8
Merge branch 'master' of https://github.com/Microsoft/python-language…
Sep 16, 2019
80fcca7
Merge branch 'db' of https://github.com/Microsoft/python-language-ser…
Sep 16, 2019
52e4beb
Unpacking typing Tuples and Lists correctly (#1498)
CTrando Sep 16, 2019
b164dbe
Merge branch 'master' of https://github.com/Microsoft/python-language…
Sep 16, 2019
e9ca834
Fix merge issue
Sep 16, 2019
881be72
Comment typo
Sep 16, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ public IMember GetValueFromCallable(CallExpression expr) {
// so we can invoke Call over the instance. Second, an type info
// so we can create an instance of the type (as in C() where C is class).
IMember value = null;
var args = ArgumentSet.Empty(expr, this);
switch (target) {
case IPythonBoundType bt: // Bound property, method or an iterator.
value = GetValueFromBound(bt, expr);
Expand All @@ -52,7 +53,7 @@ public IMember GetValueFromCallable(CallExpression expr) {
value = GetValueFromInstanceCall(pi, expr);
break;
case IPythonFunctionType ft: // Standalone function or a class method call.
var instance = ft.DeclaringType != null ? new PythonInstance(ft.DeclaringType) : null;
var instance = ft.DeclaringType?.CreateInstance(args);
value = GetValueFromFunctionType(ft, instance, expr);
break;
case IPythonClassType cls:
Expand All @@ -61,7 +62,7 @@ public IMember GetValueFromCallable(CallExpression expr) {
case IPythonType t:
// Target is type (info), the call creates instance.
// For example, 'x = C; y = x()' or 'x = C()' where C is class
value = new PythonInstance(t);
value = t.CreateInstance(args);
break;
}

Expand Down Expand Up @@ -93,14 +94,14 @@ public IMember GetValueFromClassCtor(IPythonClassType cls, CallExpression expr)
var init = cls.GetMember<IPythonFunctionType>(@"__init__");
if (init != null) {
using (OpenScope(cls.DeclaringModule, cls.ClassDefinition, out _)) {
var a = new ArgumentSet(init, 0, new PythonInstance(cls), expr, this);
var a = new ArgumentSet(init, 0, cls, expr, this);
if (a.Errors.Count > 0) {
// AddDiagnostics(Module.Uri, a.Errors);
}
args = a.Evaluate();
}
}
return cls.CreateInstance(cls.Name, args);
return cls.CreateInstance(args);
}

private IMember GetValueFromBound(IPythonBoundType t, CallExpression expr) {
Expand Down Expand Up @@ -146,12 +147,14 @@ public IMember GetValueFromFunctionType(IPythonFunctionType fn, IPythonInstance
// Pick the best overload.
FunctionDefinition fd;
ArgumentSet args;
var instanceType = instance?.GetPythonType();

if (fn.Overloads.Count == 1) {
fd = fn.Overloads[0].FunctionDefinition;
args = new ArgumentSet(fn, 0, instance, expr, this);
args = new ArgumentSet(fn, 0, instanceType, expr, this);
args = args.Evaluate();
} else {
args = FindOverload(fn, instance, expr);
args = FindOverload(fn, instanceType, expr);
if (args == null) {
return UnknownType;
}
Expand All @@ -167,21 +170,20 @@ public IMember GetValueFromFunctionType(IPythonFunctionType fn, IPythonInstance
}
}

// If instance is not the same as the declaring type, then call most probably comes
// If instance type is not the same as the declaring type, then call most probably comes
// from the derived class which means that the original 'self' and 'cls' variables
// are no longer valid and function has to be re-evaluated with new arguments.
// Note that there is nothing to re-evaluate in stubs.
var instanceType = instance?.GetPythonType();
if (instanceType == null || fn.DeclaringType == null || fn.IsSpecialized ||
instanceType.IsSpecialized || fn.DeclaringType.IsSpecialized ||
instanceType.Equals(fn.DeclaringType) ||
fn.IsStub || !string.IsNullOrEmpty(fn.Overloads[args.OverloadIndex].GetReturnDocumentation())) {

LoadFunctionDependencyModules(fn);

var t = instance?.Call(fn.Name, args) ?? fn.Call(null, fn.Name, args);
if (!t.IsUnknown()) {
return t;
var m = instance?.Call(fn.Name, args) ?? fn.Call(null, fn.Name, args);
if (!m.IsUnknown()) {
return m;
}
}

Expand Down Expand Up @@ -244,14 +246,14 @@ private IMember TryEvaluateWithArguments(IPythonFunctionType fn, IArgumentSet ar
return result;
}

private ArgumentSet FindOverload(IPythonFunctionType fn, IPythonInstance instance, CallExpression expr) {
private ArgumentSet FindOverload(IPythonFunctionType fn, IPythonType instanceType, CallExpression expr) {
if (fn.Overloads.Count == 1) {
return null;
}

var sets = new List<ArgumentSet>();
for (var i = 0; i < fn.Overloads.Count; i++) {
var a = new ArgumentSet(fn, i, instance, expr, this);
var a = new ArgumentSet(fn, i, instanceType, expr, this);
var args = a.Evaluate();
sets.Add(args);
}
Expand Down Expand Up @@ -329,7 +331,8 @@ public IReadOnlyList<IParameterInfo> CreateFunctionParameters(IPythonClassType s
// The reason is that if method might be called on a derived class.
// Declare self or cls in this scope.
if (declareVariables) {
DeclareVariable(p0.Name, new PythonInstance(self), VariableSource.Declaration, p0.NameExpression);
DeclareVariable(p0.Name, self.CreateInstance(ArgumentSet.Empty(p0.NameExpression, this)),
VariableSource.Declaration, p0.NameExpression);
}
// Set parameter info, declare type as annotation type for generic self
// e.g def test(self: T)
Expand Down Expand Up @@ -377,7 +380,8 @@ private void DeclareParameter(Parameter p, ParameterInfo pi) {
} else {
paramType = pi?.DefaultValue?.GetPythonType() ?? UnknownType;
}
DeclareVariable(p.Name, new PythonInstance(paramType), VariableSource.Declaration, p.NameExpression);
DeclareVariable(p.Name, paramType.CreateInstance(ArgumentSet.Empty(p.NameExpression, this)),
VariableSource.Declaration, p.NameExpression);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,11 @@ public IMember GetValueFromIndex(IndexExpression expr) {
var type = target.GetPythonType();
if (type != null) {
if (!(target is IPythonInstance instance)) {
instance = new PythonInstance(type);
instance = type.CreateInstance(ArgumentSet.Empty(expr, this));
}
var index = GetValueFromExpression(expr.Index);
if (index != null) {
return type.Index(instance, new ArgumentSet(new []{index}, expr, this));
return type.Index(instance, new ArgumentSet(new[] { index }, expr, this));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public IPythonInstance GetConstantFromLiteral(Expression expr) {
}

var t = SuppressBuiltinLookup ? UnknownType : (GetTypeFromLiteral(expr) ?? UnknownType);
return new PythonInstance(t);
return t.CreateInstance(ArgumentSet.Empty(expr, this));
}

public IPythonType GetTypeFromLiteral(Expression expr) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -172,12 +172,12 @@ private IMember CreateClassInstance(PythonClassType cls, IReadOnlyList<IMember>
var initOverload = initFunc?.DeclaringType == cls ? initFunc.Overloads.FirstOrDefault() : null;

var argSet = initOverload != null
? new ArgumentSet(initFunc, 0, null, callExpr, this)
? new ArgumentSet(initFunc, 0, cls, callExpr, this)
: new ArgumentSet(constructorArguments, callExpr, this);

argSet.Evaluate();
var specificType = cls.CreateSpecificType(argSet);
return new PythonInstance(specificType);
return specificType.CreateInstance(argSet);
}

private ScopeStatement GetScope(IMember m) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ private IMember GetValueFromMember(MemberExpression expr) {
f.AddReference(GetLocationOfName(expr));
return f.ToUnbound();
}
instance = new PythonInstance(type);
instance = type.CreateInstance(ArgumentSet.Empty(expr, this));
}

instance = instance ?? m as IPythonInstance;
Expand Down
79 changes: 43 additions & 36 deletions src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.

using System.Diagnostics;
using System.Linq;
using Microsoft.Python.Analysis.Types;
using Microsoft.Python.Analysis.Values;
using Microsoft.Python.Core;
using Microsoft.Python.Parsing.Ast;
using System.Diagnostics;
using System.Linq;

namespace Microsoft.Python.Analysis.Analyzer.Handlers {
internal sealed class AssignmentHandler : StatementHandler {
Expand All @@ -30,10 +30,12 @@ public void HandleAssignment(AssignmentStatement node) {
}

var value = Eval.GetValueFromExpression(node.Right) ?? Eval.UnknownType;
// Filter out parenthesis expression in assignment because it makes no difference.
var lhs = node.Left.Select(s => s.RemoveParenthesis());
// Check PEP hint first
var valueType = Eval.GetTypeFromPepHint(node.Right);
if (valueType != null) {
HandleTypedVariable(valueType, value, node.Left.FirstOrDefault());
HandleTypedVariable(valueType, value, lhs.FirstOrDefault());
return;
}

Expand All @@ -44,44 +46,49 @@ public void HandleAssignment(AssignmentStatement node) {
value = Eval.UnknownType;
}

if (node.Left.FirstOrDefault() is SequenceExpression seq) {
// Tuple = Tuple. Transfer values.
var seqHandler = new SequenceExpressionHandler(Walker);
seqHandler.HandleAssignment(seq.Items, node.Right, value);
return;
foreach (var expr in lhs) {
switch (expr) {
case SequenceExpression seq:
// Tuple = Tuple. Transfer values.
var seqHandler = new SequenceExpressionHandler(Walker);
seqHandler.HandleAssignment(seq, value);
break;
case ExpressionWithAnnotation annExpr:
HandleAnnotatedExpression(annExpr, value);
break;
case NameExpression nameExpr:
HandleNameExpression(nameExpr, value);
break;
case MemberExpression memberExpr:
TryHandleClassVariable(memberExpr, value);
break;
}
}
}

// Process annotations, if any.
foreach (var expr in node.Left.OfType<ExpressionWithAnnotation>()) {
// x: List[str] = [...]
HandleAnnotatedExpression(expr, value);
}
private bool IsValidAssignment(string name, Location loc) => !Eval.GetInScope(name).IsDeclaredAfter(loc);

foreach (var ne in node.Left.OfType<NameExpression>()) {
IScope scope;
if (Eval.CurrentScope.NonLocals[ne.Name] != null) {
Eval.LookupNameInScopes(ne.Name, out scope, LookupOptions.Nonlocal);
scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne));
continue;
}
if (Eval.CurrentScope.Globals[ne.Name] != null) {
Eval.LookupNameInScopes(ne.Name, out scope, LookupOptions.Global);
scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne));
continue;
}
private void HandleNameExpression(NameExpression ne, IMember value) {
IScope scope;
if (Eval.CurrentScope.NonLocals[ne.Name] != null) {
Eval.LookupNameInScopes(ne.Name, out scope, LookupOptions.Nonlocal);
scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne));
return;
}

var source = value.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration;
var location = Eval.GetLocationOfName(ne);
if (IsValidAssignment(ne.Name, location)) {
Eval.DeclareVariable(ne.Name, value ?? Module.Interpreter.UnknownType, source, location);
}
if (Eval.CurrentScope.Globals[ne.Name] != null) {
Eval.LookupNameInScopes(ne.Name, out scope, LookupOptions.Global);
scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne));
return;
}

TryHandleClassVariable(node, value);
var source = value.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration;
var location = Eval.GetLocationOfName(ne);
if (IsValidAssignment(ne.Name, location)) {
Eval.DeclareVariable(ne.Name, value ?? Module.Interpreter.UnknownType, source, location);
}
}

private bool IsValidAssignment(string name, Location loc) => !Eval.GetInScope(name).IsDeclaredAfter(loc);

public void HandleAnnotatedExpression(ExpressionWithAnnotation expr, IMember value) {
if (expr?.Annotation == null) {
return;
Expand All @@ -95,8 +102,7 @@ public void HandleAnnotatedExpression(ExpressionWithAnnotation expr, IMember val
HandleTypedVariable(variableType, value, expr.Expression);
}

private void TryHandleClassVariable(AssignmentStatement node, IMember value) {
var mex = node.Left.OfType<MemberExpression>().FirstOrDefault();
private void TryHandleClassVariable(MemberExpression mex, IMember value) {
if (!string.IsNullOrEmpty(mex?.Name) && mex.Target is NameExpression nex && nex.Name.EqualsOrdinal("self")) {
var m = Eval.LookupNameInScopes(nex.Name, out _, LookupOptions.Local);
var cls = m.GetPythonType<IPythonClassType>();
Expand All @@ -121,7 +127,8 @@ private void HandleTypedVariable(IPythonType variableType, IMember value, Expres
instance = value;
}
}
instance = instance ?? variableType?.CreateInstance(variableType.Name, ArgumentSet.Empty(expr, Eval)) ?? Eval.UnknownType;
var args = ArgumentSet.Empty(expr, Eval);
instance = instance ?? variableType?.CreateInstance(args) ?? Eval.UnknownType.CreateInstance(ArgumentSet.WithoutContext);

if (expr is NameExpression ne) {
Eval.DeclareVariable(ne.Name, instance, VariableSource.Declaration, ne);
Expand Down
2 changes: 1 addition & 1 deletion src/Analysis/Ast/Impl/Analyzer/Handlers/LoopHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public bool HandleFor(ForStatement node) {
// x = [('abc', 42, True), ('abc', 23, False)]
// for some_str, (some_int, some_bool) in x:
var h = new SequenceExpressionHandler(Walker);
h.HandleAssignment(seq.Items, node.List, value);
h.HandleAssignment(seq, value);
break;
}

Expand Down
Loading