Skip to content
This repository was archived by the owner on Apr 14, 2022. It is now read-only.

Commit d61f63d

Browse files
CTrandoCTrando
authored
Trying to augment argument set with Evaluator and Expression context (#1259)
* Moving anyStr to its own method to prevent FromTypeVar from being called with an ArgumentSet that has no context * Trying to augment argument set with context wherever possible * Setting expression in argument set, removing null * Renaming to WithoutContext and updating comment * Check for null args in PythonFunctionOverload
1 parent 53de3cd commit d61f63d

18 files changed

+80
-49
lines changed

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ public IMember GetValueFromLambda(LambdaExpression expr) {
9090
public IMember GetValueFromClassCtor(IPythonClassType cls, CallExpression expr) {
9191
SymbolTable.Evaluate(cls.ClassDefinition);
9292
// Determine argument types
93-
var args = ArgumentSet.Empty;
93+
var args = ArgumentSet.Empty(expr, this);
9494
var init = cls.GetMember<IPythonFunctionType>(@"__init__");
9595
if (init != null) {
9696
using (OpenScope(cls.DeclaringModule, cls.ClassDefinition, out _)) {
@@ -109,7 +109,7 @@ private IMember GetValueFromBound(IPythonBoundType t, CallExpression expr) {
109109
case IPythonFunctionType fn:
110110
return GetValueFromFunctionType(fn, t.Self, expr);
111111
case IPythonPropertyType p:
112-
return GetValueFromProperty(p, t.Self);
112+
return GetValueFromProperty(p, t.Self, expr);
113113
case IPythonIteratorType _ when t.Self is IPythonCollection seq:
114114
return seq.GetIterator();
115115
}
@@ -202,10 +202,10 @@ public IMember GetValueFromFunctionType(IPythonFunctionType fn, IPythonInstance
202202
return UnknownType;
203203
}
204204

205-
private IMember GetValueFromProperty(IPythonPropertyType p, IPythonInstance instance) {
205+
private IMember GetValueFromProperty(IPythonPropertyType p, IPythonInstance instance, CallExpression expr) {
206206
// Function may not have been walked yet. Do it now.
207207
SymbolTable.Evaluate(p.FunctionDefinition);
208-
return instance.Call(p.Name, ArgumentSet.Empty);
208+
return instance.Call(p.Name, ArgumentSet.Empty(expr, this));
209209
}
210210

211211

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ private IMember CreateSpecificTypeFromIndex(IGenericType gt, IReadOnlyList<IMemb
8888

8989
// For other types just use supplied arguments
9090
if (indices.Count > 0) {
91-
return gt.CreateSpecificType(new ArgumentSet(indices));
91+
return gt.CreateSpecificType(new ArgumentSet(indices, expr, this));
9292
}
9393
// TODO: report too few type arguments for the generic expression.
9494
return UnknownType;
@@ -132,7 +132,7 @@ private IMember CreateClassInstance(PythonClassType cls, IReadOnlyList<IMember>
132132

133133
var argSet = initOverload != null
134134
? new ArgumentSet(initFunc, 0, null, callExpr, this)
135-
: new ArgumentSet(constructorArguments);
135+
: new ArgumentSet(constructorArguments, callExpr, this);
136136

137137
argSet.Evaluate();
138138
var specificType = cls.CreateSpecificType(argSet);

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ private IMember GetValueFromUnaryOp(UnaryExpression expr, string op) {
4747
var fn = instance.GetPythonType()?.GetMember<IPythonFunctionType>(op);
4848
// Process functions declared in code modules. Scraped/compiled/stub modules do not actually perform any operations.
4949
if (fn?.DeclaringModule != null && (fn.DeclaringModule.ModuleType == ModuleType.User || fn.DeclaringModule.ModuleType == ModuleType.Library)) {
50-
var result = fn.Call(instance, op, ArgumentSet.Empty);
50+
var result = fn.Call(instance, op, ArgumentSet.Empty(expr, this));
5151
if (!result.IsUnknown()) {
5252
return result;
5353
}
@@ -132,9 +132,9 @@ private IMember GetValueFromBinaryOp(Expression expr) {
132132
if (op.IsComparison()) {
133133
// If the op is a comparison, and the thing on the left is the builtin,
134134
// flip the operation and call it instead.
135-
ret = CallOperator(op.InvertComparison(), right, rightType, left, leftType, tryRight: false);
135+
ret = CallOperator(op.InvertComparison(), right, rightType, left, leftType, expr, tryRight: false);
136136
} else {
137-
ret = CallOperator(op, left, leftType, right, rightType, tryLeft: false);
137+
ret = CallOperator(op, left, leftType, right, rightType, expr, tryLeft: false);
138138
}
139139

140140
if (!ret.IsUnknown()) {
@@ -146,7 +146,7 @@ private IMember GetValueFromBinaryOp(Expression expr) {
146146

147147
if (rightIsSupported) {
148148
// Try calling the function on the left side, otherwise just return right.
149-
var ret = CallOperator(op, left, leftType, right, rightType, tryRight: false);
149+
var ret = CallOperator(op, left, leftType, right, rightType, expr, tryRight: false);
150150

151151
if (!ret.IsUnknown()) {
152152
return ret;
@@ -155,13 +155,13 @@ private IMember GetValueFromBinaryOp(Expression expr) {
155155
return op.IsComparison() ? Interpreter.GetBuiltinType(BuiltinTypeId.Bool) : right;
156156
}
157157

158-
var callRet = CallOperator(op, left, leftType, right, rightType);
158+
var callRet = CallOperator(op, left, leftType, right, rightType, expr);
159159
if (!callRet.IsUnknown()) {
160160
return callRet;
161161
}
162162

163163
if (op.IsComparison()) {
164-
callRet = CallOperator(op.InvertComparison(), right, rightType, left, leftType);
164+
callRet = CallOperator(op.InvertComparison(), right, rightType, left, leftType, expr);
165165

166166
if (!callRet.IsUnknown()) {
167167
return callRet;
@@ -197,18 +197,18 @@ private IMember GetValueFromBinaryOp(Expression expr) {
197197
return left.IsUnknown() ? right : left;
198198
}
199199

200-
private IMember CallOperator(PythonOperator op, IMember left, IPythonType leftType, IMember right, IPythonType rightType, bool tryLeft = true, bool tryRight = true) {
200+
private IMember CallOperator(PythonOperator op, IMember left, IPythonType leftType, IMember right, IPythonType rightType, Expression expr, bool tryLeft = true, bool tryRight = true) {
201201
var (funcName, swappedFuncName) = OpMethodName(op);
202202

203203
if (tryLeft && funcName != null && left is IPythonInstance lpi) {
204-
var ret = leftType.Call(lpi, funcName, new ArgumentSet(new[] { right }));
204+
var ret = leftType.Call(lpi, funcName, new ArgumentSet(new[] { right }, expr, this));
205205
if (!ret.IsUnknown()) {
206206
return ret;
207207
}
208208
}
209209

210210
if (tryRight && swappedFuncName != null && right is IPythonInstance rpi) {
211-
var ret = rightType.Call(rpi, swappedFuncName, new ArgumentSet(new[] { left }));
211+
var ret = rightType.Call(rpi, swappedFuncName, new ArgumentSet(new[] { left }, expr, this));
212212
if (!ret.IsUnknown()) {
213213
return ret;
214214
}

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -271,7 +271,7 @@ private IMember GetValueFromMember(MemberExpression expr) {
271271
case IPythonClassType _:
272272
return value;
273273
case IPythonPropertyType prop:
274-
return prop.Call(instance, prop.Name, ArgumentSet.Empty);
274+
return prop.Call(instance, prop.Name, ArgumentSet.Empty(expr, this));
275275
case IPythonType p:
276276
return new PythonBoundType(p, instance);
277277
case null:

‎src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ private void HandleTypedVariable(IPythonType variableType, IMember value, Expres
116116
instance = value;
117117
}
118118
}
119-
instance = instance ?? variableType?.CreateInstance(variableType.Name, ArgumentSet.Empty) ?? Eval.UnknownType;
119+
instance = instance ?? variableType?.CreateInstance(variableType.Name, ArgumentSet.Empty(expr, Eval)) ?? Eval.UnknownType;
120120

121121
if (expr is NameExpression ne) {
122122
Eval.DeclareVariable(ne.Name, instance, VariableSource.Declaration, ne);

‎src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ internal sealed class FunctionEvaluator : MemberEvaluator {
3535

3636
public FunctionEvaluator(ExpressionEval eval, PythonFunctionOverload overload)
3737
: base(eval, overload.FunctionDefinition) {
38-
3938
_overload = overload;
4039
_function = overload.ClassMember ?? throw new NullReferenceException(nameof(overload.ClassMember));
4140
_self = _function.DeclaringType as PythonClassType;
@@ -78,7 +77,7 @@ private IPythonType TryDetermineReturnValue() {
7877
if (!annotationType.IsUnknown()) {
7978
// Annotations are typically types while actually functions return
8079
// instances unless specifically annotated to a type such as Type[T].
81-
var t = annotationType.CreateInstance(annotationType.Name, ArgumentSet.Empty);
80+
var t = annotationType.CreateInstance(annotationType.Name, ArgumentSet.WithoutContext);
8281
// If instance could not be created, such as when return type is List[T] and
8382
// type of T is not yet known, just use the type.
8483
var instance = t.IsUnknown() ? annotationType : t;

‎src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -132,16 +132,7 @@ private void SpecializeMembers() {
132132

133133
_members["Any"] = new AnyType(this);
134134

135-
// AnyStr
136-
var str = Interpreter.GetBuiltinType(BuiltinTypeId.Str);
137-
var bytes = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes);
138-
var unicode = Interpreter.GetBuiltinType(BuiltinTypeId.Unicode);
139-
var anyStrName = new PythonConstant("AnyStr", str);
140-
141-
var anyStrArgs = Interpreter.LanguageVersion.Is3x()
142-
? new IMember[] { anyStrName, str, bytes }
143-
: new IMember[] { anyStrName, str, unicode };
144-
_members["AnyStr"] = GenericTypeParameter.FromTypeVar(new ArgumentSet(anyStrArgs), this);
135+
_members["AnyStr"] = CreateAnyStr();
145136

146137
_members["Optional"] = new GenericType("Optional", CreateOptional, this);
147138
_members["Type"] = new GenericType("Type", CreateType, this);
@@ -310,6 +301,21 @@ private IPythonType CreateType(IReadOnlyList<IPythonType> typeArgs) {
310301
return Interpreter.UnknownType;
311302
}
312303

304+
private IPythonType CreateAnyStr() {
305+
var str = Interpreter.GetBuiltinType(BuiltinTypeId.Str);
306+
var bytes = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes);
307+
var unicode = Interpreter.GetBuiltinType(BuiltinTypeId.Unicode);
308+
var name = "AnyStr";
309+
310+
var constraints = Interpreter.LanguageVersion.Is3x()
311+
? new IPythonType[] { str, bytes }
312+
: new IPythonType[] { str, unicode };
313+
var docArgs = new[] { $"'{name}'" }.Concat(constraints.Select(c => c.Name));
314+
var documentation = CodeFormatter.FormatSequence("TypeVar", '(', docArgs);
315+
316+
return new GenericTypeParameter(name, this, constraints, documentation, default);
317+
}
318+
313319
private IPythonType CreateGenericClassParameter(IReadOnlyList<IPythonType> typeArgs) {
314320
// Handle Generic[_T1, _T2, ...]. _T1, et al are IGenericTypeParameter from TypeVar.
315321
// Hold the parameter until concrete type is provided at the time of the class instantiation.

‎src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingIterator.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ public override IMember Next {
3939
} else if (_index < _iteratorType.ItemTypes.Count) {
4040
itemType = _iteratorType.ItemTypes[_index++];
4141
}
42-
return itemType?.CreateInstance(itemType.Name, ArgumentSet.Empty) ?? UnknownType;
42+
return itemType?.CreateInstance(itemType.Name, ArgumentSet.WithoutContext) ?? UnknownType;
4343
}
4444
}
4545
}

‎src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingTuple.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,6 @@ public override IPythonIterator GetIterator() {
3333
}
3434

3535
public override IMember Index(object index)
36-
=> _collectionType.Index(this, index).GetPythonType().CreateInstance(null, ArgumentSet.Empty);
36+
=> _collectionType.Index(this, index).GetPythonType().CreateInstance(null, ArgumentSet.WithoutContext);
3737
}
3838
}

‎src/Analysis/Ast/Impl/Types/ArgumentSet.cs

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
using System.Collections.Generic;
1818
using System.IO;
1919
using System.Linq;
20+
using System.Runtime.CompilerServices;
2021
using Microsoft.Python.Analysis.Analyzer;
2122
using Microsoft.Python.Analysis.Analyzer.Evaluation;
2223
using Microsoft.Python.Analysis.Diagnostics;
@@ -38,7 +39,7 @@ internal sealed class ArgumentSet : IArgumentSet {
3839
private readonly DictArg _dictArgument;
3940
private bool _evaluated;
4041

41-
public static IArgumentSet Empty = new ArgumentSet();
42+
public static IArgumentSet WithoutContext = new ArgumentSet();
4243

4344
/// <summary>Module that declares the function</summary>
4445
public IPythonModule DeclaringModule { get; }
@@ -49,19 +50,38 @@ internal sealed class ArgumentSet : IArgumentSet {
4950
public int OverloadIndex { get; }
5051
public IExpressionEvaluator Eval { get; }
5152

53+
public Expression Expression { get; }
5254

5355
private ArgumentSet() { }
5456

55-
public ArgumentSet(IReadOnlyList<IPythonType> typeArgs) {
56-
_arguments = typeArgs.Select(t => new Argument(t)).ToList();
57-
_evaluated = true;
57+
/// <summary>
58+
/// Creates an empty argument set with some context in how the argument set was used.
59+
/// </summary>
60+
/// <param name="expr">Expression associated with argument set.</param>
61+
/// <param name="eval">Evaluator for the expression involving the argument set.</param>
62+
/// <returns></returns>
63+
public static ArgumentSet Empty(Expression expr, IExpressionEvaluator eval) {
64+
return new ArgumentSet(new List<IMember>(), expr, eval);
5865
}
66+
67+
/// <summary>
68+
/// Creates a set of arguments for a call
69+
///
70+
/// Use in the cases a corresponding function is unknown, but it is still convenient to have the context
71+
/// of the expression which the arguments are needed for and the evaluator that is analyzing
72+
/// that expression.
73+
///
74+
/// </summary>
75+
/// <param name="args">Arguments for the call.</param>
76+
/// <param name="expr">Expression for the call.</param>
77+
/// <param name="eval">Evaluator of the current analysis.</param>
78+
public ArgumentSet(IReadOnlyList<IMember> args, Expression expr, IExpressionEvaluator eval) {
79+
_arguments = args.Select(t => new Argument(t)).ToList();
80+
Expression = expr;
81+
Eval = eval;
5982

60-
public ArgumentSet(IReadOnlyList<IMember> memberArgs) {
61-
_arguments = memberArgs.Select(t => new Argument(t)).ToList();
6283
_evaluated = true;
6384
}
64-
6585
public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance instance, CallExpression callExpr, ExpressionEval eval) :
6686
this(fn, overloadIndex, instance, callExpr, eval.Module, eval) { }
6787

@@ -81,6 +101,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in
81101
Eval = eval;
82102
OverloadIndex = overloadIndex;
83103
DeclaringModule = fn.DeclaringModule;
104+
Expression = callExpr;
84105

85106
if (callExpr == null) {
86107
// Typically invoked by specialization code without call expression in the code.

‎src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,8 +138,13 @@ public interface IArgumentSet {
138138
int OverloadIndex { get; }
139139

140140
/// <summary>
141-
/// Evaluator associated with the set.
141+
/// Evaluator associated with the argument set.
142142
/// </summary>
143143
IExpressionEvaluator Eval { get; }
144+
145+
/// <summary>
146+
/// Expression associated with the argument set
147+
/// </summary>
148+
Expression Expression { get; }
144149
}
145150
}

‎src/Analysis/Ast/Impl/Types/PythonClassType.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -402,7 +402,7 @@ public IPythonType CreateSpecificType(IArgumentSet args) {
402402
.Where(p => !p.IsUnknown())
403403
.ToArray();
404404
if (st.Length > 0) {
405-
var type = gt.CreateSpecificType(new ArgumentSet(st));
405+
var type = gt.CreateSpecificType(new ArgumentSet(st, args.Expression, args.Eval));
406406
if (!type.IsUnknown()) {
407407
bases.Add(type);
408408
}

‎src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ private IMember CreateSpecificReturnFromClassType(IPythonClassType selfClassType
167167
}
168168

169169
if (typeArgs != null) {
170-
var specificReturnValue = returnClassType.CreateSpecificType(new ArgumentSet(typeArgs));
170+
var specificReturnValue = returnClassType.CreateSpecificType(new ArgumentSet(typeArgs, args?.Expression, args?.Eval));
171171
return new PythonInstance(specificReturnValue);
172172
}
173173

‎src/Analysis/Ast/Impl/Types/PythonPropertyType.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,10 @@ public PythonPropertyType(string name, Location location, IPythonType declaringT
4444
public string Description
4545
=> Type == null ? Resources.PropertyOfUnknownType : Resources.PropertyOfType.FormatUI(Type.Name);
4646
public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args)
47-
=> _getter.Call(args, instance?.GetPythonType() ?? DeclaringType);
47+
=> _getter?.Call(args, instance?.GetPythonType() ?? DeclaringType);
4848
#endregion
4949

5050
internal void AddOverload(IPythonFunctionOverload overload) => _getter = _getter ?? overload;
51-
private IPythonType Type => _getter?.Call(ArgumentSet.Empty, DeclaringType)?.GetPythonType();
51+
private IPythonType Type => _getter?.Call(ArgumentSet.WithoutContext, DeclaringType)?.GetPythonType();
5252
}
5353
}

‎src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ public IReadOnlyList<IPythonCollection> Items
6262
_contents.TryGetValue(key, out var value) ? value : UnknownType;
6363

6464
public override IPythonIterator GetIterator() =>
65-
Call(@"iterkeys", ArgumentSet.Empty) as IPythonIterator ?? new EmptyIterator(Type.DeclaringModule.Interpreter.UnknownType);
65+
Call(@"iterkeys", ArgumentSet.WithoutContext) as IPythonIterator ?? new EmptyIterator(Type.DeclaringModule.Interpreter.UnknownType);
6666

6767
public override IMember Index(object key) => key is IMember m ? this[m] : UnknownType;
6868

‎src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ public PythonInstanceIterator(IMember instance, IPythonInterpreter interpreter)
2929
__next__ = instance.GetPythonType().GetMember(@"__next__") as IPythonFunctionType;
3030
}
3131

32-
public IMember Next => __next__?.Call(null, @"__next__", ArgumentSet.Empty) ?? UnknownType;
32+
public IMember Next => __next__?.Call(null, @"__next__", ArgumentSet.WithoutContext) ?? UnknownType;
3333

3434
public override IMember Call(string memberName, IArgumentSet args) {
3535
// Specializations

‎src/Analysis/Ast/Impl/Values/PythonInstance.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ public virtual IMember Call(string memberName, IArgumentSet args) {
5656
public virtual IPythonIterator GetIterator() {
5757
var iteratorFunc = Type.GetMember(@"__iter__") as IPythonFunctionType;
5858
var o = iteratorFunc?.Overloads.FirstOrDefault();
59-
var instance = o?.Call(ArgumentSet.Empty, Type);
59+
var instance = o?.Call(ArgumentSet.WithoutContext, Type);
6060
if (instance != null) {
6161
return new PythonInstanceIterator(instance, Type.DeclaringModule.Interpreter);
6262
}

0 commit comments

Comments
 (0)