diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index 5e6aa6b49..1d5966bc0 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -9,9 +9,6 @@ in [Filing an issue](#filing-an-issue). There are a few known issues in the current version of the language server: -- Find references and rename functionality is not yet implemented. - - See [#699](https://github.com/Microsoft/python-language-server/issues/699), - [#577](https://github.com/Microsoft/python-language-server/issues/577). - Not all `__all__` statements can be handled. - Some modules may have an incorrect list of exported names. See [#620](https://github.com/Microsoft/python-language-server/issues/620), diff --git a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs index 836b45eab..c976ffdbd 100644 --- a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs @@ -64,6 +64,13 @@ public override bool Walk(ExpressionStatement node) { case Comprehension comp: Eval.ProcessComprehension(comp); return false; + case CallExpression callex when callex.Target is NameExpression nex && !string.IsNullOrEmpty(nex.Name): + Eval.LookupNameInScopes(nex.Name)?.AddReference(Eval.GetLocationOfName(nex)); + return false; + case CallExpression callex when callex.Target is MemberExpression mex && !string.IsNullOrEmpty(mex.Name): + var t = Eval.GetValueFromExpression(mex.Target)?.GetPythonType(); + t?.GetMember(mex.Name).AddReference(Eval.GetLocationOfName(mex)); + return false; default: return base.Walk(node); } diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs index 1c0c53c0c..8d4b9751d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs @@ -13,13 +13,16 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; - namespace Microsoft.Python.Analysis.Analyzer { /// /// Represents document that can be analyzed asynchronously. /// internal interface IAnalyzable { + /// + /// Notifies document that analysis is about to begin. + /// + void NotifyAnalysisBegins(); + /// /// Notifies document that its analysis is now complete. /// diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IExpressionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IExpressionEvaluator.cs index 285d9d156..724037bb1 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IExpressionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IExpressionEvaluator.cs @@ -15,7 +15,6 @@ using System; using System.Collections.Generic; -using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; @@ -56,18 +55,39 @@ public interface IExpressionEvaluator { /// /// Evaluates expression in the currently open scope. /// - IMember GetValueFromExpression(Expression expr); + IMember GetValueFromExpression(Expression expr, LookupOptions options = LookupOptions.Normal); - IMember LookupNameInScopes(string name, out IScope scope, LookupOptions options = LookupOptions.Normal); + IMember LookupNameInScopes(string name, out IScope scope, out IVariable v, LookupOptions options = LookupOptions.Normal); IPythonType GetTypeFromString(string typeString); + /// + /// Module AST. + /// PythonAst Ast { get; } + + /// + /// Associated module. + /// IPythonModule Module { get; } + + /// + /// Interpreter used in the module analysis. + /// IPythonInterpreter Interpreter { get; } + + /// + /// Application service container. + /// IServiceContainer Services { get; } void ReportDiagnostics(Uri documentUri, DiagnosticsEntry entry); + IEnumerable Diagnostics { get; } + + /// + /// Represents built-in 'unknown' type. . + /// + IPythonType UnknownType { get; } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs index 6d7f61419..16ccd5c94 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs @@ -59,5 +59,10 @@ public interface IPythonAnalyzer { /// Removes all the modules from the analysis, except Typeshed and builtin /// void ResetAnalyzer(); + + /// + /// Returns list of currently loaded modules. + /// + IReadOnlyList LoadedModules { get; } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/LookupOptions.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/LookupOptions.cs index 32956751e..4b327faa0 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/LookupOptions.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/LookupOptions.cs @@ -19,10 +19,10 @@ namespace Microsoft.Python.Analysis.Analyzer { [Flags] public enum LookupOptions { None = 0, - Local, - Nonlocal, - Global, - Builtins, + Local = 1, + Nonlocal = 2, + Global = 4, + Builtins = 8, Normal = Local | Nonlocal | Global | Builtins } } diff --git a/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs index 52c90e1d0..c5b97a433 100644 --- a/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs +++ b/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs @@ -86,7 +86,7 @@ public EmptyAnalysis(IServiceContainer services, IDocument document) { Document = document ?? throw new ArgumentNullException(nameof(document)); GlobalScope = new EmptyGlobalScope(document); - _emptyAst = _emptyAst ?? (_emptyAst = Parser.CreateParser(new StringReader(string.Empty), PythonLanguageVersion.None).ParseFile()); + _emptyAst = _emptyAst ?? (_emptyAst = Parser.CreateParser(new StringReader(string.Empty), PythonLanguageVersion.None).ParseFile(document.Uri)); ExpressionEvaluator = new ExpressionEval(services, document, Ast); } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs index 77e3eb020..1794a0309 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs @@ -32,6 +32,8 @@ public IMember GetValueFromCallable(CallExpression expr) { } var target = GetValueFromExpression(expr.Target); + target?.AddReference(GetLocationOfName(expr.Target)); + var result = GetValueFromGeneric(target, expr); if (result != null) { return result; @@ -58,7 +60,7 @@ public IMember GetValueFromCallable(CallExpression expr) { case IPythonType t: // Target is type (info), the call creates instance. // For example, 'x = C; y = x()' or 'x = C()' where C is class - value = new PythonInstance(t, GetLoc(expr)); + value = new PythonInstance(t); break; } @@ -74,9 +76,9 @@ public IMember GetValueFromLambda(LambdaExpression expr) { return null; } - var loc = GetLoc(expr); - var ft = new PythonFunctionType(expr.Function, Module, null, loc); - var overload = new PythonFunctionOverload(expr.Function, ft, Module, GetLoc(expr)); + var location = GetLocationOfName(expr.Function); + var ft = new PythonFunctionType(expr.Function, null, location); + var overload = new PythonFunctionOverload(expr.Function, ft, location); ft.AddOverload(overload); return ft; } @@ -93,7 +95,7 @@ public IMember GetValueFromClassCtor(IPythonClassType cls, CallExpression expr) } args = a.Evaluate(); } - return cls.CreateInstance(cls.Name, GetLoc(expr), args); + return cls.CreateInstance(cls.Name, args); } public IMember GetValueFromBound(IPythonBoundType t, CallExpression expr) { @@ -190,7 +192,7 @@ public IMember GetValueFromFunctionType(IPythonFunctionType fn, IPythonInstance // Try and evaluate with specific arguments. Note that it does not // make sense to evaluate stubs since they already should be annotated. - if (fn.DeclaringModule is IDocument doc && fd?.IsInAst(doc.GetAnyAst()) == true) { + if (fn.DeclaringModule is IDocument doc && fd?.Ast == doc.GetAnyAst()) { // Stubs are coming from another module. return TryEvaluateWithArguments(fn.DeclaringModule, fd, args); } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs index 78d0ecc8f..146ae5256 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs @@ -60,7 +60,7 @@ public IMember GetValueFromList(ListExpression expression) { var value = GetValueFromExpression(item) ?? UnknownType; contents.Add(value); } - return PythonCollectionType.CreateList(Module.Interpreter, GetLoc(expression), contents, exact: true); + return PythonCollectionType.CreateList(Module.Interpreter, contents); } public IMember GetValueFromDictionary(DictionaryExpression expression) { @@ -70,7 +70,7 @@ public IMember GetValueFromDictionary(DictionaryExpression expression) { var value = GetValueFromExpression(item.SliceStop) ?? UnknownType; contents[key] = value; } - return new PythonDictionary(Interpreter, GetLoc(expression), contents, exact: true); + return new PythonDictionary(Interpreter, contents); } private IMember GetValueFromTuple(TupleExpression expression) { @@ -79,7 +79,7 @@ private IMember GetValueFromTuple(TupleExpression expression) { var value = GetValueFromExpression(item) ?? UnknownType; contents.Add(value); } - return PythonCollectionType.CreateTuple(Module.Interpreter, GetLoc(expression), contents, exact: true); + return PythonCollectionType.CreateTuple(Module.Interpreter, contents); } public IMember GetValueFromSet(SetExpression expression) { @@ -88,7 +88,7 @@ public IMember GetValueFromSet(SetExpression expression) { var value = GetValueFromExpression(item) ?? UnknownType; contents.Add(value); } - return PythonCollectionType.CreateSet(Interpreter, GetLoc(expression), contents, exact: true); + return PythonCollectionType.CreateSet(Interpreter, contents); } public IMember GetValueFromGenerator(GeneratorExpression expression) { @@ -108,14 +108,14 @@ public IMember GetValueFromComprehension(Comprehension node) { switch (node) { case ListComprehension lc: var v1 = GetValueFromExpression(lc.Item) ?? UnknownType; - return PythonCollectionType.CreateList(Interpreter, GetLoc(lc), new[] { v1 }); + return PythonCollectionType.CreateList(Interpreter, new[] { v1 }); case SetComprehension sc: var v2 = GetValueFromExpression(sc.Item) ?? UnknownType; - return PythonCollectionType.CreateSet(Interpreter, GetLoc(sc), new[] { v2 }); + return PythonCollectionType.CreateSet(Interpreter, new[] { v2 }); case DictionaryComprehension dc: var k = GetValueFromExpression(dc.Key) ?? UnknownType; var v = GetValueFromExpression(dc.Value) ?? UnknownType; - return new PythonDictionary(new PythonDictionaryType(Interpreter), GetLoc(dc), new Dictionary { { k, v } }); + return new PythonDictionary(new PythonDictionaryType(Interpreter), new Dictionary { { k, v } }); } return UnknownType; @@ -138,20 +138,20 @@ internal void ProcessComprehension(Comprehension node) { if (value != null) { switch (cfor.Left) { case NameExpression nex when value is IPythonCollection coll: - DeclareVariable(nex.Name, coll.GetIterator().Next, VariableSource.Declaration, GetLoc(nex)); + DeclareVariable(nex.Name, coll.GetIterator().Next, VariableSource.Declaration, nex); break; case NameExpression nex: - DeclareVariable(nex.Name, UnknownType, VariableSource.Declaration, GetLoc(nex)); + DeclareVariable(nex.Name, UnknownType, VariableSource.Declaration, nex); break; case TupleExpression tex when value is IPythonDictionary dict && tex.Items.Count > 0: if (tex.Items[0] is NameExpression nx0 && !string.IsNullOrEmpty(nx0.Name)) { - DeclareVariable(nx0.Name, dict.Keys.FirstOrDefault() ?? UnknownType, VariableSource.Declaration, GetLoc(nx0)); + DeclareVariable(nx0.Name, dict.Keys.FirstOrDefault() ?? UnknownType, VariableSource.Declaration, nx0); } if (tex.Items.Count > 1 && tex.Items[1] is NameExpression nx1 && !string.IsNullOrEmpty(nx1.Name)) { - DeclareVariable(nx1.Name, dict.Values.FirstOrDefault() ?? UnknownType, VariableSource.Declaration, GetLoc(nx1)); + DeclareVariable(nx1.Name, dict.Values.FirstOrDefault() ?? UnknownType, VariableSource.Declaration, nx1); } foreach (var item in tex.Items.Skip(2).OfType().Where(x => !string.IsNullOrEmpty(x.Name))) { - DeclareVariable(item.Name, UnknownType, VariableSource.Declaration, GetLoc(item)); + DeclareVariable(item.Name, UnknownType, VariableSource.Declaration, item); } break; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Constants.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Constants.cs index f32ed745c..3bf7c2603 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Constants.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Constants.cs @@ -24,20 +24,19 @@ namespace Microsoft.Python.Analysis.Analyzer.Evaluation { internal sealed partial class ExpressionEval { public IPythonInstance GetConstantFromLiteral(Expression expr, LookupOptions options) { - var location = GetLoc(expr); if (expr is ConstantExpression ce) { switch (ce.Value) { case string s: - return new PythonUnicodeString(s, Interpreter, location); + return new PythonUnicodeString(s, Interpreter); case AsciiString b: - return new PythonAsciiString(b, Interpreter, location); + return new PythonAsciiString(b, Interpreter); case int integer: - return new PythonConstant(integer, Interpreter.GetBuiltinType(BuiltinTypeId.Int), location); + return new PythonConstant(integer, Interpreter.GetBuiltinType(BuiltinTypeId.Int)); } } var t = SuppressBuiltinLookup ? UnknownType : (GetTypeFromLiteral(expr) ?? UnknownType); - return new PythonInstance(t, location); + return new PythonInstance(t); } public IPythonType GetTypeFromLiteral(Expression expr) { diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs index b0f58e0f1..017314781 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs @@ -79,7 +79,7 @@ private IMember CreateSpecificTypeFromIndex(IGenericType gt, IReadOnlyList 0) { - return new GenericClassParameter(genericTypeArgs, Module, GetLoc(expr)); + return new GenericClassParameter(genericTypeArgs, Module); } else { // TODO: report too few type arguments for Generic[]. return UnknownType; @@ -88,7 +88,7 @@ private IMember CreateSpecificTypeFromIndex(IGenericType gt, IReadOnlyList 0) { - return gt.CreateSpecificType(new ArgumentSet(indices), Module, GetLoc(expr)); + return gt.CreateSpecificType(new ArgumentSet(indices)); } // TODO: report too few type arguments for the generic expression. return UnknownType; @@ -127,7 +127,6 @@ private IReadOnlyList EvaluateCallArgs(CallExpression expr) { private IMember CreateClassInstance(PythonClassType cls, IReadOnlyList constructorArguments, CallExpression callExpr) { // Look at the constructor arguments and create argument set // based on the __init__ definition. - var location = GetLoc(callExpr); var initFunc = cls.GetMember(@"__init__") as IPythonFunctionType; var initOverload = initFunc?.DeclaringType == cls ? initFunc.Overloads.FirstOrDefault() : null; @@ -136,8 +135,8 @@ private IMember CreateClassInstance(PythonClassType cls, IReadOnlyList : new ArgumentSet(constructorArguments); argSet.Evaluate(); - var specificType = cls.CreateSpecificType(argSet, Module, location); - return new PythonInstance(specificType, location); + var specificType = cls.CreateSpecificType(argSet); + return new PythonInstance(specificType); } private ScopeStatement GetScope(IMember m) { diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Hints.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Hints.cs index 74c1ef655..55d9e25b7 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Hints.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Hints.cs @@ -27,7 +27,7 @@ namespace Microsoft.Python.Analysis.Analyzer.Evaluation { /// internal sealed partial class ExpressionEval { public IPythonType GetTypeFromPepHint(Node node) { - var location = GetLoc(node); + var location = GetLocationInfo(node); var content = (Module as IDocument)?.Content; if (string.IsNullOrEmpty(content) || !location.EndLine.HasValue) { return null; diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs index e0e16e4b2..e000235ca 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs @@ -13,8 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; -using System.Collections.Generic; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Types.Collections; @@ -54,14 +52,16 @@ private IMember GetValueFromUnaryOp(UnaryExpression expr, string op) { } } return instance is IPythonConstant c && instance.TryGetConstant(out var value) - ? new PythonConstant(-value, c.Type, GetLoc(expr)) + ? new PythonConstant(-value, c.Type) : instance; } return UnknownType; } private IMember GetValueFromBinaryOp(Expression expr) { - if (expr is AndExpression) { + if (expr is AndExpression a) { + GetValueFromExpression(a.Left); + GetValueFromExpression(a.Right); return Interpreter.GetBuiltinType(BuiltinTypeId.Bool); } @@ -108,7 +108,6 @@ private IMember GetValueFromBinaryOp(Expression expr) { var left = GetValueFromExpression(binop.Left) ?? UnknownType; var right = GetValueFromExpression(binop.Right) ?? UnknownType; - var rightType = right.GetPythonType(); if (rightType?.TypeId == BuiltinTypeId.Float) { return right; @@ -131,7 +130,7 @@ private IMember GetValueFromBinaryOp(Expression expr) { && leftType?.TypeId == BuiltinTypeId.List && rightType?.TypeId == BuiltinTypeId.List && left is IPythonCollection lc && right is IPythonCollection rc) { - return PythonCollectionType.CreateConcatenatedList(Module.Interpreter, GetLoc(expr), lc, rc); + return PythonCollectionType.CreateConcatenatedList(Module.Interpreter, lc, rc); } return left.IsUnknown() ? right : left; diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs index c6c939838..a2060a301 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs @@ -15,13 +15,13 @@ using System; using System.Collections.Concurrent; -using System.Collections.Generic; using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Core.Disposables; +using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer.Evaluation { @@ -37,10 +37,20 @@ public T GetInScope(string name, IScope scope) where T : class, IMember public IMember GetInScope(string name) => GetInScope(name, CurrentScope); public T GetInScope(string name) where T : class, IMember => GetInScope(name, CurrentScope); - public void DeclareVariable(string name, IMember value, VariableSource source, Node expression) - => DeclareVariable(name, value, source, GetLoc(expression)); + public void DeclareVariable(string name, IMember value, VariableSource source) + => DeclareVariable(name, value, source, default(Location)); - public void DeclareVariable(string name, IMember value, VariableSource source, LocationInfo location, bool overwrite = false) { + public void DeclareVariable(string name, IMember value, VariableSource source, IPythonModule module) + => DeclareVariable(name, value, source, new Location(module, default)); + + public void DeclareVariable(string name, IMember value, VariableSource source, Node location, bool overwrite = false) + => DeclareVariable(name, value, source, GetLocationOfName(location), overwrite); + + public void DeclareVariable(string name, IMember value, VariableSource source, Location location, bool overwrite = false) { + if (source == VariableSource.Import && value is IVariable v) { + CurrentScope.LinkVariable(name, v, location); + return; + } var member = GetInScope(name); if (member != null) { if (!value.IsUnknown()) { @@ -51,52 +61,50 @@ public void DeclareVariable(string name, IMember value, VariableSource source, L } } - [DebuggerStepThrough] - public IMember LookupNameInScopes(string name, out IScope scope) => LookupNameInScopes(name, out scope, DefaultLookupOptions); - - [DebuggerStepThrough] - public IMember LookupNameInScopes(string name, LookupOptions options) => LookupNameInScopes(name, out _, options); - - public IMember LookupNameInScopes(string name, out IScope scope, LookupOptions options) { + public IMember LookupNameInScopes(string name, out IScope scope, out IVariable v, LookupOptions options) { scope = null; - if (options == LookupOptions.Normal) { - // Default mode, no skipping, do direct search - for (var s = CurrentScope; s != null ; s = (Scope)s.OuterScope) { - if (s.Variables.Contains(name)) { - scope = s; - break; + switch (options) { + case LookupOptions.Normal: + // Regular lookup: all scopes and builtins. + for (var s = CurrentScope; s != null; s = (Scope)s.OuterScope) { + if (s.Variables.Contains(name)) { + scope = s; + break; + } } - } - } else { - var scopes = new List(); - for (var s = CurrentScope; s != null; s = (Scope)s.OuterScope) { - scopes.Add(s); - } - - if (scopes.Count == 1) { - if (!options.HasFlag(LookupOptions.Local) && !options.HasFlag(LookupOptions.Global)) { - scopes.Clear(); + break; + case LookupOptions.Global: + case LookupOptions.Global | LookupOptions.Builtins: + // Global scope only. + if (GlobalScope.Variables.Contains(name)) { + scope = GlobalScope; } - } else if (scopes.Count >= 2) { - if (!options.HasFlag(LookupOptions.Nonlocal)) { - while (scopes.Count > 2) { - scopes.RemoveAt(1); + break; + case LookupOptions.Nonlocal: + case LookupOptions.Nonlocal | LookupOptions.Builtins: + // All scopes but current and global ones. + for (var s = CurrentScope.OuterScope as Scope; s != null && s != GlobalScope; s = (Scope)s.OuterScope) { + if (s.Variables.Contains(name)) { + scope = s; + break; } } - - if (!options.HasFlag(LookupOptions.Local)) { - scopes.RemoveAt(0); - } - - if (!options.HasFlag(LookupOptions.Global)) { - scopes.RemoveAt(scopes.Count - 1); + break; + case LookupOptions.Local: + case LookupOptions.Local | LookupOptions.Builtins: + // Just the current scope + if (CurrentScope.Variables.Contains(name)) { + scope = CurrentScope; } - } - scope = scopes.FirstOrDefault(s => s.Variables.Contains(name)); + break; + default: + Debug.Fail("Unsupported name lookup combination"); + break; } - var value = scope?.Variables[name].Value; + v = scope?.Variables[name]; + var value = v?.Value; if (value == null && options.HasFlag(LookupOptions.Builtins)) { var builtins = Interpreter.ModuleResolution.BuiltinsModule; value = Interpreter.ModuleResolution.BuiltinsModule.GetMember(name); diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index 3700f7316..c1464424a 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -16,7 +16,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.Linq; +using System.Reflection; using Microsoft.Python.Analysis.Analyzer.Symbols; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Modules; @@ -25,6 +25,7 @@ using Microsoft.Python.Core; using Microsoft.Python.Core.Disposables; using Microsoft.Python.Core.Logging; +using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer.Evaluation { @@ -32,10 +33,10 @@ namespace Microsoft.Python.Analysis.Analyzer.Evaluation { /// Helper class that provides methods for looking up variables /// and types in a chain of scopes during analysis. /// - internal sealed partial class ExpressionEval: IExpressionEvaluator { + internal sealed partial class ExpressionEval : IExpressionEvaluator { private readonly Stack _openScopes = new Stack(); private readonly object _lock = new object(); - private List _diagnostics = new List(); + private readonly List _diagnostics = new List(); public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAst ast) { Services = services ?? throw new ArgumentNullException(nameof(services)); @@ -44,21 +45,47 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs GlobalScope = new GlobalScope(module); CurrentScope = GlobalScope; - DefaultLookupOptions = LookupOptions.Normal; - + DefaultLocation = new Location(module); //Log = services.GetService(); } - public LookupOptions DefaultLookupOptions { get; set; } public GlobalScope GlobalScope { get; } public Scope CurrentScope { get; private set; } public bool SuppressBuiltinLookup => Module.ModuleType == ModuleType.Builtins; public ILogger Log { get; } public ModuleSymbolTable SymbolTable { get; } = new ModuleSymbolTable(); public IPythonType UnknownType => Interpreter.UnknownType; + public Location DefaultLocation { get; } + + public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(Module) ?? LocationInfo.Empty; + + public Location GetLocationOfName(Node node) { + if (node == null || (Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library)) { + return DefaultLocation; + } + + IndexSpan indexSpan; + switch (node) { + case MemberExpression mex: + indexSpan = mex.GetNameSpan().ToIndexSpan(mex.Ast); + break; + case ClassDefinition cd: + indexSpan = cd.NameExpression.IndexSpan; + break; + case FunctionDefinition fd: + indexSpan = fd.NameExpression.IndexSpan; + break; + case NameExpression nex: + indexSpan = nex.IndexSpan; + break; + default: + indexSpan = node.IndexSpan; + break; + } - public LocationInfo GetLoc(Node node) => node?.GetLocation(Module, Ast) ?? LocationInfo.Empty; - public LocationInfo GetLocOfName(Node node, NameExpression header) => node?.GetLocationOfName(header, Module, Ast) ?? LocationInfo.Empty; + Debug.Assert(indexSpan.ToSourceSpan(node.Ast).Start.Column < 500); + return new Location(Module, indexSpan); + } #region IExpressionEvaluator public PythonAst Ast { get; } @@ -67,7 +94,7 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs public IServiceContainer Services { get; } IScope IExpressionEvaluator.CurrentScope => CurrentScope; IGlobalScope IExpressionEvaluator.GlobalScope => GlobalScope; - public LocationInfo GetLocation(Node node) => node?.GetLocation(Module, Ast) ?? LocationInfo.Empty; + public LocationInfo GetLocation(Node node) => node?.GetLocation(Module) ?? LocationInfo.Empty; public IEnumerable Diagnostics => _diagnostics; public void ReportDiagnostics(Uri documentUri, DiagnosticsEntry entry) { @@ -79,12 +106,6 @@ public void ReportDiagnostics(Uri documentUri, DiagnosticsEntry entry) { } } - public void RemoveDiagnostics(DiagnosticSource source) - => _diagnostics = _diagnostics.Where(d => d.Source != source).ToList(); - - public IMember GetValueFromExpression(Expression expr) - => GetValueFromExpression(expr, DefaultLookupOptions); - public IDisposable OpenScope(IScope scope) { if (!(scope is Scope s)) { return Disposable.Empty; @@ -97,7 +118,7 @@ public IDisposable OpenScope(IScope scope) { public IDisposable OpenScope(IPythonModule module, ScopeStatement scope) => OpenScope(module, scope, out _); #endregion - public IMember GetValueFromExpression(Expression expr, LookupOptions options) { + public IMember GetValueFromExpression(Expression expr, LookupOptions options = LookupOptions.Normal) { if (expr == null) { return null; } @@ -169,21 +190,20 @@ public IMember GetValueFromExpression(Expression expr, LookupOptions options) { internal void ClearCache() => _scopeLookupCache.Clear(); - private IMember GetValueFromFormatSpecifier(FormatSpecifier formatSpecifier) { - return new PythonFString(formatSpecifier.Unparsed, Interpreter, GetLoc(formatSpecifier)); - } + private IMember GetValueFromFormatSpecifier(FormatSpecifier formatSpecifier) + => new PythonFString(formatSpecifier.Unparsed, Interpreter); - private IMember GetValueFromFString(FString fString) { - return new PythonFString(fString.Unparsed, Interpreter, GetLoc(fString)); - } + private IMember GetValueFromFString(FString fString) + => new PythonFString(fString.Unparsed, Interpreter); - private IMember GetValueFromName(NameExpression expr, LookupOptions options) { + private IMember GetValueFromName(NameExpression expr, LookupOptions options = LookupOptions.Normal) { if (expr == null || string.IsNullOrEmpty(expr.Name)) { return null; } - var member = LookupNameInScopes(expr.Name, options); + var member = LookupNameInScopes(expr.Name, out _, out var v, options); if (member != null) { + v?.AddReference(GetLocationOfName(expr)); switch (member.GetPythonType()) { case IPythonClassType cls: SymbolTable.Evaluate(cls.ClassDefinition); @@ -214,6 +234,7 @@ private IMember GetValueFromMember(MemberExpression expr) { // If container is class/type info rather than the instance, then the method is an unbound function. // Example: C.f where f is a method of C. Compare to C().f where f is bound to the instance of C. if (member is PythonFunctionType f && !f.IsStatic && !f.IsClassMethod) { + f.AddReference(GetLocationOfName(expr)); return f.ToUnbound(); } instance = new PythonInstance(typeInfo); @@ -221,7 +242,9 @@ private IMember GetValueFromMember(MemberExpression expr) { instance = instance ?? m as IPythonInstance; var type = m?.GetPythonType(); // Try inner type + var value = type?.GetMember(expr.Name); + type?.AddMemberReference(expr.Name, this, GetLocationOfName(expr)); if (type is IPythonModule) { return value; @@ -241,7 +264,7 @@ private IMember GetValueFromMember(MemberExpression expr) { case IPythonPropertyType prop: return prop.Call(instance, prop.Name, ArgumentSet.Empty); case IPythonType p: - return new PythonBoundType(p, instance, GetLoc(expr)); + return new PythonBoundType(p, instance); case null: Log?.Log(TraceEventType.Verbose, $"Unknown member {expr.ToCodeString(Ast).Trim()}"); return UnknownType; diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/TypeAnnotationConverter.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/TypeAnnotationConverter.cs index 9cca0ed4f..80f52ae7c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/TypeAnnotationConverter.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/TypeAnnotationConverter.cs @@ -51,7 +51,7 @@ public override IPythonType GetTypeMember(IPythonType baseType, string member) public override IPythonType MakeGeneric(IPythonType baseType, IReadOnlyList args) { if (baseType is IGenericType gt) { - return gt.CreateSpecificType(args, _eval.Module, LocationInfo.Empty); + return gt.CreateSpecificType(args); } if(baseType is IPythonClassType cls && cls.IsGeneric()) { // Type is not yet known for generic classes. Resolution is delayed diff --git a/src/Analysis/Ast/Impl/Analyzer/Expressions/ExpressionFinder.cs b/src/Analysis/Ast/Impl/Analyzer/Expressions/ExpressionFinder.cs index 7ab19c754..c574c40ef 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Expressions/ExpressionFinder.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Expressions/ExpressionFinder.cs @@ -29,23 +29,16 @@ public ExpressionFinder(PythonAst ast, FindExpressionOptions options) { public ExpressionFinder(string expression, PythonLanguageVersion version, FindExpressionOptions options) { var parser = Parser.CreateParser(new StringReader(expression), version, ParserOptions.Default); - Ast = parser.ParseTopExpression(); + Ast = parser.ParseTopExpression(null); Ast.Body.SetLoc(0, expression.Length); Options = options; } - public static Node GetNode(PythonAst ast, SourceLocation location, FindExpressionOptions options) { - var finder = new ExpressionFinder(ast, options); - return finder.GetExpression(location); - } - public PythonAst Ast { get; } public FindExpressionOptions Options { get; } public Node GetExpression(int index) => GetExpression(index, index); public Node GetExpression(SourceLocation location) => GetExpression(new SourceSpan(location, location)); - public SourceSpan? GetExpressionSpan(int index) => GetExpression(index, index)?.GetSpan(Ast); - public SourceSpan? GetExpressionSpan(SourceLocation location) => GetExpression(new SourceSpan(location, location))?.GetSpan(Ast); public void Get(int startIndex, int endIndex, out Node node, out Node statement, out ScopeStatement scope) { ExpressionWalker walker; @@ -81,9 +74,6 @@ public Node GetExpression(SourceSpan range) { return GetExpression(startIndex, endIndex); } - public SourceSpan? GetExpressionSpan(int startIndex, int endIndex) => GetExpression(startIndex, endIndex)?.GetSpan(Ast); - public SourceSpan? GetExpressionSpan(SourceSpan range) => GetExpression(range)?.GetSpan(Ast); - private abstract class ExpressionWalker : PythonWalkerWithLocation { protected ExpressionWalker(int location) : base(location) { } public Node Expression { get; protected set; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs index 15068b59f..1626e8c5d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs @@ -44,10 +44,10 @@ public void HandleAssignment(AssignmentStatement node) { value = Eval.UnknownType; } - if (node.Left.FirstOrDefault() is TupleExpression lhs) { + if (node.Left.FirstOrDefault() is SequenceExpression seq) { // Tuple = Tuple. Transfer values. - var texHandler = new TupleExpressionHandler(Walker); - texHandler.HandleTupleAssignment(lhs, node.Right, value); + var seqHandler = new SequenceExpressionHandler(Walker); + seqHandler.HandleAssignment(seq.Items, node.Right, value); return; } @@ -60,26 +60,18 @@ public void HandleAssignment(AssignmentStatement node) { foreach (var ne in node.Left.OfType()) { if (Eval.CurrentScope.NonLocals[ne.Name] != null) { Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Nonlocal); - if (scope != null) { - scope.Variables[ne.Name].Assign(value, Eval.GetLoc(ne)); - } else { - // TODO: report variable is not declared in outer scopes. - } + scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne)); continue; } if (Eval.CurrentScope.Globals[ne.Name] != null) { Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Global); - if (scope != null) { - scope.Variables[ne.Name].Assign(value, Eval.GetLoc(ne)); - } else { - // TODO: report variable is not declared in global scope. - } + scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne)); continue; } var source = value.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration; - Eval.DeclareVariable(ne.Name, value ?? Module.Interpreter.UnknownType, source, Eval.GetLoc(ne)); + Eval.DeclareVariable(ne.Name, value ?? Module.Interpreter.UnknownType, source, Eval.GetLocationOfName(ne)); } TryHandleClassVariable(node, value); @@ -100,12 +92,12 @@ public void HandleAnnotatedExpression(ExpressionWithAnnotation expr, IMember val private void TryHandleClassVariable(AssignmentStatement node, IMember value) { var mex = node.Left.OfType().FirstOrDefault(); - if (!string.IsNullOrEmpty(mex?.Name) && mex?.Target is NameExpression nex && nex.Name.EqualsOrdinal("self")) { + if (!string.IsNullOrEmpty(mex?.Name) && mex.Target is NameExpression nex && nex.Name.EqualsOrdinal("self")) { var m = Eval.LookupNameInScopes(nex.Name, out var scope, LookupOptions.Local); var cls = m.GetPythonType(); if (cls != null) { using (Eval.OpenScope(Eval.Module, cls.ClassDefinition, out _)) { - Eval.DeclareVariable(mex.Name, value, VariableSource.Declaration, Eval.GetLoc(node), true); + Eval.DeclareVariable(mex.Name, value, VariableSource.Declaration, Eval.GetLocationOfName(mex), true); } } } @@ -124,10 +116,10 @@ private void HandleTypedVariable(IPythonType variableType, IMember value, Expres instance = value; } } - instance = instance ?? variableType?.CreateInstance(variableType.Name, Eval.GetLoc(expr), ArgumentSet.Empty) ?? Eval.UnknownType; + instance = instance ?? variableType?.CreateInstance(variableType.Name, ArgumentSet.Empty) ?? Eval.UnknownType; if (expr is NameExpression ne) { - Eval.DeclareVariable(ne.Name, instance, VariableSource.Declaration, expr); + Eval.DeclareVariable(ne.Name, instance, VariableSource.Declaration, ne); return; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs index 2578087eb..dd89f0bea 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs @@ -40,9 +40,8 @@ public bool HandleFromImport(FromImportStatement node) { } } - var location = Eval.GetLoc(node.Root); var imports = ModuleResolution.CurrentPathResolver.FindImports(Module.FilePath, node); - if (HandleImportSearchResult(imports, null, null, location, out var variableModule)) { + if (HandleImportSearchResult(imports, null, null, node.Root, out var variableModule)) { AssignVariables(node, imports, variableModule); } return false; @@ -67,10 +66,11 @@ private void AssignVariables(FromImportStatement node, IImportSearchResult impor for (var i = 0; i < names.Count; i++) { var memberName = names[i].Name; if (!string.IsNullOrEmpty(memberName)) { - var variableName = asNames[i]?.Name ?? memberName; - var value = variableModule.GetMember(memberName) ?? GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName); - - Eval.DeclareVariable(variableName, value, VariableSource.Import, names[i]); + var nameExpression = asNames[i] ?? names[i]; + var variableName = nameExpression?.Name ?? memberName; + var exported = variableModule.Analysis?.GlobalScope.Variables[memberName] ?? variableModule.GetMember(memberName); + var value = exported ?? GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName); + Eval.DeclareVariable(variableName, value, VariableSource.Import, nameExpression); } } } @@ -94,7 +94,8 @@ private void HandleModuleImportStar(PythonVariableModule variableModule) { ModuleResolution.GetOrLoadModule(m.Name); } - Eval.DeclareVariable(memberName, member, VariableSource.Import, variableModule.Location); + var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName]; + Eval.DeclareVariable(memberName, variable ?? member, VariableSource.Import); } } @@ -121,8 +122,8 @@ private void SpecializeFuture(FromImportStatement node) { var printNameExpression = node.Names.FirstOrDefault(n => n?.Name == "print_function"); if (printNameExpression != null) { - var fn = new PythonFunctionType("print", Module, null, string.Empty, LocationInfo.Empty); - var o = new PythonFunctionOverload(fn.Name, Module, _ => LocationInfo.Empty); + var fn = new PythonFunctionType("print", new Location(Module, default), null, string.Empty); + var o = new PythonFunctionOverload(fn.Name, new Location(Module, default)); var parameters = new List { new ParameterInfo("*values", Interpreter.GetBuiltinType(BuiltinTypeId.Object), ParameterKind.List, null), new ParameterInfo("sep", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.KeywordOnly, null), diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs index 66a82f076..f5fe93800 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs @@ -50,8 +50,6 @@ public bool HandleImport(ImportStatement node) { } private void HandleImport(ModuleName moduleImportExpression, NameExpression asNameExpression, bool forceAbsolute) { - var location = Eval.GetLoc(moduleImportExpression); - // "import fob.oar.baz" means // import_module('fob') // import_module('fob.oar') @@ -61,22 +59,22 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa foreach (var nameExpression in moduleImportExpression.Names) { importNames = importNames.Add(nameExpression.Name); var imports = ModuleResolution.CurrentPathResolver.GetImportsFromAbsoluteName(Module.FilePath, importNames, forceAbsolute); - if (!HandleImportSearchResult(imports, variableModule, asNameExpression, location, out variableModule)) { + if (!HandleImportSearchResult(imports, variableModule, asNameExpression, moduleImportExpression, out variableModule)) { return; } } // "import fob.oar.baz as baz" is handled as baz = import_module('fob.oar.baz') // "import fob.oar.baz" is handled as fob = import_module('fob') - if (asNameExpression != null) { + if (!string.IsNullOrEmpty(asNameExpression?.Name)) { Eval.DeclareVariable(asNameExpression.Name, variableModule, VariableSource.Import, asNameExpression); - } else if (importNames.Count > 0 && _variableModules.TryGetValue(importNames[0], out variableModule)) { + } else if (importNames.Count > 0 && !string.IsNullOrEmpty(importNames[0]) && _variableModules.TryGetValue(importNames[0], out variableModule)) { var firstName = moduleImportExpression.Names[0]; Eval.DeclareVariable(importNames[0], variableModule, VariableSource.Import, firstName); } } - private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonVariableModule parent, in NameExpression asNameExpression, in LocationInfo location, out PythonVariableModule variableModule) { + private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonVariableModule parent, in NameExpression asNameExpression, in Node location, out PythonVariableModule variableModule) { switch (imports) { case ModuleImport moduleImport when Module.ModuleType == ModuleType.Stub && moduleImport.FullName == Module.Name: return TryGetModuleFromSelf(parent, moduleImport.Name, out variableModule); @@ -88,7 +86,8 @@ private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonV return TryGetPackageFromImport(packageImport, parent, out variableModule); case RelativeImportBeyondTopLevel importBeyondTopLevel: var message = Resources.ErrorRelativeImportBeyondTopLevel.FormatInvariant(importBeyondTopLevel.RelativeImportName); - Eval.ReportDiagnostics(Eval.Module.Uri, new DiagnosticsEntry(message, location.Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis)); + Eval.ReportDiagnostics(Eval.Module.Uri, + new DiagnosticsEntry(message, location.GetLocation(Eval.Module).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis)); variableModule = default; return false; case ImportNotFound importNotFound: @@ -107,7 +106,7 @@ private bool TryGetModuleFromSelf(in PythonVariableModule parent, in string memb return true; } - private bool TryGetModuleFromImport(in ModuleImport moduleImport, in PythonVariableModule parent, LocationInfo location, out PythonVariableModule variableModule) { + private bool TryGetModuleFromImport(in ModuleImport moduleImport, in PythonVariableModule parent, Node location, out PythonVariableModule variableModule) { var module = ModuleResolution.GetOrLoadModule(moduleImport.FullName); if (module != null) { variableModule = GetOrCreateVariableModule(module, parent, moduleImport.Name); @@ -119,7 +118,7 @@ private bool TryGetModuleFromImport(in ModuleImport moduleImport, in PythonVaria return false; } - private bool TryGetModulePossibleImport(PossibleModuleImport possibleModuleImport, PythonVariableModule parent, LocationInfo location, out PythonVariableModule variableModule) { + private bool TryGetModulePossibleImport(PossibleModuleImport possibleModuleImport, PythonVariableModule parent, Node location, out PythonVariableModule variableModule) { if (_variableModules.TryGetValue(possibleModuleImport.PossibleModuleFullName, out variableModule)) { return true; } @@ -161,11 +160,13 @@ private bool TryGetPackageFromImport(in ImplicitPackageImport implicitPackageImp return true; } - private void MakeUnresolvedImport(string variableName, string moduleName, LocationInfo location) { + private void MakeUnresolvedImport(string variableName, string moduleName, Node location) { if (!string.IsNullOrEmpty(variableName)) { Eval.DeclareVariable(variableName, new SentinelModule(moduleName, Eval.Services), VariableSource.Import, location); } - Eval.ReportDiagnostics(Eval.Module.Uri, new DiagnosticsEntry(Resources.ErrorUnresolvedImport.FormatInvariant(moduleName), location.Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis)); + Eval.ReportDiagnostics(Eval.Module.Uri, + new DiagnosticsEntry(Resources.ErrorUnresolvedImport.FormatInvariant(moduleName), + Eval.GetLocationInfo(location).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis)); } private PythonVariableModule GetOrCreateVariableModule(in string fullName, in PythonVariableModule parentModule, in string memberName) { diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopHandler.cs index 2e359fd4b..018d1e558 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopHandler.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Linq; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Parsing.Ast; @@ -29,14 +28,14 @@ public bool HandleFor(ForStatement node) { case NameExpression nex: // for x in y: if (!string.IsNullOrEmpty(nex.Name)) { - Eval.DeclareVariable(nex.Name, value, VariableSource.Declaration, Eval.GetLoc(nex)); + Eval.DeclareVariable(nex.Name, value, VariableSource.Declaration, nex); } break; - case TupleExpression tex: + case SequenceExpression seq: // x = [('abc', 42, True), ('abc', 23, False)] // for some_str, (some_int, some_bool) in x: - var h = new TupleExpressionHandler(Walker); - h.HandleTupleAssignment(tex, node.List, value); + var h = new SequenceExpressionHandler(Walker); + h.HandleAssignment(seq.Items, node.List, value); break; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/NonLocalHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/NonLocalHandler.cs index 9d75c0852..e14d96981 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/NonLocalHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/NonLocalHandler.cs @@ -21,14 +21,24 @@ public NonLocalHandler(AnalysisWalker walker) : base(walker) { } public bool HandleNonLocal(NonlocalStatement node) { foreach (var nex in node.Names) { - Eval.CurrentScope.DeclareNonLocal(nex.Name, Eval.GetLoc(nex)); + var m = Eval.LookupNameInScopes(nex.Name, out _, out var v, LookupOptions.Nonlocal); + if (m != null) { + var location = Eval.GetLocationOfName(nex); + Eval.CurrentScope.DeclareNonLocal(nex.Name, location); + v?.AddReference(location); + } } return false; } public bool HandleGlobal(GlobalStatement node) { foreach (var nex in node.Names) { - Eval.CurrentScope.DeclareGlobal(nex.Name, Eval.GetLoc(nex)); + var m = Eval.LookupNameInScopes(nex.Name, out _, out var v, LookupOptions.Global); + if (m != null) { + var location = Eval.GetLocationOfName(nex); + Eval.CurrentScope.DeclareGlobal(nex.Name, location); + v?.AddReference(location); + } } return false; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/SequenceExpressionHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/SequenceExpressionHandler.cs new file mode 100644 index 000000000..26ed365ee --- /dev/null +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/SequenceExpressionHandler.cs @@ -0,0 +1,136 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Analyzer.Handlers { + internal sealed class SequenceExpressionHandler : StatementHandler { + public SequenceExpressionHandler(AnalysisWalker walker) : base(walker) { } + + public void HandleAssignment(IEnumerable lhs, Expression rhs, IMember value) { + if (rhs is TupleExpression tex) { + Assign(lhs, tex, Eval); + } else { + Assign(lhs, value, Eval); + } + } + + internal static void Assign(IEnumerable lhs, TupleExpression rhs, ExpressionEval eval) { + var names = NamesFromSequenceExpression(lhs).ToArray(); + var values = ValuesFromSequenceExpression(rhs.Items, eval).ToArray(); + for (var i = 0; i < names.Length; i++) { + IMember value = null; + if (values.Length > 0) { + value = i < values.Length ? values[i] : values[values.Length - 1]; + } + + if (!string.IsNullOrEmpty(names[i]?.Name)) { + eval.DeclareVariable(names[i].Name, value ?? eval.UnknownType, VariableSource.Declaration, names[i]); + } + } + } + + internal static void Assign(IEnumerable lhs, IMember value, ExpressionEval eval) { + // Tuple = 'tuple value' (such as from callable). Transfer values. + IMember[] values; + if (value is IPythonCollection seq) { + values = seq.Contents.ToArray(); + } else { + values = new[] { value }; + } + + var typeEnum = new ValueEnumerator(values, eval.UnknownType); + Assign(lhs, typeEnum, eval); + } + + private static void Assign(IEnumerable items, ValueEnumerator valueEnum, ExpressionEval eval) { + foreach (var item in items) { + switch (item) { + case StarredExpression stx when stx.Expression is NameExpression nex && !string.IsNullOrEmpty(nex.Name): + eval.DeclareVariable(nex.Name, valueEnum.Next, VariableSource.Declaration, nex); + break; + case NameExpression nex when !string.IsNullOrEmpty(nex.Name): + eval.DeclareVariable(nex.Name, valueEnum.Next, VariableSource.Declaration, nex); + break; + case TupleExpression te: + Assign(te.Items, valueEnum, eval); + break; + } + } + } + + private static IEnumerable NamesFromSequenceExpression(IEnumerable items) { + var names = new List(); + foreach (var item in items) { + var expr = item.RemoveParenthesis(); + switch (expr) { + case SequenceExpression seq: + names.AddRange(NamesFromSequenceExpression(seq.Items)); + break; + case NameExpression nex: + names.Add(nex); + break; + } + } + return names; + } + + private static IEnumerable ValuesFromSequenceExpression(IEnumerable items, ExpressionEval eval) { + var members = new List(); + foreach (var item in items) { + var value = eval.GetValueFromExpression(item); + switch (value) { + case IPythonCollection coll: + members.AddRange(coll.Contents); + break; + default: + members.Add(value); + break; + } + } + return members; + } + + private class ValueEnumerator { + private readonly IMember[] _values; + private readonly IMember _filler; + private int _index; + + public ValueEnumerator(IMember[] values, IMember filler) { + _values = values; + _filler = filler; + } + + public IMember Next { + get { + IMember t; + if (_values.Length > 0) { + t = _index < _values.Length ? _values[_index] : _values[_values.Length - 1]; + } else { + t = _filler; + } + + _index++; + return t; + } + } + } + } +} diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/TupleExpressionHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/TupleExpressionHandler.cs deleted file mode 100644 index 124cac1bb..000000000 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/TupleExpressionHandler.cs +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Linq; -using Microsoft.Python.Analysis.Analyzer.Evaluation; -using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Parsing.Ast; - -namespace Microsoft.Python.Analysis.Analyzer.Handlers { - internal sealed class TupleExpressionHandler : StatementHandler { - public TupleExpressionHandler(AnalysisWalker walker) : base(walker) { } - - public void HandleTupleAssignment(TupleExpression lhs, Expression rhs, IMember value) { - if (rhs is TupleExpression tex) { - AssignTuple(lhs, tex, Eval); - } else { - AssignTuple(lhs, value, Eval); - } - } - - internal static void AssignTuple(TupleExpression lhs, TupleExpression rhs, ExpressionEval eval) { - var returnedExpressions = rhs.Items.ToArray(); - var names = lhs.Items.OfType().Select(x => x.Name).ToArray(); - for (var i = 0; i < names.Length; i++) { - Expression e = null; - if (returnedExpressions.Length > 0) { - e = i < returnedExpressions.Length ? returnedExpressions[i] : returnedExpressions[returnedExpressions.Length - 1]; - } - - if (e != null && !string.IsNullOrEmpty(names[i])) { - var v = eval.GetValueFromExpression(e); - eval.DeclareVariable(names[i], v ?? eval.UnknownType, VariableSource.Declaration, e); - } - } - } - - internal static void AssignTuple(TupleExpression lhs, IMember value, ExpressionEval eval) { - // Tuple = 'tuple value' (such as from callable). Transfer values. - IMember[] values; - if (value is IPythonCollection seq) { - values = seq.Contents.ToArray(); - } else { - values = new[] { value }; - } - - var typeEnum = new ValueEnumerator(values, eval.UnknownType); - AssignTuple(lhs, typeEnum, eval); - } - - private static void AssignTuple(TupleExpression tex, ValueEnumerator valueEnum, ExpressionEval eval) { - foreach (var item in tex.Items) { - switch (item) { - case NameExpression nex when !string.IsNullOrEmpty(nex.Name): - eval.DeclareVariable(nex.Name, valueEnum.Next, VariableSource.Declaration, nex); - break; - case TupleExpression te: - AssignTuple(te, valueEnum, eval); - break; - } - } - } - - private class ValueEnumerator { - private readonly IMember[] _values; - private readonly IMember _filler; - private int _index; - - public ValueEnumerator(IMember[] values, IMember filler) { - _values = values; - _filler = filler; - } - - public IMember Next { - get { - IMember t; - if (_values.Length > 0) { - t = _index < _values.Length ? _values[_index] : _values[_values.Length - 1]; - } else { - t = _filler; - } - - _index++; - return t; - } - } - } - } -} diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/WithHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/WithHandler.cs index 3027026ef..3687ee345 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/WithHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/WithHandler.cs @@ -39,7 +39,7 @@ public void HandleWith(WithStatement node) { } if (item.Variable is NameExpression nex && !string.IsNullOrEmpty(nex.Name)) { - Eval.DeclareVariable(nex.Name, context, VariableSource.Declaration, Eval.GetLoc(item)); + Eval.DeclareVariable(nex.Name, context, VariableSource.Declaration, item); } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 09f53e905..03852fbe6 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; @@ -110,7 +109,7 @@ private void HandleAllAppendExtend(CallExpression node) { switch (me.Name) { case "append": - values = PythonCollectionType.CreateList(Module.Interpreter, Eval.GetLoc(arg), new List() { v }, exact: true); + values = PythonCollectionType.CreateList(Module.Interpreter, new List { v }, exact: true); break; case "extend": values = v as IPythonCollection; @@ -125,20 +124,17 @@ private void HandleAllAppendExtend(CallExpression node) { ExtendAll(node, values); } - private void ExtendAll(Node declNode, IPythonCollection values) { + private void ExtendAll(Node location, IPythonCollection values) { Eval.LookupNameInScopes(AllVariableName, out var scope, LookupOptions.Global); if (scope == null) { return; } - var loc = Eval.GetLoc(declNode); - var all = scope.Variables[AllVariableName]?.Value as IPythonCollection; - - var list = PythonCollectionType.CreateConcatenatedList(Module.Interpreter, loc, all, values); + var list = PythonCollectionType.CreateConcatenatedList(Module.Interpreter, all, values); var source = list.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration; - Eval.DeclareVariable(AllVariableName, list, source, loc); + Eval.DeclareVariable(AllVariableName, list, source, location); } private bool IsHandleableAll(Node node) { @@ -270,7 +266,7 @@ private void MergeStub() { sourceType.TransferDocumentationAndLocation(stubType); // TODO: choose best type between the scrape and the stub. Stub probably should always win. var source = Eval.CurrentScope.Variables[v.Name]?.Source ?? VariableSource.Declaration; - Eval.DeclareVariable(v.Name, v.Value, source, LocationInfo.Empty, overwrite: true); + Eval.DeclareVariable(v.Name, v.Value, source, Module); } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 30eadf0a7..100831e6b 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Dependencies; @@ -168,12 +169,13 @@ public IReadOnlyList LintModule(IPythonModule module) { return Array.Empty(); } - var optionsProvider = _services.GetService(); - if (optionsProvider?.Options?.LintingEnabled == false) { - return Array.Empty(); - } + // Linter always runs no matter of the option since it looks up variables + // which also enumerates and updates variable references for find all + // references and rename operations. + var result = new LinterAggregator().Lint(module.Analysis, _services); - return new LinterAggregator().Lint(module.Analysis, _services); + var optionsProvider = _services.GetService(); + return optionsProvider?.Options?.LintingEnabled == false ? Array.Empty() : result; } public void ResetAnalyzer() { @@ -188,6 +190,9 @@ public void ResetAnalyzer() { } } + public IReadOnlyList LoadedModules + => _analysisEntries.Values.ExcludeDefault().Select(v => v.Module).ExcludeDefault().ToArray(); + private void AnalyzeDocument(AnalysisModuleKey key, PythonAnalyzerEntry entry, ImmutableArray dependencies, CancellationToken cancellationToken) { _analysisCompleteEvent.Reset(); _log?.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name}({entry.Module.ModuleType}) queued"); diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 3658e4755..11effd123 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -312,21 +312,24 @@ private void Analyze(PythonAnalyzerEntry entry, int version, CancellationToken c private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, PythonAst ast, int version, CancellationToken cancellationToken) { // Now run the analysis. - var walker = new ModuleWalker(_services, module, ast); + var analyzable = module as IAnalyzable; + analyzable?.NotifyAnalysisBegins(); + var walker = new ModuleWalker(_services, module, ast); ast.Walk(walker); + cancellationToken.ThrowIfCancellationRequested(); walker.Complete(); cancellationToken.ThrowIfCancellationRequested(); var analysis = new DocumentAnalysis((IDocument)module, version, walker.GlobalScope, walker.Eval, walker.ExportedMemberNames); - (module as IAnalyzable)?.NotifyAnalysisComplete(analysis); + analyzable?.NotifyAnalysisComplete(analysis); entry.TrySetAnalysis(analysis, version); if (module.ModuleType == ModuleType.User) { var linterDiagnostics = _analyzer.LintModule(module); - _diagnosticsService.Replace(entry.Module.Uri, linterDiagnostics, DiagnosticSource.Linter); + _diagnosticsService?.Replace(entry.Module.Uri, linterDiagnostics, DiagnosticSource.Linter); } } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs b/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs index 4864de8ec..e7cd7c3f0 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs @@ -50,9 +50,9 @@ private async Task LoadBuiltinTypesAsync(string root, IServiceManager sm, Cancel lock (_lock) { var builtinModule = _moduleResolution.CreateBuiltinsModule(); _builtinTypes[BuiltinTypeId.NoneType] - = new PythonType("NoneType", builtinModule, string.Empty, LocationInfo.Empty, BuiltinTypeId.NoneType); + = new PythonType("NoneType", new Location(builtinModule, default), string.Empty, BuiltinTypeId.NoneType); _builtinTypes[BuiltinTypeId.Unknown] - = UnknownType = new PythonType("Unknown", builtinModule, string.Empty, LocationInfo.Empty); + = UnknownType = new PythonType("Unknown", new Location(builtinModule, default), string.Empty); } await _moduleResolution.InitializeAsync(cancellationToken); diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs index e2ce3991f..37eed92a4 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs @@ -56,13 +56,15 @@ public void EvaluateClass() { // We cheat slightly and treat base classes as annotations. var b = Eval.GetTypeFromAnnotation(a.Expression); if (b != null) { - bases.Add(b.GetPythonType()); + var t = b.GetPythonType(); + bases.Add(t); + t.AddReference(Eval.GetLocationOfName(a.Expression)); } } _class.SetBases(bases); // Declare __class__ variable in the scope. - Eval.DeclareVariable("__class__", _class, VariableSource.Declaration, _classDef); + Eval.DeclareVariable("__class__", _class, VariableSource.Declaration); ProcessClassBody(); } diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index 6f5945a87..871e9acd0 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -56,7 +56,7 @@ public override void Evaluate() { if (!annotationType.IsUnknown()) { // Annotations are typically types while actually functions return // instances unless specifically annotated to a type such as Type[T]. - var instance = annotationType.CreateInstance(annotationType.Name, Eval.GetLoc(FunctionDefinition), ArgumentSet.Empty); + var instance = annotationType.CreateInstance(annotationType.Name, ArgumentSet.Empty); _overload.SetReturnValue(instance, true); } else { // Check if function is a generator @@ -114,10 +114,11 @@ public override bool Walk(ReturnStatement node) { // Classes and functions are walked by their respective evaluators public override bool Walk(ClassDefinition node) => false; public override bool Walk(FunctionDefinition node) { - // Inner function, declare as variable. + // Inner function, declare as variable. Do not set variable location + // since it is not an assignment visible to the user. var m = SymbolTable.Evaluate(node); if (m != null) { - Eval.DeclareVariable(node.NameExpression.Name, m, VariableSource.Declaration, Eval.GetLoc(node)); + Eval.DeclareVariable(node.NameExpression.Name, m, VariableSource.Declaration, node.NameExpression); } return false; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/MemberEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/MemberEvaluator.cs index 43c16f114..b2f63639c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/MemberEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/MemberEvaluator.cs @@ -27,8 +27,6 @@ protected MemberEvaluator(ExpressionEval eval, ScopeStatement target) : base(eva } public ScopeStatement Target { get; } - public bool IsClass => Target is ClassDefinition; - public bool IsFunction => Target is FunctionDefinition; public IMember Result { get; protected set; } public abstract void Evaluate(); } diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index 8affdced8..ee527720e 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -48,7 +48,9 @@ private SymbolCollector(ModuleSymbolTable table, ExpressionEval eval) { public override bool Walk(ClassDefinition cd) { if (!string.IsNullOrEmpty(cd.NameExpression?.Name)) { var classInfo = CreateClass(cd); - _eval.DeclareVariable(cd.Name, classInfo, VariableSource.Declaration, GetLoc(cd)); + // The variable is transient (non-user declared) hence it does not have location. + // Class type is tracking locations for references and renaming. + _eval.DeclareVariable(cd.Name, classInfo, VariableSource.Declaration); _table.Add(new ClassEvaluator(_eval, cd)); // Open class scope _scopes.Push(_eval.OpenScope(_eval.Module, cd, out _)); @@ -79,51 +81,50 @@ public override void PostWalk(FunctionDefinition fd) { base.PostWalk(fd); } - private PythonClassType CreateClass(ClassDefinition node) { - var cls = new PythonClassType(node, _eval.Module, GetLoc(node), + private PythonClassType CreateClass(ClassDefinition cd) { + var cls = new PythonClassType(cd, _eval.GetLocationOfName(cd), _eval.SuppressBuiltinLookup ? BuiltinTypeId.Unknown : BuiltinTypeId.Type); - _typeMap[node] = cls; + _typeMap[cd] = cls; return cls; } private void AddFunctionOrProperty(FunctionDefinition fd) { var declaringType = fd.Parent != null && _typeMap.TryGetValue(fd.Parent, out var t) ? t : null; - var loc = GetLoc(fd); - if (!TryAddProperty(fd, declaringType, loc)) { - AddFunction(fd, declaringType, loc); + if (!TryAddProperty(fd, declaringType)) { + AddFunction(fd, declaringType); } } - private IMember AddFunction(FunctionDefinition node, IPythonType declaringType, LocationInfo loc) { - if (!(_eval.LookupNameInScopes(node.Name, LookupOptions.Local) is PythonFunctionType existing)) { - existing = new PythonFunctionType(node, _eval.Module, declaringType, loc); - _eval.DeclareVariable(node.Name, existing, VariableSource.Declaration, loc); + private void AddFunction(FunctionDefinition fd, IPythonType declaringType) { + if (!(_eval.LookupNameInScopes(fd.Name, LookupOptions.Local) is PythonFunctionType existing)) { + existing = new PythonFunctionType(fd, declaringType, _eval.GetLocationOfName(fd)); + // The variable is transient (non-user declared) hence it does not have location. + // Function type is tracking locations for references and renaming. + _eval.DeclareVariable(fd.Name, existing, VariableSource.Declaration); } - AddOverload(node, existing, o => existing.AddOverload(o)); - return existing; + AddOverload(fd, existing, o => existing.AddOverload(o)); } - private void AddOverload(FunctionDefinition node, IPythonClassMember function, Action addOverload) { + private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Action addOverload) { // Check if function exists in stubs. If so, take overload from stub // and the documentation from this actual module. - if (!_table.ReplacedByStubs.Contains(node)) { - var stubOverload = GetOverloadFromStub(node); + if (!_table.ReplacedByStubs.Contains(fd)) { + var stubOverload = GetOverloadFromStub(fd); if (stubOverload != null) { - if (!string.IsNullOrEmpty(node.GetDocumentation())) { - stubOverload.SetDocumentationProvider(_ => node.GetDocumentation()); + if (!string.IsNullOrEmpty(fd.GetDocumentation())) { + stubOverload.SetDocumentationProvider(_ => fd.GetDocumentation()); } addOverload(stubOverload); - _table.ReplacedByStubs.Add(node); + _table.ReplacedByStubs.Add(fd); return; } } - if (!_table.Contains(node)) { + if (!_table.Contains(fd)) { // Do not evaluate parameter types just yet. During light-weight top-level information // collection types cannot be determined as imports haven't been processed. - var location = _eval.GetLocOfName(node, node.NameExpression); - var overload = new PythonFunctionOverload(node, function, _eval.Module, location); + var overload = new PythonFunctionOverload(fd, function, _eval.GetLocationOfName(fd)); addOverload(overload); _table.Add(new FunctionEvaluator(_eval, overload)); } @@ -139,44 +140,33 @@ private PythonFunctionOverload GetOverloadFromStub(FunctionDefinition node) { return null; } - private bool TryAddProperty(FunctionDefinition node, IPythonType declaringType, LocationInfo location) { + private bool TryAddProperty(FunctionDefinition node, IPythonType declaringType) { var dec = node.Decorators?.Decorators; var decorators = dec != null ? dec.ExcludeDefault().ToArray() : Array.Empty(); foreach (var d in decorators.OfType()) { switch (d.Name) { case @"property": - AddProperty(node, _eval.Module, declaringType, false, location); + AddProperty(node, declaringType, false); return true; case @"abstractproperty": - AddProperty(node, _eval.Module, declaringType, true, location); + AddProperty(node, declaringType, true); return true; } } return false; } - private PythonPropertyType AddProperty(FunctionDefinition node, IPythonModule declaringModule, IPythonType declaringType, bool isAbstract, LocationInfo loc) { + private void AddProperty(FunctionDefinition node, IPythonType declaringType, bool isAbstract) { if (!(_eval.LookupNameInScopes(node.Name, LookupOptions.Local) is PythonPropertyType existing)) { - existing = new PythonPropertyType(node, declaringModule, declaringType, isAbstract, loc); - _eval.DeclareVariable(node.Name, existing, VariableSource.Declaration, loc); + existing = new PythonPropertyType(node, _eval.GetLocationOfName(node), declaringType, isAbstract); + // The variable is transient (non-user declared) hence it does not have location. + // Property type is tracking locations for references and renaming. + _eval.DeclareVariable(node.Name, existing, VariableSource.Declaration); } AddOverload(node, existing, o => existing.AddOverload(o)); - return existing; } - private LocationInfo GetLoc(ClassDefinition node) { - if (node == null || node.StartIndex >= node.EndIndex) { - return null; - } - - var start = node.NameExpression?.GetStart(_eval.Ast) ?? node.GetStart(_eval.Ast); - var end = node.GetEnd(_eval.Ast); - return new LocationInfo(_eval.Module.FilePath, _eval.Module.Uri, start.Line, start.Column, end.Line, end.Column); - } - - private LocationInfo GetLoc(Node node) => _eval.GetLoc(node); - private IMember GetMemberFromStub(string name) { if (_eval.Module.Stub == null) { return null; diff --git a/src/LanguageServer/Impl/Implementation/Server.Rename.cs b/src/Analysis/Ast/Impl/Definitions/IReferenceCollection.cs similarity index 65% rename from src/LanguageServer/Impl/Implementation/Server.Rename.cs rename to src/Analysis/Ast/Impl/Definitions/IReferenceCollection.cs index ddc8db6f7..de1c46302 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Rename.cs +++ b/src/Analysis/Ast/Impl/Definitions/IReferenceCollection.cs @@ -13,14 +13,12 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Python.LanguageServer.Protocol; +using System.Collections.Generic; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing.Ast; -namespace Microsoft.Python.LanguageServer.Implementation { - public sealed partial class Server { - public async Task Rename(RenameParams @params, CancellationToken cancellationToken) { - return new WorkspaceEdit(); - } +namespace Microsoft.Python.Analysis { + public interface IReferenceCollection { + IReadOnlyList GetReferences(IPythonType type); } } diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs index f053b05a0..84cab4711 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs @@ -13,9 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Core.Collections; namespace Microsoft.Python.Analysis.Dependencies { diff --git a/src/Analysis/Ast/Impl/Diagnostics/DiagnosticsEntry.cs b/src/Analysis/Ast/Impl/Diagnostics/DiagnosticsEntry.cs index c7f3e7c00..25b174e69 100644 --- a/src/Analysis/Ast/Impl/Diagnostics/DiagnosticsEntry.cs +++ b/src/Analysis/Ast/Impl/Diagnostics/DiagnosticsEntry.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing; diff --git a/src/Analysis/Ast/Impl/Diagnostics/ErrorCodes.cs b/src/Analysis/Ast/Impl/Diagnostics/ErrorCodes.cs index dcebb7356..45c6a1407 100644 --- a/src/Analysis/Ast/Impl/Diagnostics/ErrorCodes.cs +++ b/src/Analysis/Ast/Impl/Diagnostics/ErrorCodes.cs @@ -23,5 +23,7 @@ public static class ErrorCodes { public const string ParameterMissing = "parameter-missing"; public const string UnresolvedImport = "unresolved-import"; public const string UndefinedVariable = "undefined-variable"; + public const string VariableNotDefinedGlobally= "variable-not-defined-globally"; + public const string VariableNotDefinedNonLocal = "variable-not-defined-nonlocal"; } } diff --git a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs index 74173a3ae..8e53962f2 100644 --- a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs @@ -22,7 +22,7 @@ namespace Microsoft.Python.Analysis { public static class AnalysisExtensions { public static IScope FindScope(this IDocumentAnalysis analysis, SourceLocation location) - => analysis.GlobalScope.FindScope(analysis.Ast, location); + => analysis.GlobalScope.FindScope(analysis.Document, location); /// /// Provides ability to specialize function return type manually. @@ -54,8 +54,8 @@ private static PythonFunctionType GetOrCreateFunction(this IDocumentAnalysis ana // 'type()' in code is a function call, not a type class instantiation. if (!(analysis.GlobalScope.Variables[name]?.Value is PythonFunctionType f)) { f = PythonFunctionType.ForSpecialization(name, analysis.Document); - f.AddOverload(new PythonFunctionOverload(name, analysis.Document, _ => LocationInfo.Empty)); - analysis.GlobalScope.DeclareVariable(name, f, VariableSource.Declaration, LocationInfo.Empty); + f.AddOverload(new PythonFunctionOverload(name, new Location(analysis.Document))); + analysis.GlobalScope.DeclareVariable(name, f, VariableSource.Declaration); } return f; } diff --git a/src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs b/src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs index a3af57273..bd2cdf0d4 100644 --- a/src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs @@ -52,19 +52,19 @@ internal static void DeclareParametersInScope(this IArgumentSet args, Expression foreach (var a in args.Arguments) { if (a.Value is IMember m && !string.IsNullOrEmpty(a.Name)) { - eval.DeclareVariable(a.Name, m, VariableSource.Declaration, a.Location, false); + eval.DeclareVariable(a.Name, m, VariableSource.Declaration, a.Location); } } if (args.ListArgument != null && !string.IsNullOrEmpty(args.ListArgument.Name)) { var type = new PythonCollectionType(null, BuiltinTypeId.List, eval.Interpreter, false); - var list = new PythonCollection(type, LocationInfo.Empty, args.ListArgument.Values); - eval.DeclareVariable(args.ListArgument.Name, list, VariableSource.Declaration, args.ListArgument.Location, false); + var list = new PythonCollection(type, args.ListArgument.Values); + eval.DeclareVariable(args.ListArgument.Name, list, VariableSource.Declaration, args.ListArgument.Location); } if (args.DictionaryArgument != null) { foreach (var kvp in args.DictionaryArgument.Arguments) { - eval.DeclareVariable(kvp.Key, kvp.Value, VariableSource.Declaration, args.DictionaryArgument.Location, false); + eval.DeclareVariable(kvp.Key, kvp.Value, VariableSource.Declaration, args.DictionaryArgument.Location); } } } diff --git a/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs b/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs index 578535cbd..c4a220000 100644 --- a/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs @@ -67,12 +67,19 @@ public static bool IsInsideString(this PythonAst ast, SourceLocation location) { public static bool IsInParameter(this FunctionDefinition fd, PythonAst tree, SourceLocation location) { var index = tree.LocationToIndex(location); - if (index < fd.StartIndex - || (fd.Body != null && index >= fd.Body.StartIndex) - || (fd.NameExpression != null && index > fd.NameExpression.EndIndex)) { - // Not within the def line + if (index < fd.StartIndex) { + return false; // before the node + } + + if (fd.Body != null && index >= fd.Body.StartIndex) { + return false; // in the body of the function + } + + if (fd.NameExpression != null && index < fd.NameExpression.EndIndex) { + // before the name end return false; } + return fd.Parameters.Any(p => { var paramName = p.GetVerbatimImage(tree) ?? p.Name; return index >= p.StartIndex && index <= p.StartIndex + paramName.Length; diff --git a/src/LanguageServer/Impl/Implementation/Server.FindReferences.cs b/src/Analysis/Ast/Impl/Extensions/EvaluatorExtensions.cs similarity index 52% rename from src/LanguageServer/Impl/Implementation/Server.FindReferences.cs rename to src/Analysis/Ast/Impl/Extensions/EvaluatorExtensions.cs index d31d25d38..31a092efb 100644 --- a/src/LanguageServer/Impl/Implementation/Server.FindReferences.cs +++ b/src/Analysis/Ast/Impl/Extensions/EvaluatorExtensions.cs @@ -13,20 +13,15 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; -using System.Diagnostics; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Python.LanguageServer.Protocol; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; -namespace Microsoft.Python.LanguageServer.Implementation { - public sealed partial class Server { - public async Task FindReferences(ReferencesParams @params, CancellationToken cancellationToken) { - - var uri = @params.textDocument.uri; - _log?.Log(TraceEventType.Verbose, $"References in {uri} at {@params.position}"); - - return Array.Empty(); - } +namespace Microsoft.Python.Analysis { + public static class EvaluatorExtensions { + public static IMember LookupNameInScopes(this IExpressionEvaluator eval, string name, out IScope scope, LookupOptions options = LookupOptions.Normal) + => eval.LookupNameInScopes(name, out scope, out _, options); + public static IMember LookupNameInScopes(this IExpressionEvaluator eval, string name, LookupOptions options = LookupOptions.Normal) + => eval.LookupNameInScopes(name, out _, out _, options); } } diff --git a/src/Analysis/Ast/Impl/Extensions/MemberExtensions.cs b/src/Analysis/Ast/Impl/Extensions/MemberExtensions.cs index 962c6f8e4..e5b1563db 100644 --- a/src/Analysis/Ast/Impl/Extensions/MemberExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/MemberExtensions.cs @@ -13,6 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Diagnostics; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; @@ -41,6 +42,7 @@ public static IPythonType GetPythonType(this IMember m) { case IPythonInstance pi: return pi.Type; case IVariable v when v.Value != null: + Debug.Assert(!(v.Value is IVariable)); return v.Value.GetPythonType(); } return null; @@ -71,5 +73,18 @@ public static bool TryGetConstant(this IMember m, out T value) { value = default; return false; } + + public static void AddReference(this IMember m, Location location) + => (m as ILocatedMember)?.AddReference(location); + + public static string GetName(this IMember m) { + switch (m) { + case IVariable v: + return v.Name; + case IPythonType t: + return t.Name; + } + return null; + } } } diff --git a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs index 281c38744..4610d32ec 100644 --- a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs @@ -13,53 +13,19 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis { public static class NodeExtensions { - public static LocationInfo GetLocation(this Node node, IPythonModule module, PythonAst ast = null) { + public static LocationInfo GetLocation(this Node node, IPythonModule module) { if (node == null || node.StartIndex >= node.EndIndex) { return LocationInfo.Empty; } - ast = ast ?? (module as IDocument)?.GetAnyAst(); - if (ast != null) { - var start = node.GetStart(ast); - var end = node.GetEnd(ast); - return new LocationInfo(module.FilePath, module.Uri, start.Line, start.Column, end.Line, end.Column); - } - - return LocationInfo.Empty; - } - - public static LocationInfo GetLocationOfName(this Node node, NameExpression header, IPythonModule module, PythonAst ast = null) { - if (header == null) { - return LocationInfo.Empty; - } - - var loc = node.GetLocation(module, ast); - if (!loc.Equals(LocationInfo.Empty)) { - ast = ast ?? (module as IDocument)?.GetAnyAst(); - if (ast != null) { - var nameStart = header.GetStart(ast); - if (!nameStart.IsValid) { - return loc; - } - if (nameStart.Line > loc.StartLine || (nameStart.Line == loc.StartLine && nameStart.Column > loc.StartColumn)) { - return new LocationInfo(loc.FilePath, loc.DocumentUri, nameStart.Line, nameStart.Column, loc.EndLine, loc.EndColumn); - } - } - } - return LocationInfo.Empty; - } - - public static bool IsInAst(this ScopeStatement node, PythonAst ast) { - while (node.Parent != null) { - node = node.Parent; - } - return ast == node; + var start = node.GetStart(); + var end = node.GetEnd(); + return new LocationInfo(module.FilePath, module.Uri, start.Line, start.Column, end.Line, end.Column); } public static Expression RemoveParenthesis(this Expression e) { diff --git a/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs index 124807467..1ab34be54 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using System.Linq; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; @@ -21,5 +22,17 @@ namespace Microsoft.Python.Analysis { public static class PythonClassExtensions { public static bool IsGeneric(this IPythonClassType cls) => cls.Bases != null && cls.Bases.Any(b => b is IGenericType || b is IGenericClassParameter); + + public static void AddMemberReference(this IPythonType type, string name, IExpressionEvaluator eval, Location location) { + var m = type.GetMember(name); + if (m is LocatedMember lm) { + lm.AddReference(location); + } else if(type is IPythonClassType cls) { + using (eval.OpenScope(cls.DeclaringModule, cls.ClassDefinition)) { + eval.LookupNameInScopes(name, out _, out var v, LookupOptions.Local); + v?.AddReference(location); + } + } + } } } diff --git a/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs index fedc05ad0..185f549bf 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs @@ -34,7 +34,7 @@ public static void TransferDocumentationAndLocation(this IPythonType s, IPythonT if (!string.IsNullOrEmpty(documentation)) { dst.SetDocumentation(documentation); } - dst.SetLocation(src.Location); + dst.Location = src.Location; } } diff --git a/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs index 3e42839d6..70b88f287 100644 --- a/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs @@ -13,15 +13,13 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { public static class ScopeExtensions { - public static bool IsClassScope(this IScope scope) => scope.Node is ClassDefinition; - public static bool IsFunctionScope(this IScope scope) => scope.Node is FunctionDefinition; - public static int GetBodyStartIndex(this IScope scope, PythonAst ast) { switch (scope.Node) { case ClassDefinition cd: @@ -29,20 +27,20 @@ public static int GetBodyStartIndex(this IScope scope, PythonAst ast) { case FunctionDefinition fd: return fd.HeaderIndex; default: - return ast.LocationToIndex(scope.Node.GetStart(ast)); + return ast.LocationToIndex(scope.Node.GetStart()); } } - public static IScope FindScope(this IScope parent, PythonAst ast, SourceLocation location) { + public static IScope FindScope(this IScope parent, IDocument document, SourceLocation location) { var children = parent.Children; + var ast = document.Analysis.Ast; var index = ast.LocationToIndex(location); IScope candidate = null; for (var i = 0; i < children.Count; ++i) { if (children[i].Node is FunctionDefinition fd && fd.IsInParameter(ast, location)) { // In parameter name scope, so consider the function scope. - candidate = children[i]; - continue; + return children[i]; } var start = children[i].GetBodyStartIndex(ast); @@ -69,15 +67,17 @@ public static IScope FindScope(this IScope parent, PythonAst ast, SourceLocation return parent; } - var scopeIndent = GetParentScopeIndent(candidate, ast); - if (location.Column <= scopeIndent) { + var scopeIndent = GetParentScopeIndent(candidate, document.Analysis.Ast); + var indent = GetLineIndent(document, index, out var lineIsEmpty); + indent = lineIsEmpty ? location.Column : indent; // Take into account virtual space. + if (indent <= scopeIndent) { // Candidate is at deeper indentation than location and the // candidate is scoped, so return the parent instead. return parent; } // Recurse to check children of candidate scope - var child = FindScope(candidate, ast, location); + var child = FindScope(candidate, document, location); if (child.Node is FunctionDefinition fd1 && fd1.IsLambda && child.Node.EndIndex < index) { // Do not want to extend a lambda function's scope to the end of @@ -88,14 +88,32 @@ public static IScope FindScope(this IScope parent, PythonAst ast, SourceLocation return child; } + private static int GetLineIndent(IDocument document, int index, out bool lineIsEmpty) { + var content = document.Content; + lineIsEmpty = true; + if (!string.IsNullOrEmpty(content)) { + var i = index - 1; + for (; i >= 0 && content[i] != '\n' && content[i] != '\r'; i--) { } + var lineStart = i + 1; + for (i = lineStart; i < content.Length && content[i] != '\n' && content[i] != '\r'; i++) { + if (!char.IsWhiteSpace(content[i])) { + lineIsEmpty = false; + break; + } + } + return i - lineStart + 1; + } + return 1; + } + private static int GetParentScopeIndent(IScope scope, PythonAst ast) { switch (scope.Node) { case ClassDefinition cd: // Return column of "class" statement - return cd.GetStart(ast).Column; + return cd.GetStart().Column; case FunctionDefinition fd when !fd.IsLambda: // Return column of "def" statement - return fd.GetStart(ast).Column; + return fd.GetStart().Column; default: return -1; } diff --git a/src/Analysis/Ast/Impl/Linting/LinterWalker.cs b/src/Analysis/Ast/Impl/Linting/LinterWalker.cs index 5666dfb2d..242b5e51e 100644 --- a/src/Analysis/Ast/Impl/Linting/LinterWalker.cs +++ b/src/Analysis/Ast/Impl/Linting/LinterWalker.cs @@ -20,7 +20,7 @@ using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Linting { - internal abstract class LinterWalker: PythonWalker { + internal abstract class LinterWalker : PythonWalker { private readonly Stack _scopeStack = new Stack(); public IDocumentAnalysis Analysis { get; } @@ -32,16 +32,7 @@ protected LinterWalker(IDocumentAnalysis analysis, IServiceContainer services) { Services = services; } - public override bool Walk(ClassDefinition cd) { - _scopeStack.Push(Eval.OpenScope(Analysis.Document, cd)); - return true; - } - public override void PostWalk(ClassDefinition cd) => _scopeStack.Pop().Dispose(); - - public override bool Walk(FunctionDefinition fd) { - _scopeStack.Push(Eval.OpenScope(Analysis.Document, fd)); - return true; - } - public override void PostWalk(FunctionDefinition cd) => _scopeStack.Pop().Dispose(); + protected void OpenScope(ScopeStatement node) => _scopeStack.Push(Eval.OpenScope(Analysis.Document, node)); + protected void CloseScope() => _scopeStack.Pop().Dispose(); } } diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ComprehensionWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ComprehensionWalker.cs index bdabf2bbf..eed2e756c 100644 --- a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ComprehensionWalker.cs +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ComprehensionWalker.cs @@ -51,8 +51,8 @@ public override bool Walk(ForStatement node) { } public override bool Walk(DictionaryComprehension node) { - CollectNames(node); - var ew = new ExpressionWalker(_walker, _localNames, _localNameNodes); + var nc = CollectNames(node); + var ew = new ExpressionWalker(_walker, nc.Names, nc.NameExpressions); node.Key?.Walk(ew); node.Value?.Walk(ew); foreach (var iter in node.Iterators) { @@ -61,16 +61,17 @@ public override bool Walk(DictionaryComprehension node) { return true; } - private void CollectNames(Comprehension c) { + private NameCollectorWalker CollectNames(Comprehension c) { var nc = new NameCollectorWalker(_localNames, _localNameNodes); foreach (var cfor in c.Iterators.OfType()) { cfor.Left?.Walk(nc); } + return nc; } private void ProcessComprehension(Comprehension c, Node item, IEnumerable iterators) { - CollectNames(c); - var ew = new ExpressionWalker(_walker, _localNames, _localNameNodes); + var nc = CollectNames(c); + var ew = new ExpressionWalker(_walker, nc.Names, nc.NameExpressions); item?.Walk(ew); foreach (var iter in iterators) { iter.Walk(ew); diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ExpressionWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ExpressionWalker.cs index 311a2eb51..5e0e497aa 100644 --- a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ExpressionWalker.cs +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ExpressionWalker.cs @@ -24,7 +24,7 @@ namespace Microsoft.Python.Analysis.Linting.UndefinedVariables { internal sealed class ExpressionWalker : PythonWalker { private readonly UndefinedVariablesWalker _walker; private readonly HashSet _localNames; - private readonly HashSet _localNameNodes; + private readonly HashSet _localNameExpressions; public ExpressionWalker(UndefinedVariablesWalker walker) : this(walker, null, null) { } @@ -34,18 +34,11 @@ public ExpressionWalker(UndefinedVariablesWalker walker) /// /// Undefined variables walker. /// Locally defined names, such as variables in a comprehension. - /// Name nodes for local names. - public ExpressionWalker(UndefinedVariablesWalker walker, HashSet localNames, HashSet localNameNodes) { + /// Name nodes for local names. + public ExpressionWalker(UndefinedVariablesWalker walker, IEnumerable localNames, IEnumerable localNameExpressions) { _walker = walker; - _localNames = localNames ?? new HashSet(); - _localNameNodes = localNameNodes ?? new HashSet(); - } - - public override bool Walk(CallExpression node) { - foreach (var arg in node.Args) { - arg?.Expression?.Walk(this); - } - return false; + _localNames = new HashSet(localNames ?? Enumerable.Empty()); + _localNameExpressions = new HashSet(localNameExpressions ?? Enumerable.Empty()); } public override bool Walk(LambdaExpression node) { @@ -54,21 +47,22 @@ public override bool Walk(LambdaExpression node) { } public override bool Walk(ListComprehension node) { - node.Walk(new ComprehensionWalker(_walker, _localNames, _localNameNodes)); + node.Walk(new ComprehensionWalker(_walker, _localNames, _localNameExpressions)); return false; } public override bool Walk(SetComprehension node) { - node.Walk(new ComprehensionWalker(_walker, _localNames, _localNameNodes)); + node.Walk(new ComprehensionWalker(_walker, _localNames, _localNameExpressions)); return false; } + public override bool Walk(DictionaryComprehension node) { - node.Walk(new ComprehensionWalker(_walker, _localNames, _localNameNodes)); + node.Walk(new ComprehensionWalker(_walker, _localNames, _localNameExpressions)); return false; } public override bool Walk(GeneratorExpression node) { - node.Walk(new ComprehensionWalker(_walker, _localNames, _localNameNodes)); + node.Walk(new ComprehensionWalker(_walker, _localNames, _localNameExpressions)); return false; } @@ -76,24 +70,25 @@ public override bool Walk(NameExpression node) { if (_localNames?.Contains(node.Name) == true) { return false; } - if (_localNameNodes?.Contains(node) == true) { + if (_localNameExpressions?.Contains(node) == true) { return false; } var analysis = _walker.Analysis; - var m = analysis.ExpressionEvaluator.LookupNameInScopes(node.Name, out var scope); + var m = analysis.ExpressionEvaluator.LookupNameInScopes(node.Name, out _, out var v); if (m == null) { _walker.ReportUndefinedVariable(node); } + v?.AddReference(new Location(analysis.Document, node.IndexSpan)); + // Take into account where variable is defined so we do detect // undefined x in // y = x // x = 1 - var v = scope?.Variables[node.Name]; - if (v != null && v.Location.DocumentUri == analysis.Document.Uri) { + if (v?.Definition != null && v.Definition.DocumentUri == analysis.Document.Uri) { // Do not complain about functions and classes that appear later in the file if (!(v.Value is IPythonFunctionType || v.Value is IPythonClassType)) { - var span = v.Locations.First().Span; + var span = v.Definition.Span; var nodeLoc = node.GetLocation(analysis.Document); // Exclude same-name variables declared within the same statement // like 'e' that appears before its declaration in '[e in for e in {}]' diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/LambdaWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/LambdaWalker.cs index 18e833c6a..b501fbb2b 100644 --- a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/LambdaWalker.cs +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/LambdaWalker.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; using System.Linq; using Microsoft.Python.Core; using Microsoft.Python.Parsing.Ast; @@ -21,24 +20,23 @@ namespace Microsoft.Python.Analysis.Linting.UndefinedVariables { internal sealed class LambdaWalker : PythonWalker { private readonly UndefinedVariablesWalker _walker; - private readonly HashSet _names = new HashSet(); - private readonly HashSet _additionalNameNodes = new HashSet(); public LambdaWalker(UndefinedVariablesWalker walker) { _walker = walker; } public override bool Walk(FunctionDefinition node) { - CollectNames(node); - node.Body?.Walk(new ExpressionWalker(_walker, _names, _additionalNameNodes)); + var nc = CollectNames(node); + node.Body?.Walk(new ExpressionWalker(_walker, nc.Names, nc.NameExpressions)); return false; } - private void CollectNames(FunctionDefinition fd) { - var nc = new NameCollectorWalker(_names, _additionalNameNodes); + private NameCollectorWalker CollectNames(FunctionDefinition fd) { + var nc = new NameCollectorWalker(); foreach (var nex in fd.Parameters.Select(p => p.NameExpression).ExcludeDefault()) { nex.Walk(nc); } + return nc; } } } diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/NameCollectorWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/NameCollectorWalker.cs index 01fc065c8..2caeee528 100644 --- a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/NameCollectorWalker.cs +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/NameCollectorWalker.cs @@ -14,22 +14,29 @@ // permissions and limitations under the License. using System.Collections.Generic; +using System.Linq; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Linting.UndefinedVariables { internal sealed class NameCollectorWalker : PythonWalker { private readonly HashSet _names; - private readonly HashSet _additionalNameNodes; + private readonly HashSet _nameExpressions; - public NameCollectorWalker(HashSet names, HashSet additionalNameNodes) { - _names = names; - _additionalNameNodes = additionalNameNodes; + public NameCollectorWalker() + : this(Enumerable.Empty(), Enumerable.Empty()) { } + + public NameCollectorWalker(IEnumerable names, IEnumerable nameExpressions) { + _names = new HashSet(names); + _nameExpressions = new HashSet(nameExpressions); } + public IEnumerable Names => _names; + public IEnumerable NameExpressions => _nameExpressions; + public override bool Walk(NameExpression nex) { if (!string.IsNullOrEmpty(nex.Name)) { _names.Add(nex.Name); - _additionalNameNodes.Add(nex); + _nameExpressions.Add(nex); } return false; } diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs index 2e09af24b..7365bf08d 100644 --- a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs @@ -14,9 +14,11 @@ // permissions and limitations under the License. using System.Collections.Generic; +using System.Linq; using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Core; +using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; using ErrorCodes = Microsoft.Python.Analysis.Diagnostics.ErrorCodes; @@ -24,60 +26,92 @@ namespace Microsoft.Python.Analysis.Linting.UndefinedVariables { internal sealed class UndefinedVariablesWalker : LinterWalker { private readonly List _diagnostics = new List(); + private bool _suppressDiagnostics; public UndefinedVariablesWalker(IDocumentAnalysis analysis, IServiceContainer services) : base(analysis, services) { } public IReadOnlyList Diagnostics => _diagnostics; - public override bool Walk(AssignmentStatement node) { - if (node.Right is ErrorExpression) { - return false; + public override bool Walk(SuiteStatement node) { + foreach (var statement in node.Statements) { + switch (statement) { + case ClassDefinition cd: + HandleScope(cd); + break; + case FunctionDefinition fd: + HandleScope(fd); + break; + case GlobalStatement gs: + HandleGlobal(gs); + break; + case NonlocalStatement nls: + HandleNonLocal(nls); + break; + case AugmentedAssignStatement augs: + _suppressDiagnostics = true; + augs.Left?.Walk(new ExpressionWalker(this)); + _suppressDiagnostics = false; + augs.Right?.Walk(new ExpressionWalker(this)); + break; + case AssignmentStatement asst: + _suppressDiagnostics = true; + foreach (var lhs in asst.Left ?? Enumerable.Empty()) { + lhs?.Walk(new ExpressionWalker(this)); + } + _suppressDiagnostics = false; + asst.Right?.Walk(new ExpressionWalker(this)); + break; + default: + statement.Walk(new ExpressionWalker(this)); + break; + } } - node.Right?.Walk(new ExpressionWalker(this)); return false; } - public override bool Walk(CallExpression node) { - node.Target?.Walk(new ExpressionWalker(this)); - foreach (var arg in node.Args) { - arg?.Expression?.Walk(new ExpressionWalker(this)); - } - return false; + public void ReportUndefinedVariable(NameExpression node) { + var eval = Analysis.ExpressionEvaluator; + ReportUndefinedVariable(node.Name, eval.GetLocation(node).Span); } - public override bool Walk(IfStatement node) { - foreach (var test in node.Tests) { - test.Test.Walk(new ExpressionWalker(this)); + public void ReportUndefinedVariable(string name, SourceSpan span) { + if (!_suppressDiagnostics) { + _diagnostics.Add(new DiagnosticsEntry( + Resources.UndefinedVariable.FormatInvariant(name), + span, ErrorCodes.UndefinedVariable, Severity.Warning, DiagnosticSource.Linter)); } - return true; } - public override bool Walk(GlobalStatement node) { + private void HandleGlobal(GlobalStatement node) { foreach (var nex in node.Names) { var m = Eval.LookupNameInScopes(nex.Name, out _, LookupOptions.Global); if (m == null) { - ReportUndefinedVariable(nex); + _diagnostics.Add(new DiagnosticsEntry( + Resources.ErrorVariableNotDefinedGlobally.FormatInvariant(nex.Name), + Eval.GetLocation(nex).Span, ErrorCodes.VariableNotDefinedGlobally, Severity.Warning, DiagnosticSource.Linter)); } } - return false; } - public override bool Walk(NonlocalStatement node) { + private void HandleNonLocal(NonlocalStatement node) { foreach (var nex in node.Names) { var m = Eval.LookupNameInScopes(nex.Name, out _, LookupOptions.Nonlocal); if (m == null) { - ReportUndefinedVariable(nex); + _diagnostics.Add(new DiagnosticsEntry( + Resources.ErrorVariableNotDefinedNonLocal.FormatInvariant(nex.Name), + Eval.GetLocation(nex).Span, ErrorCodes.VariableNotDefinedNonLocal, Severity.Warning, DiagnosticSource.Linter)); } } - return false; } - public void ReportUndefinedVariable(NameExpression node) { - var eval = Analysis.ExpressionEvaluator; - _diagnostics.Add(new DiagnosticsEntry( - Resources.UndefinedVariable.FormatInvariant(node.Name), - eval.GetLocation(node).Span, ErrorCodes.UndefinedVariable, Severity.Warning, DiagnosticSource.Linter)); + private void HandleScope(ScopeStatement node) { + try { + OpenScope(node); + node.Walk(this); + } finally { + CloseScope(); + } } } } diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index b8140940b..60cc720b7 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -122,12 +122,12 @@ private void SpecializeTypes() { _hiddenNames.Add("__builtin_module_names__"); if (_boolType != null) { - Analysis.GlobalScope.DeclareVariable("True", _boolType, VariableSource.Declaration, LocationInfo.Empty); - Analysis.GlobalScope.DeclareVariable("False", _boolType, VariableSource.Declaration, LocationInfo.Empty); + Analysis.GlobalScope.DeclareVariable("True", _boolType, VariableSource.Builtin, new Location(this, default)); + Analysis.GlobalScope.DeclareVariable("False", _boolType, VariableSource.Builtin, new Location(this, default)); } if (noneType != null) { - Analysis.GlobalScope.DeclareVariable("None", noneType, VariableSource.Declaration, LocationInfo.Empty); + Analysis.GlobalScope.DeclareVariable("None", noneType, VariableSource.Builtin, new Location(this, default)); } foreach (var n in GetMemberNames()) { diff --git a/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs b/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs index a7e50f38d..e020e7b26 100644 --- a/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs @@ -13,9 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; using System.IO; -using System.Runtime.InteropServices.ComTypes; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; using Microsoft.Python.Core.IO; diff --git a/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs b/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs index 7b4ad204a..d9864fc9e 100644 --- a/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs @@ -16,14 +16,10 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Text; -using System.Threading; -using System.Threading.Tasks; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; using Microsoft.Python.Core.IO; -using Microsoft.Python.Core.OS; namespace Microsoft.Python.Analysis.Modules { internal class CompiledPythonModule : PythonModule { diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs index bbe8230f1..a463afd1b 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs @@ -38,7 +38,7 @@ public interface IModuleManagement: IModuleResolution { bool TryAddModulePath(in string path, out string fullName); /// - /// Sets user search paths. This changes . + /// Sets user search paths. This changes . /// /// Added roots. IEnumerable SetUserSearchPaths(in IEnumerable searchPaths); @@ -58,5 +58,15 @@ public interface IModuleManagement: IModuleResolution { /// Returns specialized module, if any. /// IPythonModule GetSpecializedModule(string name); + + /// + /// Root directory of the path resolver. + /// + string Root { get; } + + /// + /// Set of interpreter paths. + /// + IEnumerable InterpreterPaths { get; } } } diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs index e0959f44d..ef66855c8 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs @@ -50,6 +50,9 @@ public interface IModuleResolution { /// IPythonModule GetOrLoadModule(string name); + /// + /// Reloads all modules. Typically after installation or removal of packages. + /// Task ReloadAsync(CancellationToken token = default); } } diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index cf5366b1e..6e8fa7f94 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -42,7 +42,7 @@ namespace Microsoft.Python.Analysis.Modules { /// to AST and the module analysis. /// [DebuggerDisplay("{Name} : {ModuleType}")] - public class PythonModule : IDocument, IAnalyzable, IEquatable { + internal class PythonModule : LocatedMember, IDocument, IAnalyzable, IEquatable { private enum State { None, Loading, @@ -54,7 +54,7 @@ private enum State { } private readonly DocumentBuffer _buffer = new DocumentBuffer(); - private readonly DisposeToken _disposeToken = DisposeToken.Create< PythonModule>(); + private readonly DisposeToken _disposeToken = DisposeToken.Create(); private IReadOnlyList _parseErrors = Array.Empty(); private readonly IDiagnosticsService _diagnosticsService; @@ -63,6 +63,7 @@ private enum State { private CancellationTokenSource _linkedParseCts; // combined with 'dispose' cts private Task _parsingTask; private PythonAst _ast; + private bool _updated; protected ILogger Log { get; } protected IFileSystem FileSystem { get; } @@ -70,7 +71,8 @@ private enum State { private object AnalysisLock { get; } = new object(); private State ContentState { get; set; } = State.None; - protected PythonModule(string name, ModuleType moduleType, IServiceContainer services) { + protected PythonModule(string name, ModuleType moduleType, IServiceContainer services) + : base(PythonMemberType.Module) { Name = name ?? throw new ArgumentNullException(nameof(name)); Services = services ?? throw new ArgumentNullException(nameof(services)); ModuleType = moduleType; @@ -80,6 +82,7 @@ protected PythonModule(string name, ModuleType moduleType, IServiceContainer ser Analysis = new EmptyAnalysis(services, this); _diagnosticsService = services.GetService(); + SetDeclaringModule(this); } protected PythonModule(string moduleName, string filePath, ModuleType moduleType, IPythonModule stub, IServiceContainer services) : @@ -95,7 +98,6 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s Check.ArgumentNotNull(nameof(services), services); FileSystem = services.GetService(); - Location = new LocationInfo(creationOptions.FilePath, creationOptions.Uri, 1, 1); var uri = creationOptions.Uri; if (uri == null && !string.IsNullOrEmpty(creationOptions.FilePath)) { @@ -116,14 +118,13 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s #region IPythonType public string Name { get; } - public virtual IPythonModule DeclaringModule => null; public BuiltinTypeId TypeId => BuiltinTypeId.Module; public bool IsBuiltin => true; public bool IsAbstract => false; public virtual bool IsSpecialized => false; - public IMember CreateInstance(string typeName, LocationInfo location, IArgumentSet args) => this; - public PythonMemberType MemberType => PythonMemberType.Module; + public IMember CreateInstance(string typeName, IArgumentSet args) => this; + public override PythonMemberType MemberType => PythonMemberType.Module; public IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => GetMember(memberName); public IMember Index(IPythonInstance instance, object index) => Interpreter.UnknownType; @@ -163,6 +164,10 @@ public virtual IEnumerable GetMemberNames() { } #endregion + #region ILocatedMember + public override LocationInfo Definition => new LocationInfo(Uri.ToAbsolutePath(), Uri, 0, 0); + #endregion + #region IPythonFile public virtual string FilePath { get; } public virtual Uri Uri { get; } @@ -225,10 +230,6 @@ private void LoadContent(string content, int version) { } #endregion - #region ILocatedMember - public virtual LocationInfo Location { get; } - #endregion - #region IDisposable public void Dispose() => Dispose(true); @@ -304,6 +305,8 @@ public void Update(IEnumerable changes) { _linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_disposeToken.CancellationToken, _parseCts.Token); _buffer.Update(changes); + _updated = true; + Parse(); } @@ -351,7 +354,7 @@ private void Parse(CancellationToken cancellationToken) { parser = Parser.CreateParser(new StringReader(_buffer.Text), Interpreter.LanguageVersion, options); } - var ast = parser.ParseFile(); + var ast = parser.ParseFile(Uri); //Log?.Log(TraceEventType.Verbose, $"Parse complete: {Name}"); @@ -395,6 +398,21 @@ public override void Add(string message, SourceSpan span, int errorCode, Severit #endregion #region IAnalyzable + + public void NotifyAnalysisBegins() { + lock (AnalysisLock) { + if (_updated) { + _updated = false; + var analyzer = Services.GetService(); + foreach (var gs in analyzer.LoadedModules.Select(m => m.GlobalScope).OfType().ExcludeDefault()) { + foreach (var v in gs.TraverseDepthFirst(c => c.Children).SelectMany(s => s.Variables)) { + v.RemoveReferences(this); + } + } + } + } + } + public void NotifyAnalysisComplete(IDocumentAnalysis analysis) { lock (AnalysisLock) { if (analysis.Version < Analysis.Version) { diff --git a/src/Analysis/Ast/Impl/Modules/PythonPackage.cs b/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs similarity index 85% rename from src/Analysis/Ast/Impl/Modules/PythonPackage.cs rename to src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs index 12a579895..d28233904 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonPackage.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs @@ -28,22 +28,19 @@ namespace Microsoft.Python.Analysis.Modules { /// Contains either module members, members + imported children of explicit package or imported implicit package children /// Instance is unique for each module analysis /// - internal sealed class PythonVariableModule : IPythonModule, IEquatable { + internal sealed class PythonVariableModule : LocatedMember, IPythonModule, IEquatable { private readonly Dictionary _children = new Dictionary(); public string Name { get; } public IPythonModule Module { get; } public IPythonInterpreter Interpreter { get; } - public LocationInfo Location { get; } public IDocumentAnalysis Analysis => Module?.Analysis; - public IPythonModule DeclaringModule => null; public string Documentation => Module?.Documentation ?? string.Empty; public string FilePath => Module?.FilePath; public bool IsBuiltin => true; public bool IsAbstract => false; public bool IsSpecialized => Module?.IsSpecialized ?? false; - public PythonMemberType MemberType => PythonMemberType.Module; public ModuleType ModuleType => Module?.ModuleType ?? ModuleType.Package; public IPythonModule PrimaryModule => null; public IPythonModule Stub => null; @@ -51,17 +48,16 @@ internal sealed class PythonVariableModule : IPythonModule, IEquatable BuiltinTypeId.Module; public Uri Uri => Module?.Uri; - public PythonVariableModule(string name, IPythonInterpreter interpreter) { + public PythonVariableModule(string name, IPythonInterpreter interpreter) + : base(PythonMemberType.Module) { Name = name; - Location = LocationInfo.Empty; Interpreter = interpreter; + SetDeclaringModule(this); } - public PythonVariableModule(IPythonModule module) { + public PythonVariableModule(IPythonModule module): base(PythonMemberType.Module, module) { Name = module.Name; - Location = module.Location; Interpreter = module.Interpreter; - Module = module; } @@ -72,7 +68,7 @@ public PythonVariableModule(IPythonModule module) { public IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => GetMember(memberName); public IMember Index(IPythonInstance instance, object index) => Interpreter.UnknownType; - public IMember CreateInstance(string typeName = null, LocationInfo location = null, IArgumentSet args = null) => this; + public IMember CreateInstance(string typeName = null, IArgumentSet args = null) => this; public bool Equals(IPythonModule other) => other is PythonVariableModule module && Name.EqualsOrdinal(module.Name); } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index 19f5439b9..14d6d89ff 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -26,6 +26,7 @@ using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Core.IO; @@ -181,7 +182,8 @@ internal async Task LoadBuiltinTypesAsync(CancellationToken cancellationToken = // Add built-in module names var builtinModuleNamesMember = BuiltinsModule.GetAnyMember("__builtin_module_names__"); - if (builtinModuleNamesMember.TryGetConstant(out var s)) { + var value = (builtinModuleNamesMember as IVariable)?.Value ?? builtinModuleNamesMember; + if (value.TryGetConstant(out var s)) { var builtinModuleNames = s.Split(',').Select(n => n.Trim()); PathResolver.SetBuiltins(builtinModuleNames); } @@ -203,17 +205,17 @@ public async Task ReloadAsync(CancellationToken cancellationToken = default) { PathResolver = new PathResolver(_interpreter.LanguageVersion); var addedRoots = new HashSet(); - addedRoots.UnionWith(PathResolver.SetRoot(_root)); + addedRoots.UnionWith(PathResolver.SetRoot(Root)); - var interpreterPaths = await GetSearchPathsAsync(cancellationToken); - addedRoots.UnionWith(PathResolver.SetInterpreterSearchPaths(interpreterPaths)); + InterpreterPaths = await GetSearchPathsAsync(cancellationToken); + addedRoots.UnionWith(PathResolver.SetInterpreterSearchPaths(InterpreterPaths)); - var userSearchPaths = _interpreter.Configuration.SearchPaths.Except(interpreterPaths, StringExtensions.PathsStringComparer); + var userSearchPaths = _interpreter.Configuration.SearchPaths.Except(InterpreterPaths, StringExtensions.PathsStringComparer); addedRoots.UnionWith(SetUserSearchPaths(userSearchPaths)); ReloadModulePaths(addedRoots); } - public IEnumerable SetUserSearchPaths(in IEnumerable searchPaths) + public IEnumerable SetUserSearchPaths(in IEnumerable searchPaths) => PathResolver.SetUserSearchPaths(searchPaths); // For tests diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs index 123378f5b..fb6b1d793 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs @@ -35,7 +35,6 @@ internal abstract class ModuleResolutionBase { protected readonly ILogger _log; protected readonly IUIService _ui; protected readonly bool _requireInitPy; - protected string _root; protected ConcurrentDictionary Modules { get; } = new ConcurrentDictionary(); protected PathResolver PathResolver { get; set; } @@ -43,7 +42,8 @@ internal abstract class ModuleResolutionBase { protected InterpreterConfiguration Configuration => _interpreter.Configuration; protected ModuleResolutionBase(string root, IServiceContainer services) { - _root = root; + Root = root; + _services = services; _interpreter = services.GetService(); _fs = services.GetService(); @@ -53,6 +53,9 @@ protected ModuleResolutionBase(string root, IServiceContainer services) { _requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(_interpreter.Configuration.Version); } + public string Root { get; protected set; } + public IEnumerable InterpreterPaths { get; protected set; } = Enumerable.Empty(); + public IModuleCache ModuleCache { get; protected set; } public string BuiltinModuleName => BuiltinTypeId.Unknown.GetModuleName(_interpreter.LanguageVersion); diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs index b87ac799f..b03ee3cbe 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs @@ -36,9 +36,9 @@ public TypeshedResolution(IServiceContainer services) : base(null, services) { Modules[BuiltinModuleName] = new ModuleRef(BuiltinsModule); var stubs = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Stubs"); - _root = _interpreter.Configuration?.TypeshedPath; + Root = _interpreter.Configuration?.TypeshedPath; // TODO: merge with user-provided stub paths - _typeStubPaths = GetTypeShedPaths(_root) + _typeStubPaths = GetTypeShedPaths(Root) .Concat(GetTypeShedPaths(stubs)) .Where(services.GetService().DirectoryExists) .ToArray(); @@ -76,7 +76,7 @@ public Task ReloadAsync(CancellationToken cancellationToken = default) { Modules.Clear(); PathResolver = new PathResolver(_interpreter.LanguageVersion); - var addedRoots = PathResolver.SetRoot(_root); + var addedRoots = PathResolver.SetRoot(Root); ReloadModulePaths(addedRoots); addedRoots = PathResolver.SetInterpreterSearchPaths(_typeStubPaths); diff --git a/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs b/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs index 14ee21fb1..0e3e5fbcb 100644 --- a/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs +++ b/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs @@ -29,7 +29,7 @@ namespace Microsoft.Python.Analysis.Modules { /// module. Specialized module can use actual library module as a source /// of documentation for its members. See . /// - public abstract class SpecializedModule : PythonModule { + internal abstract class SpecializedModule : PythonModule { protected SpecializedModule(string name, string modulePath, IServiceContainer services) : base(name, modulePath, ModuleType.Specialized, null, services) { } diff --git a/src/Analysis/Ast/Impl/Resources.Designer.cs b/src/Analysis/Ast/Impl/Resources.Designer.cs index ea5718777..ccd0005c2 100644 --- a/src/Analysis/Ast/Impl/Resources.Designer.cs +++ b/src/Analysis/Ast/Impl/Resources.Designer.cs @@ -195,6 +195,24 @@ internal static string ErrorUseBeforeDef { } } + /// + /// Looks up a localized string similar to '{0}' is not defined in the global scope. + /// + internal static string ErrorVariableNotDefinedGlobally { + get { + return ResourceManager.GetString("ErrorVariableNotDefinedGlobally", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to '{0}' is not defined in non-local scopes. + /// + internal static string ErrorVariableNotDefinedNonLocal { + get { + return ResourceManager.GetString("ErrorVariableNotDefinedNonLocal", resourceCulture); + } + } + /// /// Looks up a localized string similar to An exception occured while discovering search paths; analysis will not be available.. /// diff --git a/src/Analysis/Ast/Impl/Resources.resx b/src/Analysis/Ast/Impl/Resources.resx index 945ea3d4b..3cbed5879 100644 --- a/src/Analysis/Ast/Impl/Resources.resx +++ b/src/Analysis/Ast/Impl/Resources.resx @@ -348,6 +348,12 @@ Undefined variable: '{0}' + + '{0}' is not defined in the global scope + + + '{0}' is not defined in non-local scopes + An exception occured while discovering search paths; analysis will not be available. diff --git a/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs b/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs index dffff9fc2..1464932dd 100644 --- a/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs +++ b/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs @@ -23,17 +23,17 @@ namespace Microsoft.Python.Analysis.Specializations { public static class BuiltinsSpecializations { - public static IMember Identity(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { + public static IMember Identity(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { var args = argSet.Values(); return args.Count > 0 ? args.FirstOrDefault(a => !a.IsUnknown()) ?? args[0] : null; } - public static IMember TypeInfo(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { + public static IMember TypeInfo(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { var args = argSet.Values(); return args.Count > 0 ? args[0].GetPythonType() : module.Interpreter.GetBuiltinType(BuiltinTypeId.Type); } - public static IMember Iterator(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { + public static IMember Iterator(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { var args = argSet.Values(); if (args.Count > 0) { if (args[0] is IPythonCollection seq) { @@ -47,48 +47,49 @@ public static IMember Iterator(IPythonModule module, IPythonFunctionOverload ove return null; } - public static IMember List(IPythonInterpreter interpreter, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) - => PythonCollectionType.CreateList(interpreter, location, argSet); + public static IMember List(IPythonInterpreter interpreter, IPythonFunctionOverload overload, IArgumentSet argSet) + => PythonCollectionType.CreateList(interpreter, argSet); - public static IMember ListOfStrings(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { + public static IMember ListOfStrings(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { var type = new TypingListType("List", module.Interpreter.GetBuiltinType(BuiltinTypeId.Str), module.Interpreter, false); - return new TypingList(type, location); + return new TypingList(type); } - public static IMember DictStringToObject(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { + public static IMember DictStringToObject(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { var str = module.Interpreter.GetBuiltinType(BuiltinTypeId.Str); var obj = module.Interpreter.GetBuiltinType(BuiltinTypeId.Object); var type = new TypingDictionaryType("Dict", str, obj, module.Interpreter, false); - return new TypingDictionary(type, location); + return new TypingDictionary(type); } - public static IMember Next(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { + public static IMember Next(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { var args = argSet.Values(); return args.Count > 0 && args[0] is IPythonIterator it ? it.Next : null; } public static IMember __iter__(IPythonInterpreter interpreter, BuiltinTypeId contentTypeId) { - var fn = new PythonFunctionType(@"__iter__", interpreter.ModuleResolution.BuiltinsModule, null, string.Empty, LocationInfo.Empty); - var o = new PythonFunctionOverload(fn.Name, interpreter.ModuleResolution.BuiltinsModule, _ => fn.Location); + var location = new Location(interpreter.ModuleResolution.BuiltinsModule, default); + var fn = new PythonFunctionType(@"__iter__", location, null, string.Empty); + var o = new PythonFunctionOverload(fn.Name, location); o.AddReturnValue(PythonTypeIterator.FromTypeId(interpreter, contentTypeId)); fn.AddOverload(o); return fn; } - public static IMember Range(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { + public static IMember Range(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { var args = argSet.Values(); if (args.Count > 0) { var type = new PythonCollectionType(null, BuiltinTypeId.List, module.Interpreter, false); - return new PythonCollection(type, location, new[] { args[0] }); + return new PythonCollection(type, new[] { args[0] }); } return null; } - public static IMember CollectionItem(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { + public static IMember CollectionItem(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { var args = argSet.Values(); return args.Count > 0 && args[0] is PythonCollection c ? c.Contents.FirstOrDefault() : null; } - public static IMember Open(IPythonModule declaringModule, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { + public static IMember Open(IPythonModule declaringModule, IPythonFunctionOverload overload, IArgumentSet argSet) { var mode = argSet.GetArgumentValue("mode"); var bytes = false; diff --git a/src/Analysis/Ast/Impl/Specializations/Specialized.cs b/src/Analysis/Ast/Impl/Specializations/Specialized.cs index 9b918e69b..4da22b82c 100644 --- a/src/Analysis/Ast/Impl/Specializations/Specialized.cs +++ b/src/Analysis/Ast/Impl/Specializations/Specialized.cs @@ -18,16 +18,18 @@ namespace Microsoft.Python.Analysis.Specializations { internal static class Specialized { public static IPythonPropertyType Property(string name, IPythonModule declaringModule, IPythonType declaringType, IMember returnValue) { - var prop = new PythonPropertyType(name, declaringModule, declaringType, false, LocationInfo.Empty); - var o = new PythonFunctionOverload(prop.Name, declaringModule, _ => LocationInfo.Empty); + var location = new Location(declaringModule); + var prop = new PythonPropertyType(name, location, declaringType, false); + var o = new PythonFunctionOverload(prop.Name, location); o.AddReturnValue(returnValue); prop.AddOverload(o); return prop; } public static IPythonFunctionType Function(string name, IPythonModule declaringModule, IPythonType declaringType, string documentation, IMember returnValue) { - var prop = new PythonFunctionType(name, declaringModule, declaringType, documentation, LocationInfo.Empty); - var o = new PythonFunctionOverload(prop.Name, declaringModule, _ => LocationInfo.Empty); + var location = new Location(declaringModule); + var prop = new PythonFunctionType(name, location, declaringType, documentation); + var o = new PythonFunctionOverload(prop.Name, location); o.AddReturnValue(returnValue); prop.AddOverload(o); return prop; diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericType.cs index e9cd02efb..11b5a6fbb 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericType.cs @@ -28,6 +28,6 @@ public interface IGenericType : IPythonTemplateType { /// IReadOnlyList Parameters { get; } - IPythonType CreateSpecificType(IReadOnlyList typeArguments, IPythonModule declaringModule, LocationInfo location = null); + IPythonType CreateSpecificType(IReadOnlyList typeArguments); } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/AnyType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/AnyType.cs index b2267a486..091fa6659 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/AnyType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/AnyType.cs @@ -19,23 +19,20 @@ using Microsoft.Python.Analysis.Values; namespace Microsoft.Python.Analysis.Specializations.Typing.Types { - internal sealed class AnyType : IPythonType { - public AnyType(IPythonModule declaringModule) { - DeclaringModule = declaringModule; - } + internal sealed class AnyType : LocatedMember, IPythonType { + public AnyType(IPythonModule declaringModule) + : base(PythonMemberType.Class, declaringModule) { } + public string Name => "Any"; - public IPythonModule DeclaringModule { get; } public BuiltinTypeId TypeId => BuiltinTypeId.Type; public string Documentation => Name; public bool IsBuiltin => false; public bool IsAbstract => false; public bool IsSpecialized => true; - public PythonMemberType MemberType => PythonMemberType.Class; - public IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) + public IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => DeclaringModule.Interpreter.UnknownType; - public IMember CreateInstance(string typeName, LocationInfo location, IArgumentSet args) - => new PythonInstance(this, location); + public IMember CreateInstance(string typeName, IArgumentSet args) => new PythonInstance(this); public IMember GetMember(string name) => null; public IEnumerable GetMemberNames() => Array.Empty(); diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassParameter.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassParameter.cs index b1f52a716..9e61ee84f 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassParameter.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassParameter.cs @@ -23,8 +23,8 @@ namespace Microsoft.Python.Analysis.Specializations.Typing.Types { /// generic type parameters from TypeVar. /// internal sealed class GenericClassParameter : PythonClassType, IGenericClassParameter { - internal GenericClassParameter(IReadOnlyList typeArgs, IPythonModule declaringModule, LocationInfo location) - : base("GenericParameter", declaringModule, location) { + internal GenericClassParameter(IReadOnlyList typeArgs, IPythonModule declaringModule) + : base("GenericParameter", new Location(declaringModule)) { TypeDefinitions = typeArgs; } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericType.cs index 9fddf0d2d..f211139f8 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericType.cs @@ -20,22 +20,19 @@ using Microsoft.Python.Analysis.Values; namespace Microsoft.Python.Analysis.Specializations.Typing.Types { - internal delegate IPythonType SpecificTypeConstructor( - IReadOnlyList typeArgs, - IPythonModule declaringModule, - LocationInfo location); + internal delegate IPythonType SpecificTypeConstructor(IReadOnlyList typeArgs); /// /// Base class for generic types and type declarations. /// - internal class GenericType : IGenericType { + internal class GenericType : LocatedMember, IGenericType { internal SpecificTypeConstructor SpecificTypeConstructor { get; } /// /// Constructs generic type with generic type parameters. Typically used /// in generic classes such as when handling Generic[_T] base. /// - public GenericType(string name, IPythonModule declaringModule, IReadOnlyList parameters) + public GenericType(string name, IReadOnlyList parameters, IPythonModule declaringModule) : this(name, declaringModule) { Parameters = parameters ?? throw new ArgumentNullException(nameof(parameters)); } @@ -45,15 +42,15 @@ public GenericType(string name, IPythonModule declaringModule, IReadOnlyList /// Type name including parameters, such as Iterator[T] - /// Declaring module. /// Constructor of specific types. + /// Declaring module. /// Type id. Used in type comparisons such as when matching /// function arguments. For example, Iterator[T] normally has type id of ListIterator. /// Optional type parameters as declared by TypeVar. public GenericType( string name, - IPythonModule declaringModule, SpecificTypeConstructor specificTypeConstructor, + IPythonModule declaringModule, BuiltinTypeId typeId = BuiltinTypeId.Unknown, IReadOnlyList parameters = null ) : this(name, declaringModule) { @@ -62,9 +59,9 @@ public GenericType( Parameters = parameters ?? Array.Empty(); } - private GenericType(string name, IPythonModule declaringModule) { + private GenericType(string name, IPythonModule declaringModule) + : base(PythonMemberType.Generic, declaringModule) { Name = name ?? throw new ArgumentNullException(nameof(name)); - DeclaringModule = declaringModule ?? throw new ArgumentNullException(nameof(declaringModule)); } /// @@ -77,13 +74,11 @@ private GenericType(string name, IPythonModule declaringModule) { /// Creates instance of a type information with the specific /// type arguments from a generic template. /// - public IPythonType CreateSpecificType(IReadOnlyList typeArguments, IPythonModule declaringModule, LocationInfo location = null) - => SpecificTypeConstructor(typeArguments, declaringModule, location); + public IPythonType CreateSpecificType(IReadOnlyList typeArguments) + => SpecificTypeConstructor(typeArguments); #region IPythonType public string Name { get; } - public IPythonModule DeclaringModule { get; } - public PythonMemberType MemberType => PythonMemberType.Generic; public IMember GetMember(string name) => null; public IEnumerable GetMemberNames() => Enumerable.Empty(); public BuiltinTypeId TypeId { get; } = BuiltinTypeId.Unknown; @@ -92,22 +87,22 @@ public IPythonType CreateSpecificType(IReadOnlyList typeArguments, public bool IsAbstract => true; public bool IsSpecialized => true; - public IMember CreateInstance(string typeName, LocationInfo location, IArgumentSet args) { + public IMember CreateInstance(string typeName, IArgumentSet args) { var types = args.Values(); if (types.Count != args.Arguments.Count) { throw new ArgumentException(@"Generic type instance construction arguments must be all of IPythonType", nameof(args)); } - var specific = CreateSpecificType(types, DeclaringModule, location); + var specific = CreateSpecificType(types); return specific == null ? DeclaringModule.Interpreter.UnknownType - : specific.CreateInstance(typeName, location, null); + : specific.CreateInstance(typeName); } public virtual IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => DeclaringModule.Interpreter.UnknownType; public virtual IMember Index(IPythonInstance instance, object index) => DeclaringModule.Interpreter.UnknownType; - public IPythonType CreateSpecificType(IArgumentSet typeArguments, IPythonModule declaringModule, LocationInfo location) - => CreateSpecificType(typeArguments.Arguments.Select(a => a.Value).OfType().ToArray(), declaringModule, location); + public IPythonType CreateSpecificType(IArgumentSet typeArguments) + => CreateSpecificType(typeArguments.Arguments.Select(a => a.Value).OfType().ToArray()); #endregion public override bool Equals(object other) { diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs index 752d8388d..3a3df5631 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs @@ -19,11 +19,12 @@ using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Utilities; using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core.Text; namespace Microsoft.Python.Analysis.Specializations.Typing.Types { internal sealed class GenericTypeParameter : PythonType, IGenericTypeDefinition { - public GenericTypeParameter(string name, IPythonModule declaringModule, IReadOnlyList constraints, string documentation, LocationInfo location) - : base(name, declaringModule, documentation, location) { + public GenericTypeParameter(string name, IPythonModule declaringModule, IReadOnlyList constraints, string documentation, IndexSpan location) + : base(name, new Location(declaringModule), documentation) { Constraints = constraints ?? Array.Empty(); } public IReadOnlyList Constraints { get; } @@ -33,14 +34,14 @@ public GenericTypeParameter(string name, IPythonModule declaringModule, IReadOnl public override bool IsSpecialized => true; - public static IPythonType FromTypeVar(IArgumentSet argSet, IPythonModule declaringModule, LocationInfo location) { + public static IPythonType FromTypeVar(IArgumentSet argSet, IPythonModule declaringModule, IndexSpan location = default) { var args = argSet.Values(); if (args.Count == 0) { // TODO: report that at least one argument is required. return declaringModule.Interpreter.UnknownType; } - var name = (args[0] as IPythonConstant)?.Value as string; + var name = (args[0] as IPythonConstant)?.GetString(); if (string.IsNullOrEmpty(name)) { // TODO: report that type name is not a string. return declaringModule.Interpreter.UnknownType; @@ -52,7 +53,7 @@ public static IPythonType FromTypeVar(IArgumentSet argSet, IPythonModule declari return !string.IsNullOrEmpty(typeString) ? argSet.Eval.GetTypeFromString(typeString) : a.GetPythonType(); }).ToArray(); if (constraints.Any(c => c.IsUnknown())) { - // TODO: report that some constraints could be be resolved. + // TODO: report that some constraints could not be resolved. } var docArgs = new[] { $"'{name}'" }.Concat(constraints.Select(c => c.IsUnknown() ? "?" : c.Name)); diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs index 2178c700d..2343bec6b 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs @@ -43,9 +43,7 @@ public NamedTupleType(string tupleName, IReadOnlyList itemNames, IReadOn public override string Name { get; } public override bool IsSpecialized => true; - - public override IMember CreateInstance(string typeName, LocationInfo location, IArgumentSet args) - => new TypingTuple(this, location); + public override IMember CreateInstance(string typeName, IArgumentSet args) => new TypingTuple(this); // NamedTuple does not create instances, it defines a type. public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => this; diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/OptionalType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/OptionalType.cs index 99ad93829..a4ba593a8 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/OptionalType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/OptionalType.cs @@ -36,7 +36,7 @@ public IEnumerator GetEnumerator() public IPythonUnionType Add(IPythonType t) => this; public IPythonUnionType Add(IPythonUnionType types) => this; - public override IMember CreateInstance(string typeName, LocationInfo location, IArgumentSet args) - => InnerType.CreateInstance(typeName, location, args); + public override IMember CreateInstance(string typeName, IArgumentSet args) + => InnerType.CreateInstance(typeName, args); } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingDictionaryType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingDictionaryType.cs index 5705476c6..0f618e96c 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingDictionaryType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingDictionaryType.cs @@ -46,8 +46,7 @@ public TypingDictionaryType(string name, IPythonType keyType, IPythonType valueT public override string Name { get; } - public override IMember CreateInstance(string typeName, LocationInfo location, IArgumentSet args) - => new TypingDictionary(this, location); + public override IMember CreateInstance(string typeName, IArgumentSet args) => new TypingDictionary(this); public override IMember Index(IPythonInstance instance, object index) => new PythonInstance(ValueType); public override bool IsSpecialized => true; diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingIteratorType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingIteratorType.cs index 6946be738..81762ac42 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingIteratorType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingIteratorType.cs @@ -18,7 +18,6 @@ using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Types.Collections; using Microsoft.Python.Analysis.Utilities; -using Microsoft.Python.Core.Diagnostics; namespace Microsoft.Python.Analysis.Specializations.Typing.Types { /// diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingListType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingListType.cs index 7b0e27857..818ed2aa4 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingListType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingListType.cs @@ -50,8 +50,7 @@ public TypingListType(string typeName, BuiltinTypeId typeId, IPythonType itemTyp public override bool IsAbstract => false; public override bool IsSpecialized => true; - public override IMember CreateInstance(string typeName, LocationInfo location, IArgumentSet args) - => new TypingList(this, location); + public override IMember CreateInstance(string typeName, IArgumentSet args) => new TypingList(this); public IPythonType ItemType { get; } public override IMember Index(IPythonInstance instance, object index) => new PythonInstance(ItemType); diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingTupleType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingTupleType.cs index 5994ae8c3..3fda6b0f4 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingTupleType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingTupleType.cs @@ -41,8 +41,8 @@ public TypingTupleType(IReadOnlyList itemTypes, IPythonInterpreter public override bool IsAbstract => false; public override bool IsSpecialized => true; - public override IMember CreateInstance(string typeName, LocationInfo location, IArgumentSet args) - => new TypingTuple(this, location); + public override IMember CreateInstance(string typeName, IArgumentSet args) + => new TypingTuple(this); public override IMember Index(IPythonInstance instance, object index) { var n = PythonCollection.GetIndex(index); diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs index a981e4dee..d91afe25c 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs @@ -43,84 +43,80 @@ public static IPythonModule Create(IServiceContainer services) { #endregion private void SpecializeMembers() { + var location = new Location(this, default); + // TypeVar - var fn = new PythonFunctionType("TypeVar", this, null, GetMemberDocumentation, GetMemberLocation); - var o = new PythonFunctionOverload(fn.Name, this, _ => fn.Location); + var fn = new PythonFunctionType("TypeVar", location, null, GetMemberDocumentation); + var o = new PythonFunctionOverload(fn.Name, location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. - o.SetReturnValueProvider((interpreter, overload, location, args) - => GenericTypeParameter.FromTypeVar(args, interpreter, location)); + o.SetReturnValueProvider((interpreter, overload, args) + => GenericTypeParameter.FromTypeVar(args, interpreter)); fn.AddOverload(o); _members["TypeVar"] = fn; // NewType - fn = new PythonFunctionType("NewType", this, null, GetMemberDocumentation, GetMemberLocation); - o = new PythonFunctionOverload(fn.Name, this, _ => fn.Location); + fn = new PythonFunctionType("NewType", location, null, GetMemberDocumentation); + o = new PythonFunctionOverload(fn.Name, location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. - o.SetReturnValueProvider((interpreter, overload, location, args) => CreateTypeAlias(args.Values())); + o.SetReturnValueProvider((interpreter, overload, args) => CreateTypeAlias(args.Values())); fn.AddOverload(o); _members["NewType"] = fn; // NewType - fn = new PythonFunctionType("Type", this, null, GetMemberDocumentation, GetMemberLocation); - o = new PythonFunctionOverload(fn.Name, this, _ => fn.Location); + fn = new PythonFunctionType("Type", location, null, GetMemberDocumentation); + o = new PythonFunctionOverload(fn.Name, location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. - o.SetReturnValueProvider((interpreter, overload, location, args) => { + o.SetReturnValueProvider((interpreter, overload, args) => { var a = args.Values(); return a.Count == 1 ? a[0] : Interpreter.UnknownType; }); fn.AddOverload(o); _members["Type"] = fn; - _members["Iterator"] = new GenericType("Iterator", this, - (typeArgs, module, location) => CreateIteratorType(typeArgs)); - - _members["Iterable"] = new GenericType("Iterable", this, - (typeArgs, module, location) => CreateListType("Iterable", BuiltinTypeId.List, typeArgs, false)); - _members["Sequence"] = new GenericType("Sequence", this, - (typeArgs, module, location) => CreateListType("Sequence", BuiltinTypeId.List, typeArgs, false)); - _members["MutableSequence"] = new GenericType("MutableSequence", this, - (typeArgs, module, location) => CreateListType("MutableSequence", BuiltinTypeId.List, typeArgs, true)); - _members["List"] = new GenericType("List", this, - (typeArgs, module, location) => CreateListType("List", BuiltinTypeId.List, typeArgs, true)); - - _members["MappingView"] = new GenericType("MappingView", this, - (typeArgs, module, location) => CreateDictionary("MappingView", typeArgs, false)); - _members["KeysView"] = new GenericType("KeysView", this, - (typeArgs, module, location) => CreateKeysViewType(typeArgs)); - _members["ValuesView"] = new GenericType("ValuesView", this, - (typeArgs, module, location) => CreateValuesViewType(typeArgs)); - _members["ItemsView"] = new GenericType("ItemsView", this, - (typeArgs, module, location) => CreateItemsViewType(typeArgs)); - - _members["Set"] = new GenericType("Set", this, - (typeArgs, module, location) => CreateListType("Set", BuiltinTypeId.Set, typeArgs, true)); - _members["MutableSet"] = new GenericType("MutableSet", this, - (typeArgs, module, location) => CreateListType("MutableSet", BuiltinTypeId.Set, typeArgs, true)); - _members["FrozenSet"] = new GenericType("FrozenSet", this, - (typeArgs, module, location) => CreateListType("FrozenSet", BuiltinTypeId.Set, typeArgs, false)); - - _members["Tuple"] = new GenericType("Tuple", this, - (typeArgs, module, location) => CreateTupleType(typeArgs)); - - _members["Mapping"] = new GenericType("Mapping", this, - (typeArgs, module, location) => CreateDictionary("Mapping", typeArgs, false)); - _members["MutableMapping"] = new GenericType("MutableMapping", this, - (typeArgs, module, location) => CreateDictionary("MutableMapping", typeArgs, true)); - _members["Dict"] = new GenericType("Dict", this, - (typeArgs, module, location) => CreateDictionary("Dict", typeArgs, true)); - _members["OrderedDict"] = new GenericType("OrderedDict", this, - (typeArgs, module, location) => CreateDictionary("OrderedDict", typeArgs, true)); - _members["DefaultDict"] = new GenericType("DefaultDict", this, - (typeArgs, module, location) => CreateDictionary("DefaultDict", typeArgs, true)); - - _members["Union"] = new GenericType("Union", this, - (typeArgs, module, location) => CreateUnion(typeArgs)); - - _members["Counter"] = Specialized.Function("Counter", this, null, "Counter", new PythonInstance(Interpreter.GetBuiltinType(BuiltinTypeId.Int))); + _members["Iterator"] = new GenericType("Iterator", CreateIteratorType, this); + + _members["Iterable"] = new GenericType("Iterable", typeArgs => CreateListType("Iterable", BuiltinTypeId.List, typeArgs, false), this); + _members["Sequence"] = new GenericType("Sequence", typeArgs => CreateListType("Sequence", BuiltinTypeId.List, typeArgs, false), this); + _members["MutableSequence"] = new GenericType("MutableSequence", + typeArgs => CreateListType("MutableSequence", BuiltinTypeId.List, typeArgs, true), this); + _members["List"] = new GenericType("List", + typeArgs => CreateListType("List", BuiltinTypeId.List, typeArgs, true), this); + + _members["MappingView"] = new GenericType("MappingView", + typeArgs => CreateDictionary("MappingView", typeArgs, false), this); + + _members["KeysView"] = new GenericType("KeysView", CreateKeysViewType, this); + _members["ValuesView"] = new GenericType("ValuesView", CreateValuesViewType, this); + _members["ItemsView"] = new GenericType("ItemsView", CreateItemsViewType, this); + + _members["Set"] = new GenericType("Set", + typeArgs => CreateListType("Set", BuiltinTypeId.Set, typeArgs, true), this); + _members["MutableSet"] = new GenericType("MutableSet", + typeArgs => CreateListType("MutableSet", BuiltinTypeId.Set, typeArgs, true), this); + _members["FrozenSet"] = new GenericType("FrozenSet", + typeArgs => CreateListType("FrozenSet", BuiltinTypeId.Set, typeArgs, false), this); + + _members["Tuple"] = new GenericType("Tuple", CreateTupleType, this); + + _members["Mapping"] = new GenericType("Mapping", + typeArgs => CreateDictionary("Mapping", typeArgs, false), this); + _members["MutableMapping"] = new GenericType("MutableMapping", + typeArgs => CreateDictionary("MutableMapping", typeArgs, true), this); + _members["Dict"] = new GenericType("Dict", + typeArgs => CreateDictionary("Dict", typeArgs, true), this); + _members["OrderedDict"] = new GenericType("OrderedDict", + typeArgs => CreateDictionary("OrderedDict", typeArgs, true), this); + _members["DefaultDict"] = new GenericType("DefaultDict", + typeArgs => CreateDictionary("DefaultDict", typeArgs, true), this); + + _members["Union"] = new GenericType("Union", CreateUnion, this); + + _members["Counter"] = Specialized.Function("Counter", this, null, "Counter", + new PythonInstance(Interpreter.GetBuiltinType(BuiltinTypeId.Int))); _members["SupportsInt"] = Interpreter.GetBuiltinType(BuiltinTypeId.Int); _members["SupportsFloat"] = Interpreter.GetBuiltinType(BuiltinTypeId.Float); @@ -128,9 +124,9 @@ private void SpecializeMembers() { _members["SupportsBytes"] = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); _members["ByteString"] = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); - fn = new PythonFunctionType("NamedTuple", this, null, GetMemberDocumentation, GetMemberLocation); - o = new PythonFunctionOverload(fn.Name, this, _ => fn.Location); - o.SetReturnValueProvider((interpreter, overload, location, args) => CreateNamedTuple(args.Values())); + fn = new PythonFunctionType("NamedTuple", location, null, GetMemberDocumentation); + o = new PythonFunctionOverload(fn.Name, location); + o.SetReturnValueProvider((interpreter, overload, args) => CreateNamedTuple(args.Values())); fn.AddOverload(o); _members["NamedTuple"] = fn; @@ -140,24 +136,22 @@ private void SpecializeMembers() { var str = Interpreter.GetBuiltinType(BuiltinTypeId.Str); var bytes = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); var unicode = Interpreter.GetBuiltinType(BuiltinTypeId.Unicode); - var anyStrName = new PythonConstant("AnyStr", str, LocationInfo.Empty); + var anyStrName = new PythonConstant("AnyStr", str); var anyStrArgs = Interpreter.LanguageVersion.Is3x() ? new IMember[] { anyStrName, str, bytes } : new IMember[] { anyStrName, str, unicode }; - _members["AnyStr"] = GenericTypeParameter.FromTypeVar(new ArgumentSet(anyStrArgs), this, LocationInfo.Empty); + _members["AnyStr"] = GenericTypeParameter.FromTypeVar(new ArgumentSet(anyStrArgs), this); - _members["Optional"] = new GenericType("Optional", this, (typeArgs, module, location) => CreateOptional(typeArgs)); - _members["Type"] = new GenericType("Type", this, (typeArgs, module, location) => CreateType(typeArgs)); + _members["Optional"] = new GenericType("Optional", CreateOptional, this); + _members["Type"] = new GenericType("Type", CreateType, this); - _members["Generic"] = new GenericType("Generic", this, CreateGenericClassParameter); + _members["Generic"] = new GenericType("Generic", CreateGenericClassParameter, this); } private string GetMemberDocumentation(string name) => base.GetMember(name)?.GetPythonType()?.Documentation; - private LocationInfo GetMemberLocation(string name) - => (base.GetMember(name)?.GetPythonType() as ILocatedMember)?.Location ?? LocationInfo.Empty; private IPythonType CreateListType(string typeName, BuiltinTypeId typeId, IReadOnlyList typeArgs, bool isMutable) { if (typeArgs.Count == 1) { @@ -246,7 +240,7 @@ private IPythonType CreateTypeAlias(IReadOnlyList typeArgs) { private IPythonType CreateUnion(IReadOnlyList typeArgs) { if (typeArgs.Count > 0) { - return TypingTypeFactory.CreateUnionType(Interpreter, typeArgs.Select(a => a.GetPythonType()).ToArray()); + return TypingTypeFactory.CreateUnionType(Interpreter, typeArgs.Select(a => a.GetPythonType()).ToArray(), this); } // TODO: report wrong number of arguments return Interpreter.UnknownType; @@ -316,13 +310,13 @@ private IPythonType CreateType(IReadOnlyList typeArgs) { return Interpreter.UnknownType; } - private IPythonType CreateGenericClassParameter(IReadOnlyList typeArgs, IPythonModule declaringModule, LocationInfo location) { + private IPythonType CreateGenericClassParameter(IReadOnlyList typeArgs) { // Handle Generic[_T1, _T2, ...]. _T1, et al are IGenericTypeParameter from TypeVar. // Hold the parameter until concrete type is provided at the time of the class instantiation. if (typeArgs.Count > 0) { var typeDefs = typeArgs.OfType().ToArray(); if (typeDefs.Length == typeArgs.Count) { - return new GenericClassParameter(typeDefs, declaringModule, location); + return new GenericClassParameter(typeDefs, this); } else { // TODO: report argument mismatch } @@ -333,7 +327,7 @@ private IPythonType CreateGenericClassParameter(IReadOnlyList typeA private IPythonType ToGenericTemplate(string typeName, IGenericTypeDefinition[] typeArgs, BuiltinTypeId typeId) => _members[typeName] is GenericType gt - ? new GenericType(CodeFormatter.FormatSequence(typeName, '[', typeArgs), this, gt.SpecificTypeConstructor, typeId, typeArgs) + ? new GenericType(CodeFormatter.FormatSequence(typeName, '[', typeArgs), gt.SpecificTypeConstructor, this, typeId, typeArgs) : Interpreter.UnknownType; } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/TypingTypeFactory.cs b/src/Analysis/Ast/Impl/Specializations/Typing/TypingTypeFactory.cs index a2bf09d7f..afed8ded6 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/TypingTypeFactory.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/TypingTypeFactory.cs @@ -52,8 +52,8 @@ public static ITypingListType CreateItemsViewType(IPythonInterpreter interpreter return new TypingListType(typeName, BuiltinTypeId.DictItems, itemType, interpreter, false, false); } - public static IPythonType CreateUnionType(IPythonInterpreter interpreter, IReadOnlyList types) - => new PythonUnionType(types.Select(a => a.GetPythonType())); + public static IPythonType CreateUnionType(IPythonInterpreter interpreter, IReadOnlyList types, IPythonModule declaringModule) + => new PythonUnionType(types.Select(a => a.GetPythonType()), declaringModule); public static ITypingNamedTupleType CreateNamedTupleType(IPythonInterpreter interpreter, string tupleName, IReadOnlyList itemNames, IReadOnlyList itemTypes) => new NamedTupleType(tupleName, itemNames, itemTypes, interpreter); diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingDictionary.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingDictionary.cs index be5927aaf..407fe8d8a 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingDictionary.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingDictionary.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; using Microsoft.Python.Analysis.Specializations.Typing.Types; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; @@ -27,8 +26,8 @@ namespace Microsoft.Python.Analysis.Specializations.Typing.Values { internal class TypingDictionary : PythonDictionary { private readonly TypingDictionaryType _dictType; - public TypingDictionary(TypingDictionaryType dictType, LocationInfo location = null) - : base(dictType, location ?? LocationInfo.Empty, EmptyDictionary.Instance) { + public TypingDictionary(TypingDictionaryType dictType) + : base(dictType, EmptyDictionary.Instance) { _dictType = dictType; } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingIterator.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingIterator.cs index d5ce5312b..16d172527 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingIterator.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingIterator.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using Microsoft.Python.Analysis.Specializations.Typing.Types; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; @@ -40,7 +39,7 @@ public override IMember Next { } else if (_index < _iteratorType.ItemTypes.Count) { itemType = _iteratorType.ItemTypes[_index++]; } - return itemType?.CreateInstance(itemType.Name, LocationInfo.Empty, ArgumentSet.Empty) ?? UnknownType; + return itemType?.CreateInstance(itemType.Name, ArgumentSet.Empty) ?? UnknownType; } } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingList.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingList.cs index 783b59a8b..ed6156f61 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingList.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingList.cs @@ -23,8 +23,8 @@ namespace Microsoft.Python.Analysis.Specializations.Typing.Values { internal class TypingList : PythonCollection { private readonly ITypingListType _collectionType; - public TypingList(ITypingListType collectionType, LocationInfo location = null) - : base(collectionType, location ?? LocationInfo.Empty, Array.Empty()) { + public TypingList(ITypingListType collectionType) + : base(collectionType, Array.Empty()) { _collectionType = collectionType; } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingTuple.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingTuple.cs index 099d6e5f7..93c9b52a2 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingTuple.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingTuple.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using Microsoft.Python.Analysis.Specializations.Typing.Types; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; @@ -22,8 +21,8 @@ namespace Microsoft.Python.Analysis.Specializations.Typing.Values { internal class TypingTuple : PythonCollection { private readonly TypingTupleType _collectionType; - public TypingTuple(TypingTupleType collectionType, LocationInfo location = null) - : base(collectionType, location ?? LocationInfo.Empty, collectionType.ItemTypes) { + public TypingTuple(TypingTupleType collectionType) + : base(collectionType, collectionType.ItemTypes) { _collectionType = collectionType; } @@ -34,6 +33,6 @@ public override IPythonIterator GetIterator() { } public override IMember Index(object index) - => _collectionType.Index(this, index).GetPythonType().CreateInstance(null, LocationInfo.Empty, ArgumentSet.Empty); + => _collectionType.Index(this, index).GetPythonType().CreateInstance(null, ArgumentSet.Empty); } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingType.cs index 7400ca83c..572a17ca5 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingType.cs @@ -22,26 +22,24 @@ namespace Microsoft.Python.Analysis.Specializations.Typing.Values { /// /// Holds type info of the generic parameter. /// - internal sealed class TypingType : IPythonType { + internal sealed class TypingType : LocatedMember, IPythonType { private readonly IPythonType _type; - public TypingType(IPythonModule declaringModule, IPythonType type) { - _type = type ?? throw new ArgumentNullException(nameof(type)); - DeclaringModule = declaringModule; + public TypingType(IPythonModule declaringModule, IPythonType type) + : base(PythonMemberType.Class, declaringModule) { + _type = type ?? throw new ArgumentNullException(nameof(type)); Name = $"Type[{_type.Name}]"; } public string Name { get; } - public IPythonModule DeclaringModule { get; } public BuiltinTypeId TypeId => BuiltinTypeId.Type; public string Documentation => Name; public bool IsBuiltin => false; public bool IsAbstract => false; public bool IsSpecialized => true; - public PythonMemberType MemberType => PythonMemberType.Class; public IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => _type.Call(instance, memberName, args); - public IMember CreateInstance(string typeName, LocationInfo location, IArgumentSet args ) => _type; + public IMember CreateInstance(string typeName, IArgumentSet args) => _type; public IMember GetMember(string name) => _type.GetMember(name); public IEnumerable GetMemberNames() => _type.GetMemberNames(); public IMember Index(IPythonInstance instance, object index) => _type.Index(instance, index); diff --git a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs index a344f89f3..37cca8722 100644 --- a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs +++ b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs @@ -35,28 +35,29 @@ internal sealed class ArgumentSet : IArgumentSet { private readonly List _errors = new List(); private readonly ListArg _listArgument; private readonly DictArg _dictArgument; - private readonly ExpressionEval _eval; private bool _evaluated; public static IArgumentSet Empty = new ArgumentSet(); + /// Module that declares the function + public IPythonModule DeclaringModule { get; } public IReadOnlyList Arguments => _arguments; public IListArgument ListArgument => _listArgument; public IDictionaryArgument DictionaryArgument => _dictArgument; public IReadOnlyList Errors => _errors; public int OverloadIndex { get; } - public IExpressionEvaluator Eval => _eval; + public IExpressionEvaluator Eval { get; } private ArgumentSet() { } public ArgumentSet(IReadOnlyList typeArgs) { - _arguments = typeArgs.Select(t => new Argument(t, LocationInfo.Empty)).ToList(); + _arguments = typeArgs.Select(t => new Argument(t)).ToList(); _evaluated = true; } public ArgumentSet(IReadOnlyList memberArgs) { - _arguments = memberArgs.Select(t => new Argument(t, LocationInfo.Empty)).ToList(); + _arguments = memberArgs.Select(t => new Argument(t)).ToList(); _evaluated = true; } @@ -75,9 +76,10 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in /// Call expression that invokes the function. /// Module that contains the call expression. /// Evaluator that can calculate values of arguments from their respective expressions. - public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance instance, CallExpression callExpr, IPythonModule module, ExpressionEval eval) { - _eval = eval; + public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance instance, CallExpression callExpr, IPythonModule module, IExpressionEvaluator eval) { + Eval = eval; OverloadIndex = overloadIndex; + DeclaringModule = fn.DeclaringModule; var overload = fn.Overloads[overloadIndex]; var fd = overload.FunctionDefinition; @@ -89,12 +91,12 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in _arguments = new List(); for (var i = 0; i < callExpr.Args.Count; i++) { var name = callExpr.Args[i].Name; - var location = fd != null && i < fd.Parameters.Length ? fd.Parameters[i].GetLocation(fn.DeclaringModule) : LocationInfo.Empty; if (string.IsNullOrEmpty(name)) { name = fd != null && i < fd.Parameters.Length ? fd.Parameters[i].Name : null; } name = name ?? $"arg{i}"; - _arguments.Add(new Argument(name, ParameterKind.Normal, callExpr.Args[i].Expression, null, location)); + var parameter = fd != null && i < fd.Parameters.Length ? fd.Parameters[i] : null; + _arguments.Add(new Argument(name, ParameterKind.Normal, callExpr.Args[i].Expression, null, parameter)); } return; } @@ -113,7 +115,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in // had values assigned to them are marked as 'filled'.Slots which have // no value assigned to them yet are considered 'empty'. - var slots = fd.Parameters.Select(p => new Argument(p, p.GetLocation(module))).ToArray(); + var slots = fd.Parameters.Select(p => new Argument(p, p)).ToArray(); // Locate sequence argument, if any var sa = slots.Where(s => s.Kind == ParameterKind.List).ToArray(); if (sa.Length > 1) { @@ -252,8 +254,10 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in _errors.Add(new DiagnosticsEntry(Resources.Analysis_ParameterMissing.FormatUI(slot.Name), callLocation.Span, ErrorCodes.ParameterMissing, Severity.Warning, DiagnosticSource.Analysis)); } - + // Note that parameter default value expression is from the function definition AST + // while actual argument values are from the calling file AST. slot.ValueExpression = parameter.DefaultValue; + slot.ValueIsDefault = true; } } } finally { @@ -268,20 +272,20 @@ public ArgumentSet Evaluate() { } foreach (var a in _arguments.Where(x => x.Value == null)) { - a.Value = Eval.GetValueFromExpression(a.ValueExpression) ?? _eval.UnknownType; + a.Value = GetArgumentValue(a); a.Type = Eval.GetValueFromExpression(a.TypeExpression) as IPythonType; } if (_listArgument != null) { foreach (var e in _listArgument.Expressions) { - var value = Eval.GetValueFromExpression(e) ?? _eval.UnknownType; + var value = Eval.GetValueFromExpression(e) ?? Eval.UnknownType; _listArgument._Values.Add(value); } } if (_dictArgument != null) { foreach (var e in _dictArgument.Expressions) { - var value = Eval.GetValueFromExpression(e.Value) ?? _eval.UnknownType; + var value = Eval.GetValueFromExpression(e.Value) ?? Eval.UnknownType; _dictArgument._Args[e.Key] = value; } } @@ -290,19 +294,70 @@ public ArgumentSet Evaluate() { return this; } + private IMember GetArgumentValue(Argument arg) { + // Evaluates expression in the specific module context. Typically used to evaluate + // expressions representing default values of function arguments since they are + // are defined in the function declaring module rather than in the caller context. + if (arg.ValueExpression == null) { + return Eval.UnknownType; + } + + if (arg.ValueIsDefault) { + using (Eval.OpenScope(DeclaringModule.Analysis.GlobalScope)) { + return Eval.GetValueFromExpression(arg.ValueExpression) ?? Eval.UnknownType; + } + } + return Eval.GetValueFromExpression(arg.ValueExpression) ?? Eval.UnknownType; + } + private sealed class Argument : IArgument { + /// + /// Argument name. + /// public string Name { get; } + + /// + /// Argument value, if known. + /// public object Value { get; internal set; } + + /// + /// Argument kind, . + /// public ParameterKind Kind { get; } + + /// + /// Expression that represents value of the argument in the + /// call expression. . + /// public Expression ValueExpression { get; set; } - public LocationInfo Location { get; } + + /// + /// Indicates if value is a default value expression. Default values + /// should be evaluated in the context of the file that declared + /// the function rather than in the caller context. + /// + public bool ValueIsDefault { get; set; } + + /// + /// Type of the argument, if annotated. + /// public IPythonType Type { get; internal set; } + + /// + /// Type annotation expression. + /// public Expression TypeExpression { get; } - public Argument(Parameter p, LocationInfo location) : + /// + /// AST node that defines the argument. + /// + public Node Location { get; } + + public Argument(Parameter p, Node location) : this(p.Name, p.Kind, null, p.Annotation, location) { } - public Argument(string name, ParameterKind kind, Expression valueValueExpression, Expression typeExpression, LocationInfo location) { + public Argument(string name, ParameterKind kind, Expression valueValueExpression, Expression typeExpression, Node location) { Name = name; Kind = kind; ValueExpression = valueValueExpression; @@ -310,20 +365,19 @@ public Argument(string name, ParameterKind kind, Expression valueValueExpression Location = location; } - public Argument(IPythonType type, LocationInfo location) : this(type.Name, type, location) { } - public Argument(IMember member, LocationInfo location) : this(string.Empty, member, location) { } + public Argument(IPythonType type) : this(type.Name, type) { } + public Argument(IMember member) : this(string.Empty, member) { } - private Argument(string name, object value, LocationInfo location) { + private Argument(string name, object value) { Name = name; Value = value; - Location = location; } } private sealed class ListArg : IListArgument { public string Name { get; } public Expression Expression { get; } - public LocationInfo Location { get; } + public Node Location { get; } public IReadOnlyList Values => _Values; public IReadOnlyList Expressions => _Expressions; @@ -331,7 +385,7 @@ private sealed class ListArg : IListArgument { public List _Values { get; } = new List(); public List _Expressions { get; } = new List(); - public ListArg(string name, Expression expression, LocationInfo location) { + public ListArg(string name, Expression expression, Node location) { Name = name; Expression = expression; Location = location; @@ -341,7 +395,7 @@ public ListArg(string name, Expression expression, LocationInfo location) { private sealed class DictArg : IDictionaryArgument { public string Name { get; } public Expression Expression { get; } - public LocationInfo Location { get; } + public Node Location { get; } public IReadOnlyDictionary Arguments => _Args; public IReadOnlyDictionary Expressions => _Expressions; @@ -349,10 +403,10 @@ private sealed class DictArg : IDictionaryArgument { public Dictionary _Args { get; } = new Dictionary(); public Dictionary _Expressions { get; } = new Dictionary(); - public DictArg(string name, Expression expression, LocationInfo location) { + public DictArg(string name, Expression expression, Node location) { Name = name; Expression = expression; - Location = location; + Location = Location; } } } diff --git a/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs b/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs index 67044546c..a3dc8979a 100644 --- a/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs +++ b/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs @@ -72,21 +72,21 @@ public override string Name { public override PythonMemberType MemberType => PythonMemberType.Class; public override IMember GetMember(string name) => name == @"__iter__" ? IteratorType : base.GetMember(name); - public override IMember CreateInstance(string typeName, LocationInfo location, IArgumentSet args) - => new PythonCollection(this, location, args.Arguments.Select(a => a.Value).OfType().ToArray()); + public override IMember CreateInstance(string typeName, IArgumentSet args) + => new PythonCollection(this, args.Arguments.Select(a => a.Value).OfType().ToArray()); // Constructor call public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) - => CreateInstance(Name, instance?.Location ?? LocationInfo.Empty, args); + => CreateInstance(Name, args); public override IMember Index(IPythonInstance instance, object index) => (instance as IPythonCollection)?.Index(index) ?? UnknownType; #endregion - public static IPythonCollection CreateList(IPythonInterpreter interpreter, LocationInfo location, IArgumentSet args) { + + public static IPythonCollection CreateList(IPythonInterpreter interpreter, IArgumentSet args) { var exact = true; IReadOnlyList contents; - if (args.Arguments.Count > 1) { // self and list like in list.__init__ and 'list([1, 'str', 3.0])' var arg = args.Arguments[1].Value as PythonCollection; @@ -95,32 +95,30 @@ public static IPythonCollection CreateList(IPythonInterpreter interpreter, Locat } else { contents = args.ListArgument?.Values; } - - return CreateList(interpreter, location, contents ?? Array.Empty(), exact: exact); + return CreateList(interpreter, contents ?? Array.Empty()); } - public static IPythonCollection CreateList(IPythonInterpreter interpreter, LocationInfo location, IReadOnlyList contents, bool flatten = true, bool exact = false) { + public static IPythonCollection CreateList(IPythonInterpreter interpreter, IReadOnlyList contents, bool flatten = true, bool exact = false) { var collectionType = new PythonCollectionType(null, BuiltinTypeId.List, interpreter, true); - return new PythonCollection(collectionType, location, contents, flatten, exact); + return new PythonCollection(collectionType, contents, flatten, exact: exact); } - public static IPythonCollection CreateConcatenatedList(IPythonInterpreter interpreter, LocationInfo location, params IPythonCollection[] many) { + public static IPythonCollection CreateConcatenatedList(IPythonInterpreter interpreter, params IPythonCollection[] many) { var exact = many?.All(c => c != null && c.IsExact) ?? false; var contents = many?.ExcludeDefault().Select(c => c.Contents).SelectMany().ToList() ?? new List(); - return CreateList(interpreter, location, contents, false, exact: exact); + return CreateList(interpreter, contents, false, exact: exact); } - - public static IPythonCollection CreateTuple(IPythonInterpreter interpreter, LocationInfo location, IReadOnlyList contents, bool exact = false) { + public static IPythonCollection CreateTuple(IPythonInterpreter interpreter, IReadOnlyList contents, bool exact = false) { var collectionType = new PythonCollectionType(null, BuiltinTypeId.Tuple, interpreter, false); - return new PythonCollection(collectionType, location, contents, exact: exact); + return new PythonCollection(collectionType, contents, exact: exact); } - public static IPythonCollection CreateSet(IPythonInterpreter interpreter, LocationInfo location, IReadOnlyList contents, bool flatten = true, bool exact = false) { + public static IPythonCollection CreateSet(IPythonInterpreter interpreter, IReadOnlyList contents, bool flatten = true, bool exact = false) { var collectionType = new PythonCollectionType(null, BuiltinTypeId.Set, interpreter, true); - return new PythonCollection(collectionType, location, contents, flatten, exact: exact); + return new PythonCollection(collectionType, contents, flatten, exact: exact); } public override bool Equals(object obj) - => obj is IPythonType pt && (PythonTypeComparer.Instance.Equals(pt, this) || PythonTypeComparer.Instance.Equals(pt, this.InnerType)); + => obj is IPythonType pt && (PythonTypeComparer.Instance.Equals(pt, this) || PythonTypeComparer.Instance.Equals(pt, InnerType)); public override int GetHashCode() => PythonTypeComparer.Instance.GetHashCode(this); } } diff --git a/src/Analysis/Ast/Impl/Types/Collections/PythonDictionaryType.cs b/src/Analysis/Ast/Impl/Types/Collections/PythonDictionaryType.cs index df6433fff..7b2efc92e 100644 --- a/src/Analysis/Ast/Impl/Types/Collections/PythonDictionaryType.cs +++ b/src/Analysis/Ast/Impl/Types/Collections/PythonDictionaryType.cs @@ -24,16 +24,16 @@ public PythonDictionaryType(IPythonInterpreter interpreter, bool isMutable = tru : base(null, BuiltinTypeId.Dict, interpreter, isMutable) { } - public override IMember CreateInstance(string typeName, LocationInfo location, IArgumentSet args) { + public override IMember CreateInstance(string typeName, IArgumentSet args) { var contents = args.Arguments.Count == 1 ? args.Arguments[0].Value as IReadOnlyDictionary : EmptyDictionary.Instance; - return new PythonDictionary(this, location, contents); + return new PythonDictionary(this, contents); } // Constructor call public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) - => CreateInstance(Name, instance?.Location ?? LocationInfo.Empty, args); + => CreateInstance(Name, args); public override BuiltinTypeId TypeId => BuiltinTypeId.Dict; public override PythonMemberType MemberType => PythonMemberType.Class; diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs b/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs index 83650a185..e4b952b6c 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs @@ -34,12 +34,6 @@ public interface IArgument { /// Expression ValueExpression { get; } - /// - /// Location of the argument in the function definition. - /// Not the same as location of the . - /// - LocationInfo Location { get; } - /// /// Value of the argument. /// @@ -54,6 +48,11 @@ public interface IArgument { /// Annotation expression. /// Expression TypeExpression { get; } + + /// + /// Parameter location in the AST. + /// + Node Location { get; } } /// @@ -71,12 +70,6 @@ public interface IListArgument { /// Expression Expression { get; } - /// - /// Location of the argument in the function definition. - /// Not the same as location of the . - /// - LocationInfo Location { get; } - /// /// Expressions that evaluate to the elements of the list. /// @@ -86,6 +79,11 @@ public interface IListArgument { /// Values of the elements of the list. /// IReadOnlyList Values { get; } + + /// + /// Parameter location in the AST. + /// + Node Location { get; } } /// @@ -103,12 +101,6 @@ public interface IDictionaryArgument { /// Expression Expression { get; } - /// - /// Location of the argument in the function definition. - /// Not the same as location of the . - /// - LocationInfo Location { get; } - /// /// Dictionary arguments. /// @@ -119,6 +111,11 @@ public interface IDictionaryArgument { /// Function call parameters. /// IReadOnlyDictionary Expressions { get; } + + /// + /// Parameter location in the AST. + /// + Node Location { get; } } /// diff --git a/src/Analysis/Ast/Impl/Types/Definitions/ILocatedMember.cs b/src/Analysis/Ast/Impl/Types/Definitions/ILocatedMember.cs index 9a87ac5e7..e585c4092 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/ILocatedMember.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/ILocatedMember.cs @@ -13,15 +13,48 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Collections.Generic; +using Microsoft.Python.Analysis.Values; + namespace Microsoft.Python.Analysis.Types { /// /// Provides the location of a member. This should be implemented on a class - /// which also implements IPythonType. + /// which also implements or . /// public interface ILocatedMember: IMember { /// - /// Returns where the member is located or null if the location is not known. + /// Module that defines the member. + /// + IPythonModule DeclaringModule { get; } + + /// + /// Location where the member is defined. + /// + Location Location { get; } + + /// + /// Location where the member is defined. + /// + LocationInfo Definition { get; } + + /// + /// Link to the primary definition such as when variable is imported from another file. + /// + ILocatedMember Parent { get; } + + /// + /// List of references to the member. + /// + IReadOnlyList References { get; } + + /// + /// Add member reference. + /// + void AddReference(Location location); + + /// + /// Removes references to the module variable recorded in other modules. /// - LocationInfo Location { get; } + void RemoveReferences(IPythonModule module); } } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IMemberContainer.cs b/src/Analysis/Ast/Impl/Types/Definitions/IMemberContainer.cs index 62d77e272..3a1227584 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IMemberContainer.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IMemberContainer.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System.Collections.Generic; -using System.Runtime.InteropServices.ComTypes; namespace Microsoft.Python.Analysis.Types { /// diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonFunctionOverload.cs index aab6fa5bf..24799ad81 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonFunctionOverload.cs @@ -54,7 +54,7 @@ public interface IPythonFunctionOverload { /// Invoking class instance. In case of generics it is instance of the specific type /// as opposed to declaring type which is the generic template class. /// Call expression location, if any. - IMember Call(IArgumentSet args, IPythonType self, LocationInfo callLocation = null); + IMember Call(IArgumentSet args, IPythonType self, Node callLocation = null); /// /// Return value documentation. diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonTemplateType.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonTemplateType.cs index af36af8b7..19fb01272 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonTemplateType.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonTemplateType.cs @@ -23,6 +23,6 @@ public interface IPythonTemplateType: IPythonType { /// Creates instance of a type information with the specific /// type arguments from a generic template. /// - IPythonType CreateSpecificType(IArgumentSet typeArguments, IPythonModule declaringModule, LocationInfo location); + IPythonType CreateSpecificType(IArgumentSet typeArguments); } } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonType.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonType.cs index 0231dd9fb..2f3cb2aa0 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonType.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonType.cs @@ -19,17 +19,12 @@ namespace Microsoft.Python.Analysis.Types { /// /// Type information of an instance. /// - public interface IPythonType : IMember, IMemberContainer { + public interface IPythonType : ILocatedMember, IMemberContainer { /// /// Type name. /// string Name { get; } - /// - /// Module the type is declared in. - /// - IPythonModule DeclaringModule { get; } - /// /// Indicates built-in type id such as 'int' or 'str' /// or 'type' for user-defined entities. @@ -61,9 +56,8 @@ public interface IPythonType : IMember, IMemberContainer { /// /// Name of the type. Used in specialization scenarios /// where constructor may want to create specialized type. - /// Instance location /// Any custom arguments required to create the instance. - IMember CreateInstance(string typeName = null, LocationInfo location = null, IArgumentSet args = null); + IMember CreateInstance(string typeName = null, IArgumentSet args = null); /// /// Invokes method or property on the specified instance. diff --git a/src/Analysis/Ast/Impl/Types/Definitions/LocationInfo.cs b/src/Analysis/Ast/Impl/Types/Definitions/LocationInfo.cs index 375e522c6..05fac0948 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/LocationInfo.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/LocationInfo.cs @@ -29,6 +29,10 @@ public LocationInfo(string path, Uri documentUri, int line, int column) : this(path, documentUri, line, column, null, null) { } + public LocationInfo(string path, Uri documentUri, SourceSpan span) : + this(path, documentUri, span.Start.Line, span.Start.Column, span.End.Line, span.End.Column) { + } + public LocationInfo(string path, Uri documentUri, int line, int column, int? endLine, int? endColumn) { FilePath = path; DocumentUri = documentUri; diff --git a/src/Analysis/Ast/Impl/Types/LocatedMember.cs b/src/Analysis/Ast/Impl/Types/LocatedMember.cs new file mode 100644 index 000000000..33bb79a0d --- /dev/null +++ b/src/Analysis/Ast/Impl/Types/LocatedMember.cs @@ -0,0 +1,105 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Modules; + +namespace Microsoft.Python.Analysis.Types { + internal abstract class LocatedMember : ILocatedMember { + private HashSet _references; + + protected LocatedMember(PythonMemberType memberType) { + MemberType = memberType; + } + + protected LocatedMember(PythonMemberType memberType, IPythonModule module) + : this(memberType, new Location(module, default)) { } + + protected LocatedMember(PythonMemberType memberType, Location location, ILocatedMember parent = null) + : this(location, parent) { + MemberType = memberType; + } + + private LocatedMember(Location location, ILocatedMember parent = null) { + Parent = parent; + Parent?.AddReference(location); + Location = location; + } + + public virtual PythonMemberType MemberType { get; } = PythonMemberType.Unknown; + + public virtual IPythonModule DeclaringModule => Location.Module; + + public virtual LocationInfo Definition => Location.LocationInfo; + + public ILocatedMember Parent { get; } + + public virtual IReadOnlyList References { + get { + lock (this) { + if (_references == null) { + return new[] { Definition }; + } + + var refs = _references + .GroupBy(x => x.LocationInfo.DocumentUri) + .SelectMany(g => g.Select(x => x.LocationInfo).OrderBy(x => x.Span)); + return Enumerable.Repeat(Definition, 1).Concat(refs).ToArray(); + } + } + } + + public virtual void AddReference(Location location) { + lock (this) { + // Don't add references to library code. + if (location.Module?.ModuleType == ModuleType.User && !location.Equals(Location)) { + _references = _references ?? new HashSet(); + _references.Add(location); + } + } + } + + public virtual void RemoveReferences(IPythonModule module) { + lock (this) { + if (_references != null) { + foreach (var r in _references.ToArray().Where(r => r.Module == module)) { + _references.Remove(r); + } + } + } + } + + public Location Location { get; internal set; } + + protected void SetDeclaringModule(IPythonModule module) => Location = new Location(module, Location.IndexSpan); + } + + internal abstract class EmptyLocatedMember : ILocatedMember { + protected EmptyLocatedMember(PythonMemberType memberType) { + MemberType = memberType; + } + + public PythonMemberType MemberType { get; } + public IPythonModule DeclaringModule => null; + public LocationInfo Definition => LocationInfo.Empty; + public ILocatedMember Parent => null; + public IReadOnlyList References => Array.Empty(); + public void AddReference(Location location) { } + public void RemoveReferences(IPythonModule module) { } + public Location Location { get; internal set; } + } +} diff --git a/src/Analysis/Ast/Impl/Types/Location.cs b/src/Analysis/Ast/Impl/Types/Location.cs new file mode 100644 index 000000000..d875216ca --- /dev/null +++ b/src/Analysis/Ast/Impl/Types/Location.cs @@ -0,0 +1,42 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Core.Text; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Types { + public struct Location { + public Location(IPythonModule module) : this(module, default) { } + + public Location(IPythonModule module, IndexSpan indexSpan) { + Module = module; + IndexSpan = indexSpan; + } + + public IPythonModule Module { get; } + public IndexSpan IndexSpan { get; } + + public LocationInfo LocationInfo => Module?.Analysis?.Ast != null + ? new LocationInfo(Module.FilePath, Module.Uri, IndexSpan.ToSourceSpan(Module.Analysis?.Ast)) + : LocationInfo.Empty; + + public bool IsValid => Module != null; + + public override bool Equals(object obj) + => obj is Location other && other.Module == Module && other.IndexSpan == IndexSpan; + + public override int GetHashCode() => (IndexSpan.GetHashCode() * 397) ^ Module?.GetHashCode() ?? 0; + } +} diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs index 4aeb45d4b..dded0f17d 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs @@ -25,6 +25,7 @@ using Microsoft.Python.Analysis.Values; using Microsoft.Python.Analysis.Values.Collections; using Microsoft.Python.Core; +using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; @@ -39,15 +40,17 @@ internal class PythonClassType : PythonType, IPythonClassType, IPythonTemplateTy private Dictionary _genericParameters; // For tests - internal PythonClassType(string name, IPythonModule declaringModule, LocationInfo location = null) - : base(name, declaringModule, string.Empty, location ?? LocationInfo.Empty, BuiltinTypeId.Type) { } + internal PythonClassType(string name, Location location) + : base(name, location, string.Empty, BuiltinTypeId.Type) { + Check.ArgumentNotNull(nameof(location), location.Module); + } public PythonClassType( ClassDefinition classDefinition, - IPythonModule declaringModule, - LocationInfo location, + Location location, BuiltinTypeId builtinTypeId = BuiltinTypeId.Type - ) : base(classDefinition.Name, declaringModule, classDefinition.GetDocumentation(), location, builtinTypeId) { + ) : base(classDefinition.Name, location, classDefinition.GetDocumentation(), builtinTypeId) { + Check.ArgumentNotNull(nameof(location), location.Module); ClassDefinition = classDefinition; } @@ -77,7 +80,7 @@ public override IMember GetMember(string name) { switch (name) { case "__mro__": case "mro": - return is3x ? PythonCollectionType.CreateList(DeclaringModule.Interpreter, LocationInfo.Empty, Mro) : UnknownType; + return is3x ? PythonCollectionType.CreateList(DeclaringModule.Interpreter, Mro) : UnknownType; case "__dict__": return is3x ? DeclaringModule.Interpreter.GetBuiltinType(BuiltinTypeId.Dict) : UnknownType; case @"__weakref__": @@ -118,22 +121,22 @@ public override string Documentation { } // Constructor call - public override IMember CreateInstance(string typeName, LocationInfo location, IArgumentSet args) { + public override IMember CreateInstance(string typeName, IArgumentSet args) { // Specializations switch (typeName) { case "list": - return PythonCollectionType.CreateList(DeclaringModule.Interpreter, location, args); + return PythonCollectionType.CreateList(DeclaringModule.Interpreter, args); case "dict": { // self, then contents var contents = args.Values().Skip(1).FirstOrDefault(); - return new PythonDictionary(DeclaringModule.Interpreter, location, contents); + return new PythonDictionary(DeclaringModule.Interpreter, contents); } case "tuple": { var contents = args.Values(); - return PythonCollectionType.CreateTuple(DeclaringModule.Interpreter, location, contents); + return PythonCollectionType.CreateTuple(DeclaringModule.Interpreter, contents); } } - return new PythonInstance(this, location); + return new PythonInstance(this); } public override IMember Index(IPythonInstance instance, object index) { @@ -202,7 +205,7 @@ internal void SetBases(IEnumerable bases) { return; } - AddMember("__bases__", PythonCollectionType.CreateList(DeclaringModule.Interpreter, LocationInfo.Empty, _bases), true); + AddMember("__bases__", PythonCollectionType.CreateList(DeclaringModule.Interpreter, _bases), true); } } @@ -272,8 +275,7 @@ private bool Push(IPythonClassType cls) { public bool Equals(IPythonClassType other) => Name == other?.Name && DeclaringModule.Equals(other?.DeclaringModule); - public IPythonType CreateSpecificType(IArgumentSet args, IPythonModule declaringModule, LocationInfo location = null) { - location = location ?? LocationInfo.Empty; + public IPythonType CreateSpecificType(IArgumentSet args) { // Get declared generic parameters of the class, i.e. list of Ts in Generic[T1, T2, ...] var genericClassParameters = Bases.OfType().ToArray(); // Optimistically use the first one @@ -364,7 +366,7 @@ public IPythonType CreateSpecificType(IArgumentSet args, IPythonModule declaring .ToArray(); var specificName = CodeFormatter.FormatSequence(Name, '[', specificTypes); - var classType = new PythonClassType(specificName, declaringModule); + var classType = new PythonClassType(specificName, new Location(DeclaringModule)); // Methods returning generic types need to know how to match generic // parameter name to the actual supplied type. @@ -392,7 +394,7 @@ public IPythonType CreateSpecificType(IArgumentSet args, IPythonModule declaring .Where(p => !p.IsUnknown()) .ToArray(); if (st.Length > 0) { - var type = gt.CreateSpecificType(new ArgumentSet(st), classType.DeclaringModule, location); + var type = gt.CreateSpecificType(new ArgumentSet(st)); if (!type.IsUnknown()) { bases.Add(type); } @@ -401,7 +403,7 @@ public IPythonType CreateSpecificType(IArgumentSet args, IPythonModule declaring // Set specific class bases classType.SetBases(bases.Concat(newBases)); // Transfer members from generic to specific type. - SetClassMembers(classType, args, declaringModule, location); + SetClassMembers(classType, args); } finally { Pop(); } @@ -461,7 +463,7 @@ private void GetSpecificTypeFromArgumentValue(IGenericType gt, object argumentVa /// Transfers members from generic class to the specific class type /// while instantiating specific types for the members. /// - private void SetClassMembers(PythonClassType classType, IArgumentSet args, IPythonModule declaringModule, LocationInfo location) { + private void SetClassMembers(PythonClassType classType, IArgumentSet args) { // Add members from the template class (this one). // Members must be clones rather than references since // we are going to set specific types on them. @@ -477,7 +479,7 @@ private void SetClassMembers(PythonClassType classType, IArgumentSet args, IPyth foreach (var m in members) { switch (m.Value) { case IPythonTemplateType tt: { - var specificType = tt.CreateSpecificType(args, declaringModule, location); + var specificType = tt.CreateSpecificType(args); classType.AddMember(m.Key, specificType, true); break; } @@ -486,7 +488,7 @@ private void SetClassMembers(PythonClassType classType, IArgumentSet args, IPyth IPythonType specificType = null; switch (t) { case IPythonTemplateType tt when tt.IsGeneric(): - specificType = tt.CreateSpecificType(args, declaringModule, location); + specificType = tt.CreateSpecificType(args); break; case IGenericTypeDefinition gtd: classType.GenericParameters.TryGetValue(gtd.Name, out specificType); @@ -494,7 +496,7 @@ private void SetClassMembers(PythonClassType classType, IArgumentSet args, IPyth } if (specificType != null) { - classType.AddMember(m.Key, new PythonInstance(specificType, location), true); + classType.AddMember(m.Key, new PythonInstance(specificType), true); } break; } diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index 087894c73..94773c43b 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -30,18 +30,14 @@ namespace Microsoft.Python.Analysis.Types { /// /// Module making the call. /// Function overload the return value is requested for. - /// Call location, if any. /// Call arguments. /// public delegate IMember ReturnValueProvider( IPythonModule declaringModule, IPythonFunctionOverload overload, - LocationInfo location, IArgumentSet args); - internal sealed class PythonFunctionOverload : IPythonFunctionOverload, ILocatedMember { - private readonly Func _locationProvider; - private readonly IPythonModule _declaringModule; + internal sealed class PythonFunctionOverload : LocatedMember, IPythonFunctionOverload { private readonly string _returnDocumentation; // Allow dynamic function specialization, such as defining return types for builtin @@ -54,18 +50,17 @@ internal sealed class PythonFunctionOverload : IPythonFunctionOverload, ILocated private Func _documentationProvider; private bool _fromAnnotation; - public PythonFunctionOverload(FunctionDefinition fd, IPythonClassMember classMember, IPythonModule declaringModule, LocationInfo location) - : this(fd.Name, declaringModule, _ => location) { + public PythonFunctionOverload(FunctionDefinition fd, IPythonClassMember classMember, Location location) + : this(fd.Name, location) { FunctionDefinition = fd; ClassMember = classMember; - var ast = (declaringModule as IDocument)?.Analysis.Ast; + var ast = (location.Module as IDocument)?.Analysis.Ast; _returnDocumentation = ast != null ? fd.ReturnAnnotation?.ToCodeString(ast) : null; } - public PythonFunctionOverload(string name, IPythonModule declaringModule, Func locationProvider) { + public PythonFunctionOverload(string name, Location location) + : base(PythonMemberType.Function, location) { Name = name ?? throw new ArgumentNullException(nameof(name)); - _declaringModule = declaringModule; - _locationProvider = locationProvider; } internal void SetParameters(IReadOnlyList parameters) => Parameters = parameters; @@ -127,7 +122,7 @@ public string GetReturnDocumentation(IPythonType self = null) { .Select(n => cls.GenericParameters.TryGetValue(n, out var t) ? t : null) .ExcludeDefault() .ToArray(); - var specificReturnValue = cls.CreateSpecificType(new ArgumentSet(typeArgs), _declaringModule); + var specificReturnValue = cls.CreateSpecificType(new ArgumentSet(typeArgs)); return specificReturnValue.Name; } case IGenericTypeDefinition gtp1 when self is IPythonClassType cls: { @@ -138,7 +133,7 @@ public string GetReturnDocumentation(IPythonType self = null) { // Try returning the constraint // TODO: improve this, the heuristic is pretty basic and tailored to simple func(_T) -> _T var name = StaticReturnValue.GetPythonType()?.Name; - var typeDefVar = _declaringModule.Analysis.GlobalScope.Variables[name]; + var typeDefVar = DeclaringModule.Analysis.GlobalScope.Variables[name]; if (typeDefVar?.Value is IGenericTypeDefinition gtp2) { var t = gtp2.Constraints.FirstOrDefault(); if (t != null) { @@ -152,15 +147,13 @@ public string GetReturnDocumentation(IPythonType self = null) { } public IReadOnlyList Parameters { get; private set; } = Array.Empty(); - public LocationInfo Location => _locationProvider?.Invoke(Name) ?? LocationInfo.Empty; - public PythonMemberType MemberType => PythonMemberType.Function; + public override PythonMemberType MemberType => PythonMemberType.Function; public IMember StaticReturnValue { get; private set; } - public IMember Call(IArgumentSet args, IPythonType self, LocationInfo callLocation = null) { - callLocation = callLocation ?? LocationInfo.Empty; + public IMember Call(IArgumentSet args, IPythonType self, Node callLocation = null) { if (!_fromAnnotation) { // First try supplied specialization callback. - var rt = _returnValueProvider?.Invoke(_declaringModule, this, callLocation, args); + var rt = _returnValueProvider?.Invoke(DeclaringModule, this, args); if (!rt.IsUnknown()) { return rt; } @@ -193,20 +186,20 @@ public IMember Call(IArgumentSet args, IPythonType self, LocationInfo callLocati } if (typeArgs != null) { - var specificReturnValue = cls.CreateSpecificType(new ArgumentSet(typeArgs), _declaringModule, callLocation); - return new PythonInstance(specificReturnValue, callLocation); + var specificReturnValue = cls.CreateSpecificType(new ArgumentSet(typeArgs)); + return new PythonInstance(specificReturnValue); } break; case IGenericTypeDefinition gtp1: { // -> _T if (selfClassType.GenericParameters.TryGetValue(gtp1.Name, out var specificType)) { - return new PythonInstance(specificType, callLocation); + return new PythonInstance(specificType); } // Try returning the constraint // TODO: improve this, the heuristic is pretty basic and tailored to simple func(_T) -> _T var name = StaticReturnValue.GetPythonType()?.Name; - var typeDefVar = _declaringModule.Analysis.GlobalScope.Variables[name]; + var typeDefVar = DeclaringModule.Analysis.GlobalScope.Variables[name]; if (typeDefVar?.Value is IGenericTypeDefinition gtp2) { return gtp2.Constraints.FirstOrDefault(); } diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs index b87ad2330..26674cc1c 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs @@ -20,6 +20,7 @@ using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Core.Collections; +using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Types { @@ -35,11 +36,11 @@ internal class PythonFunctionType : PythonType, IPythonFunctionType { /// Creates function for specializations /// public static PythonFunctionType ForSpecialization(string name, IPythonModule declaringModule) - => new PythonFunctionType(name, declaringModule, true); + => new PythonFunctionType(name, new Location(declaringModule, default), true); - private PythonFunctionType(string name, IPythonModule declaringModule, bool isSpecialized = false) : - base(name, declaringModule, null, LocationInfo.Empty, BuiltinTypeId.Function) { - DeclaringType = declaringModule; + private PythonFunctionType(string name, Location location, bool isSpecialized = false) : + base(name, location, string.Empty, BuiltinTypeId.Function) { + Check.ArgumentNotNull(nameof(location), location.Module); _isSpecialized = isSpecialized; } @@ -50,11 +51,12 @@ private PythonFunctionType(string name, IPythonModule declaringModule, bool isSp /// public PythonFunctionType( string name, - IPythonModule declaringModule, + Location location, IPythonType declaringType, - string documentation, - LocationInfo location = null - ) : this(name, declaringModule, declaringType, _ => documentation, _ => location ?? LocationInfo.Empty) { } + string documentation + ) : this(name, location, declaringType, _ => documentation) { + Check.ArgumentNotNull(nameof(location), location.Module); + } /// /// Creates function type to use in special cases when function is dynamically @@ -63,26 +65,19 @@ public PythonFunctionType( /// public PythonFunctionType( string name, - IPythonModule declaringModule, + Location location, IPythonType declaringType, - Func documentationProvider, - Func locationProvider, - IPythonFunctionOverload overload = null - ) : base(name, declaringModule, documentationProvider, locationProvider, - declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { + Func documentationProvider + ) : base(name, location, documentationProvider, declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { DeclaringType = declaringType; - if (overload != null) { - AddOverload(overload); - } } public PythonFunctionType( FunctionDefinition fd, - IPythonModule declaringModule, IPythonType declaringType, - LocationInfo location = null - ) : base(fd.Name, declaringModule, fd.Documentation, location ?? LocationInfo.Empty, - declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { + Location location + ) : base(fd.Name, location, fd.Documentation, + declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { FunctionDefinition = fd; DeclaringType = declaringType; @@ -101,7 +96,7 @@ public override PythonMemberType MemberType public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) { // Now we can go and find overload with matching arguments. var overload = Overloads[args.OverloadIndex]; - return overload?.Call(args, instance?.GetPythonType() ?? DeclaringType, instance?.Location); + return overload?.Call(args, instance?.GetPythonType() ?? DeclaringType); } internal override void SetDocumentationProvider(Func provider) { diff --git a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs index 6d91a8d74..87ee5f713 100644 --- a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs @@ -21,13 +21,13 @@ namespace Microsoft.Python.Analysis.Types { class PythonPropertyType : PythonType, IPythonPropertyType { private IPythonFunctionOverload _getter; - public PythonPropertyType(FunctionDefinition fd, IPythonModule declaringModule, IPythonType declaringType, bool isAbstract, LocationInfo location) - : this(fd.Name, declaringModule, declaringType, isAbstract, location) { + public PythonPropertyType(FunctionDefinition fd, Location location, IPythonType declaringType, bool isAbstract) + : this(fd.Name, location, declaringType, isAbstract) { FunctionDefinition = fd; } - public PythonPropertyType(string name, IPythonModule declaringModule, IPythonType declaringType, bool isAbstract, LocationInfo location) - : base(name, declaringModule, null, location) { + public PythonPropertyType(string name, Location location, IPythonType declaringType, bool isAbstract) + : base(name, location, string.Empty, BuiltinTypeId.Property) { DeclaringType = declaringType; IsAbstract = isAbstract; } @@ -44,7 +44,7 @@ public PythonPropertyType(string name, IPythonModule declaringModule, IPythonTyp public string Description => Type == null ? Resources.PropertyOfUnknownType : Resources.PropertyOfType.FormatUI(Type.Name); public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) - => _getter.Call(args, instance?.GetPythonType() ?? DeclaringType, instance?.Location); + => _getter.Call(args, instance?.GetPythonType() ?? DeclaringType); #endregion internal void AddOverload(IPythonFunctionOverload overload) => _getter = _getter ?? overload; diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index a18cc7d9e..252165952 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -18,16 +18,15 @@ using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core.Diagnostics; namespace Microsoft.Python.Analysis.Types { [DebuggerDisplay("{Name}")] - internal class PythonType : IPythonType, ILocatedMember, IHasQualifiedName, IEquatable { + internal class PythonType : LocatedMember, IPythonType, IHasQualifiedName, IEquatable { private readonly object _lock = new object(); private readonly string _name; private Func _documentationProvider; - private readonly Func _locationProvider; private string _documentation; - private LocationInfo _location; private Dictionary _members; private BuiltinTypeId _typeId; private bool _readonly; @@ -39,29 +38,26 @@ internal class PythonType : IPythonType, ILocatedMember, IHasQualifiedName, IEqu public PythonType( string name, - IPythonModule declaringModule, + Location location, string documentation, - LocationInfo location, BuiltinTypeId typeId = BuiltinTypeId.Unknown - ) : this(name, declaringModule, typeId) { + ) : this(name, location, typeId) { _documentation = documentation; - _location = location; } public PythonType( string name, - IPythonModule declaringModule, + Location location, Func documentationProvider, - Func locationProvider, BuiltinTypeId typeId = BuiltinTypeId.Unknown - ) : this(name, declaringModule, typeId) { + ) : this(name, location, typeId) { _documentationProvider = documentationProvider; - _locationProvider = locationProvider; } - private PythonType(string name, IPythonModule declaringModule, BuiltinTypeId typeId = BuiltinTypeId.Unknown) { + private PythonType(string name, Location location, BuiltinTypeId typeId) + : base(typeId.GetMemberId(), location) { + Check.ArgumentNotNull(nameof(location), location.Module); _name = name ?? throw new ArgumentNullException(nameof(name)); - DeclaringModule = declaringModule; _typeId = typeId; } @@ -69,8 +65,6 @@ private PythonType(string name, IPythonModule declaringModule, BuiltinTypeId typ public virtual string Name => TypeId == BuiltinTypeId.Ellipsis ? "..." : _name; public virtual string Documentation => _documentationProvider != null ? _documentationProvider.Invoke(Name) : _documentation; - public IPythonModule DeclaringModule { get; } - public virtual PythonMemberType MemberType => _typeId.GetMemberId(); public virtual BuiltinTypeId TypeId => _typeId; public bool IsBuiltin => DeclaringModule == null || DeclaringModule is IBuiltinsPythonModule; public virtual bool IsAbstract => false; @@ -81,10 +75,8 @@ private PythonType(string name, IPythonModule declaringModule, BuiltinTypeId typ /// /// Name of the type. Used in specialization scenarios /// where constructor may want to create specialized type. - /// Instance location /// Any custom arguments required to create the instance. - public virtual IMember CreateInstance(string typeName, LocationInfo location, IArgumentSet args) - => new PythonInstance(this, location); + public virtual IMember CreateInstance(string typeName, IArgumentSet args) => new PythonInstance(this); /// /// Invokes method or property on the specified instance. @@ -103,11 +95,6 @@ public virtual IMember Call(IPythonInstance instance, string memberName, IArgume public virtual IMember Index(IPythonInstance instance, object index) => instance?.Index(index) ?? UnknownType; #endregion - #region ILocatedMember - public virtual LocationInfo Location => _locationProvider != null - ? _locationProvider?.Invoke(Name) : _location ?? LocationInfo.Empty; - #endregion - #region IHasQualifiedName public virtual string FullyQualifiedName => FullyQualifiedNamePair.CombineNames(); public virtual KeyValuePair FullyQualifiedNamePair @@ -129,12 +116,13 @@ internal bool TrySetTypeId(BuiltinTypeId typeId) { internal virtual void SetDocumentationProvider(Func provider) => _documentationProvider = provider; internal virtual void SetDocumentation(string documentation) => _documentation = documentation; - internal virtual void SetLocation(LocationInfo location) => _location = location; internal void AddMembers(IEnumerable variables, bool overwrite) { lock (_lock) { if (!_readonly) { foreach (var v in variables.Where(m => overwrite || !Members.ContainsKey(m.Name))) { + // If variable holds function or a class, use value as member. + // If it holds an instance, use the variable itself (i.e. it is a data member). WritableMembers[v.Name] = v.Value; } } diff --git a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs index 0a47f8d7a..9ba61a893 100644 --- a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs +++ b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs @@ -16,12 +16,13 @@ using System; using System.Collections.Generic; using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core.Text; namespace Microsoft.Python.Analysis.Types { /// /// Delegates most of the methods to the wrapped/inner class. /// - internal class PythonTypeWrapper : IPythonType, ILocatedMember, IHasQualifiedName { + internal class PythonTypeWrapper : IPythonType, IHasQualifiedName { private readonly BuiltinTypeId _builtinTypeId; private IPythonType _innerType; @@ -32,8 +33,7 @@ protected IPythonType InnerType /// Creates delegate type wrapper over an existing type. /// Use dedicated constructor for wrapping builtin types. /// - public PythonTypeWrapper(IPythonType type) - : this(type, type.DeclaringModule) { + public PythonTypeWrapper(IPythonType type) : this(type, type.DeclaringModule) { } /// @@ -60,13 +60,13 @@ public PythonTypeWrapper(BuiltinTypeId builtinTypeId, IPythonModule declaringMod public IPythonModule DeclaringModule { get; } public virtual string Documentation => InnerType.Documentation; public virtual BuiltinTypeId TypeId => InnerType.TypeId; - public virtual PythonMemberType MemberType => InnerType.MemberType; + public virtual PythonMemberType MemberType => InnerType.MemberType; public virtual bool IsBuiltin => InnerType.IsBuiltin; public virtual bool IsAbstract => InnerType.IsAbstract; public virtual bool IsSpecialized => InnerType.IsSpecialized; - public virtual IMember CreateInstance(string typeName, LocationInfo location, IArgumentSet args) - => IsAbstract ? null : InnerType.CreateInstance(typeName, location, args); + public virtual IMember CreateInstance(string typeName, IArgumentSet args) + => IsAbstract ? null : InnerType.CreateInstance(typeName, args); public virtual IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => InnerType.Call(instance, memberName, args); public virtual IMember Index(IPythonInstance instance, object index) @@ -74,7 +74,12 @@ public virtual IMember Index(IPythonInstance instance, object index) #endregion #region ILocatedMember - public virtual LocationInfo Location => (InnerType as ILocatedMember)?.Location ?? LocationInfo.Empty; + public Location Location => InnerType?.Location ?? default; + public LocationInfo Definition => InnerType?.Definition ?? LocationInfo.Empty; + public ILocatedMember Parent => InnerType?.Parent; + public IReadOnlyList References => InnerType?.References ?? Array.Empty(); + public void AddReference(Location location) => InnerType?.AddReference(location); + public void RemoveReferences(IPythonModule module) => InnerType?.RemoveReferences(module); #endregion #region IMemberContainer diff --git a/src/Analysis/Ast/Impl/Types/PythonUnionType.cs b/src/Analysis/Ast/Impl/Types/PythonUnionType.cs index 90808767d..2e7c69865 100644 --- a/src/Analysis/Ast/Impl/Types/PythonUnionType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonUnionType.cs @@ -22,15 +22,17 @@ using Microsoft.Python.Core.Diagnostics; namespace Microsoft.Python.Analysis.Types { - internal sealed class PythonUnionType : IPythonUnionType { + internal sealed class PythonUnionType : LocatedMember, IPythonUnionType { private readonly HashSet _types = new HashSet(PythonTypeComparer.Instance); private readonly object _lock = new object(); - public PythonUnionType(IEnumerable types) { + public PythonUnionType(IEnumerable types, IPythonModule declaringModule) + : base(PythonMemberType.Union, declaringModule) { _types.UnionWith(types); } - private PythonUnionType(IPythonType x, IPythonType y) { + private PythonUnionType(IPythonType x, IPythonType y) + : base(PythonMemberType.Union, x.DeclaringModule) { Check.Argument(nameof(x), () => !(x is IPythonUnionType)); Check.Argument(nameof(y), () => !(y is IPythonUnionType)); _types.Add(x); @@ -47,12 +49,11 @@ public string Name { } } - public IPythonModule DeclaringModule { + public override IPythonModule DeclaringModule { get { lock (_lock) { return _types.First().DeclaringModule; } } } public BuiltinTypeId TypeId => BuiltinTypeId.Type; - public PythonMemberType MemberType => PythonMemberType.Union; public string Documentation => Name; public bool IsBuiltin { @@ -62,8 +63,7 @@ public bool IsBuiltin { public bool IsAbstract => false; public bool IsSpecialized => true; - public IMember CreateInstance(string typeName, LocationInfo location, IArgumentSet args) - => new PythonUnion(this, location); + public IMember CreateInstance(string typeName, IArgumentSet args) => new PythonUnion(this); public IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) { IPythonType[] types; diff --git a/src/Analysis/Ast/Impl/Values/Collections/PythonCollection.cs b/src/Analysis/Ast/Impl/Values/Collections/PythonCollection.cs index bc393fade..fb25819c9 100644 --- a/src/Analysis/Ast/Impl/Values/Collections/PythonCollection.cs +++ b/src/Analysis/Ast/Impl/Values/Collections/PythonCollection.cs @@ -24,18 +24,16 @@ internal class PythonCollection : PythonInstance, IPythonCollection { /// /// Collection type. /// Contents of the collection (typically elements from the initialization). - /// Declaring location. /// If true and contents is a single element /// True if the contents are an exact representation of the collection contents. /// and is a sequence, the sequence elements are copied rather than creating /// a sequence of sequences with a single element. public PythonCollection( IPythonType collectionType, - LocationInfo location, IReadOnlyList contents, bool flatten = true, bool exact = false - ) : base(collectionType, location) { + ) : base(collectionType) { var c = contents ?? Array.Empty(); if (flatten && c.Count == 1 && c[0] is IPythonCollection seq) { Contents = seq.Contents; diff --git a/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs b/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs index 1077fc5e0..53d9fef35 100644 --- a/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs +++ b/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs @@ -28,16 +28,16 @@ internal class PythonDictionary : PythonCollection, IPythonDictionary { private readonly Dictionary _contents = new Dictionary(new KeyComparer()); private readonly IPythonInterpreter _interpreter; - public PythonDictionary(PythonDictionaryType dictType, LocationInfo location, IReadOnlyDictionary contents, bool exact = false) : - base(dictType, location, contents.Keys.ToArray(), exact: exact) { + public PythonDictionary(PythonDictionaryType dictType, IReadOnlyDictionary contents, bool exact = false) : + base(dictType, contents.Keys.ToArray(), exact: exact) { foreach (var kvp in contents) { _contents[kvp.Key] = kvp.Value; } _interpreter = dictType.DeclaringModule.Interpreter; } - public PythonDictionary(IPythonInterpreter interpreter, LocationInfo location, IMember contents, bool exact = false) : - base(new PythonDictionaryType(interpreter), location, Array.Empty(), exact: exact) { + public PythonDictionary(IPythonInterpreter interpreter, IMember contents, bool exact = false) : + base(new PythonDictionaryType(interpreter), Array.Empty(), exact: exact) { if (contents is IPythonDictionary dict) { foreach (var key in dict.Keys) { _contents[key] = dict[key]; @@ -47,8 +47,8 @@ public PythonDictionary(IPythonInterpreter interpreter, LocationInfo location, I _interpreter = interpreter; } - public PythonDictionary(IPythonInterpreter interpreter, LocationInfo location, IReadOnlyDictionary contents, bool exact = false) : - this(new PythonDictionaryType(interpreter), location, contents, exact: exact) { + public PythonDictionary(IPythonInterpreter interpreter, IReadOnlyDictionary contents, bool exact = false) : + this(new PythonDictionaryType(interpreter), contents, exact: exact) { _interpreter = interpreter; } @@ -56,12 +56,12 @@ public PythonDictionary(IPythonInterpreter interpreter, LocationInfo location, I public IEnumerable Values => _contents.Values.ToArray(); public IReadOnlyList Items - => _contents.Select(kvp => PythonCollectionType.CreateTuple(Type.DeclaringModule.Interpreter, Location, new[] { kvp.Key, kvp.Value })).ToArray(); + => _contents.Select(kvp => PythonCollectionType.CreateTuple(Type.DeclaringModule.Interpreter, new[] { kvp.Key, kvp.Value })).ToArray(); public IMember this[IMember key] => _contents.TryGetValue(key, out var value) ? value : UnknownType; - public override IPythonIterator GetIterator() => + public override IPythonIterator GetIterator() => Call(@"iterkeys", ArgumentSet.Empty) as IPythonIterator ?? new EmptyIterator(Type.DeclaringModule.Interpreter.UnknownType); public override IMember Index(object key) => key is IMember m ? this[m] : UnknownType; @@ -92,7 +92,7 @@ public override IMember Call(string memberName, IArgumentSet args) { } private IPythonCollection CreateList(IReadOnlyList items) - => PythonCollectionType.CreateList(Type.DeclaringModule.Interpreter, LocationInfo.Empty, items, false); + => PythonCollectionType.CreateList(Type.DeclaringModule.Interpreter, items, false); private sealed class KeyComparer : IEqualityComparer { public bool Equals(IMember x, IMember y) { diff --git a/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs b/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs index c03520361..9dbdafffe 100644 --- a/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs +++ b/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs @@ -45,18 +45,15 @@ public override IMember Call(string memberName, IArgumentSet args) { /// /// Empty iterator /// - internal sealed class EmptyIterator : IPythonIterator { - public EmptyIterator(IPythonType unknownType) { + internal sealed class EmptyIterator : EmptyLocatedMember, IPythonIterator { + public EmptyIterator(IPythonType unknownType): base(PythonMemberType.Class) { Type = unknownType; } - public PythonMemberType MemberType => PythonMemberType.Class; - public LocationInfo Location => LocationInfo.Empty; public IPythonIterator GetIterator() => this; public IPythonType Type { get; } public IMember Call(string memberName, IArgumentSet args) => Type; public IMember Index(object index) => Type; public IMember Next => Type; - } } diff --git a/src/Analysis/Ast/Impl/Values/Definitions/IPythonConstant.cs b/src/Analysis/Ast/Impl/Values/Definitions/IPythonConstant.cs index 44d1bc6cf..96b9cce20 100644 --- a/src/Analysis/Ast/Impl/Values/Definitions/IPythonConstant.cs +++ b/src/Analysis/Ast/Impl/Values/Definitions/IPythonConstant.cs @@ -13,8 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; - namespace Microsoft.Python.Analysis.Values { public interface IPythonConstant: IPythonInstance { object Value { get; } diff --git a/src/Analysis/Ast/Impl/Values/Definitions/IPythonInstance.cs b/src/Analysis/Ast/Impl/Values/Definitions/IPythonInstance.cs index 046c90545..79198ec71 100644 --- a/src/Analysis/Ast/Impl/Values/Definitions/IPythonInstance.cs +++ b/src/Analysis/Ast/Impl/Values/Definitions/IPythonInstance.cs @@ -13,14 +13,13 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; using Microsoft.Python.Analysis.Types; namespace Microsoft.Python.Analysis.Values { /// /// Represents instance of a type. /// - public interface IPythonInstance : ILocatedMember, IPythonIterable { + public interface IPythonInstance : IMember, IPythonIterable { /// /// Type of the object the instance represents. /// diff --git a/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs b/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs index 617178b91..08f9ec8f3 100644 --- a/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs +++ b/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs @@ -22,17 +22,74 @@ namespace Microsoft.Python.Analysis.Values { /// Represents scope where variables can be declared. /// public interface IScope { + /// + /// Scope name. Typically name of the scope-defining + /// string Name { get; } + + /// + /// Node defining the scope. Typically + /// or + /// ScopeStatement Node { get; } + + /// + /// Immediate parent of this scope. + /// IScope OuterScope { get; } + + /// + /// Module global scope. + /// IGlobalScope GlobalScope { get; } + + /// + /// Child scopes. + /// IReadOnlyList Children { get; } + + /// + /// Enumerates scopes from this one to global scope. + /// IEnumerable EnumerateTowardsGlobal { get; } + + /// + /// Enumerates scopes from global to this one. + /// IEnumerable EnumerateFromGlobal { get; } + + /// + /// Collection of variables declared in the scope. + /// IVariableCollection Variables { get; } + + /// + /// Collection of variables declared via 'nonlocal' statement. + /// IVariableCollection NonLocals { get; } + + /// + /// Collection of global variables declared via 'global' statement. + /// IVariableCollection Globals { get; } + + /// + /// Module the scope belongs to. + /// IPythonModule Module { get; } - void DeclareVariable(string name, IMember value, VariableSource source, LocationInfo location); + + /// + /// Declares variable in the scope. + /// + /// Variable name. + /// Variable value. + /// Variable source. + /// Variable name node location. + void DeclareVariable(string name, IMember value, VariableSource source, Location location = default); + + /// + /// Links variable from another module such as when it is imported. + /// + void LinkVariable(string name, IVariable v, Location location); } } diff --git a/src/Analysis/Ast/Impl/Values/Definitions/IVariable.cs b/src/Analysis/Ast/Impl/Values/Definitions/IVariable.cs index b47d7f61a..8431ac86e 100644 --- a/src/Analysis/Ast/Impl/Values/Definitions/IVariable.cs +++ b/src/Analysis/Ast/Impl/Values/Definitions/IVariable.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; using Microsoft.Python.Analysis.Types; namespace Microsoft.Python.Analysis.Values { @@ -39,11 +38,6 @@ public interface IVariable: ILocatedMember { /// /// Assigns value to the variable. /// - void Assign(IMember value, LocationInfo location); - - /// - /// Provides list of all known assignment locations along the path of analysis. - /// - IReadOnlyList Locations { get; } + void Assign(IMember value, Location location); } } diff --git a/src/Analysis/Ast/Impl/Values/GlobalScope.cs b/src/Analysis/Ast/Impl/Values/GlobalScope.cs index 11d8c3401..df375b923 100644 --- a/src/Analysis/Ast/Impl/Values/GlobalScope.cs +++ b/src/Analysis/Ast/Impl/Values/GlobalScope.cs @@ -35,13 +35,13 @@ private void DeclareBuiltinVariables() { var listType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.List); var dictType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Dict); - VariableCollection.DeclareVariable("__debug__", boolType, VariableSource.Builtin, LocationInfo.Empty); - VariableCollection.DeclareVariable("__doc__", strType, VariableSource.Builtin, LocationInfo.Empty); - VariableCollection.DeclareVariable("__file__", strType, VariableSource.Builtin, LocationInfo.Empty); - VariableCollection.DeclareVariable("__name__", strType, VariableSource.Builtin, LocationInfo.Empty); - VariableCollection.DeclareVariable("__package__", strType, VariableSource.Builtin, LocationInfo.Empty); - VariableCollection.DeclareVariable("__path__", listType, VariableSource.Builtin, LocationInfo.Empty); - VariableCollection.DeclareVariable("__dict__", dictType, VariableSource.Builtin, LocationInfo.Empty); + VariableCollection.DeclareVariable("__debug__", boolType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__doc__", strType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__file__", strType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__name__", strType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__package__", strType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__path__", listType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__dict__", dictType, VariableSource.Builtin); } } } diff --git a/src/Analysis/Ast/Impl/Values/PythonBoundType.cs b/src/Analysis/Ast/Impl/Values/PythonBoundType.cs index 588db96da..0c5030636 100644 --- a/src/Analysis/Ast/Impl/Values/PythonBoundType.cs +++ b/src/Analysis/Ast/Impl/Values/PythonBoundType.cs @@ -25,7 +25,7 @@ namespace Microsoft.Python.Analysis.Values { internal sealed class PythonBoundType : PythonInstance, IPythonBoundType { public IPythonInstance Self { get; } - public PythonBoundType(IPythonType fn, IPythonInstance self, LocationInfo location) : base(fn, location) { + public PythonBoundType(IPythonType fn, IPythonInstance self) : base(fn) { Self = self; } } diff --git a/src/Analysis/Ast/Impl/Values/PythonConstant.cs b/src/Analysis/Ast/Impl/Values/PythonConstant.cs index 39763dad9..056ac0cd9 100644 --- a/src/Analysis/Ast/Impl/Values/PythonConstant.cs +++ b/src/Analysis/Ast/Impl/Values/PythonConstant.cs @@ -18,8 +18,7 @@ namespace Microsoft.Python.Analysis.Values { internal class PythonConstant : PythonInstance, IPythonConstant, IEquatable { - public PythonConstant(object value, IPythonType type, LocationInfo location) - : base(type, location) { + public PythonConstant(object value, IPythonType type) : base(type) { Value = value; } public object Value { get; } @@ -34,7 +33,7 @@ public bool TryGetValue(out T value) { } public bool Equals(IPythonConstant other) { - if(!base.Equals(other)) { + if (!base.Equals(other)) { return false; } return Value?.Equals(other?.Value) == true; diff --git a/src/Analysis/Ast/Impl/Values/PythonInstance.cs b/src/Analysis/Ast/Impl/Values/PythonInstance.cs index 5395f9e71..55249a70a 100644 --- a/src/Analysis/Ast/Impl/Values/PythonInstance.cs +++ b/src/Analysis/Ast/Impl/Values/PythonInstance.cs @@ -22,19 +22,17 @@ namespace Microsoft.Python.Analysis.Values { /// /// Represents an instance of type or the type information. - /// Actual instance has set to . + /// Actual instance has set to . /// Type information is marked as the type it describes, such as . /// [DebuggerDisplay("Instance of {Type.Name}")] internal class PythonInstance : IPythonInstance, IEquatable { - public PythonInstance(IPythonType type, LocationInfo location = null) { + public PythonInstance(IPythonType type) { Type = type ?? throw new ArgumentNullException(nameof(type)); - Location = location ?? LocationInfo.Empty; } public virtual IPythonType Type { get; } - public LocationInfo Location { get; } - public virtual PythonMemberType MemberType => PythonMemberType.Instance; + public PythonMemberType MemberType => PythonMemberType.Instance; public virtual IMember Call(string memberName, IArgumentSet args) { var t = Type.GetMember(memberName).GetPythonType(); @@ -59,7 +57,7 @@ public virtual IPythonIterator GetIterator() { var iteratorFunc = Type.GetMember(@"__iter__") as IPythonFunctionType; var o = iteratorFunc?.Overloads.FirstOrDefault(); var instance = o?.Call(ArgumentSet.Empty, Type); - if (instance != null) { + if (instance != null) { return new PythonInstanceIterator(instance, Type.DeclaringModule.Interpreter); } diff --git a/src/Analysis/Ast/Impl/Values/PythonString.cs b/src/Analysis/Ast/Impl/Values/PythonString.cs index c98af8701..01df5e74b 100644 --- a/src/Analysis/Ast/Impl/Values/PythonString.cs +++ b/src/Analysis/Ast/Impl/Values/PythonString.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System; -using Microsoft.Python.Analysis.Types; using Microsoft.Python.Parsing; namespace Microsoft.Python.Analysis.Values { @@ -22,23 +21,23 @@ namespace Microsoft.Python.Analysis.Values { /// Python ASCII (bytes) string (default string in 2.x). /// internal sealed class PythonAsciiString : PythonConstant { - public PythonAsciiString(AsciiString s, IPythonInterpreter interpreter, LocationInfo location = null) - : base(s, interpreter.GetBuiltinType(interpreter.GetAsciiTypeId()), location) { } + public PythonAsciiString(AsciiString s, IPythonInterpreter interpreter) + : base(s, interpreter.GetBuiltinType(interpreter.GetAsciiTypeId())) { } } /// /// Python Unicode string (default string in 3.x+) /// internal sealed class PythonUnicodeString : PythonConstant { - public PythonUnicodeString(string s, IPythonInterpreter interpreter, LocationInfo location = null) - : base(s, interpreter.GetBuiltinType(interpreter.GetUnicodeTypeId()), location) { } + public PythonUnicodeString(string s, IPythonInterpreter interpreter) + : base(s, interpreter.GetBuiltinType(interpreter.GetUnicodeTypeId())) { } } internal sealed class PythonFString : PythonInstance, IEquatable { public readonly string UnparsedFString; - public PythonFString(string unparsedFString, IPythonInterpreter interpreter, LocationInfo location = null) - : base(interpreter.GetBuiltinType(interpreter.GetUnicodeTypeId()), location) { + public PythonFString(string unparsedFString, IPythonInterpreter interpreter) + : base(interpreter.GetBuiltinType(interpreter.GetUnicodeTypeId())) { UnparsedFString = unparsedFString; } diff --git a/src/Analysis/Ast/Impl/Values/PythonUnion.cs b/src/Analysis/Ast/Impl/Values/PythonUnion.cs index 4f7f994a5..f94874d3d 100644 --- a/src/Analysis/Ast/Impl/Values/PythonUnion.cs +++ b/src/Analysis/Ast/Impl/Values/PythonUnion.cs @@ -21,8 +21,7 @@ namespace Microsoft.Python.Analysis.Values { /// Primarily used to describe multiple types that a function /// may be returning in different cases. /// - internal sealed class PythonUnion: PythonInstance { - public PythonUnion(IPythonUnionType unionType, LocationInfo location = null) - : base(unionType, location) { } + internal sealed class PythonUnion : PythonInstance { + public PythonUnion(IPythonUnionType unionType) : base(unionType) { } } } diff --git a/src/Analysis/Ast/Impl/Values/Scope.cs b/src/Analysis/Ast/Impl/Values/Scope.cs index f44289ef4..fc3933ede 100644 --- a/src/Analysis/Ast/Impl/Values/Scope.cs +++ b/src/Analysis/Ast/Impl/Values/Scope.cs @@ -46,7 +46,7 @@ public Scope(ScopeStatement node, IScope outerScope, IPythonModule module) { public IScope OuterScope { get; } public IPythonModule Module { get; } - public IReadOnlyList Children => (IReadOnlyList)_childScopes ?? Array.Empty(); + public IReadOnlyList Children => _childScopes?.ToArray() ?? Array.Empty(); public IVariableCollection Variables => VariableCollection; public IVariableCollection NonLocals => _nonLocals ?? VariableCollection.Empty; public IVariableCollection Globals => _globals ?? VariableCollection.Empty; @@ -73,13 +73,16 @@ public IEnumerable EnumerateTowardsGlobal { public IEnumerable EnumerateFromGlobal => EnumerateTowardsGlobal.Reverse(); - public void DeclareVariable(string name, IMember value, VariableSource source, LocationInfo location) + public void DeclareVariable(string name, IMember value, VariableSource source, Location location = default) => VariableCollection.DeclareVariable(name, value, source, location); - public void DeclareNonLocal(string name, LocationInfo location) + public void LinkVariable(string name, IVariable v, Location location) + => VariableCollection.LinkVariable(name, v, location); + + public void DeclareNonLocal(string name, Location location) => (_nonLocals ?? (_nonLocals = new VariableCollection())).DeclareVariable(name, null, VariableSource.Locality, location); - public void DeclareGlobal(string name, LocationInfo location) + public void DeclareGlobal(string name, Location location) => (_globals ?? (_globals = new VariableCollection())).DeclareVariable(name, null, VariableSource.Locality, location); #endregion @@ -87,28 +90,28 @@ public void DeclareGlobal(string name, LocationInfo location) internal void AddChildScope(Scope s) => (_childScopes ?? (_childScopes = new List())).Add(s); private void DeclareBuiltinVariables() { - if(Node == null || Module.ModuleType != ModuleType.User || this is IGlobalScope) { + if (Node == null || Module.ModuleType != ModuleType.User || this is IGlobalScope) { return; } var strType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Str); var objType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Object); - VariableCollection.DeclareVariable("__name__", strType, VariableSource.Builtin, LocationInfo.Empty); + VariableCollection.DeclareVariable("__name__", strType, VariableSource.Builtin); if (Node is FunctionDefinition) { var dictType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Dict); var tupleType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Tuple); - VariableCollection.DeclareVariable("__closure__", tupleType, VariableSource.Builtin, LocationInfo.Empty); - VariableCollection.DeclareVariable("__code__", objType, VariableSource.Builtin, LocationInfo.Empty); - VariableCollection.DeclareVariable("__defaults__", tupleType, VariableSource.Builtin, LocationInfo.Empty); - VariableCollection.DeclareVariable("__dict__", dictType, VariableSource.Builtin, LocationInfo.Empty); - VariableCollection.DeclareVariable("__doc__", strType, VariableSource.Builtin, LocationInfo.Empty); - VariableCollection.DeclareVariable("__func__", objType, VariableSource.Builtin, LocationInfo.Empty); - VariableCollection.DeclareVariable("__globals__", dictType, VariableSource.Builtin, LocationInfo.Empty); - } else if(Node is ClassDefinition) { - VariableCollection.DeclareVariable("__self__", objType, VariableSource.Builtin, LocationInfo.Empty); + VariableCollection.DeclareVariable("__closure__", tupleType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__code__", objType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__defaults__", tupleType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__dict__", dictType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__doc__", strType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__func__", objType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__globals__", dictType, VariableSource.Builtin); + } else if (Node is ClassDefinition) { + VariableCollection.DeclareVariable("__self__", objType, VariableSource.Builtin); } } } @@ -128,9 +131,9 @@ public EmptyGlobalScope(IPythonModule module) { public IEnumerable EnumerateFromGlobal => Enumerable.Repeat(this, 1); public IVariableCollection Variables => VariableCollection.Empty; public IVariableCollection NonLocals => VariableCollection.Empty; - public IVariableCollection Globals => VariableCollection.Empty; - - public void DeclareVariable(string name, IMember value, VariableSource source, LocationInfo location) { } + public IVariableCollection Globals => VariableCollection.Empty; + public void DeclareVariable(string name, IMember value, VariableSource source, Location location) { } + public void LinkVariable(string name, IVariable v, Location location) { } } } diff --git a/src/Analysis/Ast/Impl/Values/Variable.cs b/src/Analysis/Ast/Impl/Values/Variable.cs index 1780fc9b7..cc01e5a9d 100644 --- a/src/Analysis/Ast/Impl/Values/Variable.cs +++ b/src/Analysis/Ast/Impl/Values/Variable.cs @@ -17,38 +17,125 @@ using System.Collections.Generic; using System.Diagnostics; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; namespace Microsoft.Python.Analysis.Values { [DebuggerDisplay("{DebuggerDisplay}")] - internal sealed class Variable : IVariable { - private List _locations; - - public Variable(string name, IMember value, VariableSource source, LocationInfo location = null) { - Name = name; - Value = value; + internal sealed class Variable : LocatedMember, IVariable { + public Variable(string name, IMember value, VariableSource source, Location location) + : base(PythonMemberType.Variable, location) { + Name = name ?? throw new ArgumentNullException(nameof(name)); + Value = value is IVariable v ? v.Value : value; Source = source; - Location = location ?? (value is ILocatedMember lm ? lm.Location : LocationInfo.Empty); } + public Variable(string name, IVariable parent, Location location) + : base(PythonMemberType.Variable, location, parent) { + Name = name ?? throw new ArgumentNullException(nameof(name)); + Value = parent.Value; + Source = VariableSource.Import; + } + + #region IVariable public string Name { get; } public VariableSource Source { get; } public IMember Value { get; private set; } - public LocationInfo Location { get; } - public PythonMemberType MemberType => PythonMemberType.Variable; - public IReadOnlyList Locations => _locations as IReadOnlyList ?? new[] { Location }; - public void Assign(IMember value, LocationInfo location) { + public void Assign(IMember value, Location location) { + if (value is IVariable v) { + value = v.Value; + } if (Value == null || Value.GetPythonType().IsUnknown() || value?.GetPythonType().IsUnknown() == false) { + Debug.Assert(!(value is IVariable)); Value = value; } - AddLocation(location); + AddReference(location); + } + #endregion + + #region ILocatedMember + public override LocationInfo Definition { + get { + if (!Location.IsValid) { + if (Parent != null) { + return Parent?.Definition ?? LocationInfo.Empty; + } + var lmv = GetImplicitlyDeclaredValue(); + if (lmv != null) { + return lmv.Definition; + } + } + return base.Definition ?? LocationInfo.Empty; + } + } + + public override IReadOnlyList References { + get { + if (!Location.IsValid) { + if (Parent != null) { + return Parent?.References ?? Array.Empty(); + } + var lmv = GetImplicitlyDeclaredValue(); + if (lmv != null) { + return lmv.References; + } + } + return base.References ?? Array.Empty(); + } } - private void AddLocation(LocationInfo location) { - _locations = _locations ?? new List { Location }; - _locations.Add(location); + public override void AddReference(Location location) { + if (location.IsValid) { + if (!AddOrRemoveReference(lm => lm?.AddReference(location))) { + base.AddReference(location); + } + } + } + + public override void RemoveReferences(IPythonModule module) { + if (module == null) { + return; + } + if (!AddOrRemoveReference(lm => lm?.RemoveReferences(module))) { + base.RemoveReferences(module); + } } + private bool AddOrRemoveReference(Action action) { + // Variable can be + // a) Declared locally in the module. In this case it has non-default + // definition location and no parent (link). + // b) Imported from another module via 'from module import X'. + // In this case it has non-default definition location and non-null parent (link). + // c) Imported from another module via 'from module import *'. + // In this case it has default location (which means it is not explicitly declared) + // and the non-null parent (link). + var explicitlyDeclared = Location.IsValid; + if (!explicitlyDeclared && Parent != null) { + action(Parent); + return true; + } + // Explicitly declared. + // Values: + // a) If value is not a located member, then add reference to the variable. + // b) If variable name is the same as the value member name, then the variable + // is implicit declaration (like declared function or a class) and we need + // to add reference to the actual type instead. + var lmv = GetImplicitlyDeclaredValue(); + if (lmv != null) { + // Variable is not user-declared and rather is holder of a function or class definition. + action(lmv); + return true; + } + action(Parent); + return false; + } + + #endregion + + private ILocatedMember GetImplicitlyDeclaredValue() + => Value is ILocatedMember lm && Name.EqualsOrdinal(lm.GetPythonType()?.Name) && !Location.IsValid ? lm : null; + private string DebuggerDisplay { get { switch (Value) { diff --git a/src/Analysis/Ast/Impl/Values/VariableCollection.cs b/src/Analysis/Ast/Impl/Values/VariableCollection.cs index 73dea055f..e6645c09a 100644 --- a/src/Analysis/Ast/Impl/Values/VariableCollection.cs +++ b/src/Analysis/Ast/Impl/Values/VariableCollection.cs @@ -53,7 +53,7 @@ public bool TryGetVariable(string key, out IVariable value) { public IEnumerable GetMemberNames() => _variables.Keys.ToArray(); #endregion - internal void DeclareVariable(string name, IMember value, VariableSource source, LocationInfo location) { + internal void DeclareVariable(string name, IMember value, VariableSource source, Location location = default) { name = !string.IsNullOrWhiteSpace(name) ? name : throw new ArgumentException(nameof(name)); if (_variables.TryGetValue(name, out var existing)) { existing.Assign(value, location); @@ -62,7 +62,11 @@ internal void DeclareVariable(string name, IMember value, VariableSource source, } } - internal void DeclareVariable(Variable variable) =>_variables[variable.Name] = variable; + internal void LinkVariable(string name, IVariable v, Location location) { + name = !string.IsNullOrWhiteSpace(name) ? name : throw new ArgumentException(nameof(name)); + _variables[name] = new Variable(name, v, location); + } + internal void RemoveVariable(string name) => _variables.TryRemove(name, out _); } } diff --git a/src/Analysis/Ast/Test/AnalysisTestBase.cs b/src/Analysis/Ast/Test/AnalysisTestBase.cs index eaa71973d..81cd82a9b 100644 --- a/src/Analysis/Ast/Test/AnalysisTestBase.cs +++ b/src/Analysis/Ast/Test/AnalysisTestBase.cs @@ -160,5 +160,11 @@ protected async Task GetAnalysisAsync( return analysis; } + + protected async Task GetDocumentAnalysisAsync(IDocument document) { + var analyzer = Services.GetService(); + await analyzer.WaitForCompleteAnalysisAsync(); + return await document.GetAnalysisAsync(Timeout.Infinite); + } } } diff --git a/src/Analysis/Ast/Test/ArgumentSetTests.cs b/src/Analysis/Ast/Test/ArgumentSetTests.cs index 62ef48404..fc613182a 100644 --- a/src/Analysis/Ast/Test/ArgumentSetTests.cs +++ b/src/Analysis/Ast/Test/ArgumentSetTests.cs @@ -26,7 +26,7 @@ namespace Microsoft.Python.Analysis.Tests { [TestClass] - public class ArgumentSetTests: AnalysisTestBase { + public class ArgumentSetTests : AnalysisTestBase { public TestContext TestContext { get; set; } [TestInitialize] @@ -234,7 +234,7 @@ def f(a, b): ... "; var argSet = await GetArgSetAsync(code); // Collected arguments are optimistically returned even if there are errors; - argSet.Arguments.Count.Should().Be(2); + argSet.Arguments.Count.Should().Be(2); argSet.Errors.Count.Should().Be(1); argSet.Errors[0].ErrorCode.Should().Be(ErrorCodes.ParameterMissing); } @@ -344,18 +344,36 @@ from builtins import pow argSet.Arguments[2].ValueExpression.Should().BeOfType().Which.Value.Should().BeNull(); // Value has not been evaluated yet. } + [TestMethod, Priority(0)] + public async Task DefaultArgumentAnotherFile() { + const string code = @" +from DefaultArgument import func +func() +"; + var argSet = await GetArgSetAsync(code, "func"); + argSet.Arguments.Count.Should().Be(1); + argSet.Evaluate(); + var v = argSet.Arguments[0].Value; + var m = v.Should().BeAssignableTo().Which; + + var t = m.GetPythonType(); + t.Name.Should().Be("A"); + t.MemberType.Should().Be(PythonMemberType.Class); + } + + private async Task GetArgSetAsync(string code, string funcName = "f") { var analysis = await GetAnalysisAsync(code); var f = analysis.Should().HaveFunction(funcName).Which; var call = GetCall(analysis.Ast); - return new ArgumentSet(f, 0, null, call, analysis.Document, null); + return new ArgumentSet(f, 0, null, call, analysis.Document, analysis.ExpressionEvaluator); } private async Task GetUnboundArgSetAsync(string code, string funcName = "f") { var analysis = await GetAnalysisAsync(code); var f = analysis.Should().HaveVariable(funcName).Which; var call = GetCall(analysis.Ast); - return new ArgumentSet(f.Value.GetPythonType(), 0, null, call, analysis.Document, null); + return new ArgumentSet(f.Value.GetPythonType(), 0, null, call, analysis.Document, analysis.ExpressionEvaluator); } private async Task GetClassArgSetAsync(string code, string className = "A", string funcName = "f") { @@ -363,7 +381,7 @@ private async Task GetClassArgSetAsync(string code, string classNam var cls = analysis.Should().HaveClass(className).Which; var f = cls.Should().HaveMethod(funcName).Which; var call = GetCall(analysis.Ast); - return new ArgumentSet(f, 0, new PythonInstance(cls), call, analysis.Document, null); + return new ArgumentSet(f, 0, new PythonInstance(cls), call, analysis.Document, analysis.ExpressionEvaluator); } private CallExpression GetCall(PythonAst ast) { diff --git a/src/Analysis/Ast/Test/AssignmentTests.cs b/src/Analysis/Ast/Test/AssignmentTests.cs index f80d0c7a0..9c443b1e4 100644 --- a/src/Analysis/Ast/Test/AssignmentTests.cs +++ b/src/Analysis/Ast/Test/AssignmentTests.cs @@ -149,7 +149,7 @@ public async Task Tuple() { const string code = @" x, y, z = 1, 'str', 3.0 "; - var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var analysis = await GetAnalysisAsync(code); analysis.Should().HaveVariable("x").OfType(BuiltinTypeId.Int) .And.HaveVariable("y").OfType(BuiltinTypeId.Str) .And.HaveVariable("z").OfType(BuiltinTypeId.Float); @@ -160,10 +160,40 @@ public async Task TupleUnknownReturn() { const string code = @" x, y, z = func() "; - var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var analysis = await GetAnalysisAsync(code); analysis.Should().HaveVariable("x").And.HaveVariable("y").And.HaveVariable("z"); } + [TestMethod, Priority(0)] + public async Task NestedTuple() { + const string code = @" +((x, r), y, z) = (1, 1), '', False, +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable("x").Which.Should().HaveType(BuiltinTypeId.Int); + analysis.Should().HaveVariable("r").Which.Should().HaveType(BuiltinTypeId.Int); + analysis.Should().HaveVariable("y").Which.Should().HaveType(BuiltinTypeId.Str); + analysis.Should().HaveVariable("z").Which.Should().HaveType(BuiltinTypeId.Bool); + } + + [TestMethod, Priority(0)] + public async Task NestedTupleSingleValue() { + const string code = @" +(x, (y, (z))) = 1, +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable("x").And.HaveVariable("y").And.HaveVariable("z"); + } + + [TestMethod, Priority(0)] + public async Task List() { + const string code = @" +[year, month] = (1, 2) +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + analysis.Should().HaveVariable("year").And.HaveVariable("month"); + } + [TestMethod, Priority(0)] public async Task AnnotatedAssign() { const string code = @" diff --git a/src/Analysis/Ast/Test/ClassesTests.cs b/src/Analysis/Ast/Test/ClassesTests.cs index 54058c8b7..982cdac7a 100644 --- a/src/Analysis/Ast/Test/ClassesTests.cs +++ b/src/Analysis/Ast/Test/ClassesTests.cs @@ -89,13 +89,14 @@ public async Task Mro() { var o = interpreter.GetBuiltinType(BuiltinTypeId.Object); var m = new SentinelModule("test", s); - var O = new PythonClassType("O", m); - var A = new PythonClassType("A", m); - var B = new PythonClassType("B", m); - var C = new PythonClassType("C", m); - var D = new PythonClassType("D", m); - var E = new PythonClassType("E", m); - var F = new PythonClassType("F", m); + var location = new Location(m, default); + var O = new PythonClassType("O", location); + var A = new PythonClassType("A", location); + var B = new PythonClassType("B", location); + var C = new PythonClassType("C", location); + var D = new PythonClassType("D", location); + var E = new PythonClassType("E", location); + var F = new PythonClassType("F", location); O.SetBases(new[] { o }); F.SetBases(new[] { O }); diff --git a/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs b/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs index d9818ab81..2990fe8e5 100644 --- a/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs +++ b/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs @@ -18,6 +18,7 @@ using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Parsing.Tests; @@ -287,7 +288,7 @@ def func(): "; var d = await LintAsync(code); d.Should().HaveCount(1); - d[0].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); + d[0].ErrorCode.Should().Be(ErrorCodes.VariableNotDefinedNonLocal); d[0].SourceSpan.Should().Be(5, 21, 5, 22); } @@ -303,7 +304,7 @@ def func(): "; var d = await LintAsync(code); d.Should().HaveCount(1); - d[0].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); + d[0].ErrorCode.Should().Be(ErrorCodes.VariableNotDefinedGlobally); d[0].SourceSpan.Should().Be(6, 19, 6, 20); } @@ -394,12 +395,6 @@ public async Task Lambda() { d.Should().BeEmpty(); } - private async Task> LintAsync(string code) { - var analysis = await GetAnalysisAsync(code); - var a = Services.GetService(); - return a.LintModule(analysis.Document); - } - [TestMethod, Priority(0)] public async Task BuiltinModuleVariables() { const string code = @" @@ -410,8 +405,8 @@ public async Task BuiltinModuleVariables() { x5 = __path__ x6 = __dict__ "; - var analysis = await GetAnalysisAsync(code); - analysis.Diagnostics.Should().BeEmpty(); + var d = await LintAsync(code); + d.Should().BeEmpty(); } [TestMethod, Priority(0)] @@ -428,8 +423,8 @@ def func(): x8 = __globals__ x9 = __name__ "; - var analysis = await GetAnalysisAsync(code); - analysis.Diagnostics.Should().BeEmpty(); + var d = await LintAsync(code); + d.Should().BeEmpty(); } [TestMethod, Priority(0)] @@ -449,17 +444,17 @@ def func(): x9 = __name__ x10 = __self__ "; - var analysis = await GetAnalysisAsync(code); - analysis.Diagnostics.Should().BeEmpty(); + var d = await LintAsync(code); + d.Should().BeEmpty(); } [TestMethod, Priority(0)] - public async Task LambdaComrehension() { + public async Task LambdaComprehension() { const string code = @" y = lambda x: [e for e in x if e == 1] "; - var analysis = await GetAnalysisAsync(code); - analysis.Diagnostics.Should().BeEmpty(); + var d = await LintAsync(code); + d.Should().BeEmpty(); } [TestMethod, Priority(0)] @@ -468,8 +463,8 @@ public async Task FromFuture() { from __future__ import print_function print() "; - var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable2X); - analysis.Diagnostics.Should().BeEmpty(); + var d = await LintAsync(code, PythonVersions.LatestAvailable2X); + d.Should().BeEmpty(); } [TestMethod, Priority(0)] @@ -483,8 +478,93 @@ def __init__(self, path): with (self.path / 'object.xml').open() as f: self.root = ElementTree.parse(f) "; - var analysis = await GetAnalysisAsync(code); - analysis.Diagnostics.Should().BeEmpty(); + var d = await LintAsync(code); + d.Should().BeEmpty(); + } + + + [TestMethod, Priority(0)] + public async Task Various() { + const string code = @" +print x +assert x +del x +"; + var d = await LintAsync(code, PythonVersions.LatestAvailable2X); + d.Should().HaveCount(3); + } + + [TestMethod, Priority(0)] + public async Task FString() { + const string code = @" +print(f'Hello, {name}. You are {age}.') +"; + var d = await LintAsync(code, PythonVersions.LatestAvailable3X); + d.Should().HaveCount(2); + } + + [TestMethod, Priority(0)] + public async Task ClassMemberAssignment() { + const string code = @" +class A: + x: int + +a = A() +a.x = 1 +a.y = 2 +"; + var d = await LintAsync(code); + d.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task ListAssignment() { + const string code = @" +from datetime import date +[year, month] = date.split(' - ')"; + var d = await LintAsync(code); + d.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task NoRightSideCheck() { + const string code = @" +x = +y = "; + var d = await LintAsync(code); + d.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task ListComprehensionFunction() { + const string code = @" +def func_a(value): + return value + +def func_b(): + list_a = [1, 2, 3, 4] + + return [func_a(value=elem) for elem in list_a] +"; + var d = await LintAsync(code); + d.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task NoLeakFromComprehension() { + const string code = @" +len([1 for e in [1, 2]]) + len([e]) +"; + var d = await LintAsync(code); + d.Should().HaveCount(1); + d[0].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); + d[0].SourceSpan.Should().Be(2, 33, 2, 34); + } + + private async Task> LintAsync(string code, InterpreterConfiguration configuration = null) { + var analysis = await GetAnalysisAsync(code, configuration ?? PythonVersions.LatestAvailable3X); + var a = Services.GetService(); + return a.LintModule(analysis.Document); } private class AnalysisOptionsProvider : IAnalysisOptionsProvider { diff --git a/src/Analysis/Ast/Test/LocationTests.cs b/src/Analysis/Ast/Test/LocationTests.cs deleted file mode 100644 index 46f8d496c..000000000 --- a/src/Analysis/Ast/Test/LocationTests.cs +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Threading.Tasks; -using FluentAssertions; -using Microsoft.Python.Analysis.Tests.FluentAssertions; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using TestUtilities; - -namespace Microsoft.Python.Analysis.Tests { - [TestClass] - public class LocationTests : AnalysisTestBase { - public TestContext TestContext { get; set; } - - [TestInitialize] - public void TestInitialize() - => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); - - [TestCleanup] - public void Cleanup() => TestEnvironmentImpl.TestCleanup(); - - [TestMethod, Priority(0)] - public async Task Assignments() { - const string code = @" -a = 1 -a = 2 - -for x in y: - a = 3 - -if True: - a = 4 -elif False: - a = 5 -else: - a = 6 - -def func(): - a = 0 - -def func(a): - a = 0 - -def func(): - global a - a = 7 - -class A: - a: int -"; - var analysis = await GetAnalysisAsync(code); - var a = analysis.Should().HaveVariable("a").Which; - a.Locations.Should().HaveCount(7); - a.Locations[0].Span.Should().Be(2, 1, 2, 2); - a.Locations[1].Span.Should().Be(3, 1, 3, 2); - a.Locations[2].Span.Should().Be(6, 5, 6, 6); - a.Locations[3].Span.Should().Be(9, 5, 9, 6); - a.Locations[4].Span.Should().Be(11, 5, 11, 6); - a.Locations[5].Span.Should().Be(13, 5, 13, 6); - a.Locations[6].Span.Should().Be(23, 5, 23, 6); - } - - [TestMethod, Priority(0)] - public async Task NonLocal() { - const string code = @" -def outer(): - b = 1 - def inner(): - nonlocal b - b = 2 -"; - var analysis = await GetAnalysisAsync(code); - var outer = analysis.Should().HaveFunction("outer").Which; - var b = outer.Should().HaveVariable("b").Which; - b.Locations.Should().HaveCount(2); - b.Locations[0].Span.Should().Be(3, 5, 3, 6); - b.Locations[1].Span.Should().Be(6, 9, 6, 10); - } - - [TestMethod, Priority(0)] - public async Task FunctionParameter() { - const string code = @" -def func(a): - a = 1 - if True: - a = 2 -"; - var analysis = await GetAnalysisAsync(code); - var outer = analysis.Should().HaveFunction("func").Which; - var a = outer.Should().HaveVariable("a").Which; - a.Locations.Should().HaveCount(3); - a.Locations[0].Span.Should().Be(2, 10, 2, 11); - a.Locations[1].Span.Should().Be(3, 5, 3, 6); - a.Locations[2].Span.Should().Be(5, 9, 5, 10); - } - } -} diff --git a/src/Analysis/Ast/Test/ReferencesTests.cs b/src/Analysis/Ast/Test/ReferencesTests.cs new file mode 100644 index 000000000..79d9bc884 --- /dev/null +++ b/src/Analysis/Ast/Test/ReferencesTests.cs @@ -0,0 +1,433 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.ComponentModel.DataAnnotations; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.Python.Analysis.Types; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class ReferencesTests : AnalysisTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod, Priority(0)] + public async Task Methods() { + const string code = @" +class A: + def methodA(self): + return func() + +class B(A): + def methodB(self): + return self.methodA() + +a = A() +a.methodA() + +b = B() +b.methodB() +b.methodA() + +ma1 = a.methodA +ma2 = b.methodA +mb1 = b.methodB +"; + var analysis = await GetAnalysisAsync(code); + + var classA = analysis.Should().HaveVariable("A").Which; + var methodA = classA.Should().HaveMember("methodA").Which; + methodA.Definition.Span.Should().Be(3, 9, 3, 16); + methodA.References.Should().HaveCount(6); + methodA.References[0].Span.Should().Be(3, 9, 3, 16); + methodA.References[1].Span.Should().Be(8, 21, 8, 28); + methodA.References[2].Span.Should().Be(11, 3, 11, 10); + methodA.References[3].Span.Should().Be(15, 3, 15, 10); + methodA.References[4].Span.Should().Be(17, 9, 17, 16); + methodA.References[5].Span.Should().Be(18, 9, 18, 16); + + var classB = analysis.Should().HaveVariable("B").Which; + var methodB = classB.Should().HaveMember("methodB").Which; + methodB.Definition.Span.Should().Be(7, 9, 7, 16); + methodB.References.Should().HaveCount(3); + methodB.References[0].Span.Should().Be(7, 9, 7, 16); + methodB.References[1].Span.Should().Be(14, 3, 14, 10); + methodB.References[2].Span.Should().Be(19, 9, 19, 16); + } + + [TestMethod, Priority(0)] + public async Task Function() { + const string code = @" +def func(): + return 1 + +class A: + def methodA(self): + return func() + +func() +a = func +"; + var analysis = await GetAnalysisAsync(code); + + var func = analysis.Should().HaveVariable("func").Which; + func.Definition.Span.Should().Be(2, 5, 2, 9); + func.References.Should().HaveCount(4); + func.References[0].Span.Should().Be(2, 5, 2, 9); + func.References[1].Span.Should().Be(7, 16, 7, 20); + func.References[2].Span.Should().Be(9, 1, 9, 5); + func.References[3].Span.Should().Be(10, 5, 10, 9); + } + + [TestMethod, Priority(0)] + public async Task Property() { + const string code = @" +class A: + @property + def propA(self): ... + +class B(A): ... + +x = B().propA +"; + var analysis = await GetAnalysisAsync(code); + + var classA = analysis.Should().HaveVariable("A").Which; + var propA = classA.Should().HaveMember("propA").Which; + propA.Definition.Span.Should().Be(4, 9, 4, 14); + propA.References.Should().HaveCount(2); + propA.References[0].Span.Should().Be(4, 9, 4, 14); + propA.References[1].Span.Should().Be(8, 9, 8, 14); + } + + [TestMethod, Priority(0)] + public async Task ClassBase() { + const string code = @" +class A: ... +class B(A): ... +"; + var analysis = await GetAnalysisAsync(code); + + var classA = analysis.Should().HaveVariable("A").Which; + classA.Definition.Span.Should().Be(2, 7, 2, 8); + classA.References.Should().HaveCount(2); + classA.References[0].Span.Should().Be(2, 7, 2, 8); + classA.References[1].Span.Should().Be(3, 9, 3, 10); + } + + [TestMethod, Priority(0)] + public async Task Global() { + const string code = @" +z1 = 1 +z2 = 1 + +class A: + def method(self): + z1 = 3 + global z2 + z2 = 4 +"; + var analysis = await GetAnalysisAsync(code); + + var z1 = analysis.Should().HaveVariable("z1").Which; + z1.Definition.Span.Should().Be(2, 1, 2, 3); + z1.References.Should().HaveCount(1); + z1.References[0].Span.Should().Be(2, 1, 2, 3); + + var z2 = analysis.Should().HaveVariable("z2").Which; + z2.Definition.Span.Should().Be(3, 1, 3, 3); + z2.References.Should().HaveCount(3); + z2.References[0].Span.Should().Be(3, 1, 3, 3); + z2.References[1].Span.Should().Be(8, 16, 8, 18); + z2.References[2].Span.Should().Be(9, 9, 9, 11); + } + + [TestMethod, Priority(0)] + public async Task Nonlocal() { + const string code = @" +z1 = 1 + +class A: + def method(self): + z1 = 2 + def inner(self): + nonlocal z1 + z1 = 3 +"; + var analysis = await GetAnalysisAsync(code); + + var z1 = analysis.Should().HaveVariable("z1").Which; + z1.Definition.Span.Should().Be(2, 1, 2, 3); + z1.References.Should().HaveCount(1); + z1.References[0].Span.Should().Be(2, 1, 2, 3); + + var classA = analysis.Should().HaveVariable("A").Which; + var method = classA.Should().HaveMember("method").Which; + var z1Method = method.Should().HaveVariable("z1").Which; + z1Method.Definition.Span.Should().Be(6, 9, 6, 11); + z1Method.References.Should().HaveCount(3); + z1Method.References[0].Span.Should().Be(6, 9, 6, 11); + z1Method.References[1].Span.Should().Be(8, 22, 8, 24); + z1Method.References[2].Span.Should().Be(9, 13, 9, 15); + } + + [TestMethod, Priority(0)] + public async Task Import() { + const string code = @" +import sys as s +x = s.path +"; + var analysis = await GetAnalysisAsync(code); + var s = analysis.Should().HaveVariable("s").Which; + s.Definition.Span.Should().Be(2, 15, 2, 16); + s.References.Should().HaveCount(2); + s.References[0].Span.Should().Be(2, 15, 2, 16); + s.References[1].Span.Should().Be(3, 5, 3, 6); + } + + [TestMethod, Priority(0)] + public async Task FromImportAs() { + const string code = @" +from sys import path as p +x = p +"; + var analysis = await GetAnalysisAsync(code); + var p = analysis.Should().HaveVariable("p").Which; + p.Definition.Span.Should().Be(2, 25, 2, 26); + p.References.Should().HaveCount(2); + p.References[0].Span.Should().Be(2, 25, 2, 26); + p.References[1].Span.Should().Be(3, 5, 3, 6); + } + + [TestMethod, Priority(0)] + public async Task FromImport() { + const string code = @" +from sys import path +x = path +"; + var analysis = await GetAnalysisAsync(code); + var p = analysis.Should().HaveVariable("path").Which; + p.Definition.Span.Should().Be(2, 17, 2, 21); + p.References.Should().HaveCount(2); + p.References[0].Span.Should().Be(2, 17, 2, 21); + p.References[1].Span.Should().Be(3, 5, 3, 9); + } + + [TestMethod, Priority(0)] + public async Task FunctionParameter() { + const string code = @" +def func(a): + a = 1 + if True: + a = 2 +"; + var analysis = await GetAnalysisAsync(code); + var outer = analysis.Should().HaveFunction("func").Which; + var a = outer.Should().HaveVariable("a").Which; + a.References.Should().HaveCount(3); + a.References[0].Span.Should().Be(2, 10, 2, 11); + a.References[1].Span.Should().Be(3, 5, 3, 6); + a.References[2].Span.Should().Be(5, 9, 5, 10); + } + + [TestMethod, Priority(0)] + public async Task ClassField() { + const string code = @" +class A: + x = 0 + +def func(a: A): + return a.x +"; + var analysis = await GetAnalysisAsync(code); + var x = analysis.GlobalScope.Children[0].Should().HaveVariable("x").Which; + x.Should().NotBeNull(); + x.References.Should().HaveCount(2); + x.References[0].Span.Should().Be(3, 5, 3, 6); + x.References[1].Span.Should().Be(6, 14, 6, 15); + } + + [TestMethod, Priority(0)] + public async Task Assignments() { + const string code = @" +a = 1 +a = 2 + +for x in y: + a = 3 + +if True: + a = 4 +elif False: + a = 5 +else: + a = 6 + +def func(): + a = 0 + +def func(a): + a = 0 + +def func(): + global a + a = 7 + +class A: + a: int +"; + var analysis = await GetAnalysisAsync(code); + var a = analysis.Should().HaveVariable("a").Which; + a.References.Should().HaveCount(8); + a.References[0].Span.Should().Be(2, 1, 2, 2); + a.References[1].Span.Should().Be(3, 1, 3, 2); + a.References[2].Span.Should().Be(6, 5, 6, 6); + a.References[3].Span.Should().Be(9, 5, 9, 6); + a.References[4].Span.Should().Be(11, 5, 11, 6); + a.References[5].Span.Should().Be(13, 5, 13, 6); + a.References[6].Span.Should().Be(22, 12, 22, 13); + a.References[7].Span.Should().Be(23, 5, 23, 6); + } + + [TestMethod, Priority(0)] + public async Task ImportSpecific() { + const string code = @" +from MultiValues import t +x = t +"; + var analysis = await GetAnalysisAsync(code); + var t = analysis.Should().HaveVariable("t").Which; + t.Definition.Span.Should().Be(2, 25, 2, 26); + t.Definition.DocumentUri.AbsolutePath.Should().Contain("module.py"); + t.References.Should().HaveCount(2); + t.References[0].Span.Should().Be(2, 25, 2, 26); + t.References[0].DocumentUri.AbsolutePath.Should().Contain("module.py"); + t.References[1].Span.Should().Be(3, 5, 3, 6); + t.References[1].DocumentUri.AbsolutePath.Should().Contain("module.py"); + + var parent = t.Parent; + parent.Should().NotBeNull(); + parent.References.Should().HaveCount(4); + parent.References[0].Span.Should().Be(3, 1, 3, 2); + parent.References[0].DocumentUri.AbsolutePath.Should().Contain("MultiValues.py"); + parent.References[1].Span.Should().Be(12, 5, 12, 6); + parent.References[1].DocumentUri.AbsolutePath.Should().Contain("MultiValues.py"); + parent.References[2].Span.Should().Be(2, 25, 2, 26); + parent.References[2].DocumentUri.AbsolutePath.Should().Contain("module.py"); + parent.References[3].Span.Should().Be(3, 5, 3, 6); + parent.References[3].DocumentUri.AbsolutePath.Should().Contain("module.py"); + } + + [TestMethod, Priority(0)] + public async Task ImportStar() { + const string code = @" +from MultiValues import * +x = t +"; + var analysis = await GetAnalysisAsync(code); + var t = analysis.Should().HaveVariable("t").Which; + t.Definition.Span.Should().Be(3, 1, 3, 2); + t.Definition.DocumentUri.AbsolutePath.Should().Contain("MultiValues.py"); + t.References.Should().HaveCount(3); + t.References[0].Span.Should().Be(3, 1, 3, 2); + t.References[0].DocumentUri.AbsolutePath.Should().Contain("MultiValues.py"); + t.References[1].Span.Should().Be(12, 5, 12, 6); + t.References[1].DocumentUri.AbsolutePath.Should().Contain("MultiValues.py"); + t.References[2].Span.Should().Be(3, 5, 3, 6); + t.References[2].DocumentUri.AbsolutePath.Should().Contain("module.py"); + } + + [TestMethod, Priority(0)] + public async Task Conditional() { + const string code = @" +x = 1 +y = 2 + +if x < y: + pass +"; + var analysis = await GetAnalysisAsync(code); + var x = analysis.Should().HaveVariable("x").Which; + x.Definition.Span.Should().Be(2, 1, 2, 2); + x.References.Should().HaveCount(2); + x.References[0].Span.Should().Be(2, 1, 2, 2); + x.References[1].Span.Should().Be(5, 4, 5, 5); + + var y = analysis.Should().HaveVariable("y").Which; + y.Definition.Span.Should().Be(3, 1, 3, 2); + y.References.Should().HaveCount(2); + y.References[0].Span.Should().Be(3, 1, 3, 2); + y.References[1].Span.Should().Be(5, 8, 5, 9); + } + + [TestMethod, Priority(0)] + public async Task AugmentedAssign() { + const string code = @" +def func(a, b): + dest = 1 + dest += src + dest[dest < src] = np.iinfo(dest.dtype).max +"; + var analysis = await GetAnalysisAsync(code); + var dest = analysis.GlobalScope.Children[0].Should().HaveVariable("dest").Which; + dest.Definition.Span.Should().Be(3, 5, 3, 9); + dest.References.Should().HaveCount(5); + dest.References[0].Span.Should().Be(3, 5, 3, 9); + dest.References[1].Span.Should().Be(4, 5, 4, 9); + dest.References[2].Span.Should().Be(5, 5, 5, 9); + dest.References[3].Span.Should().Be(5, 10, 5, 14); + dest.References[4].Span.Should().Be(5, 33, 5, 37); + } + + [TestMethod, Priority(0)] + public async Task ExtendAllAssignment() { + const string code = @" +x_a = 1 +x_b = 2 +x_c = 3 +x_d = 4 +x_e = 5 +x_f = 6 +x_g = 7 +__all__ = ['x_a', 'x_b'] +__all__ += ['x_c'] +__all__ += ['x_d'] + ['x_e'] +__all__.extend(['x_f']) +__all__.append('x_g') +x_h = 8 +# __all__ += ['x_h'] +"; + var analysis = await GetAnalysisAsync(code); + var all = analysis.Should().HaveVariable("__all__").Which; + all.Definition.Span.Should().Be(9, 1, 9, 8); + all.References.Should().HaveCount(5); + all.References[0].Span.Should().Be(9, 1, 9, 8); + all.References[1].Span.Should().Be(10, 1, 10, 8); + all.References[2].Span.Should().Be(11, 1, 11, 8); + all.References[3].Span.Should().Be(12, 1, 12, 8); + all.References[4].Span.Should().Be(13, 1, 13, 8); + } + } +} diff --git a/src/Analysis/Ast/Test/ScopesTests.cs b/src/Analysis/Ast/Test/ScopesTests.cs new file mode 100644 index 000000000..6da4b9223 --- /dev/null +++ b/src/Analysis/Ast/Test/ScopesTests.cs @@ -0,0 +1,120 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Core.Text; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class ScopesTests : AnalysisTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod, Priority(0)] + public async Task FindScope() { + const string code = @" +a +class A: + x: int + def method(self): + + b = 1 + + + aa + +def func(x, y): + a + def inner(c): + z + + + + +"; + var analysis = await GetAnalysisAsync(code); + var gs = analysis.GlobalScope; + + var locations = new[] { + (new SourceLocation(2, 1), ""), + (new SourceLocation(2, 2), ""), + (new SourceLocation(3, 1), ""), + (new SourceLocation(3, 3), ""), + (new SourceLocation(4, 11), "A"), + (new SourceLocation(5, 1), "A"), + (new SourceLocation(5, 5), "A"), + (new SourceLocation(5, 6), "A"), + (new SourceLocation(5, 17), "method"), + (new SourceLocation(6, 9), "method"), + (new SourceLocation(7, 9), "method"), + (new SourceLocation(7, 14), "method"), + (new SourceLocation(8, 9), "method"), + (new SourceLocation(9, 5), "A"), + (new SourceLocation(10, 5), "A"), + (new SourceLocation(10, 7), "A"), + (new SourceLocation(11, 1), ""), + (new SourceLocation(12, 1), ""), + (new SourceLocation(12, 11), "func"), + (new SourceLocation(13, 5), "func"), + (new SourceLocation(13, 6), "func"), + (new SourceLocation(14, 5), "func"), + (new SourceLocation(14, 15), "inner"), + (new SourceLocation(15, 9), "inner"), + (new SourceLocation(15, 10), "inner"), + (new SourceLocation(16, 9), "inner"), + (new SourceLocation(17, 5), "func"), + (new SourceLocation(18, 1), "") + }; + + foreach (var loc in locations) { + var scope = gs.FindScope(analysis.Document, loc.Item1); + scope.Name.Should().Be(loc.Item2, $"location {loc.Item1.Line}, {loc.Item1.Column}"); + } + } + + [TestMethod, Priority(0)] + public async Task EmptyLines() { + const string code = @" +class A: + x: int + def method(self): + +"; + var analysis = await GetAnalysisAsync(code); + var gs = analysis.GlobalScope; + + var locations = new[] { + (new SourceLocation(5, 1), ""), + (new SourceLocation(5, 5), "A"), + (new SourceLocation(5, 9), "method") + }; + + foreach (var loc in locations) { + var scope = gs.FindScope(analysis.Document, loc.Item1); + scope.Name.Should().Be(loc.Item2, $"location {loc.Item1.Line}, {loc.Item1.Column}"); + } + } + } +} diff --git a/src/Analysis/Ast/Test/TypingTests.cs b/src/Analysis/Ast/Test/TypingTests.cs index 6c0940fdb..52bd9d96f 100644 --- a/src/Analysis/Ast/Test/TypingTests.cs +++ b/src/Analysis/Ast/Test/TypingTests.cs @@ -309,7 +309,7 @@ private static TypeAnnotation Parse(string expr, PythonLanguageVersion version = var errors = new CollectingErrorSink(); var ops = new ParserOptions { ErrorSink = errors }; var p = Parser.CreateParser(new StringReader(expr), version, ops); - var ast = p.ParseTopExpression(); + var ast = p.ParseTopExpression(null); if (errors.Errors.Any()) { foreach (var e in errors.Errors) { Console.WriteLine(e); diff --git a/src/Core/Impl/Extensions/IOExtensions.cs b/src/Core/Impl/Extensions/IOExtensions.cs index 4342a6506..3aaaca90b 100644 --- a/src/Core/Impl/Extensions/IOExtensions.cs +++ b/src/Core/Impl/Extensions/IOExtensions.cs @@ -116,6 +116,8 @@ public static string ReadTextWithRetry(this IFileSystem fs, string file) { return fs.ReadAllText(file); } catch (UnauthorizedAccessException) { Thread.Sleep(10); + } catch (IOException) { + Thread.Sleep(10); } } return null; diff --git a/src/Core/Impl/Text/SourceSpan.cs b/src/Core/Impl/Text/SourceSpan.cs index 8fa9b2294..d408cacdb 100644 --- a/src/Core/Impl/Text/SourceSpan.cs +++ b/src/Core/Impl/Text/SourceSpan.cs @@ -24,7 +24,7 @@ namespace Microsoft.Python.Core.Text { /// [Serializable] [DebuggerDisplay("({Start.Line}, {Start.Column})-({End.Line}, {End.Column})")] - public struct SourceSpan { + public struct SourceSpan: IComparable { /// /// Constructs a new span with a specific start and end location. /// @@ -107,6 +107,19 @@ public SourceSpan Union(SourceSpan other) { public static bool operator !=(SourceSpan left, SourceSpan right) => left.Start != right.Start || left.End != right.End; + public int CompareTo(SourceSpan other) { + if (Start.Line < other.Start.Line) { + return -1; + } + if (Start.Line == other.Start.Line) { + if (Start.Column < other.Start.Column) { + return -1; + } + return Start.Column == other.Start.Column ? 0 : 1; + } + return 1; + } + public override bool Equals(object obj) { if (!(obj is SourceSpan)) { return false; diff --git a/src/LanguageServer/Impl/Completion/ErrorExpressionCompletion.cs b/src/LanguageServer/Impl/Completion/ErrorExpressionCompletion.cs index 9508c2ba2..3430c64c6 100644 --- a/src/LanguageServer/Impl/Completion/ErrorExpressionCompletion.cs +++ b/src/LanguageServer/Impl/Completion/ErrorExpressionCompletion.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.IO; using System.Linq; using Microsoft.Python.Analysis; @@ -97,7 +96,7 @@ private static Expression GetExpressionFromText(string text, CompletionContext c scope = context.Analysis.FindScope(context.Location); using (var reader = new StringReader(text)) { var parser = Parser.CreateParser(reader, context.Ast.LanguageVersion, new ParserOptions()); - expressionAst = parser.ParseTopExpression(); + expressionAst = parser.ParseTopExpression(null); return Statement.GetExpression(expressionAst.Body); } } diff --git a/src/LanguageServer/Impl/Completion/ExpressionLocator.cs b/src/LanguageServer/Impl/Completion/ExpressionLocator.cs index e8aacb22f..9ce9cd0f6 100644 --- a/src/LanguageServer/Impl/Completion/ExpressionLocator.cs +++ b/src/LanguageServer/Impl/Completion/ExpressionLocator.cs @@ -21,6 +21,14 @@ namespace Microsoft.Python.LanguageServer.Completion { internal static class ExpressionLocator { public static void FindExpression(PythonAst ast, SourceLocation position, FindExpressionOptions options, out Node expression, out Node statement, out ScopeStatement scope) { + expression = null; + statement = null; + scope = null; + + if (ast == null) { + return; + } + var finder = new ExpressionFinder(ast, options); var index = ast.LocationToIndex(position); @@ -43,10 +51,10 @@ private static bool CanBackUp(PythonAst tree, Node node, Node statement, ScopeSt var top = 1; if (scope != null) { - var scopeStart = scope.GetStart(tree); + var scopeStart = scope.GetStart(); if (scope.Body != null) { - top = scope.Body.GetEnd(tree).Line == scopeStart.Line - ? scope.Body.GetStart(tree).Column + top = scope.Body.GetEnd().Line == scopeStart.Line + ? scope.Body.GetStart().Column : scopeStart.Column; } else { top = scopeStart.Column; diff --git a/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs b/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs index 9e3802dc1..844cf3dda 100644 --- a/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs +++ b/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -34,7 +33,7 @@ public static bool TryGetCompletionsForOverride(FunctionDefinition function, Com } if (function.Parent is ClassDefinition cd && function.NameExpression != null && context.Position > function.NameExpression.StartIndex) { - var loc = function.GetStart(context.Ast); + var loc = function.GetStart(); var overrideable = GetOverrideable(context, location).ToArray(); overrideable = !string.IsNullOrEmpty(function.Name) ? overrideable.Where(o => o.Name.StartsWithOrdinal(function.Name)).ToArray() diff --git a/src/LanguageServer/Impl/Completion/ImportCompletion.cs b/src/LanguageServer/Impl/Completion/ImportCompletion.cs index 5171c5e46..0b60509ab 100644 --- a/src/LanguageServer/Impl/Completion/ImportCompletion.cs +++ b/src/LanguageServer/Impl/Completion/ImportCompletion.cs @@ -18,10 +18,8 @@ using System.IO; using System.Linq; using Microsoft.Python.Analysis.Core.DependencyResolution; -using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; -using Microsoft.Python.Core.Collections; using Microsoft.Python.Core.Text; using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.Parsing.Ast; @@ -81,7 +79,7 @@ public static CompletionResult GetCompletionsInFromImport(FromImportStatement fr } if (context.Position >= name.StartIndex) { - var applicableSpan = name.GetSpan(context.Ast); + var applicableSpan = name.GetSpan(); var importSearchResult = mres.CurrentPathResolver.FindImports(document.FilePath, fromImport); return GetResultFromImportSearch(importSearchResult, context, false, applicableSpan); } diff --git a/src/LanguageServer/Impl/Definitions/ServerSettings.cs b/src/LanguageServer/Impl/Definitions/ServerSettings.cs index 32428d0bf..442538191 100644 --- a/src/LanguageServer/Impl/Definitions/ServerSettings.cs +++ b/src/LanguageServer/Impl/Definitions/ServerSettings.cs @@ -15,7 +15,6 @@ using System; using System.Collections.Generic; -using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.LanguageServer.Protocol; namespace Microsoft.Python.LanguageServer { diff --git a/src/LanguageServer/Impl/Documents/Document.cs b/src/LanguageServer/Impl/Documents/Document.cs new file mode 100644 index 000000000..7da841602 --- /dev/null +++ b/src/LanguageServer/Impl/Documents/Document.cs @@ -0,0 +1,38 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Logging; + +namespace Microsoft.Python.LanguageServer.Documents { + internal static class Document { + public static async Task GetAnalysisAsync(Uri uri, IServiceContainer services, int msTimeout = 300, CancellationToken cancellationToken = default) { + var rdt = services.GetService(); + var document = rdt.GetDocument(uri); + if (document == null) { + var log = services.GetService(); + log?.Log(TraceEventType.Error, $"Unable to find document {uri}"); + return null; + } + return await document.GetAnalysisAsync(msTimeout, cancellationToken); + } + } +} diff --git a/src/LanguageServer/Impl/Implementation/Server.Documents.cs b/src/LanguageServer/Impl/Implementation/Server.Documents.cs index 5aac074de..4ab9b5159 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Documents.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Documents.cs @@ -13,12 +13,8 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.Diagnostics; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Python.Analysis; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core; using Microsoft.Python.LanguageServer.Protocol; @@ -72,15 +68,5 @@ public void DidCloseTextDocument(DidCloseTextDocumentParams @params) { _rdt.CloseDocument(uri); _indexManager.ProcessClosedFile(uri.AbsolutePath); } - - private Task GetAnalysisAsync(Uri uri, CancellationToken cancellationToken) { - var document = _rdt.GetDocument(uri); - if (document == null) { - _log?.Log(TraceEventType.Error, $"Unable to find document {uri}"); - return Task.FromResult(default(IDocumentAnalysis)); - } - - return document.GetAnalysisAsync(200, cancellationToken); - } } } diff --git a/src/LanguageServer/Impl/Implementation/Server.Editor.cs b/src/LanguageServer/Impl/Implementation/Server.Editor.cs index 5467be1d1..f18384745 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Editor.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Editor.cs @@ -19,12 +19,15 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.LanguageServer.Completion; +using Microsoft.Python.LanguageServer.Documents; using Microsoft.Python.LanguageServer.Extensibility; using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.LanguageServer.Sources; namespace Microsoft.Python.LanguageServer.Implementation { public sealed partial class Server { + private const int CompletionAnalysisTimeout = 200; + private CompletionSource _completionSource; private HoverSource _hoverSource; private SignatureSource _signatureSource; @@ -34,10 +37,10 @@ public async Task Completion(CompletionParams @params, Cancellat _log?.Log(TraceEventType.Verbose, $"Completions in {uri} at {@params.position}"); var res = new CompletionList(); - var analysis = await GetAnalysisAsync(uri, cancellationToken); - if(analysis != null) { + var analysis = await Document.GetAnalysisAsync(uri, Services, CompletionAnalysisTimeout, cancellationToken); + if (analysis != null) { var result = _completionSource.GetCompletions(analysis, @params.position); - res.items = result.Completions.ToArray(); + res.items = result?.Completions?.ToArray() ?? Array.Empty(); await InvokeExtensionsAsync((ext, token) => (ext as ICompletionExtension)?.HandleCompletionAsync(analysis, @params.position, res.items.OfType().ToArray(), cancellationToken), cancellationToken); @@ -50,7 +53,7 @@ public async Task Hover(TextDocumentPositionParams @params, CancellationT var uri = @params.textDocument.uri; _log?.Log(TraceEventType.Verbose, $"Hover in {uri} at {@params.position}"); - var analysis = await GetAnalysisAsync(uri, cancellationToken); + var analysis = await Document.GetAnalysisAsync(uri, Services, CompletionAnalysisTimeout, cancellationToken); if (analysis != null) { return _hoverSource.GetHover(analysis, @params.position); } @@ -61,7 +64,7 @@ public async Task SignatureHelp(TextDocumentPositionParams @param var uri = @params.textDocument.uri; _log?.Log(TraceEventType.Verbose, $"Signatures in {uri} at {@params.position}"); - var analysis = await GetAnalysisAsync(uri, cancellationToken); + var analysis = await Document.GetAnalysisAsync(uri, Services, CompletionAnalysisTimeout, cancellationToken); if (analysis != null) { return _signatureSource.GetSignature(analysis, @params.position); } @@ -72,10 +75,21 @@ public async Task GotoDefinition(TextDocumentPositionParams @params var uri = @params.textDocument.uri; _log?.Log(TraceEventType.Verbose, $"Goto Definition in {uri} at {@params.position}"); - var analysis = await GetAnalysisAsync(uri, cancellationToken); - var ds = new DefinitionSource(); - var reference = ds.FindDefinition(analysis, @params.position); + var analysis = await Document.GetAnalysisAsync(uri, Services, CompletionAnalysisTimeout, cancellationToken); + var reference = new DefinitionSource(Services).FindDefinition(analysis, @params.position, out _); return reference != null ? new[] { reference } : Array.Empty(); } + + public Task FindReferences(ReferencesParams @params, CancellationToken cancellationToken) { + var uri = @params.textDocument.uri; + _log?.Log(TraceEventType.Verbose, $"References in {uri} at {@params.position}"); + return new ReferenceSource(Services).FindAllReferencesAsync(uri, @params.position, ReferenceSearchOptions.All, cancellationToken); + } + + public Task Rename(RenameParams @params, CancellationToken cancellationToken) { + var uri = @params.textDocument.uri; + _log?.Log(TraceEventType.Verbose, $"Rename in {uri} at {@params.position}"); + return new RenameSource(Services).RenameAsync(uri, @params.position, @params.newName, cancellationToken); + } } } diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index b99a4b918..7599da0c3 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -23,9 +23,7 @@ using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; -using Microsoft.Python.Core.Collections; using Microsoft.Python.Core.Disposables; using Microsoft.Python.Core.Idle; using Microsoft.Python.Core.IO; @@ -48,6 +46,7 @@ public sealed partial class Server : IDisposable { private ClientCapabilities _clientCaps; private ILogger _log; private IIndexManager _indexManager; + private string _rootDir; public Server(IServiceManager services) { _services = services; @@ -103,9 +102,7 @@ public async Task InitializeAsync(InitializeParams @params, Ca _services.AddService(new RunningDocumentTable(_services)); _rdt = _services.GetService(); - // TODO: multi-root workspaces. - var rootDir = @params.rootUri != null ? @params.rootUri.ToAbsolutePath() : PathUtils.NormalizePath(@params.rootPath); - + _rootDir = @params.rootUri != null ? @params.rootUri.ToAbsolutePath() : PathUtils.NormalizePath(@params.rootPath); Version.TryParse(@params.initializationOptions.interpreter.properties?.Version, out var version); var configuration = new InterpreterConfiguration(null, null, @@ -119,11 +116,11 @@ public async Task InitializeAsync(InitializeParams @params, Ca TypeshedPath = @params.initializationOptions.typeStubSearchPaths.FirstOrDefault() }; - _interpreter = await PythonInterpreter.CreateAsync(configuration, rootDir, _services, cancellationToken); + _interpreter = await PythonInterpreter.CreateAsync(configuration, _rootDir, _services, cancellationToken); _services.AddService(_interpreter); var fileSystem = _services.GetService(); - _indexManager = new IndexManager(fileSystem, _interpreter.LanguageVersion, rootDir, + _indexManager = new IndexManager(fileSystem, _interpreter.LanguageVersion, _rootDir, @params.initializationOptions.includeFiles, @params.initializationOptions.excludeFiles, _services.GetService()); diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index c79371de1..f5da88912 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -13,6 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -61,7 +62,7 @@ private async Task Parse(string path, CancellationToken parseCt) { try { using (var stream = _fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read)) { var parser = Parser.CreateParser(stream, _version); - ast = parser.ParseFile(); + ast = parser.ParseFile(new Uri(path)); } } finally { _semaphore.Release(); diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs index ddd1565df..0ebe6dda1 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs @@ -41,8 +41,8 @@ public override bool Walk(ClassDefinition node) { _stack.AddSymbol(new HierarchicalSymbol( node.Name, SymbolKind.Class, - node.GetSpan(_ast), - node.NameExpression.GetSpan(_ast), + node.GetSpan(), + node.NameExpression.GetSpan(), children, FunctionKind.Class )); @@ -58,13 +58,13 @@ public override bool Walk(FunctionDefinition node) { node.Body?.Walk(this); var children = _stack.Exit(); - var span = node.GetSpan(_ast); + var span = node.GetSpan(); var ds = new HierarchicalSymbol( node.Name, SymbolKind.Function, span, - node.IsLambda ? span : node.NameExpression.GetSpan(_ast), + node.IsLambda ? span : node.NameExpression.GetSpan(), children, FunctionKind.Function ); @@ -102,7 +102,7 @@ public override bool Walk(FunctionDefinition node) { public override bool Walk(ImportStatement node) { foreach (var (nameNode, nameString) in node.Names.Zip(node.AsNames, (name, asName) => asName != null ? (asName, asName.Name) : ((Node)name, name.MakeString()))) { - var span = nameNode.GetSpan(_ast); + var span = nameNode.GetSpan(); _stack.AddSymbol(new HierarchicalSymbol(nameString, SymbolKind.Module, span)); } @@ -115,7 +115,7 @@ public override bool Walk(FromImportStatement node) { } foreach (var name in node.Names.Zip(node.AsNames, (name, asName) => asName ?? name)) { - var span = name.GetSpan(_ast); + var span = name.GetSpan(); _stack.AddSymbol(new HierarchicalSymbol(name.Name, SymbolKind.Module, span)); } @@ -202,7 +202,7 @@ public override bool Walk(GeneratorExpression node) { private void ExitComprehension(Comprehension node) { var children = _stack.Exit(); - var span = node.GetSpan(_ast); + var span = node.GetSpan(); _stack.AddSymbol(new HierarchicalSymbol( $"<{node.NodeName}>", @@ -229,7 +229,7 @@ private void AddVarSymbol(NameExpression node) { break; } - var span = node.GetSpan(_ast); + var span = node.GetSpan(); _stack.AddSymbol(new HierarchicalSymbol(node.Name, kind, span)); } diff --git a/src/LanguageServer/Impl/Protocol/Classes.cs b/src/LanguageServer/Impl/Protocol/Classes.cs index b240c5605..417655caa 100644 --- a/src/LanguageServer/Impl/Protocol/Classes.cs +++ b/src/LanguageServer/Impl/Protocol/Classes.cs @@ -63,13 +63,6 @@ public sealed class TextEdit { /// empty string. /// public string newText; - - /// - /// Extended version information specifying the source version - /// that range applies to. Should be used by the client to - /// adjust range before applying the edit. - /// - public int? _version; } [Serializable] @@ -610,11 +603,6 @@ public sealed class ParameterInformation { public sealed class Reference { public Uri uri; public Range range; - - /// - /// The kind of reference - /// - public ReferenceKind? _kind; } [Serializable] diff --git a/src/LanguageServer/Impl/Protocol/Enums.cs b/src/LanguageServer/Impl/Protocol/Enums.cs index e332a9fa6..1bc87af14 100644 --- a/src/LanguageServer/Impl/Protocol/Enums.cs +++ b/src/LanguageServer/Impl/Protocol/Enums.cs @@ -179,11 +179,4 @@ public enum DocumentHighlightKind { Read = 2, Write = 3 } - - // Not in the LSP spec. - public enum ReferenceKind { - Definition = 1, - Reference = 2, - Value = 3 - } } diff --git a/src/LanguageServer/Impl/Resources.Designer.cs b/src/LanguageServer/Impl/Resources.Designer.cs index fbe92a2bf..e286522a2 100644 --- a/src/LanguageServer/Impl/Resources.Designer.cs +++ b/src/LanguageServer/Impl/Resources.Designer.cs @@ -87,15 +87,6 @@ internal static string Done { } } - /// - /// Looks up a localized string similar to Failed to create interpreter. - /// - internal static string Error_FailedToCreateInterpreter { - get { - return ResourceManager.GetString("Error_FailedToCreateInterpreter", resourceCulture); - } - } - /// /// Looks up a localized string similar to Initializing for generic interpreter. /// @@ -149,41 +140,5 @@ internal static string RenameVariable_CannotRename { return ResourceManager.GetString("RenameVariable_CannotRename", resourceCulture); } } - - /// - /// Looks up a localized string similar to Cannot rename modules. - /// - internal static string RenameVariable_CannotRenameModuleName { - get { - return ResourceManager.GetString("RenameVariable_CannotRenameModuleName", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to No information is available for the variable '{0}'.. - /// - internal static string RenameVariable_NoInformationAvailableForVariable { - get { - return ResourceManager.GetString("RenameVariable_NoInformationAvailableForVariable", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Please select a symbol to be renamed.. - /// - internal static string RenameVariable_SelectSymbol { - get { - return ResourceManager.GetString("RenameVariable_SelectSymbol", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Unable to get analysis for the selected expression.. - /// - internal static string RenameVariable_UnableGetExpressionAnalysis { - get { - return ResourceManager.GetString("RenameVariable_UnableGetExpressionAnalysis", resourceCulture); - } - } } } diff --git a/src/LanguageServer/Impl/Resources.resx b/src/LanguageServer/Impl/Resources.resx index acbc39276..b95437192 100644 --- a/src/LanguageServer/Impl/Resources.resx +++ b/src/LanguageServer/Impl/Resources.resx @@ -126,9 +126,6 @@ done. - - Failed to create interpreter - Initializing for generic interpreter @@ -147,16 +144,4 @@ Cannot rename - - Cannot rename modules - - - No information is available for the variable '{0}'. - - - Please select a symbol to be renamed. - - - Unable to get analysis for the selected expression. - \ No newline at end of file diff --git a/src/LanguageServer/Impl/Services/CoreShell.cs b/src/LanguageServer/Impl/Services/CoreShell.cs index 352f7be0d..580c47c33 100644 --- a/src/LanguageServer/Impl/Services/CoreShell.cs +++ b/src/LanguageServer/Impl/Services/CoreShell.cs @@ -18,7 +18,6 @@ using Microsoft.Python.Core; using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Core.Services; -using Microsoft.Python.Core.Shell; using Microsoft.PythonTools.LanguageServer.Services; namespace Microsoft.Python.LanguageServer.Services { diff --git a/src/LanguageServer/Impl/Sources/DefinitionSource.cs b/src/LanguageServer/Impl/Sources/DefinitionSource.cs index 4df412c85..f65ccb4c3 100644 --- a/src/LanguageServer/Impl/Sources/DefinitionSource.cs +++ b/src/LanguageServer/Impl/Sources/DefinitionSource.cs @@ -13,8 +13,10 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Linq; using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Analyzer.Expressions; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; @@ -28,49 +30,112 @@ namespace Microsoft.Python.LanguageServer.Sources { internal sealed class DefinitionSource { - public Reference FindDefinition(IDocumentAnalysis analysis, SourceLocation location) { + private readonly IServiceContainer _services; + + public DefinitionSource(IServiceContainer services) { + _services = services; + } + + public Reference FindDefinition(IDocumentAnalysis analysis, SourceLocation location, out ILocatedMember member) { + member = null; + if(analysis?.Ast == null) { + return null; + } + ExpressionLocator.FindExpression(analysis.Ast, location, FindExpressionOptions.Hover, out var exprNode, out var statement, out var exprScope); - if (exprNode is ConstantExpression) { + if (exprNode is ConstantExpression || !(exprNode is Expression expr)) { return null; // No hover for literals. } - if (!(exprNode is Expression expr)) { - return null; - } var eval = analysis.ExpressionEvaluator; - var name = (expr as NameExpression)?.Name; - using (eval.OpenScope(analysis.Document, exprScope)) { - // First try variables, except in imports - if (!string.IsNullOrEmpty(name) && !(statement is ImportStatement) && !(statement is FromImportStatement)) { - var m = eval.LookupNameInScopes(name, out var scope); - if (m != null && scope.Variables[name] is IVariable v) { - var type = v.Value.GetPythonType(); - var module = type as IPythonModule ?? type?.DeclaringModule; - if (CanNavigateToModule(module, analysis)) { - return new Reference { range = v.Location.Span, uri = v.Location.DocumentUri }; - } - } + if (expr is MemberExpression mex) { + return FromMemberExpression(mex, analysis, out member); } - var value = eval.GetValueFromExpression(expr); - if (value.IsUnknown() && !string.IsNullOrEmpty(name)) { - var reference = FromImport(statement, name, analysis, out value); + // Try variables + var name = (expr as NameExpression)?.Name; + IMember value = null; + if (!string.IsNullOrEmpty(name)) { + var reference = TryFromVariable(name, analysis, location, statement, out member); if (reference != null) { return reference; } + + if (statement is ImportStatement || statement is FromImportStatement) { + reference = TryFromImport(statement, name, analysis, out value); + if (reference != null) { + member = value as ILocatedMember; + return reference; + } + } } + value = value ?? eval.GetValueFromExpression(expr); if (value.IsUnknown()) { return null; } - return FromMember(value, expr, statement, analysis); + member = value as ILocatedMember; + return FromMember(value); + } + } + + private Reference TryFromVariable(string name, IDocumentAnalysis analysis, SourceLocation location, Node statement, out ILocatedMember member) { + member = null; + + var m = analysis.ExpressionEvaluator.LookupNameInScopes(name, out var scope); + if (m != null && scope.Variables[name] is IVariable v) { + member = v; + var definition = v.Definition; + if (statement is ImportStatement || statement is FromImportStatement) { + // If we are on the variable definition in this module, + // then goto definition should go to the parent, if any. + var indexSpan = v.Definition.Span.ToIndexSpan(analysis.Ast); + var index = location.ToIndex(analysis.Ast); + if (indexSpan.Start <= index && index < indexSpan.End) { + if (v.Parent == null) { + return null; + } + definition = v.Parent.Definition; + } + } + + if (CanNavigateToModule(definition.DocumentUri)) { + return new Reference { range = definition.Span, uri = definition.DocumentUri }; + } + } + return null; + } + + private Reference FromMemberExpression(MemberExpression mex, IDocumentAnalysis analysis, out ILocatedMember member) { + member = null; + + var eval = analysis.ExpressionEvaluator; + var target = eval.GetValueFromExpression(mex.Target); + var type = target?.GetPythonType(); + + if (type?.GetMember(mex.Name) is ILocatedMember lm) { + member = lm; + return FromMember(lm); + } + + if (type is IPythonClassType cls) { + // Data members may be instances that are not tracking locations. + // In this case we'll try look up the respective variable instead. + using (eval.OpenScope(analysis.Document, cls.ClassDefinition)) { + eval.LookupNameInScopes(mex.Name, out _, out var v, LookupOptions.Local); + if (v != null) { + member = v; + return FromMember(v); + } + } } + return null; } - private Reference FromImport(Node statement, string name, IDocumentAnalysis analysis, out IMember value) { + private Reference TryFromImport(Node statement, string name, IDocumentAnalysis analysis, out IMember value) { value = null; string moduleName = null; switch (statement) { @@ -83,109 +148,62 @@ private Reference FromImport(Node statement, string name, IDocumentAnalysis anal if (moduleName != null) { var module = analysis.Document.Interpreter.ModuleResolution.GetImportedModule(moduleName); - if (module != null && CanNavigateToModule(module, analysis)) { - return new Reference { range = default, uri = module.Uri }; + if (module != null) { + value = module; + return CanNavigateToModule(module) ? new Reference { range = default, uri = module.Uri } : null; } } // Perhaps it is a member such as A in 'from X import A as B' switch (statement) { case ImportStatement imp: { - // Import A as B - var index = imp.Names.IndexOf(x => x?.MakeString() == name); - if (index >= 0 && index < imp.AsNames.Count) { - value = analysis.ExpressionEvaluator.GetValueFromExpression(imp.AsNames[index]); - return null; + // Import A as B + var index = imp.Names.IndexOf(x => x?.MakeString() == name); + if (index >= 0 && index < imp.AsNames.Count) { + value = analysis.ExpressionEvaluator.GetValueFromExpression(imp.AsNames[index]); + return null; + } + break; } - break; - } case FromImportStatement fimp: { - // From X import A as B - var index = fimp.Names.IndexOf(x => x?.Name == name); - if (index >= 0 && index < fimp.AsNames.Count) { - value = analysis.ExpressionEvaluator.GetValueFromExpression(fimp.AsNames[index]); - return null; + // From X import A as B + var index = fimp.Names.IndexOf(x => x?.Name == name); + if (index >= 0 && index < fimp.AsNames.Count) { + value = analysis.ExpressionEvaluator.GetValueFromExpression(fimp.AsNames[index]); + return null; + } + break; } - break; - } } return null; } - private Reference FromMember(IMember value, Expression expr, Node statement, IDocumentAnalysis analysis) { - Node node = null; - IPythonModule module = null; - LocationInfo location = null; - var eval = analysis.ExpressionEvaluator; - switch (value) { - case IPythonClassType cls: - node = cls.ClassDefinition; - module = cls.DeclaringModule; - location = cls.Location; - break; - case IPythonFunctionType fn: - node = fn.FunctionDefinition; - module = fn.DeclaringModule; - location = fn.Location; - break; - case IPythonPropertyType prop: - node = prop.FunctionDefinition; - module = prop.DeclaringModule; - location = prop.Location; - break; - case IPythonModule mod: - return HandleModule(mod, analysis, statement); - case IPythonInstance instance when instance.Type is IPythonFunctionType ft: - node = ft.FunctionDefinition; - module = ft.DeclaringModule; - location = ft.Location; - break; - case IPythonInstance instance when instance.Type is IPythonFunctionType ft: - node = ft.FunctionDefinition; - module = ft.DeclaringModule; - break; - case IPythonInstance _ when expr is NameExpression nex: - var m1 = eval.LookupNameInScopes(nex.Name, out var scope); - if (m1 != null && scope != null) { - var v = scope.Variables[nex.Name]; - if (v != null) { - return new Reference { range = v.Location.Span, uri = v.Location.DocumentUri }; - } - } - break; - case IPythonInstance _ when expr is MemberExpression mex: { - var target = eval.GetValueFromExpression(mex.Target); - var type = target?.GetPythonType(); - var m2 = type?.GetMember(mex.Name); - if (m2 is IPythonInstance v) { - return new Reference { range = v.Location.Span, uri = v.Location.DocumentUri }; - } - return FromMember(m2, null, statement, analysis); - } + private Reference FromMember(IMember m) { + var definition = (m as ILocatedMember)?.Definition; + var moduleUri = definition?.DocumentUri; + // Make sure module we are looking for is not a stub + if (m is IPythonType t) { + moduleUri = t.DeclaringModule.ModuleType == ModuleType.Stub + ? t.DeclaringModule.PrimaryModule.Uri + : t.DeclaringModule.Uri; } - - module = module?.ModuleType == ModuleType.Stub ? module.PrimaryModule : module; - if (node != null && module is IDocument doc && CanNavigateToModule(module, analysis)) { - return new Reference { - range = location?.Span ?? node.GetSpan(doc.GetAnyAst()), uri = doc.Uri - }; + if (definition != null && CanNavigateToModule(moduleUri)) { + return new Reference { range = definition.Span, uri = moduleUri }; } - return null; } - private static Reference HandleModule(IPythonModule module, IDocumentAnalysis analysis, Node statement) { - // If we are in import statement, open the module source if available. - if (statement is ImportStatement || statement is FromImportStatement) { - if (module.Uri != null && CanNavigateToModule(module, analysis)) { - return new Reference { range = default, uri = module.Uri }; - } + private bool CanNavigateToModule(Uri uri) { + if (uri == null) { + return false; } - return null; + var rdt = _services.GetService(); + var doc = rdt.GetDocument(uri); + return CanNavigateToModule(doc); } - private static bool CanNavigateToModule(IPythonModule m, IDocumentAnalysis analysis) { + private static bool CanNavigateToModule(IPythonModule m) { if (m == null) { return false; } diff --git a/src/LanguageServer/Impl/Sources/HoverSource.cs b/src/LanguageServer/Impl/Sources/HoverSource.cs index 0dc513e49..729b69c97 100644 --- a/src/LanguageServer/Impl/Sources/HoverSource.cs +++ b/src/LanguageServer/Impl/Sources/HoverSource.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using Microsoft.Python.Analysis; using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Analyzer.Expressions; @@ -46,8 +45,8 @@ public Hover GetHover(IDocumentAnalysis analysis, SourceLocation location) { } var range = new Range { - start = expr.GetStart(analysis.Ast), - end = expr.GetEnd(analysis.Ast) + start = expr.GetStart(), + end = expr.GetEnd() }; var eval = analysis.ExpressionEvaluator; @@ -92,7 +91,7 @@ public Hover GetHover(IDocumentAnalysis analysis, SourceLocation location) { if (expr is MemberExpression mex) { name = mex.Name; range = new Range { - start = mex.Target.GetEnd(analysis.Ast), + start = mex.Target.GetEnd(), end = range.end }; diff --git a/src/LanguageServer/Impl/Sources/ReferenceSource.cs b/src/LanguageServer/Impl/Sources/ReferenceSource.cs new file mode 100644 index 000000000..310981188 --- /dev/null +++ b/src/LanguageServer/Impl/Sources/ReferenceSource.cs @@ -0,0 +1,198 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; +using Microsoft.Python.Core.IO; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Documents; +using Microsoft.Python.LanguageServer.Protocol; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Sources { + internal enum ReferenceSearchOptions { + All, + ExcludeLibraries + } + + internal sealed class ReferenceSource { + private const int FindReferencesAnalysisTimeout = 10000; + private readonly IServiceContainer _services; + + public ReferenceSource(IServiceContainer services) { + _services = services; + } + + public async Task FindAllReferencesAsync(Uri uri, SourceLocation location, ReferenceSearchOptions options, CancellationToken cancellationToken = default) { + if (uri != null) { + var analysis = await Document.GetAnalysisAsync(uri, _services, FindReferencesAnalysisTimeout, cancellationToken); + + var definition = new DefinitionSource(_services).FindDefinition(analysis, location, out var definingMember); + if (definition == null) { + return Array.Empty(); + } + + var rootDefinition = GetRootDefinition(definingMember); + var name = definingMember.GetName(); + + Debug.Assert(rootDefinition.DeclaringModule != null); + if (rootDefinition.DeclaringModule == null) { + return Array.Empty(); + } + + if (!string.IsNullOrEmpty(name) && (rootDefinition.DeclaringModule.ModuleType == ModuleType.User || options == ReferenceSearchOptions.All)) { + return await FindAllReferencesAsync(name, rootDefinition, cancellationToken); + } + } + return Array.Empty(); + } + + private async Task FindAllReferencesAsync(string name, ILocatedMember rootDefinition, CancellationToken cancellationToken) { + var candidateFiles = ScanClosedFiles(name, cancellationToken); + await AnalyzeFiles(candidateFiles, cancellationToken); + + return rootDefinition.References + .Select(r => new Reference { uri = new Uri(r.FilePath), range = r.Span }) + .ToArray(); + } + + private IEnumerable ScanClosedFiles(string name, CancellationToken cancellationToken) { + var fs = _services.GetService(); + var rdt = _services.GetService(); + var interpreter = _services.GetService(); + + var root = interpreter.ModuleResolution.Root; + var interpreterPaths = interpreter.ModuleResolution.InterpreterPaths.ToArray(); + var files = new List(); + + foreach (var filePath in fs.GetFiles(root, "*.py", SearchOption.AllDirectories).Select(Path.GetFullPath)) { + try { + cancellationToken.ThrowIfCancellationRequested(); + // Exclude files that are inside interpreter paths such as when + // virtual environment is inside the user workspace folder. + var fileDirectory = Path.GetDirectoryName(filePath); + if (interpreterPaths.Any(p => fs.IsPathUnderRoot(p, fileDirectory))) { + continue; + } + + var uri = new Uri(filePath); + var doc = rdt.GetDocument(uri); + if (doc != null) { + continue; + } + + var content = fs.ReadTextWithRetry(filePath); + if (content.Contains(name)) { + files.Add(uri); + } + } catch (IOException) { } catch (UnauthorizedAccessException) { } + } + + return files; + } + + private IEnumerable ScanFiles(IDictionary closedFiles, string name, CancellationToken cancellationToken) { + var candidateNames = new HashSet { name }; + var candidateFiles = new HashSet(); + + while (candidateNames.Count > 0) { + var nextCandidateNames = new HashSet(); + + foreach (var kvp in closedFiles.ToArray()) { + cancellationToken.ThrowIfCancellationRequested(); + + var w = new ImportsWalker(candidateNames); + try { + kvp.Value.Walk(w); + } catch (OperationCanceledException) { } + + if (w.IsCandidate) { + candidateFiles.Add(kvp.Key); + nextCandidateNames.Add(Path.GetFileNameWithoutExtension(kvp.Key)); + closedFiles.Remove(kvp.Key); + } + } + candidateNames = nextCandidateNames; + } + return candidateFiles; + } + + private async Task AnalyzeFiles(IEnumerable files, CancellationToken cancellationToken) { + var rdt = _services.GetService(); + foreach (var f in files) { + Analyze(f, rdt); + } + var analyzer = _services.GetService(); + await analyzer.WaitForCompleteAnalysisAsync(cancellationToken); + } + + private static void Analyze(Uri uri, IRunningDocumentTable rdt) { + if (rdt.GetDocument(uri) != null) { + return; // Already opened by another analysis. + } + + var filePath = uri.ToAbsolutePath(); + var mco = new ModuleCreationOptions { + ModuleName = Path.GetFileNameWithoutExtension(filePath), + FilePath = filePath, + Uri = uri, + ModuleType = ModuleType.User + }; + rdt.AddModule(mco); + } + + private ILocatedMember GetRootDefinition(ILocatedMember lm) { + for (; lm.Parent != null; lm = lm.Parent) { } + return lm; + } + + private class ImportsWalker : PythonWalker { + private readonly HashSet _names; + + public bool IsCandidate { get; private set; } + + public ImportsWalker(HashSet names) { + _names = names; + } + + public override bool Walk(ImportStatement node) { + if (node.Names.ExcludeDefault().Any(n => _names.Any(x => n.MakeString().Contains(x)))) { + IsCandidate = true; + throw new OperationCanceledException(); + } + return false; + } + + public override bool Walk(FromImportStatement node) { + if (_names.Any(x => node.Root.MakeString().Contains(x))) { + IsCandidate = true; + throw new OperationCanceledException(); + } + return false; + } + } + } +} diff --git a/src/LanguageServer/Impl/Sources/RenameSource.cs b/src/LanguageServer/Impl/Sources/RenameSource.cs new file mode 100644 index 000000000..6f6068f62 --- /dev/null +++ b/src/LanguageServer/Impl/Sources/RenameSource.cs @@ -0,0 +1,54 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Protocol; + +namespace Microsoft.Python.LanguageServer.Sources { + internal sealed class RenameSource { + private readonly IServiceContainer _services; + + public RenameSource(IServiceContainer services) { + _services = services; + } + + public async Task RenameAsync(Uri uri, SourceLocation location, string newName, CancellationToken cancellationToken = default) { + var rs = new ReferenceSource(_services); + var references = await rs.FindAllReferencesAsync(uri, location, ReferenceSearchOptions.ExcludeLibraries, cancellationToken); + if (references.Length == 0) { + return null; + } + + var changes = new Dictionary>(); + foreach (var r in references) { + if (!changes.TryGetValue(r.uri, out var edits)) { + changes[r.uri] = edits = new List(); + } + edits.Add(new TextEdit { newText = newName, range = r.range }); + } + + return new WorkspaceEdit { + changes = changes.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.ToArray()) + }; + } + } +} diff --git a/src/LanguageServer/Test/CompletionTests.cs b/src/LanguageServer/Test/CompletionTests.cs index fd8e57dde..60c9a24bd 100644 --- a/src/LanguageServer/Test/CompletionTests.cs +++ b/src/LanguageServer/Test/CompletionTests.cs @@ -16,7 +16,6 @@ using System; using System.IO; using System.Linq; -using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis; @@ -147,7 +146,7 @@ def fob(self): [TestMethod, Priority(0)] public async Task TypeAtEndOfIncompleteMethod() { - var code = @" + const string code = @" class Fob(object): def oar(self, a): @@ -1122,7 +1121,6 @@ public async Task NoCompletionForCurrentModuleName(bool empty) { } [TestMethod, Priority(0)] - [Ignore] public async Task OddNamedFile() { const string code = @" import sys @@ -1133,11 +1131,24 @@ import sys var rdt = Services.GetService(); var doc = rdt.OpenDocument(uri, null, uri.AbsolutePath); - var analysis = await doc.GetAnalysisAsync(Timeout.Infinite); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var analysis = await GetDocumentAnalysisAsync(doc); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); var completions = cs.GetCompletions(analysis, new SourceLocation(3, 5)); completions.Should().HaveLabels("argv", "path", "exit"); } + + [TestMethod, Priority(0)] + public async Task FunctionScope() { + const string code = @" +def func(): + aaa = 1 +a"; + var analysis = await GetAnalysisAsync(code); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = cs.GetCompletions(analysis, new SourceLocation(4, 2)); + result.Completions.Select(c => c.label).Should().NotContain("aaa"); + } } } diff --git a/src/LanguageServer/Test/GoToDefinitionTests.cs b/src/LanguageServer/Test/GoToDefinitionTests.cs index 21f6ace79..323dc2e44 100644 --- a/src/LanguageServer/Test/GoToDefinitionTests.cs +++ b/src/LanguageServer/Test/GoToDefinitionTests.cs @@ -13,10 +13,9 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; -using System.IO; using System.Threading.Tasks; using FluentAssertions; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core.Text; using Microsoft.Python.LanguageServer.Sources; @@ -61,34 +60,43 @@ def func(a, b): c.method(1, 2) "; var analysis = await GetAnalysisAsync(code); - var ds = new DefinitionSource(); + var ds = new DefinitionSource(Services); - var reference = ds.FindDefinition(analysis, new SourceLocation(4, 5)); + var reference = ds.FindDefinition(analysis, new SourceLocation(4, 5), out _); + reference.Should().NotBeNull(); reference.range.Should().Be(1, 7, 1, 9); - reference = ds.FindDefinition(analysis, new SourceLocation(9, 9)); + reference = ds.FindDefinition(analysis, new SourceLocation(9, 9), out _); + reference.Should().NotBeNull(); reference.range.Should().Be(7, 15, 7, 19); - reference = ds.FindDefinition(analysis, new SourceLocation(9, 14)); + reference = ds.FindDefinition(analysis, new SourceLocation(9, 14), out _); + reference.Should().NotBeNull(); reference.range.Should().Be(6, 4, 6, 5); - reference = ds.FindDefinition(analysis, new SourceLocation(13, 5)); + reference = ds.FindDefinition(analysis, new SourceLocation(13, 5), out _); + reference.Should().NotBeNull(); reference.range.Should().Be(11, 9, 11, 10); - reference = ds.FindDefinition(analysis, new SourceLocation(14, 9)); + reference = ds.FindDefinition(analysis, new SourceLocation(14, 9), out _); + reference.Should().NotBeNull(); reference.range.Should().Be(11, 9, 11, 10); - reference = ds.FindDefinition(analysis, new SourceLocation(17, 5)); - reference.range.Should().Be(11, 0, 14, 12); + reference = ds.FindDefinition(analysis, new SourceLocation(17, 5), out _); + reference.Should().NotBeNull(); + reference.range.Should().Be(11, 4, 11, 8); - reference = ds.FindDefinition(analysis, new SourceLocation(18, 1)); - reference.range.Should().Be(3, 0, 3, 1); // TODO: store all locations + reference = ds.FindDefinition(analysis, new SourceLocation(18, 1), out _); + reference.Should().NotBeNull(); + reference.range.Should().Be(3, 0, 3, 1); - reference = ds.FindDefinition(analysis, new SourceLocation(19, 5)); - reference.range.Should().Be(5, 6, 9, 18); + reference = ds.FindDefinition(analysis, new SourceLocation(19, 5), out _); + reference.Should().NotBeNull(); + reference.range.Should().Be(5, 6, 5, 7); - reference = ds.FindDefinition(analysis, new SourceLocation(20, 5)); - reference.range.Should().Be(7, 4, 9, 18); + reference = ds.FindDefinition(analysis, new SourceLocation(20, 5), out _); + reference.Should().NotBeNull(); + reference.range.Should().Be(7, 8, 7, 14); } [TestMethod, Priority(0)] @@ -100,20 +108,23 @@ import logging logging.info('') "; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); - var ds = new DefinitionSource(); + var ds = new DefinitionSource(Services); - var reference = ds.FindDefinition(analysis, new SourceLocation(2, 9)); + var reference = ds.FindDefinition(analysis, new SourceLocation(2, 9), out _); reference.Should().BeNull(); - reference = ds.FindDefinition(analysis, new SourceLocation(5, 3)); + reference = ds.FindDefinition(analysis, new SourceLocation(5, 3), out _); + reference.Should().NotBeNull(); reference.range.Should().Be(2, 7, 2, 14); - reference = ds.FindDefinition(analysis, new SourceLocation(3, 10)); + reference = ds.FindDefinition(analysis, new SourceLocation(3, 10), out _); + reference.Should().NotBeNull(); reference.range.Should().Be(0, 0, 0, 0); reference.uri.AbsolutePath.Should().Contain("logging"); reference.uri.AbsolutePath.Should().NotContain("pyi"); - reference = ds.FindDefinition(analysis, new SourceLocation(5, 11)); + reference = ds.FindDefinition(analysis, new SourceLocation(5, 11), out _); + reference.Should().NotBeNull(); reference.uri.AbsolutePath.Should().Contain("logging"); reference.uri.AbsolutePath.Should().NotContain("pyi"); } @@ -125,40 +136,63 @@ import logging as log log "; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); - var ds = new DefinitionSource(); + var ds = new DefinitionSource(Services); - var reference = ds.FindDefinition(analysis, new SourceLocation(2, 10)); + var reference = ds.FindDefinition(analysis, new SourceLocation(2, 10), out _); + reference.Should().NotBeNull(); reference.range.Should().Be(0, 0, 0, 0); reference.uri.AbsolutePath.Should().Contain("logging"); reference.uri.AbsolutePath.Should().NotContain("pyi"); - reference = ds.FindDefinition(analysis, new SourceLocation(3, 2)); + reference = ds.FindDefinition(analysis, new SourceLocation(3, 2), out _); + reference.Should().NotBeNull(); reference.range.Should().Be(1, 18, 1, 21); - reference = ds.FindDefinition(analysis, new SourceLocation(2, 20)); + reference = ds.FindDefinition(analysis, new SourceLocation(2, 20), out _); + reference.Should().NotBeNull(); reference.uri.AbsolutePath.Should().Contain("logging"); reference.uri.AbsolutePath.Should().NotContain("pyi"); } [TestMethod, Priority(0)] - public async Task GotoModuleSourceFromImport() { + public async Task GotoModuleSourceFromImport1() { const string code = @"from logging import A"; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); - var ds = new DefinitionSource(); + var ds = new DefinitionSource(Services); - var reference = ds.FindDefinition(analysis, new SourceLocation(1, 7)); + var reference = ds.FindDefinition(analysis, new SourceLocation(1, 7), out _); + reference.Should().NotBeNull(); reference.range.Should().Be(0, 0, 0, 0); reference.uri.AbsolutePath.Should().Contain("logging"); reference.uri.AbsolutePath.Should().NotContain("pyi"); } + [TestMethod, Priority(0)] + public async Task GotoModuleSourceFromImport2() { + const string code = @" +from MultiValues import t +x = t +"; + var analysis = await GetAnalysisAsync(code); + var ds = new DefinitionSource(Services); + + var reference = ds.FindDefinition(analysis, new SourceLocation(3, 5), out _); + reference.Should().NotBeNull(); + reference.range.Should().Be(1, 24, 1, 25); + + reference = ds.FindDefinition(analysis, new SourceLocation(2, 25), out _); + reference.Should().NotBeNull(); + reference.range.Should().Be(2, 0, 2, 1); + reference.uri.AbsolutePath.Should().Contain("MultiValues.py"); + } + [TestMethod, Priority(0)] public async Task GotoModuleSourceFromImportAs() { const string code = @"from logging import RootLogger as rl"; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); - var ds = new DefinitionSource(); + var ds = new DefinitionSource(Services); - var reference = ds.FindDefinition(analysis, new SourceLocation(1, 23)); + var reference = ds.FindDefinition(analysis, new SourceLocation(1, 23), out _); reference.Should().NotBeNull(); reference.range.start.line.Should().BeGreaterThan(500); reference.uri.AbsolutePath.Should().Contain("logging"); @@ -172,9 +206,9 @@ class A(object): pass "; var analysis = await GetAnalysisAsync(code); - var ds = new DefinitionSource(); + var ds = new DefinitionSource(Services); - var reference = ds.FindDefinition(analysis, new SourceLocation(2, 12)); + var reference = ds.FindDefinition(analysis, new SourceLocation(2, 12), out _); reference.Should().BeNull(); } @@ -195,9 +229,38 @@ public async Task GotoRelativeImportInExplicitPackage() { var submod = rdt.OpenDocument(submodPath, "from .. import mod"); var analysis = await submod.GetAnalysisAsync(-1); - var ds = new DefinitionSource(); - var reference = ds.FindDefinition(analysis, new SourceLocation(1, 18)); + var ds = new DefinitionSource(Services); + var reference = ds.FindDefinition(analysis, new SourceLocation(1, 18), out _); + reference.Should().NotBeNull(); reference.uri.Should().Be(modPath); } + + [TestMethod, Priority(0)] + public async Task EmptyAnalysis() { + var analysis = await GetAnalysisAsync(string.Empty); + var ds = new DefinitionSource(Services); + + var reference = ds.FindDefinition(new EmptyAnalysis(Services, analysis.Document), new SourceLocation(1, 1), out _); + reference.Should().BeNull(); + + reference = ds.FindDefinition(null, new SourceLocation(1, 1), out _); + reference.Should().BeNull(); + } + + [TestMethod, Priority(0)] + public async Task ReCompile() { + const string code = @" +import re +x = re.compile(r'hello', re.IGNORECASE) +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var ds = new DefinitionSource(Services); + + var reference = ds.FindDefinition(analysis, new SourceLocation(3, 10), out _); + reference.Should().NotBeNull(); + reference.range.start.line.Should().BeGreaterThan(0); + reference.uri.AbsolutePath.Should().Contain("re.py"); + reference.uri.AbsolutePath.Should().NotContain("pyi"); + } } } diff --git a/src/LanguageServer/Test/ImportsTests.cs b/src/LanguageServer/Test/ImportsTests.cs index 43d7fc622..a55852c57 100644 --- a/src/LanguageServer/Test/ImportsTests.cs +++ b/src/LanguageServer/Test/ImportsTests.cs @@ -16,7 +16,6 @@ using System; using System.IO; using System.Threading.Tasks; -using FluentAssertions; using Microsoft.Python.Analysis; using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Documents; @@ -24,7 +23,6 @@ using Microsoft.Python.LanguageServer.Completion; using Microsoft.Python.LanguageServer.Sources; using Microsoft.Python.LanguageServer.Tests.FluentAssertions; -using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; diff --git a/src/LanguageServer/Test/LinterTests.cs b/src/LanguageServer/Test/LinterTests.cs index ec6c62e47..678b12c46 100644 --- a/src/LanguageServer/Test/LinterTests.cs +++ b/src/LanguageServer/Test/LinterTests.cs @@ -13,24 +13,11 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis; using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Diagnostics; -using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Core.IO; -using Microsoft.Python.Core.OS; -using Microsoft.Python.Core.Services; -using Microsoft.Python.Core.Text; -using Microsoft.Python.LanguageServer.Completion; -using Microsoft.Python.LanguageServer.Sources; -using Microsoft.Python.LanguageServer.Tests.FluentAssertions; -using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; using NSubstitute; using TestUtilities; diff --git a/src/LanguageServer/Test/RdtTests.cs b/src/LanguageServer/Test/RdtTests.cs index 39e028e83..738dc740f 100644 --- a/src/LanguageServer/Test/RdtTests.cs +++ b/src/LanguageServer/Test/RdtTests.cs @@ -15,7 +15,6 @@ using System; using System.Linq; -using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Analyzer; @@ -23,9 +22,9 @@ using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.LanguageServer.Tests.FluentAssertions; using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; -using Microsoft.Python.LanguageServer.Tests.FluentAssertions; using TestUtilities; namespace Microsoft.Python.LanguageServer.Tests { @@ -125,7 +124,7 @@ public async Task OpenCloseAnalysis() { rdt.OpenDocument(docLc1.Uri, null); PublishDiagnostics(); ds.Diagnostics[uri].Should().BeEmpty(); - ds.Diagnostics[docLc1.Uri].Count.Should().Be(2); + ds.Diagnostics[docLc1.Uri].Count.Should().Be(3); ds.Diagnostics[docLc2.Uri].Should().BeEmpty(); ds.Diagnostics[docLc3.Uri].Should().BeEmpty(); @@ -142,7 +141,7 @@ public async Task OpenCloseAnalysis() { PublishDiagnostics(); ds.Diagnostics[uri].Should().BeEmpty(); - ds.Diagnostics[docLc1.Uri].Count.Should().Be(2); + ds.Diagnostics[docLc1.Uri].Count.Should().Be(3); ds.Diagnostics[docLc2.Uri].Should().BeEmpty(); ds.Diagnostics[docLc3.Uri].Should().BeEmpty(); } diff --git a/src/LanguageServer/Test/ReferencesTests.cs b/src/LanguageServer/Test/ReferencesTests.cs new file mode 100644 index 000000000..995f5b87f --- /dev/null +++ b/src/LanguageServer/Test/ReferencesTests.cs @@ -0,0 +1,334 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Sources; +using Microsoft.Python.LanguageServer.Tests.FluentAssertions; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.LanguageServer.Tests { + [TestClass] + public class ReferencesTests : LanguageServerTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + + [TestMethod, Priority(0)] + public async Task SingleFile() { + const string code = @" +x = 1 + +def func(x): + return x + +y = func(x) +x = 2 +"; + var analysis = await GetAnalysisAsync(code); + var rs = new ReferenceSource(Services); + var refs = await rs.FindAllReferencesAsync(analysis.Document.Uri, new SourceLocation(8, 1), ReferenceSearchOptions.All); + + refs.Should().HaveCount(3); + refs[0].range.Should().Be(1, 0, 1, 1); + refs[1].range.Should().Be(6, 9, 6, 10); + refs[2].range.Should().Be(7, 0, 7, 1); + } + + [TestMethod, Priority(0)] + public async Task TwoOpenFiles() { + const string code1 = @" +x = 1 + +def func(x): + return x + +y = func(x) +x = 2 +"; + var uri1 = TestData.GetDefaultModuleUri(); + var uri2 = TestData.GetNextModuleUri(); + + var code2 = $@" +from {Path.GetFileNameWithoutExtension(uri1.AbsolutePath)} import x +y = x +"; + await CreateServicesAsync(PythonVersions.LatestAvailable3X, uri1.AbsolutePath); + + var rdt = Services.GetService(); + rdt.OpenDocument(uri1, code1); + rdt.OpenDocument(uri2, code2); + + var doc1 = rdt.GetDocument(uri1); + var analysis = await GetDocumentAnalysisAsync(doc1); + + var rs = new ReferenceSource(Services); + var refs = await rs.FindAllReferencesAsync(analysis.Document.Uri, new SourceLocation(7, 10), ReferenceSearchOptions.All); + + refs.Should().HaveCount(5); + + refs[0].range.Should().Be(1, 0, 1, 1); + refs[0].uri.Should().Be(uri1); + refs[1].range.Should().Be(6, 9, 6, 10); + refs[1].uri.Should().Be(uri1); + refs[2].range.Should().Be(7, 0, 7, 1); + refs[2].uri.Should().Be(uri1); + + refs[3].range.Should().Be(1, 19, 1, 20); + refs[3].uri.Should().Be(uri2); + refs[4].range.Should().Be(2, 4, 2, 5); + refs[4].uri.Should().Be(uri2); + } + + [TestMethod, Priority(0)] + public async Task ClosedFiles() { + const string code = @" +x = 1 + +def func(x): + return x + +y = func(x) +x = 2 +"; + const string mod2Code = @" +from module import x +y = x +"; + const string mod3Code = @" +from module import x +y = x +"; + var uri2 = await TestData.CreateTestSpecificFileAsync("module2.py", mod2Code); + var uri3 = await TestData.CreateTestSpecificFileAsync("module3.py", mod3Code); + + var analysis = await GetAnalysisAsync(code); + var rs = new ReferenceSource(Services); + var refs = await rs.FindAllReferencesAsync(analysis.Document.Uri, new SourceLocation(7, 10), ReferenceSearchOptions.All); + + refs.Should().HaveCount(7); + + refs[0].range.Should().Be(1, 0, 1, 1); + refs[0].uri.Should().Be(analysis.Document.Uri); + refs[1].range.Should().Be(6, 9, 6, 10); + refs[1].uri.Should().Be(analysis.Document.Uri); + refs[2].range.Should().Be(7, 0, 7, 1); + refs[2].uri.Should().Be(analysis.Document.Uri); + + refs[3].range.Should().Be(1, 19, 1, 20); + refs[3].uri.Should().Be(uri2); + refs[4].range.Should().Be(2, 4, 2, 5); + refs[4].uri.Should().Be(uri2); + + refs[5].range.Should().Be(1, 19, 1, 20); + refs[5].uri.Should().Be(uri3); + refs[6].range.Should().Be(2, 4, 2, 5); + refs[6].uri.Should().Be(uri3); + } + + [TestMethod, Priority(0)] + public async Task NestedClosedFiles() { + const string code = @" +x = 1 + +def func(x): + return x + +y = func(x) +x = 2 +"; + const string mod2Code = @" +from module import x +y = x +"; + const string mod3Code = @" +from module2 import x +y = x +"; + var uri2 = await TestData.CreateTestSpecificFileAsync("module2.py", mod2Code); + var uri3 = await TestData.CreateTestSpecificFileAsync("module3.py", mod3Code); + + var analysis = await GetAnalysisAsync(code); + + var rs = new ReferenceSource(Services); + var refs = await rs.FindAllReferencesAsync(analysis.Document.Uri, new SourceLocation(7, 10), ReferenceSearchOptions.All); + + refs.Should().HaveCount(7); + + refs[0].range.Should().Be(1, 0, 1, 1); + refs[0].uri.Should().Be(analysis.Document.Uri); + refs[1].range.Should().Be(6, 9, 6, 10); + refs[1].uri.Should().Be(analysis.Document.Uri); + refs[2].range.Should().Be(7, 0, 7, 1); + refs[2].uri.Should().Be(analysis.Document.Uri); + + refs[3].range.Should().Be(1, 19, 1, 20); + refs[3].uri.Should().Be(uri2); + refs[4].range.Should().Be(2, 4, 2, 5); + refs[4].uri.Should().Be(uri2); + + refs[5].range.Should().Be(1, 20, 1, 21); + refs[5].uri.Should().Be(uri3); + refs[6].range.Should().Be(2, 4, 2, 5); + refs[6].uri.Should().Be(uri3); + } + + [TestMethod, Priority(0)] + public async Task UnrelatedFiles() { + const string code = @" +from bar import baz + +class spam: + __bug__ = 0 + +def eggs(ham: spam): + return baz(ham.__bug__) +"; + const string barCode = @" +def baz(quux): + pass +"; + await TestData.CreateTestSpecificFileAsync("bar.py", barCode); + var analysis = await GetAnalysisAsync(code); + + var rs = new ReferenceSource(Services); + var refs = await rs.FindAllReferencesAsync(analysis.Document.Uri, new SourceLocation(5, 8), ReferenceSearchOptions.All); + + refs.Should().HaveCount(2); + + refs[0].range.Should().Be(4, 4, 4, 11); + refs[0].uri.Should().Be(analysis.Document.Uri); + refs[1].range.Should().Be(7, 19, 7, 26); + refs[1].uri.Should().Be(analysis.Document.Uri); + } + + [TestMethod, Priority(0)] + public async Task RemoveReference() { + const string code1 = @" +x = 1 + +def func(x): + return x + +y = func(x) +x = 2 +"; + var uri1 = TestData.GetDefaultModuleUri(); + var uri2 = TestData.GetNextModuleUri(); + + var code2 = $@" +from module import x, y +a = x +b = y +"; + await CreateServicesAsync(PythonVersions.LatestAvailable3X, uri1.AbsolutePath); + + var rdt = Services.GetService(); + var doc1 = rdt.OpenDocument(uri1, code1); + var doc2 = rdt.OpenDocument(uri2, code2); + + var analysis = await GetDocumentAnalysisAsync(doc1); + + var rs = new ReferenceSource(Services); + var refs = await rs.FindAllReferencesAsync(analysis.Document.Uri, new SourceLocation(7, 1), ReferenceSearchOptions.All); + + refs.Should().HaveCount(3); + refs[0].range.Should().Be(6, 0, 6, 1); + refs[0].uri.Should().Be(uri1); + refs[1].range.Should().Be(1, 22, 1, 23); + refs[1].uri.Should().Be(uri2); + refs[2].range.Should().Be(3, 4, 3, 5); + refs[2].uri.Should().Be(uri2); + + doc2.Update(new[] { + new DocumentChange { + InsertedText = string.Empty, + ReplacedSpan = new SourceSpan(4, 1, 4, 6) + }, + new DocumentChange { + InsertedText = string.Empty, + ReplacedSpan = new SourceSpan(2, 21, 2, 24) + } + }); + await GetDocumentAnalysisAsync(doc2); + + refs = await rs.FindAllReferencesAsync(analysis.Document.Uri, new SourceLocation(7, 1), ReferenceSearchOptions.All); + + refs.Should().HaveCount(1); + refs[0].range.Should().Be(6, 0, 6, 1); + refs[0].uri.Should().Be(uri1); + } + + [TestMethod, Priority(0)] + public async Task UpdateReferencesOnEdit() { + const string code = @" +import logging + +logging.getLogger() +"; + var uri = TestData.GetDefaultModuleUri(); + await CreateServicesAsync(PythonVersions.LatestAvailable3X, uri.AbsolutePath); + + var rdt = Services.GetService(); + var doc = rdt.OpenDocument(uri, code); + var analysis = await GetDocumentAnalysisAsync(doc); + + var rs = new ReferenceSource(Services); + var refs = await rs.FindAllReferencesAsync(analysis.Document.Uri, new SourceLocation(4, 12), ReferenceSearchOptions.All); + + refs.Should().HaveCount(2); + refs[0].range.start.line.Should().BeGreaterThan(1); + refs[0].uri.AbsolutePath.Should().Contain("logging"); + refs[1].range.Should().Be(3, 8, 3, 17); + refs[1].uri.Should().Be(uri); + + doc.Update(new[] { + new DocumentChange { + InsertedText = Environment.NewLine, + ReplacedSpan = new SourceSpan(3, 1, 3, 1) + }, + }); + await GetDocumentAnalysisAsync(doc); + + refs = await rs.FindAllReferencesAsync(analysis.Document.Uri, new SourceLocation(5, 12), ReferenceSearchOptions.All); + + refs.Should().HaveCount(2); + refs[0].range.start.line.Should().BeGreaterThan(1); + refs[0].uri.AbsolutePath.Should().Contain("logging"); + refs[1].range.Should().Be(4, 8, 4, 17); + refs[1].uri.Should().Be(uri); + } + + [TestMethod, Priority(0)] + public async Task EmptyAnalysis() { + await GetAnalysisAsync(string.Empty); + var rs = new ReferenceSource(Services); + var references = await rs.FindAllReferencesAsync(null, new SourceLocation(1, 1), ReferenceSearchOptions.All); + references.Should().BeEmpty(); + } + } +} diff --git a/src/LanguageServer/Test/RenameTests.cs b/src/LanguageServer/Test/RenameTests.cs new file mode 100644 index 000000000..a69d5755c --- /dev/null +++ b/src/LanguageServer/Test/RenameTests.cs @@ -0,0 +1,179 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Sources; +using Microsoft.Python.LanguageServer.Tests.FluentAssertions; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.LanguageServer.Tests { + [TestClass] + public class RenameTests : LanguageServerTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + + [TestMethod, Priority(0)] + public async Task SingleFile() { + const string code = @" +x = 1 + +def func(x): + return x + +y = func(x) +x = 2 +"; + var analysis = await GetAnalysisAsync(code); + var rs = new RenameSource(Services); + var wse = await rs.RenameAsync(analysis.Document.Uri, new SourceLocation(8, 1), "z"); + + var uri = analysis.Document.Uri; + wse.changes.Should().HaveCount(1); + wse.changes[uri].Should().HaveCount(3); + wse.changes[uri][0].range.Should().Be(1, 0, 1, 1); + wse.changes[uri][1].range.Should().Be(6, 9, 6, 10); + wse.changes[uri][2].range.Should().Be(7, 0, 7, 1); + wse.changes[uri].All(x => x.newText.EqualsOrdinal("z")).Should().BeTrue(); + } + + [TestMethod, Priority(0)] + public async Task TwoOpenFiles() { + const string code1 = @" +x = 1 + +def func(x): + return x + +y = func(x) +x = 2 +"; + var uri1 = TestData.GetDefaultModuleUri(); + var uri2 = TestData.GetNextModuleUri(); + + var code2 = $@" +from {Path.GetFileNameWithoutExtension(uri1.AbsolutePath)} import x +y = x +"; + await CreateServicesAsync(PythonVersions.LatestAvailable3X, uri1.AbsolutePath); + + var rdt = Services.GetService(); + var doc1 = rdt.OpenDocument(uri1, code1); + var doc2 = rdt.OpenDocument(uri2, code2); + + var analysis = await GetDocumentAnalysisAsync(doc1); + await GetDocumentAnalysisAsync(doc2); + + var rs = new RenameSource(Services); + var wse = await rs.RenameAsync(analysis.Document.Uri, new SourceLocation(7, 10), "z"); + + wse.changes.Should().HaveCount(2); + + wse.changes[uri1].Should().HaveCount(3); + wse.changes[uri1][0].range.Should().Be(1, 0, 1, 1); + wse.changes[uri1][1].range.Should().Be(6, 9, 6, 10); + wse.changes[uri1][2].range.Should().Be(7, 0, 7, 1); + + wse.changes[uri2].Should().HaveCount(2); + wse.changes[uri2][0].range.Should().Be(1, 19, 1, 20); + wse.changes[uri2][1].range.Should().Be(2, 4, 2, 5); + } + + [TestMethod, Priority(0)] + public async Task ClosedFiles() { + const string code = @" +x = 1 + +def func(x): + return x + +y = func(x) +x = 2 +"; + const string mod2Code = @" +from module import x +y = x +"; + const string mod3Code = @" +from module import x +y = x +"; + var uri2 = await TestData.CreateTestSpecificFileAsync("module2.py", mod2Code); + var uri3 = await TestData.CreateTestSpecificFileAsync("module3.py", mod3Code); + + var analysis = await GetAnalysisAsync(code); + var uri1 = analysis.Document.Uri; + + var rs = new RenameSource(Services); + var wse = await rs.RenameAsync(analysis.Document.Uri, new SourceLocation(7, 10), "z"); + + wse.changes.Should().HaveCount(3); + + wse.changes[uri1].Should().HaveCount(3); + wse.changes[uri1][0].range.Should().Be(1, 0, 1, 1); + wse.changes[uri1][1].range.Should().Be(6, 9, 6, 10); + wse.changes[uri1][2].range.Should().Be(7, 0, 7, 1); + + wse.changes[uri2].Should().HaveCount(2); + wse.changes[uri2][0].range.Should().Be(1, 19, 1, 20); + wse.changes[uri2][1].range.Should().Be(2, 4, 2, 5); + + wse.changes[uri3].Should().HaveCount(2); + wse.changes[uri3][0].range.Should().Be(1, 19, 1, 20); + wse.changes[uri3][1].range.Should().Be(2, 4, 2, 5); + } + + [TestMethod, Priority(0)] + public async Task NoRenameInCompiled() { + const string code = "from sys import path"; + + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var rs = new RenameSource(Services); + + var wse = await rs.RenameAsync(analysis.Document.Uri, new SourceLocation(1, 7), "z"); + wse.Should().BeNull(); + + wse = await rs.RenameAsync(analysis.Document.Uri, new SourceLocation(1, 18), "z"); + wse.Should().BeNull(); + } + + [TestMethod, Priority(0)] + public async Task NoRenameInLibrary() { + const string code = @"from logging import BASIC_FORMAT"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var rs = new RenameSource(Services); + + var wse = await rs.RenameAsync(analysis.Document.Uri, new SourceLocation(1, 10), "z"); + wse.Should().BeNull(); + + wse = await rs.RenameAsync(analysis.Document.Uri, new SourceLocation(1, 23), "z"); + wse.Should().BeNull(); + } + } +} diff --git a/src/LanguageServer/Test/SignatureTests.cs b/src/LanguageServer/Test/SignatureTests.cs index 1511105a9..adadc8d43 100644 --- a/src/LanguageServer/Test/SignatureTests.cs +++ b/src/LanguageServer/Test/SignatureTests.cs @@ -15,7 +15,6 @@ using System.Threading.Tasks; using FluentAssertions; -using Microsoft.Python.Analysis; using Microsoft.Python.Core.Text; using Microsoft.Python.LanguageServer.Sources; using Microsoft.Python.Parsing.Tests; diff --git a/src/Parsing/Impl/Ast/DottedName.cs b/src/Parsing/Impl/Ast/DottedName.cs index 6ddc5e6ef..f0e6e4b96 100644 --- a/src/Parsing/Impl/Ast/DottedName.cs +++ b/src/Parsing/Impl/Ast/DottedName.cs @@ -44,6 +44,16 @@ public virtual string MakeString() { public override IEnumerable GetChildNodes() => Names; + public override PythonAst Ast { + get => base.Ast; + internal set { + base.Ast = value; + foreach (var n in Names) { + n.Ast = value; + } + } + } + public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { } @@ -58,7 +68,7 @@ public override async Task WalkAsync(PythonWalkerAsync walker, CancellationToken internal override void AppendCodeString(StringBuilder res, PythonAst ast, CodeFormattingOptions format) { var whitespace = this.GetNamesWhiteSpace(ast); - + for (int i = 0, whitespaceIndex = 0; i < Names.Count; i++) { if (whitespace != null) { res.Append(whitespace[whitespaceIndex++]); diff --git a/src/Parsing/Impl/Ast/FromImportStatement.cs b/src/Parsing/Impl/Ast/FromImportStatement.cs index ef89925bf..d4bbdb97d 100644 --- a/src/Parsing/Impl/Ast/FromImportStatement.cs +++ b/src/Parsing/Impl/Ast/FromImportStatement.cs @@ -13,13 +13,13 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Core; using Microsoft.Python.Core.Collections; namespace Microsoft.Python.Parsing.Ast { @@ -54,6 +54,20 @@ public FromImportStatement(ModuleName/*!*/ root, ImmutableArray // TODO: return names and aliases when they are united into one node public override IEnumerable GetChildNodes() => Enumerable.Empty(); + public override PythonAst Ast { + get => base.Ast; + internal set { + foreach (var n in Names.ExcludeDefault()) { + n.Ast = value; + } + foreach (var n in AsNames.ExcludeDefault()) { + n.Ast = value; + } + + Root.Ast = value; + base.Ast = value; + } + } public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { } diff --git a/src/Parsing/Impl/Ast/IfStatement.cs b/src/Parsing/Impl/Ast/IfStatement.cs index ef3cb1fa2..c57749fb9 100644 --- a/src/Parsing/Impl/Ast/IfStatement.cs +++ b/src/Parsing/Impl/Ast/IfStatement.cs @@ -17,7 +17,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Core; using Microsoft.Python.Core.Collections; namespace Microsoft.Python.Parsing.Ast { diff --git a/src/Parsing/Impl/Ast/ImportStatement.cs b/src/Parsing/Impl/Ast/ImportStatement.cs index 1160e6182..a7126ba49 100644 --- a/src/Parsing/Impl/Ast/ImportStatement.cs +++ b/src/Parsing/Impl/Ast/ImportStatement.cs @@ -20,6 +20,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Core; using Microsoft.Python.Core.Collections; namespace Microsoft.Python.Parsing.Ast { @@ -37,14 +38,25 @@ public ImportStatement(ImmutableArray names, ImmutableArray GetVariableReferences(this, ast); - public ImmutableArray Names { get; } public ImmutableArray AsNames { get; } // TODO: return names and aliases when they are united into one node public override IEnumerable GetChildNodes() => Enumerable.Empty(); + public override PythonAst Ast { + get => base.Ast; + internal set { + foreach (var n in Names.ExcludeDefault()) { + n.Ast = value; + } + foreach (var n in AsNames.ExcludeDefault()) { + n.Ast = value; + } + base.Ast = value; + } + } + public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { } @@ -63,10 +75,10 @@ internal override void AppendCodeStringStmt(StringBuilder res, PythonAst ast, Co if (format.ReplaceMultipleImportsWithMultipleStatements) { var proceeding = this.GetPreceedingWhiteSpace(ast); var additionalProceeding = format.GetNextLineProceedingText(proceeding); - + for (int i = 0, asIndex = 0; i < Names.Count; i++) { if (i == 0) { - format.ReflowComment(res, proceeding) ; + format.ReflowComment(res, proceeding); } else { res.Append(additionalProceeding); } diff --git a/src/Parsing/Impl/Ast/MemberExpression.cs b/src/Parsing/Impl/Ast/MemberExpression.cs index 4c3f8cbd3..b11db83c2 100644 --- a/src/Parsing/Impl/Ast/MemberExpression.cs +++ b/src/Parsing/Impl/Ast/MemberExpression.cs @@ -95,6 +95,6 @@ public override void SetLeadingWhiteSpace(PythonAst ast, string whiteSpace) /// /// Returns the span of the name component of the expression /// - public SourceSpan GetNameSpan(PythonAst parent) => new SourceSpan(parent.IndexToLocation(NameHeader), GetEnd(parent)); + public SourceSpan GetNameSpan() => new SourceSpan(Ast.IndexToLocation(NameHeader), GetEnd()); } } diff --git a/src/Parsing/Impl/Ast/Node.cs b/src/Parsing/Impl/Ast/Node.cs index 7edf6da23..02dadffa4 100644 --- a/src/Parsing/Impl/Ast/Node.cs +++ b/src/Parsing/Impl/Ast/Node.cs @@ -21,10 +21,9 @@ namespace Microsoft.Python.Parsing.Ast { public abstract class Node { - internal Node() { - } #region Public API + public virtual PythonAst Ast { get; internal set; } public int EndIndex { get => IndexSpan.End; @@ -50,14 +49,11 @@ public string ToCodeString(PythonAst ast, CodeFormattingOptions format) { return res.ToString(); } - public SourceLocation GetStart(PythonAst parent) => parent.IndexToLocation(StartIndex); - - public SourceLocation GetEnd(PythonAst parent) => parent.IndexToLocation(EndIndex); + public SourceLocation GetStart() => Ast.IndexToLocation(StartIndex); - public SourceSpan GetSpan(PythonAst parent) => new SourceSpan(GetStart(parent), GetEnd(parent)); + public SourceLocation GetEnd() => Ast.IndexToLocation(EndIndex); - public static void CopyLeadingWhiteSpace(PythonAst parentNode, Node fromNode, Node toNode) - => parentNode.SetAttribute(toNode, NodeAttributes.PreceedingWhiteSpace, fromNode.GetLeadingWhiteSpace(parentNode)); + public SourceSpan GetSpan() => new SourceSpan(GetStart(), GetEnd()); /// /// Returns the proceeding whitespace (newlines and comments) that @@ -74,27 +70,9 @@ public static void CopyLeadingWhiteSpace(PythonAst parentNode, Node fromNode, No /// /// public virtual void SetLeadingWhiteSpace(PythonAst ast, string whiteSpace) => ast.SetAttribute(this, NodeAttributes.PreceedingWhiteSpace, whiteSpace); - - /// - /// Gets the indentation level for the current statement. Requires verbose - /// mode when parsing the trees. - /// - public string GetIndentationLevel(PythonAst parentNode) { - var leading = GetLeadingWhiteSpace(parentNode); - // we only want the trailing leading space for the current line... - for (var i = leading.Length - 1; i >= 0; i--) { - if (leading[i] == '\r' || leading[i] == '\n') { - leading = leading.Substring(i + 1); - break; - } - } - return leading; - } - #endregion #region Internal APIs - /// /// Appends the code representation of the node to the string builder. /// diff --git a/src/Parsing/Impl/Ast/Parameter.cs b/src/Parsing/Impl/Ast/Parameter.cs index 624d3dc4d..080c5aa2c 100644 --- a/src/Parsing/Impl/Ast/Parameter.cs +++ b/src/Parsing/Impl/Ast/Parameter.cs @@ -55,6 +55,7 @@ public Parameter(NameExpression name, ParameterKind kind) { public ParameterKind Kind { get; } public override IEnumerable GetChildNodes() { + if (NameExpression != null) yield return NameExpression; if (Annotation != null) yield return Annotation; if (DefaultValue != null) yield return DefaultValue; } diff --git a/src/Parsing/Impl/Ast/PythonAst.cs b/src/Parsing/Impl/Ast/PythonAst.cs index 142c352ab..8ce4a97de 100644 --- a/src/Parsing/Impl/Ast/PythonAst.cs +++ b/src/Parsing/Impl/Ast/PythonAst.cs @@ -30,11 +30,13 @@ public sealed class PythonAst : ScopeStatement { private readonly Statement _body; private readonly Dictionary> _attributes = new Dictionary>(); - public PythonAst(Statement body, NewLineLocation[] lineLocations, PythonLanguageVersion langVersion, SourceLocation[] commentLocations) { + public PythonAst(Uri module, Statement body, NewLineLocation[] lineLocations, PythonLanguageVersion langVersion, SourceLocation[] commentLocations) { _body = body ?? throw new ArgumentNullException(nameof(body)); + Module = module; LanguageVersion = langVersion; NewLineLocations = lineLocations; CommentLocations = commentLocations; + } public PythonAst(IEnumerable existingAst) { @@ -57,6 +59,7 @@ public PythonAst(IEnumerable existingAst) { CommentLocations = comments.ToArray(); } + public Uri Module { get; } public NewLineLocation[] NewLineLocations { get; } public SourceLocation[] CommentLocations { get; } diff --git a/src/Parsing/Impl/Ast/SourceLocationExtensions.cs b/src/Parsing/Impl/Ast/SourceLocationExtensions.cs index 89cebe6fc..4c8df3e07 100644 --- a/src/Parsing/Impl/Ast/SourceLocationExtensions.cs +++ b/src/Parsing/Impl/Ast/SourceLocationExtensions.cs @@ -27,4 +27,9 @@ public static IndexSpan ToIndexSpan(this SourceSpan span, PythonAst ast) public static IndexSpan ToIndexSpan(this Range range, PythonAst ast) => IndexSpan.FromBounds(ast.LocationToIndex(range.start), ast.LocationToIndex(range.end)); } + + public static class IndexSpanExtensions { + public static SourceSpan ToSourceSpan(this IndexSpan span, PythonAst ast) + => ast != null ? new SourceSpan(ast.IndexToLocation(span.Start), ast.IndexToLocation(span.End)) : default; + } } diff --git a/src/Parsing/Impl/Ast/TypeAnnotation.cs b/src/Parsing/Impl/Ast/TypeAnnotation.cs index 9ec88420a..b7d387cde 100644 --- a/src/Parsing/Impl/Ast/TypeAnnotation.cs +++ b/src/Parsing/Impl/Ast/TypeAnnotation.cs @@ -35,7 +35,7 @@ public static TypeAnnotation FromType(TypeAnnotationConverter converter, T private Expression ParseSubExpression(string expr) { var parser = Parser.CreateParser(new StringReader(expr), LanguageVersion); - return Statement.GetExpression(parser.ParseTopExpression()?.Body); + return Statement.GetExpression(parser.ParseTopExpression(null)?.Body); } /// diff --git a/src/Parsing/Impl/Parser.cs b/src/Parsing/Impl/Parser.cs index f7bd0aad9..e20d6efe4 100644 --- a/src/Parsing/Impl/Parser.cs +++ b/src/Parsing/Impl/Parser.cs @@ -52,7 +52,6 @@ public class Parser { private bool _parsingStarted, _allowIncomplete; private bool _inLoop, _inFinally, _isGenerator, _inGeneratorExpression; private List _returnsWithValue; - private int _errorCode; private readonly bool _verbatim; // true if we're in verbatim mode and the ASTs can be turned back into source code, preserving white space / comments private readonly bool _bindReferences; // true if we should bind the references in the ASTs private string _tokenWhiteSpace, _lookaheadWhiteSpace; // the whitespace for the current and lookahead tokens as provided from the parser @@ -146,56 +145,7 @@ public static Parser CreateParser(Stream stream, PythonLanguageVersion version, //single_input: Newline | simple_stmt | compound_stmt Newline //eval_input: testlist Newline* ENDMARKER //file_input: (Newline | stmt)* ENDMARKER - public PythonAst ParseFile() => ParseFileWorker(); - - //[stmt_list] Newline | compound_stmt Newline - //stmt_list ::= simple_stmt (";" simple_stmt)* [";"] - //compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef - //Returns a simple or coumpound_stmt or null if input is incomplete - /// - /// Parse one or more lines of interactive input - /// - /// null if input is not yet valid but could be with more lines - public PythonAst ParseInteractiveCode(out ParseResult properties) { - - properties = ParseResult.Complete; - - StartParsing(); - var ret = InternalParseInteractiveInput(out var parsingMultiLineCmpdStmt, out var isEmptyStmt); - - if (_errorCode == 0) { - if (isEmptyStmt) { - properties = ParseResult.Empty; - } else if (parsingMultiLineCmpdStmt) { - properties = ParseResult.IncompleteStatement; - } - - if (isEmptyStmt) { - return null; - } - - return CreateAst(ret); - } else { - if ((_errorCode & ErrorCodes.IncompleteMask) != 0) { - if ((_errorCode & ErrorCodes.IncompleteToken) != 0) { - properties = ParseResult.IncompleteToken; - return null; - } - - if ((_errorCode & ErrorCodes.IncompleteStatement) != 0) { - if (parsingMultiLineCmpdStmt) { - properties = ParseResult.IncompleteStatement; - } else { - properties = ParseResult.IncompleteToken; - } - return null; - } - } - - properties = ParseResult.Invalid; - return null; - } - } + public PythonAst ParseFile(Uri module = null) => ParseFileWorker(module); public Expression ParseFStrSubExpr() { _alwaysAllowContextDependentSyntax = true; @@ -227,7 +177,7 @@ public Expression ParseFStrSubExpr() { ); } - if (_errorCode == 0) { + if (ErrorCode == 0) { // Detect if there are unexpected tokens EatEndOfInput(); } @@ -236,10 +186,8 @@ public Expression ParseFStrSubExpr() { return node; } - private PythonAst CreateAst(Statement ret) { - var ast = new PythonAst(ret, _tokenizer.GetLineLocations(), _tokenizer.LanguageVersion, _tokenizer.GetCommentLocations()); - ast.HasVerbatim = _verbatim; - ast.PrivatePrefix = _privatePrefix; + private PythonAst CreateAst(Uri module, Statement ret) { + var ast = new PythonAst(module, ret, _tokenizer.GetLineLocations(), _tokenizer.LanguageVersion, _tokenizer.GetCommentLocations()) { HasVerbatim = _verbatim, PrivatePrefix = _privatePrefix }; if (_token.Token != null) { ast.SetLoc(0, GetEndForStatement()); } @@ -250,25 +198,20 @@ private PythonAst CreateAst(Statement ret) { ast.SetAttributes(_attributes); PythonNameBinder.BindAst(_langVersion, ast, _errors, _bindReferences); - return ast; - } - - public PythonAst ParseSingleStatement() { - StartParsing(); + foreach (var n in ((Node)ast).TraverseDepthFirst(c => c.GetChildNodes())) { + n.Ast = ast; + } - MaybeEatNewLine(); - var statement = ParseStmt(); - EatEndOfInput(); - return CreateAst(statement); + return ast; } - public PythonAst ParseTopExpression() { + public PythonAst ParseTopExpression(Uri module) { // TODO: move from source unit .TrimStart(' ', '\t') _alwaysAllowContextDependentSyntax = true; var ret = new ReturnStatement(ParseTestListAsExpression()); _alwaysAllowContextDependentSyntax = false; ret.SetLoc(0, 0); - return CreateAst(ret); + return CreateAst(module, ret); } internal ErrorSink ErrorSink { @@ -279,7 +222,7 @@ internal ErrorSink ErrorSink { } } - public int ErrorCode => _errorCode; + public int ErrorCode { get; private set; } public void Reset(FutureOptions languageFeatures) { _languageFeatures = languageFeatures; @@ -291,7 +234,7 @@ public void Reset(FutureOptions languageFeatures) { _privatePrefix = null; _parsingStarted = false; - _errorCode = 0; + ErrorCode = 0; } public void Reset() => Reset(_languageFeatures); @@ -336,8 +279,8 @@ private static string GetErrorMessage(Token t, int errorCode) { internal void ReportSyntaxError(int start, int end, string message, int errorCode) { // save the first one, the next error codes may be induced errors: - if (_errorCode == 0) { - _errorCode = errorCode; + if (ErrorCode == 0) { + ErrorCode = errorCode; } _errors.Add( message, @@ -4621,7 +4564,7 @@ private CallExpression FinishCallExpr(Expression target, ImmutableArray arg #region Implementation Details - private PythonAst ParseFileWorker() { + private PythonAst ParseFileWorker(Uri module) { StartParsing(); var l = new List(); @@ -4686,7 +4629,7 @@ private PythonAst ParseFileWorker() { if (_token.Token != null) { ret.SetLoc(0, GetEndForStatement()); } - return CreateAst(ret); + return CreateAst(module, ret); } private bool IsString(ConstantExpression ce) { @@ -4707,7 +4650,7 @@ private Statement InternalParseInteractiveInput(out bool parsingMultiLineCmpdStm Eat(TokenKind.EndOfFile); if (_tokenizer.EndContinues) { parsingMultiLineCmpdStmt = true; - _errorCode = ErrorCodes.IncompleteStatement; + ErrorCode = ErrorCodes.IncompleteStatement; } else { isEmptyStmt = true; } @@ -4952,8 +4895,8 @@ public TokenizerErrorSink(Parser parser) { } public override void Add(string message, SourceSpan span, int errorCode, Severity severity) { - if (_parser._errorCode == 0 && severity == Severity.Error) { - _parser._errorCode = errorCode; + if (_parser.ErrorCode == 0 && severity == Severity.Error) { + _parser.ErrorCode = errorCode; } _parser.ErrorSink.Add(message, span, errorCode, severity); diff --git a/src/Parsing/Test/ParserTests.cs b/src/Parsing/Test/ParserTests.cs index a4a19f61a..96aade77d 100644 --- a/src/Parsing/Test/ParserTests.cs +++ b/src/Parsing/Test/ParserTests.cs @@ -3591,7 +3591,7 @@ public void FindArgument() { private static Action ParseCall(string code) { var parser = Parser.CreateParser(new StringReader(code), PythonLanguageVersion.V36, new ParserOptions { Verbatim = true }); - var tree = parser.ParseTopExpression(); + var tree = parser.ParseTopExpression(null); if (Statement.GetExpression(tree.Body) is CallExpression ce) { return (index, expected) => { var actual = ce.GetArgumentAtIndex(tree, index, out var i) ? i : (int?)null; diff --git a/src/UnitTests/TestData/AstAnalysis/DefaultArgument.py b/src/UnitTests/TestData/AstAnalysis/DefaultArgument.py new file mode 100644 index 000000000..3f3cd4f6a --- /dev/null +++ b/src/UnitTests/TestData/AstAnalysis/DefaultArgument.py @@ -0,0 +1,3 @@ +class A: ... + +def func(a = A()): ...