diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs index 01724bbd2..9eecd8018 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs @@ -9,20 +9,21 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using Microsoft.PythonTools.Analysis.Infrastructure; using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Interpreter.Ast { + [DebuggerDisplay("{Target.Name}")] class AstAnalysisFunctionWalker : PythonWalker { - private readonly FunctionDefinition _target; private readonly NameLookupContext _scope; private readonly AstPythonFunctionOverload _overload; private AstPythonType _selfType; @@ -33,12 +34,11 @@ public AstAnalysisFunctionWalker( AstPythonFunctionOverload overload ) { _scope = scope ?? throw new ArgumentNullException(nameof(scope)); - _target = targetFunction ?? throw new ArgumentNullException(nameof(targetFunction)); + Target = targetFunction ?? throw new ArgumentNullException(nameof(targetFunction)); _overload = overload ?? throw new ArgumentNullException(nameof(overload)); } - public IList ReturnTypes => _overload.ReturnTypes; - public IPythonFunctionOverload Overload => _overload; + public FunctionDefinition Target { get; } private void GetMethodType(FunctionDefinition node, out bool classmethod, out bool staticmethod) { classmethod = false; @@ -51,7 +51,7 @@ private void GetMethodType(FunctionDefinition node, out bool classmethod, out bo var classmethodObj = _scope.Interpreter.GetBuiltinType(BuiltinTypeId.ClassMethod); var staticmethodObj = _scope.Interpreter.GetBuiltinType(BuiltinTypeId.StaticMethod); - foreach (var d in (_target.Decorators?.Decorators).MaybeEnumerate().ExcludeDefault()) { + foreach (var d in (Target.Decorators?.Decorators).MaybeEnumerate().ExcludeDefault()) { var m = _scope.GetValueFromExpression(d); if (m == classmethodObj) { classmethod = true; @@ -65,21 +65,31 @@ public void Walk() { var self = GetSelf(); _selfType = (self as AstPythonConstant)?.Type as AstPythonType; - _overload.ReturnTypes.AddRange(_scope.GetTypesFromAnnotation(_target.ReturnAnnotation).ExcludeDefault()); - + _overload.ReturnTypes.AddRange(_scope.GetTypesFromAnnotation(Target.ReturnAnnotation).ExcludeDefault()); _scope.PushScope(); + + // Declare self, if any + var skip = 0; if (self != null) { - var p0 = _target.Parameters.FirstOrDefault(); + var p0 = Target.Parameters.FirstOrDefault(); if (p0 != null && !string.IsNullOrEmpty(p0.Name)) { _scope.SetInScope(p0.Name, self); + skip++; } } - _target.Walk(this); + + // Declare parameters in scope + foreach(var p in Target.Parameters.Skip(skip).Where(p => !string.IsNullOrEmpty(p.Name))) { + var value = _scope.GetValueFromExpression(p.DefaultValue); + _scope.SetInScope(p.Name, value ?? _scope.UnknownType); + } + + Target.Walk(this); _scope.PopScope(); } public override bool Walk(FunctionDefinition node) { - if (node != _target) { + if (node != Target) { // Do not walk nested functions (yet) return false; } @@ -163,19 +173,28 @@ public override bool Walk(IfStatement node) { } public override bool Walk(ReturnStatement node) { - foreach (var type in _scope.GetTypesFromValue(_scope.GetValueFromExpression(node.Expression)).ExcludeDefault()) { + var types = _scope.GetTypesFromValue(_scope.GetValueFromExpression(node.Expression)).ExcludeDefault(); + foreach (var type in types) { _overload.ReturnTypes.Add(type); } + + // Clean up: if there are None or Unknown types along with real ones, remove them. + var realTypes = _overload.ReturnTypes + .Where(t => t.TypeId != BuiltinTypeId.Unknown && t.TypeId != BuiltinTypeId.NoneType) + .ToList(); + + if (realTypes.Count > 0) { + _overload.ReturnTypes.Clear(); + _overload.ReturnTypes.AddRange(realTypes); + } return true; // We want to evaluate all code so all private variables in __new__ get defined } private IMember GetSelf() { - bool classmethod, staticmethod; - GetMethodType(_target, out classmethod, out staticmethod); + GetMethodType(Target, out var classmethod, out var staticmethod); var self = _scope.LookupNameInScopes("__class__", NameLookupContext.LookupOptions.Local); if (!staticmethod && !classmethod) { - var cls = self as IPythonType; - if (cls == null) { + if (!(self is IPythonType cls)) { self = null; } else { self = new AstPythonConstant(cls, ((cls as ILocatedMember)?.Locations).MaybeEnumerate().ToArray()); diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs new file mode 100644 index 000000000..9d026442b --- /dev/null +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs @@ -0,0 +1,68 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.PythonTools.Analysis.Infrastructure; +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Interpreter.Ast { + /// + /// Represents set of function body walkers. Functions are walked after + /// all classes are collected. If function or property return type is unknown, + /// it can be walked, and so on recursively, until return type is determined + /// or there is nothing left to walk. + /// + class AstAnalysisFunctionWalkerSet { + private readonly Dictionary _functionWalkers + = new Dictionary(); + + public void Add(AstAnalysisFunctionWalker walker) + => _functionWalkers[walker.Target] = walker; + + public void ProcessSet() { + // Do not use foreach since walker list is dynamically modified and walkers are removed + // after processing. Handle __init__ and __new__ first so class variables are initialized. + var constructors = _functionWalkers + .Where(kvp => kvp.Key.Name == "__init__" || kvp.Key.Name == "__new__") + .Select(c => c.Value) + .ExcludeDefault() + .ToArray(); + + foreach (var ctor in constructors) { + ProcessWalker(ctor); + } + + while (_functionWalkers.Count > 0) { + var walker = _functionWalkers.First().Value; + ProcessWalker(walker); + } + } + + public void ProcessFunction(FunctionDefinition fn) { + if (_functionWalkers.TryGetValue(fn, out var w)) { + ProcessWalker(w); + } + } + + private void ProcessWalker(AstAnalysisFunctionWalker walker) { + // Remove walker before processing as to prevent reentrancy. + _functionWalkers.Remove(walker.Target); + var z = walker.Target.Name == "day"; + walker.Walk(); + } + } +} diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs index 596a93512..6d9798b65 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs @@ -9,7 +9,7 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. @@ -27,13 +27,12 @@ namespace Microsoft.PythonTools.Interpreter.Ast { class AstAnalysisWalker : PythonWalker { private readonly IPythonModule _module; private readonly Dictionary _members; - private readonly List _postWalkers = new List(); private readonly AnalysisLogWriter _log; - private readonly Dictionary _typingScope; - - private IPythonInterpreter _interpreter => Scope.Interpreter; - private PythonAst _ast => Scope.Ast; - private IPythonType _unknownType => Scope._unknownType; + private readonly Dictionary _typingScope = new Dictionary(); + private readonly AstAnalysisFunctionWalkerSet _functionWalkers = new AstAnalysisFunctionWalkerSet(); + private readonly NameLookupContext _scope; + private readonly PythonAst _ast; + private readonly IPythonInterpreter _interpreter; public AstAnalysisWalker( IPythonInterpreter interpreter, @@ -44,12 +43,13 @@ public AstAnalysisWalker( Dictionary members, bool includeLocationInfo, bool warnAboutUndefinedValues, + bool suppressBuiltinLookup, AnalysisLogWriter log = null ) { _log = log ?? (interpreter as AstPythonInterpreter)?.Log; _module = module ?? throw new ArgumentNullException(nameof(module)); _members = members ?? throw new ArgumentNullException(nameof(members)); - Scope = new NameLookupContext( + _scope = new NameLookupContext( interpreter ?? throw new ArgumentNullException(nameof(interpreter)), interpreter.CreateModuleContext(), ast ?? throw new ArgumentNullException(nameof(ast)), @@ -57,46 +57,46 @@ public AstAnalysisWalker( filePath, documentUri, includeLocationInfo, + _functionWalkers, log: warnAboutUndefinedValues ? _log : null ); - _typingScope = new Dictionary(); - Scope.PushScope(_typingScope); + _ast = ast; + _interpreter = interpreter; + _scope.SuppressBuiltinLookup = suppressBuiltinLookup; + _scope.PushScope(_typingScope); WarnAboutUndefinedValues = warnAboutUndefinedValues; } public bool CreateBuiltinTypes { get; set; } public bool WarnAboutUndefinedValues { get; } - public NameLookupContext Scope { get; } public override bool Walk(PythonAst node) { if (_ast != node) { throw new InvalidOperationException("walking wrong AST"); } - FirstPassCollectClasses(); - Scope.PushScope(_members); + CollectTopLevelDefinitions(); + _scope.PushScope(_members); return base.Walk(node); } public override void PostWalk(PythonAst node) { - Scope.PopScope(); + _scope.PopScope(); base.PostWalk(node); } public void Complete() { - foreach (var walker in _postWalkers) { - walker.Walk(); - } + _functionWalkers.ProcessSet(); - if (_module.Name != "typing" && Scope.FilePath.EndsWithOrdinal(".pyi", ignoreCase: true)) { + if (_module.Name != "typing" && _scope.FilePath.EndsWithOrdinal(".pyi", ignoreCase: true)) { // Do not expose members directly imported from typing _typingScope.Clear(); } } internal LocationInfo GetLoc(ClassDefinition node) { - if (!Scope.IncludeLocationInfo) { + if (!_scope.IncludeLocationInfo) { return null; } if (node == null || node.StartIndex >= node.EndIndex) { @@ -105,34 +105,33 @@ internal LocationInfo GetLoc(ClassDefinition node) { var start = node.NameExpression?.GetStart(_ast) ?? node.GetStart(_ast); var end = node.GetEnd(_ast); - return new LocationInfo(Scope.FilePath, Scope.DocumentUri, start.Line, start.Column, end.Line, end.Column); + return new LocationInfo(_scope.FilePath, _scope.DocumentUri, start.Line, start.Column, end.Line, end.Column); } - internal LocationInfo GetLoc(Node node) => Scope.GetLoc(node); - private IPythonType CurrentClass => Scope.GetInScope("__class__") as IPythonType; + private LocationInfo GetLoc(Node node) => _scope.GetLoc(node); private IMember Clone(IMember member) => member is IPythonMultipleMembers mm ? AstPythonMultipleMembers.Create(mm.Members) : member; public override bool Walk(AssignmentStatement node) { - var value = Scope.GetValueFromExpression(node.Right); + var value = _scope.GetValueFromExpression(node.Right); if ((value == null || value.MemberType == PythonMemberType.Unknown) && WarnAboutUndefinedValues) { _log?.Log(TraceLevel.Warning, "UndefinedValue", node.Right.ToCodeString(_ast).Trim()); } if ((value as IPythonConstant)?.Type?.TypeId == BuiltinTypeId.Ellipsis) { - value = _unknownType; + value = _scope.UnknownType; } foreach (var expr in node.Left.OfType()) { AssignFromAnnotation(expr); - if (value != _unknownType && expr.Expression is NameExpression ne) { - Scope.SetInScope(ne.Name, Clone(value)); + if (value != _scope.UnknownType && expr.Expression is NameExpression ne) { + _scope.SetInScope(ne.Name, Clone(value)); } } foreach (var ne in node.Left.OfType()) { - Scope.SetInScope(ne.Name, Clone(value)); + _scope.SetInScope(ne.Name, Clone(value)); } return base.Walk(node); @@ -149,13 +148,13 @@ private void AssignFromAnnotation(ExpressionWithAnnotation expr) { } if (expr.Expression is NameExpression ne) { - bool any = false; - foreach (var annType in Scope.GetTypesFromAnnotation(expr.Annotation)) { - Scope.SetInScope(ne.Name, new AstPythonConstant(annType, GetLoc(expr.Expression))); + var any = false; + foreach (var annType in _scope.GetTypesFromAnnotation(expr.Annotation)) { + _scope.SetInScope(ne.Name, new AstPythonConstant(annType, GetLoc(expr.Expression))); any = true; } if (!any) { - Scope.SetInScope(ne.Name, _unknownType); + _scope.SetInScope(ne.Name, _scope.UnknownType); } } } @@ -166,18 +165,18 @@ public override bool Walk(ImportStatement node) { } try { - for (int i = 0; i < node.Names.Count; ++i) { + for (var i = 0; i < node.Names.Count; ++i) { var n = node.AsNames?[i] ?? node.Names[i].Names[0]; if (n != null) { if (n.Name == "typing") { - Scope.SetInScope(n.Name, new AstTypingModule(), scope: _typingScope); + _scope.SetInScope(n.Name, new AstTypingModule(), scope: _typingScope); } else if (n.Name == _module.Name) { - Scope.SetInScope(n.Name, _module); + _scope.SetInScope(n.Name, _module); } else { - Scope.SetInScope(n.Name, new AstNestedPythonModule( + _scope.SetInScope(n.Name, new AstNestedPythonModule( _interpreter, n.Name, - ModuleResolver.ResolvePotentialModuleNames(_module.Name, Scope.FilePath, node.Names[i].MakeString(), true).ToArray() + ModuleResolver.ResolvePotentialModuleNames(_module.Name, _scope.FilePath, node.Names[i].MakeString(), true).ToArray() )); } } @@ -213,7 +212,7 @@ public override bool Walk(FromImportStatement node) { IPythonModule mod = null; Dictionary scope = null; - bool isTyping = modName == "typing"; + var isTyping = modName == "typing"; var warnAboutUnknownValues = WarnAboutUndefinedValues; if (isTyping) { mod = new AstTypingModule(); @@ -226,44 +225,44 @@ public override bool Walk(FromImportStatement node) { mod = mod ?? new AstNestedPythonModule( _interpreter, modName, - ModuleResolver.ResolvePotentialModuleNames(_module.Name, Scope.FilePath, modName, true).ToArray() + ModuleResolver.ResolvePotentialModuleNames(_module.Name, _scope.FilePath, modName, true).ToArray() ); foreach (var name in GetImportNames(node.Names, node.AsNames)) { if (name.Key == "*") { - mod.Imported(Scope.Context); + mod.Imported(_scope.Context); // Ensure child modules have been loaded mod.GetChildrenModules(); - foreach (var member in mod.GetMemberNames(Scope.Context)) { - var mem = mod.GetMember(Scope.Context, member); + foreach (var member in mod.GetMemberNames(_scope.Context)) { + var mem = mod.GetMember(_scope.Context, member); if (mem == null) { if (WarnAboutUndefinedValues) { _log?.Log(TraceLevel.Warning, "UndefinedImport", modName, member); } - mem = new AstPythonConstant(_unknownType, ((mod as ILocatedMember)?.Locations).MaybeEnumerate().ToArray()); + mem = new AstPythonConstant(_scope.UnknownType, ((mod as ILocatedMember)?.Locations).MaybeEnumerate().ToArray()); } else if (mem.MemberType == PythonMemberType.Unknown && warnAboutUnknownValues) { _log?.Log(TraceLevel.Warning, "UnknownImport", modName, member); } - Scope.SetInScope(member, mem, scope: scope); - (mem as IPythonModule)?.Imported(Scope.Context); + _scope.SetInScope(member, mem, scope: scope); + (mem as IPythonModule)?.Imported(_scope.Context); } } else { IMember mem; if (mod is AstNestedPythonModule m && !m.IsLoaded) { - mem = new AstNestedPythonModuleMember(name.Key, m, Scope.Context, GetLoc(name.Value)); + mem = new AstNestedPythonModuleMember(name.Key, m, _scope.Context, GetLoc(name.Value)); } else { - mem = mod.GetMember(Scope.Context, name.Key); + mem = mod.GetMember(_scope.Context, name.Key); if (mem == null) { if (WarnAboutUndefinedValues) { _log?.Log(TraceLevel.Warning, "UndefinedImport", modName, name); } - mem = new AstPythonConstant(_unknownType, GetLoc(name.Value)); + mem = new AstPythonConstant(_scope.UnknownType, GetLoc(name.Value)); } else if (mem.MemberType == PythonMemberType.Unknown && warnAboutUnknownValues) { _log?.Log(TraceLevel.Warning, "UnknownImport", modName, name); } - (mem as IPythonModule)?.Imported(Scope.Context); + (mem as IPythonModule)?.Imported(_scope.Context); } - Scope.SetInScope(name.Value.Name, mem, scope: scope); + _scope.SetInScope(name.Value.Name, mem, scope: scope); } } @@ -271,7 +270,7 @@ public override bool Walk(FromImportStatement node) { } public override bool Walk(IfStatement node) { - bool allValidComparisons = true; + var allValidComparisons = true; foreach (var test in node.TestsInternal) { if (test.Test is BinaryExpression cmp && cmp.Left is MemberExpression me && (me.Target as NameExpression)?.Name == "sys" && me.Name == "version_info" && @@ -287,7 +286,7 @@ public override bool Walk(IfStatement node) { return true; } - bool shouldWalk = false; + var shouldWalk = false; switch (cmp.Operator) { case PythonOperator.LessThan: shouldWalk = _ast.LanguageVersion.ToVersion() < v; @@ -321,7 +320,7 @@ public override bool Walk(FunctionDefinition node) { var dec = (node.Decorators?.Decorators).MaybeEnumerate().ExcludeDefault(); foreach (var d in dec) { - var obj = Scope.GetValueFromExpression(d); + var obj = _scope.GetValueFromExpression(d); if (obj == _interpreter.GetBuiltinType(BuiltinTypeId.Property)) { AddProperty(node); return false; @@ -335,7 +334,7 @@ public override bool Walk(FunctionDefinition node) { } foreach (var setter in dec.OfType().Where(n => n.Name == "setter")) { if (setter.Target is NameExpression src) { - var existingProp = Scope.LookupNameInScopes(src.Name, NameLookupContext.LookupOptions.Local) as AstPythonProperty; + var existingProp = _scope.LookupNameInScopes(src.Name, NameLookupContext.LookupOptions.Local) as AstPythonProperty; if (existingProp != null) { // Setter for an existing property, so don't create a function existingProp.MakeSettable(); @@ -344,50 +343,33 @@ public override bool Walk(FunctionDefinition node) { } } - var existing = Scope.LookupNameInScopes(node.Name, NameLookupContext.LookupOptions.Local) as AstPythonFunction; - if (existing == null) { - existing = new AstPythonFunction(_ast, _module, CurrentClass, node, GetLoc(node)); - Scope.SetInScope(node.Name, existing); - } - - var funcScope = Scope.Clone(); - if (CreateBuiltinTypes) { - funcScope.SuppressBuiltinLookup = true; - } - existing.AddOverload(CreateFunctionOverload(funcScope, node)); - + ProcessFunctionDefinition(node); // Do not recurse into functions return false; } private void AddProperty(FunctionDefinition node) { - var existing = Scope.LookupNameInScopes(node.Name, NameLookupContext.LookupOptions.Local) as AstPythonProperty; - + var existing = _scope.LookupNameInScopes(node.Name, NameLookupContext.LookupOptions.Local) as AstPythonProperty; if (existing == null) { existing = new AstPythonProperty(_ast, node, GetLoc(node)); - Scope.SetInScope(node.Name, existing); + _scope.SetInScope(node.Name, existing); } // Treat the rest of the property as a function. "AddOverload" takes the return type // and sets it as the property type. - var funcScope = Scope.Clone(); - if (CreateBuiltinTypes) { - funcScope.SuppressBuiltinLookup = true; - } + var funcScope = _scope.Clone(); + funcScope.SuppressBuiltinLookup = CreateBuiltinTypes; + existing.AddOverload(CreateFunctionOverload(funcScope, node)); } private IPythonFunctionOverload CreateFunctionOverload(NameLookupContext funcScope, FunctionDefinition node) { - var parameters = new List(); - foreach (var p in node.Parameters) { - var annType = Scope.GetTypesFromAnnotation(p.Annotation); - parameters.Add(new AstPythonParameterInfo(_ast, p, annType)); - } + var parameters = node.Parameters + .Select(p => new AstPythonParameterInfo(_ast, p, _scope.GetTypesFromAnnotation(p.Annotation))) + .ToArray(); var overload = new AstPythonFunctionOverload(parameters, funcScope.GetLocOfName(node, node.NameExpression)); - - var funcWalk = new AstAnalysisFunctionWalker(funcScope, node, overload); - _postWalkers.Add(funcWalk); + _functionWalkers.Add(new AstAnalysisFunctionWalker(funcScope, node, overload)); return overload; } @@ -402,23 +384,23 @@ private AstPythonType CreateType(ClassDefinition node) { if (node == null) { throw new ArgumentNullException(nameof(node)); } - if (CreateBuiltinTypes) { - return new AstPythonBuiltinType(_ast, _module, node, GetDoc(node.Body as SuiteStatement), GetLoc(node)); - } - - return new AstPythonType(_ast, _module, node, GetDoc(node.Body as SuiteStatement), GetLoc(node)); + return CreateBuiltinTypes + ? new AstPythonBuiltinType(_ast, _module, node, GetDoc(node.Body as SuiteStatement), GetLoc(node)) + : new AstPythonType(_ast, _module, node, GetDoc(node.Body as SuiteStatement), GetLoc(node)); } + private void CollectTopLevelDefinitions() { + var s = (_ast.Body as SuiteStatement).Statements.ToArray(); - public void FirstPassCollectClasses() { - foreach (var node in (_ast.Body as SuiteStatement).Statements.OfType()) { + foreach (var node in s.OfType()) { + ProcessFunctionDefinition(node); + } + + foreach (var node in s.OfType()) { _members[node.Name] = CreateType(node); } - foreach (var node in (_ast.Body as SuiteStatement).Statements.OfType()) { - var rhs = node.Right as NameExpression; - if (rhs == null) { - continue; - } + foreach (var node in s.OfType().Where(n => n.Right is NameExpression)) { + var rhs = (NameExpression)node.Right; if (_members.TryGetValue(rhs.Name, out var member)) { foreach (var lhs in node.Left.OfType()) { _members[lhs.Name] = member; @@ -428,17 +410,17 @@ public void FirstPassCollectClasses() { } public override bool Walk(ClassDefinition node) { - var member = Scope.GetInScope(node.Name); - AstPythonType t = member as AstPythonType ?? + var member = _scope.GetInScope(node.Name); + var t = member as AstPythonType ?? (member as IPythonMultipleMembers)?.Members.OfType().FirstOrDefault(pt => pt.StartIndex == node.StartIndex); if (t == null) { t = CreateType(node); - Scope.SetInScope(node.Name, t); + _scope.SetInScope(node.Name, t); } var bases = node.Bases.Where(a => string.IsNullOrEmpty(a.Name)) // We cheat slightly and treat base classes as annotations. - .SelectMany(a => Scope.GetTypesFromAnnotation(a.Expression)) + .SelectMany(a => _scope.GetTypesFromAnnotation(a.Expression)) .ToArray(); try { @@ -447,19 +429,33 @@ public override bool Walk(ClassDefinition node) { // Bases were set while we were working } - Scope.PushScope(); - Scope.SetInScope("__class__", t); + _scope.PushScope(); + _scope.SetInScope("__class__", t); return true; } public override void PostWalk(ClassDefinition node) { - var cls = CurrentClass as AstPythonType; - var m = Scope.PopScope(); + var cls = _scope.GetInScope("__class__") as AstPythonType; // Store before popping the scope + var m = _scope.PopScope(); if (cls != null && m != null) { cls.AddMembers(m, true); } base.PostWalk(node); } + + public void ProcessFunctionDefinition(FunctionDefinition node) { + var existing = _scope.LookupNameInScopes(node.Name, NameLookupContext.LookupOptions.Local) as AstPythonFunction; + if (existing == null) { + var cls = _scope.GetInScope("__class__") as IPythonType; + existing = new AstPythonFunction(_ast, _module, cls, node, GetLoc(node)); + _scope.SetInScope(node.Name, existing); + } + + var funcScope = _scope.Clone(); + funcScope.SuppressBuiltinLookup = CreateBuiltinTypes; + + existing.AddOverload(CreateFunctionOverload(funcScope, node)); + } } } diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstBuiltinsPythonModule.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstBuiltinsPythonModule.cs index d1472f981..2e0f5d5d0 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstBuiltinsPythonModule.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstBuiltinsPythonModule.cs @@ -89,9 +89,13 @@ protected override PythonWalker PrepareWalker(IPythonInterpreter interpreter, Py const bool includeLocations = false; #endif - var walker = new AstAnalysisWalker(interpreter, ast, this, filePath, null, _members, includeLocations, true); + var walker = new AstAnalysisWalker( + interpreter, ast, this, filePath, null, _members, + includeLocations, + warnAboutUndefinedValues: true, + suppressBuiltinLookup: true + ); walker.CreateBuiltinTypes = true; - walker.Scope.SuppressBuiltinLookup = true; return walker; } diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs index 1d3285fd4..0976d4c08 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs @@ -9,12 +9,11 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using Microsoft.PythonTools.Analysis; using Microsoft.PythonTools.Parsing.Ast; diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonConstant.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonConstant.cs index 504b6abbb..42e8009ee 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonConstant.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonConstant.cs @@ -29,7 +29,6 @@ public AstPythonConstant(IPythonType type, params ILocationInfo[] locations) { } public IEnumerable Locations { get; } - public PythonMemberType MemberType => PythonMemberType.Constant; public IPythonType Type { get; } @@ -42,19 +41,16 @@ public IMember GetMember(IModuleContext context, string name) { } m = Type?.GetMember(context, name); - if (m is IPythonFunction f && !f.IsStatic) { m = new AstPythonBoundMethod(f, Type); lock (_cachedMembers) { _cachedMembers[name] = m; } } - return m; } - public IEnumerable GetMemberNames(IModuleContext moduleContext) { - return Type?.GetMemberNames(moduleContext); - } + public IEnumerable GetMemberNames(IModuleContext moduleContext) + => Type?.GetMemberNames(moduleContext); } } diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunction.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunction.cs index 9458d6a15..f9c210a39 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunction.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunction.cs @@ -22,7 +22,7 @@ using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Interpreter.Ast { - class AstPythonFunction : IPythonFunction, ILocatedMember, IHasQualifiedName { + class AstPythonFunction : IPythonFunction2, ILocatedMember, IHasQualifiedName { private readonly List _overloads; private readonly string _doc; @@ -35,13 +35,14 @@ ILocationInfo loc ) { DeclaringModule = declModule ?? throw new ArgumentNullException(nameof(declModule)); DeclaringType = declType; + FunctionDefinition = def; - Name = def.Name; + Name = FunctionDefinition.Name; if (Name == "__init__") { _doc = declType?.Documentation; } - foreach (var dec in (def.Decorators?.Decorators).MaybeEnumerate().ExcludeDefault().OfType()) { + foreach (var dec in (FunctionDefinition.Decorators?.Decorators).MaybeEnumerate().ExcludeDefault().OfType()) { if (dec.Name == "classmethod") { IsClassMethod = true; } else if (dec.Name == "staticmethod") { @@ -54,21 +55,9 @@ ILocationInfo loc Locations = loc != null ? new[] { loc } : Array.Empty(); } - internal AstPythonFunction(IPythonFunction original) { - DeclaringModule = original.DeclaringModule; - DeclaringType = original.DeclaringType; - Name = original.Name; - // Copy the null if _doc isn't set in the original; otherwise calculate the docs - _doc = (original is AstPythonFunction apf) ? apf._doc : original.Documentation; - IsClassMethod = original.IsClassMethod; - IsStatic = original.IsStatic; - _overloads = original.Overloads.ToList(); - Locations = (original as ILocatedMember)?.Locations ?? Array.Empty(); - } + public FunctionDefinition FunctionDefinition { get; } - internal void AddOverload(IPythonFunctionOverload overload) { - _overloads.Add(overload); - } + internal void AddOverload(IPythonFunctionOverload overload) => _overloads.Add(overload); public IPythonModule DeclaringModule {get;} public IPythonType DeclaringType {get;} diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunctionOverload.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunctionOverload.cs index 56ac7c1e2..a0fe051a8 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunctionOverload.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunctionOverload.cs @@ -9,7 +9,7 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonModule.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonModule.cs index 3c4d73e39..db99531bd 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonModule.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonModule.cs @@ -9,7 +9,7 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. @@ -22,105 +22,20 @@ using System.Threading; using Microsoft.PythonTools.Analysis; using Microsoft.PythonTools.Analysis.Infrastructure; -using Microsoft.PythonTools.Parsing; using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Interpreter.Ast { - public static class PythonModuleLoader { - public static IPythonModule FromFile( - IPythonInterpreter interpreter, - string sourceFile, - PythonLanguageVersion langVersion - ) { - return FromFile(interpreter, sourceFile, langVersion, null); - } - - public static IPythonModule FromFile( - IPythonInterpreter interpreter, - string sourceFile, - PythonLanguageVersion langVersion, - string moduleFullName - ) { - Stream stream = null; - try { - if(Directory.Exists(sourceFile)) { - stream = new MemoryStream(); // Module without __init__.py, create empty stream - } else { - stream = new FileStream(sourceFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); - } - return FromStream(interpreter, stream, sourceFile, langVersion, moduleFullName); - } finally { - stream?.Dispose(); - } - } - - public static IPythonModule FromStream( - IPythonInterpreter interpreter, - Stream sourceFile, - string fileName, - PythonLanguageVersion langVersion - ) { - return FromStream(interpreter, sourceFile, fileName, langVersion, null); - } - - public static IPythonModule FromStream( - IPythonInterpreter interpreter, - Stream sourceFile, - string fileName, - PythonLanguageVersion langVersion, - string moduleFullName - ) { - PythonAst ast; - var sink = KeepParseErrors ? new CollectingErrorSink() : ErrorSink.Null; - var parser = Parser.CreateParser(sourceFile, langVersion, new ParserOptions { - StubFile = fileName.EndsWithOrdinal(".pyi", ignoreCase: true), - ErrorSink = sink - }); - ast = parser.ParseFile(); - - return new AstPythonModule( - moduleFullName ?? ModulePath.FromFullPath(fileName, isPackage: IsPackageCheck).FullName, - interpreter, - ast, - fileName, - (sink as CollectingErrorSink)?.Errors.Select(e => "{0} ({1}): {2}".FormatUI(fileName ?? "(builtins)", e.Span, e.Message)) - ); - } - - public static IPythonModule FromTypeStub( - IPythonInterpreter interpreter, - string stubFile, - PythonLanguageVersion langVersion, - string moduleFullName - ) { - return new AstCachedPythonModule(moduleFullName, stubFile); - } - - // Avoid hitting the filesystem, but exclude non-importable - // paths. Ideally, we'd stop at the first path that's a known - // search path, except we don't know search paths here. - private static bool IsPackageCheck(string path) { - return ModulePath.IsImportable(PathUtils.GetFileName(path.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar))); - } - - internal static bool KeepParseErrors = false; - } - sealed class AstPythonModule : IPythonModule, IProjectEntry, ILocatedMember { private readonly IPythonInterpreter _interpreter; - private readonly Dictionary _properties; - private readonly List _childModules; - private readonly Dictionary _members; + private readonly List _childModules = new List(); + private readonly Dictionary _members = new Dictionary(); private bool _foundChildModules; private string _documentation = string.Empty; internal AstPythonModule() { Name = string.Empty; FilePath = string.Empty; - _properties = new Dictionary(); - _childModules = new List(); _foundChildModules = true; - _members = new Dictionary(); } internal AstPythonModule(string moduleName, IPythonInterpreter interpreter, PythonAst ast, string filePath, IEnumerable parseErrors) { @@ -131,16 +46,16 @@ internal AstPythonModule(string moduleName, IPythonInterpreter interpreter, Pyth Locations = new[] { new LocationInfo(filePath, DocumentUri, 1, 1) }; _interpreter = interpreter; - _properties = new Dictionary(); - _childModules = new List(); - _members = new Dictionary(); - // Do not allow children of named modules - if (!ModulePath.IsInitPyFile(FilePath)) { - _foundChildModules = true; - } + _foundChildModules = !ModulePath.IsInitPyFile(FilePath); + + var walker = new AstAnalysisWalker( + interpreter, ast, this, filePath, DocumentUri, _members, + includeLocationInfo: true, + warnAboutUndefinedValues: true, + suppressBuiltinLookup: false + ); - var walker = new AstAnalysisWalker(interpreter, ast, this, filePath, DocumentUri, _members, true, true); ast.Walk(walker); walker.Complete(); @@ -178,7 +93,7 @@ public string Documentation { public string FilePath { get; } public Uri DocumentUri { get; } public PythonMemberType MemberType => PythonMemberType.Module; - public Dictionary Properties => _properties; + public Dictionary Properties { get; } = new Dictionary(); public IEnumerable Locations { get; } public int AnalysisVersion => 1; diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonProperty.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonProperty.cs index 60c4fac7b..69f79bb1d 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonProperty.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonProperty.cs @@ -5,34 +5,31 @@ using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Interpreter.Ast { - class AstPythonProperty : IBuiltinProperty, ILocatedMember { + class AstPythonProperty : IBuiltinProperty2, ILocatedMember { private IPythonFunctionOverload _getter; public AstPythonProperty( PythonAst ast, - FunctionDefinition getter, + FunctionDefinition definition, ILocationInfo location ) { - Documentation = getter.Documentation; IsReadOnly = true; Locations = new[] { location }; + FunctionDefinition = definition; } - public void AddOverload(IPythonFunctionOverload overload) { - if (_getter == null) { - _getter = overload; - } - } + public FunctionDefinition FunctionDefinition { get; } - public void MakeSettable() { - IsReadOnly = false; - } + public void AddOverload(IPythonFunctionOverload overload) + => _getter = _getter ?? overload; + + public void MakeSettable() => IsReadOnly = false; public IPythonType Type => _getter?.ReturnType.FirstOrDefault(); public bool IsStatic => false; - public string Documentation { get; } + public string Documentation => FunctionDefinition.Documentation; public string Description => Type == null ? "property of unknown type" : "property of type {0}".FormatUI(Type.Name); diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs index d2f9cbd51..269f7b090 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs @@ -9,7 +9,7 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstScrapedPythonModule.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstScrapedPythonModule.cs index 0fc2b462d..ab6210430 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstScrapedPythonModule.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstScrapedPythonModule.cs @@ -127,7 +127,12 @@ protected virtual PythonWalker PrepareWalker(IPythonInterpreter interpreter, Pyt const Uri uri = null; const bool includeLocations = false; #endif - return new AstAnalysisWalker(interpreter, ast, this, filePath, uri, _members, includeLocations, true); + return new AstAnalysisWalker( + interpreter, ast, this, filePath, uri, _members, + includeLocations, + warnAboutUndefinedValues: true, + suppressBuiltinLookup: false + ); } protected virtual void PostWalk(PythonWalker walker) { diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs index cf4a80797..1d9df306e 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs @@ -50,7 +50,7 @@ public override IPythonType Finalize(IPythonType type) { return null; } - if (type == _scope._unknownType) { + if (type == _scope.UnknownType) { return null; } @@ -86,21 +86,17 @@ public override IPythonType LookupName(string name) { return m as IPythonType; } - public override IPythonType GetTypeMember(IPythonType baseType, string member) { - return AsIPythonType(baseType.GetMember(_scope.Context, member)); - } + public override IPythonType GetTypeMember(IPythonType baseType, string member) + => AsIPythonType(baseType.GetMember(_scope.Context, member)); public override IPythonType MakeNameType(string name) => new NameType(name); public override string GetName(IPythonType type) => (type as NameType)?.Name; - public override IPythonType MakeUnion(IReadOnlyList types) { - return new UnionType(types); - } + public override IPythonType MakeUnion(IReadOnlyList types) => new UnionType(types); - public override IReadOnlyList GetUnionTypes(IPythonType unionType) { - return (unionType as UnionType)?.Types ?? - (unionType as IPythonMultipleMembers)?.Members.OfType().ToArray(); - } + public override IReadOnlyList GetUnionTypes(IPythonType unionType) => + (unionType as UnionType)?.Types ?? + (unionType as IPythonMultipleMembers)?.Members.OfType().ToArray(); public override IPythonType MakeGeneric(IPythonType baseType, IReadOnlyList args) { if (args == null || args.Count == 0 || baseType == null) { @@ -127,7 +123,7 @@ public override IPythonType MakeGeneric(IPythonType baseType, IReadOnlyList> _scopes = new Stack>(); + private readonly AstAnalysisFunctionWalkerSet _functionWalkers; private readonly Lazy _builtinModule; private readonly AnalysisLogWriter _log; - internal readonly IPythonType _unknownType; + internal IPythonType UnknownType { get; } public NameLookupContext( IPythonInterpreter interpreter, @@ -40,6 +41,7 @@ public NameLookupContext( string filePath, Uri documentUri, bool includeLocationInfo, + AstAnalysisFunctionWalkerSet functionWalkers, IPythonModule builtinModule = null, AnalysisLogWriter log = null ) { @@ -51,9 +53,10 @@ public NameLookupContext( DocumentUri = documentUri; IncludeLocationInfo = includeLocationInfo; + _functionWalkers = functionWalkers; DefaultLookupOptions = LookupOptions.Normal; - _unknownType = Interpreter.GetBuiltinType(BuiltinTypeId.Unknown) ?? + UnknownType = Interpreter.GetBuiltinType(BuiltinTypeId.Unknown) ?? new FallbackBuiltinPythonType(new FallbackBuiltinModule(Ast.LanguageVersion), BuiltinTypeId.Unknown); _builtinModule = builtinModule == null ? new Lazy(ImportBuiltinModule) : new Lazy(() => builtinModule); @@ -80,12 +83,13 @@ public NameLookupContext Clone(bool copyScopeContents = false) { FilePath, DocumentUri, IncludeLocationInfo, + _functionWalkers, _builtinModule.IsValueCreated ? _builtinModule.Value : null, _log - ); - - ctxt.DefaultLookupOptions = DefaultLookupOptions; - ctxt.SuppressBuiltinLookup = SuppressBuiltinLookup; + ) { + DefaultLookupOptions = DefaultLookupOptions, + SuppressBuiltinLookup = SuppressBuiltinLookup + }; foreach (var scope in _scopes.Reverse()) { if (copyScopeContents) { @@ -112,11 +116,9 @@ public Dictionary PushScope(Dictionary scope = return scope; } - public Dictionary PopScope() { - return _scopes.Pop(); - } + public Dictionary PopScope() => _scopes.Pop(); - internal LocationInfo GetLoc(Node node) { + public LocationInfo GetLoc(Node node) { if (!IncludeLocationInfo) { return null; } @@ -129,7 +131,7 @@ internal LocationInfo GetLoc(Node node) { return new LocationInfo(FilePath, DocumentUri, start.Line, start.Column, end.Line, end.Column); } - internal LocationInfo GetLocOfName(Node node, NameExpression header) { + public LocationInfo GetLocOfName(Node node, NameExpression header) { var loc = GetLoc(node); if (loc == null || header == null) { return null; @@ -147,26 +149,6 @@ internal LocationInfo GetLocOfName(Node node, NameExpression header) { return loc; } - private string GetNameFromExpressionWorker(Expression expr) { - if (expr is NameExpression ne) { - return ne.Name; - } - - if (expr is MemberExpression me) { - return "{0}.{1}".FormatInvariant(GetNameFromExpressionWorker(me.Target), me.Name); - } - - throw new FormatException(); - } - - public string GetNameFromExpression(Expression expr) { - try { - return GetNameFromExpressionWorker(expr); - } catch (FormatException) { - return null; - } - } - public IEnumerable GetTypesFromAnnotation(Expression expr) { if (expr == null) { return Enumerable.Empty(); @@ -194,28 +176,42 @@ public IMember GetValueFromExpression(Expression expr, LookupOptions options) { return null; } - var m = GetValueFromName(expr as NameExpression, options) ?? - GetValueFromMember(expr as MemberExpression, options) ?? - GetValueFromCallable(expr as CallExpression, options) ?? - GetValueFromUnaryOp(expr as UnaryExpression, options) ?? - GetValueFromBinaryOp(expr, options) ?? - GetValueFromIndex(expr as IndexExpression, options) ?? - GetValueFromConditional(expr as ConditionalExpression, options) ?? - GetConstantFromLiteral(expr, options); - if (m != null) { - return m; + IMember m; + switch (expr) { + case NameExpression nex: + m = GetValueFromName(nex, options); + break; + case MemberExpression mex: + m = GetValueFromMember(mex, options); + break; + case CallExpression cex: + m = GetValueFromCallable(cex, options); + break; + case UnaryExpression uex: + m = GetValueFromUnaryOp(uex, options); + break; + case IndexExpression iex: + m = GetValueFromIndex(iex, options); + break; + case ConditionalExpression coex: + m = GetValueFromConditional(coex, options); + break; + default: + m = GetValueFromBinaryOp(expr, options) ?? GetConstantFromLiteral(expr, options); + break; } - - _log?.Log(TraceLevel.Verbose, "UnknownExpression", expr.ToCodeString(Ast).Trim()); - return null; + if (m == null) { + _log?.Log(TraceLevel.Verbose, "UnknownExpression", expr.ToCodeString(Ast).Trim()); + } + return m; } private IMember GetValueFromName(NameExpression expr, LookupOptions options) { - if (expr == null || string.IsNullOrEmpty(expr.Name)) { + if (string.IsNullOrEmpty(expr?.Name)) { return null; } - IMember existing = LookupNameInScopes(expr.Name, options); + var existing = LookupNameInScopes(expr.Name, options); if (existing != null) { return existing; } @@ -224,32 +220,44 @@ private IMember GetValueFromName(NameExpression expr, LookupOptions options) { return Module; } _log?.Log(TraceLevel.Verbose, "UnknownName", expr.Name); - return new AstPythonConstant(_unknownType, GetLoc(expr)); + return new AstPythonConstant(UnknownType, GetLoc(expr)); } private IMember GetValueFromMember(MemberExpression expr, LookupOptions options) { - if (expr == null || expr.Target == null || string.IsNullOrEmpty(expr.Name)) { + if (expr?.Target == null || string.IsNullOrEmpty(expr?.Name)) { return null; } var e = GetValueFromExpression(expr.Target); - if (e is IMemberContainer mc) { - var m = mc.GetMember(Context, expr.Name); - if (m != null) { - return m; - } + IMember value = null; + switch (e) { + case IMemberContainer mc: + value = mc.GetMember(Context, expr.Name); + break; + case IPythonMultipleMembers mm: + value = mm.Members.OfType() + .Select(x => x.GetMember(Context, expr.Name)) + .ExcludeDefault() + .FirstOrDefault(); + break; + default: + value = GetValueFromPropertyOrFunction(e, expr); + break; + } + + if (value is IBuiltinProperty2 p) { + value = GetPropertyReturnType(p, expr); + } else if (value == null) { _log?.Log(TraceLevel.Verbose, "UnknownMember", expr.ToCodeString(Ast).Trim()); - } else { - _log?.Log(TraceLevel.Verbose, "UnknownMemberContainer", expr.Target.ToCodeString(Ast).Trim()); + return new AstPythonConstant(UnknownType, GetLoc(expr)); } - return new AstPythonConstant(_unknownType, GetLoc(expr)); + return value; } private IMember GetValueFromUnaryOp(UnaryExpression expr, LookupOptions options) { if (expr == null || expr.Expression == null) { return null; } - // Assume that the type after the op is the same as before return GetValueFromExpression(expr.Expression); } @@ -300,7 +308,7 @@ private IMember GetValueFromIndex(IndexExpression expr, LookupOptions options) { } var type = GetTypeFromValue(GetValueFromExpression(expr.Target)); - if (type != null && type != _unknownType) { + if (!IsUnknown(type)) { if (AstTypingModule.IsTypingType(type)) { return type; } @@ -326,7 +334,7 @@ private IMember GetValueFromIndex(IndexExpression expr, LookupOptions options) { } else { _log?.Log(TraceLevel.Verbose, "UnknownIndex", expr.ToCodeString(Ast, CodeFormattingOptions.Traditional).Trim()); } - return new AstPythonConstant(_unknownType, GetLoc(expr)); + return null; } private IMember GetValueFromConditional(ConditionalExpression expr, LookupOptions options) { @@ -334,10 +342,12 @@ private IMember GetValueFromConditional(ConditionalExpression expr, LookupOption return null; } - return AstPythonMultipleMembers.Combine( - GetValueFromExpression(expr.TrueExpression), - GetValueFromExpression(expr.FalseExpression) - ); + var trueValue = GetValueFromExpression(expr.TrueExpression); + var falseValue = GetValueFromExpression(expr.FalseExpression); + + return trueValue != null || falseValue != null + ? AstPythonMultipleMembers.Combine(trueValue, falseValue) + : null; } private IMember GetValueFromCallable(CallExpression expr, LookupOptions options) { @@ -346,27 +356,58 @@ private IMember GetValueFromCallable(CallExpression expr, LookupOptions options) } var m = GetValueFromExpression(expr.Target); - if (m is IPythonType type) { - if (type.TypeId == BuiltinTypeId.Type && type == Interpreter.GetBuiltinType(BuiltinTypeId.Type) && expr.Args.Count >= 1) { - var aType = GetTypeFromValue(GetValueFromExpression(expr.Args[0].Expression)); - if (aType != null) { - return aType; - } - } - return new AstPythonConstant(type, GetLoc(expr)); + IMember value = null; + switch (m) { + case IPythonType type when type == Interpreter.GetBuiltinType(BuiltinTypeId.Type) && expr.Args.Count >= 1: + value = GetTypeFromValue(GetValueFromExpression(expr.Args[0].Expression, options)); + break; + case IPythonType type: + value = new AstPythonConstant(type, GetLoc(expr)); + break; + default: + value = GetValueFromPropertyOrFunction(m, expr); + break; } - if (m is IPythonFunction fn) { - // TODO: Select correct overload and handle multiple return types - if (fn.Overloads.Count > 0 && fn.Overloads[0].ReturnType.Count > 0) { - return new AstPythonConstant(fn.Overloads[0].ReturnType[0]); - } - _log?.Log(TraceLevel.Verbose, "NoReturn", expr.Target.ToCodeString(Ast).Trim()); - return new AstPythonConstant(Interpreter.GetBuiltinType(BuiltinTypeId.NoneType), GetLoc(expr)); + if (value == null) { + _log?.Log(TraceLevel.Verbose, "UnknownCallable", expr.Target.ToCodeString(Ast).Trim()); + } + return value; + } + + private IMember GetValueFromPropertyOrFunction(IMember fn, Expression expr) { + switch (fn) { + case IPythonBoundFunction bf when bf.Function is IPythonFunction2 f: + return GetValueFromFunction(f, expr); + case IPythonFunction2 f: + return GetValueFromFunction(f, expr); + case IBuiltinProperty2 p: + return GetPropertyReturnType(p, expr); + case IPythonConstant c when c.Type?.MemberType == PythonMemberType.Class: + return c.Type; // typically cls() + } + return null; + } + + private IMember GetValueFromFunction(IPythonFunction2 fn, Expression expr) { + var returnType = GetFunctionReturnType(fn.Overloads.FirstOrDefault()); + if (IsUnknown(returnType)) { + // Function may not have been walked yet. Do it now. + _functionWalkers.ProcessFunction(fn.FunctionDefinition); + returnType = GetFunctionReturnType(fn.Overloads.FirstOrDefault()); } + return !IsUnknown(returnType) ? new AstPythonConstant(returnType, GetLoc(expr)) : null; + } - _log?.Log(TraceLevel.Verbose, "UnknownCallable", expr.Target.ToCodeString(Ast).Trim()); - return new AstPythonConstant(_unknownType, GetLoc(expr)); + private IPythonType GetFunctionReturnType(IPythonFunctionOverload o) + => o != null && o.ReturnType.Count > 0 ? o.ReturnType[0] : UnknownType; + + private IMember GetPropertyReturnType(IBuiltinProperty2 p, Expression expr) { + if (IsUnknown(p.Type)) { + // Function may not have been walked yet. Do it now. + _functionWalkers.ProcessFunction(p.FunctionDefinition); + } + return !IsUnknown(p.Type) ? new AstPythonConstant(p.Type, GetLoc(expr)) : null; } public IPythonConstant GetConstantFromLiteral(Expression expr, LookupOptions options) { @@ -379,11 +420,7 @@ public IPythonConstant GetConstantFromLiteral(Expression expr, LookupOptions opt } var type = GetTypeFromLiteral(expr); - if (type != null) { - return new AstPythonConstant(type, GetLoc(expr)); - } - - return null; + return type != null ? new AstPythonConstant(type, GetLoc(expr)) : null; } public IEnumerable GetTypesFromValue(IMember value) { @@ -525,17 +562,16 @@ public IMember GetInScope(string name, Dictionary scope = null) return null; } - IMember obj; - if ((scope ?? _scopes.Peek()).TryGetValue(name, out obj)) { + if ((scope ?? _scopes.Peek()).TryGetValue(name, out var obj)) { return obj; } return null; } - private static bool IsUnknown(IMember value) { - return (value as IPythonType)?.TypeId == BuiltinTypeId.Unknown || - (value as IPythonConstant)?.Type?.TypeId == BuiltinTypeId.Unknown; - } + private static bool IsUnknown(IMember value) => + value == null || + (value as IPythonType)?.TypeId == BuiltinTypeId.Unknown || + (value as IPythonConstant)?.Type?.TypeId == BuiltinTypeId.Unknown; public void SetInScope(string name, IMember value, bool mergeWithExisting = true, Dictionary scope = null) { Debug.Assert(_scopes.Count > 0); @@ -547,7 +583,7 @@ public void SetInScope(string name, IMember value, bool mergeWithExisting = true s.Remove(name); return; } - if (mergeWithExisting && s.TryGetValue(name, out IMember existing) && existing != null) { + if (mergeWithExisting && s.TryGetValue(name, out var existing) && existing != null) { if (IsUnknown(existing)) { s[name] = value; } else if (!IsUnknown(value)) { @@ -568,13 +604,9 @@ public enum LookupOptions { Normal = Local | Nonlocal | Global | Builtins } - public IMember LookupNameInScopes(string name) { - return LookupNameInScopes(name, DefaultLookupOptions); - } + public IMember LookupNameInScopes(string name) => LookupNameInScopes(name, DefaultLookupOptions); public IMember LookupNameInScopes(string name, LookupOptions options) { - IMember value; - var scopes = _scopes.ToList(); if (scopes.Count == 1) { if (!options.HasFlag(LookupOptions.Local) && !options.HasFlag(LookupOptions.Global)) { @@ -596,7 +628,7 @@ public IMember LookupNameInScopes(string name, LookupOptions options) { if (scopes != null) { foreach (var scope in scopes) { - if (scope.TryGetValue(name, out value) && value != null) { + if (scope.TryGetValue(name, out var value) && value != null) { if (value is ILazyMember lm) { value = lm.Get(); scope[name] = value; @@ -609,7 +641,6 @@ public IMember LookupNameInScopes(string name, LookupOptions options) { if (!SuppressBuiltinLookup && options.HasFlag(LookupOptions.Builtins)) { return _builtinModule.Value.GetMember(Context, name); } - return null; } } diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/PythonModuleLoader.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/PythonModuleLoader.cs new file mode 100644 index 000000000..1ddc468e1 --- /dev/null +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/PythonModuleLoader.cs @@ -0,0 +1,99 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.IO; +using System.Linq; +using Microsoft.PythonTools.Analysis; +using Microsoft.PythonTools.Analysis.Infrastructure; +using Microsoft.PythonTools.Parsing; +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Interpreter.Ast { + public static class PythonModuleLoader { + public static IPythonModule FromFile( + IPythonInterpreter interpreter, + string sourceFile, + PythonLanguageVersion langVersion + ) => FromFile(interpreter, sourceFile, langVersion, null); + + public static IPythonModule FromFile( + IPythonInterpreter interpreter, + string sourceFile, + PythonLanguageVersion langVersion, + string moduleFullName + ) { + Stream stream = null; + try { + if (Directory.Exists(sourceFile)) { + stream = new MemoryStream(); // Module without __init__.py, create empty stream + } else { + stream = new FileStream(sourceFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + } + return FromStream(interpreter, stream, sourceFile, langVersion, moduleFullName); + } finally { + stream?.Dispose(); + } + } + + public static IPythonModule FromStream( + IPythonInterpreter interpreter, + Stream sourceFile, + string fileName, + PythonLanguageVersion langVersion + ) { + return FromStream(interpreter, sourceFile, fileName, langVersion, null); + } + + public static IPythonModule FromStream( + IPythonInterpreter interpreter, + Stream sourceFile, + string fileName, + PythonLanguageVersion langVersion, + string moduleFullName + ) { + PythonAst ast; + var sink = KeepParseErrors ? new CollectingErrorSink() : ErrorSink.Null; + var parser = Parser.CreateParser(sourceFile, langVersion, new ParserOptions { + StubFile = fileName.EndsWithOrdinal(".pyi", ignoreCase: true), + ErrorSink = sink + }); + ast = parser.ParseFile(); + + return new AstPythonModule( + moduleFullName ?? ModulePath.FromFullPath(fileName, isPackage: IsPackageCheck).FullName, + interpreter, + ast, + fileName, + (sink as CollectingErrorSink)?.Errors.Select(e => "{0} ({1}): {2}".FormatUI(fileName ?? "(builtins)", e.Span, e.Message)) + ); + } + + public static IPythonModule FromTypeStub( + IPythonInterpreter interpreter, + string stubFile, + PythonLanguageVersion langVersion, + string moduleFullName + ) => new AstCachedPythonModule(moduleFullName, stubFile); + + // Avoid hitting the filesystem, but exclude non-importable + // paths. Ideally, we'd stop at the first path that's a known + // search path, except we don't know search paths here. + private static bool IsPackageCheck(string path) + => ModulePath.IsImportable(PathUtils.GetFileName(path.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar))); + + internal static bool KeepParseErrors = false; + } +} diff --git a/src/Analysis/Engine/Impl/Interpreter/Definitions/IBuiltinProperty.cs b/src/Analysis/Engine/Impl/Interpreter/Definitions/IBuiltinProperty.cs index 0e5098879..e9cc0ab1f 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Definitions/IBuiltinProperty.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Definitions/IBuiltinProperty.cs @@ -9,11 +9,12 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Interpreter { /// @@ -23,29 +24,25 @@ public interface IBuiltinProperty : IMember { /// /// The type of the value the property gets/sets. /// - IPythonType Type { - get; - } + IPythonType Type { get; } /// /// True if the property is static (declared on the class) not the instance. /// - bool IsStatic { - get; - } + bool IsStatic { get; } /// /// Documentation for the property. /// - string Documentation { - get; - } + string Documentation { get; } /// /// A user readable description of the property. /// - string Description { - get; - } + string Description { get; } + } + + public interface IBuiltinProperty2: IBuiltinProperty { + FunctionDefinition FunctionDefinition { get; } } } diff --git a/src/Analysis/Engine/Impl/Interpreter/Definitions/IPythonFunction.cs b/src/Analysis/Engine/Impl/Interpreter/Definitions/IPythonFunction.cs index fadf86826..69036e271 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Definitions/IPythonFunction.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Definitions/IPythonFunction.cs @@ -15,46 +15,29 @@ // permissions and limitations under the License. using System.Collections.Generic; +using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Interpreter { /// /// Represents an object which is a function. Provides documentation for signature help. /// public interface IPythonFunction : IMember { - string Name { - get; - } + string Name { get; } + string Documentation { get; } + bool IsBuiltin { get; } - string Documentation { - get; - } - - bool IsBuiltin { - get; - } - /// /// False if binds instance when in a class, true if always static. /// - bool IsStatic { - get; - } - - bool IsClassMethod { - get; - } - - IReadOnlyList Overloads { - get; - } - - IPythonType DeclaringType { - get; - } + bool IsStatic { get; } + bool IsClassMethod { get; } + IReadOnlyList Overloads { get; } + IPythonType DeclaringType { get; } + IPythonModule DeclaringModule { get; } + } - IPythonModule DeclaringModule { - get; - } + public interface IPythonFunction2 : IPythonFunction { + FunctionDefinition FunctionDefinition { get; } } /// diff --git a/src/Analysis/Engine/Impl/Parsing/Ast/FunctionDefinition.cs b/src/Analysis/Engine/Impl/Parsing/Ast/FunctionDefinition.cs index d0c32ee05..c1aa96710 100644 --- a/src/Analysis/Engine/Impl/Parsing/Ast/FunctionDefinition.cs +++ b/src/Analysis/Engine/Impl/Parsing/Ast/FunctionDefinition.cs @@ -15,53 +15,38 @@ // permissions and limitations under the License. using System; -using System.Collections.Generic; +using System.Diagnostics; using System.Text; using Microsoft.PythonTools.Analysis.Infrastructure; namespace Microsoft.PythonTools.Parsing.Ast { - + [DebuggerDisplay("{Name}")] public class FunctionDefinition : ScopeStatement, IMaybeAsyncStatement { - protected Statement _body; - private readonly NameExpression/*!*/ _name; + internal static readonly object WhitespaceAfterAsync = new object(); + private readonly Parameter[] _parameters; - private Expression _returnAnnotation; - private DecoratorStatement _decorators; - private bool _generator; // The function is a generator - private bool _coroutine; - private bool _isLambda; - - private PythonVariable _variable; // The variable corresponding to the function name or null for lambdas - internal PythonVariable _nameVariable; // the variable that refers to the global __name__ - internal bool _hasReturn; - private int _headerIndex; - private int _defIndex; private int? _keywordEndIndex; - internal static readonly object WhitespaceAfterAsync = new object(); + protected Statement _body; public FunctionDefinition(NameExpression name, Parameter[] parameters) - : this(name, parameters, (Statement)null) { + : this(name, parameters, (Statement)null) { } - + public FunctionDefinition(NameExpression name, Parameter[] parameters, Statement body, DecoratorStatement decorators = null) { if (name == null) { - _name = new NameExpression(""); - _isLambda = true; + NameExpression = new NameExpression(""); + IsLambda = true; } else { - _name = name; + NameExpression = name; } _parameters = parameters; _body = body; - _decorators = decorators; + Decorators = decorators; } - public bool IsLambda { - get { - return _isLambda; - } - } + public bool IsLambda { get; } public Parameter[] Parameters => _parameters ?? Array.Empty(); @@ -71,41 +56,21 @@ public bool IsLambda { public override int KeywordEndIndex => _keywordEndIndex ?? (DefIndex + (IsCoroutine ? 9 : 3)); public override int KeywordLength => KeywordEndIndex - StartIndex; - public Expression ReturnAnnotation { - get { return _returnAnnotation; } - set { _returnAnnotation = value; } - } + public Expression ReturnAnnotation { get; set; } - public override Statement Body { - get { return _body; } - } + public override Statement Body => _body; - internal void SetBody(Statement body) { - _body = body; - } + internal void SetBody(Statement body) => _body = body; - public int HeaderIndex { - get { return _headerIndex; } - set { _headerIndex = value; } - } + public int HeaderIndex { get; set; } - public int DefIndex { - get { return _defIndex; } - set { _defIndex = value; } - } + public int DefIndex { get; set; } - public override string/*!*/ Name { - get { return _name.Name ?? ""; } - } + public override string/*!*/ Name => NameExpression.Name ?? ""; - public NameExpression NameExpression { - get { return _name; } - } + public NameExpression NameExpression { get; } - public DecoratorStatement Decorators { - get { return _decorators; } - internal set { _decorators = value; } - } + public DecoratorStatement Decorators { get; internal set; } internal LambdaExpression LambdaExpression { get; set; } @@ -114,29 +79,20 @@ public DecoratorStatement Decorators { /// expression and instead of returning a value when called they return a generator /// object which implements the iterator protocol. /// - public bool IsGenerator { - get { return _generator; } - set { _generator = value; } - } + public bool IsGenerator { get; set; } /// /// True if the function is a coroutine. Coroutines are defined using /// 'async def'. /// - public bool IsCoroutine { - get { return _coroutine; } - set { _coroutine = value; } - } + public bool IsCoroutine { get; set; } bool IMaybeAsyncStatement.IsAsync => IsCoroutine; /// /// Gets the variable that this function is assigned to. /// - public PythonVariable Variable { - get { return _variable; } - set { _variable = value; } - } + public PythonVariable Variable { get; set; } /// /// Gets the variable reference for the specific assignment to the variable for this function definition. @@ -146,7 +102,7 @@ public PythonReference GetVariableReference(PythonAst ast) { } internal override bool ExposesLocalVariable(PythonVariable variable) { - return NeedsLocalsDictionary; + return NeedsLocalsDictionary; } internal override bool TryBindOuter(ScopeStatement from, string name, bool allowGlobals, out PythonVariable variable) { @@ -163,7 +119,7 @@ internal override bool TryBindOuter(ScopeStatement from, string name, bool allow } AddCellVariable(variable); - } else if(allowGlobals) { + } else if (allowGlobals) { from.AddReferencedGlobal(name); } return true; @@ -197,7 +153,7 @@ internal override void Bind(PythonNameBinder binder) { base.Bind(binder); Verify(binder); } - + private void Verify(PythonNameBinder binder) { if (ContainsImportStar && IsClosure) { binder.ReportSyntaxError( @@ -235,27 +191,19 @@ public int GetIndexOfDef(PythonAst ast) { public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { - _name?.Walk(walker); - if (_parameters != null) { - foreach (Parameter p in _parameters) { - p.Walk(walker); - } - } - if (_decorators != null) { - _decorators.Walk(walker); - } - if (_body != null) { - _body.Walk(walker); - } - if (_returnAnnotation != null) { - _returnAnnotation.Walk(walker); + NameExpression?.Walk(walker); + foreach (var p in _parameters.MaybeEnumerate()) { + p.Walk(walker); } + Decorators?.Walk(walker); + _body?.Walk(walker); + ReturnAnnotation?.Walk(walker); } walker.PostWalk(this); } public SourceLocation Header { - get { return GlobalParent.IndexToLocation(_headerIndex); } + get { return GlobalParent.IndexToLocation(HeaderIndex); } } public override string GetLeadingWhiteSpace(PythonAst ast) { @@ -290,10 +238,10 @@ internal override void AppendCodeStringStmt(StringBuilder res, PythonAst ast, Co res.Append(name); if (!this.IsIncompleteNode(ast)) { format.Append( - res, - format.SpaceBeforeFunctionDeclarationParen, - " ", - "", + res, + format.SpaceBeforeFunctionDeclarationParen, + " ", + "", this.GetThirdWhiteSpaceDefaultNull(ast) ); @@ -317,13 +265,13 @@ internal override void AppendCodeStringStmt(StringBuilder res, PythonAst ast, Co format.Append( res, - Parameters.Length != 0 ? + Parameters.Length != 0 ? format.SpaceWithinFunctionDeclarationParens : format.SpaceWithinEmptyParameterList, " ", "", this.GetFourthWhiteSpaceDefaultNull(ast) - ); + ); if (!this.IsMissingCloseGrouping(ast)) { res.Append(')'); @@ -336,11 +284,11 @@ internal override void AppendCodeStringStmt(StringBuilder res, PythonAst ast, Co " ", "", this.GetFifthWhiteSpace(ast) - ); + ); res.Append("->"); - _returnAnnotation.AppendCodeString( - res, - ast, + ReturnAnnotation.AppendCodeString( + res, + ast, format, format.SpaceAroundAnnotationArrow != null ? format.SpaceAroundAnnotationArrow.Value ? " " : "" : diff --git a/src/Analysis/Engine/Impl/Parsing/Ast/PythonNameBinder.cs b/src/Analysis/Engine/Impl/Parsing/Ast/PythonNameBinder.cs index 76503b40c..21ee0e84b 100644 --- a/src/Analysis/Engine/Impl/Parsing/Ast/PythonNameBinder.cs +++ b/src/Analysis/Engine/Impl/Parsing/Ast/PythonNameBinder.cs @@ -346,14 +346,6 @@ public override bool Walk(WhileStatement node) { return false; } - public override bool Walk(ReturnStatement node) { - FunctionDefinition funcDef = _currentScope as FunctionDefinition; - if (funcDef != null) { - funcDef._hasReturn = true; - } - return base.Walk(node); - } - // WithStatement public override bool Walk(WithStatement node) { _currentScope.ContainsExceptionHandling = true; @@ -393,7 +385,7 @@ public override bool Walk(FromImportStatement node) { // FunctionDefinition public override bool Walk(FunctionDefinition node) { - node._nameVariable = _globalScope.EnsureGlobalVariable("__name__"); + _globalScope.EnsureGlobalVariable("__name__"); // Name is defined in the enclosing context if (!node.IsLambda) { diff --git a/src/Analysis/Engine/Impl/PythonAnalyzer.cs b/src/Analysis/Engine/Impl/PythonAnalyzer.cs index d03fc9ab8..9911dca5e 100644 --- a/src/Analysis/Engine/Impl/PythonAnalyzer.cs +++ b/src/Analysis/Engine/Impl/PythonAnalyzer.cs @@ -221,8 +221,7 @@ public IPythonProjectEntry AddModule(string moduleName, string filePath, Uri doc /// /// New in 2.1 public void AddModuleAlias(string moduleName, string moduleAlias) { - ModuleReference modRef; - if (Modules.TryImport(moduleName, out modRef)) { + if (Modules.TryImport(moduleName, out var modRef)) { Modules[moduleAlias] = modRef; } } @@ -279,9 +278,8 @@ public void RemoveModule(IProjectEntry entry, Action onImpo /// '__init__'. /// public IEnumerable GetEntriesThatImportModule(string moduleName, bool includeUnresolved) { - ModuleReference modRef; var entries = new List(); - if (Modules.TryImport(moduleName, out modRef) && modRef.HasReferences) { + if (Modules.TryImport(moduleName, out var modRef) && modRef.HasReferences) { entries.AddRange(modRef.References.Select(m => m.ProjectEntry).OfType()); } @@ -365,8 +363,7 @@ public IMemberResult[] GetModules() { } if (moduleRef.IsValid) { - List l; - if (!d.TryGetValue(modName, out l)) { + if (!d.TryGetValue(modName, out var l)) { d[modName] = l = new List(); } if (moduleRef.HasModule) { @@ -381,7 +378,7 @@ public IMemberResult[] GetModules() { private static IMemberResult[] ModuleDictToMemberResult(Dictionary> d) { var result = new IMemberResult[d.Count]; - int pos = 0; + var pos = 0; foreach (var kvp in d) { var lazyEnumerator = new LazyModuleEnumerator(kvp.Value); result[pos++] = new MemberResult( @@ -437,7 +434,7 @@ public IEnumerable FindNameInAllModules(string name) { if (Interpreter is ICanFindModuleMembers finder) { foreach (var modName in finder.GetModulesNamed(name)) { - int dot = modName.LastIndexOf('.'); + var dot = modName.LastIndexOf('.'); if (dot < 0) { yield return new ExportedMemberInfo(null, modName); } else { @@ -500,7 +497,7 @@ public IEnumerable FindNameInAllModules(string name) { } private static bool GetPackageNameIfMatch(string name, string fullName, out string packageName) { - int lastDot = fullName.LastIndexOf('.'); + var lastDot = fullName.LastIndexOf('.'); if (lastDot < 0) { packageName = null; return false; @@ -529,10 +526,8 @@ private static bool GetPackageNameIfMatch(string name, string fullName, out stri /// /// public IMemberResult[] GetModuleMembers(IModuleContext moduleContext, string[] names, bool includeMembers = false) { - ModuleReference moduleRef; - if (Modules.TryImport(names[0], out moduleRef)) { - var module = moduleRef.Module as IModule; - if (module != null) { + if (Modules.TryImport(names[0], out var moduleRef)) { + if (moduleRef.Module is IModule module) { return GetModuleMembers(moduleContext, names, includeMembers, module); } @@ -542,7 +537,7 @@ public IMemberResult[] GetModuleMembers(IModuleContext moduleContext, string[] n } internal static IMemberResult[] GetModuleMembers(IModuleContext moduleContext, string[] names, bool includeMembers, IModule module) { - for (int i = 1; i < names.Length && module != null; i++) { + for (var i = 1; i < names.Length && module != null; i++) { module = module.GetChildPackage(moduleContext, names[i]); } @@ -564,7 +559,7 @@ internal static IMemberResult[] GetModuleMembers(IModuleContext moduleContext, s results.Add(child.Value); } foreach (var keyValue in module.GetAllMembers(moduleContext)) { - bool anyModules = false; + var anyModules = false; foreach (var ns in keyValue.Value.OfType()) { if (ns.Members.OfType().Any(mod => !(mod is MultipleMemberInfo))) { anyModules = true; @@ -584,10 +579,8 @@ internal static IMemberResult[] GetModuleMembers(IModuleContext moduleContext, s return new IMemberResult[0]; } - private static IMemberResult[] MemberDictToMemberResult(Dictionary> results) { - return results.Select(r => new MemberResult(r.Key, r.Value.SelectMany()) as IMemberResult).ToArray(); - } - + private static IMemberResult[] MemberDictToMemberResult(Dictionary> results) + => results.Select(r => new MemberResult(r.Key, r.Value.SelectMany()) as IMemberResult).ToArray(); /// /// Gets the list of directories which should be analyzed. @@ -602,7 +595,7 @@ private static IMemberResult[] MemberDictToMemberResult(Dictionary TypeStubDirectories => _typeStubPaths.AsLockedEnumerable().ToArray(); public AnalysisLimits Limits { - get { return _limits; } + get => _limits; set { value = value ?? AnalysisLimits.GetDefaultLimits(); var limits = _limits; @@ -707,8 +700,7 @@ internal IKnownClasses ClassInfos { /// Function to create the value. /// The cached value or null. internal AnalysisValue GetCached(object key, Func maker) { - AnalysisValue result; - if (!_itemCache.TryGetValue(key, out result)) { + if (!_itemCache.TryGetValue(key, out var result)) { // Set the key to prevent recursion _itemCache[key] = null; _itemCache[key] = result = maker(); @@ -718,15 +710,12 @@ internal AnalysisValue GetCached(object key, Func maker) { internal BuiltinModule BuiltinModule => _builtinModule; - internal BuiltinInstanceInfo GetInstance(IPythonType type) { - return GetBuiltinType(type).Instance; - } + internal BuiltinInstanceInfo GetInstance(IPythonType type) => GetBuiltinType(type).Instance; - internal BuiltinClassInfo GetBuiltinType(IPythonType type) { - return (BuiltinClassInfo)GetCached(type, + internal BuiltinClassInfo GetBuiltinType(IPythonType type) => + (BuiltinClassInfo)GetCached(type, () => MakeBuiltinType(type) ) ?? ClassInfos[BuiltinTypeId.Object]; - } private BuiltinClassInfo MakeBuiltinType(IPythonType type) { switch (type.TypeId) { @@ -739,8 +728,7 @@ private BuiltinClassInfo MakeBuiltinType(IPythonType type) { } internal IAnalysisSet GetAnalysisSetFromObjects(object objects) { - var typeList = objects as IEnumerable; - if (typeList == null) { + if (!(objects is IEnumerable typeList)) { return AnalysisSet.Empty; } return AnalysisSet.UnionAll(typeList.Select(GetAnalysisValueFromObjects)); @@ -814,8 +802,7 @@ internal IDictionary GetAllMembers(IMemberContainer contai var children = (container as IModule)?.GetChildrenPackages(moduleContext); if (children?.Any() ?? false) { foreach (var child in children) { - IAnalysisSet existing; - if (result.TryGetValue(child.Key, out existing)) { + if (result.TryGetValue(child.Key, out var existing)) { result[child.Key] = existing.Add(child.Value); } else { result[child.Key] = child.Value; @@ -831,8 +818,7 @@ internal IDictionary GetAllMembers(IMemberContainer contai internal ConcurrentDictionary ModulesByFilename { get; } public bool TryGetProjectEntryByPath(string path, out IProjectEntry projEntry) { - ModuleInfo modInfo; - if (ModulesByFilename.TryGetValue(path, out modInfo)) { + if (ModulesByFilename.TryGetValue(path, out var modInfo)) { projEntry = modInfo.ProjectEntry; return true; } @@ -842,7 +828,7 @@ public bool TryGetProjectEntryByPath(string path, out IProjectEntry projEntry) { } internal IAnalysisSet GetConstant(object value) { - object key = value ?? _nullKey; + var key = value ?? _nullKey; return GetCached(key, () => { var constant = value as IPythonConstant; var constantType = constant?.Type; @@ -1012,14 +998,13 @@ internal AggregateProjectEntry GetAggregate(params IProjectEntry[] aggregating) return GetAggregateWorker(aggregating); } - private static void SortAggregates(IProjectEntry[] aggregating) { - Array.Sort(aggregating, (x, y) => x.GetHashCode() - y.GetHashCode()); - } + private static void SortAggregates(IProjectEntry[] aggregating) + => Array.Sort(aggregating, (x, y) => x.GetHashCode() - y.GetHashCode()); internal AggregateProjectEntry GetAggregate(HashSet from, IProjectEntry with) { Debug.Assert(!from.Contains(with)); - IProjectEntry[] all = new IProjectEntry[from.Count + 1]; + var all = new IProjectEntry[from.Count + 1]; from.CopyTo(all); all[from.Count] = with; @@ -1036,13 +1021,11 @@ internal void ClearAggregate(AggregateProjectEntry entry) { } private AggregateProjectEntry GetAggregateWorker(IProjectEntry[] all) { - AggregateProjectEntry agg; - if (!_aggregates.TryGetValue(all, out agg)) { + if (!_aggregates.TryGetValue(all, out var agg)) { _aggregates[all] = agg = new AggregateProjectEntry(new HashSet(all)); foreach (var proj in all) { - IAggregateableProjectEntry aggretable = proj as IAggregateableProjectEntry; - if (aggretable != null) { + if (proj is IAggregateableProjectEntry aggretable) { aggretable.AggregatedInto(agg); } } @@ -1058,7 +1041,7 @@ public bool Equals(IProjectEntry[] x, IProjectEntry[] y) { if (x.Length != y.Length) { return false; } - for (int i = 0; i < x.Length; i++) { + for (var i = 0; i < x.Length; i++) { if (x[i] != y[i]) { return false; } @@ -1067,8 +1050,8 @@ public bool Equals(IProjectEntry[] x, IProjectEntry[] y) { } public int GetHashCode(IProjectEntry[] obj) { - int res = 0; - for (int i = 0; i < obj.Length; i++) { + var res = 0; + for (var i = 0; i < obj.Length; i++) { res ^= obj[i].GetHashCode(); } return res; diff --git a/src/Analysis/Engine/Test/AstAnalysisTests.cs b/src/Analysis/Engine/Test/AstAnalysisTests.cs index 81a3b268c..fb64f8443 100644 --- a/src/Analysis/Engine/Test/AstAnalysisTests.cs +++ b/src/Analysis/Engine/Test/AstAnalysisTests.cs @@ -9,7 +9,7 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. @@ -212,6 +212,50 @@ public async Task AstMultiValues() { } } + [TestMethod, Priority(0)] + public async Task AstForwardRefProperty1() { + using (var server = await CreateServerAsync()) { + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(@" +from ForwardRefProp1 import * +x = B().getA().methodA() +"); + analysis.Should().HaveVariable("x").OfTypes(BuiltinTypeId.Str); + } + } + + [TestMethod, Priority(0)] + public async Task AstForwardRefGlobalFunction() { + using (var server = await CreateServerAsync()) { + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(@" +from ForwardRefGlobalFunc import * +x = func1() +"); + analysis.Should().HaveVariable("x").OfTypes(BuiltinTypeId.Str); + } + } + + [TestMethod, Priority(0)] + public async Task AstForwardRefFunction1() { + using (var server = await CreateServerAsync()) { + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(@" +from ForwardRefFunc1 import * +x = B().getA().methodA() +"); + analysis.Should().HaveVariable("x").OfTypes(BuiltinTypeId.Str); + } + } + + [TestMethod, Priority(0)] + public async Task AstForwardRefFunction2() { + using (var server = await CreateServerAsync()) { + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(@" +from ForwardRefFunc2 import * +x = B().getA().methodA() +"); + analysis.Should().HaveVariable("x").OfTypes(BuiltinTypeId.Str); + } + } + [TestMethod, Priority(0)] public void AstImports() { var mod = Parse("Imports.py", PythonLanguageVersion.V35); @@ -777,7 +821,7 @@ private async Task FullStdLibTest(InterpreterConfiguration configuration, params foreach (var r in modules .Where(m => !skip.Contains(m.ModuleName)) .GroupBy(m => { - int i = m.FullName.IndexOf('.'); + var i = m.FullName.IndexOf('.'); return i <= 0 ? m.FullName : m.FullName.Remove(i); }) .AsParallel() @@ -934,13 +978,20 @@ public async Task TypeShedJsonMakeScanner() { scanner = _json.make_scanner()"; var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); - analysis.Should().HaveVariable("scanner").WithValue() - .Which.Should().HaveSingleOverload() + var v0 = analysis.Should().HaveVariable("scanner").WithValueAt(0); + var v1 = analysis.Should().HaveVariable("scanner").WithValueAt(1); + + v0.Which.Should().HaveSingleOverload() .Which.Should().HaveName("__call__") .And.HaveParameters("string", "index") .And.HaveParameterAt(0).WithName("string").WithType("str").WithNoDefaultValue() .And.HaveParameterAt(1).WithName("index").WithType("int").WithNoDefaultValue() .And.HaveSingleReturnType("tuple[None, int]"); + + v1.Which.Should().HaveSingleOverload() + .Which.Should().HaveName("__call__") + .And.HaveParameters("*args", "**kwargs") + .And.HaveSingleReturnType("type"); } } diff --git a/src/Analysis/Engine/Test/FluentAssertions/VariableDefAssertions.cs b/src/Analysis/Engine/Test/FluentAssertions/VariableDefAssertions.cs index 9965e9d8c..a7fbeb637 100644 --- a/src/Analysis/Engine/Test/FluentAssertions/VariableDefAssertions.cs +++ b/src/Analysis/Engine/Test/FluentAssertions/VariableDefAssertions.cs @@ -211,7 +211,19 @@ public AndWhichConstraint> var testInfo = new AnalysisValueTestInfo((TValue)value, GetScopeDescription(), _scope); return new AndWhichConstraint>(this, testInfo); } - + + public AndWhichConstraint> HaveValueAt(int index, string because = "", params object[] reasonArgs) + where TValue : IAnalysisValue { + var value = FlattenAnalysisValues(Subject.Types).ToArray()[index]; + + Execute.Assertion.ForCondition(value is TValue) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected {_moduleName}.{_name} to have value of type {typeof(TValue)}{{reason}}, but its value has type {value.GetType()}."); + + var testInfo = new AnalysisValueTestInfo((TValue)value, GetScopeDescription(), _scope); + return new AndWhichConstraint>(this, testInfo); + } + private IAnalysisValue AssertSingle(string because, object[] reasonArgs, IAnalysisValue[] values) { Execute.Assertion.ForCondition(values.Length == 1) .BecauseOf(because, reasonArgs) @@ -222,6 +234,16 @@ private IAnalysisValue AssertSingle(string because, object[] reasonArgs, IAnalys return values[0]; } + private IAnalysisValue[] AssertCount(int count, string because, object[] reasonArgs, IAnalysisValue[] values) { + Execute.Assertion.ForCondition(values.Length != count) + .BecauseOf(because, reasonArgs) + .FailWith(values.Length == count + ? $"Expected variable '{_moduleName}.{_name}' to have {count} values{{reason}}, but found none." + : $"Expected variable '{_moduleName}.{_name}' to have {count} values{{reason}}, but found {values.Length}: {GetQuotedNames(values)}"); + + return values; + } + private string GetScopeDescription() { switch (_scope) { case IFunctionScope functionScope: diff --git a/src/Analysis/Engine/Test/FluentAssertions/VariableDefAssertionsExtensions.cs b/src/Analysis/Engine/Test/FluentAssertions/VariableDefAssertionsExtensions.cs index fc5504561..881a0bf3a 100644 --- a/src/Analysis/Engine/Test/FluentAssertions/VariableDefAssertionsExtensions.cs +++ b/src/Analysis/Engine/Test/FluentAssertions/VariableDefAssertionsExtensions.cs @@ -122,11 +122,23 @@ public static AndWhichConstraint> return new AndWhichConstraint>(andWhichConstraint.And, testInfo); } + public static AndWhichConstraint> + WithValueAt(this AndWhichConstraint andWhichConstraint, int index, string because = "", params object[] reasonArgs) + where TValue : IAnalysisValue { + var testInfo = andWhichConstraint.Which.Should().HaveValueAt(index, because, reasonArgs).Which; + return new AndWhichConstraint>(andWhichConstraint.And, testInfo); + } + public static AndWhichConstraint> WithValue(this AndWhichConstraint constraint, string because = "", params object[] reasonArgs) where TValue : IAnalysisValue => constraint.WithValue(because, reasonArgs); + public static AndWhichConstraint> + WithValueAt(this AndWhichConstraint constraint, int index, string because = "", params object[] reasonArgs) + where TValue : IAnalysisValue + => constraint.WithValueAt(index, because, reasonArgs); + public static AndWhichConstraint> WithValue(this AndWhichConstraint constraint, string because = "", params object[] reasonArgs) where TValue : IAnalysisValue diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 27ab814f4..2021df3c6 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -9,7 +9,7 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. diff --git a/src/UnitTests/TestData/AstAnalysis/ForwardRefFunc1.py b/src/UnitTests/TestData/AstAnalysis/ForwardRefFunc1.py new file mode 100644 index 000000000..8a3c55b08 --- /dev/null +++ b/src/UnitTests/TestData/AstAnalysis/ForwardRefFunc1.py @@ -0,0 +1,10 @@ +class A(object): + def methodA(self): + return 's' + +class B(object): + def getA(self): + return self.funcA() + + def funcA(self): + return A() diff --git a/src/UnitTests/TestData/AstAnalysis/ForwardRefFunc2.py b/src/UnitTests/TestData/AstAnalysis/ForwardRefFunc2.py new file mode 100644 index 000000000..a149c8c15 --- /dev/null +++ b/src/UnitTests/TestData/AstAnalysis/ForwardRefFunc2.py @@ -0,0 +1,16 @@ +class A(object): + def methodA(self): + return 's' + +class B(object): + def getA(self): + return self.func1() + + def func1(self): + return self.func2() + + def func2(self): + return self.func3() + + def func3(self): + return A() diff --git a/src/UnitTests/TestData/AstAnalysis/ForwardRefGlobalFunc.py b/src/UnitTests/TestData/AstAnalysis/ForwardRefGlobalFunc.py new file mode 100644 index 000000000..0eef85ba8 --- /dev/null +++ b/src/UnitTests/TestData/AstAnalysis/ForwardRefGlobalFunc.py @@ -0,0 +1,8 @@ +def func1(): + return func2() + +def func2(): + return func3() + +def func3(): + return 's' diff --git a/src/UnitTests/TestData/AstAnalysis/ForwardRefProp1.py b/src/UnitTests/TestData/AstAnalysis/ForwardRefProp1.py new file mode 100644 index 000000000..47e2051c9 --- /dev/null +++ b/src/UnitTests/TestData/AstAnalysis/ForwardRefProp1.py @@ -0,0 +1,11 @@ +class A(object): + def methodA(self): + return 's' + +class B(object): + def getA(self): + return self.propA + + @property + def propA(self): + return A()