Fix processing of forward references

src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs

@@ -9,20 +9,21 @@
 // THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
 // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
 // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
-// MERCHANTABLITY OR NON-INFRINGEMENT.
+// MERCHANTABILITY OR NON-INFRINGEMENT.
 //
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using Microsoft.PythonTools.Analysis.Infrastructure;
 using Microsoft.PythonTools.Parsing.Ast;
 
 namespace Microsoft.PythonTools.Interpreter.Ast {
+    [DebuggerDisplay("{Target.Name}")]
     class AstAnalysisFunctionWalker : PythonWalker {
-        private readonly FunctionDefinition _target;
         private readonly NameLookupContext _scope;
         private readonly AstPythonFunctionOverload _overload;
         private AstPythonType _selfType;
@@ -33,12 +34,11 @@ public AstAnalysisFunctionWalker(
             AstPythonFunctionOverload overload
         ) {
             _scope = scope ?? throw new ArgumentNullException(nameof(scope));
-            _target = targetFunction ?? throw new ArgumentNullException(nameof(targetFunction));
+            Target = targetFunction ?? throw new ArgumentNullException(nameof(targetFunction));
             _overload = overload ?? throw new ArgumentNullException(nameof(overload));
         }
 
-        public IList<IPythonType> ReturnTypes => _overload.ReturnTypes;
-        public IPythonFunctionOverload Overload => _overload;
+        public FunctionDefinition Target { get; } 
 
         private void GetMethodType(FunctionDefinition node, out bool classmethod, out bool staticmethod) {
             classmethod = false;
@@ -51,7 +51,7 @@ private void GetMethodType(FunctionDefinition node, out bool classmethod, out bo
 
             var classmethodObj = _scope.Interpreter.GetBuiltinType(BuiltinTypeId.ClassMethod);
             var staticmethodObj = _scope.Interpreter.GetBuiltinType(BuiltinTypeId.StaticMethod);
-            foreach (var d in (_target.Decorators?.Decorators).MaybeEnumerate().ExcludeDefault()) {
+            foreach (var d in (Target.Decorators?.Decorators).MaybeEnumerate().ExcludeDefault()) {
                 var m = _scope.GetValueFromExpression(d);
                 if (m == classmethodObj) {
                     classmethod = true;
@@ -65,21 +65,31 @@ public void Walk() {
             var self = GetSelf();
             _selfType = (self as AstPythonConstant)?.Type as AstPythonType;
 
-            _overload.ReturnTypes.AddRange(_scope.GetTypesFromAnnotation(_target.ReturnAnnotation).ExcludeDefault());
-
+            _overload.ReturnTypes.AddRange(_scope.GetTypesFromAnnotation(Target.ReturnAnnotation).ExcludeDefault());
             _scope.PushScope();
+
+            // Declare self, if any
+            var skip = 0;
             if (self != null) {
-                var p0 = _target.Parameters.FirstOrDefault();
+                var p0 = Target.Parameters.FirstOrDefault();
                 if (p0 != null && !string.IsNullOrEmpty(p0.Name)) {
                     _scope.SetInScope(p0.Name, self);
+                    skip++;
                 }
             }
-            _target.Walk(this);
+
+            // Declare parameters in scope
+            foreach(var p in Target.Parameters.Skip(skip).Where(p => !string.IsNullOrEmpty(p.Name))) {
+                 var value = _scope.GetValueFromExpression(p.DefaultValue);
+                _scope.SetInScope(p.Name, value ?? _scope.UnknownType);
+            }
+
+            Target.Walk(this);
             _scope.PopScope();
         }
 
         public override bool Walk(FunctionDefinition node) {
-            if (node != _target) {
+            if (node != Target) {
                 // Do not walk nested functions (yet)
                 return false;
             }
@@ -163,19 +173,28 @@ public override bool Walk(IfStatement node) {
         }
 
         public override bool Walk(ReturnStatement node) {
-            foreach (var type in _scope.GetTypesFromValue(_scope.GetValueFromExpression(node.Expression)).ExcludeDefault()) {
+            var types = _scope.GetTypesFromValue(_scope.GetValueFromExpression(node.Expression)).ExcludeDefault();
+            foreach (var type in types) {
                 _overload.ReturnTypes.Add(type);
             }
+
+            // Clean up: if there are None or Unknown types along with real ones, remove them.
+            var realTypes = _overload.ReturnTypes
+                .Where(t => t.TypeId != BuiltinTypeId.Unknown && t.TypeId != BuiltinTypeId.NoneType)
+                .ToList();
+
+            if (realTypes.Count > 0) {
+                _overload.ReturnTypes.Clear();
+                _overload.ReturnTypes.AddRange(realTypes);
+            }
             return true; // We want to evaluate all code so all private variables in __new__ get defined
         }
 
         private IMember GetSelf() {
-            bool classmethod, staticmethod;
-            GetMethodType(_target, out classmethod, out staticmethod);
+            GetMethodType(Target, out var classmethod, out var staticmethod);
             var self = _scope.LookupNameInScopes("__class__", NameLookupContext.LookupOptions.Local);
             if (!staticmethod && !classmethod) {
-                var cls = self as IPythonType;
-                if (cls == null) {
+                if (!(self is IPythonType cls)) {
                     self = null;
                 } else {
                     self = new AstPythonConstant(cls, ((cls as ILocatedMember)?.Locations).MaybeEnumerate().ToArray());

src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs

@@ -0,0 +1,68 @@
+// Python Tools for Visual Studio
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.PythonTools.Analysis.Infrastructure;
+using Microsoft.PythonTools.Parsing.Ast;
+
+namespace Microsoft.PythonTools.Interpreter.Ast {
+    /// <summary>
+    /// Represents set of function body walkers. Functions are walked after
+    /// all classes are collected. If function or property return type is unknown,
+    /// it can be walked, and so on recursively, until return type is determined
+    /// or there is nothing left to walk.
+    /// </summary>
+    class AstAnalysisFunctionWalkerSet {
+        private readonly Dictionary<FunctionDefinition, AstAnalysisFunctionWalker> _functionWalkers
+            = new Dictionary<FunctionDefinition, AstAnalysisFunctionWalker>();
+
+        public void Add(AstAnalysisFunctionWalker walker)
+            => _functionWalkers[walker.Target] = walker;
+
+        public void ProcessSet() {
+            // Do not use foreach since walker list is dynamically modified and walkers are removed
+            // after processing. Handle __init__ and __new__ first so class variables are initialized.
+            var constructors = _functionWalkers
+                .Where(kvp => kvp.Key.Name == "__init__" || kvp.Key.Name == "__new__")
+                .Select(c => c.Value)
+                .ExcludeDefault()
+                .ToArray();
+
+            foreach (var ctor in constructors) {
+                ProcessWalker(ctor);
+            }
+
+            while (_functionWalkers.Count > 0) {
+                var walker = _functionWalkers.First().Value;
+                ProcessWalker(walker);
+            }
+        }
+
+        public void ProcessFunction(FunctionDefinition fn) {
+            if (_functionWalkers.TryGetValue(fn, out var w)) {
+                ProcessWalker(w);
+            }
+        }
+
+        private void ProcessWalker(AstAnalysisFunctionWalker walker) {
+            // Remove walker before processing as to prevent reentrancy.
+            _functionWalkers.Remove(walker.Target);
+            var z = walker.Target.Name == "day";
+            walker.Walk();
+        }
+    }
+}