Drop AST for libraries after final analysis (microsoft#1134)

* Remove old qualified name

* Node storage

* Class and scope to use AST map

* Library analysis

* Fix SO

* Keep small AST with imports

* AST reduction

* Final field

* Reload

* Ignore post-final requests

* Drop AST

* Remove local variables

* Test fixes

* Fix overload match

* Tests

* Add locks

* Remove local variables

* Drop file content to save memory

* Cache PEP hints

* Recreate AST

* Fix specialization

* Fix locations

* usings

* Test fixes

* Add options to keep data in memory

* Fix test

* Fix lambda parameters

* Fix argument set
Fix global scope node

* Fix overload doc

* Fix stub merge errors

* Fix async issues

* Undo some changes

* Fix test

* Fix race condition

* Restore log null checks

* Fix merge conflict

* Fix merge issue

* Null check

* Fix test

Author: Mikhail Arkhipov

src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs

@@ -69,7 +69,7 @@ public override bool Walk(ExpressionStatement node) {
                     return true;
                 case CallExpression callex when callex.Target is MemberExpression mex && !string.IsNullOrEmpty(mex.Name):
                     var t = Eval.GetValueFromExpression(mex.Target)?.GetPythonType();
-                    t?.GetMember(mex.Name).AddReference(Eval.GetLocationOfName(mex));
+                    t?.GetMember(mex.Name)?.AddReference(Eval.GetLocationOfName(mex));
                     return true;
                 default:
                     return base.Walk(node);

src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs

@@ -26,7 +26,6 @@ internal interface IAnalyzable {
         /// <summary>
         /// Notifies document that its analysis is now complete.
         /// </summary>
-        /// <param name="analysis">Document analysis</param>
-        void NotifyAnalysisComplete(IDocumentAnalysis analysis);
+        void NotifyAnalysisComplete(int version, ModuleWalker walker, bool isFinalPass);
     }
 }

src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs

@@ -20,15 +20,14 @@
 using Microsoft.Python.Analysis.Diagnostics;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Core.Collections;
-using Microsoft.Python.Parsing.Ast;
 
 namespace Microsoft.Python.Analysis.Analyzer {
     public interface IPythonAnalyzer {
         Task WaitForCompleteAnalysisAsync(CancellationToken cancellationToken = default);
         /// <summary>
         /// Schedules module for analysis. Module will be scheduled if version of AST is greater than the one used to get previous analysis
         /// </summary>
-        void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int version);
+        void EnqueueDocumentForAnalysis(IPythonModule module, int version);
 
         /// <summary>
         /// Schedules module for analysis for its existing AST, but with new dependencies.

src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs

@@ -13,17 +13,11 @@
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
 
-using System;
 using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using Microsoft.Python.Analysis.Analyzer.Evaluation;
 using Microsoft.Python.Analysis.Diagnostics;
 using Microsoft.Python.Analysis.Documents;
 using Microsoft.Python.Analysis.Values;
-using Microsoft.Python.Core;
 using Microsoft.Python.Core.Diagnostics;
-using Microsoft.Python.Parsing;
 using Microsoft.Python.Parsing.Ast;
 
 namespace Microsoft.Python.Analysis.Analyzer {
@@ -77,24 +71,4 @@ public DocumentAnalysis(IDocument document, int version, IGlobalScope globalScop
         public IEnumerable<DiagnosticsEntry> Diagnostics => ExpressionEvaluator.Diagnostics;
         #endregion
     }
-
-    public sealed class EmptyAnalysis : IDocumentAnalysis {
-        private static PythonAst _emptyAst;
-
-        public EmptyAnalysis(IServiceContainer services, IDocument document) {
-            Document = document ?? throw new ArgumentNullException(nameof(document));
-            GlobalScope = new EmptyGlobalScope(document);
-
-            _emptyAst = _emptyAst ?? (_emptyAst = Parser.CreateParser(new StringReader(string.Empty), PythonLanguageVersion.None).ParseFile(document.Uri));
-            ExpressionEvaluator = new ExpressionEval(services, document, Ast);
-        }
-
-        public IDocument Document { get; }
-        public int Version { get; } = -1;
-        public IGlobalScope GlobalScope { get; }
-        public PythonAst Ast => _emptyAst;
-        public IExpressionEvaluator ExpressionEvaluator { get; }
-        public IReadOnlyList<string> StarImportMemberNames => null;
-        public IEnumerable<DiagnosticsEntry> Diagnostics => Enumerable.Empty<DiagnosticsEntry>();
-    }
 }

src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs

@@ -0,0 +1,44 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Python.Analysis.Analyzer.Evaluation;
+using Microsoft.Python.Analysis.Diagnostics;
+using Microsoft.Python.Analysis.Documents;
+using Microsoft.Python.Analysis.Utilities;
+using Microsoft.Python.Analysis.Values;
+using Microsoft.Python.Core;
+using Microsoft.Python.Parsing.Ast;
+
+namespace Microsoft.Python.Analysis.Analyzer {
+    public sealed class EmptyAnalysis : IDocumentAnalysis {
+        public EmptyAnalysis(IServiceContainer services, IDocument document) {
+            Document = document ?? throw new ArgumentNullException(nameof(document));
+            GlobalScope = new EmptyGlobalScope(document);
+            Ast = AstUtilities.MakeEmptyAst(document.Uri);
+            ExpressionEvaluator = new ExpressionEval(services, document);
+        }
+
+        public IDocument Document { get; }
+        public int Version { get; } = -1;
+        public IGlobalScope GlobalScope { get; }
+        public PythonAst Ast { get; }
+        public IExpressionEvaluator ExpressionEvaluator { get; }
+        public IReadOnlyList<string> StarImportMemberNames => null;
+        public IEnumerable<DiagnosticsEntry> Diagnostics => Enumerable.Empty<DiagnosticsEntry>();
+    }
+}

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Annotations.cs

@@ -0,0 +1,48 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Parsing.Ast;
+
+namespace Microsoft.Python.Analysis.Analyzer.Evaluation {
+    internal sealed partial class ExpressionEval {
+        public IPythonType GetTypeFromAnnotation(Expression expr, LookupOptions options = LookupOptions.Global | LookupOptions.Builtins)
+            => GetTypeFromAnnotation(expr, out _, options);
+
+        public IPythonType GetTypeFromAnnotation(Expression expr, out bool isGeneric, LookupOptions options = LookupOptions.Global | LookupOptions.Builtins) {
+            isGeneric = false;
+            switch (expr) {
+                case null:
+                    return null;
+                case CallExpression callExpr:
+                    // x: NamedTuple(...)
+                    return GetValueFromCallable(callExpr)?.GetPythonType() ?? UnknownType;
+                case IndexExpression indexExpr:
+                    // Try generics
+                    var target = GetValueFromExpression(indexExpr.Target);
+                    var result = GetValueFromGeneric(target, indexExpr);
+                    if (result != null) {
+                        isGeneric = true;
+                        return result.GetPythonType();
+                    }
+                    break;
+            }
+
+            // Look at specialization and typing first
+            var ann = new TypeAnnotation(Ast.LanguageVersion, expr);
+            return ann.GetValue(new TypeAnnotationConverter(this, options));
+        }
+    }
+}

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs

@@ -17,6 +17,7 @@
 using System.Diagnostics;
 using System.Linq;
 using Microsoft.Python.Analysis.Documents;
+using Microsoft.Python.Analysis.Extensions;
 using Microsoft.Python.Analysis.Modules;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Values;
@@ -77,9 +78,11 @@ public IMember GetValueFromLambda(LambdaExpression expr) {
                 return null;
             }
 
-            var location = GetLocationOfName(expr.Function);
-            var ft = new PythonFunctionType(expr.Function, null, location);
-            var overload = new PythonFunctionOverload(expr.Function, ft, location);
+            var fd = expr.Function;
+            var location = GetLocationOfName(fd);
+            var ft = new PythonFunctionType(fd, null, location);
+            var overload = new PythonFunctionOverload(ft, fd, location, expr.Function.ReturnAnnotation?.ToCodeString(Ast));
+            overload.SetParameters(CreateFunctionParameters(null, ft, fd, false));
             ft.AddOverload(overload);
             return ft;
         }
@@ -101,13 +104,13 @@ public IMember GetValueFromClassCtor(IPythonClassType cls, CallExpression expr)
             return cls.CreateInstance(cls.Name, args);
         }
 
-        public IMember GetValueFromBound(IPythonBoundType t, CallExpression expr) {
+        private IMember GetValueFromBound(IPythonBoundType t, CallExpression expr) {
             switch (t.Type) {
                 case IPythonFunctionType fn:
                     return GetValueFromFunctionType(fn, t.Self, expr);
                 case IPythonPropertyType p:
                     return GetValueFromProperty(p, t.Self);
-                case IPythonIteratorType it when t.Self is IPythonCollection seq:
+                case IPythonIteratorType _ when t.Self is IPythonCollection seq:
                     return seq.GetIterator();
             }
             return UnknownType;
@@ -173,7 +176,7 @@ public IMember GetValueFromFunctionType(IPythonFunctionType fn, IPythonInstance
             if (instanceType == null || fn.DeclaringType == null || fn.IsSpecialized ||
                 instanceType.IsSpecialized || fn.DeclaringType.IsSpecialized ||
                 instanceType.Equals(fn.DeclaringType) ||
-                fn.IsStub || !string.IsNullOrEmpty(fn.Overloads[args.OverloadIndex].GetReturnDocumentation(null))) {
+                fn.IsStub || !string.IsNullOrEmpty(fn.Overloads[args.OverloadIndex].GetReturnDocumentation())) {
 
                 LoadFunctionDependencyModules(fn);
 
@@ -192,14 +195,14 @@ public IMember GetValueFromFunctionType(IPythonFunctionType fn, IPythonInstance
             //      def func(a, b): return a + b
             // from working in libraries, but this is small sacrifice for significant performance
             // increase in library analysis.
-            if (fn.DeclaringModule is IDocument doc && fd?.Ast == doc.GetAnyAst() && EvaluateFunctionBody(fn)) {
+            if (fn.DeclaringModule is IDocument && EvaluateFunctionBody(fn)) {
                 // Stubs are coming from another module.
                 return TryEvaluateWithArguments(fn, args);
             }
             return UnknownType;
         }
 
-        public IMember GetValueFromProperty(IPythonPropertyType p, IPythonInstance instance) {
+        private IMember GetValueFromProperty(IPythonPropertyType p, IPythonInstance instance) {
             // Function may not have been walked yet. Do it now.
             SymbolTable.Evaluate(p.FunctionDefinition);
             return instance.Call(p.Name, ArgumentSet.Empty);
@@ -311,5 +314,69 @@ private void LoadFunctionDependencyModules(IPythonFunctionType fn) {
                 Services.GetService<IPythonAnalyzer>().EnqueueDocumentForAnalysis(Module, dependencies);
             }
         }
+
+        public IReadOnlyList<IParameterInfo> CreateFunctionParameters(IPythonClassType self, IPythonClassMember function, FunctionDefinition fd, bool declareVariables) {
+            // For class method no need to add extra parameters, but first parameter type should be the class.
+            // For static and unbound methods do not add or set anything.
+            // For regular bound methods add first parameter and set it to the class.
+
+            var parameters = new List<ParameterInfo>();
+            var skip = 0;
+            if (self != null && function.HasClassFirstArgument()) {
+                var p0 = fd.Parameters.FirstOrDefault();
+                if (p0 != null && !string.IsNullOrEmpty(p0.Name)) {
+                    // Actual parameter type will be determined when method is invoked.
+                    // The reason is that if method might be called on a derived class.
+                    // Declare self or cls in this scope.
+                    if (declareVariables) {
+                        DeclareVariable(p0.Name, new PythonInstance(self), VariableSource.Declaration, p0.NameExpression);
+                    }
+                    // Set parameter info.
+                    var pi = new ParameterInfo(Ast, p0, self, null, false);
+                    parameters.Add(pi);
+                    skip++;
+                }
+            }
+
+            // Declare parameters in scope
+            for (var i = skip; i < fd.Parameters.Length; i++) {
+                var p = fd.Parameters[i];
+                if (!string.IsNullOrEmpty(p.Name)) {
+                    var defaultValue = GetValueFromExpression(p.DefaultValue);
+                    var paramType = GetTypeFromAnnotation(p.Annotation, out var isGeneric) ?? UnknownType;
+                    if (paramType.IsUnknown()) {
+                        // If parameter has default value, look for the annotation locally first
+                        // since outer type may be getting redefined. Consider 's = None; def f(s: s = 123): ...
+                        paramType = GetTypeFromAnnotation(p.Annotation, out isGeneric, LookupOptions.Local | LookupOptions.Builtins);
+                        // Default value of None does not mean the parameter is None, just says it can be missing.
+                        defaultValue = defaultValue.IsUnknown() || defaultValue.IsOfType(BuiltinTypeId.NoneType) ? null : defaultValue;
+                        if (paramType == null && defaultValue != null) {
+                            paramType = defaultValue.GetPythonType();
+                        }
+                    }
+                    // If all else fails, look up globally.
+                    var pi = new ParameterInfo(Ast, p, paramType, defaultValue, isGeneric | paramType.IsGeneric());
+                    if (declareVariables) {
+                        DeclareParameter(p, pi);
+                    }
+                    parameters.Add(pi);
+                } else if (p.IsList || p.IsDictionary) {
+                    parameters.Add(new ParameterInfo(Ast, p, null, null, false));
+                }
+            }
+            return parameters;
+        }
+
+        private void DeclareParameter(Parameter p, ParameterInfo pi) {
+            IPythonType paramType;
+            // If type is known from annotation, use it.
+            if (pi != null && !pi.Type.IsUnknown() && !pi.Type.IsGenericParameter()) {
+                // TODO: technically generics may have constraints. Should we consider them?
+                paramType = pi.Type;
+            } else {
+                paramType = pi?.DefaultValue?.GetPythonType() ?? UnknownType;
+            }
+            DeclareVariable(p.Name, new PythonInstance(paramType), VariableSource.Declaration, p.NameExpression);
+        }
     }
 }

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs

@@ -117,33 +117,6 @@ public IMember LookupNameInScopes(string name, out IScope scope, out IVariable v
             return value;
         }
 
-        public IPythonType GetTypeFromAnnotation(Expression expr, LookupOptions options = LookupOptions.Global | LookupOptions.Builtins)
-            => GetTypeFromAnnotation(expr, out _, options);
-
-        public IPythonType GetTypeFromAnnotation(Expression expr, out bool isGeneric, LookupOptions options = LookupOptions.Global | LookupOptions.Builtins) {
-            isGeneric = false;
-            switch (expr) {
-                case null:
-                    return null;
-                case CallExpression callExpr:
-                    // x: NamedTuple(...)
-                    return GetValueFromCallable(callExpr)?.GetPythonType() ?? UnknownType;
-                case IndexExpression indexExpr:
-                    // Try generics
-                    var target = GetValueFromExpression(indexExpr.Target);
-                    var result = GetValueFromGeneric(target, indexExpr);
-                    if (result != null) {
-                        isGeneric = true;
-                        return result.GetPythonType();
-                    }
-                    break;
-            }
-
-            // Look at specialization and typing first
-            var ann = new TypeAnnotation(Ast.LanguageVersion, expr);
-            return ann.GetValue(new TypeAnnotationConverter(this, options));
-        }
-
         /// <summary>
         /// Locates and opens existing scope for a node or creates a new scope
         /// as a child of the specified scope. Scope is pushed on the stack

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs

@@ -20,6 +20,7 @@
 using Microsoft.Python.Analysis.Diagnostics;
 using Microsoft.Python.Analysis.Modules;
 using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Analysis.Utilities;
 using Microsoft.Python.Analysis.Values;
 using Microsoft.Python.Core;
 using Microsoft.Python.Core.Disposables;
@@ -37,12 +38,15 @@ internal sealed partial class ExpressionEval : IExpressionEvaluator {
         private readonly object _lock = new object();
         private readonly List<DiagnosticsEntry> _diagnostics = new List<DiagnosticsEntry>();
 
-        public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAst ast) {
+        public ExpressionEval(IServiceContainer services, IPythonModule module)
+            : this(services, module, new GlobalScope(module)) { }
+
+        public ExpressionEval(IServiceContainer services, IPythonModule module, GlobalScope gs) {
             Services = services ?? throw new ArgumentNullException(nameof(services));
             Module = module ?? throw new ArgumentNullException(nameof(module));
-            Ast = ast ?? throw new ArgumentNullException(nameof(ast));
+            Ast = module.GetAst();
 
-            GlobalScope = new GlobalScope(module);
+            GlobalScope = gs;
             CurrentScope = GlobalScope;
             DefaultLocation = new Location(module);
             //Log = services.GetService<ILogger>();
@@ -59,14 +63,15 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs
         public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(Module) ?? LocationInfo.Empty;
 
         public Location GetLocationOfName(Node node) {
-            if (node == null || (Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library)) {
+            node = node ?? throw new ArgumentNullException(nameof(node));
+            if (Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library) {
                 return DefaultLocation;
             }
 
             IndexSpan indexSpan;
             switch (node) {
                 case MemberExpression mex:
-                    indexSpan = mex.GetNameSpan().ToIndexSpan(mex.Ast);
+                    indexSpan = mex.GetNameSpan(Ast).ToIndexSpan(Ast);
                     break;
                 case ClassDefinition cd:
                     indexSpan = cd.NameExpression.IndexSpan;
@@ -86,7 +91,7 @@ public Location GetLocationOfName(Node node) {
             // turns into span at line 1 and very large column. This DOES can
             // produce false positives occasionally.
 #if DEBUG
-            var sourceSpan = indexSpan.ToSourceSpan(node.Ast);
+            var sourceSpan = indexSpan.ToSourceSpan(Ast);
             Debug.Assert(sourceSpan.Start.Line > 1 || sourceSpan.Start.Column < 1000);
 #endif
             return new Location(Module, indexSpan);