Language server using new analysis (microsoft#546)

* Fix tests

* Baselines

* Null ref

* Build break

* Overrides signatures

* Fix typeinfo tests

* PR feedback

* Infra to separate assembly

* Parser to own assembly

* Parser tests to own assembly

* Unify to IndexSpan

* Better handle overloads

* Reorg, part I

* Part II

* Part III

* Part 4

* Part 5

* Build breaks

* Baselines

* PR fix

* Merge build issues

* Part 6

* Part 7

* Buildable

* PR feedback

* Merge conflict

* Fix microsoft#446

* Fix microsoft#446

* Part 8

* Part 9

* Buildable

* Part 10

* Part 11

* Part 12

* Buildable

* Part 14

* First passing test

* Simplify configuration

* Style

* Fix test and move code to folders

* Builtins import

* Fluents

* Add search path

* Import analysis, part I

* Simplify builtins handling

* Remove IMember

* Handle import specific

* More tests

* Add typeshed

* Renames

* Make sure lazy modules are loaded

* Renames

* Move/rename

* Rework importing

* Derivation rework

* Part 2

* Part 3

* Buildable

* Module members

* Async walk

* Imports test pass

* Remove lazy types

* Fix from import

* Stubs

* Double overloads

* Fix datetime test

* Couple more tests + fluents

* Few more tests

* Additionl test + union type

* Built-in scrape tests

* Full stdlib scrape test

* Complete async AST walker

* Conditional defines test + variable loc cleanup

* More stub tests
Fix stub loading for packages (port from DDG)
Split walker into multiple files

* Add some (broken mostly) tests from DDG

* Move document tests

* Function arg eval, part I

* Instance/factory

* Builds

* Test fixes

* Fix static and instance call eval

* More tests

* More ported tests

* Specialize builtin functions

* Make walkers common and handle nested functions

* Moar tests

* Parser fixes + more tests

* Handle negative numbers

* Fix null ref

* Basic list support

* Few more list tests

* Basic iterators

* Support __iter__

* Iterators

* Fix couple of tests

* Add decorator test

* Generics, part I

* Generics, part 2

* Generics, part 3

* Basic TypeVar test

* Typings, part 4

* Fix test

* Generics, part 6

* Generics, part 7

* More tests (failing)

* Forward ref fixes

* Reorg

* Improve symbol resolution + test fixes

* Test fixes

* Dictionary, part I

* Part 11

* Fix test

* Tests

* Tests

* More dict work

* List ctor

* Skip some tests for now

* Fix iterators

* Tuple slicing

* Polish type comparo in return types

* Add Mapping and tests

* Add Iterable

* Fix typo

* Add Iterator[T] + test

* Simplify typing types

* Class reduction

* Fix tests

* Test fix

* Handle 'with' statement

* Handle try-except

* Class method inheritance + NewType

* Container types

* Containers test

* Tests

* Handle generic type alias

* Named tuple

* Global/non-local

* Handle tuples in for
Handle custom iterators

* Basic generator

* Any/AnyStr

* Test fixes

* Type/Optional/etc handling

* Proper doc population

* Tests + range

* Argument match

* Basic argset and diagnostics

* Argset tests

* Exclude WIP

* Exclude WIP

* Arg eval

* Arg match, part 2

* Tests and generic arg comparisons

* Function eval with arguments

* Baselines

* Fix test

* Undo AST formatting change and update baseline

* LS cleanup 1

* Fix list ctor argument unpacking

* Cleanup 2

* Builds

* Partial completions

* Partial

* Partial

* Simple test

* Tests

* Basic startup

* Clean up a bit

* Remove debug code

* Port formatter tests

* Fix tokenizer crash

* Async fixes

* Hover

* Basic hover

* Adjust expression options

* Basic signature help

* Fix class/instance

* Update test

* Fix builtin creation exception

* Fix tests

* Actually provide declared module

* Completion test (partial)

* Undo

* Fix null await
Fix override completions + test

* Exports filtering
Prevent augmenting imported types

* Filter variables & exports

* Ported tests

* Test fixes

* More ported tests

* Fix exception completions

* Import completions

* Scope completions

* With completions

* Test fixes

* WIP

* Test fix

* Better arg match

* Temp disable WIP

* First cut

* Fix type leak

* WIP

* Remove ConfigureAwait and handle canceled and failed in the analysis notifications

* WIP

* Generic class base

* Generic forward reference resolution

* Suppress completion in strings + test

* Prevent recursion on generic resolution
Better match arguments

* Handle call expression in generics

* Relax condition as it happens in tensorflow

* Fix typeshed version search
Make writing cached modules async
Fix module doc fetching

* Hover tests

* Fix prom import hover

* Hover tests

* Synchronize test cache writing

* First cut

* Test

* Fixes

* Add tests for os.path
Null ref fix

* Fix cache check

* Improve resolution of builtins and typing in stubs

* Merge tests

* Add ntst for requests

* Handle typeshed better

* Fix custom stub handling

* Better sync

* Move files

* Fix parameter locations

* Hover improvement

* PEP hints

* One more test for PEP hints

* Better handle hover over import as

* Text based generic constraints

* Handle with better with generic stubs

* Undo debug

* Handle non-binary open()
Temporary fix 'with' handler since we haven't specialized IO/TextIO/BinaryIO yet.

Author: Mikhail Arkhipov

src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs

@@ -30,7 +30,6 @@ namespace Microsoft.Python.Analysis.Analyzer {
     /// </summary>
     internal abstract class AnalysisWalker : PythonWalkerAsync {
         protected ImportHandler ImportHandler { get; }
-        protected FromImportHandler FromImportHandler { get; }
         protected LoopHandler LoopHandler { get; }
         protected ConditionalHandler ConditionalHandler { get; }
         protected AssignmentHandler AssignmentHandler { get; }
@@ -47,7 +46,6 @@ internal abstract class AnalysisWalker : PythonWalkerAsync {
         protected AnalysisWalker(ExpressionEval eval) {
             Eval = eval;
             ImportHandler = new ImportHandler(this);
-            FromImportHandler = new FromImportHandler(this);
             AssignmentHandler = new AssignmentHandler(this);
             LoopHandler = new LoopHandler(this);
             ConditionalHandler = new ConditionalHandler(this);
@@ -77,7 +75,7 @@ public override async Task<bool> WalkAsync(ForStatement node, CancellationToken
         }
 
         public override Task<bool> WalkAsync(FromImportStatement node, CancellationToken cancellationToken = default)
-            => FromImportHandler.HandleFromImportAsync(node, cancellationToken);
+            => ImportHandler.HandleFromImportAsync(node, cancellationToken);
 
         public override Task<bool> WalkAsync(GlobalStatement node, CancellationToken cancellationToken = default)
             => NonLocalHandler.HandleGlobalAsync(node, cancellationToken);

src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs

@@ -13,6 +13,8 @@
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
 
+using System;
+
 namespace Microsoft.Python.Analysis.Analyzer {
     /// <summary>
     /// Represents document that can be analyzed asynchronously.
@@ -40,8 +42,17 @@ internal interface IAnalyzable {
         /// Notifies document that its analysis is now complete.
         /// </summary>
         /// <param name="analysis">Document analysis</param>
-        /// (version of the snapshot in the beginning of analysis).</param>
         /// <returns>True if analysis was accepted, false if is is out of date.</returns>
         bool NotifyAnalysisComplete(IDocumentAnalysis analysis);
+
+        /// <summary>
+        /// Notifies module that analysis has been canceled.
+        /// </summary>
+        void NotifyAnalysisCanceled();
+
+        /// <summary>
+        /// Notifies module that analysis has thrown an exception.
+        /// </summary>
+        void NotifyAnalysisFailed(Exception ex);
     }
 }

src/Analysis/Ast/Impl/Analyzer/Definitions/IExpressionEvaluator.cs

@@ -0,0 +1,70 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Analysis.Values;
+using Microsoft.Python.Core;
+using Microsoft.Python.Parsing.Ast;
+
+namespace Microsoft.Python.Analysis.Analyzer {
+    public interface IExpressionEvaluator {
+        /// <summary>
+        /// Opens existing scope for a node. The scope is pushed
+        /// on the stack and will be removed when the returned
+        /// disposable is disposed.
+        /// </summary>
+        IDisposable OpenScope(IScope scope);
+
+        /// <summary>
+        /// Opens existing scope for a node. The scope is pushed
+        /// on the stack and will be removed when the returned
+        /// disposable is disposed.
+        /// </summary>
+        IDisposable OpenScope(IPythonModule module, ScopeStatement scope);
+
+        /// <summary>
+        /// Currently opened (deep-most) scope.
+        /// </summary>
+        IScope CurrentScope { get; }
+
+        /// <summary>
+        /// Module global scope.
+        /// </summary>
+        IGlobalScope GlobalScope { get; }
+
+        /// <summary>
+        /// Determines node location in the module source code.
+        /// </summary>
+        LocationInfo GetLocation(Node node);
+
+        /// <summary>
+        /// Evaluates expression in the currently open scope.
+        /// </summary>
+        Task<IMember> GetValueFromExpressionAsync(Expression expr, CancellationToken cancellationToken = default);
+
+        IMember LookupNameInScopes(string name, out IScope scope);
+
+        IPythonType GetTypeFromPepHint(Node node);
+        IPythonType GetTypeFromString(string typeString);
+
+        PythonAst Ast { get; }
+        IPythonModule Module { get; }
+        IPythonInterpreter Interpreter { get; }
+        IServiceContainer Services { get; }
+    }
+}

src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs

@@ -13,28 +13,31 @@
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
 
+using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
+using Microsoft.Python.Analysis.Analyzer.Evaluation;
+using Microsoft.Python.Analysis.Analyzer.Expressions;
 using Microsoft.Python.Analysis.Diagnostics;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Documents;
 using Microsoft.Python.Analysis.Values;
 using Microsoft.Python.Core;
 using Microsoft.Python.Core.Diagnostics;
 using Microsoft.Python.Core.Text;
+using Microsoft.Python.Parsing;
 using Microsoft.Python.Parsing.Ast;
 
 namespace Microsoft.Python.Analysis.Analyzer {
     internal sealed class DocumentAnalysis : IDocumentAnalysis {
-        public static readonly IDocumentAnalysis Empty = new EmptyAnalysis();
-
-        public DocumentAnalysis(IDocument document, int version, IGlobalScope globalScope, PythonAst ast) {
+        public DocumentAnalysis(IDocument document, int version, IGlobalScope globalScope, IExpressionEvaluator eval) {
             Check.ArgumentNotNull(nameof(document), document);
             Check.ArgumentNotNull(nameof(globalScope), globalScope);
             Document = document;
             Version = version;
             GlobalScope = globalScope;
-            Ast = ast;
+            ExpressionEvaluator = eval;
         }
 
         #region IDocumentAnalysis
@@ -53,47 +56,37 @@ public DocumentAnalysis(IDocument document, int version, IGlobalScope globalScop
         /// <summary>
         /// AST that was used in the analysis.
         /// </summary>
-        public PythonAst Ast { get; }
+        public PythonAst Ast => ExpressionEvaluator.Ast;
 
         /// <summary>
         /// Document/module global scope.
         /// </summary>
         public IGlobalScope GlobalScope { get; }
 
         /// <summary>
-        /// Module top-level members
-        /// </summary>
-        public IVariableCollection TopLevelVariables => GlobalScope.Variables;
-
-        /// <summary>
-        /// All module members from all scopes.
+        /// Expression evaluator used in the analysis.
         /// </summary>
-        public IEnumerable<IVariable> AllVariables
-            => (GlobalScope as IScope).TraverseBreadthFirst(s => s.Children).SelectMany(s => s.Variables);
-
-        public IEnumerable<IPythonType> GetAllAvailableItems(SourceLocation location) => Enumerable.Empty<IPythonType>();
-        public IEnumerable<IPythonType> GetMembers(SourceLocation location) => Enumerable.Empty<IPythonType>();
-        public IEnumerable<IPythonFunctionOverload> GetSignatures(SourceLocation location) => Enumerable.Empty<IPythonFunctionOverload>();
-        public IEnumerable<IPythonType> GetValues(SourceLocation location) => Enumerable.Empty<IPythonType>();
+        public IExpressionEvaluator ExpressionEvaluator { get; }
         #endregion
+    }
 
-        private sealed class EmptyAnalysis : IDocumentAnalysis {
-            public EmptyAnalysis(IDocument document = null) {
-                Document = document;
-                GlobalScope = new EmptyGlobalScope(document);
-            }
+    public sealed class EmptyAnalysis : IDocumentAnalysis {
+        private static PythonAst _emptyAst;
 
-            public IDocument Document { get; }
-            public int Version { get; } = -1;
-            public IGlobalScope GlobalScope { get; }
-            public PythonAst Ast => null;
-            public IEnumerable<IPythonType> GetAllAvailableItems(SourceLocation location) => Enumerable.Empty<IPythonType>();
-            public IEnumerable<DiagnosticsEntry> Diagnostics => Enumerable.Empty<DiagnosticsEntry>();
-            public IVariableCollection TopLevelVariables => VariableCollection.Empty;
-            public IEnumerable<IVariable> AllVariables => Enumerable.Empty<IVariable>();
-            public IEnumerable<IPythonType> GetMembers(SourceLocation location) => Enumerable.Empty<IPythonType>();
-            public IEnumerable<IPythonFunctionOverload> GetSignatures(SourceLocation location) => Enumerable.Empty<IPythonFunctionOverload>();
-            public IEnumerable<IPythonType> GetValues(SourceLocation location) => Enumerable.Empty<IPythonType>();
+        public EmptyAnalysis(IServiceContainer services, IDocument document) {
+            Document = document ?? throw new ArgumentNullException(nameof(document));
+            GlobalScope = new EmptyGlobalScope(document);
+
+            _emptyAst = _emptyAst ?? (_emptyAst = Parser.CreateParser(new StringReader(string.Empty), PythonLanguageVersion.None).ParseFile());
+            ExpressionEvaluator = new ExpressionEval(services, document, Ast);
         }
+
+        public IDocument Document { get; }
+        public int Version { get; } = -1;
+        public IGlobalScope GlobalScope { get; }
+        public PythonAst Ast => _emptyAst;
+        public IExpressionEvaluator ExpressionEvaluator { get; }
+        public IEnumerable<DiagnosticsEntry> Diagnostics => Enumerable.Empty<DiagnosticsEntry>();
     }
+
 }