AST analyzer and assemblies split (microsoft#465)

* Types hierarchy cleanup

* Test fixes, part I

* Restore IPythonBoundFunction

* Fix function instance types

* More fixes

* Fix overload handling

* Fix properties

* Fix type factory

* Remove class built-in type

* Simplify builtins, remove obsolete method

* Fix typeshed merge

* Allow declaring module to be null

* Correct assertion

* Simplify functions/methods

* Fix stub merge tests

* Fix tests

* Baselines

* Null ref

* Build break

* Overrides signatures

* Fix typeinfo tests

* PR feedback

* Infra to separate assembly

* Parser to own assembly

* Parser tests to own assembly

* Unify to IndexSpan

* Better handle overloads

* Reorg, part I

* Part II

* Part III

* Part 4

* Part 5

* Build breaks

* Baselines

* PR fix

* Merge build issues

* Part 6

* Part 7

* Buildable

* PR feedback

* Merge conflict

* Fix microsoft#446

* Fix microsoft#446

* Part 8

* Part 9

* Buildable

* Part 10

* Part 11

* Part 12

* Buildable

* Part 14

* First passing test

* Simplify configuration

* Style

* Fix test and move code to folders

* Builtins import

* Fluents

* Add search path

* Import analysis, part I

* Simplify builtins handling

* Remove IMember

* Handle import specific

* More tests

* Add typeshed

* Renames

* Make sure lazy modules are loaded

* Renames

* Move/rename

* Rework importing

* Derivation rework

* Part 2

* Part 3

* Buildable

* Module members

* Async walk

* Imports test pass

* Remove lazy types

* Fix from import

* Stubs

* Double overloads

* Fix datetime test

* Couple more tests + fluents

* Few more tests

* Additionl test + union type

* Built-in scrape tests

* Full stdlib scrape test

* Complete async AST walker

* Conditional defines test + variable loc cleanup

* More stub tests
Fix stub loading for packages (port from DDG)
Split walker into multiple files

* Add some (broken mostly) tests from DDG

* Move document tests

* Function arg eval, part I

* Instance/factory

* Builds

* Test fixes

* Fix static and instance call eval

* More tests

* More ported tests

* Specialize builtin functions

* Make walkers common and handle nested functions

* Moar tests

* Parser fixes + more tests

* Handle negative numbers

* Fix null ref

* Basic list support

* Few more list tests

* Basic iterators

* Support __iter__

* Iterators

* Fix couple of tests

* Add decorator test

* Generics, part I

* Generics, part 2

* Generics, part 3

* Basic TypeVar test

* Typings, part 4

* Fix test

* Generics, part 6

* Generics, part 7

* More tests (failing)

* Forward ref fixes

* Reorg

* Improve symbol resolution + test fixes

* Test fixes

* Dictionary, part I

* Part 11

* Fix test

* Tests

* Tests

* More dict work

* List ctor

* Skip some tests for now

* Fix iterators

* Tuple slicing

* Polish type comparo in return types

* Add Mapping and tests

* Add Iterable

* Fix typo

* Add Iterator[T] + test

* Simplify typing types

* Class reduction

* Fix tests

* Test fix

* Handle 'with' statement

* Handle try-except

* Class method inheritance + NewType

* Container types

* Containers test

* Tests

* Handle generic type alias

* Named tuple

* Global/non-local

* Handle tuples in for
Handle custom iterators

* Basic generator

* Any/AnyStr

* Test fixes

* Type/Optional/etc handling

* Proper doc population

* Tests + range

* Argument match

* Basic argset and diagnostics

* Argset tests

* Exclude WIP

* Exclude WIP

* Arg eval

* Arg match, part 2

* Tests and generic arg comparisons

* Function eval with arguments

* Baselines

* Fix test

* Undo AST formatting change and update baseline

* Fix list ctor argument unpacking

Author: Mikhail Arkhipov

build/Common.Build.Core.settings

@@ -16,18 +16,10 @@
     <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">10.0</VisualStudioVersion>
     <VSTarget Condition="$(VSTarget)=='' and '$(VisualStudioVersion)'=='16.0'">16.0</VSTarget>
     <VSTarget Condition="$(VSTarget)=='' and '$(VisualStudioVersion)'=='15.0'">15.0</VSTarget>
-    <VSTarget Condition="$(VSTarget)=='' and '$(VisualStudioVersion)'=='14.0'">14.0</VSTarget>
-    <VSTarget Condition="$(VSTarget)=='' and '$(VisualStudioVersion)'=='12.0'">12.0</VSTarget>
-    <VSTarget Condition="$(VSTarget)=='' and '$(VisualStudioVersion)'=='11.0'">11.0</VSTarget>
-    <VSTarget Condition="$(VSTarget)==''">10.0</VSTarget>
-    
+
     <VSTargetName Condition="'$(VSTarget)' == '16.0'">2019</VSTargetName>
     <VSTargetName Condition="'$(VSTarget)' == '15.0'">2017</VSTargetName>
-    <VSTargetName Condition="'$(VSTarget)' == '14.0'">2015</VSTargetName>
-    <VSTargetName Condition="'$(VSTarget)' == '12.0'">2013</VSTargetName>
-    <VSTargetName Condition="'$(VSTarget)' == '11.0'">2012</VSTargetName>
-    <VSTargetName Condition="'$(VSTarget)' == '10.0'">2010</VSTargetName>
-    
+
     <BuildNumber Condition="'$(BuildNumber)' == ''">$(BUILD_BUILDNUMBER)</BuildNumber>
     <BuildNumber Condition="'$(BuildNumber)' == ''">1000.00</BuildNumber>
     <!--

src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs

@@ -0,0 +1,113 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Python.Analysis.Analyzer.Evaluation;
+using Microsoft.Python.Analysis.Analyzer.Handlers;
+using Microsoft.Python.Analysis.Analyzer.Symbols;
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Core;
+using Microsoft.Python.Parsing.Ast;
+
+namespace Microsoft.Python.Analysis.Analyzer {
+    /// <summary>
+    /// Base class with common functionality to module and function analysis walkers.
+    /// </summary>
+    internal abstract class AnalysisWalker : PythonWalkerAsync {
+        protected ImportHandler ImportHandler { get; }
+        protected FromImportHandler FromImportHandler { get; }
+        protected LoopHandler LoopHandler { get; }
+        protected ConditionalHandler ConditionalHandler { get; }
+        protected AssignmentHandler AssignmentHandler { get; }
+        protected WithHandler WithHandler { get; }
+        protected TryExceptHandler TryExceptHandler { get; }
+        protected NonLocalHandler NonLocalHandler { get; }
+
+        public ExpressionEval Eval { get; }
+        public IPythonModule Module => Eval.Module;
+        public IPythonInterpreter Interpreter => Eval.Interpreter;
+        public PythonAst Ast => Eval.Ast;
+        protected ModuleSymbolTable SymbolTable => Eval.SymbolTable;
+
+        protected AnalysisWalker(ExpressionEval eval) {
+            Eval = eval;
+            ImportHandler = new ImportHandler(this);
+            FromImportHandler = new FromImportHandler(this);
+            AssignmentHandler = new AssignmentHandler(this);
+            LoopHandler = new LoopHandler(this);
+            ConditionalHandler = new ConditionalHandler(this);
+            WithHandler = new WithHandler(this);
+            TryExceptHandler = new TryExceptHandler(this);
+            NonLocalHandler = new NonLocalHandler(this);
+        }
+
+        protected AnalysisWalker(IServiceContainer services, IPythonModule module, PythonAst ast)
+            : this(new ExpressionEval(services, module, ast)) {
+        }
+
+        #region AST walker overrides
+        public override async Task<bool> WalkAsync(AssignmentStatement node, CancellationToken cancellationToken = default) {
+            await AssignmentHandler.HandleAssignmentAsync(node, cancellationToken);
+            return await base.WalkAsync(node, cancellationToken);
+        }
+
+        public override async Task<bool> WalkAsync(ExpressionStatement node, CancellationToken cancellationToken = default) {
+            await AssignmentHandler.HandleAnnotatedExpressionAsync(node.Expression as ExpressionWithAnnotation, null, cancellationToken);
+            return false;
+        }
+
+        public override async Task<bool> WalkAsync(ForStatement node, CancellationToken cancellationToken = default) {
+            await LoopHandler.HandleForAsync(node, cancellationToken);
+            return await base.WalkAsync(node, cancellationToken);
+        }
+
+        public override Task<bool> WalkAsync(FromImportStatement node, CancellationToken cancellationToken = default)
+            => FromImportHandler.HandleFromImportAsync(node, cancellationToken);
+
+        public override Task<bool> WalkAsync(GlobalStatement node, CancellationToken cancellationToken = default)
+            => NonLocalHandler.HandleGlobalAsync(node, cancellationToken);
+
+        public override Task<bool> WalkAsync(IfStatement node, CancellationToken cancellationToken = default)
+            => ConditionalHandler.HandleIfAsync(node, cancellationToken);
+
+        public override Task<bool> WalkAsync(ImportStatement node, CancellationToken cancellationToken = default)
+            => ImportHandler.HandleImportAsync(node, cancellationToken);
+
+        public override Task<bool> WalkAsync(NonlocalStatement node, CancellationToken cancellationToken = default)
+            => NonLocalHandler.HandleNonLocalAsync(node, cancellationToken);
+
+        public override async Task<bool> WalkAsync(TryStatement node, CancellationToken cancellationToken = default) {
+            await TryExceptHandler.HandleTryExceptAsync(node, cancellationToken);
+            return await base.WalkAsync(node, cancellationToken);
+        }
+
+        public override async Task<bool> WalkAsync(WhileStatement node, CancellationToken cancellationToken = default) {
+            await LoopHandler.HandleWhileAsync(node, cancellationToken);
+            return await base.WalkAsync(node, cancellationToken);
+        }
+
+        public override async Task<bool> WalkAsync(WithStatement node, CancellationToken cancellationToken = default) {
+            await WithHandler.HandleWithAsync(node, cancellationToken);
+            return await base.WalkAsync(node, cancellationToken);
+        }
+        #endregion
+
+        protected T[] GetStatements<T>(ScopeStatement s)
+            => (s.Body as SuiteStatement)?.Statements.OfType<T>().ToArray() ?? Array.Empty<T>();
+    }
+}

src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs

@@ -0,0 +1,47 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+namespace Microsoft.Python.Analysis.Analyzer {
+    /// <summary>
+    /// Represents document that can be analyzed asynchronously.
+    /// </summary>
+    internal interface IAnalyzable {
+        /// <summary>
+        /// Expected version of the analysis when asynchronous operations complete.
+        /// Typically every change to the document or documents that depend on it
+        /// increment the expected version. At the end of the analysis if the expected
+        /// version is still the same, the analysis is applied to the document and
+        /// becomes available to consumers.
+        /// </summary>
+        int ExpectedAnalysisVersion { get; }
+
+        /// <summary>
+        /// Notifies document that analysis is now pending. Typically document increments 
+        /// the expected analysis version. The method can be called repeatedly without
+        /// calling `CompleteAnalysis` first. The method is invoked for every dependency
+        /// in the chain to ensure that objects know that their dependencies have been
+        /// modified and the current analysis is no longer up to date.
+        /// </summary>
+        void NotifyAnalysisPending();
+
+        /// <summary>
+        /// Notifies document that its analysis is now complete.
+        /// </summary>
+        /// <param name="analysis">Document analysis</param>
+        /// (version of the snapshot in the beginning of analysis).</param>
+        /// <returns>True if analysis was accepted, false if is is out of date.</returns>
+        bool NotifyAnalysisComplete(IDocumentAnalysis analysis);
+    }
+}

src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs

@@ -0,0 +1,32 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Python.Analysis.Documents;
+
+namespace Microsoft.Python.Analysis.Analyzer {
+    public interface IPythonAnalyzer {
+        /// <summary>
+        /// Analyze single document.
+        /// </summary>
+        Task AnalyzeDocumentAsync(IDocument document, CancellationToken cancellationToken);
+
+        /// <summary>
+        /// Analyze document with dependents.
+        /// </summary>
+        Task AnalyzeDocumentDependencyChainAsync(IDocument document, CancellationToken cancellationToken);
+    }
+}

src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs

@@ -0,0 +1,99 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Python.Analysis.Diagnostics;
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Analysis.Documents;
+using Microsoft.Python.Analysis.Values;
+using Microsoft.Python.Core;
+using Microsoft.Python.Core.Diagnostics;
+using Microsoft.Python.Core.Text;
+using Microsoft.Python.Parsing.Ast;
+
+namespace Microsoft.Python.Analysis.Analyzer {
+    internal sealed class DocumentAnalysis : IDocumentAnalysis {
+        public static readonly IDocumentAnalysis Empty = new EmptyAnalysis();
+
+        public DocumentAnalysis(IDocument document, int version, IGlobalScope globalScope, PythonAst ast) {
+            Check.ArgumentNotNull(nameof(document), document);
+            Check.ArgumentNotNull(nameof(globalScope), globalScope);
+            Document = document;
+            Version = version;
+            GlobalScope = globalScope;
+            Ast = ast;
+        }
+
+        #region IDocumentAnalysis
+        /// <summary>
+        /// Analyzed document.
+        /// </summary>
+        public IDocument Document { get; }
+
+        /// <summary>
+        /// Version of the analysis. Usually matches document version,
+        /// but can be lower when document or its dependencies were
+        /// updated since.
+        /// </summary>
+        public int Version { get; }
+
+        /// <summary>
+        /// AST that was used in the analysis.
+        /// </summary>
+        public PythonAst Ast { get; }
+
+        /// <summary>
+        /// Document/module global scope.
+        /// </summary>
+        public IGlobalScope GlobalScope { get; }
+
+        /// <summary>
+        /// Module top-level members
+        /// </summary>
+        public IVariableCollection TopLevelVariables => GlobalScope.Variables;
+
+        /// <summary>
+        /// All module members from all scopes.
+        /// </summary>
+        public IEnumerable<IVariable> AllVariables
+            => (GlobalScope as IScope).TraverseBreadthFirst(s => s.Children).SelectMany(s => s.Variables);
+
+        public IEnumerable<IPythonType> GetAllAvailableItems(SourceLocation location) => Enumerable.Empty<IPythonType>();
+        public IEnumerable<IPythonType> GetMembers(SourceLocation location) => Enumerable.Empty<IPythonType>();
+        public IEnumerable<IPythonFunctionOverload> GetSignatures(SourceLocation location) => Enumerable.Empty<IPythonFunctionOverload>();
+        public IEnumerable<IPythonType> GetValues(SourceLocation location) => Enumerable.Empty<IPythonType>();
+        #endregion
+
+        private sealed class EmptyAnalysis : IDocumentAnalysis {
+            public EmptyAnalysis(IDocument document = null) {
+                Document = document;
+                GlobalScope = new EmptyGlobalScope(document);
+            }
+
+            public IDocument Document { get; }
+            public int Version { get; } = -1;
+            public IGlobalScope GlobalScope { get; }
+            public PythonAst Ast => null;
+            public IEnumerable<IPythonType> GetAllAvailableItems(SourceLocation location) => Enumerable.Empty<IPythonType>();
+            public IEnumerable<DiagnosticsEntry> Diagnostics => Enumerable.Empty<DiagnosticsEntry>();
+            public IVariableCollection TopLevelVariables => VariableCollection.Empty;
+            public IEnumerable<IVariable> AllVariables => Enumerable.Empty<IVariable>();
+            public IEnumerable<IPythonType> GetMembers(SourceLocation location) => Enumerable.Empty<IPythonType>();
+            public IEnumerable<IPythonFunctionOverload> GetSignatures(SourceLocation location) => Enumerable.Empty<IPythonFunctionOverload>();
+            public IEnumerable<IPythonType> GetValues(SourceLocation location) => Enumerable.Empty<IPythonType>();
+        }
+    }
+}