AST analyzer and assemblies split

build/Common.Build.Core.settings

@@ -16,18 +16,10 @@
     <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">10.0</VisualStudioVersion>
     <VSTarget Condition="$(VSTarget)=='' and '$(VisualStudioVersion)'=='16.0'">16.0</VSTarget>
     <VSTarget Condition="$(VSTarget)=='' and '$(VisualStudioVersion)'=='15.0'">15.0</VSTarget>
-    <VSTarget Condition="$(VSTarget)=='' and '$(VisualStudioVersion)'=='14.0'">14.0</VSTarget>
-    <VSTarget Condition="$(VSTarget)=='' and '$(VisualStudioVersion)'=='12.0'">12.0</VSTarget>
-    <VSTarget Condition="$(VSTarget)=='' and '$(VisualStudioVersion)'=='11.0'">11.0</VSTarget>
-    <VSTarget Condition="$(VSTarget)==''">10.0</VSTarget>
-
+
     <VSTargetName Condition="'$(VSTarget)' == '16.0'">2019</VSTargetName>
     <VSTargetName Condition="'$(VSTarget)' == '15.0'">2017</VSTargetName>
-    <VSTargetName Condition="'$(VSTarget)' == '14.0'">2015</VSTargetName>
-    <VSTargetName Condition="'$(VSTarget)' == '12.0'">2013</VSTargetName>
-    <VSTargetName Condition="'$(VSTarget)' == '11.0'">2012</VSTargetName>
-    <VSTargetName Condition="'$(VSTarget)' == '10.0'">2010</VSTargetName>
-
+
     <BuildNumber Condition="'$(BuildNumber)' == ''">$(BUILD_BUILDNUMBER)</BuildNumber>
     <BuildNumber Condition="'$(BuildNumber)' == ''">1000.00</BuildNumber>
     <!--

src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs

@@ -0,0 +1,113 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Python.Analysis.Analyzer.Evaluation;
+using Microsoft.Python.Analysis.Analyzer.Handlers;
+using Microsoft.Python.Analysis.Analyzer.Symbols;
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Core;
+using Microsoft.Python.Parsing.Ast;
+
+namespace Microsoft.Python.Analysis.Analyzer {
+    /// <summary>
+    /// Base class with common functionality to module and function analysis walkers.
+    /// </summary>
+    internal abstract class AnalysisWalker : PythonWalkerAsync {
+        protected ImportHandler ImportHandler { get; }
+        protected FromImportHandler FromImportHandler { get; }
+        protected LoopHandler LoopHandler { get; }
+        protected ConditionalHandler ConditionalHandler { get; }
+        protected AssignmentHandler AssignmentHandler { get; }
+        protected WithHandler WithHandler { get; }
+        protected TryExceptHandler TryExceptHandler { get; }
+        protected NonLocalHandler NonLocalHandler { get; }
+
+        public ExpressionEval Eval { get; }
+        public IPythonModule Module => Eval.Module;
+        public IPythonInterpreter Interpreter => Eval.Interpreter;
+        public PythonAst Ast => Eval.Ast;
+        protected ModuleSymbolTable SymbolTable => Eval.SymbolTable;
+
+        protected AnalysisWalker(ExpressionEval eval) {
+            Eval = eval;
+            ImportHandler = new ImportHandler(this);
+            FromImportHandler = new FromImportHandler(this);
+            AssignmentHandler = new AssignmentHandler(this);
+            LoopHandler = new LoopHandler(this);
+            ConditionalHandler = new ConditionalHandler(this);
+            WithHandler = new WithHandler(this);
+            TryExceptHandler = new TryExceptHandler(this);
+            NonLocalHandler = new NonLocalHandler(this);
+        }
+
+        protected AnalysisWalker(IServiceContainer services, IPythonModule module, PythonAst ast)
+            : this(new ExpressionEval(services, module, ast)) {
+        }
+
+        #region AST walker overrides
+        public override async Task<bool> WalkAsync(AssignmentStatement node, CancellationToken cancellationToken = default) {
+            await AssignmentHandler.HandleAssignmentAsync(node, cancellationToken);
+            return await base.WalkAsync(node, cancellationToken);
+        }
+
+        public override async Task<bool> WalkAsync(ExpressionStatement node, CancellationToken cancellationToken = default) {
+            await AssignmentHandler.HandleAnnotatedExpressionAsync(node.Expression as ExpressionWithAnnotation, null, cancellationToken);
+            return false;
+        }
+
+        public override async Task<bool> WalkAsync(ForStatement node, CancellationToken cancellationToken = default) {
+            await LoopHandler.HandleForAsync(node, cancellationToken);
+            return await base.WalkAsync(node, cancellationToken);
+        }
+
+        public override Task<bool> WalkAsync(FromImportStatement node, CancellationToken cancellationToken = default)
+            => FromImportHandler.HandleFromImportAsync(node, cancellationToken);
+
+        public override Task<bool> WalkAsync(GlobalStatement node, CancellationToken cancellationToken = default)
+            => NonLocalHandler.HandleGlobalAsync(node, cancellationToken);
+
+        public override Task<bool> WalkAsync(IfStatement node, CancellationToken cancellationToken = default)
+            => ConditionalHandler.HandleIfAsync(node, cancellationToken);
+
+        public override Task<bool> WalkAsync(ImportStatement node, CancellationToken cancellationToken = default)
+            => ImportHandler.HandleImportAsync(node, cancellationToken);
+
+        public override Task<bool> WalkAsync(NonlocalStatement node, CancellationToken cancellationToken = default)
+            => NonLocalHandler.HandleNonLocalAsync(node, cancellationToken);
+
+        public override async Task<bool> WalkAsync(TryStatement node, CancellationToken cancellationToken = default) {
+            await TryExceptHandler.HandleTryExceptAsync(node, cancellationToken);
+            return await base.WalkAsync(node, cancellationToken);
+        }
+
+        public override async Task<bool> WalkAsync(WhileStatement node, CancellationToken cancellationToken = default) {
+            await LoopHandler.HandleWhileAsync(node, cancellationToken);
+            return await base.WalkAsync(node, cancellationToken);
+        }
+
+        public override async Task<bool> WalkAsync(WithStatement node, CancellationToken cancellationToken = default) {
+            await WithHandler.HandleWithAsync(node, cancellationToken);
+            return await base.WalkAsync(node, cancellationToken);
+        }
+        #endregion
+
+        protected T[] GetStatements<T>(ScopeStatement s)
+            => (s.Body as SuiteStatement)?.Statements.OfType<T>().ToArray() ?? Array.Empty<T>();
+    }
+}

src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs

@@ -0,0 +1,47 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+namespace Microsoft.Python.Analysis.Analyzer {
+    /// <summary>
+    /// Represents document that can be analyzed asynchronously.
+    /// </summary>
+    internal interface IAnalyzable {
+        /// <summary>
+        /// Expected version of the analysis when asynchronous operations complete.
+        /// Typically every change to the document or documents that depend on it
+        /// increment the expected version. At the end of the analysis if the expected
+        /// version is still the same, the analysis is applied to the document and
+        /// becomes available to consumers.
+        /// </summary>
+        int ExpectedAnalysisVersion { get; }
+
+        /// <summary>
+        /// Notifies document that analysis is now pending. Typically document increments 
+        /// the expected analysis version. The method can be called repeatedly without
+        /// calling `CompleteAnalysis` first. The method is invoked for every dependency
+        /// in the chain to ensure that objects know that their dependencies have been
+        /// modified and the current analysis is no longer up to date.
+        /// </summary>
+        void NotifyAnalysisPending();
+
+        /// <summary>
+        /// Notifies document that its analysis is now complete.
+        /// </summary>
+        /// <param name="analysis">Document analysis</param>
+        /// (version of the snapshot in the beginning of analysis).</param>
+        /// <returns>True if analysis was accepted, false if is is out of date.</returns>
+        bool NotifyAnalysisComplete(IDocumentAnalysis analysis);
+    }
+}

src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs

@@ -0,0 +1,32 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Python.Analysis.Documents;
+
+namespace Microsoft.Python.Analysis.Analyzer {
+    public interface IPythonAnalyzer {
+        /// <summary>
+        /// Analyze single document.
+        /// </summary>
+        Task AnalyzeDocumentAsync(IDocument document, CancellationToken cancellationToken);
+
+        /// <summary>
+        /// Analyze document with dependents.
+        /// </summary>
+        Task AnalyzeDocumentDependencyChainAsync(IDocument document, CancellationToken cancellationToken);
+    }
+}

src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs

@@ -0,0 +1,99 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Python.Analysis.Diagnostics;
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Analysis.Documents;
+using Microsoft.Python.Analysis.Values;
+using Microsoft.Python.Core;
+using Microsoft.Python.Core.Diagnostics;
+using Microsoft.Python.Core.Text;
+using Microsoft.Python.Parsing.Ast;
+
+namespace Microsoft.Python.Analysis.Analyzer {
+    internal sealed class DocumentAnalysis : IDocumentAnalysis {
+        public static readonly IDocumentAnalysis Empty = new EmptyAnalysis();
+
+        public DocumentAnalysis(IDocument document, int version, IGlobalScope globalScope, PythonAst ast) {
+            Check.ArgumentNotNull(nameof(document), document);
+            Check.ArgumentNotNull(nameof(globalScope), globalScope);
+            Document = document;
+            Version = version;
+            GlobalScope = globalScope;
+            Ast = ast;
+        }
+
+        #region IDocumentAnalysis
+        /// <summary>
+        /// Analyzed document.
+        /// </summary>
+        public IDocument Document { get; }
+
+        /// <summary>
+        /// Version of the analysis. Usually matches document version,
+        /// but can be lower when document or its dependencies were
+        /// updated since.
+        /// </summary>
+        public int Version { get; }
+
+        /// <summary>
+        /// AST that was used in the analysis.
+        /// </summary>
+        public PythonAst Ast { get; }
+
+        /// <summary>
+        /// Document/module global scope.
+        /// </summary>
+        public IGlobalScope GlobalScope { get; }
+
+        /// <summary>
+        /// Module top-level members
+        /// </summary>
+        public IVariableCollection TopLevelVariables => GlobalScope.Variables;
+
+        /// <summary>
+        /// All module members from all scopes.
+        /// </summary>
+        public IEnumerable<IVariable> AllVariables
+            => (GlobalScope as IScope).TraverseBreadthFirst(s => s.Children).SelectMany(s => s.Variables);
+
+        public IEnumerable<IPythonType> GetAllAvailableItems(SourceLocation location) => Enumerable.Empty<IPythonType>();
+        public IEnumerable<IPythonType> GetMembers(SourceLocation location) => Enumerable.Empty<IPythonType>();
+        public IEnumerable<IPythonFunctionOverload> GetSignatures(SourceLocation location) => Enumerable.Empty<IPythonFunctionOverload>();
+        public IEnumerable<IPythonType> GetValues(SourceLocation location) => Enumerable.Empty<IPythonType>();
+        #endregion
+
+        private sealed class EmptyAnalysis : IDocumentAnalysis {
+            public EmptyAnalysis(IDocument document = null) {
+                Document = document;
+                GlobalScope = new EmptyGlobalScope(document);
+            }
+
+            public IDocument Document { get; }
+            public int Version { get; } = -1;
+            public IGlobalScope GlobalScope { get; }
+            public PythonAst Ast => null;
+            public IEnumerable<IPythonType> GetAllAvailableItems(SourceLocation location) => Enumerable.Empty<IPythonType>();
+            public IEnumerable<DiagnosticsEntry> Diagnostics => Enumerable.Empty<DiagnosticsEntry>();
+            public IVariableCollection TopLevelVariables => VariableCollection.Empty;
+            public IEnumerable<IVariable> AllVariables => Enumerable.Empty<IVariable>();
+            public IEnumerable<IPythonType> GetMembers(SourceLocation location) => Enumerable.Empty<IPythonType>();
+            public IEnumerable<IPythonFunctionOverload> GetSignatures(SourceLocation location) => Enumerable.Empty<IPythonFunctionOverload>();
+            public IEnumerable<IPythonType> GetValues(SourceLocation location) => Enumerable.Empty<IPythonType>();
+        }
+    }
+}