diff --git a/src/Analysis/Engine/Impl/Analyzer/ExpressionEvaluator.cs b/src/Analysis/Engine/Impl/Analyzer/ExpressionEvaluator.cs index c45691a34..78508cc01 100644 --- a/src/Analysis/Engine/Impl/Analyzer/ExpressionEvaluator.cs +++ b/src/Analysis/Engine/Impl/Analyzer/ExpressionEvaluator.cs @@ -149,8 +149,19 @@ public IAnalysisSet LookupAnalysisSetByName(Node node, string name, bool addRef refs = createIn.CreateVariable(node, _unit, name, addRef); res = refs.Types; } else { - // ... warn the user - warn = true; + switch (name) { + // "atom" in Python grammar. + case "True": + case "False": + case "None": + case "...": + Debug.Fail($"Known good name '{name}' not found in scope"); + break; + default: + // ... warn the user + warn = true; + break; + } } } } diff --git a/src/Analysis/Engine/Impl/Definitions/IGroupableAnalysisProjectEntry.cs b/src/Analysis/Engine/Impl/Definitions/IGroupableAnalysisProjectEntry.cs index f0012554c..a8e39eb24 100644 --- a/src/Analysis/Engine/Impl/Definitions/IGroupableAnalysisProjectEntry.cs +++ b/src/Analysis/Engine/Impl/Definitions/IGroupableAnalysisProjectEntry.cs @@ -22,14 +22,14 @@ namespace Microsoft.PythonTools.Analysis { /// more efficient analysis. /// /// To analyze the full group you call Analyze(true) on all the items in the same group (determined - /// by looking at the identity of the AnalysGroup object). Then you call AnalyzeQueuedEntries on the + /// by looking at the identity of the AnalysisGroup object). Then you call AnalyzeQueuedEntries on the /// group. /// public interface IGroupableAnalysisProjectEntry { /// /// Analyzes this project entry optionally just adding it to the queue shared by the project. /// - void Analyze(CancellationToken cancel, bool enqueueOnly); + void PreAnalyze(); IGroupableAnalysisProject AnalysisGroup { get; } } diff --git a/src/Analysis/Engine/Impl/DependencyResolution/PathResolverSnapshot.cs b/src/Analysis/Engine/Impl/DependencyResolution/PathResolverSnapshot.cs index f3bebb454..c1f69c132 100644 --- a/src/Analysis/Engine/Impl/DependencyResolution/PathResolverSnapshot.cs +++ b/src/Analysis/Engine/Impl/DependencyResolution/PathResolverSnapshot.cs @@ -359,11 +359,11 @@ private void CreateRootsWithDefault(string rootDirectory, string[] userSearchPat .ToArray(); var filteredInterpreterSearchPaths = interpreterSearchPaths.Select(FixPath) - .Except(filteredUserSearchPaths.Prepend(rootDirectory)) + .Except(filteredUserSearchPaths.Append(rootDirectory)) .ToArray(); userRootsCount = filteredUserSearchPaths.Length + 1; - nodes = AddRootsFromSearchPaths(ImmutableArray.Empty.Add(GetOrCreateRoot(rootDirectory)), filteredUserSearchPaths, filteredInterpreterSearchPaths); + nodes = AddRootsFromSearchPaths(rootDirectory, filteredUserSearchPaths, filteredInterpreterSearchPaths); string FixPath(string p) => Path.IsPathRooted(p) ? PathUtils.NormalizePath(p) : PathUtils.NormalizePath(Path.Combine(rootDirectory, p)); } @@ -381,11 +381,18 @@ private void CreateRootsWithoutDefault(string[] userSearchPaths, string[] interp .ToArray(); userRootsCount = filteredUserSearchPaths.Length; - nodes = AddRootsFromSearchPaths(ImmutableArray.Empty, filteredUserSearchPaths, filteredInterpreterSearchPaths); + nodes = AddRootsFromSearchPaths(filteredUserSearchPaths, filteredInterpreterSearchPaths); } - private ImmutableArray AddRootsFromSearchPaths(ImmutableArray roots, string[] userSearchPaths, string[] interpreterSearchPaths) { - return roots + private ImmutableArray AddRootsFromSearchPaths(string rootDirectory, string[] userSearchPaths, string[] interpreterSearchPaths) { + return ImmutableArray.Empty + .AddRange(userSearchPaths.Select(GetOrCreateRoot).ToArray()) + .Add(GetOrCreateRoot(rootDirectory)) + .AddRange(interpreterSearchPaths.Select(GetOrCreateRoot).ToArray()); + } + + private ImmutableArray AddRootsFromSearchPaths(string[] userSearchPaths, string[] interpreterSearchPaths) { + return ImmutableArray.Empty .AddRange(userSearchPaths.Select(GetOrCreateRoot).ToArray()) .AddRange(interpreterSearchPaths.Select(GetOrCreateRoot).ToArray()); } diff --git a/src/Analysis/Engine/Impl/Infrastructure/Extensions/StackExtensions.cs b/src/Analysis/Engine/Impl/Infrastructure/Extensions/StackExtensions.cs new file mode 100644 index 000000000..662a9002a --- /dev/null +++ b/src/Analysis/Engine/Impl/Infrastructure/Extensions/StackExtensions.cs @@ -0,0 +1,31 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; + +namespace Microsoft.PythonTools.Analysis.Infrastructure { + internal static class StackExtensions { + public static bool TryPeek(this Stack stack, out T result) { + if (stack.Count == 0) { + result = default; + return false; + } + + result = stack.Peek(); + return true; + } + } +} diff --git a/src/Analysis/Engine/Impl/Intellisense/AnalysisQueue.cs b/src/Analysis/Engine/Impl/Intellisense/AnalysisQueue.cs index 143e66ed8..8cc4d0a54 100644 --- a/src/Analysis/Engine/Impl/Intellisense/AnalysisQueue.cs +++ b/src/Analysis/Engine/Impl/Intellisense/AnalysisQueue.cs @@ -137,6 +137,8 @@ public void Enqueue(IAnalyzable item, AnalysisPriority priority) { } private async Task HandleAnalyzable(IAnalyzable item, AnalysisPriority priority, CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); + if (item is IGroupableAnalysisProjectEntry groupable) { var added = _enqueuedGroups.Add(groupable.AnalysisGroup); if (added) { @@ -147,7 +149,7 @@ private async Task HandleAnalyzable(IAnalyzable item, AnalysisPriority priority, } } - groupable.Analyze(cancellationToken, true); + groupable.PreAnalyze(); } else { item.Analyze(cancellationToken); } diff --git a/src/Analysis/Engine/Impl/Interpreter/Definitions/PythonMemberType.cs b/src/Analysis/Engine/Impl/Interpreter/Definitions/PythonMemberType.cs index 6b464037a..020ebfcc9 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Definitions/PythonMemberType.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Definitions/PythonMemberType.cs @@ -64,7 +64,6 @@ public enum PythonMemberType { /// An instance of a namespace object that was imported from .NET. /// Namespace, - /// /// A constant defined in source code. /// diff --git a/src/Analysis/Engine/Impl/Interpreter/InterpreterConfiguration.cs b/src/Analysis/Engine/Impl/Interpreter/InterpreterConfiguration.cs index e6012dfa1..a2542b99b 100644 --- a/src/Analysis/Engine/Impl/Interpreter/InterpreterConfiguration.cs +++ b/src/Analysis/Engine/Impl/Interpreter/InterpreterConfiguration.cs @@ -21,7 +21,7 @@ using Microsoft.PythonTools.Analysis.Infrastructure; namespace Microsoft.PythonTools.Interpreter { - public sealed class InterpreterConfiguration : IEquatable { + public class InterpreterConfiguration : IEquatable { private readonly string _description; private string _fullDescription; @@ -48,29 +48,6 @@ public InterpreterConfiguration( SitePackagesPath = sitePackagesPath ?? string.Empty; } - [Obsolete] - public InterpreterConfiguration( - string id, - string description, - string prefixPath = null, - string path = null, - string winPath = "", - string pathVar = "", - InterpreterArchitecture arch = default(InterpreterArchitecture), - Version version = null, - InterpreterUIMode uiMode = InterpreterUIMode.Normal - ) { - Id = id; - _description = description ?? ""; - PrefixPath = prefixPath; - InterpreterPath = path; - WindowsInterpreterPath = string.IsNullOrEmpty(winPath) ? path : winPath; - PathEnvironmentVariable = pathVar; - Architecture = arch ?? InterpreterArchitecture.Unknown; - Version = version ?? new Version(); - UIMode = uiMode; - } - private static string Read(Dictionary d, string k) => d.TryGetValue(k, out var o) ? o as string : null; @@ -155,22 +132,12 @@ public void SwitchToFullDescription() { } } - [Obsolete("Prefix path only applies to Windows.")] - public string PrefixPath { get; } - /// /// Returns the path to the interpreter executable for launching Python /// applications. /// public string InterpreterPath { get; } - - /// - /// Returns the path to the interpreter executable for launching Python - /// applications which are windows applications (pythonw.exe, ipyw.exe). - /// - [Obsolete("Python Language Server is platform-agnostic and does not use Windows-specific settings.")] - public string WindowsInterpreterPath { get; } - + /// /// Gets the environment variable which should be used to set sys.path. /// @@ -198,12 +165,6 @@ public void SwitchToFullDescription() { /// public string SitePackagesPath { get; } - /// - /// The UI behavior of the interpreter. - /// - [Obsolete("Language Server does not support UI features related to the interpreter.")] - public InterpreterUIMode UIMode { get; } - /// /// The fixed search paths of the interpreter. /// diff --git a/src/Analysis/Engine/Impl/Interpreter/InterpreterUIMode.cs b/src/Analysis/Engine/Impl/Interpreter/InterpreterUIMode.cs deleted file mode 100644 index ab7aba8dd..000000000 --- a/src/Analysis/Engine/Impl/Interpreter/InterpreterUIMode.cs +++ /dev/null @@ -1,56 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; - -namespace Microsoft.PythonTools.Interpreter { - /// - /// Specifies the interpreter's behavior in the UI. - /// - [Flags] - [Obsolete("Language Server does not support UI features related to the interpreter.")] - public enum InterpreterUIMode : int { - /// - /// Interpreter can be set or selected as the default, and is visible to - /// the user. - /// - Normal = 0x00, - - /// - /// Interpreter is not displayed in the user interface, but can still be - /// added to a project if the ID is known. - /// - Hidden = 0x01, - - /// - /// Interpreter cannot be selected as the default. Implies - /// . - /// - CannotBeDefault = 0x02, - - /// - /// Interpreter cannot be automatically selected as the default. - /// - CannotBeAutoDefault = 0x04, - - /// - /// Interpreter has no user-modifiable settings. - /// - CannotBeConfigured = 0x08, - - SupportsDatabase = 0x10, - } -} diff --git a/src/Analysis/Engine/Impl/Interpreter/PythonInterpreterFactoryExtensions.cs b/src/Analysis/Engine/Impl/Interpreter/PythonInterpreterFactoryExtensions.cs deleted file mode 100644 index b41fd914f..000000000 --- a/src/Analysis/Engine/Impl/Interpreter/PythonInterpreterFactoryExtensions.cs +++ /dev/null @@ -1,112 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.IO; - -namespace Microsoft.PythonTools.Interpreter { - public static class PythonInterpreterFactoryExtensions { - /// - /// Determines whether two interpreter factories are equivalent. - /// - public static bool IsEqual(this IPythonInterpreterFactory x, IPythonInterpreterFactory y) { - if (x == null || y == null) { - return x == null && y == null; - } - if (x.GetType() != y.GetType()) { - return false; - } - - return x.Configuration.Equals(y.Configuration); - } - - /// - /// Returns true if the factory can be run. This checks whether the - /// configured InterpreterPath value is an actual file. - /// - public static bool IsRunnable(this IPythonInterpreterFactory factory) { - return factory != null && factory.Configuration.IsRunnable(); - } - - /// - /// Returns true if the configuration can be run. This checks whether - /// the configured InterpreterPath value is an actual file. - /// - public static bool IsRunnable(this InterpreterConfiguration config) { - return config != null && - !InterpreterRegistryConstants.IsNoInterpretersFactory(config.Id) && - File.Exists(config.InterpreterPath); - } - - /// - /// Checks whether the factory can be run and throws the appropriate - /// exception if it cannot. - /// - /// - /// factory is null and parameterName is provided. - /// - /// - /// factory is null and parameterName is not provided, or the factory - /// has no configuration. - /// - /// - /// factory is the sentinel used when no environments are installed. - /// - /// - /// factory's InterpreterPath does not exist on disk. - /// - public static void ThrowIfNotRunnable(this IPythonInterpreterFactory factory, string parameterName = null) { - if (factory == null) { - if (string.IsNullOrEmpty(parameterName)) { - throw new NullReferenceException(); - } else { - throw new ArgumentNullException(parameterName); - } - } - factory.Configuration.ThrowIfNotRunnable(); - } - - /// - /// Checks whether the configuration can be run and throws the - /// appropriate exception if it cannot. - /// - /// - /// config is null and parameterName is provided. - /// - /// - /// config is null and parameterName is not provided. - /// - /// - /// config is the sentinel used when no environments are installed. - /// - /// - /// config's InterpreterPath does not exist on disk. - /// - public static void ThrowIfNotRunnable(this InterpreterConfiguration config, string parameterName = null) { - if (config == null) { - if (string.IsNullOrEmpty(parameterName)) { - throw new NullReferenceException(); - } else { - throw new ArgumentNullException(parameterName); - } - } else if (InterpreterRegistryConstants.IsNoInterpretersFactory(config.Id)) { - throw new NoInterpretersException(); - } else if (!File.Exists(config.InterpreterPath)) { - throw new FileNotFoundException(config.InterpreterPath ?? "(null)"); - } - } - } -} diff --git a/src/Analysis/Engine/Impl/LocationInfo.cs b/src/Analysis/Engine/Impl/LocationInfo.cs index c34854ea6..52c25ecad 100644 --- a/src/Analysis/Engine/Impl/LocationInfo.cs +++ b/src/Analysis/Engine/Impl/LocationInfo.cs @@ -18,7 +18,7 @@ using System.Collections.Generic; namespace Microsoft.PythonTools.Analysis { - internal class LocationInfo : ILocationInfo, IEquatable { + public class LocationInfo : ILocationInfo, IEquatable { internal static readonly LocationInfo[] Empty = new LocationInfo[0]; public LocationInfo(string path, Uri documentUri, int line, int column) : diff --git a/src/Analysis/Engine/Impl/ModuleAnalysis.cs b/src/Analysis/Engine/Impl/ModuleAnalysis.cs index 3c328585c..702ddb4eb 100644 --- a/src/Analysis/Engine/Impl/ModuleAnalysis.cs +++ b/src/Analysis/Engine/Impl/ModuleAnalysis.cs @@ -857,17 +857,6 @@ private IEnumerable GetKeywordMembers(GetMemberOptions options, I #endregion - /// - /// Gets the available names at the given location. This includes - /// global variables and locals, but not built-in variables. - /// - /// - /// The 0-based absolute index into the file where the available members - /// should be looked up. - /// - /// TODO: Remove; this is only used for tests - internal IEnumerable GetVariablesNoBuiltinsByIndex(int index) => GetVariablesNoBuiltins(_unit.Tree.IndexToLocation(index)); - /// /// Gets the available names at the given location. This includes /// global variables and locals, but not built-in variables. diff --git a/src/Analysis/Engine/Impl/ProjectEntry.cs b/src/Analysis/Engine/Impl/ProjectEntry.cs index 755726ed5..ec16a777c 100644 --- a/src/Analysis/Engine/Impl/ProjectEntry.cs +++ b/src/Analysis/Engine/Impl/ProjectEntry.cs @@ -21,6 +21,7 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -45,11 +46,10 @@ internal sealed class ProjectEntry : IPythonProjectEntry, IAggregateableProjectE private readonly ConcurrentQueue> _backReferences = new ConcurrentQueue>(); private readonly HashSet _aggregates = new HashSet(); - private TaskCompletionSource _analysisTcs = new TaskCompletionSource(); + private AnalysisCompletionToken _analysisCompletionToken; private AnalysisUnit _unit; private readonly ManualResetEventSlim _pendingParse = new ManualResetEventSlim(true); private long _expectedParseVersion; - private long _expectedAnalysisVersion; internal ProjectEntry( PythonAnalyzer state, @@ -66,6 +66,7 @@ IAnalysisCookie cookie MyScope = new ModuleInfo(ModuleName, this, state.Interpreter.CreateModuleContext()); _unit = new AnalysisUnit(null, MyScope.Scope); + _analysisCompletionToken = AnalysisCompletionToken.Default; _buffers = new SortedDictionary { [0] = new DocumentBuffer() }; if (Cookie is InitialContentCookie c) { @@ -137,10 +138,10 @@ public IPythonParse GetCurrentParse() { } } - internal Task GetAnalysisAsync(int waitingTimeout = -1, CancellationToken cancellationToken = default(CancellationToken)) { + internal Task GetAnalysisAsync(int waitingTimeout = -1, CancellationToken cancellationToken = default) { Task task; lock (this) { - task = _analysisTcs.Task; + task = _analysisCompletionToken.Task; } if (task.IsCompleted || waitingTimeout == -1 && !cancellationToken.CanBeCanceled) { @@ -158,22 +159,15 @@ public IPythonParse GetCurrentParse() { internal void SetCompleteAnalysis() { lock (this) { - if (_expectedAnalysisVersion != Analysis.Version) { - return; - } - _analysisTcs.TrySetResult(Analysis); + _analysisCompletionToken.TrySetAnalysis(Analysis); } RaiseNewAnalysis(); } - internal void ResetCompleteAnalysis() { - TaskCompletionSource analysisTcs = null; + internal void NewAnalysisAwaitableOnParse() { lock (this) { - _expectedAnalysisVersion = AnalysisVersion + 1; - analysisTcs = _analysisTcs; - _analysisTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _analysisCompletionToken = _analysisCompletionToken.NewParse(); } - analysisTcs?.TrySetCanceled(); } public void SetCurrentParse(PythonAst tree, IAnalysisCookie cookie, bool notify = true) { @@ -195,38 +189,58 @@ public void SetCurrentParse(PythonAst tree, IAnalysisCookie cookie, bool notify internal bool IsVisible(ProjectEntry assigningScope) => true; - public void Analyze(CancellationToken cancel) => Analyze(cancel, false); - - public void Analyze(CancellationToken cancel, bool enqueueOnly) { + public void Analyze(CancellationToken cancel) { if (cancel.IsCancellationRequested) { return; } lock (this) { - AnalysisVersion++; + PrepareForAnalysis(); - foreach (var aggregate in _aggregates) { - aggregate?.BumpVersion(); - } + ProjectState.AnalyzeQueuedEntries(cancel); - Parse(enqueueOnly, cancel); + // publish the analysis now that it's complete/running + Analysis = new ModuleAnalysis( + _unit, + ((ModuleScope)_unit.Scope).CloneForPublish(), + DocumentUri, + AnalysisVersion + ); } - if (!enqueueOnly) { - RaiseNewAnalysis(); + RaiseNewAnalysis(); + } + + public void PreAnalyze() { + lock (this) { + PrepareForAnalysis(); + + // publish the analysis now that it's complete/running + Analysis = new ModuleAnalysis( + _unit, + ((ModuleScope)_unit.Scope).CloneForPublish(), + DocumentUri, + AnalysisVersion + ); } } private void RaiseNewAnalysis() => NewAnalysis?.Invoke(this, EventArgs.Empty); - public int AnalysisVersion { get; private set; } + public int AnalysisVersion => _analysisCompletionToken.Version; public bool IsAnalyzed => Analysis != null; - private void Parse(bool enqueueOnly, CancellationToken cancel) { + private void PrepareForAnalysis() { #if DEBUG Debug.Assert(Monitor.IsEntered(this)); #endif + _analysisCompletionToken = _analysisCompletionToken.NewAnalysis(); + + foreach (var aggregate in _aggregates) { + aggregate?.BumpVersion(); + } + var parse = GetCurrentParse(); var tree = parse?.Tree; var cookie = parse?.Cookie; @@ -306,18 +320,6 @@ where lastDot > 0 } _unit.Enqueue(); - - if (!enqueueOnly) { - ProjectState.AnalyzeQueuedEntries(cancel); - } - - // publish the analysis now that it's complete/running - Analysis = new ModuleAnalysis( - _unit, - ((ModuleScope)_unit.Scope).CloneForPublish(), - DocumentUri, - AnalysisVersion - ); } public IGroupableAnalysisProject AnalysisGroup => ProjectState; @@ -348,7 +350,7 @@ where lastDot > 0 public void Dispose() { lock (this) { - AnalysisVersion = -1; + _analysisCompletionToken = AnalysisCompletionToken.Disposed; var state = ProjectState; foreach (var aggregatedInto in _aggregates) { @@ -494,6 +496,55 @@ public void ResetDocument(int version, string content) { public void AddBackReference(ReferenceDict referenceDict) { _backReferences.Enqueue(new WeakReference(referenceDict)); } + + private struct AnalysisCompletionToken { + public static readonly AnalysisCompletionToken Default; + public static readonly AnalysisCompletionToken Disposed; + + static AnalysisCompletionToken() { + var cancelledTcs = CreateTcs(); + cancelledTcs.SetCanceled(); + + Default = new AnalysisCompletionToken(CreateTcs(), -1, false); + Disposed = new AnalysisCompletionToken(cancelledTcs, -1, true); + } + + private readonly bool _isParse; + private readonly TaskCompletionSource _tcs; + + public int Version { get; } + public Task Task => _tcs.Task; + + public AnalysisCompletionToken NewParse() { + if (_isParse) { + return this; + } + + var tcs = CreateTcs(); + _tcs.TrySetCanceled(); + return new AnalysisCompletionToken(tcs, Version + 1, true); + } + + public AnalysisCompletionToken NewAnalysis() { + var tcs = _tcs.Task.IsCompleted ? CreateTcs() : _tcs; + return new AnalysisCompletionToken(tcs, _isParse ? Version : Version + 1, false); + } + + private AnalysisCompletionToken(TaskCompletionSource tcs, int version, bool isParse) { + _tcs = tcs; + Version = version; + _isParse = isParse; + } + + public void TrySetAnalysis(IModuleAnalysis analysis) { + if (Version == analysis.Version) { + _tcs.TrySetResult(analysis); + } + } + + private static TaskCompletionSource CreateTcs() + => new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } } class InitialContentCookie : IAnalysisCookie { diff --git a/src/Analysis/Engine/Impl/Projects/ProjectAnalyzer.cs b/src/Analysis/Engine/Impl/Projects/ProjectAnalyzer.cs deleted file mode 100644 index 126bfa37c..000000000 --- a/src/Analysis/Engine/Impl/Projects/ProjectAnalyzer.cs +++ /dev/null @@ -1,53 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Generic; -using System.Threading.Tasks; - -namespace Microsoft.PythonTools.Projects { - public abstract class ProjectAnalyzer { - /// - /// Registers the extension type with the analyzer. The extension must have a - /// public default constructor, as it will be recreated in the out-of-process - /// analyzer. - /// - public abstract Task RegisterExtensionAsync(Type extensionType); - - /// - /// Sends a command to an analysis extension with the specified input and returns - /// the result. - /// - /// The name of the analysis extension, as attributed with - /// AnalysisExtensionNameAttribute. - /// The command that the extension supports and will execute. - /// The input to the command. - /// - public abstract Task SendExtensionCommandAsync(string extensionName, string commandId, string body); - - /// - /// Raised when the analysis is complete for the specified file. - /// - public abstract event EventHandler AnalysisComplete; - - /// - /// Gets the list of files which are being analyzed by this ProjectAnalyzer. - /// - public abstract IEnumerable Files { - get; - } - } -} diff --git a/src/Analysis/Engine/Impl/Projects/PythonProject.cs b/src/Analysis/Engine/Impl/Projects/PythonProject.cs deleted file mode 100644 index 519ddf769..000000000 --- a/src/Analysis/Engine/Impl/Projects/PythonProject.cs +++ /dev/null @@ -1,79 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Threading.Tasks; -using Microsoft.PythonTools.Interpreter; - -namespace Microsoft.PythonTools.Projects { - /// - /// Provides information about a Python project. This is an abstract base class that - /// different project systems can implement. Tools which want to plug in an extend the - /// Python analysis system can work with the PythonProject to get information about - /// the project. - /// - /// This differs from the ProjectAnalyzer class in that it contains more rich information - /// about the configuration of the project related to running and executing. - /// - public abstract class PythonProject { - /// - /// Gets a property for the project. Users can get/set their own properties, also these properties - /// are available: - /// - /// CommandLineArguments -> arguments to be passed to the debugged program. - /// InterpreterPath -> gets a configured directory where the interpreter should be launched from. - /// IsWindowsApplication -> determines whether or not the application is a windows application (for which no console window should be created) - /// - /// - /// - public abstract string GetProperty(string name); - - public abstract string GetUnevaluatedProperty(string name); - - /// - /// Sets a property for the project. See GetProperty for more information on common properties. - /// - /// - /// - public abstract void SetProperty(string name, string value); - - public abstract IPythonInterpreterFactory GetInterpreterFactory(); - - /// - /// Gets the current analyzer for the project, or null if no analyzer is available. - /// - [Obsolete("Use the async version if possible")] - public abstract ProjectAnalyzer Analyzer { get; } - - /// - /// Gets the current analyzer for the project. May wait while creating an analyzer - /// if necessary, where the property would return null. - /// - public abstract Task GetAnalyzerAsync(); - - public abstract event EventHandler ProjectAnalyzerChanged; - - public abstract string ProjectHome { get; } - - /// - /// Attempts to retrieve a PythonProject from the provided object, which - /// should implement . - /// - public static PythonProject FromObject(object source) { - return (source as IPythonProjectProvider)?.Project; - } - } -} diff --git a/src/Analysis/Engine/Impl/Properties/AssemblyInfo.cs b/src/Analysis/Engine/Impl/Properties/AssemblyInfo.cs index 060ebe28d..114d2b066 100644 --- a/src/Analysis/Engine/Impl/Properties/AssemblyInfo.cs +++ b/src/Analysis/Engine/Impl/Properties/AssemblyInfo.cs @@ -17,40 +17,6 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Microsoft.PythonTools.Analyzer, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.BuildTasks, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.Debugger, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.Django, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.EnvironmentsList, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.IronPython.Interpreter, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.TestAdapter, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.TestAdapter.Analysis, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.TestAdapter.Executor, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.Uwp.Interpreter, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.VSInterpreters, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.Python.LanguageServer, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] - -[assembly: InternalsVisibleTo("AnalysisTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("CookiecutterTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("DebuggerTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("DebuggerUITests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("DjangoTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("DjangoUITests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("FastCgiTest, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("IpcJsonTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("IronPythonTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("ProfilingTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("ProfilingUITests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("ProjectUITests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("PythonToolsMockTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("PythonToolsTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("PythonToolsUITests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("ReplWindowUITests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("TestAdapterTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("TestUtilities, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("TestUtilities.Python, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("TestUtilities.Python.Analysis, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("VSInterpretersTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("AnalysisMemoryTester, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.Analysis.Browser, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("Microsoft.Python.LanguageServer.Core, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.Python.Analysis.Engine.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/src/Analysis/Engine/Impl/PythonAnalyzer.cs b/src/Analysis/Engine/Impl/PythonAnalyzer.cs index 14a4a8ec0..982fde416 100644 --- a/src/Analysis/Engine/Impl/PythonAnalyzer.cs +++ b/src/Analysis/Engine/Impl/PythonAnalyzer.cs @@ -129,8 +129,8 @@ private async Task LoadKnownTypesAsync(CancellationToken token) { } private void ReloadModulePaths(in IEnumerable rootPaths) { - foreach (var modulePath in rootPaths.Where(Directory.Exists).SelectMany(p => ModulePath.GetModulesInPath(p))) { - _pathResolver.TryAddModulePath(modulePath.SourceFile, out _); + foreach (var modulePath in rootPaths.Where(Directory.Exists).SelectMany(p => PathUtils.EnumerateFiles(p))) { + _pathResolver.TryAddModulePath(modulePath, out _); } } @@ -201,9 +201,7 @@ public IPythonProjectEntry AddModule(string moduleName, string filePath, Uri doc } return entry; } - - public void RemoveModule(IProjectEntry entry) => RemoveModule(entry, null); - + /// /// Removes the specified project entry from the current analysis. /// @@ -213,9 +211,8 @@ public IPythonProjectEntry AddModule(string moduleName, string filePath, Uri doc /// Action to perform on each module that /// had imported the one being removed. public void RemoveModule(IProjectEntry entry, Action onImporter) { - if (entry == null) { - throw new ArgumentNullException(nameof(entry)); - } + Check.ArgumentNotNull(nameof(entry), entry); + Check.ArgumentNotNull(nameof(onImporter), onImporter); Contract.EndContractBlock(); var pyEntry = entry as IPythonProjectEntry; @@ -237,9 +234,6 @@ public void RemoveModule(IProjectEntry entry, Action onImpo entry.Dispose(); ClearDiagnostics(entry); - if (onImporter == null) { - onImporter = e => e.Analyze(CancellationToken.None, enqueueOnly: true); - } if (!string.IsNullOrEmpty(pyEntry?.ModuleName)) { Modules.TryRemove(pyEntry.ModuleName, out _); @@ -377,17 +371,6 @@ public PythonMemberType GetModuleType() { } } - private static bool GetPackageNameIfMatch(string name, string fullName, out string packageName) { - var lastDot = fullName.LastIndexOf('.'); - if (lastDot < 0) { - packageName = null; - return false; - } - - packageName = fullName.Remove(lastDot); - return String.Compare(fullName, lastDot + 1, name, 0, name.Length, StringComparison.Ordinal) == 0; - } - /// /// Returns the interpreter that the analyzer is using. /// diff --git a/src/Analysis/Engine/Test/AnalysisTest.cs b/src/Analysis/Engine/Test/AnalysisTest.cs index 54d6917ed..b6f0275b2 100644 --- a/src/Analysis/Engine/Test/AnalysisTest.cs +++ b/src/Analysis/Engine/Test/AnalysisTest.cs @@ -29,6 +29,7 @@ using Microsoft.PythonTools.Analysis.Analyzer; using Microsoft.PythonTools.Analysis.FluentAssertions; using Microsoft.PythonTools.Analysis.Values; +using Microsoft.PythonTools.Intellisense; using Microsoft.PythonTools.Interpreter; using Microsoft.PythonTools.Interpreter.Ast; using Microsoft.PythonTools.Parsing; @@ -5576,6 +5577,52 @@ def f(s = 123) -> s: .And.HaveReturnValue().OfTypes(BuiltinTypeId.Int, BuiltinTypeId.NoneType, BuiltinTypeId.Unknown); } } + + + [TestMethod, Priority(0)] + public async Task SysModulesSetSpecialization() { + var code = @"import sys +modules = sys.modules + +modules['name_in_modules'] = None +"; + code += string.Join( + Environment.NewLine, + Enumerable.Range(0, 100).Select(i => $"sys.modules['name{i}'] = None") + ); + + using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); + analysis.Should().HaveVariable("sys").WithValue() + .And.HaveVariable("modules").WithValue(); + } + + } + + + [DataRow("import abc", 7, "abc", "")] + [DataRow("import abc", 8, "abc", "")] + [DataRow("import abc", 9, "abc", "")] + [DataRow("import abc", 10, "abc", "")] + [DataRow("import deg, abc as A",12, "abc", "")] + [DataRow("from abc import A", 6, "abc", "")] + [DataRow("from .deg import A", 9, "deg", "abc")] + [DataRow("from .hij import A", 9, "abc.hij", "abc.deg")] + [DataRow("from ..hij import A", 10, "hij", "abc.deg")] + [DataRow("from ..hij import A", 10, "abc.hij", "abc.deg.HIJ")] + [DataTestMethod, Priority(0)] + public async Task ModuleNameWalker(string code, int index, string expected, string baseCode) { + using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); + var entry = (IPythonProjectEntry)server.GetEntry(analysis.DocumentUri); + var walker = new ImportedModuleNameWalker(baseCode, string.Empty, index, null); + entry.Tree.Walk(walker); + walker.ImportedModules.Should().NotBeEmpty() + .And.NotContainNulls() + .And.Subject.First().Name.Should().Be(expected); + } + } + /* [TestMethod, Priority(0)] public async Task Super() { @@ -5842,32 +5889,6 @@ def fn(self): ); } - [TestMethod, Priority(0)] - public async Task SysModulesSetSpecialization() { - var code = @"import sys -modules = sys.modules - -modules['name_in_modules'] = None -"; - code += string.Join( - Environment.NewLine, - Enumerable.Range(0, 100).Select(i => string.Format("sys.modules['name{0}'] = None", i)) - ); - - var entry = ProcessTextV2(code); - - var sys = entry.GetValue("sys"); - - var modules = entry.GetValue("modules"); - Assert.IsInstanceOfType(modules, typeof(SysModuleInfo.SysModulesDictionaryInfo)); - - AssertUtil.ContainsExactly( - sys.Modules.Keys, - Enumerable.Range(0, 100).Select(i => string.Format("name{0}", i)) - .Concat(new[] { "name_in_modules" }) - ); - } - [TestMethod, Priority(0)] public async Task SysModulesGetSpecialization() { var code = @"import sys @@ -6030,28 +6051,6 @@ public void NullNamedArgument() { } } - [TestMethod, Priority(0)] - public void ModuleNameWalker() { - foreach (var item in new[] { - new { Code="import abc", Index=7, Expected="abc", Base="" }, - new { Code="import abc", Index=8, Expected="abc", Base="" }, - new { Code="import abc", Index=9, Expected="abc", Base="" }, - new { Code="import abc", Index=10, Expected="abc", Base="" }, - new { Code="import deg, abc as A", Index=12, Expected="abc", Base="" }, - new { Code="from abc import A", Index=6, Expected="abc", Base="" }, - new { Code="from .deg import A", Index=9, Expected="deg", Base="abc" }, - new { Code="from .hij import A", Index=9, Expected="abc.hij", Base="abc.deg" }, - new { Code="from ..hij import A", Index=10, Expected="hij", Base="abc.deg" }, - new { Code="from ..hij import A", Index=10, Expected="abc.hij", Base="abc.deg.HIJ" }, - }) { - var entry = ProcessTextV3(item.Code); - var walker = new ImportedModuleNameWalker(item.Base, string.Empty, item.Index, null); - entry.Modules[entry.DefaultModule].Tree.Walk(walker); - - Assert.AreEqual(item.Expected, walker.ImportedModules.FirstOrDefault()?.Name); - } - } - [TestMethod, Priority(0)] public void CrossModuleFunctionCallMemLeak() { var modA = @"from B import h diff --git a/src/Analysis/Engine/Test/ImportTests.cs b/src/Analysis/Engine/Test/ImportTests.cs index a3dc62a6e..4f1ad0acd 100644 --- a/src/Analysis/Engine/Test/ImportTests.cs +++ b/src/Analysis/Engine/Test/ImportTests.cs @@ -98,6 +98,67 @@ import package.sub_package.module2 completionModule2.Should().HaveLabels("Y").And.NotContainLabels("X"); } + [ServerTestMethod(LatestAvailable3X = true, TestSpecificRootUri = true), Priority(0)] + public async Task Completions_ImportResolution_UserSearchPathsInsideWorkspace(Server server) { + var folder1 = TestData.GetTestSpecificPath("folder1"); + var folder2 = TestData.GetTestSpecificPath("folder2"); + var packageInFolder1 = Path.Combine(folder1, "package"); + var packageInFolder2 = Path.Combine(folder2, "package"); + var module1Path = Path.Combine(packageInFolder1, "module1.py"); + var module2Path = Path.Combine(packageInFolder2, "module2.py"); + var module1Content = @"class A(): + @staticmethod + def method1(): + pass"; + var module2Content = @"class B(): + @staticmethod + def method2(): + pass"; + var mainContent = @"from package import module1 as mod1, module2 as mod2 +mod1. +mod2. +mod1.A. +mod2.B."; + + server.Analyzer.SetSearchPaths(new[] { folder1, folder2 }); + + await server.OpenDocumentAndGetUriAsync(module1Path, module1Content); + await server.OpenDocumentAndGetUriAsync(module2Path, module2Content); + var uri = await server.OpenDocumentAndGetUriAsync("main.py", mainContent); + + await server.WaitForCompleteAnalysisAsync(CancellationToken.None); + + var completionMod1 = await server.SendCompletion(uri, 1, 5); + var completionMod2 = await server.SendCompletion(uri, 2, 5); + var completionA = await server.SendCompletion(uri, 3, 7); + var completionB = await server.SendCompletion(uri, 4, 7); + completionMod1.Should().HaveLabels("A").And.NotContainLabels("B"); + completionMod2.Should().HaveLabels("B").And.NotContainLabels("A"); + completionA.Should().HaveLabels("method1"); + completionB.Should().HaveLabels("method2"); + } + + [ServerTestMethod(LatestAvailable3X = true, TestSpecificRootUri = true), Priority(0)] + [Ignore("https://github.com/Microsoft/python-language-server/issues/468")] + public async Task Completions_ImportResolution_ModuleInWorkspaceAndInUserSearchPath(Server server) { + var extraSearchPath = TestData.GetTestSpecificPath(Path.Combine("some", "other")); + var module1Path = TestData.GetTestSpecificPath("module.py"); + var module2Path = Path.Combine(extraSearchPath, "module.py"); + var module1Content = "A = 1"; + var module2Content = "B = 2"; + var mainContent = @"import module as mod; mod."; + + server.Analyzer.SetSearchPaths(new[] { extraSearchPath }); + + await server.OpenDocumentAndGetUriAsync(module1Path, module1Content); + await server.OpenDocumentAndGetUriAsync(module2Path, module2Content); + var uri = await server.OpenDocumentAndGetUriAsync("main.py", mainContent); + + await server.WaitForCompleteAnalysisAsync(CancellationToken.None); + var completion = await server.SendCompletion(uri, 0, 26); + completion.Should().HaveLabels("A").And.NotContainLabels("B"); + } + [Ignore("https://github.com/Microsoft/python-language-server/issues/443")] [ServerTestMethod(LatestAvailable3X = true, TestSpecificRootUri = true), Priority(0)] public async Task Completions_ImportResolution_OneSearchPathInsideAnother(Server server) { diff --git a/src/Analysis/Engine/Test/ServerExtensions.cs b/src/Analysis/Engine/Test/ServerExtensions.cs index b6041b26a..52813934b 100644 --- a/src/Analysis/Engine/Test/ServerExtensions.cs +++ b/src/Analysis/Engine/Test/ServerExtensions.cs @@ -164,6 +164,14 @@ await server.DidOpenTextDocument(new DidOpenTextDocumentParams { }, GetCancellationToken()); } + public static async Task OpenDocumentAndGetAnalysisAsync(this Server server, string relativePath, string content, int failAfter = 30000, string languageId = null) { + var cancellationToken = GetCancellationToken(failAfter); + var uri = TestData.GetTestSpecificUri(relativePath); + await server.SendDidOpenTextDocument(uri, content, languageId); + cancellationToken.ThrowIfCancellationRequested(); + return await server.GetAnalysisAsync(uri, cancellationToken); + } + public static async Task OpenDefaultDocumentAndGetAnalysisAsync(this Server server, string content, int failAfter = 30000, string languageId = null) { var cancellationToken = GetCancellationToken(failAfter); await server.SendDidOpenTextDocument(TestData.GetDefaultModuleUri(), content, languageId); diff --git a/src/Analysis/Engine/Test/ThreadingTest.cs b/src/Analysis/Engine/Test/ThreadingTest.cs index 5069dddea..23b954368 100644 --- a/src/Analysis/Engine/Test/ThreadingTest.cs +++ b/src/Analysis/Engine/Test/ThreadingTest.cs @@ -131,11 +131,11 @@ class MyClass: // One analysis before we start foreach (var e in entries) { - e.Analyze(cancel, true); + e.PreAnalyze(); } state.AnalyzeQueuedEntries(cancel); - // Repeatedly re-analyse the code + // Repeatedly re-analyze the code yield return Task.Run(() => { var rnd = new Random(); while (!cancel.IsCancellationRequested) { @@ -146,7 +146,7 @@ class MyClass: .Select(t => t.Item2) .ToList(); foreach (var e in shufEntries) { - e.Analyze(cancel, true); + e.PreAnalyze(); } state.AnalyzeQueuedEntries(cancel); diff --git a/src/LanguageServer/Impl/Definitions/CallbackEventArgs.cs b/src/LanguageServer/Core/Impl/Definitions/CallbackEventArgs.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/CallbackEventArgs.cs rename to src/LanguageServer/Core/Impl/Definitions/CallbackEventArgs.cs diff --git a/src/LanguageServer/Impl/Definitions/EditorOperationException.cs b/src/LanguageServer/Core/Impl/Definitions/EditorOperationException.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/EditorOperationException.cs rename to src/LanguageServer/Core/Impl/Definitions/EditorOperationException.cs diff --git a/src/LanguageServer/Impl/Definitions/Enums.cs b/src/LanguageServer/Core/Impl/Definitions/Enums.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/Enums.cs rename to src/LanguageServer/Core/Impl/Definitions/Enums.cs diff --git a/src/LanguageServer/Impl/Definitions/IDocumentReader.cs b/src/LanguageServer/Core/Impl/Definitions/IDocumentReader.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/IDocumentReader.cs rename to src/LanguageServer/Core/Impl/Definitions/IDocumentReader.cs diff --git a/src/LanguageServer/Impl/Definitions/ILanguageServerExtensionProvider.cs b/src/LanguageServer/Core/Impl/Definitions/ILanguageServerExtensionProvider.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/ILanguageServerExtensionProvider.cs rename to src/LanguageServer/Core/Impl/Definitions/ILanguageServerExtensionProvider.cs diff --git a/src/LanguageServer/Impl/Definitions/ILogger.cs b/src/LanguageServer/Core/Impl/Definitions/ILogger.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/ILogger.cs rename to src/LanguageServer/Core/Impl/Definitions/ILogger.cs diff --git a/src/LanguageServer/Impl/Definitions/IProgressService.cs b/src/LanguageServer/Core/Impl/Definitions/IProgressService.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/IProgressService.cs rename to src/LanguageServer/Core/Impl/Definitions/IProgressService.cs diff --git a/src/LanguageServer/Impl/Definitions/IPythonLanguageServer.cs b/src/LanguageServer/Core/Impl/Definitions/IPythonLanguageServer.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/IPythonLanguageServer.cs rename to src/LanguageServer/Core/Impl/Definitions/IPythonLanguageServer.cs diff --git a/src/LanguageServer/Impl/Definitions/IPythonLanguageServerProtocol.cs b/src/LanguageServer/Core/Impl/Definitions/IPythonLanguageServerProtocol.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/IPythonLanguageServerProtocol.cs rename to src/LanguageServer/Core/Impl/Definitions/IPythonLanguageServerProtocol.cs diff --git a/src/LanguageServer/Impl/Definitions/IServiceContainer.cs b/src/LanguageServer/Core/Impl/Definitions/IServiceContainer.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/IServiceContainer.cs rename to src/LanguageServer/Core/Impl/Definitions/IServiceContainer.cs diff --git a/src/LanguageServer/Impl/Definitions/ITelemetryService.cs b/src/LanguageServer/Core/Impl/Definitions/ITelemetryService.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/ITelemetryService.cs rename to src/LanguageServer/Core/Impl/Definitions/ITelemetryService.cs diff --git a/src/LanguageServer/Impl/Definitions/ITextChangeNotifications.cs b/src/LanguageServer/Core/Impl/Definitions/ITextChangeNotifications.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/ITextChangeNotifications.cs rename to src/LanguageServer/Core/Impl/Definitions/ITextChangeNotifications.cs diff --git a/src/LanguageServer/Impl/Definitions/IUIService.cs b/src/LanguageServer/Core/Impl/Definitions/IUIService.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/IUIService.cs rename to src/LanguageServer/Core/Impl/Definitions/IUIService.cs diff --git a/src/LanguageServer/Impl/Definitions/Messages.cs b/src/LanguageServer/Core/Impl/Definitions/Messages.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/Messages.cs rename to src/LanguageServer/Core/Impl/Definitions/Messages.cs diff --git a/src/LanguageServer/Impl/Definitions/ServerSettings.cs b/src/LanguageServer/Core/Impl/Definitions/ServerSettings.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/ServerSettings.cs rename to src/LanguageServer/Core/Impl/Definitions/ServerSettings.cs diff --git a/src/LanguageServer/Impl/Definitions/Structures.cs b/src/LanguageServer/Core/Impl/Definitions/Structures.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/Structures.cs rename to src/LanguageServer/Core/Impl/Definitions/Structures.cs diff --git a/src/LanguageServer/Impl/Extensions/ICompletionExtension.cs b/src/LanguageServer/Core/Impl/Extensions/ICompletionExtension.cs similarity index 100% rename from src/LanguageServer/Impl/Extensions/ICompletionExtension.cs rename to src/LanguageServer/Core/Impl/Extensions/ICompletionExtension.cs diff --git a/src/LanguageServer/Impl/Extensions/ILanguageServerExtension.cs b/src/LanguageServer/Core/Impl/Extensions/ILanguageServerExtension.cs similarity index 100% rename from src/LanguageServer/Impl/Extensions/ILanguageServerExtension.cs rename to src/LanguageServer/Core/Impl/Extensions/ILanguageServerExtension.cs diff --git a/src/LanguageServer/Impl/Implementation/BlockFormatter.cs b/src/LanguageServer/Core/Impl/Implementation/BlockFormatter.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/BlockFormatter.cs rename to src/LanguageServer/Core/Impl/Implementation/BlockFormatter.cs diff --git a/src/LanguageServer/Impl/Implementation/CodeActionProvider.cs b/src/LanguageServer/Core/Impl/Implementation/CodeActionProvider.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/CodeActionProvider.cs rename to src/LanguageServer/Core/Impl/Implementation/CodeActionProvider.cs diff --git a/src/LanguageServer/Impl/Implementation/CompletionAnalysis.cs b/src/LanguageServer/Core/Impl/Implementation/CompletionAnalysis.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/CompletionAnalysis.cs rename to src/LanguageServer/Core/Impl/Implementation/CompletionAnalysis.cs diff --git a/src/LanguageServer/Impl/Implementation/DiagnosticsErrorSink.cs b/src/LanguageServer/Core/Impl/Implementation/DiagnosticsErrorSink.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/DiagnosticsErrorSink.cs rename to src/LanguageServer/Core/Impl/Implementation/DiagnosticsErrorSink.cs diff --git a/src/LanguageServer/Impl/Implementation/DocumentReader.cs b/src/LanguageServer/Core/Impl/Implementation/DocumentReader.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/DocumentReader.cs rename to src/LanguageServer/Core/Impl/Implementation/DocumentReader.cs diff --git a/src/LanguageServer/Impl/Implementation/EditorFiles.cs b/src/LanguageServer/Core/Impl/Implementation/EditorFiles.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/EditorFiles.cs rename to src/LanguageServer/Core/Impl/Implementation/EditorFiles.cs diff --git a/src/LanguageServer/Impl/Implementation/LineFormatter.cs b/src/LanguageServer/Core/Impl/Implementation/LineFormatter.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/LineFormatter.cs rename to src/LanguageServer/Core/Impl/Implementation/LineFormatter.cs diff --git a/src/LanguageServer/Impl/Implementation/ParseQueue.cs b/src/LanguageServer/Core/Impl/Implementation/ParseQueue.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/ParseQueue.cs rename to src/LanguageServer/Core/Impl/Implementation/ParseQueue.cs diff --git a/src/LanguageServer/Impl/Implementation/ProjectFiles.cs b/src/LanguageServer/Core/Impl/Implementation/ProjectFiles.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/ProjectFiles.cs rename to src/LanguageServer/Core/Impl/Implementation/ProjectFiles.cs diff --git a/src/LanguageServer/Impl/Implementation/RestTextConverter.cs b/src/LanguageServer/Core/Impl/Implementation/RestTextConverter.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/RestTextConverter.cs rename to src/LanguageServer/Core/Impl/Implementation/RestTextConverter.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.Completion.cs b/src/LanguageServer/Core/Impl/Implementation/Server.Completion.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.Completion.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.Completion.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.Extensions.cs b/src/LanguageServer/Core/Impl/Implementation/Server.Extensions.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.Extensions.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.Extensions.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.FindReferences.cs b/src/LanguageServer/Core/Impl/Implementation/Server.FindReferences.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.FindReferences.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.FindReferences.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.GoToDefinition.cs b/src/LanguageServer/Core/Impl/Implementation/Server.GoToDefinition.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.GoToDefinition.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.GoToDefinition.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.Hover.cs b/src/LanguageServer/Core/Impl/Implementation/Server.Hover.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.Hover.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.Hover.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.OnTypeFormatting.cs b/src/LanguageServer/Core/Impl/Implementation/Server.OnTypeFormatting.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.OnTypeFormatting.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.OnTypeFormatting.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.PrivateHelpers.cs b/src/LanguageServer/Core/Impl/Implementation/Server.PrivateHelpers.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.PrivateHelpers.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.PrivateHelpers.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.Rename.cs b/src/LanguageServer/Core/Impl/Implementation/Server.Rename.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.Rename.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.Rename.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.SignatureHelp.cs b/src/LanguageServer/Core/Impl/Implementation/Server.SignatureHelp.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.SignatureHelp.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.SignatureHelp.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Core/Impl/Implementation/Server.WorkspaceSymbols.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.WorkspaceSymbols.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Core/Impl/Implementation/Server.cs similarity index 99% rename from src/LanguageServer/Impl/Implementation/Server.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.cs index 8a6a1e6c0..d5fde147d 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Core/Impl/Implementation/Server.cs @@ -583,7 +583,7 @@ internal Task EnqueueItemAsync(IDocument doc, AnalysisPriority priority = Analys var pending = _pendingAnalysisEnqueue.Incremented(); try { var entry = doc as ProjectEntry; - entry?.ResetCompleteAnalysis(); + entry?.NewAnalysisAwaitableOnParse(); // If we don't need to parse, use null cookie var cookieTask = Task.FromResult(null); diff --git a/src/LanguageServer/Impl/Implementation/ServerBase.cs b/src/LanguageServer/Core/Impl/Implementation/ServerBase.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/ServerBase.cs rename to src/LanguageServer/Core/Impl/Implementation/ServerBase.cs diff --git a/src/LanguageServer/Impl/Implementation/VolatileCounter.cs b/src/LanguageServer/Core/Impl/Implementation/VolatileCounter.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/VolatileCounter.cs rename to src/LanguageServer/Core/Impl/Implementation/VolatileCounter.cs diff --git a/src/LanguageServer/Core/Impl/Microsoft.Python.LanguageServer.Core.csproj b/src/LanguageServer/Core/Impl/Microsoft.Python.LanguageServer.Core.csproj new file mode 100644 index 000000000..631c3e87a --- /dev/null +++ b/src/LanguageServer/Core/Impl/Microsoft.Python.LanguageServer.Core.csproj @@ -0,0 +1,41 @@ + + + netstandard2.0 + Microsoft.Python.LanguageServer + Microsoft.Python.LanguageServer.Core + + + + 1701;1702;1998;$(NoWarn) + 7.2 + + + + + + all + runtime; build; native; contentfiles; analyzers + + + + + + + + True + True + Resources.resx + + + + + ResXFileCodeGenerator + Resources.Designer.cs + + + + + diff --git a/src/LanguageServer/Core/Impl/Properties/AssemblyInfo.cs b/src/LanguageServer/Core/Impl/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..1d7853194 --- /dev/null +++ b/src/LanguageServer/Core/Impl/Properties/AssemblyInfo.cs @@ -0,0 +1,21 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Microsoft.PythonTools.Analyzer, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("Microsoft.Python.LanguageServer, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("Microsoft.Python.Analysis.Engine.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/src/LanguageServer/Core/Impl/Resources.Designer.cs b/src/LanguageServer/Core/Impl/Resources.Designer.cs new file mode 100644 index 000000000..902a60fed --- /dev/null +++ b/src/LanguageServer/Core/Impl/Resources.Designer.cs @@ -0,0 +1,168 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace Microsoft.Python.LanguageServer { + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "15.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + internal class Resources { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal Resources() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("Microsoft.Python.LanguageServer.Resources", typeof(Resources).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to done.. + /// + internal static string Done { + get { + return ResourceManager.GetString("Done", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Failed to create interpreter. + /// + internal static string Error_FailedToCreateInterpreter { + get { + return ResourceManager.GetString("Error_FailedToCreateInterpreter", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Initializing for generic interpreter. + /// + internal static string InitializingForGenericInterpreter { + get { + return ResourceManager.GetString("InitializingForGenericInterpreter", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Initializing for {0}. + /// + internal static string InitializingForPythonInterpreter { + get { + return ResourceManager.GetString("InitializingForPythonInterpreter", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Microsoft Python Language Server version {0}. + /// + internal static string LanguageServerVersion { + get { + return ResourceManager.GetString("LanguageServerVersion", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Unmatched token '{0}' on line {1}; line formatting may not be accurate.. + /// + internal static string LineFormatter_UnmatchedToken { + get { + return ResourceManager.GetString("LineFormatter_UnmatchedToken", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Reloading modules... . + /// + internal static string ReloadingModules { + get { + return ResourceManager.GetString("ReloadingModules", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Cannot rename. + /// + internal static string RenameVariable_CannotRename { + get { + return ResourceManager.GetString("RenameVariable_CannotRename", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Cannot rename modules. + /// + internal static string RenameVariable_CannotRenameModuleName { + get { + return ResourceManager.GetString("RenameVariable_CannotRenameModuleName", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to No information is available for the variable '{0}'.. + /// + internal static string RenameVariable_NoInformationAvailableForVariable { + get { + return ResourceManager.GetString("RenameVariable_NoInformationAvailableForVariable", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Please select a symbol to be renamed.. + /// + internal static string RenameVariable_SelectSymbol { + get { + return ResourceManager.GetString("RenameVariable_SelectSymbol", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Unable to get analysis for the selected expression.. + /// + internal static string RenameVariable_UnableGetExpressionAnalysis { + get { + return ResourceManager.GetString("RenameVariable_UnableGetExpressionAnalysis", resourceCulture); + } + } + } +} diff --git a/src/LanguageServer/Core/Impl/Resources.resx b/src/LanguageServer/Core/Impl/Resources.resx new file mode 100644 index 000000000..6d9943476 --- /dev/null +++ b/src/LanguageServer/Core/Impl/Resources.resx @@ -0,0 +1,137 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 1.3 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=2.0.3500.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=2.0.3500.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + done. + + + Failed to create interpreter + + + Initializing for generic interpreter + + + Initializing for {0} + + + Microsoft Python Language Server version {0} + + + Unmatched token '{0}' on line {1}; line formatting may not be accurate. + + + Reloading modules... + + + Cannot rename + + + Cannot rename modules + + + No information is available for the variable '{0}'. + + + Please select a symbol to be renamed. + + + Unable to get analysis for the selected expression. + + \ No newline at end of file diff --git a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj index 77702d03c..ab661828f 100644 --- a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj +++ b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj @@ -33,6 +33,7 @@ + diff --git a/src/LanguageServer/Impl/Resources.Designer.cs b/src/LanguageServer/Impl/Resources.Designer.cs index 33c50439f..5f9eb5335 100644 --- a/src/LanguageServer/Impl/Resources.Designer.cs +++ b/src/LanguageServer/Impl/Resources.Designer.cs @@ -60,15 +60,6 @@ internal Resources() { } } - /// - /// Looks up a localized string similar to done.. - /// - internal static string Done { - get { - return ResourceManager.GetString("Done", resourceCulture); - } - } - /// /// Looks up a localized string similar to Failed to create interpreter. /// @@ -77,95 +68,5 @@ internal static string Error_FailedToCreateInterpreter { return ResourceManager.GetString("Error_FailedToCreateInterpreter", resourceCulture); } } - - /// - /// Looks up a localized string similar to Initializing for generic interpreter. - /// - internal static string InitializingForGenericInterpreter { - get { - return ResourceManager.GetString("InitializingForGenericInterpreter", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Initializing for {0}. - /// - internal static string InitializingForPythonInterpreter { - get { - return ResourceManager.GetString("InitializingForPythonInterpreter", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Microsoft Python Language Server version {0}. - /// - internal static string LanguageServerVersion { - get { - return ResourceManager.GetString("LanguageServerVersion", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Unmatched token '{0}' on line {1}; line formatting may not be accurate.. - /// - internal static string LineFormatter_UnmatchedToken { - get { - return ResourceManager.GetString("LineFormatter_UnmatchedToken", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Reloading modules... . - /// - internal static string ReloadingModules { - get { - return ResourceManager.GetString("ReloadingModules", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Cannot rename. - /// - internal static string RenameVariable_CannotRename { - get { - return ResourceManager.GetString("RenameVariable_CannotRename", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Cannot rename modules. - /// - internal static string RenameVariable_CannotRenameModuleName { - get { - return ResourceManager.GetString("RenameVariable_CannotRenameModuleName", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to No information is available for the variable '{0}'.. - /// - internal static string RenameVariable_NoInformationAvailableForVariable { - get { - return ResourceManager.GetString("RenameVariable_NoInformationAvailableForVariable", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Please select a symbol to be renamed.. - /// - internal static string RenameVariable_SelectSymbol { - get { - return ResourceManager.GetString("RenameVariable_SelectSymbol", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Unable to get analysis for the selected expression.. - /// - internal static string RenameVariable_UnableGetExpressionAnalysis { - get { - return ResourceManager.GetString("RenameVariable_UnableGetExpressionAnalysis", resourceCulture); - } - } } } diff --git a/src/LanguageServer/Impl/Resources.resx b/src/LanguageServer/Impl/Resources.resx index 630472020..f78a79bc5 100644 --- a/src/LanguageServer/Impl/Resources.resx +++ b/src/LanguageServer/Impl/Resources.resx @@ -117,40 +117,7 @@ System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - done. - Failed to create interpreter - - Initializing for generic interpreter - - - Initializing for {0} - - - Microsoft Python Language Server version {0} - - - Unmatched token '{0}' on line {1}; line formatting may not be accurate. - - - Reloading modules... - - - Cannot rename - - - Cannot rename modules - - - No information is available for the variable '{0}'. - - - Please select a symbol to be renamed. - - - Unable to get analysis for the selected expression. - \ No newline at end of file diff --git a/src/PLS.sln b/src/PLS.sln index a5f064534..499b9218e 100644 --- a/src/PLS.sln +++ b/src/PLS.sln @@ -15,6 +15,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Analysis.E EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Analysis.Engine.Tests", "Analysis\Engine\Test\Microsoft.Python.Analysis.Engine.Tests.csproj", "{1CFA416B-6932-432F-8C75-34B5615D7664}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.PythonTools.Analyzer", "PTVS\Microsoft.PythonTools.Analyzer\Impl\Microsoft.PythonTools.Analyzer.csproj", "{467679B2-D043-4C7D-855C-5A833B635EEE}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.PythonTools.Ipc.Json", "PTVS\Microsoft.PythonTools.Ipc.Json\Impl\Microsoft.PythonTools.Ipc.Json.csproj", "{A5E04CC1-F600-45BA-B493-F7DCBA8C7E1A}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.LanguageServer.Core", "LanguageServer\Core\Impl\Microsoft.Python.LanguageServer.Core.csproj", "{0EA1F1C3-2733-423C-BEC5-059BD77F0A3F}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "PTVS", "PTVS", "{99A92748-7947-412A-BA92-5F4E2AA63918}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -37,6 +45,18 @@ Global {1CFA416B-6932-432F-8C75-34B5615D7664}.Debug|Any CPU.Build.0 = Debug|Any CPU {1CFA416B-6932-432F-8C75-34B5615D7664}.Release|Any CPU.ActiveCfg = Release|Any CPU {1CFA416B-6932-432F-8C75-34B5615D7664}.Release|Any CPU.Build.0 = Release|Any CPU + {467679B2-D043-4C7D-855C-5A833B635EEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {467679B2-D043-4C7D-855C-5A833B635EEE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {467679B2-D043-4C7D-855C-5A833B635EEE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {467679B2-D043-4C7D-855C-5A833B635EEE}.Release|Any CPU.Build.0 = Release|Any CPU + {A5E04CC1-F600-45BA-B493-F7DCBA8C7E1A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A5E04CC1-F600-45BA-B493-F7DCBA8C7E1A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A5E04CC1-F600-45BA-B493-F7DCBA8C7E1A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A5E04CC1-F600-45BA-B493-F7DCBA8C7E1A}.Release|Any CPU.Build.0 = Release|Any CPU + {0EA1F1C3-2733-423C-BEC5-059BD77F0A3F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0EA1F1C3-2733-423C-BEC5-059BD77F0A3F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0EA1F1C3-2733-423C-BEC5-059BD77F0A3F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0EA1F1C3-2733-423C-BEC5-059BD77F0A3F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -46,6 +66,9 @@ Global {B1F6F2EA-6465-4D63-9457-D856F17EDF38} = {80AA38A1-3E82-4B87-BB21-FDEDD2CC87E6} {55679AE9-8C3D-4F26-A0B5-10A5B2F1789F} = {C465393D-145E-4695-A7DB-AF55951BD533} {1CFA416B-6932-432F-8C75-34B5615D7664} = {80AA38A1-3E82-4B87-BB21-FDEDD2CC87E6} + {467679B2-D043-4C7D-855C-5A833B635EEE} = {99A92748-7947-412A-BA92-5F4E2AA63918} + {A5E04CC1-F600-45BA-B493-F7DCBA8C7E1A} = {99A92748-7947-412A-BA92-5F4E2AA63918} + {0EA1F1C3-2733-423C-BEC5-059BD77F0A3F} = {C465393D-145E-4695-A7DB-AF55951BD533} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {ABC12ED7-0EC8-4219-8A14-A058F7942D92} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/AnalyzerScrapers.pyproj b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/AnalyzerScrapers.pyproj new file mode 100644 index 000000000..ba7bd6733 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/AnalyzerScrapers.pyproj @@ -0,0 +1,41 @@ + + + + Debug + 2.0 + {1fc96711-4e42-4c41-b7fc-b4141d57d0bb} + + PythonScraper.py + + . + . + {888888a0-9f3d-457c-b088-3a5042f75d52} + Standard Python launcher + {2af0f10d-7135-4994-9156-5d01c9c11b7e} + 2.7 + "C:\Users\steve_000\AppData\Local\Python Tools\CompletionDB\Debug\12.0\2af0f10d-7135-4994-9156-5d01c9c11b7e\2.7" "C:\USERS\STEVE_000\APPDATA\LOCAL\MICROSOFT\VISUALSTUDIO\12.0EXP\EXTENSIONS\MICROSOFT\PYTHON TOOLS FOR VISUAL STUDIO\2.1\CompletionDB" + False + + + + + 10.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets + + + + + Code + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/AnalyzerScrapers.sln b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/AnalyzerScrapers.sln new file mode 100644 index 000000000..c5f284c7a --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/AnalyzerScrapers.sln @@ -0,0 +1,20 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2013 +VisualStudioVersion = 12.0.30501.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "AnalyzerScrapers", "AnalyzerScrapers.pyproj", "{1FC96711-4E42-4C41-B7FC-B4141D57D0BB}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {1FC96711-4E42-4C41-B7FC-B4141D57D0BB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1FC96711-4E42-4C41-B7FC-B4141D57D0BB}.Release|Any CPU.ActiveCfg = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/App.config b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/App.config new file mode 100644 index 000000000..fb43a1004 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/App.config @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/AssemblyInfo.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/AssemblyInfo.cs new file mode 100644 index 000000000..2732f17e2 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/AssemblyInfo.cs @@ -0,0 +1,22 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyDescription("Performs analysis of the Python standard library and installed site packages.")] + +[assembly: ComVisible(false)] diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/BuiltinScraper.py b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/BuiltinScraper.py new file mode 100644 index 000000000..3ec651b36 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/BuiltinScraper.py @@ -0,0 +1,554 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABILITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +import re +import sys +import types +import PythonScraper +try: + import thread +except: + import _thread as thread + +try: + import __builtin__ as __builtins__ +except ImportError: + import builtins as __builtins__ + +def safe_dir(obj): + try: + return frozenset(obj.__dict__) | frozenset(dir(obj)) + except: + # Some types crash when we access __dict__ and/or dir() + pass + try: + return frozenset(dir(obj)) + except: + pass + try: + return frozenset(obj.__dict__) + except: + pass + return frozenset() + +def builtins_keys(): + if isinstance(__builtins__, dict): + return __builtins__.keys() + return dir(__builtins__) + +def get_builtin(name): + if isinstance(__builtins__, dict): + return __builtins__[name] + + return getattr(__builtins__, name) + +safe_getattr = PythonScraper.safe_getattr + +BUILTIN_TYPES = [type_name for type_name in builtins_keys() if type(get_builtin(type_name)) is type] +if sys.version_info[0] >= 3: + BUILTIN = 'builtins' + unicode = str +else: + BUILTIN = '__builtin__' + +TYPE_OVERRIDES = { + 'string': PythonScraper.type_to_typeref(types.CodeType), + 's': PythonScraper.type_to_typeref(str), + 'integer': PythonScraper.type_to_typeref(int), + 'boolean': PythonScraper.type_to_typeref(bool), + 'number': PythonScraper.type_to_typeref(int), + 'pid': PythonScraper.type_to_typeref(int), + 'ppid': PythonScraper.type_to_typeref(int), + 'fd': PythonScraper.type_to_typeref(int), + 'handle': PythonScraper.type_to_typeref(int), + 'Exit': PythonScraper.type_to_typeref(int), + 'fd2': PythonScraper.type_to_typeref(int), + 'Integral': PythonScraper.type_to_typeref(int), + 'exit_status':PythonScraper.type_to_typeref(int), + 'old_mask': PythonScraper.type_to_typeref(int), + 'source': PythonScraper.type_to_typeref(str), + 'newpos': PythonScraper.type_to_typeref(int), + 'key': PythonScraper.type_to_typeref(str), + 'dictionary': PythonScraper.type_to_typeref(dict), + 'None': PythonScraper.type_to_typeref(type(None)), + 'floating': PythonScraper.type_to_typeref(float), + 'filename': PythonScraper.type_to_typeref(str), + 'path': PythonScraper.type_to_typeref(str), + 'byteswritten': PythonScraper.type_to_typeref(int), + 'unicode': PythonScraper.type_to_typeref(unicode), + 'Unicode': PythonScraper.type_to_typeref(unicode), + 'True': PythonScraper.type_to_typeref(bool), + 'False': PythonScraper.type_to_typeref(bool), + 'lock': PythonScraper.type_to_typeref(thread.LockType), + 'code': PythonScraper.type_to_typeref(types.CodeType), + 'module': PythonScraper.type_to_typeref(types.ModuleType), + 'size': PythonScraper.type_to_typeref(int), + 'INT': PythonScraper.type_to_typeref(int), + 'STRING': PythonScraper.type_to_typeref(str), + 'TUPLE': PythonScraper.type_to_typeref(tuple), + 'OBJECT': PythonScraper.type_to_typeref(object), + 'LIST': PythonScraper.type_to_typeref(list), + 'DICT': PythonScraper.type_to_typeref(dict), + 'char *': PythonScraper.type_to_typeref(str), + 'wchar_t *': PythonScraper.type_to_typeref(unicode), + 'CHAR *': PythonScraper.type_to_typeref(str), + 'TCHAR *': PythonScraper.type_to_typeref(str), + 'WCHAR *': PythonScraper.type_to_typeref(unicode), + 'LPSTR': PythonScraper.type_to_typeref(str), + 'LPCSTR': PythonScraper.type_to_typeref(str), + 'LPTSTR': PythonScraper.type_to_typeref(str), + 'LPCTSTR': PythonScraper.type_to_typeref(str), + 'LPWSTR': PythonScraper.type_to_typeref(unicode), + 'LPCWSTR': PythonScraper.type_to_typeref(unicode), +} + +try: + TYPE_OVERRIDES['file object'] = PythonScraper.type_to_typeref(file) +except NameError: + try: + import _io + TYPE_OVERRIDES['file object'] = PythonScraper.type_to_typeref(_io._IOBase) + except (NameError, ImportError): + pass + +RETURN_TYPE_OVERRIDES = dict(TYPE_OVERRIDES) +RETURN_TYPE_OVERRIDES.update({'string': PythonScraper.type_to_typeref(str)}) + +def type_name_to_typeref(name, mod, type_overrides = TYPE_OVERRIDES): + arg_type = type_overrides.get(name, None) + if arg_type is None: + if name in BUILTIN_TYPES: + arg_type = PythonScraper.type_to_typeref(get_builtin(name)) + elif mod is not None and name in mod.__dict__: + arg_type = PythonScraper.typename_to_typeref(mod.__name__, name) + elif name.startswith('list'): + arg_type = PythonScraper.type_to_typeref(list) + else: + # see if we can find it in any module we've imported... + for mod_name, mod in list(sys.modules.items()): + if mod is not None and name in mod.__dict__ and isinstance(mod.__dict__[name], type): + arg_type = PythonScraper.typename_to_typeref(mod_name, name) + break + else: + first_space = name.find(' ') + if first_space != -1: + return type_name_to_typeref(name[:first_space], mod, type_overrides) + arg_type = PythonScraper.typename_to_typeref(name) + return arg_type + +OBJECT_TYPE = PythonScraper.type_to_typeref(object) + +TOKENS_REGEX = '(' + '|'.join([ + r'(?:[a-zA-Z_][0-9a-zA-Z_-]*)', # identifier + r'(?:-?[0-9]+[lL]?(?!\.))', # integer value + r'(?:-?[0-9]*\.[0-9]+)', # floating point value + r'(?:-?[0-9]+\.[0-9]*)', # floating point value + r'(?:\s*\'.*?(?)', # return value + r'(?:->)', # return value + r'(?:=>)', # return value + r'(?:,)', # comma + r'(?:=)', # assignment (default value) + r'(?:\[)', + r'(?:\])', + r'(?:\*\*)', + r'(?:\*)', +]) + ')' + +def get_ret_type(ret_type, obj_class, mod): + if ret_type is not None: + if ret_type == 'copy' and obj_class is not None: + # returns a copy of self + return PythonScraper.type_to_typelist(obj_class) + else: + return [type_name_to_typeref(ret_type, mod, RETURN_TYPE_OVERRIDES)] + + +RETURNS_REGEX = [r'^\s*returns?[\s\-]*[a-z_]\w*\s*:\s*([a-z_]\w*)'] + +def update_overload_from_doc_str(overload, doc_str, obj_class, mod): + # see if we can get additional information from the doc string + if 'ret_type' not in overload: + for ret_regex in RETURNS_REGEX: + match = re.search(ret_regex, doc_str, re.MULTILINE | re.IGNORECASE) + if match: + ret_type = match.groups(0)[0] + overload['ret_type'] = get_ret_type(ret_type, obj_class, mod) + break + + +def parse_doc_str(input_str, module_name, mod, func_name, extra_args = [], obj_class = None): + # we split, so as long as we have all tokens every other item is a token, and the + # rest are empty space. If we have unrecognized tokens (for example during the description + # of the function) those will show up in the even locations. We do join's and bring the + # entire range together in that case. + tokens = re.split(TOKENS_REGEX, input_str) + start_token = 0 + last_identifier = None + cur_token = 1 + overloads = [] + while cur_token < len(tokens): + token = tokens[cur_token] + # see if we have modname.funcname( + if (cur_token + 10 < len(tokens) and + token == module_name and + tokens[cur_token + 2] == '.' and + tokens[cur_token + 4] == func_name and + tokens[cur_token + 6] == '('): + sig_start = cur_token + args, ret_type, cur_token = parse_args(tokens, cur_token + 8, mod) + + doc_str = ''.join(tokens[start_token:sig_start]) + if doc_str.find(' ') == -1: + doc_str = '' + if (not args or doc_str) and overloads: + # if we already parsed an overload, and are now getting an argless + # overload we're likely just seeing a reference to the function in + # a doc string, let's ignore that. This is betting on the idea that + # people list overloads first, then doc strings, and that people tend + # to list overloads from simplest to more complex. an example of this + # is the future_builtins.ascii doc string + # We also skip it if we have a doc string, this comes up in overloads + # like isinstance which has example calls embedded in the doc string + continue + + start_token = cur_token + overload = {'args': tuple(extra_args + args), 'doc': doc_str} + ret_types = get_ret_type(ret_type, obj_class, mod) + if ret_types is not None: + overload['ret_type'] = ret_types + update_overload_from_doc_str(overload, doc_str, obj_class, mod) + overloads.append(overload) + # see if we have funcname( + elif (cur_token + 4 < len(tokens) and + token == func_name and + tokens[cur_token + 2] == '('): + sig_start = cur_token + args, ret_type, cur_token = parse_args(tokens, cur_token + 4, mod) + + doc_str = ''.join(tokens[start_token:sig_start]) + if doc_str.find(' ') == -1: + doc_str = '' + if (not args or doc_str) and overloads: + # if we already parsed an overload, and are now getting an argless + # overload we're likely just seeing a reference to the function in + # a doc string, let's ignore that. This is betting on the idea that + # people list overloads first, then doc strings, and that people tend + # to list overloads from simplest to more complex. an example of this + # is the future_builtins.ascii doc string + # We also skip it if we have a doc string, this comes up in overloads + # like isinstance which has example calls embedded in the doc string + continue + + start_token = cur_token + overload = {'args': tuple(extra_args + args), 'doc': doc_str} + ret_types = get_ret_type(ret_type, obj_class, mod) + if ret_types is not None: + overload['ret_type'] = ret_types + update_overload_from_doc_str(overload, doc_str, obj_class, mod) + overloads.append(overload) + + else: + # append to doc string + cur_token += 2 + + finish_doc = ''.join(tokens[start_token:cur_token]) + if finish_doc: + if not overloads: + # This occurs when the docstring does not include a function spec + overloads.append({ + 'args': ({'name': 'args', 'arg_format': '*'}, {'name': 'kwargs', 'arg_format': '**'}), + 'doc': '' + }) + for overload in overloads: + overload['doc'] += finish_doc + update_overload_from_doc_str(overload, overload['doc'], obj_class, mod) + + return overloads + + +IDENTIFIER_REGEX = re.compile('^[a-zA-Z_][a-zA-Z_0-9-]*$') + +def is_identifier(token): + if IDENTIFIER_REGEX.match(token): + return True + return False + +RETURN_TOKENS = set(['-->', '->', '=>', 'return']) + +def parse_args(tokens, cur_token, module): + args = [] + + arg = [] + annotation = None + default_value = None + ignore = False + arg_tokens = [] + next_is_optional = False + is_optional = False + paren_nesting = 0 + while cur_token < len(tokens): + token = tokens[cur_token] + cur_token += 1 + + if token in (',', ')') and paren_nesting == 0: + arg_tokens.append((arg, annotation, default_value, is_optional)) + is_optional = False + arg = [] + annotation = None + default_value = None + if token == ')': + cur_token += 1 + break + elif ignore: + continue + elif token == '=': + if default_value is None: + default_value = [] + else: + ignore = True + elif token == ':': + if annotation is None and default_value is None: + annotation = [] + else: + ignore = True + elif default_value is not None: + default_value.append(token) + elif annotation is not None: + annotation.append(token) + elif token == '[': + next_is_optional = True + elif token in (']', ' ', ''): + pass + else: + arg.append(token) + if next_is_optional: + is_optional, next_is_optional = True, False + + if token == '(': + paren_nesting += 1 + elif token == ')': + paren_nesting -= 1 + + #from pprint import pprint; pprint(arg_tokens) + + for arg, annotation, default_value, is_optional in arg_tokens: + if not arg or arg[0] == '/': + continue + + arg_name = None + star_args = None + + if arg[0] == '(': + names = [arg.pop(0)] + while names[-1] != ')' and arg: + names.append(arg.pop(0)) + if names[-1] == ')': + names.pop() + arg_name = ', '.join(n for n in names[1:] if is_identifier(n)) + elif is_identifier(arg[-1]): + arg_name = arg.pop() + elif arg[-1] == '...': + arg_name = 'args' + star_args = '*' + + if not annotation and arg: + if len(arg) > 1 and arg[-1] == '*': + # C style prototype + annotation = [' '.join(a for a in arg if a != 'const')] + elif is_identifier(arg[-1]): + annotation = arg[-1:] + elif arg[-1] == ')': + annotation = [arg.pop()] + while annotation[0] != '(': + annotation.insert(0, arg.pop()) + + if arg and arg[0] in ('*', '**'): + star_args = arg[0] + + data = { } + + if arg_name: + data['name'] = arg_name + elif star_args == '*': + data['name'] = 'args' + elif star_args == '**': + data['name'] = 'kwargs' + else: + data['name'] = 'arg' + + if annotation and len(annotation) == 1: + data['type'] = [type_name_to_typeref(annotation[0], module)] + + if default_value: + default_value = [d for d in default_value if d] + + if is_optional and default_value[-1] == ']': + default_value.pop() + + data['default_value'] = ''.join(default_value).strip() + elif is_optional: + data['default_value'] = 'None' + + if star_args: + data['arg_format'] = star_args + + args.append(data) + + + # end of params, check for ret value + ret_type = None + + if cur_token + 2 < len(tokens) and tokens[cur_token] in RETURN_TOKENS: + ret_type_start = cur_token + 2 + # we might have a descriptive return value, 'list of fob' + while ret_type_start < len(tokens) and is_identifier(tokens[ret_type_start]): + if tokens[ret_type_start - 1].find('\n') != -1: + break + ret_type_start += 2 + + if ret_type_start < len(tokens) and ',' in tokens[ret_type_start]: + # fob(oar, baz) -> some info about the return, and more info, and more info. + # "some info" is unlikely to be a return type + ret_type = '' + cur_token += 2 + else: + ret_type = ''.join(tokens[cur_token + 2:ret_type_start]).strip() + cur_token = ret_type_start + elif (cur_token + 4 < len(tokens) and + tokens[cur_token] == ':' and tokens[cur_token + 2] in RETURN_TOKENS): + ret_type_start = cur_token + 4 + # we might have a descriptive return value, 'list of fob' + while ret_type_start < len(tokens) and is_identifier(tokens[ret_type_start]): + if tokens[ret_type_start - 1].find('\n') != -1: + break + ret_type_start += 2 + + if ret_type_start < len(tokens) and ',' in tokens[ret_type_start]: + # fob(oar, baz) -> some info about the return, and more info, and more info. + # "some info" is unlikely to be a return type + ret_type = '' + cur_token += 4 + else: + ret_type = ''.join(tokens[cur_token + 4:ret_type_start]).strip() + cur_token = ret_type_start + + return args, ret_type, cur_token + + +if sys.version > '3.': + str_types = (str, bytes) +else: + str_types = (str, unicode) + + +def get_overloads_from_doc_string(doc_str, mod, obj_class, func_name, extra_args = []): + if isinstance(doc_str, str_types): + decl_mod = None + if isinstance(mod, types.ModuleType): + decl_mod = mod + mod = decl_mod.__name__ + elif mod is not None: + decl_mod = sys.modules.get(mod, None) + + res = parse_doc_str(doc_str, mod, decl_mod, func_name, extra_args, obj_class) + if res: + for i, v in enumerate(res): + if 'ret_type' not in v or (not v['ret_type'] or v['ret_type'] == ('', '')): + alt_ret_type = v['doc'].find('returned as a ') + if alt_ret_type != -1: + last_space = v['doc'].find(' ', alt_ret_type + 14) + last_new_line = v['doc'].find('\n', alt_ret_type + 14) + if last_space == -1: + if last_new_line == -1: + last_space = None + else: + last_space = last_new_line + elif last_new_line == -1: + last_space = None + else: + last_space = last_new_line + + ret_type_str = v['doc'][alt_ret_type+14:last_space] + if ret_type_str.endswith('.') or ret_type_str.endswith(','): + ret_type_str = ret_type_str[:-1] + new_ret_type = get_ret_type(ret_type_str, obj_class, decl_mod) + res[i]['ret_type'] = new_ret_type + + return res + return None + + +def get_overloads(func, is_method = False): + if is_method: + extra_args = [{'type': PythonScraper.type_to_typelist(object), 'name': 'self'}] + else: + extra_args = [] + + func_doc = safe_getattr(func, '__doc__', None) + if not func_doc: + return None + + return get_overloads_from_doc_string( + func_doc, + safe_getattr(func, '__module__', None), + safe_getattr(func, '__objclass__', None), + safe_getattr(func, '__name__', None), + extra_args, + ) + +def get_descriptor_type(descriptor): + return object + +def get_new_overloads(type_obj, obj): + try: + type_doc = safe_getattr(type_obj, '__doc__', None) + type_type = type(type_obj) + except: + return None + + res = get_overloads_from_doc_string( + type_doc, + safe_getattr(type_obj, '__module__', None), + type_type, + safe_getattr(type_obj, '__name__', None), + [{'type': PythonScraper.type_to_typelist(type), 'name': 'cls'}], + ) + + if not res: + obj_doc = safe_getattr(obj, '__doc__', None) + if not obj_doc: + return None + res = get_overloads_from_doc_string( + obj_doc, + safe_getattr(type_obj, '__module__', None), + type_type, + safe_getattr(type_obj, '__name__', None), + ) + + return res + +def should_include_module(name): + return True diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/BuiltinScraperTests.py b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/BuiltinScraperTests.py new file mode 100644 index 000000000..67ef67d51 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/BuiltinScraperTests.py @@ -0,0 +1,491 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABILITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +import re +import unittest +from pprint import pformat +from BuiltinScraper import parse_doc_str, BUILTIN, __builtins__, get_overloads_from_doc_string, TOKENS_REGEX + +try: + unicode +except NameError: + from BuiltinScraper import unicode +import sys + +class Test_BuiltinScraperTests(unittest.TestCase): + def check_doc_str(self, doc, module_name, func_name, expected, mod=None, extra_args=[], obj_class=None): + r = parse_doc_str(doc, module_name, mod, func_name, extra_args, obj_class) + + # Quick pass if everything matches + if r == expected: + return + + msg = 'Expected:\n%s\nActual\n%s' % (pformat(expected), pformat(r)) + + self.assertEqual(len(r), len(expected), msg) + + def check_dict(e, a, indent): + if e == a: + return + missing_keys = set(e.keys()) - set(a.keys()) + extra_keys = set(a.keys()) - set(e.keys()) + mismatched_keys = [k for k in set(a.keys()) & set(e.keys()) if a[k] != e[k]] + if missing_keys: + print('%sDid not received following keys: %s' % (indent, ', '.join(missing_keys))) + if extra_keys: + print('%sDid not expect following keys: %s' % (indent, ', '.join(extra_keys))) + for k in mismatched_keys: + if isinstance(e[k], dict) and isinstance(a[k], dict): + check_dict(e[k], a[k], indent + ' ') + elif (isinstance(e[k], tuple) and isinstance(a[k], tuple) or isinstance(e[k], list) and isinstance(a[k], list)): + check_seq(e[k], a[k], indent + ' ') + else: + print('%sExpected "%s": "%s"' % (indent, k, e[k])) + print('%sActual "%s": "%s"' % (indent, k, a[k])) + print('') + + def check_seq(e, a, indent): + if e == a: + return + for i, (e2, a2) in enumerate(zip(e, a)): + if isinstance(e2, dict) and isinstance(a2, dict): + check_dict(e2, a2, indent + ' ') + elif (isinstance(e2, tuple) and isinstance(a2, tuple) or isinstance(e2, list) and isinstance(a2, list)): + check_seq(e2, a2, indent + ' ') + elif e1 != a1: + print('%sExpected "%s"' % (indent, e2)) + print('%sActual "%s"' % (indent, a2)) + print('') + + for e1, a1 in zip(expected, r): + check_dict(e1, a1, '') + self.fail(msg) + + def test_regex(self): + self.assertSequenceEqual( + [i.strip() for i in re.split(TOKENS_REGEX, 'f(\'\', \'a\', \'a\\\'b\', "", "a", "a\\\"b")') if i.strip()], + ['f', '(', "''", ',', "'a'", ',', "'a\\'b'", ',', '""', ',', '"a"', ',', '"a\\"b"', ')'] + ) + self.assertSequenceEqual( + [i.strip() for i in re.split(TOKENS_REGEX, 'f(1, 1., -1, -1.)') if i.strip()], + ['f', '(', '1', ',', '1.', ',', '-1', ',', '-1.', ')'] + ) + self.assertSequenceEqual( + [i.strip() for i in re.split(TOKENS_REGEX, 'f(a, *a, **a, ...)') if i.strip()], + ['f', '(', 'a', ',', '*', 'a', ',', '**', 'a', ',', '...', ')'] + ) + self.assertSequenceEqual( + [i.strip() for i in re.split(TOKENS_REGEX, 'f(a:123, a=123) --> => ->') if i.strip()], + ['f', '(', 'a', ':', '123', ',', 'a', '=', '123', ')', '-->', '=>', '->'] + ) + + + def test_numpy_1(self): + self.check_doc_str( + """arange([start,] stop[, step,], dtype=None) + + Returns + ------- + out : ndarray""", + 'numpy', + 'arange', + [{ + 'doc': 'Returns\n -------\n out : ndarray', + 'ret_type': [('', 'ndarray')], + 'args': ( + {'name': 'start', 'default_value':'None'}, + {'name': 'stop'}, + {'name': 'step', 'default_value': 'None'}, + {'name': 'dtype', 'default_value':'None'}, + ) + }] + ) + + def test_numpy_2(self): + self.check_doc_str( + """arange([start,] stop[, step,], dtype=None) + + Return - out : ndarray""", + 'numpy', + 'arange', + [{ + 'doc': 'Return - out : ndarray', + 'ret_type': [('', 'ndarray')], + 'args': ( + {'name': 'start', 'default_value':'None'}, + {'name': 'stop'}, + {'name': 'step', 'default_value': 'None'}, + {'name': 'dtype', 'default_value':'None'}, + ) + }] + ) + + def test_reduce(self): + self.check_doc_str( + 'reduce(function, sequence[, initial]) -> value', + BUILTIN, + 'reduce', + mod=__builtins__, + expected = [{ + 'args': ( + {'name': 'function'}, + {'name': 'sequence'}, + {'default_value': 'None', 'name': 'initial'} + ), + 'doc': '', + 'ret_type': [('', 'value')] + }] + ) + + def test_pygame_draw_arc(self): + self.check_doc_str( + 'pygame.draw.arc(Surface, color, Rect, start_angle, stop_angle, width=1): return Rect', + 'draw', + 'arc', + [{ + 'args': ( + {'name': 'Surface'}, + {'name': 'color'}, + {'name': 'Rect'}, + {'name': 'start_angle'}, + {'name': 'stop_angle'}, + {'default_value': '1', 'name': 'width'} + ), + 'doc': '', + 'ret_type': [('', 'Rect')] + }] + ) + + def test_isdigit(self): + self.check_doc_str( + '''B.isdigit() -> bool + +Return True if all characters in B are digits +and there is at least one character in B, False otherwise.''', + 'bytes', + 'isdigit', + [{ + 'args': (), + 'doc': 'Return True if all characters in B are digits\nand there is at least one character in B, False otherwise.', + 'ret_type': [(BUILTIN, 'bool')] + }] + ) + + def test_init(self): + self.check_doc_str( + 'x.__init__(...) initializes x; see help(type(x)) for signature', + 'str', + '__init__', + [{'args': ({'arg_format': '*', 'name': 'args'},), 'doc': 'initializes x; see help(type(x)) for signature'}] + ) + + def test_find(self): + self.check_doc_str( + 'S.find(sub [,start [,end]]) -> int', + 'str', + 'find', + [{ + 'args': ( + {'name': 'sub'}, + {'default_value': 'None', 'name': 'start'}, + {'default_value': 'None', 'name': 'end'} + ), + 'doc': '', + 'ret_type': [(BUILTIN, 'int')] + }] + ) + + def test_format(self): + self.check_doc_str( + 'S.format(*args, **kwargs) -> unicode', + 'str', + 'format', + [{ + 'args': ( + {'arg_format': '*', 'name': 'args'}, + {'arg_format': '**', 'name': 'kwargs'} + ), + 'doc': '', + 'ret_type': [(BUILTIN, unicode.__name__)] + }] + ) + + def test_ascii(self): + self.check_doc_str( + "'ascii(object) -> string\n\nReturn the same as repr(). In Python 3.x, the repr() result will\\ncontain printable characters unescaped, while the ascii() result\\nwill have such characters backslash-escaped.'", + 'future_builtins', + 'ascii', + [{ + 'args': ({'name': 'object'},), + 'doc': "Return the same as repr(). In Python 3.x, the repr() result will\\ncontain printable characters unescaped, while the ascii() result\\nwill have such characters backslash-escaped.'", + 'ret_type': [(BUILTIN, 'str')] + }] + ) + + def test_preannotation(self): + self.check_doc_str( + 'f(INT class_code) => SpaceID', + 'fob', + 'f', + [{ + 'args': ({'name': 'class_code', 'type': [(BUILTIN, 'int')]},), + 'doc': '', + 'ret_type': [('', 'SpaceID')] + }]) + + def test_compress(self): + self.check_doc_str( + 'compress(data, selectors) --> iterator over selected data\n\nReturn data elements', + 'itertools', + 'compress', + [{ + 'args': ({'name': 'data'}, {'name': 'selectors'}), + 'doc': 'Return data elements', + 'ret_type': [('', 'iterator')] + }] + ) + + def test_isinstance(self): + self.check_doc_str( + 'isinstance(object, class-or-type-or-tuple) -> bool\n\nReturn whether an object is an ' + 'instance of a class or of a subclass thereof.\nWith a type as second argument, ' + 'return whether that is the object\'s type.\nThe form using a tuple, isinstance(x, (A, B, ...)),' + ' is a shortcut for\nisinstance(x, A) or isinstance(x, B) or ... (etc.).', + BUILTIN, + 'isinstance', + [{ + 'args': ({'name': 'object'}, {'name': 'class-or-type-or-tuple'}), + 'doc': "Return whether an object is an instance of a class or of a subclass thereof.\n" + "With a type as second argument, return whether that is the object's type.\n" + "The form using a tuple, isinstance(x, (A, B, ...)), is a shortcut for\n" + "isinstance(x, A) or isinstance(x, B) or ... (etc.).", + 'ret_type': [(BUILTIN, 'bool')] + }] + ) + + def test_tuple_parameters(self): + self.check_doc_str( + 'pygame.Rect(left, top, width, height): return Rect\n' + 'pygame.Rect((left, top), (width, height)): return Rect\n' + 'pygame.Rect(object): return Rect\n' + 'pygame object for storing rectangular coordinates', + 'pygame', + 'Rect', + [{ + 'args': ({'name': 'left'}, {'name': 'top'}, {'name': 'width'}, {'name': 'height'}), + 'doc': 'pygame object for storing rectangular coordinates', + 'ret_type': [('', 'Rect')] + }, + { + 'args': ({'name': 'left, top'}, {'name': 'width, height'}), + 'doc': 'pygame object for storing rectangular coordinates', + 'ret_type': [('', 'Rect')] + }, + { + 'args': ({'name': 'object'},), + 'doc': 'pygame object for storing rectangular coordinates', + 'ret_type': [('', 'Rect')] + }] + ) + + def test_read(self): + self.check_doc_str( + 'read([size]) -> read at most size bytes, returned as a string.\n\n' + 'If the size argument is negative or omitted, read until EOF is reached.\n' + 'Notice that when in non-blocking mode, less data than what was requested\n' + 'may be returned, even if no size parameter was given.', + BUILTIN, + 'read', + mod=__builtins__, + expected=[{ + 'args': ({'default_value': 'None', 'name': 'size'},), + 'doc': 'read at most size bytes, returned as a string.\n\nIf the size argument is negative or omitted, read until EOF is reached.\nNotice that when in non-blocking mode, less data than what was requested\nmay be returned, even if no size parameter was given.', + 'ret_type': [('', '')] + }] + ) + + + r = get_overloads_from_doc_string( + 'read([size]) -> read at most size bytes, returned as a string.\n\n' + 'If the size argument is negative or omitted, read until EOF is reached.\n' + 'Notice that when in non-blocking mode, less data than what was requested\n' + 'may be returned, even if no size parameter was given.', + __builtins__, + None, + 'read' + ) + + self.assertEqual( + r, + [{ + 'args': ({'default_value': 'None', 'name': 'size'},), + 'doc': 'read at most size bytes, returned as a string.\n\nIf the size argument is negative or omitted, read until EOF is reached.\nNotice that when in non-blocking mode, less data than what was requested\nmay be returned, even if no size parameter was given.', + 'ret_type': [('', '')] + }], + repr(r) + ) + + def test_new(self): + self.check_doc_str( + 'T.__new__(S, ...) -> a new object with type S, a subtype of T', + 'struct', + '__new__', + [{ + 'ret_type': [('', '')], + 'doc': 'a new object with type S, a subtype of T', + 'args': ({'name': 'S'}, {'arg_format': '*', 'name': 'args'}) + }] + ) + + def test_C_prototype(self): + self.check_doc_str( + 'GetDriverByName(char const * name) -> Driver', + '', + 'GetDriverByName', + [{ + 'ret_type': [('', 'Driver')], + 'doc': '', + 'args': ({'name': 'name', 'type': [(BUILTIN, 'str')]},), + }] + ) + + def test_chmod(self): + self.check_doc_str( + 'chmod(path, mode, *, dir_fd=None, follow_symlinks=True)', + 'nt', + 'chmod', + [{ + 'doc': '', + 'args': ( + {'name': 'path'}, + {'name': 'mode'}, + {'name': 'args', 'arg_format': '*'}, + {'name': 'dir_fd', 'default_value': 'None'}, + {'name': 'follow_symlinks', 'default_value': 'True'} + ) + }] + ) + + def test_open(self): + if sys.version_info[0] >= 3: + expect_ret_type = ('_io', '_IOBase') + else: + expect_ret_type = (BUILTIN, 'file') + + self.check_doc_str( + 'open(file, mode=\'r\', buffering=-1, encoding=None,\n' + + ' errors=None, newline=None, closefd=True, opener=None)' + + ' -> file object\n\nOpen file', + BUILTIN, + 'open', + [{ + 'doc': 'Open file', + 'ret_type': [expect_ret_type], + 'args': ( + {'name': 'file'}, + {'name': 'mode', 'default_value': "'r'"}, + {'name': 'buffering', 'default_value': '-1'}, + {'name': 'encoding', 'default_value': 'None'}, + {'name': 'errors', 'default_value': 'None'}, + {'name': 'newline', 'default_value': 'None'}, + {'name': 'closefd', 'default_value': 'True'}, + {'name': 'opener', 'default_value': 'None'}, + ) + }] + ) + + def test_optional_with_default(self): + self.check_doc_str( + 'max(iterable[, key=func]) -> value', + BUILTIN, + 'max', + [{ + 'doc': '', + 'ret_type': [('', 'value')], + 'args': ( + {'name': 'iterable'}, + {'name': 'key', 'default_value': 'func'} + ) + }] + ) + + def test_pyplot_figure(self): + pyplot_doc = """ + Creates a new figure. + + Parameters + ---------- + + num : integer or string, optional, default: none + If not provided, a new figure will be created, and a the figure number + will be increamted. The figure objects holds this number in a `number` + attribute. + If num is provided, and a figure with this id already exists, make + it active, and returns a reference to it. If this figure does not + exists, create it and returns it. + If num is a string, the window title will be set to this figure's + `num`. + + figsize : tuple of integers, optional, default : None + width, height in inches. If not provided, defaults to rc + figure.figsize. + + dpi : integer, optional, default ; None + resolution of the figure. If not provided, defaults to rc figure.dpi. + + facecolor : + the background color; If not provided, defaults to rc figure.facecolor + + edgecolor : + the border color. If not provided, defaults to rc figure.edgecolor + + Returns + ------- + figure : Figure + The Figure instance returned will also be passed to new_figure_manager + in the backends, which allows to hook custom Figure classes into the + pylab interface. Additional kwargs will be passed to the figure init + function. + + Note + ---- + If you are creating many figures, make sure you explicitly call "close" + on the figures you are not using, because this will enable pylab + to properly clean up the memory. + + rcParams defines the default values, which can be modified in the + matplotlibrc file + + """ + self.check_doc_str( + pyplot_doc, + 'matplotlib.pyplot', + 'figure', + [{ + 'doc': pyplot_doc, + 'ret_type': [('', 'Figure')], + 'args': ( + {'name': 'args', 'arg_format': '*'}, + {'name': 'kwargs', 'arg_format': '**'} + ) + }] + ) + +if __name__ == '__main__': + unittest.main() diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/ExtensionScraper.py b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/ExtensionScraper.py new file mode 100644 index 000000000..29952ecaa --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/ExtensionScraper.py @@ -0,0 +1,79 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABILITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +import sys +import PythonScraper + +try: + # disable error reporting in our process, bad extension modules can crash us, and we don't + # want a bunch of Watson boxes popping up... + import ctypes + ctypes.windll.kernel32.SetErrorMode(3) # SEM_FAILCRITICALERRORS / SEM_NOGPFAULTERRORBOX +except: + pass + +# Scrapes the file and saves the analysis to the specified filename, exits w/ nonzero exit code if anything goes wrong. +# Usage: ExtensionScraper.py scrape [mod_name or '-'] [mod_path or '-'] [output_path] + +if len(sys.argv) != 5 or sys.argv[1].lower() != 'scrape': + raise ValueError('Expects "ExtensionScraper.py scrape [mod_name|'-'] [mod_path|'-'] [output_path]"') + +mod_name, mod_path, output_path = sys.argv[2:] +module = None + +if mod_name and mod_name != '-': + remove_sys_path_0 = False + try: + if mod_path and mod_path != '-': + import os.path + if os.path.exists(mod_path): + sys.path.insert(0, mod_path) + remove_sys_path_0 = True + __import__(mod_name) + module = sys.modules[mod_name] + finally: + if remove_sys_path_0: + del sys.path[0] + + if not module: + print('__import__("' + mod_name + '")') + PythonScraper.write_analysis(output_path, {"members": {}, "doc": "Could not import compiled module"}) +elif mod_path and mod_path != '-': + try: + import os.path + mod_name = os.path.split(mod_path)[1].partition('.')[0] + try: + import importlib + module = importlib.import_module(mod_name) + except ImportError: + # Don't really care which import failed - we'll try imp + pass + if not module: + import imp + module = imp.load_dynamic(mod_name, mod_path) + finally: + if not module: + print('imp.load_dynamic("' + mod_name + '", "' + mod_path + '")') + PythonScraper.write_analysis(output_path, {"members": {}, "doc": "Could not import compiled module", "filename": mod_path}) +else: + raise ValueError('No module name or path provided') + +if module: + analysis = PythonScraper.generate_module(module) + PythonScraper.write_analysis(output_path, analysis) diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AnalysisLimitsConverter.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AnalysisLimitsConverter.cs new file mode 100644 index 000000000..1cd095217 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AnalysisLimitsConverter.cs @@ -0,0 +1,114 @@ +using System.Collections.Generic; +using Microsoft.PythonTools.Analysis; +using Microsoft.Win32; + +namespace Microsoft.PythonTools.Intellisense { + public static class AnalysisLimitsConverter { + // We use string literals here rather than nameof() to ensure back-compat + // (though we need to preserve the names of the properties as well for + // the same reason, so just don't change anything :) ) + private const string CrossModuleId = "CrossModule"; + private const string CallDepthId = "CallDepth"; + private const string DecreaseCallDepthId = "DecreaseCallDepth"; + private const string NormalArgumentTypesId = "NormalArgumentTypes"; + private const string ListArgumentTypesId = "ListArgumentTypes"; + private const string DictArgumentTypesId = "DictArgumentTypes"; + private const string ReturnTypesId = "ReturnTypes"; + private const string YieldTypesId = "YieldTypes"; + private const string InstanceMembersId = "InstanceMembers"; + private const string DictKeyTypesId = "DictKeyTypes"; + private const string DictValueTypesId = "DictValueTypes"; + private const string IndexTypesId = "IndexTypes"; + private const string AssignedTypesId = "AssignedTypes"; + private const string UnifyCallsToNewId = "UnifyCallsToNew"; + private const string ProcessCustomDecoratorsId = "ProcessCustomDecorators"; + private const string UseTypeStubPackagesId = "UseTypeStubPackages"; + private const string UseTypeStubPackagesExclusivelyId = "UseTypeStubPackagesExclusively"; + + + /// + /// Loads a new instance from the specified registry key. + /// + /// + /// The key to load settings from. Each setting is a DWORD value. If + /// null, all settings are assumed to be unspecified and the default + /// values are used. + /// + /// + /// If True, unspecified settings are taken from the defaults for + /// standard library analysis. Otherwise, they are taken from the + /// usual defaults. + /// + public static AnalysisLimits LoadFromStorage(RegistryKey key, bool defaultToStdLib = false) { + var limits = defaultToStdLib ? AnalysisLimits.GetStandardLibraryLimits() : new AnalysisLimits(); + + if (key != null) { + limits.CrossModule = (key.GetValue(CrossModuleId) as int?) ?? limits.CrossModule; + limits.CallDepth = (key.GetValue(CallDepthId) as int?) ?? limits.CallDepth; + limits.DecreaseCallDepth = (key.GetValue(DecreaseCallDepthId) as int?) ?? limits.DecreaseCallDepth; + limits.NormalArgumentTypes = (key.GetValue(NormalArgumentTypesId) as int?) ?? limits.NormalArgumentTypes; + limits.ListArgumentTypes = (key.GetValue(ListArgumentTypesId) as int?) ?? limits.ListArgumentTypes; + limits.DictArgumentTypes = (key.GetValue(DictArgumentTypesId) as int?) ?? limits.DictArgumentTypes; + limits.ReturnTypes = (key.GetValue(ReturnTypesId) as int?) ?? limits.ReturnTypes; + limits.YieldTypes = (key.GetValue(YieldTypesId) as int?) ?? limits.YieldTypes; + limits.InstanceMembers = (key.GetValue(InstanceMembersId) as int?) ?? limits.InstanceMembers; + limits.DictKeyTypes = (key.GetValue(DictKeyTypesId) as int?) ?? limits.DictKeyTypes; + limits.DictValueTypes = (key.GetValue(DictValueTypesId) as int?) ?? limits.DictValueTypes; + limits.IndexTypes = (key.GetValue(IndexTypesId) as int?) ?? limits.IndexTypes; + limits.AssignedTypes = (key.GetValue(AssignedTypesId) as int?) ?? limits.AssignedTypes; + limits.UnifyCallsToNew = ((key.GetValue(UnifyCallsToNewId) as int?) ?? (limits.UnifyCallsToNew ? 1 : 0)) != 0; + limits.ProcessCustomDecorators = ((key.GetValue(ProcessCustomDecoratorsId) as int?) ?? (limits.ProcessCustomDecorators ? 1 : 0)) != 0; + limits.UseTypeStubPackages = ((key.GetValue(UseTypeStubPackagesId) as int?) ?? (limits.UseTypeStubPackages ? 1 : 0)) != 0; + limits.UseTypeStubPackagesExclusively = ((key.GetValue(UseTypeStubPackagesExclusivelyId) as int?) ?? (limits.UseTypeStubPackagesExclusively ? 1 : 0)) != 0; + } + + return limits; + } + + public static AnalysisLimits FromDictionary(Dictionary limits) { + var analysisLimits = new AnalysisLimits(); + int i; + if (limits.TryGetValue(CrossModuleId, out i)) analysisLimits.CrossModule = i; + if (limits.TryGetValue(CallDepthId, out i)) analysisLimits.CallDepth = i; + if (limits.TryGetValue(DecreaseCallDepthId, out i)) analysisLimits.DecreaseCallDepth = i; + if (limits.TryGetValue(NormalArgumentTypesId, out i)) analysisLimits.NormalArgumentTypes = i; + if (limits.TryGetValue(ListArgumentTypesId, out i)) analysisLimits.ListArgumentTypes = i; + if (limits.TryGetValue(DictArgumentTypesId, out i)) analysisLimits.DictArgumentTypes = i; + if (limits.TryGetValue(ReturnTypesId, out i)) analysisLimits.ReturnTypes = i; + if (limits.TryGetValue(YieldTypesId, out i)) analysisLimits.YieldTypes = i; + if (limits.TryGetValue(InstanceMembersId, out i)) analysisLimits.InstanceMembers = i; + if (limits.TryGetValue(DictKeyTypesId, out i)) analysisLimits.DictKeyTypes = i; + if (limits.TryGetValue(DictValueTypesId, out i)) analysisLimits.DictValueTypes = i; + if (limits.TryGetValue(IndexTypesId, out i)) analysisLimits.IndexTypes = i; + if (limits.TryGetValue(AssignedTypesId, out i)) analysisLimits.AssignedTypes = i; + if (limits.TryGetValue(UnifyCallsToNewId, out i)) analysisLimits.UnifyCallsToNew = i != 0; + if (limits.TryGetValue(ProcessCustomDecoratorsId, out i)) analysisLimits.ProcessCustomDecorators = i != 0; + if (limits.TryGetValue(UseTypeStubPackagesId, out i)) analysisLimits.UseTypeStubPackages = i != 0; + if (limits.TryGetValue(UseTypeStubPackagesExclusivelyId, out i)) analysisLimits.UseTypeStubPackagesExclusively = i != 0; + + return analysisLimits; + } + + public static Dictionary ToDictionary(this AnalysisLimits analysisLimits) { + return new Dictionary { + { CrossModuleId, analysisLimits.CrossModule }, + { CallDepthId, analysisLimits.CallDepth }, + { DecreaseCallDepthId, analysisLimits.DecreaseCallDepth }, + { NormalArgumentTypesId, analysisLimits.NormalArgumentTypes }, + { ListArgumentTypesId, analysisLimits.ListArgumentTypes }, + { DictArgumentTypesId, analysisLimits.DictArgumentTypes }, + { ReturnTypesId, analysisLimits.ReturnTypes }, + { YieldTypesId, analysisLimits.YieldTypes }, + { InstanceMembersId, analysisLimits.InstanceMembers }, + { DictKeyTypesId, analysisLimits.DictKeyTypes }, + { DictValueTypesId, analysisLimits.DictValueTypes }, + { IndexTypesId, analysisLimits.IndexTypes }, + { AssignedTypesId, analysisLimits.AssignedTypes }, + { UnifyCallsToNewId, analysisLimits.UnifyCallsToNew ? 1 : 0 }, + { ProcessCustomDecoratorsId, analysisLimits.ProcessCustomDecorators ? 1 : 0 }, + { UseTypeStubPackagesId, analysisLimits.UseTypeStubPackages ? 1 : 0 }, + { UseTypeStubPackagesExclusivelyId, analysisLimits.UseTypeStubPackagesExclusively ? 1 : 0 } + }; + } + } +} \ No newline at end of file diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AnalysisProtocol.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AnalysisProtocol.cs new file mode 100644 index 000000000..46873b01a --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AnalysisProtocol.cs @@ -0,0 +1,965 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Reflection; +using Microsoft.PythonTools.Analysis; +using Microsoft.PythonTools.Analysis.Infrastructure; +using Microsoft.PythonTools.Interpreter; +using Microsoft.PythonTools.Ipc.Json; +using Microsoft.PythonTools.Parsing; +using Newtonsoft.Json; +using LS = Microsoft.Python.LanguageServer; + +namespace Microsoft.PythonTools.Intellisense { + public static class AnalysisProtocol { + public static readonly Dictionary RegisteredTypes = CollectCommands(); + + private static Dictionary CollectCommands() { + Dictionary all = new Dictionary(); + foreach (var type in typeof(AnalysisProtocol).GetNestedTypes()) { + if (type.IsSubclassOf(typeof(Request))) { + var command = type.GetField("Command"); + if (command != null) { + all["request." + (string)command.GetRawConstantValue()] = type; + } + } else if (type.IsSubclassOf(typeof(Event))) { + var name = type.GetField("Name"); + if (name != null) { + all["event." + (string)name.GetRawConstantValue()] = type; + } + } + } + return all; + } + + public sealed class InitializeRequest : Request { + public const string Command = "initialize"; + + public override string command => Command; + + public InterpreterInfo interpreter; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri rootUri; + public bool analyzeAllFiles; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public bool traceLogging; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public bool liveLinting; + } + + public sealed class InterpreterInfo { + public string assembly, typeName; + public Dictionary properties; + } + + public sealed class InitializeResponse : Response { + [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public string error; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public string fullError; + } + + public sealed class ExitRequest : GenericRequest { + public const string Command = "exit"; + + public override string command => Command; + } + + public sealed class GetReferencesResponse : Response { + public ProjectReference[] references; + } + + public sealed class ProjectReference { + public string name, kind, assemblyName; + + public static ProjectReference Convert(Microsoft.PythonTools.Interpreter.ProjectReference reference) { + return new ProjectReference() { + name = reference.Name, + kind = GetReferenceKind(reference.Kind), + assemblyName = GetReferenceAssembly(reference) + }; + } + + public static Microsoft.PythonTools.Interpreter.ProjectReference Convert(ProjectReference reference) { + switch (reference.kind) { + case "extension": + return new Microsoft.PythonTools.Interpreter.ProjectReference( + reference.name, + ProjectReferenceKind.ExtensionModule + ); + case "assembly": + return new ProjectAssemblyReference( + new AssemblyName(reference.assemblyName), + reference.name + ); + default: + throw new InvalidOperationException("Unsupported reference type " + reference.kind); + } + } + + private static string GetReferenceAssembly(Microsoft.PythonTools.Interpreter.ProjectReference reference) { + switch (reference.Kind) { + case ProjectReferenceKind.Assembly: + return ((ProjectAssemblyReference)reference).AssemblyName.FullName; + default: return null; + } + } + + public static string GetReferenceKind(ProjectReferenceKind kind) { + switch (kind) { + case ProjectReferenceKind.Assembly: return "assembly"; + case ProjectReferenceKind.ExtensionModule: return "extension"; + default: return null; + } + } + + } + + public sealed class SetAnalysisLimitsRequest : Request { + public const string Command = "setAnalysisLimits"; + + public override string command => Command; + + } + + public sealed class ValueDescriptionRequest : Request { + public const string Command = "valueDescriptions"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string expr; + public int line, column; + + public override string command => Command; + } + + public sealed class ValueDescriptionResponse : Response { + public string[] descriptions; + } + + public sealed class AddReferenceRequest : Request { + public const string Command = "addReference"; + public ProjectReference reference; + + public override string command => Command; + } + + public sealed class AddReferenceResponse : Response { + } + + public sealed class RemoveReferenceRequest : Request { + public const string Command = "removeReference"; + public ProjectReference reference; + + public override string command => Command; + } + + public sealed class RemoveReferenceResponse : Response { + } + + public sealed class AnalysisClassificationsRequest : Request { + public const string Command = "analysisClassify"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public bool colorNames; + + public override string command => Command; + } + + /// + /// Gets a location where a method can safely be inserted into a top level class + /// + public sealed class MethodInsertionLocationRequest : Request { + public const string Command = "methodInsertion"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string className; + + public override string command => Command; + } + + public sealed class MethodInsertionLocationResponse : Response { + public int line, column; + public int version; + } + + public sealed class MethodInfoRequest : Request { + public const string Command = "methodInfo"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string className; + public string methodName; + + public override string command => Command; + } + + public sealed class MethodInfoResponse : Response { + public int start, end; + public int version; + public bool found; + } + + public sealed class FindMethodsRequest : Request { + public const string Command = "findMethods"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string className; + + /// + /// Optional filter of the number of parameters + /// + public int? paramCount; + + public override string command => Command; + } + + public sealed class FindMethodsResponse : Response { + public string[] names; + } + + public sealed class AnalysisClassificationsResponse : Response { + public AnalysisClassification[] classifications; + + public int version; + } + + public sealed class AnalysisClassification { + public int startLine, startColumn; + public int endLine, endColumn; + public string type; + } + + public class QuickInfoRequest : Request { + public const string Command = "quickInfo"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string expr; + public int line, column; + + public override string command => Command; + } + + public class QuickInfoResponse : Response { + public string text; + } + + public class FileParsedEvent : Event { + public const string Name = "fileParsed"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int version; + + public override string name => Name; + } + + public class DiagnosticsEvent : Event { + public const string Name = "diagnostics"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int version; + public Diagnostic[] diagnostics; + + public bool ShouldSerializediagnostics() => (diagnostics?.Length ?? 0) > 0; + + public override string name => Name; + } + + public sealed class FormatCodeRequest : Request { + public const string Command = "formatCode"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int startLine, startColumn; + public int endLine, endColumn; + public string newLine; + public CodeFormattingOptions options; + + public override string command => Command; + } + + public sealed class FormatCodeResponse : Response { + public ChangeInfo[] changes; + public int version; + } + + public struct CodeSpan { + public int start, length; + } + + public class AddFileRequest : Request { + public const string Command = "addFile"; + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public string path; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public string addingFromDir; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public bool isTemporaryFile; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public bool suppressErrorLists; + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + [JsonConverter(typeof(UriJsonConverter))] + public Uri uri; + + public override string command => Command; + } + + public class AddFileResponse : Response { + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + } + + public class AddBulkFileRequest : Request { + public const string Command = "addBulkFile"; + + public string[] path; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public string addingFromDir; + + public override string command => Command; + } + + public class AddBulkFileResponse : Response { + [JsonProperty(ItemConverterType = typeof(UriJsonConverter))] + public Uri[] documentUri; + } + + public sealed class SetSearchPathRequest : Request { + public const string Command = "setSearchPath"; + + public string[] dir; + public override string command => Command; + } + + public sealed class UnloadFileRequest : Request { + public const string Command = "unloadFile"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public override string command => Command; + + public override string ToString() => "{0}:{1}".FormatUI(command, documentUri); + } + + + public sealed class DirectoryFileAddedEvent : Event { + public const string Name = "directoryFileAdded"; + + public string filename; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + + public override string name => Name; + } + + public sealed class FileUpdateRequest : Request { + public const string Command = "fileUpdate"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public FileUpdate[] updates; + + public override string command => Command; + + public override string ToString() => "{0}:{1} ({2} updates)".FormatUI(command, documentUri, updates.Length); + } + + public enum FileUpdateKind { + none, + /// + /// Reset the content to the specified content string + /// + reset, + /// + /// Apply the list of changes to the content + /// + changes + } + + public sealed class AddImportRequest : Request { + public const string Command = "addImport"; + + public string fromModule, name, newLine; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + + public override string command => Command; + } + + public sealed class AddImportResponse : Response { + public ChangeInfo[] changes; + public int version = -1; + } + + public sealed class IsMissingImportRequest : Request { + public const string Command = "isMissingImport"; + + public string text; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int line, column; + + public override string command => Command; + } + + public sealed class IsMissingImportResponse : Response { + public bool isMissing; + } + + public sealed class AvailableImportsRequest : Request { + public const string Command = "availableImports"; + + public string name; + + public override string command => Command; + } + + public sealed class AvailableImportsResponse : Response { + public ImportInfo[] imports; + } + + public sealed class ImportInfo { + public string fromName, importName; + + + // Provide Equals so we can easily uniquify sequences of ImportInfo + + public override bool Equals(object obj) { + if (obj is ImportInfo ii) { + return fromName == ii.fromName && importName == ii.importName; + } + return false; + } + + public override int GetHashCode() { + return ((fromName ?? "") + "." + (importName ?? "")).GetHashCode(); + } + } + + public sealed class FileUpdate { + public FileUpdateKind kind; + + // Unlike most version numbers, this is what the version will be + // _after_ applying the update, not before. The target file is + // assumed to be at version-1 when applying this change. + public int version; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public ChangeInfo[] changes; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public string content; + } + + public sealed class FileUpdateResponse : Response { + public int version; +#if DEBUG + public string newCode; +#endif + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public bool? failed; + } + + public sealed class UnresolvedImport { + public string name; + public int startLine, endLine, startColumn, endColumn; + } + + public sealed class ChangeInfo { + public string newText; + public int startLine, startColumn; + public int endLine, endColumn; + + public static ChangeInfo FromDocumentChange(DocumentChange c) { + return new ChangeInfo { + startLine = c.ReplacedSpan.Start.Line, + startColumn = c.ReplacedSpan.Start.Column, + endLine = c.ReplacedSpan.End.Line, + endColumn = c.ReplacedSpan.End.Column, + newText = c.InsertedText + }; + } + + public DocumentChange ToDocumentChange() { + return new DocumentChange { + InsertedText = newText, + ReplacedSpan = new SourceSpan(new SourceLocation(startLine, startColumn), new SourceLocation(endLine, endColumn)) + }; + } + } + + public sealed class LocationNameRequest : Request { + public const string Command = "locationName"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int line, column; + + public override string command => Command; + } + + public sealed class LocationNameResponse : Response { + public string name; + public int lineOffset; + } + + + public sealed class ProximityExpressionsRequest : Request { + public const string Command = "proximityExpressions"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int line, column, lineCount; + + public override string command => Command; + } + + public sealed class ProximityExpressionsResponse : Response { + public string[] names; + } + + public sealed class RemoveImportsRequest : Request { + public const string Command = "removeImports"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int version; + public int line, column; + public bool allScopes; + + public override string command => Command; + } + + public sealed class RemoveImportsResponse : Response { + public ChangeInfo[] changes; + public int version = -1; + } + + public sealed class ExtractMethodRequest : Request { + public const string Command = "extractMethod"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int startIndex, endIndex; + public int indentSize; + public string name; + public string[] parameters; + public bool convertTabsToSpaces; + public string newLine; + public bool shouldExpandSelection; + public int? scope; + + public override string command => Command; + } + + public sealed class ExtractMethodResponse : Response { + public CannotExtractReason cannotExtractReason; + public ChangeInfo[] changes; + /// + /// available scopes the user can retarget to + /// + public ScopeInfo[] scopes; + public bool wasExpanded; + public int startLine, startCol; + public int endLine, endCol; + public int version; + public string methodBody; + public string[] variables; + } + + public enum CannotExtractReason { + None = 0, + InvalidTargetSelected = 1, + InvalidExpressionSelected = 2, + MethodAssignsVariablesAndReturns = 3, + StatementsFromClassDefinition = 4, + SelectionContainsBreakButNotEnclosingLoop = 5, + SelectionContainsContinueButNotEnclosingLoop = 6, + ContainsYieldExpression = 7, + ContainsFromImportStar = 8, + SelectionContainsReturn = 9 + } + + public class ScopeInfo { + public string type, name; + public int id; + public string[] variables; + } + + public sealed class ModuleImportsRequest : Request { + public const string Command = "moduleImports"; + + public string moduleName; + public bool includeUnresolved; + + public override string command => Command; + } + + public sealed class ModuleImportsResponse : Response { + public ModuleInfo[] modules; + } + + public sealed class ModuleInfo { + public string moduleName; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string filename; + } + + public class EnqueueFileResponse : Response { + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + } + + public class GetModulesRequest : Request { + public const string Command = "getModules"; + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string[] package; + + public override string command => Command; + + public bool ShouldSerializepackage() => (package?.Length ?? 0) > 0; + } + + public class GetAllMembersRequest : Request { + public const string Command = "getAllMembers"; + + public string prefix; + public GetMemberOptions options; + + public override string command => Command; + } + + public class CompletionsRequest : Request { + public const string Command = "completions"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string text; + public int line, column; + public GetMemberOptions options; + public bool forceCompletions; + + public override string command => Command; + } + + public class SignaturesRequest : Request { + public const string Command = "sigs"; + + public override string command => Command; + + public string text; + public int line, column; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + } + + public sealed class ModulesChangedEvent : Event { + public const string Name = "modulesChanged"; + + public override string name => Name; + } + + public struct FileEvent { + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public LS.FileChangeType kind; + } + + public sealed class FileChangedEvent : Event { + public const string Name = "fileChanged"; + + public FileEvent[] changes; + + public override string name => Name; + } + + public sealed class SignaturesResponse : Response { + public Signature[] sigs; + } + + public class Signature { + public string name; + public string doc; + public Parameter[] parameters; + } + + public class Parameter { + public string name, defaultValue, doc, type; + public bool optional; + } + + public class FileAnalysisCompleteEvent : Event { + public const string Name = "fileAnalysisComplete"; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + + public int version; + + public override string name => Name; + public override string ToString() => "{0}:{1} ({2})".FormatUI(name, documentUri, version); + } + + public sealed class LoadExtensionRequest : Request { + public const string Command = "loadExtensionRequest"; + + public override string command => Command; + + public string extension; + public string assembly; + public string typeName; + } + + public sealed class LoadExtensionResponse : Response { + public string error; + public string fullError; + } + + public sealed class ExtensionRequest : Request { + public const string Command = "extensionRequest"; + + public override string command => Command; + + public string extension; + public string commandId; + public string body; + } + + public sealed class ExtensionResponse : Response { + public string response; + public string error; + } + + /// + /// Signals all files are analyzed + /// + public class AnalysisCompleteEvent : Event { + public const string Name = "analysisComplete"; + + public override string name => Name; + } + + public sealed class AnalysisStatusRequest : Request { + public const string Command = "analysisStatus"; + + public override string command => Command; + } + + public sealed class AnalysisStatusResponse : Response { + public int itemsLeft; + } + + + public class CompletionsResponse : Response { + public Completion[] completions; + } + + public class Completion { + public string name; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public string completion; // when null, use "name" + public string doc; + public PythonMemberType memberType; + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public CompletionValue[] detailedValues; + + public bool ShouldSerializedetailedValues() => (detailedValues?.Length ?? 0) > 0; + } + + public sealed class CompletionValue { + public DescriptionComponent[] description; + public string doc; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public AnalysisReference[] locations; + + public bool ShouldSerializelocations() => (locations?.Length ?? 0) > 0; + } + + public sealed class DescriptionComponent { + public string text, kind; + + public DescriptionComponent() { + } + + public DescriptionComponent(string text, string kind) { + this.text = text; + this.kind = kind; + } + } + + public sealed class SetAnalysisOptionsRequest : Request { + public const string Command = "setAnalysisOptions"; + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public AnalysisOptions options; + + public override string command => Command; + } + + public sealed class AnalysisOptions { + public Severity indentationInconsistencySeverity; + public Dictionary commentTokens; + public Dictionary analysisLimits; + public LS.MessageType? traceLevel; + public string[] typeStubPaths; + } + + public class AnalysisReference { + public string kind; // definition, reference, value + public string expr; + public string file; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int startLine, startColumn, endLine, endColumn; + public int? version; + } + + public sealed class AnalyzeExpressionRequest : Request { + public const string Command = "findDefs"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string expr; + public int line, column; + + public override string command => Command; + } + + public sealed class AnalyzeExpressionResponse : Response { + public AnalysisReference[] variables; + /// + /// The private prefix for the member if defined inside a class with name mangling. + /// + public string privatePrefix; + } + + public sealed class OutliningRegionsRequest : Request { + public const string Command = "outliningRegions"; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + + public override string command => Command; + } + + public sealed class OutliningRegionsResponse : Response { + public int version = -1; + public OutliningTag[] tags; + } + + public sealed class OutliningTag { + public int startLine, startCol; + public int endLine, endCol; + } + + public sealed class NavigationRequest : Request { + public const string Command = "navigation"; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + + public override string command => Command; + } + + public sealed class NavigationResponse : Response { + public int version; + public Navigation[] navigations; + } + + public sealed class Navigation { + public string name, type; + public int startLine, startColumn; + public int endLine, endColumn; + public Navigation[] children; + } + + public class AnalyzerWarningEvent : Event { + public string message; + public const string Name = "analyzerWarning"; + + public override string name => Name; + } + + public class UnhandledExceptionEvent : Event { + public string message; + public const string Name = "unhandledException"; + + public UnhandledExceptionEvent(Exception ex) { + message = ex.ToString(); + } + + public UnhandledExceptionEvent(string message) { + this.message = message; + } + + public override string name => Name; + } + + public enum ExpressionAtPointPurpose : int { + Hover = 1, + Evaluate = 2, + EvaluateMembers = 3, + FindDefinition = 4, + Rename = 5 + } + + public sealed class ExpressionAtPointRequest : Request { + public const string Command = "exprAtPoint"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int line, column; + public ExpressionAtPointPurpose purpose; + + public override string command => Command; + } + + public sealed class ExpressionAtPointResponse : Response { + public string expression; + public string type; + public int bufferVersion; + public int startLine, startColumn; + public int endLine, endColumn; + } + + public class LanguageServerRequest : Request { + public const string Command = "languageServer"; + + public string name; + public object body; + + public override string command => Command; + } + + public class LanguageServerResponse : Response { + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public object body; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public string error; + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AssignmentWalker.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AssignmentWalker.cs new file mode 100644 index 000000000..4bf76dfc7 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AssignmentWalker.cs @@ -0,0 +1,137 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Intellisense { + + /// + /// A walker which handles all nodes which can result in assignments and + /// calls back on a special walker for defining the variable names. + /// + /// Note this class only handles things which show up as name expressions, + /// other implicit assignments (such as class and function definitions, + /// import/from import statements, need to be handled by the derived binder). + /// + public abstract class AssignmentWalker : PythonWalker { + public abstract AssignedNameWalker Define { + get; + } + + #region Assignment Walkers + + public override bool Walk(AssignmentStatement node) { + foreach (var lhs in node.Left) { + DefineExpr(lhs); + } + node.Right.Walk(this); + return false; + } + + private void DefineExpr(Expression lhs) { + if (lhs is NameExpression) { + lhs.Walk(Define); + } else { + // fob.oar = 42, fob[oar] = 42, we don't actually define any variables + lhs.Walk(this); + } + } + + public override bool Walk(AugmentedAssignStatement node) { + DefineExpr(node.Left); + node.Right.Walk(this); + return false; + } + + public override bool Walk(DelStatement node) { + foreach (var expr in node.Expressions) { + DefineExpr(expr); + } + return false; + } + + public override bool Walk(ComprehensionFor node) { + if (node.Left != null) { + node.Left.Walk(Define); + } + if (node.List != null) { + node.List.Walk(this); + } + return false; + } + + private bool WalkIterators(Comprehension node) { + if (node.Iterators != null) { + foreach (ComprehensionIterator ci in node.Iterators) { + ci.Walk(this); + } + } + + return false; + } + + public override bool Walk(ForStatement node) { + if (node.Left != null) { + node.Left.Walk(Define); + } + if (node.List != null) { + node.List.Walk(this); + } + if (node.Body != null) { + node.Body.Walk(this); + } + if (node.Else != null) { + node.Else.Walk(this); + } + return false; + } + + public override bool Walk(WithStatement node) { + foreach (var item in node.Items) { + if (item.Variable != null) { + item.Variable.Walk(Define); + } + if (item.ContextManager != null) { + item.ContextManager.Walk(this); + } + } + if (node.Body != null) { + node.Body.Walk(this); + } + return false; + } + + #endregion + } + + public abstract class AssignedNameWalker : PythonWalkerNonRecursive { + + public override abstract bool Walk(NameExpression node); + + public override bool Walk(ParenthesisExpression node) { + return true; + } + + public override bool Walk(TupleExpression node) { + return true; + } + + public override bool Walk(ListExpression node) { + return true; + } + } + +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/ClassifierWalker.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/ClassifierWalker.cs new file mode 100644 index 000000000..12cd32225 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/ClassifierWalker.cs @@ -0,0 +1,480 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Microsoft.PythonTools.Analysis; +using Microsoft.PythonTools.Interpreter; +using Microsoft.PythonTools.Parsing; +using Microsoft.PythonTools.Parsing.Ast; +using Microsoft.PythonTools.Analysis.Values; + +namespace Microsoft.PythonTools.Intellisense { + class ClassifierWalker : PythonWalker { + class StackData { + public readonly string Name; + public readonly HashSet Parameters; + public readonly HashSet Functions; + public readonly HashSet Types; + public readonly HashSet TypeHints; + public readonly HashSet Modules; + public readonly List> Names; + public readonly StackData Previous; + + public StackData(string name, StackData previous) { + Name = name; + Previous = previous; + Parameters = new HashSet(); + Functions = new HashSet(); + Types = new HashSet(); + TypeHints = new HashSet(); + Modules = new HashSet(); + Names = new List>(); + } + + public IEnumerable EnumerateTowardsGlobal { + get { + for (var sd = this; sd != null; sd = sd.Previous) { + yield return sd; + } + } + } + } + + private readonly PythonAst _ast; + private readonly IModuleAnalysis _analysis; + private StackData _head; + public readonly List Spans; + + public static class Classifications { + public const string Keyword = "keyword"; + public const string Class = "class"; + public const string Function = "function"; + public const string Module = "module"; + public const string Parameter = "parameter"; + public const string RegexLiteral = "regexliteral"; + public const string DocString = "docstring"; + public const string TypeHint = "class"; + } + + public ClassifierWalker(PythonAst ast, IModuleAnalysis analysis) { + _ast = ast; + _analysis = analysis; + Spans = new List(); + } + + private void AddSpan(Tuple node, string type) { + Spans.Add(new TaggedSpan( + new SourceSpan( + _ast.IndexToLocation(node.Item2.Start), + _ast.IndexToLocation(node.Item2.Start + node.Item2.Length) + ), + type + )); + } + + private void BeginScope(string name = null) { + if (_head != null) { + if (name == null) { + name = _head.Name; + } else if (_head.Name != null) { + name = _head.Name + "." + name; + } + } + _head = new StackData(name, _head); + } + + private void AddParameter(Parameter node) { + Debug.Assert(_head != null); + _head.Parameters.Add(node.Name); + _head.Names.Add(Tuple.Create(node.Name, new Span(node.NameSpan.Start, node.NameSpan.Length))); + } + + private void AddParameter(Node node) { + NameExpression name; + TupleExpression tuple; + Debug.Assert(_head != null); + if ((name = node as NameExpression) != null) { + _head.Parameters.Add(name.Name); + } else if ((tuple = node as TupleExpression) != null) { + foreach (var expr in tuple.Items) { + AddParameter(expr); + } + } else { + Trace.TraceWarning("Unable to find parameter in {0}", node); + } + } + + public override bool Walk(NameExpression node) { + _head.Names.Add(Tuple.Create(node.Name, Span.FromBounds(node.StartIndex, node.EndIndex))); + return base.Walk(node); + } + + private static string GetFullName(MemberExpression expr) { + var ne = expr.Target as NameExpression; + if (ne != null) { + return ne.Name + "." + expr.Name ?? string.Empty; + } + var me = expr.Target as MemberExpression; + if (me != null) { + var baseName = GetFullName(me); + if (baseName == null) { + return null; + } + return baseName + "." + expr.Name ?? string.Empty; + } + return null; + } + + public override bool Walk(MemberExpression node) { + var fullname = GetFullName(node); + if (fullname != null) { + _head.Names.Add(Tuple.Create(fullname, Span.FromBounds(node.NameHeader, node.EndIndex))); + } + return base.Walk(node); + } + + public override bool Walk(DottedName node) { + string totalName = ""; + foreach (var name in node.Names) { + _head.Names.Add(Tuple.Create(totalName + name.Name, Span.FromBounds(name.StartIndex, name.EndIndex))); + totalName += name.Name + "."; + } + return base.Walk(node); + } + + private BuiltinTypeId GetTypeId(AnalysisValue v) { + if (v.TypeId != BuiltinTypeId.Type) { + return v.TypeId; + } + + if (v.MemberType == PythonMemberType.Instance && + v.IsOfType(_analysis.ProjectState.ClassInfos[BuiltinTypeId.Type])) { + return BuiltinTypeId.Type; + } + + return BuiltinTypeId.Unknown; + } + + private string ClassifyName(Tuple node) { + var name = node.Item1; + foreach (var sd in _head.EnumerateTowardsGlobal) { + if (sd.Parameters.Contains(name)) { + return Classifications.Parameter; + } else if (sd.Functions.Contains(name)) { + return Classifications.Function; + } else if (sd.Types.Contains(name)) { + return Classifications.Class; + } else if (sd.TypeHints.Contains(name)) { + return Classifications.TypeHint; + } else if (sd.Modules.Contains(name)) { + return Classifications.Module; + } + } + + if (_analysis != null) { + var memberType = PythonMemberType.Unknown; + var typeId = BuiltinTypeId.Unknown; + bool isTypeHint = false; + lock (_analysis) { + var values = _analysis.GetValuesByIndex(name, node.Item2.Start).ToArray(); + isTypeHint = values.Any(v => v is TypingTypeInfo || v.DeclaringModule?.ModuleName == "typing"); + memberType = values.Select(v => v.MemberType) + .DefaultIfEmpty(PythonMemberType.Unknown) + .Aggregate((a, b) => a == b || b == PythonMemberType.Unknown ? a : PythonMemberType.Unknown); + typeId = values.Select(GetTypeId) + .DefaultIfEmpty(BuiltinTypeId.Unknown) + .Aggregate((a, b) => a == b || b == BuiltinTypeId.Unknown ? a : BuiltinTypeId.Unknown); + } + + if (isTypeHint) { + return Classifications.TypeHint; + } + if (memberType == PythonMemberType.Module || typeId == BuiltinTypeId.Module) { + return Classifications.Module; + } else if (memberType == PythonMemberType.Class || typeId == BuiltinTypeId.Type) { + return Classifications.Class; + } else if (memberType == PythonMemberType.Function || memberType == PythonMemberType.Method || + typeId == BuiltinTypeId.Function || typeId == BuiltinTypeId.BuiltinFunction) { + return Classifications.Function; + } + } + + return null; + } + + private void EndScope(bool mergeNames) { + var sd = _head; + foreach (var node in sd.Names) { + var classificationName = ClassifyName(node); + if (classificationName != null) { + AddSpan(node, classificationName); + if (mergeNames && sd.Previous != null) { + if (classificationName == Classifications.Module) { + sd.Previous.Modules.Add(sd.Name + "." + node.Item1); + } else if (classificationName == Classifications.Class) { + sd.Previous.Types.Add(sd.Name + "." + node.Item1); + } else if (classificationName == Classifications.Function) { + sd.Previous.Functions.Add(sd.Name + "." + node.Item1); + } + } + } + } + _head = sd.Previous; + } + + public override bool Walk(PythonAst node) { + Debug.Assert(_head == null); + _head = new StackData(string.Empty, null); + return base.Walk(node); + } + + public override void PostWalk(PythonAst node) { + EndScope(false); + Debug.Assert(_head == null); + base.PostWalk(node); + } + + private void MaybeAddDocstring(Node body) { + var docString = (body as SuiteStatement)?.Statements?[0] as ExpressionStatement; + if (docString?.Expression is ConstantExpression ce && (ce.Value is string || ce.Value is AsciiString)) { + AddSpan(Tuple.Create("", Span.FromBounds(ce.StartIndex, ce.EndIndex)), Classifications.DocString); + } + } + + public override bool Walk(ClassDefinition node) { + Debug.Assert(_head != null); + _head.Types.Add(node.NameExpression.Name); + node.NameExpression.Walk(this); + BeginScope(node.NameExpression.Name); + MaybeAddDocstring(node.Body); + return base.Walk(node); + } + + public override bool Walk(FunctionDefinition node) { + if (node.IsCoroutine) { + AddSpan(Tuple.Create("", new Span(node.DefIndex, 5)), Classifications.Keyword); + } + + Debug.Assert(_head != null); + _head.Functions.Add(node.NameExpression.Name); + node.NameExpression.Walk(this); + BeginScope(); + MaybeAddDocstring(node.Body); + return base.Walk(node); + } + + public override bool Walk(DictionaryComprehension node) { + BeginScope(); + return base.Walk(node); + } + + public override bool Walk(ListComprehension node) { + BeginScope(); + return base.Walk(node); + } + + public override bool Walk(GeneratorExpression node) { + BeginScope(); + return base.Walk(node); + } + + public override bool Walk(ComprehensionFor node) { + AddParameter(node.Left); + + if (node.IsAsync) { + AddSpan(Tuple.Create("", new Span(node.StartIndex, 5)), Classifications.Keyword); + } + + return base.Walk(node); + } + + public override bool Walk(Parameter node) { + AddParameter(node); + return base.Walk(node); + } + + public override bool Walk(ImportStatement node) { + Debug.Assert(_head != null); + if (node.AsNames != null) { + foreach (var name in node.AsNames) { + if (name != null && !string.IsNullOrEmpty(name.Name)) { + _head.Modules.Add(name.Name); + _head.Names.Add(Tuple.Create(name.Name, Span.FromBounds(name.StartIndex, name.EndIndex))); + } + } + } + if (node.Names != null) { + for (int i = 0; i < node.Names.Count; ++i) { + var dottedName = node.Names[i]; + var hasAsName = (node.AsNames != null && node.AsNames.Count > i) ? node.AsNames[i] != null : false; + foreach (var name in dottedName.Names) { + if (name != null && !string.IsNullOrEmpty(name.Name)) { + if (!hasAsName) { + _head.Modules.Add(name.Name); + _head.Names.Add(Tuple.Create(name.Name, Span.FromBounds(name.StartIndex, name.EndIndex))); + } else { + // Only want to highlight this instance of the + // name, since it isn't going to be bound in the + // rest of the module. + AddSpan(Tuple.Create(name.Name, Span.FromBounds(name.StartIndex, name.EndIndex)), Classifications.Module); + } + } + } + } + } + return base.Walk(node); + } + + public override bool Walk(FromImportStatement node) { + Debug.Assert(_head != null); + + if (node.Root != null) { + foreach (var name in node.Root.Names) { + if (name != null && !string.IsNullOrEmpty(name.Name)) { + AddSpan(Tuple.Create(name.Name, Span.FromBounds(name.StartIndex, name.EndIndex)), Classifications.Module); + } + } + } + if (node.Names != null) { + for (int i = 0; i < node.Names.Count; ++i) { + var name = node.Names[i]; + var asName = (i < node.AsNames?.Count) ? node.AsNames[i] : null; + if (!string.IsNullOrEmpty(asName?.Name)) { + _head.Names.Add(Tuple.Create(asName.Name, Span.FromBounds(name.StartIndex, name.EndIndex))); + _head.Names.Add(Tuple.Create(asName.Name, Span.FromBounds(asName.StartIndex, asName.EndIndex))); + } else if (!string.IsNullOrEmpty(name?.Name)) { + _head.Names.Add(Tuple.Create(name.Name, Span.FromBounds(name.StartIndex, name.EndIndex))); + } + } + } + return base.Walk(node); + } + + + + public override void PostWalk(ClassDefinition node) { + EndScope(true); + Debug.Assert(_head != null); + base.PostWalk(node); + } + + public override void PostWalk(FunctionDefinition node) { + EndScope(false); + Debug.Assert(_head != null); + base.PostWalk(node); + } + + public override void PostWalk(DictionaryComprehension node) { + EndScope(false); + Debug.Assert(_head != null); + base.PostWalk(node); + } + + public override void PostWalk(ListComprehension node) { + EndScope(false); + Debug.Assert(_head != null); + base.PostWalk(node); + } + + public override void PostWalk(GeneratorExpression node) { + EndScope(false); + Debug.Assert(_head != null); + base.PostWalk(node); + } + + + public override bool Walk(AwaitExpression node) { + AddSpan(Tuple.Create("", new Span(node.StartIndex, 5)), Classifications.Keyword); + return base.Walk(node); + } + + public override bool Walk(ForStatement node) { + if (node.IsAsync) { + AddSpan(Tuple.Create("", new Span(node.StartIndex, 5)), Classifications.Keyword); + } + return base.Walk(node); + } + + public override bool Walk(WithStatement node) { + if (node.IsAsync) { + AddSpan(Tuple.Create("", new Span(node.StartIndex, 5)), Classifications.Keyword); + } + return base.Walk(node); + } + + private static readonly HashSet RegexFunctionNames = new HashSet { + "compile", + "escape", + "findall", + "finditer", + "fullmatch", + "match", + "search", + "split", + "sub", + "subn" + }; + + public override bool Walk(CallExpression node) { + bool isRegex = false; + + if (node.Target is MemberExpression me && RegexFunctionNames.Contains(me.Name) && me.Target is NameExpression target) { + if (_analysis.GetValues(target.Name, me.GetStart(_ast)).Any(m => m is IModule && m.Name == "re")) { + isRegex = true; + } + } else if (node.Target is NameExpression ne && RegexFunctionNames.Contains(ne.Name)) { + if (_analysis.GetValues(ne.Name, ne.GetStart(_ast)).OfType() + .Any(f => f.Function?.DeclaringType == null && f.Function?.DeclaringModule.Name == "re")) { + isRegex = true; + } + } + + if (isRegex && node.Args != null && node.Args.Count > 0 && node.Args[0].Expression is ConstantExpression ce) { + if (ce.Value is string || ce.Value is AsciiString) { + AddSpan(Tuple.Create("", Span.FromBounds(ce.StartIndex, ce.EndIndex)), Classifications.RegexLiteral); + } + } + + return base.Walk(node); + } + + public struct Span { + public readonly int Start, Length; + + public Span(int start, int length) { + Start = start; + Length = length; + } + + public static Span FromBounds(int start, int end) { + return new Span(start, end - start); + } + } + + public struct Classification { + public readonly Span Span; + public readonly string Type; + + public Classification(Span span, string type) { + Span = span; + Type = type; + } + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/MethodExtractor.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/MethodExtractor.cs new file mode 100644 index 000000000..df22da077 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/MethodExtractor.cs @@ -0,0 +1,762 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Microsoft.PythonTools.Analysis; +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Intellisense { + using AP = AnalysisProtocol; + + // TODO: Move this class to Analysis and make it public + + class MethodExtractor { + private readonly PythonAst _ast; + private readonly string _code; + + public MethodExtractor(PythonAst ast, string code) { + _code = code ?? throw new ArgumentNullException(nameof(code)); + _ast = ast ?? throw new ArgumentNullException(nameof(ast)); + } + + public AP.ExtractMethodResponse ExtractMethod(AP.ExtractMethodRequest input, int version) { + // tighten up the selection so that we don't expand into any enclosing nodes because we overlap w/ their white space... + var selectionStart = input.startIndex; + while (selectionStart < _code.Length && + Char.IsWhiteSpace(_code[selectionStart])) { + selectionStart++; + } + + var selectionEnd = input.endIndex; + if (selectionEnd == _code.Length) { + selectionEnd -= 1; + } + while (selectionEnd >= 0 && char.IsWhiteSpace(_code[selectionEnd])) { + selectionEnd -= 1; + } + + var walker = new EnclosingNodeWalker(_ast, selectionStart, selectionEnd); + _ast.Walk(walker); + + Debug.Assert(walker.Target != null); + if (walker.Target == null) { + return new AP.ExtractMethodResponse() { + cannotExtractReason = AP.CannotExtractReason.InvalidTargetSelected + }; + } + + bool expanded = false; + // expand the selection if we aren't currently covering a full expression/statement + if (!walker.Target.IsValidSelection) { + return new AP.ExtractMethodResponse() { + cannotExtractReason = AP.CannotExtractReason.InvalidExpressionSelected + }; + } + + expanded = WasSelectionExpanded(walker.Target, selectionStart, selectionEnd); + + // check for things we cannot handle + if (!IsValidExtraction(walker.Target, out var failureReason)) { + return new AP.ExtractMethodResponse() { + // Note: this returns the unformatted error + cannotExtractReason = failureReason + }; + } + + // get the variables which are read by the selected statement(s) + var varCollector = new InnerVariableWalker(_ast); + walker.Target.Walk(varCollector); + + // Walk the target to understand flow control and definite assignment to further understand + // what we need to flow in. For example if a variable is assigned to both in the un-extracted + // and extracted code but it's definitely assigned in the extracted code then we don't need + // to flow the variable in. + + // Run flow checker, first on any nested scopes... + foreach (ScopeStatement scope in varCollector._scopes) { + FlowChecker.Check(scope); + } + + // then on our extracted code + var parent = walker.Target.Parents[walker.Target.Parents.Count - 1]; + HashSet readBeforeInit; + FlowChecker extractedChecker = null; + if (parent.ScopeVariables != null) { + extractedChecker = new FlowChecker(parent); + + walker.Target.Walk(extractedChecker); + readBeforeInit = extractedChecker.ReadBeforeInitializedVariables; + } else { + readBeforeInit = new HashSet(); + } + + // then on code which follows our extracted body + var afterStmts = walker.Target.GetStatementsAfter(); + HashSet readByFollowingCodeBeforeInit = null; + var parentNode = walker.Target.Parents[walker.Target.Parents.Count - 1]; + var outputVars = new HashSet(); + if (parentNode.ScopeVariables != null) { + var checker = new FlowChecker(parentNode); + + foreach (var afterStmt in afterStmts) { + afterStmt.Walk(checker); + } + + readByFollowingCodeBeforeInit = checker.ReadBeforeInitializedVariables; + + foreach (var variable in varCollector._allWrittenVariables) { + if (variable != null && variable.Scope is PythonAst) { + // global variable assigned to in outer scope, and left with + // a valid value (not deleted) from the extracted code. We + // need to pass the value back out and assign to it in the + // global scope. + if (!checker.IsInitialized(variable) && + extractedChecker.IsAssigned(variable)) { + outputVars.Add(variable); + } + } + } + } + + // collect any nested scopes, and see if they read any free variables + var scopeCollector = new ScopeCollector(); + foreach (var afterStmt in afterStmts) { + afterStmt.Walk(scopeCollector); + } + + foreach (var scope in scopeCollector._scopes) { + if (scope.FreeVariables != null) { + foreach (var freeVar in scope.FreeVariables) { + if (varCollector._allWrittenVariables.Contains(freeVar)) { + // we've assigned to a variable accessed from an inner + // scope, we need to get the value of the variable back out. + outputVars.Add(freeVar); + } + } + } + } + + + // discover any variables which are consumed and need to be available as outputs... + var outputCollector = new OuterVariableWalker(_ast, walker.Target, varCollector, readBeforeInit, readByFollowingCodeBeforeInit, outputVars); + _ast.Walk(outputCollector); + + if (outputCollector._outputVars.Count > 0 && + walker.Target.ContainsReturn) { + return new AP.ExtractMethodResponse { + cannotExtractReason = AP.CannotExtractReason.MethodAssignsVariablesAndReturns + }; + } + + var targetScope = walker.Target.Parents[input.scope ?? 0]; + var creator = new OutOfProcExtractedMethodCreator( + _ast, + walker.Target.Parents, + outputCollector._inputVars, + outputCollector._outputVars, + walker.Target, + input.indentSize, + !input.convertTabsToSpaces, + input.newLine, + input.name, + input.parameters ?? new string[0], + walker.Target.Parents[input.scope ?? 0] + ); + + // get the new method body... + var newMethod = creator.GetExtractionResult(); + + var changes = new List(); + + var callChange = DocumentChange.Replace(walker.Target.StartIncludingIndentation, walker.Target.End, newMethod.Call); + var methodChange = DocumentChange.Insert(newMethod.Method, walker.Target.InsertLocations[targetScope]); + if (callChange.ReplacedSpan.Start < methodChange.ReplacedSpan.Start) { + changes.Add(callChange); + changes.Add(methodChange); + } else { + changes.Add(methodChange); + changes.Add(callChange); + } + + List scopes = new List(); + for(int i = 0; i x.Name).ToArray(), + scopes = scopes.ToArray(), + wasExpanded = expanded, + startLine = walker.Target.StartIncludingIndentation.Line, + startCol = walker.Target.StartIncludingIndentation.Column, + endLine = walker.Target.End.Line, + endCol = walker.Target.End.Column, + version = version + }; + } + + private string[] GetScopeVariables(ScopeStatement scope, HashSet inputVars) { + List res = new List(); + foreach (var variable in inputVars) { + var variableScope = variable.Scope; + + var parentScope = scope; + // are these variables a child of the target scope so we can close over them? + while (parentScope != null && parentScope != variableScope) { + parentScope = parentScope.Parent; + } + + if (parentScope != null) { + res.Add(variable.Name); + } + } + return res.ToArray(); + } + + private string GetScopeType(ScopeStatement scope) { + if (scope is ClassDefinition) { + return "class"; + } else if (scope is FunctionDefinition) { + return "function"; + } + + return "global"; + } + + private bool WasSelectionExpanded(SelectionTarget target, int selectionStart, int selectionEnd) { + int startIndex = target.Ast.LocationToIndex(target.StartIncludingIndentation); + if (startIndex != selectionStart) { + for (var curChar = selectionStart - 1; curChar >= startIndex; curChar -= 1) { + if (!Char.IsWhiteSpace(_code[curChar])) { + return true; + } + } + } + int endIndex = target.Ast.LocationToIndex(target.End); + if (endIndex != selectionEnd) { + for (var curChar = selectionEnd + 1; curChar < endIndex; curChar += 1) { + if (!Char.IsWhiteSpace(_code[curChar])) { + return true; + } + } + } + return false; + } + + private static bool IsValidExtraction(SelectionTarget target, out AP.CannotExtractReason failureReason) { + if (target.Parents[target.Parents.Count - 1] is ClassDefinition) { + failureReason = AP.CannotExtractReason.StatementsFromClassDefinition; + return false; + } + + var breakContinueWalker = new ContinueBreakWalker(); + target.Walk(breakContinueWalker); + if (breakContinueWalker.ContainsBreak) { + failureReason = AP.CannotExtractReason.SelectionContainsBreakButNotEnclosingLoop; + return false; + } else if (breakContinueWalker.ContainsContinue) { + failureReason = AP.CannotExtractReason.SelectionContainsContinueButNotEnclosingLoop; + return false; + } + + var yieldWalker = new YieldWalker(); + target.Walk(yieldWalker); + if (yieldWalker.ContainsYield) { + failureReason = AP.CannotExtractReason.ContainsYieldExpression; + return false; + } + + var importStarWalker = new ImportStarWalker(); + target.Walk(importStarWalker); + if (importStarWalker.ContainsImportStar) { + failureReason = AP.CannotExtractReason.ContainsFromImportStar; + return false; + } + + var returnWalker = new ReturnWalker(); + target.Walk(returnWalker); + if (returnWalker.ContainsReturn && !returnWalker.Returns) { + failureReason = AP.CannotExtractReason.SelectionContainsReturn; + return false; + } + + target.ContainsReturn = returnWalker.ContainsReturn; + failureReason = AP.CannotExtractReason.None; + return true; + } + + class ReturnWalker : PythonWalker { + public bool ContainsReturn, Returns; + private bool _raises; + + public override bool Walk(ReturnStatement node) { + Returns = true; + ContainsReturn = true; + return base.Walk(node); + } + + public override bool Walk(RaiseStatement node) { + _raises = true; + return base.Walk(node); + } + + public override bool Walk(IfStatement node) { + bool allReturn = true; + for (int i = 0; i < node.TestsInternal.Length; i++) { + _raises = Returns = false; + node.TestsInternal[i].Body.Walk(this); + + allReturn &= Returns || _raises; + } + + _raises = Returns = false; + if (node.ElseStatement != null) { + node.ElseStatement.Walk(this); + + allReturn &= Returns || _raises; + } else { + allReturn = false; + } + + + Returns = allReturn; + return false; + } + + public override bool Walk(ForStatement node) { + WalkLoop(node.Body, node.Else); + return false; + } + + public override bool Walk(WhileStatement node) { + WalkLoop(node.Body, node.ElseStatement); + return false; + } + + private void WalkLoop(Statement body, Statement elseStmt) { + bool allReturn = true; + + _raises = Returns = false; + body.Walk(this); + + allReturn &= Returns || _raises; + + if (elseStmt != null) { + _raises = false; + elseStmt.Walk(this); + allReturn &= Returns || _raises; + } + + Returns = allReturn; + } + + public override bool Walk(SuiteStatement node) { + foreach (var statement in node.Statements) { + if (statement is BreakStatement || statement is ContinueStatement) { + // code after here is unreachable + break; + } + + Returns = false; + statement.Walk(this); + if (Returns) { + // rest of the code is unreachable... + break; + } + } + return false; + } + + public override bool Walk(TryStatement node) { + node.Body.Walk(this); + + if (node.Handlers != null && node.Handlers.Count > 0) { + // treat any exceptions from the body as handled, any exceptions + // from the handlers/else are not handled. + _raises = false; + + foreach (var handler in node.Handlers) { + handler.Walk(this); + } + } + + if (node.Finally != null) { + node.Finally.Walk(this); + } + + if (node.Else != null) { + node.Else.Walk(this); + } + + return false; + } + + public override bool Walk(FunctionDefinition node) { + return false; + } + + public override bool Walk(ClassDefinition node) { + return false; + } + } + + class ContinueBreakWalker : PythonWalker { + public bool ContainsBreak, ContainsContinue; + + public override bool Walk(ForStatement node) { + return false; + } + + public override bool Walk(WhileStatement node) { + return false; + } + + public override bool Walk(FunctionDefinition node) { + return false; + } + + public override bool Walk(ContinueStatement node) { + if (!ContainsBreak) { + ContainsContinue = true; + } + return base.Walk(node); + } + + public override bool Walk(BreakStatement node) { + if (!ContainsContinue) { + ContainsBreak = true; + } + return base.Walk(node); + } + } + + class YieldWalker : PythonWalker { + public bool ContainsYield; + + public override bool Walk(FunctionDefinition node) { + return false; + } + + public override bool Walk(YieldExpression node) { + ContainsYield = true; + return base.Walk(node); + } + + public override bool Walk(YieldFromExpression node) { + ContainsYield = true; + return base.Walk(node); + } + } + + class ImportStarWalker : PythonWalker { + public bool ContainsImportStar; + + public override bool Walk(FunctionDefinition node) { + return false; + } + + public override bool Walk(ClassDefinition node) { + return false; + } + + public override bool Walk(FromImportStatement node) { + if (node.Names.Count == 1 && node.Names[0].Name == "*") { + ContainsImportStar = true; + } + return base.Walk(node); + } + } + + /// + /// Inspects the variables used in the surrounding code to figure out which ones need to be flowed in + /// and which ones need to be returned based upon the variables we collected which are read/assigned + /// from the code being extracted. + /// + class OuterVariableWalker : AssignmentWalker { + private readonly PythonAst _root; + private readonly InnerVariableWalker _inputCollector; + private readonly DefineWalker _define; + private readonly SelectionTarget _target; + internal readonly HashSet _outputVars; + internal readonly HashSet _inputVars = new HashSet(); + private readonly HashSet _readBeforeInitialized, _readByFollowingCodeBeforeInit; + private bool _inLoop = false; + + public OuterVariableWalker(PythonAst root, SelectionTarget target, InnerVariableWalker inputCollector, HashSet readBeforeInitialized, HashSet readByFollowingCodeBeforeInit, HashSet outputVars) { + _root = root; + _target = target; + _inputCollector = inputCollector; + _readBeforeInitialized = readBeforeInitialized; + _readByFollowingCodeBeforeInit = readByFollowingCodeBeforeInit; + _outputVars = outputVars; + _define = new DefineWalker(this); + } + + public override AssignedNameWalker Define { + get { return _define; } + } + + public override bool Walk(FunctionDefinition node) { + if (!node.IsLambda) { + _define.WalkName(node.NameExpression, node.GetVariableReference(_root)); + } + + bool oldInLoop = _inLoop; + _inLoop = false; + var res = base.Walk(node); + _inLoop = oldInLoop; + return res; + } + + public override bool Walk(ClassDefinition node) { + _define.WalkName(node.NameExpression, node.GetVariableReference(_root)); + + bool oldInLoop = _inLoop; + _inLoop = false; + var res = base.Walk(node); + _inLoop = oldInLoop; + return res; + } + + public override bool Walk(WhileStatement node) { + if (node.Test != null) { + node.Test.Walk(this); + } + if (node.Body != null) { + bool oldInLoop = _inLoop; + _inLoop = true; + node.Body.Walk(this); + _inLoop = oldInLoop; + } + if (node.ElseStatement != null) { + node.ElseStatement.Walk(this); + } + return false; + } + + public override bool Walk(ForStatement node) { + if (node.Left != null) { + node.Left.Walk(Define); + } + + if (node.List != null) { + node.List.Walk(this); + } + if (node.Body != null) { + bool oldInLoop = _inLoop; + _inLoop = true; + node.Body.Walk(this); + _inLoop = oldInLoop; + } + if (node.Else != null) { + node.Else.Walk(this); + } + return false; + } + + public override bool Walk(NameExpression node) { + var reference = node.GetVariableReference(_root); + if (reference != null && !_inputCollector._allReads.Contains(reference) && !_inputCollector._allWrites.Contains(reference)) { + // this variable is referenced outside of the refactored code + if (node.GetStart(_root) < _target.StartIncludingIndentation) { + // it's read before the extracted code, we don't care... + } else { + Debug.Assert(node.GetEnd(_root) > _target.End, "didn't reference variable in extracted range"); + + // it's read after the extracted code, if its written to in the refactored + // code we need to include it as an output + if (_inputCollector._allWrittenVariables.Contains(reference.Variable) && + (_readByFollowingCodeBeforeInit == null || _readByFollowingCodeBeforeInit.Contains(reference.Variable))) { + // the variable is written to by the refactored code + _outputVars.Add(reference.Variable); + } + } + } + + return true; + } + + public override bool Walk(Parameter node) { + var variable = node.GetVariable(_root); + if (ReadFromExtractedCode(variable)) { + _inputVars.Add(variable); + } + + return base.Walk(node); + } + + private bool ReadFromExtractedCode(PythonVariable variable) { + return _readBeforeInitialized.Contains(variable) && + _inputCollector._allReadVariables.Contains(variable); + } + + class DefineWalker : AssignedNameWalker { + private readonly OuterVariableWalker _collector; + + public DefineWalker(OuterVariableWalker collector) { + _collector = collector; + } + + public override bool Walk(NameExpression node) { + var reference = node.GetVariableReference(_collector._root); + + return WalkName(node, reference); + } + + internal bool WalkName(NameExpression node, PythonReference reference) { + if (_collector.ReadFromExtractedCode(reference.Variable)) { + if ((!_collector._inputCollector._allReads.Contains(reference) && + !_collector._inputCollector._allWrites.Contains(reference))) { + + // the variable is assigned outside the refactored code + if (node.GetStart(_collector._root) < _collector._target.StartIncludingIndentation) { + // it's assigned before the extracted code + _collector._inputVars.Add(reference.Variable); + } else { + Debug.Assert(node.GetEnd(_collector._root) > _collector._target.End); + // it's assigned afterwards, we don't care... + } + } else if (_collector._readBeforeInitialized.Contains(reference.Variable) && + _collector._inputCollector._allWrites.Contains(reference) && + _collector._inLoop) { + // we read an un-initialized value, so it needs to be passed in. If we + // write to it as well then we need to pass it back out for future calls. + _collector._outputVars.Add(reference.Variable); + } + } + + return false; + } + } + } + + class ScopeCollector : PythonWalker { + internal readonly List _scopes = new List(); + + public override void PostWalk(ClassDefinition node) { + _scopes.Add(node); + base.PostWalk(node); + } + + public override void PostWalk(FunctionDefinition node) { + _scopes.Add(node); + base.PostWalk(node); + } + } + + /// + /// Walks the code which is being extracted and collects all the variables which are read from and written to. + /// + class InnerVariableWalker : AssignmentWalker { + private readonly PythonAst _root; + internal readonly HashSet _allReads = new HashSet(); + internal readonly HashSet _allWrites = new HashSet(); + internal readonly HashSet _allWrittenVariables = new HashSet(); + internal readonly HashSet _allReadVariables = new HashSet(); + internal readonly List _scopes = new List(); + + private readonly DefineWalker _define; + + public InnerVariableWalker(PythonAst root) { + _root = root; + _define = new DefineWalker(this); + } + + public override AssignedNameWalker Define { + get { return _define; } + } + + public override bool Walk(NameExpression node) { + var reference = node.GetVariableReference(_root); + + if (reference != null) { + _allReads.Add(reference); + _allReadVariables.Add(reference.Variable); + } + + return true; + } + + public override void PostWalk(ClassDefinition node) { + _scopes.Add(node); + _allWrites.Add(node.GetVariableReference(_root)); + _allWrittenVariables.Add(node.Variable); + base.PostWalk(node); + } + + public override void PostWalk(FunctionDefinition node) { + _scopes.Add(node); + _allWrites.Add(node.GetVariableReference(_root)); + _allWrittenVariables.Add(node.Variable); + base.PostWalk(node); + } + + public override bool Walk(FromImportStatement node) { + var vars = node.Variables; + var refs = node.GetReferences(_root); + if (refs != null) { // from .. import * will have null refs + for (int i = 0; i < vars.Length; i++) { + if (vars[i] != null) { + _allWrites.Add(refs[i]); + _allWrittenVariables.Add(vars[i]); + } + } + } + return base.Walk(node); + } + + public override bool Walk(ImportStatement node) { + var vars = node.Variables; + var refs = node.GetReferences(_root); + for (int i = 0; i < vars.Length; i++) { + if (vars[i] != null) { + _allWrites.Add(refs[i]); + _allWrittenVariables.Add(vars[i]); + } + } + return base.Walk(node); + } + + class DefineWalker : AssignedNameWalker { + private readonly InnerVariableWalker _collector; + + public DefineWalker(InnerVariableWalker collector) { + _collector = collector; + } + + public override bool Walk(NameExpression node) { + var reference = node.GetVariableReference(_collector._root); + + _collector._allWrites.Add(reference); + _collector._allWrittenVariables.Add(reference.Variable); + return false; + } + } + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutOfProcMethodExtractor.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutOfProcMethodExtractor.cs new file mode 100644 index 000000000..d9d3247d6 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutOfProcMethodExtractor.cs @@ -0,0 +1,38 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Globalization; +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Intellisense { + using AP = AnalysisProtocol; + + // TODO: This class should transform the request and pass to MethodExtractor + // MethodExtractor should move to Analysis and remove all direct depnedencies + // on AnalysisProtocol. For now, we are keeping it here to save effort. + + class OutOfProcMethodExtractor { + private readonly MethodExtractor _extractor; + + public OutOfProcMethodExtractor(PythonAst ast, string code) { + _extractor = new MethodExtractor(ast, code); + } + + public AP.ExtractMethodResponse ExtractMethod(AP.ExtractMethodRequest input, int version) { + return _extractor.ExtractMethod(input, version); + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutOfProcProjectAnalyzer.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutOfProcProjectAnalyzer.cs new file mode 100644 index 000000000..f1346fbf9 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutOfProcProjectAnalyzer.cs @@ -0,0 +1,1698 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.LanguageServer.Implementation; +using Microsoft.PythonTools.Analysis; +using Microsoft.PythonTools.Analysis.Analyzer; +using Microsoft.PythonTools.Analysis.Documentation; +using Microsoft.PythonTools.Analysis.Infrastructure; +using Microsoft.PythonTools.Interpreter; +using Microsoft.PythonTools.Ipc.Json; +using Microsoft.PythonTools.Parsing; +using Microsoft.PythonTools.Parsing.Ast; +using Microsoft.PythonTools.Projects; + +namespace Microsoft.PythonTools.Intellisense { + using AP = AnalysisProtocol; + using LS = Microsoft.Python.LanguageServer; + + /// + /// Performs centralized parsing and analysis of Python source code for a remotely running process. + /// + /// This class is responsible for maintaining the up-to-date analysis of the active files being worked + /// on inside of a single proejct. + /// + /// This class is built upon the core PythonAnalyzer class which provides basic analysis services. This class + /// maintains the thread safety invarients of working with that class, handles parsing of files as they're + /// updated via interfacing w/ the remote process editor APIs, and supports adding additional files to the + /// analysis. + /// + public sealed class OutOfProcProjectAnalyzer : IDisposable { + private readonly Server _server; + private readonly Dictionary _extensions; + private readonly Action _log; + + private bool _isDisposed; + + private readonly Connection _connection; + + public OutOfProcProjectAnalyzer(Stream writer, Stream reader, Action log) { + _server = new Server(); + _server.OnParseComplete += OnParseComplete; + _server.OnAnalysisComplete += OnAnalysisComplete; + _server.OnLogMessage += Server_OnLogMessage; + _server.OnPublishDiagnostics += OnPublishDiagnostics; + _server.AnalysisQueue.AnalysisComplete += AnalysisQueue_Complete; + _server.AnalysisQueue.AnalysisAborted += AnalysisQueue_Aborted; + + _log = log; + Options = new AP.AnalysisOptions(); + + _extensions = new Dictionary(); + + _connection = new Connection( + writer, + true, + reader, + true, + RequestHandler, + AP.RegisteredTypes, + "Analyzer" + ); + _connection.EventReceived += ConectionReceivedEvent; + if (!string.IsNullOrEmpty(_connection.LogFilename)) { + _log?.Invoke($"Connection log: {_connection.LogFilename}"); + } + } + + private void Server_OnLogMessage(object sender, LS.LogMessageEventArgs e) { + if (_log != null && Options.traceLevel.HasValue && e.type <= Options.traceLevel.Value) { + _log(e.message); + _connection?.SendEventAsync(new AP.AnalyzerWarningEvent { message = e.message }).DoNotWait(); + } + } + + private void AnalysisQueue_Aborted(object sender, EventArgs e) { + _connection.Dispose(); + } + + private void ConectionReceivedEvent(object sender, EventReceivedEventArgs e) { + switch (e.Event.name) { + case AP.ModulesChangedEvent.Name: OnModulesChanged(this, EventArgs.Empty); break; + case AP.FileChangedEvent.Name: OnFileChanged((AP.FileChangedEvent)e.Event); break; + } + } + + private async Task RequestHandler(RequestArgs requestArgs, Func done) { + Response response; + var command = requestArgs.Command; + var request = requestArgs.Request; + + // These commands send their own responses, and then we return. + switch (command) { + case AP.AddFileRequest.Command: await AnalyzeFileAsync((AP.AddFileRequest)request, done); return; + case AP.AddBulkFileRequest.Command: await AnalyzeFileAsync((AP.AddBulkFileRequest)request, done); return; + } + + // These commands return a response, which we then send. + switch (command) { + case AP.UnloadFileRequest.Command: response = await UnloadFile((AP.UnloadFileRequest)request); break; + case AP.CompletionsRequest.Command: response = await GetCompletions(request); break; + case AP.GetAllMembersRequest.Command: response = await GetAllMembers(request); break; + case AP.GetModulesRequest.Command: response = await GetModules(request); break; + case AP.SignaturesRequest.Command: response = await GetSignatures((AP.SignaturesRequest)request); break; + case AP.QuickInfoRequest.Command: response = await GetQuickInfo((AP.QuickInfoRequest)request); break; + case AP.AnalyzeExpressionRequest.Command: response = await AnalyzeExpression((AP.AnalyzeExpressionRequest)request); break; + case AP.OutliningRegionsRequest.Command: response = GetOutliningRegions((AP.OutliningRegionsRequest)request); break; + case AP.NavigationRequest.Command: response = await GetNavigationsAsync((AP.NavigationRequest)request); break; + case AP.FileUpdateRequest.Command: response = await UpdateContent((AP.FileUpdateRequest)request); break; + case AP.AddImportRequest.Command: response = AddImportRequest((AP.AddImportRequest)request); break; + case AP.IsMissingImportRequest.Command: response = IsMissingImport((AP.IsMissingImportRequest)request); break; + case AP.AvailableImportsRequest.Command: response = AvailableImports((AP.AvailableImportsRequest)request); break; + case AP.FormatCodeRequest.Command: response = FormatCode((AP.FormatCodeRequest)request); break; + case AP.RemoveImportsRequest.Command: response = RemoveImports((AP.RemoveImportsRequest)request); break; + case AP.ExtractMethodRequest.Command: response = ExtractMethod((AP.ExtractMethodRequest)request); break; + case AP.AnalysisStatusRequest.Command: response = AnalysisStatus(); break; + case AP.LocationNameRequest.Command: response = GetLocationName((AP.LocationNameRequest)request); break; + case AP.ProximityExpressionsRequest.Command: response = GetProximityExpressions((AP.ProximityExpressionsRequest)request); break; + case AP.AnalysisClassificationsRequest.Command: response = GetAnalysisClassifications((AP.AnalysisClassificationsRequest)request); break; + case AP.MethodInsertionLocationRequest.Command: response = GetMethodInsertionLocation((AP.MethodInsertionLocationRequest)request); break; + case AP.MethodInfoRequest.Command: response = GetMethodInfo((AP.MethodInfoRequest)request); break; + case AP.FindMethodsRequest.Command: response = FindMethods((AP.FindMethodsRequest)request); break; + case AP.AddReferenceRequest.Command: response = AddReference((AP.AddReferenceRequest)request); break; + case AP.RemoveReferenceRequest.Command: response = RemoveReference((AP.RemoveReferenceRequest)request); break; + case AP.SetSearchPathRequest.Command: response = SetSearchPath((AP.SetSearchPathRequest)request); break; + case AP.ModuleImportsRequest.Command: response = GetModuleImports((AP.ModuleImportsRequest)request); break; + case AP.ValueDescriptionRequest.Command: response = GetValueDescriptions((AP.ValueDescriptionRequest)request); break; + case AP.LoadExtensionRequest.Command: response = LoadExtensionRequest((AP.LoadExtensionRequest)request); break; + case AP.ExtensionRequest.Command: response = ExtensionRequest((AP.ExtensionRequest)request); break; + case AP.ExpressionAtPointRequest.Command: response = ExpressionAtPoint((AP.ExpressionAtPointRequest)request); break; + case AP.InitializeRequest.Command: response = await Initialize((AP.InitializeRequest)request); break; + case AP.SetAnalysisOptionsRequest.Command: response = SetAnalysisOptions((AP.SetAnalysisOptionsRequest)request); break; + case AP.LanguageServerRequest.Command: response = await ProcessLanguageServerRequest((AP.LanguageServerRequest)request); break; + case AP.ExitRequest.Command: throw new OperationCanceledException(); + default: + throw new InvalidOperationException("Unknown command"); + } + + await done(response); + } + + private async Task ProcessLanguageServerRequest(AP.LanguageServerRequest request) { + try { + var body = (Newtonsoft.Json.Linq.JObject)request.body; + object result = null; + + switch (request.name) { + case "textDocument/completion": result = await _server.Completion(body.ToObject(), CancellationToken.None); break; + case "textDocument/hover": result = await _server.Hover(body.ToObject(), CancellationToken.None); break; + case "textDocument/definition": result = await _server.GotoDefinition(body.ToObject(), CancellationToken.None); break; + case "textDocument/references": result = await _server.FindReferences(body.ToObject(), CancellationToken.None); break; + case "textDocument/signatureHelp": result = await _server.SignatureHelp(body.ToObject(), CancellationToken.None); break; + } + + if (result != null) { + return new AP.LanguageServerResponse { body = result }; + } + + return new AP.LanguageServerResponse { error = "Unknown command: " + request.name }; + } catch (Exception ex) { + return new AP.LanguageServerResponse { error = ex.ToString() }; + } + } + + internal void ReportUnhandledException(Exception ex) { + SendUnhandledExceptionAsync(ex).DoNotWait(); + // Allow some time for the other threads to write the event before + // we (probably) come crashing down. + Thread.Sleep(100); + } + + private async Task SendUnhandledExceptionAsync(Exception ex) { + try { + Debug.Fail(ex.ToString()); + await _connection.SendEventAsync( + new AP.UnhandledExceptionEvent(ex) + ).ConfigureAwait(false); + } catch (Exception) { + // We're in pretty bad state, but nothing useful we can do about + // it. + Debug.Fail("Unhandled exception reporting unhandled exception"); + } + } + + private Response IncorrectFileType() { + throw new InvalidOperationException("File was not correct type"); + } + + private Response IncorrectBufferId(Uri documentUri) { + throw new InvalidOperationException($"Buffer was not valid in file {documentUri?.AbsoluteUri ?? "(null)"}"); + } + + private IPythonInterpreterFactory LoadInterpreterFactory(AP.InterpreterInfo info) { + if (string.IsNullOrEmpty(info?.assembly) || string.IsNullOrEmpty(info?.typeName)) { + return null; + } + + var assembly = File.Exists(info.assembly) ? AssemblyName.GetAssemblyName(info.assembly) : new AssemblyName(info.assembly); + var type = Assembly.Load(assembly).GetType(info.typeName, true); + + return (IPythonInterpreterFactory)Activator.CreateInstance( + type, + BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance, + null, + new object[] { info.properties }, + CultureInfo.CurrentCulture + ); + } + + private IAnalysisExtension LoadAnalysisExtension(AP.LoadExtensionRequest info) { + if (string.IsNullOrEmpty(info?.assembly) || string.IsNullOrEmpty(info?.typeName)) { + return null; + } + + var assembly = File.Exists(info.assembly) ? AssemblyName.GetAssemblyName(info.assembly) : new AssemblyName(info.assembly); + var type = Assembly.Load(assembly).GetType(info.typeName, true); + + return (IAnalysisExtension)Activator.CreateInstance( + type, + BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance, + null, + new object[0], + CultureInfo.CurrentCulture + ); + } + + private async Task Initialize(AP.InitializeRequest request) { + try { + await _server.Initialize(new LS.InitializeParams { + rootUri = request.rootUri, + initializationOptions = new LS.PythonInitializationOptions { + interpreter = new LS.PythonInitializationOptions.Interpreter { + assembly = request.interpreter?.assembly, + typeName = request.interpreter?.typeName, + properties = request.interpreter?.properties + }, + displayOptions = new InformationDisplayOptions { + maxDocumentationLineLength = 30, + trimDocumentationLines = true, + maxDocumentationTextLength = 1024, + trimDocumentationText = true, + maxDocumentationLines = 100 + }, + analysisUpdates = true, + traceLogging = request.traceLogging, + }, + capabilities = new LS.ClientCapabilities { + python = new LS.PythonClientCapabilities { + manualFileLoad = !request.analyzeAllFiles, + liveLinting = request.liveLinting + }, + textDocument = new LS.TextDocumentClientCapabilities { + completion = new LS.TextDocumentClientCapabilities.CompletionCapabilities { + completionItem = new LS.TextDocumentClientCapabilities.CompletionCapabilities.CompletionItemCapabilities { + documentationFormat = new[] { MarkupKind.PlainText }, + snippetSupport = false + } + }, + signatureHelp = new LS.TextDocumentClientCapabilities.SignatureHelpCapabilities { + signatureInformation = new LS.TextDocumentClientCapabilities.SignatureHelpCapabilities.SignatureInformationCapabilities { + documentationFormat = new[] { MarkupKind.PlainText }, + _shortLabel = true + } + }, + hover = new LS.TextDocumentClientCapabilities.HoverCapabilities { + contentFormat = new[] { MarkupKind.PlainText } + } + } + } + }, CancellationToken.None); + } catch (Exception ex) { + return new AP.InitializeResponse { + error = ex.Message, + fullError = ex.ToString() + }; + } + + return new AP.InitializeResponse(); + } + + private Response LoadExtensionRequest(AP.LoadExtensionRequest request) { + IAnalysisExtension extension, oldExtension; + + if (Project == null) { + return new AP.LoadExtensionResponse { + error = "Uninitialized analyzer", + fullError = $"Uninitialized analyzer{Environment.NewLine}{new StackTrace()}" + }; + } + + try { + extension = LoadAnalysisExtension(request); + extension.Register(Project); + } catch (Exception ex) { + return new AP.LoadExtensionResponse { + error = ex.Message, + fullError = ex.ToString() + }; + } + + lock (_extensions) { + _extensions.TryGetValue(request.extension, out oldExtension); + _extensions[request.extension] = extension; + } + (oldExtension as IDisposable)?.Dispose(); + + return new AP.LoadExtensionResponse(); + } + + private Response ExtensionRequest(AP.ExtensionRequest request) { + IAnalysisExtension extension; + lock (_extensions) { + if (!_extensions.TryGetValue(request.extension, out extension)) { + return new AP.ExtensionResponse { + error = $"Unknown extension: {request.extension}" + }; + } + } + + try { + return new AP.ExtensionResponse { + response = extension.HandleCommand(request.commandId, request.body) + }; + } catch (Exception ex) { + return new AP.ExtensionResponse { + error = ex.ToString() + }; + } + } + + private Response GetValueDescriptions(AP.ValueDescriptionRequest request) { + var entry = GetPythonEntry(request.documentUri); + if (entry == null) { + return IncorrectFileType(); + } + string[] descriptions = Array.Empty(); + if (entry.Analysis != null) { + var values = entry.Analysis.GetValues( + request.expr, + new SourceLocation( + request.line, + request.column + ) + ); + + descriptions = values.Select(x => x.Description).ToArray(); + } + + return new AP.ValueDescriptionResponse() { + descriptions = descriptions + }; + } + + private Response GetModuleImports(AP.ModuleImportsRequest request) { + var res = Analyzer.GetEntriesThatImportModule( + request.moduleName, + request.includeUnresolved + ); + + return new AP.ModuleImportsResponse() { + modules = res.Select(entry => new AP.ModuleInfo() { + filename = entry.FilePath, + moduleName = entry.ModuleName, + documentUri = entry.DocumentUri + }).ToArray() + }; + } + + private Response SetSearchPath(AP.SetSearchPathRequest request) { + Analyzer.SetSearchPaths(request.dir); + + return new Response(); + } + + private Response RemoveReference(AP.RemoveReferenceRequest request) { + var interp = Interpreter as IPythonInterpreterWithProjectReferences; + if (interp != null) { + interp.RemoveReference(AP.ProjectReference.Convert(request.reference)); + } + return new AP.RemoveReferenceResponse(); + } + + private Response AddReference(AP.AddReferenceRequest request) { + var interp = Interpreter as IPythonInterpreterWithProjectReferences; + if (interp != null) { + interp.AddReferenceAsync(AP.ProjectReference.Convert(request.reference)).Wait(); + } + return new AP.AddReferenceResponse(); + } + + private Response FindMethods(AP.FindMethodsRequest request) { + var analysis = GetPythonEntry(request.documentUri); + + List names = new List(); + if (analysis != null) { + int version; + string code; + var ast = analysis.GetVerbatimAstAndCode( + Analyzer.LanguageVersion, + out version, + out code + ); + + if (ast != null) { + foreach (var classDef in FindClassDef(request.className, ast)) { + SuiteStatement suite = classDef.Body as SuiteStatement; + if (suite != null) { + foreach (var methodCandidate in suite.Statements) { + FunctionDefinition funcDef = methodCandidate as FunctionDefinition; + if (funcDef != null) { + if (request.paramCount != null && request.paramCount != funcDef.Parameters.Length) { + continue; + } + + names.Add(funcDef.Name); + } + } + } + } + } + } + + return new AP.FindMethodsResponse() { + names = names.ToArray() + }; + } + + private Response GetMethodInsertionLocation(AP.MethodInsertionLocationRequest request) { + var analysis = GetPythonEntry(request.documentUri); + if (analysis == null) { + return IncorrectFileType(); + } + + int version; + string code; + var ast = analysis.GetVerbatimAstAndCode( + Analyzer.LanguageVersion, + out version, + out code + ); + + if (ast == null) { + return new AP.MethodInsertionLocationResponse(); + } + + foreach (var classDef in FindClassDef(request.className, ast)) { + int end = classDef.Body.EndIndex; + // insert after the newline at the end of the last statement of the class def + if (code[end] == '\r') { + if (end + 1 < code.Length && + code[end + 1] == '\n') { + end += 2; + } else { + end++; + } + } else if (code[end] == '\n') { + end++; + } + + return new AP.MethodInsertionLocationResponse() { + line = ast.IndexToLocation(end).Line, + column = classDef.Body.GetStart(ast).Column, + version = version + }; + } + + throw new InvalidOperationException("Failed to find class definition"); + } + + private static IEnumerable FindClassDef(string name, PythonAst ast) { + var suiteStmt = ast.Body as SuiteStatement; + foreach (var stmt in suiteStmt.Statements) { + var classDef = stmt as ClassDefinition; + if (classDef != null && + (classDef.Name == name || name == null)) { + yield return classDef; + } + } + } + + private Response GetMethodInfo(AP.MethodInfoRequest request) { + var analysis = GetPythonEntry(request.documentUri); + + if (analysis != null) { + int version; + string code; + var ast = analysis.GetVerbatimAstAndCode( + Project.LanguageVersion, + out version, + out code + ); + + if (ast != null) { + foreach (var classDef in FindClassDef(request.className, ast)) { + SuiteStatement suite = classDef.Body as SuiteStatement; + + if (suite != null) { + foreach (var methodCandidate in suite.Statements) { + FunctionDefinition funcDef = methodCandidate as FunctionDefinition; + if (funcDef != null) { + if (funcDef.Name == request.methodName) { + return new AP.MethodInfoResponse() { + start = funcDef.StartIndex, + end = funcDef.EndIndex, + version = version, + found = true + }; + } + } + } + } + } + } + } + + return new AP.MethodInfoResponse() { + found = false + }; + } + + private Response GetAnalysisClassifications(AP.AnalysisClassificationsRequest request) { + var projEntry = GetPythonEntry(request.documentUri); + + if (projEntry == null) { + return IncorrectFileType(); + } + + var bufferVersion = GetPythonBuffer(request.documentUri); + if (bufferVersion.Ast == null) { + return new AP.AnalysisClassificationsResponse(); + } + + var moduleAnalysis = request.colorNames ? projEntry.Analysis : null; + + var walker = new ClassifierWalker(bufferVersion.Ast, moduleAnalysis); + bufferVersion.Ast.Walk(walker); + + return new AP.AnalysisClassificationsResponse() { + version = bufferVersion.Version, + classifications = walker.Spans.Select(s => new AP.AnalysisClassification { + startLine = s.Span.Start.Line, + startColumn = s.Span.Start.Column, + endLine = s.Span.End.Line, + endColumn = s.Span.End.Column, + type = s.Tag + }).ToArray() + }; + } + + private Response GetProximityExpressions(AP.ProximityExpressionsRequest request) { + var projEntry = GetPythonEntry(request.documentUri); + + var res = new AP.ProximityExpressionsResponse(); + + var tree = projEntry?.Tree; + if (tree == null) { + return res; + } + + int startLine = Math.Max(request.line - request.lineCount + 1, 0); + if (startLine <= request.line) { + var walker = new ProximityExpressionWalker(tree, startLine, request.line); + tree.Walk(walker); + res.names = walker.GetExpressions().ToArray(); + } + + return res; + } + + private Response GetLocationName(AP.LocationNameRequest request) { + var projEntry = GetPythonEntry(request.documentUri); + + var res = new AP.LocationNameResponse(); + + var tree = projEntry?.Tree; + if (tree == null) { + return res; + } + + string foundName = FindNodeInTree(tree, tree.Body as SuiteStatement, request.line); + if (foundName != null) { + res.name = projEntry.ModuleName + "." + foundName; + res.lineOffset = request.column; + } else { + res.name = projEntry.ModuleName; + res.lineOffset = request.column; + } + + return res; + } + + private static string FindNodeInTree(PythonAst tree, SuiteStatement statement, int line) { + if (statement == null) { + return null; + } + + foreach (var node in statement.Statements) { + if (node is FunctionDefinition funcDef) { + var span = funcDef.GetSpan(tree); + if (span.Start.Line <= line && line <= span.End.Line) { + var res = FindNodeInTree(tree, funcDef.Body as SuiteStatement, line); + if (res != null) { + return funcDef.Name + "." + res; + } + return funcDef.Name; + } + } else if (node is ClassDefinition classDef) { + var span = classDef.GetSpan(tree); + if (span.Start.Line <= line && line <= span.End.Line) { + var res = FindNodeInTree(tree, classDef.Body as SuiteStatement, line); + if (res != null) { + return classDef.Name + "." + res; + } + return classDef.Name; + } + } + } + return null; + } + + + private Response AnalysisStatus() { + return new AP.AnalysisStatusResponse() { + itemsLeft = _server.EstimateRemainingWork() + }; + } + + private Response ExtractMethod(AP.ExtractMethodRequest request) { + var projectFile = GetPythonEntry(request.documentUri); + if (projectFile == null) { + return IncorrectFileType(); + } + + int version; + string code; + var ast = projectFile.GetVerbatimAstAndCode( + Project.LanguageVersion, + out version, + out code + ); + + return new OutOfProcMethodExtractor( + ast, + code + ).ExtractMethod(request, version); + } + + private Response RemoveImports(AP.RemoveImportsRequest request) { + var projectFile = GetPythonEntry(request.documentUri); + if (projectFile == null) { + return IncorrectFileType(); + } + + int version; + string code; + var ast = projectFile.GetVerbatimAstAndCode( + Project.LanguageVersion, + out version, + out code + ); + if (ast == null) { + return new AP.RemoveImportsResponse(); + } + var remover = new ImportRemover(ast, code, request.allScopes, ast.LocationToIndex(new SourceLocation(request.line, request.column))); + + return new AP.RemoveImportsResponse() { + changes = remover.RemoveImports().Select(AP.ChangeInfo.FromDocumentChange).ToArray(), + version = version + }; + } + + private Response FormatCode(AP.FormatCodeRequest request) { + var projectFile = GetPythonEntry(request.documentUri); + if (projectFile == null) { + return IncorrectFileType(); + } + + int version; + string code; + var ast = projectFile.GetVerbatimAstAndCode( + Project.LanguageVersion, + out version, + out code + ); + if (ast == null) { + return new AP.FormatCodeResponse(); + } + + int startIndex = ast.LocationToIndex(new SourceLocation(request.startLine, request.startColumn)); + int endIndex = ast.LocationToIndex(new SourceLocation(request.endLine, request.endColumn)); + + var walker = new EnclosingNodeWalker(ast, startIndex, endIndex); + ast.Walk(walker); + + if (walker.Target == null || !walker.Target.IsValidSelection) { + return new AP.FormatCodeResponse(); + } + + var body = walker.Target.GetNode(); + + + var whitspaceStart = walker.Target.StartIncludingIndentation; + + int start = ast.LocationToIndex(walker.Target.StartIncludingLeadingWhiteSpace); + int end = ast.LocationToIndex(walker.Target.End); + if (startIndex > start) { + // the user didn't have any comments selected, don't reformat them + body.SetLeadingWhiteSpace(ast, body.GetIndentationLevel(ast)); + + start = ast.LocationToIndex(walker.Target.StartIncludingIndentation); + } + + int length = end - start; + if (end < code.Length) { + if (code[end] == '\r') { + end++; + length++; + if (end < code.Length && + code[end] == '\n') { + end++; + length++; + } + } else if (code[end] == '\n') { + length++; + } + } + + var selectedCode = code.Substring(start, length); + + return new AP.FormatCodeResponse() { + version = version, + changes = selectedCode.ReplaceByLines( + walker.Target.StartIncludingLeadingWhiteSpace, + body.ToCodeString(ast, request.options), + request.newLine + ).Select(AP.ChangeInfo.FromDocumentChange).ToArray() + }; + } + + private Response AvailableImports(AP.AvailableImportsRequest request) { + return new AP.AvailableImportsResponse { + imports = FindNameInAllModules(request.name) + .Select( + x => new AP.ImportInfo { + importName = x.ImportName, + fromName = x.FromName + } + ) + .Distinct() + .ToArray() + }; + } + + private IEnumerable FindNameInAllModules(string name) { + string pkgName; + + // provide module names first + foreach (var keyValue in Analyzer.Modules.GetModuleStates()) { + var modName = keyValue.Key; + var moduleRef = keyValue.Value; + + if (moduleRef.IsValid) { + // include modules which can be imported + if (modName == name) { + yield return new ExportedMemberInfo(null, modName); + } else if (GetPackageNameIfMatch(name, modName, out pkgName)) { + yield return new ExportedMemberInfo(pkgName, name); + } + } + } + + foreach (var modName in Interpreter.GetModuleNames()) { + if (modName == name) { + yield return new ExportedMemberInfo(null, modName); + } else if (GetPackageNameIfMatch(name, modName, out pkgName)) { + yield return new ExportedMemberInfo(pkgName, name); + } + } + + // then include imported module members + foreach (var keyValue in Analyzer.Modules.GetModuleStates()) { + var modName = keyValue.Key; + var moduleRef = keyValue.Value; + + if (moduleRef.IsValid && moduleRef.ModuleContainsMember(Analyzer._defaultContext, name)) { + yield return new ExportedMemberInfo(modName, name); + } + } + } + + private static bool GetPackageNameIfMatch(string name, string fullName, out string packageName) { + var lastDot = fullName.LastIndexOf('.'); + if (lastDot < 0) { + packageName = null; + return false; + } + + packageName = fullName.Remove(lastDot); + return String.Compare(fullName, lastDot + 1, name, 0, name.Length, StringComparison.Ordinal) == 0; + } + + private Response IsMissingImport(AP.IsMissingImportRequest request) { + var entry = GetPythonEntry(request.documentUri); + var analysis = entry?.Analysis; + if (analysis == null) { + return new AP.IsMissingImportResponse(); + } + + var location = new SourceLocation(request.line, request.column); + var nameExpr = GetFirstNameExpression( + analysis.GetAstFromText( + request.text, + location + ).Body + ); + + if (nameExpr != null && !IsImplicitlyDefinedName(nameExpr)) { + var name = nameExpr.Name; + var hasVariables = analysis.GetVariables(name, location).Any(IsDefinition); + var hasValues = analysis.GetValues(name, location).Any(); + + // if we have type information or an assignment to the variable we won't offer + // an import smart tag. + if (!hasValues && !hasVariables) { + return new AP.IsMissingImportResponse() { + isMissing = true + }; + } + } + + return new AP.IsMissingImportResponse(); + } + + private Response AddImportRequest(AP.AddImportRequest request) { + var projectFile = GetPythonEntry(request.documentUri); + if (projectFile == null) { + return IncorrectFileType(); + } + + string name = request.name; + string fromModule = request.fromModule; + + int version; + var curAst = projectFile.GetVerbatimAst(Project.LanguageVersion, out version); + if (curAst == null) { + return new AP.AddImportResponse(); + } + + var suiteBody = curAst.Body as SuiteStatement; + int start = 0; + if (suiteBody != null) { + foreach (var statement in suiteBody.Statements) { + if (IsDocString(statement as ExpressionStatement)) { + // doc string, import comes after this... + start = statement.EndIndex; + continue; + } + + FromImportStatement fromImport; + + if (statement is ImportStatement) { + if (fromModule == "__future__") { + // we need to insert before normal imports + break; + } + + // we insert after this + start = statement.EndIndex; + } else if ((fromImport = (statement as FromImportStatement)) != null) { + // we might update this, we might insert after + if (fromModule != "__future__" && fromImport.Root.MakeString() == fromModule) { + // update the existing from ... import statement to include the new name. + return new AP.AddImportResponse() { + changes = new[] { UpdateFromImport(curAst, fromImport, name) }, + version = version + }; + } + + start = statement.EndIndex; + } + + break; + } + } + + string newText = MakeImportCode(fromModule, name); + if (start == 0) { + // we're adding it at the beginning of the file, we need a new line + // after the import statement + newText += request.newLine; + } else { + // we're adding it after the end of a statement, we need a newline after + // the statement we're appending after. + newText = request.newLine + newText; + } + + return new AP.AddImportResponse() { + changes = new[] { + AP.ChangeInfo.FromDocumentChange(DocumentChange.Insert(newText, curAst.IndexToLocation(start))) + }, + version = version + }; + } + + public static string MakeImportCode(string fromModule, string name) { + if (string.IsNullOrEmpty(fromModule)) { + return string.Format("import {0}", name); + } else { + return string.Format("from {0} import {1}", fromModule, name); + } + } + + private static AP.ChangeInfo UpdateFromImport( + PythonAst curAst, + FromImportStatement fromImport, + string name + ) { + NameExpression[] names = new NameExpression[fromImport.Names.Count + 1]; + NameExpression[] asNames = fromImport.AsNames == null ? null : new NameExpression[fromImport.AsNames.Count + 1]; + NameExpression newName = new NameExpression(name); + for (int i = 0; i < fromImport.Names.Count; i++) { + names[i] = fromImport.Names[i]; + } + names[fromImport.Names.Count] = newName; + + if (asNames != null) { + for (int i = 0; i < fromImport.AsNames.Count; i++) { + asNames[i] = fromImport.AsNames[i]; + } + } + + var newImport = new FromImportStatement((ModuleName)fromImport.Root, names, asNames, fromImport.IsFromFuture, fromImport.ForceAbsolute, -1); + curAst.CopyAttributes(fromImport, newImport); + + var newCode = newImport.ToCodeString(curAst); + + var span = fromImport.GetSpan(curAst); + int leadingWhiteSpaceLength = (fromImport.GetLeadingWhiteSpace(curAst) ?? "").Length; + return AP.ChangeInfo.FromDocumentChange(DocumentChange.Replace( + new SourceSpan( + span.Start.AddColumns(-leadingWhiteSpaceLength), + span.End + ), newCode + )); + } + + private static bool IsDocString(ExpressionStatement exprStmt) { + ConstantExpression constExpr; + return exprStmt != null && + (constExpr = exprStmt.Expression as ConstantExpression) != null && + (constExpr.Value is string || constExpr.Value is AsciiString); + } + + private IPythonProjectEntry GetPythonEntry(Uri documentUri) { + if (documentUri == null) { + return null; + } + return _server.GetEntry(documentUri) as IPythonProjectEntry; + } + + private VersionedAst GetPythonBuffer(Uri documentUri) { + var entry = GetPythonEntry(documentUri); + if (entry == null) { + return default(VersionedAst); + } + + var parse = entry.GetCurrentParse(); + var ast = parse?.Tree; + var cookie = parse?.Cookie; + + if (cookie is VersionCookie vc) { + int i = _server.GetPart(documentUri); + if (vc.Versions.TryGetValue(i, out var bv)) { + return new VersionedAst { Ast = bv.Ast, Version = bv.Version }; + } + } + return new VersionedAst { Ast = ast, Version = 0 }; + } + + private struct VersionedAst { + public PythonAst Ast; + public int Version; + } + + private Response GetOutliningRegions(AP.OutliningRegionsRequest request) { + var bufferVersion = GetPythonBuffer(request.documentUri); + if (bufferVersion.Ast == null) { + return IncorrectBufferId(request.documentUri); + } + + var walker = new OutliningWalker(bufferVersion.Ast); + bufferVersion.Ast.Walk(walker); + + return new AP.OutliningRegionsResponse() { + tags = walker.GetTags().Select(t => new AP.OutliningTag { + startLine = t.Span.Start.Line, + startCol = t.Span.Start.Column, + endLine = t.Span.End.Line, + endCol = t.Span.End.Column + }).ToArray(), + version = bufferVersion.Version + }; + } + + private async Task GetNavigationsAsync(AP.NavigationRequest request) { + var symbols = await _server.HierarchicalDocumentSymbol(new LS.DocumentSymbolParams { + textDocument = new LS.TextDocumentIdentifier { uri = request.documentUri } + }, CancellationToken.None); + + var navs = symbols.Select(ToNavigation).ToArray(); + var bufferVersion = GetPythonBuffer(request.documentUri); + return new AP.NavigationResponse() { + version = bufferVersion.Version, + navigations = navs + }; + } + + private AP.Navigation ToNavigation(LS.DocumentSymbol symbol) => + new AP.Navigation { + name = symbol.name, + startLine = symbol.range.start.line + 1, + startColumn = symbol.range.start.character + 1, + endLine = symbol.range.end.line + 1, + endColumn = symbol.range.end.character + 1, + type = symbol._functionKind, + children = symbol.children.Select(ToNavigation).ToArray() + }; + + private Response ExpressionAtPoint(AP.ExpressionAtPointRequest request) { + var buffer = GetPythonBuffer(request.documentUri); + if (buffer.Ast == null) { + return null; + } + + var res = new AP.ExpressionAtPointResponse(); + if (!GetExpressionAtPoint(buffer.Ast, request.line, request.column, request.purpose, out SourceSpan span, out res.type)) { + return null; + } + res.startLine = span.Start.Line; + res.startColumn = span.Start.Column; + res.endLine = span.End.Line; + res.endColumn = span.End.Column; + res.bufferVersion = buffer.Version; + return res; + } + + private bool GetExpressionAtPoint(PythonAst ast, int line, int column, AP.ExpressionAtPointPurpose purpose, out SourceSpan span, out string type) { + span = default(SourceSpan); + type = null; + + if (ast == null) { + return false; + } + + GetExpressionOptions options; + switch (purpose) { + case AP.ExpressionAtPointPurpose.Evaluate: + options = GetExpressionOptions.Evaluate; + break; + case AP.ExpressionAtPointPurpose.EvaluateMembers: + options = GetExpressionOptions.EvaluateMembers; + break; + case AP.ExpressionAtPointPurpose.Hover: + options = GetExpressionOptions.Hover; + break; + case AP.ExpressionAtPointPurpose.FindDefinition: + options = GetExpressionOptions.FindDefinition; + break; + case AP.ExpressionAtPointPurpose.Rename: + options = GetExpressionOptions.Rename; + break; + default: + options = new GetExpressionOptions(); + break; + } + + var exprFinder = new ExpressionFinder(ast, options); + var expr = exprFinder.GetExpression(new SourceLocation(line, column)); + if (expr == null) { + return false; + } + + span = expr.GetSpan(ast); + type = expr.NodeName; + return true; + } + + private async Task AnalyzeExpression(AP.AnalyzeExpressionRequest request) { + var entry = GetPythonEntry(request.documentUri); + if (entry == null) { + return IncorrectFileType(); + } + + var references = await _server.FindReferences(new LS.ReferencesParams { + textDocument = request.documentUri, + position = new SourceLocation(request.line, request.column), + context = new LS.ReferenceContext { + includeDeclaration = true, + _includeValues = true + } + }, CancellationToken.None); + + var privatePrefix = entry.Analysis.GetPrivatePrefix(new SourceLocation(request.line, request.column)); + + return new AP.AnalyzeExpressionResponse { + variables = references.Select(MakeReference).ToArray(), + privatePrefix = privatePrefix + }; + } + + private AP.AnalysisReference MakeReference(LS.Reference r) { + var range = (SourceSpan)r.range; + + return new AP.AnalysisReference { + documentUri = r.uri, + file = (_server.GetEntry(r.uri, throwIfMissing: false)?.FilePath) ?? r.uri?.LocalPath, + startLine = range.Start.Line, + startColumn = range.Start.Column, + endLine = range.End.Line, + endColumn = range.End.Column, + kind = GetVariableType(r._kind), + version = r._version + }; + } + + private static string GetVariableType(VariableType type) { + switch (type) { + case VariableType.Definition: return "definition"; + case VariableType.Reference: return "reference"; + case VariableType.Value: return "value"; + } + return null; + } + + private static string GetVariableType(LS.ReferenceKind? type) { + if (!type.HasValue) { + return null; + } + switch (type.Value) { + case LS.ReferenceKind.Definition: return "definition"; + case LS.ReferenceKind.Reference: return "reference"; + case LS.ReferenceKind.Value: return "value"; + } + return null; + } + + private async Task GetQuickInfo(AP.QuickInfoRequest request) { + LS.Hover hover = await _server.Hover(new LS.TextDocumentPositionParams { + textDocument = request.documentUri, + position = new SourceLocation(request.line, request.column), + _expr = request.expr, + }, CancellationToken.None); + + return new AP.QuickInfoResponse { + text = hover.contents.value + }; + } + + private async Task GetSignatures(AP.SignaturesRequest request) { + var sigs = await _server.SignatureHelp(new LS.TextDocumentPositionParams { + textDocument = request.documentUri, + position = new SourceLocation(request.line, request.column), + _expr = request.text + }, CancellationToken.None); + + return new AP.SignaturesResponse { + sigs = sigs?.signatures?.Select( + s => new AP.Signature { + name = s.label, + doc = s.documentation?.value, + parameters = s.parameters.MaybeEnumerate().Select( + p => new AP.Parameter { + name = p.label, + defaultValue = p._defaultValue, + optional = p._isOptional ?? false, + doc = p.documentation?.value, + type = p._type + } + ).ToArray() + } + ).ToArray() + }; + } + + private async Task GetModules(Request request) { + var getModules = (AP.GetModulesRequest)request; + var prefix = getModules.package == null ? null : (string.Join(".", getModules.package)); + + var modules = await _server.Completion(new LS.CompletionParams { + textDocument = getModules.documentUri, + _expr = prefix, + context = new LS.CompletionContext { + triggerKind = LS.CompletionTriggerKind.Invoked, + _filterKind = LS.CompletionItemKind.Module, + //_includeAllModules = getModules.package == null + } + }, CancellationToken.None); + + return new AP.CompletionsResponse { + completions = await ToCompletions(modules.items, GetMemberOptions.None) + }; + } + + private async Task GetCompletions(Request request) { + var req = (AP.CompletionsRequest)request; + + var members = await _server.Completion(new LS.CompletionParams { + position = new Position { line = req.line - 1, character = req.column - 1 }, + textDocument = req.documentUri, + context = new LS.CompletionContext { + _intersection = req.options.HasFlag(GetMemberOptions.IntersectMultipleResults), + //_statementKeywords = req.options.HasFlag(GetMemberOptions.IncludeStatementKeywords), + //_expressionKeywords = req.options.HasFlag(GetMemberOptions.IncludeExpressionKeywords), + //_includeArgumentNames = true + }, + _expr = req.text + }, CancellationToken.None); + + return new AP.CompletionsResponse() { + completions = await ToCompletions(members.items, req.options) + }; + } + + private async Task GetAllMembers(Request request) { + var req = (AP.GetAllMembersRequest)request; + + var members = await _server.WorkspaceSymbols(new LS.WorkspaceSymbolParams { + query = req.prefix + }, CancellationToken.None).ConfigureAwait(false); + + return new AP.CompletionsResponse() { + completions = await ToCompletions(members) + }; + } + + private async Task ToCompletions(IEnumerable symbols) { + if (symbols == null) { + return null; + } + + var res = new List(); + foreach (var s in symbols) { + var m = new AP.Completion { + name = s.name, + memberType = ToMemberType(s._kind, s.kind) + }; + + if (s.location.uri != null) { + m.detailedValues = new[] { + new AP.CompletionValue { + locations = new [] { + new AP.AnalysisReference { + file = s.location.uri.AbsolutePath, + documentUri = s.location.uri, + startLine = s.location.range.start.line + 1, + startColumn = s.location.range.start.character + 1, + endLine = s.location.range.end.line + 1, + endColumn = s.location.range.end.character + 1, + kind = "definition", + } + } + } + }; + } + + res.Add(m); + } + + return res.ToArray(); + } + + + private async Task ToCompletions(IEnumerable completions, GetMemberOptions options) { + if (completions == null) { + return null; + } + + var res = new List(); + foreach (var c in completions) { + var m = new AP.Completion { + name = c.label, + completion = (c.label == c.insertText) ? null : c.insertText, + doc = c.documentation?.value, + memberType = ToMemberType(c._kind, c.kind) + }; + + if (options.HasFlag(GetMemberOptions.DetailedInformation)) { + var c2 = await _server.CompletionItemResolve(c, CancellationToken.None); + var vars = new List(); + foreach (var v in c2._values.MaybeEnumerate()) { + vars.Add(new AP.CompletionValue { + description = new[] { new AP.DescriptionComponent { kind = null, text = v.description } }, + doc = v.documentation, + locations = v.references?.Where(r => r.uri.IsFile).Select(r => new AP.AnalysisReference { + file = r.uri.AbsolutePath, + documentUri = r.uri, + startLine = r.range.start.line + 1, + startColumn = r.range.start.character + 1, + endLine = r.range.end.line + 1, + endColumn = r.range.end.character + 1, + kind = GetVariableType(r._kind) + }).ToArray() + }); + } + } + + res.Add(m); + } + + return res.ToArray(); + } + + private PythonMemberType ToMemberType(string originalKind, LS.CompletionItemKind kind) { + PythonMemberType res; + if (!string.IsNullOrEmpty(originalKind) && Enum.TryParse(originalKind, true, out res)) { + return res; + } + + switch (kind) { + case LS.CompletionItemKind.None: return PythonMemberType.Unknown; + case LS.CompletionItemKind.Text: return PythonMemberType.Constant; + case LS.CompletionItemKind.Method: return PythonMemberType.Method; + case LS.CompletionItemKind.Function: return PythonMemberType.Function; + case LS.CompletionItemKind.Constructor: return PythonMemberType.Function; + case LS.CompletionItemKind.Field: return PythonMemberType.Field; + case LS.CompletionItemKind.Variable: return PythonMemberType.Instance; + case LS.CompletionItemKind.Class: return PythonMemberType.Class; + case LS.CompletionItemKind.Interface: return PythonMemberType.Class; + case LS.CompletionItemKind.Module: return PythonMemberType.Module; + case LS.CompletionItemKind.Property: return PythonMemberType.Property; + case LS.CompletionItemKind.Unit: return PythonMemberType.Unknown; + case LS.CompletionItemKind.Value: return PythonMemberType.Instance; + case LS.CompletionItemKind.Enum: return PythonMemberType.Enum; + case LS.CompletionItemKind.Keyword: return PythonMemberType.Keyword; + case LS.CompletionItemKind.Snippet: return PythonMemberType.CodeSnippet; + case LS.CompletionItemKind.Color: return PythonMemberType.Instance; + case LS.CompletionItemKind.File: return PythonMemberType.Module; + case LS.CompletionItemKind.Reference: return PythonMemberType.Unknown; + case LS.CompletionItemKind.Folder: return PythonMemberType.Module; + case LS.CompletionItemKind.EnumMember: return PythonMemberType.EnumInstance; + case LS.CompletionItemKind.Constant: return PythonMemberType.Constant; + case LS.CompletionItemKind.Struct: return PythonMemberType.Class; + case LS.CompletionItemKind.Event: return PythonMemberType.Delegate; + case LS.CompletionItemKind.Operator: return PythonMemberType.Unknown; + case LS.CompletionItemKind.TypeParameter: return PythonMemberType.Class; + default: return PythonMemberType.Unknown; + } + } + + private PythonMemberType ToMemberType(string originalKind, LS.SymbolKind kind) { + PythonMemberType res; + if (!string.IsNullOrEmpty(originalKind) && Enum.TryParse(originalKind, true, out res)) { + return res; + } + + switch (kind) { + case LS.SymbolKind.None: return PythonMemberType.Unknown; + case LS.SymbolKind.File: return PythonMemberType.Module; + case LS.SymbolKind.Module: return PythonMemberType.Module; + case LS.SymbolKind.Namespace: return PythonMemberType.Namespace; + case LS.SymbolKind.Package: return PythonMemberType.Module; + case LS.SymbolKind.Class: return PythonMemberType.Class; + case LS.SymbolKind.Method: return PythonMemberType.Method; + case LS.SymbolKind.Property: return PythonMemberType.Property; + case LS.SymbolKind.Field: return PythonMemberType.Field; + case LS.SymbolKind.Constructor: return PythonMemberType.Method; + case LS.SymbolKind.Enum: return PythonMemberType.Enum; + case LS.SymbolKind.Interface: return PythonMemberType.Class; + case LS.SymbolKind.Function: return PythonMemberType.Function; + case LS.SymbolKind.Variable: return PythonMemberType.Field; + case LS.SymbolKind.Constant: return PythonMemberType.Constant; + case LS.SymbolKind.String: return PythonMemberType.Constant; + case LS.SymbolKind.Number: return PythonMemberType.Constant; + case LS.SymbolKind.Boolean: return PythonMemberType.Constant; + case LS.SymbolKind.Array: return PythonMemberType.Instance; + case LS.SymbolKind.Object: return PythonMemberType.Instance; + case LS.SymbolKind.Key: return PythonMemberType.Unknown; + case LS.SymbolKind.Null: return PythonMemberType.Unknown; + case LS.SymbolKind.EnumMember: return PythonMemberType.EnumInstance; + case LS.SymbolKind.Struct: return PythonMemberType.Class; + case LS.SymbolKind.Event: return PythonMemberType.Event; + case LS.SymbolKind.Operator: return PythonMemberType.Method; + case LS.SymbolKind.TypeParameter: return PythonMemberType.NamedArgument; + default: return PythonMemberType.Unknown; + } + } + + private async Task AnalyzeFileAsync(AP.AddFileRequest request, Func done) { + var uri = request.uri ?? ProjectEntry.MakeDocumentUri(request.path); + var entry = await AddNewFile(uri, request.path, request.addingFromDir); + + await done(new AP.AddFileResponse { documentUri = uri }); + } + + private async Task AnalyzeFileAsync(AP.AddBulkFileRequest request, Func done) { + var entries = new IProjectEntry[request.path.Length]; + var response = new AP.AddBulkFileResponse { + documentUri = new Uri[request.path.Length] + }; + + for (int i = 0; i < request.path.Length; ++i) { + if (!string.IsNullOrEmpty(request.path[i])) { + var documentUri = ProjectEntry.MakeDocumentUri(request.path[i]); + entries[i] = await AddNewFile(documentUri, request.path[i], request.addingFromDir); + response.documentUri[i] = documentUri; + } + } + + await done(response); + } + + private async Task AddNewFile(Uri documentUri, string path, string addingFromDir) { + if (documentUri.IsFile && Path.GetExtension(documentUri.LocalPath).Equals(".xaml", StringComparison.OrdinalIgnoreCase) && Project.Interpreter is IDotNetPythonInterpreter interpreter) { + return interpreter.AddXamlEntry(path, documentUri); + } + + return await _server.LoadFileAsync(documentUri).ConfigureAwait(false); + } + + private async Task UnloadFile(AP.UnloadFileRequest command) { + await _server.UnloadFileAsync(command.documentUri); + return new Response(); + } + + abstract class CodeInfo { + public readonly int Version; + + public CodeInfo(int version) { + Version = version; + } + + public abstract Parser CreateParser(PythonLanguageVersion version, ParserOptions options); + + public abstract TextReader GetReader(); + } + + class StreamCodeInfo : CodeInfo { + private readonly Stream _stream; + private readonly bool _isStubFile; + + public StreamCodeInfo(int version, Stream stream, bool isStubFile) : base(version) { + _stream = stream; + _isStubFile = isStubFile; + } + + public override Parser CreateParser(PythonLanguageVersion version, ParserOptions options) { + if (_isStubFile) { + options = options?.Clone() ?? new ParserOptions(); + options.StubFile = true; + } + return Parser.CreateParser(_stream, version, options); + } + + public override TextReader GetReader() { + return new StreamReader(_stream); + } + } + + class TextCodeInfo : CodeInfo { + private readonly TextReader _text; + private readonly bool _isStubFile; + + public TextCodeInfo(int version, TextReader text, bool isStubFile) : base(version) { + _text = text; + _isStubFile = isStubFile; + } + + public override Parser CreateParser(PythonLanguageVersion version, ParserOptions options) { + if (_isStubFile) { + options = options?.Clone() ?? new ParserOptions(); + options.StubFile = true; + } + return Parser.CreateParser(_text, version, options); + } + + public override TextReader GetReader() { + return _text; + } + } + + private async Task UpdateContent(AP.FileUpdateRequest request) { + int version = -1; + foreach (var fileChange in request.updates) { + var changes = new List(); + if (fileChange.kind == AP.FileUpdateKind.reset) { + changes.Add(new LS.TextDocumentContentChangedEvent { + text = fileChange.content + }); + version = fileChange.version; + } else if (fileChange.kind == AP.FileUpdateKind.changes) { + changes.AddRange(fileChange.changes.Select(c => new LS.TextDocumentContentChangedEvent { + range = new SourceSpan( + new SourceLocation(c.startLine, c.startColumn), + new SourceLocation(c.endLine, c.endColumn) + ), + text = c.newText + })); + version = fileChange.version; + } else { + continue; + } + + _server.DidChangeTextDocument(new LS.DidChangeTextDocumentParams { + textDocument = new LS.VersionedTextDocumentIdentifier { + uri = request.documentUri, + version = version, + _fromVersion = Math.Max(version - 1, 0) + }, + contentChanges = changes.ToArray() + }, CancellationToken.None).DoNotWait(); + } + +#if DEBUG + var entry = _server.GetEntry(request.documentUri); + int part = _server.GetPart(request.documentUri); + return new AP.FileUpdateResponse { + version = version, + newCode = entry.Document?.ReadDocument(part, out _)?.ReadToEnd() + }; +#else + return new AP.FileUpdateResponse { + version = version + }; +#endif + } + + public Task ProcessMessages() { + return _connection.ProcessMessages(); + } + + private Response SetAnalysisOptions(AP.SetAnalysisOptionsRequest request) { + Options = request.options ?? new AP.AnalysisOptions(); + + Project.Limits = AnalysisLimitsConverter.FromDictionary(Options.analysisLimits); + _server.ParseQueue.InconsistentIndentation = DiagnosticsErrorSink.GetSeverity(Options.indentationInconsistencySeverity); + _server.ParseQueue.TaskCommentMap = Options.commentTokens; + _server.Analyzer.SetTypeStubPaths(Options.typeStubPaths ?? Enumerable.Empty()); + + return new Response(); + } + + + public AP.AnalysisOptions Options { get; set; } + + private void AnalysisQueue_Complete(object sender, EventArgs e) { + _connection?.SendEventAsync(new AP.AnalysisCompleteEvent()).DoNotWait(); + } + + private void OnModulesChanged(object sender, EventArgs args) { + _server.DidChangeConfiguration(new LS.DidChangeConfigurationParams(), CancellationToken.None).DoNotWait(); + } + + private void OnFileChanged(AP.FileChangedEvent e) { + _server.DidChangeWatchedFiles(new LS.DidChangeWatchedFilesParams { + changes = e.changes.MaybeEnumerate().Select(c => new LS.FileEvent { uri = c.documentUri, type = c.kind }).ToArray() + }, CancellationToken.None).DoNotWait(); + } + + private void OnAnalysisComplete(object sender, LS.AnalysisCompleteEventArgs e) { + _connection.SendEventAsync( + new AP.FileAnalysisCompleteEvent { + documentUri = e.uri, + version = e.version + } + ).DoNotWait(); + } + + private static NameExpression GetFirstNameExpression(Statement stmt) { + return GetFirstNameExpression(Statement.GetExpression(stmt)); + } + + private static NameExpression GetFirstNameExpression(Expression expr) { + NameExpression nameExpr; + CallExpression callExpr; + MemberExpression membExpr; + + if ((nameExpr = expr as NameExpression) != null) { + return nameExpr; + } + if ((callExpr = expr as CallExpression) != null) { + return GetFirstNameExpression(callExpr.Target); + } + if ((membExpr = expr as MemberExpression) != null) { + return GetFirstNameExpression(membExpr.Target); + } + + return null; + } + + private static bool IsDefinition(IAnalysisVariable variable) { + return variable.Type == VariableType.Definition; + } + + private static bool IsImplicitlyDefinedName(NameExpression nameExpr) { + return nameExpr.Name == "__all__" || + nameExpr.Name == "__file__" || + nameExpr.Name == "__doc__" || + nameExpr.Name == "__name__"; + } + + internal IPythonInterpreterFactory InterpreterFactory => Project?.InterpreterFactory; + + internal IPythonInterpreter Interpreter => Project?.Interpreter; + + // Returns the current analyzer or throws InvalidOperationException. + // This should be used in request handlers that should fail when + // analysis is impossible. Callers that explicitly check for null before + // use should use _pyAnalyzer directly. + private PythonAnalyzer Analyzer { + get { + if (Project == null) { + throw new InvalidOperationException("Unable to analyze code"); + } + + return Project; + } + } + + /// + /// Returns the current analyzer or null if unable to analyze code. + /// + /// + /// This is for public consumption only and should not be used within + /// . + /// + public PythonAnalyzer Project => _server.Analyzer; + + private void OnPublishDiagnostics(object sender, LS.PublishDiagnosticsEventArgs e) { + _connection.SendEventAsync( + new AP.DiagnosticsEvent { + documentUri = e.uri, + version = e._version ?? -1, + diagnostics = e.diagnostics?.ToArray() + } + ).DoNotWait(); + } + + private void OnParseComplete(object sender, LS.ParseCompleteEventArgs e) { + _connection.SendEventAsync( + new AP.FileParsedEvent { + documentUri = e.uri, + version = e.version + } + ).DoNotWait(); + } + + + #region IDisposable Members + + public void Dispose() { + if (_isDisposed) { + return; + } + + _isDisposed = true; + _server.AnalysisQueue.AnalysisComplete -= AnalysisQueue_Complete; + if (Project != null) { + Project.Interpreter.ModuleNamesChanged -= OnModulesChanged; + Project.Dispose(); + } + + lock (_extensions) { + foreach (var extension in _extensions.Values) { + (extension as IDisposable)?.Dispose(); + } + } + + _server.Dispose(); + + _connection.Dispose(); + } + + #endregion + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutliningWalker.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutliningWalker.cs new file mode 100644 index 000000000..b9da4121d --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutliningWalker.cs @@ -0,0 +1,196 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.PythonTools.Parsing; +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Intellisense { + public class OutliningWalker : PythonWalker { + private readonly PythonAst _ast; + private readonly List _tagSpans; + + public OutliningWalker(PythonAst ast) { + _ast = ast; + _tagSpans = new List(); + } + + // Compound Statements: if, while, for, try, with, func, class, decorated + public override bool Walk(IfStatement node) { + if (node.ElseStatement != null) { + AddTagIfNecessary(node.ElseIndex, node.ElseStatement.EndIndex); + } + + return base.Walk(node); + } + + public override bool Walk(IfStatementTest node) { + AddTagIfNecessary(node.HeaderIndex + 1, node.Body?.EndIndex); + // Only walk body, not the condition. + node.Body?.Walk(this); + return false; + } + + public override bool Walk(WhileStatement node) { + // Walk while statements manually so we don't traverse the test. + // This prevents the test from being collapsed ever. + if (node.Body != null) { + AddTagIfNecessary( + _ast.GetLineEndFromPosition(node.StartIndex), + node.Body.EndIndex + ); + node.Body.Walk(this); + } + if (node.ElseStatement != null) { + AddTagIfNecessary(node.ElseIndex, node.ElseStatement.EndIndex); + node.ElseStatement.Walk(this); + } + return false; + } + + public override bool Walk(ForStatement node) { + // Walk for statements manually so we don't traverse the list. + // This prevents the list and/or left from being collapsed ever. + node.List?.Walk(this); + + if (node.Body != null) { + AddTagIfNecessary(node.HeaderIndex + 1, node.Body.EndIndex); + node.Body.Walk(this); + } + if (node.Else != null) { + AddTagIfNecessary(node.ElseIndex, node.Else.EndIndex); + node.Else.Walk(this); + } + return false; + } + + public override bool Walk(TryStatement node) { + AddTagIfNecessary(node.HeaderIndex, node.Body?.EndIndex); + if (node.Handlers != null) { + foreach (var h in node.Handlers) { + AddTagIfNecessary(h.HeaderIndex, h.EndIndex); + } + } + AddTagIfNecessary(node.FinallyIndex, node.Finally?.EndIndex); + AddTagIfNecessary(node.ElseIndex, node.Else?.EndIndex); + + return base.Walk(node); + } + + public override bool Walk(WithStatement node) { + AddTagIfNecessary(node.HeaderIndex + 1, node.Body?.EndIndex); + return base.Walk(node); + } + + public override bool Walk(FunctionDefinition node) { + // Walk manually so collapsing is not enabled for params. + if (node.Parameters != null) { + AddTagIfNecessary(node.Parameters.FirstOrDefault()?.StartIndex, node.Parameters.LastOrDefault()?.EndIndex); + } + if (node.Body != null) { + AddTagIfNecessary(node.HeaderIndex + 1, node.EndIndex); + node.Body.Walk(this); + } + + return false; + } + + public override bool Walk(ClassDefinition node) { + AddTagIfNecessary(node.HeaderIndex + 1, node.EndIndex); + + return base.Walk(node); + } + + // Not-Compound Statements + public override bool Walk(CallExpression node) { + AddTagIfNecessary(node.Args?.FirstOrDefault()?.StartIndex, node.Args?.LastOrDefault()?.EndIndex); + return base.Walk(node); + } + + public override bool Walk(FromImportStatement node) { + if (node.Names != null && node.Names.Any()) { + int lastName = node.Names.Count - 1; + int? nameEnd = node.Names[lastName]?.EndIndex; + if (node.AsNames != null && node.AsNames.Count >= lastName && node.AsNames[lastName] != null) { + nameEnd = node.AsNames[lastName].EndIndex; + } + AddTagIfNecessary(node.Names[0].StartIndex, nameEnd); + } + return base.Walk(node); + } + + public override bool Walk(ListExpression node) { + AddTagIfNecessary(node.Items?.FirstOrDefault()?.StartIndex, node.Items?.LastOrDefault()?.EndIndex); + return base.Walk(node); + } + + public override bool Walk(TupleExpression node) { + AddTagIfNecessary(node.Items?.FirstOrDefault()?.StartIndex, node.Items?.LastOrDefault()?.EndIndex); + return base.Walk(node); + } + + public override bool Walk(DictionaryExpression node) { + AddTagIfNecessary(node.Items?.FirstOrDefault()?.StartIndex, node.Items?.LastOrDefault()?.EndIndex); + return base.Walk(node); + } + + public override bool Walk(SetExpression node) { + AddTagIfNecessary(node.Items?.FirstOrDefault()?.StartIndex, node.Items?.LastOrDefault()?.EndIndex); + return base.Walk(node); + } + + public override bool Walk(ParenthesisExpression node) { + AddTagIfNecessary(node.Expression?.StartIndex, node.Expression?.EndIndex); + return base.Walk(node); + } + + public override bool Walk(ConstantExpression node) { + AddTagIfNecessary(_ast.GetLineEndFromPosition(node.StartIndex), node.EndIndex); + return base.Walk(node); + } + + private void AddTagIfNecessary(int? startIndex, int? endIndex, int minLinesToCollapse = 3) { + if (!startIndex.HasValue || !endIndex.HasValue) { + return; + } + + if (startIndex < 0 || endIndex < 0) { + return; + } + + var start = _ast.IndexToLocation(startIndex.Value); + var end = _ast.IndexToLocation(endIndex.Value); + var lines = end.Line - start.Line + 1; + + // Collapse if more than 3 lines. + if (lines < minLinesToCollapse) { + return; + } + + var tagSpan = new TaggedSpan(new SourceSpan(start, end), null); + _tagSpans.Add(tagSpan); + } + + public IEnumerable GetTags() { + return _tagSpans + .GroupBy(s => s.Span.Start.Line) + .Select(ss => ss.OrderByDescending(s => s.Span.End.Line).First()) + .ToArray(); + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/ProximityExpressionWalker.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/ProximityExpressionWalker.cs new file mode 100644 index 000000000..a646b61e1 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/ProximityExpressionWalker.cs @@ -0,0 +1,144 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.PythonTools.Parsing; +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Intellisense { + /// + /// Walks the AST, and returns all expressions in the given line range that are eligible to be displayed + /// in the Autos debugger tool window. + /// + /// + /// The following expressions are considered eligible: + /// + /// standalone names: x + /// member access, so long as the target is side-effect free: x.y, 123.real, (x + 1).imag + /// indexing/slicing, so long as target and index are side-effect free: x.y[z], ('abc' + 'd')[1:len(s)-1] + /// function calls, if function is one of the several hardcoded builtin functions (e.g. abs, len, str, repr), + /// and all arguments are side-effect free: len('a' * n) + /// + /// + /// All error-free expressions are considered side-effect-free except those involving function calls, backquotes, yield + /// or yield from. Function calls are considered side-effect-free if they are eligible according to the definition above. + /// + /// + /// For member access and indexing, if an expression is eligible, all its direct nested targets become non-eligible. For example, + /// given a.b.c, neither a nor a.b are eligible. + /// + /// + /// For function calls, the immediate target becomes ineligible. For example, given a.b.c(d), a.b.c is not eligible, + /// but a.b is (and hence a is not, per the earlier rule). + /// + /// + public class ProximityExpressionWalker : PythonWalker { + private readonly PythonAst _ast; + private readonly int _startLine, _endLine; + + // Keys are expressions that have been walked and found to be eligible. + // For a given key, a value is either null, or an expression that was removed from keys when this key was + // added, because it is a subexpression of this key that we wanted to exclude. For example, given A.B.C, + // we will first walk A and add it as a key with null as value; then walk A.B, remove A, and add A.B with + // A as value; then walk A.B.C, remove A.B, and add A.B.C with A.B as value. + // The information about the excluded node is used by PostWalk(CallExpression). + private readonly Dictionary _expressions = new Dictionary(); + + public ProximityExpressionWalker(PythonAst ast, int startLine, int endLine) { + _ast = ast; + _startLine = startLine; + _endLine = endLine; + } + + public override void PostWalk(NameExpression node) { + if (IsInRange(node)) { + if(_ast.LanguageVersion.Is2x()) { + // In 2.7 True and False are constants, we made an exception to not show them Autos window. + if(node.Name == "True" || node.Name == "False") { + return; + } + } + + _expressions.Add(node, null); + } + + } + + public override void PostWalk(MemberExpression node) { + if (IsInRange(node) && IsValidTarget(node.Target)) { + _expressions.Remove(node.Target); + _expressions.Add(node, node.Target); + } + } + + public override void PostWalk(IndexExpression node) { + if (IsInRange(node) && IsValidTarget(node.Target) && IsValidTarget(node.Index)) { + _expressions.Remove(node.Target); + _expressions.Add(node, node.Target); + } + } + + public override void PostWalk(CallExpression node) { + if (IsInRange(node) && node.Target != null) { + // For call nodes, we don't want either the call nor the called function to show up, + // but if it is a method, then we do want to show the object on which it is called. + // For example, given A.B.C(42), we want A.B to show. By the time we get here, we + // already have A.B.C in the list, so we need to remove it and reinstate A.B, which + // will be stored as a value in the dictionary with A.B.C as key. + Expression oldNode; + _expressions.TryGetValue(node.Target, out oldNode); + _expressions.Remove(node.Target); + if (oldNode != null) { + _expressions.Add(oldNode, null); + } + + // Hardcode some commonly used side-effect-free builtins to show even in calls. + var name = node.Target as NameExpression; + if (name != null && DetectSideEffectsWalker.IsSideEffectFreeCall(name.Name) && node.Args.All(arg => IsValidTarget(arg.Expression))) { + _expressions.Add(node, null); + } + } + } + + private bool IsInRange(Expression node) { + var span = node.GetSpan(_ast); + int isct0 = Math.Max(span.Start.Line, _startLine); + int isct1 = Math.Min(span.End.Line, _endLine); + return isct0 <= isct1; + } + + private bool IsValidTarget(Node node) { + if (node == null || node is ConstantExpression || node is NameExpression) { + return true; + } + + var expr = node as Expression; + if (expr != null && _expressions.ContainsKey(expr)) { + return true; + } + + var walker = new DetectSideEffectsWalker(); + node.Walk(walker); + return !walker.HasSideEffects; + } + + public IEnumerable GetExpressions() { + return _expressions.Keys.Select(expr => expr.ToCodeString(_ast, CodeFormattingOptions.Traditional).Trim()).OrderBy(s => s).Distinct(); + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/TaggedSpan.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/TaggedSpan.cs new file mode 100644 index 000000000..95c4b643e --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/TaggedSpan.cs @@ -0,0 +1,27 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.PythonTools.Intellisense { + public sealed class TaggedSpan { + public SourceSpan Span { get; } + public string Tag { get; } + + public TaggedSpan(SourceSpan span, string tag) { + Span = span; + Tag = tag; + } + } +} diff --git a/src/Analysis/Engine/Impl/Interpreter/Definitions/IDotNetPythonInterpreter.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Interpreter/IDotNetPythonInterpreter.cs similarity index 75% rename from src/Analysis/Engine/Impl/Interpreter/Definitions/IDotNetPythonInterpreter.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Interpreter/IDotNetPythonInterpreter.cs index 6ccd37072..df07e38b5 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Definitions/IDotNetPythonInterpreter.cs +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Interpreter/IDotNetPythonInterpreter.cs @@ -9,18 +9,24 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. using System; +using Microsoft.PythonTools.Analysis; namespace Microsoft.PythonTools.Interpreter { public interface IDotNetPythonInterpreter { /// - /// Gets the IPythonType object for the specifed .NET type; + /// Gets the IPythonType object for the specified .NET type; /// IPythonType GetBuiltinType(Type type); + + /// + /// Adds xaml entry + /// + IProjectEntry AddXamlEntry(string filePath, Uri documentUri); } } diff --git a/src/Analysis/Engine/Impl/Interpreter/Definitions/IPythonInterpreterWithProjectReferences.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Interpreter/IPythonInterpreterWithProjectReferences.cs similarity index 100% rename from src/Analysis/Engine/Impl/Interpreter/Definitions/IPythonInterpreterWithProjectReferences.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Interpreter/IPythonInterpreterWithProjectReferences.cs diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Interpreter/ProjectAssemblyReference.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Interpreter/ProjectAssemblyReference.cs new file mode 100644 index 000000000..b2daa0f3e --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Interpreter/ProjectAssemblyReference.cs @@ -0,0 +1,62 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Reflection; + +namespace Microsoft.PythonTools.Interpreter { + public sealed class ProjectAssemblyReference : ProjectReference, IEquatable { + private readonly AssemblyName _asmName; + + public ProjectAssemblyReference(AssemblyName assemblyName, string filename) + : base(filename, ProjectReferenceKind.Assembly) { + _asmName = assemblyName; + } + + public AssemblyName AssemblyName => _asmName; + + public override int GetHashCode() { + return base.GetHashCode() ^ _asmName.GetHashCode(); + } + + public override bool Equals(object obj) { + ProjectAssemblyReference asmRef = obj as ProjectAssemblyReference; + if (asmRef != null) { + return Equals(asmRef); + } + return false; + } + + public override bool Equals(ProjectReference other) { + ProjectAssemblyReference asmRef = other as ProjectAssemblyReference; + if (asmRef != null) { + return Equals(asmRef); + } + return false; + } + + #region IEquatable Members + + public bool Equals(ProjectAssemblyReference other) { + if (base.Equals(other)) { + return other._asmName == this._asmName; + } + return false; + } + + #endregion + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/IronPythonScraper.py b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/IronPythonScraper.py new file mode 100644 index 000000000..f0be30975 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/IronPythonScraper.py @@ -0,0 +1,177 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABILITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +import clr +import PythonScraper + +from System import ParamArrayAttribute, DBNull, Void +from System.Reflection import Missing + +clr.AddReference('IronPython') +from IronPython.Runtime.Operations import PythonOps +from IronPython.Runtime import SiteLocalStorage +from IronPython.Runtime.Operations import InstanceOps + +clr.AddReference('Microsoft.Dynamic') +clr.AddReference('Microsoft.Scripting') +from Microsoft.Scripting import ParamDictionaryAttribute +from Microsoft.Scripting.Generation import CompilerHelpers + +class NonPythonTypeException(Exception): + pass + +def safe_dir(obj): + try: + return frozenset(obj.__dict__) | frozenset(clr.Dir(obj)) + except: + # Some types crash when we access __dict__ and/or dir() + pass + try: + return frozenset(clr.Dir(obj)) + except: + pass + try: + return frozenset(obj.__dict__) + except: + pass + return frozenset() + +def type_to_typelist(type_obj): + if type_obj.IsArray: + return PythonScraper.type_to_typelist(tuple) + elif type_obj == Void: + return PythonScraper.type_to_typelist(type(None)) + elif not PythonOps.IsPythonType(clr.GetPythonType(type_obj)): + raise NonPythonTypeException + + return PythonScraper.type_to_typelist(clr.GetPythonType(type_obj)) + + +def get_default_value(param): + if param.DefaultValue is not DBNull.Value and param.DefaultValue is not Missing.Value: + return repr(param.DefaultValue) + elif param.IsOptional: + missing = CompilerHelpers.GetMissingValue(param.ParameterType) + if missing != Missing.Value: + return repr(missing) + return "" + + +def get_arg_format(param): + if param.IsDefined(ParamArrayAttribute, False): + return "*" + elif param.IsDefined(ParamDictionaryAttribute, False): + return "**" + + +def sanitize_name(param): + for v in param.Name: + if not ((v >= '0' and v <= '9') or (v >= 'A' and v <= 'Z') or (v >= 'a' and v <= 'z') or v == '_'): + break + else: + return param.Name + + letters = [] + for v in param.Name: + if ((v >= '0' and v <= '9') or (v >= 'A' and v <= 'Z') or (v >= 'a' and v <= 'z') or v == '_'): + letters.append(v) + return ''.join(letters) + +def get_parameter_info(param): + parameter_table = { + 'type': type_to_typelist(param.ParameterType), + 'name': sanitize_name(param), + } + + default_value = get_default_value(param) + if default_value is not None: + parameter_table['default_value'] = default_value + + arg_format = get_arg_format(param) + if arg_format is not None: + parameter_table['arg_format'] = arg_format + + return parameter_table + +def get_return_type(target): + if hasattr(target, 'ReturnType'): + return target.ReturnType + # constructor + return target.DeclaringType + +def get_function_overloads(targets): + res = [] + for target in targets: + try: + args = list(target.GetParameters()) + except: + # likely a failure to load an assembly... + continue + if args and args[0].ParameterType.FullName == 'IronPython.Runtime.CodeContext': + del args[0] + if args and args[0].ParameterType.IsSubclassOf(SiteLocalStorage): + del args[0] + + try: + arg_info = [get_parameter_info(arg) for arg in args] + if not target.IsStatic and not target.IsConstructor: + arg_info.insert(0, {'type' : type_to_typelist(target.DeclaringType), 'name': 'self'}) + + res.append({ + 'args' : tuple(arg_info), + 'ret_type' : type_to_typelist(get_return_type(target)) + }) + except NonPythonTypeException: + pass + + + return res + +def get_overloads(func, is_method = False): + if type(func) == type(list.append): + func = PythonOps.GetBuiltinMethodDescriptorTemplate(func) + + targets = func.Targets + + res = get_function_overloads(targets) + + return res + + +def get_descriptor_type(descriptor): + if hasattr(descriptor, 'PropertyType'): + return clr.GetPythonType(descriptor.PropertyType) + elif hasattr(descriptor, 'FieldType'): + return clr.GetPythonType(descriptor.FieldType) + return object + + +def get_new_overloads(type_obj, func): + if func.Targets and func.Targets[0].DeclaringType == clr.GetClrType(InstanceOps): + print('has instance ops ' + str(type_obj)) + clrType = clr.GetClrType(type_obj) + + return get_function_overloads(clrType.GetConstructors()) + + return None + +SPECIAL_MODULES = ('wpf', 'clr') + +def should_include_module(name): + return name not in SPECIAL_MODULES diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Microsoft.PythonTools.Analyzer.csproj b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Microsoft.PythonTools.Analyzer.csproj new file mode 100644 index 000000000..de8669058 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Microsoft.PythonTools.Analyzer.csproj @@ -0,0 +1,77 @@ + + + net472 + Microsoft.PythonTools.Analysis + Visual Studio - Python background analyzer + Microsoft.PythonTools.Analyzer + python-language-server-ptvs.nuspec + version=$(NugetVersion) + + + + + Exe + portable + $(OutputPath) + true + + + ..\..\..\PLS.ruleset + + + ..\..\..\PLS.ruleset + + + + + + + + + all + runtime; build; native; contentfiles; analyzers + + + + + $(AnalysisReference)/Microsoft.Python.Analysis.Engine.dll + + + PreserveNewest + + + + + PreserveNewest + True + + + PreserveNewest + True + + + PreserveNewest + True + + + PreserveNewest + True + + + + + + + + + + + + + + + + + + + diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/PyLibAnalyzer.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/PyLibAnalyzer.cs new file mode 100644 index 000000000..9e74fc4b7 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/PyLibAnalyzer.cs @@ -0,0 +1,244 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Threading.Tasks; +using Microsoft.PythonTools.Analysis.Infrastructure; +using Microsoft.PythonTools.Intellisense; + +namespace Microsoft.PythonTools.Analysis { + internal class PyLibAnalyzer : IDisposable { + private static void Help() { + var assembly = typeof(PyLibAnalyzer).Assembly; + string version = FileVersionInfo.GetVersionInfo(assembly.Location).FileVersion; + Console.WriteLine("Python Library Analyzer {0} ", version); + Console.WriteLine("Python analysis server."); + Console.WriteLine(); +#if DEBUG + Console.WriteLine(" /unittest - run from tests, Debug.Listeners will be replaced"); +#endif + } + + private static IEnumerable> ParseArguments(IEnumerable args) { + string currentKey = null; + + using (var e = args.GetEnumerator()) { + while (e.MoveNext()) { + if (e.Current.StartsWithOrdinal("/")) { + if (currentKey != null) { + yield return new KeyValuePair(currentKey, null); + } + currentKey = e.Current.Substring(1).Trim(); + } else { + yield return new KeyValuePair(currentKey, e.Current); + currentKey = null; + } + } + + if (currentKey != null) { + yield return new KeyValuePair(currentKey, null); + } + } + } + + /// + /// The exit code returned when database generation fails due to an + /// invalid argument. + /// + public const int InvalidArgumentExitCode = -1; + + /// + /// The exit code returned when database generation fails due to a + /// non-specific error. + /// + public const int InvalidOperationExitCode = -2; + + public static int Main(string[] args) { + PyLibAnalyzer inst; + try { + inst = MakeFromArguments(args); + } catch (ArgumentNullException ex) { + Console.Error.WriteLine("{0} is a required argument", ex.Message); + Help(); + return InvalidArgumentExitCode; + } catch (ArgumentException ex) { + Console.Error.WriteLine("'{0}' is not valid for {1}", ex.Message, ex.ParamName); + Help(); + return InvalidArgumentExitCode; + } catch (InvalidOperationException ex) { + Console.Error.WriteLine(ex.Message); + Help(); + return InvalidOperationExitCode; + } + + using (inst) { + return inst.Run().GetAwaiter().GetResult(); + } + } + + private async Task Run() { +#if DEBUG + // Running with the debugger attached will skip the + // unhandled exception handling to allow easier debugging. + if (Debugger.IsAttached) { + await RunWorker(); + } else { +#endif + try { + await RunWorker(); + } catch (Exception e) { + Console.Error.WriteLine("Error during analysis: {0}{1}", Environment.NewLine, e.ToString()); + return -10; + } +#if DEBUG + } +#endif + + return 0; + } + + private async Task RunWorker() { + var analyzer = new OutOfProcProjectAnalyzer( + Console.OpenStandardOutput(), + Console.OpenStandardInput(), + Console.Error.WriteLine + ); + + await analyzer.ProcessMessages(); + } + + public PyLibAnalyzer() { + } + + public void Dispose() { + } + + private static PyLibAnalyzer MakeFromArguments(IEnumerable args) { + var options = ParseArguments(args) + .ToDictionary(kv => kv.Key, kv => kv.Value, StringComparer.OrdinalIgnoreCase); + +#if DEBUG + if (options.ContainsKey("unittest")) { + AssertListener.Initialize(); + } +#endif + + return new PyLibAnalyzer(); + } + +#if DEBUG + class AssertListener : TraceListener { + private AssertListener() { + } + + public override string Name { + get { return "Microsoft.PythonTools.AssertListener"; } + set { } + } + + public static bool LogObjectDisposedExceptions { get; set; } = true; + + public static void Initialize() { + var listener = new AssertListener(); + if (null == Debug.Listeners[listener.Name]) { + Debug.Listeners.Add(listener); + Debug.Listeners.Remove("Default"); + + AppDomain.CurrentDomain.FirstChanceException += CurrentDomain_FirstChanceException; + } + } + + static void CurrentDomain_FirstChanceException(object sender, FirstChanceExceptionEventArgs e) { + if (e.Exception is NullReferenceException || (e.Exception is ObjectDisposedException && LogObjectDisposedExceptions)) { + // Exclude safe handle messages because they are noisy + if (!e.Exception.Message.Contains("Safe handle has been closed")) { + var log = new EventLog("Application"); + log.Source = "Application Error"; + log.WriteEntry( + "First-chance exception: " + e.Exception.ToString(), + EventLogEntryType.Warning + ); + } + } + } + + public override void Fail(string message) { + Fail(message, null); + } + + public override void Fail(string message, string detailMessage) { + Trace.WriteLine("Debug.Assert failed in analyzer"); + if (!string.IsNullOrEmpty(message)) { + Trace.WriteLine(message); + } else { + Trace.WriteLine("(No message provided)"); + } + if (!string.IsNullOrEmpty(detailMessage)) { + Trace.WriteLine(detailMessage); + } + var trace = new StackTrace(true); + bool seenDebugAssert = false; + foreach (var frame in trace.GetFrames()) { + var mi = frame.GetMethod(); + if (!seenDebugAssert) { + seenDebugAssert = (mi.DeclaringType == typeof(Debug) && + (mi.Name == "Assert" || mi.Name == "Fail")); + } else if (mi.DeclaringType == typeof(System.RuntimeMethodHandle)) { + break; + } else { + var filename = frame.GetFileName(); + Console.WriteLine(string.Format( + " at {0}.{1}({2}) in {3}:line {4}", + mi.DeclaringType.FullName, + mi.Name, + string.Join(", ", mi.GetParameters().Select(p => p.ToString())), + filename ?? "", + frame.GetFileLineNumber() + )); + if (File.Exists(filename)) { + try { + Console.WriteLine( + " " + + File.ReadLines(filename).ElementAt(frame.GetFileLineNumber() - 1).Trim() + ); + } catch { + } + } + } + } + + message = string.IsNullOrEmpty(message) ? "Debug.Assert failed" : message; + if (Debugger.IsAttached) { + Debugger.Break(); + } + + Console.WriteLine(message); + } + + public override void WriteLine(string message) { + } + + public override void Write(string message) { + } + } +#endif + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/PythonScraper.py b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/PythonScraper.py new file mode 100644 index 000000000..8758a556c --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/PythonScraper.py @@ -0,0 +1,999 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABILITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +""" +Generates information for supporting completion and analysis of Python code. + +Outputs a pickled set of dictionaries. The dictionaries are in the format: + +top-level: module_table + +module_table: + { + 'members': member_table, + 'doc': doc_string, + } + +type_table: + { + 'mro' : type_list, + 'bases' : type_list, + 'members' : member_table, + 'doc' : doc_string, + 'is_hidden': bool, + 'builtin': bool + } + +member_table: + { member_name : member_entry } + +member_name: str + +member_entry: + { + 'kind': member_kind + 'value': member_value + 'version': version_spec # optional + } + +member_kind: 'function' | 'funcref' | 'method' | 'property' | 'data' | 'type' | 'multiple' | 'typeref' | 'moduleref' +member_value: builtin_function | getset_descriptor | data | type_table | multiple_value | typeref | moduleref + +moduleref: + { 'module_name' : name } + +typeref: + ( + module name, # use '' to omit + type name, + type_list # index types; optional + ) + +funcref: + { + 'func_name' : fully-qualified function name + } + +multiple_value: + { 'members' : (member_entry, ... ) } + +builtin_function: + { + 'doc': doc string, + 'overloads': overload_table, + 'builtin' : bool, + 'static': bool, + } + +overload_table: + [overload, ...] + +overload: + { + 'args': args_info, + 'ret_type': type_list + } + +args_info: + (arg_info, ...) + +arg_info: + { + 'type': type_list, + 'name': argument name, + 'default_value': repr of default value, + 'arg_format' : ('*' | '**') + } + +getset_descriptor: + { + 'doc': doc string, + 'type': type_list + } + +data: + { 'type': type_list } + +type_list: + [type_ref, ...] +""" + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +try: + import cPickle as pickle +except ImportError: + import pickle # Py3k +import datetime +import os +import subprocess +import sys +import traceback +import types + +# The values in KNOWN_METHOD_TYPES and KNOWN_FUNCTION_TYPES are used when +# detecting the types of members. The names are ('module.name', 'type.name') +# pairs, matching the contents of a typeref. +# +# If the type's name matches an item in KNOWN_METHOD_TYPES, the object is +# treated as a method descriptor. +# +# If the type's name matches an item in KNOWN_FUNCTION_TYPES, the object is +# treated as a method if defined on a type or a function otherwise. +KNOWN_METHOD_TYPES = frozenset( + ('sip', 'methoddescriptor'), +) + +KNOWN_FUNCTION_TYPES = frozenset( + ('numpy', 'ufunc'), +) + + +# Safe member access methods because PyQt4 contains compiled types that crash +# on operations that should be completely safe, such as getattr() and dir(). +# These should be used throughout when accessing members on type objects. +def safe_getattr(obj, attr, default): + try: + return getattr(obj, attr, default) + except: + return default + +def safe_hasattr(obj, attr): + try: + return hasattr(obj, attr) + except: + return False + +def safe_isinstance(obj, types): + try: + return isinstance(obj, types) + except: + return False + +# safe_dir is imported from BuiltinScraper/IronPythonScraper +def safe_repr(obj): + try: + return repr(obj) + except: + return 'invalid object' + +if sys.version_info[0] == 2: + builtin_name = '__builtin__' +else: + builtin_name = 'builtins' + + +TYPE_NAMES = {} + +def types_to_typelist(iterable): + return [type_to_typeref(type) for type in iterable] + +def type_to_typelist(type): + return [type_to_typeref(type)] + +def typename_to_typeref(n1, n2=None): + '''If both n1 and n2 are specified, n1 is the module name and n2 is the type name. + If only n1 is specified, it is the type name. + ''' + if n2 is None: + name = ('', n1) + elif n1 == '__builtin__': + name = (builtin_name, n2) + else: + name = (n1, n2) + return memoize_type_name(name) + +def type_to_typeref(type): + type_name = safe_getattr(type, '__name__', None) + if not type_name: + print('Cannot get type name of ' + safe_repr(type)) + type = object + type_name = 'object' + if safe_hasattr(type, '__module__'): + if safe_getattr(type, '__module__', '') == '__builtin__': + name = (builtin_name, type_name) + else: + name = (type.__module__, type_name) + elif safe_isinstance(type, types.ModuleType): + name = (type_name, '') + else: + name = ('', type_name) + # memoize so when we pickle we can share refs + return memoize_type_name(name) + +def memoize_type_name(name): + key = repr(name) + if key in TYPE_NAMES: + return TYPE_NAMES[key] + TYPE_NAMES[key] = name + return name + +def maybe_to_typelist(type): + if isinstance(type, list) and len(type) > 0 and isinstance(type[0], tuple) and len(type[0]) > 1 and type[0][1]: + return type + elif isinstance(type, tuple) and len(type) > 1 and type[1]: + return [type] + else: + return type_to_typelist(type) + +def generate_overload(ret_type, *args): + '''ret_type is either a type suitable for type_to_typelist, or it is the result of + one of the *_to_typelist() or *_to_typeref() functions. + + Each arg is a tuple of ('name', type or type_ref or type_list, '' or '*' or '**', 'default value string') + The last two elements are optional, but the format is required if the default value + is specified. + ''' + res = { 'ret_type': maybe_to_typelist(ret_type) } + arglist = [] + for arg in args: + arglist.append({ 'name': arg[0], 'type': maybe_to_typelist(arg[1]) }) + if len(arg) >= 3 and arg[2]: + arglist[-1]['arg_format'] = arg[2] + if len(arg) >= 4: + arglist[-1]['default_value'] = arg[3] + res['args'] = tuple(arglist) + return res + +if sys.platform == "cli": + # provides extra type info when generating against IronPython which can be + # used w/ CPython completions + import IronPythonScraper as BuiltinScraper +else: + import BuiltinScraper + +def generate_builtin_function(function, is_method=False): + function_table = {} + + func_doc = safe_getattr(function, '__doc__', None) + if safe_isinstance(func_doc, str): + function_table['doc'] = func_doc + + function_table['overloads'] = BuiltinScraper.get_overloads(function, is_method) + + return function_table + +def generate_getset_descriptor(descriptor): + descriptor_table = {} + + desc_doc = safe_getattr(descriptor, '__doc__', None) + if safe_isinstance(desc_doc, str): + descriptor_table['doc'] = desc_doc + + desc_type = BuiltinScraper.get_descriptor_type(descriptor) + descriptor_table['type'] = type_to_typelist(desc_type) + + return descriptor_table + +NoneType = type(None) +slot_wrapper_type = type(int.__add__) +method_descriptor_type = type(str.center) +member_descriptor_type = type(property.fdel) +try: + getset_descriptor_type = type(file.closed) +except NameError: + getset_descriptor_type = type(Exception.args) # Py3k, no file +class_method_descriptor_type = type(datetime.date.__dict__['today']) +class OldStyleClass: pass +OldStyleClassType = type(OldStyleClass) + +def generate_member_table(obj, is_hidden=False, from_type=False, extra_types=None): + '''Generates a table of members of `obj`. + + `is_hidden` determines whether all the members are hidden from IntelliSense. + + `from_type` determines whether method descriptors are retained (True) or + ignored (False). + + `extra_types` is a sequence of ``(type_name, object)`` pairs to add as types + to this table. These types are always hidden. + ''' + + sentinel = object() + members = [] + for name in BuiltinScraper.safe_dir(obj): + member = safe_getattr(obj, name, sentinel) + if member is not sentinel: + members.append((name, member)) + + dependencies = {} + table = {} + if extra_types: + for name, member in extra_types: + member_kind, member_value = generate_member(member, is_hidden = True, from_type = from_type) + if member_kind == 'typeref': + actual_name = type_to_typeref(member) + if actual_name not in dependencies: + dependencies[actual_name] = member + table[name] = { 'kind': member_kind, 'value': member_value } + + for name, member in members: + member_kind, member_value = generate_member(member, is_hidden, from_type) + if member_kind == 'typeref': + actual_name = type_to_typeref(member) + if actual_name not in dependencies: + dependencies[actual_name] = member + table[name] = { 'kind': member_kind, 'value': member_value } + + if dependencies: + obj_mod, obj_name = type_to_typeref(obj) + def needs_type_info(other_mod, other_name): + if obj_mod != other_mod: + if other_mod == builtin_name: + # Never embed builtins (unless obj_mod is builtins, in + # which case the first comparison failed) + return False + + # Always embed external types + return True + + # We know obj_mod == other_mod at this point + + if not obj_name: + # Writing ourselves in the expected place + return True + elif obj_name.startswith(other_name + '.'): + # Always write references to outer types + return False + elif other_name and other_name.startswith(obj_name + '.'): + # Always write type info for inner types + return True + + # Otherwise, use a typeref + return False + + for (dep_mod, dep_name), dep_obj in dependencies.items(): + if needs_type_info(dep_mod, dep_name): + table[dep_name] = { + 'kind': 'type', + 'value': generate_type(dep_obj, is_hidden = dep_name not in table), + } + + return table + +def generate_member(obj, is_hidden=False, from_type=False): + try: + # Already handling all exceptions here, so don't worry about using the + # 'safe_*' functions. + + if isinstance(obj, (types.BuiltinFunctionType, class_method_descriptor_type)): + return 'function', generate_builtin_function(obj) + elif isinstance(obj, types.FunctionType): + # PyPy - we see plain old Python functions in addition to built-ins + return 'method' if from_type else 'function', generate_builtin_function(obj, from_type) + elif isinstance(obj, (type, OldStyleClassType)): + return 'typeref', type_to_typelist(obj) + elif isinstance(obj, (types.BuiltinMethodType, slot_wrapper_type, method_descriptor_type)): + return 'method', generate_builtin_function(obj, True) + elif isinstance(obj, (getset_descriptor_type, member_descriptor_type)): + return 'property', generate_getset_descriptor(obj) + + # Check whether we recognize the type name as one that does not respond + # correctly to isinstance checks. + type_name = type_to_typeref(type(obj)) + + if type_name in KNOWN_METHOD_TYPES: + return 'method', generate_builtin_function(obj, True) + + if type_name in KNOWN_FUNCTION_TYPES: + return 'method' if from_type else 'function', generate_builtin_function(obj, from_type) + + # Callable objects with a docstring that provides us with at least one + # overload will be treated as functions rather than data. + if safe_hasattr(obj, '__call__'): + try: + info = generate_builtin_function(obj, from_type) + if info and info['overloads']: + return 'method' if from_type else 'function', info + except: + pass + except: + # Some compiled types fail here, so catch everything and treat the + # object as data. + traceback.print_exc() + print('Treating type as data') + + # We don't have any special handling for this object type, so treat it as + # a constant. + return 'data', generate_data(obj) + + +if sys.version > '3.': + str_types = (str, bytes) +else: + str_types = (str, unicode) + + +def generate_type_new(type_obj, obj): + if safe_isinstance(obj, (types.BuiltinFunctionType, class_method_descriptor_type)): + function_info = generate_builtin_function(obj) + + new_overloads = BuiltinScraper.get_new_overloads(type_obj, obj) + if new_overloads is not None: + # replace overloads with better version if available + function_info['overloads'] = new_overloads + return 'function', function_info + + if safe_getattr(obj, '__doc__', '') == 'T.__new__(S, ...) -> a new object with type S, a subtype of T': + doc_str = safe_getattr(type_obj, '__doc__', None) + if not safe_isinstance(doc_str, str_types): + doc_str = '' + return ( + 'function', + { + 'doc': doc_str, + 'overloads' : [{'doc': doc_str, 'args': [{'arg_format': '*', 'name': 'args'}] }] + } + ) + + return generate_member(obj) + +def oldstyle_mro(type_obj, res): + type_bases = safe_getattr(type_obj, '__bases__', None) + + if not type_bases: + return res + + for base in type_bases: + if base not in res: + res.append(type_to_typeref(base)) + + for base in type_bases: + oldstyle_mro(base, res) + return res + +def generate_type(type_obj, is_hidden=False): + type_table = {} + + type_mro = safe_getattr(type_obj, '__mro__', None) + if type_mro: + type_table['mro'] = types_to_typelist(type_mro) + else: + type_table['mro'] = oldstyle_mro(type_obj, []) + + type_bases = safe_getattr(type_obj, '__bases__', None) + if type_bases: + type_table['bases'] = types_to_typelist(type_bases) + + type_doc = safe_getattr(type_obj, '__doc__', None) + if safe_isinstance(type_doc, str): + type_table['doc'] = type_doc + + if is_hidden: + type_table['is_hidden'] = True + + + type_table['members'] = member_table = generate_member_table(type_obj) + + if type_obj is object: + member_table['__new__'] = { + 'kind' : 'function', + 'value': { 'overloads': [generate_overload(object, ('cls', type))] } + } + elif '__new__' not in member_table: + member_table['__new__'] = generate_type_new(type_obj, + safe_getattr(type_obj, '__new__', object.__new__),) + + if ('__getattribute__' in member_table and + type_obj is not object and + safe_isinstance(safe_getattr(type_obj, '__getattribute__', None), slot_wrapper_type)): + # skip __getattribute__ on types other than object if it's just a slot + # wrapper. + del member_table['__getattribute__'] + + return type_table + +def generate_data(data_value): + data_table = {} + + data_type = type(data_value) + data_table['type'] = type_to_typelist(data_type) + + return data_table + +def lookup_module(module_name): + try: + module = __import__(module_name) + except: + return None + if '.' in module_name: + for name in module_name.split('.')[1:]: + module = safe_getattr(module, name, None) + if not module: + module = sys.modules[module_name] + + return module + +def generate_module(module, extra_types=None): + module_table = {} + + module_doc = safe_getattr(module, '__doc__', None) + if safe_isinstance(module_doc, str): + module_table['doc'] = module_doc + + module_table['members'] = generate_member_table(module, extra_types = extra_types) + + return module_table + + +def get_module_members(module): + """returns an iterable which gives the names of the module which should be exposed""" + module_all = safe_getattr(module, '__all__', None) + if module_all: + return frozenset(module_all) + + return BuiltinScraper.safe_dir(module) + +def generate_builtin_module(): + extra_types = {} + extra_types['object'] = type(object) + extra_types['function'] = types.FunctionType + extra_types['builtin_function'] = types.BuiltinFunctionType + extra_types['builtin_method_descriptor'] = types.BuiltinMethodType + extra_types['generator'] = types.GeneratorType + extra_types['NoneType'] = NoneType + extra_types['ellipsis'] = type(Ellipsis) + extra_types['module_type'] = types.ModuleType + if sys.version_info[0] == 2: + extra_types['dict_keys'] = type({}.iterkeys()) + extra_types['dict_values'] = type({}.itervalues()) + extra_types['dict_items'] = type({}.iteritems()) + else: + extra_types['dict_keys'] = type({}.keys()) + extra_types['dict_values'] = type({}.values()) + extra_types['dict_items'] = type({}.items()) + + extra_types['list_iterator'] = type(iter(list())) + extra_types['tuple_iterator'] = type(iter(tuple())) + extra_types['set_iterator'] = type(iter(set())) + extra_types['str_iterator'] = type(iter("")) + if sys.version_info[0] == 2: + extra_types['bytes_iterator'] = type(iter("")) + extra_types['unicode_iterator'] = type(iter(unicode())) + else: + extra_types['bytes_iterator'] = type(iter(bytes())) + extra_types['unicode_iterator'] = type(iter("")) + extra_types['callable_iterator'] = type(iter(lambda: None, None)) + + res = generate_module(lookup_module(builtin_name), extra_types = extra_types.items()) + + if res and 'members' in res and 'object' in res['members']: + assert res['members']['object']['kind'] == 'type', "Unexpected: " + repr(res['members']['object']) + res['members']['object']['value']['doc'] = "The most base type" + + return res + + +def merge_type(baseline_type, new_type): + if 'doc' not in new_type and 'doc' in baseline_type: + new_type['doc'] = baseline_type['doc'] + + merge_member_table(baseline_type['members'], new_type['members']) + + return new_type + +def merge_function(baseline_func, new_func): + new_func['overloads'].extend(baseline_func['overloads']) + return new_func + +def merge_property(baseline_prop, new_prop): + new_prop['type'].extend(baseline_prop['type']) + return new_prop + +def merge_data(baseline_data, new_data): + new_data['type'].extend(baseline_data['type']) + return new_data + +def merge_method(baseline_method, new_method): + if baseline_method.get('overloads') is not None: + if new_method.get('overloads') is None: + new_method['overloads'] = baseline_method['overloads'] + else: + new_method['overloads'].extend(baseline_method['overloads']) + + if 'doc' in baseline_method and 'doc' not in new_method: + new_method['doc'] = baseline_method['doc'] + #print 'new doc string' + + return new_method + +_MERGES = {'type' : merge_type, + 'function': merge_method, + 'property': merge_property, + 'data': merge_data, + 'method': merge_method} + +def merge_member_table(baseline_table, new_table): + for name, member_table in new_table.items(): + base_member_table = baseline_table.get(name, None) + kind = member_table['kind'] + + if base_member_table is not None and base_member_table['kind'] == kind: + merger = _MERGES.get(kind, None) + if merger is not None: + member_table['value'] = merger(base_member_table['value'], member_table['value']) + #else: + # print('unknown kind') + #elif base_member_table is not None: + # print('kinds differ', kind, base_member_table['kind'], name) + +InitMethodEntry = { + 'kind': 'method', + 'value': { + 'doc': 'x.__init__(...) initializes x; see help(type(x)) for signature', + 'overloads': [generate_overload(NoneType, ('self', object), ('args', object, '*'), ('kwargs', object, '**'))] + } +} + +NewMethodEntry = { + 'kind': 'function', + 'value': { + 'doc': 'T.__new__(S, ...) -> a new object with type S, a subtype of T', + 'overloads': [generate_overload(object, ('self', object), ('args', object, '*'), ('kwargs', object, '**'))] + } +} + +ReprMethodEntry = { + 'kind': 'method', + 'value': { + 'doc': 'x.__repr__() <==> repr(x)', + 'overloads': [generate_overload(str, ('self', object))] + } +} + + + +def _sre_post_fixer(mod): + if sys.platform == 'cli': + # IronPython doesn't have a real _sre module + return mod + + mod['members']['compile'] = { + 'kind': 'function', + 'value': { + 'overloads': [generate_overload(typename_to_typeref('_sre', 'SRE_Pattern'), + ('pattern', object), ('flags', object), ('code', object), ('groups', object), + ('groupindex', object), ('indexgroup', object))], + 'builtin' : True, + 'static': True, + } + } + mod['members']['SRE_Match'] = { + 'kind': 'type', + 'value': { + 'bases': [(builtin_name, 'object')], + 'doc': 'SRE_Match(m: Match, pattern: SRE_Pattern, text: str)\r\nRE_Match(m: Match, pattern: SRE_Pattern, text: str, pos: int, endpos: int)\r\n', + 'members': { + '__new__': { + 'kind': 'function', + 'value': { + 'doc': '__new__(cls: type, m: Match, pattern: SRE_Pattern, text: str)\r\n__new__(cls: type, m: Match, pattern: SRE_Pattern, text: str, pos: int, endpos: int)\r\n', + 'overloads': None + } + }, + 'end': { + 'kind': 'method', + 'value': { + 'doc': 'end(self: SRE_Match, group: object) -> int\r\nend(self: SRE_Match) -> int\r\n', + 'overloads': [ + generate_overload(int, ('self', typename_to_typeref('re', 'SRE_Match'))), + generate_overload(int, ('self', typename_to_typeref('re', 'SRE_Match')), ('group', object)) + ], + } + }, + 'endpos': { + 'kind': 'property', + 'value': { + 'doc': 'Get: endpos(self: SRE_Match) -> int\r\n\r\n', + 'type': type_to_typelist(int) + } + }, + 'expand': { + 'kind': 'method', + 'value': { + 'doc': 'expand(self: SRE_Match, template: object) -> str\r\n', + 'overloads': [generate_overload(str, ('self', typename_to_typeref('re', 'SRE_Match')), ('template', object))], + } + }, + 'group': { + 'kind': 'method', + 'value': { + 'doc': 'group(self: SRE_Match) -> str\r\ngroup(self: SRE_Match, index: object) -> str\r\ngroup(self: SRE_Match, index: object, *additional: Array[object]) -> object\r\n', + 'overloads': [ + generate_overload(str, ('self', typename_to_typeref('re', 'SRE_Match'))), + generate_overload(str, ('self', typename_to_typeref('re', 'SRE_Match')), ('index', object)), + generate_overload(object, ('self', typename_to_typeref('re', 'SRE_Match')), ('index', object), ('additional', tuple, '*')) + ], + }, + }, + 'groupdict': { + 'kind': 'method', + 'value': { + 'doc': 'groupdict(self: SRE_Match, value: object) -> dict (of str to object)\r\ngroupdict(self: SRE_Match, value: str) -> dict (of str to str)\r\ngroupdict(self: SRE_Match) -> dict (of str to str)\r\n', + 'overloads': [ + generate_overload(dict, ('self', typename_to_typeref('re', 'SRE_Match')), ('value', types_to_typelist([object, str]))), + generate_overload(dict, ('self', typename_to_typeref('re', 'SRE_Match'))) + ], + } + }, + 'groups': { + 'kind': 'method', + 'value': { + 'doc': 'groups(self: SRE_Match, default: object) -> tuple\r\ngroups(self: SRE_Match) -> tuple (of str)\r\n', + 'overloads': [ + generate_overload(tuple, ('self', typename_to_typeref('re', 'SRE_Match')), ('default', object)), + generate_overload(tuple, ('self', typename_to_typeref('re', 'SRE_Match'))) + ], + } + }, + 'lastgroup': { + 'kind': 'property', + 'value': { + 'doc': 'Get: lastgroup(self: SRE_Match) -> str\r\n\r\n', + 'type': type_to_typelist(str) + } + }, + 'lastindex': { + 'kind': 'property', + 'value': { + 'doc': 'Get: lastindex(self: SRE_Match) -> object\r\n\r\n', + 'type': type_to_typelist(object) + } + }, + 'pos': { + 'kind': 'property', + 'value': { + 'doc': 'Get: pos(self: SRE_Match) -> int\r\n\r\n', + 'type': type_to_typelist(int) + } + }, + 're': { + 'kind': 'property', + 'value': { + 'doc': 'Get: re(self: SRE_Match) -> SRE_Pattern\r\n\r\n', + 'type': [typename_to_typeref('re', 'SRE_Pattern')] + } + }, + 'regs': { + 'kind': 'property', + 'value': { + 'doc': 'Get: regs(self: SRE_Match) -> tuple\r\n\r\n', + 'type': type_to_typelist(tuple) + } + }, + 'span': { + 'kind': 'method', + 'value': { + 'doc': 'span(self: SRE_Match, group: object) -> tuple (of int)\r\nspan(self: SRE_Match) -> tuple (of int)\r\n', + 'overloads': [ + generate_overload(tuple, ('self', typename_to_typeref('re', 'SRE_Match'))), + generate_overload(tuple, ('self', typename_to_typeref('re', 'SRE_Match')), ('group', object)) + ] + } + }, + 'start': { + 'kind': 'method', + 'value': { + 'doc': 'start(self: SRE_Match, group: object) -> int\r\nstart(self: SRE_Match) -> int\r\n', + 'overloads': [ + generate_overload(int, ('self', typename_to_typeref('re', 'SRE_Match'))), + generate_overload(int, ('self', typename_to_typeref('re', 'SRE_Match')), ('group', object)) + ] + } + }, + 'string': { + 'kind': 'property', + 'value': { + 'doc': 'Get: string(self: SRE_Match) -> str\r\n\r\n', + 'type': type_to_typelist(str) + } + } + }, + 'mro': [typename_to_typeref('re', 'SRE_Match'), type_to_typeref(object)] + } + } + mod['members']['SRE_Pattern'] = { + 'kind': 'type', + 'value': {'bases': [type_to_typeref(object)], + 'doc': '', + 'members': { + '__eq__': { + 'kind': 'method', + 'value': { + 'doc': 'x.__eq__(y) <==> x==y', + 'overloads': [generate_overload(bool, ('self', typename_to_typeref('_sre', 'SRE_Pattern')), ('obj', object))], + } + }, + '__ne__': { + 'kind': 'method', + 'value': { + 'doc': '__ne__(x: object, y: object) -> bool\r\n', + 'overloads': [generate_overload(bool, ('x', object), ('y', object))] + } + }, + '__new__': NewMethodEntry, + 'findall': { + 'kind': 'method', + 'value': { + 'doc': 'findall(self: SRE_Pattern, string: object, pos: int, endpos: object) -> object\r\nfindall(self: SRE_Pattern, string: str, pos: int) -> object\r\nfindall(self: SRE_Pattern, string: str) -> object\r\n', + 'overloads': [generate_overload(bool, ('self', typename_to_typeref('_sre', 'SRE_Pattern')), ('string', str), ('pos', int, '', '0'), ('endpos', object, '', 'None'))] + } + }, + 'finditer': { + 'kind': 'method', + 'value': { + 'doc': 'finditer(self: SRE_Pattern, string: object, pos: int, endpos: int) -> object\r\nfinditer(self: SRE_Pattern, string: object, pos: int) -> object\r\nfinditer(self: SRE_Pattern, string: object) -> object\r\n', + 'overloads': [generate_overload(object, ('self', typename_to_typeref('_sre', 'SRE_Pattern')), ('string', str), ('pos', int, '', '0'), ('endpos', int, '', 'None'))] + } + }, + 'flags': { + 'kind': 'property', + 'value': { + 'doc': 'Get: flags(self: SRE_Pattern) -> int\r\n\r\n', + 'type': type_to_typelist(int) + } + }, + 'groupindex': { + 'kind': 'property', + 'value': { + 'doc': 'Get: groupindex(self: SRE_Pattern) -> dict\r\n\r\n', + 'type': type_to_typelist(dict) + } + }, + 'groups': { + 'kind': 'property', + 'value': { + 'doc': 'Get: groups(self: SRE_Pattern) -> int\r\n\r\n', + 'type': type_to_typelist(int) + } + }, + 'match': { + 'kind': 'method', + 'value': { + 'doc': 'match(self: SRE_Pattern, text: object, pos: int, endpos: int) -> RE_Match\r\nmatch(self: SRE_Pattern, text: object, pos: int) -> RE_Match\r\nmatch(self: SRE_Pattern, text: object) -> RE_Match\r\n', + 'overloads': [generate_overload(object, ('self', typename_to_typeref('_sre', 'SRE_Pattern')), ('text', str), ('pos', int, '', '0'), ('endpos', int, '', 'None'))], + } + }, + 'pattern': { + 'kind': 'property', + 'value': { + 'doc': 'Get: pattern(self: SRE_Pattern) -> str\r\n\r\n', + 'type': type_to_typelist(str) + } + }, + 'search': { + 'kind': 'method', + 'value': { + 'doc': 'search(self: SRE_Pattern, text: object, pos: int, endpos: int) -> RE_Match\r\nsearch(self: SRE_Pattern,text: object, pos: int) -> RE_Match\r\nsearch(self: SRE_Pattern, text: object) -> RE_Match\r\n', + 'overloads': [generate_overload(typename_to_typeref('_sre', 'RE_Match'), ('self', typename_to_typeref('_sre', 'SRE_Pattern')), ('text', str), ('pos', int, '', '0'), ('endpos', int, '', 'None'))] + } + }, + 'split': { + 'kind': 'method', + 'value': { + 'doc': 'split(self: SRE_Pattern, string: object, maxsplit: int) -> list (of str)\r\nsplit(self: SRE_Pattern, string: str) -> list (of str)\r\n', + 'overloads': [generate_overload(list, ('self', typename_to_typeref('_sre', 'SRE_Pattern')), ('string', str), ('maxsplit', int, '', 'None'))] + } + }, + 'sub': { + 'kind': 'method', + 'value': { + 'doc': 'sub(self: SRE_Pattern, repl: object, string: object, count: int) -> str\r\nsub(self: SRE_Pattern, repl: object, string: object) -> str\r\n', + 'overloads': [generate_overload(str, ('self', typename_to_typeref('_sre', 'SRE_Pattern')), ('repl', object), ('string', str), ('count', int, '', 'None'))] + } + }, + 'subn': { + 'kind': 'method', + 'value': {'doc': 'subn(self: SRE_Pattern, repl: object, string: object, count: int) -> object\r\nsubn(self: SRE_Pattern, repl: object, string: str) -> object\r\n', + 'overloads': [generate_overload(object, ('self', typename_to_typeref('_sre', 'SRE_Pattern')), ('repl', object), ('string', str), ('count', int, '', 'None'))] + } + }, + }, + 'mro': [typename_to_typeref('_sre', 'SRE_Pattern'), + type_to_typeref(object)] + } + } + + return mod + + +# fixers which run on the newly generated file, not on the baseline file. +post_module_fixers = { + '_sre' : _sre_post_fixer, +} + +def merge_with_baseline(mod_name, baselinepath, final): + baseline_file = os.path.join(baselinepath, mod_name + '.idb') + if os.path.exists(baseline_file): + print(baseline_file) + f = open(baseline_file, 'rb') + baseline = pickle.load(f) + f.close() + + #import pprint + #pp = pprint.PrettyPrinter() + #pp.pprint(baseline['members']) + fixer = post_module_fixers.get(mod_name, None) + if fixer is not None: + final = fixer(final) + + merge_member_table(baseline['members'], final['members']) + + return final + +def write_analysis(out_filename, analysis): + out_file = open(out_filename + '.idb', 'wb') + saved_analysis = pickle.dumps(analysis, 2) + if sys.platform == 'cli': + # work around strings always being unicode on IronPython, we fail to + # write back out here because IronPython doesn't like the non-ascii + # characters in the unicode string + import System + data = System.Array.CreateInstance(System.Byte, len(saved_analysis)) + for i, v in enumerate(saved_analysis): + try: + data[i] = ord(v) + except: + pass + + saved_analysis = data + out_file.write(saved_analysis) + out_file.close() + + # write a list of members which we can load to check for member existance + out_file = open(out_filename + '.idb.$memlist', 'wb') + for member in sorted(analysis['members']): + if sys.version_info >= (3, 0): + out_file.write((member + '\n').encode('utf8')) + else: + out_file.write(member + '\n') + + out_file.close() + +def write_module(mod_name, outpath, analysis): + write_analysis(os.path.join(outpath, mod_name), analysis) + + + +if __name__ == "__main__": + outpath = sys.argv[1] + if len(sys.argv) > 2: + baselinepath = sys.argv[2] + else: + baselinepath = None + + res = generate_builtin_module() + if not res: + raise RuntimeError("Unable to scrape builtins") + res = merge_with_baseline(builtin_name, baselinepath, res) + + write_module(builtin_name, outpath, res) + + for mod_name in sys.builtin_module_names: + if (mod_name == builtin_name or + mod_name == '__main__' or + not BuiltinScraper.should_include_module(mod_name)): + continue + + res = generate_module(lookup_module(mod_name)) + if res is not None: + try: + res = merge_with_baseline(mod_name, baselinepath, res) + + write_module(mod_name, outpath, res) + except ValueError: + pass diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/VersionDiff.py b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/VersionDiff.py new file mode 100644 index 000000000..e2c28f59e --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/VersionDiff.py @@ -0,0 +1,68 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABLITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +# Compares two different completion databases, dumping the results. +# Used for generating fixers for later versions of Python. +# Usage: +# First run: +# python.exe PythonScraper.py OutDir1 +# pythonv2.exe PythonScraper.py OutDir2 +# python.exe VersionDiff.py OutDir1 OutDir2 +# +# This will then dump a list of removed and new members which can be added to PythonScraper.py. +# A person needs to manually merge these in and make sure the version fields are all correct. +# In general this should be low overhead once every couple of years when a new version of Python +# ships. +# +# In theory this could be completely automated but we'd need to automatically run against all Python +# versions to figure out the minimum version. + +import os, pprint, sys +try: + import cPickle +except: + import _pickle as cPickle + +dir1 = sys.argv[1] +dir2 = sys.argv[2] + +files3 = set(os.listdir(dir1)) +files2 = set(os.listdir(dir2)) + +for filename in files3: + if filename.endswith('.idb') and filename in files2: + b2 = cPickle.load(file(dir2 + '\\' + filename, 'rb')) + b3 = cPickle.load(file(dir1 + '\\' + filename, 'rb')) + + removed_three = set(b2['members']) - set(b3['members']) + added_three = set(b3['members']) - set(b2['members']) + + if removed_three or added_three: + print(filename[:-4]) + if removed_three: + print('Removed in ' + dir1, removed_three) + #if added_three: + # print('New in ' + dir1, added_three) + + #for added in added_three: + # sys.stdout.write("mod['members']['%s'] = " % added) + # b3['members'][added]['value']['version'] = '>=3.2' + # pprint.pprint(b3['members'][added]) + # + diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/python-language-server-ptvs.nuspec b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/python-language-server-ptvs.nuspec new file mode 100644 index 000000000..dfe1b8838 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/python-language-server-ptvs.nuspec @@ -0,0 +1,35 @@ + + + + python-language-server-ptvs$os$ + $version$ + Python Language Server for PTVS + Microsoft + Microsoft + Apache-2.0 + https://github.com/Microsoft/PTVS + false + Microsoft Python Language Server for PTVS + Copyright (c) 2018 + Python;VisualStudio + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Connection.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Connection.cs new file mode 100644 index 000000000..5914597df --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Connection.cs @@ -0,0 +1,709 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Runtime.ExceptionServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Microsoft.PythonTools.Ipc.Json { + public sealed class Connection : IDisposable { + private readonly object _cacheLock = new object(); + private readonly SemaphoreSlim _writeLock = new SemaphoreSlim(1); + private readonly Dictionary _requestCache; + private readonly Dictionary _types; + private readonly Func, Task> _requestHandler; + private readonly bool _disposeWriter, _disposeReader; + private readonly Stream _writer, _reader; + private readonly TextWriter _basicLog; + private readonly TextWriter _logFile; + private readonly object _logFileLock; + private int _seq; + private static readonly char[] _headerSeparator = new[] { ':' }; + + // Exposed settings for tests + internal static bool AlwaysLog = false; + internal static string LoggingBaseDirectory = Path.GetTempPath(); + + private const string LoggingRegistrySubkey = @"Software\Microsoft\PythonTools\ConnectionLog"; + + private static readonly Encoding TextEncoding = new UTF8Encoding(false); + + /// + /// Creates a new connection object for doing client/server communication. + /// + /// The stream that we write to for sending requests and responses. + /// The writer stream is disposed when this object is disposed. + /// The stream that we read from for reading responses and events. + /// The reader stream is disposed when this object is disposed. + /// The callback that is invoked when a request is received. + /// A set of registered types for receiving requests and events. The key is "request.command_name" or "event.event_name" + /// where command_name and event_name correspond to the fields in the Request and Event objects. The type is the type of object + /// which will be deserialized and instantiated. If a type is not registered a GenericRequest or GenericEvent object will be created + /// which will include the complete body of the request as a dictionary. + /// + /// Name of registry key used to determine if we should log messages and exceptions to disk. + /// Text writer to use for basic logging (message ids only). If null then output will go to . + public Connection( + Stream writer, + bool disposeWriter, + Stream reader, + bool disposeReader, + Func, Task> requestHandler = null, + Dictionary types = null, + string connectionLogKey = null, + TextWriter basicLog = null + ) { + _requestCache = new Dictionary(); + _requestHandler = requestHandler; + _types = types; + _writer = writer; + _disposeWriter = disposeWriter; + _reader = reader; + _disposeReader = disposeReader; + _basicLog = basicLog ?? new DebugTextWriter(); + _logFile = OpenLogFile(connectionLogKey, out var filename); + // FxCop won't let us lock a MarshalByRefObject, so we create + // a plain old object that we can log against. + if (_logFile != null) { + _logFileLock = new object(); + LogFilename = filename; + } + } + + public string LogFilename { get; } + + /// + /// Opens the log file for this connection. The log must be enabled in + /// the registry under HKCU\Software\Microsoft\PythonTools\ConnectionLog + /// with the connectionLogKey value set to a non-zero integer or + /// non-empty string. + /// + private static TextWriter OpenLogFile(string connectionLogKey, out string filename) { + filename = null; + if (!AlwaysLog) { + if (string.IsNullOrEmpty(connectionLogKey)) { + return null; + } + + using (var root = Win32.Registry.CurrentUser.OpenSubKey(LoggingRegistrySubkey, false)) { + var value = root?.GetValue(connectionLogKey); + int? asInt = value as int?; + if (asInt.HasValue) { + if (asInt.GetValueOrDefault() == 0) { + // REG_DWORD but 0 means no logging + return null; + } + } else if (string.IsNullOrEmpty(value as string)) { + // Empty string or no value means no logging + return null; + } + } + } + + var filenameBase = Path.Combine( + LoggingBaseDirectory, + string.Format("PythonTools_{0}_{1}_{2:yyyyMMddHHmmss}", connectionLogKey, Process.GetCurrentProcess().Id.ToString(), DateTime.Now) + ); + + filename = filenameBase + ".log"; + for (int counter = 0; counter < int.MaxValue; ++counter) { + try { + var file = new FileStream(filename, FileMode.CreateNew, FileAccess.Write, FileShare.ReadWrite); + return new StreamWriter(file, TextEncoding); + } catch (IOException) { + } catch (UnauthorizedAccessException) { + } + filename = string.Format("{0}_{1}.log", filenameBase, ++counter); + } + return null; + } + + private void LogToDisk(string message) { + if (_logFile == null || _logFileLock == null) { + return; + } + try { + lock (_logFileLock) { + _logFile.WriteLine(message); + _logFile.Flush(); + } + } catch (IOException) { + } catch (ObjectDisposedException) { + } + } + + private void LogToDisk(Exception ex) { + if (_logFile == null || _logFileLock == null) { + return; + } + // Should have immediately logged a message before, so the exception + // will have context automatically. Stack trace will not be + // interesting because of the async-ness. + LogToDisk(string.Format("{0}: {1}", ex.GetType().Name, ex.Message)); + } + + /// + /// When a fire and forget notification is received from the other side this event is raised. + /// + public event EventHandler EventReceived; + + /// + /// When a fire and forget error notification is received from the other side this event is raised. + /// + public event EventHandler ErrorReceived; + + /// + /// Sends a request from the client to the listening server. + /// + /// All request payloads inherit from Request<TResponse> where the TResponse generic parameter + /// specifies the .NET type used for serializing the response. + /// + /// Request to send to the server. + /// + /// An action that you want to invoke after the SendRequestAsync task + /// has completed, but before the message processing loop reads the + /// next message from the read stream. This is currently used to cancel + /// the message loop immediately on reception of a response. This is + /// necessary when you want to stop processing messages without closing + /// the read stream. + /// + public async Task SendRequestAsync(Request request, CancellationToken cancellationToken = default(CancellationToken), Action onResponse = null) + where T : Response, new() { + int seq = Interlocked.Increment(ref _seq); + + var r = new RequestInfo(request, seq, onResponse); + if (cancellationToken.CanBeCanceled) { + cancellationToken.Register(() => r._task.TrySetCanceled()); + } + + lock (_cacheLock) { + _requestCache[seq] = r; + } + + T res; + try { + _basicLog.WriteLine("Sending request {0}: {1}", seq, request.command); + await SendMessage( + new RequestMessage() { + command = r.Request.command, + arguments = r.Request, + seq = seq, + type = PacketType.Request + }, + cancellationToken + ).ConfigureAwait(false); + + res = await r._task.Task.ConfigureAwait(false); + if (r.success) { + return res; + } + + throw new FailedRequestException(r.message, res); + } finally { + lock (_cacheLock) { + _requestCache.Remove(seq); + } + } + } + + /// + /// Send a fire and forget event to the other side. + /// + /// The event value to be sent. + public async Task SendEventAsync(Event eventValue) { + int seq = Interlocked.Increment(ref _seq); + _basicLog.WriteLine("Sending event {0}: {1}", seq, eventValue.name); + try { + await SendMessage( + new EventMessage() { + @event = eventValue.name, + body = eventValue, + seq = seq, + type = PacketType.Event + }, + CancellationToken.None + ).ConfigureAwait(false); + } catch (ObjectDisposedException) { + } + } + + /// + /// Returns a task which will process incoming messages. This can be started on another thread or + /// in whatever form of synchronization context you like. StartProcessing is a convenience helper + /// for starting this running asynchronously using Task.Run. + /// + /// + public async Task ProcessMessages() { + try { + var reader = new ProtocolReader(_reader); + while (true) { + var packet = await ReadPacketAsJObject(reader); + if (packet == null) { + break; + } + + var type = packet["type"].ToObject(); + switch (type) { + case PacketType.Request: { + var seq = packet["seq"].ToObject(); + if (seq == null) { + throw new InvalidDataException("Request is missing seq attribute"); + } + await ProcessRequest(packet, seq); + } + break; + case PacketType.Response: + ProcessResponse(packet); + break; + case PacketType.Event: + ProcessEvent(packet); + break; + case PacketType.Error: + ProcessError(packet); + break; + default: + throw new InvalidDataException("Bad packet type: " + type ?? ""); + } + } + } catch (InvalidDataException ex) { + // UNDONE: Skipping assert to see if that fixes broken tests + //Debug.Assert(false, "Terminating ProcessMessages loop due to InvalidDataException", ex.Message); + // TODO: unsure that it makes sense to do this, but it maintains existing behavior + await WriteError(ex.Message); + } catch (OperationCanceledException) { + } catch (ObjectDisposedException) { + } + + _basicLog.WriteLine("ProcessMessages ended"); + } + + private void ProcessError(JObject packet) { + var eventBody = packet["body"]; + string message; + try { + message = eventBody["message"].Value(); + } catch (Exception e) { + message = e.Message; + } + try { + ErrorReceived?.Invoke(this, new ErrorReceivedEventArgs(message)); + } catch (Exception e) { + // TODO: Report unhandled exception? + Debug.Fail(e.Message); + } + } + + private void ProcessEvent(JObject packet) { + Type requestType; + var name = packet["event"].ToObject(); + var eventBody = packet["body"]; + Event eventObj; + try { + if (name != null && + _types != null && + _types.TryGetValue("event." + name, out requestType)) { + // We have a strongly typed event type registered, use that. + eventObj = eventBody.ToObject(requestType) as Event; + } else { + // We have no strongly typed event type, so give the user a + // GenericEvent and they can look through the body manually. + eventObj = new GenericEvent() { + body = eventBody.ToObject>() + }; + } + } catch (Exception e) { + // TODO: Notify receiver of invalid message + Debug.Fail(e.Message); + return; + } + try { + EventReceived?.Invoke(this, new EventReceivedEventArgs(name, eventObj)); + } catch (Exception e) { + // TODO: Report unhandled exception? + Debug.Fail(e.Message); + } + } + + private void ProcessResponse(JObject packet) { + var body = packet["body"]; + + var reqSeq = packet["request_seq"].ToObject(); + + _basicLog.WriteLine("Received response {0}", reqSeq); + + RequestInfo r; + lock (_cacheLock) { + // We might not find the entry in the request cache if the CancellationSource + // passed into SendRequestAsync was signaled before the request + // was completed. That's okay, there's no one waiting on the + // response anymore. + if (_requestCache.TryGetValue(reqSeq.Value, out r)) { + r.message = packet["message"]?.ToObject() ?? string.Empty; + r.success = packet["success"]?.ToObject() ?? false; + r.SetResponse(body); + } + } + } + + private async Task ProcessRequest(JObject packet, int? seq) { + var command = packet["command"].ToObject(); + var arguments = packet["arguments"]; + + Request request; + Type requestType; + if (command != null && + _types != null && + _types.TryGetValue("request." + command, out requestType)) { + // We have a strongly typed request type registered, use that... + request = (Request)arguments.ToObject(requestType); + } else { + // There's no strongly typed request type, give the user a generic + // request object and they can look through the dictionary. + request = new GenericRequest() { + body = arguments.ToObject>() + }; + } + + bool success = true; + string message = null; + try { + await _requestHandler( + new RequestArgs(command, request), + result => SendResponseAsync(seq.Value, command, success, message, result, CancellationToken.None) + ); + } catch (OperationCanceledException) { + throw; + } catch (ObjectDisposedException) { + throw; + } catch (Exception e) { + success = false; + message = e.ToString(); + Trace.TraceError(message); + await SendResponseAsync(seq.Value, command, success, message, null, CancellationToken.None).ConfigureAwait(false); + } + } + + public void Dispose() { + if (_disposeWriter) { + _writer.Dispose(); + } + if (_disposeReader) { + _reader.Dispose(); + } + _writeLock.Dispose(); + lock (_cacheLock) { + foreach (var r in _requestCache.Values) { + r.Cancel(); + } + } + try { + _logFile?.Dispose(); + } catch (ObjectDisposedException) { + } + } + + internal static async Task ReadPacketAsJObject(ProtocolReader reader) { + var line = await ReadPacket(reader).ConfigureAwait(false); + if (line == null) { + return null; + } + + string message = ""; + JObject packet = null; + try { + // JObject.Parse is more strict than JsonConvert.DeserializeObject, + // the latter happily deserializes malformed json. + packet = JObject.Parse(line); + } catch (JsonSerializationException ex) { + message = ": " + ex.Message; + } catch (JsonReaderException ex) { + message = ": " + ex.Message; + } + + if (packet == null) { + Debug.WriteLine("Failed to parse {0}{1}", line, message); + throw new InvalidDataException("Failed to parse packet" + message); + } + + return packet; + } + + /// + /// Reads a single message from the protocol buffer. First reads in any headers until a blank + /// line is received. Then reads in the body of the message. The headers must include a Content-Length + /// header specifying the length of the body. + /// + private static async Task ReadPacket(ProtocolReader reader) { + var headers = new Dictionary(StringComparer.OrdinalIgnoreCase); + var lines = new List(); + string line; + while ((line = await reader.ReadHeaderLineAsync().ConfigureAwait(false)) != null) { + lines.Add(line ?? "(null)"); + if (String.IsNullOrEmpty(line)) { + if (headers.Count == 0) { + continue; + } + // end of headers for this request... + break; + } + var split = line.Split(_headerSeparator, 2); + if (split.Length != 2) { + // Probably getting an error message, so read all available text + var error = line; + try { + // Encoding is uncertain since this is malformed + error += TextEncoding.GetString(await reader.ReadToEndAsync()); + } catch (ArgumentException) { + } + throw new InvalidDataException("Malformed header, expected 'name: value'" + Environment.NewLine + error); + } + headers[split[0]] = split[1]; + } + + if (line == null) { + return null; + } + + string contentLengthStr; + int contentLength; + + if (!headers.TryGetValue(Headers.ContentLength, out contentLengthStr)) { + // HACK: Attempting to find problem with message content + Console.Error.WriteLine("Content-Length not specified on request. Lines follow:"); + foreach (var l in lines) { + Console.Error.WriteLine($"> {l}"); + } + Console.Error.Flush(); + throw new InvalidDataException("Content-Length not specified on request"); + } + + if (!Int32.TryParse(contentLengthStr, out contentLength) || contentLength < 0) { + throw new InvalidDataException("Invalid Content-Length: " + contentLengthStr); + } + + var contentBinary = await reader.ReadContentAsync(contentLength); + if (contentBinary.Length == 0 && contentLength > 0) { + // The stream was closed, so let's abort safely + return null; + } + if (contentBinary.Length != contentLength) { + throw new InvalidDataException(string.Format("Content length does not match Content-Length header. Expected {0} bytes but read {1} bytes.", contentLength, contentBinary.Length)); + } + + try { + var text = TextEncoding.GetString(contentBinary); + return text; + } catch (ArgumentException ex) { + throw new InvalidDataException("Content is not valid UTF-8.", ex); + } + } + + private async Task SendResponseAsync( + int sequence, + string command, + bool success, + string message, + Response response, + CancellationToken cancel + ) { + int newSeq = Interlocked.Increment(ref _seq); + _basicLog.WriteLine("Sending response {0}", newSeq); + await SendMessage( + new ResponseMessage() { + request_seq = sequence, + success = success, + message = message, + command = command, + body = response, + seq = newSeq, + type = PacketType.Response + }, + cancel + ).ConfigureAwait(false); + } + + /// + /// Sends a single message across the wire. + /// + /// + /// Base protocol defined at https://github.com/Microsoft/language-server-protocol/blob/master/protocol.md#base-protocol + /// + private async Task SendMessage(ProtocolMessage packet, CancellationToken cancel) { + var str = JsonConvert.SerializeObject(packet, UriJsonConverter.Instance); + + try { + try { + await _writeLock.WaitAsync(cancel).ConfigureAwait(false); + } catch (ArgumentNullException) { + throw new ObjectDisposedException(nameof(_writeLock)); + } catch (ObjectDisposedException) { + throw new ObjectDisposedException(nameof(_writeLock)); + } + try { + LogToDisk(str); + + // The content part is encoded using the charset provided in the Content-Type field. + // It defaults to utf-8, which is the only encoding supported right now. + var contentBytes = TextEncoding.GetBytes(str); + + // The header part is encoded using the 'ascii' encoding. + // This includes the '\r\n' separating the header and content part. + var header = "Content-Length: " + contentBytes.Length + "\r\n\r\n"; + var headerBytes = Encoding.ASCII.GetBytes(header); + + await _writer.WriteAsync(headerBytes, 0, headerBytes.Length).ConfigureAwait(false); + await _writer.WriteAsync(contentBytes, 0, contentBytes.Length).ConfigureAwait(false); + await _writer.FlushAsync().ConfigureAwait(false); + } finally { + _writeLock.Release(); + } + } catch (Exception ex) { + LogToDisk(ex); + throw; + } + } + + /// + /// Writes an error on a malformed request. + /// + private async Task WriteError(string message) { + try { + await SendMessage( + new ErrorMessage() { + seq = Interlocked.Increment(ref _seq), + type = PacketType.Error, + body = new Dictionary() { { "message", message } } + }, + CancellationToken.None + ); + throw new InvalidOperationException(message); + } catch (ObjectDisposedException) { + } catch (IOException) { + } + } + + /// + /// Class used for serializing each packet of information we send across. + /// + /// Each packet consists of a packet type (defined in the PacketType class), a sequence + /// number (requests and responses have linked sequence numbers), and a body which is + /// the rest of the JSON payload. + /// + private class ProtocolMessage { + public string type; + public int seq; + } + + private class RequestMessage : ProtocolMessage { + public string command; + public object arguments; + } + + private class ResponseMessage : ProtocolMessage { + public int request_seq; + public bool success; + public string command; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public string message; + public object body; + } + + private class EventMessage : ProtocolMessage { + public string @event; + public object body; + } + + private class ErrorMessage : ProtocolMessage { + public object body; + } + + private class PacketType { + public const string Request = "request"; + public const string Response = "response"; + public const string Event = "event"; + public const string Error = "error"; + } + + /// + /// Provides constants for known header types, currently just includs the Content-Length + /// header for specifying the length of the body of the request in bytes. + /// + private class Headers { + public const string ContentLength = "Content-Length"; + } + + /// + /// Base class for tracking state of our requests. This is a non-generic class so we can have + /// a dictionary of these and call the abstract methods which are actually implemented by the + /// generic version. + /// + private abstract class RequestInfo { + private readonly Request _request; + private readonly int _sequence; + public bool success; + public string message; + + internal RequestInfo(Request request, int sequence) { + _request = request; + _sequence = sequence; + } + + public Request Request => _request; + + internal abstract void SetResponse(JToken obj); + internal abstract void Cancel(); + } + + /// + /// Generic version of the request info type which includes the type information for + /// the type of response we should return. This response type is inferred from the + /// TRespones type parameter of the Request<TResponse> generic type which is + /// passed on a SendRequestAsync call. The caller this receives the strongly typed + /// response without any extra need to specify any information beyond the request + /// payload. + /// + private sealed class RequestInfo : RequestInfo where TResponse : Response, new() { + internal readonly TaskCompletionSource _task; + private readonly Action _postResponseAction; + + internal RequestInfo(Request request, int sequence, Action postResponseAction) : base(request, sequence) { + // Make it run continuation asynchronously so that the thread calling SetResponse + // doesn't end up being hijacked to run the code after SendRequest, which would + // prevent it from processing any more messages. + _task = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _postResponseAction = postResponseAction; + } + + internal override void SetResponse(JToken obj) { + var res = obj.ToObject(); + _task.TrySetResult(res); + _postResponseAction?.Invoke(res); + } + + internal override void Cancel() { + _task.TrySetCanceled(); + } + } + } +} + diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/DebugTextWriter.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/DebugTextWriter.cs new file mode 100644 index 000000000..ce8b70be8 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/DebugTextWriter.cs @@ -0,0 +1,38 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Diagnostics; +using System.IO; +using System.Text; + +namespace Microsoft.PythonTools.Ipc.Json { + public class DebugTextWriter : TextWriter { + public override Encoding Encoding => Encoding.UTF8; + public override void Write(char value) { + // Technically this is the only Write/WriteLine overload we need to + // implement. We override the string versions for better performance. + Debug.Write(value); + } + + public override void Write(string value) { + Debug.Write(value); + } + + public override void WriteLine(string value) { + Debug.WriteLine(value); + } + } +} diff --git a/src/Analysis/Engine/Impl/Projects/AnalysisCompleteEventArgs.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/ErrorReceivedEventArgs.cs similarity index 69% rename from src/Analysis/Engine/Impl/Projects/AnalysisCompleteEventArgs.cs rename to src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/ErrorReceivedEventArgs.cs index a93c6343f..bb3a9bef9 100644 --- a/src/Analysis/Engine/Impl/Projects/AnalysisCompleteEventArgs.cs +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/ErrorReceivedEventArgs.cs @@ -9,21 +9,22 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. using System; -namespace Microsoft.PythonTools.Projects { - public sealed class AnalysisCompleteEventArgs : EventArgs { - private readonly string _path; +namespace Microsoft.PythonTools.Ipc.Json { + public sealed class ErrorReceivedEventArgs : EventArgs { + private readonly string _message; - public string Path => _path; - - public AnalysisCompleteEventArgs(string path) { - _path = path; + public ErrorReceivedEventArgs(string message) { + _message = message; } + + public string Message => _message; } } + diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Event.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Event.cs new file mode 100644 index 000000000..016a4936f --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Event.cs @@ -0,0 +1,32 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace Microsoft.PythonTools.Ipc.Json { + public class Event { + + [JsonIgnore] + public virtual string name => null; + } + + + public class GenericEvent : Event { + public Dictionary body; + } +} + diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/EventReceivedEventArgs.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/EventReceivedEventArgs.cs new file mode 100644 index 000000000..dc03d6a89 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/EventReceivedEventArgs.cs @@ -0,0 +1,33 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; + +namespace Microsoft.PythonTools.Ipc.Json { + public sealed class EventReceivedEventArgs : EventArgs { + private readonly string _name; + private readonly Event _event; + + public EventReceivedEventArgs(string name, Event event_) { + _name = name; + _event = event_; + } + + public Event Event => _event; + public string Name => _name; + } +} + diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/FailedRequestException.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/FailedRequestException.cs new file mode 100644 index 000000000..41dfa6467 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/FailedRequestException.cs @@ -0,0 +1,38 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Runtime.Serialization; +using System.Security.Permissions; + +namespace Microsoft.PythonTools.Ipc.Json { + [Serializable] + public sealed class FailedRequestException : Exception { + private readonly Response _response; + + public FailedRequestException(string message, Response response) : base(message) { + _response = response; + } + + [SecurityPermission(SecurityAction.Demand, SerializationFormatter = true)] + public override void GetObjectData(SerializationInfo info, StreamingContext context) { + base.GetObjectData(info, context); + info.AddValue("Response", _response); + } + + public Response Response => _response; + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Microsoft.PythonTools.Ipc.Json.csproj b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Microsoft.PythonTools.Ipc.Json.csproj new file mode 100644 index 000000000..08ac149b7 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Microsoft.PythonTools.Ipc.Json.csproj @@ -0,0 +1,15 @@ + + + net472 + Microsoft.PythonTools.Ipc.Json + Microsoft.PythonTools.Ipc.Json + + + + + + + + + + diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Properties/AssemblyInfo.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..fe60dedfe --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Properties/AssemblyInfo.cs @@ -0,0 +1,24 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +[assembly: ComVisible(false)] +[assembly: Guid("e1e1613d-0c96-42f9-9f2d-052c72533297")] + +[assembly: InternalsVisibleTo("IpcJsonTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/ProtocolReader.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/ProtocolReader.cs new file mode 100644 index 000000000..942525613 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/ProtocolReader.cs @@ -0,0 +1,117 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.PythonTools.Ipc.Json { + class ProtocolReader { + private Stream _stream; + private List _buffer = new List(); + + public ProtocolReader(Stream stream) { + if (stream == null) { + throw new ArgumentNullException(nameof(stream)); + } + _stream = stream; + } + + /// + /// Reads an ASCII encoded header line asynchronously from the current stream + /// and returns the data as a string. Line termination chars are '\r\n' and + /// are excluded from the return value. Keeps reading until it finds it, and + /// if it reaches the end of the stream (no more data is read) without finding + /// it then it returns null. + /// + public async Task ReadHeaderLineAsync() { + // Keep reading into the buffer until it contains the '\r\n'. + int searchStartPos = 0; + int newLineIndex; + while ((newLineIndex = IndexOfNewLineInBuffer(searchStartPos)) < 0) { + searchStartPos = Math.Max(0, _buffer.Count - 1); + if (await ReadIntoBuffer() == 0) { + return null; + } + } + + var line = _buffer.Take(newLineIndex).ToArray(); + _buffer.RemoveRange(0, newLineIndex + 2); + return Encoding.ASCII.GetString(line); + } + + /// + /// Reads all bytes from the current position to the end of the + /// stream asynchronously. + /// + public async Task ReadToEndAsync() { + int read; + while ((read = await ReadIntoBuffer()) > 0) { + } + + var all = _buffer.ToArray(); + _buffer.Clear(); + return all; + } + + /// + /// Reads a number of bytes asynchronously from the current stream. + /// + /// Number of bytes to read. + /// + /// May return fewer bytes than requested. + /// + public async Task ReadContentAsync(int byteCount) { + if (byteCount < 0) { + throw new ArgumentOutOfRangeException(nameof(byteCount), byteCount, "Value cannot be negative."); + } + + while (_buffer.Count < byteCount) { + if (await ReadIntoBuffer() == 0) { + break; + } + } + + var actualCount = Math.Min(byteCount, _buffer.Count); + var result = _buffer.Take(actualCount).ToArray(); + _buffer.RemoveRange(0, actualCount); + return result; + } + + private int IndexOfNewLineInBuffer(int searchStartPos) { + for (int i = searchStartPos; i < _buffer.Count - 1; i++) { + if (_buffer[i] == 13 && _buffer[i + 1] == 10) { + return i; + } + } + return -1; + } + + /// + /// Reads bytes from the stream into the buffer, in chunks. + /// + /// Number of bytes that were added to the buffer. + private async Task ReadIntoBuffer() { + var temp = new byte[1024]; + var read = await _stream.ReadAsync(temp, 0, temp.Length); + _buffer.AddRange(temp.Take(read).ToArray()); + return read; + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Request.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Request.cs new file mode 100644 index 000000000..bc38fa1d8 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Request.cs @@ -0,0 +1,37 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Microsoft.PythonTools.Ipc.Json { + public class Request { + [JsonIgnore] + public virtual string command => null; + + public override string ToString() => command; + } + + public class GenericRequest : Request { + public Dictionary body; + } + + public class Request : Request where TResponse : Response, new() { + } +} \ No newline at end of file diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/RequestArgs.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/RequestArgs.cs new file mode 100644 index 000000000..93f1fe538 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/RequestArgs.cs @@ -0,0 +1,31 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + + +namespace Microsoft.PythonTools.Ipc.Json { + public sealed class RequestArgs { + private readonly string _command; + private readonly Request _request; + + public RequestArgs(string command, Request request) { + _command = command; + _request = request; + } + + public Request Request => _request; + public string Command => _command; + } +} diff --git a/src/Analysis/Engine/Impl/Projects/IPythonProjectProvider.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Response.cs similarity index 72% rename from src/Analysis/Engine/Impl/Projects/IPythonProjectProvider.cs rename to src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Response.cs index 0a3145b17..1a73e78a4 100644 --- a/src/Analysis/Engine/Impl/Projects/IPythonProjectProvider.cs +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Response.cs @@ -9,16 +9,14 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -namespace Microsoft.PythonTools.Projects { - /// - /// Provides access to an abstract Python project. - /// - public interface IPythonProjectProvider { - PythonProject Project { get; } +namespace Microsoft.PythonTools.Ipc.Json { + public class Response { + public Response() { + } } -} +} \ No newline at end of file diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/UriJsonConverter.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/UriJsonConverter.cs new file mode 100644 index 000000000..bece70ae5 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/UriJsonConverter.cs @@ -0,0 +1,41 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using Newtonsoft.Json; + +namespace Microsoft.PythonTools.Ipc.Json { + public sealed class UriJsonConverter : JsonConverter { + public static readonly JsonConverter Instance = new UriJsonConverter(); + + public override bool CanConvert(Type objectType) { + return typeof(Uri).IsAssignableFrom(objectType); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) { + var uri = serializer.Deserialize(reader); + return string.IsNullOrEmpty(uri) ? null : new Uri(uri); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) { + if (value is Uri u) { + serializer.Serialize(writer, u.AbsoluteUri); + } else { + serializer.Serialize(writer, null); + } + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonConnectionTests.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonConnectionTests.cs new file mode 100644 index 000000000..32c4e534f --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonConnectionTests.cs @@ -0,0 +1,396 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.PythonTools.Infrastructure; +using Microsoft.PythonTools.Ipc.Json; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace IpcJsonTests { + [TestClass] + public class IpcJsonConnectionTests { + private Connection _client; + private Connection _server; + private readonly AutoResetEvent _connected = new AutoResetEvent(false); + + private static string PythonSocketSendEventPath => Path.Combine(TestData.GetPath("TestData"), "Ipc.Json", "socket_send_event.py"); + private static string PythonSocketHandleRequest => Path.Combine(TestData.GetPath("TestData"), "Ipc.Json", "socket_handle_request.py"); + + // Change this to true if you want to have an easier time debugging the python script + // You'll be expected to start it manually, which you can do by opening + // IpcJsonServers.sln, adjusting the debug script arguments to have the correct port number and pressing F5 + private static readonly bool StartPythonProcessManually = false; + + [ClassInitialize] + public static void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + [TestMethod, Priority(0)] + public async Task DotNetHandleRequest() { + InitConnection(async (requestArgs, done) => { + switch (requestArgs.Command) { + case TestDataProtocol.TestRequest.Command: { + var testRequest = requestArgs.Request as TestDataProtocol.TestRequest; + await done(new TestDataProtocol.TestResponse() { + requestText = testRequest.dataText, + responseText = "test response text", + }); + break; + } + default: + throw new InvalidOperationException(); + } + }); + + _client.EventReceived += (object sender, EventReceivedEventArgs e) => { + Assert.Fail(); + }; + + var response = await _client.SendRequestAsync(new TestDataProtocol.TestRequest() { + dataText = "request data text", + dataTextList = new string[] { "value 1", "value 2" }, + }, CancellationTokens.After5s); + + Assert.AreEqual("request data text", response.requestText); + Assert.AreEqual("test response text", response.responseText); + } + + [TestMethod, Priority(0)] + public async Task DotNetHandleRequestUnicode() { + InitConnection(async (requestArgs, done) => { + switch (requestArgs.Command) { + case TestDataProtocol.TestRequest.Command: { + var testRequest = requestArgs.Request as TestDataProtocol.TestRequest; + await done(new TestDataProtocol.TestResponse() { + requestText = testRequest.dataText, + responseText = "この文は、テストです。私はこれがうまく願っています。", + }); + break; + } + default: + throw new InvalidOperationException(); + } + }); + + _client.EventReceived += (object sender, EventReceivedEventArgs e) => { + Assert.Fail(); + }; + + var response = await _client.SendRequestAsync(new TestDataProtocol.TestRequest() { + dataText = "データテキストを要求する", + dataTextList = new string[] { "value 1", "value 2" }, + }, CancellationTokens.After5s); + + Assert.AreEqual("データテキストを要求する", response.requestText); + Assert.AreEqual("この文は、テストです。私はこれがうまく願っています。", response.responseText); + } + + [TestMethod, Priority(0)] + public async Task PythonHandleRequest() { + using (InitConnection(PythonSocketHandleRequest)) { + _client.EventReceived += (object sender, EventReceivedEventArgs e) => { + Assert.Fail(); + }; + + var response = await _client.SendRequestAsync(new TestDataProtocol.TestRequest() { + dataText = "request data text", + dataTextList = new string[] { "value 1", "value 2" }, + }, StartPythonProcessManually ? CancellationTokens.After60s : CancellationTokens.After5s); + + await _client.SendRequestAsync( + new TestDataProtocol.DisconnectRequest(), + StartPythonProcessManually ? CancellationTokens.After60s : CancellationTokens.After5s + ); + + Assert.AreEqual("request data text", response.requestText); + Assert.AreEqual("test response text", response.responseText); + } + } + + [TestMethod, Priority(0)] + public async Task PythonHandleRequestUnicode() { + using (InitConnection(PythonSocketHandleRequest)) { + _client.EventReceived += (object sender, EventReceivedEventArgs e) => { + Assert.Fail(); + }; + + var response = await _client.SendRequestAsync(new TestDataProtocol.TestRequest() { + dataText = "データテキストを要求する 请输入", + dataTextList = new string[] { "value 1", "value 2" }, + }, StartPythonProcessManually ? CancellationTokens.After60s : CancellationTokens.After5s); + + await _client.SendRequestAsync( + new TestDataProtocol.DisconnectRequest(), + StartPythonProcessManually ? CancellationTokens.After60s : CancellationTokens.After5s + ); + + Assert.AreEqual("データテキストを要求する 请输入", response.requestText); + Assert.AreEqual("test response text", response.responseText); + } + } + + [TestMethod, Priority(0)] + public async Task DotNetSendEvent() { + InitConnection((requestArgs, done) => { + Assert.Fail(); + return Task.CompletedTask; + }); + + var eventReceived = new AutoResetEvent(false); + var eventsReceived = new List(); + _client.EventReceived += (object sender, EventReceivedEventArgs e) => { + eventsReceived.Add(e); + if (eventsReceived.Count == 1) { + eventReceived.Set(); + } + }; + + await _server.SendEventAsync(new TestDataProtocol.TestEvent() { + dataText = "event data text" + }); + + eventReceived.WaitOne(2000); + + Assert.AreEqual(1, eventsReceived.Count); + Assert.AreEqual(TestDataProtocol.TestEvent.Name, eventsReceived[0].Name); + Assert.AreEqual("event data text", ((TestDataProtocol.TestEvent)eventsReceived[0].Event).dataText); + } + + [TestMethod, Priority(0)] + public async Task PythonSendEvent() { + using (InitConnection(PythonSocketSendEventPath)) { + var eventReceived = new AutoResetEvent(false); + var eventsReceived = new List(); + _client.EventReceived += (object sender, EventReceivedEventArgs e) => { + eventsReceived.Add(e); + if (eventsReceived.Count == 1) { + eventReceived.Set(); + } + }; + + eventReceived.WaitOne(2000); + + Assert.AreEqual(1, eventsReceived.Count); + Assert.AreEqual(TestDataProtocol.TestEvent.Name, eventsReceived[0].Name); + Assert.AreEqual("python event data text", ((TestDataProtocol.TestEvent)eventsReceived[0].Event).dataText); + Assert.AreEqual(76, ((TestDataProtocol.TestEvent)eventsReceived[0].Event).dataInt32); + } + } + + private void InitConnection(Func, Task> serverRequestHandler) { + // Client sends requests, receives responses and events + // Server receives requests, sends back responses and events + // Client creates the socket on an available port, + // passes the port number to the server which connects back to it. + var portNum = StartClient(); + + StartServer(portNum, serverRequestHandler); + + _connected.WaitOne(); + } + + private sealed class KillAndDisposeProcess : IDisposable { + public KillAndDisposeProcess(ProcessOutput process) { + Process = process; + } + + public void Dispose() { + Process.Kill(); + Process.Dispose(); + } + + public ProcessOutput Process { get; } + } + + private IDisposable InitConnection(string serverScriptPath) { + // Client sends requests, receives responses and events + // Server receives requests, sends back responses and events + // Client creates the socket on an available port, + // passes the port number to the server which connects back to it. + var portNum = StartClient(); + + if (!StartPythonProcessManually) { + Assert.IsTrue(File.Exists(serverScriptPath), "Python test data script '{0}' was not found.".FormatUI(serverScriptPath)); + + var workingDir = Path.GetDirectoryName(serverScriptPath); + + var searchPaths = new HashSet(); + searchPaths.Add(PtvsdSearchPath); + searchPaths.Add(workingDir); + + var env = new List>(); + env.Add(new KeyValuePair("PYTHONPATH", string.Join(";", searchPaths))); + + var arguments = new List(); + arguments.Add(serverScriptPath); + arguments.Add("-r"); + arguments.Add(portNum.ToString()); + var proc = ProcessOutput.Run( + (PythonPaths.Python27 ?? PythonPaths.Python27_x64).InterpreterPath, + arguments, + workingDir, + env, + false, + null + ); + try { + if (proc.ExitCode.HasValue) { + // Process has already exited + proc.Wait(); + if (proc.StandardErrorLines.Any()) { + Assert.Fail(String.Join(Environment.NewLine, proc.StandardErrorLines)); + } + return null; + } else { + _connected.WaitOne(); + var p = proc; + proc = null; + return new KillAndDisposeProcess(p); + } + } finally { + if (proc != null) { + proc.Dispose(); + } + } + } else { + // Check the port number variable assigned above if you want to + // start the python process manually + Debugger.Break(); + return null; + } + } + + internal static string PtvsdSearchPath { + get { + return Path.GetDirectoryName(Path.GetDirectoryName(PythonToolsInstallPath.GetFile("ptvsd\\__init__.py", typeof(Connection).Assembly))); + } + } + + private void StartServer(int portNum, Func, Task> requestHandler) { + IPEndPoint endPoint = new IPEndPoint(IPAddress.Loopback, portNum); + + var socket = new Socket(endPoint.AddressFamily, SocketType.Stream, ProtocolType.IP); + socket.Connect(endPoint); + + var stream = new NetworkStream(socket, ownsSocket: true); + + _server = new Connection( + stream, + true, + stream, + true, + requestHandler, + TestDataProtocol.RegisteredTypes + ); + Task.Run(_server.ProcessMessages).DoNotWait(); + } + + private int StartClient() { + new SocketPermission(NetworkAccess.Accept, TransportType.All, "", SocketPermission.AllPorts).Demand(); + + // Use an available port + IPEndPoint endPoint = new IPEndPoint(IPAddress.Loopback, 0); + + var socket = new Socket(endPoint.AddressFamily, SocketType.Stream, ProtocolType.IP); + socket.Bind(endPoint); + socket.Listen(10); + + // Find out which port is being used + var portNum = ((IPEndPoint)socket.LocalEndPoint).Port; + + Task.Run(() => { socket.BeginAccept(StartClientAcceptCallback, socket); }); + return portNum; + } + + private void StartClientAcceptCallback(IAsyncResult ar) { + var socket = ((Socket)ar.AsyncState).EndAccept(ar); + var stream = new NetworkStream(socket, ownsSocket: true); + _client = new Connection(stream, true, stream, true, null, TestDataProtocol.RegisteredTypes); + Task.Run(_client.ProcessMessages).DoNotWait(); + _connected.Set(); + } + } + + static class TestDataProtocol { + public static readonly Dictionary RegisteredTypes = CollectCommands(); + + private static Dictionary CollectCommands() { + Dictionary all = new Dictionary(); + foreach (var type in typeof(TestDataProtocol).GetNestedTypes()) { + if (type.IsSubclassOf(typeof(Request))) { + var command = type.GetField("Command"); + if (command != null) { + all["request." + (string)command.GetRawConstantValue()] = type; + } + } else if (type.IsSubclassOf(typeof(Event))) { + var name = type.GetField("Name"); + if (name != null) { + all["event." + (string)name.GetRawConstantValue()] = type; + } + } + } + return all; + } + +#pragma warning disable 0649 // Field 'field' is never assigned to, and will always have its default value 'value' + + public sealed class TestRequest : Request { + public const string Command = "testRequest"; + + public override string command => Command; + + public string dataText; + public string[] dataTextList; + + } + + public sealed class TestResponse : Response { + public string requestText; + public string responseText; + public string[] responseTextList; + } + + public sealed class DisconnectRequest : GenericRequest { + public const string Command = "disconnect"; + + public override string command => Command; + } + + public class TestEvent : Event { + public const string Name = "testEvent"; + public string dataText; + public int dataInt32; + public long dataInt64; + public float dataFloat32; + public double dataFloat64; + + public override string name => Name; + } + +#pragma warning restore 0649 + + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonPacketReadCSharpTests.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonPacketReadCSharpTests.cs new file mode 100644 index 000000000..319c321f5 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonPacketReadCSharpTests.cs @@ -0,0 +1,122 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.IO; +using System.Threading.Tasks; +using Microsoft.PythonTools.Ipc.Json; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace IpcJsonTests { + [TestClass] + public class IpcJsonPacketReadCSharpTests { + [TestMethod, Priority(0)] + public async Task ValidPackets() { + await TestValidPacketAsync(PacketProvider.GetValidPacket1()); + await TestValidPacketAsync(PacketProvider.GetValidPacket2()); + } + + [TestMethod, Priority(0)] + public async Task ValidUnicodePackets() { + await TestValidPacketAsync(PacketProvider.GetValidUnicodePacket1()); + await TestValidPacketAsync(PacketProvider.GetValidUnicodePacket2()); + } + + [TestMethod, Priority(0)] + public async Task TruncatedJson() { + foreach (var packet in PacketProvider.GetTruncatedJsonPackets()) { + await TestInvalidPacketAsync(packet); + } + } + + [TestMethod, Priority(0)] + public async Task IncorrectContentLengthUnderread() { + foreach (var packet in PacketProvider.GetIncorrectContentLengthUnderreadPackets()) { + await TestInvalidPacketAsync(packet); + } + } + + [TestMethod, Priority(0)] + public async Task IncorrectContentLengthOverread() { + foreach (var packet in PacketProvider.GetIncorrectContentLengthOverreadPackets()) { + await TestInvalidPacketAsync(packet); + } + } + + [TestMethod, Priority(0)] + public async Task IncorrectContentLengthOverreadEndOfStream() { + foreach (var packet in PacketProvider.GetIncorrectContentLengthOverreadEndOfStreamPackets()) { + await TestInvalidPacketAsync(packet); + } + } + + [TestMethod, Priority(0)] + public async Task InvalidContentLengthType() { + await TestInvalidPacketAsync(PacketProvider.GetInvalidContentLengthIntegerTooLargePacket()); + await TestInvalidPacketAsync(PacketProvider.GetInvalidContentLengthNegativeIntegerPacket()); + await TestInvalidPacketAsync(PacketProvider.GetInvalidContentLengthNotIntegerPacket()); + } + + [TestMethod, Priority(0)] + public async Task MissingContentLength() { + await TestInvalidPacketAsync(PacketProvider.GetMissingContentLengthPacket()); + } + + [TestMethod, Priority(0)] + public async Task MalformedHeader() { + await TestInvalidPacketAsync(PacketProvider.GetMalformedHeaderPacket()); + } + + [TestMethod, Priority(0)] + public async Task AdditionalHeaders() { + await TestValidPacketAsync(PacketProvider.GetAdditionalHeadersPacket()); + } + + [TestMethod, Priority(0)] + public async Task EmptyStream() { + await TestNoPacketAsync(PacketProvider.GetNoPacket()); + } + + [TestMethod, Priority(0)] + public async Task UnterminatedHeader() { + await TestNoPacketAsync(PacketProvider.GetUnterminatedPacket()); + await TestNoPacketAsync(PacketProvider.GetIncorrectlyTerminatedPacket()); + } + + private static async Task TestValidPacketAsync(Packet packet) { + Assert.IsFalse(packet.BadHeaders || packet.BadContent); + var reader = new ProtocolReader(packet.AsStream()); + var obj = await Connection.ReadPacketAsJObject(reader); + Assert.IsNotNull(obj); + } + + private static async Task TestNoPacketAsync(Packet packet) { + var reader = new ProtocolReader(packet.AsStream()); + var obj = await Connection.ReadPacketAsJObject(reader); + Assert.IsNull(obj); + } + + private static async Task TestInvalidPacketAsync(Packet packet) { + Assert.IsTrue(packet.BadHeaders || packet.BadContent); + var reader = new ProtocolReader(packet.AsStream()); + try { + var obj = await Connection.ReadPacketAsJObject(reader); + Assert.IsNotNull(obj); + Assert.Fail("Failed to raise InvalidDataException."); + } catch (InvalidDataException) { + } + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonPacketReadPythonTests.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonPacketReadPythonTests.cs new file mode 100644 index 000000000..bdc06a4c3 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonPacketReadPythonTests.cs @@ -0,0 +1,362 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.PythonTools.Infrastructure; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; +using TestUtilities.Python; + +namespace IpcJsonTests { + [TestClass] + public class IpcJsonPacketReadPythonTests { + private Stream _clientStream; + private readonly AutoResetEvent _connected = new AutoResetEvent(false); + + private static string PythonParsingTestPath => Path.Combine(TestData.GetPath("TestData"), "Ipc.Json", "parsing_test.py"); + + [ClassInitialize] + public static void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + [TestInitialize] + public void TestInit() { + Version.AssertInstalled(); + } + + internal virtual PythonVersion Version { + get { + return PythonPaths.Python26 ?? PythonPaths.Python26_x64; + } + } + + [TestMethod, Priority(0)] + public async Task ValidPackets() { + await TestValidPacketAsync(PacketProvider.GetValidPacket1()); + await TestValidPacketAsync(PacketProvider.GetValidPacket2()); + } + + [TestMethod, Priority(0)] + public async Task ValidUnicodePackets() { + await TestValidPacketAsync(PacketProvider.GetValidUnicodePacket1()); + await TestValidPacketAsync(PacketProvider.GetValidUnicodePacket2()); + } + + [TestMethod, Priority(0)] + public async Task TruncatedJson() { + foreach (var packet in PacketProvider.GetTruncatedJsonPackets()) { + await TestInvalidPacketAsync(packet); + } + } + + [TestMethod, Priority(0)] + public async Task IncorrectContentLengthUnderread() { + foreach (var packet in PacketProvider.GetIncorrectContentLengthUnderreadPackets()) { + await TestInvalidPacketAsync(packet); + } + } + + [TestMethod, Priority(0)] + public async Task IncorrectContentLengthOverread() { + foreach (var packet in PacketProvider.GetIncorrectContentLengthOverreadPackets()) { + await TestInvalidPacketAsync(packet); + } + } + + [TestMethod, Priority(0)] + public async Task IncorrectContentLengthOverreadEndOfStream() { + foreach (var packet in PacketProvider.GetIncorrectContentLengthOverreadEndOfStreamPackets()) { + await TestInvalidPacketAsync(packet); + } + } + + [TestMethod, Priority(0)] + public async Task InvalidContentLengthType() { + await TestInvalidPacketAsync(PacketProvider.GetInvalidContentLengthIntegerTooLargePacket()); + await TestInvalidPacketAsync(PacketProvider.GetInvalidContentLengthNegativeIntegerPacket()); + await TestInvalidPacketAsync(PacketProvider.GetInvalidContentLengthNotIntegerPacket()); + } + + [TestMethod, Priority(0)] + public async Task MissingContentLength() { + await TestInvalidPacketAsync(PacketProvider.GetMissingContentLengthPacket()); + } + + [TestMethod, Priority(0)] + public async Task MalformedHeader() { + await TestInvalidPacketAsync(PacketProvider.GetMalformedHeaderPacket()); + } + + [TestMethod, Priority(0)] + public async Task AdditionalHeaders() { + await TestValidPacketAsync(PacketProvider.GetAdditionalHeadersPacket()); + } + + [TestMethod, Priority(0)] + public async Task EmptyStream() { + await TestNoPacketAsync(PacketProvider.GetNoPacket()); + } + + [TestMethod, Priority(0)] + public async Task UnterminatedHeader() { + await TestNoPacketAsync(PacketProvider.GetUnterminatedPacket()); + await TestNoPacketAsync(PacketProvider.GetIncorrectlyTerminatedPacket()); + } + + private Task TestValidPacketAsync(Packet packet) { + Assert.IsFalse(packet.BadHeaders || packet.BadContent); + return TestPacketAsync(packet); + } + + private Task TestInvalidPacketAsync(Packet packet) { + Assert.IsTrue(packet.BadHeaders || packet.BadContent); + return TestPacketAsync(packet, + packet.BadHeaders ? "ptvsd.ipcjson.InvalidHeaderError" : "ptvsd.ipcjson.InvalidContentError", + closeStream: packet.ReadPastEndOfStream + ); + } + + private Task TestNoPacketAsync(Packet packet) { + string expectedError = null; + if (packet.BadHeaders) { + expectedError = "ptvsd.ipcjson.InvalidHeaderError"; + } else if (packet.BadContent) { + expectedError = "ptvsd.ipcjson.InvalidContentError"; + } + return TestPacketAsync(packet, expectedError, closeStream: true); + } + + private async Task TestPacketAsync(Packet packet, string expectedError = null, bool closeStream = false) { + using (var proc = InitConnection(PythonParsingTestPath)) { + var bytes = packet.AsBytes(); + await _clientStream.WriteAsync(bytes, 0, bytes.Length); + if (closeStream) { + // We expect the process to be busy reading headers and not exit + var closed = proc.Wait(TimeSpan.FromMilliseconds(500)); + Assert.IsFalse(closed); + + // Close the stream so it gets unblocked + _clientStream.Close(); + } + + if (!proc.Wait(TimeSpan.FromSeconds(2))) { + proc.Kill(); + Assert.Fail("Python process did not exit"); + } + + CheckProcessResult(proc, expectedError); + } + } + + private ProcessOutput InitConnection(string serverScriptPath) { + var portNum = StartClient(); + + Assert.IsTrue(File.Exists(serverScriptPath), "Python test data script '{0}' was not found.".FormatUI(serverScriptPath)); + + var workingDir = Path.GetDirectoryName(serverScriptPath); + + var searchPaths = new HashSet(); + searchPaths.Add(IpcJsonConnectionTests.PtvsdSearchPath); + searchPaths.Add(workingDir); + + var env = new List>(); + env.Add(new KeyValuePair(Version.Configuration.PathEnvironmentVariable, string.Join(";", searchPaths))); + + var arguments = new List(); + arguments.Add(serverScriptPath); + arguments.Add("-r"); + arguments.Add(portNum.ToString()); + var proc = ProcessOutput.Run( + Version.InterpreterPath, + arguments, + workingDir, + env, + false, + null + ); + + if (proc.ExitCode.HasValue) { + // Process has already exited + proc.Wait(); + CheckProcessResult(proc); + } + + _connected.WaitOne(10000); + + return proc; + } + + private void CheckProcessResult(ProcessOutput proc, string expectedError = null) { + Console.WriteLine(String.Join(Environment.NewLine, proc.StandardOutputLines)); + Debug.WriteLine(String.Join(Environment.NewLine, proc.StandardErrorLines)); + if (!string.IsNullOrEmpty(expectedError)) { + var matches = proc.StandardErrorLines.Where(err => err.Contains(expectedError)); + if (matches.Count() == 0) { + Assert.Fail(String.Join(Environment.NewLine, proc.StandardErrorLines)); + } + } else { + if (proc.StandardErrorLines.Any()) { + Assert.Fail(String.Join(Environment.NewLine, proc.StandardErrorLines)); + } + Assert.AreEqual(0, proc.ExitCode); + } + } + + private int StartClient() { + new SocketPermission(NetworkAccess.Accept, TransportType.All, "", SocketPermission.AllPorts).Demand(); + + // Use an available port + IPEndPoint endPoint = new IPEndPoint(IPAddress.Loopback, 0); + + var socket = new Socket(endPoint.AddressFamily, SocketType.Stream, ProtocolType.IP); + socket.Bind(endPoint); + socket.Listen(10); + + // Find out which port is being used + var portNum = ((IPEndPoint)socket.LocalEndPoint).Port; + + Task.Run(() => { socket.BeginAccept(StartClientAcceptCallback, socket); }); + return portNum; + } + + private void StartClientAcceptCallback(IAsyncResult ar) { + var socket = ((Socket)ar.AsyncState).EndAccept(ar); + _clientStream = new NetworkStream(socket, ownsSocket: true); + _connected.Set(); + } + } + + [TestClass] + public class IpcJsonPacketReadPythonTestsIpy : IpcJsonPacketReadPythonTests { + [ClassInitialize] + public static new void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + internal override PythonVersion Version { + get { + return PythonPaths.IronPython27 ?? PythonPaths.IronPython27_x64; + } + } + } + + [TestClass] + public class IpcJsonPacketReadPythonTests27 : IpcJsonPacketReadPythonTests { + [ClassInitialize] + public static new void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + internal override PythonVersion Version { + get { + return PythonPaths.Python27 ?? PythonPaths.Python27_x64; + } + } + } + + [TestClass] + public class IpcJsonPacketReadPythonTests31 : IpcJsonPacketReadPythonTests { + [ClassInitialize] + public static new void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + internal override PythonVersion Version { + get { + return PythonPaths.Python31 ?? PythonPaths.Python31_x64; + } + } + } + + [TestClass] + public class IpcJsonPacketReadPythonTests32 : IpcJsonPacketReadPythonTests { + [ClassInitialize] + public static new void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + internal override PythonVersion Version { + get { + return PythonPaths.Python32 ?? PythonPaths.Python32_x64; + } + } + } + + [TestClass] + public class IpcJsonPacketReadPythonTests33 : IpcJsonPacketReadPythonTests { + [ClassInitialize] + public static new void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + internal override PythonVersion Version { + get { + return PythonPaths.Python33 ?? PythonPaths.Python33_x64; + } + } + } + + [TestClass] + public class IpcJsonPacketReadPythonTests34 : IpcJsonPacketReadPythonTests { + [ClassInitialize] + public static new void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + internal override PythonVersion Version { + get { + return PythonPaths.Python34 ?? PythonPaths.Python34_x64; + } + } + } + + [TestClass] + public class IpcJsonPacketReadPythonTests35 : IpcJsonPacketReadPythonTests { + [ClassInitialize] + public static new void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + internal override PythonVersion Version { + get { + return PythonPaths.Python35 ?? PythonPaths.Python35_x64; + } + } + } + + [TestClass] + public class IpcJsonPacketReadPythonTests36 : IpcJsonPacketReadPythonTests { + [ClassInitialize] + public static new void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + internal override PythonVersion Version { + get { + return PythonPaths.Python36 ?? PythonPaths.Python36_x64; + } + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonTests.csproj b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonTests.csproj new file mode 100644 index 000000000..f5801cdd4 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonTests.csproj @@ -0,0 +1,85 @@ + + + + + + 15.0 + + + + + 14.0 + + + + + 16.0 + + + + + 16.0 + + + + + + Debug + AnyCPU + + + 2.0 + {BE9F11BF-EDD4-4138-A8AD-C8126BF2C77F} + Library + Properties + IpcJsonTests + IpcJsonTests + 512 + {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + + + AnyCPU + + + + + + + 3.5 + + + + $(PackagesPath)\Newtonsoft.Json\lib\net45\Newtonsoft.Json.dll + True + + + + + + + + + + + + {D092D54E-FF29-4D32-9AEE-4EF704C92F67} + TestUtilities + + + {e1e1613d-0c96-42f9-9f2d-052c72533297} + Ipc.Json + + + {a731c4c3-3741-4080-a946-c47574c1f3bf} + TestUtilities.Python.Analysis + + + {b3db0521-d9e3-4f48-9e2e-e5ecae886049} + Common + + + + + + + \ No newline at end of file diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/PacketProvider.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/PacketProvider.cs new file mode 100644 index 000000000..90ddb56f5 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/PacketProvider.cs @@ -0,0 +1,162 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace IpcJsonTests { + class PacketProvider { + private static string validJson1 = @"{'request_seq':1,'success':true,'command':'initialize','message':null,'body':{'failedLoads':[],'error':null},'type':'response','seq':2}".Replace('\'', '"'); + private static string validJson2 = @"{'event':'childFileAnalyzed','body':{'filename':'C:\\Projects\\RemoteDebugApp\\RemoteDebugApp.py','fileId':1,'isTemporaryFile':false,'suppressErrorList':false},'type':'event','seq':5}".Replace('\'', '"'); + private static string validUtf8Json1 = @"{'event':'childFileAnalyzed','body':{'filename':'C:\\Projects\\RemoteDebugApp\\この文は、テストです。私はこれがうまく願っています。.py','fileId':1,'isTemporaryFile':false,'suppressErrorList':false},'type':'event','seq':5}".Replace('\'', '"'); + private static string validUtf8Json2 = @"{'event':'childFileAnalyzed','body':{'filename':'C:\\Projects\\RemoteDebugApp\\é.py','fileId':1,'isTemporaryFile':false,'suppressErrorList':false},'type':'event','seq':5}".Replace('\'', '"'); + + public static Packet GetValidPacket1() => MakePacketFromJson(validJson1); + + public static Packet GetValidPacket2() => MakePacketFromJson(validJson2); + + public static Packet GetValidUnicodePacket1() => MakePacketFromJson(validUtf8Json1); + + public static Packet GetValidUnicodePacket2() => MakePacketFromJson(validUtf8Json2); + + public static Packet GetNoPacket() => MakePacket(new byte[0], new byte[0]); + + public static Packet GetUnterminatedPacket() => MakePacket(Encoding.ASCII.GetBytes("NoTerminator"), new byte[0], badHeaders: true); + + public static Packet GetInvalidContentLengthIntegerTooLargePacket() { + return MakePacket(Encoding.ASCII.GetBytes("Content-Length: 2147483649\r\n\r\n"), MakeBody(validJson1), badHeaders: true); + } + + public static Packet GetInvalidContentLengthNegativeIntegerPacket() { + return MakePacket(Encoding.ASCII.GetBytes("Content-Length: -1\r\n\r\n"), MakeBody(validJson1), badHeaders: true); + } + + public static Packet GetInvalidContentLengthNotIntegerPacket() { + return MakePacket(Encoding.ASCII.GetBytes("Content-Length: BAD\r\n\r\n"), MakeBody(validJson1), badHeaders: true); + } + + public static Packet GetMissingContentLengthPacket() { + return MakePacket(Encoding.ASCII.GetBytes("From: Test\r\n\r\n"), MakeBody(validJson1), badHeaders: true); + } + + public static Packet GetMalformedHeaderPacket() { + return MakePacket(Encoding.ASCII.GetBytes("Content-Length\r\n\r\n"), MakeBody(validJson1), badHeaders: true); + } + + public static Packet GetAdditionalHeadersPacket() { + // Other headers are fine, we only care that Content-Length is there and valid + var body = MakeBody(validJson1); + return MakePacket(Encoding.ASCII.GetBytes(string.Format("From: Test\r\nContent-Length:{0}\r\nTo: You\r\n\r\n", body.Length)), body); + } + + public static Packet GetIncorrectlyTerminatedPacket() { + var body = MakeBody(validJson1); + return MakePacket(Encoding.ASCII.GetBytes(string.Format("Content-Length:{0}\n\n", body.Length)), body, badHeaders: true); + } + + public static IEnumerable GetTruncatedJsonPackets() { + // Valid packet, but the json is invalid because it's truncated + for (int i = 1; i < validJson1.Length; i += 3) { + yield return MakePacketFromJson(validJson1.Substring(0, validJson1.Length - i), badContent: true); + } + } + + public static IEnumerable GetIncorrectContentLengthUnderreadPackets() { + // Full json is in the stream, but the header was corrupted and the + // Content-Length value is SMALLER than it should be, so the packet body + // will miss parts of the json at the end. + for (int i = 1; i < validJson1.Length; i += 3) { + var json = MakeBody(validJson1); + var headers = MakeHeaders(i); + yield return MakePacket(headers, json, badContent: true); + } + } + + public static IEnumerable GetIncorrectContentLengthOverreadPackets() { + // Full json is in the stream, but the header was corrupted and the + // Content-Length value is LARGER than it should be, so the packet body + // will include junk at the end from the next message. + var endJunk = MakeHeaders(5); + for (int i = 1; i < endJunk.Length; i++) { + var json = MakeBody(validJson1); + var headers = MakeHeaders(json.Length + i); + yield return MakePacket(headers, json, endJunk, badContent: true); + } + } + + public static IEnumerable GetIncorrectContentLengthOverreadEndOfStreamPackets() { + // Full json is in the stream, but the header was corrupted and the + // Content-Length value is LARGER than it should be, and there's no + // more data in the stream after this. + for (int i = 1; i < 5; i++) { + var json = MakeBody(validJson1); + var headers = MakeHeaders(json.Length + i); + yield return MakePacket(headers, json, badContent: true, blocked: true); + } + } + + private static Packet MakePacketFromJson(string json, bool badContent = false) { + var encoded = MakeBody(json); + var headers = MakeHeaders(encoded.Length); + + return MakePacket(headers, encoded, badContent: badContent); + } + + private static Packet MakePacket(byte[] headers, byte[] encoded, byte[] endJunk = null, bool badHeaders = false, bool badContent = false, bool blocked = false) { + return new Packet(headers, encoded, endJunk, badHeaders, badContent, blocked); + } + + private static byte[] MakeBody(string json) { + return Encoding.UTF8.GetBytes(json); + } + + private static byte[] MakeHeaders(int contentLength) { + return Encoding.ASCII.GetBytes(string.Format("Content-Length: {0}\r\n\r\n", contentLength)); + } + } + + class Packet { + private List _data = new List(); + public bool BadHeaders { get; } + public bool BadContent { get; } + public bool ReadPastEndOfStream { get; } + + public Packet(byte[] headers, byte[] content, byte[] endJunk = null, bool badHeaders = false, bool badContent = false, bool readPastEndOfStream = false) { + _data.AddRange(headers); + _data.AddRange(content); + if (endJunk != null) { + _data.AddRange(endJunk); + } + BadHeaders = badHeaders; + BadContent = badContent; + ReadPastEndOfStream = readPastEndOfStream; + } + + public Stream AsStream() { + var stream = new MemoryStream(); + var data = AsBytes(); + stream.Write(data, 0, data.Length); + stream.Flush(); + stream.Seek(0, SeekOrigin.Begin); + return stream; + } + + public byte[] AsBytes() { + return _data.ToArray(); + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/Properties/AssemblyInfo.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..c29e2e86b --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/Properties/AssemblyInfo.cs @@ -0,0 +1,25 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("IpcJsonTests")] +[assembly: AssemblyDescription("")] + +[assembly: ComVisible(false)] +[assembly: Guid("5D889F27-FE98-43AC-AE9F-509B47B2484D")]