diff --git a/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj b/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj index 805e064e4..3925c3a58 100644 --- a/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj +++ b/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj @@ -18,6 +18,7 @@ all runtime; build; native; contentfiles; analyzers + diff --git a/src/Core/Impl/Extensions/EnumerableExtensions.cs b/src/Core/Impl/Extensions/EnumerableExtensions.cs index 709b4ee43..5e40af9eb 100644 --- a/src/Core/Impl/Extensions/EnumerableExtensions.cs +++ b/src/Core/Impl/Extensions/EnumerableExtensions.cs @@ -88,10 +88,30 @@ public static IEnumerable IndexWhere(this IEnumerable source, Func TraverseBreadthFirst(this T root, Func> selectChildren) + => Enumerable.Repeat(root, 1).TraverseBreadthFirst(selectChildren); + + public static IEnumerable TraverseBreadthFirst(this IEnumerable roots, Func> selectChildren) { + var items = new Queue(roots); + while (items.Count > 0) { + var item = items.Dequeue(); + yield return item; + + var children = selectChildren(item); + if (children == null) { + continue; + } + + foreach (var child in children) { + items.Enqueue(child); + } + } + } public static Dictionary ToDictionary(this IEnumerable source, Func keySelector, Func valueSelector) { - var dictionary = source is IReadOnlyCollection collection - ? new Dictionary(collection.Count) + var dictionary = source is IReadOnlyCollection collection + ? new Dictionary(collection.Count) : new Dictionary(); var index = 0; @@ -130,23 +150,5 @@ public static IEnumerable TraverseDepthFirst(this T root, Func TraverseBreadthFirst(this T root, Func> selectChildren) { - var items = new Queue(); - items.Enqueue(root); - while (items.Count > 0) { - var item = items.Dequeue(); - yield return item; - - var children = selectChildren(item); - if (children == null) { - continue; - } - - foreach (var child in children) { - items.Enqueue(child); - } - } - } } } diff --git a/src/Core/Impl/Extensions/StringExtensions.cs b/src/Core/Impl/Extensions/StringExtensions.cs index 7eee22152..5f380aaac 100644 --- a/src/Core/Impl/Extensions/StringExtensions.cs +++ b/src/Core/Impl/Extensions/StringExtensions.cs @@ -206,6 +206,8 @@ public static bool PathEquals(this string s, string other) public static bool EqualsOrdinal(this string s, int index, string other, int otherIndex, int length, bool ignoreCase = false) => string.Compare(s, index, other, otherIndex, length, ignoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal) == 0; + public static bool ContainsOrdinal(this string s, string value, bool ignoreCase = false) + => s.IndexOfOrdinal(value, ignoreCase: ignoreCase) != -1; public static int GetPathHashCode(this string s) => IgnoreCaseInPaths ? StringComparer.OrdinalIgnoreCase.GetHashCode(s) : StringComparer.Ordinal.GetHashCode(s); diff --git a/src/Core/Impl/Extensions/TaskExtensions.cs b/src/Core/Impl/Extensions/TaskExtensions.cs index ca83cb836..0e39d3495 100644 --- a/src/Core/Impl/Extensions/TaskExtensions.cs +++ b/src/Core/Impl/Extensions/TaskExtensions.cs @@ -21,11 +21,11 @@ namespace Microsoft.Python.Core { public static class TaskExtensions { - public static void SetCompletionResultTo(this Task task, TaskCompletionSourceEx tcs) + public static void SetCompletionResultTo(this Task task, TaskCompletionSource tcs) => task.ContinueWith(SetCompletionResultToContinuation, tcs, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); - + private static void SetCompletionResultToContinuation(Task task, object state) { - var tcs = (TaskCompletionSourceEx) state; + var tcs = (TaskCompletionSource)state; switch (task.Status) { case TaskStatus.RanToCompletion: tcs.TrySetResult(task.Result); @@ -34,7 +34,7 @@ private static void SetCompletionResultToContinuation(Task task, object st try { task.GetAwaiter().GetResult(); } catch (OperationCanceledException ex) { - tcs.TrySetCanceled(ex); + tcs.TrySetCanceled(ex.CancellationToken); } break; case TaskStatus.Faulted: diff --git a/src/Core/Impl/IO/DirectoryInfoProxy.cs b/src/Core/Impl/IO/DirectoryInfoProxy.cs index 1295804bf..d2a7a34d9 100644 --- a/src/Core/Impl/IO/DirectoryInfoProxy.cs +++ b/src/Core/Impl/IO/DirectoryInfoProxy.cs @@ -13,9 +13,12 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; using System.IO; using System.Linq; +using Microsoft.Extensions.FileSystemGlobbing; +using Microsoft.Extensions.FileSystemGlobbing.Abstractions; namespace Microsoft.Python.Core.IO { public sealed class DirectoryInfoProxy : IDirectoryInfo { @@ -40,6 +43,39 @@ public IEnumerable EnumerateFileSystemInfos() => _directoryInfo .EnumerateFileSystemInfos() .Select(CreateFileSystemInfoProxy); + public IEnumerable EnumerateFileSystemInfos(string[] includePatterns, string[] excludePatterns) { + var matcher = GetMatcher(includePatterns, excludePatterns); + PatternMatchingResult matchResult = SafeExecuteMatcher(matcher); + return matchResult.Files.Select((filePatternMatch) => { + var fileSystemInfo = _directoryInfo.GetFileSystemInfos(filePatternMatch.Stem).First(); + return CreateFileSystemInfoProxy(fileSystemInfo); + }); + } + + public bool Match(string path, string[] includePatterns = default, string[] excludePatterns = default) { + var matcher = GetMatcher(includePatterns, excludePatterns); + return matcher.Match(FullName, path).HasMatches; + } + + private static Matcher GetMatcher(string[] includePatterns, string[] excludePatterns) { + Matcher matcher = new Matcher(); + matcher.AddIncludePatterns(includePatterns.IsNullOrEmpty() ? new[] { "**/*" } : includePatterns); + if (!excludePatterns.IsNullOrEmpty()) { + matcher.AddExcludePatterns(excludePatterns); + } + return matcher; + } + + private PatternMatchingResult SafeExecuteMatcher(Matcher matcher) { + var directoryInfo = new DirectoryInfoWrapper(_directoryInfo); + for (var retries = 5; retries > 0; retries--) { + try { + return matcher.Execute(directoryInfo); + } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) { } + } + return new PatternMatchingResult(Enumerable.Empty()); + } + private static IFileSystemInfo CreateFileSystemInfoProxy(FileSystemInfo fileSystemInfo) => fileSystemInfo is DirectoryInfo directoryInfo ? (IFileSystemInfo)new DirectoryInfoProxy(directoryInfo) diff --git a/src/Core/Impl/IO/FileSystem.cs b/src/Core/Impl/IO/FileSystem.cs index ad865dfaf..a88edb77c 100644 --- a/src/Core/Impl/IO/FileSystem.cs +++ b/src/Core/Impl/IO/FileSystem.cs @@ -32,6 +32,7 @@ public long FileSize(string path) { public void FileWriteAllBytes(string path, byte[] bytes) => File.WriteAllBytes(path, bytes); public Stream CreateFile(string path) => File.Create(path); public Stream FileOpen(string path, FileMode mode) => File.Open(path, mode); + public Stream FileOpen(string path, FileMode mode, FileAccess access, FileShare share) => File.Open(path, mode, access, share); public bool DirectoryExists(string path) => Directory.Exists(path); public FileAttributes GetFileAttributes(string path) => File.GetAttributes(path); public void SetFileAttributes(string fullPath, FileAttributes attributes) => File.SetAttributes(fullPath, attributes); diff --git a/src/Core/Impl/IO/IDirectoryInfo.cs b/src/Core/Impl/IO/IDirectoryInfo.cs index bb97fb3f2..0ee8bed59 100644 --- a/src/Core/Impl/IO/IDirectoryInfo.cs +++ b/src/Core/Impl/IO/IDirectoryInfo.cs @@ -19,5 +19,7 @@ namespace Microsoft.Python.Core.IO { public interface IDirectoryInfo : IFileSystemInfo { IDirectoryInfo Parent { get; } IEnumerable EnumerateFileSystemInfos(); + IEnumerable EnumerateFileSystemInfos(string[] includeFiles, string[] excludeFiles); + bool Match(string path, string[] includePatterns = default, string[] excludePatterns = default); } } diff --git a/src/Core/Impl/IO/IFileSystem.cs b/src/Core/Impl/IO/IFileSystem.cs index ecdb55065..509c7c282 100644 --- a/src/Core/Impl/IO/IFileSystem.cs +++ b/src/Core/Impl/IO/IFileSystem.cs @@ -41,6 +41,7 @@ public interface IFileSystem { Stream CreateFile(string path); Stream FileOpen(string path, FileMode mode); + Stream FileOpen(string path, FileMode mode, FileAccess access, FileShare share); Version GetFileVersion(string path); void DeleteFile(string path); diff --git a/src/Core/Impl/Microsoft.Python.Core.csproj b/src/Core/Impl/Microsoft.Python.Core.csproj index 6bcb47038..f41668645 100644 --- a/src/Core/Impl/Microsoft.Python.Core.csproj +++ b/src/Core/Impl/Microsoft.Python.Core.csproj @@ -23,6 +23,7 @@ all runtime; build; native; contentfiles; analyzers + diff --git a/src/LanguageServer/Impl/Implementation/Server.Documents.cs b/src/LanguageServer/Impl/Implementation/Server.Documents.cs index 2d359eb10..6305645d7 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Documents.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Documents.cs @@ -27,14 +27,17 @@ namespace Microsoft.Python.LanguageServer.Implementation { public sealed partial class Server { public void DidOpenTextDocument(DidOpenTextDocumentParams @params) { _disposableBag.ThrowIfDisposed(); - _log?.Log(TraceEventType.Verbose, $"Opening document {@params.textDocument.uri}"); + var uri = @params.textDocument.uri; + _log?.Log(TraceEventType.Verbose, $"Opening document {uri}"); - _rdt.OpenDocument(@params.textDocument.uri, @params.textDocument.text); + var doc = _rdt.OpenDocument(uri, @params.textDocument.text); + _indexManager.ProcessNewFile(uri.AbsolutePath, doc); } public void DidChangeTextDocument(DidChangeTextDocumentParams @params) { _disposableBag.ThrowIfDisposed(); - var doc = _rdt.GetDocument(@params.textDocument.uri); + var uri = @params.textDocument.uri; + var doc = _rdt.GetDocument(uri); if (doc != null) { var changes = new List(); foreach (var c in @params.contentChanges) { @@ -46,6 +49,7 @@ public void DidChangeTextDocument(DidChangeTextDocumentParams @params) { changes.Add(change); } doc.Update(changes); + _indexManager.AddPendingDoc(doc); } else { _log?.Log(TraceEventType.Warning, $"Unable to find document for {@params.textDocument.uri}"); } @@ -60,7 +64,9 @@ public void DidChangeWatchedFiles(DidChangeWatchedFilesParams @params) { public void DidCloseTextDocument(DidCloseTextDocumentParams @params) { _disposableBag.ThrowIfDisposed(); - _rdt.CloseDocument(@params.textDocument.uri); + var uri = @params.textDocument.uri; + _rdt.CloseDocument(uri); + _indexManager.ProcessClosedFile(uri.AbsolutePath); } private Task GetAnalysisAsync(Uri uri, CancellationToken cancellationToken) { diff --git a/src/LanguageServer/Impl/Implementation/Server.Symbols.cs b/src/LanguageServer/Impl/Implementation/Server.Symbols.cs new file mode 100644 index 000000000..09a000228 --- /dev/null +++ b/src/LanguageServer/Impl/Implementation/Server.Symbols.cs @@ -0,0 +1,127 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Core; +using Microsoft.Python.LanguageServer.Indexing; +using Microsoft.Python.LanguageServer.Protocol; + +namespace Microsoft.Python.LanguageServer.Implementation { + public sealed partial class Server { + private static int _symbolHierarchyMaxSymbols = 1000; + + public async Task WorkspaceSymbols(WorkspaceSymbolParams @params, CancellationToken cancellationToken) { + var symbols = await _indexManager.WorkspaceSymbolsAsync(@params.query, + _symbolHierarchyMaxSymbols, + cancellationToken); + return symbols.Select(MakeSymbolInfo).ToArray(); + } + + public async Task HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { + var path = @params.textDocument.uri.AbsolutePath; + var symbols = await _indexManager.HierarchicalDocumentSymbolsAsync(path, cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + return symbols.Select(hSym => MakeDocumentSymbol(hSym)).ToArray(); + } + + private static SymbolInformation MakeSymbolInfo(FlatSymbol s) { + return new SymbolInformation { + name = s.Name, + kind = (Protocol.SymbolKind)s.Kind, + location = new Location { + range = s.Range, + uri = new Uri(s.DocumentPath), + }, + containerName = s.ContainerName, + }; + } + + private DocumentSymbol MakeDocumentSymbol(HierarchicalSymbol hSym) { + return new DocumentSymbol { + name = hSym.Name, + detail = hSym.Detail, + kind = ToSymbolKind(hSym.Kind), + deprecated = hSym.Deprecated ?? false, + range = hSym.Range, + selectionRange = hSym.SelectionRange, + children = hSym.Children.MaybeEnumerate().Select(MakeDocumentSymbol).ToArray(), + }; + } + + private Protocol.SymbolKind ToSymbolKind(Indexing.SymbolKind kind) { + switch (kind) { + case Indexing.SymbolKind.None: + return Protocol.SymbolKind.None; + case Indexing.SymbolKind.File: + return Protocol.SymbolKind.File; + case Indexing.SymbolKind.Module: + return Protocol.SymbolKind.Module; + case Indexing.SymbolKind.Namespace: + return Protocol.SymbolKind.Namespace; + case Indexing.SymbolKind.Package: + return Protocol.SymbolKind.Package; + case Indexing.SymbolKind.Class: + return Protocol.SymbolKind.Class; + case Indexing.SymbolKind.Method: + return Protocol.SymbolKind.Method; + case Indexing.SymbolKind.Property: + return Protocol.SymbolKind.Property; + case Indexing.SymbolKind.Field: + return Protocol.SymbolKind.Field; + case Indexing.SymbolKind.Constructor: + return Protocol.SymbolKind.Constructor; + case Indexing.SymbolKind.Enum: + return Protocol.SymbolKind.Enum; + case Indexing.SymbolKind.Interface: + return Protocol.SymbolKind.Interface; + case Indexing.SymbolKind.Function: + return Protocol.SymbolKind.Function; + case Indexing.SymbolKind.Variable: + return Protocol.SymbolKind.Variable; + case Indexing.SymbolKind.Constant: + return Protocol.SymbolKind.Constant; + case Indexing.SymbolKind.String: + return Protocol.SymbolKind.String; + case Indexing.SymbolKind.Number: + return Protocol.SymbolKind.Number; + case Indexing.SymbolKind.Boolean: + return Protocol.SymbolKind.Boolean; + case Indexing.SymbolKind.Array: + return Protocol.SymbolKind.Array; + case Indexing.SymbolKind.Object: + return Protocol.SymbolKind.Object; + case Indexing.SymbolKind.Key: + return Protocol.SymbolKind.Key; + case Indexing.SymbolKind.Null: + return Protocol.SymbolKind.Null; + case Indexing.SymbolKind.EnumMember: + return Protocol.SymbolKind.EnumMember; + case Indexing.SymbolKind.Struct: + return Protocol.SymbolKind.Struct; + case Indexing.SymbolKind.Event: + return Protocol.SymbolKind.Event; + case Indexing.SymbolKind.Operator: + return Protocol.SymbolKind.Operator; + case Indexing.SymbolKind.TypeParameter: + return Protocol.SymbolKind.TypeParameter; + default: + throw new NotImplementedException($"{kind} is not a LSP's SymbolKind"); + } + } + } +} diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs deleted file mode 100644 index b5f658a06..000000000 --- a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Python.Analysis.Types; -using Microsoft.Python.LanguageServer.Protocol; - -namespace Microsoft.Python.LanguageServer.Implementation { - public sealed partial class Server { - private static int _symbolHierarchyDepthLimit = 10; - private static int _symbolHierarchyMaxSymbols = 1000; - - public async Task WorkspaceSymbols(WorkspaceSymbolParams @params, CancellationToken cancellationToken) { - return Array.Empty< SymbolInformation>(); - } - - public async Task DocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { - return Array.Empty(); - } - - public async Task HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { - return Array.Empty(); - } - - - private static SymbolKind ToSymbolKind(PythonMemberType memberType) { - switch (memberType) { - case PythonMemberType.Unknown: return SymbolKind.None; - case PythonMemberType.Class: return SymbolKind.Class; - case PythonMemberType.Instance: return SymbolKind.Variable; - case PythonMemberType.Function: return SymbolKind.Function; - case PythonMemberType.Method: return SymbolKind.Method; - case PythonMemberType.Module: return SymbolKind.Module; - case PythonMemberType.Property: return SymbolKind.Property; - case PythonMemberType.Union: return SymbolKind.Object; - case PythonMemberType.Variable: return SymbolKind.Variable; - case PythonMemberType.Generic: return SymbolKind.TypeParameter; - default: return SymbolKind.None; - } - } - } -} diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index a0241f6ce..63121ff0a 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -25,11 +25,13 @@ using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core; using Microsoft.Python.Core.Disposables; +using Microsoft.Python.Core.Idle; using Microsoft.Python.Core.IO; using Microsoft.Python.Core.Logging; using Microsoft.Python.Core.Services; using Microsoft.Python.LanguageServer.Completion; using Microsoft.Python.LanguageServer.Diagnostics; +using Microsoft.Python.LanguageServer.Indexing; using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.LanguageServer.Sources; @@ -43,6 +45,7 @@ public sealed partial class Server : IDisposable { private IRunningDocumentTable _rdt; private ClientCapabilities _clientCaps; private ILogger _log; + private IIndexManager _indexManager; public static InformationDisplayOptions DisplayOptions { get; private set; } = new InformationDisplayOptions { preferredFormat = MarkupKind.PlainText, @@ -120,6 +123,14 @@ public async Task InitializeAsync(InitializeParams @params, Ca _interpreter = await PythonInterpreter.CreateAsync(configuration, rootDir, _services, cancellationToken); _services.AddService(_interpreter); + var fileSystem = _services.GetService(); + _indexManager = new IndexManager(fileSystem, _interpreter.LanguageVersion, rootDir, + @params.initializationOptions.includeFiles, + @params.initializationOptions.excludeFiles, + _services.GetService()); + _services.AddService(_indexManager); + _disposableBag.Add(_indexManager); + DisplayStartupInfo(); // TODO: Pass different documentation sources to completion/hover/signature @@ -171,7 +182,6 @@ private bool HandleConfigurationChanges(ServerSettings newSettings) { var oldSettings = Settings; Settings = newSettings; - _symbolHierarchyDepthLimit = Settings.analysis.symbolsHierarchyDepthLimit; _symbolHierarchyMaxSymbols = Settings.analysis.symbolsHierarchyMaxSymbols; if (oldSettings == null) { diff --git a/src/LanguageServer/Impl/Indexing/IIndexManager.cs b/src/LanguageServer/Impl/Indexing/IIndexManager.cs new file mode 100644 index 000000000..0451efc42 --- /dev/null +++ b/src/LanguageServer/Impl/Indexing/IIndexManager.cs @@ -0,0 +1,31 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Documents; + +namespace Microsoft.Python.LanguageServer.Indexing { + internal interface IIndexManager : IDisposable { + void ProcessNewFile(string path, IDocument doc); + void ProcessClosedFile(string path); + void ReIndexFile(string path, IDocument doc); + void AddPendingDoc(IDocument doc); + Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken cancellationToken = default); + Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken cancellationToken = default); + } +} diff --git a/src/LanguageServer/Impl/Indexing/IIndexParser.cs b/src/LanguageServer/Impl/Indexing/IIndexParser.cs new file mode 100644 index 000000000..7330ef521 --- /dev/null +++ b/src/LanguageServer/Impl/Indexing/IIndexParser.cs @@ -0,0 +1,25 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Indexing { + internal interface IIndexParser : IDisposable { + Task ParseAsync(string path, CancellationToken cancellationToken = default); + } +} diff --git a/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs new file mode 100644 index 000000000..6e64ea40a --- /dev/null +++ b/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs @@ -0,0 +1,29 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Documents; + +namespace Microsoft.Python.LanguageServer.Indexing { + interface IMostRecentDocumentSymbols : IDisposable { + void Parse(); + void Index(IDocument doc); + Task> GetSymbolsAsync(CancellationToken ct = default); + void MarkAsPending(); + } +} diff --git a/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs new file mode 100644 index 000000000..e7cf22459 --- /dev/null +++ b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs @@ -0,0 +1,32 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Documents; + +namespace Microsoft.Python.LanguageServer.Indexing { + internal interface ISymbolIndex : IDisposable { + Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken ct = default); + Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken ct = default); + void Add(string path, IDocument doc); + void Parse(string path); + void Delete(string path); + void ReIndex(string path, IDocument doc); + void MarkAsPending(string path); + } +} diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs new file mode 100644 index 000000000..e6a8dfcfc --- /dev/null +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -0,0 +1,142 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Core.Interpreter; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Core.Diagnostics; +using Microsoft.Python.Core.Disposables; +using Microsoft.Python.Core.Idle; +using Microsoft.Python.Core.IO; +using Microsoft.Python.Parsing; + +namespace Microsoft.Python.LanguageServer.Indexing { + internal class IndexManager : IIndexManager { + private const int DefaultReIndexDelay = 350; + private readonly ISymbolIndex _symbolIndex; + private readonly IFileSystem _fileSystem; + private readonly string _workspaceRootPath; + private readonly string[] _includeFiles; + private readonly string[] _excludeFiles; + private readonly DisposableBag _disposables = new DisposableBag(nameof(IndexManager)); + private readonly ConcurrentDictionary _pendingDocs = new ConcurrentDictionary(new UriDocumentComparer()); + + public IndexManager(IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, + string[] excludeFiles, IIdleTimeService idleTimeService) { + Check.ArgumentNotNull(nameof(fileSystem), fileSystem); + Check.ArgumentNotNull(nameof(rootPath), rootPath); + Check.ArgumentNotNull(nameof(includeFiles), includeFiles); + Check.ArgumentNotNull(nameof(excludeFiles), excludeFiles); + Check.ArgumentNotNull(nameof(idleTimeService), idleTimeService); + + _fileSystem = fileSystem; + _workspaceRootPath = rootPath; + _includeFiles = includeFiles; + _excludeFiles = excludeFiles; + + _symbolIndex = new SymbolIndex(_fileSystem, version); + idleTimeService.Idle += OnIdle; + + _disposables + .Add(_symbolIndex) + .Add(() => idleTimeService.Idle -= OnIdle); + + StartAddRootDir(); + } + + public int ReIndexingDelay { get; set; } = DefaultReIndexDelay; + + private void StartAddRootDir() { + foreach (var fileInfo in WorkspaceFiles()) { + if (ModulePath.IsPythonSourceFile(fileInfo.FullName)) { + _symbolIndex.Parse(fileInfo.FullName); + } + } + } + + private IEnumerable WorkspaceFiles() { + if (string.IsNullOrEmpty(_workspaceRootPath)) { + return Enumerable.Empty(); + } + return _fileSystem.GetDirectoryInfo(_workspaceRootPath).EnumerateFileSystemInfos(_includeFiles, _excludeFiles); + } + + public void ProcessClosedFile(string path) { + if (IsFileOnWorkspace(path)) { + _symbolIndex.Parse(path); + } else { + _symbolIndex.Delete(path); + } + } + + private bool IsFileOnWorkspace(string path) { + if (string.IsNullOrEmpty(_workspaceRootPath)) { + return false; + } + return _fileSystem.GetDirectoryInfo(_workspaceRootPath) + .Match(path, _includeFiles, _excludeFiles); + } + + public void ProcessNewFile(string path, IDocument doc) { + _symbolIndex.Add(path, doc); + } + + public void ReIndexFile(string path, IDocument doc) { + _symbolIndex.ReIndex(path, doc); + } + + public void Dispose() { + _disposables.TryDispose(); + } + + public Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken cancellationToken = default) { + return _symbolIndex.HierarchicalDocumentSymbolsAsync(path, cancellationToken); + } + + public Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken cancellationToken = default) { + return _symbolIndex.WorkspaceSymbolsAsync(query, maxLength, cancellationToken); + } + + public void AddPendingDoc(IDocument doc) { + _pendingDocs.TryAdd(doc, DateTime.Now); + _symbolIndex.MarkAsPending(doc.Uri.AbsolutePath); + } + + private void OnIdle(object sender, EventArgs _) { + ReIndexPendingDocsAsync(); + } + + private void ReIndexPendingDocsAsync() { + foreach (var (doc, lastTime) in _pendingDocs) { + if ((DateTime.Now - lastTime).TotalMilliseconds > ReIndexingDelay) { + ReIndexFile(doc.Uri.AbsolutePath, doc); + _pendingDocs.TryRemove(doc, out var _); + } + } + } + + private class UriDocumentComparer : IEqualityComparer { + public bool Equals(IDocument x, IDocument y) => x.Uri.Equals(y.Uri); + + public int GetHashCode(IDocument obj) => obj.Uri.GetHashCode(); + } + } + +} diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs new file mode 100644 index 000000000..c79371de1 --- /dev/null +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -0,0 +1,78 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Diagnostics; +using Microsoft.Python.Core.Disposables; +using Microsoft.Python.Core.IO; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Indexing { + internal sealed class IndexParser : IIndexParser { + private DisposableBag disposables = new DisposableBag(nameof(IndexParser)); + private const int MaxConcurrentParsings = 10; + private readonly IFileSystem _fileSystem; + private readonly PythonLanguageVersion _version; + private readonly SemaphoreSlim _semaphore; + private readonly CancellationTokenSource _allProcessingCts = new CancellationTokenSource(); + + public IndexParser(IFileSystem fileSystem, PythonLanguageVersion version) { + Check.ArgumentNotNull(nameof(fileSystem), fileSystem); + + _fileSystem = fileSystem; + _version = version; + _semaphore = new SemaphoreSlim(MaxConcurrentParsings); + + disposables + .Add(_semaphore) + .Add(() => { + _allProcessingCts.Cancel(); + _allProcessingCts.Dispose(); + }); + } + + public Task ParseAsync(string path, CancellationToken cancellationToken = default) { + var linkedParseCts = + CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, cancellationToken); + var parseTask = Parse(path, linkedParseCts.Token); + parseTask.ContinueWith(_ => linkedParseCts.Dispose()).DoNotWait(); + return parseTask; + } + + private async Task Parse(string path, CancellationToken parseCt) { + await _semaphore.WaitAsync(parseCt); + PythonAst ast; + try { + using (var stream = _fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read)) { + var parser = Parser.CreateParser(stream, _version); + ast = parser.ParseFile(); + } + } finally { + _semaphore.Release(); + } + + parseCt.ThrowIfCancellationRequested(); + return ast; + } + + public void Dispose() { + disposables.TryDispose(); + } + } +} diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs new file mode 100644 index 000000000..277267d09 --- /dev/null +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -0,0 +1,160 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Diagnostics; + +namespace Microsoft.Python.LanguageServer.Indexing { + class MostRecentDocumentSymbols : IMostRecentDocumentSymbols { + private readonly object _syncObj = new object(); + private readonly IIndexParser _indexParser; + private readonly string _path; + + private CancellationTokenSource _fileCts = new CancellationTokenSource(); + + private TaskCompletionSource> _fileTcs = new TaskCompletionSource>(); + private WorkQueueState state = WorkQueueState.WaitingForWork; + + public MostRecentDocumentSymbols(string path, IIndexParser indexParser) { + _path = path; + _indexParser = indexParser; + } + + public void Parse() { + WorkAndSetTcs(ParseAsync); + } + + public void Index(IDocument doc) { + WorkAndSetTcs(ct => IndexAsync(doc, ct)); + } + + public void WorkAndSetTcs(Func>> asyncWork) { + CancellationTokenSource currentCts; + TaskCompletionSource> currentTcs; + lock (_syncObj) { + switch (state) { + case WorkQueueState.Working: + CancelExistingWork(); + RenewTcs(); + break; + case WorkQueueState.WaitingForWork: + break; + case WorkQueueState.FinishedWork: + RenewTcs(); + break; + default: + throw new InvalidOperationException(); + } + state = WorkQueueState.Working; + currentCts = _fileCts; + currentTcs = _fileTcs; + } + + DoWork(currentCts, asyncWork).SetCompletionResultTo(currentTcs); + } + + private async Task> DoWork(CancellationTokenSource tcs, Func>> asyncWork) { + var token = tcs.Token; + + try { + return await asyncWork(token); + } finally { + lock (_syncObj) { + tcs.Dispose(); + if (!token.IsCancellationRequested) { + state = WorkQueueState.FinishedWork; + } + } + } + } + + public Task> GetSymbolsAsync(CancellationToken ct = default) { + TaskCompletionSource> currentTcs; + lock (_syncObj) { + currentTcs = _fileTcs; + } + return currentTcs.Task.ContinueWith(t => t.GetAwaiter().GetResult(), ct); + } + + public void MarkAsPending() { + lock (_syncObj) { + switch (state) { + case WorkQueueState.WaitingForWork: + break; + case WorkQueueState.Working: + CancelExistingWork(); + RenewTcs(); + break; + case WorkQueueState.FinishedWork: + RenewTcs(); + break; + default: + throw new InvalidOperationException(); + } + state = WorkQueueState.WaitingForWork; + } + } + + public void Dispose() { + lock (_syncObj) { + switch (state) { + case WorkQueueState.Working: + CancelExistingWork(); + break; + case WorkQueueState.WaitingForWork: + CancelExistingWork(); + // Manually cancel tcs, in case any task is awaiting + _fileTcs.TrySetCanceled(); + break; + case WorkQueueState.FinishedWork: + break; + default: + throw new InvalidOperationException(); + } + state = WorkQueueState.FinishedWork; + } + } + + private async Task> IndexAsync(IDocument doc, + CancellationToken indexCt) { + var ast = await doc.GetAstAsync(indexCt); + indexCt.ThrowIfCancellationRequested(); + var walker = new SymbolIndexWalker(ast); + ast.Walk(walker); + return walker.Symbols; + } + + private async Task> ParseAsync(CancellationToken parseCancellationToken) { + try { + var ast = await _indexParser.ParseAsync(_path, parseCancellationToken); + parseCancellationToken.ThrowIfCancellationRequested(); + var walker = new SymbolIndexWalker(ast); + ast.Walk(walker); + return walker.Symbols; + } catch (Exception e) when (e is IOException || e is UnauthorizedAccessException) { + Trace.TraceError(e.Message); + } + + return new List(); + } + + private void RenewTcs() { + Check.InvalidOperation(Monitor.IsEntered(_syncObj)); + _fileCts = new CancellationTokenSource(); + _fileTcs = new TaskCompletionSource>(); + } + + private void CancelExistingWork() { + Check.InvalidOperation(Monitor.IsEntered(_syncObj)); + _fileCts.Cancel(); + } + + /* It's easier to think of it as a queue of work + * but it maintains only one item at a time in the queue */ + private enum WorkQueueState { WaitingForWork, Working, FinishedWork }; + } +} diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs new file mode 100644 index 000000000..0680c563a --- /dev/null +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -0,0 +1,114 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Disposables; +using Microsoft.Python.Core.IO; +using Microsoft.Python.Parsing; + +namespace Microsoft.Python.LanguageServer.Indexing { + internal sealed class SymbolIndex : ISymbolIndex { + private readonly DisposableBag _disposables = new DisposableBag(nameof(SymbolIndex)); + private readonly ConcurrentDictionary _index; + private readonly IIndexParser _indexParser; + + public SymbolIndex(IFileSystem fileSystem, PythonLanguageVersion version) { + var comparer = PathEqualityComparer.Instance; + _index = new ConcurrentDictionary(comparer); + _indexParser = new IndexParser(fileSystem, version); + _disposables + .Add(_indexParser) + .Add(() => { + foreach (var recentSymbols in _index.Values) { + recentSymbols.Dispose(); + } + }); + } + + public Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken ct = default) { + if (_index.TryGetValue(path, out var mostRecentSymbols)) { + return mostRecentSymbols.GetSymbolsAsync(ct); + } else { + return Task.FromResult>(new List()); + } + } + + public async Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken ct = default) { + var results = new List(); + foreach (var filePathAndRecent in _index) { + var symbols = await filePathAndRecent.Value.GetSymbolsAsync(ct); + results.AddRange(WorkspaceSymbolsQuery(filePathAndRecent.Key, query, symbols).Take(maxLength - results.Count)); + ct.ThrowIfCancellationRequested(); + if (results.Count >= maxLength) { + break; + } + } + return results; + } + + public void Add(string path, IDocument doc) { + _index.GetOrAdd(path, MakeMostRecentDocSymbols(path)).Index(doc); + } + + public void Parse(string path) { + _index.GetOrAdd(path, MakeMostRecentDocSymbols(path)).Parse(); + } + + public void Delete(string path) { + _index.Remove(path, out var mostRecentDocSymbols); + mostRecentDocSymbols.Dispose(); + } + + public void ReIndex(string path, IDocument doc) { + if (_index.TryGetValue(path, out var currentSymbols)) { + currentSymbols.Index(doc); + } + } + + public void MarkAsPending(string path) { + _index[path].MarkAsPending(); + } + + private IEnumerable WorkspaceSymbolsQuery(string path, string query, + IReadOnlyList symbols) { + var rootSymbols = DecorateWithParentsName(symbols, null); + var treeSymbols = rootSymbols.TraverseBreadthFirst((symAndPar) => { + var sym = symAndPar.symbol; + return DecorateWithParentsName((sym.Children ?? Enumerable.Empty()).ToList(), sym.Name); + }); + return treeSymbols.Where(sym => sym.symbol.Name.ContainsOrdinal(query, ignoreCase: true)) + .Select(sym => new FlatSymbol(sym.symbol.Name, sym.symbol.Kind, path, sym.symbol.SelectionRange, sym.parentName)); + } + + private static IEnumerable<(HierarchicalSymbol symbol, string parentName)> DecorateWithParentsName( + IEnumerable symbols, string parentName) { + return symbols.Select((symbol) => (symbol, parentName)).ToList(); + } + + private IMostRecentDocumentSymbols MakeMostRecentDocSymbols(string path) { + return new MostRecentDocumentSymbols(path, _indexParser); + } + + public void Dispose() { + _disposables.TryDispose(); + } + } +} diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs new file mode 100644 index 000000000..0f9d1d1f3 --- /dev/null +++ b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs @@ -0,0 +1,347 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Indexing { + internal class SymbolIndexWalker : PythonWalker { + private static readonly Regex DoubleUnderscore = new Regex(@"^__.*__$", RegexOptions.Compiled); + private static readonly Regex ConstantLike = new Regex(@"^[\p{Lu}\p{N}_]+$", RegexOptions.Compiled); + + private readonly PythonAst _ast; + private readonly SymbolStack _stack = new SymbolStack(); + + public SymbolIndexWalker(PythonAst ast) { + _ast = ast; + } + + public IReadOnlyList Symbols => _stack.Root; + + public override bool Walk(ClassDefinition node) { + _stack.Enter(SymbolKind.Class); + node.Body?.Walk(this); + var children = _stack.Exit(); + + _stack.AddSymbol(new HierarchicalSymbol( + node.Name, + SymbolKind.Class, + node.GetSpan(_ast), + node.NameExpression.GetSpan(_ast), + children, + FunctionKind.Class + )); + + return false; + } + + public override bool Walk(FunctionDefinition node) { + _stack.Enter(SymbolKind.Function); + foreach (var p in node.Parameters) { + AddVarSymbol(p.NameExpression); + } + node.Body?.Walk(this); + var children = _stack.Exit(); + + var span = node.GetSpan(_ast); + + var ds = new HierarchicalSymbol( + node.Name, + SymbolKind.Function, + span, + node.IsLambda ? span : node.NameExpression.GetSpan(_ast), + children, + FunctionKind.Function + ); + + if (_stack.Parent == SymbolKind.Class) { + switch (ds.Name) { + case "__init__": + ds.Kind = SymbolKind.Constructor; + break; + case var name when DoubleUnderscore.IsMatch(name): + ds.Kind = SymbolKind.Operator; + break; + default: + ds.Kind = SymbolKind.Method; + + if (node.Decorators != null) { + foreach (var dec in node.Decorators.Decorators) { + var maybeKind = DecoratorExpressionToKind(dec); + if (maybeKind.HasValue) { + ds.Kind = maybeKind.Value.kind; + ds._functionKind = maybeKind.Value.functionKind; + break; + } + } + } + + break; + } + } + + _stack.AddSymbol(ds); + + return false; + } + + public override bool Walk(ImportStatement node) { + foreach (var (nameNode, nameString) in node.Names.Zip(node.AsNames, (name, asName) => asName != null ? (asName, asName.Name) : ((Node)name, name.MakeString()))) { + var span = nameNode.GetSpan(_ast); + _stack.AddSymbol(new HierarchicalSymbol(nameString, SymbolKind.Module, span)); + } + + return false; + } + + public override bool Walk(FromImportStatement node) { + if (node.IsFromFuture) { + return false; + } + + foreach (var name in node.Names.Zip(node.AsNames, (name, asName) => asName ?? name)) { + var span = name.GetSpan(_ast); + _stack.AddSymbol(new HierarchicalSymbol(name.Name, SymbolKind.Module, span)); + } + + return false; + } + + public override bool Walk(AssignmentStatement node) { + node.Right?.Walk(this); + foreach (var exp in node.Left) { + switch (exp) { + case ExpressionWithAnnotation ewa when ewa.Expression is NameExpression ne: + AddVarSymbol(ne); + break; + case NameExpression ne: + AddVarSymbol(ne); + break; + } + } + + return false; + } + + public override bool Walk(AugmentedAssignStatement node) { + node.Right?.Walk(this); + AddVarSymbol(node.Left as NameExpression); + return false; + } + + public override bool Walk(IfStatement node) { + WalkAndDeclareAll(node.Tests); + WalkAndDeclare(node.ElseStatement); + + return false; + } + + public override bool Walk(TryStatement node) { + WalkAndDeclare(node.Body); + WalkAndDeclareAll(node.Handlers); + WalkAndDeclare(node.Else); + WalkAndDeclare(node.Finally); + + return false; + } + + public override bool Walk(ForStatement node) { + _stack.EnterDeclared(); + AddVarSymbolRecursive(node.Left); + node.List?.Walk(this); + node.Body?.Walk(this); + _stack.ExitDeclaredAndMerge(); + + _stack.EnterDeclared(); + node.Else?.Walk(this); + _stack.ExitDeclaredAndMerge(); + + return false; + } + + public override bool Walk(ComprehensionFor node) { + AddVarSymbolRecursive(node.Left); + return base.Walk(node); + } + + public override bool Walk(ListComprehension node) { + _stack.Enter(SymbolKind.None); + return base.Walk(node); + } + + public override void PostWalk(ListComprehension node) => ExitComprehension(node); + + public override bool Walk(DictionaryComprehension node) { + _stack.Enter(SymbolKind.None); + return base.Walk(node); + } + + public override void PostWalk(DictionaryComprehension node) => ExitComprehension(node); + + public override bool Walk(SetComprehension node) { + _stack.Enter(SymbolKind.None); + return base.Walk(node); + } + + public override void PostWalk(SetComprehension node) => ExitComprehension(node); + + public override bool Walk(GeneratorExpression node) { + _stack.Enter(SymbolKind.None); + return base.Walk(node); + } + + public override void PostWalk(GeneratorExpression node) => ExitComprehension(node); + + private void ExitComprehension(Comprehension node) { + var children = _stack.Exit(); + var span = node.GetSpan(_ast); + + _stack.AddSymbol(new HierarchicalSymbol( + $"<{node.NodeName}>", + SymbolKind.None, + span, + children: children + )); + } + + + private void AddVarSymbol(NameExpression node) { + if (node == null) { + return; + } + + var kind = SymbolKind.Variable; + + switch (node.Name) { + case "*": + case "_": + return; + case var s when (_stack.Parent == null || _stack.Parent == SymbolKind.Class) && ConstantLike.IsMatch(s): + kind = SymbolKind.Constant; + break; + } + + var span = node.GetSpan(_ast); + + _stack.AddSymbol(new HierarchicalSymbol(node.Name, kind, span)); + } + + private void AddVarSymbolRecursive(Expression node) { + if (node == null) { + return; + } + + switch (node) { + case NameExpression ne: + AddVarSymbol(ne); + return; + + case SequenceExpression se: + foreach (var item in se.Items.MaybeEnumerate()) { + AddVarSymbolRecursive(item); + } + return; + } + } + + private (SymbolKind kind, string functionKind)? DecoratorExpressionToKind(Expression exp) { + switch (exp) { + case NameExpression ne when NameIsProperty(ne.Name): + case MemberExpression me when NameIsProperty(me.Name): + return (SymbolKind.Property, FunctionKind.Property); + + case NameExpression ne when NameIsStaticMethod(ne.Name): + case MemberExpression me when NameIsStaticMethod(me.Name): + return (SymbolKind.Method, FunctionKind.StaticMethod); + + case NameExpression ne when NameIsClassMethod(ne.Name): + case MemberExpression me when NameIsClassMethod(me.Name): + return (SymbolKind.Method, FunctionKind.ClassMethod); + } + + return null; + } + + private bool NameIsProperty(string name) => + name == "property" + || name == "abstractproperty" + || name == "classproperty" + || name == "abstractclassproperty"; + + private bool NameIsStaticMethod(string name) => + name == "staticmethod" + || name == "abstractstaticmethod"; + + private bool NameIsClassMethod(string name) => + name == "classmethod" + || name == "abstractclassmethod"; + + private void WalkAndDeclare(Node node) { + if (node == null) { + return; + } + + _stack.EnterDeclared(); + node.Walk(this); + _stack.ExitDeclaredAndMerge(); + } + + private void WalkAndDeclareAll(IEnumerable nodes) { + foreach (var node in nodes.MaybeEnumerate()) { + WalkAndDeclare(node); + } + } + + private class SymbolStack { + private readonly Stack<(SymbolKind? kind, List symbols)> _symbols; + private readonly Stack> _declared = new Stack>(new[] { new HashSet() }); + + public List Root { get; } = new List(); + + public SymbolKind? Parent => _symbols.Peek().kind; + + public SymbolStack() { + _symbols = new Stack<(SymbolKind?, List)>(new (SymbolKind?, List)[] { (null, Root) }); + } + + public void Enter(SymbolKind parent) { + _symbols.Push((parent, new List())); + EnterDeclared(); + } + + public List Exit() { + ExitDeclared(); + return _symbols.Pop().symbols; + } + public void EnterDeclared() => _declared.Push(new HashSet()); + + public void ExitDeclared() => _declared.Pop(); + + public void ExitDeclaredAndMerge() => _declared.Peek().UnionWith(_declared.Pop()); + + public void AddSymbol(HierarchicalSymbol sym) { + if (sym.Kind == SymbolKind.Variable && _declared.Peek().Contains(sym.Name)) { + return; + } + + _symbols.Peek().symbols.Add(sym); + _declared.Peek().Add(sym.Name); + } + } + } +} diff --git a/src/LanguageServer/Impl/Indexing/Symbols.cs b/src/LanguageServer/Impl/Indexing/Symbols.cs new file mode 100644 index 000000000..0cdac218d --- /dev/null +++ b/src/LanguageServer/Impl/Indexing/Symbols.cs @@ -0,0 +1,112 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using Microsoft.Python.Core.Text; + +namespace Microsoft.Python.LanguageServer.Indexing { + // From LSP. + internal enum SymbolKind { + None = 0, + File = 1, + Module = 2, + Namespace = 3, + Package = 4, + Class = 5, + Method = 6, + Property = 7, + Field = 8, + Constructor = 9, + Enum = 10, + Interface = 11, + Function = 12, + Variable = 13, + Constant = 14, + String = 15, + Number = 16, + Boolean = 17, + Array = 18, + Object = 19, + Key = 20, + Null = 21, + EnumMember = 22, + Struct = 23, + Event = 24, + Operator = 25, + TypeParameter = 26 + } + + internal class FunctionKind { + public const string None = ""; + public const string Function = "function"; + public const string Property = "property"; + public const string StaticMethod = "staticmethod"; + public const string ClassMethod = "classmethod"; + public const string Class = "class"; + } + + // Analagous to LSP's DocumentSymbol. + internal class HierarchicalSymbol { + public string Name; + public string Detail; + public SymbolKind Kind; + public bool? Deprecated; + public SourceSpan Range; + public SourceSpan SelectionRange; + public IList Children; + + public string _functionKind; + + public HierarchicalSymbol( + string name, + SymbolKind kind, + SourceSpan range, + SourceSpan? selectionRange = null, + IList children = null, + string functionKind = FunctionKind.None + ) { + Name = name; + Kind = kind; + Range = range; + SelectionRange = selectionRange ?? range; + Children = children; + _functionKind = functionKind; + } + } + + // Analagous to LSP's SymbolInformation. + internal class FlatSymbol { + public string Name; + public SymbolKind Kind; + public bool? Deprecated; + public string DocumentPath; + public SourceSpan Range; + public string ContainerName; + + public FlatSymbol( + string name, + SymbolKind kind, + string documentPath, + SourceSpan range, + string containerName = null + ) { + Name = name; + Kind = kind; + DocumentPath = documentPath; + Range = range; + ContainerName = containerName; + } + } +} diff --git a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj index a462754b9..50cb57b0c 100644 --- a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj +++ b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj @@ -29,7 +29,7 @@ all runtime; build; native; contentfiles; analyzers - + diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs new file mode 100644 index 000000000..c0c3f85fb --- /dev/null +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -0,0 +1,364 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Core.Idle; +using Microsoft.Python.Core.IO; +using Microsoft.Python.Core.Services; +using Microsoft.Python.LanguageServer.Indexing; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using NSubstitute; +using TestUtilities; + +namespace Microsoft.Python.LanguageServer.Tests { + [TestClass] + public class IndexManagerTests : LanguageServerTestBase { + private readonly int maxSymbolsCount = 1000; + private const string _rootPath = "C:/root"; + + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() { + TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + } + + [TestCleanup] + public void Cleanup() { + TestEnvironmentImpl.TestCleanup(); + } + + [TestMethod, Priority(0)] + public async Task AddsRootDirectoryAsync() { + var context = new IndexTestContext(this); + context.FileWithXVarInRootDir(); + context.AddFileToRoot($"{_rootPath}\foo.py", MakeStream("y = 1")); + + var indexManager = context.GetDefaultIndexManager(); + + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + symbols.Should().HaveCount(2); + } + + [TestMethod, Priority(0)] + public void NullDirectoryThrowsException() { + var context = new IndexTestContext(this); + Action construct = () => { + PythonLanguageVersion version = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); + IIndexManager indexManager = new IndexManager(context.FileSystem, + version, null, new string[] { }, new string[] { }, + new IdleTimeService()); + }; + construct.Should().Throw(); + } + + [TestMethod, Priority(0)] + public async Task IgnoresNonPythonFiles() { + var context = new IndexTestContext(this); + + var nonPythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}/bla.txt"); + context.AddFileInfoToRootTestFS(nonPythonTestFileInfo); + + IIndexManager indexManager = context.GetDefaultIndexManager(); + await WaitForWorkspaceAddedAsync(indexManager); + + context.FileSystem.DidNotReceive().FileExists(nonPythonTestFileInfo.FullName); + } + + [TestMethod, Priority(0)] + public async Task CanOpenFiles() { + string nonRootPath = "C:/nonRoot"; + var context = new IndexTestContext(this); + var pythonTestFileInfo = MakeFileInfoProxy($"{nonRootPath}/bla.py"); + IDocument doc = DocumentWithAst("x = 1"); + + IIndexManager indexManager = context.GetDefaultIndexManager(); + indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); + + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + SymbolsShouldBeOnlyX(symbols); + } + + [TestMethod, Priority(0)] + public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { + var context = new IndexTestContext(this); + var pythonTestFilePath = context.FileWithXVarInRootDir(); + + var indexManager = context.GetDefaultIndexManager(); + await WaitForWorkspaceAddedAsync(indexManager); + + indexManager.ReIndexFile(pythonTestFilePath, DocumentWithAst("y = 1")); + + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("y"); + } + + [TestMethod, Priority(0)] + public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { + var context = new IndexTestContext(this); + var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); + context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false); + + var indexManager = context.GetDefaultIndexManager(); + indexManager.ProcessNewFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); + indexManager.ProcessClosedFile(pythonTestFileInfo.FullName); + + await SymbolIndexShouldBeEmpty(indexManager); + } + + [TestMethod, Priority(0)] + public async Task CloseWorkspaceFilesReUpdatesIndexAsync() { + var context = new IndexTestContext(this); + var pythonTestFilePath = context.FileWithXVarInRootDir(); + context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); + + var indexManager = context.GetDefaultIndexManager(); + await WaitForWorkspaceAddedAsync(indexManager); + + indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("r = 1")); + // It Needs to remake the stream for the file, previous one is closed + context.FileSystem.FileExists(pythonTestFilePath).Returns(true); + context.SetFileOpen(pythonTestFilePath, MakeStream("x = 1")); + context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); + indexManager.ProcessClosedFile(pythonTestFilePath); + + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + SymbolsShouldBeOnlyX(symbols); + } + + [TestMethod, Priority(0)] + public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() { + // If events get to index manager in the order: [open, close, update] + // it should not reindex file + + var context = new IndexTestContext(this); + var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); + + var indexManager = context.GetDefaultIndexManager(); + indexManager.ProcessNewFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); + indexManager.ProcessClosedFile(pythonTestFileInfo.FullName); + + context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false); + indexManager.ReIndexFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); + + await SymbolIndexShouldBeEmpty(indexManager); + } + + [TestMethod, Priority(0)] + public async Task WorkspaceSymbolsAddsRootDirectory() { + var context = new IndexTestContext(this); + + var pythonTestFilePath = context.FileWithXVarInRootDir(); + + var indexManager = context.GetDefaultIndexManager(); + + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + SymbolsShouldBeOnlyX(symbols); + } + + [TestMethod, Priority(0)] + public async Task WorkspaceSymbolsLimited() { + var context = new IndexTestContext(this); + + for (int fileNumber = 0; fileNumber < 10; fileNumber++) { + context.AddFileToRoot($"{_rootPath}\bla{fileNumber}.py", MakeStream($"x{fileNumber} = 1")); + } + var indexManager = context.GetDefaultIndexManager(); + + const int amountOfSymbols = 3; + + var symbols = await indexManager.WorkspaceSymbolsAsync("", amountOfSymbols); + symbols.Should().HaveCount(amountOfSymbols); + } + + [TestMethod, Priority(0)] + public async Task HierarchicalDocumentSymbolsAsync() { + var context = new IndexTestContext(this); + var pythonTestFilePath = context.FileWithXVarInRootDir(); + + var indexManager = context.GetDefaultIndexManager(); + + var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath); + SymbolsShouldBeOnlyX(symbols); + } + + [TestMethod, Priority(0)] + public async Task LatestVersionASTVersionIsIndexed() { + var context = new IndexTestContext(this); + var pythonTestFilePath = context.FileWithXVarInRootDir(); + + var indexManager = context.GetDefaultIndexManager(); + indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("y = 1")); + indexManager.ProcessClosedFile(pythonTestFilePath); + indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("z = 1")); + + var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath); + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("z"); + } + + [TestMethod, Priority(0)] + public async Task AddFilesToPendingChanges() { + var context = new IndexTestContext(this); + var f1 = context.AddFileToRoot($"{_rootPath}/fileA.py", MakeStream("")); + var f2 = context.AddFileToRoot($"{_rootPath}/fileB.py", MakeStream("")); + + var indexManager = context.GetDefaultIndexManager(); + await WaitForWorkspaceAddedAsync(indexManager); + + indexManager.AddPendingDoc(DocumentWithAst("y = 1", f1)); + indexManager.AddPendingDoc(DocumentWithAst("x = 1", f2)); + + context.SetIdleEvent(Raise.Event()); + + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + symbols.Should().HaveCount(2); + } + + private static void SymbolsShouldBeOnlyX(IEnumerable symbols) { + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("x"); + } + + private static void SymbolsShouldBeOnlyX(IEnumerable symbols) { + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("x"); + } + + + private class IndexTestContext : IDisposable { + private readonly List _rootFileList = new List(); + private readonly IIdleTimeService _idleTimeService = Substitute.For(); + private readonly PythonLanguageVersion _pythonLanguageVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); + private IIndexManager _indexM; + private IndexManagerTests _tests; + + public IndexTestContext(IndexManagerTests tests) { + _tests = tests; + Setup(); + } + + private void Setup() { + FileSystem = Substitute.For(); + SymbolIndex = new SymbolIndex(FileSystem, _pythonLanguageVersion); + SetupRootDir(); + } + + public IFileSystem FileSystem { get; private set; } + public SymbolIndex SymbolIndex { get; private set; } + + public void AddFileInfoToRootTestFS(FileInfoProxy fileInfo) { + _rootFileList.Add(fileInfo); + FileSystem.FileExists(fileInfo.FullName).Returns(true); + } + + public string FileWithXVarInRootDir() { + return AddFileToRoot($"{_rootPath}\bla.py", _tests.MakeStream("x = 1")); + } + + public IIndexManager GetDefaultIndexManager() { + _indexM = new IndexManager(FileSystem, _pythonLanguageVersion, + _rootPath, new string[] { }, new string[] { }, + _idleTimeService) { + ReIndexingDelay = 1 + }; + + return _indexM; + } + + public string AddFileToRoot(string filePath, Stream stream) { + var fileInfo = _tests.MakeFileInfoProxy(filePath); + AddFileInfoToRootTestFS(fileInfo); + SetFileOpen(fileInfo.FullName, stream); + // FileInfo fullName is used everywhere as path + // Otherwise, path discrepancies might appear + return fileInfo.FullName; + } + + public void SetIdleEvent(EventHandler handler) { + _idleTimeService.Idle += handler; + } + + private void SetupRootDir() { + var directoryInfo = Substitute.For(); + directoryInfo.Match("", new string[] { }, new string[] { }).ReturnsForAnyArgs(callInfo => { + string path = callInfo.ArgAt(0); + return _rootFileList + .Where(fsInfo => PathEqualityComparer.Instance.Equals(fsInfo.FullName, path)) + .Count() > 0; + }); + // Doesn't work without 'forAnyArgs' + directoryInfo.EnumerateFileSystemInfos(new string[] { }, new string[] { }).ReturnsForAnyArgs(_rootFileList); + FileSystem.GetDirectoryInfo(_rootPath).Returns(directoryInfo); + } + + public void Dispose() { + _indexM?.Dispose(); + } + + public void SetFileOpen(string pythonTestFilePath, Func returnFunc) { + FileSystem.FileOpen(pythonTestFilePath, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(returnFunc); + } + + internal void SetFileOpen(string path, Stream stream) { + FileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(stream); + } + } + + private IDocument DocumentWithAst(string testCode, string filePath = null) { + filePath = filePath ?? $"{_rootPath}/{testCode}.py"; + IDocument doc = Substitute.For(); + doc.GetAstAsync().ReturnsForAnyArgs(Task.FromResult(MakeAst(testCode))); + doc.Uri.Returns(new Uri(filePath)); + return doc; + } + + private static async Task WaitForWorkspaceAddedAsync(IIndexManager indexManager) { + await indexManager.WorkspaceSymbolsAsync("", 1000); + } + + private async Task SymbolIndexShouldBeEmpty(IIndexManager indexManager) { + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + symbols.Should().HaveCount(0); + } + + public PythonAst MakeAst(string testCode) { + PythonLanguageVersion latestVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); + return Parser.CreateParser(MakeStream(testCode), latestVersion).ParseFile(); + } + + public Stream MakeStream(string str) { + return new MemoryStream(Encoding.UTF8.GetBytes(str)); + } + + public FileInfoProxy MakeFileInfoProxy(string filePath) + => new FileInfoProxy(new FileInfo(filePath)); + } +} diff --git a/src/LanguageServer/Test/IndexParserTests.cs b/src/LanguageServer/Test/IndexParserTests.cs new file mode 100644 index 000000000..ecd86654d --- /dev/null +++ b/src/LanguageServer/Test/IndexParserTests.cs @@ -0,0 +1,119 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Core.IO; +using Microsoft.Python.LanguageServer.Indexing; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using NSubstitute; +using TestUtilities; + +namespace Microsoft.Python.LanguageServer.Tests { + + [TestClass] + public class IndexParserTests : LanguageServerTestBase { + private IFileSystem _fileSystem; + private PythonLanguageVersion _pythonLanguageVersion; + + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() { + TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + _fileSystem = Substitute.For(); + _pythonLanguageVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); + } + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod, Priority(0)] + public async Task ParseVariableInFileAsync() { + const string testFilePath = "C:/bla.py"; + _fileSystem.FileExists(testFilePath).Returns(true); + + using (var fileStream = MakeStream("x = 1")) { + SetFileOpen(_fileSystem, testFilePath, fileStream); + IIndexParser indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion); + var ast = await indexParser.ParseAsync(testFilePath); + + var symbols = GetIndexSymbols(ast); + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("x"); + } + + } + + private IReadOnlyList GetIndexSymbols(PythonAst ast) { + var walker = new SymbolIndexWalker(ast); + ast.Walk(walker); + return walker.Symbols; + } + + + [TestMethod, Priority(0)] + [ExpectedException(typeof(FileNotFoundException))] + public async Task ParseFileThatStopsExisting() { + const string testFilePath = "C:/bla.py"; + _fileSystem.FileExists(testFilePath).Returns(true); + SetFileOpen(_fileSystem, testFilePath, _ => throw new FileNotFoundException()); + + IIndexParser indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion); + var ast = await indexParser.ParseAsync(testFilePath); + } + + [TestMethod, Priority(0)] + public void CancelParsingAsync() { + const string testFilePath = "C:/bla.py"; + _fileSystem.FileExists(testFilePath).Returns(true); + + using (var fileStream = MakeStream("x = 1")) { + SetFileOpen(_fileSystem, testFilePath, fileStream); + } + + IIndexParser indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + cancellationTokenSource.Cancel(); + + Func parse = async () => { + await indexParser.ParseAsync(testFilePath, cancellationTokenSource.Token); + }; + parse.Should().Throw(); + } + + private void SetFileOpen(IFileSystem fileSystem, string path, Stream stream) { + fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(stream); + } + + private void SetFileOpen(IFileSystem fileSystem, string path, Func p) { + fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(p); + } + + private Stream MakeStream(string str) { + return new MemoryStream(Encoding.UTF8.GetBytes(str)); + } + } +} diff --git a/src/LanguageServer/Test/SymbolIndexTests.cs b/src/LanguageServer/Test/SymbolIndexTests.cs new file mode 100644 index 000000000..0a91ba2b0 --- /dev/null +++ b/src/LanguageServer/Test/SymbolIndexTests.cs @@ -0,0 +1,218 @@ +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Core.IO; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Indexing; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; +using Microsoft.Python.Parsing.Tests; +using Microsoft.Python.Tests.Utilities.FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using NSubstitute; +using TestUtilities; + +namespace Microsoft.Python.LanguageServer.Tests { + [TestClass] + public class SymbolIndexTests { + private const int maxSymbols = 1000; + private readonly string _rootPath = "C:/root"; + + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() { + TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + } + + [TestCleanup] + public void TestCleanup() { + TestEnvironmentImpl.TestCleanup(); + } + + [TestMethod, Priority(0)] + public async Task IndexHierarchicalDocumentAsync() { + ISymbolIndex index = MakeSymbolIndex(); + var path = TestData.GetDefaultModulePath(); + index.Add(path, DocumentWithAst("x = 1")); + + var symbols = await index.HierarchicalDocumentSymbolsAsync(path); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + private static ISymbolIndex MakeSymbolIndex() { + return new SymbolIndex(Substitute.For(), PythonLanguageVersion.V38); + } + + [TestMethod, Priority(0)] + public async Task IndexHierarchicalDocumentUpdate() { + ISymbolIndex index = MakeSymbolIndex(); + var path = TestData.GetDefaultModulePath(); + + index.Add(path, DocumentWithAst("x = 1")); + + var symbols = await index.HierarchicalDocumentSymbolsAsync(path); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + + index.Add(path, DocumentWithAst("y = 1")); + + symbols = await index.HierarchicalDocumentSymbolsAsync(path); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public async Task IndexHierarchicalDocumentNotFoundAsync() { + ISymbolIndex index = MakeSymbolIndex(); + var path = TestData.GetDefaultModulePath(); + + var symbols = await index.HierarchicalDocumentSymbolsAsync(path); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task IndexWorkspaceSymbolsFlattenAsync() { + var code = @"class Foo(object): + def foo(self, x): ..."; + + ISymbolIndex index = MakeSymbolIndex(); + var path = TestData.GetDefaultModulePath(); + + index.Add(path, DocumentWithAst(code)); + + var symbols = await index.WorkspaceSymbolsAsync("", maxSymbols); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("Foo", SymbolKind.Class, path, new SourceSpan(1, 7, 1, 10)), + new FlatSymbol("foo", SymbolKind.Method, path, new SourceSpan(2, 9, 2, 12), "Foo"), + new FlatSymbol("self", SymbolKind.Variable, path, new SourceSpan(2, 13, 2, 17), "foo"), + new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(2, 19, 2, 20), "foo"), + }); + } + + [TestMethod, Priority(0)] + public async Task IndexWorkspaceSymbolsFilteredAsync() { + var code = @"class Foo(object): + def foo(self, x): ..."; + + ISymbolIndex index = MakeSymbolIndex(); + var path = TestData.GetDefaultModulePath(); + + index.Add(path, DocumentWithAst(code)); + + var symbols = await index.WorkspaceSymbolsAsync("x", maxSymbols); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(2, 19, 2, 20), "foo"), + }); + } + + [TestMethod, Priority(0)] + public async Task IndexWorkspaceSymbolsNotFoundAsync() { + ISymbolIndex index = MakeSymbolIndex(); + var path = TestData.GetDefaultModulePath(); + + var symbols = await index.WorkspaceSymbolsAsync("", maxSymbols); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task IndexWorkspaceSymbolsCaseInsensitiveAsync() { + var code = @"class Foo(object): + def foo(self, x): ..."; + + ISymbolIndex index = MakeSymbolIndex(); + var path = TestData.GetDefaultModulePath(); + + index.Add(path, DocumentWithAst(code)); + + var symbols = await index.WorkspaceSymbolsAsync("foo", maxSymbols); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("Foo", SymbolKind.Class, path, new SourceSpan(1, 7, 1, 10)), + new FlatSymbol("foo", SymbolKind.Method, path, new SourceSpan(2, 9, 2, 12), "Foo"), + }); + } + + [TestMethod, Priority(0)] + public void MarkAsPendingWaitsForUpdates() { + ISymbolIndex index = MakeSymbolIndex(); + var path = TestData.GetDefaultModulePath(); + + index.Add(path, DocumentWithAst("x = 1")); + index.MarkAsPending(path); + var cts = new CancellationTokenSource(); + var t = index.HierarchicalDocumentSymbolsAsync(path, cts.Token); + t.IsCompleted.Should().BeFalse(); + cts.Cancel(); + Func cancelled = async () => { + await t; + }; + cancelled.Should().Throw(); + } + + [TestMethod, Priority(0)] + public async Task SymbolsAfterPendingWaitsForUpdateAsync() { + ISymbolIndex index = MakeSymbolIndex(); + var path = TestData.GetDefaultModulePath(); + + index.Add(path, DocumentWithAst("x = 1")); + index.MarkAsPending(path); + var t = index.WorkspaceSymbolsAsync("", maxSymbols); + index.ReIndex(path, DocumentWithAst("x = 1")); + var symbols = await t; + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(1, 1, 1, 2)), + }); + } + + private PythonAst GetParse(string code, PythonLanguageVersion version = PythonLanguageVersion.V37) + => Parser.CreateParser(new StringReader(code), version).ParseFile(); + + private IReadOnlyList WalkSymbols(string code, PythonLanguageVersion version = PythonLanguageVersion.V37) { + var ast = GetParse(code); + var walker = new SymbolIndexWalker(ast); + ast.Walk(walker); + return walker.Symbols; + } + + private IDocument DocumentWithAst(string testCode, string filePath = null) { + filePath = filePath ?? $"{_rootPath}/{testCode}.py"; + IDocument doc = Substitute.For(); + doc.GetAstAsync().ReturnsForAnyArgs(Task.FromResult(MakeAst(testCode))); + doc.Uri.Returns(new Uri(filePath)); + return doc; + } + + private PythonAst MakeAst(string testCode) { + PythonLanguageVersion latestVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); + return Parser.CreateParser(MakeStream(testCode), latestVersion).ParseFile(); + } + + private Stream MakeStream(string str) { + return new MemoryStream(Encoding.UTF8.GetBytes(str)); + } + + } +} diff --git a/src/LanguageServer/Test/SymbolIndexWalkerTests.cs b/src/LanguageServer/Test/SymbolIndexWalkerTests.cs new file mode 100644 index 000000000..fefb63601 --- /dev/null +++ b/src/LanguageServer/Test/SymbolIndexWalkerTests.cs @@ -0,0 +1,634 @@ +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.IO; +using FluentAssertions; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Indexing; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; +using Microsoft.Python.Tests.Utilities.FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; +namespace Microsoft.Python.LanguageServer.Tests { + [TestClass] + public class SymbolIndexWalkerTests { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() { + TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + } + + [TestCleanup] + public void TestCleanup() { + TestEnvironmentImpl.TestCleanup(); + } + + [TestMethod, Priority(0)] + public void WalkerAssignments() { + var code = @"x = 1 +y = x +z = y"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 1, 2, 2)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(3, 1, 3, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerMultipleAssignments() { + var code = @"x = y = z = 1"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 5, 1, 6)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(1, 9, 1, 10)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerUnderscore() { + var code = @"_ = 1"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public void WalkerIfStatement() { + var code = @"if foo(): + x = 1 +elif bar(): + x = 2 +else: + y = 3 +"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(4, 5, 4, 6)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(6, 5, 6, 6)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerTryExceptFinally() { + var code = @"try: + x = 1 +except Exception: + x = 2 +finally: + y = 3 +"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(4, 5, 4, 6)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(6, 5, 6, 6)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerReassign() { + var code = @"x = 1 +x = 2"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerAugmentedAssign() { + var code = @"x += 1"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerTopLevelConstant() { + var code = @"FOO_BAR_3 = 1234"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("FOO_BAR_3", SymbolKind.Constant, new SourceSpan(1, 1, 1, 10)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerFunction() { + var code = @"def func(x, y): + z = x + y + return z"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 3, 13), new SourceSpan(1, 5, 1, 9), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 10, 1, 11)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerFunctionStarredArgs() { + var code = @"def func(*args, **kwargs): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 31), new SourceSpan(1, 5, 1, 9), new[] { + new HierarchicalSymbol("args", SymbolKind.Variable, new SourceSpan(1, 11, 1, 15)), + new HierarchicalSymbol("kwargs", SymbolKind.Variable, new SourceSpan(1, 19, 1, 25)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerFunctionUnderscoreArg() { + var code = @"def func(_): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 17), new SourceSpan(1, 5, 1, 9), new List(), FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerImports() { + var code = @"import sys +import numpy as np +from os.path import join as osjoin +from os.path import ( join as osjoin2, exists as osexists, expanduser ) +"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("sys", SymbolKind.Module, new SourceSpan(1, 8, 1, 11)), + new HierarchicalSymbol("np", SymbolKind.Module, new SourceSpan(2, 17, 2, 19)), + new HierarchicalSymbol("osjoin", SymbolKind.Module, new SourceSpan(3, 29, 3, 35)), + new HierarchicalSymbol("osjoin2", SymbolKind.Module, new SourceSpan(4, 31, 4, 38)), + new HierarchicalSymbol("osexists", SymbolKind.Module, new SourceSpan(4, 50, 4, 58)), + new HierarchicalSymbol("expanduser", SymbolKind.Module, new SourceSpan(4, 60, 4, 70)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerImportFromFuture() { + var code = @"from __future__ import print_function"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public void WalkerClass() { + var code = @"class Foo(object): + ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 8), new SourceSpan(1, 7, 1, 10), new List(), FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassConstant() { + var code = @"class Foo(object): + CONSTANT = 1234"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 20), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("CONSTANT", SymbolKind.Constant, new SourceSpan(2, 5, 2, 13)), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerConstructor() { + var code = @"class Foo(object): + def __init__(self, x): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 31), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("__init__", SymbolKind.Constructor, new SourceSpan(2, 5, 2, 31), new SourceSpan(2, 9, 2, 17), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 18, 2, 22)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 24, 2, 25)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerMethod() { + var code = @"class Foo(object): + def foo(self, x): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 26), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("foo", SymbolKind.Method, new SourceSpan(2, 5, 2, 26), new SourceSpan(2, 9, 2, 12), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 13, 2, 17)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 19, 2, 20)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerDoubleUnderscoreMethod() { + var code = @"class Foo(object): + def __lt__(self, x): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 29), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("__lt__", SymbolKind.Operator, new SourceSpan(2, 5, 2, 29), new SourceSpan(2, 9, 2, 15), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 16, 2, 20)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 22, 2, 23)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerProperties() { + var code = @"class Foo(object): + @property + def func1(self): ... + + @abstractproperty + def func2(self): ... + + @classproperty + def func3(self): ... + + @abstractclassproperty + def func4(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 12, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func3", SymbolKind.Property, new SourceSpan(8, 5, 9, 25), new SourceSpan(9, 9, 9, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(9, 15, 9, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func4", SymbolKind.Property, new SourceSpan(11, 5, 12, 25), new SourceSpan(12, 9, 12, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(12, 15, 12, 19)), + }, FunctionKind.Property), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerAbcProperties() { + var code = @"class Foo(object): + @abc.abstractproperty + def func1(self): ... + + @abc.abstractclassproperty + def func2(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 6, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), + }, FunctionKind.Property), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerStaticMethods() { + var code = @"class Foo(object): + @staticmethod + def func1(arg): ... + + @abstractstaticmethod + def func2(arg): ... + + @abc.abstractstaticmethod + def func3(arg): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 9, 24), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 24), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(3, 15, 3, 18)), + }, FunctionKind.StaticMethod), + new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 24), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(6, 15, 6, 18)), + }, FunctionKind.StaticMethod), + new HierarchicalSymbol("func3", SymbolKind.Method, new SourceSpan(8, 5, 9, 24), new SourceSpan(9, 9, 9, 14), new[] { + new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(9, 15, 9, 18)), + }, FunctionKind.StaticMethod), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassMethods() { + var code = @"class Foo(object): + @classmethod + def func1(cls): ... + + @abstractclassmethod + def func2(cls): ... + + @abc.abstractclassmethod + def func3(cls): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 9, 24), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 24), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(3, 15, 3, 18)), + }, FunctionKind.ClassMethod), + new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 24), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(6, 15, 6, 18)), + }, FunctionKind.ClassMethod), + new HierarchicalSymbol("func3", SymbolKind.Method, new SourceSpan(8, 5, 9, 24), new SourceSpan(9, 9, 9, 14), new[] { + new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(9, 15, 9, 18)), + }, FunctionKind.ClassMethod), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerTopLevelFunctionDecorator() { + var code = @"@something +def func1(x, y): ... + +@something_else() +def func2(x, y): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func1", SymbolKind.Function, new SourceSpan(1, 1, 2, 21), new SourceSpan(2, 5, 2, 10), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 11, 2, 12)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 14, 2, 15)), + }, FunctionKind.Function), + new HierarchicalSymbol("func2", SymbolKind.Function, new SourceSpan(4, 1, 5, 21), new SourceSpan(5, 5, 5, 10), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(5, 11, 5, 12)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(5, 14, 5, 15)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassFunctionDecorator() { + var code = @"class Foo(object): + @something + def func1(self): ... + + @something_else() + def func2(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 6, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), + }, FunctionKind.Function), + new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassFunctionMultiDecorator() { + var code = @"class Foo(object): + @property + @something + def func1(self): ... + + @something + @property + def func2(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 8, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 4, 25), new SourceSpan(4, 9, 4, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(4, 15, 4, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(6, 5, 8, 25), new SourceSpan(8, 9, 8, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(8, 15, 8, 19)), + }, FunctionKind.Property), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerLambda() { + var code = @"f = lambda x, y: x + y"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.Function, new SourceSpan(1, 5, 1, 23), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 12, 1, 13)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 15, 1, 16)), + }, functionKind: FunctionKind.Function), + new HierarchicalSymbol("f", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerForLoop() { + var code = @"z = False +for [x, y, (p, q)] in [[1, 2, [3, 4]]]: + z += x +else: + z = None"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 6, 2, 7)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 9, 2, 10)), + new HierarchicalSymbol("p", SymbolKind.Variable, new SourceSpan(2, 13, 2, 14)), + new HierarchicalSymbol("q", SymbolKind.Variable, new SourceSpan(2, 16, 2, 17)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(3, 5, 3, 6)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(5, 5, 5, 6)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerListComprehension() { + var code = @"flat_list = [item for sublist in l for item in sublist]"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 13, 1, 56), children: new[] { + new HierarchicalSymbol("sublist", SymbolKind.Variable, new SourceSpan(1, 23, 1, 30)), + new HierarchicalSymbol("item", SymbolKind.Variable, new SourceSpan(1, 40, 1, 44)), + }), + new HierarchicalSymbol("flat_list", SymbolKind.Variable, new SourceSpan(1, 1, 1, 10)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerDictionaryComprehension() { + var code = @"d = { x: y for x, y in zip(range(10), range(10)) }"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 51), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 19, 1, 20)), + }), + new HierarchicalSymbol("d", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerSetComprehension() { + var code = @"s = { x for x in range(10) }"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 29), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + }), + new HierarchicalSymbol("s", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerGenerator() { + var code = @"g = (x + y for x, y in zip(range(10), range(10)))"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 50), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 19, 1, 20)), + }), + new HierarchicalSymbol("g", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerNestedListComprehension() { + var code = @"l = [ + x for x in [ + y for y in range(10) + ] +]"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 5, 2), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 11, 2, 12)), + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(2, 16, 4, 6), children: new[] { + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(3, 15, 3, 16)), + }), + }), + new HierarchicalSymbol("l", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerIncompleteFunction() { + var code = @"def func(x, y):"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 16), new SourceSpan(1, 5, 1, 9), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 10, 1, 11)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerIncompleteClass() { + var code = @"class Foo(object):"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 1, 19), new SourceSpan(1, 7, 1, 10), new List(), FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerIncompleteAssign() { + var code = @"x ="; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerAugmentedAssignLambda() { + var code = @"x += lambda x, y: x + y"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.Function, new SourceSpan(1, 6, 1, 24), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), + }, functionKind: FunctionKind.Function), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerAnnotatedAssignments() { + var code = @"x:int = 1"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)) + }); + } + + private PythonAst GetParse(string code, PythonLanguageVersion version = PythonLanguageVersion.V37) + => Parser.CreateParser(new StringReader(code), version).ParseFile(); + + private IReadOnlyList WalkSymbols(string code, PythonLanguageVersion version = PythonLanguageVersion.V37) { + var ast = GetParse(code); + var walker = new SymbolIndexWalker(ast); + ast.Walk(walker); + return walker.Symbols; + } + } +} diff --git a/src/UnitTests/Core/Impl/FluentAssertions/CollectionAssertionsExtensions.cs b/src/UnitTests/Core/Impl/FluentAssertions/CollectionAssertionsExtensions.cs index 4ddcf50d9..d91e6d1a0 100644 --- a/src/UnitTests/Core/Impl/FluentAssertions/CollectionAssertionsExtensions.cs +++ b/src/UnitTests/Core/Impl/FluentAssertions/CollectionAssertionsExtensions.cs @@ -35,5 +35,9 @@ public static AndConstraint OnlyContain( public static AndConstraint OnlyContain( this StringCollectionAssertions assertions, IReadOnlyCollection expected, string because = "", params object[] reasonArgs) => assertions.HaveCount(expected.Count, because, reasonArgs).And.Contain(expected, because, reasonArgs); + + public static AndConstraint> BeEquivalentToWithStrictOrdering( + this GenericCollectionAssertions assertions, IEnumerable expected, string because = "", params object[] reasonArgs) + => assertions.BeEquivalentTo(expected, options => options.WithStrictOrdering(), because, reasonArgs); } }