Async Indexer for new LanguageServer

src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj

@@ -18,6 +18,7 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
+    <PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" Version="2.2.0" />
   </ItemGroup>
   <ItemGroup>
     <Content Include="get_search_paths.py">

src/Core/Impl/Extensions/EnumerableExtensions.cs

@@ -88,10 +88,30 @@ public static IEnumerable<int> IndexWhere<T>(this IEnumerable<T> source, Func<T,
                 i++;
             }
         }
+
+        public static IEnumerable<T> TraverseBreadthFirst<T>(this T root, Func<T, IEnumerable<T>> selectChildren)
+            => Enumerable.Repeat(root, 1).TraverseBreadthFirst(selectChildren);
+
+        public static IEnumerable<T> TraverseBreadthFirst<T>(this IEnumerable<T> roots, Func<T, IEnumerable<T>> selectChildren) {
+            var items = new Queue<T>(roots);
+            while (items.Count > 0) {
+                var item = items.Dequeue();
+                yield return item;
+
+                var children = selectChildren(item);
+                if (children == null) {
+                    continue;
+                }
+
+                foreach (var child in children) {
+                    items.Enqueue(child);
+                }
+            }
+        }
 
         public static Dictionary<TKey, TValue> ToDictionary<TSource, TKey, TValue>(this IEnumerable<TSource> source, Func<TSource, int, TKey> keySelector, Func<TSource, int, TValue> valueSelector) {
-            var dictionary = source is IReadOnlyCollection<TSource> collection 
-                ? new Dictionary<TKey, TValue>(collection.Count) 
+            var dictionary = source is IReadOnlyCollection<TSource> collection
+                ? new Dictionary<TKey, TValue>(collection.Count)
                 : new Dictionary<TKey, TValue>();
 
             var index = 0;
@@ -130,23 +150,5 @@ public static IEnumerable<T> TraverseDepthFirst<T>(this T root, Func<T, IEnumera
                 reverseChildren.Clear();
             }
         }
-
-        public static IEnumerable<T> TraverseBreadthFirst<T>(this T root, Func<T, IEnumerable<T>> selectChildren) {
-            var items = new Queue<T>();
-            items.Enqueue(root);
-            while (items.Count > 0) {
-                var item = items.Dequeue();
-                yield return item;
-
-                var children = selectChildren(item);
-                if (children == null) {
-                    continue;
-                }
-
-                foreach (var child in children) {
-                    items.Enqueue(child);
-                }
-            }
-        }
     }
 }

src/Core/Impl/Extensions/StringExtensions.cs

@@ -206,6 +206,8 @@ public static bool PathEquals(this string s, string other)
 
         public static bool EqualsOrdinal(this string s, int index, string other, int otherIndex, int length, bool ignoreCase = false)
             => string.Compare(s, index, other, otherIndex, length, ignoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal) == 0;
+        public static bool ContainsOrdinal(this string s, string value, bool ignoreCase = false)
+            => s.IndexOfOrdinal(value, ignoreCase: ignoreCase) != -1;
 
         public static int GetPathHashCode(this string s)
             => IgnoreCaseInPaths ? StringComparer.OrdinalIgnoreCase.GetHashCode(s) : StringComparer.Ordinal.GetHashCode(s);

src/Core/Impl/Extensions/TaskExtensions.cs

@@ -21,11 +21,11 @@
 
 namespace Microsoft.Python.Core {
     public static class TaskExtensions {
-        public static void SetCompletionResultTo<T>(this Task<T> task, TaskCompletionSourceEx<T> tcs) 
+        public static void SetCompletionResultTo<T>(this Task<T> task, TaskCompletionSource<T> tcs)
             => task.ContinueWith(SetCompletionResultToContinuation, tcs, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
-        
+
         private static void SetCompletionResultToContinuation<T>(Task<T> task, object state) {
-            var tcs = (TaskCompletionSourceEx<T>) state;
+            var tcs = (TaskCompletionSource<T>)state;
             switch (task.Status) {
                 case TaskStatus.RanToCompletion:
                     tcs.TrySetResult(task.Result);
@@ -34,7 +34,7 @@ private static void SetCompletionResultToContinuation<T>(Task<T> task, object st
                     try {
                         task.GetAwaiter().GetResult();
                     } catch (OperationCanceledException ex) {
-                        tcs.TrySetCanceled(ex);
+                        tcs.TrySetCanceled(ex.CancellationToken);
                     }
                     break;
                 case TaskStatus.Faulted:

src/Core/Impl/IO/DirectoryInfoProxy.cs

@@ -13,9 +13,12 @@
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
 
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using Microsoft.Extensions.FileSystemGlobbing;
+using Microsoft.Extensions.FileSystemGlobbing.Abstractions;
 
 namespace Microsoft.Python.Core.IO {
     public sealed class DirectoryInfoProxy : IDirectoryInfo {
@@ -40,6 +43,39 @@ public IEnumerable<IFileSystemInfo> EnumerateFileSystemInfos() => _directoryInfo
                 .EnumerateFileSystemInfos()
                 .Select(CreateFileSystemInfoProxy);
 
+        public IEnumerable<IFileSystemInfo> EnumerateFileSystemInfos(string[] includePatterns, string[] excludePatterns) {
+            var matcher = GetMatcher(includePatterns, excludePatterns);
+            PatternMatchingResult matchResult = SafeExecuteMatcher(matcher);
+            return matchResult.Files.Select((filePatternMatch) => {
+                var fileSystemInfo = _directoryInfo.GetFileSystemInfos(filePatternMatch.Stem).First();
+                return CreateFileSystemInfoProxy(fileSystemInfo);
+            });
+        }
+
+        public bool Match(string path, string[] includePatterns = default, string[] excludePatterns = default) {
+            var matcher = GetMatcher(includePatterns, excludePatterns);
+            return matcher.Match(FullName, path).HasMatches;
+        }
+
+        private static Matcher GetMatcher(string[] includePatterns, string[] excludePatterns) {
+            Matcher matcher = new Matcher();
+            matcher.AddIncludePatterns(includePatterns.IsNullOrEmpty() ? new[] { "**/*" } : includePatterns);
+            if (!excludePatterns.IsNullOrEmpty()) {
+                matcher.AddExcludePatterns(excludePatterns);
+            }
+            return matcher;
+        }
+
+        private PatternMatchingResult SafeExecuteMatcher(Matcher matcher) {
+            var directoryInfo = new DirectoryInfoWrapper(_directoryInfo);
+            for (var retries = 5; retries > 0; retries--) {
+                try {
+                    return matcher.Execute(directoryInfo);
+                } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) { }
+            }
+            return new PatternMatchingResult(Enumerable.Empty<FilePatternMatch>());
+        }
+
         private static IFileSystemInfo CreateFileSystemInfoProxy(FileSystemInfo fileSystemInfo)
             => fileSystemInfo is DirectoryInfo directoryInfo
                 ? (IFileSystemInfo)new DirectoryInfoProxy(directoryInfo)

src/Core/Impl/IO/FileSystem.cs

@@ -32,6 +32,7 @@ public long FileSize(string path) {
         public void FileWriteAllBytes(string path, byte[] bytes) => File.WriteAllBytes(path, bytes);
         public Stream CreateFile(string path) => File.Create(path);
         public Stream FileOpen(string path, FileMode mode) => File.Open(path, mode);
+        public Stream FileOpen(string path, FileMode mode, FileAccess access, FileShare share) => File.Open(path, mode, access, share);
         public bool DirectoryExists(string path) => Directory.Exists(path);
         public FileAttributes GetFileAttributes(string path) => File.GetAttributes(path);
         public void SetFileAttributes(string fullPath, FileAttributes attributes) => File.SetAttributes(fullPath, attributes);

src/Core/Impl/IO/IDirectoryInfo.cs

@@ -19,5 +19,7 @@ namespace Microsoft.Python.Core.IO {
     public interface IDirectoryInfo : IFileSystemInfo {
         IDirectoryInfo Parent { get; }
         IEnumerable<IFileSystemInfo> EnumerateFileSystemInfos();
+        IEnumerable<IFileSystemInfo> EnumerateFileSystemInfos(string[] includeFiles, string[] excludeFiles);
+        bool Match(string path, string[] includePatterns = default, string[] excludePatterns = default);
     }
 }

src/Core/Impl/IO/IFileSystem.cs

@@ -41,6 +41,7 @@ public interface IFileSystem {
 
         Stream CreateFile(string path);
         Stream FileOpen(string path, FileMode mode);
+        Stream FileOpen(string path, FileMode mode, FileAccess access, FileShare share);
 
         Version GetFileVersion(string path);
         void DeleteFile(string path);

src/Core/Impl/Microsoft.Python.Core.csproj

@@ -23,6 +23,7 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
+    <PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" Version="2.2.0" />
   </ItemGroup>
   <ItemGroup>
     <Compile Update="Resources.Designer.cs">

src/LanguageServer/Impl/Implementation/Server.Documents.cs

@@ -27,14 +27,17 @@ namespace Microsoft.Python.LanguageServer.Implementation {
     public sealed partial class Server {
         public void DidOpenTextDocument(DidOpenTextDocumentParams @params) {
             _disposableBag.ThrowIfDisposed();
-            _log?.Log(TraceEventType.Verbose, $"Opening document {@params.textDocument.uri}");
+            var uri = @params.textDocument.uri;
+            _log?.Log(TraceEventType.Verbose, $"Opening document {uri}");
 
-            _rdt.OpenDocument(@params.textDocument.uri, @params.textDocument.text);
+            var doc = _rdt.OpenDocument(uri, @params.textDocument.text);
+            _indexManager.ProcessNewFile(uri.AbsolutePath, doc);
         }
 
         public void DidChangeTextDocument(DidChangeTextDocumentParams @params) {
             _disposableBag.ThrowIfDisposed();
-            var doc = _rdt.GetDocument(@params.textDocument.uri);
+            var uri = @params.textDocument.uri;
+            var doc = _rdt.GetDocument(uri);
             if (doc != null) {
                 var changes = new List<DocumentChange>();
                 foreach (var c in @params.contentChanges) {
@@ -46,6 +49,7 @@ public void DidChangeTextDocument(DidChangeTextDocumentParams @params) {
                     changes.Add(change);
                 }
                 doc.Update(changes);
+                _indexManager.AddPendingDoc(doc);
             } else {
                 _log?.Log(TraceEventType.Warning, $"Unable to find document for {@params.textDocument.uri}");
             }
@@ -60,7 +64,9 @@ public void DidChangeWatchedFiles(DidChangeWatchedFilesParams @params) {
 
         public void DidCloseTextDocument(DidCloseTextDocumentParams @params) {
             _disposableBag.ThrowIfDisposed();
-            _rdt.CloseDocument(@params.textDocument.uri);
+            var uri = @params.textDocument.uri;
+            _rdt.CloseDocument(uri);
+            _indexManager.ProcessClosedFile(uri.AbsolutePath);
         }
 
         private Task<IDocumentAnalysis> GetAnalysisAsync(Uri uri, CancellationToken cancellationToken) {

src/LanguageServer/Impl/Implementation/Server.Symbols.cs

@@ -0,0 +1,127 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Python.Core;
+using Microsoft.Python.LanguageServer.Indexing;
+using Microsoft.Python.LanguageServer.Protocol;
+
+namespace Microsoft.Python.LanguageServer.Implementation {
+    public sealed partial class Server {
+        private static int _symbolHierarchyMaxSymbols = 1000;
+
+        public async Task<SymbolInformation[]> WorkspaceSymbols(WorkspaceSymbolParams @params, CancellationToken cancellationToken) {
+            var symbols = await _indexManager.WorkspaceSymbolsAsync(@params.query,
+                                                                    _symbolHierarchyMaxSymbols,
+                                                                    cancellationToken);
+            return symbols.Select(MakeSymbolInfo).ToArray();
+        }
+
+        public async Task<DocumentSymbol[]> HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) {
+            var path = @params.textDocument.uri.AbsolutePath;
+            var symbols = await _indexManager.HierarchicalDocumentSymbolsAsync(path, cancellationToken);
+            cancellationToken.ThrowIfCancellationRequested();
+            return symbols.Select(hSym => MakeDocumentSymbol(hSym)).ToArray();
+        }
+
+        private static SymbolInformation MakeSymbolInfo(FlatSymbol s) {
+            return new SymbolInformation {
+                name = s.Name,
+                kind = (Protocol.SymbolKind)s.Kind,
+                location = new Location {
+                    range = s.Range,
+                    uri = new Uri(s.DocumentPath),
+                },
+                containerName = s.ContainerName,
+            };
+        }
+
+        private DocumentSymbol MakeDocumentSymbol(HierarchicalSymbol hSym) {
+            return new DocumentSymbol {
+                name = hSym.Name,
+                detail = hSym.Detail,
+                kind = ToSymbolKind(hSym.Kind),
+                deprecated = hSym.Deprecated ?? false,
+                range = hSym.Range,
+                selectionRange = hSym.SelectionRange,
+                children = hSym.Children.MaybeEnumerate().Select(MakeDocumentSymbol).ToArray(),
+            };
+        }
+
+        private Protocol.SymbolKind ToSymbolKind(Indexing.SymbolKind kind) {
+            switch (kind) {
+                case Indexing.SymbolKind.None:
+                    return Protocol.SymbolKind.None;
+                case Indexing.SymbolKind.File:
+                    return Protocol.SymbolKind.File;
+                case Indexing.SymbolKind.Module:
+                    return Protocol.SymbolKind.Module;
+                case Indexing.SymbolKind.Namespace:
+                    return Protocol.SymbolKind.Namespace;
+                case Indexing.SymbolKind.Package:
+                    return Protocol.SymbolKind.Package;
+                case Indexing.SymbolKind.Class:
+                    return Protocol.SymbolKind.Class;
+                case Indexing.SymbolKind.Method:
+                    return Protocol.SymbolKind.Method;
+                case Indexing.SymbolKind.Property:
+                    return Protocol.SymbolKind.Property;
+                case Indexing.SymbolKind.Field:
+                    return Protocol.SymbolKind.Field;
+                case Indexing.SymbolKind.Constructor:
+                    return Protocol.SymbolKind.Constructor;
+                case Indexing.SymbolKind.Enum:
+                    return Protocol.SymbolKind.Enum;
+                case Indexing.SymbolKind.Interface:
+                    return Protocol.SymbolKind.Interface;
+                case Indexing.SymbolKind.Function:
+                    return Protocol.SymbolKind.Function;
+                case Indexing.SymbolKind.Variable:
+                    return Protocol.SymbolKind.Variable;
+                case Indexing.SymbolKind.Constant:
+                    return Protocol.SymbolKind.Constant;
+                case Indexing.SymbolKind.String:
+                    return Protocol.SymbolKind.String;
+                case Indexing.SymbolKind.Number:
+                    return Protocol.SymbolKind.Number;
+                case Indexing.SymbolKind.Boolean:
+                    return Protocol.SymbolKind.Boolean;
+                case Indexing.SymbolKind.Array:
+                    return Protocol.SymbolKind.Array;
+                case Indexing.SymbolKind.Object:
+                    return Protocol.SymbolKind.Object;
+                case Indexing.SymbolKind.Key:
+                    return Protocol.SymbolKind.Key;
+                case Indexing.SymbolKind.Null:
+                    return Protocol.SymbolKind.Null;
+                case Indexing.SymbolKind.EnumMember:
+                    return Protocol.SymbolKind.EnumMember;
+                case Indexing.SymbolKind.Struct:
+                    return Protocol.SymbolKind.Struct;
+                case Indexing.SymbolKind.Event:
+                    return Protocol.SymbolKind.Event;
+                case Indexing.SymbolKind.Operator:
+                    return Protocol.SymbolKind.Operator;
+                case Indexing.SymbolKind.TypeParameter:
+                    return Protocol.SymbolKind.TypeParameter;
+                default:
+                    throw new NotImplementedException($"{kind} is not a LSP's SymbolKind");
+            }
+        }
+    }
+}