Async Indexer for new LanguageServer

src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj

@@ -18,6 +18,7 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
+    <PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" Version="2.2.0" />
   </ItemGroup>
   <ItemGroup>
     <Content Include="get_search_paths.py">

src/Core/Impl/Diagnostics/Check.cs

@@ -66,6 +66,13 @@ public static void InvalidOperation(Func<bool> predicate, string message = null)
             }
         }
 
+        [DebuggerStepThrough]
+        public static void InvalidOperation(bool condition, string message = null) {
+            if (!condition) {
+                throw new InvalidOperationException(message ?? string.Empty);
+            }
+        }
+
         [DebuggerStepThrough]
         public static void Argument(string argumentName, Func<bool> predicate) {
             if (!predicate()) {

src/Core/Impl/Extensions/EnumerableExtensions.cs

@@ -89,10 +89,13 @@ public static IEnumerable<int> IndexWhere<T>(this IEnumerable<T> source, Func<T,
             }
         }
 
-        public static IEnumerable<T> TraverseBreadthFirst<T>(this T root, Func<T, IEnumerable<T>> selectChildren) {
-            var items = new Queue<T>();
-            items.Enqueue(root);
-            while (items.Count > 0) {
+        public static IEnumerable<T> TraverseBreadthFirst<T>(this T root, Func<T, IEnumerable<T>> selectChildren)
+            => Enumerable.Repeat(root, 1).TraverseBreadthFirst(selectChildren);
+
+        public static IEnumerable<T> TraverseBreadthFirst<T>(this IEnumerable<T> roots, Func<T, IEnumerable<T>> selectChildren) {
+            var items = new Queue<T>(roots);
+
+            while (!items.IsNullOrEmpty()) {
                 var item = items.Dequeue();
                 yield return item;
 

src/Core/Impl/Extensions/StringExtensions.cs

@@ -192,6 +192,8 @@ public static bool EqualsOrdinal(this string s, string other)
 
         public static bool EqualsOrdinal(this string s, int index, string other, int otherIndex, int length, bool ignoreCase = false)
             => string.Compare(s, index, other, otherIndex, length, ignoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal) == 0;
+        public static bool ContainsOrdinal(this string s, string value, bool ignoreCase = false)
+            => s.IndexOfOrdinal(value, ignoreCase: ignoreCase) != -1;
 
         public static string[] Split(this string s, char separator, int startIndex, int length) {
             var count = 0;

src/Core/Impl/Extensions/TaskExtensions.cs

@@ -22,9 +22,9 @@
 namespace Microsoft.Python.Core {
     public static class TaskExtensions {
         public static void SetCompletionResultTo<T>(this Task<T> task, TaskCompletionSourceEx<T> tcs) 
-            => task.ContinueWith(SetCompletionResultToContinuation, tcs, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
+            => task.ContinueWith(SetCompletionResultToContinuationEx, tcs, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
 
-        private static void SetCompletionResultToContinuation<T>(Task<T> task, object state) {
+        private static void SetCompletionResultToContinuationEx<T>(Task<T> task, object state) {
             var tcs = (TaskCompletionSourceEx<T>) state;
             switch (task.Status) {
                 case TaskStatus.RanToCompletion:
@@ -45,6 +45,30 @@ private static void SetCompletionResultToContinuation<T>(Task<T> task, object st
             }
         }
 
+        public static void SetCompletionResultTo<T>(this Task<T> task, TaskCompletionSource<T> tcs)
+            => task.ContinueWith(SetCompletionResultToContinuation, tcs, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
+
+        private static void SetCompletionResultToContinuation<T>(Task<T> task, object state) {
+            var tcs = (TaskCompletionSource<T>)state;
+            switch (task.Status) {
+                case TaskStatus.RanToCompletion:
+                    tcs.TrySetResult(task.Result);
+                    break;
+                case TaskStatus.Canceled:
+                    try {
+                        task.GetAwaiter().GetResult();
+                    } catch (OperationCanceledException ex) {
+                        tcs.TrySetCanceled(ex.CancellationToken);
+                    }
+                    break;
+                case TaskStatus.Faulted:
+                    tcs.TrySetException(task.Exception);
+                    break;
+                default:
+                    throw new ArgumentOutOfRangeException();
+            }
+        }
+
         /// <summary>
         /// Suppresses warnings about unawaited tasks and rethrows task exceptions back to the callers synchronization context if it is possible
         /// </summary>

src/Core/Impl/IO/DirectoryInfoProxy.cs

@@ -13,9 +13,12 @@
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
 
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using Microsoft.Extensions.FileSystemGlobbing;
+using Microsoft.Extensions.FileSystemGlobbing.Abstractions;
 
 namespace Microsoft.Python.Core.IO {
     public sealed class DirectoryInfoProxy : IDirectoryInfo {
@@ -40,6 +43,27 @@ public IEnumerable<IFileSystemInfo> EnumerateFileSystemInfos() => _directoryInfo
                 .EnumerateFileSystemInfos()
                 .Select(CreateFileSystemInfoProxy);
 
+        public IEnumerable<IFileSystemInfo> EnumerateFileSystemInfos(string[] includeFiles, string[] excludeFiles) {
+            Matcher matcher = new Matcher();
+            matcher.AddIncludePatterns(includeFiles.IsNullOrEmpty() ? new[] { "**/*" } : includeFiles);
+            matcher.AddExcludePatterns(excludeFiles ?? Enumerable.Empty<string>());
+            PatternMatchingResult matchResult = SafeExecuteMatcher(matcher);
+            return matchResult.Files.Select((filePatternMatch) => {
+                var fileSystemInfo = _directoryInfo.GetFileSystemInfos(filePatternMatch.Stem).First();
+                return CreateFileSystemInfoProxy(fileSystemInfo);
+            });
+        }
+
+        private PatternMatchingResult SafeExecuteMatcher(Matcher matcher) {
+            var directoryInfo = new DirectoryInfoWrapper(_directoryInfo);
+            for (var retries = 5; retries > 0; retries--) {
+                try {
+                    return matcher.Execute(directoryInfo);
+                } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) { }
+            }
+            return new PatternMatchingResult(Enumerable.Empty<FilePatternMatch>());
+        }
+
         private static IFileSystemInfo CreateFileSystemInfoProxy(FileSystemInfo fileSystemInfo)
             => fileSystemInfo is DirectoryInfo directoryInfo
                 ? (IFileSystemInfo)new DirectoryInfoProxy(directoryInfo)

src/Core/Impl/IO/FileSystem.cs

@@ -31,7 +31,8 @@ public long FileSize(string path) {
         public byte[] FileReadAllBytes(string path) => File.ReadAllBytes(path);
         public void FileWriteAllBytes(string path, byte[] bytes) => File.WriteAllBytes(path, bytes);
         public Stream CreateFile(string path) => File.Create(path);
-        public Stream FileOpen(string path, FileMode mode) => File.Open(path, mode);
+        public Stream FileOpen(string path, FileMode mode) => FileOpen(path, mode);
+        public Stream FileOpen(string path, FileMode mode, FileAccess access, FileShare share) => File.Open(path, mode, access, share);
         public bool DirectoryExists(string path) => Directory.Exists(path);
         public FileAttributes GetFileAttributes(string path) => File.GetAttributes(path);
         public void SetFileAttributes(string fullPath, FileAttributes attributes) => File.SetAttributes(fullPath, attributes);

src/Core/Impl/IO/IDirectoryInfo.cs

@@ -19,5 +19,6 @@ namespace Microsoft.Python.Core.IO {
     public interface IDirectoryInfo : IFileSystemInfo {
         IDirectoryInfo Parent { get; }
         IEnumerable<IFileSystemInfo> EnumerateFileSystemInfos();
+        IEnumerable<IFileSystemInfo> EnumerateFileSystemInfos(string[] includeFiles, string[] excludeFiles);
     }
 }

src/Core/Impl/IO/IFileSystem.cs

@@ -41,6 +41,7 @@ public interface IFileSystem {
 
         Stream CreateFile(string path);
         Stream FileOpen(string path, FileMode mode);
+        Stream FileOpen(string path, FileMode mode, FileAccess access, FileShare share);
 
         Version GetFileVersion(string path);
         void DeleteFile(string path);

src/Core/Impl/Microsoft.Python.Core.csproj

@@ -23,6 +23,7 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
+    <PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" Version="2.2.0" />
   </ItemGroup>
   <ItemGroup>
     <Compile Update="Resources.Designer.cs">

src/LanguageServer/Impl/Implementation/Server.Documents.cs

@@ -16,27 +16,27 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
 using System.Threading;
-using System.Threading.Tasks;
 using Microsoft.Python.Analysis;
 using Microsoft.Python.Analysis.Documents;
 using Microsoft.Python.Core;
-using Microsoft.Python.Core.Text;
 using Microsoft.Python.LanguageServer.Protocol;
 
 namespace Microsoft.Python.LanguageServer.Implementation {
     public sealed partial class Server {
         public void DidOpenTextDocument(DidOpenTextDocumentParams @params) {
             _disposableBag.ThrowIfDisposed();
-            _log?.Log(TraceEventType.Verbose, $"Opening document {@params.textDocument.uri}");
+            var uri = @params.textDocument.uri;
+            _log?.Log(TraceEventType.Verbose, $"Opening document {uri}");
 
-            _rdt.OpenDocument(@params.textDocument.uri, @params.textDocument.text);
+            var doc = _rdt.OpenDocument(uri, @params.textDocument.text);
+            _indexManager.ProcessNewFile(uri.AbsolutePath, doc);
         }
 
         public void DidChangeTextDocument(DidChangeTextDocumentParams @params) {
             _disposableBag.ThrowIfDisposed();
-            var doc = _rdt.GetDocument(@params.textDocument.uri);
+            var uri = @params.textDocument.uri;
+            var doc = _rdt.GetDocument(uri);
             if (doc != null) {
                 var changes = new List<DocumentChange>();
                 foreach (var c in @params.contentChanges) {
@@ -48,6 +48,7 @@ public void DidChangeTextDocument(DidChangeTextDocumentParams @params) {
                     changes.Add(change);
                 }
                 doc.Update(changes);
+                _indexManager.AddPendingDoc(doc);
             } else {
                 _log?.Log(TraceEventType.Warning, $"Unable to find document for {@params.textDocument.uri}");
             }
@@ -62,7 +63,9 @@ public void DidChangeWatchedFiles(DidChangeWatchedFilesParams @params) {
 
         public void DidCloseTextDocument(DidCloseTextDocumentParams @params) {
             _disposableBag.ThrowIfDisposed();
-            _rdt.CloseDocument(@params.textDocument.uri);
+            var uri = @params.textDocument.uri;
+            _rdt.CloseDocument(uri);
+            _indexManager.ProcessClosedFile(uri.AbsolutePath);
         }
 
         private IDocumentAnalysis GetAnalysis(Uri uri, CancellationToken cancellationToken) {

src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs

@@ -14,9 +14,11 @@
 // permissions and limitations under the License.
 
 using System;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Core;
+using Microsoft.Python.LanguageServer.Indexing;
 using Microsoft.Python.LanguageServer.Protocol;
 
 namespace Microsoft.Python.LanguageServer.Implementation {
@@ -25,31 +27,100 @@ public sealed partial class Server {
         private static int _symbolHierarchyMaxSymbols = 1000;
 
         public async Task<SymbolInformation[]> WorkspaceSymbols(WorkspaceSymbolParams @params, CancellationToken cancellationToken) {
-            return Array.Empty< SymbolInformation>();
+            var symbols = await _indexManager.WorkspaceSymbolsAsync(@params.query,
+                                                                    _symbolHierarchyMaxSymbols,
+                                                                    cancellationToken);
+            return symbols.Select(MakeSymbolInfo).ToArray();
         }
 
-        public async Task<SymbolInformation[]> DocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) {
-            return Array.Empty<SymbolInformation>();
+        public async Task<DocumentSymbol[]> HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) {
+            var path = @params.textDocument.uri.AbsolutePath;
+            var symbols = await _indexManager.HierarchicalDocumentSymbolsAsync(path, cancellationToken);
+            return symbols.MaybeEnumerate().Select(hSym => MakeDocumentSymbol(hSym)).ToArray();
         }
 
-        public async Task<DocumentSymbol[]> HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) {
-            return Array.Empty<DocumentSymbol>();
+        private static SymbolInformation MakeSymbolInfo(Indexing.FlatSymbol s) {
+            return new SymbolInformation {
+                name = s.Name,
+                kind = (Protocol.SymbolKind)s.Kind,
+                location = new Location {
+                    range = s.Range,
+                    uri = new Uri(s.DocumentPath),
+                },
+                containerName = s.ContainerName,
+            };
         }
 
+        private DocumentSymbol MakeDocumentSymbol(HierarchicalSymbol hSym) {
+            return new DocumentSymbol {
+                name = hSym.Name,
+                detail = hSym.Detail,
+                kind = ToSymbolKind(hSym.Kind),
+                deprecated = hSym.Deprecated ?? false,
+                range = hSym.Range,
+                selectionRange = hSym.SelectionRange,
+                children = hSym.Children.MaybeEnumerate().Select(MakeDocumentSymbol).ToArray(),
+            };
+        }
 
-        private static SymbolKind ToSymbolKind(PythonMemberType memberType) {
-            switch (memberType) {
-                case PythonMemberType.Unknown: return SymbolKind.None;
-                case PythonMemberType.Class: return SymbolKind.Class;
-                case PythonMemberType.Instance: return SymbolKind.Variable;
-                case PythonMemberType.Function: return SymbolKind.Function;
-                case PythonMemberType.Method: return SymbolKind.Method;
-                case PythonMemberType.Module: return SymbolKind.Module;
-                case PythonMemberType.Property: return SymbolKind.Property;
-                case PythonMemberType.Union: return SymbolKind.Object;
-                case PythonMemberType.Variable: return SymbolKind.Variable;
-                case PythonMemberType.Generic: return SymbolKind.TypeParameter;
-                default: return SymbolKind.None;
+        private Protocol.SymbolKind ToSymbolKind(Indexing.SymbolKind kind) {
+            switch (kind) {
+                case Indexing.SymbolKind.None:
+                    return Protocol.SymbolKind.None;
+                case Indexing.SymbolKind.File:
+                    return Protocol.SymbolKind.File;
+                case Indexing.SymbolKind.Module:
+                    return Protocol.SymbolKind.Module;
+                case Indexing.SymbolKind.Namespace:
+                    return Protocol.SymbolKind.Namespace;
+                case Indexing.SymbolKind.Package:
+                    return Protocol.SymbolKind.Package;
+                case Indexing.SymbolKind.Class:
+                    return Protocol.SymbolKind.Class;
+                case Indexing.SymbolKind.Method:
+                    return Protocol.SymbolKind.Method;
+                case Indexing.SymbolKind.Property:
+                    return Protocol.SymbolKind.Property;
+                case Indexing.SymbolKind.Field:
+                    return Protocol.SymbolKind.Field;
+                case Indexing.SymbolKind.Constructor:
+                    return Protocol.SymbolKind.Constructor;
+                case Indexing.SymbolKind.Enum:
+                    return Protocol.SymbolKind.Enum;
+                case Indexing.SymbolKind.Interface:
+                    return Protocol.SymbolKind.Interface;
+                case Indexing.SymbolKind.Function:
+                    return Protocol.SymbolKind.Function;
+                case Indexing.SymbolKind.Variable:
+                    return Protocol.SymbolKind.Variable;
+                case Indexing.SymbolKind.Constant:
+                    return Protocol.SymbolKind.Constant;
+                case Indexing.SymbolKind.String:
+                    return Protocol.SymbolKind.String;
+                case Indexing.SymbolKind.Number:
+                    return Protocol.SymbolKind.Number;
+                case Indexing.SymbolKind.Boolean:
+                    return Protocol.SymbolKind.Boolean;
+                case Indexing.SymbolKind.Array:
+                    return Protocol.SymbolKind.Array;
+                case Indexing.SymbolKind.Object:
+                    return Protocol.SymbolKind.Object;
+                case Indexing.SymbolKind.Key:
+                    return Protocol.SymbolKind.Key;
+                case Indexing.SymbolKind.Null:
+                    return Protocol.SymbolKind.Null;
+                case Indexing.SymbolKind.EnumMember:
+                    return Protocol.SymbolKind.EnumMember;
+                case Indexing.SymbolKind.Struct:
+                    return Protocol.SymbolKind.Struct;
+                case Indexing.SymbolKind.Event:
+                    return Protocol.SymbolKind.Event;
+                case Indexing.SymbolKind.Operator:
+                    return Protocol.SymbolKind.Operator;
+                case Indexing.SymbolKind.TypeParameter:
+                    return Protocol.SymbolKind.TypeParameter;
+                default:
+                    throw new NotImplementedException($"{kind} is not a LSP's SymbolKind");
             }
         }
     }

src/LanguageServer/Impl/Implementation/Server.cs

@@ -25,11 +25,13 @@
 using Microsoft.Python.Analysis.Documents;
 using Microsoft.Python.Core;
 using Microsoft.Python.Core.Disposables;
+using Microsoft.Python.Core.Idle;
 using Microsoft.Python.Core.IO;
 using Microsoft.Python.Core.Logging;
 using Microsoft.Python.Core.Services;
 using Microsoft.Python.LanguageServer.Completion;
 using Microsoft.Python.LanguageServer.Diagnostics;
+using Microsoft.Python.LanguageServer.Indexing;
 using Microsoft.Python.LanguageServer.Protocol;
 using Microsoft.Python.LanguageServer.Sources;
 
@@ -43,6 +45,7 @@ public sealed partial class Server : IDisposable {
         private IRunningDocumentTable _rdt;
         private ClientCapabilities _clientCaps;
         private ILogger _log;
+        private IIndexManager _indexManager;
 
         public static InformationDisplayOptions DisplayOptions { get; private set; } = new InformationDisplayOptions {
             preferredFormat = MarkupKind.PlainText,
@@ -120,6 +123,14 @@ public async Task<InitializeResult> InitializeAsync(InitializeParams @params, Ca
             _interpreter = await PythonInterpreter.CreateAsync(configuration, rootDir, _services, cancellationToken);
             _services.AddService(_interpreter);
 
+            var fileSystem = _services.GetService<IFileSystem>();
+            var symbolIndex = new SymbolIndex(fileSystem, _interpreter.LanguageVersion);
+            _indexManager = new IndexManager(symbolIndex, fileSystem, _interpreter.LanguageVersion, rootDir,
+                                            @params.initializationOptions.includeFiles,
+                                            @params.initializationOptions.excludeFiles,
+                                            _services.GetService<IIdleTimeService>());
+            _services.AddService(_indexManager);
+
             DisplayStartupInfo();
 
             // TODO: Pass different documentation sources to completion/hover/signature