Symbol indexer (#521)

For #433.
Closes #483.
Closes #484.
Closes #485.

General overview:
- Indexing happens on each parse, so with the current LS, every file in the workspace gets indexed. This will need to be reworked.
- All symbol queries go to the index, I have not yet approached #486.
- I do some extra heuristics to highlight some of the more interesting symbols, namely:
  - `__init__` in a `class` will be marked as a constructor.
  - Functions annotated with a property decorator are given the kind `Property`.
  - Functions that fit the pattern `__.*__` are given the kind `Operator`.
  - Other functions in a class are given the kind `Method`.
  - Variables that look like `UPPERCASE_UNDERSCORE_123` at the top level or directly inside a class are given the kind `Constant`.
- Any parse updates the index. I don't do any version management other than to look up the AST on the parse complete event.
- VS-specific "functionKind" is preserved.

Things to do in future PRs:
- Fallback to analysis when it's present. I left all of the functions previously used there.
- Using the import system to answer the question "is this thing being imported a module or not", probably augmented/modified outside the indexer.
- Parse version management.
- Asynchronous behavior (switching the dictionary to be a map from `Uri` to `TaskCompletionSource<HierarchicalSymbol>`).

Author: jakebailey

src/Analysis/Engine/Impl/Indexing/ISymbolIndex.cs

@@ -0,0 +1,26 @@
+// Python Tools for Visual Studio
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Collections.Generic;
+
+namespace Microsoft.PythonTools.Analysis.Indexing {
+    internal interface ISymbolIndex {
+        IEnumerable<HierarchicalSymbol> HierarchicalDocumentSymbols(Uri uri);
+
+        IEnumerable<FlatSymbol> WorkspaceSymbols(string query);
+    }
+}

src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs

@@ -0,0 +1,39 @@
+// Python Tools for Visual Studio
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Collections.Generic;
+
+namespace Microsoft.PythonTools.Analysis.Indexing {
+    internal static class SymbolExtensions {
+        public static IEnumerable<FlatSymbol> Flatten(this IEnumerable<HierarchicalSymbol> docSyms, Uri uri, string parent = null, int? depthLimit = null) {
+            foreach (var sym in docSyms) {
+                yield return new FlatSymbol(sym.Name, sym.Kind, uri, sym.SelectionRange, parent);
+
+                if (depthLimit != null) {
+                    if (depthLimit < 1) {
+                        yield break;
+                    }
+                    depthLimit--;
+                }
+
+                foreach (var si in sym.Children.Flatten(uri, sym.Name, depthLimit)) {
+                    yield return si;
+                }
+            }
+        }
+    }
+}

src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs

@@ -0,0 +1,64 @@
+// Python Tools for Visual Studio
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.PythonTools.Analysis.Infrastructure;
+using Microsoft.PythonTools.Parsing.Ast;
+
+namespace Microsoft.PythonTools.Analysis.Indexing {
+    internal class SymbolIndex : ISymbolIndex {
+        private readonly ConcurrentDictionary<Uri, IReadOnlyList<HierarchicalSymbol>> _index = new ConcurrentDictionary<Uri, IReadOnlyList<HierarchicalSymbol>>();
+
+        public SymbolIndex() { }
+
+        public IEnumerable<HierarchicalSymbol> HierarchicalDocumentSymbols(Uri uri)
+            => _index.TryGetValue(uri, out var list) ? list : Enumerable.Empty<HierarchicalSymbol>();
+
+        public IEnumerable<FlatSymbol> WorkspaceSymbols(string query) {
+            foreach (var kvp in _index) {
+                foreach (var found in WorkspaceSymbolsQuery(query, kvp.Key, kvp.Value)) {
+                    yield return found;
+                }
+            }
+        }
+
+        private IEnumerable<FlatSymbol> WorkspaceSymbolsQuery(string query, Uri uri, IEnumerable<HierarchicalSymbol> symbols) {
+            // Some semblance of a BFS.
+            var queue = new Queue<(HierarchicalSymbol, string)>(symbols.Select(s => (s, (string)null)));
+
+            while (queue.Count > 0) {
+                var (sym, parent) = queue.Dequeue();
+
+                if (sym.Name.ContainsOrdinal(query, ignoreCase: true)) {
+                    yield return new FlatSymbol(sym.Name, sym.Kind, uri, sym.SelectionRange, parent);
+                }
+
+                foreach (var child in sym.Children.MaybeEnumerate()) {
+                    queue.Enqueue((child, sym.Name));
+                }
+            }
+        }
+
+        public void UpdateParseTree(Uri uri, PythonAst ast) {
+            var walker = new SymbolIndexWalker(ast);
+            ast.Walk(walker);
+            _index[uri] = walker.Symbols;
+        }
+    }
+}