From b9b255470a8845ef34bb8d9afb23fb2154586d0e Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 25 Jan 2019 14:42:45 -0800 Subject: [PATCH 001/123] IndexParser --- .../Ast/Impl/Indexing/IIndexParser.cs | 9 + .../Ast/Impl/Indexing/ISymbolIndex.cs | 12 + src/Analysis/Ast/Impl/Indexing/IndexParser.cs | 30 ++ src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs | 57 +++ .../Ast/Impl/Indexing/SymbolIndexWalker.cs | 326 ++++++++++++++++++ src/Analysis/Ast/Impl/Indexing/Symbols.cs | 99 ++++++ src/Analysis/Ast/Test/IndexParserTests.cs | 77 +++++ .../Microsoft.Python.Analysis.Tests.csproj | 1 + src/Core/Impl/Extensions/StringExtensions.cs | 2 + 9 files changed, 613 insertions(+) create mode 100644 src/Analysis/Ast/Impl/Indexing/IIndexParser.cs create mode 100644 src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs create mode 100644 src/Analysis/Ast/Impl/Indexing/IndexParser.cs create mode 100644 src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs create mode 100644 src/Analysis/Ast/Impl/Indexing/SymbolIndexWalker.cs create mode 100644 src/Analysis/Ast/Impl/Indexing/Symbols.cs create mode 100644 src/Analysis/Ast/Test/IndexParserTests.cs diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs b/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs new file mode 100644 index 000000000..5c12d9777 --- /dev/null +++ b/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs @@ -0,0 +1,9 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.Python.Analysis.Indexing { + internal interface IIndexParser { + void ParseFile(Uri uri); + } +} diff --git a/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs new file mode 100644 index 000000000..c41237bff --- /dev/null +++ b/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Indexing { + internal interface ISymbolIndex { + void UpdateIndex(Uri uri, PythonAst pythonAst); + bool isNotEmpty(); + IEnumerable WorkspaceSymbols(string query); + } +} diff --git a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs new file mode 100644 index 000000000..63d3ae2e5 --- /dev/null +++ b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs @@ -0,0 +1,30 @@ +using Microsoft.Python.Core.IO; +using Microsoft.Python.Parsing; +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace Microsoft.Python.Analysis.Indexing { + internal sealed class IndexParser: IIndexParser { + private readonly ISymbolIndex _symbolIndex; + private readonly IFileSystem _fileSystem; + private readonly PythonLanguageVersion _version; + + public IndexParser(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version) { + _symbolIndex = symbolIndex ?? throw new ArgumentNullException(nameof(symbolIndex)); + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + _version = version; + } + + public void ParseFile(Uri uri) { + if (!_fileSystem.FileExists(uri.AbsolutePath)) { + throw new ArgumentException($"{uri.AbsolutePath} does not exist", nameof(uri)); + } + using (var stream = _fileSystem.FileOpen(uri.AbsolutePath, FileMode.Open)) { + var parser = Parser.CreateParser(stream, _version); + _symbolIndex.UpdateIndex(uri, parser.ParseFile()); + } + } + } +} diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs new file mode 100644 index 000000000..fcee73c1a --- /dev/null +++ b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Indexing { + internal sealed class SymbolIndex : ISymbolIndex { + private readonly ConcurrentDictionary> _index = new ConcurrentDictionary>(); + private bool _empty; + + public SymbolIndex() { + _empty = false; + } + + public bool isNotEmpty() { + return _empty; + } + + + public IEnumerable HierarchicalDocumentSymbols(Uri uri) + => _index.TryGetValue(uri, out var list) ? list : Enumerable.Empty(); + + public IEnumerable WorkspaceSymbols(string query) { + foreach (var kvp in _index) { + foreach (var found in WorkspaceSymbolsQuery(query, kvp.Key, kvp.Value)) { + yield return found; + } + } + } + + private IEnumerable WorkspaceSymbolsQuery(string query, Uri uri, IEnumerable symbols) { + // Some semblance of a BFS. + var queue = new Queue<(HierarchicalSymbol, string)>(symbols.Select(s => (s, (string)null))); + + while (queue.Count > 0) { + var (sym, parent) = queue.Dequeue(); + + if (sym.Name.ContainsOrdinal(query, ignoreCase: true)) { + yield return new FlatSymbol(sym.Name, sym.Kind, uri, sym.SelectionRange, parent); + } + + foreach (var child in sym.Children.MaybeEnumerate()) { + queue.Enqueue((child, sym.Name)); + } + } + } + + public void UpdateIndex(Uri uri, PythonAst ast) { + var walker = new SymbolIndexWalker(ast); + ast.Walk(walker); + _index[uri] = walker.Symbols; + } + } +} diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndexWalker.cs b/src/Analysis/Ast/Impl/Indexing/SymbolIndexWalker.cs new file mode 100644 index 000000000..e4aba431a --- /dev/null +++ b/src/Analysis/Ast/Impl/Indexing/SymbolIndexWalker.cs @@ -0,0 +1,326 @@ +using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +namespace Microsoft.Python.Analysis.Indexing { + internal class SymbolIndexWalker : PythonWalker { + private static readonly Regex DoubleUnderscore = new Regex(@"^__.*__$", RegexOptions.Compiled); + private static readonly Regex ConstantLike = new Regex(@"^[\p{Lu}\p{N}_]+$", RegexOptions.Compiled); + + private readonly PythonAst _ast; + private readonly SymbolStack _stack = new SymbolStack(); + + public SymbolIndexWalker(PythonAst ast) { + _ast = ast; + } + + public IReadOnlyList Symbols => _stack.Root; + + public override bool Walk(ClassDefinition node) { + _stack.Enter(SymbolKind.Class); + node.Body?.Walk(this); + var children = _stack.Exit(); + + _stack.AddSymbol(new HierarchicalSymbol( + node.Name, + SymbolKind.Class, + node.GetSpan(_ast), + node.NameExpression.GetSpan(_ast), + children, + FunctionKind.Class + )); + + return false; + } + + public override bool Walk(FunctionDefinition node) { + _stack.Enter(SymbolKind.Function); + foreach (var p in node.Parameters) { + AddVarSymbol(p.NameExpression); + } + node.Body?.Walk(this); + var children = _stack.Exit(); + + var span = node.GetSpan(_ast); + + var ds = new HierarchicalSymbol( + node.Name, + SymbolKind.Function, + span, + node.IsLambda ? span : node.NameExpression.GetSpan(_ast), + children, + FunctionKind.Function + ); + + if (_stack.Parent == SymbolKind.Class) { + switch (ds.Name) { + case "__init__": + ds.Kind = SymbolKind.Constructor; + break; + case var name when DoubleUnderscore.IsMatch(name): + ds.Kind = SymbolKind.Operator; + break; + default: + ds.Kind = SymbolKind.Method; + + if (node.Decorators != null) { + foreach (var dec in node.Decorators.Decorators) { + var maybeKind = DecoratorExpressionToKind(dec); + if (maybeKind.HasValue) { + ds.Kind = maybeKind.Value.kind; + ds._functionKind = maybeKind.Value.functionKind; + break; + } + } + } + + break; + } + } + + _stack.AddSymbol(ds); + + return false; + } + + public override bool Walk(ImportStatement node) { + foreach (var (nameNode, nameString) in node.Names.Zip(node.AsNames, (name, asName) => asName != null ? (asName, asName.Name) : ((Node)name, name.MakeString()))) { + var span = nameNode.GetSpan(_ast); + _stack.AddSymbol(new HierarchicalSymbol(nameString, SymbolKind.Module, span)); + } + + return false; + } + + public override bool Walk(FromImportStatement node) { + if (node.IsFromFuture) { + return false; + } + + foreach (var name in node.Names.Zip(node.AsNames, (name, asName) => asName ?? name)) { + var span = name.GetSpan(_ast); + _stack.AddSymbol(new HierarchicalSymbol(name.Name, SymbolKind.Module, span)); + } + + return false; + } + + public override bool Walk(AssignmentStatement node) { + node.Right?.Walk(this); + foreach (var ne in node.Left.OfType()) { + AddVarSymbol(ne); + } + return false; + } + + public override bool Walk(AugmentedAssignStatement node) { + node.Right?.Walk(this); + AddVarSymbol(node.Left as NameExpression); + return false; + } + + public override bool Walk(IfStatement node) { + WalkAndDeclareAll(node.Tests); + WalkAndDeclare(node.ElseStatement); + + return false; + } + + public override bool Walk(TryStatement node) { + WalkAndDeclare(node.Body); + WalkAndDeclareAll(node.Handlers); + WalkAndDeclare(node.Else); + WalkAndDeclare(node.Finally); + + return false; + } + + public override bool Walk(ForStatement node) { + _stack.EnterDeclared(); + AddVarSymbolRecursive(node.Left); + node.List?.Walk(this); + node.Body?.Walk(this); + _stack.ExitDeclaredAndMerge(); + + _stack.EnterDeclared(); + node.Else?.Walk(this); + _stack.ExitDeclaredAndMerge(); + + return false; + } + + public override bool Walk(ComprehensionFor node) { + AddVarSymbolRecursive(node.Left); + return base.Walk(node); + } + + public override bool Walk(ListComprehension node) { + _stack.Enter(SymbolKind.None); + return base.Walk(node); + } + + public override void PostWalk(ListComprehension node) => ExitComprehension(node); + + public override bool Walk(DictionaryComprehension node) { + _stack.Enter(SymbolKind.None); + return base.Walk(node); + } + + public override void PostWalk(DictionaryComprehension node) => ExitComprehension(node); + + public override bool Walk(SetComprehension node) { + _stack.Enter(SymbolKind.None); + return base.Walk(node); + } + + public override void PostWalk(SetComprehension node) => ExitComprehension(node); + + public override bool Walk(GeneratorExpression node) { + _stack.Enter(SymbolKind.None); + return base.Walk(node); + } + + public override void PostWalk(GeneratorExpression node) => ExitComprehension(node); + + private void ExitComprehension(Comprehension node) { + var children = _stack.Exit(); + var span = node.GetSpan(_ast); + + _stack.AddSymbol(new HierarchicalSymbol( + $"<{node.NodeName}>", + SymbolKind.None, + span, + children: children + )); + } + + + private void AddVarSymbol(NameExpression node) { + if (node == null) { + return; + } + + var kind = SymbolKind.Variable; + + switch (node.Name) { + case "*": + case "_": + return; + case var s when (_stack.Parent == null || _stack.Parent == SymbolKind.Class) && ConstantLike.IsMatch(s): + kind = SymbolKind.Constant; + break; + } + + var span = node.GetSpan(_ast); + + _stack.AddSymbol(new HierarchicalSymbol(node.Name, kind, span)); + } + + private void AddVarSymbolRecursive(Expression node) { + if (node == null) { + return; + } + + switch (node) { + case NameExpression ne: + AddVarSymbol(ne); + return; + + case SequenceExpression se: + foreach (var item in se.Items.MaybeEnumerate()) { + AddVarSymbolRecursive(item); + } + return; + } + } + + private (SymbolKind kind, string functionKind)? DecoratorExpressionToKind(Expression exp) { + switch (exp) { + case NameExpression ne when NameIsProperty(ne.Name): + case MemberExpression me when NameIsProperty(me.Name): + return (SymbolKind.Property, FunctionKind.Property); + + case NameExpression ne when NameIsStaticMethod(ne.Name): + case MemberExpression me when NameIsStaticMethod(me.Name): + return (SymbolKind.Method, FunctionKind.StaticMethod); + + case NameExpression ne when NameIsClassMethod(ne.Name): + case MemberExpression me when NameIsClassMethod(me.Name): + return (SymbolKind.Method, FunctionKind.ClassMethod); + } + + return null; + } + + private bool NameIsProperty(string name) => + name == "property" + || name == "abstractproperty" + || name == "classproperty" + || name == "abstractclassproperty"; + + private bool NameIsStaticMethod(string name) => + name == "staticmethod" + || name == "abstractstaticmethod"; + + private bool NameIsClassMethod(string name) => + name == "classmethod" + || name == "abstractclassmethod"; + + private void WalkAndDeclare(Node node) { + if (node == null) { + return; + } + + _stack.EnterDeclared(); + node.Walk(this); + _stack.ExitDeclaredAndMerge(); + } + + private void WalkAndDeclareAll(IEnumerable nodes) { + foreach (var node in nodes.MaybeEnumerate()) { + WalkAndDeclare(node); + } + } + + private class SymbolStack { + private readonly Stack<(SymbolKind? kind, List symbols)> _symbols; + private readonly Stack> _declared = new Stack>(new[] { new HashSet() }); + + public List Root { get; } = new List(); + + public SymbolKind? Parent => _symbols.Peek().kind; + + public SymbolStack() { + _symbols = new Stack<(SymbolKind?, List)>(new (SymbolKind?, List)[] { (null, Root) }); + } + + public void Enter(SymbolKind parent) { + _symbols.Push((parent, new List())); + EnterDeclared(); + } + + public List Exit() { + ExitDeclared(); + return _symbols.Pop().symbols; + } + public void EnterDeclared() => _declared.Push(new HashSet()); + + public void ExitDeclared() => _declared.Pop(); + + public void ExitDeclaredAndMerge() => _declared.Peek().UnionWith(_declared.Pop()); + + public void AddSymbol(HierarchicalSymbol sym) { + if (sym.Kind == SymbolKind.Variable && _declared.Peek().Contains(sym.Name)) { + return; + } + + _symbols.Peek().symbols.Add(sym); + _declared.Peek().Add(sym.Name); + } + } + } +} diff --git a/src/Analysis/Ast/Impl/Indexing/Symbols.cs b/src/Analysis/Ast/Impl/Indexing/Symbols.cs new file mode 100644 index 000000000..a5f1e1876 --- /dev/null +++ b/src/Analysis/Ast/Impl/Indexing/Symbols.cs @@ -0,0 +1,99 @@ +using Microsoft.Python.Core.Text; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.Python.Analysis.Indexing { + // From LSP. + internal enum SymbolKind { + None = 0, + File = 1, + Module = 2, + Namespace = 3, + Package = 4, + Class = 5, + Method = 6, + Property = 7, + Field = 8, + Constructor = 9, + Enum = 10, + Interface = 11, + Function = 12, + Variable = 13, + Constant = 14, + String = 15, + Number = 16, + Boolean = 17, + Array = 18, + Object = 19, + Key = 20, + Null = 21, + EnumMember = 22, + Struct = 23, + Event = 24, + Operator = 25, + TypeParameter = 26 + } + + internal class FunctionKind { + public const string None = ""; + public const string Function = "function"; + public const string Property = "property"; + public const string StaticMethod = "staticmethod"; + public const string ClassMethod = "classmethod"; + public const string Class = "class"; + } + + // Analagous to LSP's DocumentSymbol. + internal class HierarchicalSymbol { + public string Name; + public string Detail; + public SymbolKind Kind; + public bool? Deprecated; + public SourceSpan Range; + public SourceSpan SelectionRange; + public IList Children; + + public string _functionKind; + + public HierarchicalSymbol( + string name, + SymbolKind kind, + SourceSpan range, + SourceSpan? selectionRange = null, + IList children = null, + string functionKind = FunctionKind.None + ) { + Name = name; + Kind = kind; + Range = range; + SelectionRange = selectionRange ?? range; + Children = children; + _functionKind = functionKind; + } + } + + // Analagous to LSP's SymbolInformation. + internal class FlatSymbol { + public string Name; + public SymbolKind Kind; + public bool? Deprecated; + public Uri DocumentUri; + public SourceSpan Range; + public string ContainerName; + + public FlatSymbol( + string name, + SymbolKind kind, + Uri documentUri, + SourceSpan range, + string containerName = null + ) { + Name = name; + Kind = kind; + DocumentUri = documentUri; + Range = range; + ContainerName = containerName; + } + } +} diff --git a/src/Analysis/Ast/Test/IndexParserTests.cs b/src/Analysis/Ast/Test/IndexParserTests.cs new file mode 100644 index 000000000..a782b83f5 --- /dev/null +++ b/src/Analysis/Ast/Test/IndexParserTests.cs @@ -0,0 +1,77 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.Text; +using TestUtilities; +using Microsoft.Python.Analysis.Indexing; +using Microsoft.Python.Core.IO; +using System.IO; +using NSubstitute; +using FluentAssertions; +using Microsoft.Python.Parsing; +using System.Linq; + +namespace Microsoft.Python.Analysis.Tests { + + [TestClass] + public class IndexParserTests : AnalysisTestBase { + public TestContext TestContext { get; set; } + ISymbolIndex _symbolIndex; + IFileSystem _fileSystem; + + [TestInitialize] + public void TestInitialize() { + TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + _symbolIndex = new SymbolIndex(); + _fileSystem = Substitute.For(); + } + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + [TestMethod, Priority(0)] + public void NoSymbols() { + ISymbolIndex symbolIndex = new SymbolIndex(); + var symbols = symbolIndex.WorkspaceSymbols(""); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public void NullIndexThrowsException() { + Action build = () => { + IIndexParser indexParser = new IndexParser(null, _fileSystem, PythonLanguageVersion.V37); + }; + build.Should().Throw(); + } + + [TestMethod, Priority(0)] + public void ParseVariableInFile() { + const string testFilePath = "C:/bla.py"; + _fileSystem.FileExists(testFilePath).Returns(true); + _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(MakeStream("x = 1")); + + IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, PythonLanguageVersion.V37); + indexParser.ParseFile(new Uri(testFilePath)); + + var symbols = _symbolIndex.WorkspaceSymbols(""); + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("x"); + } + + [TestMethod, Priority(0)] + public void ParseNonexistentFile() { + const string testFilePath = "C:/bla.py"; + _fileSystem.FileExists(testFilePath).Returns(false); + + IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, PythonLanguageVersion.V37); + Action parse = () => { + indexParser.ParseFile(new Uri(testFilePath)); + }; + parse.Should().Throw(); + } + + private Stream MakeStream(string str) { + return new MemoryStream(Encoding.UTF8.GetBytes(str)); + } + } +} diff --git a/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj b/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj index 2772e926b..a24f66322 100644 --- a/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj +++ b/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj @@ -36,6 +36,7 @@ all runtime; build; native; contentfiles; analyzers + diff --git a/src/Core/Impl/Extensions/StringExtensions.cs b/src/Core/Impl/Extensions/StringExtensions.cs index 82c383e9d..1020329b0 100644 --- a/src/Core/Impl/Extensions/StringExtensions.cs +++ b/src/Core/Impl/Extensions/StringExtensions.cs @@ -192,6 +192,8 @@ public static bool EqualsOrdinal(this string s, string other) public static bool EqualsOrdinal(this string s, int index, string other, int otherIndex, int length, bool ignoreCase = false) => string.Compare(s, index, other, otherIndex, length, ignoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal) == 0; + public static bool ContainsOrdinal(this string s, string value, bool ignoreCase = false) + => s.IndexOfOrdinal(value, ignoreCase: ignoreCase) != -1; public static string[] Split(this string s, char separator, int startIndex, int length) { var count = 0; From cd1878d9b2d49f4f26530d43f4d8d74c84159930 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 25 Jan 2019 17:04:07 -0800 Subject: [PATCH 002/123] Workspace Index Manage --- .../Ast/Impl/Indexing/DirectoryFileReader.cs | 41 ++++++++++ .../Ast/Impl/Indexing/IDirectoryFileReader.cs | 9 ++ .../Impl/Indexing/IWorkspaceIndexManager.cs | 9 ++ .../Impl/Indexing/WorkspaceIndexManager.cs | 34 ++++++++ .../Ast/Impl/Microsoft.Python.Analysis.csproj | 1 + .../Ast/Test/WorkspaceIndexManagerTests.cs | 82 +++++++++++++++++++ 6 files changed, 176 insertions(+) create mode 100644 src/Analysis/Ast/Impl/Indexing/DirectoryFileReader.cs create mode 100644 src/Analysis/Ast/Impl/Indexing/IDirectoryFileReader.cs create mode 100644 src/Analysis/Ast/Impl/Indexing/IWorkspaceIndexManager.cs create mode 100644 src/Analysis/Ast/Impl/Indexing/WorkspaceIndexManager.cs create mode 100644 src/Analysis/Ast/Test/WorkspaceIndexManagerTests.cs diff --git a/src/Analysis/Ast/Impl/Indexing/DirectoryFileReader.cs b/src/Analysis/Ast/Impl/Indexing/DirectoryFileReader.cs new file mode 100644 index 000000000..e2a0f50a1 --- /dev/null +++ b/src/Analysis/Ast/Impl/Indexing/DirectoryFileReader.cs @@ -0,0 +1,41 @@ +using Microsoft.Extensions.FileSystemGlobbing; +using Microsoft.Extensions.FileSystemGlobbing.Abstractions; +using Microsoft.Python.Core; +using Microsoft.Python.Core.IO; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace Microsoft.Python.Analysis.Indexing { + internal class DirectoryFileReader { + private readonly string _directoryPath; + private readonly string[] _includeFiles; + private readonly string[] _excludeFiles; + + public DirectoryFileReader(string directoryPath, string[] includeFiles, string[] excludeFiles) { + _directoryPath = directoryPath; + _includeFiles = includeFiles; + _excludeFiles = excludeFiles; + } + + private Matcher BuildMatcher() { + Matcher matcher = new Matcher(); + matcher.AddIncludePatterns(_includeFiles.IsNullOrEmpty() ? new[] { "**/*" } : _includeFiles); + matcher.AddExcludePatterns(_excludeFiles ?? Enumerable.Empty()); + return matcher; + } + + public IEnumerable DirectoryFilePaths() { + var matcher = BuildMatcher(); + var dib = new DirectoryInfoWrapper(new DirectoryInfo(_directoryPath)); + var matchResult = matcher.Execute(dib); + + foreach (var file in matchResult.Files) { + var path = Path.Combine(_directoryPath, PathUtils.NormalizePath(file.Path)); + yield return path; + } + } + } +} diff --git a/src/Analysis/Ast/Impl/Indexing/IDirectoryFileReader.cs b/src/Analysis/Ast/Impl/Indexing/IDirectoryFileReader.cs new file mode 100644 index 000000000..6701e6f3c --- /dev/null +++ b/src/Analysis/Ast/Impl/Indexing/IDirectoryFileReader.cs @@ -0,0 +1,9 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.Python.Analysis.Indexing { + public interface IDirectoryFileReader { + IEnumerable DirectoryFilePaths(); + } +} diff --git a/src/Analysis/Ast/Impl/Indexing/IWorkspaceIndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IWorkspaceIndexManager.cs new file mode 100644 index 000000000..6f2e9b2c6 --- /dev/null +++ b/src/Analysis/Ast/Impl/Indexing/IWorkspaceIndexManager.cs @@ -0,0 +1,9 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.Python.Analysis.Indexing { + internal interface IWorkspaceIndexManager { + void AddRootDirectory(); + } +} diff --git a/src/Analysis/Ast/Impl/Indexing/WorkspaceIndexManager.cs b/src/Analysis/Ast/Impl/Indexing/WorkspaceIndexManager.cs new file mode 100644 index 000000000..7db8bce07 --- /dev/null +++ b/src/Analysis/Ast/Impl/Indexing/WorkspaceIndexManager.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.Python.Core.IO; +using Microsoft.Extensions.FileSystemGlobbing; +using Microsoft.Extensions.FileSystemGlobbing.Abstractions; +using Microsoft.Python.Analysis.Core.Interpreter; +using System.IO; +using Microsoft.Python.Parsing; + +namespace Microsoft.Python.Analysis.Indexing { + internal class WorkspaceIndexManager : IWorkspaceIndexManager { + private readonly ISymbolIndex _symbolIndex; + private readonly IDirectoryFileReader _rootFileReader; + private readonly IFileSystem _fileSystem; + private readonly IndexParser _indexParser; + + public WorkspaceIndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, IDirectoryFileReader rootFileReader) { + _rootFileReader = rootFileReader ?? throw new ArgumentNullException($"rootFileReader is null", nameof(rootFileReader)); + _symbolIndex = symbolIndex; + _fileSystem = fileSystem; + _indexParser = new IndexParser(symbolIndex, fileSystem, version); + } + + + public void AddRootDirectory() { + foreach (var path in _rootFileReader.DirectoryFilePaths()) { + if (ModulePath.IsPythonSourceFile(path)) { + _indexParser.ParseFile(new Uri(path)); + } + } + } + } +} diff --git a/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj b/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj index 0d264f7df..8c0a7ec4b 100644 --- a/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj +++ b/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj @@ -19,6 +19,7 @@ all runtime; build; native; contentfiles; analyzers + diff --git a/src/Analysis/Ast/Test/WorkspaceIndexManagerTests.cs b/src/Analysis/Ast/Test/WorkspaceIndexManagerTests.cs new file mode 100644 index 000000000..afa47e19b --- /dev/null +++ b/src/Analysis/Ast/Test/WorkspaceIndexManagerTests.cs @@ -0,0 +1,82 @@ +using FluentAssertions; +using Microsoft.Python.Analysis.Indexing; +using Microsoft.Python.Core.IO; +using Microsoft.Python.Parsing; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using NSubstitute; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class WorkspaceIndexManagerTests : AnalysisTestBase { + private IDirectoryFileReader _rootFileReader; + private IFileSystem _fileSystem; + private ISymbolIndex _symbolIndex; + + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() { + TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + _fileSystem = Substitute.For(); + _rootFileReader = Substitute.For(); + _symbolIndex = new SymbolIndex(); + } + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod, Priority(0)] + public void AddsRootDirectory() { + var rootPath = "C:/root"; + string pythonTestFile = $"{rootPath}/bla.py"; + AddFileToTestFileSystem(pythonTestFile); + _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); + + IWorkspaceIndexManager workspaceIndexManager = new WorkspaceIndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootFileReader); + workspaceIndexManager.AddRootDirectory(); + + var symbols = _symbolIndex.WorkspaceSymbols(""); + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("x"); + } + + [TestMethod, Priority(0)] + public void NullDirectoryThrowsException() { + Action construct = () => { + IWorkspaceIndexManager workspaceIndexManager = new WorkspaceIndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, null); + }; + construct.Should().Throw(); + } + + [TestMethod, Priority(0)] + public void IgnoresNonPythonFiles() { + var rootPath = "C:/root"; + string nonPythonTestFile = $"{rootPath}/bla.txt"; + AddFileToTestFileSystem(nonPythonTestFile); + + IWorkspaceIndexManager workspaceIndexManager = new WorkspaceIndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootFileReader); + workspaceIndexManager.AddRootDirectory(); + + _fileSystem.DidNotReceive().FileExists(nonPythonTestFile); + } + + private void AddFileToTestFileSystem(string filePath) { + _rootFileReader.DirectoryFilePaths().Returns(new List() { + filePath + }); + _fileSystem.FileExists(filePath).Returns(true); + } + + private Stream MakeStream(string str) { + return new MemoryStream(Encoding.UTF8.GetBytes(str)); + } + + } +} From 6a6a38d3b211a51e49d8cfd3a27e50f720dacd86 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Mon, 28 Jan 2019 10:33:23 -0800 Subject: [PATCH 003/123] Sorting using as configured on repo --- .../Ast/Impl/Indexing/DirectoryFileReader.cs | 12 +++++------- .../Ast/Impl/Indexing/IDirectoryFileReader.cs | 4 +--- .../Ast/Impl/Indexing/IIndexParser.cs | 2 -- .../Ast/Impl/Indexing/ISymbolIndex.cs | 1 - .../Impl/Indexing/IWorkspaceIndexManager.cs | 6 +----- src/Analysis/Ast/Impl/Indexing/IndexParser.cs | 10 ++++------ src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs | 5 ++--- .../Ast/Impl/Indexing/SymbolIndexWalker.cs | 8 +++----- src/Analysis/Ast/Impl/Indexing/Symbols.cs | 5 ++--- .../Impl/Indexing/WorkspaceIndexManager.cs | 7 +------ src/Analysis/Ast/Test/IndexParserTests.cs | 19 +++++++++---------- .../Ast/Test/WorkspaceIndexManagerTests.cs | 14 +++++++------- 12 files changed, 35 insertions(+), 58 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/DirectoryFileReader.cs b/src/Analysis/Ast/Impl/Indexing/DirectoryFileReader.cs index e2a0f50a1..1f92d819f 100644 --- a/src/Analysis/Ast/Impl/Indexing/DirectoryFileReader.cs +++ b/src/Analysis/Ast/Impl/Indexing/DirectoryFileReader.cs @@ -1,12 +1,10 @@ -using Microsoft.Extensions.FileSystemGlobbing; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Extensions.FileSystemGlobbing; using Microsoft.Extensions.FileSystemGlobbing.Abstractions; using Microsoft.Python.Core; using Microsoft.Python.Core.IO; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; namespace Microsoft.Python.Analysis.Indexing { internal class DirectoryFileReader { @@ -18,7 +16,7 @@ public DirectoryFileReader(string directoryPath, string[] includeFiles, string[] _directoryPath = directoryPath; _includeFiles = includeFiles; _excludeFiles = excludeFiles; - } + } private Matcher BuildMatcher() { Matcher matcher = new Matcher(); diff --git a/src/Analysis/Ast/Impl/Indexing/IDirectoryFileReader.cs b/src/Analysis/Ast/Impl/Indexing/IDirectoryFileReader.cs index 6701e6f3c..d08f69877 100644 --- a/src/Analysis/Ast/Impl/Indexing/IDirectoryFileReader.cs +++ b/src/Analysis/Ast/Impl/Indexing/IDirectoryFileReader.cs @@ -1,6 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; +using System.Collections.Generic; namespace Microsoft.Python.Analysis.Indexing { public interface IDirectoryFileReader { diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs b/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs index 5c12d9777..17d106f22 100644 --- a/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs +++ b/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs @@ -1,6 +1,4 @@ using System; -using System.Collections.Generic; -using System.Text; namespace Microsoft.Python.Analysis.Indexing { internal interface IIndexParser { diff --git a/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs index c41237bff..5a085b899 100644 --- a/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Text; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Indexing { diff --git a/src/Analysis/Ast/Impl/Indexing/IWorkspaceIndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IWorkspaceIndexManager.cs index 6f2e9b2c6..74a352d09 100644 --- a/src/Analysis/Ast/Impl/Indexing/IWorkspaceIndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IWorkspaceIndexManager.cs @@ -1,8 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace Microsoft.Python.Analysis.Indexing { +namespace Microsoft.Python.Analysis.Indexing { internal interface IWorkspaceIndexManager { void AddRootDirectory(); } diff --git a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs index 63d3ae2e5..2cc129856 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs @@ -1,12 +1,10 @@ -using Microsoft.Python.Core.IO; -using Microsoft.Python.Parsing; -using System; -using System.Collections.Generic; +using System; using System.IO; -using System.Text; +using Microsoft.Python.Core.IO; +using Microsoft.Python.Parsing; namespace Microsoft.Python.Analysis.Indexing { - internal sealed class IndexParser: IIndexParser { + internal sealed class IndexParser : IIndexParser { private readonly ISymbolIndex _symbolIndex; private readonly IFileSystem _fileSystem; private readonly PythonLanguageVersion _version; diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs index fcee73c1a..0006e6d71 100644 --- a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs @@ -2,7 +2,6 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using System.Text; using Microsoft.Python.Core; using Microsoft.Python.Parsing.Ast; @@ -10,11 +9,11 @@ namespace Microsoft.Python.Analysis.Indexing { internal sealed class SymbolIndex : ISymbolIndex { private readonly ConcurrentDictionary> _index = new ConcurrentDictionary>(); private bool _empty; - + public SymbolIndex() { _empty = false; } - + public bool isNotEmpty() { return _empty; } diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndexWalker.cs b/src/Analysis/Ast/Impl/Indexing/SymbolIndexWalker.cs index e4aba431a..3f645f5e0 100644 --- a/src/Analysis/Ast/Impl/Indexing/SymbolIndexWalker.cs +++ b/src/Analysis/Ast/Impl/Indexing/SymbolIndexWalker.cs @@ -1,10 +1,8 @@ -using Microsoft.Python.Core; -using Microsoft.Python.Parsing.Ast; -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; -using System.Text; using System.Text.RegularExpressions; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Indexing { internal class SymbolIndexWalker : PythonWalker { diff --git a/src/Analysis/Ast/Impl/Indexing/Symbols.cs b/src/Analysis/Ast/Impl/Indexing/Symbols.cs index a5f1e1876..8324a07a0 100644 --- a/src/Analysis/Ast/Impl/Indexing/Symbols.cs +++ b/src/Analysis/Ast/Impl/Indexing/Symbols.cs @@ -1,7 +1,6 @@ -using Microsoft.Python.Core.Text; -using System; +using System; using System.Collections.Generic; -using System.Text; +using Microsoft.Python.Core.Text; namespace Microsoft.Python.Analysis.Indexing { // From LSP. diff --git a/src/Analysis/Ast/Impl/Indexing/WorkspaceIndexManager.cs b/src/Analysis/Ast/Impl/Indexing/WorkspaceIndexManager.cs index 7db8bce07..44a00e388 100644 --- a/src/Analysis/Ast/Impl/Indexing/WorkspaceIndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/WorkspaceIndexManager.cs @@ -1,11 +1,6 @@ using System; -using System.Collections.Generic; -using System.Text; -using Microsoft.Python.Core.IO; -using Microsoft.Extensions.FileSystemGlobbing; -using Microsoft.Extensions.FileSystemGlobbing.Abstractions; using Microsoft.Python.Analysis.Core.Interpreter; -using System.IO; +using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; namespace Microsoft.Python.Analysis.Indexing { diff --git a/src/Analysis/Ast/Test/IndexParserTests.cs b/src/Analysis/Ast/Test/IndexParserTests.cs index a782b83f5..ce64e22cd 100644 --- a/src/Analysis/Ast/Test/IndexParserTests.cs +++ b/src/Analysis/Ast/Test/IndexParserTests.cs @@ -1,15 +1,14 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; -using System; -using System.Collections.Generic; +using System; +using System.IO; +using System.Linq; using System.Text; -using TestUtilities; +using FluentAssertions; using Microsoft.Python.Analysis.Indexing; using Microsoft.Python.Core.IO; -using System.IO; -using NSubstitute; -using FluentAssertions; using Microsoft.Python.Parsing; -using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using NSubstitute; +using TestUtilities; namespace Microsoft.Python.Analysis.Tests { @@ -69,9 +68,9 @@ public void ParseNonexistentFile() { }; parse.Should().Throw(); } - + private Stream MakeStream(string str) { return new MemoryStream(Encoding.UTF8.GetBytes(str)); } - } + } } diff --git a/src/Analysis/Ast/Test/WorkspaceIndexManagerTests.cs b/src/Analysis/Ast/Test/WorkspaceIndexManagerTests.cs index afa47e19b..48c46c9eb 100644 --- a/src/Analysis/Ast/Test/WorkspaceIndexManagerTests.cs +++ b/src/Analysis/Ast/Test/WorkspaceIndexManagerTests.cs @@ -1,14 +1,14 @@ -using FluentAssertions; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using FluentAssertions; using Microsoft.Python.Analysis.Indexing; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; using Microsoft.VisualStudio.TestTools.UnitTesting; using NSubstitute; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; using TestUtilities; namespace Microsoft.Python.Analysis.Tests { @@ -66,7 +66,7 @@ public void IgnoresNonPythonFiles() { _fileSystem.DidNotReceive().FileExists(nonPythonTestFile); } - + private void AddFileToTestFileSystem(string filePath) { _rootFileReader.DirectoryFilePaths().Returns(new List() { filePath From 765653b82bf2cd5452248464e71f8832a424b57f Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Mon, 28 Jan 2019 10:42:16 -0800 Subject: [PATCH 004/123] Removing unuseful empty property --- src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs | 1 - src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs | 10 ---------- 2 files changed, 11 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs index 5a085b899..000d87e22 100644 --- a/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs @@ -5,7 +5,6 @@ namespace Microsoft.Python.Analysis.Indexing { internal interface ISymbolIndex { void UpdateIndex(Uri uri, PythonAst pythonAst); - bool isNotEmpty(); IEnumerable WorkspaceSymbols(string query); } } diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs index 0006e6d71..50e979d98 100644 --- a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs @@ -8,16 +8,6 @@ namespace Microsoft.Python.Analysis.Indexing { internal sealed class SymbolIndex : ISymbolIndex { private readonly ConcurrentDictionary> _index = new ConcurrentDictionary>(); - private bool _empty; - - public SymbolIndex() { - _empty = false; - } - - public bool isNotEmpty() { - return _empty; - } - public IEnumerable HierarchicalDocumentSymbols(Uri uri) => _index.TryGetValue(uri, out var list) ? list : Enumerable.Empty(); From b42d1f47b50c7b746c69b8d2b59dea9bba5eb6c4 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Mon, 28 Jan 2019 13:03:00 -0800 Subject: [PATCH 005/123] Moving directory reader to DirectoryInfo --- .../Ast/Impl/Indexing/DirectoryFileReader.cs | 39 ------------------- .../Ast/Impl/Indexing/IDirectoryFileReader.cs | 7 ---- .../Impl/Indexing/WorkspaceIndexManager.cs | 20 ++++++---- .../Ast/Test/WorkspaceIndexManagerTests.cs | 26 +++++++------ src/Core/Impl/IO/DirectoryInfoProxy.cs | 13 +++++++ src/Core/Impl/IO/IDirectoryInfo.cs | 1 + src/Core/Impl/Microsoft.Python.Core.csproj | 1 + .../Microsoft.Python.LanguageServer.csproj | 2 +- 8 files changed, 43 insertions(+), 66 deletions(-) delete mode 100644 src/Analysis/Ast/Impl/Indexing/DirectoryFileReader.cs delete mode 100644 src/Analysis/Ast/Impl/Indexing/IDirectoryFileReader.cs diff --git a/src/Analysis/Ast/Impl/Indexing/DirectoryFileReader.cs b/src/Analysis/Ast/Impl/Indexing/DirectoryFileReader.cs deleted file mode 100644 index 1f92d819f..000000000 --- a/src/Analysis/Ast/Impl/Indexing/DirectoryFileReader.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Microsoft.Extensions.FileSystemGlobbing; -using Microsoft.Extensions.FileSystemGlobbing.Abstractions; -using Microsoft.Python.Core; -using Microsoft.Python.Core.IO; - -namespace Microsoft.Python.Analysis.Indexing { - internal class DirectoryFileReader { - private readonly string _directoryPath; - private readonly string[] _includeFiles; - private readonly string[] _excludeFiles; - - public DirectoryFileReader(string directoryPath, string[] includeFiles, string[] excludeFiles) { - _directoryPath = directoryPath; - _includeFiles = includeFiles; - _excludeFiles = excludeFiles; - } - - private Matcher BuildMatcher() { - Matcher matcher = new Matcher(); - matcher.AddIncludePatterns(_includeFiles.IsNullOrEmpty() ? new[] { "**/*" } : _includeFiles); - matcher.AddExcludePatterns(_excludeFiles ?? Enumerable.Empty()); - return matcher; - } - - public IEnumerable DirectoryFilePaths() { - var matcher = BuildMatcher(); - var dib = new DirectoryInfoWrapper(new DirectoryInfo(_directoryPath)); - var matchResult = matcher.Execute(dib); - - foreach (var file in matchResult.Files) { - var path = Path.Combine(_directoryPath, PathUtils.NormalizePath(file.Path)); - yield return path; - } - } - } -} diff --git a/src/Analysis/Ast/Impl/Indexing/IDirectoryFileReader.cs b/src/Analysis/Ast/Impl/Indexing/IDirectoryFileReader.cs deleted file mode 100644 index d08f69877..000000000 --- a/src/Analysis/Ast/Impl/Indexing/IDirectoryFileReader.cs +++ /dev/null @@ -1,7 +0,0 @@ -using System.Collections.Generic; - -namespace Microsoft.Python.Analysis.Indexing { - public interface IDirectoryFileReader { - IEnumerable DirectoryFilePaths(); - } -} diff --git a/src/Analysis/Ast/Impl/Indexing/WorkspaceIndexManager.cs b/src/Analysis/Ast/Impl/Indexing/WorkspaceIndexManager.cs index 44a00e388..e36a7c249 100644 --- a/src/Analysis/Ast/Impl/Indexing/WorkspaceIndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/WorkspaceIndexManager.cs @@ -6,22 +6,28 @@ namespace Microsoft.Python.Analysis.Indexing { internal class WorkspaceIndexManager : IWorkspaceIndexManager { private readonly ISymbolIndex _symbolIndex; - private readonly IDirectoryFileReader _rootFileReader; private readonly IFileSystem _fileSystem; private readonly IndexParser _indexParser; + private readonly string _rootPath; + private readonly string[] _includeFiles; + private readonly string[] _excludeFiles; - public WorkspaceIndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, IDirectoryFileReader rootFileReader) { - _rootFileReader = rootFileReader ?? throw new ArgumentNullException($"rootFileReader is null", nameof(rootFileReader)); + public WorkspaceIndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, + string[] excludeFiles) { _symbolIndex = symbolIndex; - _fileSystem = fileSystem; + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); _indexParser = new IndexParser(symbolIndex, fileSystem, version); + _rootPath = rootPath ?? throw new ArgumentNullException(nameof(rootPath)); + _includeFiles = includeFiles ?? throw new ArgumentNullException(nameof(rootPath)); + _excludeFiles = excludeFiles ?? throw new ArgumentNullException(nameof(excludeFiles)); } public void AddRootDirectory() { - foreach (var path in _rootFileReader.DirectoryFilePaths()) { - if (ModulePath.IsPythonSourceFile(path)) { - _indexParser.ParseFile(new Uri(path)); + var files = _fileSystem.GetDirectoryInfo(_rootPath).EnumerateFileSystemInfos(_includeFiles, _excludeFiles); + foreach (var fileInfo in files) { + if (ModulePath.IsPythonSourceFile(fileInfo.FullName)) { + _indexParser.ParseFile(new Uri(fileInfo.FullName)); } } } diff --git a/src/Analysis/Ast/Test/WorkspaceIndexManagerTests.cs b/src/Analysis/Ast/Test/WorkspaceIndexManagerTests.cs index 48c46c9eb..3f4957af4 100644 --- a/src/Analysis/Ast/Test/WorkspaceIndexManagerTests.cs +++ b/src/Analysis/Ast/Test/WorkspaceIndexManagerTests.cs @@ -14,9 +14,10 @@ namespace Microsoft.Python.Analysis.Tests { [TestClass] public class WorkspaceIndexManagerTests : AnalysisTestBase { - private IDirectoryFileReader _rootFileReader; private IFileSystem _fileSystem; private ISymbolIndex _symbolIndex; + private string _rootPath; + private List _fileList; public TestContext TestContext { get; set; } @@ -24,8 +25,13 @@ public class WorkspaceIndexManagerTests : AnalysisTestBase { public void TestInitialize() { TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); _fileSystem = Substitute.For(); - _rootFileReader = Substitute.For(); _symbolIndex = new SymbolIndex(); + _rootPath = "C:/root"; + _fileList = new List(); + IDirectoryInfo directoryInfo = Substitute.For(); + // Doesn't work without 'forAnyArgs' + directoryInfo.EnumerateFileSystemInfos(new string[] { }, new string[] { }).ReturnsForAnyArgs(_fileList); + _fileSystem.GetDirectoryInfo(_rootPath).Returns(directoryInfo); } [TestCleanup] @@ -33,12 +39,11 @@ public void TestInitialize() { [TestMethod, Priority(0)] public void AddsRootDirectory() { - var rootPath = "C:/root"; - string pythonTestFile = $"{rootPath}/bla.py"; + string pythonTestFile = $"{_rootPath}/bla.py"; AddFileToTestFileSystem(pythonTestFile); _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); - IWorkspaceIndexManager workspaceIndexManager = new WorkspaceIndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootFileReader); + IWorkspaceIndexManager workspaceIndexManager = new WorkspaceIndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); workspaceIndexManager.AddRootDirectory(); var symbols = _symbolIndex.WorkspaceSymbols(""); @@ -50,27 +55,24 @@ public void AddsRootDirectory() { [TestMethod, Priority(0)] public void NullDirectoryThrowsException() { Action construct = () => { - IWorkspaceIndexManager workspaceIndexManager = new WorkspaceIndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, null); + IWorkspaceIndexManager workspaceIndexManager = new WorkspaceIndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, null, new string[] { }, new string[] { }); }; construct.Should().Throw(); } [TestMethod, Priority(0)] public void IgnoresNonPythonFiles() { - var rootPath = "C:/root"; - string nonPythonTestFile = $"{rootPath}/bla.txt"; + string nonPythonTestFile = $"{_rootPath}/bla.txt"; AddFileToTestFileSystem(nonPythonTestFile); - IWorkspaceIndexManager workspaceIndexManager = new WorkspaceIndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootFileReader); + IWorkspaceIndexManager workspaceIndexManager = new WorkspaceIndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, this._rootPath, new string[] { }, new string[] { }); workspaceIndexManager.AddRootDirectory(); _fileSystem.DidNotReceive().FileExists(nonPythonTestFile); } private void AddFileToTestFileSystem(string filePath) { - _rootFileReader.DirectoryFilePaths().Returns(new List() { - filePath - }); + _fileList.Add(new FileInfoProxy(new FileInfo(filePath))); _fileSystem.FileExists(filePath).Returns(true); } diff --git a/src/Core/Impl/IO/DirectoryInfoProxy.cs b/src/Core/Impl/IO/DirectoryInfoProxy.cs index 1295804bf..4b2b3671f 100644 --- a/src/Core/Impl/IO/DirectoryInfoProxy.cs +++ b/src/Core/Impl/IO/DirectoryInfoProxy.cs @@ -16,6 +16,8 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using Microsoft.Extensions.FileSystemGlobbing; +using Microsoft.Extensions.FileSystemGlobbing.Abstractions; namespace Microsoft.Python.Core.IO { public sealed class DirectoryInfoProxy : IDirectoryInfo { @@ -40,6 +42,17 @@ public IEnumerable EnumerateFileSystemInfos() => _directoryInfo .EnumerateFileSystemInfos() .Select(CreateFileSystemInfoProxy); + public IEnumerable EnumerateFileSystemInfos(string[] includeFiles, string[] excludeFiles) { + Matcher matcher = new Matcher(); + matcher.AddIncludePatterns(includeFiles.IsNullOrEmpty() ? new[] { "**/*" } : includeFiles); + matcher.AddExcludePatterns(excludeFiles ?? Enumerable.Empty()); + var matchResult = matcher.Execute(new DirectoryInfoWrapper(_directoryInfo)); + + foreach (var file in matchResult.Files) { + yield return CreateFileSystemInfoProxy(_directoryInfo.GetFileSystemInfos(file.Stem).First()); + } + } + private static IFileSystemInfo CreateFileSystemInfoProxy(FileSystemInfo fileSystemInfo) => fileSystemInfo is DirectoryInfo directoryInfo ? (IFileSystemInfo)new DirectoryInfoProxy(directoryInfo) diff --git a/src/Core/Impl/IO/IDirectoryInfo.cs b/src/Core/Impl/IO/IDirectoryInfo.cs index bb97fb3f2..b8ccd5410 100644 --- a/src/Core/Impl/IO/IDirectoryInfo.cs +++ b/src/Core/Impl/IO/IDirectoryInfo.cs @@ -19,5 +19,6 @@ namespace Microsoft.Python.Core.IO { public interface IDirectoryInfo : IFileSystemInfo { IDirectoryInfo Parent { get; } IEnumerable EnumerateFileSystemInfos(); + IEnumerable EnumerateFileSystemInfos(string[] includeFiles, string[] excludeFiles); } } diff --git a/src/Core/Impl/Microsoft.Python.Core.csproj b/src/Core/Impl/Microsoft.Python.Core.csproj index 4b3da4419..4d4e1bcb1 100644 --- a/src/Core/Impl/Microsoft.Python.Core.csproj +++ b/src/Core/Impl/Microsoft.Python.Core.csproj @@ -23,6 +23,7 @@ all runtime; build; native; contentfiles; analyzers + diff --git a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj index de6dba4a4..94140782f 100644 --- a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj +++ b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj @@ -27,7 +27,7 @@ all runtime; build; native; contentfiles; analyzers - + From 8a4d3d6b52491b37cb5241de7622a06822fab873 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Mon, 28 Jan 2019 13:18:21 -0800 Subject: [PATCH 006/123] EnumerateFiles returns non-lazily --- src/Core/Impl/IO/DirectoryInfoProxy.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Core/Impl/IO/DirectoryInfoProxy.cs b/src/Core/Impl/IO/DirectoryInfoProxy.cs index 4b2b3671f..6f6689dd3 100644 --- a/src/Core/Impl/IO/DirectoryInfoProxy.cs +++ b/src/Core/Impl/IO/DirectoryInfoProxy.cs @@ -47,10 +47,10 @@ public IEnumerable EnumerateFileSystemInfos(string[] includeFil matcher.AddIncludePatterns(includeFiles.IsNullOrEmpty() ? new[] { "**/*" } : includeFiles); matcher.AddExcludePatterns(excludeFiles ?? Enumerable.Empty()); var matchResult = matcher.Execute(new DirectoryInfoWrapper(_directoryInfo)); - - foreach (var file in matchResult.Files) { - yield return CreateFileSystemInfoProxy(_directoryInfo.GetFileSystemInfos(file.Stem).First()); - } + return matchResult.Files.Select((filePatternMatch) => { + FileSystemInfo fileSystemInfo = _directoryInfo.GetFileSystemInfos(filePatternMatch.Stem).First(); + return CreateFileSystemInfoProxy(fileSystemInfo); + }); } private static IFileSystemInfo CreateFileSystemInfoProxy(FileSystemInfo fileSystemInfo) From 33d768af7cab97ae7e4d45e2fc5e1d2f96b7fb34 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Mon, 28 Jan 2019 13:31:45 -0800 Subject: [PATCH 007/123] IndexParser throws FileNotFoundException --- src/Analysis/Ast/Impl/Indexing/IndexParser.cs | 2 +- src/Analysis/Ast/Test/IndexParserTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs index 2cc129856..fecb88336 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs @@ -17,7 +17,7 @@ public IndexParser(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLangu public void ParseFile(Uri uri) { if (!_fileSystem.FileExists(uri.AbsolutePath)) { - throw new ArgumentException($"{uri.AbsolutePath} does not exist", nameof(uri)); + throw new FileNotFoundException($"{uri.AbsolutePath} does not exist", uri.AbsolutePath); } using (var stream = _fileSystem.FileOpen(uri.AbsolutePath, FileMode.Open)) { var parser = Parser.CreateParser(stream, _version); diff --git a/src/Analysis/Ast/Test/IndexParserTests.cs b/src/Analysis/Ast/Test/IndexParserTests.cs index ce64e22cd..c32aa9116 100644 --- a/src/Analysis/Ast/Test/IndexParserTests.cs +++ b/src/Analysis/Ast/Test/IndexParserTests.cs @@ -66,7 +66,7 @@ public void ParseNonexistentFile() { Action parse = () => { indexParser.ParseFile(new Uri(testFilePath)); }; - parse.Should().Throw(); + parse.Should().Throw(); } private Stream MakeStream(string str) { From 1db803058870392b256d9c638f36bf5a77f642ce Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Mon, 28 Jan 2019 14:50:21 -0800 Subject: [PATCH 008/123] Events on files --- .../Ast/Impl/Indexing/IIndexManager.cs | 10 ++ .../Ast/Impl/Indexing/ISymbolIndex.cs | 1 + .../Impl/Indexing/IWorkspaceIndexManager.cs | 5 - ...rkspaceIndexManager.cs => IndexManager.cs} | 26 ++- src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs | 2 + src/Analysis/Ast/Test/IndexManagerTests.cs | 162 ++++++++++++++++++ .../Ast/Test/WorkspaceIndexManagerTests.cs | 84 --------- 7 files changed, 196 insertions(+), 94 deletions(-) create mode 100644 src/Analysis/Ast/Impl/Indexing/IIndexManager.cs delete mode 100644 src/Analysis/Ast/Impl/Indexing/IWorkspaceIndexManager.cs rename src/Analysis/Ast/Impl/Indexing/{WorkspaceIndexManager.cs => IndexManager.cs} (52%) create mode 100644 src/Analysis/Ast/Test/IndexManagerTests.cs delete mode 100644 src/Analysis/Ast/Test/WorkspaceIndexManagerTests.cs diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs new file mode 100644 index 000000000..bcdf794a1 --- /dev/null +++ b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs @@ -0,0 +1,10 @@ +using System; +using Microsoft.Python.Analysis.Documents; + +namespace Microsoft.Python.Analysis.Indexing { + internal interface IIndexManager { + void AddRootDirectory(); + void ProcessFile(Uri uri, IDocument doc); + void ProcessClosedFile(Uri uri); + } +} diff --git a/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs index 000d87e22..39fb81e42 100644 --- a/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs @@ -6,5 +6,6 @@ namespace Microsoft.Python.Analysis.Indexing { internal interface ISymbolIndex { void UpdateIndex(Uri uri, PythonAst pythonAst); IEnumerable WorkspaceSymbols(string query); + void Delete(Uri uri); } } diff --git a/src/Analysis/Ast/Impl/Indexing/IWorkspaceIndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IWorkspaceIndexManager.cs deleted file mode 100644 index 74a352d09..000000000 --- a/src/Analysis/Ast/Impl/Indexing/IWorkspaceIndexManager.cs +++ /dev/null @@ -1,5 +0,0 @@ -namespace Microsoft.Python.Analysis.Indexing { - internal interface IWorkspaceIndexManager { - void AddRootDirectory(); - } -} diff --git a/src/Analysis/Ast/Impl/Indexing/WorkspaceIndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs similarity index 52% rename from src/Analysis/Ast/Impl/Indexing/WorkspaceIndexManager.cs rename to src/Analysis/Ast/Impl/Indexing/IndexManager.cs index e36a7c249..e523e8ea7 100644 --- a/src/Analysis/Ast/Impl/Indexing/WorkspaceIndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs @@ -1,35 +1,51 @@ using System; using Microsoft.Python.Analysis.Core.Interpreter; +using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; namespace Microsoft.Python.Analysis.Indexing { - internal class WorkspaceIndexManager : IWorkspaceIndexManager { + internal class IndexManager : IIndexManager { private readonly ISymbolIndex _symbolIndex; private readonly IFileSystem _fileSystem; private readonly IndexParser _indexParser; - private readonly string _rootPath; + private readonly string _workspaceRootPath; private readonly string[] _includeFiles; private readonly string[] _excludeFiles; - public WorkspaceIndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, + public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, string[] excludeFiles) { _symbolIndex = symbolIndex; _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); _indexParser = new IndexParser(symbolIndex, fileSystem, version); - _rootPath = rootPath ?? throw new ArgumentNullException(nameof(rootPath)); + _workspaceRootPath = rootPath ?? throw new ArgumentNullException(nameof(rootPath)); _includeFiles = includeFiles ?? throw new ArgumentNullException(nameof(rootPath)); _excludeFiles = excludeFiles ?? throw new ArgumentNullException(nameof(excludeFiles)); } public void AddRootDirectory() { - var files = _fileSystem.GetDirectoryInfo(_rootPath).EnumerateFileSystemInfos(_includeFiles, _excludeFiles); + var files = _fileSystem.GetDirectoryInfo(_workspaceRootPath).EnumerateFileSystemInfos(_includeFiles, _excludeFiles); foreach (var fileInfo in files) { if (ModulePath.IsPythonSourceFile(fileInfo.FullName)) { _indexParser.ParseFile(new Uri(fileInfo.FullName)); } } } + + public void ProcessClosedFile(Uri uri) { + // If path is on workspace + if (_fileSystem.IsPathUnderRoot(_workspaceRootPath, uri.AbsolutePath)) { + // updates index and ignores previous AST + _indexParser.ParseFile(uri); + } else { + // remove file from index + _symbolIndex.Delete(uri); + } + } + + public void ProcessFile(Uri uri, IDocument doc) { + _symbolIndex.UpdateIndex(uri, doc.GetAnyAst()); + } } } diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs index 50e979d98..28677c4f4 100644 --- a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs @@ -42,5 +42,7 @@ public void UpdateIndex(Uri uri, PythonAst ast) { ast.Walk(walker); _index[uri] = walker.Symbols; } + + public void Delete(Uri uri) => _index.TryRemove(uri, out var _); } } diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/Analysis/Ast/Test/IndexManagerTests.cs new file mode 100644 index 000000000..b17d5774a --- /dev/null +++ b/src/Analysis/Ast/Test/IndexManagerTests.cs @@ -0,0 +1,162 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using FluentAssertions; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Indexing; +using Microsoft.Python.Core.IO; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using NSubstitute; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class IndexManagerTests : AnalysisTestBase { + private IFileSystem _fileSystem; + private ISymbolIndex _symbolIndex; + private string _rootPath; + private List _rootFileList; + + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() { + TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + _fileSystem = Substitute.For(); + _symbolIndex = new SymbolIndex(); + _rootPath = "C:/root"; + _rootFileList = new List(); + IDirectoryInfo directoryInfo = Substitute.For(); + // Doesn't work without 'forAnyArgs' + directoryInfo.EnumerateFileSystemInfos(new string[] { }, new string[] { }).ReturnsForAnyArgs(_rootFileList); + _fileSystem.GetDirectoryInfo(_rootPath).Returns(directoryInfo); + } + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod, Priority(0)] + public void AddsRootDirectory() { + string pythonTestFile = $"{_rootPath}/bla.py"; + AddFileToRootTestFileSystem(pythonTestFile); + _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); + + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); + indexManager.AddRootDirectory(); + + var symbols = _symbolIndex.WorkspaceSymbols(""); + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("x"); + } + + [TestMethod, Priority(0)] + public void NullDirectoryThrowsException() { + Action construct = () => { + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, null, new string[] { }, new string[] { }); + }; + construct.Should().Throw(); + } + + [TestMethod, Priority(0)] + public void IgnoresNonPythonFiles() { + string nonPythonTestFile = $"{_rootPath}/bla.txt"; + AddFileToRootTestFileSystem(nonPythonTestFile); + + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, this._rootPath, new string[] { }, new string[] { }); + indexManager.AddRootDirectory(); + + _fileSystem.DidNotReceive().FileExists(nonPythonTestFile); + } + + [TestMethod, Priority(0)] + public void CanOpenFiles() { + string nonRootPath = "C:/nonRoot"; + string pythonTestFile = $"{nonRootPath}/bla.py"; + IDocument doc = Substitute.For(); + doc.GetAnyAst().Returns(MakeAst("x = 1")); + + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); + indexManager.ProcessFile(new Uri(pythonTestFile), doc); + + var symbols = _symbolIndex.WorkspaceSymbols(""); + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("x"); + } + + [TestMethod, Priority(0)] + public void UpdateFilesOnWorkspaceIndexesLatest() { + string pythonTestFile = $"{_rootPath}/bla.py"; + AddFileToRootTestFileSystem(pythonTestFile); + _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); + + IDocument latestDoc = Substitute.For(); + latestDoc.GetAnyAst().Returns(MakeAst("y = 1")); + + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); + indexManager.AddRootDirectory(); + indexManager.ProcessFile(new Uri(pythonTestFile), latestDoc); + + var symbols = _symbolIndex.WorkspaceSymbols(""); + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("y"); + } + + [TestMethod, Priority(0)] + public void CloseNonWorkspaceFilesRemovesFromIndex() { + string nonRootPath = "C:/nonRoot"; + string pythonTestFile = $"{nonRootPath}/bla.py"; + IDocument doc = Substitute.For(); + doc.GetAnyAst().Returns(MakeAst("x = 1")); + + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); + indexManager.ProcessFile(new Uri(pythonTestFile), doc); + indexManager.ProcessClosedFile(new Uri(pythonTestFile)); + + var symbols = _symbolIndex.WorkspaceSymbols(""); + symbols.Should().HaveCount(0); + } + + [TestMethod, Priority(0)] + public void CloseWorkspaceFilesReUpdatesIndex() { + string pythonTestFile = $"{_rootPath}/bla.py"; + AddFileToRootTestFileSystem(pythonTestFile); + _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); + _fileSystem.IsPathUnderRoot(_rootPath, pythonTestFile).Returns(true); + + IDocument latestDoc = Substitute.For(); + latestDoc.GetAnyAst().Returns(MakeAst("y = 1")); + + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); + indexManager.AddRootDirectory(); + indexManager.ProcessFile(new Uri(pythonTestFile), latestDoc); + // It Needs to remake the stream for the file, previous one is closed + _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); + indexManager.ProcessClosedFile(new Uri(pythonTestFile)); + + var symbols = _symbolIndex.WorkspaceSymbols(""); + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("x"); + } + + private PythonAst MakeAst(string testCode) { + return Parser.CreateParser(MakeStream(testCode), PythonLanguageVersion.V37).ParseFile(); + } + + private void AddFileToRootTestFileSystem(string filePath) { + _rootFileList.Add(new FileInfoProxy(new FileInfo(filePath))); + _fileSystem.FileExists(filePath).Returns(true); + } + + private Stream MakeStream(string str) { + return new MemoryStream(Encoding.UTF8.GetBytes(str)); + } + } +} diff --git a/src/Analysis/Ast/Test/WorkspaceIndexManagerTests.cs b/src/Analysis/Ast/Test/WorkspaceIndexManagerTests.cs deleted file mode 100644 index 3f4957af4..000000000 --- a/src/Analysis/Ast/Test/WorkspaceIndexManagerTests.cs +++ /dev/null @@ -1,84 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using FluentAssertions; -using Microsoft.Python.Analysis.Indexing; -using Microsoft.Python.Core.IO; -using Microsoft.Python.Parsing; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using NSubstitute; -using TestUtilities; - -namespace Microsoft.Python.Analysis.Tests { - [TestClass] - public class WorkspaceIndexManagerTests : AnalysisTestBase { - private IFileSystem _fileSystem; - private ISymbolIndex _symbolIndex; - private string _rootPath; - private List _fileList; - - public TestContext TestContext { get; set; } - - [TestInitialize] - public void TestInitialize() { - TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); - _fileSystem = Substitute.For(); - _symbolIndex = new SymbolIndex(); - _rootPath = "C:/root"; - _fileList = new List(); - IDirectoryInfo directoryInfo = Substitute.For(); - // Doesn't work without 'forAnyArgs' - directoryInfo.EnumerateFileSystemInfos(new string[] { }, new string[] { }).ReturnsForAnyArgs(_fileList); - _fileSystem.GetDirectoryInfo(_rootPath).Returns(directoryInfo); - } - - [TestCleanup] - public void Cleanup() => TestEnvironmentImpl.TestCleanup(); - - [TestMethod, Priority(0)] - public void AddsRootDirectory() { - string pythonTestFile = $"{_rootPath}/bla.py"; - AddFileToTestFileSystem(pythonTestFile); - _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); - - IWorkspaceIndexManager workspaceIndexManager = new WorkspaceIndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); - workspaceIndexManager.AddRootDirectory(); - - var symbols = _symbolIndex.WorkspaceSymbols(""); - symbols.Should().HaveCount(1); - symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); - symbols.First().Name.Should().BeEquivalentTo("x"); - } - - [TestMethod, Priority(0)] - public void NullDirectoryThrowsException() { - Action construct = () => { - IWorkspaceIndexManager workspaceIndexManager = new WorkspaceIndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, null, new string[] { }, new string[] { }); - }; - construct.Should().Throw(); - } - - [TestMethod, Priority(0)] - public void IgnoresNonPythonFiles() { - string nonPythonTestFile = $"{_rootPath}/bla.txt"; - AddFileToTestFileSystem(nonPythonTestFile); - - IWorkspaceIndexManager workspaceIndexManager = new WorkspaceIndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, this._rootPath, new string[] { }, new string[] { }); - workspaceIndexManager.AddRootDirectory(); - - _fileSystem.DidNotReceive().FileExists(nonPythonTestFile); - } - - private void AddFileToTestFileSystem(string filePath) { - _fileList.Add(new FileInfoProxy(new FileInfo(filePath))); - _fileSystem.FileExists(filePath).Returns(true); - } - - private Stream MakeStream(string str) { - return new MemoryStream(Encoding.UTF8.GetBytes(str)); - } - - } -} From 52dd81251e3ad719fc9479d5a24176ad61190869 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Mon, 28 Jan 2019 16:08:49 -0800 Subject: [PATCH 009/123] Adding update event to index manager --- .../Ast/Impl/Indexing/IIndexManager.cs | 1 + .../Ast/Impl/Indexing/IndexManager.cs | 25 +++++++++++++++++-- src/Analysis/Ast/Test/IndexManagerTests.cs | 22 +++++++++++++++- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs index bcdf794a1..1366b6434 100644 --- a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs @@ -6,5 +6,6 @@ internal interface IIndexManager { void AddRootDirectory(); void ProcessFile(Uri uri, IDocument doc); void ProcessClosedFile(Uri uri); + void ProcessFileIfIndexed(Uri uri, IDocument doc); } } diff --git a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs index e523e8ea7..0a05b94d3 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core.IO; @@ -12,6 +13,7 @@ internal class IndexManager : IIndexManager { private readonly string _workspaceRootPath; private readonly string[] _includeFiles; private readonly string[] _excludeFiles; + private readonly ConcurrentDictionary _indexedFiles = new ConcurrentDictionary(); public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, string[] excludeFiles) { @@ -28,24 +30,43 @@ public void AddRootDirectory() { var files = _fileSystem.GetDirectoryInfo(_workspaceRootPath).EnumerateFileSystemInfos(_includeFiles, _excludeFiles); foreach (var fileInfo in files) { if (ModulePath.IsPythonSourceFile(fileInfo.FullName)) { - _indexParser.ParseFile(new Uri(fileInfo.FullName)); + Uri uri = new Uri(fileInfo.FullName); + _indexParser.ParseFile(uri); + _indexedFiles[uri] = true; } } } + private bool IsFileIndexed(Uri uri) { + _indexedFiles.TryGetValue(uri, out var val); + return val; + } + public void ProcessClosedFile(Uri uri) { // If path is on workspace - if (_fileSystem.IsPathUnderRoot(_workspaceRootPath, uri.AbsolutePath)) { + if (IsFileOnWorkspace(uri)) { // updates index and ignores previous AST _indexParser.ParseFile(uri); } else { // remove file from index + _indexedFiles.TryRemove(uri, out _); _symbolIndex.Delete(uri); } } + private bool IsFileOnWorkspace(Uri uri) { + return _fileSystem.IsPathUnderRoot(_workspaceRootPath, uri.AbsolutePath); + } + public void ProcessFile(Uri uri, IDocument doc) { + _indexedFiles[uri] = true; _symbolIndex.UpdateIndex(uri, doc.GetAnyAst()); } + + public void ProcessFileIfIndexed(Uri uri, IDocument doc) { + if (IsFileIndexed(uri)) { + ProcessFile(uri, doc); + } + } } } diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/Analysis/Ast/Test/IndexManagerTests.cs index b17d5774a..137f618cc 100644 --- a/src/Analysis/Ast/Test/IndexManagerTests.cs +++ b/src/Analysis/Ast/Test/IndexManagerTests.cs @@ -100,7 +100,7 @@ public void UpdateFilesOnWorkspaceIndexesLatest() { IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); indexManager.AddRootDirectory(); - indexManager.ProcessFile(new Uri(pythonTestFile), latestDoc); + indexManager.ProcessFileIfIndexed(new Uri(pythonTestFile), latestDoc); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(1); @@ -146,6 +146,26 @@ public void CloseWorkspaceFilesReUpdatesIndex() { symbols.First().Name.Should().BeEquivalentTo("x"); } + [TestMethod, Priority(0)] + public void ProcessFileIfIndexedAfterCloseIgnoresUpdate() { + // If events get to index manager in the order: [open, close, update] + // it should not reindex file + + string nonRootPath = "C:/nonRoot"; + string pythonTestFile = $"{nonRootPath}/bla.py"; + IDocument doc = Substitute.For(); + doc.GetAnyAst().Returns(MakeAst("x = 1")); + + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); + indexManager.ProcessFile(new Uri(pythonTestFile), doc); + indexManager.ProcessClosedFile(new Uri(pythonTestFile)); + doc.GetAnyAst().Returns(MakeAst("x = 1")); + indexManager.ProcessFileIfIndexed(new Uri(pythonTestFile), doc); + + var symbols = _symbolIndex.WorkspaceSymbols(""); + symbols.Should().HaveCount(0); + } + private PythonAst MakeAst(string testCode) { return Parser.CreateParser(MakeStream(testCode), PythonLanguageVersion.V37).ParseFile(); } From 7c535c3aef2c51ffb6134c1c1020c54cf97cbcc0 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 29 Jan 2019 11:08:02 -0800 Subject: [PATCH 010/123] Async IndexParser --- .../Ast/Impl/Indexing/IIndexParser.cs | 6 +- .../Ast/Impl/Indexing/IndexManager.cs | 11 +++- src/Analysis/Ast/Impl/Indexing/IndexParser.cs | 29 +++++++--- src/Analysis/Ast/Test/IndexParserTests.cs | 57 +++++++++++++++---- 4 files changed, 80 insertions(+), 23 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs b/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs index 17d106f22..659739e98 100644 --- a/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs +++ b/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs @@ -1,7 +1,9 @@ using System; +using System.Threading; +using System.Threading.Tasks; namespace Microsoft.Python.Analysis.Indexing { - internal interface IIndexParser { - void ParseFile(Uri uri); + internal interface IIndexParser : IDisposable { + Task ParseAsync(Uri uri, CancellationToken cancellationToken = default); } } diff --git a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs index 0a05b94d3..ae1f8f768 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Concurrent; +using System.Collections.Generic; using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core.IO; @@ -27,16 +28,20 @@ public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLang public void AddRootDirectory() { - var files = _fileSystem.GetDirectoryInfo(_workspaceRootPath).EnumerateFileSystemInfos(_includeFiles, _excludeFiles); + var files = WorkspaceFiles(); foreach (var fileInfo in files) { if (ModulePath.IsPythonSourceFile(fileInfo.FullName)) { Uri uri = new Uri(fileInfo.FullName); - _indexParser.ParseFile(uri); + _indexParser.ParseAsync(uri); _indexedFiles[uri] = true; } } } + private IEnumerable WorkspaceFiles() { + return _fileSystem.GetDirectoryInfo(_workspaceRootPath).EnumerateFileSystemInfos(_includeFiles, _excludeFiles); + } + private bool IsFileIndexed(Uri uri) { _indexedFiles.TryGetValue(uri, out var val); return val; @@ -46,7 +51,7 @@ public void ProcessClosedFile(Uri uri) { // If path is on workspace if (IsFileOnWorkspace(uri)) { // updates index and ignores previous AST - _indexParser.ParseFile(uri); + _indexParser.ParseAsync(uri); } else { // remove file from index _indexedFiles.TryRemove(uri, out _); diff --git a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs index fecb88336..36b70784a 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs @@ -1,5 +1,7 @@ using System; using System.IO; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; @@ -8,6 +10,7 @@ internal sealed class IndexParser : IIndexParser { private readonly ISymbolIndex _symbolIndex; private readonly IFileSystem _fileSystem; private readonly PythonLanguageVersion _version; + private readonly CancellationTokenSource _allProcessingCts = new CancellationTokenSource(); public IndexParser(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version) { _symbolIndex = symbolIndex ?? throw new ArgumentNullException(nameof(symbolIndex)); @@ -15,14 +18,24 @@ public IndexParser(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLangu _version = version; } - public void ParseFile(Uri uri) { - if (!_fileSystem.FileExists(uri.AbsolutePath)) { - throw new FileNotFoundException($"{uri.AbsolutePath} does not exist", uri.AbsolutePath); - } - using (var stream = _fileSystem.FileOpen(uri.AbsolutePath, FileMode.Open)) { - var parser = Parser.CreateParser(stream, _version); - _symbolIndex.UpdateIndex(uri, parser.ParseFile()); - } + public void Dispose() { + _allProcessingCts.Cancel(); + _allProcessingCts.Dispose(); + } + + public Task ParseAsync(Uri uri, CancellationToken parseCancellationToken = default) { + var linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, parseCancellationToken); + var linkedParseToken = linkedParseCts.Token; + return Task.Run(() => { + if (!_fileSystem.FileExists(uri.AbsolutePath)) { + throw new FileNotFoundException($"{uri.AbsolutePath} does not exist", uri.AbsolutePath); + } + using (var stream = _fileSystem.FileOpen(uri.AbsolutePath, FileMode.Open)) { + var parser = Parser.CreateParser(stream, _version); + linkedParseToken.ThrowIfCancellationRequested(); + _symbolIndex.UpdateIndex(uri, parser.ParseFile()); + } + }, linkedParseToken); } } } diff --git a/src/Analysis/Ast/Test/IndexParserTests.cs b/src/Analysis/Ast/Test/IndexParserTests.cs index c32aa9116..326b2688d 100644 --- a/src/Analysis/Ast/Test/IndexParserTests.cs +++ b/src/Analysis/Ast/Test/IndexParserTests.cs @@ -2,6 +2,8 @@ using System.IO; using System.Linq; using System.Text; +using System.Threading; +using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Indexing; using Microsoft.Python.Core.IO; @@ -27,12 +29,6 @@ public void TestInitialize() { [TestCleanup] public void Cleanup() => TestEnvironmentImpl.TestCleanup(); - [TestMethod, Priority(0)] - public void NoSymbols() { - ISymbolIndex symbolIndex = new SymbolIndex(); - var symbols = symbolIndex.WorkspaceSymbols(""); - symbols.Should().BeEmpty(); - } [TestMethod, Priority(0)] public void NullIndexThrowsException() { @@ -43,13 +39,13 @@ public void NullIndexThrowsException() { } [TestMethod, Priority(0)] - public void ParseVariableInFile() { + public async Task ParseVariableInFileAsync() { const string testFilePath = "C:/bla.py"; _fileSystem.FileExists(testFilePath).Returns(true); _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(MakeStream("x = 1")); IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, PythonLanguageVersion.V37); - indexParser.ParseFile(new Uri(testFilePath)); + await indexParser.ParseAsync(new Uri(testFilePath)); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(1); @@ -63,10 +59,51 @@ public void ParseNonexistentFile() { _fileSystem.FileExists(testFilePath).Returns(false); IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, PythonLanguageVersion.V37); - Action parse = () => { - indexParser.ParseFile(new Uri(testFilePath)); + Func parse = async () => { + await indexParser.ParseAsync(new Uri(testFilePath)); }; + parse.Should().Throw(); + var symbols = _symbolIndex.WorkspaceSymbols(""); + symbols.Should().HaveCount(0); + } + + [TestMethod, Priority(0)] + public void CancellParsing() { + const string testFilePath = "C:/bla.py"; + _fileSystem.FileExists(testFilePath).Returns(true); + _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(MakeStream("x = 1")); + + IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, PythonLanguageVersion.V37); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + cancellationTokenSource.Cancel(); + Func parse = async () => { + await indexParser.ParseAsync(new Uri(testFilePath), cancellationTokenSource.Token); + }; + + parse.Should().Throw(); + var symbols = _symbolIndex.WorkspaceSymbols(""); + symbols.Should().HaveCount(0); + } + + [TestMethod, Priority(0)] + public void DisposeParserCancelesParsing() { + const string testFilePath = "C:/bla.py"; + _fileSystem.FileExists(testFilePath).Returns(true); + MemoryStream stream = new MemoryStream(); + // no writing to the stream, will block when read + _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(stream); + + IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, PythonLanguageVersion.V37); + Func parse = async () => { + Task t = indexParser.ParseAsync(new Uri(testFilePath)); + indexParser.Dispose(); + await t; + }; + + parse.Should().Throw(); + var symbols = _symbolIndex.WorkspaceSymbols(""); + symbols.Should().HaveCount(0); } private Stream MakeStream(string str) { From 05f3b4efb358dbd919729fbea455afed86c484dc Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 29 Jan 2019 13:20:19 -0800 Subject: [PATCH 011/123] Async IndexManager --- .../Ast/Impl/Indexing/IIndexManager.cs | 8 +++-- .../Ast/Impl/Indexing/IndexManager.cs | 29 ++++++++++++++----- src/Analysis/Ast/Test/IndexManagerTests.cs | 23 ++++++++------- src/Core/Impl/IO/DirectoryInfoProxy.cs | 2 +- 4 files changed, 39 insertions(+), 23 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs index 1366b6434..b7c5374a1 100644 --- a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs @@ -1,11 +1,13 @@ using System; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Python.Analysis.Documents; namespace Microsoft.Python.Analysis.Indexing { - internal interface IIndexManager { - void AddRootDirectory(); + internal interface IIndexManager : IDisposable { + Task AddRootDirectory(CancellationToken workspaceCancellationToken = default); void ProcessFile(Uri uri, IDocument doc); - void ProcessClosedFile(Uri uri); + Task ProcessClosedFile(Uri uri, CancellationToken fileCancellationToken = default); void ProcessFileIfIndexed(Uri uri, IDocument doc); } } diff --git a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs index ae1f8f768..9b1e6ea85 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core.IO; @@ -15,6 +17,7 @@ internal class IndexManager : IIndexManager { private readonly string[] _includeFiles; private readonly string[] _excludeFiles; private readonly ConcurrentDictionary _indexedFiles = new ConcurrentDictionary(); + private readonly CancellationTokenSource _allIndexCts = new CancellationTokenSource(); public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, string[] excludeFiles) { @@ -26,16 +29,19 @@ public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLang _excludeFiles = excludeFiles ?? throw new ArgumentNullException(nameof(excludeFiles)); } - - public void AddRootDirectory() { - var files = WorkspaceFiles(); - foreach (var fileInfo in files) { + public Task AddRootDirectory(CancellationToken workspaceCancellationToken = default) { + var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(workspaceCancellationToken, _allIndexCts.Token); + var parseTasks = new List(); + foreach (var fileInfo in WorkspaceFiles()) { if (ModulePath.IsPythonSourceFile(fileInfo.FullName)) { Uri uri = new Uri(fileInfo.FullName); - _indexParser.ParseAsync(uri); - _indexedFiles[uri] = true; + parseTasks.Add(_indexParser.ParseAsync(uri, linkedCts.Token).ContinueWith((task) => { + linkedCts.Token.ThrowIfCancellationRequested(); + _indexedFiles[uri] = true; + })); } } + return Task.WhenAll(parseTasks.ToArray()); } private IEnumerable WorkspaceFiles() { @@ -47,15 +53,17 @@ private bool IsFileIndexed(Uri uri) { return val; } - public void ProcessClosedFile(Uri uri) { + public Task ProcessClosedFile(Uri uri, CancellationToken fileCancellationToken = default) { + var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(fileCancellationToken, _allIndexCts.Token); // If path is on workspace if (IsFileOnWorkspace(uri)) { // updates index and ignores previous AST - _indexParser.ParseAsync(uri); + return _indexParser.ParseAsync(uri, linkedCts.Token); } else { // remove file from index _indexedFiles.TryRemove(uri, out _); _symbolIndex.Delete(uri); + return Task.CompletedTask; } } @@ -73,5 +81,10 @@ public void ProcessFileIfIndexed(Uri uri, IDocument doc) { ProcessFile(uri, doc); } } + + public void Dispose() { + _allIndexCts.Cancel(); + _allIndexCts.Dispose(); + } } } diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/Analysis/Ast/Test/IndexManagerTests.cs index 137f618cc..a0bf8ed8a 100644 --- a/src/Analysis/Ast/Test/IndexManagerTests.cs +++ b/src/Analysis/Ast/Test/IndexManagerTests.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Indexing; @@ -40,13 +41,13 @@ public void TestInitialize() { public void Cleanup() => TestEnvironmentImpl.TestCleanup(); [TestMethod, Priority(0)] - public void AddsRootDirectory() { + public async Task AddsRootDirectoryAsync() { string pythonTestFile = $"{_rootPath}/bla.py"; AddFileToRootTestFileSystem(pythonTestFile); _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); - indexManager.AddRootDirectory(); + await indexManager.AddRootDirectory(); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(1); @@ -90,7 +91,7 @@ public void CanOpenFiles() { } [TestMethod, Priority(0)] - public void UpdateFilesOnWorkspaceIndexesLatest() { + public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { string pythonTestFile = $"{_rootPath}/bla.py"; AddFileToRootTestFileSystem(pythonTestFile); _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); @@ -99,7 +100,7 @@ public void UpdateFilesOnWorkspaceIndexesLatest() { latestDoc.GetAnyAst().Returns(MakeAst("y = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); - indexManager.AddRootDirectory(); + await indexManager.AddRootDirectory(); indexManager.ProcessFileIfIndexed(new Uri(pythonTestFile), latestDoc); var symbols = _symbolIndex.WorkspaceSymbols(""); @@ -109,7 +110,7 @@ public void UpdateFilesOnWorkspaceIndexesLatest() { } [TestMethod, Priority(0)] - public void CloseNonWorkspaceFilesRemovesFromIndex() { + public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { string nonRootPath = "C:/nonRoot"; string pythonTestFile = $"{nonRootPath}/bla.py"; IDocument doc = Substitute.For(); @@ -117,14 +118,14 @@ public void CloseNonWorkspaceFilesRemovesFromIndex() { IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); indexManager.ProcessFile(new Uri(pythonTestFile), doc); - indexManager.ProcessClosedFile(new Uri(pythonTestFile)); + await indexManager.ProcessClosedFile(new Uri(pythonTestFile)); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(0); } [TestMethod, Priority(0)] - public void CloseWorkspaceFilesReUpdatesIndex() { + public async Task CloseWorkspaceFilesReUpdatesIndexAsync() { string pythonTestFile = $"{_rootPath}/bla.py"; AddFileToRootTestFileSystem(pythonTestFile); _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); @@ -134,11 +135,11 @@ public void CloseWorkspaceFilesReUpdatesIndex() { latestDoc.GetAnyAst().Returns(MakeAst("y = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); - indexManager.AddRootDirectory(); + await indexManager.AddRootDirectory(); indexManager.ProcessFile(new Uri(pythonTestFile), latestDoc); // It Needs to remake the stream for the file, previous one is closed _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); - indexManager.ProcessClosedFile(new Uri(pythonTestFile)); + await indexManager.ProcessClosedFile(new Uri(pythonTestFile)); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(1); @@ -147,7 +148,7 @@ public void CloseWorkspaceFilesReUpdatesIndex() { } [TestMethod, Priority(0)] - public void ProcessFileIfIndexedAfterCloseIgnoresUpdate() { + public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() { // If events get to index manager in the order: [open, close, update] // it should not reindex file @@ -158,7 +159,7 @@ public void ProcessFileIfIndexedAfterCloseIgnoresUpdate() { IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); indexManager.ProcessFile(new Uri(pythonTestFile), doc); - indexManager.ProcessClosedFile(new Uri(pythonTestFile)); + await indexManager.ProcessClosedFile(new Uri(pythonTestFile)); doc.GetAnyAst().Returns(MakeAst("x = 1")); indexManager.ProcessFileIfIndexed(new Uri(pythonTestFile), doc); diff --git a/src/Core/Impl/IO/DirectoryInfoProxy.cs b/src/Core/Impl/IO/DirectoryInfoProxy.cs index 6f6689dd3..fea90d9f5 100644 --- a/src/Core/Impl/IO/DirectoryInfoProxy.cs +++ b/src/Core/Impl/IO/DirectoryInfoProxy.cs @@ -48,7 +48,7 @@ public IEnumerable EnumerateFileSystemInfos(string[] includeFil matcher.AddExcludePatterns(excludeFiles ?? Enumerable.Empty()); var matchResult = matcher.Execute(new DirectoryInfoWrapper(_directoryInfo)); return matchResult.Files.Select((filePatternMatch) => { - FileSystemInfo fileSystemInfo = _directoryInfo.GetFileSystemInfos(filePatternMatch.Stem).First(); + var fileSystemInfo = _directoryInfo.GetFileSystemInfos(filePatternMatch.Stem).First(); return CreateFileSystemInfoProxy(fileSystemInfo); }); } From e5791d36905b7609a285f6fca1cd328f224111b2 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 29 Jan 2019 13:26:16 -0800 Subject: [PATCH 012/123] Check ArgumentNotNull using Diagnostics --- src/Analysis/Ast/Impl/Indexing/IndexManager.cs | 14 ++++++++++---- src/Analysis/Ast/Impl/Indexing/IndexParser.cs | 8 ++++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs index 9b1e6ea85..e4849f9df 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; @@ -21,12 +22,17 @@ internal class IndexManager : IIndexManager { public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, string[] excludeFiles) { + Check.ArgumentNotNull(nameof(fileSystem), fileSystem); + Check.ArgumentNotNull(nameof(rootPath), rootPath); + Check.ArgumentNotNull(nameof(includeFiles), includeFiles); + Check.ArgumentNotNull(nameof(excludeFiles), excludeFiles); + _symbolIndex = symbolIndex; - _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + _fileSystem = fileSystem; _indexParser = new IndexParser(symbolIndex, fileSystem, version); - _workspaceRootPath = rootPath ?? throw new ArgumentNullException(nameof(rootPath)); - _includeFiles = includeFiles ?? throw new ArgumentNullException(nameof(rootPath)); - _excludeFiles = excludeFiles ?? throw new ArgumentNullException(nameof(excludeFiles)); + _workspaceRootPath = rootPath; + _includeFiles = includeFiles; + _excludeFiles = excludeFiles; } public Task AddRootDirectory(CancellationToken workspaceCancellationToken = default) { diff --git a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs index 36b70784a..76346f70e 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs @@ -2,6 +2,7 @@ using System.IO; using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; @@ -13,8 +14,11 @@ internal sealed class IndexParser : IIndexParser { private readonly CancellationTokenSource _allProcessingCts = new CancellationTokenSource(); public IndexParser(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version) { - _symbolIndex = symbolIndex ?? throw new ArgumentNullException(nameof(symbolIndex)); - _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + Check.ArgumentNotNull(nameof(symbolIndex), symbolIndex); + Check.ArgumentNotNull(nameof(fileSystem), fileSystem); + + _symbolIndex = symbolIndex; + _fileSystem = fileSystem; _version = version; } From b39ecf49d4541e8e10f73b97b721e660c30960cf Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 29 Jan 2019 14:50:41 -0800 Subject: [PATCH 013/123] Async fixes --- .../Ast/Impl/Indexing/IIndexManager.cs | 8 +++--- .../Ast/Impl/Indexing/ISymbolIndex.cs | 1 + .../Ast/Impl/Indexing/IndexManager.cs | 24 +++++------------ src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs | 2 ++ src/Analysis/Ast/Test/IndexManagerTests.cs | 26 +++++++++---------- 5 files changed, 27 insertions(+), 34 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs index b7c5374a1..3742aad2f 100644 --- a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs @@ -5,9 +5,9 @@ namespace Microsoft.Python.Analysis.Indexing { internal interface IIndexManager : IDisposable { - Task AddRootDirectory(CancellationToken workspaceCancellationToken = default); - void ProcessFile(Uri uri, IDocument doc); - Task ProcessClosedFile(Uri uri, CancellationToken fileCancellationToken = default); - void ProcessFileIfIndexed(Uri uri, IDocument doc); + Task AddRootDirectoryAsync(CancellationToken workspaceCancellationToken = default); + void ProcessNewFile(Uri uri, IDocument doc); + Task ProcessClosedFileAsync(Uri uri, CancellationToken fileCancellationToken = default); + void ReIndexFile(Uri uri, IDocument doc); } } diff --git a/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs index 39fb81e42..624b55c69 100644 --- a/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs @@ -7,5 +7,6 @@ internal interface ISymbolIndex { void UpdateIndex(Uri uri, PythonAst pythonAst); IEnumerable WorkspaceSymbols(string query); void Delete(Uri uri); + bool IsIndexed(Uri uri); } } diff --git a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs index e4849f9df..523ca352c 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -17,7 +16,6 @@ internal class IndexManager : IIndexManager { private readonly string _workspaceRootPath; private readonly string[] _includeFiles; private readonly string[] _excludeFiles; - private readonly ConcurrentDictionary _indexedFiles = new ConcurrentDictionary(); private readonly CancellationTokenSource _allIndexCts = new CancellationTokenSource(); public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, @@ -35,16 +33,13 @@ public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLang _excludeFiles = excludeFiles; } - public Task AddRootDirectory(CancellationToken workspaceCancellationToken = default) { + public Task AddRootDirectoryAsync(CancellationToken workspaceCancellationToken = default) { var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(workspaceCancellationToken, _allIndexCts.Token); var parseTasks = new List(); foreach (var fileInfo in WorkspaceFiles()) { if (ModulePath.IsPythonSourceFile(fileInfo.FullName)) { Uri uri = new Uri(fileInfo.FullName); - parseTasks.Add(_indexParser.ParseAsync(uri, linkedCts.Token).ContinueWith((task) => { - linkedCts.Token.ThrowIfCancellationRequested(); - _indexedFiles[uri] = true; - })); + parseTasks.Add(_indexParser.ParseAsync(uri, linkedCts.Token)); } } return Task.WhenAll(parseTasks.ToArray()); @@ -54,12 +49,9 @@ private IEnumerable WorkspaceFiles() { return _fileSystem.GetDirectoryInfo(_workspaceRootPath).EnumerateFileSystemInfos(_includeFiles, _excludeFiles); } - private bool IsFileIndexed(Uri uri) { - _indexedFiles.TryGetValue(uri, out var val); - return val; - } + private bool IsFileIndexed(Uri uri) => _symbolIndex.IsIndexed(uri); - public Task ProcessClosedFile(Uri uri, CancellationToken fileCancellationToken = default) { + public Task ProcessClosedFileAsync(Uri uri, CancellationToken fileCancellationToken = default) { var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(fileCancellationToken, _allIndexCts.Token); // If path is on workspace if (IsFileOnWorkspace(uri)) { @@ -67,7 +59,6 @@ public Task ProcessClosedFile(Uri uri, CancellationToken fileCancellationToken = return _indexParser.ParseAsync(uri, linkedCts.Token); } else { // remove file from index - _indexedFiles.TryRemove(uri, out _); _symbolIndex.Delete(uri); return Task.CompletedTask; } @@ -77,14 +68,13 @@ private bool IsFileOnWorkspace(Uri uri) { return _fileSystem.IsPathUnderRoot(_workspaceRootPath, uri.AbsolutePath); } - public void ProcessFile(Uri uri, IDocument doc) { - _indexedFiles[uri] = true; + public void ProcessNewFile(Uri uri, IDocument doc) { _symbolIndex.UpdateIndex(uri, doc.GetAnyAst()); } - public void ProcessFileIfIndexed(Uri uri, IDocument doc) { + public void ReIndexFile(Uri uri, IDocument doc) { if (IsFileIndexed(uri)) { - ProcessFile(uri, doc); + ProcessNewFile(uri, doc); } } diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs index 28677c4f4..00f18f4fe 100644 --- a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs @@ -44,5 +44,7 @@ public void UpdateIndex(Uri uri, PythonAst ast) { } public void Delete(Uri uri) => _index.TryRemove(uri, out var _); + + public bool IsIndexed(Uri uri) => _index.ContainsKey(uri); } } diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/Analysis/Ast/Test/IndexManagerTests.cs index a0bf8ed8a..62f2655d5 100644 --- a/src/Analysis/Ast/Test/IndexManagerTests.cs +++ b/src/Analysis/Ast/Test/IndexManagerTests.cs @@ -47,7 +47,7 @@ public async Task AddsRootDirectoryAsync() { _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); - await indexManager.AddRootDirectory(); + await indexManager.AddRootDirectoryAsync(); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(1); @@ -69,7 +69,7 @@ public void IgnoresNonPythonFiles() { AddFileToRootTestFileSystem(nonPythonTestFile); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, this._rootPath, new string[] { }, new string[] { }); - indexManager.AddRootDirectory(); + indexManager.AddRootDirectoryAsync(); _fileSystem.DidNotReceive().FileExists(nonPythonTestFile); } @@ -82,7 +82,7 @@ public void CanOpenFiles() { doc.GetAnyAst().Returns(MakeAst("x = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); - indexManager.ProcessFile(new Uri(pythonTestFile), doc); + indexManager.ProcessNewFile(new Uri(pythonTestFile), doc); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(1); @@ -100,8 +100,8 @@ public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { latestDoc.GetAnyAst().Returns(MakeAst("y = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); - await indexManager.AddRootDirectory(); - indexManager.ProcessFileIfIndexed(new Uri(pythonTestFile), latestDoc); + await indexManager.AddRootDirectoryAsync(); + indexManager.ReIndexFile(new Uri(pythonTestFile), latestDoc); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(1); @@ -117,8 +117,8 @@ public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { doc.GetAnyAst().Returns(MakeAst("x = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); - indexManager.ProcessFile(new Uri(pythonTestFile), doc); - await indexManager.ProcessClosedFile(new Uri(pythonTestFile)); + indexManager.ProcessNewFile(new Uri(pythonTestFile), doc); + await indexManager.ProcessClosedFileAsync(new Uri(pythonTestFile)); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(0); @@ -135,11 +135,11 @@ public async Task CloseWorkspaceFilesReUpdatesIndexAsync() { latestDoc.GetAnyAst().Returns(MakeAst("y = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); - await indexManager.AddRootDirectory(); - indexManager.ProcessFile(new Uri(pythonTestFile), latestDoc); + await indexManager.AddRootDirectoryAsync(); + indexManager.ProcessNewFile(new Uri(pythonTestFile), latestDoc); // It Needs to remake the stream for the file, previous one is closed _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); - await indexManager.ProcessClosedFile(new Uri(pythonTestFile)); + await indexManager.ProcessClosedFileAsync(new Uri(pythonTestFile)); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(1); @@ -158,10 +158,10 @@ public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() { doc.GetAnyAst().Returns(MakeAst("x = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); - indexManager.ProcessFile(new Uri(pythonTestFile), doc); - await indexManager.ProcessClosedFile(new Uri(pythonTestFile)); + indexManager.ProcessNewFile(new Uri(pythonTestFile), doc); + await indexManager.ProcessClosedFileAsync(new Uri(pythonTestFile)); doc.GetAnyAst().Returns(MakeAst("x = 1")); - indexManager.ProcessFileIfIndexed(new Uri(pythonTestFile), doc); + indexManager.ReIndexFile(new Uri(pythonTestFile), doc); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(0); From b767fbd28506cde2c031f2a9bfbb5555e184118c Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 29 Jan 2019 15:00:11 -0800 Subject: [PATCH 014/123] PythonVersions.LatestAvailable3X --- src/Analysis/Ast/Test/IndexManagerTests.cs | 21 ++++++++++++--------- src/Analysis/Ast/Test/IndexParserTests.cs | 18 +++++++++++------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/Analysis/Ast/Test/IndexManagerTests.cs index 62f2655d5..200579a0e 100644 --- a/src/Analysis/Ast/Test/IndexManagerTests.cs +++ b/src/Analysis/Ast/Test/IndexManagerTests.cs @@ -10,6 +10,7 @@ using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; +using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; using NSubstitute; using TestUtilities; @@ -21,6 +22,7 @@ public class IndexManagerTests : AnalysisTestBase { private ISymbolIndex _symbolIndex; private string _rootPath; private List _rootFileList; + private PythonLanguageVersion _pythonLanguageVersion; public TestContext TestContext { get; set; } @@ -30,6 +32,7 @@ public void TestInitialize() { _fileSystem = Substitute.For(); _symbolIndex = new SymbolIndex(); _rootPath = "C:/root"; + _pythonLanguageVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); _rootFileList = new List(); IDirectoryInfo directoryInfo = Substitute.For(); // Doesn't work without 'forAnyArgs' @@ -46,7 +49,7 @@ public async Task AddsRootDirectoryAsync() { AddFileToRootTestFileSystem(pythonTestFile); _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); await indexManager.AddRootDirectoryAsync(); var symbols = _symbolIndex.WorkspaceSymbols(""); @@ -58,7 +61,7 @@ public async Task AddsRootDirectoryAsync() { [TestMethod, Priority(0)] public void NullDirectoryThrowsException() { Action construct = () => { - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, null, new string[] { }, new string[] { }); + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), null, new string[] { }, new string[] { }); }; construct.Should().Throw(); } @@ -68,7 +71,7 @@ public void IgnoresNonPythonFiles() { string nonPythonTestFile = $"{_rootPath}/bla.txt"; AddFileToRootTestFileSystem(nonPythonTestFile); - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, this._rootPath, new string[] { }, new string[] { }); + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), this._rootPath, new string[] { }, new string[] { }); indexManager.AddRootDirectoryAsync(); _fileSystem.DidNotReceive().FileExists(nonPythonTestFile); @@ -81,7 +84,7 @@ public void CanOpenFiles() { IDocument doc = Substitute.For(); doc.GetAnyAst().Returns(MakeAst("x = 1")); - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), _rootPath, new string[] { }, new string[] { }); indexManager.ProcessNewFile(new Uri(pythonTestFile), doc); var symbols = _symbolIndex.WorkspaceSymbols(""); @@ -99,7 +102,7 @@ public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { IDocument latestDoc = Substitute.For(); latestDoc.GetAnyAst().Returns(MakeAst("y = 1")); - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), _rootPath, new string[] { }, new string[] { }); await indexManager.AddRootDirectoryAsync(); indexManager.ReIndexFile(new Uri(pythonTestFile), latestDoc); @@ -116,7 +119,7 @@ public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { IDocument doc = Substitute.For(); doc.GetAnyAst().Returns(MakeAst("x = 1")); - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), _rootPath, new string[] { }, new string[] { }); indexManager.ProcessNewFile(new Uri(pythonTestFile), doc); await indexManager.ProcessClosedFileAsync(new Uri(pythonTestFile)); @@ -134,7 +137,7 @@ public async Task CloseWorkspaceFilesReUpdatesIndexAsync() { IDocument latestDoc = Substitute.For(); latestDoc.GetAnyAst().Returns(MakeAst("y = 1")); - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), _rootPath, new string[] { }, new string[] { }); await indexManager.AddRootDirectoryAsync(); indexManager.ProcessNewFile(new Uri(pythonTestFile), latestDoc); // It Needs to remake the stream for the file, previous one is closed @@ -157,7 +160,7 @@ public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() { IDocument doc = Substitute.For(); doc.GetAnyAst().Returns(MakeAst("x = 1")); - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonLanguageVersion.V37, _rootPath, new string[] { }, new string[] { }); + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), _rootPath, new string[] { }, new string[] { }); indexManager.ProcessNewFile(new Uri(pythonTestFile), doc); await indexManager.ProcessClosedFileAsync(new Uri(pythonTestFile)); doc.GetAnyAst().Returns(MakeAst("x = 1")); @@ -168,7 +171,7 @@ public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() { } private PythonAst MakeAst(string testCode) { - return Parser.CreateParser(MakeStream(testCode), PythonLanguageVersion.V37).ParseFile(); + return Parser.CreateParser(MakeStream(testCode), PythonVersions.LatestAvailable3X.Version.ToLanguageVersion()).ParseFile(); } private void AddFileToRootTestFileSystem(string filePath) { diff --git a/src/Analysis/Ast/Test/IndexParserTests.cs b/src/Analysis/Ast/Test/IndexParserTests.cs index 326b2688d..d38c5d0c8 100644 --- a/src/Analysis/Ast/Test/IndexParserTests.cs +++ b/src/Analysis/Ast/Test/IndexParserTests.cs @@ -8,6 +8,7 @@ using Microsoft.Python.Analysis.Indexing; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; using NSubstitute; using TestUtilities; @@ -16,15 +17,18 @@ namespace Microsoft.Python.Analysis.Tests { [TestClass] public class IndexParserTests : AnalysisTestBase { + private ISymbolIndex _symbolIndex; + private IFileSystem _fileSystem; + private PythonLanguageVersion _pythonLanguageVersion; + public TestContext TestContext { get; set; } - ISymbolIndex _symbolIndex; - IFileSystem _fileSystem; [TestInitialize] public void TestInitialize() { TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); _symbolIndex = new SymbolIndex(); _fileSystem = Substitute.For(); + _pythonLanguageVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); } [TestCleanup] @@ -33,7 +37,7 @@ public void TestInitialize() { [TestMethod, Priority(0)] public void NullIndexThrowsException() { Action build = () => { - IIndexParser indexParser = new IndexParser(null, _fileSystem, PythonLanguageVersion.V37); + IIndexParser indexParser = new IndexParser(null, _fileSystem, _pythonLanguageVersion); }; build.Should().Throw(); } @@ -44,7 +48,7 @@ public async Task ParseVariableInFileAsync() { _fileSystem.FileExists(testFilePath).Returns(true); _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(MakeStream("x = 1")); - IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, PythonLanguageVersion.V37); + IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); await indexParser.ParseAsync(new Uri(testFilePath)); var symbols = _symbolIndex.WorkspaceSymbols(""); @@ -58,7 +62,7 @@ public void ParseNonexistentFile() { const string testFilePath = "C:/bla.py"; _fileSystem.FileExists(testFilePath).Returns(false); - IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, PythonLanguageVersion.V37); + IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); Func parse = async () => { await indexParser.ParseAsync(new Uri(testFilePath)); }; @@ -74,7 +78,7 @@ public void CancellParsing() { _fileSystem.FileExists(testFilePath).Returns(true); _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(MakeStream("x = 1")); - IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, PythonLanguageVersion.V37); + IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); cancellationTokenSource.Cancel(); Func parse = async () => { @@ -94,7 +98,7 @@ public void DisposeParserCancelesParsing() { // no writing to the stream, will block when read _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(stream); - IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, PythonLanguageVersion.V37); + IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); Func parse = async () => { Task t = indexParser.ParseAsync(new Uri(testFilePath)); indexParser.Dispose(); From 703de21dd63dff43fdec15a2e41c523fa43fb5f3 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 30 Jan 2019 13:07:16 -0800 Subject: [PATCH 015/123] SymbolIndex fixes --- src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs | 25 +++++++------------ src/Analysis/Ast/Impl/Indexing/Symbols.cs | 4 +++ .../Impl/Extensions/EnumerableExtensions.cs | 20 +++++++-------- 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs index 00f18f4fe..ce6a26917 100644 --- a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs @@ -13,26 +13,19 @@ public IEnumerable HierarchicalDocumentSymbols(Uri uri) => _index.TryGetValue(uri, out var list) ? list : Enumerable.Empty(); public IEnumerable WorkspaceSymbols(string query) { - foreach (var kvp in _index) { - foreach (var found in WorkspaceSymbolsQuery(query, kvp.Key, kvp.Value)) { - yield return found; - } - } + return _index.SelectMany(kvp => WorkspaceSymbolsQuery(query, kvp.Key, kvp.Value)); } private IEnumerable WorkspaceSymbolsQuery(string query, Uri uri, IEnumerable symbols) { - // Some semblance of a BFS. - var queue = new Queue<(HierarchicalSymbol, string)>(symbols.Select(s => (s, (string)null))); - - while (queue.Count > 0) { - var (sym, parent) = queue.Dequeue(); - + var rootSymbols = symbols.Select((symbol) => { + return (symbol, parentName: (string) null); + }); + var treeSymbols = rootSymbols.TraverseBreadthFirst((symbolAndParent) => { + return symbolAndParent.symbol.ChildrenWithParentName(); + }); + foreach (var (sym, parentName) in treeSymbols) { if (sym.Name.ContainsOrdinal(query, ignoreCase: true)) { - yield return new FlatSymbol(sym.Name, sym.Kind, uri, sym.SelectionRange, parent); - } - - foreach (var child in sym.Children.MaybeEnumerate()) { - queue.Enqueue((child, sym.Name)); + yield return new FlatSymbol(sym.Name, sym.Kind, uri, sym.SelectionRange, parentName); } } } diff --git a/src/Analysis/Ast/Impl/Indexing/Symbols.cs b/src/Analysis/Ast/Impl/Indexing/Symbols.cs index 8324a07a0..78779c384 100644 --- a/src/Analysis/Ast/Impl/Indexing/Symbols.cs +++ b/src/Analysis/Ast/Impl/Indexing/Symbols.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Core; using Microsoft.Python.Core.Text; namespace Microsoft.Python.Analysis.Indexing { @@ -70,6 +72,8 @@ public HierarchicalSymbol( Children = children; _functionKind = functionKind; } + + public IEnumerable<(HierarchicalSymbol, string)> ChildrenWithParentName() => Children.MaybeEnumerate().Select((child) => (child, Name)); } // Analagous to LSP's SymbolInformation. diff --git a/src/Core/Impl/Extensions/EnumerableExtensions.cs b/src/Core/Impl/Extensions/EnumerableExtensions.cs index f64014627..b2dd6320a 100644 --- a/src/Core/Impl/Extensions/EnumerableExtensions.cs +++ b/src/Core/Impl/Extensions/EnumerableExtensions.cs @@ -90,19 +90,19 @@ public static IEnumerable IndexWhere(this IEnumerable source, Func TraverseBreadthFirst(this T root, Func> selectChildren) { - var items = new Queue(); - items.Enqueue(root); - while (items.Count > 0) { - var item = items.Dequeue(); - yield return item; + return new List() { root }.TraverseBreadthFirst(selectChildren); + } - var children = selectChildren(item); - if (children == null) { - continue; - } + public static IEnumerable TraverseBreadthFirst(this IEnumerable roots, Func> selectChildren) { + var queue = new Queue(roots); + + while (!queue.IsNullOrEmpty()) { + var item = queue.Dequeue(); + yield return item; + var children = selectChildren(item) ?? new List(); foreach (var child in children) { - items.Enqueue(child); + queue.Enqueue(child); } } } From af8596667b1ebf7654fad4f8d58ec47d6a4ba2e5 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 30 Jan 2019 13:21:48 -0800 Subject: [PATCH 016/123] SymbolIndex moving method from HierarchicalSymbol --- src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs | 5 ++++- src/Analysis/Ast/Impl/Indexing/Symbols.cs | 2 -- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs index ce6a26917..0d294e489 100644 --- a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs @@ -21,7 +21,10 @@ private IEnumerable WorkspaceSymbolsQuery(string query, Uri uri, IEn return (symbol, parentName: (string) null); }); var treeSymbols = rootSymbols.TraverseBreadthFirst((symbolAndParent) => { - return symbolAndParent.symbol.ChildrenWithParentName(); + var symbol = symbolAndParent.symbol; + return symbol.Children.MaybeEnumerate().Select((child) => { + return (child, symbol.Name); + }); }); foreach (var (sym, parentName) in treeSymbols) { if (sym.Name.ContainsOrdinal(query, ignoreCase: true)) { diff --git a/src/Analysis/Ast/Impl/Indexing/Symbols.cs b/src/Analysis/Ast/Impl/Indexing/Symbols.cs index 78779c384..177e21636 100644 --- a/src/Analysis/Ast/Impl/Indexing/Symbols.cs +++ b/src/Analysis/Ast/Impl/Indexing/Symbols.cs @@ -72,8 +72,6 @@ public HierarchicalSymbol( Children = children; _functionKind = functionKind; } - - public IEnumerable<(HierarchicalSymbol, string)> ChildrenWithParentName() => Children.MaybeEnumerate().Select((child) => (child, Name)); } // Analagous to LSP's SymbolInformation. From 2c7dc2d75eac9219c1570379a1f5bd844bb7e2ee Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 30 Jan 2019 13:31:59 -0800 Subject: [PATCH 017/123] SymbolIndex corrections --- src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs | 16 +++++++++------- src/Core/Impl/Extensions/EnumerableExtensions.cs | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs index 0d294e489..e5a40ae1f 100644 --- a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs @@ -17,14 +17,10 @@ public IEnumerable WorkspaceSymbols(string query) { } private IEnumerable WorkspaceSymbolsQuery(string query, Uri uri, IEnumerable symbols) { - var rootSymbols = symbols.Select((symbol) => { - return (symbol, parentName: (string) null); - }); + var rootSymbols = DecorateWithParentsName(symbols, null); var treeSymbols = rootSymbols.TraverseBreadthFirst((symbolAndParent) => { - var symbol = symbolAndParent.symbol; - return symbol.Children.MaybeEnumerate().Select((child) => { - return (child, symbol.Name); - }); + var sym = symbolAndParent.symbol; + return DecorateWithParentsName(sym.Children.MaybeEnumerate(), sym.Name); }); foreach (var (sym, parentName) in treeSymbols) { if (sym.Name.ContainsOrdinal(query, ignoreCase: true)) { @@ -42,5 +38,11 @@ public void UpdateIndex(Uri uri, PythonAst ast) { public void Delete(Uri uri) => _index.TryRemove(uri, out var _); public bool IsIndexed(Uri uri) => _index.ContainsKey(uri); + + private static IEnumerable<(HierarchicalSymbol symbol, string parentName)> DecorateWithParentsName(IEnumerable symbols, string parentName) { + return symbols.Select((symbol) => { + return (symbol, parentName); + }); + } } } diff --git a/src/Core/Impl/Extensions/EnumerableExtensions.cs b/src/Core/Impl/Extensions/EnumerableExtensions.cs index b2dd6320a..3aa9ca645 100644 --- a/src/Core/Impl/Extensions/EnumerableExtensions.cs +++ b/src/Core/Impl/Extensions/EnumerableExtensions.cs @@ -90,7 +90,7 @@ public static IEnumerable IndexWhere(this IEnumerable source, Func TraverseBreadthFirst(this T root, Func> selectChildren) { - return new List() { root }.TraverseBreadthFirst(selectChildren); + return new[] { root }.TraverseBreadthFirst(selectChildren); } public static IEnumerable TraverseBreadthFirst(this IEnumerable roots, Func> selectChildren) { @@ -100,7 +100,7 @@ public static IEnumerable TraverseBreadthFirst(this IEnumerable roots, var item = queue.Dequeue(); yield return item; - var children = selectChildren(item) ?? new List(); + var children = selectChildren(item) ?? Enumerable.Empty(); foreach (var child in children) { queue.Enqueue(child); } From 26bef41ef3245fae37e518535f2011593e551919 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 30 Jan 2019 13:36:17 -0800 Subject: [PATCH 018/123] SymbolIndex return style --- src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs index e5a40ae1f..d71b64b97 100644 --- a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs @@ -40,9 +40,7 @@ public void UpdateIndex(Uri uri, PythonAst ast) { public bool IsIndexed(Uri uri) => _index.ContainsKey(uri); private static IEnumerable<(HierarchicalSymbol symbol, string parentName)> DecorateWithParentsName(IEnumerable symbols, string parentName) { - return symbols.Select((symbol) => { - return (symbol, parentName); - }); + return symbols.Select((symbol) => (symbol, parentName)); } } } From 93d046437a699a139bf6229b13708cac030aa28a Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 30 Jan 2019 14:10:14 -0800 Subject: [PATCH 019/123] Adding symbolIndex tests --- src/Analysis/Ast/Test/SymbolIndexTests.cs | 728 ++++++++++++++++++ .../CollectionAssertionsExtensions.cs | 4 + 2 files changed, 732 insertions(+) create mode 100644 src/Analysis/Ast/Test/SymbolIndexTests.cs diff --git a/src/Analysis/Ast/Test/SymbolIndexTests.cs b/src/Analysis/Ast/Test/SymbolIndexTests.cs new file mode 100644 index 000000000..ed4fe7deb --- /dev/null +++ b/src/Analysis/Ast/Test/SymbolIndexTests.cs @@ -0,0 +1,728 @@ +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.IO; +using FluentAssertions; +using Microsoft.Python.Analysis.Indexing; +using Microsoft.Python.Core.Text; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; +using Microsoft.Python.Tests.Utilities.FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class SymbolIndexTests { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() { + TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + } + + [TestCleanup] + public void TestCleanup() { + TestEnvironmentImpl.TestCleanup(); + } + + [TestMethod, Priority(0)] + public void WalkerAssignments() { + var code = @"x = 1 +y = x +z = y"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 1, 2, 2)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(3, 1, 3, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerMultipleAssignments() { + var code = @"x = y = z = 1"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 5, 1, 6)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(1, 9, 1, 10)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerUnderscore() { + var code = @"_ = 1"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public void WalkerIfStatement() { + var code = @"if foo(): + x = 1 +elif bar(): + x = 2 +else: + y = 3 +"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(4, 5, 4, 6)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(6, 5, 6, 6)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerTryExceptFinally() { + var code = @"try: + x = 1 +except Exception: + x = 2 +finally: + y = 3 +"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(4, 5, 4, 6)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(6, 5, 6, 6)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerReassign() { + var code = @"x = 1 +x = 2"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerAugmentedAssign() { + var code = @"x += 1"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerTopLevelConstant() { + var code = @"FOO_BAR_3 = 1234"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("FOO_BAR_3", SymbolKind.Constant, new SourceSpan(1, 1, 1, 10)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerFunction() { + var code = @"def func(x, y): + z = x + y + return z"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 3, 13), new SourceSpan(1, 5, 1, 9), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 10, 1, 11)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerFunctionStarredArgs() { + var code = @"def func(*args, **kwargs): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 31), new SourceSpan(1, 5, 1, 9), new[] { + new HierarchicalSymbol("args", SymbolKind.Variable, new SourceSpan(1, 11, 1, 15)), + new HierarchicalSymbol("kwargs", SymbolKind.Variable, new SourceSpan(1, 19, 1, 25)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerFunctionUnderscoreArg() { + var code = @"def func(_): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 17), new SourceSpan(1, 5, 1, 9), new List(), FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerImports() { + var code = @"import sys +import numpy as np +from os.path import join as osjoin +from os.path import ( join as osjoin2, exists as osexists, expanduser ) +"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("sys", SymbolKind.Module, new SourceSpan(1, 8, 1, 11)), + new HierarchicalSymbol("np", SymbolKind.Module, new SourceSpan(2, 17, 2, 19)), + new HierarchicalSymbol("osjoin", SymbolKind.Module, new SourceSpan(3, 29, 3, 35)), + new HierarchicalSymbol("osjoin2", SymbolKind.Module, new SourceSpan(4, 31, 4, 38)), + new HierarchicalSymbol("osexists", SymbolKind.Module, new SourceSpan(4, 50, 4, 58)), + new HierarchicalSymbol("expanduser", SymbolKind.Module, new SourceSpan(4, 60, 4, 70)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerImportFromFuture() { + var code = @"from __future__ import print_function"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public void WalkerClass() { + var code = @"class Foo(object): + ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 8), new SourceSpan(1, 7, 1, 10), new List(), FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassConstant() { + var code = @"class Foo(object): + CONSTANT = 1234"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 20), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("CONSTANT", SymbolKind.Constant, new SourceSpan(2, 5, 2, 13)), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerConstructor() { + var code = @"class Foo(object): + def __init__(self, x): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 31), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("__init__", SymbolKind.Constructor, new SourceSpan(2, 5, 2, 31), new SourceSpan(2, 9, 2, 17), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 18, 2, 22)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 24, 2, 25)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerMethod() { + var code = @"class Foo(object): + def foo(self, x): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 26), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("foo", SymbolKind.Method, new SourceSpan(2, 5, 2, 26), new SourceSpan(2, 9, 2, 12), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 13, 2, 17)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 19, 2, 20)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerDoubleUnderscoreMethod() { + var code = @"class Foo(object): + def __lt__(self, x): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 29), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("__lt__", SymbolKind.Operator, new SourceSpan(2, 5, 2, 29), new SourceSpan(2, 9, 2, 15), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 16, 2, 20)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 22, 2, 23)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerProperties() { + var code = @"class Foo(object): + @property + def func1(self): ... + + @abstractproperty + def func2(self): ... + + @classproperty + def func3(self): ... + + @abstractclassproperty + def func4(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 12, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func3", SymbolKind.Property, new SourceSpan(8, 5, 9, 25), new SourceSpan(9, 9, 9, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(9, 15, 9, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func4", SymbolKind.Property, new SourceSpan(11, 5, 12, 25), new SourceSpan(12, 9, 12, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(12, 15, 12, 19)), + }, FunctionKind.Property), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerAbcProperties() { + var code = @"class Foo(object): + @abc.abstractproperty + def func1(self): ... + + @abc.abstractclassproperty + def func2(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 6, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), + }, FunctionKind.Property), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerStaticMethods() { + var code = @"class Foo(object): + @staticmethod + def func1(arg): ... + + @abstractstaticmethod + def func2(arg): ... + + @abc.abstractstaticmethod + def func3(arg): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 9, 24), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 24), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(3, 15, 3, 18)), + }, FunctionKind.StaticMethod), + new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 24), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(6, 15, 6, 18)), + }, FunctionKind.StaticMethod), + new HierarchicalSymbol("func3", SymbolKind.Method, new SourceSpan(8, 5, 9, 24), new SourceSpan(9, 9, 9, 14), new[] { + new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(9, 15, 9, 18)), + }, FunctionKind.StaticMethod), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassMethods() { + var code = @"class Foo(object): + @classmethod + def func1(cls): ... + + @abstractclassmethod + def func2(cls): ... + + @abc.abstractclassmethod + def func3(cls): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 9, 24), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 24), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(3, 15, 3, 18)), + }, FunctionKind.ClassMethod), + new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 24), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(6, 15, 6, 18)), + }, FunctionKind.ClassMethod), + new HierarchicalSymbol("func3", SymbolKind.Method, new SourceSpan(8, 5, 9, 24), new SourceSpan(9, 9, 9, 14), new[] { + new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(9, 15, 9, 18)), + }, FunctionKind.ClassMethod), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerTopLevelFunctionDecorator() { + var code = @"@something +def func1(x, y): ... + +@something_else() +def func2(x, y): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func1", SymbolKind.Function, new SourceSpan(1, 1, 2, 21), new SourceSpan(2, 5, 2, 10), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 11, 2, 12)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 14, 2, 15)), + }, FunctionKind.Function), + new HierarchicalSymbol("func2", SymbolKind.Function, new SourceSpan(4, 1, 5, 21), new SourceSpan(5, 5, 5, 10), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(5, 11, 5, 12)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(5, 14, 5, 15)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassFunctionDecorator() { + var code = @"class Foo(object): + @something + def func1(self): ... + + @something_else() + def func2(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 6, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), + }, FunctionKind.Function), + new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassFunctionMultiDecorator() { + var code = @"class Foo(object): + @property + @something + def func1(self): ... + + @something + @property + def func2(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 8, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 4, 25), new SourceSpan(4, 9, 4, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(4, 15, 4, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(6, 5, 8, 25), new SourceSpan(8, 9, 8, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(8, 15, 8, 19)), + }, FunctionKind.Property), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerLambda() { + var code = @"f = lambda x, y: x + y"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.Function, new SourceSpan(1, 5, 1, 23), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 12, 1, 13)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 15, 1, 16)), + }, functionKind: FunctionKind.Function), + new HierarchicalSymbol("f", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerForLoop() { + var code = @"z = False +for [x, y, (p, q)] in [[1, 2, [3, 4]]]: + z += x +else: + z = None"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 6, 2, 7)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 9, 2, 10)), + new HierarchicalSymbol("p", SymbolKind.Variable, new SourceSpan(2, 13, 2, 14)), + new HierarchicalSymbol("q", SymbolKind.Variable, new SourceSpan(2, 16, 2, 17)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(3, 5, 3, 6)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(5, 5, 5, 6)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerListComprehension() { + var code = @"flat_list = [item for sublist in l for item in sublist]"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 13, 1, 56), children: new[] { + new HierarchicalSymbol("sublist", SymbolKind.Variable, new SourceSpan(1, 23, 1, 30)), + new HierarchicalSymbol("item", SymbolKind.Variable, new SourceSpan(1, 40, 1, 44)), + }), + new HierarchicalSymbol("flat_list", SymbolKind.Variable, new SourceSpan(1, 1, 1, 10)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerDictionaryComprehension() { + var code = @"d = { x: y for x, y in zip(range(10), range(10)) }"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 51), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 19, 1, 20)), + }), + new HierarchicalSymbol("d", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerSetComprehension() { + var code = @"s = { x for x in range(10) }"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 29), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + }), + new HierarchicalSymbol("s", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerGenerator() { + var code = @"g = (x + y for x, y in zip(range(10), range(10)))"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 50), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 19, 1, 20)), + }), + new HierarchicalSymbol("g", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerNestedListComprehension() { + var code = @"l = [ + x for x in [ + y for y in range(10) + ] +]"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 5, 2), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 11, 2, 12)), + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(2, 16, 4, 6), children: new[] { + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(3, 15, 3, 16)), + }), + }), + new HierarchicalSymbol("l", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerIncompleteFunction() { + var code = @"def func(x, y):"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 16), new SourceSpan(1, 5, 1, 9), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 10, 1, 11)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerIncompleteClass() { + var code = @"class Foo(object):"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 1, 19), new SourceSpan(1, 7, 1, 10), new List(), FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerIncompleteAssign() { + var code = @"x ="; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerAugmentedAssignLambda() { + var code = @"x += lambda x, y: x + y"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.Function, new SourceSpan(1, 6, 1, 24), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), + }, functionKind: FunctionKind.Function), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void IndexHierarchicalDocument() { + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + var ast = GetParse("x = 1"); + index.UpdateIndex(uri, ast); + + var symbols = index.HierarchicalDocumentSymbols(uri); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void IndexHierarchicalDocumentUpdate() { + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + var ast = GetParse("x = 1"); + index.UpdateIndex(uri, ast); + + var symbols = index.HierarchicalDocumentSymbols(uri); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + + ast = GetParse("y = 1"); + index.UpdateIndex(uri, ast); + + symbols = index.HierarchicalDocumentSymbols(uri); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void IndexHierarchicalDocumentNotFound() { + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + + var symbols = index.HierarchicalDocumentSymbols(uri); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public void IndexWorkspaceSymbolsFlatten() { + var code = @"class Foo(object): + def foo(self, x): ..."; + + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + var ast = GetParse(code); + index.UpdateIndex(uri, ast); + + var symbols = index.WorkspaceSymbols(""); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("Foo", SymbolKind.Class, uri, new SourceSpan(1, 7, 1, 10)), + new FlatSymbol("foo", SymbolKind.Method, uri, new SourceSpan(2, 9, 2, 12), "Foo"), + new FlatSymbol("self", SymbolKind.Variable, uri, new SourceSpan(2, 13, 2, 17), "foo"), + new FlatSymbol("x", SymbolKind.Variable, uri, new SourceSpan(2, 19, 2, 20), "foo"), + }); + } + + [TestMethod, Priority(0)] + public void IndexWorkspaceSymbolsFiltered() { + var code = @"class Foo(object): + def foo(self, x): ..."; + + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + var ast = GetParse(code); + index.UpdateIndex(uri, ast); + + var symbols = index.WorkspaceSymbols("x"); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("x", SymbolKind.Variable, uri, new SourceSpan(2, 19, 2, 20), "foo"), + }); + } + + [TestMethod, Priority(0)] + public void IndexWorkspaceSymbolsNotFound() { + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + + var symbols = index.WorkspaceSymbols(""); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public void IndexWorkspaceSymbolsCaseInsensitive() { + var code = @"class Foo(object): + def foo(self, x): ..."; + + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + var ast = GetParse(code); + index.UpdateIndex(uri, ast); + + var symbols = index.WorkspaceSymbols("foo"); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("Foo", SymbolKind.Class, uri, new SourceSpan(1, 7, 1, 10)), + new FlatSymbol("foo", SymbolKind.Method, uri, new SourceSpan(2, 9, 2, 12), "Foo"), + }); + } + + private PythonAst GetParse(string code, PythonLanguageVersion version = PythonLanguageVersion.V37) + => Parser.CreateParser(new StringReader(code), version).ParseFile(); + + private IReadOnlyList WalkSymbols(string code, PythonLanguageVersion version = PythonLanguageVersion.V37) { + var ast = GetParse(code); + var walker = new SymbolIndexWalker(ast); + ast.Walk(walker); + return walker.Symbols; + } + } +} diff --git a/src/UnitTests/Core/Impl/FluentAssertions/CollectionAssertionsExtensions.cs b/src/UnitTests/Core/Impl/FluentAssertions/CollectionAssertionsExtensions.cs index 4ddcf50d9..d91e6d1a0 100644 --- a/src/UnitTests/Core/Impl/FluentAssertions/CollectionAssertionsExtensions.cs +++ b/src/UnitTests/Core/Impl/FluentAssertions/CollectionAssertionsExtensions.cs @@ -35,5 +35,9 @@ public static AndConstraint OnlyContain( public static AndConstraint OnlyContain( this StringCollectionAssertions assertions, IReadOnlyCollection expected, string because = "", params object[] reasonArgs) => assertions.HaveCount(expected.Count, because, reasonArgs).And.Contain(expected, because, reasonArgs); + + public static AndConstraint> BeEquivalentToWithStrictOrdering( + this GenericCollectionAssertions assertions, IEnumerable expected, string because = "", params object[] reasonArgs) + => assertions.BeEquivalentTo(expected, options => options.WithStrictOrdering(), because, reasonArgs); } } From c93c496850cdbf5fcb05a9404aa6f4c3ceb805ae Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 30 Jan 2019 14:23:05 -0800 Subject: [PATCH 020/123] SymbolIndex return style --- src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs index d71b64b97..d932a281a 100644 --- a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs @@ -39,8 +39,7 @@ public void UpdateIndex(Uri uri, PythonAst ast) { public bool IsIndexed(Uri uri) => _index.ContainsKey(uri); - private static IEnumerable<(HierarchicalSymbol symbol, string parentName)> DecorateWithParentsName(IEnumerable symbols, string parentName) { - return symbols.Select((symbol) => (symbol, parentName)); - } + private static IEnumerable<(HierarchicalSymbol symbol, string parentName)> DecorateWithParentsName(IEnumerable symbols, string parentName) + => symbols.Select((symbol) => (symbol, parentName)); } } From 28685c6a830ccd3120f55846383522434ca0fe30 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 30 Jan 2019 15:57:15 -0800 Subject: [PATCH 021/123] Using strings (plain path) instead of Uri --- .../Ast/Impl/Indexing/IIndexManager.cs | 6 +- .../Ast/Impl/Indexing/IIndexParser.cs | 2 +- .../Ast/Impl/Indexing/ISymbolIndex.cs | 6 +- .../Ast/Impl/Indexing/IndexManager.cs | 27 ++++---- src/Analysis/Ast/Impl/Indexing/IndexParser.cs | 10 +-- src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs | 24 +++++--- src/Analysis/Ast/Test/IndexManagerTests.cs | 61 ++++++++++--------- src/Analysis/Ast/Test/IndexParserTests.cs | 8 +-- src/Analysis/Ast/Test/SymbolIndexTests.cs | 39 ++++++------ 9 files changed, 96 insertions(+), 87 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs index 3742aad2f..daa2639b5 100644 --- a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs @@ -6,8 +6,8 @@ namespace Microsoft.Python.Analysis.Indexing { internal interface IIndexManager : IDisposable { Task AddRootDirectoryAsync(CancellationToken workspaceCancellationToken = default); - void ProcessNewFile(Uri uri, IDocument doc); - Task ProcessClosedFileAsync(Uri uri, CancellationToken fileCancellationToken = default); - void ReIndexFile(Uri uri, IDocument doc); + void ProcessNewFile(string path, IDocument doc); + Task ProcessClosedFileAsync(string path, CancellationToken fileCancellationToken = default); + void ReIndexFile(string path, IDocument doc); } } diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs b/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs index 659739e98..b60b47060 100644 --- a/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs +++ b/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs @@ -4,6 +4,6 @@ namespace Microsoft.Python.Analysis.Indexing { internal interface IIndexParser : IDisposable { - Task ParseAsync(Uri uri, CancellationToken cancellationToken = default); + Task ParseAsync(string path, CancellationToken cancellationToken = default); } } diff --git a/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs index 624b55c69..9968fc8f2 100644 --- a/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs @@ -4,9 +4,9 @@ namespace Microsoft.Python.Analysis.Indexing { internal interface ISymbolIndex { - void UpdateIndex(Uri uri, PythonAst pythonAst); + void UpdateIndex(string path, PythonAst pythonAst); IEnumerable WorkspaceSymbols(string query); - void Delete(Uri uri); - bool IsIndexed(Uri uri); + void Delete(string path); + bool IsIndexed(string path); } } diff --git a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs index 523ca352c..0716f2c1f 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs @@ -38,8 +38,7 @@ public Task AddRootDirectoryAsync(CancellationToken workspaceCancellationToken = var parseTasks = new List(); foreach (var fileInfo in WorkspaceFiles()) { if (ModulePath.IsPythonSourceFile(fileInfo.FullName)) { - Uri uri = new Uri(fileInfo.FullName); - parseTasks.Add(_indexParser.ParseAsync(uri, linkedCts.Token)); + parseTasks.Add(_indexParser.ParseAsync(fileInfo.FullName, linkedCts.Token)); } } return Task.WhenAll(parseTasks.ToArray()); @@ -49,32 +48,32 @@ private IEnumerable WorkspaceFiles() { return _fileSystem.GetDirectoryInfo(_workspaceRootPath).EnumerateFileSystemInfos(_includeFiles, _excludeFiles); } - private bool IsFileIndexed(Uri uri) => _symbolIndex.IsIndexed(uri); + private bool IsFileIndexed(string path) => _symbolIndex.IsIndexed(path); - public Task ProcessClosedFileAsync(Uri uri, CancellationToken fileCancellationToken = default) { + public Task ProcessClosedFileAsync(string path, CancellationToken fileCancellationToken = default) { var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(fileCancellationToken, _allIndexCts.Token); // If path is on workspace - if (IsFileOnWorkspace(uri)) { + if (IsFileOnWorkspace(path)) { // updates index and ignores previous AST - return _indexParser.ParseAsync(uri, linkedCts.Token); + return _indexParser.ParseAsync(path, linkedCts.Token); } else { // remove file from index - _symbolIndex.Delete(uri); + _symbolIndex.Delete(path); return Task.CompletedTask; } } - private bool IsFileOnWorkspace(Uri uri) { - return _fileSystem.IsPathUnderRoot(_workspaceRootPath, uri.AbsolutePath); + private bool IsFileOnWorkspace(string path) { + return _fileSystem.IsPathUnderRoot(_workspaceRootPath, path); } - public void ProcessNewFile(Uri uri, IDocument doc) { - _symbolIndex.UpdateIndex(uri, doc.GetAnyAst()); + public void ProcessNewFile(string path, IDocument doc) { + _symbolIndex.UpdateIndex(path, doc.GetAnyAst()); } - public void ReIndexFile(Uri uri, IDocument doc) { - if (IsFileIndexed(uri)) { - ProcessNewFile(uri, doc); + public void ReIndexFile(string path, IDocument doc) { + if (IsFileIndexed(path)) { + ProcessNewFile(path, doc); } } diff --git a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs index 76346f70e..edf31c96d 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs @@ -27,17 +27,17 @@ public void Dispose() { _allProcessingCts.Dispose(); } - public Task ParseAsync(Uri uri, CancellationToken parseCancellationToken = default) { + public Task ParseAsync(string path, CancellationToken parseCancellationToken = default) { var linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, parseCancellationToken); var linkedParseToken = linkedParseCts.Token; return Task.Run(() => { - if (!_fileSystem.FileExists(uri.AbsolutePath)) { - throw new FileNotFoundException($"{uri.AbsolutePath} does not exist", uri.AbsolutePath); + if (!_fileSystem.FileExists(path)) { + throw new FileNotFoundException($"{path} does not exist", path); } - using (var stream = _fileSystem.FileOpen(uri.AbsolutePath, FileMode.Open)) { + using (var stream = _fileSystem.FileOpen(path, FileMode.Open)) { var parser = Parser.CreateParser(stream, _version); linkedParseToken.ThrowIfCancellationRequested(); - _symbolIndex.UpdateIndex(uri, parser.ParseFile()); + _symbolIndex.UpdateIndex(path, parser.ParseFile()); } }, linkedParseToken); } diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs index d932a281a..b8d447eb9 100644 --- a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs @@ -3,20 +3,26 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Python.Core; +using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Indexing { internal sealed class SymbolIndex : ISymbolIndex { - private readonly ConcurrentDictionary> _index = new ConcurrentDictionary>(); + private readonly ConcurrentDictionary> _index; - public IEnumerable HierarchicalDocumentSymbols(Uri uri) - => _index.TryGetValue(uri, out var list) ? list : Enumerable.Empty(); + public SymbolIndex(IEqualityComparer comparer = null) { + comparer = comparer ?? PathEqualityComparer.Instance; + _index = new ConcurrentDictionary>(comparer); + } + + public IEnumerable HierarchicalDocumentSymbols(string path) + => _index.TryGetValue(path, out var list) ? list : Enumerable.Empty(); public IEnumerable WorkspaceSymbols(string query) { return _index.SelectMany(kvp => WorkspaceSymbolsQuery(query, kvp.Key, kvp.Value)); } - private IEnumerable WorkspaceSymbolsQuery(string query, Uri uri, IEnumerable symbols) { + private IEnumerable WorkspaceSymbolsQuery(string query, string path, IEnumerable symbols) { var rootSymbols = DecorateWithParentsName(symbols, null); var treeSymbols = rootSymbols.TraverseBreadthFirst((symbolAndParent) => { var sym = symbolAndParent.symbol; @@ -24,20 +30,20 @@ private IEnumerable WorkspaceSymbolsQuery(string query, Uri uri, IEn }); foreach (var (sym, parentName) in treeSymbols) { if (sym.Name.ContainsOrdinal(query, ignoreCase: true)) { - yield return new FlatSymbol(sym.Name, sym.Kind, uri, sym.SelectionRange, parentName); + yield return new FlatSymbol(sym.Name, sym.Kind, new Uri(path), sym.SelectionRange, parentName); } } } - public void UpdateIndex(Uri uri, PythonAst ast) { + public void UpdateIndex(string path, PythonAst ast) { var walker = new SymbolIndexWalker(ast); ast.Walk(walker); - _index[uri] = walker.Symbols; + _index[path] = walker.Symbols; } - public void Delete(Uri uri) => _index.TryRemove(uri, out var _); + public void Delete(string path) => _index.TryRemove(path, out var _); - public bool IsIndexed(Uri uri) => _index.ContainsKey(uri); + public bool IsIndexed(string path) => _index.ContainsKey(path); private static IEnumerable<(HierarchicalSymbol symbol, string parentName)> DecorateWithParentsName(IEnumerable symbols, string parentName) => symbols.Select((symbol) => (symbol, parentName)); diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/Analysis/Ast/Test/IndexManagerTests.cs index 200579a0e..405a735e7 100644 --- a/src/Analysis/Ast/Test/IndexManagerTests.cs +++ b/src/Analysis/Ast/Test/IndexManagerTests.cs @@ -31,7 +31,7 @@ public void TestInitialize() { TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); _fileSystem = Substitute.For(); _symbolIndex = new SymbolIndex(); - _rootPath = "C:/root"; + _rootPath = "C:\root"; _pythonLanguageVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); _rootFileList = new List(); IDirectoryInfo directoryInfo = Substitute.For(); @@ -45,9 +45,9 @@ public void TestInitialize() { [TestMethod, Priority(0)] public async Task AddsRootDirectoryAsync() { - string pythonTestFile = $"{_rootPath}/bla.py"; - AddFileToRootTestFileSystem(pythonTestFile); - _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); + var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}\bla.py"); + AddFileToRootTestFileSystem(pythonTestFileInfo); + _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(MakeStream("x = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); await indexManager.AddRootDirectoryAsync(); @@ -68,24 +68,24 @@ public void NullDirectoryThrowsException() { [TestMethod, Priority(0)] public void IgnoresNonPythonFiles() { - string nonPythonTestFile = $"{_rootPath}/bla.txt"; - AddFileToRootTestFileSystem(nonPythonTestFile); + var nonPythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}/bla.txt"); + AddFileToRootTestFileSystem(nonPythonTestFileInfo); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), this._rootPath, new string[] { }, new string[] { }); indexManager.AddRootDirectoryAsync(); - _fileSystem.DidNotReceive().FileExists(nonPythonTestFile); + _fileSystem.DidNotReceive().FileExists(nonPythonTestFileInfo.FullName); } [TestMethod, Priority(0)] public void CanOpenFiles() { string nonRootPath = "C:/nonRoot"; - string pythonTestFile = $"{nonRootPath}/bla.py"; + var pythonTestFileInfo = MakeFileInfoProxy($"{nonRootPath}/bla.py"); IDocument doc = Substitute.For(); doc.GetAnyAst().Returns(MakeAst("x = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), _rootPath, new string[] { }, new string[] { }); - indexManager.ProcessNewFile(new Uri(pythonTestFile), doc); + indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(1); @@ -95,16 +95,16 @@ public void CanOpenFiles() { [TestMethod, Priority(0)] public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { - string pythonTestFile = $"{_rootPath}/bla.py"; - AddFileToRootTestFileSystem(pythonTestFile); - _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); + var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}/bla.py"); + AddFileToRootTestFileSystem(pythonTestFileInfo); + _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(MakeStream("x = 1")); IDocument latestDoc = Substitute.For(); latestDoc.GetAnyAst().Returns(MakeAst("y = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), _rootPath, new string[] { }, new string[] { }); await indexManager.AddRootDirectoryAsync(); - indexManager.ReIndexFile(new Uri(pythonTestFile), latestDoc); + indexManager.ReIndexFile(pythonTestFileInfo.FullName, latestDoc); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(1); @@ -115,13 +115,13 @@ public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { [TestMethod, Priority(0)] public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { string nonRootPath = "C:/nonRoot"; - string pythonTestFile = $"{nonRootPath}/bla.py"; + var pythonTestFileInfo = MakeFileInfoProxy($"{nonRootPath}/bla.py"); IDocument doc = Substitute.For(); doc.GetAnyAst().Returns(MakeAst("x = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), _rootPath, new string[] { }, new string[] { }); - indexManager.ProcessNewFile(new Uri(pythonTestFile), doc); - await indexManager.ProcessClosedFileAsync(new Uri(pythonTestFile)); + indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); + await indexManager.ProcessClosedFileAsync(pythonTestFileInfo.FullName); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(0); @@ -129,20 +129,20 @@ public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { [TestMethod, Priority(0)] public async Task CloseWorkspaceFilesReUpdatesIndexAsync() { - string pythonTestFile = $"{_rootPath}/bla.py"; + var pythonTestFile = MakeFileInfoProxy($"{_rootPath}/bla.py"); AddFileToRootTestFileSystem(pythonTestFile); - _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); - _fileSystem.IsPathUnderRoot(_rootPath, pythonTestFile).Returns(true); + _fileSystem.FileOpen(pythonTestFile.FullName, FileMode.Open).Returns(MakeStream("x = 1")); + _fileSystem.IsPathUnderRoot(_rootPath, pythonTestFile.FullName).Returns(true); IDocument latestDoc = Substitute.For(); latestDoc.GetAnyAst().Returns(MakeAst("y = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), _rootPath, new string[] { }, new string[] { }); await indexManager.AddRootDirectoryAsync(); - indexManager.ProcessNewFile(new Uri(pythonTestFile), latestDoc); + indexManager.ProcessNewFile(pythonTestFile.FullName, latestDoc); // It Needs to remake the stream for the file, previous one is closed - _fileSystem.FileOpen(pythonTestFile, FileMode.Open).Returns(MakeStream("x = 1")); - await indexManager.ProcessClosedFileAsync(new Uri(pythonTestFile)); + _fileSystem.FileOpen(pythonTestFile.FullName, FileMode.Open).Returns(MakeStream("x = 1")); + await indexManager.ProcessClosedFileAsync(pythonTestFile.FullName); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(1); @@ -156,15 +156,15 @@ public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() { // it should not reindex file string nonRootPath = "C:/nonRoot"; - string pythonTestFile = $"{nonRootPath}/bla.py"; + var pythonTestFileInfo = MakeFileInfoProxy($"{nonRootPath}/bla.py"); IDocument doc = Substitute.For(); doc.GetAnyAst().Returns(MakeAst("x = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), _rootPath, new string[] { }, new string[] { }); - indexManager.ProcessNewFile(new Uri(pythonTestFile), doc); - await indexManager.ProcessClosedFileAsync(new Uri(pythonTestFile)); + indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); + await indexManager.ProcessClosedFileAsync(pythonTestFileInfo.FullName); doc.GetAnyAst().Returns(MakeAst("x = 1")); - indexManager.ReIndexFile(new Uri(pythonTestFile), doc); + indexManager.ReIndexFile(pythonTestFileInfo.FullName, doc); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(0); @@ -174,9 +174,12 @@ private PythonAst MakeAst(string testCode) { return Parser.CreateParser(MakeStream(testCode), PythonVersions.LatestAvailable3X.Version.ToLanguageVersion()).ParseFile(); } - private void AddFileToRootTestFileSystem(string filePath) { - _rootFileList.Add(new FileInfoProxy(new FileInfo(filePath))); - _fileSystem.FileExists(filePath).Returns(true); + private FileInfoProxy MakeFileInfoProxy(string filePath) + => new FileInfoProxy(new FileInfo(filePath)); + + private void AddFileToRootTestFileSystem(FileInfoProxy fileInfo) { + _rootFileList.Add(fileInfo); + _fileSystem.FileExists(fileInfo.FullName).Returns(true); } private Stream MakeStream(string str) { diff --git a/src/Analysis/Ast/Test/IndexParserTests.cs b/src/Analysis/Ast/Test/IndexParserTests.cs index d38c5d0c8..032f11a68 100644 --- a/src/Analysis/Ast/Test/IndexParserTests.cs +++ b/src/Analysis/Ast/Test/IndexParserTests.cs @@ -49,7 +49,7 @@ public async Task ParseVariableInFileAsync() { _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(MakeStream("x = 1")); IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); - await indexParser.ParseAsync(new Uri(testFilePath)); + await indexParser.ParseAsync(testFilePath); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(1); @@ -64,7 +64,7 @@ public void ParseNonexistentFile() { IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); Func parse = async () => { - await indexParser.ParseAsync(new Uri(testFilePath)); + await indexParser.ParseAsync(testFilePath); }; parse.Should().Throw(); @@ -82,7 +82,7 @@ public void CancellParsing() { CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); cancellationTokenSource.Cancel(); Func parse = async () => { - await indexParser.ParseAsync(new Uri(testFilePath), cancellationTokenSource.Token); + await indexParser.ParseAsync(testFilePath, cancellationTokenSource.Token); }; parse.Should().Throw(); @@ -100,7 +100,7 @@ public void DisposeParserCancelesParsing() { IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); Func parse = async () => { - Task t = indexParser.ParseAsync(new Uri(testFilePath)); + Task t = indexParser.ParseAsync(testFilePath); indexParser.Dispose(); await t; }; diff --git a/src/Analysis/Ast/Test/SymbolIndexTests.cs b/src/Analysis/Ast/Test/SymbolIndexTests.cs index ed4fe7deb..80f4fad6b 100644 --- a/src/Analysis/Ast/Test/SymbolIndexTests.cs +++ b/src/Analysis/Ast/Test/SymbolIndexTests.cs @@ -12,6 +12,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; using System.IO; using FluentAssertions; @@ -614,11 +615,11 @@ public void WalkerAugmentedAssignLambda() { [TestMethod, Priority(0)] public void IndexHierarchicalDocument() { var index = new SymbolIndex(); - var uri = TestData.GetDefaultModuleUri(); + var path = TestData.GetDefaultModulePath(); var ast = GetParse("x = 1"); - index.UpdateIndex(uri, ast); + index.UpdateIndex(path, ast); - var symbols = index.HierarchicalDocumentSymbols(uri); + var symbols = index.HierarchicalDocumentSymbols(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); @@ -627,19 +628,19 @@ public void IndexHierarchicalDocument() { [TestMethod, Priority(0)] public void IndexHierarchicalDocumentUpdate() { var index = new SymbolIndex(); - var uri = TestData.GetDefaultModuleUri(); + var path = TestData.GetDefaultModulePath(); var ast = GetParse("x = 1"); - index.UpdateIndex(uri, ast); + index.UpdateIndex(path, ast); - var symbols = index.HierarchicalDocumentSymbols(uri); + var symbols = index.HierarchicalDocumentSymbols(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); ast = GetParse("y = 1"); - index.UpdateIndex(uri, ast); + index.UpdateIndex(path, ast); - symbols = index.HierarchicalDocumentSymbols(uri); + symbols = index.HierarchicalDocumentSymbols(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); @@ -648,9 +649,9 @@ public void IndexHierarchicalDocumentUpdate() { [TestMethod, Priority(0)] public void IndexHierarchicalDocumentNotFound() { var index = new SymbolIndex(); - var uri = TestData.GetDefaultModuleUri(); + var path = TestData.GetDefaultModulePath(); - var symbols = index.HierarchicalDocumentSymbols(uri); + var symbols = index.HierarchicalDocumentSymbols(path); symbols.Should().BeEmpty(); } @@ -662,7 +663,7 @@ public void IndexWorkspaceSymbolsFlatten() { var index = new SymbolIndex(); var uri = TestData.GetDefaultModuleUri(); var ast = GetParse(code); - index.UpdateIndex(uri, ast); + index.UpdateIndex(uri.AbsolutePath, ast); var symbols = index.WorkspaceSymbols(""); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { @@ -679,20 +680,20 @@ public void IndexWorkspaceSymbolsFiltered() { def foo(self, x): ..."; var index = new SymbolIndex(); - var uri = TestData.GetDefaultModuleUri(); + var path = TestData.GetDefaultModulePath(); var ast = GetParse(code); - index.UpdateIndex(uri, ast); + index.UpdateIndex(path, ast); var symbols = index.WorkspaceSymbols("x"); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new FlatSymbol("x", SymbolKind.Variable, uri, new SourceSpan(2, 19, 2, 20), "foo"), + new FlatSymbol("x", SymbolKind.Variable, new Uri(path), new SourceSpan(2, 19, 2, 20), "foo"), }); } [TestMethod, Priority(0)] public void IndexWorkspaceSymbolsNotFound() { var index = new SymbolIndex(); - var uri = TestData.GetDefaultModuleUri(); + var path = TestData.GetDefaultModulePath(); var symbols = index.WorkspaceSymbols(""); symbols.Should().BeEmpty(); @@ -704,14 +705,14 @@ public void IndexWorkspaceSymbolsCaseInsensitive() { def foo(self, x): ..."; var index = new SymbolIndex(); - var uri = TestData.GetDefaultModuleUri(); + var path = TestData.GetDefaultModulePath(); var ast = GetParse(code); - index.UpdateIndex(uri, ast); + index.UpdateIndex(path, ast); var symbols = index.WorkspaceSymbols("foo"); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new FlatSymbol("Foo", SymbolKind.Class, uri, new SourceSpan(1, 7, 1, 10)), - new FlatSymbol("foo", SymbolKind.Method, uri, new SourceSpan(2, 9, 2, 12), "Foo"), + new FlatSymbol("Foo", SymbolKind.Class, new Uri(path), new SourceSpan(1, 7, 1, 10)), + new FlatSymbol("foo", SymbolKind.Method, new Uri(path), new SourceSpan(2, 9, 2, 12), "Foo"), }); } From 0091933fde5d08c5286874bdc1750870ab5b6ee6 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 30 Jan 2019 16:47:25 -0800 Subject: [PATCH 022/123] More tests --- .../Ast/Impl/Indexing/IndexManager.cs | 2 +- src/Analysis/Ast/Test/IndexManagerTests.cs | 51 +++++++++++++++++++ src/Analysis/Ast/Test/IndexParserTests.cs | 2 +- 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs index 0716f2c1f..225c0b264 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs @@ -41,7 +41,7 @@ public Task AddRootDirectoryAsync(CancellationToken workspaceCancellationToken = parseTasks.Add(_indexParser.ParseAsync(fileInfo.FullName, linkedCts.Token)); } } - return Task.WhenAll(parseTasks.ToArray()); + return Task.Run(() => Task.WaitAll(parseTasks.ToArray()), linkedCts.Token); } private IEnumerable WorkspaceFiles() { diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/Analysis/Ast/Test/IndexManagerTests.cs index 405a735e7..f33c1f2df 100644 --- a/src/Analysis/Ast/Test/IndexManagerTests.cs +++ b/src/Analysis/Ast/Test/IndexManagerTests.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Documents; @@ -170,6 +171,56 @@ public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() { symbols.Should().HaveCount(0); } + [TestMethod, Priority(0)] + public void AddingRootMightThrowUnauthorizedAccess() { + var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}\bla.py"); + AddFileToRootTestFileSystem(pythonTestFileInfo); + _fileSystem.GetDirectoryInfo(_rootPath).EnumerateFileSystemInfos(new string[] { }, new string[] { }) + .ReturnsForAnyArgs(_ => throw new UnauthorizedAccessException()); + _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(MakeStream("x = 1")); + + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); + Func add = async () => await indexManager.AddRootDirectoryAsync(); + + add.Should().Throw(); + var symbols = _symbolIndex.WorkspaceSymbols(""); + symbols.Should().HaveCount(0); + } + + [TestMethod, Priority(0)] + public void CancelAdding() { + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); + + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + cancellationTokenSource.Cancel(); + Func add = async () => await indexManager.AddRootDirectoryAsync(cancellationTokenSource.Token); + + add.Should().Throw(); + var symbols = _symbolIndex.WorkspaceSymbols(""); + symbols.Should().HaveCount(0); + } + + [TestMethod, Priority(0)] + public void DisposeManagerCancelsTask() { + var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}\bla.py"); + AddFileToRootTestFileSystem(pythonTestFileInfo); + // Reading this stream will block + _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(new MemoryStream()); + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); + + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + cancellationTokenSource.Cancel(); + Func add = async () => { + var t = indexManager.AddRootDirectoryAsync(cancellationTokenSource.Token); + indexManager.Dispose(); + await t; + }; + + add.Should().Throw(); + var symbols = _symbolIndex.WorkspaceSymbols(""); + symbols.Should().HaveCount(0); + } + private PythonAst MakeAst(string testCode) { return Parser.CreateParser(MakeStream(testCode), PythonVersions.LatestAvailable3X.Version.ToLanguageVersion()).ParseFile(); } diff --git a/src/Analysis/Ast/Test/IndexParserTests.cs b/src/Analysis/Ast/Test/IndexParserTests.cs index 032f11a68..110f5252e 100644 --- a/src/Analysis/Ast/Test/IndexParserTests.cs +++ b/src/Analysis/Ast/Test/IndexParserTests.cs @@ -91,7 +91,7 @@ public void CancellParsing() { } [TestMethod, Priority(0)] - public void DisposeParserCancelesParsing() { + public void DisposeParserCancelsParsing() { const string testFilePath = "C:/bla.py"; _fileSystem.FileExists(testFilePath).Returns(true); MemoryStream stream = new MemoryStream(); From c0a0387432823b9dbd8e962c40ba2ddee6a202be Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 30 Jan 2019 16:53:51 -0800 Subject: [PATCH 023/123] More Uri to string type transforming --- src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs | 2 +- src/Analysis/Ast/Impl/Indexing/Symbols.cs | 6 +++--- src/Analysis/Ast/Test/SymbolIndexTests.cs | 18 +++++++++--------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs index b8d447eb9..fe834edac 100644 --- a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs @@ -30,7 +30,7 @@ private IEnumerable WorkspaceSymbolsQuery(string query, string path, }); foreach (var (sym, parentName) in treeSymbols) { if (sym.Name.ContainsOrdinal(query, ignoreCase: true)) { - yield return new FlatSymbol(sym.Name, sym.Kind, new Uri(path), sym.SelectionRange, parentName); + yield return new FlatSymbol(sym.Name, sym.Kind, path, sym.SelectionRange, parentName); } } } diff --git a/src/Analysis/Ast/Impl/Indexing/Symbols.cs b/src/Analysis/Ast/Impl/Indexing/Symbols.cs index 177e21636..75a608b1c 100644 --- a/src/Analysis/Ast/Impl/Indexing/Symbols.cs +++ b/src/Analysis/Ast/Impl/Indexing/Symbols.cs @@ -79,20 +79,20 @@ internal class FlatSymbol { public string Name; public SymbolKind Kind; public bool? Deprecated; - public Uri DocumentUri; + public string DocumentPath; public SourceSpan Range; public string ContainerName; public FlatSymbol( string name, SymbolKind kind, - Uri documentUri, + string documentPath, SourceSpan range, string containerName = null ) { Name = name; Kind = kind; - DocumentUri = documentUri; + DocumentPath = documentPath; Range = range; ContainerName = containerName; } diff --git a/src/Analysis/Ast/Test/SymbolIndexTests.cs b/src/Analysis/Ast/Test/SymbolIndexTests.cs index 80f4fad6b..88e20a337 100644 --- a/src/Analysis/Ast/Test/SymbolIndexTests.cs +++ b/src/Analysis/Ast/Test/SymbolIndexTests.cs @@ -661,16 +661,16 @@ public void IndexWorkspaceSymbolsFlatten() { def foo(self, x): ..."; var index = new SymbolIndex(); - var uri = TestData.GetDefaultModuleUri(); + var path = TestData.GetDefaultModulePath(); var ast = GetParse(code); - index.UpdateIndex(uri.AbsolutePath, ast); + index.UpdateIndex(path, ast); var symbols = index.WorkspaceSymbols(""); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new FlatSymbol("Foo", SymbolKind.Class, uri, new SourceSpan(1, 7, 1, 10)), - new FlatSymbol("foo", SymbolKind.Method, uri, new SourceSpan(2, 9, 2, 12), "Foo"), - new FlatSymbol("self", SymbolKind.Variable, uri, new SourceSpan(2, 13, 2, 17), "foo"), - new FlatSymbol("x", SymbolKind.Variable, uri, new SourceSpan(2, 19, 2, 20), "foo"), + new FlatSymbol("Foo", SymbolKind.Class, path, new SourceSpan(1, 7, 1, 10)), + new FlatSymbol("foo", SymbolKind.Method, path, new SourceSpan(2, 9, 2, 12), "Foo"), + new FlatSymbol("self", SymbolKind.Variable, path, new SourceSpan(2, 13, 2, 17), "foo"), + new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(2, 19, 2, 20), "foo"), }); } @@ -686,7 +686,7 @@ public void IndexWorkspaceSymbolsFiltered() { var symbols = index.WorkspaceSymbols("x"); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new FlatSymbol("x", SymbolKind.Variable, new Uri(path), new SourceSpan(2, 19, 2, 20), "foo"), + new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(2, 19, 2, 20), "foo"), }); } @@ -711,8 +711,8 @@ public void IndexWorkspaceSymbolsCaseInsensitive() { var symbols = index.WorkspaceSymbols("foo"); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new FlatSymbol("Foo", SymbolKind.Class, new Uri(path), new SourceSpan(1, 7, 1, 10)), - new FlatSymbol("foo", SymbolKind.Method, new Uri(path), new SourceSpan(2, 9, 2, 12), "Foo"), + new FlatSymbol("Foo", SymbolKind.Class, path, new SourceSpan(1, 7, 1, 10)), + new FlatSymbol("foo", SymbolKind.Method, path, new SourceSpan(2, 9, 2, 12), "Foo"), }); } From 6e4a0f50176d330bbeaab99be243280b173b55d9 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 31 Jan 2019 12:14:40 -0800 Subject: [PATCH 024/123] Removing FileNotFoundException --- .../Ast/Impl/Indexing/IIndexParser.cs | 2 +- .../Ast/Impl/Indexing/IndexManager.cs | 2 +- src/Analysis/Ast/Impl/Indexing/IndexParser.cs | 21 ++++++++++++------ src/Analysis/Ast/Test/IndexParserTests.cs | 22 +++++++++++++++---- 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs b/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs index b60b47060..8e13feb05 100644 --- a/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs +++ b/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs @@ -4,6 +4,6 @@ namespace Microsoft.Python.Analysis.Indexing { internal interface IIndexParser : IDisposable { - Task ParseAsync(string path, CancellationToken cancellationToken = default); + Task ParseAsync(string path, CancellationToken cancellationToken = default); } } diff --git a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs index 225c0b264..d67b299c0 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs @@ -35,7 +35,7 @@ public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLang public Task AddRootDirectoryAsync(CancellationToken workspaceCancellationToken = default) { var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(workspaceCancellationToken, _allIndexCts.Token); - var parseTasks = new List(); + var parseTasks = new List>(); foreach (var fileInfo in WorkspaceFiles()) { if (ModulePath.IsPythonSourceFile(fileInfo.FullName)) { parseTasks.Add(_indexParser.ParseAsync(fileInfo.FullName, linkedCts.Token)); diff --git a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs index edf31c96d..9664f4bc2 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -27,17 +28,23 @@ public void Dispose() { _allProcessingCts.Dispose(); } - public Task ParseAsync(string path, CancellationToken parseCancellationToken = default) { + public Task ParseAsync(string path, CancellationToken parseCancellationToken = default) { var linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, parseCancellationToken); var linkedParseToken = linkedParseCts.Token; - return Task.Run(() => { + return Task.Run(() => { if (!_fileSystem.FileExists(path)) { - throw new FileNotFoundException($"{path} does not exist", path); + return false; } - using (var stream = _fileSystem.FileOpen(path, FileMode.Open)) { - var parser = Parser.CreateParser(stream, _version); - linkedParseToken.ThrowIfCancellationRequested(); - _symbolIndex.UpdateIndex(path, parser.ParseFile()); + try { + using (var stream = _fileSystem.FileOpen(path, FileMode.Open)) { + var parser = Parser.CreateParser(stream, _version); + linkedParseToken.ThrowIfCancellationRequested(); + _symbolIndex.UpdateIndex(path, parser.ParseFile()); + return true; + } + } catch (FileNotFoundException e) { + Trace.TraceError(e.ToString()); + return false; } }, linkedParseToken); } diff --git a/src/Analysis/Ast/Test/IndexParserTests.cs b/src/Analysis/Ast/Test/IndexParserTests.cs index 110f5252e..154ce8432 100644 --- a/src/Analysis/Ast/Test/IndexParserTests.cs +++ b/src/Analysis/Ast/Test/IndexParserTests.cs @@ -63,11 +63,25 @@ public void ParseNonexistentFile() { _fileSystem.FileExists(testFilePath).Returns(false); IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); - Func parse = async () => { - await indexParser.ParseAsync(testFilePath); - }; + var t = indexParser.ParseAsync(testFilePath); + t.Wait(); + + t.Result.Should().BeFalse(); + var symbols = _symbolIndex.WorkspaceSymbols(""); + symbols.Should().HaveCount(0); + } + + [TestMethod, Priority(0)] + public void ParseFileThatStopsExisting() { + const string testFilePath = "C:/bla.py"; + _fileSystem.FileExists(testFilePath).Returns(true); + _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(_ => throw new FileNotFoundException()); + + IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); + var t = indexParser.ParseAsync(testFilePath); + t.Wait(); - parse.Should().Throw(); + t.Result.Should().BeFalse(); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(0); } From 2e8ecdfe49c4a14f6045840d8c439737ea493713 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 31 Jan 2019 12:39:13 -0800 Subject: [PATCH 025/123] Single Line Return --- src/Analysis/Ast/Impl/Indexing/IndexManager.cs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs index d67b299c0..079511576 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs @@ -63,13 +63,11 @@ public Task ProcessClosedFileAsync(string path, CancellationToken fileCancellati } } - private bool IsFileOnWorkspace(string path) { - return _fileSystem.IsPathUnderRoot(_workspaceRootPath, path); - } + private bool IsFileOnWorkspace(string path) + => _fileSystem.IsPathUnderRoot(_workspaceRootPath, path); - public void ProcessNewFile(string path, IDocument doc) { - _symbolIndex.UpdateIndex(path, doc.GetAnyAst()); - } + public void ProcessNewFile(string path, IDocument doc) + => _symbolIndex.UpdateIndex(path, doc.GetAnyAst()); public void ReIndexFile(string path, IDocument doc) { if (IsFileIndexed(path)) { From 332358f517c5e64b2886bb3f9b65b7f51d886ece Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 31 Jan 2019 13:22:01 -0800 Subject: [PATCH 026/123] SymbolIndex Interface --- src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs | 3 ++- src/Analysis/Ast/Test/SymbolIndexTests.cs | 14 +++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs index 9968fc8f2..b1a141769 100644 --- a/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs @@ -4,8 +4,9 @@ namespace Microsoft.Python.Analysis.Indexing { internal interface ISymbolIndex { - void UpdateIndex(string path, PythonAst pythonAst); IEnumerable WorkspaceSymbols(string query); + IEnumerable HierarchicalDocumentSymbols(string path); + void UpdateIndex(string path, PythonAst pythonAst); void Delete(string path); bool IsIndexed(string path); } diff --git a/src/Analysis/Ast/Test/SymbolIndexTests.cs b/src/Analysis/Ast/Test/SymbolIndexTests.cs index 88e20a337..bc0e30d01 100644 --- a/src/Analysis/Ast/Test/SymbolIndexTests.cs +++ b/src/Analysis/Ast/Test/SymbolIndexTests.cs @@ -614,7 +614,7 @@ public void WalkerAugmentedAssignLambda() { [TestMethod, Priority(0)] public void IndexHierarchicalDocument() { - var index = new SymbolIndex(); + ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var ast = GetParse("x = 1"); index.UpdateIndex(path, ast); @@ -627,7 +627,7 @@ public void IndexHierarchicalDocument() { [TestMethod, Priority(0)] public void IndexHierarchicalDocumentUpdate() { - var index = new SymbolIndex(); + ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var ast = GetParse("x = 1"); index.UpdateIndex(path, ast); @@ -648,7 +648,7 @@ public void IndexHierarchicalDocumentUpdate() { [TestMethod, Priority(0)] public void IndexHierarchicalDocumentNotFound() { - var index = new SymbolIndex(); + ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var symbols = index.HierarchicalDocumentSymbols(path); @@ -660,7 +660,7 @@ public void IndexWorkspaceSymbolsFlatten() { var code = @"class Foo(object): def foo(self, x): ..."; - var index = new SymbolIndex(); + ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var ast = GetParse(code); index.UpdateIndex(path, ast); @@ -679,7 +679,7 @@ public void IndexWorkspaceSymbolsFiltered() { var code = @"class Foo(object): def foo(self, x): ..."; - var index = new SymbolIndex(); + ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var ast = GetParse(code); index.UpdateIndex(path, ast); @@ -692,7 +692,7 @@ public void IndexWorkspaceSymbolsFiltered() { [TestMethod, Priority(0)] public void IndexWorkspaceSymbolsNotFound() { - var index = new SymbolIndex(); + ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var symbols = index.WorkspaceSymbols(""); @@ -704,7 +704,7 @@ public void IndexWorkspaceSymbolsCaseInsensitive() { var code = @"class Foo(object): def foo(self, x): ..."; - var index = new SymbolIndex(); + ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var ast = GetParse(code); index.UpdateIndex(path, ast); From 0048bf873f537ef8fc04811de601520a428eb1f9 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 1 Feb 2019 13:57:48 -0800 Subject: [PATCH 027/123] Fix AddRoot Test --- src/Analysis/Ast/Impl/Indexing/IndexParser.cs | 1 + src/Analysis/Ast/Test/IndexManagerTests.cs | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs index 9664f4bc2..452620633 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs @@ -36,6 +36,7 @@ public Task ParseAsync(string path, CancellationToken parseCancellationTok return false; } try { + linkedParseToken.ThrowIfCancellationRequested(); using (var stream = _fileSystem.FileOpen(path, FileMode.Open)) { var parser = Parser.CreateParser(stream, _version); linkedParseToken.ThrowIfCancellationRequested(); diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/Analysis/Ast/Test/IndexManagerTests.cs index f33c1f2df..334268c92 100644 --- a/src/Analysis/Ast/Test/IndexManagerTests.cs +++ b/src/Analysis/Ast/Test/IndexManagerTests.cs @@ -48,12 +48,15 @@ public void TestInitialize() { public async Task AddsRootDirectoryAsync() { var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}\bla.py"); AddFileToRootTestFileSystem(pythonTestFileInfo); + var fooFile = MakeFileInfoProxy($"{_rootPath}\foo.py"); + AddFileToRootTestFileSystem(fooFile); _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(MakeStream("x = 1")); + _fileSystem.FileOpen(fooFile.FullName, FileMode.Open).Returns(MakeStream("y = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); await indexManager.AddRootDirectoryAsync(); - var symbols = _symbolIndex.WorkspaceSymbols(""); + var symbols = _symbolIndex.WorkspaceSymbols("x"); symbols.Should().HaveCount(1); symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); symbols.First().Name.Should().BeEquivalentTo("x"); From 8f4343e400940744edf3cb78a496e31779dbf71b Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 1 Feb 2019 13:59:39 -0800 Subject: [PATCH 028/123] Disposing test fix --- src/Analysis/Ast/Test/IndexManagerTests.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/Analysis/Ast/Test/IndexManagerTests.cs index 334268c92..0a222f9f5 100644 --- a/src/Analysis/Ast/Test/IndexManagerTests.cs +++ b/src/Analysis/Ast/Test/IndexManagerTests.cs @@ -211,10 +211,8 @@ public void DisposeManagerCancelsTask() { _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(new MemoryStream()); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); - CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); - cancellationTokenSource.Cancel(); Func add = async () => { - var t = indexManager.AddRootDirectoryAsync(cancellationTokenSource.Token); + var t = indexManager.AddRootDirectoryAsync(); indexManager.Dispose(); await t; }; From 36ed2e6fcceb2b1c54343d691902d604c28e8241 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 1 Feb 2019 14:02:33 -0800 Subject: [PATCH 029/123] Blocking tests new fake --- src/Analysis/Ast/Test/FakeBlockingStream.cs | 54 +++++++++++++++++++++ src/Analysis/Ast/Test/IndexManagerTests.cs | 2 +- src/Analysis/Ast/Test/IndexParserTests.cs | 2 +- 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 src/Analysis/Ast/Test/FakeBlockingStream.cs diff --git a/src/Analysis/Ast/Test/FakeBlockingStream.cs b/src/Analysis/Ast/Test/FakeBlockingStream.cs new file mode 100644 index 000000000..0146b651a --- /dev/null +++ b/src/Analysis/Ast/Test/FakeBlockingStream.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; + +namespace Microsoft.Python.Analysis.Tests { + + internal class FakeBlockingStream : Stream { + ManualResetEventSlim mres1 = new ManualResetEventSlim(false); // initialize as unsignaled + private byte[] _internalBuffer; + + public FakeBlockingStream(byte[] buffer) { + _internalBuffer = new byte[buffer.Length]; + Buffer.BlockCopy(buffer, 0, _internalBuffer, 0, buffer.Length); + } + + public override bool CanRead => throw new NotImplementedException(); + + public override bool CanSeek => throw new NotImplementedException(); + + public override bool CanWrite => throw new NotImplementedException(); + + public override long Length => throw new NotImplementedException(); + + public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + + public override void Flush() { + throw new NotImplementedException(); + } + + public override int Read(byte[] buffer, int offset, int count) { + mres1.Wait(); + Buffer.BlockCopy(_internalBuffer, 0, buffer, 0, count); + return count; + } + + public override long Seek(long offset, SeekOrigin origin) { + throw new NotImplementedException(); + } + + public override void SetLength(long value) { + throw new NotImplementedException(); + } + + public override void Write(byte[] buffer, int offset, int count) { + throw new NotImplementedException(); + } + + public void Unblock() { + mres1.Set(); + } + } +} diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/Analysis/Ast/Test/IndexManagerTests.cs index 0a222f9f5..e65ddeee4 100644 --- a/src/Analysis/Ast/Test/IndexManagerTests.cs +++ b/src/Analysis/Ast/Test/IndexManagerTests.cs @@ -208,7 +208,7 @@ public void DisposeManagerCancelsTask() { var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}\bla.py"); AddFileToRootTestFileSystem(pythonTestFileInfo); // Reading this stream will block - _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(new MemoryStream()); + _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(new FakeBlockingStream(new byte[] { })); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); Func add = async () => { diff --git a/src/Analysis/Ast/Test/IndexParserTests.cs b/src/Analysis/Ast/Test/IndexParserTests.cs index 154ce8432..ec82a90b8 100644 --- a/src/Analysis/Ast/Test/IndexParserTests.cs +++ b/src/Analysis/Ast/Test/IndexParserTests.cs @@ -110,7 +110,7 @@ public void DisposeParserCancelsParsing() { _fileSystem.FileExists(testFilePath).Returns(true); MemoryStream stream = new MemoryStream(); // no writing to the stream, will block when read - _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(stream); + _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(new FakeBlockingStream(new byte[] { })); IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); Func parse = async () => { From d58cc6b34a8c7ebdb5ca0eccc92259459cdfaff9 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 1 Feb 2019 15:16:37 -0800 Subject: [PATCH 030/123] Wait for AddRootTask in symbol querying --- .../Ast/Impl/Indexing/IIndexManager.cs | 3 ++ .../Ast/Impl/Indexing/IndexManager.cs | 36 ++++++++++++++----- src/Analysis/Ast/Test/FakeBlockingStream.cs | 17 +++------ src/Analysis/Ast/Test/IndexManagerTests.cs | 21 ++++++++++- src/Analysis/Ast/Test/IndexParserTests.cs | 2 +- 5 files changed, 57 insertions(+), 22 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs index daa2639b5..a912a4ec5 100644 --- a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Documents; @@ -9,5 +10,7 @@ internal interface IIndexManager : IDisposable { void ProcessNewFile(string path, IDocument doc); Task ProcessClosedFileAsync(string path, CancellationToken fileCancellationToken = default); void ReIndexFile(string path, IDocument doc); + Task> HierarchicalDocumentSymbolsAsync(string path); + Task> WorkspaceSymbolsAsync(string query); } } diff --git a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs index 079511576..19020eacd 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Core.Interpreter; @@ -16,7 +15,9 @@ internal class IndexManager : IIndexManager { private readonly string _workspaceRootPath; private readonly string[] _includeFiles; private readonly string[] _excludeFiles; + private bool _isRootAddTaskSet; private readonly CancellationTokenSource _allIndexCts = new CancellationTokenSource(); + private Task _addRootTask; public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, string[] excludeFiles) { @@ -31,17 +32,26 @@ public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLang _workspaceRootPath = rootPath; _includeFiles = includeFiles; _excludeFiles = excludeFiles; + _isRootAddTaskSet = false; } public Task AddRootDirectoryAsync(CancellationToken workspaceCancellationToken = default) { - var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(workspaceCancellationToken, _allIndexCts.Token); - var parseTasks = new List>(); - foreach (var fileInfo in WorkspaceFiles()) { - if (ModulePath.IsPythonSourceFile(fileInfo.FullName)) { - parseTasks.Add(_indexParser.ParseAsync(fileInfo.FullName, linkedCts.Token)); + if (_isRootAddTaskSet) { + return _addRootTask; + } else { + lock (this) { + var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(workspaceCancellationToken, _allIndexCts.Token); + var parseTasks = new List>(); + foreach (var fileInfo in WorkspaceFiles()) { + if (ModulePath.IsPythonSourceFile(fileInfo.FullName)) { + parseTasks.Add(_indexParser.ParseAsync(fileInfo.FullName, linkedCts.Token)); + } + } + _addRootTask = Task.Run(() => Task.WaitAll(parseTasks.ToArray()), linkedCts.Token); + _isRootAddTaskSet = true; } + return _addRootTask; } - return Task.Run(() => Task.WaitAll(parseTasks.ToArray()), linkedCts.Token); } private IEnumerable WorkspaceFiles() { @@ -79,5 +89,15 @@ public void Dispose() { _allIndexCts.Cancel(); _allIndexCts.Dispose(); } + + public async Task> HierarchicalDocumentSymbolsAsync(string path) { + await AddRootDirectoryAsync(); + return _symbolIndex.HierarchicalDocumentSymbols(path); + } + + public async Task> WorkspaceSymbolsAsync(string query) { + await AddRootDirectoryAsync(); + return _symbolIndex.WorkspaceSymbols(query); + } } } diff --git a/src/Analysis/Ast/Test/FakeBlockingStream.cs b/src/Analysis/Ast/Test/FakeBlockingStream.cs index 0146b651a..4f5a291f6 100644 --- a/src/Analysis/Ast/Test/FakeBlockingStream.cs +++ b/src/Analysis/Ast/Test/FakeBlockingStream.cs @@ -1,18 +1,15 @@ using System; -using System.Collections.Generic; using System.IO; -using System.Text; using System.Threading; namespace Microsoft.Python.Analysis.Tests { internal class FakeBlockingStream : Stream { ManualResetEventSlim mres1 = new ManualResetEventSlim(false); // initialize as unsignaled - private byte[] _internalBuffer; + private MemoryStream _memoryStream; - public FakeBlockingStream(byte[] buffer) { - _internalBuffer = new byte[buffer.Length]; - Buffer.BlockCopy(buffer, 0, _internalBuffer, 0, buffer.Length); + public FakeBlockingStream() { + _memoryStream = new MemoryStream(); } public override bool CanRead => throw new NotImplementedException(); @@ -31,8 +28,7 @@ public override void Flush() { public override int Read(byte[] buffer, int offset, int count) { mres1.Wait(); - Buffer.BlockCopy(_internalBuffer, 0, buffer, 0, count); - return count; + return _memoryStream.Read(buffer, offset, count); } public override long Seek(long offset, SeekOrigin origin) { @@ -44,10 +40,7 @@ public override void SetLength(long value) { } public override void Write(byte[] buffer, int offset, int count) { - throw new NotImplementedException(); - } - - public void Unblock() { + _memoryStream.Write(buffer, offset, count); mres1.Set(); } } diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/Analysis/Ast/Test/IndexManagerTests.cs index e65ddeee4..d2c70aeeb 100644 --- a/src/Analysis/Ast/Test/IndexManagerTests.cs +++ b/src/Analysis/Ast/Test/IndexManagerTests.cs @@ -208,7 +208,7 @@ public void DisposeManagerCancelsTask() { var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}\bla.py"); AddFileToRootTestFileSystem(pythonTestFileInfo); // Reading this stream will block - _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(new FakeBlockingStream(new byte[] { })); + _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(new FakeBlockingStream()); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); Func add = async () => { @@ -222,6 +222,25 @@ public void DisposeManagerCancelsTask() { symbols.Should().HaveCount(0); } + [TestMethod, Priority(0)] + public async Task QueueWait() { + var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}\bla.py"); + AddFileToRootTestFileSystem(pythonTestFileInfo); + Stream stream = new FakeBlockingStream(); + _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(stream); + + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); + indexManager.AddRootDirectoryAsync(); + + stream.Write(Encoding.UTF8.GetBytes("x = 1")); + var t = indexManager.WorkspaceSymbolsAsync(""); + await t; + var symbols = t.Result; + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("x"); + } + private PythonAst MakeAst(string testCode) { return Parser.CreateParser(MakeStream(testCode), PythonVersions.LatestAvailable3X.Version.ToLanguageVersion()).ParseFile(); } diff --git a/src/Analysis/Ast/Test/IndexParserTests.cs b/src/Analysis/Ast/Test/IndexParserTests.cs index ec82a90b8..f7925950e 100644 --- a/src/Analysis/Ast/Test/IndexParserTests.cs +++ b/src/Analysis/Ast/Test/IndexParserTests.cs @@ -110,7 +110,7 @@ public void DisposeParserCancelsParsing() { _fileSystem.FileExists(testFilePath).Returns(true); MemoryStream stream = new MemoryStream(); // no writing to the stream, will block when read - _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(new FakeBlockingStream(new byte[] { })); + _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(new FakeBlockingStream()); IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); Func parse = async () => { From 2b3d45045458afbc65fc807d6ae72d3401c9c066 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 1 Feb 2019 17:06:55 -0800 Subject: [PATCH 031/123] Refactor tests and async add root --- .../Ast/Impl/Indexing/IIndexManager.cs | 6 +-- .../Ast/Impl/Indexing/IndexManager.cs | 48 +++++++++++-------- src/Analysis/Ast/Impl/Indexing/IndexParser.cs | 8 ++-- src/Analysis/Ast/Test/FakeBlockingStream.cs | 47 ------------------ src/Analysis/Ast/Test/IndexManagerTests.cs | 28 +++-------- src/Analysis/Ast/Test/IndexParserTests.cs | 3 +- 6 files changed, 42 insertions(+), 98 deletions(-) delete mode 100644 src/Analysis/Ast/Test/FakeBlockingStream.cs diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs index a912a4ec5..af29bd377 100644 --- a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs @@ -6,11 +6,11 @@ namespace Microsoft.Python.Analysis.Indexing { internal interface IIndexManager : IDisposable { - Task AddRootDirectoryAsync(CancellationToken workspaceCancellationToken = default); + Task AddRootDirectoryAsync(); void ProcessNewFile(string path, IDocument doc); Task ProcessClosedFileAsync(string path, CancellationToken fileCancellationToken = default); void ReIndexFile(string path, IDocument doc); - Task> HierarchicalDocumentSymbolsAsync(string path); - Task> WorkspaceSymbolsAsync(string query); + Task> HierarchicalDocumentSymbolsAsync(string path); + Task> WorkspaceSymbolsAsync(string query); } } diff --git a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs index 19020eacd..e98f6527a 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs @@ -1,4 +1,6 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Core.Interpreter; @@ -15,9 +17,8 @@ internal class IndexManager : IIndexManager { private readonly string _workspaceRootPath; private readonly string[] _includeFiles; private readonly string[] _excludeFiles; - private bool _isRootAddTaskSet; private readonly CancellationTokenSource _allIndexCts = new CancellationTokenSource(); - private Task _addRootTask; + private readonly TaskCompletionSource _addRootTcs; public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, string[] excludeFiles) { @@ -32,28 +33,32 @@ public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLang _workspaceRootPath = rootPath; _includeFiles = includeFiles; _excludeFiles = excludeFiles; - _isRootAddTaskSet = false; + _addRootTcs = new TaskCompletionSource(); + StartAddRootDir(); } - public Task AddRootDirectoryAsync(CancellationToken workspaceCancellationToken = default) { - if (_isRootAddTaskSet) { - return _addRootTask; - } else { - lock (this) { - var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(workspaceCancellationToken, _allIndexCts.Token); + private void StartAddRootDir() { + Task.Run(() => { + try { var parseTasks = new List>(); foreach (var fileInfo in WorkspaceFiles()) { if (ModulePath.IsPythonSourceFile(fileInfo.FullName)) { - parseTasks.Add(_indexParser.ParseAsync(fileInfo.FullName, linkedCts.Token)); + parseTasks.Add(_indexParser.ParseAsync(fileInfo.FullName, _allIndexCts.Token)); } } - _addRootTask = Task.Run(() => Task.WaitAll(parseTasks.ToArray()), linkedCts.Token); - _isRootAddTaskSet = true; + Task.WaitAll(parseTasks.ToArray(), _allIndexCts.Token); + _addRootTcs.SetResult(true); + } catch (Exception e) { + _addRootTcs.SetException(e); } - return _addRootTask; - } + }); } + public Task AddRootDirectoryAsync() { + return _addRootTcs.Task; + } + + private IEnumerable WorkspaceFiles() { return _fileSystem.GetDirectoryInfo(_workspaceRootPath).EnumerateFileSystemInfos(_includeFiles, _excludeFiles); } @@ -61,11 +66,12 @@ private IEnumerable WorkspaceFiles() { private bool IsFileIndexed(string path) => _symbolIndex.IsIndexed(path); public Task ProcessClosedFileAsync(string path, CancellationToken fileCancellationToken = default) { - var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(fileCancellationToken, _allIndexCts.Token); // If path is on workspace if (IsFileOnWorkspace(path)) { // updates index and ignores previous AST - return _indexParser.ParseAsync(path, linkedCts.Token); + var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(fileCancellationToken, _allIndexCts.Token); + return _indexParser.ParseAsync(path, linkedCts.Token) + .ContinueWith(_ => linkedCts.Dispose()); } else { // remove file from index _symbolIndex.Delete(path); @@ -90,14 +96,14 @@ public void Dispose() { _allIndexCts.Dispose(); } - public async Task> HierarchicalDocumentSymbolsAsync(string path) { + public async Task> HierarchicalDocumentSymbolsAsync(string path) { await AddRootDirectoryAsync(); - return _symbolIndex.HierarchicalDocumentSymbols(path); + return _symbolIndex.HierarchicalDocumentSymbols(path).ToList(); } - public async Task> WorkspaceSymbolsAsync(string query) { + public async Task> WorkspaceSymbolsAsync(string query) { await AddRootDirectoryAsync(); - return _symbolIndex.WorkspaceSymbols(query); + return _symbolIndex.WorkspaceSymbols(query).ToList(); } } } diff --git a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs index 452620633..d93345d83 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs @@ -1,5 +1,4 @@ -using System; -using System.Diagnostics; +using System.Diagnostics; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -47,7 +46,10 @@ public Task ParseAsync(string path, CancellationToken parseCancellationTok Trace.TraceError(e.ToString()); return false; } - }, linkedParseToken); + }, linkedParseToken).ContinueWith((task) => { + linkedParseCts.Dispose(); + return task.Result; + }); } } } diff --git a/src/Analysis/Ast/Test/FakeBlockingStream.cs b/src/Analysis/Ast/Test/FakeBlockingStream.cs deleted file mode 100644 index 4f5a291f6..000000000 --- a/src/Analysis/Ast/Test/FakeBlockingStream.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using System.IO; -using System.Threading; - -namespace Microsoft.Python.Analysis.Tests { - - internal class FakeBlockingStream : Stream { - ManualResetEventSlim mres1 = new ManualResetEventSlim(false); // initialize as unsignaled - private MemoryStream _memoryStream; - - public FakeBlockingStream() { - _memoryStream = new MemoryStream(); - } - - public override bool CanRead => throw new NotImplementedException(); - - public override bool CanSeek => throw new NotImplementedException(); - - public override bool CanWrite => throw new NotImplementedException(); - - public override long Length => throw new NotImplementedException(); - - public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } - - public override void Flush() { - throw new NotImplementedException(); - } - - public override int Read(byte[] buffer, int offset, int count) { - mres1.Wait(); - return _memoryStream.Read(buffer, offset, count); - } - - public override long Seek(long offset, SeekOrigin origin) { - throw new NotImplementedException(); - } - - public override void SetLength(long value) { - throw new NotImplementedException(); - } - - public override void Write(byte[] buffer, int offset, int count) { - _memoryStream.Write(buffer, offset, count); - mres1.Set(); - } - } -} diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/Analysis/Ast/Test/IndexManagerTests.cs index d2c70aeeb..e60018f3a 100644 --- a/src/Analysis/Ast/Test/IndexManagerTests.cs +++ b/src/Analysis/Ast/Test/IndexManagerTests.cs @@ -3,7 +3,6 @@ using System.IO; using System.Linq; using System.Text; -using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Documents; @@ -183,32 +182,20 @@ public void AddingRootMightThrowUnauthorizedAccess() { _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(MakeStream("x = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); - Func add = async () => await indexManager.AddRootDirectoryAsync(); + Func add = async () => { + await indexManager.AddRootDirectoryAsync(); + }; add.Should().Throw(); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(0); } - [TestMethod, Priority(0)] - public void CancelAdding() { - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); - - CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); - cancellationTokenSource.Cancel(); - Func add = async () => await indexManager.AddRootDirectoryAsync(cancellationTokenSource.Token); - - add.Should().Throw(); - var symbols = _symbolIndex.WorkspaceSymbols(""); - symbols.Should().HaveCount(0); - } - [TestMethod, Priority(0)] public void DisposeManagerCancelsTask() { var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}\bla.py"); AddFileToRootTestFileSystem(pythonTestFileInfo); - // Reading this stream will block - _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(new FakeBlockingStream()); + _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(MakeStream("x = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); Func add = async () => { @@ -223,16 +210,13 @@ public void DisposeManagerCancelsTask() { } [TestMethod, Priority(0)] - public async Task QueueWait() { + public async Task WorkspaceSymbolsAddsRootDirectory() { var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}\bla.py"); AddFileToRootTestFileSystem(pythonTestFileInfo); - Stream stream = new FakeBlockingStream(); - _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(stream); + _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(MakeStream("x = 1")); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); - indexManager.AddRootDirectoryAsync(); - stream.Write(Encoding.UTF8.GetBytes("x = 1")); var t = indexManager.WorkspaceSymbolsAsync(""); await t; var symbols = t.Result; diff --git a/src/Analysis/Ast/Test/IndexParserTests.cs b/src/Analysis/Ast/Test/IndexParserTests.cs index f7925950e..93b05aa5d 100644 --- a/src/Analysis/Ast/Test/IndexParserTests.cs +++ b/src/Analysis/Ast/Test/IndexParserTests.cs @@ -109,8 +109,7 @@ public void DisposeParserCancelsParsing() { const string testFilePath = "C:/bla.py"; _fileSystem.FileExists(testFilePath).Returns(true); MemoryStream stream = new MemoryStream(); - // no writing to the stream, will block when read - _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(new FakeBlockingStream()); + _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(MakeStream("x = 1")); IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); Func parse = async () => { From 36c6f8587a6125c6fb1862d67f6151b798dde53d Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 1 Feb 2019 17:11:17 -0800 Subject: [PATCH 032/123] Adding license header --- src/Analysis/Ast/Impl/Indexing/IIndexManager.cs | 17 ++++++++++++++++- src/Analysis/Ast/Impl/Indexing/IIndexParser.cs | 17 ++++++++++++++++- src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs | 17 ++++++++++++++++- src/Analysis/Ast/Impl/Indexing/IndexManager.cs | 17 ++++++++++++++++- src/Analysis/Ast/Impl/Indexing/IndexParser.cs | 17 ++++++++++++++++- src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs | 17 ++++++++++++++++- .../Ast/Impl/Indexing/SymbolIndexWalker.cs | 17 ++++++++++++++++- src/Analysis/Ast/Impl/Indexing/Symbols.cs | 17 ++++++++++++++++- src/Analysis/Ast/Test/IndexManagerTests.cs | 17 ++++++++++++++++- src/Analysis/Ast/Test/IndexParserTests.cs | 17 ++++++++++++++++- 10 files changed, 160 insertions(+), 10 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs index af29bd377..076606a67 100644 --- a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs @@ -1,4 +1,19 @@ -using System; +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs b/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs index 8e13feb05..c9fea675a 100644 --- a/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs +++ b/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs @@ -1,4 +1,19 @@ -using System; +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; using System.Threading; using System.Threading.Tasks; diff --git a/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs index b1a141769..c46507893 100644 --- a/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs @@ -1,4 +1,19 @@ -using System; +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; using System.Collections.Generic; using Microsoft.Python.Parsing.Ast; diff --git a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs index e98f6527a..cda87e871 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs @@ -1,4 +1,19 @@ -using System; +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; using System.Collections.Generic; using System.Linq; using System.Threading; diff --git a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs index d93345d83..e879effc6 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs @@ -1,4 +1,19 @@ -using System.Diagnostics; +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Diagnostics; using System.IO; using System.Threading; using System.Threading.Tasks; diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs index fe834edac..1c70244cd 100644 --- a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs +++ b/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs @@ -1,4 +1,19 @@ -using System; +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndexWalker.cs b/src/Analysis/Ast/Impl/Indexing/SymbolIndexWalker.cs index 3f645f5e0..477a457ab 100644 --- a/src/Analysis/Ast/Impl/Indexing/SymbolIndexWalker.cs +++ b/src/Analysis/Ast/Impl/Indexing/SymbolIndexWalker.cs @@ -1,4 +1,19 @@ -using System.Collections.Generic; +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; using Microsoft.Python.Core; diff --git a/src/Analysis/Ast/Impl/Indexing/Symbols.cs b/src/Analysis/Ast/Impl/Indexing/Symbols.cs index 75a608b1c..6435e0685 100644 --- a/src/Analysis/Ast/Impl/Indexing/Symbols.cs +++ b/src/Analysis/Ast/Impl/Indexing/Symbols.cs @@ -1,4 +1,19 @@ -using System; +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; using System.Collections.Generic; using System.Linq; using Microsoft.Python.Core; diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/Analysis/Ast/Test/IndexManagerTests.cs index e60018f3a..cfa59cd83 100644 --- a/src/Analysis/Ast/Test/IndexManagerTests.cs +++ b/src/Analysis/Ast/Test/IndexManagerTests.cs @@ -1,4 +1,19 @@ -using System; +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; using System.Collections.Generic; using System.IO; using System.Linq; diff --git a/src/Analysis/Ast/Test/IndexParserTests.cs b/src/Analysis/Ast/Test/IndexParserTests.cs index 93b05aa5d..e5f78c3f8 100644 --- a/src/Analysis/Ast/Test/IndexParserTests.cs +++ b/src/Analysis/Ast/Test/IndexParserTests.cs @@ -1,4 +1,19 @@ -using System; +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; using System.IO; using System.Linq; using System.Text; From a897585f54049f2c7b5344d22e1dc5eaf5f09e6c Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 1 Feb 2019 17:13:36 -0800 Subject: [PATCH 033/123] TraverseBreadth variable rename --- src/Core/Impl/Extensions/EnumerableExtensions.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Core/Impl/Extensions/EnumerableExtensions.cs b/src/Core/Impl/Extensions/EnumerableExtensions.cs index 3aa9ca645..cbb8a555a 100644 --- a/src/Core/Impl/Extensions/EnumerableExtensions.cs +++ b/src/Core/Impl/Extensions/EnumerableExtensions.cs @@ -94,15 +94,15 @@ public static IEnumerable TraverseBreadthFirst(this T root, Func TraverseBreadthFirst(this IEnumerable roots, Func> selectChildren) { - var queue = new Queue(roots); + var items = new Queue(roots); - while (!queue.IsNullOrEmpty()) { - var item = queue.Dequeue(); + while (!items.IsNullOrEmpty()) { + var item = items.Dequeue(); yield return item; var children = selectChildren(item) ?? Enumerable.Empty(); foreach (var child in children) { - queue.Enqueue(child); + items.Enqueue(child); } } } From 7a2b9c3ced45251f88297aaf51a0c95cdd8cb316 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 5 Feb 2019 10:14:09 -0800 Subject: [PATCH 034/123] deleting unused variable --- src/Analysis/Ast/Test/IndexParserTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Analysis/Ast/Test/IndexParserTests.cs b/src/Analysis/Ast/Test/IndexParserTests.cs index e5f78c3f8..5183fc09c 100644 --- a/src/Analysis/Ast/Test/IndexParserTests.cs +++ b/src/Analysis/Ast/Test/IndexParserTests.cs @@ -123,7 +123,6 @@ public void CancellParsing() { public void DisposeParserCancelsParsing() { const string testFilePath = "C:/bla.py"; _fileSystem.FileExists(testFilePath).Returns(true); - MemoryStream stream = new MemoryStream(); _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(MakeStream("x = 1")); IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); From b49329488be3b28b27fc5a083bcc5ea427c469a0 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 5 Feb 2019 11:13:10 -0800 Subject: [PATCH 035/123] Limiting API --- src/Analysis/Ast/Impl/Indexing/IIndexManager.cs | 4 ++-- src/Analysis/Ast/Impl/Indexing/IndexManager.cs | 8 +++++--- src/Analysis/Ast/Test/IndexManagerTests.cs | 17 +++++++++++++++++ src/Analysis/Ast/Test/IndexParserTests.cs | 2 +- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs index 076606a67..9c98655ce 100644 --- a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs @@ -25,7 +25,7 @@ internal interface IIndexManager : IDisposable { void ProcessNewFile(string path, IDocument doc); Task ProcessClosedFileAsync(string path, CancellationToken fileCancellationToken = default); void ReIndexFile(string path, IDocument doc); - Task> HierarchicalDocumentSymbolsAsync(string path); - Task> WorkspaceSymbolsAsync(string query); + Task> HierarchicalDocumentSymbolsAsync(string path); + Task> WorkspaceSymbolsAsync(string query, int maxLength = IndexManager.DefaultWorkspaceSymbolsLimit); } } diff --git a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs index cda87e871..a72d75d8d 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs @@ -35,6 +35,8 @@ internal class IndexManager : IIndexManager { private readonly CancellationTokenSource _allIndexCts = new CancellationTokenSource(); private readonly TaskCompletionSource _addRootTcs; + public const int DefaultWorkspaceSymbolsLimit = 50; + public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, string[] excludeFiles) { Check.ArgumentNotNull(nameof(fileSystem), fileSystem); @@ -111,14 +113,14 @@ public void Dispose() { _allIndexCts.Dispose(); } - public async Task> HierarchicalDocumentSymbolsAsync(string path) { + public async Task> HierarchicalDocumentSymbolsAsync(string path) { await AddRootDirectoryAsync(); return _symbolIndex.HierarchicalDocumentSymbols(path).ToList(); } - public async Task> WorkspaceSymbolsAsync(string query) { + public async Task> WorkspaceSymbolsAsync(string query, int maxLength = DefaultWorkspaceSymbolsLimit) { await AddRootDirectoryAsync(); - return _symbolIndex.WorkspaceSymbols(query).ToList(); + return _symbolIndex.WorkspaceSymbols(query).Take(maxLength).ToList(); } } } diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/Analysis/Ast/Test/IndexManagerTests.cs index cfa59cd83..c61c680ee 100644 --- a/src/Analysis/Ast/Test/IndexManagerTests.cs +++ b/src/Analysis/Ast/Test/IndexManagerTests.cs @@ -240,6 +240,23 @@ public async Task WorkspaceSymbolsAddsRootDirectory() { symbols.First().Name.Should().BeEquivalentTo("x"); } + [TestMethod, Priority(0)] + public async Task WorkspaceSymbolsLimited() { + for (int fileNumber = 0; fileNumber < 10; fileNumber++) { + var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}\bla{fileNumber}.py"); + AddFileToRootTestFileSystem(pythonTestFileInfo); + _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(MakeStream($"x{fileNumber} = 1")); + } + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); + + const int amountOfSymbols = 3; + var t = indexManager.WorkspaceSymbolsAsync("", amountOfSymbols); + await t; + + var symbols = t.Result; + symbols.Should().HaveCount(amountOfSymbols); + } + private PythonAst MakeAst(string testCode) { return Parser.CreateParser(MakeStream(testCode), PythonVersions.LatestAvailable3X.Version.ToLanguageVersion()).ParseFile(); } diff --git a/src/Analysis/Ast/Test/IndexParserTests.cs b/src/Analysis/Ast/Test/IndexParserTests.cs index 5183fc09c..7e2983b08 100644 --- a/src/Analysis/Ast/Test/IndexParserTests.cs +++ b/src/Analysis/Ast/Test/IndexParserTests.cs @@ -127,7 +127,7 @@ public void DisposeParserCancelsParsing() { IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); Func parse = async () => { - Task t = indexParser.ParseAsync(testFilePath); + var t = indexParser.ParseAsync(testFilePath); indexParser.Dispose(); await t; }; From fb7f8b3fd4ff8c65c21438b3bfe1549cc119b7a9 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 5 Feb 2019 11:14:15 -0800 Subject: [PATCH 036/123] Dispose test fixed --- src/Analysis/Ast/Test/IndexManagerTests.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/Analysis/Ast/Test/IndexManagerTests.cs index c61c680ee..d264451df 100644 --- a/src/Analysis/Ast/Test/IndexManagerTests.cs +++ b/src/Analysis/Ast/Test/IndexManagerTests.cs @@ -18,6 +18,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Documents; @@ -207,15 +208,20 @@ public void AddingRootMightThrowUnauthorizedAccess() { } [TestMethod, Priority(0)] - public void DisposeManagerCancelsTask() { + public async Task DisposeManagerCancelsTask() { var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}\bla.py"); AddFileToRootTestFileSystem(pythonTestFileInfo); - _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(MakeStream("x = 1")); + ManualResetEventSlim mres = new ManualResetEventSlim(false); + _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(_ => { + mres.Wait(); + return MakeStream("x = 1"); + }); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); + await Task.Delay(1000); + indexManager.Dispose(); Func add = async () => { var t = indexManager.AddRootDirectoryAsync(); - indexManager.Dispose(); await t; }; From c4dcf2376fcc7ea0ebdf6abe868b8bd23633f048 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 5 Feb 2019 12:04:57 -0800 Subject: [PATCH 037/123] Limiting API fix and styling --- src/Analysis/Ast/Impl/Indexing/IIndexManager.cs | 2 +- src/Analysis/Ast/Impl/Indexing/IndexManager.cs | 4 +--- src/Analysis/Ast/Test/IndexManagerTests.cs | 12 ++++-------- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs index 9c98655ce..8bf490f3c 100644 --- a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs @@ -26,6 +26,6 @@ internal interface IIndexManager : IDisposable { Task ProcessClosedFileAsync(string path, CancellationToken fileCancellationToken = default); void ReIndexFile(string path, IDocument doc); Task> HierarchicalDocumentSymbolsAsync(string path); - Task> WorkspaceSymbolsAsync(string query, int maxLength = IndexManager.DefaultWorkspaceSymbolsLimit); + Task> WorkspaceSymbolsAsync(string query, int maxLength); } } diff --git a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs index a72d75d8d..33d335c36 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs @@ -35,8 +35,6 @@ internal class IndexManager : IIndexManager { private readonly CancellationTokenSource _allIndexCts = new CancellationTokenSource(); private readonly TaskCompletionSource _addRootTcs; - public const int DefaultWorkspaceSymbolsLimit = 50; - public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, string[] excludeFiles) { Check.ArgumentNotNull(nameof(fileSystem), fileSystem); @@ -118,7 +116,7 @@ public async Task> HierarchicalDocumentSymbols return _symbolIndex.HierarchicalDocumentSymbols(path).ToList(); } - public async Task> WorkspaceSymbolsAsync(string query, int maxLength = DefaultWorkspaceSymbolsLimit) { + public async Task> WorkspaceSymbolsAsync(string query, int maxLength) { await AddRootDirectoryAsync(); return _symbolIndex.WorkspaceSymbols(query).Take(maxLength).ToList(); } diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/Analysis/Ast/Test/IndexManagerTests.cs index d264451df..e0c3cf8e6 100644 --- a/src/Analysis/Ast/Test/IndexManagerTests.cs +++ b/src/Analysis/Ast/Test/IndexManagerTests.cs @@ -221,8 +221,7 @@ public async Task DisposeManagerCancelsTask() { indexManager.Dispose(); Func add = async () => { - var t = indexManager.AddRootDirectoryAsync(); - await t; + await indexManager.AddRootDirectoryAsync(); }; add.Should().Throw(); @@ -238,9 +237,7 @@ public async Task WorkspaceSymbolsAddsRootDirectory() { IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); - var t = indexManager.WorkspaceSymbolsAsync(""); - await t; - var symbols = t.Result; + var symbols = await indexManager.WorkspaceSymbolsAsync("", 10); symbols.Should().HaveCount(1); symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); symbols.First().Name.Should().BeEquivalentTo("x"); @@ -256,10 +253,9 @@ public async Task WorkspaceSymbolsLimited() { IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); const int amountOfSymbols = 3; - var t = indexManager.WorkspaceSymbolsAsync("", amountOfSymbols); - await t; + await indexManager.WorkspaceSymbolsAsync("", amountOfSymbols); - var symbols = t.Result; + var symbols = indexManager.WorkspaceSymbolsAsync("", amountOfSymbols).Result; symbols.Should().HaveCount(amountOfSymbols); } From aedb206f92feb17596e96452c8d294f1b467f2b5 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 5 Feb 2019 13:16:05 -0800 Subject: [PATCH 038/123] Dispose testing --- src/Analysis/Ast/Impl/Indexing/IndexManager.cs | 2 ++ src/Analysis/Ast/Impl/Indexing/IndexParser.cs | 6 +++--- src/Analysis/Ast/Test/IndexManagerTests.cs | 16 +++++++++++----- src/Analysis/Ast/Test/IndexParserTests.cs | 10 +++++++++- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs index 33d335c36..3b94983dd 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexManager.cs @@ -15,6 +15,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -64,6 +65,7 @@ private void StartAddRootDir() { Task.WaitAll(parseTasks.ToArray(), _allIndexCts.Token); _addRootTcs.SetResult(true); } catch (Exception e) { + Trace.TraceError(e.Message); _addRootTcs.SetException(e); } }); diff --git a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs index e879effc6..33ba034b0 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs +++ b/src/Analysis/Ast/Impl/Indexing/IndexParser.cs @@ -58,13 +58,13 @@ public Task ParseAsync(string path, CancellationToken parseCancellationTok return true; } } catch (FileNotFoundException e) { - Trace.TraceError(e.ToString()); + Trace.TraceError(e.Message); return false; } - }, linkedParseToken).ContinueWith((task) => { + }).ContinueWith((task) => { linkedParseCts.Dispose(); return task.Result; - }); + }, linkedParseToken); } } } diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/Analysis/Ast/Test/IndexManagerTests.cs index e0c3cf8e6..e68fa9f58 100644 --- a/src/Analysis/Ast/Test/IndexManagerTests.cs +++ b/src/Analysis/Ast/Test/IndexManagerTests.cs @@ -208,16 +208,22 @@ public void AddingRootMightThrowUnauthorizedAccess() { } [TestMethod, Priority(0)] - public async Task DisposeManagerCancelsTask() { + public void DisposeManagerCancelsTask() { var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}\bla.py"); AddFileToRootTestFileSystem(pythonTestFileInfo); - ManualResetEventSlim mres = new ManualResetEventSlim(false); + ManualResetEventSlim neverSignaledEvent = new ManualResetEventSlim(false); + ManualResetEventSlim fileOpenedEvent = new ManualResetEventSlim(false); + _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(_ => { - mres.Wait(); - return MakeStream("x = 1"); + fileOpenedEvent.Set(); + // Wait forever + neverSignaledEvent.Wait(); + throw new InternalTestFailureException("Task should have been cancelled"); }); + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); - await Task.Delay(1000); + + fileOpenedEvent.Wait(); indexManager.Dispose(); Func add = async () => { diff --git a/src/Analysis/Ast/Test/IndexParserTests.cs b/src/Analysis/Ast/Test/IndexParserTests.cs index 7e2983b08..a08730daf 100644 --- a/src/Analysis/Ast/Test/IndexParserTests.cs +++ b/src/Analysis/Ast/Test/IndexParserTests.cs @@ -123,11 +123,19 @@ public void CancellParsing() { public void DisposeParserCancelsParsing() { const string testFilePath = "C:/bla.py"; _fileSystem.FileExists(testFilePath).Returns(true); - _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(MakeStream("x = 1")); + ManualResetEventSlim neverSignaledEvent = new ManualResetEventSlim(false); + ManualResetEventSlim fileOpenedEvent = new ManualResetEventSlim(false); + _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(_ => { + fileOpenedEvent.Set(); + // Wait forever + neverSignaledEvent.Wait(); + throw new InternalTestFailureException("Task should have been cancelled"); + }); IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); Func parse = async () => { var t = indexParser.ParseAsync(testFilePath); + fileOpenedEvent.Wait(); indexParser.Dispose(); await t; }; From 1363efc3754da79ae3c5a340e5da57b0139bbd7b Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 7 Feb 2019 10:35:57 -0800 Subject: [PATCH 039/123] Moved everything into language server --- .../Ast/Test/Microsoft.Python.Analysis.Tests.csproj | 1 - .../Ast => LanguageServer}/Impl/Indexing/IIndexManager.cs | 2 +- .../Ast => LanguageServer}/Impl/Indexing/IIndexParser.cs | 2 +- .../Ast => LanguageServer}/Impl/Indexing/ISymbolIndex.cs | 3 +-- .../Ast => LanguageServer}/Impl/Indexing/IndexManager.cs | 2 +- .../Ast => LanguageServer}/Impl/Indexing/IndexParser.cs | 2 +- .../Ast => LanguageServer}/Impl/Indexing/SymbolIndex.cs | 2 +- .../Impl/Indexing/SymbolIndexWalker.cs | 2 +- .../Ast => LanguageServer}/Impl/Indexing/Symbols.cs | 5 +---- .../Ast => LanguageServer}/Test/IndexManagerTests.cs | 6 +++--- .../Ast => LanguageServer}/Test/IndexParserTests.cs | 6 +++--- .../Test/Microsoft.Python.LanguageServer.Tests.csproj | 1 + .../Ast => LanguageServer}/Test/SymbolIndexTests.cs | 3 +-- 13 files changed, 16 insertions(+), 21 deletions(-) rename src/{Analysis/Ast => LanguageServer}/Impl/Indexing/IIndexManager.cs (96%) rename src/{Analysis/Ast => LanguageServer}/Impl/Indexing/IIndexParser.cs (94%) rename src/{Analysis/Ast => LanguageServer}/Impl/Indexing/ISymbolIndex.cs (94%) rename src/{Analysis/Ast => LanguageServer}/Impl/Indexing/IndexManager.cs (98%) rename src/{Analysis/Ast => LanguageServer}/Impl/Indexing/IndexParser.cs (98%) rename src/{Analysis/Ast => LanguageServer}/Impl/Indexing/SymbolIndex.cs (98%) rename src/{Analysis/Ast => LanguageServer}/Impl/Indexing/SymbolIndexWalker.cs (99%) rename src/{Analysis/Ast => LanguageServer}/Impl/Indexing/Symbols.cs (96%) rename src/{Analysis/Ast => LanguageServer}/Test/IndexManagerTests.cs (98%) rename src/{Analysis/Ast => LanguageServer}/Test/IndexParserTests.cs (97%) rename src/{Analysis/Ast => LanguageServer}/Test/SymbolIndexTests.cs (99%) diff --git a/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj b/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj index f8859e18e..345a3ce1d 100644 --- a/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj +++ b/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj @@ -31,7 +31,6 @@ all runtime; build; native; contentfiles; analyzers - diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs b/src/LanguageServer/Impl/Indexing/IIndexManager.cs similarity index 96% rename from src/Analysis/Ast/Impl/Indexing/IIndexManager.cs rename to src/LanguageServer/Impl/Indexing/IIndexManager.cs index 8bf490f3c..4e8829f30 100644 --- a/src/Analysis/Ast/Impl/Indexing/IIndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IIndexManager.cs @@ -19,7 +19,7 @@ using System.Threading.Tasks; using Microsoft.Python.Analysis.Documents; -namespace Microsoft.Python.Analysis.Indexing { +namespace Microsoft.Python.LanguageServer.Indexing { internal interface IIndexManager : IDisposable { Task AddRootDirectoryAsync(); void ProcessNewFile(string path, IDocument doc); diff --git a/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs b/src/LanguageServer/Impl/Indexing/IIndexParser.cs similarity index 94% rename from src/Analysis/Ast/Impl/Indexing/IIndexParser.cs rename to src/LanguageServer/Impl/Indexing/IIndexParser.cs index c9fea675a..20f0d8c8c 100644 --- a/src/Analysis/Ast/Impl/Indexing/IIndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IIndexParser.cs @@ -17,7 +17,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Microsoft.Python.Analysis.Indexing { +namespace Microsoft.Python.LanguageServer.Indexing { internal interface IIndexParser : IDisposable { Task ParseAsync(string path, CancellationToken cancellationToken = default); } diff --git a/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs similarity index 94% rename from src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs rename to src/LanguageServer/Impl/Indexing/ISymbolIndex.cs index c46507893..e63e25fd7 100644 --- a/src/Analysis/Ast/Impl/Indexing/ISymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs @@ -13,11 +13,10 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using Microsoft.Python.Parsing.Ast; -namespace Microsoft.Python.Analysis.Indexing { +namespace Microsoft.Python.LanguageServer.Indexing { internal interface ISymbolIndex { IEnumerable WorkspaceSymbols(string query); IEnumerable HierarchicalDocumentSymbols(string path); diff --git a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs similarity index 98% rename from src/Analysis/Ast/Impl/Indexing/IndexManager.cs rename to src/LanguageServer/Impl/Indexing/IndexManager.cs index 3b94983dd..e9975bcf4 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -25,7 +25,7 @@ using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; -namespace Microsoft.Python.Analysis.Indexing { +namespace Microsoft.Python.LanguageServer.Indexing { internal class IndexManager : IIndexManager { private readonly ISymbolIndex _symbolIndex; private readonly IFileSystem _fileSystem; diff --git a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs similarity index 98% rename from src/Analysis/Ast/Impl/Indexing/IndexParser.cs rename to src/LanguageServer/Impl/Indexing/IndexParser.cs index 33ba034b0..e3368304b 100644 --- a/src/Analysis/Ast/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -21,7 +21,7 @@ using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; -namespace Microsoft.Python.Analysis.Indexing { +namespace Microsoft.Python.LanguageServer.Indexing { internal sealed class IndexParser : IIndexParser { private readonly ISymbolIndex _symbolIndex; private readonly IFileSystem _fileSystem; diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs similarity index 98% rename from src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs rename to src/LanguageServer/Impl/Indexing/SymbolIndex.cs index 1c70244cd..e226bdbb0 100644 --- a/src/Analysis/Ast/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -21,7 +21,7 @@ using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing.Ast; -namespace Microsoft.Python.Analysis.Indexing { +namespace Microsoft.Python.LanguageServer.Indexing { internal sealed class SymbolIndex : ISymbolIndex { private readonly ConcurrentDictionary> _index; diff --git a/src/Analysis/Ast/Impl/Indexing/SymbolIndexWalker.cs b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs similarity index 99% rename from src/Analysis/Ast/Impl/Indexing/SymbolIndexWalker.cs rename to src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs index 477a457ab..93dc0b1bc 100644 --- a/src/Analysis/Ast/Impl/Indexing/SymbolIndexWalker.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs @@ -19,7 +19,7 @@ using Microsoft.Python.Core; using Microsoft.Python.Parsing.Ast; -namespace Microsoft.Python.Analysis.Indexing { +namespace Microsoft.Python.LanguageServer.Indexing { internal class SymbolIndexWalker : PythonWalker { private static readonly Regex DoubleUnderscore = new Regex(@"^__.*__$", RegexOptions.Compiled); private static readonly Regex ConstantLike = new Regex(@"^[\p{Lu}\p{N}_]+$", RegexOptions.Compiled); diff --git a/src/Analysis/Ast/Impl/Indexing/Symbols.cs b/src/LanguageServer/Impl/Indexing/Symbols.cs similarity index 96% rename from src/Analysis/Ast/Impl/Indexing/Symbols.cs rename to src/LanguageServer/Impl/Indexing/Symbols.cs index 6435e0685..0cdac218d 100644 --- a/src/Analysis/Ast/Impl/Indexing/Symbols.cs +++ b/src/LanguageServer/Impl/Indexing/Symbols.cs @@ -13,13 +13,10 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; -using System.Linq; -using Microsoft.Python.Core; using Microsoft.Python.Core.Text; -namespace Microsoft.Python.Analysis.Indexing { +namespace Microsoft.Python.LanguageServer.Indexing { // From LSP. internal enum SymbolKind { None = 0, diff --git a/src/Analysis/Ast/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs similarity index 98% rename from src/Analysis/Ast/Test/IndexManagerTests.cs rename to src/LanguageServer/Test/IndexManagerTests.cs index e68fa9f58..f8c78bc67 100644 --- a/src/Analysis/Ast/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -22,8 +22,8 @@ using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Analysis.Indexing; using Microsoft.Python.Core.IO; +using Microsoft.Python.LanguageServer.Indexing; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; using Microsoft.Python.Parsing.Tests; @@ -31,9 +31,9 @@ using NSubstitute; using TestUtilities; -namespace Microsoft.Python.Analysis.Tests { +namespace Microsoft.Python.LanguageServer.Tests { [TestClass] - public class IndexManagerTests : AnalysisTestBase { + public class IndexManagerTests : LanguageServerTestBase { private IFileSystem _fileSystem; private ISymbolIndex _symbolIndex; private string _rootPath; diff --git a/src/Analysis/Ast/Test/IndexParserTests.cs b/src/LanguageServer/Test/IndexParserTests.cs similarity index 97% rename from src/Analysis/Ast/Test/IndexParserTests.cs rename to src/LanguageServer/Test/IndexParserTests.cs index a08730daf..48415d880 100644 --- a/src/Analysis/Ast/Test/IndexParserTests.cs +++ b/src/LanguageServer/Test/IndexParserTests.cs @@ -20,18 +20,18 @@ using System.Threading; using System.Threading.Tasks; using FluentAssertions; -using Microsoft.Python.Analysis.Indexing; using Microsoft.Python.Core.IO; +using Microsoft.Python.LanguageServer.Indexing; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; using NSubstitute; using TestUtilities; -namespace Microsoft.Python.Analysis.Tests { +namespace Microsoft.Python.LanguageServer.Tests { [TestClass] - public class IndexParserTests : AnalysisTestBase { + public class IndexParserTests : LanguageServerTestBase { private ISymbolIndex _symbolIndex; private IFileSystem _fileSystem; private PythonLanguageVersion _pythonLanguageVersion; diff --git a/src/LanguageServer/Test/Microsoft.Python.LanguageServer.Tests.csproj b/src/LanguageServer/Test/Microsoft.Python.LanguageServer.Tests.csproj index f1a3b09f0..20a662dbb 100644 --- a/src/LanguageServer/Test/Microsoft.Python.LanguageServer.Tests.csproj +++ b/src/LanguageServer/Test/Microsoft.Python.LanguageServer.Tests.csproj @@ -31,6 +31,7 @@ all runtime; build; native; contentfiles; analyzers + diff --git a/src/Analysis/Ast/Test/SymbolIndexTests.cs b/src/LanguageServer/Test/SymbolIndexTests.cs similarity index 99% rename from src/Analysis/Ast/Test/SymbolIndexTests.cs rename to src/LanguageServer/Test/SymbolIndexTests.cs index bc0e30d01..db2e495a5 100644 --- a/src/Analysis/Ast/Test/SymbolIndexTests.cs +++ b/src/LanguageServer/Test/SymbolIndexTests.cs @@ -12,12 +12,11 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.IO; using FluentAssertions; -using Microsoft.Python.Analysis.Indexing; using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Indexing; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; using Microsoft.Python.Tests.Utilities.FluentAssertions; From d788f163a5d97c956f83b3b4180b82053bd379e7 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 7 Feb 2019 15:20:52 -0800 Subject: [PATCH 040/123] Tests refactor and connecting index to ls --- .../Implementation/Server.WorkspaceSymbols.cs | 60 ++++--- .../Impl/Implementation/Server.cs | 9 + .../Impl/Indexing/IIndexManager.cs | 6 +- .../Impl/Indexing/IndexManager.cs | 15 +- src/LanguageServer/Test/IndexManagerTests.cs | 156 ++++++++++-------- 5 files changed, 146 insertions(+), 100 deletions(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs index b5f658a06..2f680c750 100644 --- a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs @@ -14,9 +14,11 @@ // permissions and limitations under the License. using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; +using Microsoft.Python.LanguageServer.Indexing; using Microsoft.Python.LanguageServer.Protocol; namespace Microsoft.Python.LanguageServer.Implementation { @@ -25,32 +27,50 @@ public sealed partial class Server { private static int _symbolHierarchyMaxSymbols = 1000; public async Task WorkspaceSymbols(WorkspaceSymbolParams @params, CancellationToken cancellationToken) { - return Array.Empty< SymbolInformation>(); + var symbols = await _indexManager.WorkspaceSymbolsAsync(@params.query, + _symbolHierarchyMaxSymbols, + cancellationToken); + return symbols.Select(MakeSymbolInfo).ToArray(); } - public async Task DocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { - return Array.Empty(); - } + /*public async Task DocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { + var path = @params.textDocument.uri.AbsolutePath; + var symbols = await _indexManager.HierarchicalDocumentSymbolsAsync(path, cancellationToken); + return symbols.Flatten(path, depthLimit: _symbolHierarchyDepthLimit) + .Select(s => { + cancellationToken.ThrowIfCancellationRequested(); + return MakeSymbolInfo(s); + }).ToArray(); + }*/ public async Task HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { - return Array.Empty(); + var path = @params.textDocument.uri.AbsolutePath; + var symbols = await _indexManager.HierarchicalDocumentSymbolsAsync(path, cancellationToken); + return symbols.MaybeEnumerate().Select(hSym => MakeDocumentSymbol(hSym)).ToArray(); } + private static SymbolInformation MakeSymbolInfo(Indexing.FlatSymbol s) { + return new SymbolInformation { + name = s.Name, + kind = (Protocol.SymbolKind)s.Kind, + location = new Location { + range = s.Range, + uri = new Uri(s.DocumentPath), + }, + containerName = s.ContainerName, + }; + } - private static SymbolKind ToSymbolKind(PythonMemberType memberType) { - switch (memberType) { - case PythonMemberType.Unknown: return SymbolKind.None; - case PythonMemberType.Class: return SymbolKind.Class; - case PythonMemberType.Instance: return SymbolKind.Variable; - case PythonMemberType.Function: return SymbolKind.Function; - case PythonMemberType.Method: return SymbolKind.Method; - case PythonMemberType.Module: return SymbolKind.Module; - case PythonMemberType.Property: return SymbolKind.Property; - case PythonMemberType.Union: return SymbolKind.Object; - case PythonMemberType.Variable: return SymbolKind.Variable; - case PythonMemberType.Generic: return SymbolKind.TypeParameter; - default: return SymbolKind.None; - } + private DocumentSymbol MakeDocumentSymbol(HierarchicalSymbol hSym) { + return new DocumentSymbol { + name = hSym.Name, + detail = hSym.Detail, + kind = (Protocol.SymbolKind)hSym.Kind, + deprecated = hSym.Deprecated ?? false, + range = hSym.Range, + selectionRange = hSym.SelectionRange, + children = hSym.Children.MaybeEnumerate().Select(MakeDocumentSymbol).ToArray(), + }; } } } diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index 08d90d8fb..6e2e10594 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -30,6 +30,7 @@ using Microsoft.Python.Core.Shell; using Microsoft.Python.LanguageServer.Completion; using Microsoft.Python.LanguageServer.Diagnostics; +using Microsoft.Python.LanguageServer.Indexing; using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.LanguageServer.Sources; @@ -43,6 +44,7 @@ public sealed partial class Server : IDisposable { private IRunningDocumentTable _rdt; private ClientCapabilities _clientCaps; private ILogger _log; + private IIndexManager _indexManager; public static InformationDisplayOptions DisplayOptions { get; private set; } = new InformationDisplayOptions { preferredFormat = MarkupKind.PlainText, @@ -120,6 +122,13 @@ public async Task InitializeAsync(InitializeParams @params, Ca _interpreter = await PythonInterpreter.CreateAsync(configuration, rootDir, _services, cancellationToken); _services.AddService(_interpreter); + var symbolIndex = new SymbolIndex(); + var fileSystem = _services.GetService(); + _indexManager = new IndexManager(symbolIndex, fileSystem, _interpreter.LanguageVersion, rootDir, + @params.initializationOptions.includeFiles, + @params.initializationOptions.excludeFiles); + _services.AddService(_indexManager); + DisplayStartupInfo(); var ds = new PlainTextDocumentationSource(); diff --git a/src/LanguageServer/Impl/Indexing/IIndexManager.cs b/src/LanguageServer/Impl/Indexing/IIndexManager.cs index 4e8829f30..f1b2a2af6 100644 --- a/src/LanguageServer/Impl/Indexing/IIndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IIndexManager.cs @@ -21,11 +21,11 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal interface IIndexManager : IDisposable { - Task AddRootDirectoryAsync(); + Task AddRootDirectoryAsync(CancellationToken cancellationToken = default); void ProcessNewFile(string path, IDocument doc); Task ProcessClosedFileAsync(string path, CancellationToken fileCancellationToken = default); void ReIndexFile(string path, IDocument doc); - Task> HierarchicalDocumentSymbolsAsync(string path); - Task> WorkspaceSymbolsAsync(string query, int maxLength); + Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken cancellationToken = default); + Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken cancellationToken = default); } } diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index e9975bcf4..1cb2f6ed6 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -68,11 +68,12 @@ private void StartAddRootDir() { Trace.TraceError(e.Message); _addRootTcs.SetException(e); } - }); + }, _allIndexCts.Token); } - public Task AddRootDirectoryAsync() { - return _addRootTcs.Task; + public Task AddRootDirectoryAsync(CancellationToken cancellationToken = default) { + // Add cancellation token around task + return Task.Run(async () => await _addRootTcs.Task, cancellationToken); } @@ -113,13 +114,13 @@ public void Dispose() { _allIndexCts.Dispose(); } - public async Task> HierarchicalDocumentSymbolsAsync(string path) { - await AddRootDirectoryAsync(); + public async Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken cancellationToken = default) { + await AddRootDirectoryAsync(cancellationToken); return _symbolIndex.HierarchicalDocumentSymbols(path).ToList(); } - public async Task> WorkspaceSymbolsAsync(string query, int maxLength) { - await AddRootDirectoryAsync(); + public async Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken cancellationToken = default) { + await AddRootDirectoryAsync(cancellationToken); return _symbolIndex.WorkspaceSymbols(query).Take(maxLength).ToList(); } } diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index f8c78bc67..20d4a7abc 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -61,26 +61,22 @@ public void TestInitialize() { [TestMethod, Priority(0)] public async Task AddsRootDirectoryAsync() { - var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}\bla.py"); - AddFileToRootTestFileSystem(pythonTestFileInfo); - var fooFile = MakeFileInfoProxy($"{_rootPath}\foo.py"); - AddFileToRootTestFileSystem(fooFile); - _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(MakeStream("x = 1")); - _fileSystem.FileOpen(fooFile.FullName, FileMode.Open).Returns(MakeStream("y = 1")); - - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); + FileWithXVarInRootDir(); + AddFileToRoot($"{_rootPath}\foo.py", MakeStream("y = 1")); + + var indexManager = GetDefaultIndexManager(); await indexManager.AddRootDirectoryAsync(); - var symbols = _symbolIndex.WorkspaceSymbols("x"); - symbols.Should().HaveCount(1); - symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); - symbols.First().Name.Should().BeEquivalentTo("x"); + var symbols = _symbolIndex.WorkspaceSymbols(""); + symbols.Should().HaveCount(2); } [TestMethod, Priority(0)] public void NullDirectoryThrowsException() { Action construct = () => { - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), null, new string[] { }, new string[] { }); + PythonLanguageVersion version = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); + IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, + version, null, new string[] { }, new string[] { }); }; construct.Should().Throw(); } @@ -88,7 +84,7 @@ public void NullDirectoryThrowsException() { [TestMethod, Priority(0)] public void IgnoresNonPythonFiles() { var nonPythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}/bla.txt"); - AddFileToRootTestFileSystem(nonPythonTestFileInfo); + AddFileInfoToRootTestFS(nonPythonTestFileInfo); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), this._rootPath, new string[] { }, new string[] { }); indexManager.AddRootDirectoryAsync(); @@ -103,27 +99,22 @@ public void CanOpenFiles() { IDocument doc = Substitute.For(); doc.GetAnyAst().Returns(MakeAst("x = 1")); - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), _rootPath, new string[] { }, new string[] { }); + IIndexManager indexManager = GetDefaultIndexManager(); indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); var symbols = _symbolIndex.WorkspaceSymbols(""); - symbols.Should().HaveCount(1); - symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); - symbols.First().Name.Should().BeEquivalentTo("x"); + SymbolsShouldBeOnlyX(symbols); } [TestMethod, Priority(0)] public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { - var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}/bla.py"); - AddFileToRootTestFileSystem(pythonTestFileInfo); - _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(MakeStream("x = 1")); - + var pythonTestFilePath = FileWithXVarInRootDir(); IDocument latestDoc = Substitute.For(); latestDoc.GetAnyAst().Returns(MakeAst("y = 1")); - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), _rootPath, new string[] { }, new string[] { }); + var indexManager = GetDefaultIndexManager(); await indexManager.AddRootDirectoryAsync(); - indexManager.ReIndexFile(pythonTestFileInfo.FullName, latestDoc); + indexManager.ReIndexFile(pythonTestFilePath, latestDoc); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(1); @@ -133,40 +124,34 @@ public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { [TestMethod, Priority(0)] public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { - string nonRootPath = "C:/nonRoot"; - var pythonTestFileInfo = MakeFileInfoProxy($"{nonRootPath}/bla.py"); + var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); IDocument doc = Substitute.For(); doc.GetAnyAst().Returns(MakeAst("x = 1")); - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), _rootPath, new string[] { }, new string[] { }); + var indexManager = GetDefaultIndexManager(); indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); await indexManager.ProcessClosedFileAsync(pythonTestFileInfo.FullName); - var symbols = _symbolIndex.WorkspaceSymbols(""); - symbols.Should().HaveCount(0); + SymbolIndexShouldBeEmpty(); } [TestMethod, Priority(0)] public async Task CloseWorkspaceFilesReUpdatesIndexAsync() { - var pythonTestFile = MakeFileInfoProxy($"{_rootPath}/bla.py"); - AddFileToRootTestFileSystem(pythonTestFile); - _fileSystem.FileOpen(pythonTestFile.FullName, FileMode.Open).Returns(MakeStream("x = 1")); - _fileSystem.IsPathUnderRoot(_rootPath, pythonTestFile.FullName).Returns(true); + string pythonTestFilePath = FileWithXVarInRootDir(); + _fileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); IDocument latestDoc = Substitute.For(); latestDoc.GetAnyAst().Returns(MakeAst("y = 1")); - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), _rootPath, new string[] { }, new string[] { }); + var indexManager = GetDefaultIndexManager(); await indexManager.AddRootDirectoryAsync(); - indexManager.ProcessNewFile(pythonTestFile.FullName, latestDoc); + indexManager.ProcessNewFile(pythonTestFilePath, latestDoc); // It Needs to remake the stream for the file, previous one is closed - _fileSystem.FileOpen(pythonTestFile.FullName, FileMode.Open).Returns(MakeStream("x = 1")); - await indexManager.ProcessClosedFileAsync(pythonTestFile.FullName); + _fileSystem.FileOpen(pythonTestFilePath, FileMode.Open).Returns(MakeStream("x = 1")); + await indexManager.ProcessClosedFileAsync(pythonTestFilePath); var symbols = _symbolIndex.WorkspaceSymbols(""); - symbols.Should().HaveCount(1); - symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); - symbols.First().Name.Should().BeEquivalentTo("x"); + SymbolsShouldBeOnlyX(symbols); } [TestMethod, Priority(0)] @@ -174,55 +159,53 @@ public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() { // If events get to index manager in the order: [open, close, update] // it should not reindex file - string nonRootPath = "C:/nonRoot"; - var pythonTestFileInfo = MakeFileInfoProxy($"{nonRootPath}/bla.py"); + var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); IDocument doc = Substitute.For(); doc.GetAnyAst().Returns(MakeAst("x = 1")); - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), _rootPath, new string[] { }, new string[] { }); + var indexManager = GetDefaultIndexManager(); indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); await indexManager.ProcessClosedFileAsync(pythonTestFileInfo.FullName); doc.GetAnyAst().Returns(MakeAst("x = 1")); indexManager.ReIndexFile(pythonTestFileInfo.FullName, doc); - var symbols = _symbolIndex.WorkspaceSymbols(""); - symbols.Should().HaveCount(0); + SymbolIndexShouldBeEmpty(); } [TestMethod, Priority(0)] public void AddingRootMightThrowUnauthorizedAccess() { - var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}\bla.py"); - AddFileToRootTestFileSystem(pythonTestFileInfo); + string pythonTestFilePath = FileWithXVarInRootDir(); _fileSystem.GetDirectoryInfo(_rootPath).EnumerateFileSystemInfos(new string[] { }, new string[] { }) .ReturnsForAnyArgs(_ => throw new UnauthorizedAccessException()); - _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(MakeStream("x = 1")); - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); + var indexManager = GetDefaultIndexManager(); Func add = async () => { await indexManager.AddRootDirectoryAsync(); }; add.Should().Throw(); + SymbolIndexShouldBeEmpty(); + } + + private void SymbolIndexShouldBeEmpty() { var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(0); } [TestMethod, Priority(0)] public void DisposeManagerCancelsTask() { - var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}\bla.py"); - AddFileToRootTestFileSystem(pythonTestFileInfo); + string pythonTestFilePath = FileWithXVarInRootDir(); ManualResetEventSlim neverSignaledEvent = new ManualResetEventSlim(false); ManualResetEventSlim fileOpenedEvent = new ManualResetEventSlim(false); - _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(_ => { + _fileSystem.FileOpen(pythonTestFilePath, FileMode.Open).Returns(_ => { fileOpenedEvent.Set(); // Wait forever neverSignaledEvent.Wait(); throw new InternalTestFailureException("Task should have been cancelled"); }); - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); - + var indexManager = GetDefaultIndexManager(); fileOpenedEvent.Wait(); indexManager.Dispose(); @@ -231,40 +214,54 @@ public void DisposeManagerCancelsTask() { }; add.Should().Throw(); - var symbols = _symbolIndex.WorkspaceSymbols(""); - symbols.Should().HaveCount(0); + SymbolIndexShouldBeEmpty(); } [TestMethod, Priority(0)] public async Task WorkspaceSymbolsAddsRootDirectory() { - var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}\bla.py"); - AddFileToRootTestFileSystem(pythonTestFileInfo); - _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(MakeStream("x = 1")); + string pythonTestFilePath = FileWithXVarInRootDir(); - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); + var indexManager = GetDefaultIndexManager(); var symbols = await indexManager.WorkspaceSymbolsAsync("", 10); - symbols.Should().HaveCount(1); - symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); - symbols.First().Name.Should().BeEquivalentTo("x"); + SymbolsShouldBeOnlyX(symbols); } [TestMethod, Priority(0)] public async Task WorkspaceSymbolsLimited() { for (int fileNumber = 0; fileNumber < 10; fileNumber++) { - var pythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}\bla{fileNumber}.py"); - AddFileToRootTestFileSystem(pythonTestFileInfo); - _fileSystem.FileOpen(pythonTestFileInfo.FullName, FileMode.Open).Returns(MakeStream($"x{fileNumber} = 1")); + AddFileToRoot($"{_rootPath}\bla{fileNumber}.py", MakeStream($"x{fileNumber} = 1")); } - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }); + var indexManager = GetDefaultIndexManager(); const int amountOfSymbols = 3; - await indexManager.WorkspaceSymbolsAsync("", amountOfSymbols); - var symbols = indexManager.WorkspaceSymbolsAsync("", amountOfSymbols).Result; + var symbols = await indexManager.WorkspaceSymbolsAsync("", amountOfSymbols); symbols.Should().HaveCount(amountOfSymbols); } + [TestMethod, Priority(0)] + public async Task HierarchicalDocumentSymbolsAsync() { + string pythonTestFilePath = FileWithXVarInRootDir(); + + var indexManager = GetDefaultIndexManager(); + + var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath); + SymbolsShouldBeOnlyX(symbols); + } + + private static void SymbolsShouldBeOnlyX(IEnumerable symbols) { + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("x"); + } + + private static void SymbolsShouldBeOnlyX(IEnumerable symbols) { + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("x"); + } + private PythonAst MakeAst(string testCode) { return Parser.CreateParser(MakeStream(testCode), PythonVersions.LatestAvailable3X.Version.ToLanguageVersion()).ParseFile(); } @@ -272,7 +269,7 @@ private PythonAst MakeAst(string testCode) { private FileInfoProxy MakeFileInfoProxy(string filePath) => new FileInfoProxy(new FileInfo(filePath)); - private void AddFileToRootTestFileSystem(FileInfoProxy fileInfo) { + private void AddFileInfoToRootTestFS(FileInfoProxy fileInfo) { _rootFileList.Add(fileInfo); _fileSystem.FileExists(fileInfo.FullName).Returns(true); } @@ -280,5 +277,24 @@ private void AddFileToRootTestFileSystem(FileInfoProxy fileInfo) { private Stream MakeStream(string str) { return new MemoryStream(Encoding.UTF8.GetBytes(str)); } + + private string FileWithXVarInRootDir() { + return AddFileToRoot($"{_rootPath}\bla.py", MakeStream("x = 1")); + } + + private IIndexManager GetDefaultIndexManager() { + return new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, + _rootPath, new string[] { }, new string[] { }); + } + + private string AddFileToRoot(string filePath, Stream stream) { + var fileInfo = MakeFileInfoProxy(filePath); + AddFileInfoToRootTestFS(fileInfo); + string fullName = fileInfo.FullName; + _fileSystem.FileOpen(fullName, FileMode.Open).Returns(stream); + // FileInfo fullName is used everywhere as path + // Otherwise, path discrepancies might appear + return fullName; + } } } From 98c18aa1b392aadd279b25d190dbfbdb2af390c9 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 7 Feb 2019 16:03:46 -0800 Subject: [PATCH 041/123] Adding file events --- .../Impl/Implementation/Server.Documents.cs | 16 +++++++++++----- .../Implementation/Server.WorkspaceSymbols.cs | 10 ---------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.Documents.cs b/src/LanguageServer/Impl/Implementation/Server.Documents.cs index d84fe1a5b..a7d3ab282 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Documents.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Documents.cs @@ -29,14 +29,17 @@ namespace Microsoft.Python.LanguageServer.Implementation { public sealed partial class Server { public void DidOpenTextDocument(DidOpenTextDocumentParams @params) { _disposableBag.ThrowIfDisposed(); - _log?.Log(TraceEventType.Verbose, $"Opening document {@params.textDocument.uri}"); + Uri uri = @params.textDocument.uri; + _log?.Log(TraceEventType.Verbose, $"Opening document {uri}"); - _rdt.OpenDocument(@params.textDocument.uri, @params.textDocument.text); + var doc = _rdt.OpenDocument(uri, @params.textDocument.text); + _indexManager.ProcessNewFile(uri.AbsolutePath, doc); } public void DidChangeTextDocument(DidChangeTextDocumentParams @params) { _disposableBag.ThrowIfDisposed(); - var doc = _rdt.GetDocument(@params.textDocument.uri); + Uri uri = @params.textDocument.uri; + var doc = _rdt.GetDocument(uri); if (doc != null) { var changes = new List(); foreach (var c in @params.contentChanges) { @@ -48,6 +51,7 @@ public void DidChangeTextDocument(DidChangeTextDocumentParams @params) { changes.Add(change); } doc.Update(changes); + _indexManager.ReIndexFile(uri.AbsolutePath, doc); } else { _log?.Log(TraceEventType.Warning, $"Unable to find document for {@params.textDocument.uri}"); } @@ -60,9 +64,11 @@ public void DidChangeWatchedFiles(DidChangeWatchedFilesParams @params) { } } - public void DidCloseTextDocument(DidCloseTextDocumentParams @params) { + public async void DidCloseTextDocument(DidCloseTextDocumentParams @params) { _disposableBag.ThrowIfDisposed(); - _rdt.CloseDocument(@params.textDocument.uri); + Uri uri = @params.textDocument.uri; + _rdt.CloseDocument(uri); + await _indexManager.ProcessClosedFileAsync(uri.AbsolutePath); } private IDocumentAnalysis GetAnalysis(Uri uri, CancellationToken cancellationToken) { diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs index 2f680c750..f4d5c8f8d 100644 --- a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs @@ -33,16 +33,6 @@ public async Task WorkspaceSymbols(WorkspaceSymbolParams @p return symbols.Select(MakeSymbolInfo).ToArray(); } - /*public async Task DocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { - var path = @params.textDocument.uri.AbsolutePath; - var symbols = await _indexManager.HierarchicalDocumentSymbolsAsync(path, cancellationToken); - return symbols.Flatten(path, depthLimit: _symbolHierarchyDepthLimit) - .Select(s => { - cancellationToken.ThrowIfCancellationRequested(); - return MakeSymbolInfo(s); - }).ToArray(); - }*/ - public async Task HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { var path = @params.textDocument.uri.AbsolutePath; var symbols = await _indexManager.HierarchicalDocumentSymbolsAsync(path, cancellationToken); From eb83604dd39d870ef48e640621f0dc0b0144e90e Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 7 Feb 2019 16:27:04 -0800 Subject: [PATCH 042/123] async fix --- src/LanguageServer/Impl/Implementation/Server.Documents.cs | 4 ++-- src/LanguageServer/Impl/Indexing/IIndexManager.cs | 2 +- src/LanguageServer/Impl/Indexing/IndexManager.cs | 6 ++---- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.Documents.cs b/src/LanguageServer/Impl/Implementation/Server.Documents.cs index a7d3ab282..51041d54b 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Documents.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Documents.cs @@ -64,11 +64,11 @@ public void DidChangeWatchedFiles(DidChangeWatchedFilesParams @params) { } } - public async void DidCloseTextDocument(DidCloseTextDocumentParams @params) { + public void DidCloseTextDocument(DidCloseTextDocumentParams @params) { _disposableBag.ThrowIfDisposed(); Uri uri = @params.textDocument.uri; _rdt.CloseDocument(uri); - await _indexManager.ProcessClosedFileAsync(uri.AbsolutePath); + _indexManager.ProcessClosedFileAsync(uri.AbsolutePath).DoNotWait(); } private IDocumentAnalysis GetAnalysis(Uri uri, CancellationToken cancellationToken) { diff --git a/src/LanguageServer/Impl/Indexing/IIndexManager.cs b/src/LanguageServer/Impl/Indexing/IIndexManager.cs index f1b2a2af6..af2a73095 100644 --- a/src/LanguageServer/Impl/Indexing/IIndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IIndexManager.cs @@ -23,7 +23,7 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal interface IIndexManager : IDisposable { Task AddRootDirectoryAsync(CancellationToken cancellationToken = default); void ProcessNewFile(string path, IDocument doc); - Task ProcessClosedFileAsync(string path, CancellationToken fileCancellationToken = default); + Task ProcessClosedFileAsync(string path); void ReIndexFile(string path, IDocument doc); Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken cancellationToken = default); Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken cancellationToken = default); diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index 1cb2f6ed6..01371305b 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -83,13 +83,11 @@ private IEnumerable WorkspaceFiles() { private bool IsFileIndexed(string path) => _symbolIndex.IsIndexed(path); - public Task ProcessClosedFileAsync(string path, CancellationToken fileCancellationToken = default) { + public Task ProcessClosedFileAsync(string path) { // If path is on workspace if (IsFileOnWorkspace(path)) { // updates index and ignores previous AST - var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(fileCancellationToken, _allIndexCts.Token); - return _indexParser.ParseAsync(path, linkedCts.Token) - .ContinueWith(_ => linkedCts.Dispose()); + return _indexParser.ParseAsync(path, _allIndexCts.Token); } else { // remove file from index _symbolIndex.Delete(path); From 8b90ac703209af9e083a6f81ce1022664e359049 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 7 Feb 2019 17:43:28 -0800 Subject: [PATCH 043/123] Enumerable fix --- src/Core/Impl/Extensions/EnumerableExtensions.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Core/Impl/Extensions/EnumerableExtensions.cs b/src/Core/Impl/Extensions/EnumerableExtensions.cs index cbb8a555a..e2fed366c 100644 --- a/src/Core/Impl/Extensions/EnumerableExtensions.cs +++ b/src/Core/Impl/Extensions/EnumerableExtensions.cs @@ -89,9 +89,8 @@ public static IEnumerable IndexWhere(this IEnumerable source, Func TraverseBreadthFirst(this T root, Func> selectChildren) { - return new[] { root }.TraverseBreadthFirst(selectChildren); - } + public static IEnumerable TraverseBreadthFirst(this T root, Func> selectChildren) + => Enumerable.Repeat(root, 1).TraverseBreadthFirst(selectChildren); public static IEnumerable TraverseBreadthFirst(this IEnumerable roots, Func> selectChildren) { var items = new Queue(roots); @@ -100,7 +99,7 @@ public static IEnumerable TraverseBreadthFirst(this IEnumerable roots, var item = items.Dequeue(); yield return item; - var children = selectChildren(item) ?? Enumerable.Empty(); + var children = selectChildren(item).MaybeEnumerate(); foreach (var child in children) { items.Enqueue(child); } From fae4315555f45b7d125bf34084ca25af0d3d79f5 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 7 Feb 2019 17:48:02 -0800 Subject: [PATCH 044/123] Delete copy of SymbolKind --- .../Implementation/Server.WorkspaceSymbols.cs | 6 ++-- .../Impl/Indexing/SymbolIndexWalker.cs | 1 + src/LanguageServer/Impl/Indexing/Symbols.cs | 31 +------------------ src/LanguageServer/Test/IndexManagerTests.cs | 1 + src/LanguageServer/Test/IndexParserTests.cs | 1 + src/LanguageServer/Test/SymbolIndexTests.cs | 1 + 6 files changed, 8 insertions(+), 33 deletions(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs index f4d5c8f8d..cf8da50aa 100644 --- a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs @@ -39,10 +39,10 @@ public async Task HierarchicalDocumentSymbol(DocumentSymbolPar return symbols.MaybeEnumerate().Select(hSym => MakeDocumentSymbol(hSym)).ToArray(); } - private static SymbolInformation MakeSymbolInfo(Indexing.FlatSymbol s) { + private static SymbolInformation MakeSymbolInfo(FlatSymbol s) { return new SymbolInformation { name = s.Name, - kind = (Protocol.SymbolKind)s.Kind, + kind = s.Kind, location = new Location { range = s.Range, uri = new Uri(s.DocumentPath), @@ -55,7 +55,7 @@ private DocumentSymbol MakeDocumentSymbol(HierarchicalSymbol hSym) { return new DocumentSymbol { name = hSym.Name, detail = hSym.Detail, - kind = (Protocol.SymbolKind)hSym.Kind, + kind = hSym.Kind, deprecated = hSym.Deprecated ?? false, range = hSym.Range, selectionRange = hSym.SelectionRange, diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs index 93dc0b1bc..bac40516d 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs @@ -17,6 +17,7 @@ using System.Linq; using System.Text.RegularExpressions; using Microsoft.Python.Core; +using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.LanguageServer.Indexing { diff --git a/src/LanguageServer/Impl/Indexing/Symbols.cs b/src/LanguageServer/Impl/Indexing/Symbols.cs index 0cdac218d..4988027fc 100644 --- a/src/LanguageServer/Impl/Indexing/Symbols.cs +++ b/src/LanguageServer/Impl/Indexing/Symbols.cs @@ -15,38 +15,9 @@ using System.Collections.Generic; using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Protocol; namespace Microsoft.Python.LanguageServer.Indexing { - // From LSP. - internal enum SymbolKind { - None = 0, - File = 1, - Module = 2, - Namespace = 3, - Package = 4, - Class = 5, - Method = 6, - Property = 7, - Field = 8, - Constructor = 9, - Enum = 10, - Interface = 11, - Function = 12, - Variable = 13, - Constant = 14, - String = 15, - Number = 16, - Boolean = 17, - Array = 18, - Object = 19, - Key = 20, - Null = 21, - EnumMember = 22, - Struct = 23, - Event = 24, - Operator = 25, - TypeParameter = 26 - } internal class FunctionKind { public const string None = ""; diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index 20d4a7abc..854d17f0c 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -24,6 +24,7 @@ using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core.IO; using Microsoft.Python.LanguageServer.Indexing; +using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; using Microsoft.Python.Parsing.Tests; diff --git a/src/LanguageServer/Test/IndexParserTests.cs b/src/LanguageServer/Test/IndexParserTests.cs index 48415d880..bf3503fdd 100644 --- a/src/LanguageServer/Test/IndexParserTests.cs +++ b/src/LanguageServer/Test/IndexParserTests.cs @@ -22,6 +22,7 @@ using FluentAssertions; using Microsoft.Python.Core.IO; using Microsoft.Python.LanguageServer.Indexing; +using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; diff --git a/src/LanguageServer/Test/SymbolIndexTests.cs b/src/LanguageServer/Test/SymbolIndexTests.cs index db2e495a5..2c42ef602 100644 --- a/src/LanguageServer/Test/SymbolIndexTests.cs +++ b/src/LanguageServer/Test/SymbolIndexTests.cs @@ -17,6 +17,7 @@ using FluentAssertions; using Microsoft.Python.Core.Text; using Microsoft.Python.LanguageServer.Indexing; +using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; using Microsoft.Python.Tests.Utilities.FluentAssertions; From d3f7da1b85d6b847e6dd444c1eb08cf03f6f47a0 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 8 Feb 2019 12:35:39 -0800 Subject: [PATCH 045/123] Revert "Delete copy of SymbolKind" This reverts commit fae4315555f45b7d125bf34084ca25af0d3d79f5. --- .../Implementation/Server.WorkspaceSymbols.cs | 6 ++-- .../Impl/Indexing/SymbolIndexWalker.cs | 1 - src/LanguageServer/Impl/Indexing/Symbols.cs | 31 ++++++++++++++++++- src/LanguageServer/Test/IndexManagerTests.cs | 1 - src/LanguageServer/Test/IndexParserTests.cs | 1 - src/LanguageServer/Test/SymbolIndexTests.cs | 1 - 6 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs index cf8da50aa..f4d5c8f8d 100644 --- a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs @@ -39,10 +39,10 @@ public async Task HierarchicalDocumentSymbol(DocumentSymbolPar return symbols.MaybeEnumerate().Select(hSym => MakeDocumentSymbol(hSym)).ToArray(); } - private static SymbolInformation MakeSymbolInfo(FlatSymbol s) { + private static SymbolInformation MakeSymbolInfo(Indexing.FlatSymbol s) { return new SymbolInformation { name = s.Name, - kind = s.Kind, + kind = (Protocol.SymbolKind)s.Kind, location = new Location { range = s.Range, uri = new Uri(s.DocumentPath), @@ -55,7 +55,7 @@ private DocumentSymbol MakeDocumentSymbol(HierarchicalSymbol hSym) { return new DocumentSymbol { name = hSym.Name, detail = hSym.Detail, - kind = hSym.Kind, + kind = (Protocol.SymbolKind)hSym.Kind, deprecated = hSym.Deprecated ?? false, range = hSym.Range, selectionRange = hSym.SelectionRange, diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs index bac40516d..93dc0b1bc 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs @@ -17,7 +17,6 @@ using System.Linq; using System.Text.RegularExpressions; using Microsoft.Python.Core; -using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.LanguageServer.Indexing { diff --git a/src/LanguageServer/Impl/Indexing/Symbols.cs b/src/LanguageServer/Impl/Indexing/Symbols.cs index 4988027fc..0cdac218d 100644 --- a/src/LanguageServer/Impl/Indexing/Symbols.cs +++ b/src/LanguageServer/Impl/Indexing/Symbols.cs @@ -15,9 +15,38 @@ using System.Collections.Generic; using Microsoft.Python.Core.Text; -using Microsoft.Python.LanguageServer.Protocol; namespace Microsoft.Python.LanguageServer.Indexing { + // From LSP. + internal enum SymbolKind { + None = 0, + File = 1, + Module = 2, + Namespace = 3, + Package = 4, + Class = 5, + Method = 6, + Property = 7, + Field = 8, + Constructor = 9, + Enum = 10, + Interface = 11, + Function = 12, + Variable = 13, + Constant = 14, + String = 15, + Number = 16, + Boolean = 17, + Array = 18, + Object = 19, + Key = 20, + Null = 21, + EnumMember = 22, + Struct = 23, + Event = 24, + Operator = 25, + TypeParameter = 26 + } internal class FunctionKind { public const string None = ""; diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index 854d17f0c..20d4a7abc 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -24,7 +24,6 @@ using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core.IO; using Microsoft.Python.LanguageServer.Indexing; -using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; using Microsoft.Python.Parsing.Tests; diff --git a/src/LanguageServer/Test/IndexParserTests.cs b/src/LanguageServer/Test/IndexParserTests.cs index bf3503fdd..48415d880 100644 --- a/src/LanguageServer/Test/IndexParserTests.cs +++ b/src/LanguageServer/Test/IndexParserTests.cs @@ -22,7 +22,6 @@ using FluentAssertions; using Microsoft.Python.Core.IO; using Microsoft.Python.LanguageServer.Indexing; -using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; diff --git a/src/LanguageServer/Test/SymbolIndexTests.cs b/src/LanguageServer/Test/SymbolIndexTests.cs index 2c42ef602..db2e495a5 100644 --- a/src/LanguageServer/Test/SymbolIndexTests.cs +++ b/src/LanguageServer/Test/SymbolIndexTests.cs @@ -17,7 +17,6 @@ using FluentAssertions; using Microsoft.Python.Core.Text; using Microsoft.Python.LanguageServer.Indexing; -using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; using Microsoft.Python.Tests.Utilities.FluentAssertions; From 9c0cb667029e89449f3a242460ddfb7901a8a141 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Mon, 11 Feb 2019 10:13:33 -0800 Subject: [PATCH 046/123] Deleting cast --- .../Implementation/Server.WorkspaceSymbols.cs | 63 ++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs index f4d5c8f8d..21cd661dd 100644 --- a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs @@ -55,12 +55,73 @@ private DocumentSymbol MakeDocumentSymbol(HierarchicalSymbol hSym) { return new DocumentSymbol { name = hSym.Name, detail = hSym.Detail, - kind = (Protocol.SymbolKind)hSym.Kind, + kind = ToSymbolKind(hSym.Kind), deprecated = hSym.Deprecated ?? false, range = hSym.Range, selectionRange = hSym.SelectionRange, children = hSym.Children.MaybeEnumerate().Select(MakeDocumentSymbol).ToArray(), }; } + + private Protocol.SymbolKind ToSymbolKind(Indexing.SymbolKind kind) { + switch (kind) { + case Indexing.SymbolKind.None: + return Protocol.SymbolKind.None; + case Indexing.SymbolKind.File: + return Protocol.SymbolKind.File; + case Indexing.SymbolKind.Module: + return Protocol.SymbolKind.Module; + case Indexing.SymbolKind.Namespace: + return Protocol.SymbolKind.Namespace; + case Indexing.SymbolKind.Package: + return Protocol.SymbolKind.Package; + case Indexing.SymbolKind.Class: + return Protocol.SymbolKind.Class; + case Indexing.SymbolKind.Method: + return Protocol.SymbolKind.Method; + case Indexing.SymbolKind.Property: + return Protocol.SymbolKind.Property; + case Indexing.SymbolKind.Field: + return Protocol.SymbolKind.Field; + case Indexing.SymbolKind.Constructor: + return Protocol.SymbolKind.Constructor; + case Indexing.SymbolKind.Enum: + return Protocol.SymbolKind.Enum; + case Indexing.SymbolKind.Interface: + return Protocol.SymbolKind.Interface; + case Indexing.SymbolKind.Function: + return Protocol.SymbolKind.Function; + case Indexing.SymbolKind.Variable: + return Protocol.SymbolKind.Variable; + case Indexing.SymbolKind.Constant: + return Protocol.SymbolKind.Constant; + case Indexing.SymbolKind.String: + return Protocol.SymbolKind.String; + case Indexing.SymbolKind.Number: + return Protocol.SymbolKind.Number; + case Indexing.SymbolKind.Boolean: + return Protocol.SymbolKind.Boolean; + case Indexing.SymbolKind.Array: + return Protocol.SymbolKind.Array; + case Indexing.SymbolKind.Object: + return Protocol.SymbolKind.Object; + case Indexing.SymbolKind.Key: + return Protocol.SymbolKind.Key; + case Indexing.SymbolKind.Null: + return Protocol.SymbolKind.Null; + case Indexing.SymbolKind.EnumMember: + return Protocol.SymbolKind.EnumMember; + case Indexing.SymbolKind.Struct: + return Protocol.SymbolKind.Struct; + case Indexing.SymbolKind.Event: + return Protocol.SymbolKind.Event; + case Indexing.SymbolKind.Operator: + return Protocol.SymbolKind.Operator; + case Indexing.SymbolKind.TypeParameter: + return Protocol.SymbolKind.TypeParameter; + default: + throw new NotImplementedException($"{kind} is not a LSP's SymbolKind"); + } + } } } From 7d159bc28e294ab68578a98f9f58c90b4f18e06d Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Mon, 11 Feb 2019 11:50:03 -0800 Subject: [PATCH 047/123] Async handling update of AST --- src/LanguageServer/Impl/Implementation/Server.Documents.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.Documents.cs b/src/LanguageServer/Impl/Implementation/Server.Documents.cs index 51041d54b..368898632 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Documents.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Documents.cs @@ -34,6 +34,7 @@ public void DidOpenTextDocument(DidOpenTextDocumentParams @params) { var doc = _rdt.OpenDocument(uri, @params.textDocument.text); _indexManager.ProcessNewFile(uri.AbsolutePath, doc); + doc.NewAst += (d, _) => _indexManager.ReIndexFile(uri.AbsolutePath, d as IDocument); } public void DidChangeTextDocument(DidChangeTextDocumentParams @params) { @@ -51,7 +52,6 @@ public void DidChangeTextDocument(DidChangeTextDocumentParams @params) { changes.Add(change); } doc.Update(changes); - _indexManager.ReIndexFile(uri.AbsolutePath, doc); } else { _log?.Log(TraceEventType.Warning, $"Unable to find document for {@params.textDocument.uri}"); } From a03c022eb88ace8c668210842aaf302bc5686708 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Mon, 11 Feb 2019 16:44:17 -0800 Subject: [PATCH 048/123] Versioned symbolindex --- .../Impl/Indexing/ISymbolIndex.cs | 5 +- .../Impl/Indexing/IndexManager.cs | 4 +- .../Impl/Indexing/IndexParser.cs | 3 +- .../Impl/Indexing/SymbolIndex.cs | 73 +++++++++++++++++-- src/LanguageServer/Test/IndexManagerTests.cs | 33 +++++++++ src/LanguageServer/Test/SymbolIndexTests.cs | 20 +++-- 6 files changed, 118 insertions(+), 20 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs index e63e25fd7..cf09fda77 100644 --- a/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs @@ -20,8 +20,9 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal interface ISymbolIndex { IEnumerable WorkspaceSymbols(string query); IEnumerable HierarchicalDocumentSymbols(string path); - void UpdateIndex(string path, PythonAst pythonAst); - void Delete(string path); + void UpdateIndexIfNewer(string path, PythonAst pythonAst, int version); + void DeleteIfNewer(string path, int version); bool IsIndexed(string path); + int GetNewVersion(string path); } } diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index 01371305b..8a0a5b425 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -90,7 +90,7 @@ public Task ProcessClosedFileAsync(string path) { return _indexParser.ParseAsync(path, _allIndexCts.Token); } else { // remove file from index - _symbolIndex.Delete(path); + _symbolIndex.DeleteIfNewer(path, _symbolIndex.GetNewVersion(path)); return Task.CompletedTask; } } @@ -99,7 +99,7 @@ private bool IsFileOnWorkspace(string path) => _fileSystem.IsPathUnderRoot(_workspaceRootPath, path); public void ProcessNewFile(string path, IDocument doc) - => _symbolIndex.UpdateIndex(path, doc.GetAnyAst()); + => _symbolIndex.UpdateIndexIfNewer(path, doc.GetAnyAst(), _symbolIndex.GetNewVersion(path)); public void ReIndexFile(string path, IDocument doc) { if (IsFileIndexed(path)) { diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index e3368304b..11a54c591 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -45,6 +45,7 @@ public void Dispose() { public Task ParseAsync(string path, CancellationToken parseCancellationToken = default) { var linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, parseCancellationToken); var linkedParseToken = linkedParseCts.Token; + var version = _symbolIndex.GetNewVersion(path); return Task.Run(() => { if (!_fileSystem.FileExists(path)) { return false; @@ -54,7 +55,7 @@ public Task ParseAsync(string path, CancellationToken parseCancellationTok using (var stream = _fileSystem.FileOpen(path, FileMode.Open)) { var parser = Parser.CreateParser(stream, _version); linkedParseToken.ThrowIfCancellationRequested(); - _symbolIndex.UpdateIndex(path, parser.ParseFile()); + _symbolIndex.UpdateIndexIfNewer(path, parser.ParseFile(), version); return true; } } catch (FileNotFoundException e) { diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index e226bdbb0..d3cc86c59 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; @@ -23,18 +22,19 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal sealed class SymbolIndex : ISymbolIndex { - private readonly ConcurrentDictionary> _index; + private static int DefaultVersion = 0; + private readonly ConcurrentDictionary>> _index; public SymbolIndex(IEqualityComparer comparer = null) { comparer = comparer ?? PathEqualityComparer.Instance; - _index = new ConcurrentDictionary>(comparer); + _index = new ConcurrentDictionary>>(comparer); } public IEnumerable HierarchicalDocumentSymbols(string path) - => _index.TryGetValue(path, out var list) ? list : Enumerable.Empty(); + => _index.TryGetValue(path, out var list) ? list.Value : Enumerable.Empty(); public IEnumerable WorkspaceSymbols(string query) { - return _index.SelectMany(kvp => WorkspaceSymbolsQuery(query, kvp.Key, kvp.Value)); + return _index.SelectMany(kvp => WorkspaceSymbolsQuery(query, kvp.Key, kvp.Value.Value)); } private IEnumerable WorkspaceSymbolsQuery(string query, string path, IEnumerable symbols) { @@ -50,17 +50,74 @@ private IEnumerable WorkspaceSymbolsQuery(string query, string path, } } - public void UpdateIndex(string path, PythonAst ast) { + public void UpdateIndexIfNewer(string path, PythonAst ast, int version) { var walker = new SymbolIndexWalker(ast); ast.Walk(walker); - _index[path] = walker.Symbols; + if (_index.TryGetValue(path, out var versionedList)) { + versionedList.UpdateIfNewer(walker.Symbols, version); + } else { + _index[path] = new VersionedValue>(walker.Symbols, version); + } } - public void Delete(string path) => _index.TryRemove(path, out var _); + public void DeleteIfNewer(string path, int version) { + var currentVersion = _index[path].Version; + if (version > currentVersion) { + // Point A + _index.Remove(path, out var oldVersionedValue); + // Point B + if (oldVersionedValue.Version > version) { + // An update happened at point A + // Reinsert that version, only if key doesn't exist + _index.GetOrAdd(path, oldVersionedValue); + // Update could have happened at B + } + } + } public bool IsIndexed(string path) => _index.ContainsKey(path); private static IEnumerable<(HierarchicalSymbol symbol, string parentName)> DecorateWithParentsName(IEnumerable symbols, string parentName) => symbols.Select((symbol) => (symbol, parentName)); + + public int GetNewVersion(string path) { + if (_index.TryGetValue(path, out var versionedList)) { + return versionedList.GetNewVersion(); + } else { + _index[path] = new VersionedValue>(new List(), DefaultVersion); + return _index[path].GetNewVersion(); + } + } + } + + internal class VersionedValue { + private readonly object _syncObj = new object(); + private int _newVersion; + + public VersionedValue(T value, int version) { + Value = value; + Version = version; + _newVersion = version + 1; + } + + public int Version { get; private set; } + + public T Value { get; private set; } + + public void UpdateIfNewer(T value, int version) { + lock (_syncObj) { + if (version > Version) { + Value = value; + Version = version; + } + } + } + + public int GetNewVersion() { + lock (_syncObj) { + _newVersion++; + return _newVersion; + } + } } } diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index 20d4a7abc..855452083 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -250,6 +250,39 @@ public async Task HierarchicalDocumentSymbolsAsync() { SymbolsShouldBeOnlyX(symbols); } + [TestMethod, Priority(0)] + public async Task LatestVersionASTVersionIsIndexed() { + ManualResetEventSlim reOpenedFileFinished = new ManualResetEventSlim(false); + ManualResetEventSlim fileOpenedEvent = new ManualResetEventSlim(false); + + var pythonTestFilePath = FileWithXVarInRootDir(); + _fileSystem.FileOpen(pythonTestFilePath, FileMode.Open).Returns(_ => { + fileOpenedEvent.Set(); + // Wait forever + reOpenedFileFinished.Wait(); + return MakeStream("x = 1"); + }); + + var indexManager = GetDefaultIndexManager(); + + IDocument yVarDoc = Substitute.For(); + yVarDoc.GetAnyAst().Returns(MakeAst("y = 1")); + IDocument zVarDoc = Substitute.For(); + zVarDoc.GetAnyAst().Returns(MakeAst("z = 1")); + + indexManager.ProcessNewFile(pythonTestFilePath, yVarDoc); + var closeFileTask = indexManager.ProcessClosedFileAsync(pythonTestFilePath); + fileOpenedEvent.Wait(); + indexManager.ProcessNewFile(pythonTestFilePath, zVarDoc); + reOpenedFileFinished.Set(); + + await closeFileTask; + var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath); + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("z"); + } + private static void SymbolsShouldBeOnlyX(IEnumerable symbols) { symbols.Should().HaveCount(1); symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); diff --git a/src/LanguageServer/Test/SymbolIndexTests.cs b/src/LanguageServer/Test/SymbolIndexTests.cs index db2e495a5..8240e95d9 100644 --- a/src/LanguageServer/Test/SymbolIndexTests.cs +++ b/src/LanguageServer/Test/SymbolIndexTests.cs @@ -22,7 +22,7 @@ using Microsoft.Python.Tests.Utilities.FluentAssertions; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; -namespace Microsoft.Python.Analysis.Tests { +namespace Microsoft.Python.LanguageServer.Tests { [TestClass] public class SymbolIndexTests { public TestContext TestContext { get; set; } @@ -616,7 +616,8 @@ public void IndexHierarchicalDocument() { ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var ast = GetParse("x = 1"); - index.UpdateIndex(path, ast); + var version = index.GetNewVersion(path); + index.UpdateIndexIfNewer(path, ast, version); var symbols = index.HierarchicalDocumentSymbols(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { @@ -629,7 +630,8 @@ public void IndexHierarchicalDocumentUpdate() { ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var ast = GetParse("x = 1"); - index.UpdateIndex(path, ast); + var version = index.GetNewVersion(path); + index.UpdateIndexIfNewer(path, ast, version); var symbols = index.HierarchicalDocumentSymbols(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { @@ -637,7 +639,8 @@ public void IndexHierarchicalDocumentUpdate() { }); ast = GetParse("y = 1"); - index.UpdateIndex(path, ast); + version = index.GetNewVersion(path); + index.UpdateIndexIfNewer(path, ast, version); symbols = index.HierarchicalDocumentSymbols(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { @@ -662,7 +665,8 @@ public void IndexWorkspaceSymbolsFlatten() { ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var ast = GetParse(code); - index.UpdateIndex(path, ast); + var version = index.GetNewVersion(path); + index.UpdateIndexIfNewer(path, ast, version); var symbols = index.WorkspaceSymbols(""); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { @@ -681,7 +685,8 @@ public void IndexWorkspaceSymbolsFiltered() { ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var ast = GetParse(code); - index.UpdateIndex(path, ast); + var version = index.GetNewVersion(path); + index.UpdateIndexIfNewer(path, ast, version); var symbols = index.WorkspaceSymbols("x"); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { @@ -706,7 +711,8 @@ public void IndexWorkspaceSymbolsCaseInsensitive() { ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var ast = GetParse(code); - index.UpdateIndex(path, ast); + var version = index.GetNewVersion(path); + index.UpdateIndexIfNewer(path, ast, version); var symbols = index.WorkspaceSymbols("foo"); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { From f6c854b8bce8c309802526fcb3957173a13e1134 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Mon, 11 Feb 2019 23:00:00 -0800 Subject: [PATCH 049/123] More versioned syncronization --- .../Impl/Indexing/SymbolIndex.cs | 86 +++++++++---------- 1 file changed, 42 insertions(+), 44 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index d3cc86c59..783c4ca02 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -16,6 +16,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using System.Threading; using Microsoft.Python.Core; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing.Ast; @@ -37,7 +38,8 @@ public IEnumerable WorkspaceSymbols(string query) { return _index.SelectMany(kvp => WorkspaceSymbolsQuery(query, kvp.Key, kvp.Value.Value)); } - private IEnumerable WorkspaceSymbolsQuery(string query, string path, IEnumerable symbols) { + private IEnumerable WorkspaceSymbolsQuery(string query, string path, + IEnumerable symbols) { var rootSymbols = DecorateWithParentsName(symbols, null); var treeSymbols = rootSymbols.TraverseBreadthFirst((symbolAndParent) => { var sym = symbolAndParent.symbol; @@ -53,70 +55,66 @@ private IEnumerable WorkspaceSymbolsQuery(string query, string path, public void UpdateIndexIfNewer(string path, PythonAst ast, int version) { var walker = new SymbolIndexWalker(ast); ast.Walk(walker); - if (_index.TryGetValue(path, out var versionedList)) { - versionedList.UpdateIfNewer(walker.Symbols, version); - } else { - _index[path] = new VersionedValue>(walker.Symbols, version); - } + var versionedList = _index.GetOrAdd(path, MakeVersionedValue(walker.Symbols, version)); + versionedList.UpdateIfNewer(walker.Symbols, version); } public void DeleteIfNewer(string path, int version) { var currentVersion = _index[path].Version; - if (version > currentVersion) { - // Point A - _index.Remove(path, out var oldVersionedValue); - // Point B - if (oldVersionedValue.Version > version) { - // An update happened at point A - // Reinsert that version, only if key doesn't exist - _index.GetOrAdd(path, oldVersionedValue); - // Update could have happened at B - } + if (version <= currentVersion) return; + // Point A + _index.Remove(path, out var oldVersionedValue); + // Point B + if (oldVersionedValue.Version > version) { + // An update happened at point A + // Reinsert that version, only if key doesn't exist + _index.GetOrAdd(path, oldVersionedValue); + // Update could have happened at B } } public bool IsIndexed(string path) => _index.ContainsKey(path); - private static IEnumerable<(HierarchicalSymbol symbol, string parentName)> DecorateWithParentsName(IEnumerable symbols, string parentName) + private static IEnumerable<(HierarchicalSymbol symbol, string parentName)> DecorateWithParentsName( + IEnumerable symbols, string parentName) => symbols.Select((symbol) => (symbol, parentName)); public int GetNewVersion(string path) { - if (_index.TryGetValue(path, out var versionedList)) { - return versionedList.GetNewVersion(); - } else { - _index[path] = new VersionedValue>(new List(), DefaultVersion); - return _index[path].GetNewVersion(); - } + var versionedList = + _index.GetOrAdd(path, MakeVersionedValue(new List(), DefaultVersion)); + return versionedList.GetNewVersion(); } - } - - internal class VersionedValue { - private readonly object _syncObj = new object(); - private int _newVersion; - public VersionedValue(T value, int version) { - Value = value; - Version = version; - _newVersion = version + 1; + private VersionedValue> MakeVersionedValue( + IReadOnlyList symbols, int version) { + return new VersionedValue>(symbols, version); } - public int Version { get; private set; } + internal class VersionedValue { + private readonly object _syncObj = new object(); + private int _newVersion; - public T Value { get; private set; } + public VersionedValue(T value, int version) { + Value = value; + Version = version; + _newVersion = version + 1; + } + + public int Version { get; private set; } - public void UpdateIfNewer(T value, int version) { - lock (_syncObj) { - if (version > Version) { - Value = value; - Version = version; + public T Value { get; private set; } + + public void UpdateIfNewer(T value, int version) { + lock (_syncObj) { + if (version > Version) { + Value = value; + Version = version; + } } } - } - public int GetNewVersion() { - lock (_syncObj) { - _newVersion++; - return _newVersion; + public int GetNewVersion() { + return Interlocked.Increment(ref _newVersion); } } } From 6cf5057039e87fcac7d83c62a383b6c97a40c4cf Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 13 Feb 2019 11:09:28 -0800 Subject: [PATCH 050/123] Removing event handler from doc --- .../Impl/Implementation/Server.Documents.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.Documents.cs b/src/LanguageServer/Impl/Implementation/Server.Documents.cs index 368898632..7b7d8c403 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Documents.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Documents.cs @@ -16,13 +16,10 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.Linq; using System.Threading; -using System.Threading.Tasks; using Microsoft.Python.Analysis; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core; -using Microsoft.Python.Core.Text; using Microsoft.Python.LanguageServer.Protocol; namespace Microsoft.Python.LanguageServer.Implementation { @@ -34,7 +31,7 @@ public void DidOpenTextDocument(DidOpenTextDocumentParams @params) { var doc = _rdt.OpenDocument(uri, @params.textDocument.text); _indexManager.ProcessNewFile(uri.AbsolutePath, doc); - doc.NewAst += (d, _) => _indexManager.ReIndexFile(uri.AbsolutePath, d as IDocument); + doc.NewAst += DocNewAstEvent; } public void DidChangeTextDocument(DidChangeTextDocumentParams @params) { @@ -67,6 +64,7 @@ public void DidChangeWatchedFiles(DidChangeWatchedFilesParams @params) { public void DidCloseTextDocument(DidCloseTextDocumentParams @params) { _disposableBag.ThrowIfDisposed(); Uri uri = @params.textDocument.uri; + _rdt.GetDocument(uri).NewAst -= DocNewAstEvent; _rdt.CloseDocument(uri); _indexManager.ProcessClosedFileAsync(uri.AbsolutePath).DoNotWait(); } @@ -80,5 +78,10 @@ private IDocumentAnalysis GetAnalysis(Uri uri, CancellationToken cancellationTok _log?.Log(TraceEventType.Error, $"Unable to find document {uri}"); return null; } + + private void DocNewAstEvent(object d, EventArgs _) { + var document = d as IDocument; + _indexManager.ReIndexFile(document.Uri.AbsolutePath, document); + } } } From 4406a7131b52de9078b27406e8d3ee6999fb5ab5 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 13 Feb 2019 14:12:31 -0800 Subject: [PATCH 051/123] Safely executing matcher --- src/Core/Impl/IO/DirectoryInfoProxy.cs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Core/Impl/IO/DirectoryInfoProxy.cs b/src/Core/Impl/IO/DirectoryInfoProxy.cs index fea90d9f5..72af4f8cd 100644 --- a/src/Core/Impl/IO/DirectoryInfoProxy.cs +++ b/src/Core/Impl/IO/DirectoryInfoProxy.cs @@ -13,6 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -46,13 +47,27 @@ public IEnumerable EnumerateFileSystemInfos(string[] includeFil Matcher matcher = new Matcher(); matcher.AddIncludePatterns(includeFiles.IsNullOrEmpty() ? new[] { "**/*" } : includeFiles); matcher.AddExcludePatterns(excludeFiles ?? Enumerable.Empty()); - var matchResult = matcher.Execute(new DirectoryInfoWrapper(_directoryInfo)); + PatternMatchingResult matchResult = SafeExecuteMatcher(matcher); return matchResult.Files.Select((filePatternMatch) => { var fileSystemInfo = _directoryInfo.GetFileSystemInfos(filePatternMatch.Stem).First(); return CreateFileSystemInfoProxy(fileSystemInfo); }); } + private PatternMatchingResult SafeExecuteMatcher(Matcher matcher) { + PatternMatchingResult matchResult = null; + int leftTries = 5; + while (matchResult == null && leftTries > 0) { + try { + matchResult = matcher.Execute(new DirectoryInfoWrapper(_directoryInfo)); + } catch (Exception ex) when (ex is IOException // FileNotFoundException, DirectoryNotFoundException, etc + || ex is UnauthorizedAccessException) { + leftTries--; + } + } + return matchResult ?? new PatternMatchingResult(Enumerable.Empty()); + } + private static IFileSystemInfo CreateFileSystemInfoProxy(FileSystemInfo fileSystemInfo) => fileSystemInfo is DirectoryInfo directoryInfo ? (IFileSystemInfo)new DirectoryInfoProxy(directoryInfo) From ab650eb5ab81edcd0e65124346ba0b0dc5803cb3 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 13 Feb 2019 14:33:56 -0800 Subject: [PATCH 052/123] Casting instead of 'as' --- src/LanguageServer/Impl/Implementation/Server.Documents.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.Documents.cs b/src/LanguageServer/Impl/Implementation/Server.Documents.cs index 7b7d8c403..38e8cdc06 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Documents.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Documents.cs @@ -79,8 +79,8 @@ private IDocumentAnalysis GetAnalysis(Uri uri, CancellationToken cancellationTok return null; } - private void DocNewAstEvent(object d, EventArgs _) { - var document = d as IDocument; + private void DocNewAstEvent(object sender, EventArgs _) { + var document = (IDocument) sender; _indexManager.ReIndexFile(document.Uri.AbsolutePath, document); } } From 2617c925072ba05e9001b7c9c4f2dc1872c2bfdb Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 13 Feb 2019 14:50:43 -0800 Subject: [PATCH 053/123] Cast --- src/LanguageServer/Impl/Implementation/Server.Documents.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.Documents.cs b/src/LanguageServer/Impl/Implementation/Server.Documents.cs index 38e8cdc06..2c61c0398 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Documents.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Documents.cs @@ -80,7 +80,7 @@ private IDocumentAnalysis GetAnalysis(Uri uri, CancellationToken cancellationTok } private void DocNewAstEvent(object sender, EventArgs _) { - var document = (IDocument) sender; + var document = (IDocument)sender; _indexManager.ReIndexFile(document.Uri.AbsolutePath, document); } } From a62bf815d6ab98d42304aa8b39fdd87b8d7be79e Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 13 Feb 2019 14:57:13 -0800 Subject: [PATCH 054/123] Returning on execute matcher --- src/Core/Impl/IO/DirectoryInfoProxy.cs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/Core/Impl/IO/DirectoryInfoProxy.cs b/src/Core/Impl/IO/DirectoryInfoProxy.cs index 72af4f8cd..5bb81569e 100644 --- a/src/Core/Impl/IO/DirectoryInfoProxy.cs +++ b/src/Core/Impl/IO/DirectoryInfoProxy.cs @@ -55,17 +55,13 @@ public IEnumerable EnumerateFileSystemInfos(string[] includeFil } private PatternMatchingResult SafeExecuteMatcher(Matcher matcher) { - PatternMatchingResult matchResult = null; - int leftTries = 5; - while (matchResult == null && leftTries > 0) { + var directoryInfo = new DirectoryInfoWrapper(_directoryInfo); + for (var retries = 5; retries > 0; retries--) { try { - matchResult = matcher.Execute(new DirectoryInfoWrapper(_directoryInfo)); - } catch (Exception ex) when (ex is IOException // FileNotFoundException, DirectoryNotFoundException, etc - || ex is UnauthorizedAccessException) { - leftTries--; - } + return matcher.Execute(directoryInfo); + } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) { } } - return matchResult ?? new PatternMatchingResult(Enumerable.Empty()); + return new PatternMatchingResult(Enumerable.Empty()); } private static IFileSystemInfo CreateFileSystemInfoProxy(FileSystemInfo fileSystemInfo) From fa32e974a26f9d7abce1eb21b74d9a637f88d423 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 13 Feb 2019 17:04:03 -0800 Subject: [PATCH 055/123] Adding empty workspace path check --- src/LanguageServer/Impl/Indexing/IndexManager.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index 8a0a5b425..4bc0a84f7 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -78,6 +78,9 @@ public Task AddRootDirectoryAsync(CancellationToken cancellationToken = default) private IEnumerable WorkspaceFiles() { + if (string.IsNullOrEmpty(_workspaceRootPath)) { + return Enumerable.Empty(); + } return _fileSystem.GetDirectoryInfo(_workspaceRootPath).EnumerateFileSystemInfos(_includeFiles, _excludeFiles); } From 93e4d0ddf57c4a83baa1bc8c45eaf2ebbcb125db Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 13 Feb 2019 18:07:30 -0800 Subject: [PATCH 056/123] Resolving race condition --- .../Impl/Implementation/Server.Documents.cs | 2 +- .../Impl/Indexing/IIndexManager.cs | 4 +- .../Impl/Indexing/IndexManager.cs | 17 ++++--- src/LanguageServer/Test/IndexManagerTests.cs | 45 ++++++++++--------- 4 files changed, 36 insertions(+), 32 deletions(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.Documents.cs b/src/LanguageServer/Impl/Implementation/Server.Documents.cs index 37349d93d..aa2f80961 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Documents.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Documents.cs @@ -30,8 +30,8 @@ public void DidOpenTextDocument(DidOpenTextDocumentParams @params) { _log?.Log(TraceEventType.Verbose, $"Opening document {uri}"); var doc = _rdt.OpenDocument(uri, @params.textDocument.text); - _indexManager.ProcessNewFile(uri.AbsolutePath, doc); doc.NewAst += DocNewAstEvent; + _indexManager.ProcessNewFile(uri.AbsolutePath, doc).DoNotWait(); } public void DidChangeTextDocument(DidChangeTextDocumentParams @params) { diff --git a/src/LanguageServer/Impl/Indexing/IIndexManager.cs b/src/LanguageServer/Impl/Indexing/IIndexManager.cs index af2a73095..a93e1cd0a 100644 --- a/src/LanguageServer/Impl/Indexing/IIndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IIndexManager.cs @@ -22,9 +22,9 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal interface IIndexManager : IDisposable { Task AddRootDirectoryAsync(CancellationToken cancellationToken = default); - void ProcessNewFile(string path, IDocument doc); + Task ProcessNewFile(string path, IDocument doc); Task ProcessClosedFileAsync(string path); - void ReIndexFile(string path, IDocument doc); + Task ReIndexFile(string path, IDocument doc); Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken cancellationToken = default); Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken cancellationToken = default); } diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index 4bc0a84f7..222485edc 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -98,15 +98,18 @@ public Task ProcessClosedFileAsync(string path) { } } - private bool IsFileOnWorkspace(string path) - => _fileSystem.IsPathUnderRoot(_workspaceRootPath, path); - - public void ProcessNewFile(string path, IDocument doc) - => _symbolIndex.UpdateIndexIfNewer(path, doc.GetAnyAst(), _symbolIndex.GetNewVersion(path)); + private bool IsFileOnWorkspace(string path) { + if (string.IsNullOrEmpty(_workspaceRootPath)) { + return true; + } + return _fileSystem.IsPathUnderRoot(_workspaceRootPath, path); + } + public async Task ProcessNewFile(string path, IDocument doc) + => _symbolIndex.UpdateIndexIfNewer(path, await doc.GetAstAsync(), _symbolIndex.GetNewVersion(path)); - public void ReIndexFile(string path, IDocument doc) { + public async Task ReIndexFile(string path, IDocument doc) { if (IsFileIndexed(path)) { - ProcessNewFile(path, doc); + await ProcessNewFile(path, doc); } } diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index 855452083..35fbbfb05 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -93,24 +93,29 @@ public void IgnoresNonPythonFiles() { } [TestMethod, Priority(0)] - public void CanOpenFiles() { + public async Task CanOpenFiles() { string nonRootPath = "C:/nonRoot"; var pythonTestFileInfo = MakeFileInfoProxy($"{nonRootPath}/bla.py"); - IDocument doc = Substitute.For(); - doc.GetAnyAst().Returns(MakeAst("x = 1")); + const string TestCode = "x = 1"; + IDocument doc = DocumentWithAst(TestCode); IIndexManager indexManager = GetDefaultIndexManager(); - indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); + await indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); var symbols = _symbolIndex.WorkspaceSymbols(""); SymbolsShouldBeOnlyX(symbols); } + private IDocument DocumentWithAst(string TestCode) { + IDocument doc = Substitute.For(); + doc.GetAstAsync().Returns(Task.FromResult(MakeAst(TestCode))); + return doc; + } + [TestMethod, Priority(0)] public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { var pythonTestFilePath = FileWithXVarInRootDir(); - IDocument latestDoc = Substitute.For(); - latestDoc.GetAnyAst().Returns(MakeAst("y = 1")); + IDocument latestDoc = DocumentWithAst("y = 1"); var indexManager = GetDefaultIndexManager(); await indexManager.AddRootDirectoryAsync(); @@ -126,10 +131,10 @@ public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); IDocument doc = Substitute.For(); - doc.GetAnyAst().Returns(MakeAst("x = 1")); + doc.GetAstAsync().Returns(Task.FromResult(MakeAst("x = 1"))); var indexManager = GetDefaultIndexManager(); - indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); + await indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); await indexManager.ProcessClosedFileAsync(pythonTestFileInfo.FullName); SymbolIndexShouldBeEmpty(); @@ -140,12 +145,11 @@ public async Task CloseWorkspaceFilesReUpdatesIndexAsync() { string pythonTestFilePath = FileWithXVarInRootDir(); _fileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); - IDocument latestDoc = Substitute.For(); - latestDoc.GetAnyAst().Returns(MakeAst("y = 1")); + IDocument latestDoc = DocumentWithAst("y = 1"); var indexManager = GetDefaultIndexManager(); await indexManager.AddRootDirectoryAsync(); - indexManager.ProcessNewFile(pythonTestFilePath, latestDoc); + await indexManager.ProcessNewFile(pythonTestFilePath, latestDoc); // It Needs to remake the stream for the file, previous one is closed _fileSystem.FileOpen(pythonTestFilePath, FileMode.Open).Returns(MakeStream("x = 1")); await indexManager.ProcessClosedFileAsync(pythonTestFilePath); @@ -160,14 +164,13 @@ public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() { // it should not reindex file var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); - IDocument doc = Substitute.For(); - doc.GetAnyAst().Returns(MakeAst("x = 1")); + IDocument doc = DocumentWithAst("x = 1"); var indexManager = GetDefaultIndexManager(); - indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); + await indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); await indexManager.ProcessClosedFileAsync(pythonTestFileInfo.FullName); - doc.GetAnyAst().Returns(MakeAst("x = 1")); - indexManager.ReIndexFile(pythonTestFileInfo.FullName, doc); + doc = DocumentWithAst("x = 1"); + await indexManager.ReIndexFile(pythonTestFileInfo.FullName, doc); SymbolIndexShouldBeEmpty(); } @@ -265,15 +268,13 @@ public async Task LatestVersionASTVersionIsIndexed() { var indexManager = GetDefaultIndexManager(); - IDocument yVarDoc = Substitute.For(); - yVarDoc.GetAnyAst().Returns(MakeAst("y = 1")); - IDocument zVarDoc = Substitute.For(); - zVarDoc.GetAnyAst().Returns(MakeAst("z = 1")); + IDocument yVarDoc = DocumentWithAst("y = 1"); + IDocument zVarDoc = DocumentWithAst("z = 1"); - indexManager.ProcessNewFile(pythonTestFilePath, yVarDoc); + await indexManager.ProcessNewFile(pythonTestFilePath, yVarDoc); var closeFileTask = indexManager.ProcessClosedFileAsync(pythonTestFilePath); fileOpenedEvent.Wait(); - indexManager.ProcessNewFile(pythonTestFilePath, zVarDoc); + await indexManager.ProcessNewFile(pythonTestFilePath, zVarDoc); reOpenedFileFinished.Set(); await closeFileTask; From 80d4d39aca80c331bca84e1608552fbe6605a143 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 13 Feb 2019 22:22:16 -0800 Subject: [PATCH 057/123] Async rename --- .../Impl/Implementation/Server.Documents.cs | 4 ++-- .../Impl/Indexing/IIndexManager.cs | 4 ++-- src/LanguageServer/Impl/Indexing/IndexManager.cs | 8 ++++---- src/LanguageServer/Test/IndexManagerTests.cs | 16 ++++++++-------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.Documents.cs b/src/LanguageServer/Impl/Implementation/Server.Documents.cs index aa2f80961..174508d3f 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Documents.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Documents.cs @@ -31,7 +31,7 @@ public void DidOpenTextDocument(DidOpenTextDocumentParams @params) { var doc = _rdt.OpenDocument(uri, @params.textDocument.text); doc.NewAst += DocNewAstEvent; - _indexManager.ProcessNewFile(uri.AbsolutePath, doc).DoNotWait(); + _indexManager.ProcessNewFileAsync(uri.AbsolutePath, doc).DoNotWait(); } public void DidChangeTextDocument(DidChangeTextDocumentParams @params) { @@ -85,7 +85,7 @@ private IDocumentAnalysis GetAnalysis(Uri uri, CancellationToken cancellationTok private void DocNewAstEvent(object sender, EventArgs _) { var document = (IDocument)sender; - _indexManager.ReIndexFile(document.Uri.AbsolutePath, document); + _indexManager.ReIndexFileAsync(document.Uri.AbsolutePath, document); } } } diff --git a/src/LanguageServer/Impl/Indexing/IIndexManager.cs b/src/LanguageServer/Impl/Indexing/IIndexManager.cs index a93e1cd0a..6dc387cc7 100644 --- a/src/LanguageServer/Impl/Indexing/IIndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IIndexManager.cs @@ -22,9 +22,9 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal interface IIndexManager : IDisposable { Task AddRootDirectoryAsync(CancellationToken cancellationToken = default); - Task ProcessNewFile(string path, IDocument doc); + Task ProcessNewFileAsync(string path, IDocument doc); Task ProcessClosedFileAsync(string path); - Task ReIndexFile(string path, IDocument doc); + Task ReIndexFileAsync(string path, IDocument doc); Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken cancellationToken = default); Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken cancellationToken = default); } diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index 222485edc..5f6361175 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -100,16 +100,16 @@ public Task ProcessClosedFileAsync(string path) { private bool IsFileOnWorkspace(string path) { if (string.IsNullOrEmpty(_workspaceRootPath)) { - return true; + return false; } return _fileSystem.IsPathUnderRoot(_workspaceRootPath, path); } - public async Task ProcessNewFile(string path, IDocument doc) + public async Task ProcessNewFileAsync(string path, IDocument doc) => _symbolIndex.UpdateIndexIfNewer(path, await doc.GetAstAsync(), _symbolIndex.GetNewVersion(path)); - public async Task ReIndexFile(string path, IDocument doc) { + public async Task ReIndexFileAsync(string path, IDocument doc) { if (IsFileIndexed(path)) { - await ProcessNewFile(path, doc); + await ProcessNewFileAsync(path, doc); } } diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index 35fbbfb05..b5239665b 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -100,7 +100,7 @@ public async Task CanOpenFiles() { IDocument doc = DocumentWithAst(TestCode); IIndexManager indexManager = GetDefaultIndexManager(); - await indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); + await indexManager.ProcessNewFileAsync(pythonTestFileInfo.FullName, doc); var symbols = _symbolIndex.WorkspaceSymbols(""); SymbolsShouldBeOnlyX(symbols); @@ -119,7 +119,7 @@ public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { var indexManager = GetDefaultIndexManager(); await indexManager.AddRootDirectoryAsync(); - indexManager.ReIndexFile(pythonTestFilePath, latestDoc); + indexManager.ReIndexFileAsync(pythonTestFilePath, latestDoc); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(1); @@ -134,7 +134,7 @@ public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { doc.GetAstAsync().Returns(Task.FromResult(MakeAst("x = 1"))); var indexManager = GetDefaultIndexManager(); - await indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); + await indexManager.ProcessNewFileAsync(pythonTestFileInfo.FullName, doc); await indexManager.ProcessClosedFileAsync(pythonTestFileInfo.FullName); SymbolIndexShouldBeEmpty(); @@ -149,7 +149,7 @@ public async Task CloseWorkspaceFilesReUpdatesIndexAsync() { var indexManager = GetDefaultIndexManager(); await indexManager.AddRootDirectoryAsync(); - await indexManager.ProcessNewFile(pythonTestFilePath, latestDoc); + await indexManager.ProcessNewFileAsync(pythonTestFilePath, latestDoc); // It Needs to remake the stream for the file, previous one is closed _fileSystem.FileOpen(pythonTestFilePath, FileMode.Open).Returns(MakeStream("x = 1")); await indexManager.ProcessClosedFileAsync(pythonTestFilePath); @@ -167,10 +167,10 @@ public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() { IDocument doc = DocumentWithAst("x = 1"); var indexManager = GetDefaultIndexManager(); - await indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); + await indexManager.ProcessNewFileAsync(pythonTestFileInfo.FullName, doc); await indexManager.ProcessClosedFileAsync(pythonTestFileInfo.FullName); doc = DocumentWithAst("x = 1"); - await indexManager.ReIndexFile(pythonTestFileInfo.FullName, doc); + await indexManager.ReIndexFileAsync(pythonTestFileInfo.FullName, doc); SymbolIndexShouldBeEmpty(); } @@ -271,10 +271,10 @@ public async Task LatestVersionASTVersionIsIndexed() { IDocument yVarDoc = DocumentWithAst("y = 1"); IDocument zVarDoc = DocumentWithAst("z = 1"); - await indexManager.ProcessNewFile(pythonTestFilePath, yVarDoc); + await indexManager.ProcessNewFileAsync(pythonTestFilePath, yVarDoc); var closeFileTask = indexManager.ProcessClosedFileAsync(pythonTestFilePath); fileOpenedEvent.Wait(); - await indexManager.ProcessNewFile(pythonTestFilePath, zVarDoc); + await indexManager.ProcessNewFileAsync(pythonTestFilePath, zVarDoc); reOpenedFileFinished.Set(); await closeFileTask; From 57d5ec8da0652e45a6e3c21dcd4737426bf3e853 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 14 Feb 2019 13:00:52 -0800 Subject: [PATCH 058/123] Idling for index --- .../Impl/Implementation/Server.Documents.cs | 8 +-- .../Impl/Implementation/Server.cs | 7 ++- .../Impl/Indexing/IIndexManager.cs | 1 + .../Impl/Indexing/IndexManager.cs | 42 +++++++++++++- src/LanguageServer/Test/IndexManagerTests.cs | 55 ++++++++++++++----- 5 files changed, 87 insertions(+), 26 deletions(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.Documents.cs b/src/LanguageServer/Impl/Implementation/Server.Documents.cs index 174508d3f..8d7d13dbb 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Documents.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Documents.cs @@ -30,7 +30,6 @@ public void DidOpenTextDocument(DidOpenTextDocumentParams @params) { _log?.Log(TraceEventType.Verbose, $"Opening document {uri}"); var doc = _rdt.OpenDocument(uri, @params.textDocument.text); - doc.NewAst += DocNewAstEvent; _indexManager.ProcessNewFileAsync(uri.AbsolutePath, doc).DoNotWait(); } @@ -49,6 +48,7 @@ public void DidChangeTextDocument(DidChangeTextDocumentParams @params) { changes.Add(change); } doc.Update(changes); + _indexManager.AddPendingDoc(doc); } else { _log?.Log(TraceEventType.Warning, $"Unable to find document for {@params.textDocument.uri}"); } @@ -64,7 +64,6 @@ public void DidChangeWatchedFiles(DidChangeWatchedFilesParams @params) { public void DidCloseTextDocument(DidCloseTextDocumentParams @params) { _disposableBag.ThrowIfDisposed(); Uri uri = @params.textDocument.uri; - _rdt.GetDocument(uri).NewAst -= DocNewAstEvent; _rdt.CloseDocument(uri); _indexManager.ProcessClosedFileAsync(uri.AbsolutePath).DoNotWait(); } @@ -82,10 +81,5 @@ private IDocumentAnalysis GetAnalysis(Uri uri, CancellationToken cancellationTok _log?.Log(TraceEventType.Error, $"Unable to find document {uri}"); return null; } - - private void DocNewAstEvent(object sender, EventArgs _) { - var document = (IDocument)sender; - _indexManager.ReIndexFileAsync(document.Uri.AbsolutePath, document); - } } } diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index e9c622401..34f78f5c0 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -25,6 +25,7 @@ using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core; using Microsoft.Python.Core.Disposables; +using Microsoft.Python.Core.Idle; using Microsoft.Python.Core.IO; using Microsoft.Python.Core.Logging; using Microsoft.Python.Core.Services; @@ -123,10 +124,10 @@ public async Task InitializeAsync(InitializeParams @params, Ca _services.AddService(_interpreter); var symbolIndex = new SymbolIndex(); - var fileSystem = _services.GetService(); - _indexManager = new IndexManager(symbolIndex, fileSystem, _interpreter.LanguageVersion, rootDir, + _indexManager = new IndexManager(symbolIndex, _services.GetService(), _interpreter.LanguageVersion, rootDir, @params.initializationOptions.includeFiles, - @params.initializationOptions.excludeFiles); + @params.initializationOptions.excludeFiles, + _services.GetService()); _services.AddService(_indexManager); DisplayStartupInfo(); diff --git a/src/LanguageServer/Impl/Indexing/IIndexManager.cs b/src/LanguageServer/Impl/Indexing/IIndexManager.cs index 6dc387cc7..997db50f1 100644 --- a/src/LanguageServer/Impl/Indexing/IIndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IIndexManager.cs @@ -27,5 +27,6 @@ internal interface IIndexManager : IDisposable { Task ReIndexFileAsync(string path, IDocument doc); Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken cancellationToken = default); Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken cancellationToken = default); + void AddPendingDoc(IDocument doc); } } diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index 5f6361175..cc27d263f 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -21,7 +21,9 @@ using System.Threading.Tasks; using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Core; using Microsoft.Python.Core.Diagnostics; +using Microsoft.Python.Core.Idle; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; @@ -35,9 +37,11 @@ internal class IndexManager : IIndexManager { private readonly string[] _excludeFiles; private readonly CancellationTokenSource _allIndexCts = new CancellationTokenSource(); private readonly TaskCompletionSource _addRootTcs; + private readonly IIdleTimeService _idleTimeService; + private HashSet _pendingDocs; public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, - string[] excludeFiles) { + string[] excludeFiles, IIdleTimeService idleTimeService) { Check.ArgumentNotNull(nameof(fileSystem), fileSystem); Check.ArgumentNotNull(nameof(rootPath), rootPath); Check.ArgumentNotNull(nameof(includeFiles), includeFiles); @@ -49,7 +53,10 @@ public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLang _workspaceRootPath = rootPath; _includeFiles = includeFiles; _excludeFiles = excludeFiles; + _idleTimeService = idleTimeService; _addRootTcs = new TaskCompletionSource(); + _idleTimeService.Idle += ReIndexPendingDocsAsync; + _pendingDocs = new HashSet(new UriDocumentComparer()); StartAddRootDir(); } @@ -104,8 +111,11 @@ private bool IsFileOnWorkspace(string path) { } return _fileSystem.IsPathUnderRoot(_workspaceRootPath, path); } - public async Task ProcessNewFileAsync(string path, IDocument doc) - => _symbolIndex.UpdateIndexIfNewer(path, await doc.GetAstAsync(), _symbolIndex.GetNewVersion(path)); + + public async Task ProcessNewFileAsync(string path, IDocument doc) { + var ast = await doc.GetAstAsync(); + _symbolIndex.UpdateIndexIfNewer(path, ast, _symbolIndex.GetNewVersion(path)); + } public async Task ReIndexFileAsync(string path, IDocument doc) { if (IsFileIndexed(path)) { @@ -127,5 +137,31 @@ public async Task> WorkspaceSymbolsAsync(string query, await AddRootDirectoryAsync(cancellationToken); return _symbolIndex.WorkspaceSymbols(query).Take(maxLength).ToList(); } + + public void AddPendingDoc(IDocument doc) { + lock (_pendingDocs) { + _pendingDocs.Add(doc); + } + } + + private void ReIndexPendingDocsAsync(object sender, EventArgs _) { + IEnumerable pendingDocs; + lock (_pendingDocs) { + pendingDocs = _pendingDocs.ToList(); + _pendingDocs.Clear(); + } + + // Since its an event handler I have to synchronously wait + Task.WaitAll(pendingDocs.MaybeEnumerate() + .Select(doc => ReIndexFileAsync(doc.Uri.AbsolutePath, doc)) + .ToArray()); + } + + private class UriDocumentComparer : IEqualityComparer { + public bool Equals(IDocument x, IDocument y) => x.Uri.Equals(y.Uri); + + public int GetHashCode(IDocument obj) => obj.Uri.GetHashCode(); + } } + } diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index b5239665b..2c53bc228 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -21,8 +21,13 @@ using System.Threading; using System.Threading.Tasks; using FluentAssertions; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Core.Idle; using Microsoft.Python.Core.IO; +using Microsoft.Python.Core.Services; using Microsoft.Python.LanguageServer.Indexing; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; @@ -39,6 +44,8 @@ public class IndexManagerTests : LanguageServerTestBase { private string _rootPath; private List _rootFileList; private PythonLanguageVersion _pythonLanguageVersion; + private IIdleTimeService _idleTimeService; + private ServiceManager _services; public TestContext TestContext { get; set; } @@ -47,15 +54,22 @@ public void TestInitialize() { TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); _fileSystem = Substitute.For(); _symbolIndex = new SymbolIndex(); - _rootPath = "C:\root"; + _rootPath = "C:/root"; _pythonLanguageVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); _rootFileList = new List(); + _idleTimeService = Substitute.For(); + CreateServices(); IDirectoryInfo directoryInfo = Substitute.For(); // Doesn't work without 'forAnyArgs' directoryInfo.EnumerateFileSystemInfos(new string[] { }, new string[] { }).ReturnsForAnyArgs(_rootFileList); _fileSystem.GetDirectoryInfo(_rootPath).Returns(directoryInfo); } + private void CreateServices() { + _services = new ServiceManager(); + _services.AddService(_fileSystem); + } + [TestCleanup] public void Cleanup() => TestEnvironmentImpl.TestCleanup(); @@ -76,18 +90,18 @@ public void NullDirectoryThrowsException() { Action construct = () => { PythonLanguageVersion version = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, - version, null, new string[] { }, new string[] { }); + version, null, new string[] { }, new string[] { }, _idleTimeService); }; construct.Should().Throw(); } [TestMethod, Priority(0)] - public void IgnoresNonPythonFiles() { + public async Task IgnoresNonPythonFiles() { var nonPythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}/bla.txt"); AddFileInfoToRootTestFS(nonPythonTestFileInfo); - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(), this._rootPath, new string[] { }, new string[] { }); - indexManager.AddRootDirectoryAsync(); + IIndexManager indexManager = GetDefaultIndexManager(); + await indexManager.AddRootDirectoryAsync(); _fileSystem.DidNotReceive().FileExists(nonPythonTestFileInfo.FullName); } @@ -96,8 +110,7 @@ public void IgnoresNonPythonFiles() { public async Task CanOpenFiles() { string nonRootPath = "C:/nonRoot"; var pythonTestFileInfo = MakeFileInfoProxy($"{nonRootPath}/bla.py"); - const string TestCode = "x = 1"; - IDocument doc = DocumentWithAst(TestCode); + IDocument doc = DocumentWithAst("x = 1"); IIndexManager indexManager = GetDefaultIndexManager(); await indexManager.ProcessNewFileAsync(pythonTestFileInfo.FullName, doc); @@ -106,9 +119,11 @@ public async Task CanOpenFiles() { SymbolsShouldBeOnlyX(symbols); } - private IDocument DocumentWithAst(string TestCode) { + private IDocument DocumentWithAst(string testCode, string filePath = null) { + filePath = filePath ?? $"{_rootPath}/{testCode}.py"; IDocument doc = Substitute.For(); - doc.GetAstAsync().Returns(Task.FromResult(MakeAst(TestCode))); + doc.GetAstAsync().ReturnsForAnyArgs(Task.FromResult(MakeAst(testCode))); + doc.Uri.Returns(new Uri(filePath)); return doc; } @@ -119,7 +134,7 @@ public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { var indexManager = GetDefaultIndexManager(); await indexManager.AddRootDirectoryAsync(); - indexManager.ReIndexFileAsync(pythonTestFilePath, latestDoc); + await indexManager.ReIndexFileAsync(pythonTestFilePath, latestDoc); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(1); @@ -130,8 +145,7 @@ public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { [TestMethod, Priority(0)] public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); - IDocument doc = Substitute.For(); - doc.GetAstAsync().Returns(Task.FromResult(MakeAst("x = 1"))); + IDocument doc = DocumentWithAst("x = 1"); var indexManager = GetDefaultIndexManager(); await indexManager.ProcessNewFileAsync(pythonTestFileInfo.FullName, doc); @@ -284,6 +298,20 @@ public async Task LatestVersionASTVersionIsIndexed() { symbols.First().Name.Should().BeEquivalentTo("z"); } + [TestMethod, Priority(0)] + public async Task AddFilesToPendingChanges() { + var f1 = AddFileToRoot($"{_rootPath}/fileA.py", MakeStream("")); + var f2 = AddFileToRoot($"{_rootPath}/fileB.py", MakeStream("")); + + var indexManager = GetDefaultIndexManager(); + + indexManager.AddPendingDoc(DocumentWithAst("y = 1", f1)); + indexManager.AddPendingDoc(DocumentWithAst("x = 1", f2)); + _idleTimeService.Idle += Raise.Event(); + var symbols = await indexManager.WorkspaceSymbolsAsync("", 1000); + symbols.Should().HaveCount(2); + } + private static void SymbolsShouldBeOnlyX(IEnumerable symbols) { symbols.Should().HaveCount(1); symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); @@ -318,7 +346,8 @@ private string FileWithXVarInRootDir() { private IIndexManager GetDefaultIndexManager() { return new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, - _rootPath, new string[] { }, new string[] { }); + _rootPath, new string[] { }, new string[] { }, + _idleTimeService); } private string AddFileToRoot(string filePath, Stream stream) { From 78f12c8063bc62eb33619e13ea373651106ed133 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 14 Feb 2019 13:26:59 -0800 Subject: [PATCH 059/123] Adding delay to idling --- src/LanguageServer/Impl/Indexing/IndexManager.cs | 15 +++++++++++++-- src/LanguageServer/Test/IndexManagerTests.cs | 7 +++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index cc27d263f..fb76bd1eb 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -39,6 +39,7 @@ internal class IndexManager : IIndexManager { private readonly TaskCompletionSource _addRootTcs; private readonly IIdleTimeService _idleTimeService; private HashSet _pendingDocs; + private DateTime _lastPendingDocAddedTime; public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, string[] excludeFiles, IIdleTimeService idleTimeService) { @@ -55,11 +56,14 @@ public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLang _excludeFiles = excludeFiles; _idleTimeService = idleTimeService; _addRootTcs = new TaskCompletionSource(); - _idleTimeService.Idle += ReIndexPendingDocsAsync; + _idleTimeService.Idle += OnIdle; _pendingDocs = new HashSet(new UriDocumentComparer()); + ReIndexingDelay = 1000; StartAddRootDir(); } + public int ReIndexingDelay { get; set; } + private void StartAddRootDir() { Task.Run(() => { try { @@ -140,11 +144,18 @@ public async Task> WorkspaceSymbolsAsync(string query, public void AddPendingDoc(IDocument doc) { lock (_pendingDocs) { + _lastPendingDocAddedTime = DateTime.Now; _pendingDocs.Add(doc); } } - private void ReIndexPendingDocsAsync(object sender, EventArgs _) { + private void OnIdle(object sender, EventArgs _) { + if (_pendingDocs.Count > 0 && (DateTime.Now - _lastPendingDocAddedTime).TotalMilliseconds > ReIndexingDelay) { + ReIndexPendingDocsAsync(); + } + } + + private void ReIndexPendingDocsAsync() { IEnumerable pendingDocs; lock (_pendingDocs) { pendingDocs = _pendingDocs.ToList(); diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index 2c53bc228..288ba477a 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -345,9 +345,12 @@ private string FileWithXVarInRootDir() { } private IIndexManager GetDefaultIndexManager() { - return new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, + var indexM = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }, - _idleTimeService); + _idleTimeService) { + ReIndexingDelay = 1 + }; + return indexM; } private string AddFileToRoot(string filePath, Stream stream) { From 4a724c74989896e27f378e56eb188f445796b518 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 14 Feb 2019 19:10:10 -0800 Subject: [PATCH 060/123] New async interface --- .../Impl/Implementation/Server.Documents.cs | 4 +- .../Impl/Indexing/IIndexManager.cs | 9 +- .../Impl/Indexing/ISymbolIndex.cs | 5 +- .../Impl/Indexing/IndexManager.cs | 75 ++++++------- .../Impl/Indexing/IndexParser.cs | 7 +- .../Impl/Indexing/MostRecentSymbols.cs | 100 ++++++++++++++++++ .../Impl/Indexing/SymbolIndex.cs | 71 ++----------- src/LanguageServer/Test/IndexManagerTests.cs | 95 ++++++++--------- src/LanguageServer/Test/SymbolIndexTests.cs | 22 ++-- 9 files changed, 207 insertions(+), 181 deletions(-) create mode 100644 src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.Documents.cs b/src/LanguageServer/Impl/Implementation/Server.Documents.cs index 8d7d13dbb..cf6dd595c 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Documents.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Documents.cs @@ -30,7 +30,7 @@ public void DidOpenTextDocument(DidOpenTextDocumentParams @params) { _log?.Log(TraceEventType.Verbose, $"Opening document {uri}"); var doc = _rdt.OpenDocument(uri, @params.textDocument.text); - _indexManager.ProcessNewFileAsync(uri.AbsolutePath, doc).DoNotWait(); + _indexManager.ProcessNewFile(uri.AbsolutePath, doc); } public void DidChangeTextDocument(DidChangeTextDocumentParams @params) { @@ -65,7 +65,7 @@ public void DidCloseTextDocument(DidCloseTextDocumentParams @params) { _disposableBag.ThrowIfDisposed(); Uri uri = @params.textDocument.uri; _rdt.CloseDocument(uri); - _indexManager.ProcessClosedFileAsync(uri.AbsolutePath).DoNotWait(); + _indexManager.ProcessClosedFile(uri.AbsolutePath); } private IDocumentAnalysis GetAnalysis(Uri uri, CancellationToken cancellationToken) { diff --git a/src/LanguageServer/Impl/Indexing/IIndexManager.cs b/src/LanguageServer/Impl/Indexing/IIndexManager.cs index 997db50f1..0451efc42 100644 --- a/src/LanguageServer/Impl/Indexing/IIndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IIndexManager.cs @@ -21,12 +21,11 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal interface IIndexManager : IDisposable { - Task AddRootDirectoryAsync(CancellationToken cancellationToken = default); - Task ProcessNewFileAsync(string path, IDocument doc); - Task ProcessClosedFileAsync(string path); - Task ReIndexFileAsync(string path, IDocument doc); + void ProcessNewFile(string path, IDocument doc); + void ProcessClosedFile(string path); + void ReIndexFile(string path, IDocument doc); + void AddPendingDoc(IDocument doc); Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken cancellationToken = default); Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken cancellationToken = default); - void AddPendingDoc(IDocument doc); } } diff --git a/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs index cf09fda77..52adf1dbf 100644 --- a/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs @@ -20,9 +20,8 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal interface ISymbolIndex { IEnumerable WorkspaceSymbols(string query); IEnumerable HierarchicalDocumentSymbols(string path); - void UpdateIndexIfNewer(string path, PythonAst pythonAst, int version); - void DeleteIfNewer(string path, int version); + void Update(string path, PythonAst pythonAst); + void Delete(string path); bool IsIndexed(string path); - int GetNewVersion(string path); } } diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index fb76bd1eb..47688b8ef 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -14,8 +14,8 @@ // permissions and limitations under the License. using System; +using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -38,6 +38,7 @@ internal class IndexManager : IIndexManager { private readonly CancellationTokenSource _allIndexCts = new CancellationTokenSource(); private readonly TaskCompletionSource _addRootTcs; private readonly IIdleTimeService _idleTimeService; + private readonly ConcurrentDictionary _files; private HashSet _pendingDocs; private DateTime _lastPendingDocAddedTime; @@ -58,36 +59,23 @@ public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLang _addRootTcs = new TaskCompletionSource(); _idleTimeService.Idle += OnIdle; _pendingDocs = new HashSet(new UriDocumentComparer()); + _files = new ConcurrentDictionary(PathEqualityComparer.Instance); ReIndexingDelay = 1000; StartAddRootDir(); + } public int ReIndexingDelay { get; set; } private void StartAddRootDir() { - Task.Run(() => { - try { - var parseTasks = new List>(); - foreach (var fileInfo in WorkspaceFiles()) { - if (ModulePath.IsPythonSourceFile(fileInfo.FullName)) { - parseTasks.Add(_indexParser.ParseAsync(fileInfo.FullName, _allIndexCts.Token)); - } - } - Task.WaitAll(parseTasks.ToArray(), _allIndexCts.Token); - _addRootTcs.SetResult(true); - } catch (Exception e) { - Trace.TraceError(e.Message); - _addRootTcs.SetException(e); + foreach (var fileInfo in WorkspaceFiles()) { + if (ModulePath.IsPythonSourceFile(fileInfo.FullName)) { + _files.GetOrAdd(fileInfo.FullName, MakeMostRecentFileSymbols(fileInfo.FullName)); + _files[fileInfo.FullName].Parse(); } - }, _allIndexCts.Token); - } - - public Task AddRootDirectoryAsync(CancellationToken cancellationToken = default) { - // Add cancellation token around task - return Task.Run(async () => await _addRootTcs.Task, cancellationToken); + } } - private IEnumerable WorkspaceFiles() { if (string.IsNullOrEmpty(_workspaceRootPath)) { return Enumerable.Empty(); @@ -97,16 +85,8 @@ private IEnumerable WorkspaceFiles() { private bool IsFileIndexed(string path) => _symbolIndex.IsIndexed(path); - public Task ProcessClosedFileAsync(string path) { - // If path is on workspace - if (IsFileOnWorkspace(path)) { - // updates index and ignores previous AST - return _indexParser.ParseAsync(path, _allIndexCts.Token); - } else { - // remove file from index - _symbolIndex.DeleteIfNewer(path, _symbolIndex.GetNewVersion(path)); - return Task.CompletedTask; - } + public void ProcessClosedFile(string path) { + _files[path].Close(IsFileOnWorkspace(path)); } private bool IsFileOnWorkspace(string path) { @@ -116,29 +96,35 @@ private bool IsFileOnWorkspace(string path) { return _fileSystem.IsPathUnderRoot(_workspaceRootPath, path); } - public async Task ProcessNewFileAsync(string path, IDocument doc) { - var ast = await doc.GetAstAsync(); - _symbolIndex.UpdateIndexIfNewer(path, ast, _symbolIndex.GetNewVersion(path)); + public void ProcessNewFile(string path, IDocument doc) { + _files.GetOrAdd(path, MakeMostRecentFileSymbols(path)); + _files[path].Process(doc); } - public async Task ReIndexFileAsync(string path, IDocument doc) { + public void ReIndexFile(string path, IDocument doc) { if (IsFileIndexed(path)) { - await ProcessNewFileAsync(path, doc); + ProcessNewFile(path, doc); } } public void Dispose() { + foreach (var mostRecentSymbols in _files.Values) { + mostRecentSymbols.Dispose(); + } + _indexParser.Dispose(); _allIndexCts.Cancel(); _allIndexCts.Dispose(); } public async Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken cancellationToken = default) { - await AddRootDirectoryAsync(cancellationToken); - return _symbolIndex.HierarchicalDocumentSymbols(path).ToList(); + var s = await _files[path].GetSymbolsAsync(); + return s.ToList(); } public async Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken cancellationToken = default) { - await AddRootDirectoryAsync(cancellationToken); + foreach(var mostRecentSymbols in _files.Values) { + await mostRecentSymbols.GetSymbolsAsync(); + } return _symbolIndex.WorkspaceSymbols(query).Take(maxLength).ToList(); } @@ -162,10 +148,13 @@ private void ReIndexPendingDocsAsync() { _pendingDocs.Clear(); } - // Since its an event handler I have to synchronously wait - Task.WaitAll(pendingDocs.MaybeEnumerate() - .Select(doc => ReIndexFileAsync(doc.Uri.AbsolutePath, doc)) - .ToArray()); + foreach (var doc in pendingDocs.MaybeEnumerate()) { + ReIndexFile(doc.Uri.AbsolutePath, doc); + } + } + + private MostRecentDocumentSymbols MakeMostRecentFileSymbols(string path) { + return new MostRecentDocumentSymbols(path, _indexParser, _symbolIndex); } private class UriDocumentComparer : IEqualityComparer { diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index 11a54c591..9b53a2ac4 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -45,8 +45,7 @@ public void Dispose() { public Task ParseAsync(string path, CancellationToken parseCancellationToken = default) { var linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, parseCancellationToken); var linkedParseToken = linkedParseCts.Token; - var version = _symbolIndex.GetNewVersion(path); - return Task.Run(() => { + return Task.Run((System.Func)(() => { if (!_fileSystem.FileExists(path)) { return false; } @@ -55,14 +54,14 @@ public Task ParseAsync(string path, CancellationToken parseCancellationTok using (var stream = _fileSystem.FileOpen(path, FileMode.Open)) { var parser = Parser.CreateParser(stream, _version); linkedParseToken.ThrowIfCancellationRequested(); - _symbolIndex.UpdateIndexIfNewer(path, parser.ParseFile(), version); + _symbolIndex.Update(path, parser.ParseFile()); return true; } } catch (FileNotFoundException e) { Trace.TraceError(e.Message); return false; } - }).ContinueWith((task) => { + })).ContinueWith((Task task) => { linkedParseCts.Dispose(); return task.Result; }, linkedParseToken); diff --git a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs new file mode 100644 index 000000000..5969a76fb --- /dev/null +++ b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Indexing { + class MostRecentDocumentSymbols : IDisposable { + private readonly object _syncObj = new object(); + private readonly IIndexParser _indexParser; + private readonly ISymbolIndex _symbolIndex; + private readonly string _path; + + private CancellationTokenSource _fileCts; + private Task _fileTask; + private TaskCompletionSource> _fileTcs; + + public MostRecentDocumentSymbols(string path, IIndexParser indexParser, ISymbolIndex symbolIndex) { + _path = path; + _indexParser = indexParser; + _symbolIndex = symbolIndex; + _fileCts = new CancellationTokenSource(); + _fileTcs = new TaskCompletionSource>(); + } + + public void Parse() { + lock (_syncObj) { + CancelExistingTask(); + SetFileTask(_indexParser.ParseAsync(_path, _fileCts.Token)); + } + } + + public void Close(bool isOnWorkspace) { + if (isOnWorkspace) { + Parse(); + } else { + lock (_syncObj) { + _symbolIndex.Delete(_path); + } + } + } + + public void Process(PythonAst ast) { + lock (_syncObj) { + CancelExistingTask(); + _symbolIndex.Update(_path, ast); + SetFileTcsResult(); + } + } + + public void Process(IDocument doc) { + lock (_syncObj) { + CancelExistingTask(); + SetFileTask(doc.GetAstAsync(_fileCts.Token).ContinueWith(t => { + lock (_syncObj) { + _symbolIndex.Update(_path, t.Result); + } + }, _fileCts.Token)); + } + } + + public Task> GetSymbolsAsync() => _fileTcs.Task; + + private void CancelExistingTask() { + if (_fileTask != null) { + _fileCts.Cancel(); + _fileCts.Dispose(); + _fileCts = new CancellationTokenSource(); + + _fileTcs.TrySetCanceled(); + _fileTcs = new TaskCompletionSource>(); + + _fileTask = null; + } + } + + private void SetFileTcsResult() { + _fileTcs.TrySetResult(_symbolIndex.HierarchicalDocumentSymbols(_path)); + } + + private void SetFileTask(Task task) { + _fileTask = task; + _fileTask.ContinueWith(t => { + if (t.Status.Equals(TaskStatus.RanToCompletion)) { + lock (_syncObj) { + SetFileTcsResult(); + } + } + }); + } + + public void Dispose() { + lock (_syncObj) { + _fileCts.Dispose(); + _fileTask = null; + } + } + } +} diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index 783c4ca02..6da2b117a 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -17,6 +17,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; +using System.Threading.Tasks; using Microsoft.Python.Core; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing.Ast; @@ -24,18 +25,18 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal sealed class SymbolIndex : ISymbolIndex { private static int DefaultVersion = 0; - private readonly ConcurrentDictionary>> _index; + private readonly ConcurrentDictionary> _index; - public SymbolIndex(IEqualityComparer comparer = null) { - comparer = comparer ?? PathEqualityComparer.Instance; - _index = new ConcurrentDictionary>>(comparer); + public SymbolIndex() { + var comparer = PathEqualityComparer.Instance; + _index = new ConcurrentDictionary>(comparer); } public IEnumerable HierarchicalDocumentSymbols(string path) - => _index.TryGetValue(path, out var list) ? list.Value : Enumerable.Empty(); + => _index.TryGetValue(path, out var list) ? list : Enumerable.Empty(); public IEnumerable WorkspaceSymbols(string query) { - return _index.SelectMany(kvp => WorkspaceSymbolsQuery(query, kvp.Key, kvp.Value.Value)); + return _index.SelectMany(kvp => WorkspaceSymbolsQuery(query, kvp.Key, kvp.Value)); } private IEnumerable WorkspaceSymbolsQuery(string query, string path, @@ -52,25 +53,14 @@ private IEnumerable WorkspaceSymbolsQuery(string query, string path, } } - public void UpdateIndexIfNewer(string path, PythonAst ast, int version) { + public void Update(string path, PythonAst ast) { var walker = new SymbolIndexWalker(ast); ast.Walk(walker); - var versionedList = _index.GetOrAdd(path, MakeVersionedValue(walker.Symbols, version)); - versionedList.UpdateIfNewer(walker.Symbols, version); + _index[path] = walker.Symbols; } - public void DeleteIfNewer(string path, int version) { - var currentVersion = _index[path].Version; - if (version <= currentVersion) return; - // Point A - _index.Remove(path, out var oldVersionedValue); - // Point B - if (oldVersionedValue.Version > version) { - // An update happened at point A - // Reinsert that version, only if key doesn't exist - _index.GetOrAdd(path, oldVersionedValue); - // Update could have happened at B - } + public void Delete(string path) { + _index.Remove(path, out var _); } public bool IsIndexed(string path) => _index.ContainsKey(path); @@ -78,44 +68,5 @@ public void DeleteIfNewer(string path, int version) { private static IEnumerable<(HierarchicalSymbol symbol, string parentName)> DecorateWithParentsName( IEnumerable symbols, string parentName) => symbols.Select((symbol) => (symbol, parentName)); - - public int GetNewVersion(string path) { - var versionedList = - _index.GetOrAdd(path, MakeVersionedValue(new List(), DefaultVersion)); - return versionedList.GetNewVersion(); - } - - private VersionedValue> MakeVersionedValue( - IReadOnlyList symbols, int version) { - return new VersionedValue>(symbols, version); - } - - internal class VersionedValue { - private readonly object _syncObj = new object(); - private int _newVersion; - - public VersionedValue(T value, int version) { - Value = value; - Version = version; - _newVersion = version + 1; - } - - public int Version { get; private set; } - - public T Value { get; private set; } - - public void UpdateIfNewer(T value, int version) { - lock (_syncObj) { - if (version > Version) { - Value = value; - Version = version; - } - } - } - - public int GetNewVersion() { - return Interlocked.Increment(ref _newVersion); - } - } } } diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index 288ba477a..e57361128 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -21,10 +21,7 @@ using System.Threading; using System.Threading.Tasks; using FluentAssertions; -using Microsoft.Python.Analysis.Analyzer; -using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Core.Idle; using Microsoft.Python.Core.IO; using Microsoft.Python.Core.Services; @@ -47,6 +44,8 @@ public class IndexManagerTests : LanguageServerTestBase { private IIdleTimeService _idleTimeService; private ServiceManager _services; + private const int maxSymbolsCount = 1000; + public TestContext TestContext { get; set; } [TestInitialize] @@ -71,7 +70,11 @@ private void CreateServices() { } [TestCleanup] - public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + public void Cleanup() { + _services.Dispose(); + _rootFileList.Clear(); + TestEnvironmentImpl.TestCleanup(); + } [TestMethod, Priority(0)] public async Task AddsRootDirectoryAsync() { @@ -79,9 +82,8 @@ public async Task AddsRootDirectoryAsync() { AddFileToRoot($"{_rootPath}\foo.py", MakeStream("y = 1")); var indexManager = GetDefaultIndexManager(); - await indexManager.AddRootDirectoryAsync(); - var symbols = _symbolIndex.WorkspaceSymbols(""); + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); symbols.Should().HaveCount(2); } @@ -101,7 +103,7 @@ public async Task IgnoresNonPythonFiles() { AddFileInfoToRootTestFS(nonPythonTestFileInfo); IIndexManager indexManager = GetDefaultIndexManager(); - await indexManager.AddRootDirectoryAsync(); + await WaitForWorkspaceAddedAsync(indexManager); _fileSystem.DidNotReceive().FileExists(nonPythonTestFileInfo.FullName); } @@ -113,9 +115,9 @@ public async Task CanOpenFiles() { IDocument doc = DocumentWithAst("x = 1"); IIndexManager indexManager = GetDefaultIndexManager(); - await indexManager.ProcessNewFileAsync(pythonTestFileInfo.FullName, doc); + indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); - var symbols = _symbolIndex.WorkspaceSymbols(""); + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); SymbolsShouldBeOnlyX(symbols); } @@ -130,13 +132,13 @@ private IDocument DocumentWithAst(string testCode, string filePath = null) { [TestMethod, Priority(0)] public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { var pythonTestFilePath = FileWithXVarInRootDir(); - IDocument latestDoc = DocumentWithAst("y = 1"); var indexManager = GetDefaultIndexManager(); - await indexManager.AddRootDirectoryAsync(); - await indexManager.ReIndexFileAsync(pythonTestFilePath, latestDoc); + await WaitForWorkspaceAddedAsync(indexManager); - var symbols = _symbolIndex.WorkspaceSymbols(""); + indexManager.ReIndexFile(pythonTestFilePath, DocumentWithAst("y = 1")); + + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); symbols.Should().HaveCount(1); symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); symbols.First().Name.Should().BeEquivalentTo("y"); @@ -146,12 +148,13 @@ public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); IDocument doc = DocumentWithAst("x = 1"); + _fileSystem.IsPathUnderRoot("", "").ReturnsForAnyArgs(false); var indexManager = GetDefaultIndexManager(); - await indexManager.ProcessNewFileAsync(pythonTestFileInfo.FullName, doc); - await indexManager.ProcessClosedFileAsync(pythonTestFileInfo.FullName); + indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); + indexManager.ProcessClosedFile(pythonTestFileInfo.FullName); - SymbolIndexShouldBeEmpty(); + await SymbolIndexShouldBeEmpty(indexManager); } [TestMethod, Priority(0)] @@ -159,16 +162,15 @@ public async Task CloseWorkspaceFilesReUpdatesIndexAsync() { string pythonTestFilePath = FileWithXVarInRootDir(); _fileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); - IDocument latestDoc = DocumentWithAst("y = 1"); - var indexManager = GetDefaultIndexManager(); - await indexManager.AddRootDirectoryAsync(); - await indexManager.ProcessNewFileAsync(pythonTestFilePath, latestDoc); + await WaitForWorkspaceAddedAsync(indexManager); + + indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("y = 1")); // It Needs to remake the stream for the file, previous one is closed _fileSystem.FileOpen(pythonTestFilePath, FileMode.Open).Returns(MakeStream("x = 1")); - await indexManager.ProcessClosedFileAsync(pythonTestFilePath); + indexManager.ProcessClosedFile(pythonTestFilePath); - var symbols = _symbolIndex.WorkspaceSymbols(""); + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); SymbolsShouldBeOnlyX(symbols); } @@ -181,36 +183,25 @@ public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() { IDocument doc = DocumentWithAst("x = 1"); var indexManager = GetDefaultIndexManager(); - await indexManager.ProcessNewFileAsync(pythonTestFileInfo.FullName, doc); - await indexManager.ProcessClosedFileAsync(pythonTestFileInfo.FullName); + indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); + indexManager.ProcessClosedFile(pythonTestFileInfo.FullName); doc = DocumentWithAst("x = 1"); - await indexManager.ReIndexFileAsync(pythonTestFileInfo.FullName, doc); + indexManager.ReIndexFile(pythonTestFileInfo.FullName, doc); - SymbolIndexShouldBeEmpty(); + await SymbolIndexShouldBeEmpty(indexManager); } - [TestMethod, Priority(0)] - public void AddingRootMightThrowUnauthorizedAccess() { - string pythonTestFilePath = FileWithXVarInRootDir(); - _fileSystem.GetDirectoryInfo(_rootPath).EnumerateFileSystemInfos(new string[] { }, new string[] { }) - .ReturnsForAnyArgs(_ => throw new UnauthorizedAccessException()); - - var indexManager = GetDefaultIndexManager(); - Func add = async () => { - await indexManager.AddRootDirectoryAsync(); - }; - - add.Should().Throw(); - SymbolIndexShouldBeEmpty(); + private static async Task WaitForWorkspaceAddedAsync(IIndexManager indexManager) { + await indexManager.WorkspaceSymbolsAsync("", 1000); } - private void SymbolIndexShouldBeEmpty() { - var symbols = _symbolIndex.WorkspaceSymbols(""); + private async Task SymbolIndexShouldBeEmpty(IIndexManager indexManager) { + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); symbols.Should().HaveCount(0); } - + /* [TestMethod, Priority(0)] - public void DisposeManagerCancelsTask() { + public async Task DisposeManagerCancelsTaskAsync() { string pythonTestFilePath = FileWithXVarInRootDir(); ManualResetEventSlim neverSignaledEvent = new ManualResetEventSlim(false); ManualResetEventSlim fileOpenedEvent = new ManualResetEventSlim(false); @@ -227,12 +218,12 @@ public void DisposeManagerCancelsTask() { indexManager.Dispose(); Func add = async () => { - await indexManager.AddRootDirectoryAsync(); + await WaitForWorkspaceAddedAsync(indexManager); }; add.Should().Throw(); - SymbolIndexShouldBeEmpty(); - } + await SymbolIndexShouldBeEmpty(indexManager); + }*/ [TestMethod, Priority(0)] public async Task WorkspaceSymbolsAddsRootDirectory() { @@ -240,7 +231,7 @@ public async Task WorkspaceSymbolsAddsRootDirectory() { var indexManager = GetDefaultIndexManager(); - var symbols = await indexManager.WorkspaceSymbolsAsync("", 10); + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); SymbolsShouldBeOnlyX(symbols); } @@ -285,13 +276,12 @@ public async Task LatestVersionASTVersionIsIndexed() { IDocument yVarDoc = DocumentWithAst("y = 1"); IDocument zVarDoc = DocumentWithAst("z = 1"); - await indexManager.ProcessNewFileAsync(pythonTestFilePath, yVarDoc); - var closeFileTask = indexManager.ProcessClosedFileAsync(pythonTestFilePath); + indexManager.ProcessNewFile(pythonTestFilePath, yVarDoc); + indexManager.ProcessClosedFile(pythonTestFilePath); fileOpenedEvent.Wait(); - await indexManager.ProcessNewFileAsync(pythonTestFilePath, zVarDoc); + indexManager.ProcessNewFile(pythonTestFilePath, zVarDoc); reOpenedFileFinished.Set(); - await closeFileTask; var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath); symbols.Should().HaveCount(1); symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); @@ -304,11 +294,12 @@ public async Task AddFilesToPendingChanges() { var f2 = AddFileToRoot($"{_rootPath}/fileB.py", MakeStream("")); var indexManager = GetDefaultIndexManager(); + await WaitForWorkspaceAddedAsync(indexManager); indexManager.AddPendingDoc(DocumentWithAst("y = 1", f1)); indexManager.AddPendingDoc(DocumentWithAst("x = 1", f2)); _idleTimeService.Idle += Raise.Event(); - var symbols = await indexManager.WorkspaceSymbolsAsync("", 1000); + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); symbols.Should().HaveCount(2); } diff --git a/src/LanguageServer/Test/SymbolIndexTests.cs b/src/LanguageServer/Test/SymbolIndexTests.cs index 8240e95d9..b3b56d705 100644 --- a/src/LanguageServer/Test/SymbolIndexTests.cs +++ b/src/LanguageServer/Test/SymbolIndexTests.cs @@ -616,8 +616,7 @@ public void IndexHierarchicalDocument() { ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var ast = GetParse("x = 1"); - var version = index.GetNewVersion(path); - index.UpdateIndexIfNewer(path, ast, version); + index.Update(path, ast); var symbols = index.HierarchicalDocumentSymbols(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { @@ -630,8 +629,8 @@ public void IndexHierarchicalDocumentUpdate() { ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var ast = GetParse("x = 1"); - var version = index.GetNewVersion(path); - index.UpdateIndexIfNewer(path, ast, version); + + index.Update(path, ast); var symbols = index.HierarchicalDocumentSymbols(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { @@ -639,8 +638,7 @@ public void IndexHierarchicalDocumentUpdate() { }); ast = GetParse("y = 1"); - version = index.GetNewVersion(path); - index.UpdateIndexIfNewer(path, ast, version); + index.Update(path, ast); symbols = index.HierarchicalDocumentSymbols(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { @@ -665,8 +663,8 @@ public void IndexWorkspaceSymbolsFlatten() { ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var ast = GetParse(code); - var version = index.GetNewVersion(path); - index.UpdateIndexIfNewer(path, ast, version); + + index.Update(path, ast); var symbols = index.WorkspaceSymbols(""); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { @@ -685,8 +683,8 @@ public void IndexWorkspaceSymbolsFiltered() { ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var ast = GetParse(code); - var version = index.GetNewVersion(path); - index.UpdateIndexIfNewer(path, ast, version); + + index.Update(path, ast); var symbols = index.WorkspaceSymbols("x"); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { @@ -711,8 +709,8 @@ public void IndexWorkspaceSymbolsCaseInsensitive() { ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var ast = GetParse(code); - var version = index.GetNewVersion(path); - index.UpdateIndexIfNewer(path, ast, version); + + index.Update(path, ast); var symbols = index.WorkspaceSymbols("foo"); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { From 0f331c2ea87100d9f3fe5c34dd555e99e1927343 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 14 Feb 2019 21:15:31 -0800 Subject: [PATCH 061/123] Passed more tests --- .../Impl/Indexing/IndexManager.cs | 18 ++++++++---- .../Impl/Indexing/MostRecentSymbols.cs | 21 +++++++------- src/LanguageServer/Test/IndexManagerTests.cs | 28 ++++++++++--------- 3 files changed, 37 insertions(+), 30 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index 47688b8ef..bf1fabb39 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -86,7 +86,11 @@ private IEnumerable WorkspaceFiles() { private bool IsFileIndexed(string path) => _symbolIndex.IsIndexed(path); public void ProcessClosedFile(string path) { - _files[path].Close(IsFileOnWorkspace(path)); + if (IsFileOnWorkspace(path)) { + _files[path].Parse(); + } else { + _files[path].Delete(); + } } private bool IsFileOnWorkspace(string path) { @@ -121,11 +125,13 @@ public async Task> HierarchicalDocumentSymbols return s.ToList(); } - public async Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken cancellationToken = default) { - foreach(var mostRecentSymbols in _files.Values) { - await mostRecentSymbols.GetSymbolsAsync(); - } - return _symbolIndex.WorkspaceSymbols(query).Take(maxLength).ToList(); + public Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken cancellationToken = default) { + var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(_allIndexCts.Token, cancellationToken); + return Task.WhenAll(_files.Values.Select(mostRecent => mostRecent.GetSymbolsAsync()).ToArray()).ContinueWith>( + _ => { + return _symbolIndex.WorkspaceSymbols(query).Take(maxLength).ToList(); + }, linkedCts.Token); + } public void AddPendingDoc(IDocument doc) { diff --git a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs index 5969a76fb..201f73338 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs @@ -31,13 +31,9 @@ public void Parse() { } } - public void Close(bool isOnWorkspace) { - if (isOnWorkspace) { - Parse(); - } else { - lock (_syncObj) { - _symbolIndex.Delete(_path); - } + public void Delete() { + lock (_syncObj) { + _symbolIndex.Delete(_path); } } @@ -64,13 +60,16 @@ public void Process(IDocument doc) { private void CancelExistingTask() { if (_fileTask != null) { - _fileCts.Cancel(); + if (_fileTcs.Task.IsCompleted) { + _fileCts.Cancel(); + + _fileTcs.TrySetCanceled(); + _fileTcs = new TaskCompletionSource>(); + } + _fileCts.Dispose(); _fileCts = new CancellationTokenSource(); - _fileTcs.TrySetCanceled(); - _fileTcs = new TaskCompletionSource>(); - _fileTask = null; } } diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index e57361128..95e6b1340 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -37,7 +37,6 @@ namespace Microsoft.Python.LanguageServer.Tests { [TestClass] public class IndexManagerTests : LanguageServerTestBase { private IFileSystem _fileSystem; - private ISymbolIndex _symbolIndex; private string _rootPath; private List _rootFileList; private PythonLanguageVersion _pythonLanguageVersion; @@ -52,7 +51,6 @@ public class IndexManagerTests : LanguageServerTestBase { public void TestInitialize() { TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); _fileSystem = Substitute.For(); - _symbolIndex = new SymbolIndex(); _rootPath = "C:/root"; _pythonLanguageVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); _rootFileList = new List(); @@ -91,7 +89,7 @@ public async Task AddsRootDirectoryAsync() { public void NullDirectoryThrowsException() { Action construct = () => { PythonLanguageVersion version = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); - IIndexManager indexManager = new IndexManager(_symbolIndex, _fileSystem, + IIndexManager indexManager = new IndexManager(new SymbolIndex(), _fileSystem, version, null, new string[] { }, new string[] { }, _idleTimeService); }; construct.Should().Throw(); @@ -148,7 +146,7 @@ public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); IDocument doc = DocumentWithAst("x = 1"); - _fileSystem.IsPathUnderRoot("", "").ReturnsForAnyArgs(false); + _fileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false); var indexManager = GetDefaultIndexManager(); indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); @@ -159,7 +157,7 @@ public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { [TestMethod, Priority(0)] public async Task CloseWorkspaceFilesReUpdatesIndexAsync() { - string pythonTestFilePath = FileWithXVarInRootDir(); + var pythonTestFilePath = FileWithXVarInRootDir(); _fileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); var indexManager = GetDefaultIndexManager(); @@ -168,6 +166,7 @@ public async Task CloseWorkspaceFilesReUpdatesIndexAsync() { indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("y = 1")); // It Needs to remake the stream for the file, previous one is closed _fileSystem.FileOpen(pythonTestFilePath, FileMode.Open).Returns(MakeStream("x = 1")); + _fileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); indexManager.ProcessClosedFile(pythonTestFilePath); var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); @@ -180,12 +179,14 @@ public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() { // it should not reindex file var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); - IDocument doc = DocumentWithAst("x = 1"); + var doc = DocumentWithAst("x = 1"); var indexManager = GetDefaultIndexManager(); indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); indexManager.ProcessClosedFile(pythonTestFileInfo.FullName); doc = DocumentWithAst("x = 1"); + + _fileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false); indexManager.ReIndexFile(pythonTestFileInfo.FullName, doc); await SymbolIndexShouldBeEmpty(indexManager); @@ -199,10 +200,11 @@ private async Task SymbolIndexShouldBeEmpty(IIndexManager indexManager) { var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); symbols.Should().HaveCount(0); } - /* + + [TestMethod, Priority(0)] public async Task DisposeManagerCancelsTaskAsync() { - string pythonTestFilePath = FileWithXVarInRootDir(); + var pythonTestFilePath = FileWithXVarInRootDir(); ManualResetEventSlim neverSignaledEvent = new ManualResetEventSlim(false); ManualResetEventSlim fileOpenedEvent = new ManualResetEventSlim(false); @@ -220,14 +222,14 @@ public async Task DisposeManagerCancelsTaskAsync() { Func add = async () => { await WaitForWorkspaceAddedAsync(indexManager); }; - + add.Should().Throw(); await SymbolIndexShouldBeEmpty(indexManager); - }*/ + } [TestMethod, Priority(0)] public async Task WorkspaceSymbolsAddsRootDirectory() { - string pythonTestFilePath = FileWithXVarInRootDir(); + var pythonTestFilePath = FileWithXVarInRootDir(); var indexManager = GetDefaultIndexManager(); @@ -250,7 +252,7 @@ public async Task WorkspaceSymbolsLimited() { [TestMethod, Priority(0)] public async Task HierarchicalDocumentSymbolsAsync() { - string pythonTestFilePath = FileWithXVarInRootDir(); + var pythonTestFilePath = FileWithXVarInRootDir(); var indexManager = GetDefaultIndexManager(); @@ -336,7 +338,7 @@ private string FileWithXVarInRootDir() { } private IIndexManager GetDefaultIndexManager() { - var indexM = new IndexManager(_symbolIndex, _fileSystem, _pythonLanguageVersion, + var indexM = new IndexManager(new SymbolIndex(), _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }, _idleTimeService) { ReIndexingDelay = 1 From d3b4761f0b9459345708e69eb9f24682bb3a5c0a Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 15 Feb 2019 01:08:07 -0800 Subject: [PATCH 062/123] Removing fields of test --- src/LanguageServer/Test/IndexManagerTests.cs | 38 +++++++++----------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index 95e6b1340..97c3376a7 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -37,40 +37,35 @@ namespace Microsoft.Python.LanguageServer.Tests { [TestClass] public class IndexManagerTests : LanguageServerTestBase { private IFileSystem _fileSystem; - private string _rootPath; - private List _rootFileList; - private PythonLanguageVersion _pythonLanguageVersion; private IIdleTimeService _idleTimeService; - private ServiceManager _services; private const int maxSymbolsCount = 1000; + private const string _rootPath = "C:/root"; + private readonly PythonLanguageVersion _pythonLanguageVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion();; public TestContext TestContext { get; set; } [TestInitialize] public void TestInitialize() { TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + } + + private void Setup() { _fileSystem = Substitute.For(); - _rootPath = "C:/root"; - _pythonLanguageVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); - _rootFileList = new List(); _idleTimeService = Substitute.For(); - CreateServices(); + } + + private List SetupRootDir(IFileSystem _fileSystem) { + var rootFileList = new List(); IDirectoryInfo directoryInfo = Substitute.For(); // Doesn't work without 'forAnyArgs' - directoryInfo.EnumerateFileSystemInfos(new string[] { }, new string[] { }).ReturnsForAnyArgs(_rootFileList); + directoryInfo.EnumerateFileSystemInfos(new string[] { }, new string[] { }).ReturnsForAnyArgs(rootFileList); _fileSystem.GetDirectoryInfo(_rootPath).Returns(directoryInfo); - } - - private void CreateServices() { - _services = new ServiceManager(); - _services.AddService(_fileSystem); + return rootFileList; } [TestCleanup] public void Cleanup() { - _services.Dispose(); - _rootFileList.Clear(); TestEnvironmentImpl.TestCleanup(); } @@ -324,7 +319,7 @@ private PythonAst MakeAst(string testCode) { private FileInfoProxy MakeFileInfoProxy(string filePath) => new FileInfoProxy(new FileInfo(filePath)); - private void AddFileInfoToRootTestFS(FileInfoProxy fileInfo) { + private void AddFileInfoToRootTestFS(FileInfoProxy fileInfo, List_rootFileList, IFileSystem fileSystem) { _rootFileList.Add(fileInfo); _fileSystem.FileExists(fileInfo.FullName).Returns(true); } @@ -333,11 +328,12 @@ private Stream MakeStream(string str) { return new MemoryStream(Encoding.UTF8.GetBytes(str)); } - private string FileWithXVarInRootDir() { - return AddFileToRoot($"{_rootPath}\bla.py", MakeStream("x = 1")); + private string FileWithXVarInRootDir(IFileSystem fileSystem) { + return AddFileToRoot($"{_rootPath}\bla.py", MakeStream("x = 1"), fileSystem); } private IIndexManager GetDefaultIndexManager() { + var _rootFileList = SetupRootDir(_fileSystem); var indexM = new IndexManager(new SymbolIndex(), _fileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }, _idleTimeService) { @@ -346,9 +342,9 @@ private IIndexManager GetDefaultIndexManager() { return indexM; } - private string AddFileToRoot(string filePath, Stream stream) { + private string AddFileToRoot(string filePath, Stream stream, List rootFileList, IFileSystem fileSystem) { var fileInfo = MakeFileInfoProxy(filePath); - AddFileInfoToRootTestFS(fileInfo); + AddFileInfoToRootTestFS(fileInfo, rootFileList, fileSystem); string fullName = fileInfo.FullName; _fileSystem.FileOpen(fullName, FileMode.Open).Returns(stream); // FileInfo fullName is used everywhere as path From 74e9bed164872b78036ff5669a3336c773336c2e Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 15 Feb 2019 14:03:54 -0800 Subject: [PATCH 063/123] Passes every test again --- .../Impl/Indexing/ISymbolIndex.cs | 4 +- .../Impl/Indexing/IndexManager.cs | 23 +- .../Impl/Indexing/IndexParser.cs | 2 +- .../Impl/Indexing/MostRecentSymbols.cs | 65 +++-- .../Impl/Indexing/SymbolIndex.cs | 41 +-- src/LanguageServer/Test/IndexManagerTests.cs | 262 ++++++++++-------- src/LanguageServer/Test/SymbolIndexTests.cs | 12 +- 7 files changed, 237 insertions(+), 172 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs index 52adf1dbf..eb2098485 100644 --- a/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs @@ -20,8 +20,10 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal interface ISymbolIndex { IEnumerable WorkspaceSymbols(string query); IEnumerable HierarchicalDocumentSymbols(string path); - void Update(string path, PythonAst pythonAst); + void Add(string path, PythonAst pythonAst); void Delete(string path); bool IsIndexed(string path); + void Update(string path, PythonAst pythonAst); + } } diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index bf1fabb39..ad45b84da 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -88,8 +88,9 @@ private IEnumerable WorkspaceFiles() { public void ProcessClosedFile(string path) { if (IsFileOnWorkspace(path)) { _files[path].Parse(); - } else { - _files[path].Delete(); + } else if (_files.TryRemove(path, out var fileSymbols)) { + fileSymbols.Delete(); + fileSymbols.Dispose(); } } @@ -102,12 +103,12 @@ private bool IsFileOnWorkspace(string path) { public void ProcessNewFile(string path, IDocument doc) { _files.GetOrAdd(path, MakeMostRecentFileSymbols(path)); - _files[path].Process(doc); + _files[path].Add(doc); } public void ReIndexFile(string path, IDocument doc) { - if (IsFileIndexed(path)) { - ProcessNewFile(path, doc); + if (_files.TryGetValue(path, out var fileSymbols)) { + fileSymbols.ReIndex(doc); } } @@ -125,13 +126,13 @@ public async Task> HierarchicalDocumentSymbols return s.ToList(); } - public Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken cancellationToken = default) { + public async Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken cancellationToken = default) { var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(_allIndexCts.Token, cancellationToken); - return Task.WhenAll(_files.Values.Select(mostRecent => mostRecent.GetSymbolsAsync()).ToArray()).ContinueWith>( - _ => { - return _symbolIndex.WorkspaceSymbols(query).Take(maxLength).ToList(); - }, linkedCts.Token); - + await Task.WhenAny( + Task.WhenAll(_files.Values.Select(mostRecent => mostRecent.GetSymbolsAsync()).ToArray()), + Task.Delay(Timeout.Infinite, linkedCts.Token)); + linkedCts.Dispose(); + return _symbolIndex.WorkspaceSymbols(query).Take(maxLength).ToList(); } public void AddPendingDoc(IDocument doc) { diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index 9b53a2ac4..073cdc006 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -54,7 +54,7 @@ public Task ParseAsync(string path, CancellationToken parseCancellationTok using (var stream = _fileSystem.FileOpen(path, FileMode.Open)) { var parser = Parser.CreateParser(stream, _version); linkedParseToken.ThrowIfCancellationRequested(); - _symbolIndex.Update(path, parser.ParseFile()); + _symbolIndex.Add(path, parser.ParseFile()); return true; } } catch (FileNotFoundException e) { diff --git a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs index 201f73338..701cc2ce7 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs @@ -27,10 +27,19 @@ public MostRecentDocumentSymbols(string path, IIndexParser indexParser, ISymbolI public void Parse() { lock (_syncObj) { CancelExistingTask(); - SetFileTask(_indexParser.ParseAsync(_path, _fileCts.Token)); + _fileTask = ParseAsync(); } } + private async Task ParseAsync() { + var ast = await _indexParser.ParseAsync(_path, _fileCts.Token); + _fileCts.Token.ThrowIfCancellationRequested(); + lock (_syncObj) { + SetFileTcsResult(); + } + + } + public void Delete() { lock (_syncObj) { _symbolIndex.Delete(_path); @@ -40,19 +49,24 @@ public void Delete() { public void Process(PythonAst ast) { lock (_syncObj) { CancelExistingTask(); - _symbolIndex.Update(_path, ast); + _symbolIndex.Add(_path, ast); SetFileTcsResult(); } } - public void Process(IDocument doc) { + public void Add(IDocument doc) { lock (_syncObj) { CancelExistingTask(); - SetFileTask(doc.GetAstAsync(_fileCts.Token).ContinueWith(t => { - lock (_syncObj) { - _symbolIndex.Update(_path, t.Result); - } - }, _fileCts.Token)); + _fileTask = AddAsync(doc); + } + } + + private async Task AddAsync(IDocument doc) { + var ast = await doc.GetAstAsync(_fileCts.Token); + _fileCts.Token.ThrowIfCancellationRequested(); + lock (_syncObj) { + _symbolIndex.Add(_path, ast); + SetFileTcsResult(); } } @@ -61,12 +75,11 @@ public void Process(IDocument doc) { private void CancelExistingTask() { if (_fileTask != null) { if (_fileTcs.Task.IsCompleted) { - _fileCts.Cancel(); - _fileTcs.TrySetCanceled(); _fileTcs = new TaskCompletionSource>(); } + _fileCts.Cancel(); _fileCts.Dispose(); _fileCts = new CancellationTokenSource(); @@ -74,26 +87,34 @@ private void CancelExistingTask() { } } - private void SetFileTcsResult() { - _fileTcs.TrySetResult(_symbolIndex.HierarchicalDocumentSymbols(_path)); + public void ReIndex(IDocument doc) { + lock (_syncObj) { + CancelExistingTask(); + _fileTask = ReIndexAsync(doc); + } } - private void SetFileTask(Task task) { - _fileTask = task; - _fileTask.ContinueWith(t => { - if (t.Status.Equals(TaskStatus.RanToCompletion)) { - lock (_syncObj) { - SetFileTcsResult(); - } - } - }); + private async Task ReIndexAsync(IDocument doc) { + var ast = await doc.GetAstAsync(_fileCts.Token); + _fileCts.Token.ThrowIfCancellationRequested(); + lock (_syncObj) { + _symbolIndex.Update(_path, ast); + SetFileTcsResult(); + } + } + + private void SetFileTcsResult() { + _fileTcs.TrySetResult(_symbolIndex.HierarchicalDocumentSymbols(_path)); } public void Dispose() { lock (_syncObj) { - _fileCts.Dispose(); + _fileCts?.Cancel(); + _fileCts?.Dispose(); + _fileCts = null; _fileTask = null; } } } } + diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index 6da2b117a..03776a706 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -16,15 +16,12 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using System.Threading; -using System.Threading.Tasks; using Microsoft.Python.Core; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.LanguageServer.Indexing { internal sealed class SymbolIndex : ISymbolIndex { - private static int DefaultVersion = 0; private readonly ConcurrentDictionary> _index; public SymbolIndex() { @@ -39,21 +36,7 @@ public IEnumerable WorkspaceSymbols(string query) { return _index.SelectMany(kvp => WorkspaceSymbolsQuery(query, kvp.Key, kvp.Value)); } - private IEnumerable WorkspaceSymbolsQuery(string query, string path, - IEnumerable symbols) { - var rootSymbols = DecorateWithParentsName(symbols, null); - var treeSymbols = rootSymbols.TraverseBreadthFirst((symbolAndParent) => { - var sym = symbolAndParent.symbol; - return DecorateWithParentsName(sym.Children.MaybeEnumerate(), sym.Name); - }); - foreach (var (sym, parentName) in treeSymbols) { - if (sym.Name.ContainsOrdinal(query, ignoreCase: true)) { - yield return new FlatSymbol(sym.Name, sym.Kind, path, sym.SelectionRange, parentName); - } - } - } - - public void Update(string path, PythonAst ast) { + public void Add(string path, PythonAst ast) { var walker = new SymbolIndexWalker(ast); ast.Walk(walker); _index[path] = walker.Symbols; @@ -68,5 +51,27 @@ public void Delete(string path) { private static IEnumerable<(HierarchicalSymbol symbol, string parentName)> DecorateWithParentsName( IEnumerable symbols, string parentName) => symbols.Select((symbol) => (symbol, parentName)); + + public void Update(string path, PythonAst ast) { + if (_index.TryGetValue(path, out var currentSymbols)) { + var walker = new SymbolIndexWalker(ast); + ast.Walk(walker); + _index.TryUpdate(path, walker.Symbols, currentSymbols); + } + } + + private IEnumerable WorkspaceSymbolsQuery(string query, string path, + IEnumerable symbols) { + var rootSymbols = DecorateWithParentsName(symbols, null); + var treeSymbols = rootSymbols.TraverseBreadthFirst((symbolAndParent) => { + var sym = symbolAndParent.symbol; + return DecorateWithParentsName(sym.Children.MaybeEnumerate(), sym.Name); + }); + foreach (var (sym, parentName) in treeSymbols) { + if (sym.Name.ContainsOrdinal(query, ignoreCase: true)) { + yield return new FlatSymbol(sym.Name, sym.Kind, path, sym.SelectionRange, parentName); + } + } + } } } diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index 97c3376a7..db86f703a 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -36,12 +36,8 @@ namespace Microsoft.Python.LanguageServer.Tests { [TestClass] public class IndexManagerTests : LanguageServerTestBase { - private IFileSystem _fileSystem; - private IIdleTimeService _idleTimeService; - private const int maxSymbolsCount = 1000; private const string _rootPath = "C:/root"; - private readonly PythonLanguageVersion _pythonLanguageVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion();; public TestContext TestContext { get; set; } @@ -50,20 +46,6 @@ public void TestInitialize() { TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); } - private void Setup() { - _fileSystem = Substitute.For(); - _idleTimeService = Substitute.For(); - } - - private List SetupRootDir(IFileSystem _fileSystem) { - var rootFileList = new List(); - IDirectoryInfo directoryInfo = Substitute.For(); - // Doesn't work without 'forAnyArgs' - directoryInfo.EnumerateFileSystemInfos(new string[] { }, new string[] { }).ReturnsForAnyArgs(rootFileList); - _fileSystem.GetDirectoryInfo(_rootPath).Returns(directoryInfo); - return rootFileList; - } - [TestCleanup] public void Cleanup() { TestEnvironmentImpl.TestCleanup(); @@ -71,10 +53,11 @@ public void Cleanup() { [TestMethod, Priority(0)] public async Task AddsRootDirectoryAsync() { - FileWithXVarInRootDir(); - AddFileToRoot($"{_rootPath}\foo.py", MakeStream("y = 1")); + var context = new IndexTestContext(this); + context.FileWithXVarInRootDir(); + context.AddFileToRoot($"{_rootPath}\foo.py", MakeStream("y = 1")); - var indexManager = GetDefaultIndexManager(); + var indexManager = context.GetDefaultIndexManager(); var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); symbols.Should().HaveCount(2); @@ -82,51 +65,49 @@ public async Task AddsRootDirectoryAsync() { [TestMethod, Priority(0)] public void NullDirectoryThrowsException() { + var context = new IndexTestContext(this); Action construct = () => { PythonLanguageVersion version = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); - IIndexManager indexManager = new IndexManager(new SymbolIndex(), _fileSystem, - version, null, new string[] { }, new string[] { }, _idleTimeService); + IIndexManager indexManager = new IndexManager(new SymbolIndex(), context.FileSystem, + version, null, new string[] { }, new string[] { }, + new IdleTimeService()); }; construct.Should().Throw(); } [TestMethod, Priority(0)] public async Task IgnoresNonPythonFiles() { + var context = new IndexTestContext(this); + var nonPythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}/bla.txt"); - AddFileInfoToRootTestFS(nonPythonTestFileInfo); + context.AddFileInfoToRootTestFS(nonPythonTestFileInfo); - IIndexManager indexManager = GetDefaultIndexManager(); + IIndexManager indexManager = context.GetDefaultIndexManager(); await WaitForWorkspaceAddedAsync(indexManager); - _fileSystem.DidNotReceive().FileExists(nonPythonTestFileInfo.FullName); + context.FileSystem.DidNotReceive().FileExists(nonPythonTestFileInfo.FullName); } [TestMethod, Priority(0)] public async Task CanOpenFiles() { string nonRootPath = "C:/nonRoot"; + var context = new IndexTestContext(this); var pythonTestFileInfo = MakeFileInfoProxy($"{nonRootPath}/bla.py"); IDocument doc = DocumentWithAst("x = 1"); - IIndexManager indexManager = GetDefaultIndexManager(); + IIndexManager indexManager = context.GetDefaultIndexManager(); indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); SymbolsShouldBeOnlyX(symbols); } - private IDocument DocumentWithAst(string testCode, string filePath = null) { - filePath = filePath ?? $"{_rootPath}/{testCode}.py"; - IDocument doc = Substitute.For(); - doc.GetAstAsync().ReturnsForAnyArgs(Task.FromResult(MakeAst(testCode))); - doc.Uri.Returns(new Uri(filePath)); - return doc; - } - [TestMethod, Priority(0)] public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { - var pythonTestFilePath = FileWithXVarInRootDir(); + var context = new IndexTestContext(this); + var pythonTestFilePath = context.FileWithXVarInRootDir(); - var indexManager = GetDefaultIndexManager(); + var indexManager = context.GetDefaultIndexManager(); await WaitForWorkspaceAddedAsync(indexManager); indexManager.ReIndexFile(pythonTestFilePath, DocumentWithAst("y = 1")); @@ -139,11 +120,12 @@ public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { [TestMethod, Priority(0)] public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { + var context = new IndexTestContext(this); var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); IDocument doc = DocumentWithAst("x = 1"); - _fileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false); + context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false); - var indexManager = GetDefaultIndexManager(); + var indexManager = context.GetDefaultIndexManager(); indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); indexManager.ProcessClosedFile(pythonTestFileInfo.FullName); @@ -152,16 +134,18 @@ public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { [TestMethod, Priority(0)] public async Task CloseWorkspaceFilesReUpdatesIndexAsync() { - var pythonTestFilePath = FileWithXVarInRootDir(); - _fileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); + var context = new IndexTestContext(this); + var pythonTestFilePath = context.FileWithXVarInRootDir(); + context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); - var indexManager = GetDefaultIndexManager(); + var indexManager = context.GetDefaultIndexManager(); await WaitForWorkspaceAddedAsync(indexManager); - indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("y = 1")); + indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("r = 1")); // It Needs to remake the stream for the file, previous one is closed - _fileSystem.FileOpen(pythonTestFilePath, FileMode.Open).Returns(MakeStream("x = 1")); - _fileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); + context.FileSystem.FileExists(pythonTestFilePath).Returns(true); + context.FileSystem.FileOpen(pythonTestFilePath, FileMode.Open).Returns(MakeStream("x = 1")); + context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); indexManager.ProcessClosedFile(pythonTestFilePath); var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); @@ -173,16 +157,15 @@ public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() { // If events get to index manager in the order: [open, close, update] // it should not reindex file + var context = new IndexTestContext(this); var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); - var doc = DocumentWithAst("x = 1"); - var indexManager = GetDefaultIndexManager(); - indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); + var indexManager = context.GetDefaultIndexManager(); + indexManager.ProcessNewFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); indexManager.ProcessClosedFile(pythonTestFileInfo.FullName); - doc = DocumentWithAst("x = 1"); - - _fileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false); - indexManager.ReIndexFile(pythonTestFileInfo.FullName, doc); + + context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false); + indexManager.ReIndexFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); await SymbolIndexShouldBeEmpty(indexManager); } @@ -195,38 +178,40 @@ private async Task SymbolIndexShouldBeEmpty(IIndexManager indexManager) { var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); symbols.Should().HaveCount(0); } - - + + [TestMethod, Priority(0)] public async Task DisposeManagerCancelsTaskAsync() { - var pythonTestFilePath = FileWithXVarInRootDir(); + var context = new IndexTestContext(this); + var pythonTestFilePath = context.FileWithXVarInRootDir(); ManualResetEventSlim neverSignaledEvent = new ManualResetEventSlim(false); ManualResetEventSlim fileOpenedEvent = new ManualResetEventSlim(false); - _fileSystem.FileOpen(pythonTestFilePath, FileMode.Open).Returns(_ => { + context.FileSystem.FileOpen(pythonTestFilePath, FileMode.Open).Returns(_ => { fileOpenedEvent.Set(); // Wait forever neverSignaledEvent.Wait(); throw new InternalTestFailureException("Task should have been cancelled"); }); - var indexManager = GetDefaultIndexManager(); + var indexManager = context.GetDefaultIndexManager(); fileOpenedEvent.Wait(); + var t = WaitForWorkspaceAddedAsync(indexManager); indexManager.Dispose(); - Func add = async () => { - await WaitForWorkspaceAddedAsync(indexManager); - }; - - add.Should().Throw(); - await SymbolIndexShouldBeEmpty(indexManager); + await t; + + neverSignaledEvent.IsSet.Should().BeFalse(); + context.SymbolIndex.WorkspaceSymbols("").Should().HaveCount(0); } [TestMethod, Priority(0)] public async Task WorkspaceSymbolsAddsRootDirectory() { - var pythonTestFilePath = FileWithXVarInRootDir(); + var context = new IndexTestContext(this); - var indexManager = GetDefaultIndexManager(); + var pythonTestFilePath = context.FileWithXVarInRootDir(); + + var indexManager = context.GetDefaultIndexManager(); var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); SymbolsShouldBeOnlyX(symbols); @@ -234,10 +219,12 @@ public async Task WorkspaceSymbolsAddsRootDirectory() { [TestMethod, Priority(0)] public async Task WorkspaceSymbolsLimited() { + var context = new IndexTestContext(this); + for (int fileNumber = 0; fileNumber < 10; fileNumber++) { - AddFileToRoot($"{_rootPath}\bla{fileNumber}.py", MakeStream($"x{fileNumber} = 1")); + context.AddFileToRoot($"{_rootPath}\bla{fileNumber}.py", MakeStream($"x{fileNumber} = 1")); } - var indexManager = GetDefaultIndexManager(); + var indexManager = context.GetDefaultIndexManager(); const int amountOfSymbols = 3; @@ -247,9 +234,10 @@ public async Task WorkspaceSymbolsLimited() { [TestMethod, Priority(0)] public async Task HierarchicalDocumentSymbolsAsync() { - var pythonTestFilePath = FileWithXVarInRootDir(); + var context = new IndexTestContext(this); + var pythonTestFilePath = context.FileWithXVarInRootDir(); - var indexManager = GetDefaultIndexManager(); + var indexManager = context.GetDefaultIndexManager(); var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath); SymbolsShouldBeOnlyX(symbols); @@ -259,24 +247,19 @@ public async Task HierarchicalDocumentSymbolsAsync() { public async Task LatestVersionASTVersionIsIndexed() { ManualResetEventSlim reOpenedFileFinished = new ManualResetEventSlim(false); ManualResetEventSlim fileOpenedEvent = new ManualResetEventSlim(false); - - var pythonTestFilePath = FileWithXVarInRootDir(); - _fileSystem.FileOpen(pythonTestFilePath, FileMode.Open).Returns(_ => { + var context = new IndexTestContext(this); + var pythonTestFilePath = context.FileWithXVarInRootDir(); + context.FileSystem.FileOpen(pythonTestFilePath, FileMode.Open).Returns(_ => { fileOpenedEvent.Set(); - // Wait forever reOpenedFileFinished.Wait(); return MakeStream("x = 1"); }); - var indexManager = GetDefaultIndexManager(); - - IDocument yVarDoc = DocumentWithAst("y = 1"); - IDocument zVarDoc = DocumentWithAst("z = 1"); - - indexManager.ProcessNewFile(pythonTestFilePath, yVarDoc); + var indexManager = context.GetDefaultIndexManager(); + indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("y = 1")); indexManager.ProcessClosedFile(pythonTestFilePath); fileOpenedEvent.Wait(); - indexManager.ProcessNewFile(pythonTestFilePath, zVarDoc); + indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("z = 1")); reOpenedFileFinished.Set(); var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath); @@ -287,15 +270,18 @@ public async Task LatestVersionASTVersionIsIndexed() { [TestMethod, Priority(0)] public async Task AddFilesToPendingChanges() { - var f1 = AddFileToRoot($"{_rootPath}/fileA.py", MakeStream("")); - var f2 = AddFileToRoot($"{_rootPath}/fileB.py", MakeStream("")); + var context = new IndexTestContext(this); + var f1 = context.AddFileToRoot($"{_rootPath}/fileA.py", MakeStream("")); + var f2 = context.AddFileToRoot($"{_rootPath}/fileB.py", MakeStream("")); - var indexManager = GetDefaultIndexManager(); + var indexManager = context.GetDefaultIndexManager(); await WaitForWorkspaceAddedAsync(indexManager); indexManager.AddPendingDoc(DocumentWithAst("y = 1", f1)); indexManager.AddPendingDoc(DocumentWithAst("x = 1", f2)); - _idleTimeService.Idle += Raise.Event(); + + context.SetIdleEvent(Raise.Event()); + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); symbols.Should().HaveCount(2); } @@ -312,44 +298,94 @@ private static void SymbolsShouldBeOnlyX(IEnumerable symbols) { symbols.First().Name.Should().BeEquivalentTo("x"); } - private PythonAst MakeAst(string testCode) { - return Parser.CreateParser(MakeStream(testCode), PythonVersions.LatestAvailable3X.Version.ToLanguageVersion()).ParseFile(); - } - private FileInfoProxy MakeFileInfoProxy(string filePath) - => new FileInfoProxy(new FileInfo(filePath)); + private class IndexTestContext : IDisposable { + private List _rootFileList; + private IIdleTimeService _idleTimeService; + private IIndexManager _indexM; + private IndexManagerTests _tests; + private readonly PythonLanguageVersion _pythonLanguageVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); - private void AddFileInfoToRootTestFS(FileInfoProxy fileInfo, List_rootFileList, IFileSystem fileSystem) { - _rootFileList.Add(fileInfo); - _fileSystem.FileExists(fileInfo.FullName).Returns(true); - } + public IndexTestContext(IndexManagerTests tests) { + _tests = tests; + Setup(); + } - private Stream MakeStream(string str) { - return new MemoryStream(Encoding.UTF8.GetBytes(str)); + public IFileSystem FileSystem { get; private set; } + public SymbolIndex SymbolIndex { get; private set; } + + public void AddFileInfoToRootTestFS(FileInfoProxy fileInfo) { + _rootFileList.Add(fileInfo); + FileSystem.FileExists(fileInfo.FullName).Returns(true); + } + + public string FileWithXVarInRootDir() { + return AddFileToRoot($"{_rootPath}\bla.py", _tests.MakeStream("x = 1")); + } + + public IIndexManager GetDefaultIndexManager() { + _indexM = new IndexManager(SymbolIndex, FileSystem, _pythonLanguageVersion, + _rootPath, new string[] { }, new string[] { }, + _idleTimeService) { + ReIndexingDelay = 1 + }; + + return _indexM; + } + + public string AddFileToRoot(string filePath, Stream stream) { + var fileInfo = _tests.MakeFileInfoProxy(filePath); + AddFileInfoToRootTestFS(fileInfo); + string fullName = fileInfo.FullName; + FileSystem.FileOpen(fullName, FileMode.Open).Returns(stream); + // FileInfo fullName is used everywhere as path + // Otherwise, path discrepancies might appear + return fullName; + } + + public void SetIdleEvent(EventHandler handler) { + _idleTimeService.Idle += handler; + } + + private void Setup() { + FileSystem = Substitute.For(); + SymbolIndex = new SymbolIndex(); + _idleTimeService = Substitute.For(); + _rootFileList = new List(); + + SetupRootDir(); + } + + private void SetupRootDir() { + IDirectoryInfo directoryInfo = Substitute.For(); + // Doesn't work without 'forAnyArgs' + directoryInfo.EnumerateFileSystemInfos(new string[] { }, new string[] { }).ReturnsForAnyArgs(_rootFileList); + FileSystem.GetDirectoryInfo(_rootPath).Returns(directoryInfo); + } + + public void Dispose() { + _indexM?.Dispose(); + } } - private string FileWithXVarInRootDir(IFileSystem fileSystem) { - return AddFileToRoot($"{_rootPath}\bla.py", MakeStream("x = 1"), fileSystem); + private IDocument DocumentWithAst(string testCode, string filePath = null) { + filePath = filePath ?? $"{_rootPath}/{testCode}.py"; + IDocument doc = Substitute.For(); + doc.GetAstAsync().ReturnsForAnyArgs(Task.FromResult(MakeAst(testCode))); + doc.Uri.Returns(new Uri(filePath)); + return doc; } - private IIndexManager GetDefaultIndexManager() { - var _rootFileList = SetupRootDir(_fileSystem); - var indexM = new IndexManager(new SymbolIndex(), _fileSystem, _pythonLanguageVersion, - _rootPath, new string[] { }, new string[] { }, - _idleTimeService) { - ReIndexingDelay = 1 - }; - return indexM; + public PythonAst MakeAst(string testCode) { + PythonLanguageVersion latestVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); + return Parser.CreateParser(MakeStream(testCode), latestVersion).ParseFile(); } - private string AddFileToRoot(string filePath, Stream stream, List rootFileList, IFileSystem fileSystem) { - var fileInfo = MakeFileInfoProxy(filePath); - AddFileInfoToRootTestFS(fileInfo, rootFileList, fileSystem); - string fullName = fileInfo.FullName; - _fileSystem.FileOpen(fullName, FileMode.Open).Returns(stream); - // FileInfo fullName is used everywhere as path - // Otherwise, path discrepancies might appear - return fullName; + public Stream MakeStream(string str) { + return new MemoryStream(Encoding.UTF8.GetBytes(str)); } + + public FileInfoProxy MakeFileInfoProxy(string filePath) + => new FileInfoProxy(new FileInfo(filePath)); } } diff --git a/src/LanguageServer/Test/SymbolIndexTests.cs b/src/LanguageServer/Test/SymbolIndexTests.cs index b3b56d705..1a66fee65 100644 --- a/src/LanguageServer/Test/SymbolIndexTests.cs +++ b/src/LanguageServer/Test/SymbolIndexTests.cs @@ -616,7 +616,7 @@ public void IndexHierarchicalDocument() { ISymbolIndex index = new SymbolIndex(); var path = TestData.GetDefaultModulePath(); var ast = GetParse("x = 1"); - index.Update(path, ast); + index.Add(path, ast); var symbols = index.HierarchicalDocumentSymbols(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { @@ -630,7 +630,7 @@ public void IndexHierarchicalDocumentUpdate() { var path = TestData.GetDefaultModulePath(); var ast = GetParse("x = 1"); - index.Update(path, ast); + index.Add(path, ast); var symbols = index.HierarchicalDocumentSymbols(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { @@ -638,7 +638,7 @@ public void IndexHierarchicalDocumentUpdate() { }); ast = GetParse("y = 1"); - index.Update(path, ast); + index.Add(path, ast); symbols = index.HierarchicalDocumentSymbols(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { @@ -664,7 +664,7 @@ public void IndexWorkspaceSymbolsFlatten() { var path = TestData.GetDefaultModulePath(); var ast = GetParse(code); - index.Update(path, ast); + index.Add(path, ast); var symbols = index.WorkspaceSymbols(""); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { @@ -684,7 +684,7 @@ public void IndexWorkspaceSymbolsFiltered() { var path = TestData.GetDefaultModulePath(); var ast = GetParse(code); - index.Update(path, ast); + index.Add(path, ast); var symbols = index.WorkspaceSymbols("x"); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { @@ -710,7 +710,7 @@ public void IndexWorkspaceSymbolsCaseInsensitive() { var path = TestData.GetDefaultModulePath(); var ast = GetParse(code); - index.Update(path, ast); + index.Add(path, ast); var symbols = index.WorkspaceSymbols("foo"); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { From 59bccd29510f724886193ad3ad702738b253a1cb Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 15 Feb 2019 14:24:57 -0800 Subject: [PATCH 064/123] Working version --- .../Impl/Indexing/IndexManager.cs | 1 + .../Impl/Indexing/MostRecentSymbols.cs | 36 ++++++++++--------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index ad45b84da..797a38620 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -139,6 +139,7 @@ public void AddPendingDoc(IDocument doc) { lock (_pendingDocs) { _lastPendingDocAddedTime = DateTime.Now; _pendingDocs.Add(doc); + _files[doc.Uri.AbsolutePath].MarkAsPending(); } } diff --git a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs index 701cc2ce7..41e0d40fe 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs @@ -27,13 +27,13 @@ public MostRecentDocumentSymbols(string path, IIndexParser indexParser, ISymbolI public void Parse() { lock (_syncObj) { CancelExistingTask(); - _fileTask = ParseAsync(); + _fileTask = ParseAsync(_fileCts.Token); } } - private async Task ParseAsync() { - var ast = await _indexParser.ParseAsync(_path, _fileCts.Token); - _fileCts.Token.ThrowIfCancellationRequested(); + private async Task ParseAsync(CancellationToken cancellationToken) { + var ast = await _indexParser.ParseAsync(_path, cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); lock (_syncObj) { SetFileTcsResult(); } @@ -57,13 +57,13 @@ public void Process(PythonAst ast) { public void Add(IDocument doc) { lock (_syncObj) { CancelExistingTask(); - _fileTask = AddAsync(doc); + _fileTask = AddAsync(doc, _fileCts.Token); } } - private async Task AddAsync(IDocument doc) { - var ast = await doc.GetAstAsync(_fileCts.Token); - _fileCts.Token.ThrowIfCancellationRequested(); + private async Task AddAsync(IDocument doc, CancellationToken cancellationToken) { + var ast = await doc.GetAstAsync(cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); lock (_syncObj) { _symbolIndex.Add(_path, ast); SetFileTcsResult(); @@ -74,10 +74,7 @@ private async Task AddAsync(IDocument doc) { private void CancelExistingTask() { if (_fileTask != null) { - if (_fileTcs.Task.IsCompleted) { - _fileTcs.TrySetCanceled(); - _fileTcs = new TaskCompletionSource>(); - } + _fileTcs = new TaskCompletionSource>(); _fileCts.Cancel(); _fileCts.Dispose(); @@ -90,19 +87,26 @@ private void CancelExistingTask() { public void ReIndex(IDocument doc) { lock (_syncObj) { CancelExistingTask(); - _fileTask = ReIndexAsync(doc); + _fileTask = ReIndexAsync(doc, _fileCts.Token); } } - private async Task ReIndexAsync(IDocument doc) { - var ast = await doc.GetAstAsync(_fileCts.Token); - _fileCts.Token.ThrowIfCancellationRequested(); + private async Task ReIndexAsync(IDocument doc, CancellationToken cancellationToken) { + var ast = await doc.GetAstAsync(cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); lock (_syncObj) { _symbolIndex.Update(_path, ast); SetFileTcsResult(); } } + public void MarkAsPending() { + lock (_syncObj) { + CancelExistingTask(); + } + + } + private void SetFileTcsResult() { _fileTcs.TrySetResult(_symbolIndex.HierarchicalDocumentSymbols(_path)); } From 0727619f61c639ea4ce89922a0d9a1d577b1e201 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 15 Feb 2019 15:13:21 -0800 Subject: [PATCH 065/123] Readonly fields --- .../Impl/Indexing/IndexManager.cs | 15 +++---- .../Impl/Indexing/MostRecentSymbols.cs | 6 +-- src/LanguageServer/Test/IndexManagerTests.cs | 45 +++++++++---------- 3 files changed, 28 insertions(+), 38 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index 797a38620..fa5dc8c70 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -29,17 +29,18 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal class IndexManager : IIndexManager { + private static int DefaultReIndexDelay = 1000; private readonly ISymbolIndex _symbolIndex; private readonly IFileSystem _fileSystem; private readonly IndexParser _indexParser; private readonly string _workspaceRootPath; private readonly string[] _includeFiles; private readonly string[] _excludeFiles; - private readonly CancellationTokenSource _allIndexCts = new CancellationTokenSource(); - private readonly TaskCompletionSource _addRootTcs; private readonly IIdleTimeService _idleTimeService; - private readonly ConcurrentDictionary _files; - private HashSet _pendingDocs; + private readonly CancellationTokenSource _allIndexCts = new CancellationTokenSource(); + private readonly TaskCompletionSource _addRootTcs = new TaskCompletionSource(); + private readonly ConcurrentDictionary _files = new ConcurrentDictionary(PathEqualityComparer.Instance); + private readonly HashSet _pendingDocs = new HashSet(new UriDocumentComparer()); private DateTime _lastPendingDocAddedTime; public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, @@ -56,13 +57,9 @@ public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLang _includeFiles = includeFiles; _excludeFiles = excludeFiles; _idleTimeService = idleTimeService; - _addRootTcs = new TaskCompletionSource(); _idleTimeService.Idle += OnIdle; - _pendingDocs = new HashSet(new UriDocumentComparer()); - _files = new ConcurrentDictionary(PathEqualityComparer.Instance); - ReIndexingDelay = 1000; + ReIndexingDelay = DefaultReIndexDelay; StartAddRootDir(); - } public int ReIndexingDelay { get; set; } diff --git a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs index 41e0d40fe..c11f30d5f 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs @@ -12,16 +12,14 @@ class MostRecentDocumentSymbols : IDisposable { private readonly ISymbolIndex _symbolIndex; private readonly string _path; - private CancellationTokenSource _fileCts; + private CancellationTokenSource _fileCts = new CancellationTokenSource(); private Task _fileTask; - private TaskCompletionSource> _fileTcs; + private TaskCompletionSource> _fileTcs = new TaskCompletionSource>(); public MostRecentDocumentSymbols(string path, IIndexParser indexParser, ISymbolIndex symbolIndex) { _path = path; _indexParser = indexParser; _symbolIndex = symbolIndex; - _fileCts = new CancellationTokenSource(); - _fileTcs = new TaskCompletionSource>(); } public void Parse() { diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index db86f703a..7674b70c6 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -36,7 +36,7 @@ namespace Microsoft.Python.LanguageServer.Tests { [TestClass] public class IndexManagerTests : LanguageServerTestBase { - private const int maxSymbolsCount = 1000; + private readonly int maxSymbolsCount = 1000; private const string _rootPath = "C:/root"; public TestContext TestContext { get; set; } @@ -122,11 +122,10 @@ public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { var context = new IndexTestContext(this); var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); - IDocument doc = DocumentWithAst("x = 1"); context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false); var indexManager = context.GetDefaultIndexManager(); - indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); + indexManager.ProcessNewFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); indexManager.ProcessClosedFile(pythonTestFileInfo.FullName); await SymbolIndexShouldBeEmpty(indexManager); @@ -170,16 +169,6 @@ public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() { await SymbolIndexShouldBeEmpty(indexManager); } - private static async Task WaitForWorkspaceAddedAsync(IIndexManager indexManager) { - await indexManager.WorkspaceSymbolsAsync("", 1000); - } - - private async Task SymbolIndexShouldBeEmpty(IIndexManager indexManager) { - var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); - symbols.Should().HaveCount(0); - } - - [TestMethod, Priority(0)] public async Task DisposeManagerCancelsTaskAsync() { var context = new IndexTestContext(this); @@ -300,17 +289,23 @@ private static void SymbolsShouldBeOnlyX(IEnumerable symbols) { private class IndexTestContext : IDisposable { - private List _rootFileList; - private IIdleTimeService _idleTimeService; + private readonly List _rootFileList = new List(); + private readonly IIdleTimeService _idleTimeService = Substitute.For(); + private readonly PythonLanguageVersion _pythonLanguageVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); private IIndexManager _indexM; private IndexManagerTests _tests; - private readonly PythonLanguageVersion _pythonLanguageVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); public IndexTestContext(IndexManagerTests tests) { _tests = tests; Setup(); } + private void Setup() { + FileSystem = Substitute.For(); + SymbolIndex = new SymbolIndex(); + SetupRootDir(); + } + public IFileSystem FileSystem { get; private set; } public SymbolIndex SymbolIndex { get; private set; } @@ -347,15 +342,6 @@ public void SetIdleEvent(EventHandler handler) { _idleTimeService.Idle += handler; } - private void Setup() { - FileSystem = Substitute.For(); - SymbolIndex = new SymbolIndex(); - _idleTimeService = Substitute.For(); - _rootFileList = new List(); - - SetupRootDir(); - } - private void SetupRootDir() { IDirectoryInfo directoryInfo = Substitute.For(); // Doesn't work without 'forAnyArgs' @@ -376,6 +362,15 @@ private IDocument DocumentWithAst(string testCode, string filePath = null) { return doc; } + private static async Task WaitForWorkspaceAddedAsync(IIndexManager indexManager) { + await indexManager.WorkspaceSymbolsAsync("", 1000); + } + + private async Task SymbolIndexShouldBeEmpty(IIndexManager indexManager) { + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + symbols.Should().HaveCount(0); + } + public PythonAst MakeAst(string testCode) { PythonLanguageVersion latestVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); return Parser.CreateParser(MakeStream(testCode), latestVersion).ParseFile(); From dc7b9eb784d51c7e5b2757d89fe5cea1ba5f17cf Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 15 Feb 2019 16:27:41 -0800 Subject: [PATCH 066/123] IndexParser refactoring --- .../Impl/Indexing/IIndexParser.cs | 2 +- .../Impl/Indexing/IndexManager.cs | 7 +- .../Impl/Indexing/IndexParser.cs | 70 ++++++++++++++----- .../Impl/Indexing/MostRecentSymbols.cs | 11 +-- src/LanguageServer/Test/IndexParserTests.cs | 27 ++++--- 5 files changed, 76 insertions(+), 41 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/IIndexParser.cs b/src/LanguageServer/Impl/Indexing/IIndexParser.cs index 20f0d8c8c..e8d9c5dae 100644 --- a/src/LanguageServer/Impl/Indexing/IIndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IIndexParser.cs @@ -19,6 +19,6 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal interface IIndexParser : IDisposable { - Task ParseAsync(string path, CancellationToken cancellationToken = default); + Task ParseAsync(string path, CancellationToken cancellationToken = default); } } diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index fa5dc8c70..af08756bb 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -32,7 +32,6 @@ internal class IndexManager : IIndexManager { private static int DefaultReIndexDelay = 1000; private readonly ISymbolIndex _symbolIndex; private readonly IFileSystem _fileSystem; - private readonly IndexParser _indexParser; private readonly string _workspaceRootPath; private readonly string[] _includeFiles; private readonly string[] _excludeFiles; @@ -41,6 +40,7 @@ internal class IndexManager : IIndexManager { private readonly TaskCompletionSource _addRootTcs = new TaskCompletionSource(); private readonly ConcurrentDictionary _files = new ConcurrentDictionary(PathEqualityComparer.Instance); private readonly HashSet _pendingDocs = new HashSet(new UriDocumentComparer()); + private readonly PythonLanguageVersion _version; private DateTime _lastPendingDocAddedTime; public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, @@ -52,12 +52,12 @@ public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLang _symbolIndex = symbolIndex; _fileSystem = fileSystem; - _indexParser = new IndexParser(symbolIndex, fileSystem, version); _workspaceRootPath = rootPath; _includeFiles = includeFiles; _excludeFiles = excludeFiles; _idleTimeService = idleTimeService; _idleTimeService.Idle += OnIdle; + _version = version; ReIndexingDelay = DefaultReIndexDelay; StartAddRootDir(); } @@ -113,7 +113,6 @@ public void Dispose() { foreach (var mostRecentSymbols in _files.Values) { mostRecentSymbols.Dispose(); } - _indexParser.Dispose(); _allIndexCts.Cancel(); _allIndexCts.Dispose(); } @@ -159,7 +158,7 @@ private void ReIndexPendingDocsAsync() { } private MostRecentDocumentSymbols MakeMostRecentFileSymbols(string path) { - return new MostRecentDocumentSymbols(path, _indexParser, _symbolIndex); + return new MostRecentDocumentSymbols(path, _symbolIndex, _fileSystem, _version); } private class UriDocumentComparer : IEqualityComparer { diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index 073cdc006..b2ec01641 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -27,6 +27,10 @@ internal sealed class IndexParser : IIndexParser { private readonly IFileSystem _fileSystem; private readonly PythonLanguageVersion _version; private readonly CancellationTokenSource _allProcessingCts = new CancellationTokenSource(); + private readonly object _syncObj = new object(); + private CancellationTokenSource _linkedParseCts; + private Task _parseTask; + private TaskCompletionSource _tcs; public IndexParser(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version) { Check.ArgumentNotNull(nameof(symbolIndex), symbolIndex); @@ -37,34 +41,64 @@ public IndexParser(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLangu _version = version; } - public void Dispose() { - _allProcessingCts.Cancel(); - _allProcessingCts.Dispose(); + public Task ParseAsync(string path, CancellationToken cancellationToken = default) { + lock (_syncObj) { + CancelCurrentParse(); + _tcs = new TaskCompletionSource(); + _linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, cancellationToken); + _parseTask = Task.Run(() => Parse(path, _linkedParseCts)); + return _tcs.Task; + } } - public Task ParseAsync(string path, CancellationToken parseCancellationToken = default) { - var linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, parseCancellationToken); - var linkedParseToken = linkedParseCts.Token; - return Task.Run((System.Func)(() => { - if (!_fileSystem.FileExists(path)) { - return false; - } + private void CancelCurrentParse() { + _linkedParseCts?.Cancel(); + _linkedParseCts?.Dispose(); + _linkedParseCts = null; + _parseTask = null; + } + + private void Parse(string path, CancellationTokenSource parseCts) { + if (parseCts.Token.IsCancellationRequested) { + _tcs.SetCanceled(); + parseCts.Token.ThrowIfCancellationRequested(); + } + if (_fileSystem.FileExists(path)) { try { - linkedParseToken.ThrowIfCancellationRequested(); + if (parseCts.Token.IsCancellationRequested) { + _tcs.SetCanceled(); + parseCts.Token.ThrowIfCancellationRequested(); + } using (var stream = _fileSystem.FileOpen(path, FileMode.Open)) { var parser = Parser.CreateParser(stream, _version); - linkedParseToken.ThrowIfCancellationRequested(); _symbolIndex.Add(path, parser.ParseFile()); - return true; } } catch (FileNotFoundException e) { Trace.TraceError(e.Message); - return false; } - })).ContinueWith((Task task) => { - linkedParseCts.Dispose(); - return task.Result; - }, linkedParseToken); + } + lock (_syncObj) { + if (_linkedParseCts == parseCts) { + _linkedParseCts.Dispose(); + _linkedParseCts = null; + _tcs.SetResult(new object()); + } + } + } + + public void Dispose() { + lock (_syncObj) { + _tcs?.TrySetCanceled(); + _tcs = null; + _allProcessingCts.Cancel(); + _allProcessingCts.Dispose(); + + _linkedParseCts?.Cancel(); + _linkedParseCts?.Dispose(); + _linkedParseCts = null; + + _parseTask = null; + } } } } diff --git a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs index c11f30d5f..b13eb4fde 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs @@ -3,6 +3,8 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Core.IO; +using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.LanguageServer.Indexing { @@ -16,10 +18,10 @@ class MostRecentDocumentSymbols : IDisposable { private Task _fileTask; private TaskCompletionSource> _fileTcs = new TaskCompletionSource>(); - public MostRecentDocumentSymbols(string path, IIndexParser indexParser, ISymbolIndex symbolIndex) { + public MostRecentDocumentSymbols(string path, ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version) { _path = path; - _indexParser = indexParser; _symbolIndex = symbolIndex; + _indexParser = new IndexParser(_symbolIndex, fileSystem, version); } public void Parse() { @@ -30,12 +32,11 @@ public void Parse() { } private async Task ParseAsync(CancellationToken cancellationToken) { - var ast = await _indexParser.ParseAsync(_path, cancellationToken); + await _indexParser.ParseAsync(_path, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); lock (_syncObj) { SetFileTcsResult(); } - } public void Delete() { @@ -115,6 +116,8 @@ public void Dispose() { _fileCts?.Dispose(); _fileCts = null; _fileTask = null; + + _indexParser.Dispose(); } } } diff --git a/src/LanguageServer/Test/IndexParserTests.cs b/src/LanguageServer/Test/IndexParserTests.cs index 48415d880..93f6d688b 100644 --- a/src/LanguageServer/Test/IndexParserTests.cs +++ b/src/LanguageServer/Test/IndexParserTests.cs @@ -73,36 +73,32 @@ public async Task ParseVariableInFileAsync() { } [TestMethod, Priority(0)] - public void ParseNonexistentFile() { + public async Task ParseNonexistentFile() { const string testFilePath = "C:/bla.py"; _fileSystem.FileExists(testFilePath).Returns(false); IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); - var t = indexParser.ParseAsync(testFilePath); - t.Wait(); + await indexParser.ParseAsync(testFilePath); - t.Result.Should().BeFalse(); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(0); } [TestMethod, Priority(0)] - public void ParseFileThatStopsExisting() { + public async Task ParseFileThatStopsExisting() { const string testFilePath = "C:/bla.py"; _fileSystem.FileExists(testFilePath).Returns(true); _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(_ => throw new FileNotFoundException()); IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); - var t = indexParser.ParseAsync(testFilePath); - t.Wait(); + await indexParser.ParseAsync(testFilePath); - t.Result.Should().BeFalse(); var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(0); } [TestMethod, Priority(0)] - public void CancellParsing() { + public void CancellParsingAsync() { const string testFilePath = "C:/bla.py"; _fileSystem.FileExists(testFilePath).Returns(true); _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(MakeStream("x = 1")); @@ -110,11 +106,12 @@ public void CancellParsing() { IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); cancellationTokenSource.Cancel(); + Func parse = async () => { await indexParser.ParseAsync(testFilePath, cancellationTokenSource.Token); }; - parse.Should().Throw(); + var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(0); } @@ -133,14 +130,16 @@ public void DisposeParserCancelsParsing() { }); IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); + + var parseTask = indexParser.ParseAsync(testFilePath); + fileOpenedEvent.Wait(); + indexParser.Dispose(); Func parse = async () => { - var t = indexParser.ParseAsync(testFilePath); - fileOpenedEvent.Wait(); - indexParser.Dispose(); - await t; + await parseTask; }; parse.Should().Throw(); + var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(0); } From 935f6bac51f111edf722f6b718f7b13b8bc50cb1 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 15 Feb 2019 17:06:15 -0800 Subject: [PATCH 067/123] IndexParser fixes --- src/Core/Impl/IO/FileSystem.cs | 3 ++- src/Core/Impl/IO/IFileSystem.cs | 1 + .../Impl/Indexing/IndexParser.cs | 19 +++++++------------ src/LanguageServer/Test/IndexManagerTests.cs | 19 +++++++++++++------ src/LanguageServer/Test/IndexParserTests.cs | 18 +++++++++++++----- 5 files changed, 36 insertions(+), 24 deletions(-) diff --git a/src/Core/Impl/IO/FileSystem.cs b/src/Core/Impl/IO/FileSystem.cs index ad865dfaf..b78f441d4 100644 --- a/src/Core/Impl/IO/FileSystem.cs +++ b/src/Core/Impl/IO/FileSystem.cs @@ -31,7 +31,8 @@ public long FileSize(string path) { public byte[] FileReadAllBytes(string path) => File.ReadAllBytes(path); public void FileWriteAllBytes(string path, byte[] bytes) => File.WriteAllBytes(path, bytes); public Stream CreateFile(string path) => File.Create(path); - public Stream FileOpen(string path, FileMode mode) => File.Open(path, mode); + public Stream FileOpen(string path, FileMode mode) => FileOpen(path, mode); + public Stream FileOpen(string path, FileMode mode, FileAccess access, FileShare share) => File.Open(path, mode, access, share); public bool DirectoryExists(string path) => Directory.Exists(path); public FileAttributes GetFileAttributes(string path) => File.GetAttributes(path); public void SetFileAttributes(string fullPath, FileAttributes attributes) => File.SetAttributes(fullPath, attributes); diff --git a/src/Core/Impl/IO/IFileSystem.cs b/src/Core/Impl/IO/IFileSystem.cs index ecdb55065..509c7c282 100644 --- a/src/Core/Impl/IO/IFileSystem.cs +++ b/src/Core/Impl/IO/IFileSystem.cs @@ -41,6 +41,7 @@ public interface IFileSystem { Stream CreateFile(string path); Stream FileOpen(string path, FileMode mode); + Stream FileOpen(string path, FileMode mode, FileAccess access, FileShare share); Version GetFileVersion(string path); void DeleteFile(string path); diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index b2ec01641..45c1a0745 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -13,6 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Diagnostics; using System.IO; using System.Threading; @@ -63,19 +64,13 @@ private void Parse(string path, CancellationTokenSource parseCts) { _tcs.SetCanceled(); parseCts.Token.ThrowIfCancellationRequested(); } - if (_fileSystem.FileExists(path)) { - try { - if (parseCts.Token.IsCancellationRequested) { - _tcs.SetCanceled(); - parseCts.Token.ThrowIfCancellationRequested(); - } - using (var stream = _fileSystem.FileOpen(path, FileMode.Open)) { - var parser = Parser.CreateParser(stream, _version); - _symbolIndex.Add(path, parser.ParseFile()); - } - } catch (FileNotFoundException e) { - Trace.TraceError(e.Message); + try { + using (var stream = _fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read)) { + var parser = Parser.CreateParser(stream, _version); + _symbolIndex.Add(path, parser.ParseFile()); } + } catch (Exception e) when (e is IOException || e is UnauthorizedAccessException) { + Trace.TraceError(e.Message); } lock (_syncObj) { if (_linkedParseCts == parseCts) { diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index 7674b70c6..7730aa3f3 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -143,7 +143,7 @@ public async Task CloseWorkspaceFilesReUpdatesIndexAsync() { indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("r = 1")); // It Needs to remake the stream for the file, previous one is closed context.FileSystem.FileExists(pythonTestFilePath).Returns(true); - context.FileSystem.FileOpen(pythonTestFilePath, FileMode.Open).Returns(MakeStream("x = 1")); + context.SetFileOpen(pythonTestFilePath, MakeStream("x = 1")); context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); indexManager.ProcessClosedFile(pythonTestFilePath); @@ -176,7 +176,7 @@ public async Task DisposeManagerCancelsTaskAsync() { ManualResetEventSlim neverSignaledEvent = new ManualResetEventSlim(false); ManualResetEventSlim fileOpenedEvent = new ManualResetEventSlim(false); - context.FileSystem.FileOpen(pythonTestFilePath, FileMode.Open).Returns(_ => { + context.SetFileOpen(pythonTestFilePath, _ => { fileOpenedEvent.Set(); // Wait forever neverSignaledEvent.Wait(); @@ -238,7 +238,7 @@ public async Task LatestVersionASTVersionIsIndexed() { ManualResetEventSlim fileOpenedEvent = new ManualResetEventSlim(false); var context = new IndexTestContext(this); var pythonTestFilePath = context.FileWithXVarInRootDir(); - context.FileSystem.FileOpen(pythonTestFilePath, FileMode.Open).Returns(_ => { + context.SetFileOpen(pythonTestFilePath, _ => { fileOpenedEvent.Set(); reOpenedFileFinished.Wait(); return MakeStream("x = 1"); @@ -331,11 +331,10 @@ public IIndexManager GetDefaultIndexManager() { public string AddFileToRoot(string filePath, Stream stream) { var fileInfo = _tests.MakeFileInfoProxy(filePath); AddFileInfoToRootTestFS(fileInfo); - string fullName = fileInfo.FullName; - FileSystem.FileOpen(fullName, FileMode.Open).Returns(stream); + SetFileOpen(fileInfo.FullName, stream); // FileInfo fullName is used everywhere as path // Otherwise, path discrepancies might appear - return fullName; + return fileInfo.FullName; } public void SetIdleEvent(EventHandler handler) { @@ -352,6 +351,14 @@ private void SetupRootDir() { public void Dispose() { _indexM?.Dispose(); } + + public void SetFileOpen(string pythonTestFilePath, Func returnFunc) { + FileSystem.FileOpen(pythonTestFilePath, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(returnFunc); + } + + internal void SetFileOpen(string path, Stream stream) { + FileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(stream); + } } private IDocument DocumentWithAst(string testCode, string filePath = null) { diff --git a/src/LanguageServer/Test/IndexParserTests.cs b/src/LanguageServer/Test/IndexParserTests.cs index 93f6d688b..36629dcfa 100644 --- a/src/LanguageServer/Test/IndexParserTests.cs +++ b/src/LanguageServer/Test/IndexParserTests.cs @@ -61,7 +61,7 @@ public void NullIndexThrowsException() { public async Task ParseVariableInFileAsync() { const string testFilePath = "C:/bla.py"; _fileSystem.FileExists(testFilePath).Returns(true); - _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(MakeStream("x = 1")); + SetFileOpen(_fileSystem, testFilePath, MakeStream("x = 1")); IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); await indexParser.ParseAsync(testFilePath); @@ -75,7 +75,7 @@ public async Task ParseVariableInFileAsync() { [TestMethod, Priority(0)] public async Task ParseNonexistentFile() { const string testFilePath = "C:/bla.py"; - _fileSystem.FileExists(testFilePath).Returns(false); + SetFileOpen(_fileSystem, testFilePath, _ => throw new FileNotFoundException()); IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); await indexParser.ParseAsync(testFilePath); @@ -88,7 +88,7 @@ public async Task ParseNonexistentFile() { public async Task ParseFileThatStopsExisting() { const string testFilePath = "C:/bla.py"; _fileSystem.FileExists(testFilePath).Returns(true); - _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(_ => throw new FileNotFoundException()); + SetFileOpen(_fileSystem, testFilePath, _ => throw new FileNotFoundException()); IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); await indexParser.ParseAsync(testFilePath); @@ -101,7 +101,7 @@ public async Task ParseFileThatStopsExisting() { public void CancellParsingAsync() { const string testFilePath = "C:/bla.py"; _fileSystem.FileExists(testFilePath).Returns(true); - _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(MakeStream("x = 1")); + SetFileOpen(_fileSystem, testFilePath, MakeStream("x = 1")); IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); @@ -116,13 +116,17 @@ public void CancellParsingAsync() { symbols.Should().HaveCount(0); } + private void SetFileOpen(IFileSystem fileSystem, string path, Stream stream) { + fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(stream); + } + [TestMethod, Priority(0)] public void DisposeParserCancelsParsing() { const string testFilePath = "C:/bla.py"; _fileSystem.FileExists(testFilePath).Returns(true); ManualResetEventSlim neverSignaledEvent = new ManualResetEventSlim(false); ManualResetEventSlim fileOpenedEvent = new ManualResetEventSlim(false); - _fileSystem.FileOpen(testFilePath, FileMode.Open).Returns(_ => { + SetFileOpen(_fileSystem, testFilePath, _ => { fileOpenedEvent.Set(); // Wait forever neverSignaledEvent.Wait(); @@ -144,6 +148,10 @@ public void DisposeParserCancelsParsing() { symbols.Should().HaveCount(0); } + private void SetFileOpen(IFileSystem fileSystem, string path, Func p) { + fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(p); + } + private Stream MakeStream(string str) { return new MemoryStream(Encoding.UTF8.GetBytes(str)); } From 6578616c212c7f755d662d412dbaa6674ee51e0f Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 15 Feb 2019 17:13:55 -0800 Subject: [PATCH 068/123] No Tcs in IndexParser --- .../Impl/Indexing/IndexParser.cs | 9 +----- src/LanguageServer/Test/IndexParserTests.cs | 28 ------------------- 2 files changed, 1 insertion(+), 36 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index 45c1a0745..731ef28e4 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -31,7 +31,6 @@ internal sealed class IndexParser : IIndexParser { private readonly object _syncObj = new object(); private CancellationTokenSource _linkedParseCts; private Task _parseTask; - private TaskCompletionSource _tcs; public IndexParser(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version) { Check.ArgumentNotNull(nameof(symbolIndex), symbolIndex); @@ -45,10 +44,8 @@ public IndexParser(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLangu public Task ParseAsync(string path, CancellationToken cancellationToken = default) { lock (_syncObj) { CancelCurrentParse(); - _tcs = new TaskCompletionSource(); _linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, cancellationToken); - _parseTask = Task.Run(() => Parse(path, _linkedParseCts)); - return _tcs.Task; + return Task.Run(() => Parse(path, _linkedParseCts)); } } @@ -61,7 +58,6 @@ private void CancelCurrentParse() { private void Parse(string path, CancellationTokenSource parseCts) { if (parseCts.Token.IsCancellationRequested) { - _tcs.SetCanceled(); parseCts.Token.ThrowIfCancellationRequested(); } try { @@ -76,15 +72,12 @@ private void Parse(string path, CancellationTokenSource parseCts) { if (_linkedParseCts == parseCts) { _linkedParseCts.Dispose(); _linkedParseCts = null; - _tcs.SetResult(new object()); } } } public void Dispose() { lock (_syncObj) { - _tcs?.TrySetCanceled(); - _tcs = null; _allProcessingCts.Cancel(); _allProcessingCts.Dispose(); diff --git a/src/LanguageServer/Test/IndexParserTests.cs b/src/LanguageServer/Test/IndexParserTests.cs index 36629dcfa..3e01188b9 100644 --- a/src/LanguageServer/Test/IndexParserTests.cs +++ b/src/LanguageServer/Test/IndexParserTests.cs @@ -120,34 +120,6 @@ private void SetFileOpen(IFileSystem fileSystem, string path, Stream stream) { fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(stream); } - [TestMethod, Priority(0)] - public void DisposeParserCancelsParsing() { - const string testFilePath = "C:/bla.py"; - _fileSystem.FileExists(testFilePath).Returns(true); - ManualResetEventSlim neverSignaledEvent = new ManualResetEventSlim(false); - ManualResetEventSlim fileOpenedEvent = new ManualResetEventSlim(false); - SetFileOpen(_fileSystem, testFilePath, _ => { - fileOpenedEvent.Set(); - // Wait forever - neverSignaledEvent.Wait(); - throw new InternalTestFailureException("Task should have been cancelled"); - }); - - IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); - - var parseTask = indexParser.ParseAsync(testFilePath); - fileOpenedEvent.Wait(); - indexParser.Dispose(); - Func parse = async () => { - await parseTask; - }; - - parse.Should().Throw(); - - var symbols = _symbolIndex.WorkspaceSymbols(""); - symbols.Should().HaveCount(0); - } - private void SetFileOpen(IFileSystem fileSystem, string path, Func p) { fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(p); } From e5bf1d8f70701ad93aa423f753d4ca1213c3ae39 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 15 Feb 2019 17:37:15 -0800 Subject: [PATCH 069/123] Disposing streams --- src/LanguageServer/Test/IndexParserTests.cs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/LanguageServer/Test/IndexParserTests.cs b/src/LanguageServer/Test/IndexParserTests.cs index 3e01188b9..876108d4b 100644 --- a/src/LanguageServer/Test/IndexParserTests.cs +++ b/src/LanguageServer/Test/IndexParserTests.cs @@ -61,10 +61,12 @@ public void NullIndexThrowsException() { public async Task ParseVariableInFileAsync() { const string testFilePath = "C:/bla.py"; _fileSystem.FileExists(testFilePath).Returns(true); - SetFileOpen(_fileSystem, testFilePath, MakeStream("x = 1")); - IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); - await indexParser.ParseAsync(testFilePath); + using (var fileStream = MakeStream("x = 1")) { + SetFileOpen(_fileSystem, testFilePath, fileStream); + IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); + await indexParser.ParseAsync(testFilePath); + } var symbols = _symbolIndex.WorkspaceSymbols(""); symbols.Should().HaveCount(1); @@ -101,7 +103,10 @@ public async Task ParseFileThatStopsExisting() { public void CancellParsingAsync() { const string testFilePath = "C:/bla.py"; _fileSystem.FileExists(testFilePath).Returns(true); - SetFileOpen(_fileSystem, testFilePath, MakeStream("x = 1")); + + using (var fileStream = MakeStream("x = 1")) { + SetFileOpen(_fileSystem, testFilePath, fileStream); + } IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); From 108259f8d0927c7e5ca36695f78ac335fc9e1dca Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 15 Feb 2019 17:46:37 -0800 Subject: [PATCH 070/123] Implementing tcs pattern --- .../Impl/Indexing/MostRecentSymbols.cs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs index b13eb4fde..6bcee5faa 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs @@ -27,20 +27,27 @@ public MostRecentDocumentSymbols(string path, ISymbolIndex symbolIndex, IFileSys public void Parse() { lock (_syncObj) { CancelExistingTask(); - _fileTask = ParseAsync(_fileCts.Token); + _fileTask = ParseAsync(_fileCts.Token, _fileTcs); } } - private async Task ParseAsync(CancellationToken cancellationToken) { - await _indexParser.ParseAsync(_path, cancellationToken); - cancellationToken.ThrowIfCancellationRequested(); - lock (_syncObj) { - SetFileTcsResult(); + private async Task ParseAsync(CancellationToken cancellationToken, TaskCompletionSource> parseTcs) { + try { + await _indexParser.ParseAsync(_path, cancellationToken); + lock (_syncObj) { + SetFileTcsResult(); + } + } catch (OperationCanceledException) { + parseTcs.TrySetCanceled(); + } catch (Exception ex) { + parseTcs.TrySetException(ex); + throw; } } public void Delete() { lock (_syncObj) { + CancelExistingTask(); _symbolIndex.Delete(_path); } } From 1ff0935ffd849b59f6a7bfe381fa96a51d6b29a1 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 19 Feb 2019 09:32:53 -0800 Subject: [PATCH 071/123] PR comments fixes and typos --- src/LanguageServer/Impl/Indexing/IndexParser.cs | 15 +++++++-------- .../Impl/Indexing/MostRecentSymbols.cs | 2 +- src/LanguageServer/Test/IndexParserTests.cs | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index 731ef28e4..f45a4a57c 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -57,9 +57,7 @@ private void CancelCurrentParse() { } private void Parse(string path, CancellationTokenSource parseCts) { - if (parseCts.Token.IsCancellationRequested) { - parseCts.Token.ThrowIfCancellationRequested(); - } + parseCts.Token.ThrowIfCancellationRequested(); try { using (var stream = _fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read)) { var parser = Parser.CreateParser(stream, _version); @@ -67,11 +65,12 @@ private void Parse(string path, CancellationTokenSource parseCts) { } } catch (Exception e) when (e is IOException || e is UnauthorizedAccessException) { Trace.TraceError(e.Message); - } - lock (_syncObj) { - if (_linkedParseCts == parseCts) { - _linkedParseCts.Dispose(); - _linkedParseCts = null; + } finally { + lock (_syncObj) { + if (_linkedParseCts == parseCts) { + _linkedParseCts.Dispose(); + _linkedParseCts = null; + } } } } diff --git a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs index 6bcee5faa..f29de037f 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs @@ -41,7 +41,7 @@ private async Task ParseAsync(CancellationToken cancellationToken, TaskCompletio parseTcs.TrySetCanceled(); } catch (Exception ex) { parseTcs.TrySetException(ex); - throw; + throw; } } diff --git a/src/LanguageServer/Test/IndexParserTests.cs b/src/LanguageServer/Test/IndexParserTests.cs index 876108d4b..0c87707af 100644 --- a/src/LanguageServer/Test/IndexParserTests.cs +++ b/src/LanguageServer/Test/IndexParserTests.cs @@ -100,7 +100,7 @@ public async Task ParseFileThatStopsExisting() { } [TestMethod, Priority(0)] - public void CancellParsingAsync() { + public void CancelParsingAsync() { const string testFilePath = "C:/bla.py"; _fileSystem.FileExists(testFilePath).Returns(true); From 0bc72c3a64b558b0505b7dbde4200c897b5dc173 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 19 Feb 2019 12:38:32 -0800 Subject: [PATCH 072/123] Rework of tcs in indexmanager --- src/Core/Impl/Diagnostics/Check.cs | 7 ++ src/Core/Impl/Extensions/TaskExtensions.cs | 28 ++++- .../Impl/Indexing/IIndexParser.cs | 1 + .../Impl/Indexing/IndexParser.cs | 9 +- .../Impl/Indexing/MostRecentSymbols.cs | 115 +++++++++--------- 5 files changed, 98 insertions(+), 62 deletions(-) diff --git a/src/Core/Impl/Diagnostics/Check.cs b/src/Core/Impl/Diagnostics/Check.cs index 9437ca48e..fccc8ae73 100644 --- a/src/Core/Impl/Diagnostics/Check.cs +++ b/src/Core/Impl/Diagnostics/Check.cs @@ -66,6 +66,13 @@ public static void InvalidOperation(Func predicate, string message = null) } } + [DebuggerStepThrough] + public static void InvalidOperation(bool condition, string message = null) { + if (!condition) { + throw new InvalidOperationException(message ?? string.Empty); + } + } + [DebuggerStepThrough] public static void Argument(string argumentName, Func predicate) { if (!predicate()) { diff --git a/src/Core/Impl/Extensions/TaskExtensions.cs b/src/Core/Impl/Extensions/TaskExtensions.cs index ca83cb836..c96b92f87 100644 --- a/src/Core/Impl/Extensions/TaskExtensions.cs +++ b/src/Core/Impl/Extensions/TaskExtensions.cs @@ -22,9 +22,9 @@ namespace Microsoft.Python.Core { public static class TaskExtensions { public static void SetCompletionResultTo(this Task task, TaskCompletionSourceEx tcs) - => task.ContinueWith(SetCompletionResultToContinuation, tcs, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + => task.ContinueWith(SetCompletionResultToContinuationEx, tcs, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); - private static void SetCompletionResultToContinuation(Task task, object state) { + private static void SetCompletionResultToContinuationEx(Task task, object state) { var tcs = (TaskCompletionSourceEx) state; switch (task.Status) { case TaskStatus.RanToCompletion: @@ -45,6 +45,30 @@ private static void SetCompletionResultToContinuation(Task task, object st } } + public static void SetCompletionResultTo(this Task task, TaskCompletionSource tcs) + => task.ContinueWith(SetCompletionResultToContinuation, tcs, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + + private static void SetCompletionResultToContinuation(Task task, object state) { + var tcs = (TaskCompletionSource)state; + switch (task.Status) { + case TaskStatus.RanToCompletion: + tcs.TrySetResult(task.Result); + break; + case TaskStatus.Canceled: + try { + task.GetAwaiter().GetResult(); + } catch (OperationCanceledException ex) { + tcs.TrySetCanceled(ex.CancellationToken); + } + break; + case TaskStatus.Faulted: + tcs.TrySetException(task.Exception); + break; + default: + throw new ArgumentOutOfRangeException(); + } + } + /// /// Suppresses warnings about unawaited tasks and rethrows task exceptions back to the callers synchronization context if it is possible /// diff --git a/src/LanguageServer/Impl/Indexing/IIndexParser.cs b/src/LanguageServer/Impl/Indexing/IIndexParser.cs index e8d9c5dae..efe7e56a7 100644 --- a/src/LanguageServer/Impl/Indexing/IIndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IIndexParser.cs @@ -16,6 +16,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.LanguageServer.Indexing { internal interface IIndexParser : IDisposable { diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index f45a4a57c..9416ee75f 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -21,6 +21,7 @@ using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.LanguageServer.Indexing { internal sealed class IndexParser : IIndexParser { @@ -50,6 +51,8 @@ public Task ParseAsync(string path, CancellationToken cancellationToken = defaul } private void CancelCurrentParse() { + Check.InvalidOperation(Monitor.IsEntered(_syncObj)); + _linkedParseCts?.Cancel(); _linkedParseCts?.Dispose(); _linkedParseCts = null; @@ -61,7 +64,11 @@ private void Parse(string path, CancellationTokenSource parseCts) { try { using (var stream = _fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read)) { var parser = Parser.CreateParser(stream, _version); - _symbolIndex.Add(path, parser.ParseFile()); + var ast = parser.ParseFile(); + lock (_syncObj) { + parseCts.Token.ThrowIfCancellationRequested(); + _symbolIndex.Add(path, ast); + } } } catch (Exception e) when (e is IOException || e is UnauthorizedAccessException) { Trace.TraceError(e.Message); diff --git a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs index f29de037f..9f7522a14 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs @@ -3,9 +3,10 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; -using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.LanguageServer.Indexing { class MostRecentDocumentSymbols : IDisposable { @@ -15,7 +16,6 @@ class MostRecentDocumentSymbols : IDisposable { private readonly string _path; private CancellationTokenSource _fileCts = new CancellationTokenSource(); - private Task _fileTask; private TaskCompletionSource> _fileTcs = new TaskCompletionSource>(); public MostRecentDocumentSymbols(string path, ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version) { @@ -25,106 +25,103 @@ public MostRecentDocumentSymbols(string path, ISymbolIndex symbolIndex, IFileSys } public void Parse() { + CancellationToken currentCt; + TaskCompletionSource> currentTcs; lock (_syncObj) { CancelExistingTask(); - _fileTask = ParseAsync(_fileCts.Token, _fileTcs); + currentCt = _fileCts.Token; + currentTcs = _fileTcs; } + ParseAsync(currentCt).SetCompletionResultTo(currentTcs); } - private async Task ParseAsync(CancellationToken cancellationToken, TaskCompletionSource> parseTcs) { - try { - await _indexParser.ParseAsync(_path, cancellationToken); - lock (_syncObj) { - SetFileTcsResult(); - } - } catch (OperationCanceledException) { - parseTcs.TrySetCanceled(); - } catch (Exception ex) { - parseTcs.TrySetException(ex); - throw; - } - } - - public void Delete() { + public void Add(IDocument doc) { + CancellationToken currentCt; + TaskCompletionSource> currentTcs; lock (_syncObj) { CancelExistingTask(); - _symbolIndex.Delete(_path); + currentCt = _fileCts.Token; + currentTcs = _fileTcs; } + AddAsync(doc, currentCt).SetCompletionResultTo(currentTcs); } - public void Process(PythonAst ast) { + public void ReIndex(IDocument doc) { + CancellationToken currentCt; + TaskCompletionSource> currentTcs; lock (_syncObj) { CancelExistingTask(); - _symbolIndex.Add(_path, ast); - SetFileTcsResult(); + currentCt = _fileCts.Token; + currentTcs = _fileTcs; } + ReIndexAsync(doc, currentCt).SetCompletionResultTo(currentTcs); } - public void Add(IDocument doc) { + public Task> GetSymbolsAsync() => _fileTcs.Task; + + public void Delete() { lock (_syncObj) { CancelExistingTask(); - _fileTask = AddAsync(doc, _fileCts.Token); + _symbolIndex.Delete(_path); } } - private async Task AddAsync(IDocument doc, CancellationToken cancellationToken) { - var ast = await doc.GetAstAsync(cancellationToken); - cancellationToken.ThrowIfCancellationRequested(); + public void MarkAsPending() { lock (_syncObj) { - _symbolIndex.Add(_path, ast); - SetFileTcsResult(); + CancelExistingTask(); } } - public Task> GetSymbolsAsync() => _fileTcs.Task; - - private void CancelExistingTask() { - if (_fileTask != null) { - _fileTcs = new TaskCompletionSource>(); + public void Dispose() { + lock (_syncObj) { + _fileCts?.Cancel(); + _fileCts?.Dispose(); + _fileCts = null; - _fileCts.Cancel(); - _fileCts.Dispose(); - _fileCts = new CancellationTokenSource(); + _fileTcs.TrySetCanceled(); - _fileTask = null; + _indexParser.Dispose(); } } - public void ReIndex(IDocument doc) { + private void SetResultOn(TaskCompletionSource> tcs) { + tcs.TrySetResult(_symbolIndex.HierarchicalDocumentSymbols(_path)); + } + + private async Task> AddAsync(IDocument doc, CancellationToken addCancellationToken) { + var ast = await doc.GetAstAsync(addCancellationToken); lock (_syncObj) { - CancelExistingTask(); - _fileTask = ReIndexAsync(doc, _fileCts.Token); + addCancellationToken.ThrowIfCancellationRequested(); + _symbolIndex.Add(_path, ast); + return _symbolIndex.HierarchicalDocumentSymbols(_path); } } - private async Task ReIndexAsync(IDocument doc, CancellationToken cancellationToken) { - var ast = await doc.GetAstAsync(cancellationToken); - cancellationToken.ThrowIfCancellationRequested(); + private async Task> ReIndexAsync(IDocument doc, CancellationToken reIndexCancellationToken) { + var ast = await doc.GetAstAsync(reIndexCancellationToken); lock (_syncObj) { + reIndexCancellationToken.ThrowIfCancellationRequested(); _symbolIndex.Update(_path, ast); - SetFileTcsResult(); + return _symbolIndex.HierarchicalDocumentSymbols(_path); } } - public void MarkAsPending() { + private async Task> ParseAsync(CancellationToken parseCancellationToken) { + await _indexParser.ParseAsync(_path, parseCancellationToken); + parseCancellationToken.ThrowIfCancellationRequested(); lock (_syncObj) { - CancelExistingTask(); + return _symbolIndex.HierarchicalDocumentSymbols(_path); } - - } - - private void SetFileTcsResult() { - _fileTcs.TrySetResult(_symbolIndex.HierarchicalDocumentSymbols(_path)); } - public void Dispose() { - lock (_syncObj) { - _fileCts?.Cancel(); - _fileCts?.Dispose(); - _fileCts = null; - _fileTask = null; + private void CancelExistingTask() { + Check.InvalidOperation(Monitor.IsEntered(_syncObj)); - _indexParser.Dispose(); + _fileCts.Cancel(); + _fileCts.Dispose(); + _fileCts = new CancellationTokenSource(); + if (_fileTcs.Task.IsCompleted) { + _fileTcs = new TaskCompletionSource>(); } } } From 2bba9dc559a1cb4a1404e04eb8830dbc2f941340 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 19 Feb 2019 12:41:27 -0800 Subject: [PATCH 073/123] Delete unused variable --- src/LanguageServer/Impl/Indexing/IndexParser.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index 9416ee75f..31dc4b4c7 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -21,7 +21,6 @@ using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; -using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.LanguageServer.Indexing { internal sealed class IndexParser : IIndexParser { @@ -31,7 +30,6 @@ internal sealed class IndexParser : IIndexParser { private readonly CancellationTokenSource _allProcessingCts = new CancellationTokenSource(); private readonly object _syncObj = new object(); private CancellationTokenSource _linkedParseCts; - private Task _parseTask; public IndexParser(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version) { Check.ArgumentNotNull(nameof(symbolIndex), symbolIndex); @@ -56,7 +54,6 @@ private void CancelCurrentParse() { _linkedParseCts?.Cancel(); _linkedParseCts?.Dispose(); _linkedParseCts = null; - _parseTask = null; } private void Parse(string path, CancellationTokenSource parseCts) { @@ -90,8 +87,6 @@ public void Dispose() { _linkedParseCts?.Cancel(); _linkedParseCts?.Dispose(); _linkedParseCts = null; - - _parseTask = null; } } } From 9c5bb3ada83faaf2a1da8365d82a70fbfaa1c652 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 19 Feb 2019 12:59:00 -0800 Subject: [PATCH 074/123] Disposal check --- .../Impl/Indexing/MostRecentSymbols.cs | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs index 9f7522a14..3f51abfb8 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs @@ -17,6 +17,7 @@ class MostRecentDocumentSymbols : IDisposable { private CancellationTokenSource _fileCts = new CancellationTokenSource(); private TaskCompletionSource> _fileTcs = new TaskCompletionSource>(); + private bool wasLastTaskDisposed = true; public MostRecentDocumentSymbols(string path, ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version) { _path = path; @@ -31,6 +32,7 @@ public void Parse() { CancelExistingTask(); currentCt = _fileCts.Token; currentTcs = _fileTcs; + wasLastTaskDisposed = false; } ParseAsync(currentCt).SetCompletionResultTo(currentTcs); } @@ -42,6 +44,7 @@ public void Add(IDocument doc) { CancelExistingTask(); currentCt = _fileCts.Token; currentTcs = _fileTcs; + wasLastTaskDisposed = false; } AddAsync(doc, currentCt).SetCompletionResultTo(currentTcs); } @@ -53,6 +56,7 @@ public void ReIndex(IDocument doc) { CancelExistingTask(); currentCt = _fileCts.Token; currentTcs = _fileTcs; + wasLastTaskDisposed = false; } ReIndexAsync(doc, currentCt).SetCompletionResultTo(currentTcs); } @@ -74,20 +78,18 @@ public void MarkAsPending() { public void Dispose() { lock (_syncObj) { - _fileCts?.Cancel(); - _fileCts?.Dispose(); - _fileCts = null; - - _fileTcs.TrySetCanceled(); - + if (!wasLastTaskDisposed) { + _fileCts?.Cancel(); + _fileCts?.Dispose(); + _fileCts = null; + + wasLastTaskDisposed = true; + _fileTcs.TrySetCanceled(); + } _indexParser.Dispose(); } } - private void SetResultOn(TaskCompletionSource> tcs) { - tcs.TrySetResult(_symbolIndex.HierarchicalDocumentSymbols(_path)); - } - private async Task> AddAsync(IDocument doc, CancellationToken addCancellationToken) { var ast = await doc.GetAstAsync(addCancellationToken); lock (_syncObj) { @@ -117,11 +119,15 @@ private async Task> ParseAsync(CancellationToken private void CancelExistingTask() { Check.InvalidOperation(Monitor.IsEntered(_syncObj)); - _fileCts.Cancel(); - _fileCts.Dispose(); - _fileCts = new CancellationTokenSource(); - if (_fileTcs.Task.IsCompleted) { - _fileTcs = new TaskCompletionSource>(); + if (!wasLastTaskDisposed) { + _fileCts.Cancel(); + _fileCts.Dispose(); + _fileCts = new CancellationTokenSource(); + // Re use current tcs if possible + if (_fileTcs.Task.IsCompleted) { + _fileTcs = new TaskCompletionSource>(); + } + wasLastTaskDisposed = true; } } } From 8de9f3aa38f19d6ab3c9a1b7a981c3e035965675 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 19 Feb 2019 13:03:31 -0800 Subject: [PATCH 075/123] TCS no reuse --- src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs index 3f51abfb8..67c17757c 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs @@ -123,10 +123,8 @@ private void CancelExistingTask() { _fileCts.Cancel(); _fileCts.Dispose(); _fileCts = new CancellationTokenSource(); - // Re use current tcs if possible - if (_fileTcs.Task.IsCompleted) { - _fileTcs = new TaskCompletionSource>(); - } + + _fileTcs = new TaskCompletionSource>(); wasLastTaskDisposed = true; } } From e2bf0fef98984b9864e8ce835206e81e35ea2226 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 19 Feb 2019 16:21:39 -0800 Subject: [PATCH 076/123] Reorder layers --- .../Impl/Implementation/Server.cs | 5 +- .../Impl/Indexing/IIndexParser.cs | 2 +- .../Impl/Indexing/ISymbolIndex.cs | 15 +- .../Impl/Indexing/IndexManager.cs | 41 +- .../Impl/Indexing/IndexParser.cs | 40 +- .../Impl/Indexing/MostRecentSymbols.cs | 80 +-- .../Impl/Indexing/SymbolIndex.cs | 103 ++- src/LanguageServer/Test/IndexManagerTests.cs | 29 +- src/LanguageServer/Test/IndexParserTests.cs | 52 +- src/LanguageServer/Test/SymbolIndexTests.cs | 667 ++---------------- .../Test/SymbolIndexWalkerTests.cs | 624 ++++++++++++++++ 11 files changed, 855 insertions(+), 803 deletions(-) create mode 100644 src/LanguageServer/Test/SymbolIndexWalkerTests.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index 34f78f5c0..4e723c394 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -123,8 +123,9 @@ public async Task InitializeAsync(InitializeParams @params, Ca _interpreter = await PythonInterpreter.CreateAsync(configuration, rootDir, _services, cancellationToken); _services.AddService(_interpreter); - var symbolIndex = new SymbolIndex(); - _indexManager = new IndexManager(symbolIndex, _services.GetService(), _interpreter.LanguageVersion, rootDir, + var fileSystem = _services.GetService(); + var symbolIndex = new SymbolIndex(fileSystem, _interpreter.LanguageVersion); + _indexManager = new IndexManager(symbolIndex, fileSystem, _interpreter.LanguageVersion, rootDir, @params.initializationOptions.includeFiles, @params.initializationOptions.excludeFiles, _services.GetService()); diff --git a/src/LanguageServer/Impl/Indexing/IIndexParser.cs b/src/LanguageServer/Impl/Indexing/IIndexParser.cs index efe7e56a7..7330ef521 100644 --- a/src/LanguageServer/Impl/Indexing/IIndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IIndexParser.cs @@ -20,6 +20,6 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal interface IIndexParser : IDisposable { - Task ParseAsync(string path, CancellationToken cancellationToken = default); + Task ParseAsync(string path, CancellationToken cancellationToken = default); } } diff --git a/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs index eb2098485..ce71d6337 100644 --- a/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs @@ -14,16 +14,19 @@ // permissions and limitations under the License. using System.Collections.Generic; -using Microsoft.Python.Parsing.Ast; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Documents; namespace Microsoft.Python.LanguageServer.Indexing { internal interface ISymbolIndex { - IEnumerable WorkspaceSymbols(string query); - IEnumerable HierarchicalDocumentSymbols(string path); - void Add(string path, PythonAst pythonAst); + Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken ct = default); + Task> HierarchicalDocumentSymbols(string path); + void Add(string path, IDocument doc); + void Parse(string path); void Delete(string path); bool IsIndexed(string path); - void Update(string path, PythonAst pythonAst); - + void ReIndex(string path, IDocument doc); + void MarkAsPending(string path); } } diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index af08756bb..e9340e6c1 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -38,7 +37,6 @@ internal class IndexManager : IIndexManager { private readonly IIdleTimeService _idleTimeService; private readonly CancellationTokenSource _allIndexCts = new CancellationTokenSource(); private readonly TaskCompletionSource _addRootTcs = new TaskCompletionSource(); - private readonly ConcurrentDictionary _files = new ConcurrentDictionary(PathEqualityComparer.Instance); private readonly HashSet _pendingDocs = new HashSet(new UriDocumentComparer()); private readonly PythonLanguageVersion _version; private DateTime _lastPendingDocAddedTime; @@ -67,8 +65,7 @@ public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLang private void StartAddRootDir() { foreach (var fileInfo in WorkspaceFiles()) { if (ModulePath.IsPythonSourceFile(fileInfo.FullName)) { - _files.GetOrAdd(fileInfo.FullName, MakeMostRecentFileSymbols(fileInfo.FullName)); - _files[fileInfo.FullName].Parse(); + _symbolIndex.Parse(fileInfo.FullName); } } } @@ -84,10 +81,9 @@ private IEnumerable WorkspaceFiles() { public void ProcessClosedFile(string path) { if (IsFileOnWorkspace(path)) { - _files[path].Parse(); - } else if (_files.TryRemove(path, out var fileSymbols)) { - fileSymbols.Delete(); - fileSymbols.Dispose(); + _symbolIndex.Parse(path); + } else { + _symbolIndex.Delete(path); } } @@ -99,43 +95,32 @@ private bool IsFileOnWorkspace(string path) { } public void ProcessNewFile(string path, IDocument doc) { - _files.GetOrAdd(path, MakeMostRecentFileSymbols(path)); - _files[path].Add(doc); + _symbolIndex.Add(path, doc); } public void ReIndexFile(string path, IDocument doc) { - if (_files.TryGetValue(path, out var fileSymbols)) { - fileSymbols.ReIndex(doc); - } + _symbolIndex.ReIndex(path, doc); } public void Dispose() { - foreach (var mostRecentSymbols in _files.Values) { - mostRecentSymbols.Dispose(); - } + //_symbolIndex.Dispose(); _allIndexCts.Cancel(); _allIndexCts.Dispose(); } - public async Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken cancellationToken = default) { - var s = await _files[path].GetSymbolsAsync(); - return s.ToList(); + public Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken cancellationToken = default) { + return _symbolIndex.HierarchicalDocumentSymbols(path); } - public async Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken cancellationToken = default) { - var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(_allIndexCts.Token, cancellationToken); - await Task.WhenAny( - Task.WhenAll(_files.Values.Select(mostRecent => mostRecent.GetSymbolsAsync()).ToArray()), - Task.Delay(Timeout.Infinite, linkedCts.Token)); - linkedCts.Dispose(); - return _symbolIndex.WorkspaceSymbols(query).Take(maxLength).ToList(); + public Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken cancellationToken = default) { + return _symbolIndex.WorkspaceSymbolsAsync(query, maxLength, cancellationToken); } public void AddPendingDoc(IDocument doc) { lock (_pendingDocs) { _lastPendingDocAddedTime = DateTime.Now; _pendingDocs.Add(doc); - _files[doc.Uri.AbsolutePath].MarkAsPending(); + _symbolIndex.MarkAsPending(doc.Uri.AbsolutePath); } } @@ -158,7 +143,7 @@ private void ReIndexPendingDocsAsync() { } private MostRecentDocumentSymbols MakeMostRecentFileSymbols(string path) { - return new MostRecentDocumentSymbols(path, _symbolIndex, _fileSystem, _version); + return new MostRecentDocumentSymbols(path, _fileSystem, _version); } private class UriDocumentComparer : IEqualityComparer { diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index 31dc4b4c7..43b908ae5 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -13,34 +13,30 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; -using System.Diagnostics; using System.IO; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.LanguageServer.Indexing { internal sealed class IndexParser : IIndexParser { - private readonly ISymbolIndex _symbolIndex; private readonly IFileSystem _fileSystem; private readonly PythonLanguageVersion _version; private readonly CancellationTokenSource _allProcessingCts = new CancellationTokenSource(); private readonly object _syncObj = new object(); private CancellationTokenSource _linkedParseCts; - public IndexParser(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version) { - Check.ArgumentNotNull(nameof(symbolIndex), symbolIndex); + public IndexParser(IFileSystem fileSystem, PythonLanguageVersion version) { Check.ArgumentNotNull(nameof(fileSystem), fileSystem); - _symbolIndex = symbolIndex; _fileSystem = fileSystem; _version = version; } - public Task ParseAsync(string path, CancellationToken cancellationToken = default) { + public Task ParseAsync(string path, CancellationToken cancellationToken = default) { lock (_syncObj) { CancelCurrentParse(); _linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, cancellationToken); @@ -56,27 +52,21 @@ private void CancelCurrentParse() { _linkedParseCts = null; } - private void Parse(string path, CancellationTokenSource parseCts) { + private PythonAst Parse(string path, CancellationTokenSource parseCts) { parseCts.Token.ThrowIfCancellationRequested(); - try { - using (var stream = _fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read)) { - var parser = Parser.CreateParser(stream, _version); - var ast = parser.ParseFile(); - lock (_syncObj) { - parseCts.Token.ThrowIfCancellationRequested(); - _symbolIndex.Add(path, ast); - } - } - } catch (Exception e) when (e is IOException || e is UnauthorizedAccessException) { - Trace.TraceError(e.Message); - } finally { - lock (_syncObj) { - if (_linkedParseCts == parseCts) { - _linkedParseCts.Dispose(); - _linkedParseCts = null; - } + PythonAst ast = null; + using (var stream = _fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read)) { + var parser = Parser.CreateParser(stream, _version); + ast = parser.ParseFile(); + } + parseCts.Token.ThrowIfCancellationRequested(); + lock (_syncObj) { + if (_linkedParseCts == parseCts) { + _linkedParseCts.Dispose(); + _linkedParseCts = null; } } + return ast; } public void Dispose() { diff --git a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs index 67c17757c..c801e2e20 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; +using System.IO; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Documents; @@ -12,63 +14,54 @@ namespace Microsoft.Python.LanguageServer.Indexing { class MostRecentDocumentSymbols : IDisposable { private readonly object _syncObj = new object(); private readonly IIndexParser _indexParser; - private readonly ISymbolIndex _symbolIndex; private readonly string _path; private CancellationTokenSource _fileCts = new CancellationTokenSource(); - private TaskCompletionSource> _fileTcs = new TaskCompletionSource>(); - private bool wasLastTaskDisposed = true; + private TaskCompletionSource> _fileTcs = new TaskCompletionSource>(); + private bool _wasLastTaskDisposed = true; - public MostRecentDocumentSymbols(string path, ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version) { + public MostRecentDocumentSymbols(string path, IFileSystem fileSystem, PythonLanguageVersion version) { _path = path; - _symbolIndex = symbolIndex; - _indexParser = new IndexParser(_symbolIndex, fileSystem, version); + _indexParser = new IndexParser(fileSystem, version); } public void Parse() { CancellationToken currentCt; - TaskCompletionSource> currentTcs; + TaskCompletionSource> currentTcs; lock (_syncObj) { CancelExistingTask(); currentCt = _fileCts.Token; currentTcs = _fileTcs; - wasLastTaskDisposed = false; + _wasLastTaskDisposed = false; } ParseAsync(currentCt).SetCompletionResultTo(currentTcs); } public void Add(IDocument doc) { CancellationToken currentCt; - TaskCompletionSource> currentTcs; + TaskCompletionSource> currentTcs; lock (_syncObj) { CancelExistingTask(); currentCt = _fileCts.Token; currentTcs = _fileTcs; - wasLastTaskDisposed = false; + _wasLastTaskDisposed = false; } AddAsync(doc, currentCt).SetCompletionResultTo(currentTcs); } public void ReIndex(IDocument doc) { CancellationToken currentCt; - TaskCompletionSource> currentTcs; + TaskCompletionSource> currentTcs; lock (_syncObj) { CancelExistingTask(); currentCt = _fileCts.Token; currentTcs = _fileTcs; - wasLastTaskDisposed = false; + _wasLastTaskDisposed = false; } ReIndexAsync(doc, currentCt).SetCompletionResultTo(currentTcs); } - public Task> GetSymbolsAsync() => _fileTcs.Task; - - public void Delete() { - lock (_syncObj) { - CancelExistingTask(); - _symbolIndex.Delete(_path); - } - } + public Task> GetSymbolsAsync() => _fileTcs.Task; public void MarkAsPending() { lock (_syncObj) { @@ -78,54 +71,57 @@ public void MarkAsPending() { public void Dispose() { lock (_syncObj) { - if (!wasLastTaskDisposed) { + if (!_wasLastTaskDisposed) { _fileCts?.Cancel(); _fileCts?.Dispose(); _fileCts = null; - wasLastTaskDisposed = true; + _wasLastTaskDisposed = true; _fileTcs.TrySetCanceled(); } _indexParser.Dispose(); } } - private async Task> AddAsync(IDocument doc, CancellationToken addCancellationToken) { + private async Task> AddAsync(IDocument doc, CancellationToken addCancellationToken) { var ast = await doc.GetAstAsync(addCancellationToken); - lock (_syncObj) { - addCancellationToken.ThrowIfCancellationRequested(); - _symbolIndex.Add(_path, ast); - return _symbolIndex.HierarchicalDocumentSymbols(_path); - } + var walker = new SymbolIndexWalker(ast); + ast.Walk(walker); + addCancellationToken.ThrowIfCancellationRequested(); + return walker.Symbols; } - private async Task> ReIndexAsync(IDocument doc, CancellationToken reIndexCancellationToken) { + private async Task> ReIndexAsync(IDocument doc, CancellationToken reIndexCancellationToken) { var ast = await doc.GetAstAsync(reIndexCancellationToken); - lock (_syncObj) { - reIndexCancellationToken.ThrowIfCancellationRequested(); - _symbolIndex.Update(_path, ast); - return _symbolIndex.HierarchicalDocumentSymbols(_path); - } + var walker = new SymbolIndexWalker(ast); + ast.Walk(walker); + reIndexCancellationToken.ThrowIfCancellationRequested(); + return walker.Symbols; } - private async Task> ParseAsync(CancellationToken parseCancellationToken) { - await _indexParser.ParseAsync(_path, parseCancellationToken); - parseCancellationToken.ThrowIfCancellationRequested(); - lock (_syncObj) { - return _symbolIndex.HierarchicalDocumentSymbols(_path); + private async Task> ParseAsync(CancellationToken parseCancellationToken) { + try { + var ast = await _indexParser.ParseAsync(_path, parseCancellationToken); + var walker = new SymbolIndexWalker(ast); + ast.Walk(walker); + parseCancellationToken.ThrowIfCancellationRequested(); + return walker.Symbols; + } catch (Exception e) when (e is IOException || e is UnauthorizedAccessException) { + Trace.TraceError(e.Message); } + return new List(); } private void CancelExistingTask() { Check.InvalidOperation(Monitor.IsEntered(_syncObj)); - if (!wasLastTaskDisposed) { + if (!_wasLastTaskDisposed) { _fileCts.Cancel(); _fileCts.Dispose(); _fileCts = new CancellationTokenSource(); - _fileTcs = new TaskCompletionSource>(); - wasLastTaskDisposed = true; + _fileTcs = new TaskCompletionSource>(); + _wasLastTaskDisposed = true; } } } diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index 03776a706..8efb80434 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -16,62 +16,103 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core; +using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Core.IO; -using Microsoft.Python.Parsing.Ast; +using Microsoft.Python.Parsing; namespace Microsoft.Python.LanguageServer.Indexing { internal sealed class SymbolIndex : ISymbolIndex { - private readonly ConcurrentDictionary> _index; + private readonly ConcurrentDictionary _index; + private readonly IFileSystem _fileSystem; + private readonly PythonLanguageVersion _version; + + public SymbolIndex(IFileSystem fileSystem, PythonLanguageVersion version) { + _fileSystem = fileSystem; + _version = version; - public SymbolIndex() { var comparer = PathEqualityComparer.Instance; - _index = new ConcurrentDictionary>(comparer); + _index = new ConcurrentDictionary(comparer); + } + + public Task> HierarchicalDocumentSymbols(string path) { + if (_index.TryGetValue(path, out var mostRecentSymbols)) { + return mostRecentSymbols.GetSymbolsAsync(); + } else { + return Task.FromResult>(new List()); + } + } + + public async Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken ct = default) { + var tasks = _index + .Select(kvp => WorkspaceSymbolsQueryAsync(kvp.Key, query, kvp.Value, ct)) + .Take(maxLength) + .ToArray(); + var b = await Task.WhenAll(tasks); + List results = new List(); + foreach (var t in b) { + foreach (var tt in t) { + results.Add(tt); + } + } + return results; } - public IEnumerable HierarchicalDocumentSymbols(string path) - => _index.TryGetValue(path, out var list) ? list : Enumerable.Empty(); + private async Task> WorkspaceSymbolsQueryAsync(string filePath, string query, MostRecentDocumentSymbols recentSymbols, CancellationToken cancellationToken) { + var symbols = await recentSymbols.GetSymbolsAsync(); + cancellationToken.ThrowIfCancellationRequested(); + return WorkspaceSymbolsQuery(filePath, query, symbols); + } - public IEnumerable WorkspaceSymbols(string query) { - return _index.SelectMany(kvp => WorkspaceSymbolsQuery(query, kvp.Key, kvp.Value)); + public void Add(string path, IDocument doc) { + _index.GetOrAdd(path, MakeMostRecentDocSymbols(path)).Add(doc); } - public void Add(string path, PythonAst ast) { - var walker = new SymbolIndexWalker(ast); - ast.Walk(walker); - _index[path] = walker.Symbols; + public void Parse(string path) { + var mostRecentSymbols = _index.GetOrAdd(path, MakeMostRecentDocSymbols(path)); + mostRecentSymbols.Parse(); } public void Delete(string path) { - _index.Remove(path, out var _); + _index.Remove(path, out var mostRecentDocSymbols); + mostRecentDocSymbols.Dispose(); } public bool IsIndexed(string path) => _index.ContainsKey(path); - private static IEnumerable<(HierarchicalSymbol symbol, string parentName)> DecorateWithParentsName( - IEnumerable symbols, string parentName) - => symbols.Select((symbol) => (symbol, parentName)); - - public void Update(string path, PythonAst ast) { + public void ReIndex(string path, IDocument doc) { if (_index.TryGetValue(path, out var currentSymbols)) { - var walker = new SymbolIndexWalker(ast); - ast.Walk(walker); - _index.TryUpdate(path, walker.Symbols, currentSymbols); + currentSymbols.ReIndex(doc); } } - private IEnumerable WorkspaceSymbolsQuery(string query, string path, - IEnumerable symbols) { + public void MarkAsPending(string path) { + _index[path].MarkAsPending(); + } + + private IReadOnlyList WorkspaceSymbolsQuery(string path, string query, + IReadOnlyList symbols) { var rootSymbols = DecorateWithParentsName(symbols, null); - var treeSymbols = rootSymbols.TraverseBreadthFirst((symbolAndParent) => { - var sym = symbolAndParent.symbol; - return DecorateWithParentsName(sym.Children.MaybeEnumerate(), sym.Name); + var treeSymbols = rootSymbols.TraverseBreadthFirst((symAndPar) => { + var sym = symAndPar.symbol; + return DecorateWithParentsName(sym.Children.MaybeEnumerate().ToList(), sym.Name); }); - foreach (var (sym, parentName) in treeSymbols) { - if (sym.Name.ContainsOrdinal(query, ignoreCase: true)) { - yield return new FlatSymbol(sym.Name, sym.Kind, path, sym.SelectionRange, parentName); - } - } + return treeSymbols.Where(sym => sym.symbol.Name.ContainsOrdinal(query, ignoreCase: true)) + .Select(sym => new FlatSymbol(sym.symbol.Name, sym.symbol.Kind, path, sym.symbol.SelectionRange, sym.parentName)) + .ToList(); + } + + + private static IReadOnlyList<(HierarchicalSymbol symbol, string parentName)> DecorateWithParentsName( + IReadOnlyList symbols, string parentName) { + return symbols.Select((symbol) => (symbol, parentName)).ToList(); + } + + private MostRecentDocumentSymbols MakeMostRecentDocSymbols(string path) { + return new MostRecentDocumentSymbols(path, _fileSystem, _version); } } } diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index 7730aa3f3..3cd768ac7 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -68,7 +68,7 @@ public void NullDirectoryThrowsException() { var context = new IndexTestContext(this); Action construct = () => { PythonLanguageVersion version = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); - IIndexManager indexManager = new IndexManager(new SymbolIndex(), context.FileSystem, + IIndexManager indexManager = new IndexManager(new SymbolIndex(context.FileSystem, version), context.FileSystem, version, null, new string[] { }, new string[] { }, new IdleTimeService()); }; @@ -169,31 +169,6 @@ public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() { await SymbolIndexShouldBeEmpty(indexManager); } - [TestMethod, Priority(0)] - public async Task DisposeManagerCancelsTaskAsync() { - var context = new IndexTestContext(this); - var pythonTestFilePath = context.FileWithXVarInRootDir(); - ManualResetEventSlim neverSignaledEvent = new ManualResetEventSlim(false); - ManualResetEventSlim fileOpenedEvent = new ManualResetEventSlim(false); - - context.SetFileOpen(pythonTestFilePath, _ => { - fileOpenedEvent.Set(); - // Wait forever - neverSignaledEvent.Wait(); - throw new InternalTestFailureException("Task should have been cancelled"); - }); - - var indexManager = context.GetDefaultIndexManager(); - fileOpenedEvent.Wait(); - var t = WaitForWorkspaceAddedAsync(indexManager); - indexManager.Dispose(); - - await t; - - neverSignaledEvent.IsSet.Should().BeFalse(); - context.SymbolIndex.WorkspaceSymbols("").Should().HaveCount(0); - } - [TestMethod, Priority(0)] public async Task WorkspaceSymbolsAddsRootDirectory() { var context = new IndexTestContext(this); @@ -302,7 +277,7 @@ public IndexTestContext(IndexManagerTests tests) { private void Setup() { FileSystem = Substitute.For(); - SymbolIndex = new SymbolIndex(); + SymbolIndex = new SymbolIndex(FileSystem, _pythonLanguageVersion); SetupRootDir(); } diff --git a/src/LanguageServer/Test/IndexParserTests.cs b/src/LanguageServer/Test/IndexParserTests.cs index 0c87707af..4187f312d 100644 --- a/src/LanguageServer/Test/IndexParserTests.cs +++ b/src/LanguageServer/Test/IndexParserTests.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; @@ -23,6 +24,7 @@ using Microsoft.Python.Core.IO; using Microsoft.Python.LanguageServer.Indexing; using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; using NSubstitute; @@ -41,7 +43,6 @@ public class IndexParserTests : LanguageServerTestBase { [TestInitialize] public void TestInitialize() { TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); - _symbolIndex = new SymbolIndex(); _fileSystem = Substitute.For(); _pythonLanguageVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); } @@ -49,14 +50,6 @@ public void TestInitialize() { [TestCleanup] public void Cleanup() => TestEnvironmentImpl.TestCleanup(); - [TestMethod, Priority(0)] - public void NullIndexThrowsException() { - Action build = () => { - IIndexParser indexParser = new IndexParser(null, _fileSystem, _pythonLanguageVersion); - }; - build.Should().Throw(); - } - [TestMethod, Priority(0)] public async Task ParseVariableInFileAsync() { const string testFilePath = "C:/bla.py"; @@ -64,39 +57,33 @@ public async Task ParseVariableInFileAsync() { using (var fileStream = MakeStream("x = 1")) { SetFileOpen(_fileSystem, testFilePath, fileStream); - IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); - await indexParser.ParseAsync(testFilePath); + IIndexParser indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion); + var ast = await indexParser.ParseAsync(testFilePath); + + var symbols = GetIndexSymbols(ast); + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("x"); } - var symbols = _symbolIndex.WorkspaceSymbols(""); - symbols.Should().HaveCount(1); - symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); - symbols.First().Name.Should().BeEquivalentTo("x"); } - [TestMethod, Priority(0)] - public async Task ParseNonexistentFile() { - const string testFilePath = "C:/bla.py"; - SetFileOpen(_fileSystem, testFilePath, _ => throw new FileNotFoundException()); - - IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); - await indexParser.ParseAsync(testFilePath); - - var symbols = _symbolIndex.WorkspaceSymbols(""); - symbols.Should().HaveCount(0); + private IReadOnlyList GetIndexSymbols(PythonAst ast) { + var walker = new SymbolIndexWalker(ast); + ast.Walk(walker); + return walker.Symbols; } + [TestMethod, Priority(0)] + [ExpectedException(typeof(FileNotFoundException))] public async Task ParseFileThatStopsExisting() { const string testFilePath = "C:/bla.py"; _fileSystem.FileExists(testFilePath).Returns(true); SetFileOpen(_fileSystem, testFilePath, _ => throw new FileNotFoundException()); - IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); - await indexParser.ParseAsync(testFilePath); - - var symbols = _symbolIndex.WorkspaceSymbols(""); - symbols.Should().HaveCount(0); + IIndexParser indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion); + var ast = await indexParser.ParseAsync(testFilePath); } [TestMethod, Priority(0)] @@ -108,7 +95,7 @@ public void CancelParsingAsync() { SetFileOpen(_fileSystem, testFilePath, fileStream); } - IIndexParser indexParser = new IndexParser(_symbolIndex, _fileSystem, _pythonLanguageVersion); + IIndexParser indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); cancellationTokenSource.Cancel(); @@ -116,9 +103,6 @@ public void CancelParsingAsync() { await indexParser.ParseAsync(testFilePath, cancellationTokenSource.Token); }; parse.Should().Throw(); - - var symbols = _symbolIndex.WorkspaceSymbols(""); - symbols.Should().HaveCount(0); } private void SetFileOpen(IFileSystem fileSystem, string path, Stream stream) { diff --git a/src/LanguageServer/Test/SymbolIndexTests.cs b/src/LanguageServer/Test/SymbolIndexTests.cs index 1a66fee65..479f89d0a 100644 --- a/src/LanguageServer/Test/SymbolIndexTests.cs +++ b/src/LanguageServer/Test/SymbolIndexTests.cs @@ -12,19 +12,30 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; using System.IO; +using System.Text; +using System.Threading.Tasks; using FluentAssertions; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Core.IO; using Microsoft.Python.Core.Text; using Microsoft.Python.LanguageServer.Indexing; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; +using Microsoft.Python.Parsing.Tests; using Microsoft.Python.Tests.Utilities.FluentAssertions; using Microsoft.VisualStudio.TestTools.UnitTesting; +using NSubstitute; using TestUtilities; + namespace Microsoft.Python.LanguageServer.Tests { [TestClass] public class SymbolIndexTests { + private const int maxSymbols = 1000; + private readonly string _rootPath = "C:/root"; + public TestContext TestContext { get; set; } [TestInitialize] @@ -38,635 +49,61 @@ public void TestCleanup() { } [TestMethod, Priority(0)] - public void WalkerAssignments() { - var code = @"x = 1 -y = x -z = y"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 1, 2, 2)), - new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(3, 1, 3, 2)), - }); - } - - [TestMethod, Priority(0)] - public void WalkerMultipleAssignments() { - var code = @"x = y = z = 1"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 5, 1, 6)), - new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(1, 9, 1, 10)), - }); - } - - [TestMethod, Priority(0)] - public void WalkerUnderscore() { - var code = @"_ = 1"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEmpty(); - } - - [TestMethod, Priority(0)] - public void WalkerIfStatement() { - var code = @"if foo(): - x = 1 -elif bar(): - x = 2 -else: - y = 3 -"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(4, 5, 4, 6)), - new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(6, 5, 6, 6)), - }); - } - - [TestMethod, Priority(0)] - public void WalkerTryExceptFinally() { - var code = @"try: - x = 1 -except Exception: - x = 2 -finally: - y = 3 -"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(4, 5, 4, 6)), - new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(6, 5, 6, 6)), - }); - } - - [TestMethod, Priority(0)] - public void WalkerReassign() { - var code = @"x = 1 -x = 2"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - }); - } - - [TestMethod, Priority(0)] - public void WalkerAugmentedAssign() { - var code = @"x += 1"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - }); - } - - [TestMethod, Priority(0)] - public void WalkerTopLevelConstant() { - var code = @"FOO_BAR_3 = 1234"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("FOO_BAR_3", SymbolKind.Constant, new SourceSpan(1, 1, 1, 10)), - }); - } - - [TestMethod, Priority(0)] - public void WalkerFunction() { - var code = @"def func(x, y): - z = x + y - return z"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 3, 13), new SourceSpan(1, 5, 1, 9), new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 10, 1, 11)), - new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), - new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), - }, FunctionKind.Function), - }); - } - - [TestMethod, Priority(0)] - public void WalkerFunctionStarredArgs() { - var code = @"def func(*args, **kwargs): ..."; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 31), new SourceSpan(1, 5, 1, 9), new[] { - new HierarchicalSymbol("args", SymbolKind.Variable, new SourceSpan(1, 11, 1, 15)), - new HierarchicalSymbol("kwargs", SymbolKind.Variable, new SourceSpan(1, 19, 1, 25)), - }, FunctionKind.Function), - }); - } - - [TestMethod, Priority(0)] - public void WalkerFunctionUnderscoreArg() { - var code = @"def func(_): ..."; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 17), new SourceSpan(1, 5, 1, 9), new List(), FunctionKind.Function), - }); - } - - [TestMethod, Priority(0)] - public void WalkerImports() { - var code = @"import sys -import numpy as np -from os.path import join as osjoin -from os.path import ( join as osjoin2, exists as osexists, expanduser ) -"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("sys", SymbolKind.Module, new SourceSpan(1, 8, 1, 11)), - new HierarchicalSymbol("np", SymbolKind.Module, new SourceSpan(2, 17, 2, 19)), - new HierarchicalSymbol("osjoin", SymbolKind.Module, new SourceSpan(3, 29, 3, 35)), - new HierarchicalSymbol("osjoin2", SymbolKind.Module, new SourceSpan(4, 31, 4, 38)), - new HierarchicalSymbol("osexists", SymbolKind.Module, new SourceSpan(4, 50, 4, 58)), - new HierarchicalSymbol("expanduser", SymbolKind.Module, new SourceSpan(4, 60, 4, 70)), - }); - } - - [TestMethod, Priority(0)] - public void WalkerImportFromFuture() { - var code = @"from __future__ import print_function"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEmpty(); - } - - [TestMethod, Priority(0)] - public void WalkerClass() { - var code = @"class Foo(object): - ..."; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 8), new SourceSpan(1, 7, 1, 10), new List(), FunctionKind.Class), - }); - } - - [TestMethod, Priority(0)] - public void WalkerClassConstant() { - var code = @"class Foo(object): - CONSTANT = 1234"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 20), new SourceSpan(1, 7, 1, 10), new[] { - new HierarchicalSymbol("CONSTANT", SymbolKind.Constant, new SourceSpan(2, 5, 2, 13)), - }, FunctionKind.Class), - }); - } - - [TestMethod, Priority(0)] - public void WalkerConstructor() { - var code = @"class Foo(object): - def __init__(self, x): ..."; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 31), new SourceSpan(1, 7, 1, 10), new[] { - new HierarchicalSymbol("__init__", SymbolKind.Constructor, new SourceSpan(2, 5, 2, 31), new SourceSpan(2, 9, 2, 17), new[] { - new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 18, 2, 22)), - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 24, 2, 25)), - }, FunctionKind.Function), - }, FunctionKind.Class), - }); - } - - [TestMethod, Priority(0)] - public void WalkerMethod() { - var code = @"class Foo(object): - def foo(self, x): ..."; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 26), new SourceSpan(1, 7, 1, 10), new[] { - new HierarchicalSymbol("foo", SymbolKind.Method, new SourceSpan(2, 5, 2, 26), new SourceSpan(2, 9, 2, 12), new[] { - new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 13, 2, 17)), - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 19, 2, 20)), - }, FunctionKind.Function), - }, FunctionKind.Class), - }); - } - - [TestMethod, Priority(0)] - public void WalkerDoubleUnderscoreMethod() { - var code = @"class Foo(object): - def __lt__(self, x): ..."; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 29), new SourceSpan(1, 7, 1, 10), new[] { - new HierarchicalSymbol("__lt__", SymbolKind.Operator, new SourceSpan(2, 5, 2, 29), new SourceSpan(2, 9, 2, 15), new[] { - new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 16, 2, 20)), - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 22, 2, 23)), - }, FunctionKind.Function), - }, FunctionKind.Class), - }); - } - - [TestMethod, Priority(0)] - public void WalkerProperties() { - var code = @"class Foo(object): - @property - def func1(self): ... - - @abstractproperty - def func2(self): ... - - @classproperty - def func3(self): ... - - @abstractclassproperty - def func4(self): ..."; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 12, 25), new SourceSpan(1, 7, 1, 10), new[] { - new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { - new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), - }, FunctionKind.Property), - new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { - new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), - }, FunctionKind.Property), - new HierarchicalSymbol("func3", SymbolKind.Property, new SourceSpan(8, 5, 9, 25), new SourceSpan(9, 9, 9, 14), new[] { - new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(9, 15, 9, 19)), - }, FunctionKind.Property), - new HierarchicalSymbol("func4", SymbolKind.Property, new SourceSpan(11, 5, 12, 25), new SourceSpan(12, 9, 12, 14), new[] { - new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(12, 15, 12, 19)), - }, FunctionKind.Property), - }, FunctionKind.Class), - }); - } - - [TestMethod, Priority(0)] - public void WalkerAbcProperties() { - var code = @"class Foo(object): - @abc.abstractproperty - def func1(self): ... - - @abc.abstractclassproperty - def func2(self): ..."; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 6, 25), new SourceSpan(1, 7, 1, 10), new[] { - new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { - new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), - }, FunctionKind.Property), - new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { - new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), - }, FunctionKind.Property), - }, FunctionKind.Class), - }); - } - - [TestMethod, Priority(0)] - public void WalkerStaticMethods() { - var code = @"class Foo(object): - @staticmethod - def func1(arg): ... - - @abstractstaticmethod - def func2(arg): ... - - @abc.abstractstaticmethod - def func3(arg): ..."; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 9, 24), new SourceSpan(1, 7, 1, 10), new[] { - new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 24), new SourceSpan(3, 9, 3, 14), new[] { - new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(3, 15, 3, 18)), - }, FunctionKind.StaticMethod), - new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 24), new SourceSpan(6, 9, 6, 14), new[] { - new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(6, 15, 6, 18)), - }, FunctionKind.StaticMethod), - new HierarchicalSymbol("func3", SymbolKind.Method, new SourceSpan(8, 5, 9, 24), new SourceSpan(9, 9, 9, 14), new[] { - new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(9, 15, 9, 18)), - }, FunctionKind.StaticMethod), - }, FunctionKind.Class), - }); - } - - [TestMethod, Priority(0)] - public void WalkerClassMethods() { - var code = @"class Foo(object): - @classmethod - def func1(cls): ... - - @abstractclassmethod - def func2(cls): ... - - @abc.abstractclassmethod - def func3(cls): ..."; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 9, 24), new SourceSpan(1, 7, 1, 10), new[] { - new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 24), new SourceSpan(3, 9, 3, 14), new[] { - new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(3, 15, 3, 18)), - }, FunctionKind.ClassMethod), - new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 24), new SourceSpan(6, 9, 6, 14), new[] { - new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(6, 15, 6, 18)), - }, FunctionKind.ClassMethod), - new HierarchicalSymbol("func3", SymbolKind.Method, new SourceSpan(8, 5, 9, 24), new SourceSpan(9, 9, 9, 14), new[] { - new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(9, 15, 9, 18)), - }, FunctionKind.ClassMethod), - }, FunctionKind.Class), - }); - } - - [TestMethod, Priority(0)] - public void WalkerTopLevelFunctionDecorator() { - var code = @"@something -def func1(x, y): ... - -@something_else() -def func2(x, y): ..."; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("func1", SymbolKind.Function, new SourceSpan(1, 1, 2, 21), new SourceSpan(2, 5, 2, 10), new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 11, 2, 12)), - new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 14, 2, 15)), - }, FunctionKind.Function), - new HierarchicalSymbol("func2", SymbolKind.Function, new SourceSpan(4, 1, 5, 21), new SourceSpan(5, 5, 5, 10), new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(5, 11, 5, 12)), - new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(5, 14, 5, 15)), - }, FunctionKind.Function), - }); - } - - [TestMethod, Priority(0)] - public void WalkerClassFunctionDecorator() { - var code = @"class Foo(object): - @something - def func1(self): ... - - @something_else() - def func2(self): ..."; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 6, 25), new SourceSpan(1, 7, 1, 10), new[] { - new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { - new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), - }, FunctionKind.Function), - new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { - new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), - }, FunctionKind.Function), - }, FunctionKind.Class), - }); - } - - [TestMethod, Priority(0)] - public void WalkerClassFunctionMultiDecorator() { - var code = @"class Foo(object): - @property - @something - def func1(self): ... - - @something - @property - def func2(self): ..."; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 8, 25), new SourceSpan(1, 7, 1, 10), new[] { - new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 4, 25), new SourceSpan(4, 9, 4, 14), new[] { - new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(4, 15, 4, 19)), - }, FunctionKind.Property), - new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(6, 5, 8, 25), new SourceSpan(8, 9, 8, 14), new[] { - new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(8, 15, 8, 19)), - }, FunctionKind.Property), - }, FunctionKind.Class), - }); - } - - [TestMethod, Priority(0)] - public void WalkerLambda() { - var code = @"f = lambda x, y: x + y"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("", SymbolKind.Function, new SourceSpan(1, 5, 1, 23), children: new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 12, 1, 13)), - new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 15, 1, 16)), - }, functionKind: FunctionKind.Function), - new HierarchicalSymbol("f", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - }); - } - - [TestMethod, Priority(0)] - public void WalkerForLoop() { - var code = @"z = False -for [x, y, (p, q)] in [[1, 2, [3, 4]]]: - z += x -else: - z = None"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 6, 2, 7)), - new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 9, 2, 10)), - new HierarchicalSymbol("p", SymbolKind.Variable, new SourceSpan(2, 13, 2, 14)), - new HierarchicalSymbol("q", SymbolKind.Variable, new SourceSpan(2, 16, 2, 17)), - new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(3, 5, 3, 6)), - new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(5, 5, 5, 6)), - }); - } - - [TestMethod, Priority(0)] - public void WalkerListComprehension() { - var code = @"flat_list = [item for sublist in l for item in sublist]"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 13, 1, 56), children: new[] { - new HierarchicalSymbol("sublist", SymbolKind.Variable, new SourceSpan(1, 23, 1, 30)), - new HierarchicalSymbol("item", SymbolKind.Variable, new SourceSpan(1, 40, 1, 44)), - }), - new HierarchicalSymbol("flat_list", SymbolKind.Variable, new SourceSpan(1, 1, 1, 10)), - }); - } - - [TestMethod, Priority(0)] - public void WalkerDictionaryComprehension() { - var code = @"d = { x: y for x, y in zip(range(10), range(10)) }"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 51), children: new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), - new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 19, 1, 20)), - }), - new HierarchicalSymbol("d", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - }); - } - - [TestMethod, Priority(0)] - public void WalkerSetComprehension() { - var code = @"s = { x for x in range(10) }"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 29), children: new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), - }), - new HierarchicalSymbol("s", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - }); - } - - [TestMethod, Priority(0)] - public void WalkerGenerator() { - var code = @"g = (x + y for x, y in zip(range(10), range(10)))"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 50), children: new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), - new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 19, 1, 20)), - }), - new HierarchicalSymbol("g", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - }); - } - - [TestMethod, Priority(0)] - public void WalkerNestedListComprehension() { - var code = @"l = [ - x for x in [ - y for y in range(10) - ] -]"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 5, 2), children: new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 11, 2, 12)), - new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(2, 16, 4, 6), children: new[] { - new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(3, 15, 3, 16)), - }), - }), - new HierarchicalSymbol("l", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - }); - } - - [TestMethod, Priority(0)] - public void WalkerIncompleteFunction() { - var code = @"def func(x, y):"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 16), new SourceSpan(1, 5, 1, 9), new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 10, 1, 11)), - new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), - }, FunctionKind.Function), - }); - } - - [TestMethod, Priority(0)] - public void WalkerIncompleteClass() { - var code = @"class Foo(object):"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 1, 19), new SourceSpan(1, 7, 1, 10), new List(), FunctionKind.Class), - }); - } - - [TestMethod, Priority(0)] - public void WalkerIncompleteAssign() { - var code = @"x ="; + public async Task IndexHierarchicalDocumentAsync() { + ISymbolIndex index = MakeSymbolIndex(); + var path = TestData.GetDefaultModulePath(); + index.Add(path, DocumentWithAst("x = 1")); - var symbols = WalkSymbols(code); + var symbols = await index.HierarchicalDocumentSymbols(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); } - [TestMethod, Priority(0)] - public void WalkerAugmentedAssignLambda() { - var code = @"x += lambda x, y: x + y"; - - var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("", SymbolKind.Function, new SourceSpan(1, 6, 1, 24), children: new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), - new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), - }, functionKind: FunctionKind.Function), - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - }); + private static ISymbolIndex MakeSymbolIndex() { + return new SymbolIndex(Substitute.For(), PythonLanguageVersion.V38); } [TestMethod, Priority(0)] - public void IndexHierarchicalDocument() { - ISymbolIndex index = new SymbolIndex(); + public async Task IndexHierarchicalDocumentUpdate() { + ISymbolIndex index = MakeSymbolIndex(); var path = TestData.GetDefaultModulePath(); - var ast = GetParse("x = 1"); - index.Add(path, ast); - var symbols = index.HierarchicalDocumentSymbols(path); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - }); - } - - [TestMethod, Priority(0)] - public void IndexHierarchicalDocumentUpdate() { - ISymbolIndex index = new SymbolIndex(); - var path = TestData.GetDefaultModulePath(); - var ast = GetParse("x = 1"); + index.Add(path, DocumentWithAst("x = 1")); - index.Add(path, ast); - - var symbols = index.HierarchicalDocumentSymbols(path); + var symbols = await index.HierarchicalDocumentSymbols(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); - ast = GetParse("y = 1"); - index.Add(path, ast); + index.Add(path, DocumentWithAst("y = 1")); - symbols = index.HierarchicalDocumentSymbols(path); + symbols = await index.HierarchicalDocumentSymbols(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); } [TestMethod, Priority(0)] - public void IndexHierarchicalDocumentNotFound() { - ISymbolIndex index = new SymbolIndex(); + public async Task IndexHierarchicalDocumentNotFoundAsync() { + ISymbolIndex index = MakeSymbolIndex(); var path = TestData.GetDefaultModulePath(); - var symbols = index.HierarchicalDocumentSymbols(path); + var symbols = await index.HierarchicalDocumentSymbols(path); symbols.Should().BeEmpty(); } [TestMethod, Priority(0)] - public void IndexWorkspaceSymbolsFlatten() { + public async Task IndexWorkspaceSymbolsFlattenAsync() { var code = @"class Foo(object): def foo(self, x): ..."; - ISymbolIndex index = new SymbolIndex(); + ISymbolIndex index = MakeSymbolIndex(); var path = TestData.GetDefaultModulePath(); - var ast = GetParse(code); - index.Add(path, ast); + index.Add(path, DocumentWithAst(code)); - var symbols = index.WorkspaceSymbols(""); + var symbols = await index.WorkspaceSymbolsAsync("", maxSymbols); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new FlatSymbol("Foo", SymbolKind.Class, path, new SourceSpan(1, 7, 1, 10)), new FlatSymbol("foo", SymbolKind.Method, path, new SourceSpan(2, 9, 2, 12), "Foo"), @@ -676,43 +113,41 @@ public void IndexWorkspaceSymbolsFlatten() { } [TestMethod, Priority(0)] - public void IndexWorkspaceSymbolsFiltered() { + public async Task IndexWorkspaceSymbolsFilteredAsync() { var code = @"class Foo(object): def foo(self, x): ..."; - ISymbolIndex index = new SymbolIndex(); + ISymbolIndex index = MakeSymbolIndex(); var path = TestData.GetDefaultModulePath(); - var ast = GetParse(code); - index.Add(path, ast); + index.Add(path, DocumentWithAst(code)); - var symbols = index.WorkspaceSymbols("x"); + var symbols = await index.WorkspaceSymbolsAsync("x", maxSymbols); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(2, 19, 2, 20), "foo"), }); } [TestMethod, Priority(0)] - public void IndexWorkspaceSymbolsNotFound() { - ISymbolIndex index = new SymbolIndex(); + public async Task IndexWorkspaceSymbolsNotFoundAsync() { + ISymbolIndex index = MakeSymbolIndex(); var path = TestData.GetDefaultModulePath(); - var symbols = index.WorkspaceSymbols(""); + var symbols = await index.WorkspaceSymbolsAsync("", maxSymbols); symbols.Should().BeEmpty(); } [TestMethod, Priority(0)] - public void IndexWorkspaceSymbolsCaseInsensitive() { + public async Task IndexWorkspaceSymbolsCaseInsensitiveAsync() { var code = @"class Foo(object): def foo(self, x): ..."; - ISymbolIndex index = new SymbolIndex(); + ISymbolIndex index = MakeSymbolIndex(); var path = TestData.GetDefaultModulePath(); - var ast = GetParse(code); - index.Add(path, ast); + index.Add(path, DocumentWithAst(code)); - var symbols = index.WorkspaceSymbols("foo"); + var symbols = await index.WorkspaceSymbolsAsync("foo", maxSymbols); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new FlatSymbol("Foo", SymbolKind.Class, path, new SourceSpan(1, 7, 1, 10)), new FlatSymbol("foo", SymbolKind.Method, path, new SourceSpan(2, 9, 2, 12), "Foo"), @@ -728,5 +163,23 @@ private IReadOnlyList WalkSymbols(string code, PythonLanguag ast.Walk(walker); return walker.Symbols; } + + private IDocument DocumentWithAst(string testCode, string filePath = null) { + filePath = filePath ?? $"{_rootPath}/{testCode}.py"; + IDocument doc = Substitute.For(); + doc.GetAstAsync().ReturnsForAnyArgs(Task.FromResult(MakeAst(testCode))); + doc.Uri.Returns(new Uri(filePath)); + return doc; + } + + private PythonAst MakeAst(string testCode) { + PythonLanguageVersion latestVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); + return Parser.CreateParser(MakeStream(testCode), latestVersion).ParseFile(); + } + + private Stream MakeStream(string str) { + return new MemoryStream(Encoding.UTF8.GetBytes(str)); + } + } } diff --git a/src/LanguageServer/Test/SymbolIndexWalkerTests.cs b/src/LanguageServer/Test/SymbolIndexWalkerTests.cs new file mode 100644 index 000000000..a8ad2cc3b --- /dev/null +++ b/src/LanguageServer/Test/SymbolIndexWalkerTests.cs @@ -0,0 +1,624 @@ +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.IO; +using FluentAssertions; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Indexing; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; +using Microsoft.Python.Tests.Utilities.FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; +namespace Microsoft.Python.LanguageServer.Tests { + [TestClass] + public class SymbolIndexWalkerTests { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() { + TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + } + + [TestCleanup] + public void TestCleanup() { + TestEnvironmentImpl.TestCleanup(); + } + + [TestMethod, Priority(0)] + public void WalkerAssignments() { + var code = @"x = 1 +y = x +z = y"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 1, 2, 2)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(3, 1, 3, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerMultipleAssignments() { + var code = @"x = y = z = 1"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 5, 1, 6)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(1, 9, 1, 10)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerUnderscore() { + var code = @"_ = 1"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public void WalkerIfStatement() { + var code = @"if foo(): + x = 1 +elif bar(): + x = 2 +else: + y = 3 +"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(4, 5, 4, 6)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(6, 5, 6, 6)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerTryExceptFinally() { + var code = @"try: + x = 1 +except Exception: + x = 2 +finally: + y = 3 +"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(4, 5, 4, 6)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(6, 5, 6, 6)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerReassign() { + var code = @"x = 1 +x = 2"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerAugmentedAssign() { + var code = @"x += 1"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerTopLevelConstant() { + var code = @"FOO_BAR_3 = 1234"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("FOO_BAR_3", SymbolKind.Constant, new SourceSpan(1, 1, 1, 10)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerFunction() { + var code = @"def func(x, y): + z = x + y + return z"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 3, 13), new SourceSpan(1, 5, 1, 9), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 10, 1, 11)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerFunctionStarredArgs() { + var code = @"def func(*args, **kwargs): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 31), new SourceSpan(1, 5, 1, 9), new[] { + new HierarchicalSymbol("args", SymbolKind.Variable, new SourceSpan(1, 11, 1, 15)), + new HierarchicalSymbol("kwargs", SymbolKind.Variable, new SourceSpan(1, 19, 1, 25)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerFunctionUnderscoreArg() { + var code = @"def func(_): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 17), new SourceSpan(1, 5, 1, 9), new List(), FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerImports() { + var code = @"import sys +import numpy as np +from os.path import join as osjoin +from os.path import ( join as osjoin2, exists as osexists, expanduser ) +"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("sys", SymbolKind.Module, new SourceSpan(1, 8, 1, 11)), + new HierarchicalSymbol("np", SymbolKind.Module, new SourceSpan(2, 17, 2, 19)), + new HierarchicalSymbol("osjoin", SymbolKind.Module, new SourceSpan(3, 29, 3, 35)), + new HierarchicalSymbol("osjoin2", SymbolKind.Module, new SourceSpan(4, 31, 4, 38)), + new HierarchicalSymbol("osexists", SymbolKind.Module, new SourceSpan(4, 50, 4, 58)), + new HierarchicalSymbol("expanduser", SymbolKind.Module, new SourceSpan(4, 60, 4, 70)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerImportFromFuture() { + var code = @"from __future__ import print_function"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public void WalkerClass() { + var code = @"class Foo(object): + ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 8), new SourceSpan(1, 7, 1, 10), new List(), FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassConstant() { + var code = @"class Foo(object): + CONSTANT = 1234"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 20), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("CONSTANT", SymbolKind.Constant, new SourceSpan(2, 5, 2, 13)), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerConstructor() { + var code = @"class Foo(object): + def __init__(self, x): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 31), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("__init__", SymbolKind.Constructor, new SourceSpan(2, 5, 2, 31), new SourceSpan(2, 9, 2, 17), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 18, 2, 22)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 24, 2, 25)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerMethod() { + var code = @"class Foo(object): + def foo(self, x): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 26), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("foo", SymbolKind.Method, new SourceSpan(2, 5, 2, 26), new SourceSpan(2, 9, 2, 12), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 13, 2, 17)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 19, 2, 20)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerDoubleUnderscoreMethod() { + var code = @"class Foo(object): + def __lt__(self, x): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 29), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("__lt__", SymbolKind.Operator, new SourceSpan(2, 5, 2, 29), new SourceSpan(2, 9, 2, 15), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 16, 2, 20)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 22, 2, 23)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerProperties() { + var code = @"class Foo(object): + @property + def func1(self): ... + + @abstractproperty + def func2(self): ... + + @classproperty + def func3(self): ... + + @abstractclassproperty + def func4(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 12, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func3", SymbolKind.Property, new SourceSpan(8, 5, 9, 25), new SourceSpan(9, 9, 9, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(9, 15, 9, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func4", SymbolKind.Property, new SourceSpan(11, 5, 12, 25), new SourceSpan(12, 9, 12, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(12, 15, 12, 19)), + }, FunctionKind.Property), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerAbcProperties() { + var code = @"class Foo(object): + @abc.abstractproperty + def func1(self): ... + + @abc.abstractclassproperty + def func2(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 6, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), + }, FunctionKind.Property), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerStaticMethods() { + var code = @"class Foo(object): + @staticmethod + def func1(arg): ... + + @abstractstaticmethod + def func2(arg): ... + + @abc.abstractstaticmethod + def func3(arg): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 9, 24), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 24), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(3, 15, 3, 18)), + }, FunctionKind.StaticMethod), + new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 24), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(6, 15, 6, 18)), + }, FunctionKind.StaticMethod), + new HierarchicalSymbol("func3", SymbolKind.Method, new SourceSpan(8, 5, 9, 24), new SourceSpan(9, 9, 9, 14), new[] { + new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(9, 15, 9, 18)), + }, FunctionKind.StaticMethod), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassMethods() { + var code = @"class Foo(object): + @classmethod + def func1(cls): ... + + @abstractclassmethod + def func2(cls): ... + + @abc.abstractclassmethod + def func3(cls): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 9, 24), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 24), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(3, 15, 3, 18)), + }, FunctionKind.ClassMethod), + new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 24), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(6, 15, 6, 18)), + }, FunctionKind.ClassMethod), + new HierarchicalSymbol("func3", SymbolKind.Method, new SourceSpan(8, 5, 9, 24), new SourceSpan(9, 9, 9, 14), new[] { + new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(9, 15, 9, 18)), + }, FunctionKind.ClassMethod), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerTopLevelFunctionDecorator() { + var code = @"@something +def func1(x, y): ... + +@something_else() +def func2(x, y): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func1", SymbolKind.Function, new SourceSpan(1, 1, 2, 21), new SourceSpan(2, 5, 2, 10), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 11, 2, 12)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 14, 2, 15)), + }, FunctionKind.Function), + new HierarchicalSymbol("func2", SymbolKind.Function, new SourceSpan(4, 1, 5, 21), new SourceSpan(5, 5, 5, 10), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(5, 11, 5, 12)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(5, 14, 5, 15)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassFunctionDecorator() { + var code = @"class Foo(object): + @something + def func1(self): ... + + @something_else() + def func2(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 6, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), + }, FunctionKind.Function), + new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassFunctionMultiDecorator() { + var code = @"class Foo(object): + @property + @something + def func1(self): ... + + @something + @property + def func2(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 8, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 4, 25), new SourceSpan(4, 9, 4, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(4, 15, 4, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(6, 5, 8, 25), new SourceSpan(8, 9, 8, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(8, 15, 8, 19)), + }, FunctionKind.Property), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerLambda() { + var code = @"f = lambda x, y: x + y"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.Function, new SourceSpan(1, 5, 1, 23), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 12, 1, 13)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 15, 1, 16)), + }, functionKind: FunctionKind.Function), + new HierarchicalSymbol("f", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerForLoop() { + var code = @"z = False +for [x, y, (p, q)] in [[1, 2, [3, 4]]]: + z += x +else: + z = None"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 6, 2, 7)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 9, 2, 10)), + new HierarchicalSymbol("p", SymbolKind.Variable, new SourceSpan(2, 13, 2, 14)), + new HierarchicalSymbol("q", SymbolKind.Variable, new SourceSpan(2, 16, 2, 17)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(3, 5, 3, 6)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(5, 5, 5, 6)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerListComprehension() { + var code = @"flat_list = [item for sublist in l for item in sublist]"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 13, 1, 56), children: new[] { + new HierarchicalSymbol("sublist", SymbolKind.Variable, new SourceSpan(1, 23, 1, 30)), + new HierarchicalSymbol("item", SymbolKind.Variable, new SourceSpan(1, 40, 1, 44)), + }), + new HierarchicalSymbol("flat_list", SymbolKind.Variable, new SourceSpan(1, 1, 1, 10)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerDictionaryComprehension() { + var code = @"d = { x: y for x, y in zip(range(10), range(10)) }"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 51), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 19, 1, 20)), + }), + new HierarchicalSymbol("d", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerSetComprehension() { + var code = @"s = { x for x in range(10) }"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 29), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + }), + new HierarchicalSymbol("s", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerGenerator() { + var code = @"g = (x + y for x, y in zip(range(10), range(10)))"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 50), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 19, 1, 20)), + }), + new HierarchicalSymbol("g", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerNestedListComprehension() { + var code = @"l = [ + x for x in [ + y for y in range(10) + ] +]"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 5, 2), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 11, 2, 12)), + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(2, 16, 4, 6), children: new[] { + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(3, 15, 3, 16)), + }), + }), + new HierarchicalSymbol("l", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerIncompleteFunction() { + var code = @"def func(x, y):"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 16), new SourceSpan(1, 5, 1, 9), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 10, 1, 11)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerIncompleteClass() { + var code = @"class Foo(object):"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 1, 19), new SourceSpan(1, 7, 1, 10), new List(), FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerIncompleteAssign() { + var code = @"x ="; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerAugmentedAssignLambda() { + var code = @"x += lambda x, y: x + y"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.Function, new SourceSpan(1, 6, 1, 24), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), + }, functionKind: FunctionKind.Function), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + private PythonAst GetParse(string code, PythonLanguageVersion version = PythonLanguageVersion.V37) + => Parser.CreateParser(new StringReader(code), version).ParseFile(); + + private IReadOnlyList WalkSymbols(string code, PythonLanguageVersion version = PythonLanguageVersion.V37) { + var ast = GetParse(code); + var walker = new SymbolIndexWalker(ast); + ast.Walk(walker); + return walker.Symbols; + } + } +} From d04d3754b78303b9c0a3df0a13570fbbbf52e2f8 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 19 Feb 2019 16:28:04 -0800 Subject: [PATCH 077/123] Making interface for mostrecent symbols --- .../Impl/Indexing/IMostRecentDocumentSymbols.cs | 14 ++++++++++++++ ...centSymbols.cs => MostRecentDocumentSymbols.cs} | 2 +- src/LanguageServer/Impl/Indexing/SymbolIndex.cs | 8 ++++---- 3 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs rename src/LanguageServer/Impl/Indexing/{MostRecentSymbols.cs => MostRecentDocumentSymbols.cs} (98%) diff --git a/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs new file mode 100644 index 000000000..db10b973e --- /dev/null +++ b/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Documents; + +namespace Microsoft.Python.LanguageServer.Indexing { + interface IMostRecentDocumentSymbols : IDisposable { + void Parse(); + void Add(IDocument doc); + void ReIndex(IDocument doc); + Task> GetSymbolsAsync(); + void MarkAsPending(); + } +} diff --git a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs similarity index 98% rename from src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs rename to src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index c801e2e20..e8ec34791 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -11,7 +11,7 @@ using Microsoft.Python.Parsing; namespace Microsoft.Python.LanguageServer.Indexing { - class MostRecentDocumentSymbols : IDisposable { + class MostRecentDocumentSymbols : IMostRecentDocumentSymbols { private readonly object _syncObj = new object(); private readonly IIndexParser _indexParser; private readonly string _path; diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index 8efb80434..333bc2c6e 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -26,7 +26,7 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal sealed class SymbolIndex : ISymbolIndex { - private readonly ConcurrentDictionary _index; + private readonly ConcurrentDictionary _index; private readonly IFileSystem _fileSystem; private readonly PythonLanguageVersion _version; @@ -35,7 +35,7 @@ public SymbolIndex(IFileSystem fileSystem, PythonLanguageVersion version) { _version = version; var comparer = PathEqualityComparer.Instance; - _index = new ConcurrentDictionary(comparer); + _index = new ConcurrentDictionary(comparer); } public Task> HierarchicalDocumentSymbols(string path) { @@ -61,7 +61,7 @@ public async Task> WorkspaceSymbolsAsync(string query, return results; } - private async Task> WorkspaceSymbolsQueryAsync(string filePath, string query, MostRecentDocumentSymbols recentSymbols, CancellationToken cancellationToken) { + private async Task> WorkspaceSymbolsQueryAsync(string filePath, string query, IMostRecentDocumentSymbols recentSymbols, CancellationToken cancellationToken) { var symbols = await recentSymbols.GetSymbolsAsync(); cancellationToken.ThrowIfCancellationRequested(); return WorkspaceSymbolsQuery(filePath, query, symbols); @@ -111,7 +111,7 @@ private IReadOnlyList WorkspaceSymbolsQuery(string path, string quer return symbols.Select((symbol) => (symbol, parentName)).ToList(); } - private MostRecentDocumentSymbols MakeMostRecentDocSymbols(string path) { + private IMostRecentDocumentSymbols MakeMostRecentDocSymbols(string path) { return new MostRecentDocumentSymbols(path, _fileSystem, _version); } } From f7f09fb442cfc3268bf67da604075a252d0613a2 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 19 Feb 2019 16:51:32 -0800 Subject: [PATCH 078/123] Copyright comments --- .../Impl/Indexing/IMostRecentDocumentSymbols.cs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs index db10b973e..c26179dd3 100644 --- a/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs @@ -1,4 +1,19 @@ -using System; +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.Python.Analysis.Documents; From 99526816f56b337f337a158e92dcc9794d3a2d9a Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 19 Feb 2019 16:57:37 -0800 Subject: [PATCH 079/123] Unused var --- src/LanguageServer/Test/IndexParserTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/LanguageServer/Test/IndexParserTests.cs b/src/LanguageServer/Test/IndexParserTests.cs index 4187f312d..ecd86654d 100644 --- a/src/LanguageServer/Test/IndexParserTests.cs +++ b/src/LanguageServer/Test/IndexParserTests.cs @@ -34,7 +34,6 @@ namespace Microsoft.Python.LanguageServer.Tests { [TestClass] public class IndexParserTests : LanguageServerTestBase { - private ISymbolIndex _symbolIndex; private IFileSystem _fileSystem; private PythonLanguageVersion _pythonLanguageVersion; From 91b836ca52cd87e3336b49c7c01186ce86a2c019 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 19 Feb 2019 16:58:02 -0800 Subject: [PATCH 080/123] Changin default delay --- src/LanguageServer/Impl/Indexing/IndexManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index e9340e6c1..c5af2c0d2 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -28,7 +28,7 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal class IndexManager : IIndexManager { - private static int DefaultReIndexDelay = 1000; + private static int DefaultReIndexDelay = 350; private readonly ISymbolIndex _symbolIndex; private readonly IFileSystem _fileSystem; private readonly string _workspaceRootPath; From 29ecef7c2713ee8866165e3c06cbc4a9c69d47cb Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 19 Feb 2019 17:00:53 -0800 Subject: [PATCH 081/123] Rename and style correct --- src/LanguageServer/Impl/Indexing/SymbolIndex.cs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index 333bc2c6e..4d9e12d75 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -20,7 +20,6 @@ using System.Threading.Tasks; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core; -using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; @@ -51,14 +50,9 @@ public async Task> WorkspaceSymbolsAsync(string query, .Select(kvp => WorkspaceSymbolsQueryAsync(kvp.Key, query, kvp.Value, ct)) .Take(maxLength) .ToArray(); - var b = await Task.WhenAll(tasks); - List results = new List(); - foreach (var t in b) { - foreach (var tt in t) { - results.Add(tt); - } - } - return results; + var symbols = await Task.WhenAll(tasks); + // Flatten + return symbols.SelectMany(l => l).ToList(); } private async Task> WorkspaceSymbolsQueryAsync(string filePath, string query, IMostRecentDocumentSymbols recentSymbols, CancellationToken cancellationToken) { From 846f9a92d3be7e6999a1ff48d0bd572ec123a9bf Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 19 Feb 2019 17:50:38 -0800 Subject: [PATCH 082/123] Getting rid of comment --- src/LanguageServer/Impl/Indexing/IndexManager.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index c5af2c0d2..30a923969 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -103,7 +103,6 @@ public void ReIndexFile(string path, IDocument doc) { } public void Dispose() { - //_symbolIndex.Dispose(); _allIndexCts.Cancel(); _allIndexCts.Dispose(); } From 8b19e45b2955383cbd71cf8c4729c3b26853f275 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 19 Feb 2019 19:19:17 -0800 Subject: [PATCH 083/123] Fix limiting in workspace symbols query --- src/LanguageServer/Impl/Indexing/SymbolIndex.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index 4d9e12d75..ea8caa00d 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -48,11 +48,12 @@ public Task> HierarchicalDocumentSymbols(strin public async Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken ct = default) { var tasks = _index .Select(kvp => WorkspaceSymbolsQueryAsync(kvp.Key, query, kvp.Value, ct)) - .Take(maxLength) .ToArray(); var symbols = await Task.WhenAll(tasks); - // Flatten - return symbols.SelectMany(l => l).ToList(); + // Flatten and limit + return symbols.SelectMany(l => l) + .Take(maxLength) + .ToList(); } private async Task> WorkspaceSymbolsQueryAsync(string filePath, string query, IMostRecentDocumentSymbols recentSymbols, CancellationToken cancellationToken) { From bad44b1549991e1858b88146dbffeba2e16559d9 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 19 Feb 2019 19:50:37 -0800 Subject: [PATCH 084/123] Better disposal --- .../Impl/Indexing/ISymbolIndex.cs | 3 +- .../Impl/Indexing/IndexManager.cs | 1 + .../Impl/Indexing/IndexParser.cs | 43 ++++++------------- .../Impl/Indexing/SymbolIndex.cs | 6 +++ 4 files changed, 22 insertions(+), 31 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs index ce71d6337..9763e545f 100644 --- a/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs @@ -13,13 +13,14 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Documents; namespace Microsoft.Python.LanguageServer.Indexing { - internal interface ISymbolIndex { + internal interface ISymbolIndex : IDisposable { Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken ct = default); Task> HierarchicalDocumentSymbols(string path); void Add(string path, IDocument doc); diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index 30a923969..3752b03c0 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -103,6 +103,7 @@ public void ReIndexFile(string path, IDocument doc) { } public void Dispose() { + _symbolIndex.Dispose(); _allIndexCts.Cancel(); _allIndexCts.Dispose(); } diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index 43b908ae5..d6ce77080 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -27,7 +27,6 @@ internal sealed class IndexParser : IIndexParser { private readonly PythonLanguageVersion _version; private readonly CancellationTokenSource _allProcessingCts = new CancellationTokenSource(); private readonly object _syncObj = new object(); - private CancellationTokenSource _linkedParseCts; public IndexParser(IFileSystem fileSystem, PythonLanguageVersion version) { Check.ArgumentNotNull(nameof(fileSystem), fileSystem); @@ -37,46 +36,30 @@ public IndexParser(IFileSystem fileSystem, PythonLanguageVersion version) { } public Task ParseAsync(string path, CancellationToken cancellationToken = default) { - lock (_syncObj) { - CancelCurrentParse(); - _linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, cancellationToken); - return Task.Run(() => Parse(path, _linkedParseCts)); - } - } - - private void CancelCurrentParse() { - Check.InvalidOperation(Monitor.IsEntered(_syncObj)); - - _linkedParseCts?.Cancel(); - _linkedParseCts?.Dispose(); - _linkedParseCts = null; + var linkedParseCts = + CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, cancellationToken); + Task parseTask; + parseTask = Task.Run(() => Parse(path, linkedParseCts)); + parseTask.ContinueWith(_ => linkedParseCts.Dispose()); + return parseTask; } private PythonAst Parse(string path, CancellationTokenSource parseCts) { parseCts.Token.ThrowIfCancellationRequested(); - PythonAst ast = null; using (var stream = _fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read)) { var parser = Parser.CreateParser(stream, _version); - ast = parser.ParseFile(); + var ast = parser.ParseFile(); + parseCts.Token.ThrowIfCancellationRequested(); + return ast; } - parseCts.Token.ThrowIfCancellationRequested(); - lock (_syncObj) { - if (_linkedParseCts == parseCts) { - _linkedParseCts.Dispose(); - _linkedParseCts = null; - } - } - return ast; } public void Dispose() { lock (_syncObj) { - _allProcessingCts.Cancel(); - _allProcessingCts.Dispose(); - - _linkedParseCts?.Cancel(); - _linkedParseCts?.Dispose(); - _linkedParseCts = null; + if (!_allProcessingCts.IsCancellationRequested) { + _allProcessingCts.Cancel(); + _allProcessingCts.Dispose(); + } } } } diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index ea8caa00d..f03012578 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -109,5 +109,11 @@ private IReadOnlyList WorkspaceSymbolsQuery(string path, string quer private IMostRecentDocumentSymbols MakeMostRecentDocSymbols(string path) { return new MostRecentDocumentSymbols(path, _fileSystem, _version); } + + public void Dispose() { + foreach (var recentSymbols in _index.Values) { + recentSymbols.Dispose(); + } + } } } From 615e483357010f3f930af767c800cfb2c249cda4 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 19 Feb 2019 20:07:58 -0800 Subject: [PATCH 085/123] CancellationToken on querying --- .../Indexing/IMostRecentDocumentSymbols.cs | 3 ++- .../Impl/Indexing/ISymbolIndex.cs | 2 +- .../Impl/Indexing/IndexManager.cs | 2 +- .../Indexing/MostRecentDocumentSymbols.cs | 26 +++++++++++++++---- .../Impl/Indexing/SymbolIndex.cs | 7 +++-- src/LanguageServer/Test/SymbolIndexTests.cs | 8 +++--- 6 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs index c26179dd3..a0157eb7c 100644 --- a/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs @@ -15,6 +15,7 @@ using System; using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Documents; @@ -23,7 +24,7 @@ interface IMostRecentDocumentSymbols : IDisposable { void Parse(); void Add(IDocument doc); void ReIndex(IDocument doc); - Task> GetSymbolsAsync(); + Task> GetSymbolsAsync(CancellationToken ct = default); void MarkAsPending(); } } diff --git a/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs index 9763e545f..8163eac65 100644 --- a/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs @@ -22,7 +22,7 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal interface ISymbolIndex : IDisposable { Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken ct = default); - Task> HierarchicalDocumentSymbols(string path); + Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken ct = default); void Add(string path, IDocument doc); void Parse(string path); void Delete(string path); diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index 3752b03c0..3145a6620 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -109,7 +109,7 @@ public void Dispose() { } public Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken cancellationToken = default) { - return _symbolIndex.HierarchicalDocumentSymbols(path); + return _symbolIndex.HierarchicalDocumentSymbolsAsync(path, cancellationToken); } public Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken cancellationToken = default) { diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index e8ec34791..37f56e6c0 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -17,7 +17,10 @@ class MostRecentDocumentSymbols : IMostRecentDocumentSymbols { private readonly string _path; private CancellationTokenSource _fileCts = new CancellationTokenSource(); - private TaskCompletionSource> _fileTcs = new TaskCompletionSource>(); + + private TaskCompletionSource> _fileTcs = + new TaskCompletionSource>(); + private bool _wasLastTaskDisposed = true; public MostRecentDocumentSymbols(string path, IFileSystem fileSystem, PythonLanguageVersion version) { @@ -34,6 +37,7 @@ public void Parse() { currentTcs = _fileTcs; _wasLastTaskDisposed = false; } + ParseAsync(currentCt).SetCompletionResultTo(currentTcs); } @@ -46,6 +50,7 @@ public void Add(IDocument doc) { currentTcs = _fileTcs; _wasLastTaskDisposed = false; } + AddAsync(doc, currentCt).SetCompletionResultTo(currentTcs); } @@ -58,10 +63,18 @@ public void ReIndex(IDocument doc) { currentTcs = _fileTcs; _wasLastTaskDisposed = false; } + ReIndexAsync(doc, currentCt).SetCompletionResultTo(currentTcs); } - public Task> GetSymbolsAsync() => _fileTcs.Task; + public Task> GetSymbolsAsync(CancellationToken ct = default) { + ct.Register(() => { + lock (_syncObj) { + CancelExistingTask(); + } + }); + return _fileTcs.Task; + } public void MarkAsPending() { lock (_syncObj) { @@ -79,11 +92,13 @@ public void Dispose() { _wasLastTaskDisposed = true; _fileTcs.TrySetCanceled(); } + _indexParser.Dispose(); } } - private async Task> AddAsync(IDocument doc, CancellationToken addCancellationToken) { + private async Task> AddAsync(IDocument doc, + CancellationToken addCancellationToken) { var ast = await doc.GetAstAsync(addCancellationToken); var walker = new SymbolIndexWalker(ast); ast.Walk(walker); @@ -91,7 +106,8 @@ private async Task> AddAsync(IDocument doc, Ca return walker.Symbols; } - private async Task> ReIndexAsync(IDocument doc, CancellationToken reIndexCancellationToken) { + private async Task> ReIndexAsync(IDocument doc, + CancellationToken reIndexCancellationToken) { var ast = await doc.GetAstAsync(reIndexCancellationToken); var walker = new SymbolIndexWalker(ast); ast.Walk(walker); @@ -109,6 +125,7 @@ private async Task> ParseAsync(CancellationTok } catch (Exception e) when (e is IOException || e is UnauthorizedAccessException) { Trace.TraceError(e.Message); } + return new List(); } @@ -126,4 +143,3 @@ private void CancelExistingTask() { } } } - diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index f03012578..05c46e598 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -37,9 +37,9 @@ public SymbolIndex(IFileSystem fileSystem, PythonLanguageVersion version) { _index = new ConcurrentDictionary(comparer); } - public Task> HierarchicalDocumentSymbols(string path) { + public Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken ct = default) { if (_index.TryGetValue(path, out var mostRecentSymbols)) { - return mostRecentSymbols.GetSymbolsAsync(); + return mostRecentSymbols.GetSymbolsAsync(ct); } else { return Task.FromResult>(new List()); } @@ -57,8 +57,7 @@ public async Task> WorkspaceSymbolsAsync(string query, } private async Task> WorkspaceSymbolsQueryAsync(string filePath, string query, IMostRecentDocumentSymbols recentSymbols, CancellationToken cancellationToken) { - var symbols = await recentSymbols.GetSymbolsAsync(); - cancellationToken.ThrowIfCancellationRequested(); + var symbols = await recentSymbols.GetSymbolsAsync(cancellationToken); return WorkspaceSymbolsQuery(filePath, query, symbols); } diff --git a/src/LanguageServer/Test/SymbolIndexTests.cs b/src/LanguageServer/Test/SymbolIndexTests.cs index 479f89d0a..1df02edb0 100644 --- a/src/LanguageServer/Test/SymbolIndexTests.cs +++ b/src/LanguageServer/Test/SymbolIndexTests.cs @@ -54,7 +54,7 @@ public async Task IndexHierarchicalDocumentAsync() { var path = TestData.GetDefaultModulePath(); index.Add(path, DocumentWithAst("x = 1")); - var symbols = await index.HierarchicalDocumentSymbols(path); + var symbols = await index.HierarchicalDocumentSymbolsAsync(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); @@ -71,14 +71,14 @@ public async Task IndexHierarchicalDocumentUpdate() { index.Add(path, DocumentWithAst("x = 1")); - var symbols = await index.HierarchicalDocumentSymbols(path); + var symbols = await index.HierarchicalDocumentSymbolsAsync(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); index.Add(path, DocumentWithAst("y = 1")); - symbols = await index.HierarchicalDocumentSymbols(path); + symbols = await index.HierarchicalDocumentSymbolsAsync(path); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); @@ -89,7 +89,7 @@ public async Task IndexHierarchicalDocumentNotFoundAsync() { ISymbolIndex index = MakeSymbolIndex(); var path = TestData.GetDefaultModulePath(); - var symbols = await index.HierarchicalDocumentSymbols(path); + var symbols = await index.HierarchicalDocumentSymbolsAsync(path); symbols.Should().BeEmpty(); } From da295427ae95d1a4d377b53da9903683245153d0 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Tue, 19 Feb 2019 20:45:12 -0800 Subject: [PATCH 086/123] Walker walks annotated assignments --- .../Impl/Indexing/SymbolIndexWalker.cs | 13 +++++++++++-- src/LanguageServer/Test/SymbolIndexWalkerTests.cs | 10 ++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs index 93dc0b1bc..a96486015 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs @@ -124,9 +124,18 @@ public override bool Walk(FromImportStatement node) { public override bool Walk(AssignmentStatement node) { node.Right?.Walk(this); - foreach (var ne in node.Left.OfType()) { - AddVarSymbol(ne); + foreach (var exp in node.Left) { + if (exp is ExpressionWithAnnotation expWithAnnot) { + if (expWithAnnot.Expression is NameExpression nameExpression) { + AddVarSymbol(nameExpression); + } + } + + if (exp is NameExpression ne) { + AddVarSymbol(ne); + } } + return false; } diff --git a/src/LanguageServer/Test/SymbolIndexWalkerTests.cs b/src/LanguageServer/Test/SymbolIndexWalkerTests.cs index a8ad2cc3b..fefb63601 100644 --- a/src/LanguageServer/Test/SymbolIndexWalkerTests.cs +++ b/src/LanguageServer/Test/SymbolIndexWalkerTests.cs @@ -611,6 +611,16 @@ public void WalkerAugmentedAssignLambda() { }); } + [TestMethod, Priority(0)] + public void WalkerAnnotatedAssignments() { + var code = @"x:int = 1"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)) + }); + } + private PythonAst GetParse(string code, PythonLanguageVersion version = PythonLanguageVersion.V37) => Parser.CreateParser(new StringReader(code), version).ParseFile(); From 5dd8e2cfce00db884017eb6cc691d4a842920d0b Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 20 Feb 2019 09:37:17 -0800 Subject: [PATCH 087/123] Removing use of MaybeEnumerate in Traverse --- src/Core/Impl/Extensions/EnumerableExtensions.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Core/Impl/Extensions/EnumerableExtensions.cs b/src/Core/Impl/Extensions/EnumerableExtensions.cs index e2fed366c..cc0eae9c0 100644 --- a/src/Core/Impl/Extensions/EnumerableExtensions.cs +++ b/src/Core/Impl/Extensions/EnumerableExtensions.cs @@ -99,7 +99,11 @@ public static IEnumerable TraverseBreadthFirst(this IEnumerable roots, var item = items.Dequeue(); yield return item; - var children = selectChildren(item).MaybeEnumerate(); + var children = selectChildren(item); + if (children == null) { + continue; + } + foreach (var child in children) { items.Enqueue(child); } From 53766a6311bd78850153d4f7fe4276b0fc94a397 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 20 Feb 2019 09:39:24 -0800 Subject: [PATCH 088/123] Var instead of explicit Uri type --- src/LanguageServer/Impl/Implementation/Server.Documents.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.Documents.cs b/src/LanguageServer/Impl/Implementation/Server.Documents.cs index cf6dd595c..e2b3147ed 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Documents.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Documents.cs @@ -26,7 +26,7 @@ namespace Microsoft.Python.LanguageServer.Implementation { public sealed partial class Server { public void DidOpenTextDocument(DidOpenTextDocumentParams @params) { _disposableBag.ThrowIfDisposed(); - Uri uri = @params.textDocument.uri; + var uri = @params.textDocument.uri; _log?.Log(TraceEventType.Verbose, $"Opening document {uri}"); var doc = _rdt.OpenDocument(uri, @params.textDocument.text); @@ -35,7 +35,7 @@ public void DidOpenTextDocument(DidOpenTextDocumentParams @params) { public void DidChangeTextDocument(DidChangeTextDocumentParams @params) { _disposableBag.ThrowIfDisposed(); - Uri uri = @params.textDocument.uri; + var uri = @params.textDocument.uri; var doc = _rdt.GetDocument(uri); if (doc != null) { var changes = new List(); @@ -63,7 +63,7 @@ public void DidChangeWatchedFiles(DidChangeWatchedFilesParams @params) { public void DidCloseTextDocument(DidCloseTextDocumentParams @params) { _disposableBag.ThrowIfDisposed(); - Uri uri = @params.textDocument.uri; + var uri = @params.textDocument.uri; _rdt.CloseDocument(uri); _indexManager.ProcessClosedFile(uri.AbsolutePath); } From 538482506b086543b49cafc4bf1ac5c3e2155071 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 20 Feb 2019 10:30:28 -0800 Subject: [PATCH 089/123] Null check --- src/LanguageServer/Impl/Indexing/IndexManager.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index 3145a6620..a5110b2ad 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -47,6 +47,7 @@ public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLang Check.ArgumentNotNull(nameof(rootPath), rootPath); Check.ArgumentNotNull(nameof(includeFiles), includeFiles); Check.ArgumentNotNull(nameof(excludeFiles), excludeFiles); + Check.ArgumentNotNull(nameof(idleTimeService), idleTimeService); _symbolIndex = symbolIndex; _fileSystem = fileSystem; @@ -142,10 +143,6 @@ private void ReIndexPendingDocsAsync() { } } - private MostRecentDocumentSymbols MakeMostRecentFileSymbols(string path) { - return new MostRecentDocumentSymbols(path, _fileSystem, _version); - } - private class UriDocumentComparer : IEqualityComparer { public bool Equals(IDocument x, IDocument y) => x.Uri.Equals(y.Uri); From 803b67ffe4dd07017b4322a724c0a8adc22e6c74 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 20 Feb 2019 13:49:21 -0800 Subject: [PATCH 090/123] max Concurrency handling --- .../Impl/Indexing/IndexParser.cs | 27 ++++++-- .../Indexing/MostRecentDocumentSymbols.cs | 4 +- .../Impl/Indexing/SymbolIndex.cs | 68 ++++++++++++++++--- 3 files changed, 81 insertions(+), 18 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index d6ce77080..2cb6c6fe4 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -13,6 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -23,8 +24,10 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal sealed class IndexParser : IIndexParser { + private const int MaxConcurrentParsings = 50; private readonly IFileSystem _fileSystem; private readonly PythonLanguageVersion _version; + private readonly SemaphoreSlim _semaphore; private readonly CancellationTokenSource _allProcessingCts = new CancellationTokenSource(); private readonly object _syncObj = new object(); @@ -33,25 +36,35 @@ public IndexParser(IFileSystem fileSystem, PythonLanguageVersion version) { _fileSystem = fileSystem; _version = version; + _semaphore = new SemaphoreSlim(MaxConcurrentParsings); } public Task ParseAsync(string path, CancellationToken cancellationToken = default) { var linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, cancellationToken); Task parseTask; - parseTask = Task.Run(() => Parse(path, linkedParseCts)); + parseTask = Task.Run(async () => await Parse(path, linkedParseCts)); parseTask.ContinueWith(_ => linkedParseCts.Dispose()); return parseTask; } - private PythonAst Parse(string path, CancellationTokenSource parseCts) { - parseCts.Token.ThrowIfCancellationRequested(); - using (var stream = _fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read)) { - var parser = Parser.CreateParser(stream, _version); - var ast = parser.ParseFile(); + private async Task Parse(string path, CancellationTokenSource parseCts) { + await _semaphore.WaitAsync(); + PythonAst ast; + try { parseCts.Token.ThrowIfCancellationRequested(); - return ast; + using (var stream = _fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read)) { + var parser = Parser.CreateParser(stream, _version); + ast = parser.ParseFile(); + } + } catch (Exception ex) { + _semaphore.Release(); + throw ex; + } finally { + _semaphore.Release(); } + + return ast; } public void Dispose() { diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index 37f56e6c0..34b09853a 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -23,9 +23,9 @@ class MostRecentDocumentSymbols : IMostRecentDocumentSymbols { private bool _wasLastTaskDisposed = true; - public MostRecentDocumentSymbols(string path, IFileSystem fileSystem, PythonLanguageVersion version) { + public MostRecentDocumentSymbols(string path, IFileSystem fileSystem, PythonLanguageVersion version, IIndexParser indexParser) { _path = path; - _indexParser = new IndexParser(fileSystem, version); + _indexParser = indexParser; } public void Parse() { diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index 05c46e598..e5a0bd5e3 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -13,6 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; @@ -28,6 +29,7 @@ internal sealed class SymbolIndex : ISymbolIndex { private readonly ConcurrentDictionary _index; private readonly IFileSystem _fileSystem; private readonly PythonLanguageVersion _version; + private readonly IIndexParser _indexParser; public SymbolIndex(IFileSystem fileSystem, PythonLanguageVersion version) { _fileSystem = fileSystem; @@ -35,6 +37,7 @@ public SymbolIndex(IFileSystem fileSystem, PythonLanguageVersion version) { var comparer = PathEqualityComparer.Instance; _index = new ConcurrentDictionary(comparer); + _indexParser = new IndexParser(_fileSystem, _version); } public Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken ct = default) { @@ -46,14 +49,61 @@ public Task> HierarchicalDocumentSymbolsAsync( } public async Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken ct = default) { - var tasks = _index - .Select(kvp => WorkspaceSymbolsQueryAsync(kvp.Key, query, kvp.Value, ct)) - .ToArray(); - var symbols = await Task.WhenAll(tasks); - // Flatten and limit - return symbols.SelectMany(l => l) - .Take(maxLength) - .ToList(); + /*var workspaceCts = new CancellationTokenSource(); + var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(ct, workspaceCts.Token); + + var workingTasks = new List>>(); + var results = IterateDoneSymbols(query, maxLength, workingTasks, linkedCts); + await UntilMaxLength(maxLength, results, workingTasks); + + workspaceCts.Cancel(); + workspaceCts.Dispose(); + linkedCts.Dispose(); + return results.Take(maxLength) + .ToList();*/ + var results = new List(); + + foreach (var filePathAndRecent in _index.Select(kvp => kvp)) { + var symbols = await filePathAndRecent.Value.GetSymbolsAsync(ct); + results.AddRange(WorkspaceSymbolsQuery(filePathAndRecent.Key, query, symbols)); + if (results.Count >= maxLength) { + break; + } + } + return results; + } + + private List IterateDoneSymbols(string query, int maxLength, List>> workingTasks, CancellationTokenSource linkedCts) { + var results = new List(); + foreach (var filePathAndRecent in _index.Select(kvp => kvp)) { + if (results.Count >= maxLength) { + break; + } + + var symbolsTask = filePathAndRecent.Value.GetSymbolsAsync(); + if (symbolsTask.IsCompletedSuccessfully) { + results.AddRange(WorkspaceSymbolsQuery(filePathAndRecent.Key, query, symbolsTask.Result)); + } else if (!symbolsTask.IsCompleted) { + workingTasks.Add(WorkspaceSymbolsQueryAsync(filePathAndRecent.Key, query, filePathAndRecent.Value, linkedCts.Token)); + } + } + return results; + } + + private static async Task UntilMaxLength(int maxLength, List results, List>> workingTasks) { + while (results.Count < maxLength && !workingTasks.IsNullOrEmpty()) { + const int maxAwaitingTasks = 25; + var awaitTasks = workingTasks.Take(maxAwaitingTasks).ToArray(); + workingTasks.RemoveRange(0, awaitTasks.Length); + await Task.WhenAny(awaitTasks); + foreach (var t in awaitTasks) { + if (!t.IsCompleted) { + workingTasks.Add(t); + } else if (t.IsCompletedSuccessfully) { + results.AddRange(t.Result); + } + } + } } private async Task> WorkspaceSymbolsQueryAsync(string filePath, string query, IMostRecentDocumentSymbols recentSymbols, CancellationToken cancellationToken) { @@ -106,7 +156,7 @@ private IReadOnlyList WorkspaceSymbolsQuery(string path, string quer } private IMostRecentDocumentSymbols MakeMostRecentDocSymbols(string path) { - return new MostRecentDocumentSymbols(path, _fileSystem, _version); + return new MostRecentDocumentSymbols(path, _fileSystem, _version, _indexParser); } public void Dispose() { From 8f704dd434a9eefd91d45dcad98e8e97602848a9 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 20 Feb 2019 16:15:15 -0800 Subject: [PATCH 091/123] Simple workspace querying --- .../Impl/Implementation/Server.Documents.cs | 1 + .../Implementation/Server.WorkspaceSymbols.cs | 2 +- .../Impl/Indexing/IndexParser.cs | 2 +- .../Indexing/MostRecentDocumentSymbols.cs | 10 +-- .../Impl/Indexing/SymbolIndex.cs | 65 +++---------------- 5 files changed, 15 insertions(+), 65 deletions(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.Documents.cs b/src/LanguageServer/Impl/Implementation/Server.Documents.cs index 74ffea5fe..6305645d7 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Documents.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Documents.cs @@ -17,6 +17,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Threading; +using System.Threading.Tasks; using Microsoft.Python.Analysis; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core; diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs index 21cd661dd..4a10285b2 100644 --- a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs @@ -36,7 +36,7 @@ public async Task WorkspaceSymbols(WorkspaceSymbolParams @p public async Task HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { var path = @params.textDocument.uri.AbsolutePath; var symbols = await _indexManager.HierarchicalDocumentSymbolsAsync(path, cancellationToken); - return symbols.MaybeEnumerate().Select(hSym => MakeDocumentSymbol(hSym)).ToArray(); + return symbols.Select(hSym => MakeDocumentSymbol(hSym)).ToArray(); } private static SymbolInformation MakeSymbolInfo(Indexing.FlatSymbol s) { diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index 2cb6c6fe4..3a6939fdf 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -24,7 +24,7 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal sealed class IndexParser : IIndexParser { - private const int MaxConcurrentParsings = 50; + private const int MaxConcurrentParsings = 10; private readonly IFileSystem _fileSystem; private readonly PythonLanguageVersion _version; private readonly SemaphoreSlim _semaphore; diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index 34b09853a..877228bad 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -67,14 +67,8 @@ public void ReIndex(IDocument doc) { ReIndexAsync(doc, currentCt).SetCompletionResultTo(currentTcs); } - public Task> GetSymbolsAsync(CancellationToken ct = default) { - ct.Register(() => { - lock (_syncObj) { - CancelExistingTask(); - } - }); - return _fileTcs.Task; - } + public Task> GetSymbolsAsync(CancellationToken ct = default) + => _fileTcs.Task.ContinueWith(t => t.GetAwaiter().GetResult(), ct); public void MarkAsPending() { lock (_syncObj) { diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index e5a0bd5e3..c1b621524 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; @@ -49,23 +48,13 @@ public Task> HierarchicalDocumentSymbolsAsync( } public async Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken ct = default) { - /*var workspaceCts = new CancellationTokenSource(); - var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(ct, workspaceCts.Token); - - var workingTasks = new List>>(); - var results = IterateDoneSymbols(query, maxLength, workingTasks, linkedCts); - await UntilMaxLength(maxLength, results, workingTasks); - - workspaceCts.Cancel(); - workspaceCts.Dispose(); - linkedCts.Dispose(); - return results.Take(maxLength) - .ToList();*/ + var results = new List(); - foreach (var filePathAndRecent in _index.Select(kvp => kvp)) { + foreach (var filePathAndRecent in _index) { var symbols = await filePathAndRecent.Value.GetSymbolsAsync(ct); - results.AddRange(WorkspaceSymbolsQuery(filePathAndRecent.Key, query, symbols)); + results.AddRange(WorkspaceSymbolsQuery(filePathAndRecent.Key, query, symbols).Take(maxLength - results.Count)); + ct.ThrowIfCancellationRequested(); if (results.Count >= maxLength) { break; } @@ -73,42 +62,9 @@ public async Task> WorkspaceSymbolsAsync(string query, return results; } - private List IterateDoneSymbols(string query, int maxLength, List>> workingTasks, CancellationTokenSource linkedCts) { - var results = new List(); - foreach (var filePathAndRecent in _index.Select(kvp => kvp)) { - if (results.Count >= maxLength) { - break; - } - - var symbolsTask = filePathAndRecent.Value.GetSymbolsAsync(); - if (symbolsTask.IsCompletedSuccessfully) { - results.AddRange(WorkspaceSymbolsQuery(filePathAndRecent.Key, query, symbolsTask.Result)); - } else if (!symbolsTask.IsCompleted) { - workingTasks.Add(WorkspaceSymbolsQueryAsync(filePathAndRecent.Key, query, filePathAndRecent.Value, linkedCts.Token)); - } - } - return results; - } - - private static async Task UntilMaxLength(int maxLength, List results, List>> workingTasks) { - while (results.Count < maxLength && !workingTasks.IsNullOrEmpty()) { - const int maxAwaitingTasks = 25; - var awaitTasks = workingTasks.Take(maxAwaitingTasks).ToArray(); - workingTasks.RemoveRange(0, awaitTasks.Length); - await Task.WhenAny(awaitTasks); - foreach (var t in awaitTasks) { - if (!t.IsCompleted) { - workingTasks.Add(t); - } else if (t.IsCompletedSuccessfully) { - results.AddRange(t.Result); - } - } - } - } - private async Task> WorkspaceSymbolsQueryAsync(string filePath, string query, IMostRecentDocumentSymbols recentSymbols, CancellationToken cancellationToken) { var symbols = await recentSymbols.GetSymbolsAsync(cancellationToken); - return WorkspaceSymbolsQuery(filePath, query, symbols); + return WorkspaceSymbolsQuery(filePath, query, symbols).ToList(); } public void Add(string path, IDocument doc) { @@ -137,21 +93,20 @@ public void MarkAsPending(string path) { _index[path].MarkAsPending(); } - private IReadOnlyList WorkspaceSymbolsQuery(string path, string query, + private IEnumerable WorkspaceSymbolsQuery(string path, string query, IReadOnlyList symbols) { var rootSymbols = DecorateWithParentsName(symbols, null); var treeSymbols = rootSymbols.TraverseBreadthFirst((symAndPar) => { var sym = symAndPar.symbol; - return DecorateWithParentsName(sym.Children.MaybeEnumerate().ToList(), sym.Name); + return DecorateWithParentsName((sym.Children ?? Enumerable.Empty()).ToList(), sym.Name); }); return treeSymbols.Where(sym => sym.symbol.Name.ContainsOrdinal(query, ignoreCase: true)) - .Select(sym => new FlatSymbol(sym.symbol.Name, sym.symbol.Kind, path, sym.symbol.SelectionRange, sym.parentName)) - .ToList(); + .Select(sym => new FlatSymbol(sym.symbol.Name, sym.symbol.Kind, path, sym.symbol.SelectionRange, sym.parentName)); } - private static IReadOnlyList<(HierarchicalSymbol symbol, string parentName)> DecorateWithParentsName( - IReadOnlyList symbols, string parentName) { + private static IEnumerable<(HierarchicalSymbol symbol, string parentName)> DecorateWithParentsName( + IEnumerable symbols, string parentName) { return symbols.Select((symbol) => (symbol, parentName)).ToList(); } From 17a9d0a9d14295a39b3ece6c197a6aacacd6420a Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 20 Feb 2019 16:36:35 -0800 Subject: [PATCH 092/123] Simplifying disposing --- .../Indexing/MostRecentDocumentSymbols.cs | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index 877228bad..2f48716bc 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -21,8 +21,6 @@ class MostRecentDocumentSymbols : IMostRecentDocumentSymbols { private TaskCompletionSource> _fileTcs = new TaskCompletionSource>(); - private bool _wasLastTaskDisposed = true; - public MostRecentDocumentSymbols(string path, IFileSystem fileSystem, PythonLanguageVersion version, IIndexParser indexParser) { _path = path; _indexParser = indexParser; @@ -35,7 +33,6 @@ public void Parse() { CancelExistingTask(); currentCt = _fileCts.Token; currentTcs = _fileTcs; - _wasLastTaskDisposed = false; } ParseAsync(currentCt).SetCompletionResultTo(currentTcs); @@ -48,7 +45,6 @@ public void Add(IDocument doc) { CancelExistingTask(); currentCt = _fileCts.Token; currentTcs = _fileTcs; - _wasLastTaskDisposed = false; } AddAsync(doc, currentCt).SetCompletionResultTo(currentTcs); @@ -61,7 +57,6 @@ public void ReIndex(IDocument doc) { CancelExistingTask(); currentCt = _fileCts.Token; currentTcs = _fileTcs; - _wasLastTaskDisposed = false; } ReIndexAsync(doc, currentCt).SetCompletionResultTo(currentTcs); @@ -78,12 +73,11 @@ public void MarkAsPending() { public void Dispose() { lock (_syncObj) { - if (!_wasLastTaskDisposed) { + if (_fileCts != null) { _fileCts?.Cancel(); _fileCts?.Dispose(); _fileCts = null; - _wasLastTaskDisposed = true; _fileTcs.TrySetCanceled(); } @@ -126,14 +120,11 @@ private async Task> ParseAsync(CancellationTok private void CancelExistingTask() { Check.InvalidOperation(Monitor.IsEntered(_syncObj)); - if (!_wasLastTaskDisposed) { - _fileCts.Cancel(); - _fileCts.Dispose(); - _fileCts = new CancellationTokenSource(); + _fileCts.Cancel(); + _fileCts.Dispose(); + _fileCts = new CancellationTokenSource(); - _fileTcs = new TaskCompletionSource>(); - _wasLastTaskDisposed = true; - } + _fileTcs = new TaskCompletionSource>(); } } } From 88ba453510493651c428dd290eb4bb83e2741c01 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 20 Feb 2019 17:34:02 -0800 Subject: [PATCH 093/123] Simplification --- src/LanguageServer/Impl/Indexing/IndexManager.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index a5110b2ad..6601b8777 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -35,8 +35,6 @@ internal class IndexManager : IIndexManager { private readonly string[] _includeFiles; private readonly string[] _excludeFiles; private readonly IIdleTimeService _idleTimeService; - private readonly CancellationTokenSource _allIndexCts = new CancellationTokenSource(); - private readonly TaskCompletionSource _addRootTcs = new TaskCompletionSource(); private readonly HashSet _pendingDocs = new HashSet(new UriDocumentComparer()); private readonly PythonLanguageVersion _version; private DateTime _lastPendingDocAddedTime; @@ -105,8 +103,6 @@ public void ReIndexFile(string path, IDocument doc) { public void Dispose() { _symbolIndex.Dispose(); - _allIndexCts.Cancel(); - _allIndexCts.Dispose(); } public Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken cancellationToken = default) { From d9f8d576385a1646703f1cd0bb55076c84e3830d Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 20 Feb 2019 17:34:49 -0800 Subject: [PATCH 094/123] Unused method --- src/LanguageServer/Impl/Indexing/SymbolIndex.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index c1b621524..51b852219 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -62,11 +62,6 @@ public async Task> WorkspaceSymbolsAsync(string query, return results; } - private async Task> WorkspaceSymbolsQueryAsync(string filePath, string query, IMostRecentDocumentSymbols recentSymbols, CancellationToken cancellationToken) { - var symbols = await recentSymbols.GetSymbolsAsync(cancellationToken); - return WorkspaceSymbolsQuery(filePath, query, symbols).ToList(); - } - public void Add(string path, IDocument doc) { _index.GetOrAdd(path, MakeMostRecentDocSymbols(path)).Add(doc); } From 2c60d4f313f45214f68c8492da0da7fa4dbb6fff Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Wed, 20 Feb 2019 17:35:37 -0800 Subject: [PATCH 095/123] CancellationToken --- .../Impl/Implementation/Server.WorkspaceSymbols.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs index 4a10285b2..ccc785a19 100644 --- a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs @@ -36,6 +36,7 @@ public async Task WorkspaceSymbols(WorkspaceSymbolParams @p public async Task HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { var path = @params.textDocument.uri.AbsolutePath; var symbols = await _indexManager.HierarchicalDocumentSymbolsAsync(path, cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); return symbols.Select(hSym => MakeDocumentSymbol(hSym)).ToArray(); } From 4825065ff5ba55d86d4612f9f5acb81145b6cf56 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 21 Feb 2019 11:52:23 -0800 Subject: [PATCH 096/123] First state machine --- .../Indexing/IMostRecentDocumentSymbols.cs | 3 +- .../Indexing/MostRecentDocumentSymbols.cs | 130 +++++++++++------- .../Impl/Indexing/SymbolIndex.cs | 4 +- 3 files changed, 82 insertions(+), 55 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs index a0157eb7c..6e64ea40a 100644 --- a/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/IMostRecentDocumentSymbols.cs @@ -22,8 +22,7 @@ namespace Microsoft.Python.LanguageServer.Indexing { interface IMostRecentDocumentSymbols : IDisposable { void Parse(); - void Add(IDocument doc); - void ReIndex(IDocument doc); + void Index(IDocument doc); Task> GetSymbolsAsync(CancellationToken ct = default); void MarkAsPending(); } diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index 2f48716bc..bf320de22 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -18,8 +18,8 @@ class MostRecentDocumentSymbols : IMostRecentDocumentSymbols { private CancellationTokenSource _fileCts = new CancellationTokenSource(); - private TaskCompletionSource> _fileTcs = - new TaskCompletionSource>(); + private TaskCompletionSource> _fileTcs = new TaskCompletionSource>(); + private State state = State.WaitingForWork; public MostRecentDocumentSymbols(string path, IFileSystem fileSystem, PythonLanguageVersion version, IIndexParser indexParser) { _path = path; @@ -27,88 +27,113 @@ public MostRecentDocumentSymbols(string path, IFileSystem fileSystem, PythonLang } public void Parse() { - CancellationToken currentCt; - TaskCompletionSource> currentTcs; - lock (_syncObj) { - CancelExistingTask(); - currentCt = _fileCts.Token; - currentTcs = _fileTcs; - } - - ParseAsync(currentCt).SetCompletionResultTo(currentTcs); + WorkAndSetTcs(ParseAsync); } public void Add(IDocument doc) { - CancellationToken currentCt; + WorkAndSetTcs(ct => IndexAsync(doc, ct)); + } + + public void Index(IDocument doc) { + WorkAndSetTcs(ct => IndexAsync(doc, ct)); + } + + public void WorkAndSetTcs(Func>> asyncFunc) { + CancellationTokenSource currentCts; TaskCompletionSource> currentTcs; lock (_syncObj) { - CancelExistingTask(); - currentCt = _fileCts.Token; + switch (state) { + case State.Working: + CancelExistingWork(); + RenewTcs(); + state = State.Working; + break; + case State.WaitingForWork: + state = State.Working; + break; + case State.FinishedWork: + RenewTcs(); + state = State.Working; + break; + default: + break; + } + currentCts = _fileCts; currentTcs = _fileTcs; } - AddAsync(doc, currentCt).SetCompletionResultTo(currentTcs); + asyncFunc(currentCts.Token).ContinueWith(t => { + lock (_syncObj) { + currentCts.Dispose(); + if (_fileCts == currentCts) { + state = State.FinishedWork; + } + } + return t.GetAwaiter().GetResult(); + }, currentCts.Token).SetCompletionResultTo(currentTcs); } - public void ReIndex(IDocument doc) { - CancellationToken currentCt; + public Task> GetSymbolsAsync(CancellationToken ct = default) { TaskCompletionSource> currentTcs; lock (_syncObj) { - CancelExistingTask(); - currentCt = _fileCts.Token; currentTcs = _fileTcs; } - - ReIndexAsync(doc, currentCt).SetCompletionResultTo(currentTcs); + return currentTcs.Task.ContinueWith(t => t.GetAwaiter().GetResult(), ct); } - public Task> GetSymbolsAsync(CancellationToken ct = default) - => _fileTcs.Task.ContinueWith(t => t.GetAwaiter().GetResult(), ct); - public void MarkAsPending() { lock (_syncObj) { - CancelExistingTask(); + switch (state) { + case State.WaitingForWork: + state = State.WaitingForWork; + break; + case State.Working: + CancelExistingWork(); + RenewTcs(); + state = State.WaitingForWork; + break; + case State.FinishedWork: + RenewTcs(); + state = State.WaitingForWork; + break; + default: + break; + } } } public void Dispose() { lock (_syncObj) { - if (_fileCts != null) { - _fileCts?.Cancel(); - _fileCts?.Dispose(); - _fileCts = null; - - _fileTcs.TrySetCanceled(); + switch (state) { + case State.Working: + CancelExistingWork(); + state = State.FinishedWork; + break; + case State.WaitingForWork: + state = State.FinishedWork; + break; + case State.FinishedWork: + break; } - _indexParser.Dispose(); } } - private async Task> AddAsync(IDocument doc, - CancellationToken addCancellationToken) { - var ast = await doc.GetAstAsync(addCancellationToken); + private async Task> IndexAsync(IDocument doc, + CancellationToken indexCt) { + var ast = await doc.GetAstAsync(indexCt); + indexCt.ThrowIfCancellationRequested(); var walker = new SymbolIndexWalker(ast); ast.Walk(walker); - addCancellationToken.ThrowIfCancellationRequested(); - return walker.Symbols; - } - - private async Task> ReIndexAsync(IDocument doc, - CancellationToken reIndexCancellationToken) { - var ast = await doc.GetAstAsync(reIndexCancellationToken); - var walker = new SymbolIndexWalker(ast); - ast.Walk(walker); - reIndexCancellationToken.ThrowIfCancellationRequested(); return walker.Symbols; } private async Task> ParseAsync(CancellationToken parseCancellationToken) { try { var ast = await _indexParser.ParseAsync(_path, parseCancellationToken); + parseCancellationToken.ThrowIfCancellationRequested(); var walker = new SymbolIndexWalker(ast); ast.Walk(walker); - parseCancellationToken.ThrowIfCancellationRequested(); return walker.Symbols; } catch (Exception e) when (e is IOException || e is UnauthorizedAccessException) { Trace.TraceError(e.Message); @@ -117,14 +142,17 @@ private async Task> ParseAsync(CancellationTok return new List(); } - private void CancelExistingTask() { + private void RenewTcs() { Check.InvalidOperation(Monitor.IsEntered(_syncObj)); - - _fileCts.Cancel(); - _fileCts.Dispose(); _fileCts = new CancellationTokenSource(); - _fileTcs = new TaskCompletionSource>(); } + + private void CancelExistingWork() { + Check.InvalidOperation(Monitor.IsEntered(_syncObj)); + _fileCts.Cancel(); + } + + private enum State { WaitingForWork, Working, FinishedWork }; } } diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index 51b852219..d8ca220d8 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -63,7 +63,7 @@ public async Task> WorkspaceSymbolsAsync(string query, } public void Add(string path, IDocument doc) { - _index.GetOrAdd(path, MakeMostRecentDocSymbols(path)).Add(doc); + _index.GetOrAdd(path, MakeMostRecentDocSymbols(path)).Index(doc); } public void Parse(string path) { @@ -80,7 +80,7 @@ public void Delete(string path) { public void ReIndex(string path, IDocument doc) { if (_index.TryGetValue(path, out var currentSymbols)) { - currentSymbols.ReIndex(doc); + currentSymbols.Index(doc); } } From fffb215a41b7a11a69eea90f3300c7645d9b56b2 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 21 Feb 2019 13:26:20 -0800 Subject: [PATCH 097/123] Tests for states of symbolIndex --- src/LanguageServer/Test/SymbolIndexTests.cs | 33 +++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/LanguageServer/Test/SymbolIndexTests.cs b/src/LanguageServer/Test/SymbolIndexTests.cs index 1df02edb0..0a91ba2b0 100644 --- a/src/LanguageServer/Test/SymbolIndexTests.cs +++ b/src/LanguageServer/Test/SymbolIndexTests.cs @@ -16,6 +16,7 @@ using System.Collections.Generic; using System.IO; using System.Text; +using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Documents; @@ -154,6 +155,38 @@ public async Task IndexWorkspaceSymbolsCaseInsensitiveAsync() { }); } + [TestMethod, Priority(0)] + public void MarkAsPendingWaitsForUpdates() { + ISymbolIndex index = MakeSymbolIndex(); + var path = TestData.GetDefaultModulePath(); + + index.Add(path, DocumentWithAst("x = 1")); + index.MarkAsPending(path); + var cts = new CancellationTokenSource(); + var t = index.HierarchicalDocumentSymbolsAsync(path, cts.Token); + t.IsCompleted.Should().BeFalse(); + cts.Cancel(); + Func cancelled = async () => { + await t; + }; + cancelled.Should().Throw(); + } + + [TestMethod, Priority(0)] + public async Task SymbolsAfterPendingWaitsForUpdateAsync() { + ISymbolIndex index = MakeSymbolIndex(); + var path = TestData.GetDefaultModulePath(); + + index.Add(path, DocumentWithAst("x = 1")); + index.MarkAsPending(path); + var t = index.WorkspaceSymbolsAsync("", maxSymbols); + index.ReIndex(path, DocumentWithAst("x = 1")); + var symbols = await t; + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(1, 1, 1, 2)), + }); + } + private PythonAst GetParse(string code, PythonLanguageVersion version = PythonLanguageVersion.V37) => Parser.CreateParser(new StringReader(code), version).ParseFile(); From 25b6590bd6399bad0882d1a9ff5876027a8b2f90 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 21 Feb 2019 13:48:21 -0800 Subject: [PATCH 098/123] Rename of state --- .../Indexing/MostRecentDocumentSymbols.cs | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index bf320de22..6062b9739 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -19,7 +19,7 @@ class MostRecentDocumentSymbols : IMostRecentDocumentSymbols { private CancellationTokenSource _fileCts = new CancellationTokenSource(); private TaskCompletionSource> _fileTcs = new TaskCompletionSource>(); - private State state = State.WaitingForWork; + private WorkQueueState state = WorkQueueState.WaitingForWork; public MostRecentDocumentSymbols(string path, IFileSystem fileSystem, PythonLanguageVersion version, IIndexParser indexParser) { _path = path; @@ -43,17 +43,17 @@ public void WorkAndSetTcs(Func> currentTcs; lock (_syncObj) { switch (state) { - case State.Working: + case WorkQueueState.Working: CancelExistingWork(); RenewTcs(); - state = State.Working; + state = WorkQueueState.Working; break; - case State.WaitingForWork: - state = State.Working; + case WorkQueueState.WaitingForWork: + state = WorkQueueState.Working; break; - case State.FinishedWork: + case WorkQueueState.FinishedWork: RenewTcs(); - state = State.Working; + state = WorkQueueState.Working; break; default: break; @@ -66,7 +66,7 @@ public void WorkAndSetTcs(Func> GetSymbolsAsync(CancellationToken public void MarkAsPending() { lock (_syncObj) { switch (state) { - case State.WaitingForWork: - state = State.WaitingForWork; + case WorkQueueState.WaitingForWork: + state = WorkQueueState.WaitingForWork; break; - case State.Working: + case WorkQueueState.Working: CancelExistingWork(); RenewTcs(); - state = State.WaitingForWork; + state = WorkQueueState.WaitingForWork; break; - case State.FinishedWork: + case WorkQueueState.FinishedWork: RenewTcs(); - state = State.WaitingForWork; + state = WorkQueueState.WaitingForWork; break; default: break; @@ -105,14 +105,14 @@ public void MarkAsPending() { public void Dispose() { lock (_syncObj) { switch (state) { - case State.Working: + case WorkQueueState.Working: CancelExistingWork(); - state = State.FinishedWork; + state = WorkQueueState.FinishedWork; break; - case State.WaitingForWork: - state = State.FinishedWork; + case WorkQueueState.WaitingForWork: + state = WorkQueueState.FinishedWork; break; - case State.FinishedWork: + case WorkQueueState.FinishedWork: break; } _indexParser.Dispose(); @@ -153,6 +153,6 @@ private void CancelExistingWork() { _fileCts.Cancel(); } - private enum State { WaitingForWork, Working, FinishedWork }; + private enum WorkQueueState { WaitingForWork, Working, FinishedWork }; } } From 4a569efd2f9251f6f3948f9a5e1935844a1a1248 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 21 Feb 2019 13:50:58 -0800 Subject: [PATCH 099/123] Comment on name of state --- src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index 6062b9739..99e9c40a2 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -153,6 +153,8 @@ private void CancelExistingWork() { _fileCts.Cancel(); } + /* It's easier to think of it as a queue of work + * but it maintains only one item at a time in the queue */ private enum WorkQueueState { WaitingForWork, Working, FinishedWork }; } } From 08f62871e061ac2c7b0fd77f015b6f0137b453ce Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 21 Feb 2019 13:51:39 -0800 Subject: [PATCH 100/123] File Open FileSystem back to how it was --- src/Core/Impl/IO/FileSystem.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Impl/IO/FileSystem.cs b/src/Core/Impl/IO/FileSystem.cs index b78f441d4..a88edb77c 100644 --- a/src/Core/Impl/IO/FileSystem.cs +++ b/src/Core/Impl/IO/FileSystem.cs @@ -31,7 +31,7 @@ public long FileSize(string path) { public byte[] FileReadAllBytes(string path) => File.ReadAllBytes(path); public void FileWriteAllBytes(string path, byte[] bytes) => File.WriteAllBytes(path, bytes); public Stream CreateFile(string path) => File.Create(path); - public Stream FileOpen(string path, FileMode mode) => FileOpen(path, mode); + public Stream FileOpen(string path, FileMode mode) => File.Open(path, mode); public Stream FileOpen(string path, FileMode mode, FileAccess access, FileShare share) => File.Open(path, mode, access, share); public bool DirectoryExists(string path) => Directory.Exists(path); public FileAttributes GetFileAttributes(string path) => File.GetAttributes(path); From 05e711dc58e1ed122b6ec61b95f9359a3da40517 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 21 Feb 2019 13:53:46 -0800 Subject: [PATCH 101/123] Server.Symbols --- .../{Server.WorkspaceSymbols.cs => Server.Symbols.cs} | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) rename src/LanguageServer/Impl/Implementation/{Server.WorkspaceSymbols.cs => Server.Symbols.cs} (97%) diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Impl/Implementation/Server.Symbols.cs similarity index 97% rename from src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs rename to src/LanguageServer/Impl/Implementation/Server.Symbols.cs index ccc785a19..09a000228 100644 --- a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Symbols.cs @@ -23,7 +23,6 @@ namespace Microsoft.Python.LanguageServer.Implementation { public sealed partial class Server { - private static int _symbolHierarchyDepthLimit = 10; private static int _symbolHierarchyMaxSymbols = 1000; public async Task WorkspaceSymbols(WorkspaceSymbolParams @params, CancellationToken cancellationToken) { @@ -40,7 +39,7 @@ public async Task HierarchicalDocumentSymbol(DocumentSymbolPar return symbols.Select(hSym => MakeDocumentSymbol(hSym)).ToArray(); } - private static SymbolInformation MakeSymbolInfo(Indexing.FlatSymbol s) { + private static SymbolInformation MakeSymbolInfo(FlatSymbol s) { return new SymbolInformation { name = s.Name, kind = (Protocol.SymbolKind)s.Kind, From 9213ca706a32d33bd02cc1e6c84f9b625c3623f0 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 21 Feb 2019 13:55:06 -0800 Subject: [PATCH 102/123] Reference to deleted field --- src/LanguageServer/Impl/Implementation/Server.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index 9d09aa119..1ebc4cdeb 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -182,7 +182,6 @@ private bool HandleConfigurationChanges(ServerSettings newSettings) { var oldSettings = Settings; Settings = newSettings; - _symbolHierarchyDepthLimit = Settings.analysis.symbolsHierarchyDepthLimit; _symbolHierarchyMaxSymbols = Settings.analysis.symbolsHierarchyMaxSymbols; if (oldSettings == null) { From 0e3c42d80b6cb6063f351f59d7ad816f56a447b6 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 21 Feb 2019 13:56:46 -0800 Subject: [PATCH 103/123] Extra new lines --- src/LanguageServer/Impl/Indexing/SymbolIndex.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index d8ca220d8..61ae9f70b 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -48,9 +48,7 @@ public Task> HierarchicalDocumentSymbolsAsync( } public async Task> WorkspaceSymbolsAsync(string query, int maxLength, CancellationToken ct = default) { - var results = new List(); - foreach (var filePathAndRecent in _index) { var symbols = await filePathAndRecent.Value.GetSymbolsAsync(ct); results.AddRange(WorkspaceSymbolsQuery(filePathAndRecent.Key, query, symbols).Take(maxLength - results.Count)); @@ -99,7 +97,6 @@ private IEnumerable WorkspaceSymbolsQuery(string path, string query, .Select(sym => new FlatSymbol(sym.symbol.Name, sym.symbol.Kind, path, sym.symbol.SelectionRange, sym.parentName)); } - private static IEnumerable<(HierarchicalSymbol symbol, string parentName)> DecorateWithParentsName( IEnumerable symbols, string parentName) { return symbols.Select((symbol) => (symbol, parentName)).ToList(); From 9fcdbe5ee5d17daeb452f58fa98437a1deceae49 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 21 Feb 2019 13:59:38 -0800 Subject: [PATCH 104/123] Extra spaces --- src/Core/Impl/Extensions/EnumerableExtensions.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Core/Impl/Extensions/EnumerableExtensions.cs b/src/Core/Impl/Extensions/EnumerableExtensions.cs index e655fb90a..5e40af9eb 100644 --- a/src/Core/Impl/Extensions/EnumerableExtensions.cs +++ b/src/Core/Impl/Extensions/EnumerableExtensions.cs @@ -94,10 +94,7 @@ public static IEnumerable TraverseBreadthFirst(this T root, Func TraverseBreadthFirst(this IEnumerable roots, Func> selectChildren) { var items = new Queue(roots); - while (items.Count > 0) { - - var item = items.Dequeue(); yield return item; From 08858f0c0ae4dbd83a51b6cd7c6fd317fe402831 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 21 Feb 2019 14:04:59 -0800 Subject: [PATCH 105/123] Delete unused code --- src/Core/Impl/Extensions/TaskExtensions.cs | 24 ---------------------- 1 file changed, 24 deletions(-) diff --git a/src/Core/Impl/Extensions/TaskExtensions.cs b/src/Core/Impl/Extensions/TaskExtensions.cs index c96b92f87..0e39d3495 100644 --- a/src/Core/Impl/Extensions/TaskExtensions.cs +++ b/src/Core/Impl/Extensions/TaskExtensions.cs @@ -21,30 +21,6 @@ namespace Microsoft.Python.Core { public static class TaskExtensions { - public static void SetCompletionResultTo(this Task task, TaskCompletionSourceEx tcs) - => task.ContinueWith(SetCompletionResultToContinuationEx, tcs, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); - - private static void SetCompletionResultToContinuationEx(Task task, object state) { - var tcs = (TaskCompletionSourceEx) state; - switch (task.Status) { - case TaskStatus.RanToCompletion: - tcs.TrySetResult(task.Result); - break; - case TaskStatus.Canceled: - try { - task.GetAwaiter().GetResult(); - } catch (OperationCanceledException ex) { - tcs.TrySetCanceled(ex); - } - break; - case TaskStatus.Faulted: - tcs.TrySetException(task.Exception); - break; - default: - throw new ArgumentOutOfRangeException(); - } - } - public static void SetCompletionResultTo(this Task task, TaskCompletionSource tcs) => task.ContinueWith(SetCompletionResultToContinuation, tcs, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); From ccef5369b2fc6f7fc44b910ca8566279dbe1de22 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 21 Feb 2019 14:08:38 -0800 Subject: [PATCH 106/123] Pattern matching in walker --- .../Impl/Indexing/SymbolIndexWalker.cs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs index a96486015..0f9d1d1f3 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs @@ -125,14 +125,13 @@ public override bool Walk(FromImportStatement node) { public override bool Walk(AssignmentStatement node) { node.Right?.Walk(this); foreach (var exp in node.Left) { - if (exp is ExpressionWithAnnotation expWithAnnot) { - if (expWithAnnot.Expression is NameExpression nameExpression) { - AddVarSymbol(nameExpression); - } - } - - if (exp is NameExpression ne) { - AddVarSymbol(ne); + switch (exp) { + case ExpressionWithAnnotation ewa when ewa.Expression is NameExpression ne: + AddVarSymbol(ne); + break; + case NameExpression ne: + AddVarSymbol(ne); + break; } } From 121a8b0f99f221473e990462766e7d368830c75b Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 21 Feb 2019 14:23:37 -0800 Subject: [PATCH 107/123] Unused methods --- src/LanguageServer/Impl/Indexing/ISymbolIndex.cs | 1 - src/LanguageServer/Impl/Indexing/IndexManager.cs | 2 -- src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs | 4 ---- src/LanguageServer/Impl/Indexing/SymbolIndex.cs | 2 -- 4 files changed, 9 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs index 8163eac65..e7cf22459 100644 --- a/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/ISymbolIndex.cs @@ -26,7 +26,6 @@ internal interface ISymbolIndex : IDisposable { void Add(string path, IDocument doc); void Parse(string path); void Delete(string path); - bool IsIndexed(string path); void ReIndex(string path, IDocument doc); void MarkAsPending(string path); } diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index 6601b8777..06e360118 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -76,8 +76,6 @@ private IEnumerable WorkspaceFiles() { return _fileSystem.GetDirectoryInfo(_workspaceRootPath).EnumerateFileSystemInfos(_includeFiles, _excludeFiles); } - private bool IsFileIndexed(string path) => _symbolIndex.IsIndexed(path); - public void ProcessClosedFile(string path) { if (IsFileOnWorkspace(path)) { _symbolIndex.Parse(path); diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index 99e9c40a2..0f2657780 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -30,10 +30,6 @@ public void Parse() { WorkAndSetTcs(ParseAsync); } - public void Add(IDocument doc) { - WorkAndSetTcs(ct => IndexAsync(doc, ct)); - } - public void Index(IDocument doc) { WorkAndSetTcs(ct => IndexAsync(doc, ct)); } diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index 61ae9f70b..66c3322ac 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -74,8 +74,6 @@ public void Delete(string path) { mostRecentDocSymbols.Dispose(); } - public bool IsIndexed(string path) => _index.ContainsKey(path); - public void ReIndex(string path, IDocument doc) { if (_index.TryGetValue(path, out var currentSymbols)) { currentSymbols.Index(doc); From c83112b826f19108ab020105ba65d22d2e8a97b1 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 21 Feb 2019 14:55:00 -0800 Subject: [PATCH 108/123] Using matcher for checking files in workspace --- src/Core/Impl/IO/DirectoryInfoProxy.cs | 18 ++++++++++++++---- src/Core/Impl/IO/IDirectoryInfo.cs | 1 + .../Impl/Indexing/IndexManager.cs | 3 ++- src/LanguageServer/Test/IndexManagerTests.cs | 8 +++++++- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/Core/Impl/IO/DirectoryInfoProxy.cs b/src/Core/Impl/IO/DirectoryInfoProxy.cs index 5bb81569e..b40ba05e8 100644 --- a/src/Core/Impl/IO/DirectoryInfoProxy.cs +++ b/src/Core/Impl/IO/DirectoryInfoProxy.cs @@ -43,10 +43,8 @@ public IEnumerable EnumerateFileSystemInfos() => _directoryInfo .EnumerateFileSystemInfos() .Select(CreateFileSystemInfoProxy); - public IEnumerable EnumerateFileSystemInfos(string[] includeFiles, string[] excludeFiles) { - Matcher matcher = new Matcher(); - matcher.AddIncludePatterns(includeFiles.IsNullOrEmpty() ? new[] { "**/*" } : includeFiles); - matcher.AddExcludePatterns(excludeFiles ?? Enumerable.Empty()); + public IEnumerable EnumerateFileSystemInfos(string[] includePatterns, string[] excludePatterns) { + var matcher = GetMatcher(includePatterns, excludePatterns); PatternMatchingResult matchResult = SafeExecuteMatcher(matcher); return matchResult.Files.Select((filePatternMatch) => { var fileSystemInfo = _directoryInfo.GetFileSystemInfos(filePatternMatch.Stem).First(); @@ -54,6 +52,18 @@ public IEnumerable EnumerateFileSystemInfos(string[] includeFil }); } + public bool Match(string[] includePatterns, string[] excludePatterns, string path) { + var matcher = GetMatcher(includePatterns, excludePatterns); + return matcher.Match(FullName, path).HasMatches; + } + + private static Matcher GetMatcher(string[] includePatterns, string[] excludePatterns) { + Matcher matcher = new Matcher(); + matcher.AddIncludePatterns(includePatterns.IsNullOrEmpty() ? new[] { "**/*" } : includePatterns); + matcher.AddExcludePatterns(excludePatterns ?? Enumerable.Empty()); + return matcher; + } + private PatternMatchingResult SafeExecuteMatcher(Matcher matcher) { var directoryInfo = new DirectoryInfoWrapper(_directoryInfo); for (var retries = 5; retries > 0; retries--) { diff --git a/src/Core/Impl/IO/IDirectoryInfo.cs b/src/Core/Impl/IO/IDirectoryInfo.cs index b8ccd5410..9cfb97da7 100644 --- a/src/Core/Impl/IO/IDirectoryInfo.cs +++ b/src/Core/Impl/IO/IDirectoryInfo.cs @@ -20,5 +20,6 @@ public interface IDirectoryInfo : IFileSystemInfo { IDirectoryInfo Parent { get; } IEnumerable EnumerateFileSystemInfos(); IEnumerable EnumerateFileSystemInfos(string[] includeFiles, string[] excludeFiles); + bool Match(string[] includeFiles, string[] excludeFiles, string path); } } diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index 06e360118..ee17a28b7 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -88,7 +88,8 @@ private bool IsFileOnWorkspace(string path) { if (string.IsNullOrEmpty(_workspaceRootPath)) { return false; } - return _fileSystem.IsPathUnderRoot(_workspaceRootPath, path); + return _fileSystem.GetDirectoryInfo(_workspaceRootPath) + .Match(_includeFiles, _excludeFiles, path); } public void ProcessNewFile(string path, IDocument doc) { diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index 3cd768ac7..62851b0c2 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -317,7 +317,13 @@ public void SetIdleEvent(EventHandler handler) { } private void SetupRootDir() { - IDirectoryInfo directoryInfo = Substitute.For(); + var directoryInfo = Substitute.For(); + directoryInfo.Match(new string[] { }, new string[] { }, "").ReturnsForAnyArgs(callInfo => { + string path = callInfo.ArgAt(2); + return _rootFileList + .Where(fsInfo => PathEqualityComparer.Instance.Equals(fsInfo.FullName, path)) + .Count() > 0; + }); // Doesn't work without 'forAnyArgs' directoryInfo.EnumerateFileSystemInfos(new string[] { }, new string[] { }).ReturnsForAnyArgs(_rootFileList); FileSystem.GetDirectoryInfo(_rootPath).Returns(directoryInfo); From eba45f245236f8d48e1caa4bb5ec275d27b2e7c5 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 21 Feb 2019 15:06:01 -0800 Subject: [PATCH 109/123] State machine changes --- .../Impl/Indexing/MostRecentDocumentSymbols.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index 0f2657780..f0a69e817 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -52,7 +52,7 @@ public void WorkAndSetTcs(Func Date: Thu, 21 Feb 2019 15:16:10 -0800 Subject: [PATCH 110/123] Change exception type --- .../Impl/Indexing/MostRecentDocumentSymbols.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index f0a69e817..bee59f3d3 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -52,7 +52,7 @@ public void WorkAndSetTcs(Func Date: Thu, 21 Feb 2019 15:41:40 -0800 Subject: [PATCH 111/123] Double releasing semaphore fix` --- src/LanguageServer/Impl/Indexing/IndexParser.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index 3a6939fdf..2ae26231a 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -57,9 +57,6 @@ private async Task Parse(string path, CancellationTokenSource parseCt var parser = Parser.CreateParser(stream, _version); ast = parser.ParseFile(); } - } catch (Exception ex) { - _semaphore.Release(); - throw ex; } finally { _semaphore.Release(); } @@ -74,6 +71,7 @@ public void Dispose() { _allProcessingCts.Dispose(); } } + _semaphore.Dispose(); } } } From 58d53963060239762808018d4348c8fbf4d9c26e Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 21 Feb 2019 16:49:58 -0800 Subject: [PATCH 112/123] Removing continuewith in mostrecentdoc --- .../Indexing/MostRecentDocumentSymbols.cs | 23 +++++++++++-------- .../Impl/Indexing/SymbolIndex.cs | 3 +-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index bee59f3d3..9c74137a4 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -34,7 +34,7 @@ public void Index(IDocument doc) { WorkAndSetTcs(ct => IndexAsync(doc, ct)); } - public void WorkAndSetTcs(Func>> asyncFunc) { + public void WorkAndSetTcs(Func>> asyncWork) { CancellationTokenSource currentCts; TaskCompletionSource> currentTcs; lock (_syncObj) { @@ -42,31 +42,36 @@ public void WorkAndSetTcs(Func { + DoWork(currentCts, asyncWork).SetCompletionResultTo(currentTcs); + } + + private async Task> DoWork(CancellationTokenSource tcs, Func>> asyncWork) { + var token = tcs.Token; + + try { + return await asyncWork(token); + } finally { lock (_syncObj) { - currentCts.Dispose(); - if (_fileCts == currentCts) { + tcs.Dispose(); + if (!token.IsCancellationRequested) { state = WorkQueueState.FinishedWork; } } - return t.GetAwaiter().GetResult(); - }, currentCts.Token).SetCompletionResultTo(currentTcs); + } } public Task> GetSymbolsAsync(CancellationToken ct = default) { diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index 66c3322ac..05c81cf09 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -65,8 +65,7 @@ public void Add(string path, IDocument doc) { } public void Parse(string path) { - var mostRecentSymbols = _index.GetOrAdd(path, MakeMostRecentDocSymbols(path)); - mostRecentSymbols.Parse(); + _index.GetOrAdd(path, MakeMostRecentDocSymbols(path)).Parse(); } public void Delete(string path) { From 0ae25d87e7c6b3852510518140941653b7f023ec Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Thu, 21 Feb 2019 17:18:56 -0800 Subject: [PATCH 113/123] Setting state, only one assignment per method --- .../Impl/Indexing/MostRecentDocumentSymbols.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index 9c74137a4..bcddf1a6b 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -86,20 +86,18 @@ public void MarkAsPending() { lock (_syncObj) { switch (state) { case WorkQueueState.WaitingForWork: - state = WorkQueueState.WaitingForWork; break; case WorkQueueState.Working: CancelExistingWork(); RenewTcs(); - state = WorkQueueState.WaitingForWork; break; case WorkQueueState.FinishedWork: RenewTcs(); - state = WorkQueueState.WaitingForWork; break; default: throw new InvalidOperationException(); } + state = WorkQueueState.WaitingForWork; } } @@ -108,19 +106,18 @@ public void Dispose() { switch (state) { case WorkQueueState.Working: CancelExistingWork(); - state = WorkQueueState.FinishedWork; break; case WorkQueueState.WaitingForWork: CancelExistingWork(); // Manually cancel tcs, in case any task is awaiting _fileTcs.TrySetCanceled(); - state = WorkQueueState.FinishedWork; break; case WorkQueueState.FinishedWork: break; default: throw new InvalidOperationException(); } + state = WorkQueueState.FinishedWork; _indexParser.Dispose(); } } From 8ef57508d8c50fafb1afb8e7d9fd1ff0ac7ae64e Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 22 Feb 2019 09:59:27 -0800 Subject: [PATCH 114/123] Disposing index parser in symbol index --- src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs | 1 - src/LanguageServer/Impl/Indexing/SymbolIndex.cs | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index bcddf1a6b..479c984fe 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -118,7 +118,6 @@ public void Dispose() { throw new InvalidOperationException(); } state = WorkQueueState.FinishedWork; - _indexParser.Dispose(); } } diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index 05c81cf09..ad4d76475 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -107,6 +107,7 @@ public void Dispose() { foreach (var recentSymbols in _index.Values) { recentSymbols.Dispose(); } + _indexParser.Dispose(); } } } From 14d3d9038963d90958de091cd5c989304867ffa9 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 22 Feb 2019 10:17:35 -0800 Subject: [PATCH 115/123] Make includePatterns optional for match --- src/Core/Impl/IO/DirectoryInfoProxy.cs | 2 +- src/Core/Impl/IO/IDirectoryInfo.cs | 2 +- src/LanguageServer/Impl/Indexing/IndexManager.cs | 2 +- src/LanguageServer/Test/IndexManagerTests.cs | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Core/Impl/IO/DirectoryInfoProxy.cs b/src/Core/Impl/IO/DirectoryInfoProxy.cs index b40ba05e8..57e5647a2 100644 --- a/src/Core/Impl/IO/DirectoryInfoProxy.cs +++ b/src/Core/Impl/IO/DirectoryInfoProxy.cs @@ -52,7 +52,7 @@ public IEnumerable EnumerateFileSystemInfos(string[] includePat }); } - public bool Match(string[] includePatterns, string[] excludePatterns, string path) { + public bool Match(string path, string[] includePatterns = default, string[] excludePatterns = default) { var matcher = GetMatcher(includePatterns, excludePatterns); return matcher.Match(FullName, path).HasMatches; } diff --git a/src/Core/Impl/IO/IDirectoryInfo.cs b/src/Core/Impl/IO/IDirectoryInfo.cs index 9cfb97da7..0ee8bed59 100644 --- a/src/Core/Impl/IO/IDirectoryInfo.cs +++ b/src/Core/Impl/IO/IDirectoryInfo.cs @@ -20,6 +20,6 @@ public interface IDirectoryInfo : IFileSystemInfo { IDirectoryInfo Parent { get; } IEnumerable EnumerateFileSystemInfos(); IEnumerable EnumerateFileSystemInfos(string[] includeFiles, string[] excludeFiles); - bool Match(string[] includeFiles, string[] excludeFiles, string path); + bool Match(string path, string[] includePatterns = default, string[] excludePatterns = default); } } diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index ee17a28b7..98a3a147b 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -89,7 +89,7 @@ private bool IsFileOnWorkspace(string path) { return false; } return _fileSystem.GetDirectoryInfo(_workspaceRootPath) - .Match(_includeFiles, _excludeFiles, path); + .Match(path, _includeFiles, _excludeFiles); } public void ProcessNewFile(string path, IDocument doc) { diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index 62851b0c2..48fe1fb31 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -318,8 +318,8 @@ public void SetIdleEvent(EventHandler handler) { private void SetupRootDir() { var directoryInfo = Substitute.For(); - directoryInfo.Match(new string[] { }, new string[] { }, "").ReturnsForAnyArgs(callInfo => { - string path = callInfo.ArgAt(2); + directoryInfo.Match("", new string[] { }, new string[] { }).ReturnsForAnyArgs(callInfo => { + string path = callInfo.ArgAt(0); return _rootFileList .Where(fsInfo => PathEqualityComparer.Instance.Equals(fsInfo.FullName, path)) .Count() > 0; From a87deacd8c9ce5a9e33deb1c0e3662211e881fa3 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Fri, 22 Feb 2019 10:39:00 -0800 Subject: [PATCH 116/123] Not calling excludePatterns if unnecessary --- src/Core/Impl/IO/DirectoryInfoProxy.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Core/Impl/IO/DirectoryInfoProxy.cs b/src/Core/Impl/IO/DirectoryInfoProxy.cs index 57e5647a2..d2a7a34d9 100644 --- a/src/Core/Impl/IO/DirectoryInfoProxy.cs +++ b/src/Core/Impl/IO/DirectoryInfoProxy.cs @@ -60,7 +60,9 @@ public bool Match(string path, string[] includePatterns = default, string[] excl private static Matcher GetMatcher(string[] includePatterns, string[] excludePatterns) { Matcher matcher = new Matcher(); matcher.AddIncludePatterns(includePatterns.IsNullOrEmpty() ? new[] { "**/*" } : includePatterns); - matcher.AddExcludePatterns(excludePatterns ?? Enumerable.Empty()); + if (!excludePatterns.IsNullOrEmpty()) { + matcher.AddExcludePatterns(excludePatterns); + } return matcher; } From 2ee4b55e71f08e30129e8be3e8e26225b5980bd7 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Sun, 24 Feb 2019 19:47:32 -0800 Subject: [PATCH 117/123] Use of concurrent dictionary instead of lock on object --- .../Impl/Indexing/IndexManager.cs | 28 +++++++------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index 98a3a147b..a0f0f8ade 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -35,9 +36,8 @@ internal class IndexManager : IIndexManager { private readonly string[] _includeFiles; private readonly string[] _excludeFiles; private readonly IIdleTimeService _idleTimeService; - private readonly HashSet _pendingDocs = new HashSet(new UriDocumentComparer()); private readonly PythonLanguageVersion _version; - private DateTime _lastPendingDocAddedTime; + private readonly ConcurrentDictionary _pendingDocs = new ConcurrentDictionary(new UriDocumentComparer()); public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, string[] excludeFiles, IIdleTimeService idleTimeService) { @@ -113,28 +113,20 @@ public Task> WorkspaceSymbolsAsync(string query, int m } public void AddPendingDoc(IDocument doc) { - lock (_pendingDocs) { - _lastPendingDocAddedTime = DateTime.Now; - _pendingDocs.Add(doc); - _symbolIndex.MarkAsPending(doc.Uri.AbsolutePath); - } + _pendingDocs.TryAdd(doc, DateTime.Now); + _symbolIndex.MarkAsPending(doc.Uri.AbsolutePath); } private void OnIdle(object sender, EventArgs _) { - if (_pendingDocs.Count > 0 && (DateTime.Now - _lastPendingDocAddedTime).TotalMilliseconds > ReIndexingDelay) { - ReIndexPendingDocsAsync(); - } + ReIndexPendingDocsAsync(); } private void ReIndexPendingDocsAsync() { - IEnumerable pendingDocs; - lock (_pendingDocs) { - pendingDocs = _pendingDocs.ToList(); - _pendingDocs.Clear(); - } - - foreach (var doc in pendingDocs.MaybeEnumerate()) { - ReIndexFile(doc.Uri.AbsolutePath, doc); + foreach (var (doc, lastTime) in _pendingDocs) { + if ((DateTime.Now - lastTime).TotalMilliseconds > ReIndexingDelay) { + ReIndexFile(doc.Uri.AbsolutePath, doc); + _pendingDocs.TryRemove(doc, out var _); + } } } From ad258cc49c2eb59a914898b08ceb94c37d27100c Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Mon, 25 Feb 2019 09:53:46 -0800 Subject: [PATCH 118/123] Const instead of static --- src/LanguageServer/Impl/Indexing/IndexManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index 98a3a147b..fd20a8cb1 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -28,7 +28,7 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal class IndexManager : IIndexManager { - private static int DefaultReIndexDelay = 350; + private const int DefaultReIndexDelay = 350; private readonly ISymbolIndex _symbolIndex; private readonly IFileSystem _fileSystem; private readonly string _workspaceRootPath; From 4fb33dbf11ba07b5390c989df230cde0d02f1bf9 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Mon, 25 Feb 2019 11:25:59 -0800 Subject: [PATCH 119/123] IndexParser task creation in a cleaner way Deleted complicated logic in tests for checking the cancellation of previous work. It no longer works since it relied on Task.Run checking for the CancellationToken in the background --- src/LanguageServer/Impl/Indexing/IndexParser.cs | 13 ++++++------- src/LanguageServer/Test/IndexManagerTests.cs | 10 ---------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index 2ae26231a..a9f732720 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -13,10 +13,10 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.IO; using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Core; using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; @@ -42,17 +42,15 @@ public IndexParser(IFileSystem fileSystem, PythonLanguageVersion version) { public Task ParseAsync(string path, CancellationToken cancellationToken = default) { var linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, cancellationToken); - Task parseTask; - parseTask = Task.Run(async () => await Parse(path, linkedParseCts)); - parseTask.ContinueWith(_ => linkedParseCts.Dispose()); + var parseTask = Parse(path, linkedParseCts.Token); + parseTask.ContinueWith(_ => linkedParseCts.Dispose()).DoNotWait(); return parseTask; } - private async Task Parse(string path, CancellationTokenSource parseCts) { - await _semaphore.WaitAsync(); + private async Task Parse(string path, CancellationToken parseCt) { + await _semaphore.WaitAsync(parseCt); PythonAst ast; try { - parseCts.Token.ThrowIfCancellationRequested(); using (var stream = _fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read)) { var parser = Parser.CreateParser(stream, _version); ast = parser.ParseFile(); @@ -61,6 +59,7 @@ private async Task Parse(string path, CancellationTokenSource parseCt _semaphore.Release(); } + parseCt.ThrowIfCancellationRequested(); return ast; } diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index 48fe1fb31..677e87ee7 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -18,7 +18,6 @@ using System.IO; using System.Linq; using System.Text; -using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Documents; @@ -209,22 +208,13 @@ public async Task HierarchicalDocumentSymbolsAsync() { [TestMethod, Priority(0)] public async Task LatestVersionASTVersionIsIndexed() { - ManualResetEventSlim reOpenedFileFinished = new ManualResetEventSlim(false); - ManualResetEventSlim fileOpenedEvent = new ManualResetEventSlim(false); var context = new IndexTestContext(this); var pythonTestFilePath = context.FileWithXVarInRootDir(); - context.SetFileOpen(pythonTestFilePath, _ => { - fileOpenedEvent.Set(); - reOpenedFileFinished.Wait(); - return MakeStream("x = 1"); - }); var indexManager = context.GetDefaultIndexManager(); indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("y = 1")); indexManager.ProcessClosedFile(pythonTestFilePath); - fileOpenedEvent.Wait(); indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("z = 1")); - reOpenedFileFinished.Set(); var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath); symbols.Should().HaveCount(1); From 61039d55644903ca5ea2af4e92a7b9fb29899612 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Mon, 25 Feb 2019 14:20:11 -0800 Subject: [PATCH 120/123] Use of disposablebag everywhere` --- .../Impl/Implementation/Server.cs | 4 ++-- .../Impl/Indexing/IndexManager.cs | 24 ++++++++++--------- .../Impl/Indexing/IndexParser.cs | 18 +++++++------- src/LanguageServer/Test/IndexManagerTests.cs | 4 ++-- 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index 1ebc4cdeb..63121ff0a 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -124,12 +124,12 @@ public async Task InitializeAsync(InitializeParams @params, Ca _services.AddService(_interpreter); var fileSystem = _services.GetService(); - var symbolIndex = new SymbolIndex(fileSystem, _interpreter.LanguageVersion); - _indexManager = new IndexManager(symbolIndex, fileSystem, _interpreter.LanguageVersion, rootDir, + _indexManager = new IndexManager(fileSystem, _interpreter.LanguageVersion, rootDir, @params.initializationOptions.includeFiles, @params.initializationOptions.excludeFiles, _services.GetService()); _services.AddService(_indexManager); + _disposableBag.Add(_indexManager); DisplayStartupInfo(); diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index b0a8cdea0..e6a8dfcfc 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -21,8 +21,8 @@ using System.Threading.Tasks; using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Core; using Microsoft.Python.Core.Diagnostics; +using Microsoft.Python.Core.Disposables; using Microsoft.Python.Core.Idle; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; @@ -35,11 +35,10 @@ internal class IndexManager : IIndexManager { private readonly string _workspaceRootPath; private readonly string[] _includeFiles; private readonly string[] _excludeFiles; - private readonly IIdleTimeService _idleTimeService; - private readonly PythonLanguageVersion _version; + private readonly DisposableBag _disposables = new DisposableBag(nameof(IndexManager)); private readonly ConcurrentDictionary _pendingDocs = new ConcurrentDictionary(new UriDocumentComparer()); - public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, + public IndexManager(IFileSystem fileSystem, PythonLanguageVersion version, string rootPath, string[] includeFiles, string[] excludeFiles, IIdleTimeService idleTimeService) { Check.ArgumentNotNull(nameof(fileSystem), fileSystem); Check.ArgumentNotNull(nameof(rootPath), rootPath); @@ -47,19 +46,22 @@ public IndexManager(ISymbolIndex symbolIndex, IFileSystem fileSystem, PythonLang Check.ArgumentNotNull(nameof(excludeFiles), excludeFiles); Check.ArgumentNotNull(nameof(idleTimeService), idleTimeService); - _symbolIndex = symbolIndex; _fileSystem = fileSystem; _workspaceRootPath = rootPath; _includeFiles = includeFiles; _excludeFiles = excludeFiles; - _idleTimeService = idleTimeService; - _idleTimeService.Idle += OnIdle; - _version = version; - ReIndexingDelay = DefaultReIndexDelay; + + _symbolIndex = new SymbolIndex(_fileSystem, version); + idleTimeService.Idle += OnIdle; + + _disposables + .Add(_symbolIndex) + .Add(() => idleTimeService.Idle -= OnIdle); + StartAddRootDir(); } - public int ReIndexingDelay { get; set; } + public int ReIndexingDelay { get; set; } = DefaultReIndexDelay; private void StartAddRootDir() { foreach (var fileInfo in WorkspaceFiles()) { @@ -101,7 +103,7 @@ public void ReIndexFile(string path, IDocument doc) { } public void Dispose() { - _symbolIndex.Dispose(); + _disposables.TryDispose(); } public Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken cancellationToken = default) { diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index a9f732720..c79371de1 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -18,18 +18,19 @@ using System.Threading.Tasks; using Microsoft.Python.Core; using Microsoft.Python.Core.Diagnostics; +using Microsoft.Python.Core.Disposables; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.LanguageServer.Indexing { internal sealed class IndexParser : IIndexParser { + private DisposableBag disposables = new DisposableBag(nameof(IndexParser)); private const int MaxConcurrentParsings = 10; private readonly IFileSystem _fileSystem; private readonly PythonLanguageVersion _version; private readonly SemaphoreSlim _semaphore; private readonly CancellationTokenSource _allProcessingCts = new CancellationTokenSource(); - private readonly object _syncObj = new object(); public IndexParser(IFileSystem fileSystem, PythonLanguageVersion version) { Check.ArgumentNotNull(nameof(fileSystem), fileSystem); @@ -37,6 +38,13 @@ public IndexParser(IFileSystem fileSystem, PythonLanguageVersion version) { _fileSystem = fileSystem; _version = version; _semaphore = new SemaphoreSlim(MaxConcurrentParsings); + + disposables + .Add(_semaphore) + .Add(() => { + _allProcessingCts.Cancel(); + _allProcessingCts.Dispose(); + }); } public Task ParseAsync(string path, CancellationToken cancellationToken = default) { @@ -64,13 +72,7 @@ private async Task Parse(string path, CancellationToken parseCt) { } public void Dispose() { - lock (_syncObj) { - if (!_allProcessingCts.IsCancellationRequested) { - _allProcessingCts.Cancel(); - _allProcessingCts.Dispose(); - } - } - _semaphore.Dispose(); + disposables.TryDispose(); } } } diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index 677e87ee7..c0c3f85fb 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -67,7 +67,7 @@ public void NullDirectoryThrowsException() { var context = new IndexTestContext(this); Action construct = () => { PythonLanguageVersion version = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); - IIndexManager indexManager = new IndexManager(new SymbolIndex(context.FileSystem, version), context.FileSystem, + IIndexManager indexManager = new IndexManager(context.FileSystem, version, null, new string[] { }, new string[] { }, new IdleTimeService()); }; @@ -284,7 +284,7 @@ public string FileWithXVarInRootDir() { } public IIndexManager GetDefaultIndexManager() { - _indexM = new IndexManager(SymbolIndex, FileSystem, _pythonLanguageVersion, + _indexM = new IndexManager(FileSystem, _pythonLanguageVersion, _rootPath, new string[] { }, new string[] { }, _idleTimeService) { ReIndexingDelay = 1 From 2a23b61681bf77d6af349904049ecea48e6f51a8 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Mon, 25 Feb 2019 14:29:44 -0800 Subject: [PATCH 121/123] Delete unused fields and params --- .../Impl/Indexing/MostRecentDocumentSymbols.cs | 2 +- src/LanguageServer/Impl/Indexing/SymbolIndex.cs | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index 479c984fe..c0017f74d 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -21,7 +21,7 @@ class MostRecentDocumentSymbols : IMostRecentDocumentSymbols { private TaskCompletionSource> _fileTcs = new TaskCompletionSource>(); private WorkQueueState state = WorkQueueState.WaitingForWork; - public MostRecentDocumentSymbols(string path, IFileSystem fileSystem, PythonLanguageVersion version, IIndexParser indexParser) { + public MostRecentDocumentSymbols(string path, IIndexParser indexParser) { _path = path; _indexParser = indexParser; } diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index ad4d76475..200955b41 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -26,17 +26,12 @@ namespace Microsoft.Python.LanguageServer.Indexing { internal sealed class SymbolIndex : ISymbolIndex { private readonly ConcurrentDictionary _index; - private readonly IFileSystem _fileSystem; - private readonly PythonLanguageVersion _version; private readonly IIndexParser _indexParser; public SymbolIndex(IFileSystem fileSystem, PythonLanguageVersion version) { - _fileSystem = fileSystem; - _version = version; - var comparer = PathEqualityComparer.Instance; _index = new ConcurrentDictionary(comparer); - _indexParser = new IndexParser(_fileSystem, _version); + _indexParser = new IndexParser(fileSystem, version); } public Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken ct = default) { @@ -100,7 +95,7 @@ private IEnumerable WorkspaceSymbolsQuery(string path, string query, } private IMostRecentDocumentSymbols MakeMostRecentDocSymbols(string path) { - return new MostRecentDocumentSymbols(path, _fileSystem, _version, _indexParser); + return new MostRecentDocumentSymbols(path, _indexParser); } public void Dispose() { From 6e449423e1d41b8d6d4bc332e409dbbf324772e8 Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Mon, 25 Feb 2019 14:41:34 -0800 Subject: [PATCH 122/123] More disposablebag usage --- src/LanguageServer/Impl/Indexing/SymbolIndex.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs index 200955b41..0680c563a 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndex.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndex.cs @@ -20,11 +20,13 @@ using System.Threading.Tasks; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core; +using Microsoft.Python.Core.Disposables; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; namespace Microsoft.Python.LanguageServer.Indexing { internal sealed class SymbolIndex : ISymbolIndex { + private readonly DisposableBag _disposables = new DisposableBag(nameof(SymbolIndex)); private readonly ConcurrentDictionary _index; private readonly IIndexParser _indexParser; @@ -32,6 +34,13 @@ public SymbolIndex(IFileSystem fileSystem, PythonLanguageVersion version) { var comparer = PathEqualityComparer.Instance; _index = new ConcurrentDictionary(comparer); _indexParser = new IndexParser(fileSystem, version); + _disposables + .Add(_indexParser) + .Add(() => { + foreach (var recentSymbols in _index.Values) { + recentSymbols.Dispose(); + } + }); } public Task> HierarchicalDocumentSymbolsAsync(string path, CancellationToken ct = default) { @@ -99,10 +108,7 @@ private IMostRecentDocumentSymbols MakeMostRecentDocSymbols(string path) { } public void Dispose() { - foreach (var recentSymbols in _index.Values) { - recentSymbols.Dispose(); - } - _indexParser.Dispose(); + _disposables.TryDispose(); } } } From 5485c9cd8a89b97ad6ae8370defdc3bf288ab74f Mon Sep 17 00:00:00 2001 From: Brian Bokser Date: Mon, 25 Feb 2019 15:19:49 -0800 Subject: [PATCH 123/123] Unused imports --- src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index c0017f74d..277267d09 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -7,8 +7,6 @@ using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core; using Microsoft.Python.Core.Diagnostics; -using Microsoft.Python.Core.IO; -using Microsoft.Python.Parsing; namespace Microsoft.Python.LanguageServer.Indexing { class MostRecentDocumentSymbols : IMostRecentDocumentSymbols {