From 3958d610e7c3d4756afc859df7f33c73e56f0cda Mon Sep 17 00:00:00 2001 From: Alexander Sher Date: Tue, 4 Dec 2018 16:16:28 -0800 Subject: [PATCH 01/10] Clean up Completions, Find References and Hover unit tests. --- src/Analysis/Engine/Test/AnalysisTest.cs | 2 +- src/Analysis/Engine/Test/AstAnalysisTests.cs | 10 +- src/Analysis/Engine/Test/CompletionTests.cs | 275 +-- .../Engine/Test/FindReferencesTests.cs | 2011 ++++++++--------- .../CompletionListAssertions.cs | 18 + src/Analysis/Engine/Test/HoverTests.cs | 155 +- src/Analysis/Engine/Test/ServerBasedTest.cs | 5 - src/Analysis/Engine/Test/ServerExtensions.cs | 61 +- .../Engine/Test/ServerTestMethodAttribute.cs | 8 + src/UnitTests/Core/Impl/TestData.cs | 5 + 10 files changed, 1201 insertions(+), 1349 deletions(-) diff --git a/src/Analysis/Engine/Test/AnalysisTest.cs b/src/Analysis/Engine/Test/AnalysisTest.cs index fdb04ffcb..54d6917ed 100644 --- a/src/Analysis/Engine/Test/AnalysisTest.cs +++ b/src/Analysis/Engine/Test/AnalysisTest.cs @@ -3140,7 +3140,7 @@ from collections import namedtuple pt = nt(1, 2) "; - server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); + server.Analyzer.SetTypeStubPaths(new[] { TestData.GetDefaultTypeshedPath() }); server.Analyzer.Limits = new AnalysisLimits { UseTypeStubPackages = true, UseTypeStubPackagesExclusively = false }; var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); diff --git a/src/Analysis/Engine/Test/AstAnalysisTests.cs b/src/Analysis/Engine/Test/AstAnalysisTests.cs index 9a81502b5..b9a470ba2 100644 --- a/src/Analysis/Engine/Test/AstAnalysisTests.cs +++ b/src/Analysis/Engine/Test/AstAnalysisTests.cs @@ -881,7 +881,7 @@ public async Task TypeShedChildModules() { } using (var server = await CreateServerAsync()) { - server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); + server.Analyzer.SetTypeStubPaths(new[] { TestData.GetDefaultTypeshedPath() }); var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(@"import urllib"); var secondMembers = server.Analyzer.GetModuleMembers(analysis.InterpreterContext, new[] { "urllib" }) @@ -895,7 +895,7 @@ public async Task TypeShedChildModules() { [TestMethod, Priority(0)] public async Task TypeShedSysExcInfo() { using (var server = await CreateServerAsync()) { - server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); + server.Analyzer.SetTypeStubPaths(new[] { TestData.GetDefaultTypeshedPath() }); var code = @"import sys e1, e2, e3 = sys.exc_info()"; @@ -915,7 +915,7 @@ public async Task TypeShedSysExcInfo() { [TestMethod, Priority(0)] public async Task TypeShedJsonMakeScanner() { using (var server = await CreateServerAsync()) { - server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); + server.Analyzer.SetTypeStubPaths(new[] { TestData.GetDefaultTypeshedPath() }); var code = @"import _json scanner = _json.make_scanner()"; @@ -941,7 +941,7 @@ public async Task TypeShedJsonMakeScanner() { [TestMethod, Priority(0)] public async Task TypeShedSysInfo() { using (var server = await CreateServerAsync()) { - server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); + server.Analyzer.SetTypeStubPaths(new[] { TestData.GetDefaultTypeshedPath() }); server.Analyzer.Limits = new AnalysisLimits { UseTypeStubPackages = true, UseTypeStubPackagesExclusively = true }; var code = @"import sys @@ -1016,7 +1016,7 @@ from ReturnAnnotation import * [TestMethod, Priority(0)] public async Task TypeShedNamedTuple() { using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { - server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); + server.Analyzer.SetTypeStubPaths(new[] { TestData.GetDefaultTypeshedPath() }); server.Analyzer.Limits = new AnalysisLimits { UseTypeStubPackages = true, UseTypeStubPackagesExclusively = true }; var code = "from collections import namedtuple\n"; diff --git a/src/Analysis/Engine/Test/CompletionTests.cs b/src/Analysis/Engine/Test/CompletionTests.cs index 2cc1bb329..cc98b77e6 100644 --- a/src/Analysis/Engine/Test/CompletionTests.cs +++ b/src/Analysis/Engine/Test/CompletionTests.cs @@ -16,7 +16,6 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; using System.Threading; @@ -38,7 +37,7 @@ namespace AnalysisTests { [TestClass] - public class CompletionTests : ServerBasedTest { + public class CompletionTests { public TestContext TestContext { get; set; } [TestInitialize] @@ -46,7 +45,8 @@ public void TestInitialize() => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); [TestCleanup] - public void TestCleanup() => TestEnvironmentImpl.TestCleanup(); + public void TestCleanup() + => TestEnvironmentImpl.TestCleanup(); [ServerTestMethod, Priority(0)] public async Task InWithStatementDerivedClass(Server server) { @@ -613,27 +613,27 @@ public async Task InClassDefinition(Server server, PythonLanguageVersion version [ServerTestMethod, Priority(0)] public async Task InWithStatement(Server server) { var uri = await server.OpenDefaultDocumentAndGetUriAsync("with x as y, z as w: pass"); - await AssertAnyCompletion(server, uri, new SourceLocation(1, 6)); - await AssertCompletion(server, uri, new[] { "as" }, new[] { "abs", "dir" }, new SourceLocation(1, 8)); + (await server.SendCompletion(uri, 0, 5)).Should().HaveAnyCompletions(); + (await server.SendCompletion(uri, 0, 7)).Should().HaveInsertTexts("as").And.NotContainInsertTexts("abs", "dir"); (await server.SendCompletion(uri, 0, 10)).Should().HaveNoCompletion(); - await AssertAnyCompletion(server, uri, new SourceLocation(1, 14)); - await AssertCompletion(server, uri, new[] { "as" }, new[] { "abs", "dir" }, new SourceLocation(1, 17)); + (await server.SendCompletion(uri, 0, 13)).Should().HaveAnyCompletions(); + (await server.SendCompletion(uri, 0, 16)).Should().HaveInsertTexts("as").And.NotContainInsertTexts("abs", "dir"); (await server.SendCompletion(uri, 0, 19)).Should().HaveNoCompletion(); - await AssertAnyCompletion(server, uri, new SourceLocation(1, 21)); + (await server.SendCompletion(uri, 0, 20)).Should().HaveAnyCompletions(); await server.UnloadFileAsync(uri); uri = await server.OpenDefaultDocumentAndGetUriAsync("with "); - await AssertAnyCompletion(server, uri, new SourceLocation(1, 6)); + (await server.SendCompletion(uri, 0, 5)).Should().HaveAnyCompletions(); await server.UnloadFileAsync(uri); uri = await server.OpenDefaultDocumentAndGetUriAsync("with x "); - await AssertAnyCompletion(server, uri, new SourceLocation(1, 6)); - await AssertCompletion(server, uri, new[] { "as" }, new[] { "abs", "dir" }, new SourceLocation(1, 8)); + (await server.SendCompletion(uri, 0, 5)).Should().HaveAnyCompletions(); + (await server.SendCompletion(uri, 0, 7)).Should().HaveInsertTexts("as").And.NotContainInsertTexts("abs", "dir"); await server.UnloadFileAsync(uri); uri = await server.OpenDefaultDocumentAndGetUriAsync("with x as "); - await AssertAnyCompletion(server, uri, new SourceLocation(1, 6)); - await AssertCompletion(server, uri, new[] { "as" }, new[] { "abs", "dir" }, new SourceLocation(1, 8)); + (await server.SendCompletion(uri, 0, 5)).Should().HaveAnyCompletions(); + (await server.SendCompletion(uri, 0, 7)).Should().HaveInsertTexts("as").And.NotContainInsertTexts("abs", "dir"); (await server.SendCompletion(uri, 0, 10)).Should().HaveNoCompletion(); await server.UnloadFileAsync(uri); } @@ -697,10 +697,10 @@ def i(): pass (await server.SendCompletion(uri, 2, 8)).Should().HaveLabels("__init__").And.NotContainLabels("def"); } - [DataRow(PythonLanguageMajorVersion.LatestV2)] - [DataRow(PythonLanguageMajorVersion.LatestV3)] + [DataRow(PythonLanguageMajorVersion.LatestV2, "B, self")] + [DataRow(PythonLanguageMajorVersion.LatestV3, "")] [ServerTestMethod(VersionArgumentIndex = 1), Priority(0)] - public async Task ForOverrideArgs(Server server, PythonLanguageVersion version) { + public async Task ForOverrideArgs(Server server, PythonLanguageVersion version, string superArgs) { var code = @" class A(object): def foo(self, a, b=None, *args, **kwargs): @@ -712,17 +712,9 @@ class B(A): var uri = await server.OpenDefaultDocumentAndGetUriAsync(code); (await server.SendCompletion(uri, 2, 8)).Should().HaveNoCompletion(); - if (version.Is3x()) { - await AssertCompletion(server, uri, - new[] { $"foo(self, a, b=None, *args, **kwargs):{Environment.NewLine} return super().foo(a, b=b, *args, **kwargs)" }, - new[] { $"foo(self, a, b = None, *args, **kwargs):{Environment.NewLine} return super().foo(a, b = b, *args, **kwargs)" }, - new SourceLocation(7, 10)); - } else { - await AssertCompletion(server, uri, - new[] { $"foo(self, a, b=None, *args, **kwargs):{Environment.NewLine} return super(B, self).foo(a, b=b, *args, **kwargs)" }, - new[] { $"foo(self, a, b = None, *args, **kwargs):{Environment.NewLine} return super(B, self).foo(a, b = b, *args, **kwargs)" }, - new SourceLocation(7, 10)); - } + (await server.SendCompletion(uri, 6, 9)).Should() + .HaveInsertTexts($"foo(self, a, b=None, *args, **kwargs):{Environment.NewLine} return super({superArgs}).foo(a, b=b, *args, **kwargs)") + .And.NotContainInsertTexts($"foo(self, a, b = None, *args, **kwargs):{Environment.NewLine} return super({superArgs}).foo(a, b = b, *args, **kwargs)"); } [ServerTestMethod(LatestAvailable3X = true), Priority(0)] @@ -735,7 +727,7 @@ def f(): pass (await server.SendCompletion(uri, 3, 7)).Should().HaveLabels("f", "x", "property", "abs").And.NotContainLabels("def"); await server.ChangeDefaultDocumentAndGetAnalysisAsync("@"); - await AssertAnyCompletion(server, uri, new SourceLocation(1, 2)); + (await server.SendCompletion(uri, 0, 1)).Should().HaveAnyCompletions(); await server.ChangeDefaultDocumentAndGetAnalysisAsync(@"import unittest @@ -755,49 +747,45 @@ await server.ChangeDefaultDocumentAndGetAnalysisAsync(@"import unittest [ServerTestMethod(VersionArgumentIndex = 1), Priority(0)] public async Task InRaise(Server server, PythonLanguageVersion version) { var uri = await server.OpenDefaultDocumentAndGetUriAsync("raise "); - await AssertCompletion(server, uri, new[] { "Exception", "ValueError" }, new[] { "def", "abs" }, new SourceLocation(1, 7)); - await server.UnloadFileAsync(uri); + (await server.SendCompletion(uri, 0, 6)).Should().HaveInsertTexts("Exception", "ValueError").And.NotContainInsertTexts("def", "abs"); if (version.Is3x()) { - uri = await server.OpenDefaultDocumentAndGetUriAsync("raise Exception from "); - await AssertCompletion(server, uri, new[] { "Exception", "ValueError" }, new[] { "def", "abs" }, new SourceLocation(1, 7)); - await AssertCompletion(server, uri, new[] { "from" }, new[] { "Exception", "def", "abs" }, new SourceLocation(1, 17)); - await AssertAnyCompletion(server, uri, new SourceLocation(1, 22)); - await server.UnloadFileAsync(uri); - - uri = await server.OpenDefaultDocumentAndGetUriAsync("raise Exception fr"); - await AssertCompletion(server, uri, new[] { "from" }, new[] { "Exception", "def", "abs" }, new SourceLocation(1, 19), applicableSpan: new SourceSpan(1, 17, 1, 19)); - await server.UnloadFileAsync(uri); + await server.ChangeDefaultDocumentAndGetAnalysisAsync("raise Exception from "); + (await server.SendCompletion(uri, 0, 6)).Should().HaveInsertTexts("Exception", "ValueError").And.NotContainInsertTexts("def", "abs"); + (await server.SendCompletion(uri, 0, 16)).Should().HaveInsertTexts("from").And.NotContainInsertTexts("Exception", "def", "abs"); + (await server.SendCompletion(uri, 0, 21)).Should().HaveAnyCompletions(); + + await server.ChangeDefaultDocumentAndGetAnalysisAsync("raise Exception fr"); + (await server.SendCompletion(uri, 0, 18)).Should().HaveInsertTexts("from") + .And.NotContainInsertTexts("Exception", "def", "abs") + .And.Subject._applicableSpan.Should().Be(0, 16, 0, 18); } - uri = await server.OpenDefaultDocumentAndGetUriAsync("raise Exception, x, y"); - await AssertAnyCompletion(server, uri, new SourceLocation(1, 17)); - await AssertAnyCompletion(server, uri, new SourceLocation(1, 20)); - await server.UnloadFileAsync(uri); + await server.ChangeDefaultDocumentAndGetAnalysisAsync("raise Exception, x, y"); + (await server.SendCompletion(uri, 0, 16)).Should().HaveAnyCompletions(); + (await server.SendCompletion(uri, 0, 19)).Should().HaveAnyCompletions(); } [ServerTestMethod, Priority(0)] public async Task InExcept(Server server) { var uri = await server.OpenDefaultDocumentAndGetUriAsync("try:\n pass\nexcept "); - await AssertCompletion(server, uri, new[] { "Exception", "ValueError" }, new[] { "def", "abs" }, new SourceLocation(3, 8)); - await server.UnloadFileAsync(uri); + (await server.SendCompletion(uri, 2, 7)).Should().HaveInsertTexts("Exception", "ValueError").And.NotContainInsertTexts("def", "abs"); - uri = await server.OpenDefaultDocumentAndGetUriAsync("try:\n pass\nexcept ("); - await AssertCompletion(server, uri, new[] { "Exception", "ValueError" }, new[] { "def", "abs" }, new SourceLocation(3, 9)); - await server.UnloadFileAsync(uri); + await server.ChangeDefaultDocumentAndGetAnalysisAsync("try:\n pass\nexcept ("); + (await server.SendCompletion(uri, 2, 8)).Should().HaveInsertTexts("Exception", "ValueError").And.NotContainInsertTexts("def", "abs"); - uri = await server.OpenDefaultDocumentAndGetUriAsync("try:\n pass\nexcept Exception as "); - await AssertCompletion(server, uri, new[] { "Exception", "ValueError" }, new[] { "def", "abs" }, new SourceLocation(3, 8)); - await AssertCompletion(server, uri, new[] { "as" }, new[] { "Exception", "def", "abs" }, new SourceLocation(3, 18)); + await server.ChangeDefaultDocumentAndGetAnalysisAsync("try:\n pass\nexcept Exception as "); + (await server.SendCompletion(uri, 2, 7)).Should().HaveInsertTexts("Exception", "ValueError").And.NotContainInsertTexts("def", "abs"); + (await server.SendCompletion(uri, 2, 17)).Should().HaveInsertTexts("as").And.NotContainInsertTexts("def", "abs"); (await server.SendCompletion(uri, 2, 21)).Should().HaveNoCompletion(); - await server.UnloadFileAsync(uri); - uri = await server.OpenDefaultDocumentAndGetUriAsync("try:\n pass\nexc"); - await AssertCompletion(server, uri, new[] { "except", "def", "abs" }, new string[0], new SourceLocation(3, 3)); - await server.UnloadFileAsync(uri); + await server.ChangeDefaultDocumentAndGetAnalysisAsync("try:\n pass\nexc"); + (await server.SendCompletion(uri, 2, 17)).Should().HaveInsertTexts("except", "def", "abs"); - uri = await server.OpenDefaultDocumentAndGetUriAsync("try:\n pass\nexcept Exception a"); - await AssertCompletion(server, uri, new[] { "as" }, new[] { "Exception", "def", "abs" }, new SourceLocation(3, 19), applicableSpan: new SourceSpan(3, 18, 3, 19)); + await server.ChangeDefaultDocumentAndGetAnalysisAsync("try:\n pass\nexcept Exception a"); + (await server.SendCompletion(uri, 2, 18)).Should().HaveInsertTexts("as") + .And.NotContainInsertTexts("Exception", "def", "abs") + .And.Subject._applicableSpan.Should().Be(2, 17, 2, 18); await server.UnloadFileAsync(uri); } @@ -842,66 +830,51 @@ def f(self): pass var mod = await server.OpenDefaultDocumentAndGetUriAsync(code); // Completion after "mc " should normally be blank - await AssertCompletion(server, mod, - new string[0], - new string[0], - position: new Position { line = testLine, character = testChar + 1 } - ); + var completion = await server.SendCompletion(mod, testLine, testChar + 1); + completion.Should().HaveNoCompletion(); // While we're here, test with the special override field - await AssertCompletion(server, mod, - new[] { "f" }, - new[] { "abs", "bin", "int", "mc" }, - position: new Position { line = testLine, character = testChar + 1 }, - expr: "mc" - ); + completion = await server.Completion(new CompletionParams { + textDocument = new TextDocumentIdentifier { + uri = mod + }, + position = new Position { + line = testLine, + character = testChar + 1 + }, + _expr = "mc" + }, CancellationToken.None); + completion.Should().HaveInsertTexts("f").And.NotContainInsertTexts("abs", "bin", "int", "mc"); // Send the document update. - await server.DidChangeTextDocument(new DidChangeTextDocumentParams { - textDocument = new VersionedTextDocumentIdentifier { uri = mod, version = 1 }, - contentChanges = new[] { new TextDocumentContentChangedEvent { - text = ".", - range = new Range { - start = new Position { line = testLine, character = testChar }, - end = new Position { line = testLine, character = testChar } - } - } }, - }, CancellationToken.None); + await server.SendDidChangeTextDocumentAsync(mod, ".", new Position {line = testLine, character = testChar}); // Now with the "." event sent, we should see this as a dot completion - await AssertCompletion(server, mod, - new[] { "f" }, - new[] { "abs", "bin", "int", "mc" }, - position: new Position { line = testLine, character = testChar + 1 } - ); + completion = await server.SendCompletion(mod, testLine, testChar + 1); + completion.Should().HaveInsertTexts("f").And.NotContainInsertTexts("abs", "bin", "int", "mc"); } [ServerTestMethod, Priority(0)] public async Task AfterLoad(Server server) { - var mod1 = await server.OpenDocumentAndGetUriAsync("mod1.py", "import mod2\n\nmod2."); + var mod1 = await server.OpenDocumentAndGetUriAsync("mod1.py", @"import mod2 + +mod2."); + await server.WaitForCompleteAnalysisAsync(CancellationToken.None); - await AssertCompletion(server, mod1, - position: new Position { line = 2, character = 5 }, - contains: new string[0], - excludes: new[] { "value" } - ); + var completion = await server.SendCompletion(mod1, 2, 5); + completion.Should().NotContainInsertTexts("value"); var mod2 = await server.OpenDocumentAndGetUriAsync("mod2.py", "value = 123"); + await server.WaitForCompleteAnalysisAsync(CancellationToken.None); - await AssertCompletion(server, mod1, - position: new Position { line = 2, character = 5 }, - contains: new[] { "value" }, - excludes: new string[0] - ); + completion = await server.SendCompletion(mod1, 2, 5); + completion.Should().HaveInsertTexts("value"); await server.UnloadFileAsync(mod2); await server.WaitForCompleteAnalysisAsync(CancellationToken.None); - await AssertCompletion(server, mod1, - position: new Position { line = 2, character = 5 }, - contains: new string[0], - excludes: new[] { "value" } - ); + completion = await server.SendCompletion(mod1, 2, 5); + completion.Should().NotContainInsertTexts("value"); } [ServerTestMethod(LatestAvailable2X = true), Priority(0)] @@ -913,15 +886,13 @@ def test_exception(self): self.assertRaises(). "; var uri = await server.OpenDefaultDocumentAndGetUriAsync(code); + await server.WaitForCompleteAnalysisAsync(CancellationToken.None); - await AssertCompletion(server, uri, - position: new Position { line = 4, character = 28 }, - contains: new[] { "exception" }, - excludes: Array.Empty() - ); + var completion = await server.SendCompletion(uri, 4, 28); + completion.Should().HaveInsertTexts("exception"); } - [ServerTestMethod(LatestAvailable3X = true), Priority(0)] + [ServerTestMethod(LatestAvailable3X = true, DefaultTypeshedPath = true), Priority(0)] public async Task MethodFromBaseClass3X(Server server) { var code = @" import unittest @@ -929,17 +900,15 @@ class Simple(unittest.TestCase): def test_exception(self): self.assertRaises(). "; - - server.Analyzer.Limits = new AnalysisLimits { UseTypeStubPackages = true }; - server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); - + var uri = await server.OpenDefaultDocumentAndGetUriAsync(code); await server.WaitForCompleteAnalysisAsync(CancellationToken.None); + var completion = await server.SendCompletion(uri, 4, 28); completion.Should().HaveInsertTexts("exception"); } - [ServerTestMethod(LatestAvailable3X = true), Priority(0)] + [ServerTestMethod(LatestAvailable3X = true, DefaultTypeshedPath = true), Priority(0)] public async Task CollectionsNamedTuple(Server server) { var code = @" from collections import namedtuple @@ -947,7 +916,6 @@ from collections import namedtuple pt = nt(1, 2) pt. "; - server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); server.Analyzer.Limits = new AnalysisLimits { UseTypeStubPackages = true, UseTypeStubPackagesExclusively = false }; var uri = await server.OpenDefaultDocumentAndGetUriAsync(code); @@ -960,15 +928,14 @@ from collections import namedtuple [ServerTestMethod, Priority(0)] public async Task Hook(Server server) { var uri = await server.OpenDefaultDocumentAndGetUriAsync("x = 123\nx."); - - await AssertCompletion(server, uri, new[] { "real", "imag" }, new string[0], new Position { line = 1, character = 2 }); + (await server.SendCompletion(uri, 1, 2)).Should().HaveInsertTexts("real", "imag"); await server.LoadExtensionAsync(new PythonAnalysisExtensionParams { assembly = typeof(TestCompletionHookProvider).Assembly.FullName, typeName = typeof(TestCompletionHookProvider).FullName }, null, CancellationToken.None); - - await AssertCompletion(server, uri, new[] { "*real", "*imag" }, new[] { "real" }, new Position { line = 1, character = 2 }); + + (await server.SendCompletion(uri, 1, 2)).Should().HaveInsertTexts("*real", "*imag").And.NotContainInsertTexts("real"); } [ServerTestMethod, Priority(0)] @@ -979,18 +946,26 @@ public async Task MultiPartDocument(Server server) { (await server.SendCompletion(mod, 0, 0)).Should().HaveLabels("x"); - Assert.AreEqual(Tuple.Create("y = 2", 1), await ApplyChange(server, modP2, DocumentChange.Insert("y = 2", SourceLocation.MinValue))); + await server.SendDidChangeTextDocumentAsync(modP2, "y = 2", new Position()); await server.WaitForCompleteAnalysisAsync(CancellationToken.None); + var text = await ((IDocument)server.GetProjectEntry(modP2)).ReadDocument(2, out var version).ReadToEndAsync(); + text.Should().Be("y = 2"); + version.Should().Be(1); + (await server.SendCompletion(modP2, 0, 0)).Should().HaveLabels("x", "y"); - Assert.AreEqual(Tuple.Create("z = 3", 1), await ApplyChange(server, modP3, DocumentChange.Insert("z = 3", SourceLocation.MinValue))); + await server.SendDidChangeTextDocumentAsync(modP3, "z = 3", new Position()); await server.WaitForCompleteAnalysisAsync(CancellationToken.None); + text = await ((IDocument)server.GetProjectEntry(modP2)).ReadDocument(3, out version).ReadToEndAsync(); + text.Should().Be("z = 3"); + version.Should().Be(1); + (await server.SendCompletion(modP3, 0, 0)).Should().HaveLabels("x", "y", "z"); (await server.SendCompletion(mod, 0, 0)).Should().HaveLabels("x", "y", "z"); - await ApplyChange(server, mod, DocumentChange.Delete(SourceLocation.MinValue, SourceLocation.MinValue.AddColumns(5))); + await server.SendDidChangeTextDocumentAsync(mod, "", new Position(), new Position { line = 0, character = 5}); await server.WaitForCompleteAnalysisAsync(CancellationToken.None); (await server.SendCompletion(modP2, 0, 0)).Should().HaveLabels("y", "z").And.NotContainLabels("x"); @@ -999,9 +974,11 @@ public async Task MultiPartDocument(Server server) { [ServerTestMethod, Priority(0)] public async Task WithWhitespaceAroundDot(Server server) { - var u = await server.OpenDefaultDocumentAndGetUriAsync("import sys\nsys . version\n"); - await AssertCompletion(server, u, new[] { "argv" }, null, new SourceLocation(2, 7), - new CompletionContext { triggerCharacter = ".", triggerKind = CompletionTriggerKind.TriggerCharacter }); + var uri = await server.OpenDefaultDocumentAndGetUriAsync(@"import sys +sys . version +"); + await server.WaitForCompleteAnalysisAsync(CancellationToken.None); + (await server.SendCompletion(uri, 1, 6)).Should().HaveLabels("argv"); } [ServerTestMethod, Priority(0)] @@ -1069,61 +1046,7 @@ def func(a: Dict[int, str]): completions = await server.SendCompletion(uri, 5, 9); completions.Should().HaveLabels("capitalize"); } - - private static async Task AssertCompletion( - Server s, - Uri uri, - IReadOnlyCollection contains, - IReadOnlyCollection excludes, - Position? position = null, - CompletionContext? context = null, - Func cmpKey = null, - string expr = null, - Range? applicableSpan = null, - InsertTextFormat? allFormat = InsertTextFormat.PlainText) { - await s.WaitForCompleteAnalysisAsync(CancellationToken.None); - var res = await s.Completion(new CompletionParams { - textDocument = new TextDocumentIdentifier { uri = uri }, - position = position ?? new Position(), - context = context, - _expr = expr - }, CancellationToken.None); - DumpDetails(res); - - cmpKey = cmpKey ?? (c => c.insertText); - var items = res.items?.Select(i => (cmpKey(i), i.insertTextFormat)).ToList() ?? new List<(string, InsertTextFormat)>(); - - if (contains != null && contains.Any()) { - items.Select(i => i.Item1).Should().Contain(contains); - - if (allFormat != null) { - items.Where(i => contains.Contains(i.Item1)).Select(i => i.Item2).Should().AllBeEquivalentTo(allFormat); - } - } - - if (excludes != null && excludes.Any()) { - items.Should().NotContain(excludes); - } - - if (applicableSpan.HasValue) { - res._applicableSpan.Should().Be(applicableSpan.Value); - } - } - - private static async Task AssertAnyCompletion(Server s, TextDocumentIdentifier document, Position position) { - await s.WaitForCompleteAnalysisAsync(CancellationToken.None).ConfigureAwait(false); - var res = await s.Completion(new CompletionParams { textDocument = document, position = position }, CancellationToken.None); - DumpDetails(res); - if (res.items == null || !res.items.Any()) { - Assert.Fail("Completions were not returned"); - } - } - - private static void DumpDetails(CompletionList completions) { - var span = ((SourceSpan?)completions._applicableSpan) ?? SourceSpan.None; - Debug.WriteLine($"Completed {completions._expr ?? "(null)"} at {span}"); - } - + class TestCompletionHookProvider : ILanguageServerExtensionProvider { public Task CreateAsync(IPythonLanguageServer server, IReadOnlyDictionary properties, CancellationToken cancellationToken) { return Task.FromResult(new TestCompletionHook()); diff --git a/src/Analysis/Engine/Test/FindReferencesTests.cs b/src/Analysis/Engine/Test/FindReferencesTests.cs index 038d390de..495bee9c8 100644 --- a/src/Analysis/Engine/Test/FindReferencesTests.cs +++ b/src/Analysis/Engine/Test/FindReferencesTests.cs @@ -32,21 +32,17 @@ namespace AnalysisTests { [TestClass] - public class FindReferencesTests : ServerBasedTest { + public class FindReferencesTests { public TestContext TestContext { get; set; } [TestInitialize] - public void TestInitialize() { - TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); - } + public void TestInitialize() => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); [TestCleanup] - public void TestCleanup() { - TestEnvironmentImpl.TestCleanup(); - } + public void TestCleanup() => TestEnvironmentImpl.TestCleanup(); - [TestMethod, Priority(0)] - public async Task ImportStarCorrectRefs() { + [ServerTestMethod(LatestAvailable3X = true), Priority(0)] + public async Task ImportStarCorrectRefs(Server server) { var text1 = @" from mod2 import * @@ -57,26 +53,24 @@ class D(object): pass "; - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { - var uri1 = TestData.GetTestSpecificUri("mod1.py"); - var uri2 = TestData.GetTestSpecificUri("mod2.py"); - using (server.AnalysisQueue.Pause()) { - await server.SendDidOpenTextDocument(uri1, text1); - await server.SendDidOpenTextDocument(uri2, text2); - } - - var references = await server.SendFindReferences(uri2, 1, 7); - references.Should().OnlyHaveReferences( - (uri1, (3, 4, 3, 5), ReferenceKind.Reference), - (uri2, (1, 0, 2, 8), ReferenceKind.Value), - (uri2, (1, 6, 1, 7), ReferenceKind.Definition) - ); + var uri1 = TestData.GetTestSpecificUri("mod1.py"); + var uri2 = TestData.GetTestSpecificUri("mod2.py"); + using (server.AnalysisQueue.Pause()) { + await server.SendDidOpenTextDocument(uri1, text1); + await server.SendDidOpenTextDocument(uri2, text2); } + + var references = await server.SendFindReferences(uri2, 1, 7); + references.Should().OnlyHaveReferences( + (uri1, (3, 4, 3, 5), ReferenceKind.Reference), + (uri2, (1, 0, 2, 8), ReferenceKind.Value), + (uri2, (1, 6, 1, 7), ReferenceKind.Definition) + ); } - [TestMethod, Priority(0)] + [ServerTestMethod(LatestAvailable3X = true), Priority(0)] [Ignore("https://github.com/Microsoft/python-language-server/issues/47")] - public async Task MutatingReferences() { + public async Task MutatingReferences(Server server) { var text1 = @" import mod2 @@ -93,46 +87,44 @@ def __init__(self, value): self.value = value self.value.SomeMethod() "; - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { - var uri1 = TestData.GetNextModuleUri(); - var uri2 = TestData.GetNextModuleUri(); - using (server.AnalysisQueue.Pause()) { - await server.SendDidOpenTextDocument(uri1, text1); - await server.SendDidOpenTextDocument(uri2, text2); - } - - var references = await server.SendFindReferences(uri1, 4, 9); - - references.Should().OnlyHaveReferences( - (uri1, (4, 4, 5, 12), ReferenceKind.Value), - (uri1, (4, 8, 4, 18), ReferenceKind.Definition), - (uri2, (4, 19, 4, 29), ReferenceKind.Reference) - ); - - text1 = text1.Substring(0, text1.IndexOf(" def")) + Environment.NewLine + text1.Substring(text1.IndexOf(" def")); - await server.SendDidChangeTextDocumentAsync(uri1, text1); - - references = await server.SendFindReferences(uri1, 5, 9); - references.Should().OnlyHaveReferences( - (uri1, (5, 4, 6, 12), ReferenceKind.Value), - (uri1, (5, 8, 5, 18), ReferenceKind.Definition), - (uri2, (4, 19, 4, 29), ReferenceKind.Reference) - ); - - text2 = Environment.NewLine + text2; - await server.SendDidChangeTextDocumentAsync(uri2, text2); - - references = await server.SendFindReferences(uri1, 5, 9); - references.Should().OnlyHaveReferences( - (uri1, (5, 4, 6, 12), ReferenceKind.Value), - (uri1, (5, 8, 5, 18), ReferenceKind.Definition), - (uri2, (5, 19, 5, 29), ReferenceKind.Reference) - ); + var uri1 = TestData.GetNextModuleUri(); + var uri2 = TestData.GetNextModuleUri(); + using (server.AnalysisQueue.Pause()) { + await server.SendDidOpenTextDocument(uri1, text1); + await server.SendDidOpenTextDocument(uri2, text2); } + + var references = await server.SendFindReferences(uri1, 4, 9); + + references.Should().OnlyHaveReferences( + (uri1, (4, 4, 5, 12), ReferenceKind.Value), + (uri1, (4, 8, 4, 18), ReferenceKind.Definition), + (uri2, (4, 19, 4, 29), ReferenceKind.Reference) + ); + + text1 = text1.Substring(0, text1.IndexOf(" def")) + Environment.NewLine + text1.Substring(text1.IndexOf(" def")); + await server.SendDidChangeTextDocumentAsync(uri1, text1); + + references = await server.SendFindReferences(uri1, 5, 9); + references.Should().OnlyHaveReferences( + (uri1, (5, 4, 6, 12), ReferenceKind.Value), + (uri1, (5, 8, 5, 18), ReferenceKind.Definition), + (uri2, (4, 19, 4, 29), ReferenceKind.Reference) + ); + + text2 = Environment.NewLine + text2; + await server.SendDidChangeTextDocumentAsync(uri2, text2); + + references = await server.SendFindReferences(uri1, 5, 9); + references.Should().OnlyHaveReferences( + (uri1, (5, 4, 6, 12), ReferenceKind.Value), + (uri1, (5, 8, 5, 18), ReferenceKind.Definition), + (uri2, (5, 19, 5, 29), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task ListComprehensions() { + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task ListComprehensions(Server server) { // var entry = ProcessText(@" //x = [2,3,4] //y = [a for a in x] @@ -141,54 +133,51 @@ public async Task ListComprehensions() { // AssertUtil.ContainsExactly(entry.GetTypesFromName("z", 0), IntType); - string text = @" + var text = @" def f(abc): print abc [f(x) for x in [2,3,4]] "; - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - var uri = TestData.GetDefaultModuleUri(); - await server.SendDidOpenTextDocument(uri, text); - - var references = await server.SendFindReferences(uri, 4, 2); - references.Should().OnlyHaveReferences( - (uri, (1, 0, 2, 13), ReferenceKind.Value), - (uri, (1, 4, 1, 5), ReferenceKind.Definition), - (uri, (4, 1, 4, 2), ReferenceKind.Reference) - ); - } + var uri = TestData.GetDefaultModuleUri(); + await server.SendDidOpenTextDocument(uri, text); + + var references = await server.SendFindReferences(uri, 4, 2); + references.Should().OnlyHaveReferences( + (uri, (1, 0, 2, 13), ReferenceKind.Value), + (uri, (1, 4, 1, 5), ReferenceKind.Definition), + (uri, (4, 1, 4, 2), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task ExecReferences() { - string text = @" + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task ExecReferences(Server server) { + var text = @" a = {} b = """" exec b in a "; - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - var uri = TestData.GetDefaultModuleUri(); - await server.SendDidOpenTextDocument(uri, text); - - var referencesA = await server.SendFindReferences(uri, 1, 1); - var referencesB = await server.SendFindReferences(uri, 2, 1); - - referencesA.Should().OnlyHaveReferences( - (uri, (1, 0, 1, 1), ReferenceKind.Definition), - (uri, (3, 10, 3, 11), ReferenceKind.Reference) - ); - - referencesB.Should().OnlyHaveReferences( - (uri, (2, 0, 2, 1), ReferenceKind.Definition), - (uri, (3, 5, 3, 6), ReferenceKind.Reference) - ); - } + + var uri = TestData.GetDefaultModuleUri(); + await server.SendDidOpenTextDocument(uri, text); + + var referencesA = await server.SendFindReferences(uri, 1, 1); + var referencesB = await server.SendFindReferences(uri, 2, 1); + + referencesA.Should().OnlyHaveReferences( + (uri, (1, 0, 1, 1), ReferenceKind.Definition), + (uri, (3, 10, 3, 11), ReferenceKind.Reference) + ); + + referencesB.Should().OnlyHaveReferences( + (uri, (2, 0, 2, 1), ReferenceKind.Definition), + (uri, (3, 5, 3, 6), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task PrivateMemberReferences() { + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task PrivateMemberReferences(Server server) { var text = @" class C: def __x(self): @@ -201,34 +190,31 @@ def g(self): self._C__x() "; - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - var uri = TestData.GetDefaultModuleUri(); - await server.OpenDefaultDocumentAndGetAnalysisAsync(text); - var references = await server.SendFindReferences(uri, 6, 14); - - references.Should().OnlyHaveReferences( - (uri, (2, 4, 3, 12), ReferenceKind.Value), - (uri, (2, 8, 2, 11), ReferenceKind.Definition), - (uri, (6, 13, 6, 16), ReferenceKind.Reference), - (uri, (9, 13, 9, 18), ReferenceKind.Reference) - ); - } + var uri = TestData.GetDefaultModuleUri(); + await server.OpenDefaultDocumentAndGetAnalysisAsync(text); + var references = await server.SendFindReferences(uri, 6, 14); + + references.Should().OnlyHaveReferences( + (uri, (2, 4, 3, 12), ReferenceKind.Value), + (uri, (2, 8, 2, 11), ReferenceKind.Definition), + (uri, (6, 13, 6, 16), ReferenceKind.Reference), + (uri, (9, 13, 9, 18), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task GeneratorComprehensions() { - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - var text = @" + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task GeneratorComprehensions(Server server) { + var text = @" x = [2,3,4] y = (a for a in x) for z in y: print z "; - var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(text); - analysis.Should().HaveVariable("z").OfResolvedType(BuiltinTypeId.Int); + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(text); + analysis.Should().HaveVariable("z").OfResolvedType(BuiltinTypeId.Int); - text = @" + text = @" x = [2,3,4] y = (a for a in x) @@ -238,11 +224,11 @@ print z f(y) "; - analysis = await server.ChangeDefaultDocumentAndGetAnalysisAsync(text); - analysis.Should().HaveFunction("f") - .Which.Should().HaveVariable("z").OfResolvedType(BuiltinTypeId.Int); + analysis = await server.ChangeDefaultDocumentAndGetAnalysisAsync(text); + analysis.Should().HaveFunction("f") + .Which.Should().HaveVariable("z").OfResolvedType(BuiltinTypeId.Int); - text = @" + text = @" x = [True, False, None] def f(iterable): @@ -254,37 +240,35 @@ def f(iterable): y = f(i for i in x) "; - analysis = await server.ChangeDefaultDocumentAndGetAnalysisAsync(text); - analysis.Should().HaveVariable("y").OfTypes(BuiltinTypeId.Bool, BuiltinTypeId.NoneType); + analysis = await server.ChangeDefaultDocumentAndGetAnalysisAsync(text); + analysis.Should().HaveVariable("y").OfTypes(BuiltinTypeId.Bool, BuiltinTypeId.NoneType); - text = @" + text = @" def f(abc): print abc (f(x) for x in [2,3,4]) "; - analysis = await server.ChangeDefaultDocumentAndGetAnalysisAsync(text); - var uri = TestData.GetDefaultModuleUri(); - var references = await server.SendFindReferences(uri, 1, 5); - - analysis.Should().HaveFunction("f") - .Which.Should().HaveParameter("abc").OfType(BuiltinTypeId.Int); + analysis = await server.ChangeDefaultDocumentAndGetAnalysisAsync(text); + var uri = TestData.GetDefaultModuleUri(); + var references = await server.SendFindReferences(uri, 1, 5); - references.Should().OnlyHaveReferences( - (uri, (1, 0, 2, 13), ReferenceKind.Value), - (uri, (1, 4, 1, 5), ReferenceKind.Definition), - (uri, (4, 1, 4, 2), ReferenceKind.Reference) - ); + analysis.Should().HaveFunction("f") + .Which.Should().HaveParameter("abc").OfType(BuiltinTypeId.Int); - } + references.Should().OnlyHaveReferences( + (uri, (1, 0, 2, 13), ReferenceKind.Value), + (uri, (1, 4, 1, 5), ReferenceKind.Definition), + (uri, (4, 1, 4, 2), ReferenceKind.Reference) + ); } /// /// Verifies that a line in triple quoted string which ends with a \ (eating the newline) doesn't throw /// off our newline tracking. /// - [TestMethod, Priority(0)] - public async Task ReferencesTripleQuotedStringWithBackslash() { + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task ReferencesTripleQuotedStringWithBackslash(Server server) { // instance variables var text = @" '''this is a triple quoted string\ @@ -300,31 +284,28 @@ print self.abc "; - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - var uri = TestData.GetDefaultModuleUri(); - await server.SendDidOpenTextDocument(uri, text); - - var referencesAbc = await server.SendFindReferences(uri, 8, 15); - var referencesFob = await server.SendFindReferences(uri, 8, 21); - - referencesAbc.Should().OnlyHaveReferences( - (uri, (8, 13, 8, 16), ReferenceKind.Definition), - (uri, (9, 17, 9, 20), ReferenceKind.Reference), - (uri, (10, 19, 10, 22), ReferenceKind.Reference) - ); - referencesFob.Should().OnlyHaveReferences( - (uri, (7, 23, 7, 26), ReferenceKind.Definition), - (uri, (8, 19, 8, 22), ReferenceKind.Reference) - ); - } + var uri = TestData.GetDefaultModuleUri(); + await server.SendDidOpenTextDocument(uri, text); + + var referencesAbc = await server.SendFindReferences(uri, 8, 15); + var referencesFob = await server.SendFindReferences(uri, 8, 21); + + referencesAbc.Should().OnlyHaveReferences( + (uri, (8, 13, 8, 16), ReferenceKind.Definition), + (uri, (9, 17, 9, 20), ReferenceKind.Reference), + (uri, (10, 19, 10, 22), ReferenceKind.Reference) + ); + referencesFob.Should().OnlyHaveReferences( + (uri, (7, 23, 7, 26), ReferenceKind.Definition), + (uri, (8, 19, 8, 22), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task InstanceVariables() { - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - var uri = TestData.GetDefaultModuleUri(); + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task InstanceVariables(Server server) { + var uri = TestData.GetDefaultModuleUri(); - var text = @" + var text = @" # add ref w/o type info class C(object): def __init__(self, fob): @@ -334,22 +315,22 @@ print self.abc "; - await server.SendDidOpenTextDocument(uri, text); + await server.SendDidOpenTextDocument(uri, text); - var referencesAbc = await server.SendFindReferences(uri, 4, 15); - var referencesFob = await server.SendFindReferences(uri, 4, 21); + var referencesAbc = await server.SendFindReferences(uri, 4, 15); + var referencesFob = await server.SendFindReferences(uri, 4, 21); - referencesAbc.Should().OnlyHaveReferences( - (uri, (4, 13, 4, 16), ReferenceKind.Definition), - (uri, (5, 17, 5, 20), ReferenceKind.Reference), - (uri, (6, 19, 6, 22), ReferenceKind.Reference) - ); - referencesFob.Should().OnlyHaveReferences( - (uri, (3, 23, 3, 26), ReferenceKind.Definition), - (uri, (4, 19, 4, 22), ReferenceKind.Reference) - ); + referencesAbc.Should().OnlyHaveReferences( + (uri, (4, 13, 4, 16), ReferenceKind.Definition), + (uri, (5, 17, 5, 20), ReferenceKind.Reference), + (uri, (6, 19, 6, 22), ReferenceKind.Reference) + ); + referencesFob.Should().OnlyHaveReferences( + (uri, (3, 23, 3, 26), ReferenceKind.Definition), + (uri, (4, 19, 4, 22), ReferenceKind.Reference) + ); - text = @" + text = @" # add ref w/ type info class D(object): def __init__(self, fob): @@ -358,192 +339,176 @@ del self.abc print self.abc D(42)"; - await server.SendDidChangeTextDocumentAsync(uri, text); - - referencesAbc = await server.SendFindReferences(uri, 4, 15); - referencesFob = await server.SendFindReferences(uri, 4, 21); - var referencesD = await server.SendFindReferences(uri, 8, 1); - - referencesAbc.Should().OnlyHaveReferences( - (uri, (4, 13, 4, 16), ReferenceKind.Definition), - (uri, (5, 17, 5, 20), ReferenceKind.Reference), - (uri, (6, 19, 6, 22), ReferenceKind.Reference) - ); - referencesFob.Should().OnlyHaveReferences( - (uri, (3, 23, 3, 26), ReferenceKind.Definition), - (uri, (4, 19, 4, 22), ReferenceKind.Reference) - ); - referencesD.Should().OnlyHaveReferences( - (uri, (2, 6, 2, 7), ReferenceKind.Definition), - (uri, (2, 0, 6, 22), ReferenceKind.Value), - (uri, (8, 0, 8, 1), ReferenceKind.Reference) - ); - } + await server.SendDidChangeTextDocumentAsync(uri, text); + + referencesAbc = await server.SendFindReferences(uri, 4, 15); + referencesFob = await server.SendFindReferences(uri, 4, 21); + var referencesD = await server.SendFindReferences(uri, 8, 1); + + referencesAbc.Should().OnlyHaveReferences( + (uri, (4, 13, 4, 16), ReferenceKind.Definition), + (uri, (5, 17, 5, 20), ReferenceKind.Reference), + (uri, (6, 19, 6, 22), ReferenceKind.Reference) + ); + referencesFob.Should().OnlyHaveReferences( + (uri, (3, 23, 3, 26), ReferenceKind.Definition), + (uri, (4, 19, 4, 22), ReferenceKind.Reference) + ); + referencesD.Should().OnlyHaveReferences( + (uri, (2, 6, 2, 7), ReferenceKind.Definition), + (uri, (2, 0, 6, 22), ReferenceKind.Value), + (uri, (8, 0, 8, 1), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task FunctionDefinitions() { - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - var uri = TestData.GetDefaultModuleUri(); - var text = @" + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task FunctionDefinitions(Server server) { + var uri = TestData.GetDefaultModuleUri(); + var text = @" def f(): pass x = f()"; - await server.SendDidOpenTextDocument(uri, text); + await server.SendDidOpenTextDocument(uri, text); - var referencesF = await server.SendFindReferences(uri, 3, 5); - referencesF.Should().OnlyHaveReferences( - (uri, (1, 0, 1, 13), ReferenceKind.Value), - (uri, (1, 4, 1, 5), ReferenceKind.Definition), - (uri, (3, 4, 3, 5), ReferenceKind.Reference) - ); + var referencesF = await server.SendFindReferences(uri, 3, 5); + referencesF.Should().OnlyHaveReferences( + (uri, (1, 0, 1, 13), ReferenceKind.Value), + (uri, (1, 4, 1, 5), ReferenceKind.Definition), + (uri, (3, 4, 3, 5), ReferenceKind.Reference) + ); - text = @" + text = @" def f(): pass x = f"; - await server.SendDidChangeTextDocumentAsync(uri, text); - referencesF = await server.SendFindReferences(uri, 3, 5); - - referencesF.Should().OnlyHaveReferences( - (uri, (1, 0, 1, 13), ReferenceKind.Value), - (uri, (1, 4, 1, 5), ReferenceKind.Definition), - (uri, (3, 4, 3, 5), ReferenceKind.Reference) - ); - } + await server.SendDidChangeTextDocumentAsync(uri, text); + referencesF = await server.SendFindReferences(uri, 3, 5); + + referencesF.Should().OnlyHaveReferences( + (uri, (1, 0, 1, 13), ReferenceKind.Value), + (uri, (1, 4, 1, 5), ReferenceKind.Definition), + (uri, (3, 4, 3, 5), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task ClassVariables() { - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - var uri = TestData.GetDefaultModuleUri(); - var text = @" + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task ClassVariables(Server server) { + var uri = TestData.GetDefaultModuleUri(); + var text = @" class D(object): abc = 42 print abc del abc "; - await server.SendDidOpenTextDocument(uri, text); - var references = await server.SendFindReferences(uri, 3, 5); - - references.Should().OnlyHaveReferences( - (uri, (3, 4, 3, 7), ReferenceKind.Definition), - (uri, (4, 10, 4, 13), ReferenceKind.Reference), - (uri, (5, 8, 5, 11), ReferenceKind.Reference) - ); - } + await server.SendDidOpenTextDocument(uri, text); + var references = await server.SendFindReferences(uri, 3, 5); + + references.Should().OnlyHaveReferences( + (uri, (3, 4, 3, 7), ReferenceKind.Definition), + (uri, (4, 10, 4, 13), ReferenceKind.Reference), + (uri, (5, 8, 5, 11), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task ClassDefinition() { - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - var uri = TestData.GetDefaultModuleUri(); - var text = @" + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task ClassDefinition(Server server) { + var uri = TestData.GetDefaultModuleUri(); + var text = @" class D(object): pass a = D "; - await server.SendDidOpenTextDocument(uri, text); - var references = await server.SendFindReferences(uri, 3, 5); - - references.Should().OnlyHaveReferences( - (uri, (1, 0, 1, 21), ReferenceKind.Value), - (uri, (1, 6, 1, 7), ReferenceKind.Definition), - (uri, (3, 4, 3, 5), ReferenceKind.Reference) - ); - } + await server.SendDidOpenTextDocument(uri, text); + var references = await server.SendFindReferences(uri, 3, 5); + + references.Should().OnlyHaveReferences( + (uri, (1, 0, 1, 21), ReferenceKind.Value), + (uri, (1, 6, 1, 7), ReferenceKind.Definition), + (uri, (3, 4, 3, 5), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task MethodDefinition() { - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - var uri = TestData.GetDefaultModuleUri(); - var text = @" + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task MethodDefinition(Server server) { + var uri = TestData.GetDefaultModuleUri(); + var text = @" class D(object): def f(self): pass a = D().f() "; - await server.SendDidOpenTextDocument(uri, text); - - var references = await server.SendFindReferences(uri, 4, 9); - references.Should().OnlyHaveReferences( - (uri, (2, 4, 2, 21), ReferenceKind.Value), - (uri, (2, 8, 2, 9), ReferenceKind.Definition), - (uri, (4, 8, 4, 9), ReferenceKind.Reference) - ); - } + await server.SendDidOpenTextDocument(uri, text); + + var references = await server.SendFindReferences(uri, 4, 9); + references.Should().OnlyHaveReferences( + (uri, (2, 4, 2, 21), ReferenceKind.Value), + (uri, (2, 8, 2, 9), ReferenceKind.Definition), + (uri, (4, 8, 4, 9), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task Globals() { - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - var uri = TestData.GetDefaultModuleUri(); - var text = @" + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task Globals(Server server) { + var uri = TestData.GetDefaultModuleUri(); + var text = @" abc = 42 print abc del abc "; - await server.SendDidOpenTextDocument(uri, text); - - var references = await server.SendFindReferences(uri, 2, 7); - references.Should().OnlyHaveReferences( - (uri, (1, 0, 1, 3), ReferenceKind.Definition), - (uri, (2, 6, 2, 9), ReferenceKind.Reference), - (uri, (3, 4, 3, 7), ReferenceKind.Reference) - ); - } + await server.SendDidOpenTextDocument(uri, text); + + var references = await server.SendFindReferences(uri, 2, 7); + references.Should().OnlyHaveReferences( + (uri, (1, 0, 1, 3), ReferenceKind.Definition), + (uri, (2, 6, 2, 9), ReferenceKind.Reference), + (uri, (3, 4, 3, 7), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task Parameters() { - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - var uri = TestData.GetDefaultModuleUri(); - var text = @" + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task Parameters(Server server) { + var uri = TestData.GetDefaultModuleUri(); + var text = @" def f(abc): print abc abc = 42 del abc "; - await server.SendDidOpenTextDocument(uri, text); - - var references = await server.SendFindReferences(uri, 2, 11); - references.Should().OnlyHaveReferences( - (uri, (1, 6, 1, 9), ReferenceKind.Definition), - (uri, (3, 4, 3, 7), ReferenceKind.Reference), - (uri, (2, 10, 2, 13), ReferenceKind.Reference), - (uri, (4, 8, 4, 11), ReferenceKind.Reference) - ); - } + await server.SendDidOpenTextDocument(uri, text); + + var references = await server.SendFindReferences(uri, 2, 11); + references.Should().OnlyHaveReferences( + (uri, (1, 6, 1, 9), ReferenceKind.Definition), + (uri, (3, 4, 3, 7), ReferenceKind.Reference), + (uri, (2, 10, 2, 13), ReferenceKind.Reference), + (uri, (4, 8, 4, 11), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task NamedArguments() { - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - var uri = TestData.GetDefaultModuleUri(); - var text = @" + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task NamedArguments(Server server) { + var uri = TestData.GetDefaultModuleUri(); + var text = @" def f(abc): print abc f(abc = 123) "; - await server.SendDidOpenTextDocument(uri, text); - - var references = await server.SendFindReferences(uri, 2, 11); - references.Should().OnlyHaveReferences( - (uri, (1, 6, 1, 9), ReferenceKind.Definition), - (uri, (2, 10, 2, 13), ReferenceKind.Reference), - (uri, (4, 2, 4, 5), ReferenceKind.Reference) - ); - } + await server.SendDidOpenTextDocument(uri, text); + + var references = await server.SendFindReferences(uri, 2, 11); + references.Should().OnlyHaveReferences( + (uri, (1, 6, 1, 9), ReferenceKind.Definition), + (uri, (2, 10, 2, 13), ReferenceKind.Reference), + (uri, (4, 2, 4, 5), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task GrammarTest_Statements() { - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - var uri = TestData.GetDefaultModuleUri(); - var text = @" + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task GrammarTest_Statements(Server server) { + var uri = TestData.GetDefaultModuleUri(); + var text = @" def f(abc): try: pass except abc: pass @@ -592,66 +557,64 @@ print abc else: abc "; - await server.SendDidOpenTextDocument(uri, text); + await server.SendDidOpenTextDocument(uri, text); - var references = await server.SendFindReferences(uri, 3, 12); + var references = await server.SendFindReferences(uri, 3, 12); - // External module 'abc', URI varies depending on install - var externalUri = references[1].uri; - externalUri.LocalPath.Should().EndWith("abc.py"); + // External module 'abc', URI varies depending on install + var externalUri = references[1].uri; + externalUri.LocalPath.Should().EndWith("abc.py"); - references.Should().OnlyHaveReferences( - (uri, (1, 6, 1, 9), ReferenceKind.Definition), - (externalUri, (0, 0, 0, 0), ReferenceKind.Definition), + references.Should().OnlyHaveReferences( + (uri, (1, 6, 1, 9), ReferenceKind.Definition), + (externalUri, (0, 0, 0, 0), ReferenceKind.Definition), - (uri, (3, 11, 3, 14), ReferenceKind.Reference), - (uri, (6, 22, 6, 25), ReferenceKind.Reference), + (uri, (3, 11, 3, 14), ReferenceKind.Reference), + (uri, (6, 22, 6, 25), ReferenceKind.Reference), - (uri, (8, 4, 8, 7), ReferenceKind.Reference), - (uri, (9, 4, 9, 7), ReferenceKind.Reference), - (uri, (10, 4, 10, 7), ReferenceKind.Reference), - (uri, (11, 4, 11, 7), ReferenceKind.Reference), + (uri, (8, 4, 8, 7), ReferenceKind.Reference), + (uri, (9, 4, 9, 7), ReferenceKind.Reference), + (uri, (10, 4, 10, 7), ReferenceKind.Reference), + (uri, (11, 4, 11, 7), ReferenceKind.Reference), - (uri, (13, 12, 13, 15), ReferenceKind.Reference), + (uri, (13, 12, 13, 15), ReferenceKind.Reference), - (uri, (15, 13, 15, 16), ReferenceKind.Reference), + (uri, (15, 13, 15, 16), ReferenceKind.Reference), - (uri, (17, 11, 17, 14), ReferenceKind.Reference), - (uri, (18, 20, 18, 23), ReferenceKind.Reference), - (uri, (19, 27, 19, 30), ReferenceKind.Reference), + (uri, (17, 11, 17, 14), ReferenceKind.Reference), + (uri, (18, 20, 18, 23), ReferenceKind.Reference), + (uri, (19, 27, 19, 30), ReferenceKind.Reference), - (uri, (21, 7, 21, 10), ReferenceKind.Reference), - (uri, (22, 9, 22, 12), ReferenceKind.Reference), - (uri, (23, 10, 23, 13), ReferenceKind.Reference), + (uri, (21, 7, 21, 10), ReferenceKind.Reference), + (uri, (22, 9, 22, 12), ReferenceKind.Reference), + (uri, (23, 10, 23, 13), ReferenceKind.Reference), - (uri, (25, 9, 25, 12), ReferenceKind.Reference), - (uri, (26, 15, 26, 18), ReferenceKind.Reference), + (uri, (25, 9, 25, 12), ReferenceKind.Reference), + (uri, (26, 15, 26, 18), ReferenceKind.Reference), - (uri, (28, 10, 28, 13), ReferenceKind.Reference), - (uri, (29, 11, 29, 14), ReferenceKind.Reference), - (uri, (29, 16, 29, 19), ReferenceKind.Reference), + (uri, (28, 10, 28, 13), ReferenceKind.Reference), + (uri, (29, 11, 29, 14), ReferenceKind.Reference), + (uri, (29, 16, 29, 19), ReferenceKind.Reference), - (uri, (31, 10, 31, 13), ReferenceKind.Reference), - (uri, (32, 15, 32, 18), ReferenceKind.Reference), - (uri, (32, 20, 32, 23), ReferenceKind.Reference), - (uri, (32, 10, 32, 13), ReferenceKind.Reference), + (uri, (31, 10, 31, 13), ReferenceKind.Reference), + (uri, (32, 15, 32, 18), ReferenceKind.Reference), + (uri, (32, 20, 32, 23), ReferenceKind.Reference), + (uri, (32, 10, 32, 13), ReferenceKind.Reference), - (uri, (34, 10, 34, 13), ReferenceKind.Reference), - (uri, (35, 8, 35, 11), ReferenceKind.Reference), - (uri, (37, 8, 37, 11), ReferenceKind.Reference), + (uri, (34, 10, 34, 13), ReferenceKind.Reference), + (uri, (35, 8, 35, 11), ReferenceKind.Reference), + (uri, (37, 8, 37, 11), ReferenceKind.Reference), - (uri, (42, 14, 42, 17), ReferenceKind.Reference), + (uri, (42, 14, 42, 17), ReferenceKind.Reference), - (uri, (47, 8, 47, 11), ReferenceKind.Reference) - ); - } + (uri, (47, 8, 47, 11), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task GrammarTest_Expressions() { + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task GrammarTest_Expressions(Server server) { var uri = TestData.GetDefaultModuleUri(); - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - var text = @" + var text = @" def f(abc): x = abc + 2 x = 2 + abc @@ -683,60 +646,58 @@ not abc lambda : abc "; - await server.SendDidOpenTextDocument(uri, text); - - var references = await server.SendFindReferences(uri, 3, 12); - references.Should().OnlyHaveReferences( - (uri, (1, 6, 1, 9), ReferenceKind.Definition), - - (uri, (2, 8, 2, 11), ReferenceKind.Reference), - (uri, (3, 12, 3, 15), ReferenceKind.Reference), - - (uri, (4, 10, 4, 13), ReferenceKind.Reference), - (uri, (5, 8, 5, 11), ReferenceKind.Reference), - (uri, (6, 8, 6, 11), ReferenceKind.Reference), - (uri, (8, 6, 8, 9), ReferenceKind.Reference), - - (uri, (10, 11, 10, 14), ReferenceKind.Reference), - (uri, (10, 4, 10, 7), ReferenceKind.Reference), - (uri, (10, 20, 10, 23), ReferenceKind.Reference), - - (uri, (12, 5, 12, 8), ReferenceKind.Reference), - (uri, (12, 9, 12, 12), ReferenceKind.Reference), - (uri, (13, 5, 13, 8), ReferenceKind.Reference), - (uri, (13, 10, 13, 13), ReferenceKind.Reference), - (uri, (14, 5, 14, 8), ReferenceKind.Reference), - (uri, (14, 10, 14, 13), ReferenceKind.Reference), - (uri, (15, 5, 15, 8), ReferenceKind.Reference), - - (uri, (17, 10, 17, 13), ReferenceKind.Reference), - (uri, (18, 16, 18, 19), ReferenceKind.Reference), - (uri, (21, 4, 21, 7), ReferenceKind.Reference), - - (uri, (21, 11, 21, 14), ReferenceKind.Reference), - (uri, (22, 4, 22, 7), ReferenceKind.Reference), - (uri, (22, 12, 22, 15), ReferenceKind.Reference), - (uri, (24, 5, 24, 8), ReferenceKind.Reference), - - (uri, (25, 6, 25, 9), ReferenceKind.Reference), - (uri, (25, 10, 25, 13), ReferenceKind.Reference), - (uri, (25, 14, 25, 17), ReferenceKind.Reference), - (uri, (27, 4, 27, 7), ReferenceKind.Reference), - - (uri, (27, 11, 27, 14), ReferenceKind.Reference), - (uri, (28, 8, 28, 11), ReferenceKind.Reference), - (uri, (19, 16, 19, 19), ReferenceKind.Reference), - - (uri, (30, 13, 30, 16), ReferenceKind.Reference) - ); - } + await server.SendDidOpenTextDocument(uri, text); + + var references = await server.SendFindReferences(uri, 3, 12); + references.Should().OnlyHaveReferences( + (uri, (1, 6, 1, 9), ReferenceKind.Definition), + + (uri, (2, 8, 2, 11), ReferenceKind.Reference), + (uri, (3, 12, 3, 15), ReferenceKind.Reference), + + (uri, (4, 10, 4, 13), ReferenceKind.Reference), + (uri, (5, 8, 5, 11), ReferenceKind.Reference), + (uri, (6, 8, 6, 11), ReferenceKind.Reference), + (uri, (8, 6, 8, 9), ReferenceKind.Reference), + + (uri, (10, 11, 10, 14), ReferenceKind.Reference), + (uri, (10, 4, 10, 7), ReferenceKind.Reference), + (uri, (10, 20, 10, 23), ReferenceKind.Reference), + + (uri, (12, 5, 12, 8), ReferenceKind.Reference), + (uri, (12, 9, 12, 12), ReferenceKind.Reference), + (uri, (13, 5, 13, 8), ReferenceKind.Reference), + (uri, (13, 10, 13, 13), ReferenceKind.Reference), + (uri, (14, 5, 14, 8), ReferenceKind.Reference), + (uri, (14, 10, 14, 13), ReferenceKind.Reference), + (uri, (15, 5, 15, 8), ReferenceKind.Reference), + + (uri, (17, 10, 17, 13), ReferenceKind.Reference), + (uri, (18, 16, 18, 19), ReferenceKind.Reference), + (uri, (21, 4, 21, 7), ReferenceKind.Reference), + + (uri, (21, 11, 21, 14), ReferenceKind.Reference), + (uri, (22, 4, 22, 7), ReferenceKind.Reference), + (uri, (22, 12, 22, 15), ReferenceKind.Reference), + (uri, (24, 5, 24, 8), ReferenceKind.Reference), + + (uri, (25, 6, 25, 9), ReferenceKind.Reference), + (uri, (25, 10, 25, 13), ReferenceKind.Reference), + (uri, (25, 14, 25, 17), ReferenceKind.Reference), + (uri, (27, 4, 27, 7), ReferenceKind.Reference), + + (uri, (27, 11, 27, 14), ReferenceKind.Reference), + (uri, (28, 8, 28, 11), ReferenceKind.Reference), + (uri, (19, 16, 19, 19), ReferenceKind.Reference), + + (uri, (30, 13, 30, 16), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task Parameters_NestedFunction() { + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task Parameters_NestedFunction(Server server) { var uri = TestData.GetDefaultModuleUri(); - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - var text = @" + var text = @" def f(a): def g(): print(a) @@ -744,50 +705,47 @@ assert isinstance(a, int) a = 200 print(a) "; - await server.SendDidOpenTextDocument(uri, text); - - var expected = new (Uri documentUri, (int, int, int, int), ReferenceKind?)[] { - (uri, (1, 6, 1, 7), ReferenceKind.Definition), - (uri, (3, 14, 3, 15), ReferenceKind.Reference), - (uri, (4, 26, 4, 27), ReferenceKind.Reference), - (uri, (5, 8, 5, 9), ReferenceKind.Reference), - (uri, (6, 14, 6, 15), ReferenceKind.Reference) - }; - var references = await server.SendFindReferences(uri, 3, 15); - references.Should().OnlyHaveReferences(expected); - - references = await server.SendFindReferences(uri, 5, 9); - references.Should().OnlyHaveReferences(expected); - - references = await server.SendFindReferences(uri, 6, 15); - references.Should().OnlyHaveReferences(expected); - } + await server.SendDidOpenTextDocument(uri, text); + + var expected = new (Uri documentUri, (int, int, int, int), ReferenceKind?)[] { + (uri, (1, 6, 1, 7), ReferenceKind.Definition), + (uri, (3, 14, 3, 15), ReferenceKind.Reference), + (uri, (4, 26, 4, 27), ReferenceKind.Reference), + (uri, (5, 8, 5, 9), ReferenceKind.Reference), + (uri, (6, 14, 6, 15), ReferenceKind.Reference) + }; + var references = await server.SendFindReferences(uri, 3, 15); + references.Should().OnlyHaveReferences(expected); + + references = await server.SendFindReferences(uri, 5, 9); + references.Should().OnlyHaveReferences(expected); + + references = await server.SendFindReferences(uri, 6, 15); + references.Should().OnlyHaveReferences(expected); } - [TestMethod, Priority(0)] - public async Task ClassLocalVariable() { + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task ClassLocalVariable(Server server) { var uri = TestData.GetDefaultModuleUri(); - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - var text = @" + var text = @" a = 1 class B: a = 2 "; - await server.SendDidOpenTextDocument(uri, text); - var references = await server.SendFindReferences(uri, 3, 5); - references.Should().OnlyHaveReferences( - (uri, (3, 4, 3, 5), ReferenceKind.Definition) - ); - references = await server.SendFindReferences(uri, 1, 1); - references.Should().OnlyHaveReferences( - (uri, (1, 0, 1, 1), ReferenceKind.Definition) - ); - } + await server.SendDidOpenTextDocument(uri, text); + var references = await server.SendFindReferences(uri, 3, 5); + references.Should().OnlyHaveReferences( + (uri, (3, 4, 3, 5), ReferenceKind.Definition) + ); + references = await server.SendFindReferences(uri, 1, 1); + references.Should().OnlyHaveReferences( + (uri, (1, 0, 1, 1), ReferenceKind.Definition) + ); } - [TestMethod, Priority(0)] - public async Task ListDictArgReferences() { + [ServerTestMethod, Priority(0)] + public async Task ListDictArgReferences(Server server) { var text = @" def f(*a, **k): x = a[1] @@ -798,29 +756,27 @@ def f(*a, **k): k = 2 "; var uri = TestData.GetDefaultModuleUri(); - using (var server = await CreateServerAsync()) { - await server.SendDidOpenTextDocument(uri, text); - - var referencesA = await server.SendFindReferences(uri, 2, 8); - var referencesK = await server.SendFindReferences(uri, 3, 8); - referencesA.Should().OnlyHaveReferences( - (uri, (1, 7, 1, 8), ReferenceKind.Definition), - (uri, (2, 8, 2, 9), ReferenceKind.Reference) - ); - referencesK.Should().OnlyHaveReferences( - (uri, (1, 12, 1, 13), ReferenceKind.Definition), - (uri, (3, 8, 3, 9), ReferenceKind.Reference) - ); - - referencesA = await server.SendFindReferences(uri, 6, 1); - referencesK = await server.SendFindReferences(uri, 7, 1); - referencesA.Should().OnlyHaveReference(uri, (6, 0, 6, 1), ReferenceKind.Definition); - referencesK.Should().OnlyHaveReference(uri, (7, 0, 7, 1), ReferenceKind.Definition); - } + await server.SendDidOpenTextDocument(uri, text); + + var referencesA = await server.SendFindReferences(uri, 2, 8); + var referencesK = await server.SendFindReferences(uri, 3, 8); + referencesA.Should().OnlyHaveReferences( + (uri, (1, 7, 1, 8), ReferenceKind.Definition), + (uri, (2, 8, 2, 9), ReferenceKind.Reference) + ); + referencesK.Should().OnlyHaveReferences( + (uri, (1, 12, 1, 13), ReferenceKind.Definition), + (uri, (3, 8, 3, 9), ReferenceKind.Reference) + ); + + referencesA = await server.SendFindReferences(uri, 6, 1); + referencesK = await server.SendFindReferences(uri, 7, 1); + referencesA.Should().OnlyHaveReference(uri, (6, 0, 6, 1), ReferenceKind.Definition); + referencesK.Should().OnlyHaveReference(uri, (7, 0, 7, 1), ReferenceKind.Definition); } - [TestMethod, Priority(0)] - public async Task KeywordArgReferences() { + [ServerTestMethod, Priority(0)] + public async Task KeywordArgReferences(Server server) { var text = @" def f(a): pass @@ -828,19 +784,17 @@ def f(a): f(a=1) "; var uri = TestData.GetDefaultModuleUri(); - using (var server = await CreateServerAsync()) { - await server.SendDidOpenTextDocument(uri, text); - - var references = await server.SendFindReferences(uri, 4, 3); - references.Should().OnlyHaveReferences( - (uri, (1, 6, 1, 7), ReferenceKind.Definition), - (uri, (4, 2, 4, 3), ReferenceKind.Reference) - ); - } + await server.SendDidOpenTextDocument(uri, text); + + var references = await server.SendFindReferences(uri, 4, 3); + references.Should().OnlyHaveReferences( + (uri, (1, 6, 1, 7), ReferenceKind.Definition), + (uri, (4, 2, 4, 3), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task ReferencesCrossModule() { + [ServerTestMethod(LatestAvailable3X = true), Priority(0)] + public async Task ReferencesCrossModule(Server server) { var fobText = @" from oar import abc @@ -849,24 +803,22 @@ from oar import abc var oarText = "class abc(object): pass"; var fobUri = TestData.GetTestSpecificUri("fob.py"); var oarUri = TestData.GetTestSpecificUri("oar.py"); - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { - using (server.AnalysisQueue.Pause()) { - await server.SendDidOpenTextDocument(fobUri, fobText); - await server.SendDidOpenTextDocument(oarUri, oarText); - } - - var references = await server.SendFindReferences(oarUri, 0, 7); - references.Should().OnlyHaveReferences( - (oarUri, (0, 0, 0, 23), ReferenceKind.Value), - (oarUri, (0, 6, 0, 9), ReferenceKind.Definition), - (fobUri, (1, 16, 1, 19), ReferenceKind.Reference), - (fobUri, (3, 0, 3, 3), ReferenceKind.Reference) - ); + using (server.AnalysisQueue.Pause()) { + await server.SendDidOpenTextDocument(fobUri, fobText); + await server.SendDidOpenTextDocument(oarUri, oarText); } + + var references = await server.SendFindReferences(oarUri, 0, 7); + references.Should().OnlyHaveReferences( + (oarUri, (0, 0, 0, 23), ReferenceKind.Value), + (oarUri, (0, 6, 0, 9), ReferenceKind.Definition), + (fobUri, (1, 16, 1, 19), ReferenceKind.Reference), + (fobUri, (3, 0, 3, 3), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task SuperclassMemberReferencesCrossModule() { + [ServerTestMethod(LatestAvailable3X = true), Priority(0)] + public async Task SuperclassMemberReferencesCrossModule(Server server) { // https://github.com/Microsoft/PTVS/issues/2271 var fobText = @" @@ -883,22 +835,20 @@ def __init__(self): var fobUri = TestData.GetTestSpecificUri("fob.py"); var oarUri = TestData.GetTestSpecificUri("oar.py"); - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { - using (server.AnalysisQueue.Pause()) { - await server.SendDidOpenTextDocument(fobUri, fobText); - await server.SendDidOpenTextDocument(oarUri, oarText); - } - - var references = await server.SendFindReferences(oarUri, 2, 14); - references.Should().OnlyHaveReferences( - (oarUri, (2, 13, 2, 14), ReferenceKind.Definition), - (fobUri, (5, 13, 5, 14), ReferenceKind.Reference) - ); + using (server.AnalysisQueue.Pause()) { + await server.SendDidOpenTextDocument(fobUri, fobText); + await server.SendDidOpenTextDocument(oarUri, oarText); } + + var references = await server.SendFindReferences(oarUri, 2, 14); + references.Should().OnlyHaveReferences( + (oarUri, (2, 13, 2, 14), ReferenceKind.Definition), + (fobUri, (5, 13, 5, 14), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task ReferencesCrossMultiModule() { + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task ReferencesCrossMultiModule(Server server) { var fobText = @" from oarbaz import abc @@ -913,47 +863,46 @@ from oarbaz import abc var oarUri = TestData.GetTestSpecificUri("oar.py"); var bazUri = TestData.GetTestSpecificUri("baz.py"); var oarBazUri = TestData.GetTestSpecificUri("oarbaz.py"); - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - using (server.AnalysisQueue.Pause()) { - await server.SendDidOpenTextDocument(fobUri, fobText); - await server.SendDidOpenTextDocument(oarUri, oarText); - await server.SendDidOpenTextDocument(bazUri, bazText); - await server.SendDidOpenTextDocument(oarBazUri, oarBazText); - } - var referencesAbc1 = await server.SendFindReferences(oarUri, 0, 7); - var referencesAbc2 = await server.SendFindReferences(bazUri, 4, 7); - var referencesAbc = await server.SendFindReferences(fobUri, 3, 1); - - referencesAbc1.Should().OnlyHaveReferences( - (oarUri, (0, 0, 0, 24), ReferenceKind.Value), - (oarUri, (0, 6, 0, 10), ReferenceKind.Definition), - (oarBazUri, (0, 16, 0, 20), ReferenceKind.Reference), - (oarBazUri, (0, 24, 0, 27), ReferenceKind.Reference) - ); - - referencesAbc2.Should().OnlyHaveReferences( - (bazUri, (4, 0, 4, 24), ReferenceKind.Value), - (bazUri, (4, 6, 4, 10), ReferenceKind.Definition), - (oarBazUri, (1, 16, 1, 20), ReferenceKind.Reference), - (oarBazUri, (1, 24, 1, 27), ReferenceKind.Reference) - ); - - referencesAbc.Should().OnlyHaveReferences( - (oarUri, (0, 0, 0, 24), ReferenceKind.Value), - (bazUri, (4, 0, 4, 24), ReferenceKind.Value), - (oarBazUri, (0, 16, 0, 20), ReferenceKind.Reference), - (oarBazUri, (0, 24, 0, 27), ReferenceKind.Reference), - (oarBazUri, (1, 16, 1, 20), ReferenceKind.Reference), - (oarBazUri, (1, 24, 1, 27), ReferenceKind.Reference), - (fobUri, (1, 19, 1, 22), ReferenceKind.Reference), - (fobUri, (3, 0, 3, 3), ReferenceKind.Reference) - ); + using (server.AnalysisQueue.Pause()) { + await server.SendDidOpenTextDocument(fobUri, fobText); + await server.SendDidOpenTextDocument(oarUri, oarText); + await server.SendDidOpenTextDocument(bazUri, bazText); + await server.SendDidOpenTextDocument(oarBazUri, oarBazText); } + + var referencesAbc1 = await server.SendFindReferences(oarUri, 0, 7); + var referencesAbc2 = await server.SendFindReferences(bazUri, 4, 7); + var referencesAbc = await server.SendFindReferences(fobUri, 3, 1); + + referencesAbc1.Should().OnlyHaveReferences( + (oarUri, (0, 0, 0, 24), ReferenceKind.Value), + (oarUri, (0, 6, 0, 10), ReferenceKind.Definition), + (oarBazUri, (0, 16, 0, 20), ReferenceKind.Reference), + (oarBazUri, (0, 24, 0, 27), ReferenceKind.Reference) + ); + + referencesAbc2.Should().OnlyHaveReferences( + (bazUri, (4, 0, 4, 24), ReferenceKind.Value), + (bazUri, (4, 6, 4, 10), ReferenceKind.Definition), + (oarBazUri, (1, 16, 1, 20), ReferenceKind.Reference), + (oarBazUri, (1, 24, 1, 27), ReferenceKind.Reference) + ); + + referencesAbc.Should().OnlyHaveReferences( + (oarUri, (0, 0, 0, 24), ReferenceKind.Value), + (bazUri, (4, 0, 4, 24), ReferenceKind.Value), + (oarBazUri, (0, 16, 0, 20), ReferenceKind.Reference), + (oarBazUri, (0, 24, 0, 27), ReferenceKind.Reference), + (oarBazUri, (1, 16, 1, 20), ReferenceKind.Reference), + (oarBazUri, (1, 24, 1, 27), ReferenceKind.Reference), + (fobUri, (1, 19, 1, 22), ReferenceKind.Reference), + (fobUri, (3, 0, 3, 3), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task ImportStarReferences() { + [ServerTestMethod, Priority(0)] + public async Task ImportStarReferences(Server server) { var fobText = @" CONSTANT = 1 class Class: pass @@ -967,37 +916,35 @@ class Class: pass f = fn()"; var fobUri = TestData.GetTestSpecificUri("fob.py"); var oarUri = TestData.GetTestSpecificUri("oar.py"); - using (var server = await CreateServerAsync()) { - using (server.AnalysisQueue.Pause()) { - await server.SendDidOpenTextDocument(fobUri, fobText); - await server.SendDidOpenTextDocument(oarUri, oarText); - } - - var referencesConstant = await server.SendFindReferences(oarUri, 4, 5); - var referencesClass = await server.SendFindReferences(oarUri, 5, 5); - var referencesfn = await server.SendFindReferences(oarUri, 6, 5); - - referencesConstant.Should().OnlyHaveReferences( - (fobUri, (1, 0, 1, 8), ReferenceKind.Definition), - (oarUri, (4, 4, 4, 12), ReferenceKind.Reference) - ); - - referencesClass.Should().OnlyHaveReferences( - (fobUri, (2, 0, 2, 17), ReferenceKind.Value), - (fobUri, (2, 6, 2, 11), ReferenceKind.Definition), - (oarUri, (5, 4, 5, 9), ReferenceKind.Reference) - ); - - referencesfn.Should().OnlyHaveReferences( - (fobUri, (3, 0, 3, 14), ReferenceKind.Value), - (fobUri, (3, 4, 3, 6), ReferenceKind.Definition), - (oarUri, (6, 4, 6, 6), ReferenceKind.Reference) - ); + using (server.AnalysisQueue.Pause()) { + await server.SendDidOpenTextDocument(fobUri, fobText); + await server.SendDidOpenTextDocument(oarUri, oarText); } + + var referencesConstant = await server.SendFindReferences(oarUri, 4, 5); + var referencesClass = await server.SendFindReferences(oarUri, 5, 5); + var referencesfn = await server.SendFindReferences(oarUri, 6, 5); + + referencesConstant.Should().OnlyHaveReferences( + (fobUri, (1, 0, 1, 8), ReferenceKind.Definition), + (oarUri, (4, 4, 4, 12), ReferenceKind.Reference) + ); + + referencesClass.Should().OnlyHaveReferences( + (fobUri, (2, 0, 2, 17), ReferenceKind.Value), + (fobUri, (2, 6, 2, 11), ReferenceKind.Definition), + (oarUri, (5, 4, 5, 9), ReferenceKind.Reference) + ); + + referencesfn.Should().OnlyHaveReferences( + (fobUri, (3, 0, 3, 14), ReferenceKind.Value), + (fobUri, (3, 4, 3, 6), ReferenceKind.Definition), + (oarUri, (6, 4, 6, 6), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task ImportAsReferences() { + [ServerTestMethod, Priority(0)] + public async Task ImportAsReferences(Server server) { var fobText = @" CONSTANT = 1 class Class: pass @@ -1012,69 +959,67 @@ class Class: pass var fobUri = TestData.GetTestSpecificUri("fob.py"); var oarUri = TestData.GetTestSpecificUri("oar.py"); - using (var server = await CreateServerAsync()) { - using (server.AnalysisQueue.Pause()) { - await server.SendDidOpenTextDocument(fobUri, fobText); - await server.SendDidOpenTextDocument(oarUri, oarText); - } - - var referencesConstant = await server.SendFindReferences(fobUri, 1, 1); - var referencesClass = await server.SendFindReferences(fobUri, 2, 7); - var referencesFn = await server.SendFindReferences(fobUri, 3, 5); - var referencesC0 = await server.SendFindReferences(oarUri, 4, 5); - var referencesC1 = await server.SendFindReferences(oarUri, 5, 5); - var referencesF = await server.SendFindReferences(oarUri, 6, 5); - - referencesConstant.Should().OnlyHaveReferences( - (fobUri, (1, 0, 1, 8), ReferenceKind.Definition), - (oarUri, (0, 16, 0, 24), ReferenceKind.Reference), - (oarUri, (0, 28, 0, 30), ReferenceKind.Reference), - (oarUri, (4, 4, 4, 6), ReferenceKind.Reference) - ); - - referencesClass.Should().OnlyHaveReferences( - (fobUri, (2, 0, 2, 17), ReferenceKind.Value), - (fobUri, (2, 6, 2, 11), ReferenceKind.Definition), - (oarUri, (0, 32, 0, 37), ReferenceKind.Reference), - (oarUri, (0, 41, 0, 43), ReferenceKind.Reference), - (oarUri, (5, 4, 5, 6), ReferenceKind.Reference) - ); - - referencesFn.Should().OnlyHaveReferences( - (fobUri, (3, 0, 3, 14), ReferenceKind.Value), - (fobUri, (3, 4, 3, 6), ReferenceKind.Definition), - (oarUri, (0, 45, 0, 47), ReferenceKind.Reference), - (oarUri, (0, 51, 0, 52), ReferenceKind.Reference), - (oarUri, (6, 4, 6, 5), ReferenceKind.Reference) - ); - - referencesC0.Should().OnlyHaveReferences( - (fobUri, (1, 0, 1, 8), ReferenceKind.Definition), - (oarUri, (0, 16, 0, 24), ReferenceKind.Reference), - (oarUri, (0, 28, 0, 30), ReferenceKind.Reference), - (oarUri, (4, 4, 4, 6), ReferenceKind.Reference) - ); - - referencesC1.Should().OnlyHaveReferences( - (fobUri, (2, 0, 2, 17), ReferenceKind.Value), - (fobUri, (2, 6, 2, 11), ReferenceKind.Definition), - (oarUri, (0, 32, 0, 37), ReferenceKind.Reference), - (oarUri, (0, 41, 0, 43), ReferenceKind.Reference), - (oarUri, (5, 4, 5, 6), ReferenceKind.Reference) - ); - - referencesF.Should().OnlyHaveReferences( - (fobUri, (3, 0, 3, 14), ReferenceKind.Value), - (fobUri, (3, 4, 3, 6), ReferenceKind.Definition), - (oarUri, (0, 45, 0, 47), ReferenceKind.Reference), - (oarUri, (0, 51, 0, 52), ReferenceKind.Reference), - (oarUri, (6, 4, 6, 5), ReferenceKind.Reference) - ); + using (server.AnalysisQueue.Pause()) { + await server.SendDidOpenTextDocument(fobUri, fobText); + await server.SendDidOpenTextDocument(oarUri, oarText); } + + var referencesConstant = await server.SendFindReferences(fobUri, 1, 1); + var referencesClass = await server.SendFindReferences(fobUri, 2, 7); + var referencesFn = await server.SendFindReferences(fobUri, 3, 5); + var referencesC0 = await server.SendFindReferences(oarUri, 4, 5); + var referencesC1 = await server.SendFindReferences(oarUri, 5, 5); + var referencesF = await server.SendFindReferences(oarUri, 6, 5); + + referencesConstant.Should().OnlyHaveReferences( + (fobUri, (1, 0, 1, 8), ReferenceKind.Definition), + (oarUri, (0, 16, 0, 24), ReferenceKind.Reference), + (oarUri, (0, 28, 0, 30), ReferenceKind.Reference), + (oarUri, (4, 4, 4, 6), ReferenceKind.Reference) + ); + + referencesClass.Should().OnlyHaveReferences( + (fobUri, (2, 0, 2, 17), ReferenceKind.Value), + (fobUri, (2, 6, 2, 11), ReferenceKind.Definition), + (oarUri, (0, 32, 0, 37), ReferenceKind.Reference), + (oarUri, (0, 41, 0, 43), ReferenceKind.Reference), + (oarUri, (5, 4, 5, 6), ReferenceKind.Reference) + ); + + referencesFn.Should().OnlyHaveReferences( + (fobUri, (3, 0, 3, 14), ReferenceKind.Value), + (fobUri, (3, 4, 3, 6), ReferenceKind.Definition), + (oarUri, (0, 45, 0, 47), ReferenceKind.Reference), + (oarUri, (0, 51, 0, 52), ReferenceKind.Reference), + (oarUri, (6, 4, 6, 5), ReferenceKind.Reference) + ); + + referencesC0.Should().OnlyHaveReferences( + (fobUri, (1, 0, 1, 8), ReferenceKind.Definition), + (oarUri, (0, 16, 0, 24), ReferenceKind.Reference), + (oarUri, (0, 28, 0, 30), ReferenceKind.Reference), + (oarUri, (4, 4, 4, 6), ReferenceKind.Reference) + ); + + referencesC1.Should().OnlyHaveReferences( + (fobUri, (2, 0, 2, 17), ReferenceKind.Value), + (fobUri, (2, 6, 2, 11), ReferenceKind.Definition), + (oarUri, (0, 32, 0, 37), ReferenceKind.Reference), + (oarUri, (0, 41, 0, 43), ReferenceKind.Reference), + (oarUri, (5, 4, 5, 6), ReferenceKind.Reference) + ); + + referencesF.Should().OnlyHaveReferences( + (fobUri, (3, 0, 3, 14), ReferenceKind.Value), + (fobUri, (3, 4, 3, 6), ReferenceKind.Definition), + (oarUri, (0, 45, 0, 47), ReferenceKind.Reference), + (oarUri, (0, 51, 0, 52), ReferenceKind.Reference), + (oarUri, (6, 4, 6, 5), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task ReferencesGeneratorsV3() { + [ServerTestMethod(LatestAvailable3X = true), Priority(0)] + public async Task ReferencesGeneratorsV3(Server server) { var text = @" [f for f in x] [x for x in f] @@ -1082,35 +1027,33 @@ public async Task ReferencesGeneratorsV3() { (y for y in g) "; var uri = TestData.GetDefaultModuleUri(); - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { - await server.SendDidOpenTextDocument(uri, text); - - var referencesF = await server.SendFindReferences(uri, 1, 8); - var referencesX = await server.SendFindReferences(uri, 2, 8); - var referencesG = await server.SendFindReferences(uri, 3, 8); - var referencesY = await server.SendFindReferences(uri, 4, 8); - - referencesF.Should().OnlyHaveReferences( - (uri, (1, 7, 1, 8), ReferenceKind.Definition), - (uri, (1, 1, 1, 2), ReferenceKind.Reference) - ); - referencesX.Should().OnlyHaveReferences( - (uri, (2, 7, 2, 8), ReferenceKind.Definition), - (uri, (2, 1, 2, 2), ReferenceKind.Reference) - ); - referencesG.Should().OnlyHaveReferences( - (uri, (3, 7, 3, 8), ReferenceKind.Definition), - (uri, (3, 1, 3, 2), ReferenceKind.Reference) - ); - referencesY.Should().OnlyHaveReferences( - (uri, (4, 7, 4, 8), ReferenceKind.Definition), - (uri, (4, 1, 4, 2), ReferenceKind.Reference) - ); - } + await server.SendDidOpenTextDocument(uri, text); + + var referencesF = await server.SendFindReferences(uri, 1, 8); + var referencesX = await server.SendFindReferences(uri, 2, 8); + var referencesG = await server.SendFindReferences(uri, 3, 8); + var referencesY = await server.SendFindReferences(uri, 4, 8); + + referencesF.Should().OnlyHaveReferences( + (uri, (1, 7, 1, 8), ReferenceKind.Definition), + (uri, (1, 1, 1, 2), ReferenceKind.Reference) + ); + referencesX.Should().OnlyHaveReferences( + (uri, (2, 7, 2, 8), ReferenceKind.Definition), + (uri, (2, 1, 2, 2), ReferenceKind.Reference) + ); + referencesG.Should().OnlyHaveReferences( + (uri, (3, 7, 3, 8), ReferenceKind.Definition), + (uri, (3, 1, 3, 2), ReferenceKind.Reference) + ); + referencesY.Should().OnlyHaveReferences( + (uri, (4, 7, 4, 8), ReferenceKind.Definition), + (uri, (4, 1, 4, 2), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task ReferencesGeneratorsV2() { + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task ReferencesGeneratorsV2(Server server) { var text = @" [f for f in x] [x for x in f] @@ -1118,36 +1061,34 @@ public async Task ReferencesGeneratorsV2() { (y for y in g) "; var uri = TestData.GetDefaultModuleUri(); - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { - await server.SendDidOpenTextDocument(uri, text); - - var referencesF = await server.SendFindReferences(uri, 1, 8); - var referencesX = await server.SendFindReferences(uri, 2, 8); - var referencesG = await server.SendFindReferences(uri, 3, 8); - var referencesY = await server.SendFindReferences(uri, 4, 8); - - referencesF.Should().OnlyHaveReferences( - (uri, (1, 7, 1, 8), ReferenceKind.Definition), - (uri, (1, 1, 1, 2), ReferenceKind.Reference), - (uri, (2, 12, 2, 13), ReferenceKind.Reference) - ); - referencesX.Should().OnlyHaveReferences( - (uri, (2, 7, 2, 8), ReferenceKind.Definition), - (uri, (2, 1, 2, 2), ReferenceKind.Reference) - ); - referencesG.Should().OnlyHaveReferences( - (uri, (3, 7, 3, 8), ReferenceKind.Definition), - (uri, (3, 1, 3, 2), ReferenceKind.Reference) - ); - referencesY.Should().OnlyHaveReferences( - (uri, (4, 7, 4, 8), ReferenceKind.Definition), - (uri, (4, 1, 4, 2), ReferenceKind.Reference) - ); - } + await server.SendDidOpenTextDocument(uri, text); + + var referencesF = await server.SendFindReferences(uri, 1, 8); + var referencesX = await server.SendFindReferences(uri, 2, 8); + var referencesG = await server.SendFindReferences(uri, 3, 8); + var referencesY = await server.SendFindReferences(uri, 4, 8); + + referencesF.Should().OnlyHaveReferences( + (uri, (1, 7, 1, 8), ReferenceKind.Definition), + (uri, (1, 1, 1, 2), ReferenceKind.Reference), + (uri, (2, 12, 2, 13), ReferenceKind.Reference) + ); + referencesX.Should().OnlyHaveReferences( + (uri, (2, 7, 2, 8), ReferenceKind.Definition), + (uri, (2, 1, 2, 2), ReferenceKind.Reference) + ); + referencesG.Should().OnlyHaveReferences( + (uri, (3, 7, 3, 8), ReferenceKind.Definition), + (uri, (3, 1, 3, 2), ReferenceKind.Reference) + ); + referencesY.Should().OnlyHaveReferences( + (uri, (4, 7, 4, 8), ReferenceKind.Definition), + (uri, (4, 1, 4, 2), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] - public async Task FunctionScoping() { + [ServerTestMethod, Priority(0)] + public async Task FunctionScoping(Server server) { var code = @"x = 100 def f(x = x): @@ -1156,34 +1097,32 @@ def f(x = x): f('abc') "; - using (var server = await CreateServerAsync()) { - var uri = TestData.GetDefaultModuleUri(); - await server.OpenDefaultDocumentAndGetAnalysisAsync(code); - - var referencesx1 = await server.SendFindReferences(uri, 2, 6); - var referencesx2 = await server.SendFindReferences(uri, 2, 10); - var referencesx3 = await server.SendFindReferences(uri, 3, 4); - - referencesx1.Should().OnlyHaveReferences( - (uri, (2, 6, 2, 7), ReferenceKind.Definition), - (uri, (3, 4, 3, 5), ReferenceKind.Reference) - ); - - referencesx2.Should().OnlyHaveReferences( - (uri, (0, 0, 0, 1), ReferenceKind.Definition), - (uri, (2, 10, 2, 11), ReferenceKind.Reference) - ); - - referencesx3.Should().OnlyHaveReferences( - (uri, (2, 6, 2, 7), ReferenceKind.Definition), - (uri, (3, 4, 3, 5), ReferenceKind.Reference) - ); - } + var uri = TestData.GetDefaultModuleUri(); + await server.OpenDefaultDocumentAndGetAnalysisAsync(code); + + var references1 = await server.SendFindReferences(uri, 2, 6); + var references2 = await server.SendFindReferences(uri, 2, 10); + var references3 = await server.SendFindReferences(uri, 3, 4); + + references1.Should().OnlyHaveReferences( + (uri, (2, 6, 2, 7), ReferenceKind.Definition), + (uri, (3, 4, 3, 5), ReferenceKind.Reference) + ); + + references2.Should().OnlyHaveReferences( + (uri, (0, 0, 0, 1), ReferenceKind.Definition), + (uri, (2, 10, 2, 11), ReferenceKind.Reference) + ); + + references3.Should().OnlyHaveReferences( + (uri, (2, 6, 2, 7), ReferenceKind.Definition), + (uri, (3, 4, 3, 5), ReferenceKind.Reference) + ); } - [TestMethod, Priority(0)] + [ServerTestMethod, Priority(0)] [Ignore("https://github.com/Microsoft/python-language-server/issues/117")] - public async Task MoveClass() { + public async Task MoveClass(Server server) { var fobSrc = ""; var oarSrc = @" @@ -1195,49 +1134,47 @@ class C(object): class C(object): pass "; - using (var server = await CreateServerAsync()) { - var uriFob = await server.OpenDocumentAndGetUriAsync("fob.py", fobSrc); - var uriOar = await server.OpenDocumentAndGetUriAsync("oar.py", oarSrc); - var uriBaz = await server.OpenDocumentAndGetUriAsync("baz.py", bazSrc); - await server.SendDidChangeTextDocumentAsync(uriFob, "from oar import C"); - - var references = await server.SendFindReferences(uriFob, 0, 17); - references.Should().OnlyHaveReferences( - (uriFob, (0, 16, 0, 17), ReferenceKind.Reference), - (uriOar, (1, 0, 2, 8), ReferenceKind.Value), - (uriOar, (1, 6, 1, 7), ReferenceKind.Definition), - (uriOar, (0, 0, 0, 0), ReferenceKind.Definition) - ); - - await server.WaitForCompleteAnalysisAsync(CancellationToken.None); - var analysis = await server.GetAnalysisAsync(uriFob); - analysis.Should().HaveVariable("C").WithDescription("C"); - - // delete the class.. - await server.SendDidChangeTextDocumentAsync(uriOar, ""); - - await server.WaitForCompleteAnalysisAsync(CancellationToken.None); - analysis = await server.GetAnalysisAsync(uriFob); - analysis.Should().HaveVariable("C").WithNoTypes(); - - // Change location of the class - await server.SendDidChangeTextDocumentAsync(uriFob, "from baz import C"); - - references = await server.SendFindReferences(uriFob, 0, 17); - references.Should().OnlyHaveReferences( - (uriFob, (0, 16, 0, 17), ReferenceKind.Reference), - (uriBaz, (1, 0, 2, 8), ReferenceKind.Value), - (uriBaz, (1, 6, 1, 7), ReferenceKind.Definition), - (uriBaz, (0, 0, 0, 0), ReferenceKind.Definition) - ); - - analysis = await server.GetAnalysisAsync(uriFob); - analysis.Should().HaveVariable("C").WithDescription("C"); - } + var uriFob = await server.OpenDocumentAndGetUriAsync("fob.py", fobSrc); + var uriOar = await server.OpenDocumentAndGetUriAsync("oar.py", oarSrc); + var uriBaz = await server.OpenDocumentAndGetUriAsync("baz.py", bazSrc); + await server.SendDidChangeTextDocumentAsync(uriFob, "from oar import C"); + + var references = await server.SendFindReferences(uriFob, 0, 17); + references.Should().OnlyHaveReferences( + (uriFob, (0, 16, 0, 17), ReferenceKind.Reference), + (uriOar, (1, 0, 2, 8), ReferenceKind.Value), + (uriOar, (1, 6, 1, 7), ReferenceKind.Definition), + (uriOar, (0, 0, 0, 0), ReferenceKind.Definition) + ); + + await server.WaitForCompleteAnalysisAsync(CancellationToken.None); + var analysis = await server.GetAnalysisAsync(uriFob); + analysis.Should().HaveVariable("C").WithDescription("C"); + + // delete the class.. + await server.SendDidChangeTextDocumentAsync(uriOar, ""); + + await server.WaitForCompleteAnalysisAsync(CancellationToken.None); + analysis = await server.GetAnalysisAsync(uriFob); + analysis.Should().HaveVariable("C").WithNoTypes(); + + // Change location of the class + await server.SendDidChangeTextDocumentAsync(uriFob, "from baz import C"); + + references = await server.SendFindReferences(uriFob, 0, 17); + references.Should().OnlyHaveReferences( + (uriFob, (0, 16, 0, 17), ReferenceKind.Reference), + (uriBaz, (1, 0, 2, 8), ReferenceKind.Value), + (uriBaz, (1, 6, 1, 7), ReferenceKind.Definition), + (uriBaz, (0, 0, 0, 0), ReferenceKind.Definition) + ); + + analysis = await server.GetAnalysisAsync(uriFob); + analysis.Should().HaveVariable("C").WithDescription("C"); } - [TestMethod, Priority(0)] - public async Task DecoratorReferences() { + [ServerTestMethod, Priority(0)] + public async Task DecoratorReferences(Server server) { var text = @" def d1(f): return f @@ -1254,36 +1191,34 @@ class cls_d1(object): pass @d2() class cls_d2(object): pass "; - using (var server = await CreateServerAsync()) { - var uri = await server.OpenDefaultDocumentAndGetUriAsync(text); - var referencesD1 = await server.SendFindReferences(uri, 1, 5); - var referencesD2 = await server.SendFindReferences(uri, 3, 7); - var analysis = await server.GetAnalysisAsync(uri); - - referencesD1.Should().OnlyHaveReferences( - (uri, (1, 0, 2, 12), ReferenceKind.Value), - (uri, (1, 4, 1, 6), ReferenceKind.Definition), - (uri, (6, 1, 6, 3), ReferenceKind.Reference), - (uri, (11, 1, 11, 3), ReferenceKind.Reference)); - - referencesD2.Should().OnlyHaveReferences( - (uri, (3, 0, 4, 35), ReferenceKind.Value), - (uri, (3, 6, 3, 8), ReferenceKind.Definition), - (uri, (8, 1, 8, 3), ReferenceKind.Reference), - (uri, (13, 1, 13, 3), ReferenceKind.Reference)); - - analysis.Should().HaveFunction("d1").WithParameter("f").OfTypes(BuiltinTypeId.Function, BuiltinTypeId.Type) - .And.HaveClass("d2").WithFunction("__call__").WithParameter("f").OfTypes(BuiltinTypeId.Function, BuiltinTypeId.Type) - .And.HaveFunction("func_d1") - .And.HaveFunction("func_d2") - .And.HaveClass("cls_d1") - .And.HaveClass("cls_d2"); - } + var uri = await server.OpenDefaultDocumentAndGetUriAsync(text); + var referencesD1 = await server.SendFindReferences(uri, 1, 5); + var referencesD2 = await server.SendFindReferences(uri, 3, 7); + var analysis = await server.GetAnalysisAsync(uri); + + referencesD1.Should().OnlyHaveReferences( + (uri, (1, 0, 2, 12), ReferenceKind.Value), + (uri, (1, 4, 1, 6), ReferenceKind.Definition), + (uri, (6, 1, 6, 3), ReferenceKind.Reference), + (uri, (11, 1, 11, 3), ReferenceKind.Reference)); + + referencesD2.Should().OnlyHaveReferences( + (uri, (3, 0, 4, 35), ReferenceKind.Value), + (uri, (3, 6, 3, 8), ReferenceKind.Definition), + (uri, (8, 1, 8, 3), ReferenceKind.Reference), + (uri, (13, 1, 13, 3), ReferenceKind.Reference)); + + analysis.Should().HaveFunction("d1").WithParameter("f").OfTypes(BuiltinTypeId.Function, BuiltinTypeId.Type) + .And.HaveClass("d2").WithFunction("__call__").WithParameter("f").OfTypes(BuiltinTypeId.Function, BuiltinTypeId.Type) + .And.HaveFunction("func_d1") + .And.HaveFunction("func_d2") + .And.HaveClass("cls_d1") + .And.HaveClass("cls_d2"); } - [TestMethod, Priority(0)] + [ServerTestMethod(LatestAvailable3X = true), Priority(0)] [Ignore("https://github.com/Microsoft/python-language-server/issues/215")] - public async Task Nonlocal() { + public async Task Nonlocal(Server server) { var text = @" def f(): x = None @@ -1297,36 +1232,35 @@ def g(): a, b = f() "; - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { - var uri = await server.OpenDefaultDocumentAndGetUriAsync(text); - var referencesX = await server.SendFindReferences(uri, 2, 5); - var referencesY = await server.SendFindReferences(uri, 3, 5); - var analysis = await server.GetAnalysisAsync(uri); + var uri = await server.OpenDefaultDocumentAndGetUriAsync(text); + var referencesX = await server.SendFindReferences(uri, 2, 5); + var referencesY = await server.SendFindReferences(uri, 3, 5); + var analysis = await server.GetAnalysisAsync(uri); - analysis.Should().HaveVariable("a").OfTypes(BuiltinTypeId.NoneType, BuiltinTypeId.Int) - .And.HaveVariable("b").OfTypes(BuiltinTypeId.NoneType, BuiltinTypeId.Int) + analysis.Should().HaveVariable("a").OfTypes(BuiltinTypeId.NoneType, BuiltinTypeId.Int) + .And.HaveVariable("b").OfTypes(BuiltinTypeId.NoneType, BuiltinTypeId.Int) - .And.HaveFunction("f") - .Which.Should().HaveVariable("x").OfTypes(BuiltinTypeId.NoneType, BuiltinTypeId.Int) - .And.HaveVariable("y").OfTypes(BuiltinTypeId.NoneType, BuiltinTypeId.Int) + .And.HaveFunction("f") + .Which.Should().HaveVariable("x").OfTypes(BuiltinTypeId.NoneType, BuiltinTypeId.Int) + .And.HaveVariable("y").OfTypes(BuiltinTypeId.NoneType, BuiltinTypeId.Int) - .And.HaveFunction("g") - .Which.Should().HaveVariable("x").OfTypes(BuiltinTypeId.NoneType, BuiltinTypeId.Int) - .And.HaveVariable("y").OfTypes(BuiltinTypeId.NoneType, BuiltinTypeId.Int); + .And.HaveFunction("g") + .Which.Should().HaveVariable("x").OfTypes(BuiltinTypeId.NoneType, BuiltinTypeId.Int) + .And.HaveVariable("y").OfTypes(BuiltinTypeId.NoneType, BuiltinTypeId.Int); - referencesX.Should().OnlyHaveReferences( - (uri, (2, 4, 2, 5), ReferenceKind.Definition), - (uri, (5, 17, 5, 18), ReferenceKind.Reference), - (uri, (6, 8, 6, 9), ReferenceKind.Definition), - (uri, (8, 11, 8, 12), ReferenceKind.Reference)); + referencesX.Should().OnlyHaveReferences( + (uri, (2, 4, 2, 5), ReferenceKind.Definition), + (uri, (5, 17, 5, 18), ReferenceKind.Reference), + (uri, (6, 8, 6, 9), ReferenceKind.Definition), + (uri, (8, 11, 8, 12), ReferenceKind.Reference)); - referencesY.Should().OnlyHaveReferences( - (uri, (3, 4, 3, 5), ReferenceKind.Definition), - (uri, (5, 20, 5, 21), ReferenceKind.Reference), - (uri, (7, 8, 7, 9), ReferenceKind.Definition), - (uri, (8, 14, 8, 15), ReferenceKind.Reference)); + referencesY.Should().OnlyHaveReferences( + (uri, (3, 4, 3, 5), ReferenceKind.Definition), + (uri, (5, 20, 5, 21), ReferenceKind.Reference), + (uri, (7, 8, 7, 9), ReferenceKind.Definition), + (uri, (8, 14, 8, 15), ReferenceKind.Reference)); - text = @" + text = @" def f(x): def g(): nonlocal x @@ -1336,14 +1270,13 @@ nonlocal x a = f(None) "; - await server.SendDidChangeTextDocumentAsync(uri, text); - analysis = await server.GetAnalysisAsync(uri); - analysis.Should().HaveVariable("a").OfTypes(BuiltinTypeId.NoneType, BuiltinTypeId.Int); - } + await server.SendDidChangeTextDocumentAsync(uri, text); + analysis = await server.GetAnalysisAsync(uri); + analysis.Should().HaveVariable("a").OfTypes(BuiltinTypeId.NoneType, BuiltinTypeId.Int); } - [TestMethod, Priority(0)] - public async Task IsInstance() { + [ServerTestMethod, Priority(0)] + public async Task IsInstance(Server server) { var text = @" def fob(): oar = get_b() @@ -1354,29 +1287,27 @@ raise IndexError return oar"; - using (var server = await CreateServerAsync()) { - var uri = await server.OpenDefaultDocumentAndGetUriAsync(text); - var references1 = await server.SendFindReferences(uri, 2, 5); - var references2 = await server.SendFindReferences(uri, 3, 23); - var references3 = await server.SendFindReferences(uri, 5, 8); - var references4 = await server.SendFindReferences(uri, 8, 12); - - var expectations = new (Uri, (int, int, int, int), ReferenceKind?)[] { - (uri, (2, 4, 2, 7), ReferenceKind.Definition), - (uri, (3, 22, 3, 25), ReferenceKind.Reference), - (uri, (5, 7, 5, 10), ReferenceKind.Reference), - (uri, (8, 11, 8, 14), ReferenceKind.Reference) - }; - - references1.Should().OnlyHaveReferences(expectations); - references2.Should().OnlyHaveReferences(expectations); - references3.Should().OnlyHaveReferences(expectations); - references4.Should().OnlyHaveReferences(expectations); - } + var uri = await server.OpenDefaultDocumentAndGetUriAsync(text); + var references1 = await server.SendFindReferences(uri, 2, 5); + var references2 = await server.SendFindReferences(uri, 3, 23); + var references3 = await server.SendFindReferences(uri, 5, 8); + var references4 = await server.SendFindReferences(uri, 8, 12); + + var expectations = new (Uri, (int, int, int, int), ReferenceKind?)[] { + (uri, (2, 4, 2, 7), ReferenceKind.Definition), + (uri, (3, 22, 3, 25), ReferenceKind.Reference), + (uri, (5, 7, 5, 10), ReferenceKind.Reference), + (uri, (8, 11, 8, 14), ReferenceKind.Reference) + }; + + references1.Should().OnlyHaveReferences(expectations); + references2.Should().OnlyHaveReferences(expectations); + references3.Should().OnlyHaveReferences(expectations); + references4.Should().OnlyHaveReferences(expectations); } - [TestMethod, Priority(0)] - public async Task FunctoolsDecorator() { + [ServerTestMethod, Priority(0)] + public async Task FunctoolsDecorator(Server server) { var text = @"from functools import wraps def d(f): @@ -1391,19 +1322,18 @@ def g(p): n1 = g(1)"; - using (var server = await CreateServerAsync()) { - var uri = await server.OpenDefaultDocumentAndGetUriAsync(text); - var referencesD = await server.SendFindReferences(uri, 2, 5); - var referencesG = await server.SendFindReferences(uri, 9, 5); + var uri = await server.OpenDefaultDocumentAndGetUriAsync(text); + var referencesD = await server.SendFindReferences(uri, 2, 5); + var referencesG = await server.SendFindReferences(uri, 9, 5); - referencesD.Should().OnlyHaveReferences( - (uri, (2, 0, 6, 18), ReferenceKind.Value), - (uri, (2, 4, 2, 5), ReferenceKind.Definition), - (uri, (8, 1, 8, 2), ReferenceKind.Reference)); - referencesG.Should().OnlyHaveReferences( - (uri, (3, 4, 5, 26), ReferenceKind.Value), - (uri, (9, 4, 9, 5), ReferenceKind.Definition), - (uri, (12, 5, 12, 6), ReferenceKind.Reference)); + referencesD.Should().OnlyHaveReferences( + (uri, (2, 0, 6, 18), ReferenceKind.Value), + (uri, (2, 4, 2, 5), ReferenceKind.Definition), + (uri, (8, 1, 8, 2), ReferenceKind.Reference)); + referencesG.Should().OnlyHaveReferences( + (uri, (3, 4, 5, 26), ReferenceKind.Value), + (uri, (9, 4, 9, 5), ReferenceKind.Definition), + (uri, (12, 5, 12, 6), ReferenceKind.Reference)); // Decorators that don't use @wraps will expose the wrapper function // as a value. @@ -1418,26 +1348,25 @@ def g(p): n1 = g(1)"; - await server.SendDidChangeTextDocumentAsync(uri, text); - referencesD = await server.SendFindReferences(uri, 0, 5); - referencesG = await server.SendFindReferences(uri, 6, 5); - - referencesD.Should().OnlyHaveReferences( - (uri, (0, 0, 3, 18), ReferenceKind.Value), - (uri, (0, 4, 0, 5), ReferenceKind.Definition), - (uri, (5, 1, 5, 2), ReferenceKind.Reference)); - referencesG.Should().OnlyHaveReferences( - (uri, (1, 4, 2, 26), ReferenceKind.Value), - (uri, (6, 4, 6, 5), ReferenceKind.Definition), - (uri, (9, 5, 9, 6), ReferenceKind.Reference)); - } + await server.SendDidChangeTextDocumentAsync(uri, text); + referencesD = await server.SendFindReferences(uri, 0, 5); + referencesG = await server.SendFindReferences(uri, 6, 5); + + referencesD.Should().OnlyHaveReferences( + (uri, (0, 0, 3, 18), ReferenceKind.Value), + (uri, (0, 4, 0, 5), ReferenceKind.Definition), + (uri, (5, 1, 5, 2), ReferenceKind.Reference)); + referencesG.Should().OnlyHaveReferences( + (uri, (1, 4, 2, 26), ReferenceKind.Value), + (uri, (6, 4, 6, 5), ReferenceKind.Definition), + (uri, (9, 5, 9, 6), ReferenceKind.Reference)); } /// /// Variable is referred to in the base class, defined in the derived class, we should know the type information. /// - [TestMethod, Priority(0)] - public async Task BaseReferencedDerivedDefined() { + [ServerTestMethod, Priority(0)] + public async Task BaseReferencedDerivedDefined(Server server) { var text = @" class Base(object): def f(self): @@ -1452,18 +1381,16 @@ def __init__(self): derived = Derived() "; - using (var server = await CreateServerAsync()) { - var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(text); - analysis.Should() - .HaveVariable("derived") - .WithValue() - .WithMemberOfType("map", PythonMemberType.Instance); - } + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(text); + analysis.Should() + .HaveVariable("derived") + .WithValue() + .WithMemberOfType("map", PythonMemberType.Instance); } - [TestMethod, Priority(0)] + [ServerTestMethod, Priority(0)] [Ignore("https://github.com/Microsoft/python-language-server/issues/218")] - public async Task SubclassFindAllRefs() { + public async Task SubclassFindAllRefs(Server server) { var text = @" class Base(object): def __init__(self): @@ -1478,30 +1405,27 @@ def fob(self): 'x' "; - using (var server = await CreateServerAsync()) { - var uri = await server.OpenDefaultDocumentAndGetUriAsync(text); - var references1 = await server.SendFindReferences(uri, 3, 14); - var references2 = await server.SendFindReferences(uri, 5, 9); - var references3 = await server.SendFindReferences(uri, 10, 9); - - var expectedReferences = new (Uri, (int, int, int, int), ReferenceKind?)[] { - (uri, (3, 13, 3, 16), ReferenceKind.Reference), - (uri, (5, 4, 6, 12), ReferenceKind.Value), - (uri, (5, 8, 5, 11), ReferenceKind.Definition), - (uri, (10, 4, 11, 11), ReferenceKind.Value), - (uri, (10, 8, 10, 11), ReferenceKind.Definition) - }; - - references1.Should().OnlyHaveReferences(expectedReferences); - references2.Should().OnlyHaveReferences(expectedReferences); - references3.Should().OnlyHaveReferences(expectedReferences); - } + var uri = await server.OpenDefaultDocumentAndGetUriAsync(text); + var references1 = await server.SendFindReferences(uri, 3, 14); + var references2 = await server.SendFindReferences(uri, 5, 9); + var references3 = await server.SendFindReferences(uri, 10, 9); + + var expectedReferences = new (Uri, (int, int, int, int), ReferenceKind?)[] { + (uri, (3, 13, 3, 16), ReferenceKind.Reference), + (uri, (5, 4, 6, 12), ReferenceKind.Value), + (uri, (5, 8, 5, 11), ReferenceKind.Definition), + (uri, (10, 4, 11, 11), ReferenceKind.Value), + (uri, (10, 8, 10, 11), ReferenceKind.Definition) + }; + + references1.Should().OnlyHaveReferences(expectedReferences); + references2.Should().OnlyHaveReferences(expectedReferences); + references3.Should().OnlyHaveReferences(expectedReferences); } - [TestMethod, Priority(0)] - public async Task FindReferences() { - using (var s = await CreateServerAsync()) { - var mod1 = await TestData.CreateTestSpecificFileAsync("mod1.py", @" + [ServerTestMethod, Priority(0)] + public async Task FindReferences(Server server) { + var mod1 = await TestData.CreateTestSpecificFileAsync("mod1.py", @" def f(a): a.real b = 1 @@ -1512,96 +1436,95 @@ class C: c=C() f(a=c) real = None"); - Uri mod2; - using (s.AnalysisQueue.Pause()) { + Uri mod2; + using (server.AnalysisQueue.Pause()) { - await s.LoadFileAsync(mod1); + await server.LoadFileAsync(mod1); - // Add 10 blank lines to ensure the line numbers do not collide - // We only check line numbers below, and by design we only get one - // reference per location, so we disambiguate by ensuring mod2's - // line numbers are larger than mod1's - mod2 = await s.OpenDocumentAndGetUriAsync("mod2.py", @"import mod1 + // Add 10 blank lines to ensure the line numbers do not collide + // We only check line numbers below, and by design we only get one + // reference per location, so we disambiguate by ensuring mod2's + // line numbers are larger than mod1's + mod2 = await server.OpenDocumentAndGetUriAsync("mod2.py", @"import mod1 " + $"{Environment.NewLine}{Environment.NewLine}{Environment.NewLine}{Environment.NewLine}{Environment.NewLine}{Environment.NewLine}{Environment.NewLine}{Environment.NewLine}{Environment.NewLine}{Environment.NewLine}" + @" class D: real = None a = 1 b = a mod1.f(a=D)"); - } - // f - var expected = new[] { - "Definition;(1, 4) - (1, 5)", - "Value;(1, 0) - (2, 10)", - "Reference;(4, 0) - (4, 1)", - "Reference;(9, 0) - (9, 1)", - "Reference;(16, 5) - (16, 6)" - }; - var unexpected = new[] { - "Definition;(7, 4) - (7, 5)", - }; - - await AssertReferences(s, mod1, SourceLocation.MinValue, expected, unexpected, "f"); - await AssertReferences(s, mod1, new SourceLocation(2, 5), expected, unexpected); - await AssertReferences(s, mod1, new SourceLocation(5, 2), expected, unexpected); - await AssertReferences(s, mod1, new SourceLocation(10, 2), expected, unexpected); - await AssertReferences(s, mod2, new SourceLocation(17, 6), expected, unexpected); - - await AssertReferences(s, mod1, new SourceLocation(8, 5), unexpected, Enumerable.Empty()); - - // a - expected = new[] { - "Definition;(1, 6) - (1, 7)", - "Reference;(2, 4) - (2, 5)", - "Reference;(4, 2) - (4, 3)", - "Reference;(9, 2) - (9, 3)", - "Reference;(16, 7) - (16, 8)" - }; - unexpected = new[] { - "Definition;(14, 4) - (14, 5)", - "Reference;(15, 8) - (15, 9)" - }; - await AssertReferences(s, mod1, new SourceLocation(3, 8), expected, unexpected, "a"); - await AssertReferences(s, mod1, new SourceLocation(2, 8), expected, unexpected); - await AssertReferences(s, mod1, new SourceLocation(3, 5), expected, unexpected); - await AssertReferences(s, mod1, new SourceLocation(5, 3), expected, unexpected); - await AssertReferences(s, mod1, new SourceLocation(10, 3), expected, unexpected); - await AssertReferences(s, mod2, new SourceLocation(17, 8), expected, unexpected); - - await AssertReferences(s, mod2, new SourceLocation(15, 5), unexpected, expected); - await AssertReferences(s, mod2, new SourceLocation(16, 9), unexpected, expected); - - // real (in f) - expected = new[] { - "Reference;(2, 6) - (2, 10)", - "Definition;(6, 4) - (6, 8)", - "Definition;(13, 4) - (13, 8)" - }; - unexpected = new[] { - "Definition;(10, 0) - (10, 4)" - }; - await AssertReferences(s, mod1, new SourceLocation(3, 5), expected, unexpected, "a.real"); - await AssertReferences(s, mod1, new SourceLocation(3, 8), expected, unexpected); - - // C.real - expected = new[] { - "Definition;(10, 0) - (10, 4)", - "Reference;(2, 6) - (2, 10)", - "Reference;(6, 4) - (6, 8)" - }; - await AssertReferences(s, mod1, new SourceLocation(7, 8), expected, Enumerable.Empty()); - - // D.real - expected = new[] { - "Reference;(2, 6) - (2, 10)", - "Definition;(13, 4) - (13, 8)" - }; - unexpected = new[] { - "Definition;(6, 4) - (6, 8)", - "Definition;(10, 0) - (10, 4)" - }; - await AssertReferences(s, mod2, new SourceLocation(14, 8), expected, unexpected); } + // f + var expected = new[] { + "Definition;(1, 4) - (1, 5)", + "Value;(1, 0) - (2, 10)", + "Reference;(4, 0) - (4, 1)", + "Reference;(9, 0) - (9, 1)", + "Reference;(16, 5) - (16, 6)" + }; + var unexpected = new[] { + "Definition;(7, 4) - (7, 5)", + }; + + await AssertReferences(server, mod1, SourceLocation.MinValue, expected, unexpected, "f"); + await AssertReferences(server, mod1, new SourceLocation(2, 5), expected, unexpected); + await AssertReferences(server, mod1, new SourceLocation(5, 2), expected, unexpected); + await AssertReferences(server, mod1, new SourceLocation(10, 2), expected, unexpected); + await AssertReferences(server, mod2, new SourceLocation(17, 6), expected, unexpected); + + await AssertReferences(server, mod1, new SourceLocation(8, 5), unexpected, Enumerable.Empty()); + + // a + expected = new[] { + "Definition;(1, 6) - (1, 7)", + "Reference;(2, 4) - (2, 5)", + "Reference;(4, 2) - (4, 3)", + "Reference;(9, 2) - (9, 3)", + "Reference;(16, 7) - (16, 8)" + }; + unexpected = new[] { + "Definition;(14, 4) - (14, 5)", + "Reference;(15, 8) - (15, 9)" + }; + await AssertReferences(server, mod1, new SourceLocation(3, 8), expected, unexpected, "a"); + await AssertReferences(server, mod1, new SourceLocation(2, 8), expected, unexpected); + await AssertReferences(server, mod1, new SourceLocation(3, 5), expected, unexpected); + await AssertReferences(server, mod1, new SourceLocation(5, 3), expected, unexpected); + await AssertReferences(server, mod1, new SourceLocation(10, 3), expected, unexpected); + await AssertReferences(server, mod2, new SourceLocation(17, 8), expected, unexpected); + + await AssertReferences(server, mod2, new SourceLocation(15, 5), unexpected, expected); + await AssertReferences(server, mod2, new SourceLocation(16, 9), unexpected, expected); + + // real (in f) + expected = new[] { + "Reference;(2, 6) - (2, 10)", + "Definition;(6, 4) - (6, 8)", + "Definition;(13, 4) - (13, 8)" + }; + unexpected = new[] { + "Definition;(10, 0) - (10, 4)" + }; + await AssertReferences(server, mod1, new SourceLocation(3, 5), expected, unexpected, "a.real"); + await AssertReferences(server, mod1, new SourceLocation(3, 8), expected, unexpected); + + // C.real + expected = new[] { + "Definition;(10, 0) - (10, 4)", + "Reference;(2, 6) - (2, 10)", + "Reference;(6, 4) - (6, 8)" + }; + await AssertReferences(server, mod1, new SourceLocation(7, 8), expected, Enumerable.Empty()); + + // D.real + expected = new[] { + "Reference;(2, 6) - (2, 10)", + "Definition;(13, 4) - (13, 8)" + }; + unexpected = new[] { + "Definition;(6, 4) - (6, 8)", + "Definition;(10, 0) - (10, 4)" + }; + await AssertReferences(server, mod2, new SourceLocation(14, 8), expected, unexpected); } public static async Task AssertReferences(Server s, TextDocumentIdentifier document, SourceLocation position, IEnumerable contains, IEnumerable excludes, string expr = null) { @@ -1622,8 +1545,8 @@ public static async Task AssertReferences(Server s, TextDocumentIdentifier docum } } - [TestMethod, Priority(0)] - public async Task ClassMethod() { + [ServerTestMethod, Priority(0)] + public async Task ClassMethod(Server server) { var text = @" class Base(object): @classmethod @@ -1639,22 +1562,20 @@ def fob_derived(cls): Derived.fob_derived() "; - using (var server = await CreateServerAsync()) { - var uri = await server.OpenDefaultDocumentAndGetUriAsync(text); - var references = await server.SendFindReferences(uri, 11, 6); + var uri = await server.OpenDefaultDocumentAndGetUriAsync(text); + var references = await server.SendFindReferences(uri, 11, 6); - var expectedReferences = new (Uri, (int, int, int, int), ReferenceKind?)[] { - (uri, (3, 8, 3, 16), ReferenceKind.Definition), - (uri, (2, 4, 4, 12), ReferenceKind.Value), - (uri, (11, 5, 11, 13), ReferenceKind.Reference), - }; + var expectedReferences = new (Uri, (int, int, int, int), ReferenceKind?)[] { + (uri, (3, 8, 3, 16), ReferenceKind.Definition), + (uri, (2, 4, 4, 12), ReferenceKind.Value), + (uri, (11, 5, 11, 13), ReferenceKind.Reference), + }; - references.Should().OnlyHaveReferences(expectedReferences); - } + references.Should().OnlyHaveReferences(expectedReferences); } - [TestMethod, Priority(0)] - public async Task ClassMethodInBase() { + [ServerTestMethod, Priority(0)] + public async Task ClassMethodInBase(Server server) { var text = @" class Base(object): @classmethod @@ -1667,22 +1588,20 @@ class Derived(Base): Derived.fob_base() "; - using (var server = await CreateServerAsync()) { - var uri = await server.OpenDefaultDocumentAndGetUriAsync(text); - var references = await server.SendFindReferences(uri, 9, 9); + var uri = await server.OpenDefaultDocumentAndGetUriAsync(text); + var references = await server.SendFindReferences(uri, 9, 9); - var expectedReferences = new (Uri, (int, int, int, int), ReferenceKind?)[] { - (uri, (3, 8, 3, 16), ReferenceKind.Definition), - (uri, (2, 4, 4, 12), ReferenceKind.Value), - (uri, (9, 8, 9, 16), ReferenceKind.Reference), - }; + var expectedReferences = new (Uri, (int, int, int, int), ReferenceKind?)[] { + (uri, (3, 8, 3, 16), ReferenceKind.Definition), + (uri, (2, 4, 4, 12), ReferenceKind.Value), + (uri, (9, 8, 9, 16), ReferenceKind.Reference), + }; - references.Should().OnlyHaveReferences(expectedReferences); - } + references.Should().OnlyHaveReferences(expectedReferences); } - [TestMethod, Priority(0)] - public async Task ClassMethodInImportedBase() { + [ServerTestMethod, Priority(0)] + public async Task ClassMethodInImportedBase(Server server) { var text1 = @" import mod2 @@ -1699,26 +1618,24 @@ def fob_base(cls): pass "; - using (var server = await CreateServerAsync()) { - Uri uri1, uri2; - using (server.AnalysisQueue.Pause()) { - uri1 = await server.OpenDefaultDocumentAndGetUriAsync(text1); - uri2 = await TestData.CreateTestSpecificFileAsync("mod2.py", text2); - await server.LoadFileAsync(uri2); - } + Uri uri1, uri2; + using (server.AnalysisQueue.Pause()) { + uri1 = await server.OpenDefaultDocumentAndGetUriAsync(text1); + uri2 = await TestData.CreateTestSpecificFileAsync("mod2.py", text2); + await server.LoadFileAsync(uri2); + } - var references = await server.SendFindReferences(uri1, 6, 9); - var expectedReferences = new (Uri, (int, int, int, int), ReferenceKind?)[] { - (uri2, (3, 8, 3, 16), ReferenceKind.Definition), - (uri2, (2, 4, 4, 12), ReferenceKind.Value), - (uri1, (6, 8, 6, 16), ReferenceKind.Reference), - }; + var references = await server.SendFindReferences(uri1, 6, 9); + var expectedReferences = new (Uri, (int, int, int, int), ReferenceKind?)[] { + (uri2, (3, 8, 3, 16), ReferenceKind.Definition), + (uri2, (2, 4, 4, 12), ReferenceKind.Value), + (uri1, (6, 8, 6, 16), ReferenceKind.Reference), + }; - references.Should().OnlyHaveReferences(expectedReferences); - } + references.Should().OnlyHaveReferences(expectedReferences); } - [ServerTestMethod(), Priority(0)] + [ServerTestMethod, Priority(0)] public async Task ClassMethodInRelativeImportedBase(Server server) { var text1 = @" from .mod2 import Base @@ -1754,8 +1671,8 @@ def fob_base(cls): references.Should().OnlyHaveReferences(expectedReferences); } - [TestMethod, Priority(0)] - public async Task ClassMethodInStarImportedBase() { + [ServerTestMethod, Priority(0)] + public async Task ClassMethodInStarImportedBase(Server server) { var text1 = @" from mod2 import * @@ -1772,28 +1689,26 @@ def fob_base(cls): pass "; - using (var server = await CreateServerAsync()) { - Uri uri1, uri2; - using (server.AnalysisQueue.Pause()) { - uri1 = await server.OpenDefaultDocumentAndGetUriAsync(text1); - uri2 = await TestData.CreateTestSpecificFileAsync("mod2.py", text2); - await server.LoadFileAsync(uri2); - } + Uri uri1, uri2; + using (server.AnalysisQueue.Pause()) { + uri1 = await server.OpenDefaultDocumentAndGetUriAsync(text1); + uri2 = await TestData.CreateTestSpecificFileAsync("mod2.py", text2); + await server.LoadFileAsync(uri2); + } - var references = await server.SendFindReferences(uri1, 6, 9); + var references = await server.SendFindReferences(uri1, 6, 9); - var expectedReferences = new (Uri, (int, int, int, int), ReferenceKind?)[] { - (uri2, (3, 8, 3, 16), ReferenceKind.Definition), - (uri2, (2, 4, 4, 12), ReferenceKind.Value), - (uri1, (6, 8, 6, 16), ReferenceKind.Reference), - }; + var expectedReferences = new (Uri, (int, int, int, int), ReferenceKind?)[] { + (uri2, (3, 8, 3, 16), ReferenceKind.Definition), + (uri2, (2, 4, 4, 12), ReferenceKind.Value), + (uri1, (6, 8, 6, 16), ReferenceKind.Reference), + }; - references.Should().OnlyHaveReferences(expectedReferences); - } + references.Should().OnlyHaveReferences(expectedReferences); } - [TestMethod, Priority(0)] - public async Task ClassMethodInRelativeImportedBaseWithCircularReference() { + [ServerTestMethod, Priority(0)] + public async Task ClassMethodInRelativeImportedBaseWithCircularReference(Server server) { var text1 = @" from .mod2 import Derived1 @@ -1815,33 +1730,31 @@ class Derived1(Base): pass "; - using (var server = await CreateServerAsync()) { - Uri uri1, uri2; - using (server.AnalysisQueue.Pause()) { - uri1 = await TestData.CreateTestSpecificFileAsync("mod1.py", text1); - uri2 = await TestData.CreateTestSpecificFileAsync("mod2.py", text2); + Uri uri1; + using (server.AnalysisQueue.Pause()) { + uri1 = await TestData.CreateTestSpecificFileAsync("mod1.py", text1); + var uri2 = await TestData.CreateTestSpecificFileAsync("mod2.py", text2); - await server.LoadFileAsync(uri2); - uri1 = await server.OpenDocumentAndGetUriAsync("mod1.py", text1); - } + await server.LoadFileAsync(uri2); + await server.SendDidOpenTextDocument(uri1, text1); + } - var references = await server.SendFindReferences(uri1, 11, 9); - var expectedReferences = new (Uri, (int, int, int, int), ReferenceKind?)[] { - (uri1, (5, 8, 5, 16), ReferenceKind.Definition), - (uri1, (4, 4, 6, 12), ReferenceKind.Value), - (uri1, (11, 9, 11, 17), ReferenceKind.Reference), - }; + var references = await server.SendFindReferences(uri1, 11, 9); + var expectedReferences = new (Uri, (int, int, int, int), ReferenceKind?)[] { + (uri1, (5, 8, 5, 16), ReferenceKind.Definition), + (uri1, (4, 4, 6, 12), ReferenceKind.Value), + (uri1, (11, 9, 11, 17), ReferenceKind.Reference), + }; - references.Should().OnlyHaveReferences(expectedReferences); - } + references.Should().OnlyHaveReferences(expectedReferences); } /// /// Verifies that go to definition on 'self' goes to the class definition /// /// - [TestMethod, Priority(0)] - public async Task SelfReferences() { + [ServerTestMethod, Priority(0)] + public async Task SelfReferences(Server server) { var text = @" class Base(object): def fob_base(self): @@ -1853,23 +1766,21 @@ def fob_derived(self): pass "; - using (var server = await CreateServerAsync()) { - var uri1 = await server.OpenDefaultDocumentAndGetUriAsync(text); - - var references = await server.SendFindReferences(uri1, 2, 18); // on first 'self' - var expectedReferences1 = new (Uri, (int, int, int, int), ReferenceKind?)[] { - (uri1, (1, 0, 1, 0), ReferenceKind.Definition), - (uri1, (2, 17, 2, 21), ReferenceKind.Reference) - }; - references.Should().OnlyHaveReferences(expectedReferences1); - - references = await server.SendFindReferences(uri1, 7, 8); // on second 'self' - var expectedReferences2 = new (Uri, (int, int, int, int), ReferenceKind?)[] { - (uri1, (5, 0, 5, 0), ReferenceKind.Definition), - (uri1, (6, 20, 6, 24), ReferenceKind.Reference) - }; - references.Should().OnlyHaveReferences(expectedReferences2); - } + var uri1 = await server.OpenDefaultDocumentAndGetUriAsync(text); + + var references = await server.SendFindReferences(uri1, 2, 18); // on first 'self' + var expectedReferences1 = new (Uri, (int, int, int, int), ReferenceKind?)[] { + (uri1, (1, 0, 1, 0), ReferenceKind.Definition), + (uri1, (2, 17, 2, 21), ReferenceKind.Reference) + }; + references.Should().OnlyHaveReferences(expectedReferences1); + + references = await server.SendFindReferences(uri1, 7, 8); // on second 'self' + var expectedReferences2 = new (Uri, (int, int, int, int), ReferenceKind?)[] { + (uri1, (5, 0, 5, 0), ReferenceKind.Definition), + (uri1, (6, 20, 6, 24), ReferenceKind.Reference) + }; + references.Should().OnlyHaveReferences(expectedReferences2); } } } diff --git a/src/Analysis/Engine/Test/FluentAssertions/CompletionListAssertions.cs b/src/Analysis/Engine/Test/FluentAssertions/CompletionListAssertions.cs index 1ac15af33..9d8b44996 100644 --- a/src/Analysis/Engine/Test/FluentAssertions/CompletionListAssertions.cs +++ b/src/Analysis/Engine/Test/FluentAssertions/CompletionListAssertions.cs @@ -33,12 +33,30 @@ public CompletionListAssertions(CompletionList subject) { protected override string Identifier => nameof(CompletionList); + [CustomAssertion] public AndConstraint OnlyHaveLabels(params string[] labels) => OnlyHaveLabels(labels, string.Empty); + [CustomAssertion] + public AndConstraint HaveAnyCompletions(string because = "", params object[] reasonArgs) { + NotBeNull(because, reasonArgs); + + var errorMessage = Subject.items != null + ? Subject.items.Length > 0 ? null : $"Expected {GetName()} to have completion items{{reason}}, but CompletionList.items collection is empty." + : $"Expected {GetName()} to have completion items{{reason}}, but CompletionList.items collection is null."; + + Execute.Assertion.ForCondition(errorMessage == null) + .BecauseOf(because, reasonArgs) + .FailWith(errorMessage); + + return new AndConstraint(this); + } + + [CustomAssertion] public AndConstraint HaveNoCompletion(string because = "", params object[] reasonArgs) => OnlyHaveLabels(Array.Empty(), because, reasonArgs); + [CustomAssertion] public AndConstraint OnlyHaveLabels(IEnumerable labels, string because = "", params object[] reasonArgs) { NotBeNull(because, reasonArgs); diff --git a/src/Analysis/Engine/Test/HoverTests.cs b/src/Analysis/Engine/Test/HoverTests.cs index f514d64e5..364dfb023 100644 --- a/src/Analysis/Engine/Test/HoverTests.cs +++ b/src/Analysis/Engine/Test/HoverTests.cs @@ -27,13 +27,12 @@ using Microsoft.PythonTools.Analysis; using Microsoft.PythonTools.Analysis.Documentation; using Microsoft.PythonTools.Analysis.FluentAssertions; -using Microsoft.PythonTools.Interpreter; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; namespace AnalysisTests { [TestClass] - public class HoverTests : ServerBasedTest { + public class HoverTests { public TestContext TestContext { get; set; } [TestInitialize] @@ -42,10 +41,9 @@ public class HoverTests : ServerBasedTest { [TestCleanup] public void TestCleanup() => TestEnvironmentImpl.TestCleanup(); - [TestMethod, Priority(0)] - public async Task Hover() { - using (var s = await CreateServerAsync()) { - var mod = await s.OpenDefaultDocumentAndGetUriAsync(@"123 + [ServerTestMethod, Priority(0)] + public async Task Hover(Server server) { + var mod = await server.OpenDefaultDocumentAndGetUriAsync(@"123 'abc' f() def f(): pass @@ -64,69 +62,60 @@ def g(self): x = 3.14 "); - await AssertHover(s, mod, new SourceLocation(1, 1), "int", new[] { "int" }, new SourceSpan(1, 1, 1, 4)); - await AssertHover(s, mod, new SourceLocation(2, 1), "str", new[] { "str" }, new SourceSpan(2, 1, 2, 6)); - await AssertHover(s, mod, new SourceLocation(3, 1), "function module.f()", new[] { "module.f" }, new SourceSpan(3, 1, 3, 2)); - await AssertHover(s, mod, new SourceLocation(4, 6), "function module.f()", new[] { "module.f" }, new SourceSpan(4, 5, 4, 6)); + await AssertHover(server, mod, new SourceLocation(1, 1), "int", new[] { "int" }, new SourceSpan(1, 1, 1, 4)); + await AssertHover(server, mod, new SourceLocation(2, 1), "str", new[] { "str" }, new SourceSpan(2, 1, 2, 6)); + await AssertHover(server, mod, new SourceLocation(3, 1), "function module.f()", new[] { "module.f" }, new SourceSpan(3, 1, 3, 2)); + await AssertHover(server, mod, new SourceLocation(4, 6), "function module.f()", new[] { "module.f" }, new SourceSpan(4, 5, 4, 6)); - await AssertHover(s, mod, new SourceLocation(12, 1), "class module.C", new[] { "module.C" }, new SourceSpan(12, 1, 12, 2)); - await AssertHover(s, mod, new SourceLocation(13, 1), "c: C", new[] { "module.C" }, new SourceSpan(13, 1, 13, 2)); - await AssertHover(s, mod, new SourceLocation(14, 7), "c: C", new[] { "module.C" }, new SourceSpan(14, 7, 14, 8)); - await AssertHover(s, mod, new SourceLocation(14, 9), "c.f: method f of module.C objects*", new[] { "module.C.f" }, new SourceSpan(14, 7, 14, 10)); - await AssertHover(s, mod, new SourceLocation(14, 1), $"function module.C.f.g(self) {Environment.NewLine}declared in C.f", new[] { "module.C.f.g" }, new SourceSpan(14, 1, 14, 4)); + await AssertHover(server, mod, new SourceLocation(12, 1), "class module.C", new[] { "module.C" }, new SourceSpan(12, 1, 12, 2)); + await AssertHover(server, mod, new SourceLocation(13, 1), "c: C", new[] { "module.C" }, new SourceSpan(13, 1, 13, 2)); + await AssertHover(server, mod, new SourceLocation(14, 7), "c: C", new[] { "module.C" }, new SourceSpan(14, 7, 14, 8)); + await AssertHover(server, mod, new SourceLocation(14, 9), "c.f: method f of module.C objects*", new[] { "module.C.f" }, new SourceSpan(14, 7, 14, 10)); + await AssertHover(server, mod, new SourceLocation(14, 1), $"function module.C.f.g(self) {Environment.NewLine}declared in C.f", new[] { "module.C.f.g" }, new SourceSpan(14, 1, 14, 4)); - await AssertHover(s, mod, new SourceLocation(16, 1), "x: int, float", new[] { "int", "float" }, new SourceSpan(16, 1, 16, 2)); - } + await AssertHover(server, mod, new SourceLocation(16, 1), "x: int, float", new[] { "int", "float" }, new SourceSpan(16, 1, 16, 2)); } - [TestMethod, Priority(0)] - public async Task HoverSpanCheck_V2() { - using (var s = await CreateServerAsync(DefaultV2)) { - var mod = await s.OpenDefaultDocumentAndGetUriAsync(@" + [ServerTestMethod(LatestAvailable2X = true), Priority(0)] + public async Task HoverSpanCheck_V2(Server server) { + var mod = await server.OpenDefaultDocumentAndGetUriAsync(@" import datetime datetime.datetime.now().day "); - await AssertHover(s, mod, new SourceLocation(3, 1), "module datetime*", new[] { "datetime" }, new SourceSpan(3, 1, 3, 9)); - await AssertHover(s, mod, new SourceLocation(3, 11), "class datetime.datetime*", new[] { "datetime.datetime" }, new SourceSpan(3, 1, 3, 18)); - await AssertHover(s, mod, new SourceLocation(3, 20), "datetime.datetime.now: bound method now*", null, new SourceSpan(3, 1, 3, 22)); - } + await AssertHover(server, mod, new SourceLocation(3, 1), "module datetime*", new[] { "datetime" }, new SourceSpan(3, 1, 3, 9)); + await AssertHover(server, mod, new SourceLocation(3, 11), "class datetime.datetime*", new[] { "datetime.datetime" }, new SourceSpan(3, 1, 3, 18)); + await AssertHover(server, mod, new SourceLocation(3, 20), "datetime.datetime.now: bound method now*", null, new SourceSpan(3, 1, 3, 22)); } - [TestMethod, Priority(0)] - public async Task HoverSpanCheck_V3() { - using (var s = await CreateServerAsync(DefaultV3)) { - var mod = await s.OpenDefaultDocumentAndGetUriAsync(@" + [ServerTestMethod(LatestAvailable3X = true), Priority(0)] + public async Task HoverSpanCheck_V3(Server server) { + var mod = await server.OpenDefaultDocumentAndGetUriAsync(@" import datetime datetime.datetime.now().day "); - await AssertHover(s, mod, new SourceLocation(3, 1), "module datetime*", new[] { "datetime" }, new SourceSpan(3, 1, 3, 9)); - await AssertHover(s, mod, new SourceLocation(3, 11), "class datetime.datetime*", new[] { "datetime.datetime" }, new SourceSpan(3, 1, 3, 18)); - await AssertHover(s, mod, new SourceLocation(3, 20), "datetime.datetime.now: bound method now*", null, new SourceSpan(3, 1, 3, 22)); - await AssertHover(s, mod, new SourceLocation(3, 28), "datetime.datetime.now().day: int*", new[] { "int" }, new SourceSpan(3, 1, 3, 28)); - } + await AssertHover(server, mod, new SourceLocation(3, 1), "module datetime*", new[] { "datetime" }, new SourceSpan(3, 1, 3, 9)); + await AssertHover(server, mod, new SourceLocation(3, 11), "class datetime.datetime*", new[] { "datetime.datetime" }, new SourceSpan(3, 1, 3, 18)); + await AssertHover(server, mod, new SourceLocation(3, 20), "datetime.datetime.now: bound method now*", null, new SourceSpan(3, 1, 3, 22)); + await AssertHover(server, mod, new SourceLocation(3, 28), "datetime.datetime.now().day: int*", new[] { "int" }, new SourceSpan(3, 1, 3, 28)); } - [TestMethod, Priority(0)] - public async Task FromImportHover() { - using (var s = await CreateServerAsync()) { - var mod = await s.OpenDefaultDocumentAndGetUriAsync("from os import path as p\n"); - await AssertHover(s, mod, new SourceLocation(1, 6), "module os*", null, new SourceSpan(1, 6, 1, 8)); - await AssertHover(s, mod, new SourceLocation(1, 16), "module posixpath*", new[] { "posixpath" }, new SourceSpan(1, 16, 1, 20)); - await AssertHover(s, mod, new SourceLocation(1, 24), "module posixpath*", new[] { "posixpath" }, new SourceSpan(1, 24, 1, 25)); - } + [ServerTestMethod, Priority(0)] + public async Task FromImportHover(Server server) { + var mod = await server.OpenDefaultDocumentAndGetUriAsync("from os import path as p\n"); + await AssertHover(server, mod, new SourceLocation(1, 6), "module os*", null, new SourceSpan(1, 6, 1, 8)); + await AssertHover(server, mod, new SourceLocation(1, 16), "module posixpath*", new[] { "posixpath" }, new SourceSpan(1, 16, 1, 20)); + await AssertHover(server, mod, new SourceLocation(1, 24), "module posixpath*", new[] { "posixpath" }, new SourceSpan(1, 24, 1, 25)); } - [TestMethod, Priority(0)] - public async Task FromImportRelativeHover() { - using (var s = await CreateServerAsync()) { - var mod1 = await s.OpenDocumentAndGetUriAsync("mod1.py", "from . import mod2\n"); - var mod2 = await s.OpenDocumentAndGetUriAsync("mod2.py", "def foo():\n pass\n"); - await AssertHover(s, mod1, new SourceLocation(1, 16), "module mod2", null, new SourceSpan(1, 15, 1, 19)); - } + [ServerTestMethod, Priority(0)] + public async Task FromImportRelativeHover(Server server) { + var mod1 = await server.OpenDocumentAndGetUriAsync("mod1.py", "from . import mod2\n"); + var mod2 = await server.OpenDocumentAndGetUriAsync("mod2.py", "def foo():\n pass\n"); + await AssertHover(server, mod1, new SourceLocation(1, 16), "module mod2", null, new SourceSpan(1, 15, 1, 19)); } - [TestMethod, Priority(0)] - public async Task SelfHover() { + [ServerTestMethod, Priority(0)] + public async Task SelfHover(Server server) { var text = @" class Base(object): def fob_base(self): @@ -137,42 +126,34 @@ def fob_derived(self): self.fob_base() pass "; - using (var s = await CreateServerAsync()) { - var uri = await s.OpenDefaultDocumentAndGetUriAsync(text); - await AssertHover(s, uri, new SourceLocation(3, 19), "class module.Base(object)", null, new SourceSpan(2, 7, 2, 11)); - await AssertHover(s, uri, new SourceLocation(8, 8), "class module.Derived(Base)", null, new SourceSpan(6, 7, 6, 14)); - } + var uri = await server.OpenDefaultDocumentAndGetUriAsync(text); + await AssertHover(server, uri, new SourceLocation(3, 19), "class module.Base(object)", null, new SourceSpan(2, 7, 2, 11)); + await AssertHover(server, uri, new SourceLocation(8, 8), "class module.Derived(Base)", null, new SourceSpan(6, 7, 6, 14)); } - [TestMethod, Priority(0)] - public async Task TupleFunctionArgumentsHover() { - using (var s = await CreateServerAsync()) { - var uri = await s.OpenDefaultDocumentAndGetUriAsync("def what(a, b):\n return a, b, 1\n"); - await AssertHover(s, uri, new SourceLocation(1, 6), "function module.what(a, b) -> tuple[Any, Any, int]", null, new SourceSpan(1, 5, 1, 9)); - } + [ServerTestMethod, Priority(0)] + public async Task TupleFunctionArgumentsHover(Server server) { + var uri = await server.OpenDefaultDocumentAndGetUriAsync("def what(a, b):\n return a, b, 1\n"); + await AssertHover(server, uri, new SourceLocation(1, 6), "function module.what(a, b) -> tuple[Any, Any, int]", null, new SourceSpan(1, 5, 1, 9)); } - [TestMethod, Priority(0)] - public async Task TupleFunctionArgumentsWithCallHover() { - using (var s = await CreateServerAsync()) { - var uri = await s.OpenDefaultDocumentAndGetUriAsync("def what(a, b):\n return a, b, 1\n\nwhat(1, 2)"); - await AssertHover(s, uri, new SourceLocation(1, 6), "function module.what(a, b) -> tuple[int, int, int]", null, new SourceSpan(1, 5, 1, 9)); - } + [ServerTestMethod, Priority(0)] + public async Task TupleFunctionArgumentsWithCallHover(Server server) { + var uri = await server.OpenDefaultDocumentAndGetUriAsync("def what(a, b):\n return a, b, 1\n\nwhat(1, 2)"); + await AssertHover(server, uri, new SourceLocation(1, 6), "function module.what(a, b) -> tuple[int, int, int]", null, new SourceSpan(1, 5, 1, 9)); } - [TestMethod, Priority(0)] - public async Task MarkupKindValid() { - using (var s = await CreateServerAsync()) { - var u = await s.OpenDefaultDocumentAndGetUriAsync("123"); + [ServerTestMethod, Priority(0)] + public async Task MarkupKindValid(Server server) { + var u = await server.OpenDefaultDocumentAndGetUriAsync("123"); - await s.WaitForCompleteAnalysisAsync(CancellationToken.None); - var hover = await s.Hover(new TextDocumentPositionParams { - textDocument = new TextDocumentIdentifier { uri = u }, - position = new SourceLocation(1, 1), - }, CancellationToken.None); + await server.WaitForCompleteAnalysisAsync(CancellationToken.None); + var hover = await server.Hover(new TextDocumentPositionParams { + textDocument = new TextDocumentIdentifier { uri = u }, + position = new SourceLocation(1, 1), + }, CancellationToken.None); - hover.contents.kind.Should().BeOneOf(MarkupKind.PlainText, MarkupKind.Markdown); - } + hover.contents.kind.Should().BeOneOf(MarkupKind.PlainText, MarkupKind.Markdown); } private static async Task AssertHover(Server s, Uri uri, SourceLocation position, string hoverText, IEnumerable typeNames, SourceSpan? range = null, string expr = null) { @@ -198,21 +179,5 @@ private static async Task AssertHover(Server s, Uri uri, SourceLocation position hover.range.Should().Be(range.Value); } } - private static InterpreterConfiguration DefaultV3 { - get { - var ver = PythonVersions.Python36_x64 ?? PythonVersions.Python36 ?? - PythonVersions.Python35_x64 ?? PythonVersions.Python35; - ver.AssertInstalled(); - return ver; - } - } - - private static InterpreterConfiguration DefaultV2 { - get { - var ver = PythonVersions.Python27_x64 ?? PythonVersions.Python27; - ver.AssertInstalled(); - return ver; - } - } } } diff --git a/src/Analysis/Engine/Test/ServerBasedTest.cs b/src/Analysis/Engine/Test/ServerBasedTest.cs index 455da8bf6..1727caf49 100644 --- a/src/Analysis/Engine/Test/ServerBasedTest.cs +++ b/src/Analysis/Engine/Test/ServerBasedTest.cs @@ -98,10 +98,5 @@ protected async Task GetStubBasedAnalysis( var uri = await server.OpenDefaultDocumentAndGetUriAsync(code); return await server.GetAnalysisAsync(uri); } - - protected static string GetTypeshedPath() { - var asmPath = Assembly.GetExecutingAssembly().GetAssemblyPath(); - return Path.Combine(Path.GetDirectoryName(asmPath), "Typeshed"); - } } } diff --git a/src/Analysis/Engine/Test/ServerExtensions.cs b/src/Analysis/Engine/Test/ServerExtensions.cs index ece5c7ffd..b6041b26a 100644 --- a/src/Analysis/Engine/Test/ServerExtensions.cs +++ b/src/Analysis/Engine/Test/ServerExtensions.cs @@ -56,7 +56,7 @@ await server.Initialize(new InitializeParams { liveLinting = true, } } - }, CancellationToken.None); + }, GetCancellationToken()); return server; } @@ -64,15 +64,6 @@ await server.Initialize(new InitializeParams { public static Task GetAnalysisAsync(this Server server, Uri uri, int waitingTimeout = -1, int failAfter = 30000) => ((ProjectEntry)server.ProjectFiles.GetEntry(uri)).GetAnalysisAsync(waitingTimeout, GetCancellationToken(failAfter)); - public static Task EnqueueItemAsync(this Server server, Uri uri) - => server.EnqueueItemAsync((IDocument)server.ProjectFiles.GetEntry(uri)); - - public static async Task EnqueueItemsAsync(this Server server, CancellationToken cancellationToken, params IDocument[] projectEntries) { - foreach (var document in projectEntries) { - await server.EnqueueItemAsync(document); - } - } - // TODO: Replace usages of AddModuleWithContent with OpenDefaultDocumentAndGetUriAsync public static async Task AddModuleWithContentAsync(this Server server, string relativePath, string content) { var entry = (ProjectEntry)server.Analyzer.AddModule(null, TestData.GetTestSpecificPath(relativePath)); @@ -89,8 +80,24 @@ public static Task SendCompletion(this Server server, Uri uri, i position = new Position { line = line, character = character + }, + }, GetCancellationToken()); + } + + public static Task SendCompletion(this Server server, Uri uri, int line, int character, string triggerCharacter, CompletionTriggerKind triggerKind) { + return server.Completion(new CompletionParams { + textDocument = new TextDocumentIdentifier { + uri = uri + }, + position = new Position { + line = line, + character = character + }, + context = new CompletionContext { + triggerCharacter = triggerCharacter, + triggerKind = triggerKind } - }, CancellationToken.None); + }, GetCancellationToken()); } public static Task SendHover(this Server server, Uri uri, int line, int character) { @@ -102,7 +109,7 @@ public static Task SendHover(this Server server, Uri uri, int line, int c line = line, character = character } - }, CancellationToken.None); + }, GetCancellationToken()); } public static Task SendSignatureHelp(this Server server, Uri uri, int line, int character) { @@ -112,7 +119,7 @@ public static Task SendSignatureHelp(this Server server, Uri uri, line = line, character = character } - }, CancellationToken.None); + }, GetCancellationToken()); } public static Task SendFindReferences(this Server server, Uri uri, int line, int character, bool includeDeclaration = true) { @@ -126,7 +133,7 @@ public static Task SendFindReferences(this Server server, Uri uri, includeDeclaration = includeDeclaration, _includeValues = true // For compatibility with PTVS } - }, CancellationToken.None); + }, GetCancellationToken()); } public static async Task OpenDefaultDocumentAndGetUriAsync(this Server server, string content) { @@ -154,7 +161,7 @@ await server.DidOpenTextDocument(new DidOpenTextDocumentParams { text = content, languageId = languageId ?? "python" } - }, CancellationToken.None); + }, GetCancellationToken()); } public static async Task OpenDefaultDocumentAndGetAnalysisAsync(this Server server, string content, int failAfter = 30000, string languageId = null) { @@ -173,9 +180,29 @@ public static Task SendDidChangeTextDocumentAsync(this Server server, Uri uri, s contentChanges = new [] { new TextDocumentContentChangedEvent { text = text, + } + } + }, GetCancellationToken()); + } + + public static Task SendDidChangeTextDocumentAsync(this Server server, Uri uri, string text, Position position) + => server.SendDidChangeTextDocumentAsync(uri, text, position, position); + + public static Task SendDidChangeTextDocumentAsync(this Server server, Uri uri, string text, Position start, Position end) { + return server.DidChangeTextDocument(new DidChangeTextDocumentParams { + textDocument = new VersionedTextDocumentIdentifier { + uri = uri + }, + contentChanges = new [] { + new TextDocumentContentChangedEvent { + text = text, + range = new Range { + start = start, + end = end + } }, } - }, CancellationToken.None); + }, GetCancellationToken()); } public static Task SendDocumentOnTypeFormatting(this Server server, TextDocumentIdentifier textDocument, Position position, string ch) { @@ -183,7 +210,7 @@ public static Task SendDocumentOnTypeFormatting(this Server server, textDocument = textDocument, position = position, ch = ch, - }, CancellationToken.None); + }, GetCancellationToken()); } public static Task SendDidChangeConfiguration(this Server server, ServerSettings.PythonCompletionOptions pythonCompletionOptions, int failAfter = 30000) { diff --git a/src/Analysis/Engine/Test/ServerTestMethodAttribute.cs b/src/Analysis/Engine/Test/ServerTestMethodAttribute.cs index 8959c0dba..d2dffff01 100644 --- a/src/Analysis/Engine/Test/ServerTestMethodAttribute.cs +++ b/src/Analysis/Engine/Test/ServerTestMethodAttribute.cs @@ -28,6 +28,7 @@ public enum PythonLanguageMajorVersion { None, EarliestV2, EarliestV3, LatestV2, public class ServerTestMethodAttribute : TestMethodAttribute { public bool TestSpecificRootUri { get; set; } + public bool DefaultTypeshedPath { get; set; } public bool LatestAvailable3X { get; set; } public bool LatestAvailable2X { get; set; } public PythonLanguageVersion Version { get; set; } = PythonLanguageVersion.None; @@ -53,6 +54,13 @@ private TestResult ExecuteWithServer(ITestMethod testMethod) { } var server = await new Server().InitializeAsync(interpreterConfiguration, rootUri); + if (DefaultTypeshedPath) { + var limits = server.Analyzer.Limits; + limits.UseTypeStubPackages = true; + server.Analyzer.Limits = limits; + server.Analyzer.SetTypeStubPaths(new[] { TestData.GetDefaultTypeshedPath() }); + } + arguments[0] = server; return server; }); diff --git a/src/UnitTests/Core/Impl/TestData.cs b/src/UnitTests/Core/Impl/TestData.cs index ae2635848..c5f033dbd 100644 --- a/src/UnitTests/Core/Impl/TestData.cs +++ b/src/UnitTests/Core/Impl/TestData.cs @@ -65,6 +65,11 @@ private static string CalculateTestDataRoot() { private static readonly Lazy RootLazy = new Lazy(CalculateTestDataRoot); public static string Root => RootLazy.Value; + public static string GetDefaultTypeshedPath() { + var asmPath = Assembly.GetExecutingAssembly().GetAssemblyPath(); + return Path.Combine(Path.GetDirectoryName(asmPath), "Typeshed"); + } + public static Uri GetDefaultModuleUri() => new Uri(GetDefaultModulePath()); public static Uri GetNextModuleUri() => new Uri(GetNextModulePath()); public static Uri[] GetNextModuleUris(int count) { From 47ffe1ec379eec57890614bc8618b7601925bb87 Mon Sep 17 00:00:00 2001 From: Alexander Sher Date: Fri, 7 Dec 2018 11:05:36 -0800 Subject: [PATCH 02/10] Initial commit --- .../Extensions/StackExtensions.cs | 31 + .../Interpreter/InterpreterConfiguration.cs | 2 +- .../Engine/Impl/Properties/AssemblyInfo.cs | 36 +- src/Analysis/Engine/Impl/PythonAnalyzer.cs | 11 - .../Impl/Definitions/CallbackEventArgs.cs | 0 .../Definitions/EditorOperationException.cs | 0 .../{ => Core}/Impl/Definitions/Enums.cs | 0 .../Impl/Definitions/IDocumentReader.cs | 0 .../ILanguageServerExtensionProvider.cs | 0 .../{ => Core}/Impl/Definitions/ILogger.cs | 0 .../Impl/Definitions/IProgressService.cs | 0 .../Impl/Definitions/IPythonLanguageServer.cs | 0 .../IPythonLanguageServerProtocol.cs | 0 .../Impl/Definitions/IServiceContainer.cs | 0 .../Impl/Definitions/ITelemetryService.cs | 0 .../Definitions/ITextChangeNotifications.cs | 0 .../{ => Core}/Impl/Definitions/IUIService.cs | 0 .../{ => Core}/Impl/Definitions/Messages.cs | 0 .../Impl/Definitions/ServerSettings.cs | 0 .../{ => Core}/Impl/Definitions/Structures.cs | 0 .../Impl/Extensions/ICompletionExtension.cs | 0 .../Extensions/ILanguageServerExtension.cs | 0 .../Impl/Implementation/BlockFormatter.cs | 0 .../Impl/Implementation/CodeActionProvider.cs | 0 .../Impl/Implementation/CompletionAnalysis.cs | 0 .../Implementation/DiagnosticsErrorSink.cs | 0 .../Impl/Implementation/DocumentReader.cs | 0 .../Impl/Implementation/EditorFiles.cs | 0 .../Impl/Implementation/LineFormatter.cs | 0 .../Impl/Implementation/ParseQueue.cs | 0 .../Impl/Implementation/ProjectFiles.cs | 0 .../Impl/Implementation/RestTextConverter.cs | 0 .../Impl/Implementation/Server.Completion.cs | 0 .../Impl/Implementation/Server.Extensions.cs | 0 .../Implementation/Server.FindReferences.cs | 0 .../Implementation/Server.GoToDefinition.cs | 0 .../Impl/Implementation/Server.Hover.cs | 0 .../Implementation/Server.OnTypeFormatting.cs | 0 .../Implementation/Server.PrivateHelpers.cs | 0 .../Impl/Implementation/Server.Rename.cs | 0 .../Implementation/Server.SignatureHelp.cs | 0 .../Implementation/Server.WorkspaceSymbols.cs | 0 .../{ => Core}/Impl/Implementation/Server.cs | 0 .../Impl/Implementation/ServerBase.cs | 0 .../Impl/Implementation/VolatileCounter.cs | 0 ...icrosoft.Python.LanguageServer.Core.csproj | 41 + .../Core/Impl/Properties/AssemblyInfo.cs | 21 + .../Core/Impl/Resources.Designer.cs | 168 ++ src/LanguageServer/Core/Impl/Resources.resx | 137 ++ .../Microsoft.Python.LanguageServer.csproj | 1 + src/LanguageServer/Impl/Program.cs | 1 + src/LanguageServer/Impl/Resources.Designer.cs | 99 - src/LanguageServer/Impl/Resources.resx | 33 - src/PLS.sln | 23 + .../Analyzer.csproj | 125 ++ .../AnalyzerScrapers.pyproj | 41 + .../AnalyzerScrapers.sln | 20 + .../Microsoft.PythonTools.Analyzer/App.config | 12 + .../AssemblyInfo.cs | 23 + .../BuiltinScraper.py | 554 ++++++ .../BuiltinScraperTests.py | 491 +++++ .../ExtensionScraper.py | 79 + .../Intellisense/AnalysisLimitsConverter.cs | 114 ++ .../Intellisense/AnalysisProtocol.cs | 970 ++++++++++ .../Intellisense/AssignmentWalker.cs | 137 ++ .../Intellisense/ClassifierWalker.cs | 480 +++++ .../Intellisense/MethodExtractor.cs | 762 ++++++++ .../Intellisense/OutOfProcMethodExtractor.cs | 38 + .../Intellisense/OutOfProcProjectAnalyzer.cs | 1699 +++++++++++++++++ .../Intellisense/OutliningWalker.cs | 196 ++ .../Intellisense/ProximityExpressionWalker.cs | 144 ++ .../Intellisense/TaggedSpan.cs | 27 + .../Interpreter/IDotNetPythonInterpreter.cs | 32 + .../Interpreter/ProjectAssemblyReference.cs | 62 + .../IronPythonScraper.py | 177 ++ .../Microsoft.PythonTools.Analyzer.csproj | 73 + .../PyLibAnalyzer.cs | 244 +++ .../PythonScraper.py | 999 ++++++++++ .../VersionDiff.py | 68 + .../Connection.cs | 709 +++++++ .../DebugTextWriter.cs | 38 + .../ErrorReceivedEventArgs.cs | 30 + .../Microsoft.PythonTools.Ipc.Json/Event.cs | 32 + .../EventReceivedEventArgs.cs | 33 + .../FailedRequestException.cs | 38 + .../Microsoft.PythonTools.Ipc.Json.csproj | 15 + .../Properties/AssemblyInfo.cs | 24 + .../ProtocolReader.cs | 117 ++ .../Microsoft.PythonTools.Ipc.Json/Request.cs | 37 + .../RequestArgs.cs | 31 + .../Response.cs | 22 + .../UriJsonConverter.cs | 41 + 92 files changed, 9159 insertions(+), 179 deletions(-) create mode 100644 src/Analysis/Engine/Impl/Infrastructure/Extensions/StackExtensions.cs rename src/LanguageServer/{ => Core}/Impl/Definitions/CallbackEventArgs.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Definitions/EditorOperationException.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Definitions/Enums.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Definitions/IDocumentReader.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Definitions/ILanguageServerExtensionProvider.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Definitions/ILogger.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Definitions/IProgressService.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Definitions/IPythonLanguageServer.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Definitions/IPythonLanguageServerProtocol.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Definitions/IServiceContainer.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Definitions/ITelemetryService.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Definitions/ITextChangeNotifications.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Definitions/IUIService.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Definitions/Messages.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Definitions/ServerSettings.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Definitions/Structures.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Extensions/ICompletionExtension.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Extensions/ILanguageServerExtension.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/BlockFormatter.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/CodeActionProvider.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/CompletionAnalysis.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/DiagnosticsErrorSink.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/DocumentReader.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/EditorFiles.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/LineFormatter.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/ParseQueue.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/ProjectFiles.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/RestTextConverter.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/Server.Completion.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/Server.Extensions.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/Server.FindReferences.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/Server.GoToDefinition.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/Server.Hover.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/Server.OnTypeFormatting.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/Server.PrivateHelpers.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/Server.Rename.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/Server.SignatureHelp.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/Server.WorkspaceSymbols.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/Server.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/ServerBase.cs (100%) rename src/LanguageServer/{ => Core}/Impl/Implementation/VolatileCounter.cs (100%) create mode 100644 src/LanguageServer/Core/Impl/Microsoft.Python.LanguageServer.Core.csproj create mode 100644 src/LanguageServer/Core/Impl/Properties/AssemblyInfo.cs create mode 100644 src/LanguageServer/Core/Impl/Resources.Designer.cs create mode 100644 src/LanguageServer/Core/Impl/Resources.resx create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/Analyzer.csproj create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/AnalyzerScrapers.pyproj create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/AnalyzerScrapers.sln create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/App.config create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/AssemblyInfo.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/BuiltinScraper.py create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/BuiltinScraperTests.py create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/ExtensionScraper.py create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AnalysisLimitsConverter.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AnalysisProtocol.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AssignmentWalker.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/ClassifierWalker.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/MethodExtractor.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutOfProcMethodExtractor.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutOfProcProjectAnalyzer.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutliningWalker.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/ProximityExpressionWalker.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/TaggedSpan.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/Interpreter/IDotNetPythonInterpreter.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/Interpreter/ProjectAssemblyReference.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/IronPythonScraper.py create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/Microsoft.PythonTools.Analyzer.csproj create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/PyLibAnalyzer.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/PythonScraper.py create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/VersionDiff.py create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/Connection.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/DebugTextWriter.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/ErrorReceivedEventArgs.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/Event.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/EventReceivedEventArgs.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/FailedRequestException.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/Microsoft.PythonTools.Ipc.Json.csproj create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/Properties/AssemblyInfo.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/ProtocolReader.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/Request.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/RequestArgs.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/Response.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/UriJsonConverter.cs diff --git a/src/Analysis/Engine/Impl/Infrastructure/Extensions/StackExtensions.cs b/src/Analysis/Engine/Impl/Infrastructure/Extensions/StackExtensions.cs new file mode 100644 index 000000000..662a9002a --- /dev/null +++ b/src/Analysis/Engine/Impl/Infrastructure/Extensions/StackExtensions.cs @@ -0,0 +1,31 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; + +namespace Microsoft.PythonTools.Analysis.Infrastructure { + internal static class StackExtensions { + public static bool TryPeek(this Stack stack, out T result) { + if (stack.Count == 0) { + result = default; + return false; + } + + result = stack.Peek(); + return true; + } + } +} diff --git a/src/Analysis/Engine/Impl/Interpreter/InterpreterConfiguration.cs b/src/Analysis/Engine/Impl/Interpreter/InterpreterConfiguration.cs index e6012dfa1..4f6af1d68 100644 --- a/src/Analysis/Engine/Impl/Interpreter/InterpreterConfiguration.cs +++ b/src/Analysis/Engine/Impl/Interpreter/InterpreterConfiguration.cs @@ -21,7 +21,7 @@ using Microsoft.PythonTools.Analysis.Infrastructure; namespace Microsoft.PythonTools.Interpreter { - public sealed class InterpreterConfiguration : IEquatable { + public class InterpreterConfiguration : IEquatable { private readonly string _description; private string _fullDescription; diff --git a/src/Analysis/Engine/Impl/Properties/AssemblyInfo.cs b/src/Analysis/Engine/Impl/Properties/AssemblyInfo.cs index 060ebe28d..114d2b066 100644 --- a/src/Analysis/Engine/Impl/Properties/AssemblyInfo.cs +++ b/src/Analysis/Engine/Impl/Properties/AssemblyInfo.cs @@ -17,40 +17,6 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Microsoft.PythonTools.Analyzer, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.BuildTasks, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.Debugger, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.Django, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.EnvironmentsList, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.IronPython.Interpreter, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.TestAdapter, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.TestAdapter.Analysis, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.TestAdapter.Executor, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.Uwp.Interpreter, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.VSInterpreters, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.Python.LanguageServer, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] - -[assembly: InternalsVisibleTo("AnalysisTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("CookiecutterTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("DebuggerTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("DebuggerUITests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("DjangoTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("DjangoUITests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("FastCgiTest, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("IpcJsonTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("IronPythonTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("ProfilingTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("ProfilingUITests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("ProjectUITests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("PythonToolsMockTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("PythonToolsTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("PythonToolsUITests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("ReplWindowUITests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("TestAdapterTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("TestUtilities, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("TestUtilities.Python, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("TestUtilities.Python.Analysis, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("VSInterpretersTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("AnalysisMemoryTester, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.Analysis.Browser, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("Microsoft.Python.LanguageServer.Core, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.Python.Analysis.Engine.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/src/Analysis/Engine/Impl/PythonAnalyzer.cs b/src/Analysis/Engine/Impl/PythonAnalyzer.cs index 14a4a8ec0..270d13b93 100644 --- a/src/Analysis/Engine/Impl/PythonAnalyzer.cs +++ b/src/Analysis/Engine/Impl/PythonAnalyzer.cs @@ -377,17 +377,6 @@ public PythonMemberType GetModuleType() { } } - private static bool GetPackageNameIfMatch(string name, string fullName, out string packageName) { - var lastDot = fullName.LastIndexOf('.'); - if (lastDot < 0) { - packageName = null; - return false; - } - - packageName = fullName.Remove(lastDot); - return String.Compare(fullName, lastDot + 1, name, 0, name.Length, StringComparison.Ordinal) == 0; - } - /// /// Returns the interpreter that the analyzer is using. /// diff --git a/src/LanguageServer/Impl/Definitions/CallbackEventArgs.cs b/src/LanguageServer/Core/Impl/Definitions/CallbackEventArgs.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/CallbackEventArgs.cs rename to src/LanguageServer/Core/Impl/Definitions/CallbackEventArgs.cs diff --git a/src/LanguageServer/Impl/Definitions/EditorOperationException.cs b/src/LanguageServer/Core/Impl/Definitions/EditorOperationException.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/EditorOperationException.cs rename to src/LanguageServer/Core/Impl/Definitions/EditorOperationException.cs diff --git a/src/LanguageServer/Impl/Definitions/Enums.cs b/src/LanguageServer/Core/Impl/Definitions/Enums.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/Enums.cs rename to src/LanguageServer/Core/Impl/Definitions/Enums.cs diff --git a/src/LanguageServer/Impl/Definitions/IDocumentReader.cs b/src/LanguageServer/Core/Impl/Definitions/IDocumentReader.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/IDocumentReader.cs rename to src/LanguageServer/Core/Impl/Definitions/IDocumentReader.cs diff --git a/src/LanguageServer/Impl/Definitions/ILanguageServerExtensionProvider.cs b/src/LanguageServer/Core/Impl/Definitions/ILanguageServerExtensionProvider.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/ILanguageServerExtensionProvider.cs rename to src/LanguageServer/Core/Impl/Definitions/ILanguageServerExtensionProvider.cs diff --git a/src/LanguageServer/Impl/Definitions/ILogger.cs b/src/LanguageServer/Core/Impl/Definitions/ILogger.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/ILogger.cs rename to src/LanguageServer/Core/Impl/Definitions/ILogger.cs diff --git a/src/LanguageServer/Impl/Definitions/IProgressService.cs b/src/LanguageServer/Core/Impl/Definitions/IProgressService.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/IProgressService.cs rename to src/LanguageServer/Core/Impl/Definitions/IProgressService.cs diff --git a/src/LanguageServer/Impl/Definitions/IPythonLanguageServer.cs b/src/LanguageServer/Core/Impl/Definitions/IPythonLanguageServer.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/IPythonLanguageServer.cs rename to src/LanguageServer/Core/Impl/Definitions/IPythonLanguageServer.cs diff --git a/src/LanguageServer/Impl/Definitions/IPythonLanguageServerProtocol.cs b/src/LanguageServer/Core/Impl/Definitions/IPythonLanguageServerProtocol.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/IPythonLanguageServerProtocol.cs rename to src/LanguageServer/Core/Impl/Definitions/IPythonLanguageServerProtocol.cs diff --git a/src/LanguageServer/Impl/Definitions/IServiceContainer.cs b/src/LanguageServer/Core/Impl/Definitions/IServiceContainer.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/IServiceContainer.cs rename to src/LanguageServer/Core/Impl/Definitions/IServiceContainer.cs diff --git a/src/LanguageServer/Impl/Definitions/ITelemetryService.cs b/src/LanguageServer/Core/Impl/Definitions/ITelemetryService.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/ITelemetryService.cs rename to src/LanguageServer/Core/Impl/Definitions/ITelemetryService.cs diff --git a/src/LanguageServer/Impl/Definitions/ITextChangeNotifications.cs b/src/LanguageServer/Core/Impl/Definitions/ITextChangeNotifications.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/ITextChangeNotifications.cs rename to src/LanguageServer/Core/Impl/Definitions/ITextChangeNotifications.cs diff --git a/src/LanguageServer/Impl/Definitions/IUIService.cs b/src/LanguageServer/Core/Impl/Definitions/IUIService.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/IUIService.cs rename to src/LanguageServer/Core/Impl/Definitions/IUIService.cs diff --git a/src/LanguageServer/Impl/Definitions/Messages.cs b/src/LanguageServer/Core/Impl/Definitions/Messages.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/Messages.cs rename to src/LanguageServer/Core/Impl/Definitions/Messages.cs diff --git a/src/LanguageServer/Impl/Definitions/ServerSettings.cs b/src/LanguageServer/Core/Impl/Definitions/ServerSettings.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/ServerSettings.cs rename to src/LanguageServer/Core/Impl/Definitions/ServerSettings.cs diff --git a/src/LanguageServer/Impl/Definitions/Structures.cs b/src/LanguageServer/Core/Impl/Definitions/Structures.cs similarity index 100% rename from src/LanguageServer/Impl/Definitions/Structures.cs rename to src/LanguageServer/Core/Impl/Definitions/Structures.cs diff --git a/src/LanguageServer/Impl/Extensions/ICompletionExtension.cs b/src/LanguageServer/Core/Impl/Extensions/ICompletionExtension.cs similarity index 100% rename from src/LanguageServer/Impl/Extensions/ICompletionExtension.cs rename to src/LanguageServer/Core/Impl/Extensions/ICompletionExtension.cs diff --git a/src/LanguageServer/Impl/Extensions/ILanguageServerExtension.cs b/src/LanguageServer/Core/Impl/Extensions/ILanguageServerExtension.cs similarity index 100% rename from src/LanguageServer/Impl/Extensions/ILanguageServerExtension.cs rename to src/LanguageServer/Core/Impl/Extensions/ILanguageServerExtension.cs diff --git a/src/LanguageServer/Impl/Implementation/BlockFormatter.cs b/src/LanguageServer/Core/Impl/Implementation/BlockFormatter.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/BlockFormatter.cs rename to src/LanguageServer/Core/Impl/Implementation/BlockFormatter.cs diff --git a/src/LanguageServer/Impl/Implementation/CodeActionProvider.cs b/src/LanguageServer/Core/Impl/Implementation/CodeActionProvider.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/CodeActionProvider.cs rename to src/LanguageServer/Core/Impl/Implementation/CodeActionProvider.cs diff --git a/src/LanguageServer/Impl/Implementation/CompletionAnalysis.cs b/src/LanguageServer/Core/Impl/Implementation/CompletionAnalysis.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/CompletionAnalysis.cs rename to src/LanguageServer/Core/Impl/Implementation/CompletionAnalysis.cs diff --git a/src/LanguageServer/Impl/Implementation/DiagnosticsErrorSink.cs b/src/LanguageServer/Core/Impl/Implementation/DiagnosticsErrorSink.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/DiagnosticsErrorSink.cs rename to src/LanguageServer/Core/Impl/Implementation/DiagnosticsErrorSink.cs diff --git a/src/LanguageServer/Impl/Implementation/DocumentReader.cs b/src/LanguageServer/Core/Impl/Implementation/DocumentReader.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/DocumentReader.cs rename to src/LanguageServer/Core/Impl/Implementation/DocumentReader.cs diff --git a/src/LanguageServer/Impl/Implementation/EditorFiles.cs b/src/LanguageServer/Core/Impl/Implementation/EditorFiles.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/EditorFiles.cs rename to src/LanguageServer/Core/Impl/Implementation/EditorFiles.cs diff --git a/src/LanguageServer/Impl/Implementation/LineFormatter.cs b/src/LanguageServer/Core/Impl/Implementation/LineFormatter.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/LineFormatter.cs rename to src/LanguageServer/Core/Impl/Implementation/LineFormatter.cs diff --git a/src/LanguageServer/Impl/Implementation/ParseQueue.cs b/src/LanguageServer/Core/Impl/Implementation/ParseQueue.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/ParseQueue.cs rename to src/LanguageServer/Core/Impl/Implementation/ParseQueue.cs diff --git a/src/LanguageServer/Impl/Implementation/ProjectFiles.cs b/src/LanguageServer/Core/Impl/Implementation/ProjectFiles.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/ProjectFiles.cs rename to src/LanguageServer/Core/Impl/Implementation/ProjectFiles.cs diff --git a/src/LanguageServer/Impl/Implementation/RestTextConverter.cs b/src/LanguageServer/Core/Impl/Implementation/RestTextConverter.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/RestTextConverter.cs rename to src/LanguageServer/Core/Impl/Implementation/RestTextConverter.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.Completion.cs b/src/LanguageServer/Core/Impl/Implementation/Server.Completion.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.Completion.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.Completion.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.Extensions.cs b/src/LanguageServer/Core/Impl/Implementation/Server.Extensions.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.Extensions.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.Extensions.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.FindReferences.cs b/src/LanguageServer/Core/Impl/Implementation/Server.FindReferences.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.FindReferences.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.FindReferences.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.GoToDefinition.cs b/src/LanguageServer/Core/Impl/Implementation/Server.GoToDefinition.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.GoToDefinition.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.GoToDefinition.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.Hover.cs b/src/LanguageServer/Core/Impl/Implementation/Server.Hover.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.Hover.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.Hover.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.OnTypeFormatting.cs b/src/LanguageServer/Core/Impl/Implementation/Server.OnTypeFormatting.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.OnTypeFormatting.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.OnTypeFormatting.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.PrivateHelpers.cs b/src/LanguageServer/Core/Impl/Implementation/Server.PrivateHelpers.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.PrivateHelpers.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.PrivateHelpers.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.Rename.cs b/src/LanguageServer/Core/Impl/Implementation/Server.Rename.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.Rename.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.Rename.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.SignatureHelp.cs b/src/LanguageServer/Core/Impl/Implementation/Server.SignatureHelp.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.SignatureHelp.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.SignatureHelp.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Core/Impl/Implementation/Server.WorkspaceSymbols.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.WorkspaceSymbols.cs diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Core/Impl/Implementation/Server.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/Server.cs rename to src/LanguageServer/Core/Impl/Implementation/Server.cs diff --git a/src/LanguageServer/Impl/Implementation/ServerBase.cs b/src/LanguageServer/Core/Impl/Implementation/ServerBase.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/ServerBase.cs rename to src/LanguageServer/Core/Impl/Implementation/ServerBase.cs diff --git a/src/LanguageServer/Impl/Implementation/VolatileCounter.cs b/src/LanguageServer/Core/Impl/Implementation/VolatileCounter.cs similarity index 100% rename from src/LanguageServer/Impl/Implementation/VolatileCounter.cs rename to src/LanguageServer/Core/Impl/Implementation/VolatileCounter.cs diff --git a/src/LanguageServer/Core/Impl/Microsoft.Python.LanguageServer.Core.csproj b/src/LanguageServer/Core/Impl/Microsoft.Python.LanguageServer.Core.csproj new file mode 100644 index 000000000..631c3e87a --- /dev/null +++ b/src/LanguageServer/Core/Impl/Microsoft.Python.LanguageServer.Core.csproj @@ -0,0 +1,41 @@ + + + netstandard2.0 + Microsoft.Python.LanguageServer + Microsoft.Python.LanguageServer.Core + + + + 1701;1702;1998;$(NoWarn) + 7.2 + + + + + + all + runtime; build; native; contentfiles; analyzers + + + + + + + + True + True + Resources.resx + + + + + ResXFileCodeGenerator + Resources.Designer.cs + + + + + diff --git a/src/LanguageServer/Core/Impl/Properties/AssemblyInfo.cs b/src/LanguageServer/Core/Impl/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..1d7853194 --- /dev/null +++ b/src/LanguageServer/Core/Impl/Properties/AssemblyInfo.cs @@ -0,0 +1,21 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Microsoft.PythonTools.Analyzer, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("Microsoft.Python.LanguageServer, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("Microsoft.Python.Analysis.Engine.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/src/LanguageServer/Core/Impl/Resources.Designer.cs b/src/LanguageServer/Core/Impl/Resources.Designer.cs new file mode 100644 index 000000000..902a60fed --- /dev/null +++ b/src/LanguageServer/Core/Impl/Resources.Designer.cs @@ -0,0 +1,168 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace Microsoft.Python.LanguageServer { + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "15.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + internal class Resources { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal Resources() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("Microsoft.Python.LanguageServer.Resources", typeof(Resources).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to done.. + /// + internal static string Done { + get { + return ResourceManager.GetString("Done", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Failed to create interpreter. + /// + internal static string Error_FailedToCreateInterpreter { + get { + return ResourceManager.GetString("Error_FailedToCreateInterpreter", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Initializing for generic interpreter. + /// + internal static string InitializingForGenericInterpreter { + get { + return ResourceManager.GetString("InitializingForGenericInterpreter", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Initializing for {0}. + /// + internal static string InitializingForPythonInterpreter { + get { + return ResourceManager.GetString("InitializingForPythonInterpreter", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Microsoft Python Language Server version {0}. + /// + internal static string LanguageServerVersion { + get { + return ResourceManager.GetString("LanguageServerVersion", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Unmatched token '{0}' on line {1}; line formatting may not be accurate.. + /// + internal static string LineFormatter_UnmatchedToken { + get { + return ResourceManager.GetString("LineFormatter_UnmatchedToken", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Reloading modules... . + /// + internal static string ReloadingModules { + get { + return ResourceManager.GetString("ReloadingModules", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Cannot rename. + /// + internal static string RenameVariable_CannotRename { + get { + return ResourceManager.GetString("RenameVariable_CannotRename", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Cannot rename modules. + /// + internal static string RenameVariable_CannotRenameModuleName { + get { + return ResourceManager.GetString("RenameVariable_CannotRenameModuleName", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to No information is available for the variable '{0}'.. + /// + internal static string RenameVariable_NoInformationAvailableForVariable { + get { + return ResourceManager.GetString("RenameVariable_NoInformationAvailableForVariable", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Please select a symbol to be renamed.. + /// + internal static string RenameVariable_SelectSymbol { + get { + return ResourceManager.GetString("RenameVariable_SelectSymbol", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Unable to get analysis for the selected expression.. + /// + internal static string RenameVariable_UnableGetExpressionAnalysis { + get { + return ResourceManager.GetString("RenameVariable_UnableGetExpressionAnalysis", resourceCulture); + } + } + } +} diff --git a/src/LanguageServer/Core/Impl/Resources.resx b/src/LanguageServer/Core/Impl/Resources.resx new file mode 100644 index 000000000..6d9943476 --- /dev/null +++ b/src/LanguageServer/Core/Impl/Resources.resx @@ -0,0 +1,137 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 1.3 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=2.0.3500.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=2.0.3500.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + done. + + + Failed to create interpreter + + + Initializing for generic interpreter + + + Initializing for {0} + + + Microsoft Python Language Server version {0} + + + Unmatched token '{0}' on line {1}; line formatting may not be accurate. + + + Reloading modules... + + + Cannot rename + + + Cannot rename modules + + + No information is available for the variable '{0}'. + + + Please select a symbol to be renamed. + + + Unable to get analysis for the selected expression. + + \ No newline at end of file diff --git a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj index 77702d03c..ab661828f 100644 --- a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj +++ b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj @@ -33,6 +33,7 @@ + diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 7485d60bf..7b5213588 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -29,6 +29,7 @@ namespace Microsoft.Python.LanguageServer.Server { internal static class Program { public static void Main(string[] args) { CheckDebugMode(); + Debugger.Launch(); using (CoreShell.Create()) { var services = CoreShell.Current.ServiceManager; diff --git a/src/LanguageServer/Impl/Resources.Designer.cs b/src/LanguageServer/Impl/Resources.Designer.cs index 33c50439f..5f9eb5335 100644 --- a/src/LanguageServer/Impl/Resources.Designer.cs +++ b/src/LanguageServer/Impl/Resources.Designer.cs @@ -60,15 +60,6 @@ internal Resources() { } } - /// - /// Looks up a localized string similar to done.. - /// - internal static string Done { - get { - return ResourceManager.GetString("Done", resourceCulture); - } - } - /// /// Looks up a localized string similar to Failed to create interpreter. /// @@ -77,95 +68,5 @@ internal static string Error_FailedToCreateInterpreter { return ResourceManager.GetString("Error_FailedToCreateInterpreter", resourceCulture); } } - - /// - /// Looks up a localized string similar to Initializing for generic interpreter. - /// - internal static string InitializingForGenericInterpreter { - get { - return ResourceManager.GetString("InitializingForGenericInterpreter", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Initializing for {0}. - /// - internal static string InitializingForPythonInterpreter { - get { - return ResourceManager.GetString("InitializingForPythonInterpreter", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Microsoft Python Language Server version {0}. - /// - internal static string LanguageServerVersion { - get { - return ResourceManager.GetString("LanguageServerVersion", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Unmatched token '{0}' on line {1}; line formatting may not be accurate.. - /// - internal static string LineFormatter_UnmatchedToken { - get { - return ResourceManager.GetString("LineFormatter_UnmatchedToken", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Reloading modules... . - /// - internal static string ReloadingModules { - get { - return ResourceManager.GetString("ReloadingModules", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Cannot rename. - /// - internal static string RenameVariable_CannotRename { - get { - return ResourceManager.GetString("RenameVariable_CannotRename", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Cannot rename modules. - /// - internal static string RenameVariable_CannotRenameModuleName { - get { - return ResourceManager.GetString("RenameVariable_CannotRenameModuleName", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to No information is available for the variable '{0}'.. - /// - internal static string RenameVariable_NoInformationAvailableForVariable { - get { - return ResourceManager.GetString("RenameVariable_NoInformationAvailableForVariable", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Please select a symbol to be renamed.. - /// - internal static string RenameVariable_SelectSymbol { - get { - return ResourceManager.GetString("RenameVariable_SelectSymbol", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Unable to get analysis for the selected expression.. - /// - internal static string RenameVariable_UnableGetExpressionAnalysis { - get { - return ResourceManager.GetString("RenameVariable_UnableGetExpressionAnalysis", resourceCulture); - } - } } } diff --git a/src/LanguageServer/Impl/Resources.resx b/src/LanguageServer/Impl/Resources.resx index 630472020..f78a79bc5 100644 --- a/src/LanguageServer/Impl/Resources.resx +++ b/src/LanguageServer/Impl/Resources.resx @@ -117,40 +117,7 @@ System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - done. - Failed to create interpreter - - Initializing for generic interpreter - - - Initializing for {0} - - - Microsoft Python Language Server version {0} - - - Unmatched token '{0}' on line {1}; line formatting may not be accurate. - - - Reloading modules... - - - Cannot rename - - - Cannot rename modules - - - No information is available for the variable '{0}'. - - - Please select a symbol to be renamed. - - - Unable to get analysis for the selected expression. - \ No newline at end of file diff --git a/src/PLS.sln b/src/PLS.sln index a5f064534..d0f54ef1c 100644 --- a/src/PLS.sln +++ b/src/PLS.sln @@ -15,6 +15,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Analysis.E EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Analysis.Engine.Tests", "Analysis\Engine\Test\Microsoft.Python.Analysis.Engine.Tests.csproj", "{1CFA416B-6932-432F-8C75-34B5615D7664}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.PythonTools.Analyzer", "PTVS\Microsoft.PythonTools.Analyzer\Microsoft.PythonTools.Analyzer.csproj", "{467679B2-D043-4C7D-855C-5A833B635EEE}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.PythonTools.Ipc.Json", "PTVS\Microsoft.PythonTools.Ipc.Json\Microsoft.PythonTools.Ipc.Json.csproj", "{A5E04CC1-F600-45BA-B493-F7DCBA8C7E1A}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.LanguageServer.Core", "LanguageServer\Core\Impl\Microsoft.Python.LanguageServer.Core.csproj", "{0EA1F1C3-2733-423C-BEC5-059BD77F0A3F}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "PTVS", "PTVS", "{99A92748-7947-412A-BA92-5F4E2AA63918}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -37,6 +45,18 @@ Global {1CFA416B-6932-432F-8C75-34B5615D7664}.Debug|Any CPU.Build.0 = Debug|Any CPU {1CFA416B-6932-432F-8C75-34B5615D7664}.Release|Any CPU.ActiveCfg = Release|Any CPU {1CFA416B-6932-432F-8C75-34B5615D7664}.Release|Any CPU.Build.0 = Release|Any CPU + {467679B2-D043-4C7D-855C-5A833B635EEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {467679B2-D043-4C7D-855C-5A833B635EEE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {467679B2-D043-4C7D-855C-5A833B635EEE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {467679B2-D043-4C7D-855C-5A833B635EEE}.Release|Any CPU.Build.0 = Release|Any CPU + {A5E04CC1-F600-45BA-B493-F7DCBA8C7E1A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A5E04CC1-F600-45BA-B493-F7DCBA8C7E1A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A5E04CC1-F600-45BA-B493-F7DCBA8C7E1A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A5E04CC1-F600-45BA-B493-F7DCBA8C7E1A}.Release|Any CPU.Build.0 = Release|Any CPU + {0EA1F1C3-2733-423C-BEC5-059BD77F0A3F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0EA1F1C3-2733-423C-BEC5-059BD77F0A3F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0EA1F1C3-2733-423C-BEC5-059BD77F0A3F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0EA1F1C3-2733-423C-BEC5-059BD77F0A3F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -46,6 +66,9 @@ Global {B1F6F2EA-6465-4D63-9457-D856F17EDF38} = {80AA38A1-3E82-4B87-BB21-FDEDD2CC87E6} {55679AE9-8C3D-4F26-A0B5-10A5B2F1789F} = {C465393D-145E-4695-A7DB-AF55951BD533} {1CFA416B-6932-432F-8C75-34B5615D7664} = {80AA38A1-3E82-4B87-BB21-FDEDD2CC87E6} + {467679B2-D043-4C7D-855C-5A833B635EEE} = {99A92748-7947-412A-BA92-5F4E2AA63918} + {A5E04CC1-F600-45BA-B493-F7DCBA8C7E1A} = {99A92748-7947-412A-BA92-5F4E2AA63918} + {0EA1F1C3-2733-423C-BEC5-059BD77F0A3F} = {C465393D-145E-4695-A7DB-AF55951BD533} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {ABC12ED7-0EC8-4219-8A14-A058F7942D92} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Analyzer.csproj b/src/PTVS/Microsoft.PythonTools.Analyzer/Analyzer.csproj new file mode 100644 index 000000000..412fc0e83 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Analyzer.csproj @@ -0,0 +1,125 @@ + + + + + + 15.0 + + + + + 14.0 + + + + + 16.0 + + + + + 16.0 + + + + + + Debug + x86 + 8.0.30703 + 2.0 + {82b43b9b-a64c-4715-b499-d71e9ca2bd60};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + {29A4FA1F-A562-4ED1-86FB-5850EF5DA92C} + Exe + Properties + Microsoft.PythonTools.Analysis + Microsoft.PythonTools.Analyzer + 512 + true + + + x86 + + + ..\Icons\Dev$(VSTarget)\PythonProject.ico + + + + $(PackagesPath)\Newtonsoft.Json\lib\net45\Newtonsoft.Json.dll + True + + + + + + + + + + + + + + + + DebugTimer.cs + + + + + + + + + + + + + + + + + + + {A85D479D-67A9-4BDB-904A-7D86DAF68A6F} + Microsoft.PythonTools.Analysis + + + {e1e1613d-0c96-42f9-9f2d-052c72533297} + Ipc.Json + + + + + PreserveNewest + True + + + PreserveNewest + True + + + PreserveNewest + True + + + PreserveNewest + True + + + + + + + + + + + + \ No newline at end of file diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/AnalyzerScrapers.pyproj b/src/PTVS/Microsoft.PythonTools.Analyzer/AnalyzerScrapers.pyproj new file mode 100644 index 000000000..ba7bd6733 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/AnalyzerScrapers.pyproj @@ -0,0 +1,41 @@ + + + + Debug + 2.0 + {1fc96711-4e42-4c41-b7fc-b4141d57d0bb} + + PythonScraper.py + + . + . + {888888a0-9f3d-457c-b088-3a5042f75d52} + Standard Python launcher + {2af0f10d-7135-4994-9156-5d01c9c11b7e} + 2.7 + "C:\Users\steve_000\AppData\Local\Python Tools\CompletionDB\Debug\12.0\2af0f10d-7135-4994-9156-5d01c9c11b7e\2.7" "C:\USERS\STEVE_000\APPDATA\LOCAL\MICROSOFT\VISUALSTUDIO\12.0EXP\EXTENSIONS\MICROSOFT\PYTHON TOOLS FOR VISUAL STUDIO\2.1\CompletionDB" + False + + + + + 10.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets + + + + + Code + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/AnalyzerScrapers.sln b/src/PTVS/Microsoft.PythonTools.Analyzer/AnalyzerScrapers.sln new file mode 100644 index 000000000..c5f284c7a --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/AnalyzerScrapers.sln @@ -0,0 +1,20 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2013 +VisualStudioVersion = 12.0.30501.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "AnalyzerScrapers", "AnalyzerScrapers.pyproj", "{1FC96711-4E42-4C41-B7FC-B4141D57D0BB}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {1FC96711-4E42-4C41-B7FC-B4141D57D0BB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1FC96711-4E42-4C41-B7FC-B4141D57D0BB}.Release|Any CPU.ActiveCfg = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/App.config b/src/PTVS/Microsoft.PythonTools.Analyzer/App.config new file mode 100644 index 000000000..fb43a1004 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/App.config @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/AssemblyInfo.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/AssemblyInfo.cs new file mode 100644 index 000000000..9f4e620c0 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/AssemblyInfo.cs @@ -0,0 +1,23 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("Visual Studio - Python background analyzer")] +[assembly: AssemblyDescription("Performs analysis of the Python standard library and installed site packages.")] + +[assembly: ComVisible(false)] \ No newline at end of file diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/BuiltinScraper.py b/src/PTVS/Microsoft.PythonTools.Analyzer/BuiltinScraper.py new file mode 100644 index 000000000..3ec651b36 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/BuiltinScraper.py @@ -0,0 +1,554 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABILITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +import re +import sys +import types +import PythonScraper +try: + import thread +except: + import _thread as thread + +try: + import __builtin__ as __builtins__ +except ImportError: + import builtins as __builtins__ + +def safe_dir(obj): + try: + return frozenset(obj.__dict__) | frozenset(dir(obj)) + except: + # Some types crash when we access __dict__ and/or dir() + pass + try: + return frozenset(dir(obj)) + except: + pass + try: + return frozenset(obj.__dict__) + except: + pass + return frozenset() + +def builtins_keys(): + if isinstance(__builtins__, dict): + return __builtins__.keys() + return dir(__builtins__) + +def get_builtin(name): + if isinstance(__builtins__, dict): + return __builtins__[name] + + return getattr(__builtins__, name) + +safe_getattr = PythonScraper.safe_getattr + +BUILTIN_TYPES = [type_name for type_name in builtins_keys() if type(get_builtin(type_name)) is type] +if sys.version_info[0] >= 3: + BUILTIN = 'builtins' + unicode = str +else: + BUILTIN = '__builtin__' + +TYPE_OVERRIDES = { + 'string': PythonScraper.type_to_typeref(types.CodeType), + 's': PythonScraper.type_to_typeref(str), + 'integer': PythonScraper.type_to_typeref(int), + 'boolean': PythonScraper.type_to_typeref(bool), + 'number': PythonScraper.type_to_typeref(int), + 'pid': PythonScraper.type_to_typeref(int), + 'ppid': PythonScraper.type_to_typeref(int), + 'fd': PythonScraper.type_to_typeref(int), + 'handle': PythonScraper.type_to_typeref(int), + 'Exit': PythonScraper.type_to_typeref(int), + 'fd2': PythonScraper.type_to_typeref(int), + 'Integral': PythonScraper.type_to_typeref(int), + 'exit_status':PythonScraper.type_to_typeref(int), + 'old_mask': PythonScraper.type_to_typeref(int), + 'source': PythonScraper.type_to_typeref(str), + 'newpos': PythonScraper.type_to_typeref(int), + 'key': PythonScraper.type_to_typeref(str), + 'dictionary': PythonScraper.type_to_typeref(dict), + 'None': PythonScraper.type_to_typeref(type(None)), + 'floating': PythonScraper.type_to_typeref(float), + 'filename': PythonScraper.type_to_typeref(str), + 'path': PythonScraper.type_to_typeref(str), + 'byteswritten': PythonScraper.type_to_typeref(int), + 'unicode': PythonScraper.type_to_typeref(unicode), + 'Unicode': PythonScraper.type_to_typeref(unicode), + 'True': PythonScraper.type_to_typeref(bool), + 'False': PythonScraper.type_to_typeref(bool), + 'lock': PythonScraper.type_to_typeref(thread.LockType), + 'code': PythonScraper.type_to_typeref(types.CodeType), + 'module': PythonScraper.type_to_typeref(types.ModuleType), + 'size': PythonScraper.type_to_typeref(int), + 'INT': PythonScraper.type_to_typeref(int), + 'STRING': PythonScraper.type_to_typeref(str), + 'TUPLE': PythonScraper.type_to_typeref(tuple), + 'OBJECT': PythonScraper.type_to_typeref(object), + 'LIST': PythonScraper.type_to_typeref(list), + 'DICT': PythonScraper.type_to_typeref(dict), + 'char *': PythonScraper.type_to_typeref(str), + 'wchar_t *': PythonScraper.type_to_typeref(unicode), + 'CHAR *': PythonScraper.type_to_typeref(str), + 'TCHAR *': PythonScraper.type_to_typeref(str), + 'WCHAR *': PythonScraper.type_to_typeref(unicode), + 'LPSTR': PythonScraper.type_to_typeref(str), + 'LPCSTR': PythonScraper.type_to_typeref(str), + 'LPTSTR': PythonScraper.type_to_typeref(str), + 'LPCTSTR': PythonScraper.type_to_typeref(str), + 'LPWSTR': PythonScraper.type_to_typeref(unicode), + 'LPCWSTR': PythonScraper.type_to_typeref(unicode), +} + +try: + TYPE_OVERRIDES['file object'] = PythonScraper.type_to_typeref(file) +except NameError: + try: + import _io + TYPE_OVERRIDES['file object'] = PythonScraper.type_to_typeref(_io._IOBase) + except (NameError, ImportError): + pass + +RETURN_TYPE_OVERRIDES = dict(TYPE_OVERRIDES) +RETURN_TYPE_OVERRIDES.update({'string': PythonScraper.type_to_typeref(str)}) + +def type_name_to_typeref(name, mod, type_overrides = TYPE_OVERRIDES): + arg_type = type_overrides.get(name, None) + if arg_type is None: + if name in BUILTIN_TYPES: + arg_type = PythonScraper.type_to_typeref(get_builtin(name)) + elif mod is not None and name in mod.__dict__: + arg_type = PythonScraper.typename_to_typeref(mod.__name__, name) + elif name.startswith('list'): + arg_type = PythonScraper.type_to_typeref(list) + else: + # see if we can find it in any module we've imported... + for mod_name, mod in list(sys.modules.items()): + if mod is not None and name in mod.__dict__ and isinstance(mod.__dict__[name], type): + arg_type = PythonScraper.typename_to_typeref(mod_name, name) + break + else: + first_space = name.find(' ') + if first_space != -1: + return type_name_to_typeref(name[:first_space], mod, type_overrides) + arg_type = PythonScraper.typename_to_typeref(name) + return arg_type + +OBJECT_TYPE = PythonScraper.type_to_typeref(object) + +TOKENS_REGEX = '(' + '|'.join([ + r'(?:[a-zA-Z_][0-9a-zA-Z_-]*)', # identifier + r'(?:-?[0-9]+[lL]?(?!\.))', # integer value + r'(?:-?[0-9]*\.[0-9]+)', # floating point value + r'(?:-?[0-9]+\.[0-9]*)', # floating point value + r'(?:\s*\'.*?(?)', # return value + r'(?:->)', # return value + r'(?:=>)', # return value + r'(?:,)', # comma + r'(?:=)', # assignment (default value) + r'(?:\[)', + r'(?:\])', + r'(?:\*\*)', + r'(?:\*)', +]) + ')' + +def get_ret_type(ret_type, obj_class, mod): + if ret_type is not None: + if ret_type == 'copy' and obj_class is not None: + # returns a copy of self + return PythonScraper.type_to_typelist(obj_class) + else: + return [type_name_to_typeref(ret_type, mod, RETURN_TYPE_OVERRIDES)] + + +RETURNS_REGEX = [r'^\s*returns?[\s\-]*[a-z_]\w*\s*:\s*([a-z_]\w*)'] + +def update_overload_from_doc_str(overload, doc_str, obj_class, mod): + # see if we can get additional information from the doc string + if 'ret_type' not in overload: + for ret_regex in RETURNS_REGEX: + match = re.search(ret_regex, doc_str, re.MULTILINE | re.IGNORECASE) + if match: + ret_type = match.groups(0)[0] + overload['ret_type'] = get_ret_type(ret_type, obj_class, mod) + break + + +def parse_doc_str(input_str, module_name, mod, func_name, extra_args = [], obj_class = None): + # we split, so as long as we have all tokens every other item is a token, and the + # rest are empty space. If we have unrecognized tokens (for example during the description + # of the function) those will show up in the even locations. We do join's and bring the + # entire range together in that case. + tokens = re.split(TOKENS_REGEX, input_str) + start_token = 0 + last_identifier = None + cur_token = 1 + overloads = [] + while cur_token < len(tokens): + token = tokens[cur_token] + # see if we have modname.funcname( + if (cur_token + 10 < len(tokens) and + token == module_name and + tokens[cur_token + 2] == '.' and + tokens[cur_token + 4] == func_name and + tokens[cur_token + 6] == '('): + sig_start = cur_token + args, ret_type, cur_token = parse_args(tokens, cur_token + 8, mod) + + doc_str = ''.join(tokens[start_token:sig_start]) + if doc_str.find(' ') == -1: + doc_str = '' + if (not args or doc_str) and overloads: + # if we already parsed an overload, and are now getting an argless + # overload we're likely just seeing a reference to the function in + # a doc string, let's ignore that. This is betting on the idea that + # people list overloads first, then doc strings, and that people tend + # to list overloads from simplest to more complex. an example of this + # is the future_builtins.ascii doc string + # We also skip it if we have a doc string, this comes up in overloads + # like isinstance which has example calls embedded in the doc string + continue + + start_token = cur_token + overload = {'args': tuple(extra_args + args), 'doc': doc_str} + ret_types = get_ret_type(ret_type, obj_class, mod) + if ret_types is not None: + overload['ret_type'] = ret_types + update_overload_from_doc_str(overload, doc_str, obj_class, mod) + overloads.append(overload) + # see if we have funcname( + elif (cur_token + 4 < len(tokens) and + token == func_name and + tokens[cur_token + 2] == '('): + sig_start = cur_token + args, ret_type, cur_token = parse_args(tokens, cur_token + 4, mod) + + doc_str = ''.join(tokens[start_token:sig_start]) + if doc_str.find(' ') == -1: + doc_str = '' + if (not args or doc_str) and overloads: + # if we already parsed an overload, and are now getting an argless + # overload we're likely just seeing a reference to the function in + # a doc string, let's ignore that. This is betting on the idea that + # people list overloads first, then doc strings, and that people tend + # to list overloads from simplest to more complex. an example of this + # is the future_builtins.ascii doc string + # We also skip it if we have a doc string, this comes up in overloads + # like isinstance which has example calls embedded in the doc string + continue + + start_token = cur_token + overload = {'args': tuple(extra_args + args), 'doc': doc_str} + ret_types = get_ret_type(ret_type, obj_class, mod) + if ret_types is not None: + overload['ret_type'] = ret_types + update_overload_from_doc_str(overload, doc_str, obj_class, mod) + overloads.append(overload) + + else: + # append to doc string + cur_token += 2 + + finish_doc = ''.join(tokens[start_token:cur_token]) + if finish_doc: + if not overloads: + # This occurs when the docstring does not include a function spec + overloads.append({ + 'args': ({'name': 'args', 'arg_format': '*'}, {'name': 'kwargs', 'arg_format': '**'}), + 'doc': '' + }) + for overload in overloads: + overload['doc'] += finish_doc + update_overload_from_doc_str(overload, overload['doc'], obj_class, mod) + + return overloads + + +IDENTIFIER_REGEX = re.compile('^[a-zA-Z_][a-zA-Z_0-9-]*$') + +def is_identifier(token): + if IDENTIFIER_REGEX.match(token): + return True + return False + +RETURN_TOKENS = set(['-->', '->', '=>', 'return']) + +def parse_args(tokens, cur_token, module): + args = [] + + arg = [] + annotation = None + default_value = None + ignore = False + arg_tokens = [] + next_is_optional = False + is_optional = False + paren_nesting = 0 + while cur_token < len(tokens): + token = tokens[cur_token] + cur_token += 1 + + if token in (',', ')') and paren_nesting == 0: + arg_tokens.append((arg, annotation, default_value, is_optional)) + is_optional = False + arg = [] + annotation = None + default_value = None + if token == ')': + cur_token += 1 + break + elif ignore: + continue + elif token == '=': + if default_value is None: + default_value = [] + else: + ignore = True + elif token == ':': + if annotation is None and default_value is None: + annotation = [] + else: + ignore = True + elif default_value is not None: + default_value.append(token) + elif annotation is not None: + annotation.append(token) + elif token == '[': + next_is_optional = True + elif token in (']', ' ', ''): + pass + else: + arg.append(token) + if next_is_optional: + is_optional, next_is_optional = True, False + + if token == '(': + paren_nesting += 1 + elif token == ')': + paren_nesting -= 1 + + #from pprint import pprint; pprint(arg_tokens) + + for arg, annotation, default_value, is_optional in arg_tokens: + if not arg or arg[0] == '/': + continue + + arg_name = None + star_args = None + + if arg[0] == '(': + names = [arg.pop(0)] + while names[-1] != ')' and arg: + names.append(arg.pop(0)) + if names[-1] == ')': + names.pop() + arg_name = ', '.join(n for n in names[1:] if is_identifier(n)) + elif is_identifier(arg[-1]): + arg_name = arg.pop() + elif arg[-1] == '...': + arg_name = 'args' + star_args = '*' + + if not annotation and arg: + if len(arg) > 1 and arg[-1] == '*': + # C style prototype + annotation = [' '.join(a for a in arg if a != 'const')] + elif is_identifier(arg[-1]): + annotation = arg[-1:] + elif arg[-1] == ')': + annotation = [arg.pop()] + while annotation[0] != '(': + annotation.insert(0, arg.pop()) + + if arg and arg[0] in ('*', '**'): + star_args = arg[0] + + data = { } + + if arg_name: + data['name'] = arg_name + elif star_args == '*': + data['name'] = 'args' + elif star_args == '**': + data['name'] = 'kwargs' + else: + data['name'] = 'arg' + + if annotation and len(annotation) == 1: + data['type'] = [type_name_to_typeref(annotation[0], module)] + + if default_value: + default_value = [d for d in default_value if d] + + if is_optional and default_value[-1] == ']': + default_value.pop() + + data['default_value'] = ''.join(default_value).strip() + elif is_optional: + data['default_value'] = 'None' + + if star_args: + data['arg_format'] = star_args + + args.append(data) + + + # end of params, check for ret value + ret_type = None + + if cur_token + 2 < len(tokens) and tokens[cur_token] in RETURN_TOKENS: + ret_type_start = cur_token + 2 + # we might have a descriptive return value, 'list of fob' + while ret_type_start < len(tokens) and is_identifier(tokens[ret_type_start]): + if tokens[ret_type_start - 1].find('\n') != -1: + break + ret_type_start += 2 + + if ret_type_start < len(tokens) and ',' in tokens[ret_type_start]: + # fob(oar, baz) -> some info about the return, and more info, and more info. + # "some info" is unlikely to be a return type + ret_type = '' + cur_token += 2 + else: + ret_type = ''.join(tokens[cur_token + 2:ret_type_start]).strip() + cur_token = ret_type_start + elif (cur_token + 4 < len(tokens) and + tokens[cur_token] == ':' and tokens[cur_token + 2] in RETURN_TOKENS): + ret_type_start = cur_token + 4 + # we might have a descriptive return value, 'list of fob' + while ret_type_start < len(tokens) and is_identifier(tokens[ret_type_start]): + if tokens[ret_type_start - 1].find('\n') != -1: + break + ret_type_start += 2 + + if ret_type_start < len(tokens) and ',' in tokens[ret_type_start]: + # fob(oar, baz) -> some info about the return, and more info, and more info. + # "some info" is unlikely to be a return type + ret_type = '' + cur_token += 4 + else: + ret_type = ''.join(tokens[cur_token + 4:ret_type_start]).strip() + cur_token = ret_type_start + + return args, ret_type, cur_token + + +if sys.version > '3.': + str_types = (str, bytes) +else: + str_types = (str, unicode) + + +def get_overloads_from_doc_string(doc_str, mod, obj_class, func_name, extra_args = []): + if isinstance(doc_str, str_types): + decl_mod = None + if isinstance(mod, types.ModuleType): + decl_mod = mod + mod = decl_mod.__name__ + elif mod is not None: + decl_mod = sys.modules.get(mod, None) + + res = parse_doc_str(doc_str, mod, decl_mod, func_name, extra_args, obj_class) + if res: + for i, v in enumerate(res): + if 'ret_type' not in v or (not v['ret_type'] or v['ret_type'] == ('', '')): + alt_ret_type = v['doc'].find('returned as a ') + if alt_ret_type != -1: + last_space = v['doc'].find(' ', alt_ret_type + 14) + last_new_line = v['doc'].find('\n', alt_ret_type + 14) + if last_space == -1: + if last_new_line == -1: + last_space = None + else: + last_space = last_new_line + elif last_new_line == -1: + last_space = None + else: + last_space = last_new_line + + ret_type_str = v['doc'][alt_ret_type+14:last_space] + if ret_type_str.endswith('.') or ret_type_str.endswith(','): + ret_type_str = ret_type_str[:-1] + new_ret_type = get_ret_type(ret_type_str, obj_class, decl_mod) + res[i]['ret_type'] = new_ret_type + + return res + return None + + +def get_overloads(func, is_method = False): + if is_method: + extra_args = [{'type': PythonScraper.type_to_typelist(object), 'name': 'self'}] + else: + extra_args = [] + + func_doc = safe_getattr(func, '__doc__', None) + if not func_doc: + return None + + return get_overloads_from_doc_string( + func_doc, + safe_getattr(func, '__module__', None), + safe_getattr(func, '__objclass__', None), + safe_getattr(func, '__name__', None), + extra_args, + ) + +def get_descriptor_type(descriptor): + return object + +def get_new_overloads(type_obj, obj): + try: + type_doc = safe_getattr(type_obj, '__doc__', None) + type_type = type(type_obj) + except: + return None + + res = get_overloads_from_doc_string( + type_doc, + safe_getattr(type_obj, '__module__', None), + type_type, + safe_getattr(type_obj, '__name__', None), + [{'type': PythonScraper.type_to_typelist(type), 'name': 'cls'}], + ) + + if not res: + obj_doc = safe_getattr(obj, '__doc__', None) + if not obj_doc: + return None + res = get_overloads_from_doc_string( + obj_doc, + safe_getattr(type_obj, '__module__', None), + type_type, + safe_getattr(type_obj, '__name__', None), + ) + + return res + +def should_include_module(name): + return True diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/BuiltinScraperTests.py b/src/PTVS/Microsoft.PythonTools.Analyzer/BuiltinScraperTests.py new file mode 100644 index 000000000..67ef67d51 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/BuiltinScraperTests.py @@ -0,0 +1,491 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABILITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +import re +import unittest +from pprint import pformat +from BuiltinScraper import parse_doc_str, BUILTIN, __builtins__, get_overloads_from_doc_string, TOKENS_REGEX + +try: + unicode +except NameError: + from BuiltinScraper import unicode +import sys + +class Test_BuiltinScraperTests(unittest.TestCase): + def check_doc_str(self, doc, module_name, func_name, expected, mod=None, extra_args=[], obj_class=None): + r = parse_doc_str(doc, module_name, mod, func_name, extra_args, obj_class) + + # Quick pass if everything matches + if r == expected: + return + + msg = 'Expected:\n%s\nActual\n%s' % (pformat(expected), pformat(r)) + + self.assertEqual(len(r), len(expected), msg) + + def check_dict(e, a, indent): + if e == a: + return + missing_keys = set(e.keys()) - set(a.keys()) + extra_keys = set(a.keys()) - set(e.keys()) + mismatched_keys = [k for k in set(a.keys()) & set(e.keys()) if a[k] != e[k]] + if missing_keys: + print('%sDid not received following keys: %s' % (indent, ', '.join(missing_keys))) + if extra_keys: + print('%sDid not expect following keys: %s' % (indent, ', '.join(extra_keys))) + for k in mismatched_keys: + if isinstance(e[k], dict) and isinstance(a[k], dict): + check_dict(e[k], a[k], indent + ' ') + elif (isinstance(e[k], tuple) and isinstance(a[k], tuple) or isinstance(e[k], list) and isinstance(a[k], list)): + check_seq(e[k], a[k], indent + ' ') + else: + print('%sExpected "%s": "%s"' % (indent, k, e[k])) + print('%sActual "%s": "%s"' % (indent, k, a[k])) + print('') + + def check_seq(e, a, indent): + if e == a: + return + for i, (e2, a2) in enumerate(zip(e, a)): + if isinstance(e2, dict) and isinstance(a2, dict): + check_dict(e2, a2, indent + ' ') + elif (isinstance(e2, tuple) and isinstance(a2, tuple) or isinstance(e2, list) and isinstance(a2, list)): + check_seq(e2, a2, indent + ' ') + elif e1 != a1: + print('%sExpected "%s"' % (indent, e2)) + print('%sActual "%s"' % (indent, a2)) + print('') + + for e1, a1 in zip(expected, r): + check_dict(e1, a1, '') + self.fail(msg) + + def test_regex(self): + self.assertSequenceEqual( + [i.strip() for i in re.split(TOKENS_REGEX, 'f(\'\', \'a\', \'a\\\'b\', "", "a", "a\\\"b")') if i.strip()], + ['f', '(', "''", ',', "'a'", ',', "'a\\'b'", ',', '""', ',', '"a"', ',', '"a\\"b"', ')'] + ) + self.assertSequenceEqual( + [i.strip() for i in re.split(TOKENS_REGEX, 'f(1, 1., -1, -1.)') if i.strip()], + ['f', '(', '1', ',', '1.', ',', '-1', ',', '-1.', ')'] + ) + self.assertSequenceEqual( + [i.strip() for i in re.split(TOKENS_REGEX, 'f(a, *a, **a, ...)') if i.strip()], + ['f', '(', 'a', ',', '*', 'a', ',', '**', 'a', ',', '...', ')'] + ) + self.assertSequenceEqual( + [i.strip() for i in re.split(TOKENS_REGEX, 'f(a:123, a=123) --> => ->') if i.strip()], + ['f', '(', 'a', ':', '123', ',', 'a', '=', '123', ')', '-->', '=>', '->'] + ) + + + def test_numpy_1(self): + self.check_doc_str( + """arange([start,] stop[, step,], dtype=None) + + Returns + ------- + out : ndarray""", + 'numpy', + 'arange', + [{ + 'doc': 'Returns\n -------\n out : ndarray', + 'ret_type': [('', 'ndarray')], + 'args': ( + {'name': 'start', 'default_value':'None'}, + {'name': 'stop'}, + {'name': 'step', 'default_value': 'None'}, + {'name': 'dtype', 'default_value':'None'}, + ) + }] + ) + + def test_numpy_2(self): + self.check_doc_str( + """arange([start,] stop[, step,], dtype=None) + + Return - out : ndarray""", + 'numpy', + 'arange', + [{ + 'doc': 'Return - out : ndarray', + 'ret_type': [('', 'ndarray')], + 'args': ( + {'name': 'start', 'default_value':'None'}, + {'name': 'stop'}, + {'name': 'step', 'default_value': 'None'}, + {'name': 'dtype', 'default_value':'None'}, + ) + }] + ) + + def test_reduce(self): + self.check_doc_str( + 'reduce(function, sequence[, initial]) -> value', + BUILTIN, + 'reduce', + mod=__builtins__, + expected = [{ + 'args': ( + {'name': 'function'}, + {'name': 'sequence'}, + {'default_value': 'None', 'name': 'initial'} + ), + 'doc': '', + 'ret_type': [('', 'value')] + }] + ) + + def test_pygame_draw_arc(self): + self.check_doc_str( + 'pygame.draw.arc(Surface, color, Rect, start_angle, stop_angle, width=1): return Rect', + 'draw', + 'arc', + [{ + 'args': ( + {'name': 'Surface'}, + {'name': 'color'}, + {'name': 'Rect'}, + {'name': 'start_angle'}, + {'name': 'stop_angle'}, + {'default_value': '1', 'name': 'width'} + ), + 'doc': '', + 'ret_type': [('', 'Rect')] + }] + ) + + def test_isdigit(self): + self.check_doc_str( + '''B.isdigit() -> bool + +Return True if all characters in B are digits +and there is at least one character in B, False otherwise.''', + 'bytes', + 'isdigit', + [{ + 'args': (), + 'doc': 'Return True if all characters in B are digits\nand there is at least one character in B, False otherwise.', + 'ret_type': [(BUILTIN, 'bool')] + }] + ) + + def test_init(self): + self.check_doc_str( + 'x.__init__(...) initializes x; see help(type(x)) for signature', + 'str', + '__init__', + [{'args': ({'arg_format': '*', 'name': 'args'},), 'doc': 'initializes x; see help(type(x)) for signature'}] + ) + + def test_find(self): + self.check_doc_str( + 'S.find(sub [,start [,end]]) -> int', + 'str', + 'find', + [{ + 'args': ( + {'name': 'sub'}, + {'default_value': 'None', 'name': 'start'}, + {'default_value': 'None', 'name': 'end'} + ), + 'doc': '', + 'ret_type': [(BUILTIN, 'int')] + }] + ) + + def test_format(self): + self.check_doc_str( + 'S.format(*args, **kwargs) -> unicode', + 'str', + 'format', + [{ + 'args': ( + {'arg_format': '*', 'name': 'args'}, + {'arg_format': '**', 'name': 'kwargs'} + ), + 'doc': '', + 'ret_type': [(BUILTIN, unicode.__name__)] + }] + ) + + def test_ascii(self): + self.check_doc_str( + "'ascii(object) -> string\n\nReturn the same as repr(). In Python 3.x, the repr() result will\\ncontain printable characters unescaped, while the ascii() result\\nwill have such characters backslash-escaped.'", + 'future_builtins', + 'ascii', + [{ + 'args': ({'name': 'object'},), + 'doc': "Return the same as repr(). In Python 3.x, the repr() result will\\ncontain printable characters unescaped, while the ascii() result\\nwill have such characters backslash-escaped.'", + 'ret_type': [(BUILTIN, 'str')] + }] + ) + + def test_preannotation(self): + self.check_doc_str( + 'f(INT class_code) => SpaceID', + 'fob', + 'f', + [{ + 'args': ({'name': 'class_code', 'type': [(BUILTIN, 'int')]},), + 'doc': '', + 'ret_type': [('', 'SpaceID')] + }]) + + def test_compress(self): + self.check_doc_str( + 'compress(data, selectors) --> iterator over selected data\n\nReturn data elements', + 'itertools', + 'compress', + [{ + 'args': ({'name': 'data'}, {'name': 'selectors'}), + 'doc': 'Return data elements', + 'ret_type': [('', 'iterator')] + }] + ) + + def test_isinstance(self): + self.check_doc_str( + 'isinstance(object, class-or-type-or-tuple) -> bool\n\nReturn whether an object is an ' + 'instance of a class or of a subclass thereof.\nWith a type as second argument, ' + 'return whether that is the object\'s type.\nThe form using a tuple, isinstance(x, (A, B, ...)),' + ' is a shortcut for\nisinstance(x, A) or isinstance(x, B) or ... (etc.).', + BUILTIN, + 'isinstance', + [{ + 'args': ({'name': 'object'}, {'name': 'class-or-type-or-tuple'}), + 'doc': "Return whether an object is an instance of a class or of a subclass thereof.\n" + "With a type as second argument, return whether that is the object's type.\n" + "The form using a tuple, isinstance(x, (A, B, ...)), is a shortcut for\n" + "isinstance(x, A) or isinstance(x, B) or ... (etc.).", + 'ret_type': [(BUILTIN, 'bool')] + }] + ) + + def test_tuple_parameters(self): + self.check_doc_str( + 'pygame.Rect(left, top, width, height): return Rect\n' + 'pygame.Rect((left, top), (width, height)): return Rect\n' + 'pygame.Rect(object): return Rect\n' + 'pygame object for storing rectangular coordinates', + 'pygame', + 'Rect', + [{ + 'args': ({'name': 'left'}, {'name': 'top'}, {'name': 'width'}, {'name': 'height'}), + 'doc': 'pygame object for storing rectangular coordinates', + 'ret_type': [('', 'Rect')] + }, + { + 'args': ({'name': 'left, top'}, {'name': 'width, height'}), + 'doc': 'pygame object for storing rectangular coordinates', + 'ret_type': [('', 'Rect')] + }, + { + 'args': ({'name': 'object'},), + 'doc': 'pygame object for storing rectangular coordinates', + 'ret_type': [('', 'Rect')] + }] + ) + + def test_read(self): + self.check_doc_str( + 'read([size]) -> read at most size bytes, returned as a string.\n\n' + 'If the size argument is negative or omitted, read until EOF is reached.\n' + 'Notice that when in non-blocking mode, less data than what was requested\n' + 'may be returned, even if no size parameter was given.', + BUILTIN, + 'read', + mod=__builtins__, + expected=[{ + 'args': ({'default_value': 'None', 'name': 'size'},), + 'doc': 'read at most size bytes, returned as a string.\n\nIf the size argument is negative or omitted, read until EOF is reached.\nNotice that when in non-blocking mode, less data than what was requested\nmay be returned, even if no size parameter was given.', + 'ret_type': [('', '')] + }] + ) + + + r = get_overloads_from_doc_string( + 'read([size]) -> read at most size bytes, returned as a string.\n\n' + 'If the size argument is negative or omitted, read until EOF is reached.\n' + 'Notice that when in non-blocking mode, less data than what was requested\n' + 'may be returned, even if no size parameter was given.', + __builtins__, + None, + 'read' + ) + + self.assertEqual( + r, + [{ + 'args': ({'default_value': 'None', 'name': 'size'},), + 'doc': 'read at most size bytes, returned as a string.\n\nIf the size argument is negative or omitted, read until EOF is reached.\nNotice that when in non-blocking mode, less data than what was requested\nmay be returned, even if no size parameter was given.', + 'ret_type': [('', '')] + }], + repr(r) + ) + + def test_new(self): + self.check_doc_str( + 'T.__new__(S, ...) -> a new object with type S, a subtype of T', + 'struct', + '__new__', + [{ + 'ret_type': [('', '')], + 'doc': 'a new object with type S, a subtype of T', + 'args': ({'name': 'S'}, {'arg_format': '*', 'name': 'args'}) + }] + ) + + def test_C_prototype(self): + self.check_doc_str( + 'GetDriverByName(char const * name) -> Driver', + '', + 'GetDriverByName', + [{ + 'ret_type': [('', 'Driver')], + 'doc': '', + 'args': ({'name': 'name', 'type': [(BUILTIN, 'str')]},), + }] + ) + + def test_chmod(self): + self.check_doc_str( + 'chmod(path, mode, *, dir_fd=None, follow_symlinks=True)', + 'nt', + 'chmod', + [{ + 'doc': '', + 'args': ( + {'name': 'path'}, + {'name': 'mode'}, + {'name': 'args', 'arg_format': '*'}, + {'name': 'dir_fd', 'default_value': 'None'}, + {'name': 'follow_symlinks', 'default_value': 'True'} + ) + }] + ) + + def test_open(self): + if sys.version_info[0] >= 3: + expect_ret_type = ('_io', '_IOBase') + else: + expect_ret_type = (BUILTIN, 'file') + + self.check_doc_str( + 'open(file, mode=\'r\', buffering=-1, encoding=None,\n' + + ' errors=None, newline=None, closefd=True, opener=None)' + + ' -> file object\n\nOpen file', + BUILTIN, + 'open', + [{ + 'doc': 'Open file', + 'ret_type': [expect_ret_type], + 'args': ( + {'name': 'file'}, + {'name': 'mode', 'default_value': "'r'"}, + {'name': 'buffering', 'default_value': '-1'}, + {'name': 'encoding', 'default_value': 'None'}, + {'name': 'errors', 'default_value': 'None'}, + {'name': 'newline', 'default_value': 'None'}, + {'name': 'closefd', 'default_value': 'True'}, + {'name': 'opener', 'default_value': 'None'}, + ) + }] + ) + + def test_optional_with_default(self): + self.check_doc_str( + 'max(iterable[, key=func]) -> value', + BUILTIN, + 'max', + [{ + 'doc': '', + 'ret_type': [('', 'value')], + 'args': ( + {'name': 'iterable'}, + {'name': 'key', 'default_value': 'func'} + ) + }] + ) + + def test_pyplot_figure(self): + pyplot_doc = """ + Creates a new figure. + + Parameters + ---------- + + num : integer or string, optional, default: none + If not provided, a new figure will be created, and a the figure number + will be increamted. The figure objects holds this number in a `number` + attribute. + If num is provided, and a figure with this id already exists, make + it active, and returns a reference to it. If this figure does not + exists, create it and returns it. + If num is a string, the window title will be set to this figure's + `num`. + + figsize : tuple of integers, optional, default : None + width, height in inches. If not provided, defaults to rc + figure.figsize. + + dpi : integer, optional, default ; None + resolution of the figure. If not provided, defaults to rc figure.dpi. + + facecolor : + the background color; If not provided, defaults to rc figure.facecolor + + edgecolor : + the border color. If not provided, defaults to rc figure.edgecolor + + Returns + ------- + figure : Figure + The Figure instance returned will also be passed to new_figure_manager + in the backends, which allows to hook custom Figure classes into the + pylab interface. Additional kwargs will be passed to the figure init + function. + + Note + ---- + If you are creating many figures, make sure you explicitly call "close" + on the figures you are not using, because this will enable pylab + to properly clean up the memory. + + rcParams defines the default values, which can be modified in the + matplotlibrc file + + """ + self.check_doc_str( + pyplot_doc, + 'matplotlib.pyplot', + 'figure', + [{ + 'doc': pyplot_doc, + 'ret_type': [('', 'Figure')], + 'args': ( + {'name': 'args', 'arg_format': '*'}, + {'name': 'kwargs', 'arg_format': '**'} + ) + }] + ) + +if __name__ == '__main__': + unittest.main() diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/ExtensionScraper.py b/src/PTVS/Microsoft.PythonTools.Analyzer/ExtensionScraper.py new file mode 100644 index 000000000..29952ecaa --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/ExtensionScraper.py @@ -0,0 +1,79 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABILITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +import sys +import PythonScraper + +try: + # disable error reporting in our process, bad extension modules can crash us, and we don't + # want a bunch of Watson boxes popping up... + import ctypes + ctypes.windll.kernel32.SetErrorMode(3) # SEM_FAILCRITICALERRORS / SEM_NOGPFAULTERRORBOX +except: + pass + +# Scrapes the file and saves the analysis to the specified filename, exits w/ nonzero exit code if anything goes wrong. +# Usage: ExtensionScraper.py scrape [mod_name or '-'] [mod_path or '-'] [output_path] + +if len(sys.argv) != 5 or sys.argv[1].lower() != 'scrape': + raise ValueError('Expects "ExtensionScraper.py scrape [mod_name|'-'] [mod_path|'-'] [output_path]"') + +mod_name, mod_path, output_path = sys.argv[2:] +module = None + +if mod_name and mod_name != '-': + remove_sys_path_0 = False + try: + if mod_path and mod_path != '-': + import os.path + if os.path.exists(mod_path): + sys.path.insert(0, mod_path) + remove_sys_path_0 = True + __import__(mod_name) + module = sys.modules[mod_name] + finally: + if remove_sys_path_0: + del sys.path[0] + + if not module: + print('__import__("' + mod_name + '")') + PythonScraper.write_analysis(output_path, {"members": {}, "doc": "Could not import compiled module"}) +elif mod_path and mod_path != '-': + try: + import os.path + mod_name = os.path.split(mod_path)[1].partition('.')[0] + try: + import importlib + module = importlib.import_module(mod_name) + except ImportError: + # Don't really care which import failed - we'll try imp + pass + if not module: + import imp + module = imp.load_dynamic(mod_name, mod_path) + finally: + if not module: + print('imp.load_dynamic("' + mod_name + '", "' + mod_path + '")') + PythonScraper.write_analysis(output_path, {"members": {}, "doc": "Could not import compiled module", "filename": mod_path}) +else: + raise ValueError('No module name or path provided') + +if module: + analysis = PythonScraper.generate_module(module) + PythonScraper.write_analysis(output_path, analysis) diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AnalysisLimitsConverter.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AnalysisLimitsConverter.cs new file mode 100644 index 000000000..1cd095217 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AnalysisLimitsConverter.cs @@ -0,0 +1,114 @@ +using System.Collections.Generic; +using Microsoft.PythonTools.Analysis; +using Microsoft.Win32; + +namespace Microsoft.PythonTools.Intellisense { + public static class AnalysisLimitsConverter { + // We use string literals here rather than nameof() to ensure back-compat + // (though we need to preserve the names of the properties as well for + // the same reason, so just don't change anything :) ) + private const string CrossModuleId = "CrossModule"; + private const string CallDepthId = "CallDepth"; + private const string DecreaseCallDepthId = "DecreaseCallDepth"; + private const string NormalArgumentTypesId = "NormalArgumentTypes"; + private const string ListArgumentTypesId = "ListArgumentTypes"; + private const string DictArgumentTypesId = "DictArgumentTypes"; + private const string ReturnTypesId = "ReturnTypes"; + private const string YieldTypesId = "YieldTypes"; + private const string InstanceMembersId = "InstanceMembers"; + private const string DictKeyTypesId = "DictKeyTypes"; + private const string DictValueTypesId = "DictValueTypes"; + private const string IndexTypesId = "IndexTypes"; + private const string AssignedTypesId = "AssignedTypes"; + private const string UnifyCallsToNewId = "UnifyCallsToNew"; + private const string ProcessCustomDecoratorsId = "ProcessCustomDecorators"; + private const string UseTypeStubPackagesId = "UseTypeStubPackages"; + private const string UseTypeStubPackagesExclusivelyId = "UseTypeStubPackagesExclusively"; + + + /// + /// Loads a new instance from the specified registry key. + /// + /// + /// The key to load settings from. Each setting is a DWORD value. If + /// null, all settings are assumed to be unspecified and the default + /// values are used. + /// + /// + /// If True, unspecified settings are taken from the defaults for + /// standard library analysis. Otherwise, they are taken from the + /// usual defaults. + /// + public static AnalysisLimits LoadFromStorage(RegistryKey key, bool defaultToStdLib = false) { + var limits = defaultToStdLib ? AnalysisLimits.GetStandardLibraryLimits() : new AnalysisLimits(); + + if (key != null) { + limits.CrossModule = (key.GetValue(CrossModuleId) as int?) ?? limits.CrossModule; + limits.CallDepth = (key.GetValue(CallDepthId) as int?) ?? limits.CallDepth; + limits.DecreaseCallDepth = (key.GetValue(DecreaseCallDepthId) as int?) ?? limits.DecreaseCallDepth; + limits.NormalArgumentTypes = (key.GetValue(NormalArgumentTypesId) as int?) ?? limits.NormalArgumentTypes; + limits.ListArgumentTypes = (key.GetValue(ListArgumentTypesId) as int?) ?? limits.ListArgumentTypes; + limits.DictArgumentTypes = (key.GetValue(DictArgumentTypesId) as int?) ?? limits.DictArgumentTypes; + limits.ReturnTypes = (key.GetValue(ReturnTypesId) as int?) ?? limits.ReturnTypes; + limits.YieldTypes = (key.GetValue(YieldTypesId) as int?) ?? limits.YieldTypes; + limits.InstanceMembers = (key.GetValue(InstanceMembersId) as int?) ?? limits.InstanceMembers; + limits.DictKeyTypes = (key.GetValue(DictKeyTypesId) as int?) ?? limits.DictKeyTypes; + limits.DictValueTypes = (key.GetValue(DictValueTypesId) as int?) ?? limits.DictValueTypes; + limits.IndexTypes = (key.GetValue(IndexTypesId) as int?) ?? limits.IndexTypes; + limits.AssignedTypes = (key.GetValue(AssignedTypesId) as int?) ?? limits.AssignedTypes; + limits.UnifyCallsToNew = ((key.GetValue(UnifyCallsToNewId) as int?) ?? (limits.UnifyCallsToNew ? 1 : 0)) != 0; + limits.ProcessCustomDecorators = ((key.GetValue(ProcessCustomDecoratorsId) as int?) ?? (limits.ProcessCustomDecorators ? 1 : 0)) != 0; + limits.UseTypeStubPackages = ((key.GetValue(UseTypeStubPackagesId) as int?) ?? (limits.UseTypeStubPackages ? 1 : 0)) != 0; + limits.UseTypeStubPackagesExclusively = ((key.GetValue(UseTypeStubPackagesExclusivelyId) as int?) ?? (limits.UseTypeStubPackagesExclusively ? 1 : 0)) != 0; + } + + return limits; + } + + public static AnalysisLimits FromDictionary(Dictionary limits) { + var analysisLimits = new AnalysisLimits(); + int i; + if (limits.TryGetValue(CrossModuleId, out i)) analysisLimits.CrossModule = i; + if (limits.TryGetValue(CallDepthId, out i)) analysisLimits.CallDepth = i; + if (limits.TryGetValue(DecreaseCallDepthId, out i)) analysisLimits.DecreaseCallDepth = i; + if (limits.TryGetValue(NormalArgumentTypesId, out i)) analysisLimits.NormalArgumentTypes = i; + if (limits.TryGetValue(ListArgumentTypesId, out i)) analysisLimits.ListArgumentTypes = i; + if (limits.TryGetValue(DictArgumentTypesId, out i)) analysisLimits.DictArgumentTypes = i; + if (limits.TryGetValue(ReturnTypesId, out i)) analysisLimits.ReturnTypes = i; + if (limits.TryGetValue(YieldTypesId, out i)) analysisLimits.YieldTypes = i; + if (limits.TryGetValue(InstanceMembersId, out i)) analysisLimits.InstanceMembers = i; + if (limits.TryGetValue(DictKeyTypesId, out i)) analysisLimits.DictKeyTypes = i; + if (limits.TryGetValue(DictValueTypesId, out i)) analysisLimits.DictValueTypes = i; + if (limits.TryGetValue(IndexTypesId, out i)) analysisLimits.IndexTypes = i; + if (limits.TryGetValue(AssignedTypesId, out i)) analysisLimits.AssignedTypes = i; + if (limits.TryGetValue(UnifyCallsToNewId, out i)) analysisLimits.UnifyCallsToNew = i != 0; + if (limits.TryGetValue(ProcessCustomDecoratorsId, out i)) analysisLimits.ProcessCustomDecorators = i != 0; + if (limits.TryGetValue(UseTypeStubPackagesId, out i)) analysisLimits.UseTypeStubPackages = i != 0; + if (limits.TryGetValue(UseTypeStubPackagesExclusivelyId, out i)) analysisLimits.UseTypeStubPackagesExclusively = i != 0; + + return analysisLimits; + } + + public static Dictionary ToDictionary(this AnalysisLimits analysisLimits) { + return new Dictionary { + { CrossModuleId, analysisLimits.CrossModule }, + { CallDepthId, analysisLimits.CallDepth }, + { DecreaseCallDepthId, analysisLimits.DecreaseCallDepth }, + { NormalArgumentTypesId, analysisLimits.NormalArgumentTypes }, + { ListArgumentTypesId, analysisLimits.ListArgumentTypes }, + { DictArgumentTypesId, analysisLimits.DictArgumentTypes }, + { ReturnTypesId, analysisLimits.ReturnTypes }, + { YieldTypesId, analysisLimits.YieldTypes }, + { InstanceMembersId, analysisLimits.InstanceMembers }, + { DictKeyTypesId, analysisLimits.DictKeyTypes }, + { DictValueTypesId, analysisLimits.DictValueTypes }, + { IndexTypesId, analysisLimits.IndexTypes }, + { AssignedTypesId, analysisLimits.AssignedTypes }, + { UnifyCallsToNewId, analysisLimits.UnifyCallsToNew ? 1 : 0 }, + { ProcessCustomDecoratorsId, analysisLimits.ProcessCustomDecorators ? 1 : 0 }, + { UseTypeStubPackagesId, analysisLimits.UseTypeStubPackages ? 1 : 0 }, + { UseTypeStubPackagesExclusivelyId, analysisLimits.UseTypeStubPackagesExclusively ? 1 : 0 } + }; + } + } +} \ No newline at end of file diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AnalysisProtocol.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AnalysisProtocol.cs new file mode 100644 index 000000000..ac2ba6415 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AnalysisProtocol.cs @@ -0,0 +1,970 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Reflection; +using Microsoft.PythonTools.Analysis; +using Microsoft.PythonTools.Analysis.Infrastructure; +using Microsoft.PythonTools.Interpreter; +using Microsoft.PythonTools.Ipc.Json; +using Microsoft.PythonTools.Parsing; +using Newtonsoft.Json; +using LS = Microsoft.Python.LanguageServer; + +namespace Microsoft.PythonTools.Intellisense { + public static class AnalysisProtocol { + public static readonly Dictionary RegisteredTypes = CollectCommands(); + + private static Dictionary CollectCommands() { + Dictionary all = new Dictionary(); + foreach (var type in typeof(AnalysisProtocol).GetNestedTypes()) { + if (type.IsSubclassOf(typeof(Request))) { + var command = type.GetField("Command"); + if (command != null) { + all["request." + (string)command.GetRawConstantValue()] = type; + } + } else if (type.IsSubclassOf(typeof(Event))) { + var name = type.GetField("Name"); + if (name != null) { + all["event." + (string)name.GetRawConstantValue()] = type; + } + } + } + return all; + } + + public sealed class InitializeRequest : Request { + public const string Command = "initialize"; + + public override string command => Command; + + public InterpreterInfo interpreter; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri rootUri; + public bool analyzeAllFiles; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public bool traceLogging; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public bool liveLinting; + } + + public sealed class InterpreterInfo { + public string assembly, typeName; + public Dictionary properties; + } + + public sealed class InitializeResponse : Response { + [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public string error; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public string fullError; + } + + public sealed class ExitRequest : GenericRequest { + public const string Command = "exit"; + + public override string command => Command; + } + + public sealed class GetReferencesResponse : Response { + public ProjectReference[] references; + } + + public sealed class ProjectReference { + public string name, kind, assemblyName; + + public static ProjectReference Convert(Microsoft.PythonTools.Interpreter.ProjectReference reference) { + return new ProjectReference() { + name = reference.Name, + kind = GetReferenceKind(reference.Kind), + assemblyName = GetReferenceAssembly(reference) + }; + } + + public static Microsoft.PythonTools.Interpreter.ProjectReference Convert(ProjectReference reference) { + switch (reference.kind) { + case "extension": + return new Microsoft.PythonTools.Interpreter.ProjectReference( + reference.name, + ProjectReferenceKind.ExtensionModule + ); + case "assembly": + return new ProjectAssemblyReference( + new AssemblyName(reference.assemblyName), + reference.name + ); + default: + throw new InvalidOperationException("Unsupported reference type " + reference.kind); + } + } + + private static string GetReferenceAssembly(Microsoft.PythonTools.Interpreter.ProjectReference reference) { + switch (reference.Kind) { + case ProjectReferenceKind.Assembly: + return ((ProjectAssemblyReference)reference).AssemblyName.FullName; + default: return null; + } + } + + public static string GetReferenceKind(ProjectReferenceKind kind) { + switch (kind) { + case ProjectReferenceKind.Assembly: return "assembly"; + case ProjectReferenceKind.ExtensionModule: return "extension"; + default: return null; + } + } + + } + + public sealed class SetAnalysisLimitsRequest : Request { + public const string Command = "setAnalysisLimits"; + + public override string command => Command; + + } + + public sealed class ValueDescriptionRequest : Request { + public const string Command = "valueDescriptions"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string expr; + public int line, column; + + public override string command => Command; + } + + public sealed class ValueDescriptionResponse : Response { + public string[] descriptions; + } + + public sealed class AddReferenceRequest : Request { + public const string Command = "addReference"; + public ProjectReference reference; + + public override string command => Command; + } + + public sealed class AddReferenceResponse : Response { + } + + public sealed class RemoveReferenceRequest : Request { + public const string Command = "removeReference"; + public ProjectReference reference; + + public override string command => Command; + } + + public sealed class RemoveReferenceResponse : Response { + } + + public sealed class AnalysisClassificationsRequest : Request { + public const string Command = "analysisClassify"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public bool colorNames; + + public override string command => Command; + } + + /// + /// Gets a location where a method can safely be inserted into a top level class + /// + public sealed class MethodInsertionLocationRequest : Request { + public const string Command = "methodInsertion"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string className; + + public override string command => Command; + } + + public sealed class MethodInsertionLocationResponse : Response { + public int line, column; + public int version; + } + + public sealed class MethodInfoRequest : Request { + public const string Command = "methodInfo"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string className; + public string methodName; + + public override string command => Command; + } + + public sealed class MethodInfoResponse : Response { + public int start, end; + public int version; + public bool found; + } + + public sealed class FindMethodsRequest : Request { + public const string Command = "findMethods"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string className; + + /// + /// Optional filter of the number of parameters + /// + public int? paramCount; + + public override string command => Command; + } + + public sealed class FindMethodsResponse : Response { + public string[] names; + } + + public sealed class AnalysisClassificationsResponse : Response { + public AnalysisClassification[] classifications; + + public int version; + } + + public sealed class AnalysisClassification { + public int startLine, startColumn; + public int endLine, endColumn; + public string type; + } + + public class QuickInfoRequest : Request { + public const string Command = "quickInfo"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string expr; + public int line, column; + + public override string command => Command; + } + + public class QuickInfoResponse : Response { + public string text; + } + + public class FileParsedEvent : Event { + public const string Name = "fileParsed"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int version; + + public override string name => Name; + } + + public class DiagnosticsEvent : Event { + public const string Name = "diagnostics"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int version; + public Diagnostic[] diagnostics; + + public bool ShouldSerializediagnostics() => (diagnostics?.Length ?? 0) > 0; + + public override string name => Name; + } + + public sealed class FormatCodeRequest : Request { + public const string Command = "formatCode"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int startLine, startColumn; + public int endLine, endColumn; + public string newLine; + public CodeFormattingOptions options; + + public override string command => Command; + } + + public sealed class FormatCodeResponse : Response { + public ChangeInfo[] changes; + public int version; + } + + public struct CodeSpan { + public int start, length; + } + + public class AddFileRequest : Request { + public const string Command = "addFile"; + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public string path; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public string addingFromDir; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public bool isTemporaryFile; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public bool suppressErrorLists; + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + [JsonConverter(typeof(UriJsonConverter))] + public Uri uri; + + public override string command => Command; + } + + public class AddFileResponse : Response { + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + } + + public class AddBulkFileRequest : Request { + public const string Command = "addBulkFile"; + + public string[] path; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public string addingFromDir; + + public override string command => Command; + } + + public class AddBulkFileResponse : Response { + [JsonProperty(ItemConverterType = typeof(UriJsonConverter))] + public Uri[] documentUri; + } + + public sealed class SetSearchPathRequest : Request { + public const string Command = "setSearchPath"; + + public string[] dir; + public override string command => Command; + } + + public sealed class UnloadFileRequest : Request { + public const string Command = "unloadFile"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public override string command => Command; + + public override string ToString() => "{0}:{1}".FormatUI(command, documentUri); + } + + + public sealed class DirectoryFileAddedEvent : Event { + public const string Name = "directoryFileAdded"; + + public string filename; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + + public override string name => Name; + } + + public sealed class FileUpdateRequest : Request { + public const string Command = "fileUpdate"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public FileUpdate[] updates; + + public override string command => Command; + + public override string ToString() => "{0}:{1} ({2} updates)".FormatUI(command, documentUri, updates.Length); + } + + public enum FileUpdateKind { + none, + /// + /// Reset the content to the specified content string + /// + reset, + /// + /// Apply the list of changes to the content + /// + changes + } + + public sealed class AddImportRequest : Request { + public const string Command = "addImport"; + + public string fromModule, name, newLine; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + + public override string command => Command; + } + + public sealed class AddImportResponse : Response { + public ChangeInfo[] changes; + public int version = -1; + } + + public sealed class IsMissingImportRequest : Request { + public const string Command = "isMissingImport"; + + public string text; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int line, column; + + public override string command => Command; + } + + public sealed class IsMissingImportResponse : Response { + public bool isMissing; + } + + public sealed class AvailableImportsRequest : Request { + public const string Command = "availableImports"; + + public string name; + + public override string command => Command; + } + + public sealed class AvailableImportsResponse : Response { + public ImportInfo[] imports; + } + + public sealed class ImportInfo { + public string fromName, importName; + + + // Provide Equals so we can easily uniquify sequences of ImportInfo + + public override bool Equals(object obj) { + if (obj is ImportInfo ii) { + return fromName == ii.fromName && importName == ii.importName; + } + return false; + } + + public override int GetHashCode() { + return ((fromName ?? "") + "." + (importName ?? "")).GetHashCode(); + } + } + + public sealed class FileUpdate { + public FileUpdateKind kind; + + // Unlike most version numbers, this is what the version will be + // _after_ applying the update, not before. The target file is + // assumed to be at version-1 when applying this change. + public int version; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public ChangeInfo[] changes; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public string content; + } + + public sealed class FileUpdateResponse : Response { + public int version; +#if DEBUG + public string newCode; +#endif + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public bool? failed; + } + + public sealed class UnresolvedImport { + public string name; + public int startLine, endLine, startColumn, endColumn; + } + + public sealed class ChangeInfo { + public string newText; + public int startLine, startColumn; + public int endLine, endColumn; +#if DEBUG + public int _startIndex, _endIndex; +#endif + + public static ChangeInfo FromDocumentChange(DocumentChange c) { + return new ChangeInfo { + startLine = c.ReplacedSpan.Start.Line, + startColumn = c.ReplacedSpan.Start.Column, + endLine = c.ReplacedSpan.End.Line, + endColumn = c.ReplacedSpan.End.Column, + newText = c.InsertedText + }; + } + + public DocumentChange ToDocumentChange() { + return new DocumentChange { + InsertedText = newText, + ReplacedSpan = new SourceSpan(new SourceLocation(startLine, startColumn), new SourceLocation(endLine, endColumn)) + }; + } + } + + public sealed class LocationNameRequest : Request { + public const string Command = "locationName"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int line, column; + + public override string command => Command; + } + + public sealed class LocationNameResponse : Response { + public string name; + public int lineOffset; + } + + + public sealed class ProximityExpressionsRequest : Request { + public const string Command = "proximityExpressions"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int line, column, lineCount; + + public override string command => Command; + } + + public sealed class ProximityExpressionsResponse : Response { + public string[] names; + } + + public sealed class RemoveImportsRequest : Request { + public const string Command = "removeImports"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int version; + public int line, column; + public bool allScopes; + + public override string command => Command; + } + + public sealed class RemoveImportsResponse : Response { + public ChangeInfo[] changes; + public int version = -1; + } + + public sealed class ExtractMethodRequest : Request { + public const string Command = "extractMethod"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int startIndex, endIndex; + public int indentSize; + public string name; + public string[] parameters; + public bool convertTabsToSpaces; + public string newLine; + public bool shouldExpandSelection; + public int? scope; + + public override string command => Command; + } + + public sealed class ExtractMethodResponse : Response { + public CannotExtractReason cannotExtractReason; + public ChangeInfo[] changes; + /// + /// available scopes the user can retarget to + /// + public ScopeInfo[] scopes; + public bool wasExpanded; + public int startLine, startCol; + public int endLine, endCol; + public int version; + public string methodBody; + public string[] variables; + } + + public enum CannotExtractReason { + None = 0, + InvalidTargetSelected = 1, + InvalidExpressionSelected = 2, + MethodAssignsVariablesAndReturns = 3, + StatementsFromClassDefinition = 4, + SelectionContainsBreakButNotEnclosingLoop = 5, + SelectionContainsContinueButNotEnclosingLoop = 6, + ContainsYieldExpression = 7, + ContainsFromImportStar = 8, + SelectionContainsReturn = 9 + } + + public class ScopeInfo { + public string type, name; + public int id; + public string[] variables; + } + + public sealed class ModuleImportsRequest : Request { + public const string Command = "moduleImports"; + + public string moduleName; + public bool includeUnresolved; + + public override string command => Command; + } + + public sealed class ModuleImportsResponse : Response { + public ModuleInfo[] modules; + } + + public sealed class ModuleInfo { + public string moduleName; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string filename; + } + + public class EnqueueFileResponse : Response { + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + } + + public class GetModulesRequest : Request { + public const string Command = "getModules"; + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string[] package; + + public override string command => Command; + + public bool ShouldSerializepackage() => (package?.Length ?? 0) > 0; + } + + public class GetAllMembersRequest : Request { + public const string Command = "getAllMembers"; + + public string prefix; + public GetMemberOptions options; + + public override string command => Command; + } + + public class CompletionsRequest : Request { + public const string Command = "completions"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string text; + public int line, column; + public GetMemberOptions options; + public bool forceCompletions; + + public override string command => Command; + } + + public class SignaturesRequest : Request { + public const string Command = "sigs"; + + public override string command => Command; + + public string text; + public int line, column; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + } + + public sealed class ModulesChangedEvent : Event { + public const string Name = "modulesChanged"; + + public override string name => Name; + } + + public struct FileEvent { + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public LS.FileChangeType kind; + } + + public sealed class FileChangedEvent : Event { + public const string Name = "fileChanged"; + + public FileEvent[] changes; + + public override string name => Name; + } + + public sealed class SignaturesResponse : Response { + public Signature[] sigs; + } + + public class Signature { + public string name; + public string doc; + public Parameter[] parameters; + } + + public class Parameter { + public string name, defaultValue, doc, type; + public bool optional; + } + + public class FileAnalysisCompleteEvent : Event { + public const string Name = "fileAnalysisComplete"; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + + public int version; + + public override string name => Name; + public override string ToString() => "{0}:{1} ({2})".FormatUI(name, documentUri, version); + } + + public sealed class LoadExtensionRequest : Request { + public const string Command = "loadExtensionRequest"; + + public override string command => Command; + + public string extension; + public string assembly; + public string typeName; + } + + public sealed class LoadExtensionResponse : Response { + public string error; + public string fullError; + } + + public sealed class ExtensionRequest : Request { + public const string Command = "extensionRequest"; + + public override string command => Command; + + public string extension; + public string commandId; + public string body; + } + + public sealed class ExtensionResponse : Response { + public string response; + public string error; + } + + /// + /// Signals all files are analyzed + /// + public class AnalysisCompleteEvent : Event { + public const string Name = "analysisComplete"; + + public override string name => Name; + } + + public sealed class AnalysisStatusRequest : Request { + public const string Command = "analysisStatus"; + + public override string command => Command; + } + + public sealed class AnalysisStatusResponse : Response { + public int itemsLeft; + } + + + public class CompletionsResponse : Response { + public Completion[] completions; + } + + public class Completion { + public string name; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public string completion; // when null, use "name" + public string doc; + public PythonMemberType memberType; + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public CompletionValue[] detailedValues; + + public bool ShouldSerializedetailedValues() => (detailedValues?.Length ?? 0) > 0; + } + + public sealed class CompletionValue { + public DescriptionComponent[] description; + public string doc; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public AnalysisReference[] locations; + + public bool ShouldSerializelocations() => (locations?.Length ?? 0) > 0; + } + + public sealed class DescriptionComponent { + public string text, kind; + + public DescriptionComponent() { + } + + public DescriptionComponent(string text, string kind) { + this.text = text; + this.kind = kind; + } + } + + public sealed class SetAnalysisOptionsRequest : Request { + public const string Command = "setAnalysisOptions"; + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public AnalysisOptions options; + + public override string command => Command; + } + + public sealed class AnalysisOptions { + public Severity indentationInconsistencySeverity; + public Dictionary commentTokens; + public Dictionary analysisLimits; + public LS.MessageType? traceLevel; + public string[] typeStubPaths; + } + + + + public class AnalysisReference { + public string kind; // definition, reference, value + public string expr; + public string file; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int startLine, startColumn, endLine, endColumn; + public int? version; + } + + public sealed class AnalyzeExpressionRequest : Request { + public const string Command = "findDefs"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public string expr; + public int line, column; + + public override string command => Command; + } + + public sealed class AnalyzeExpressionResponse : Response { + public AnalysisReference[] variables; + /// + /// The private prefix for the member if defined inside a class with name mangling. + /// + public string privatePrefix; + } + + public sealed class OutliningRegionsRequest : Request { + public const string Command = "outliningRegions"; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + + public override string command => Command; + } + + public sealed class OutliningRegionsResponse : Response { + public int version = -1; + public OutliningTag[] tags; + } + + public sealed class OutliningTag { + public int startLine, startCol; + public int endLine, endCol; + } + + public sealed class NavigationRequest : Request { + public const string Command = "navigation"; + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + + public override string command => Command; + } + + public sealed class NavigationResponse : Response { + public int version; + public Navigation[] navigations; + } + + public sealed class Navigation { + public string name, type; + public int startLine, startColumn; + public int endLine, endColumn; + public Navigation[] children; + } + + public class AnalyzerWarningEvent : Event { + public string message; + public const string Name = "analyzerWarning"; + + public override string name => Name; + } + + public class UnhandledExceptionEvent : Event { + public string message; + public const string Name = "unhandledException"; + + public UnhandledExceptionEvent(Exception ex) { + message = ex.ToString(); + } + + public UnhandledExceptionEvent(string message) { + this.message = message; + } + + public override string name => Name; + } + + public enum ExpressionAtPointPurpose : int { + Hover = 1, + Evaluate = 2, + EvaluateMembers = 3, + FindDefinition = 4, + Rename = 5 + } + + public sealed class ExpressionAtPointRequest : Request { + public const string Command = "exprAtPoint"; + + [JsonConverter(typeof(UriJsonConverter))] + public Uri documentUri; + public int line, column; + public ExpressionAtPointPurpose purpose; + + public override string command => Command; + } + + public sealed class ExpressionAtPointResponse : Response { + public string expression; + public string type; + public int bufferVersion; + public int startLine, startColumn; + public int endLine, endColumn; + } + + public class LanguageServerRequest : Request { + public const string Command = "languageServer"; + + public string name; + public object body; + + public override string command => Command; + } + + public class LanguageServerResponse : Response { + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public object body; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public string error; + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AssignmentWalker.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AssignmentWalker.cs new file mode 100644 index 000000000..4bf76dfc7 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AssignmentWalker.cs @@ -0,0 +1,137 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Intellisense { + + /// + /// A walker which handles all nodes which can result in assignments and + /// calls back on a special walker for defining the variable names. + /// + /// Note this class only handles things which show up as name expressions, + /// other implicit assignments (such as class and function definitions, + /// import/from import statements, need to be handled by the derived binder). + /// + public abstract class AssignmentWalker : PythonWalker { + public abstract AssignedNameWalker Define { + get; + } + + #region Assignment Walkers + + public override bool Walk(AssignmentStatement node) { + foreach (var lhs in node.Left) { + DefineExpr(lhs); + } + node.Right.Walk(this); + return false; + } + + private void DefineExpr(Expression lhs) { + if (lhs is NameExpression) { + lhs.Walk(Define); + } else { + // fob.oar = 42, fob[oar] = 42, we don't actually define any variables + lhs.Walk(this); + } + } + + public override bool Walk(AugmentedAssignStatement node) { + DefineExpr(node.Left); + node.Right.Walk(this); + return false; + } + + public override bool Walk(DelStatement node) { + foreach (var expr in node.Expressions) { + DefineExpr(expr); + } + return false; + } + + public override bool Walk(ComprehensionFor node) { + if (node.Left != null) { + node.Left.Walk(Define); + } + if (node.List != null) { + node.List.Walk(this); + } + return false; + } + + private bool WalkIterators(Comprehension node) { + if (node.Iterators != null) { + foreach (ComprehensionIterator ci in node.Iterators) { + ci.Walk(this); + } + } + + return false; + } + + public override bool Walk(ForStatement node) { + if (node.Left != null) { + node.Left.Walk(Define); + } + if (node.List != null) { + node.List.Walk(this); + } + if (node.Body != null) { + node.Body.Walk(this); + } + if (node.Else != null) { + node.Else.Walk(this); + } + return false; + } + + public override bool Walk(WithStatement node) { + foreach (var item in node.Items) { + if (item.Variable != null) { + item.Variable.Walk(Define); + } + if (item.ContextManager != null) { + item.ContextManager.Walk(this); + } + } + if (node.Body != null) { + node.Body.Walk(this); + } + return false; + } + + #endregion + } + + public abstract class AssignedNameWalker : PythonWalkerNonRecursive { + + public override abstract bool Walk(NameExpression node); + + public override bool Walk(ParenthesisExpression node) { + return true; + } + + public override bool Walk(TupleExpression node) { + return true; + } + + public override bool Walk(ListExpression node) { + return true; + } + } + +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/ClassifierWalker.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/ClassifierWalker.cs new file mode 100644 index 000000000..12cd32225 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/ClassifierWalker.cs @@ -0,0 +1,480 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Microsoft.PythonTools.Analysis; +using Microsoft.PythonTools.Interpreter; +using Microsoft.PythonTools.Parsing; +using Microsoft.PythonTools.Parsing.Ast; +using Microsoft.PythonTools.Analysis.Values; + +namespace Microsoft.PythonTools.Intellisense { + class ClassifierWalker : PythonWalker { + class StackData { + public readonly string Name; + public readonly HashSet Parameters; + public readonly HashSet Functions; + public readonly HashSet Types; + public readonly HashSet TypeHints; + public readonly HashSet Modules; + public readonly List> Names; + public readonly StackData Previous; + + public StackData(string name, StackData previous) { + Name = name; + Previous = previous; + Parameters = new HashSet(); + Functions = new HashSet(); + Types = new HashSet(); + TypeHints = new HashSet(); + Modules = new HashSet(); + Names = new List>(); + } + + public IEnumerable EnumerateTowardsGlobal { + get { + for (var sd = this; sd != null; sd = sd.Previous) { + yield return sd; + } + } + } + } + + private readonly PythonAst _ast; + private readonly IModuleAnalysis _analysis; + private StackData _head; + public readonly List Spans; + + public static class Classifications { + public const string Keyword = "keyword"; + public const string Class = "class"; + public const string Function = "function"; + public const string Module = "module"; + public const string Parameter = "parameter"; + public const string RegexLiteral = "regexliteral"; + public const string DocString = "docstring"; + public const string TypeHint = "class"; + } + + public ClassifierWalker(PythonAst ast, IModuleAnalysis analysis) { + _ast = ast; + _analysis = analysis; + Spans = new List(); + } + + private void AddSpan(Tuple node, string type) { + Spans.Add(new TaggedSpan( + new SourceSpan( + _ast.IndexToLocation(node.Item2.Start), + _ast.IndexToLocation(node.Item2.Start + node.Item2.Length) + ), + type + )); + } + + private void BeginScope(string name = null) { + if (_head != null) { + if (name == null) { + name = _head.Name; + } else if (_head.Name != null) { + name = _head.Name + "." + name; + } + } + _head = new StackData(name, _head); + } + + private void AddParameter(Parameter node) { + Debug.Assert(_head != null); + _head.Parameters.Add(node.Name); + _head.Names.Add(Tuple.Create(node.Name, new Span(node.NameSpan.Start, node.NameSpan.Length))); + } + + private void AddParameter(Node node) { + NameExpression name; + TupleExpression tuple; + Debug.Assert(_head != null); + if ((name = node as NameExpression) != null) { + _head.Parameters.Add(name.Name); + } else if ((tuple = node as TupleExpression) != null) { + foreach (var expr in tuple.Items) { + AddParameter(expr); + } + } else { + Trace.TraceWarning("Unable to find parameter in {0}", node); + } + } + + public override bool Walk(NameExpression node) { + _head.Names.Add(Tuple.Create(node.Name, Span.FromBounds(node.StartIndex, node.EndIndex))); + return base.Walk(node); + } + + private static string GetFullName(MemberExpression expr) { + var ne = expr.Target as NameExpression; + if (ne != null) { + return ne.Name + "." + expr.Name ?? string.Empty; + } + var me = expr.Target as MemberExpression; + if (me != null) { + var baseName = GetFullName(me); + if (baseName == null) { + return null; + } + return baseName + "." + expr.Name ?? string.Empty; + } + return null; + } + + public override bool Walk(MemberExpression node) { + var fullname = GetFullName(node); + if (fullname != null) { + _head.Names.Add(Tuple.Create(fullname, Span.FromBounds(node.NameHeader, node.EndIndex))); + } + return base.Walk(node); + } + + public override bool Walk(DottedName node) { + string totalName = ""; + foreach (var name in node.Names) { + _head.Names.Add(Tuple.Create(totalName + name.Name, Span.FromBounds(name.StartIndex, name.EndIndex))); + totalName += name.Name + "."; + } + return base.Walk(node); + } + + private BuiltinTypeId GetTypeId(AnalysisValue v) { + if (v.TypeId != BuiltinTypeId.Type) { + return v.TypeId; + } + + if (v.MemberType == PythonMemberType.Instance && + v.IsOfType(_analysis.ProjectState.ClassInfos[BuiltinTypeId.Type])) { + return BuiltinTypeId.Type; + } + + return BuiltinTypeId.Unknown; + } + + private string ClassifyName(Tuple node) { + var name = node.Item1; + foreach (var sd in _head.EnumerateTowardsGlobal) { + if (sd.Parameters.Contains(name)) { + return Classifications.Parameter; + } else if (sd.Functions.Contains(name)) { + return Classifications.Function; + } else if (sd.Types.Contains(name)) { + return Classifications.Class; + } else if (sd.TypeHints.Contains(name)) { + return Classifications.TypeHint; + } else if (sd.Modules.Contains(name)) { + return Classifications.Module; + } + } + + if (_analysis != null) { + var memberType = PythonMemberType.Unknown; + var typeId = BuiltinTypeId.Unknown; + bool isTypeHint = false; + lock (_analysis) { + var values = _analysis.GetValuesByIndex(name, node.Item2.Start).ToArray(); + isTypeHint = values.Any(v => v is TypingTypeInfo || v.DeclaringModule?.ModuleName == "typing"); + memberType = values.Select(v => v.MemberType) + .DefaultIfEmpty(PythonMemberType.Unknown) + .Aggregate((a, b) => a == b || b == PythonMemberType.Unknown ? a : PythonMemberType.Unknown); + typeId = values.Select(GetTypeId) + .DefaultIfEmpty(BuiltinTypeId.Unknown) + .Aggregate((a, b) => a == b || b == BuiltinTypeId.Unknown ? a : BuiltinTypeId.Unknown); + } + + if (isTypeHint) { + return Classifications.TypeHint; + } + if (memberType == PythonMemberType.Module || typeId == BuiltinTypeId.Module) { + return Classifications.Module; + } else if (memberType == PythonMemberType.Class || typeId == BuiltinTypeId.Type) { + return Classifications.Class; + } else if (memberType == PythonMemberType.Function || memberType == PythonMemberType.Method || + typeId == BuiltinTypeId.Function || typeId == BuiltinTypeId.BuiltinFunction) { + return Classifications.Function; + } + } + + return null; + } + + private void EndScope(bool mergeNames) { + var sd = _head; + foreach (var node in sd.Names) { + var classificationName = ClassifyName(node); + if (classificationName != null) { + AddSpan(node, classificationName); + if (mergeNames && sd.Previous != null) { + if (classificationName == Classifications.Module) { + sd.Previous.Modules.Add(sd.Name + "." + node.Item1); + } else if (classificationName == Classifications.Class) { + sd.Previous.Types.Add(sd.Name + "." + node.Item1); + } else if (classificationName == Classifications.Function) { + sd.Previous.Functions.Add(sd.Name + "." + node.Item1); + } + } + } + } + _head = sd.Previous; + } + + public override bool Walk(PythonAst node) { + Debug.Assert(_head == null); + _head = new StackData(string.Empty, null); + return base.Walk(node); + } + + public override void PostWalk(PythonAst node) { + EndScope(false); + Debug.Assert(_head == null); + base.PostWalk(node); + } + + private void MaybeAddDocstring(Node body) { + var docString = (body as SuiteStatement)?.Statements?[0] as ExpressionStatement; + if (docString?.Expression is ConstantExpression ce && (ce.Value is string || ce.Value is AsciiString)) { + AddSpan(Tuple.Create("", Span.FromBounds(ce.StartIndex, ce.EndIndex)), Classifications.DocString); + } + } + + public override bool Walk(ClassDefinition node) { + Debug.Assert(_head != null); + _head.Types.Add(node.NameExpression.Name); + node.NameExpression.Walk(this); + BeginScope(node.NameExpression.Name); + MaybeAddDocstring(node.Body); + return base.Walk(node); + } + + public override bool Walk(FunctionDefinition node) { + if (node.IsCoroutine) { + AddSpan(Tuple.Create("", new Span(node.DefIndex, 5)), Classifications.Keyword); + } + + Debug.Assert(_head != null); + _head.Functions.Add(node.NameExpression.Name); + node.NameExpression.Walk(this); + BeginScope(); + MaybeAddDocstring(node.Body); + return base.Walk(node); + } + + public override bool Walk(DictionaryComprehension node) { + BeginScope(); + return base.Walk(node); + } + + public override bool Walk(ListComprehension node) { + BeginScope(); + return base.Walk(node); + } + + public override bool Walk(GeneratorExpression node) { + BeginScope(); + return base.Walk(node); + } + + public override bool Walk(ComprehensionFor node) { + AddParameter(node.Left); + + if (node.IsAsync) { + AddSpan(Tuple.Create("", new Span(node.StartIndex, 5)), Classifications.Keyword); + } + + return base.Walk(node); + } + + public override bool Walk(Parameter node) { + AddParameter(node); + return base.Walk(node); + } + + public override bool Walk(ImportStatement node) { + Debug.Assert(_head != null); + if (node.AsNames != null) { + foreach (var name in node.AsNames) { + if (name != null && !string.IsNullOrEmpty(name.Name)) { + _head.Modules.Add(name.Name); + _head.Names.Add(Tuple.Create(name.Name, Span.FromBounds(name.StartIndex, name.EndIndex))); + } + } + } + if (node.Names != null) { + for (int i = 0; i < node.Names.Count; ++i) { + var dottedName = node.Names[i]; + var hasAsName = (node.AsNames != null && node.AsNames.Count > i) ? node.AsNames[i] != null : false; + foreach (var name in dottedName.Names) { + if (name != null && !string.IsNullOrEmpty(name.Name)) { + if (!hasAsName) { + _head.Modules.Add(name.Name); + _head.Names.Add(Tuple.Create(name.Name, Span.FromBounds(name.StartIndex, name.EndIndex))); + } else { + // Only want to highlight this instance of the + // name, since it isn't going to be bound in the + // rest of the module. + AddSpan(Tuple.Create(name.Name, Span.FromBounds(name.StartIndex, name.EndIndex)), Classifications.Module); + } + } + } + } + } + return base.Walk(node); + } + + public override bool Walk(FromImportStatement node) { + Debug.Assert(_head != null); + + if (node.Root != null) { + foreach (var name in node.Root.Names) { + if (name != null && !string.IsNullOrEmpty(name.Name)) { + AddSpan(Tuple.Create(name.Name, Span.FromBounds(name.StartIndex, name.EndIndex)), Classifications.Module); + } + } + } + if (node.Names != null) { + for (int i = 0; i < node.Names.Count; ++i) { + var name = node.Names[i]; + var asName = (i < node.AsNames?.Count) ? node.AsNames[i] : null; + if (!string.IsNullOrEmpty(asName?.Name)) { + _head.Names.Add(Tuple.Create(asName.Name, Span.FromBounds(name.StartIndex, name.EndIndex))); + _head.Names.Add(Tuple.Create(asName.Name, Span.FromBounds(asName.StartIndex, asName.EndIndex))); + } else if (!string.IsNullOrEmpty(name?.Name)) { + _head.Names.Add(Tuple.Create(name.Name, Span.FromBounds(name.StartIndex, name.EndIndex))); + } + } + } + return base.Walk(node); + } + + + + public override void PostWalk(ClassDefinition node) { + EndScope(true); + Debug.Assert(_head != null); + base.PostWalk(node); + } + + public override void PostWalk(FunctionDefinition node) { + EndScope(false); + Debug.Assert(_head != null); + base.PostWalk(node); + } + + public override void PostWalk(DictionaryComprehension node) { + EndScope(false); + Debug.Assert(_head != null); + base.PostWalk(node); + } + + public override void PostWalk(ListComprehension node) { + EndScope(false); + Debug.Assert(_head != null); + base.PostWalk(node); + } + + public override void PostWalk(GeneratorExpression node) { + EndScope(false); + Debug.Assert(_head != null); + base.PostWalk(node); + } + + + public override bool Walk(AwaitExpression node) { + AddSpan(Tuple.Create("", new Span(node.StartIndex, 5)), Classifications.Keyword); + return base.Walk(node); + } + + public override bool Walk(ForStatement node) { + if (node.IsAsync) { + AddSpan(Tuple.Create("", new Span(node.StartIndex, 5)), Classifications.Keyword); + } + return base.Walk(node); + } + + public override bool Walk(WithStatement node) { + if (node.IsAsync) { + AddSpan(Tuple.Create("", new Span(node.StartIndex, 5)), Classifications.Keyword); + } + return base.Walk(node); + } + + private static readonly HashSet RegexFunctionNames = new HashSet { + "compile", + "escape", + "findall", + "finditer", + "fullmatch", + "match", + "search", + "split", + "sub", + "subn" + }; + + public override bool Walk(CallExpression node) { + bool isRegex = false; + + if (node.Target is MemberExpression me && RegexFunctionNames.Contains(me.Name) && me.Target is NameExpression target) { + if (_analysis.GetValues(target.Name, me.GetStart(_ast)).Any(m => m is IModule && m.Name == "re")) { + isRegex = true; + } + } else if (node.Target is NameExpression ne && RegexFunctionNames.Contains(ne.Name)) { + if (_analysis.GetValues(ne.Name, ne.GetStart(_ast)).OfType() + .Any(f => f.Function?.DeclaringType == null && f.Function?.DeclaringModule.Name == "re")) { + isRegex = true; + } + } + + if (isRegex && node.Args != null && node.Args.Count > 0 && node.Args[0].Expression is ConstantExpression ce) { + if (ce.Value is string || ce.Value is AsciiString) { + AddSpan(Tuple.Create("", Span.FromBounds(ce.StartIndex, ce.EndIndex)), Classifications.RegexLiteral); + } + } + + return base.Walk(node); + } + + public struct Span { + public readonly int Start, Length; + + public Span(int start, int length) { + Start = start; + Length = length; + } + + public static Span FromBounds(int start, int end) { + return new Span(start, end - start); + } + } + + public struct Classification { + public readonly Span Span; + public readonly string Type; + + public Classification(Span span, string type) { + Span = span; + Type = type; + } + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/MethodExtractor.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/MethodExtractor.cs new file mode 100644 index 000000000..df22da077 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/MethodExtractor.cs @@ -0,0 +1,762 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Microsoft.PythonTools.Analysis; +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Intellisense { + using AP = AnalysisProtocol; + + // TODO: Move this class to Analysis and make it public + + class MethodExtractor { + private readonly PythonAst _ast; + private readonly string _code; + + public MethodExtractor(PythonAst ast, string code) { + _code = code ?? throw new ArgumentNullException(nameof(code)); + _ast = ast ?? throw new ArgumentNullException(nameof(ast)); + } + + public AP.ExtractMethodResponse ExtractMethod(AP.ExtractMethodRequest input, int version) { + // tighten up the selection so that we don't expand into any enclosing nodes because we overlap w/ their white space... + var selectionStart = input.startIndex; + while (selectionStart < _code.Length && + Char.IsWhiteSpace(_code[selectionStart])) { + selectionStart++; + } + + var selectionEnd = input.endIndex; + if (selectionEnd == _code.Length) { + selectionEnd -= 1; + } + while (selectionEnd >= 0 && char.IsWhiteSpace(_code[selectionEnd])) { + selectionEnd -= 1; + } + + var walker = new EnclosingNodeWalker(_ast, selectionStart, selectionEnd); + _ast.Walk(walker); + + Debug.Assert(walker.Target != null); + if (walker.Target == null) { + return new AP.ExtractMethodResponse() { + cannotExtractReason = AP.CannotExtractReason.InvalidTargetSelected + }; + } + + bool expanded = false; + // expand the selection if we aren't currently covering a full expression/statement + if (!walker.Target.IsValidSelection) { + return new AP.ExtractMethodResponse() { + cannotExtractReason = AP.CannotExtractReason.InvalidExpressionSelected + }; + } + + expanded = WasSelectionExpanded(walker.Target, selectionStart, selectionEnd); + + // check for things we cannot handle + if (!IsValidExtraction(walker.Target, out var failureReason)) { + return new AP.ExtractMethodResponse() { + // Note: this returns the unformatted error + cannotExtractReason = failureReason + }; + } + + // get the variables which are read by the selected statement(s) + var varCollector = new InnerVariableWalker(_ast); + walker.Target.Walk(varCollector); + + // Walk the target to understand flow control and definite assignment to further understand + // what we need to flow in. For example if a variable is assigned to both in the un-extracted + // and extracted code but it's definitely assigned in the extracted code then we don't need + // to flow the variable in. + + // Run flow checker, first on any nested scopes... + foreach (ScopeStatement scope in varCollector._scopes) { + FlowChecker.Check(scope); + } + + // then on our extracted code + var parent = walker.Target.Parents[walker.Target.Parents.Count - 1]; + HashSet readBeforeInit; + FlowChecker extractedChecker = null; + if (parent.ScopeVariables != null) { + extractedChecker = new FlowChecker(parent); + + walker.Target.Walk(extractedChecker); + readBeforeInit = extractedChecker.ReadBeforeInitializedVariables; + } else { + readBeforeInit = new HashSet(); + } + + // then on code which follows our extracted body + var afterStmts = walker.Target.GetStatementsAfter(); + HashSet readByFollowingCodeBeforeInit = null; + var parentNode = walker.Target.Parents[walker.Target.Parents.Count - 1]; + var outputVars = new HashSet(); + if (parentNode.ScopeVariables != null) { + var checker = new FlowChecker(parentNode); + + foreach (var afterStmt in afterStmts) { + afterStmt.Walk(checker); + } + + readByFollowingCodeBeforeInit = checker.ReadBeforeInitializedVariables; + + foreach (var variable in varCollector._allWrittenVariables) { + if (variable != null && variable.Scope is PythonAst) { + // global variable assigned to in outer scope, and left with + // a valid value (not deleted) from the extracted code. We + // need to pass the value back out and assign to it in the + // global scope. + if (!checker.IsInitialized(variable) && + extractedChecker.IsAssigned(variable)) { + outputVars.Add(variable); + } + } + } + } + + // collect any nested scopes, and see if they read any free variables + var scopeCollector = new ScopeCollector(); + foreach (var afterStmt in afterStmts) { + afterStmt.Walk(scopeCollector); + } + + foreach (var scope in scopeCollector._scopes) { + if (scope.FreeVariables != null) { + foreach (var freeVar in scope.FreeVariables) { + if (varCollector._allWrittenVariables.Contains(freeVar)) { + // we've assigned to a variable accessed from an inner + // scope, we need to get the value of the variable back out. + outputVars.Add(freeVar); + } + } + } + } + + + // discover any variables which are consumed and need to be available as outputs... + var outputCollector = new OuterVariableWalker(_ast, walker.Target, varCollector, readBeforeInit, readByFollowingCodeBeforeInit, outputVars); + _ast.Walk(outputCollector); + + if (outputCollector._outputVars.Count > 0 && + walker.Target.ContainsReturn) { + return new AP.ExtractMethodResponse { + cannotExtractReason = AP.CannotExtractReason.MethodAssignsVariablesAndReturns + }; + } + + var targetScope = walker.Target.Parents[input.scope ?? 0]; + var creator = new OutOfProcExtractedMethodCreator( + _ast, + walker.Target.Parents, + outputCollector._inputVars, + outputCollector._outputVars, + walker.Target, + input.indentSize, + !input.convertTabsToSpaces, + input.newLine, + input.name, + input.parameters ?? new string[0], + walker.Target.Parents[input.scope ?? 0] + ); + + // get the new method body... + var newMethod = creator.GetExtractionResult(); + + var changes = new List(); + + var callChange = DocumentChange.Replace(walker.Target.StartIncludingIndentation, walker.Target.End, newMethod.Call); + var methodChange = DocumentChange.Insert(newMethod.Method, walker.Target.InsertLocations[targetScope]); + if (callChange.ReplacedSpan.Start < methodChange.ReplacedSpan.Start) { + changes.Add(callChange); + changes.Add(methodChange); + } else { + changes.Add(methodChange); + changes.Add(callChange); + } + + List scopes = new List(); + for(int i = 0; i x.Name).ToArray(), + scopes = scopes.ToArray(), + wasExpanded = expanded, + startLine = walker.Target.StartIncludingIndentation.Line, + startCol = walker.Target.StartIncludingIndentation.Column, + endLine = walker.Target.End.Line, + endCol = walker.Target.End.Column, + version = version + }; + } + + private string[] GetScopeVariables(ScopeStatement scope, HashSet inputVars) { + List res = new List(); + foreach (var variable in inputVars) { + var variableScope = variable.Scope; + + var parentScope = scope; + // are these variables a child of the target scope so we can close over them? + while (parentScope != null && parentScope != variableScope) { + parentScope = parentScope.Parent; + } + + if (parentScope != null) { + res.Add(variable.Name); + } + } + return res.ToArray(); + } + + private string GetScopeType(ScopeStatement scope) { + if (scope is ClassDefinition) { + return "class"; + } else if (scope is FunctionDefinition) { + return "function"; + } + + return "global"; + } + + private bool WasSelectionExpanded(SelectionTarget target, int selectionStart, int selectionEnd) { + int startIndex = target.Ast.LocationToIndex(target.StartIncludingIndentation); + if (startIndex != selectionStart) { + for (var curChar = selectionStart - 1; curChar >= startIndex; curChar -= 1) { + if (!Char.IsWhiteSpace(_code[curChar])) { + return true; + } + } + } + int endIndex = target.Ast.LocationToIndex(target.End); + if (endIndex != selectionEnd) { + for (var curChar = selectionEnd + 1; curChar < endIndex; curChar += 1) { + if (!Char.IsWhiteSpace(_code[curChar])) { + return true; + } + } + } + return false; + } + + private static bool IsValidExtraction(SelectionTarget target, out AP.CannotExtractReason failureReason) { + if (target.Parents[target.Parents.Count - 1] is ClassDefinition) { + failureReason = AP.CannotExtractReason.StatementsFromClassDefinition; + return false; + } + + var breakContinueWalker = new ContinueBreakWalker(); + target.Walk(breakContinueWalker); + if (breakContinueWalker.ContainsBreak) { + failureReason = AP.CannotExtractReason.SelectionContainsBreakButNotEnclosingLoop; + return false; + } else if (breakContinueWalker.ContainsContinue) { + failureReason = AP.CannotExtractReason.SelectionContainsContinueButNotEnclosingLoop; + return false; + } + + var yieldWalker = new YieldWalker(); + target.Walk(yieldWalker); + if (yieldWalker.ContainsYield) { + failureReason = AP.CannotExtractReason.ContainsYieldExpression; + return false; + } + + var importStarWalker = new ImportStarWalker(); + target.Walk(importStarWalker); + if (importStarWalker.ContainsImportStar) { + failureReason = AP.CannotExtractReason.ContainsFromImportStar; + return false; + } + + var returnWalker = new ReturnWalker(); + target.Walk(returnWalker); + if (returnWalker.ContainsReturn && !returnWalker.Returns) { + failureReason = AP.CannotExtractReason.SelectionContainsReturn; + return false; + } + + target.ContainsReturn = returnWalker.ContainsReturn; + failureReason = AP.CannotExtractReason.None; + return true; + } + + class ReturnWalker : PythonWalker { + public bool ContainsReturn, Returns; + private bool _raises; + + public override bool Walk(ReturnStatement node) { + Returns = true; + ContainsReturn = true; + return base.Walk(node); + } + + public override bool Walk(RaiseStatement node) { + _raises = true; + return base.Walk(node); + } + + public override bool Walk(IfStatement node) { + bool allReturn = true; + for (int i = 0; i < node.TestsInternal.Length; i++) { + _raises = Returns = false; + node.TestsInternal[i].Body.Walk(this); + + allReturn &= Returns || _raises; + } + + _raises = Returns = false; + if (node.ElseStatement != null) { + node.ElseStatement.Walk(this); + + allReturn &= Returns || _raises; + } else { + allReturn = false; + } + + + Returns = allReturn; + return false; + } + + public override bool Walk(ForStatement node) { + WalkLoop(node.Body, node.Else); + return false; + } + + public override bool Walk(WhileStatement node) { + WalkLoop(node.Body, node.ElseStatement); + return false; + } + + private void WalkLoop(Statement body, Statement elseStmt) { + bool allReturn = true; + + _raises = Returns = false; + body.Walk(this); + + allReturn &= Returns || _raises; + + if (elseStmt != null) { + _raises = false; + elseStmt.Walk(this); + allReturn &= Returns || _raises; + } + + Returns = allReturn; + } + + public override bool Walk(SuiteStatement node) { + foreach (var statement in node.Statements) { + if (statement is BreakStatement || statement is ContinueStatement) { + // code after here is unreachable + break; + } + + Returns = false; + statement.Walk(this); + if (Returns) { + // rest of the code is unreachable... + break; + } + } + return false; + } + + public override bool Walk(TryStatement node) { + node.Body.Walk(this); + + if (node.Handlers != null && node.Handlers.Count > 0) { + // treat any exceptions from the body as handled, any exceptions + // from the handlers/else are not handled. + _raises = false; + + foreach (var handler in node.Handlers) { + handler.Walk(this); + } + } + + if (node.Finally != null) { + node.Finally.Walk(this); + } + + if (node.Else != null) { + node.Else.Walk(this); + } + + return false; + } + + public override bool Walk(FunctionDefinition node) { + return false; + } + + public override bool Walk(ClassDefinition node) { + return false; + } + } + + class ContinueBreakWalker : PythonWalker { + public bool ContainsBreak, ContainsContinue; + + public override bool Walk(ForStatement node) { + return false; + } + + public override bool Walk(WhileStatement node) { + return false; + } + + public override bool Walk(FunctionDefinition node) { + return false; + } + + public override bool Walk(ContinueStatement node) { + if (!ContainsBreak) { + ContainsContinue = true; + } + return base.Walk(node); + } + + public override bool Walk(BreakStatement node) { + if (!ContainsContinue) { + ContainsBreak = true; + } + return base.Walk(node); + } + } + + class YieldWalker : PythonWalker { + public bool ContainsYield; + + public override bool Walk(FunctionDefinition node) { + return false; + } + + public override bool Walk(YieldExpression node) { + ContainsYield = true; + return base.Walk(node); + } + + public override bool Walk(YieldFromExpression node) { + ContainsYield = true; + return base.Walk(node); + } + } + + class ImportStarWalker : PythonWalker { + public bool ContainsImportStar; + + public override bool Walk(FunctionDefinition node) { + return false; + } + + public override bool Walk(ClassDefinition node) { + return false; + } + + public override bool Walk(FromImportStatement node) { + if (node.Names.Count == 1 && node.Names[0].Name == "*") { + ContainsImportStar = true; + } + return base.Walk(node); + } + } + + /// + /// Inspects the variables used in the surrounding code to figure out which ones need to be flowed in + /// and which ones need to be returned based upon the variables we collected which are read/assigned + /// from the code being extracted. + /// + class OuterVariableWalker : AssignmentWalker { + private readonly PythonAst _root; + private readonly InnerVariableWalker _inputCollector; + private readonly DefineWalker _define; + private readonly SelectionTarget _target; + internal readonly HashSet _outputVars; + internal readonly HashSet _inputVars = new HashSet(); + private readonly HashSet _readBeforeInitialized, _readByFollowingCodeBeforeInit; + private bool _inLoop = false; + + public OuterVariableWalker(PythonAst root, SelectionTarget target, InnerVariableWalker inputCollector, HashSet readBeforeInitialized, HashSet readByFollowingCodeBeforeInit, HashSet outputVars) { + _root = root; + _target = target; + _inputCollector = inputCollector; + _readBeforeInitialized = readBeforeInitialized; + _readByFollowingCodeBeforeInit = readByFollowingCodeBeforeInit; + _outputVars = outputVars; + _define = new DefineWalker(this); + } + + public override AssignedNameWalker Define { + get { return _define; } + } + + public override bool Walk(FunctionDefinition node) { + if (!node.IsLambda) { + _define.WalkName(node.NameExpression, node.GetVariableReference(_root)); + } + + bool oldInLoop = _inLoop; + _inLoop = false; + var res = base.Walk(node); + _inLoop = oldInLoop; + return res; + } + + public override bool Walk(ClassDefinition node) { + _define.WalkName(node.NameExpression, node.GetVariableReference(_root)); + + bool oldInLoop = _inLoop; + _inLoop = false; + var res = base.Walk(node); + _inLoop = oldInLoop; + return res; + } + + public override bool Walk(WhileStatement node) { + if (node.Test != null) { + node.Test.Walk(this); + } + if (node.Body != null) { + bool oldInLoop = _inLoop; + _inLoop = true; + node.Body.Walk(this); + _inLoop = oldInLoop; + } + if (node.ElseStatement != null) { + node.ElseStatement.Walk(this); + } + return false; + } + + public override bool Walk(ForStatement node) { + if (node.Left != null) { + node.Left.Walk(Define); + } + + if (node.List != null) { + node.List.Walk(this); + } + if (node.Body != null) { + bool oldInLoop = _inLoop; + _inLoop = true; + node.Body.Walk(this); + _inLoop = oldInLoop; + } + if (node.Else != null) { + node.Else.Walk(this); + } + return false; + } + + public override bool Walk(NameExpression node) { + var reference = node.GetVariableReference(_root); + if (reference != null && !_inputCollector._allReads.Contains(reference) && !_inputCollector._allWrites.Contains(reference)) { + // this variable is referenced outside of the refactored code + if (node.GetStart(_root) < _target.StartIncludingIndentation) { + // it's read before the extracted code, we don't care... + } else { + Debug.Assert(node.GetEnd(_root) > _target.End, "didn't reference variable in extracted range"); + + // it's read after the extracted code, if its written to in the refactored + // code we need to include it as an output + if (_inputCollector._allWrittenVariables.Contains(reference.Variable) && + (_readByFollowingCodeBeforeInit == null || _readByFollowingCodeBeforeInit.Contains(reference.Variable))) { + // the variable is written to by the refactored code + _outputVars.Add(reference.Variable); + } + } + } + + return true; + } + + public override bool Walk(Parameter node) { + var variable = node.GetVariable(_root); + if (ReadFromExtractedCode(variable)) { + _inputVars.Add(variable); + } + + return base.Walk(node); + } + + private bool ReadFromExtractedCode(PythonVariable variable) { + return _readBeforeInitialized.Contains(variable) && + _inputCollector._allReadVariables.Contains(variable); + } + + class DefineWalker : AssignedNameWalker { + private readonly OuterVariableWalker _collector; + + public DefineWalker(OuterVariableWalker collector) { + _collector = collector; + } + + public override bool Walk(NameExpression node) { + var reference = node.GetVariableReference(_collector._root); + + return WalkName(node, reference); + } + + internal bool WalkName(NameExpression node, PythonReference reference) { + if (_collector.ReadFromExtractedCode(reference.Variable)) { + if ((!_collector._inputCollector._allReads.Contains(reference) && + !_collector._inputCollector._allWrites.Contains(reference))) { + + // the variable is assigned outside the refactored code + if (node.GetStart(_collector._root) < _collector._target.StartIncludingIndentation) { + // it's assigned before the extracted code + _collector._inputVars.Add(reference.Variable); + } else { + Debug.Assert(node.GetEnd(_collector._root) > _collector._target.End); + // it's assigned afterwards, we don't care... + } + } else if (_collector._readBeforeInitialized.Contains(reference.Variable) && + _collector._inputCollector._allWrites.Contains(reference) && + _collector._inLoop) { + // we read an un-initialized value, so it needs to be passed in. If we + // write to it as well then we need to pass it back out for future calls. + _collector._outputVars.Add(reference.Variable); + } + } + + return false; + } + } + } + + class ScopeCollector : PythonWalker { + internal readonly List _scopes = new List(); + + public override void PostWalk(ClassDefinition node) { + _scopes.Add(node); + base.PostWalk(node); + } + + public override void PostWalk(FunctionDefinition node) { + _scopes.Add(node); + base.PostWalk(node); + } + } + + /// + /// Walks the code which is being extracted and collects all the variables which are read from and written to. + /// + class InnerVariableWalker : AssignmentWalker { + private readonly PythonAst _root; + internal readonly HashSet _allReads = new HashSet(); + internal readonly HashSet _allWrites = new HashSet(); + internal readonly HashSet _allWrittenVariables = new HashSet(); + internal readonly HashSet _allReadVariables = new HashSet(); + internal readonly List _scopes = new List(); + + private readonly DefineWalker _define; + + public InnerVariableWalker(PythonAst root) { + _root = root; + _define = new DefineWalker(this); + } + + public override AssignedNameWalker Define { + get { return _define; } + } + + public override bool Walk(NameExpression node) { + var reference = node.GetVariableReference(_root); + + if (reference != null) { + _allReads.Add(reference); + _allReadVariables.Add(reference.Variable); + } + + return true; + } + + public override void PostWalk(ClassDefinition node) { + _scopes.Add(node); + _allWrites.Add(node.GetVariableReference(_root)); + _allWrittenVariables.Add(node.Variable); + base.PostWalk(node); + } + + public override void PostWalk(FunctionDefinition node) { + _scopes.Add(node); + _allWrites.Add(node.GetVariableReference(_root)); + _allWrittenVariables.Add(node.Variable); + base.PostWalk(node); + } + + public override bool Walk(FromImportStatement node) { + var vars = node.Variables; + var refs = node.GetReferences(_root); + if (refs != null) { // from .. import * will have null refs + for (int i = 0; i < vars.Length; i++) { + if (vars[i] != null) { + _allWrites.Add(refs[i]); + _allWrittenVariables.Add(vars[i]); + } + } + } + return base.Walk(node); + } + + public override bool Walk(ImportStatement node) { + var vars = node.Variables; + var refs = node.GetReferences(_root); + for (int i = 0; i < vars.Length; i++) { + if (vars[i] != null) { + _allWrites.Add(refs[i]); + _allWrittenVariables.Add(vars[i]); + } + } + return base.Walk(node); + } + + class DefineWalker : AssignedNameWalker { + private readonly InnerVariableWalker _collector; + + public DefineWalker(InnerVariableWalker collector) { + _collector = collector; + } + + public override bool Walk(NameExpression node) { + var reference = node.GetVariableReference(_collector._root); + + _collector._allWrites.Add(reference); + _collector._allWrittenVariables.Add(reference.Variable); + return false; + } + } + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutOfProcMethodExtractor.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutOfProcMethodExtractor.cs new file mode 100644 index 000000000..d9d3247d6 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutOfProcMethodExtractor.cs @@ -0,0 +1,38 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Globalization; +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Intellisense { + using AP = AnalysisProtocol; + + // TODO: This class should transform the request and pass to MethodExtractor + // MethodExtractor should move to Analysis and remove all direct depnedencies + // on AnalysisProtocol. For now, we are keeping it here to save effort. + + class OutOfProcMethodExtractor { + private readonly MethodExtractor _extractor; + + public OutOfProcMethodExtractor(PythonAst ast, string code) { + _extractor = new MethodExtractor(ast, code); + } + + public AP.ExtractMethodResponse ExtractMethod(AP.ExtractMethodRequest input, int version) { + return _extractor.ExtractMethod(input, version); + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutOfProcProjectAnalyzer.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutOfProcProjectAnalyzer.cs new file mode 100644 index 000000000..57f0f4792 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutOfProcProjectAnalyzer.cs @@ -0,0 +1,1699 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.LanguageServer.Implementation; +using Microsoft.PythonTools.Analysis; +using Microsoft.PythonTools.Analysis.Analyzer; +using Microsoft.PythonTools.Analysis.Documentation; +using Microsoft.PythonTools.Analysis.Infrastructure; +using Microsoft.PythonTools.Interpreter; +using Microsoft.PythonTools.Ipc.Json; +using Microsoft.PythonTools.Parsing; +using Microsoft.PythonTools.Parsing.Ast; +using Microsoft.PythonTools.Projects; + +namespace Microsoft.PythonTools.Intellisense { + using AP = AnalysisProtocol; + using LS = Microsoft.PythonTools.Analysis.LanguageServer; + using LS2 = Microsoft.Python.LanguageServer; + + /// + /// Performs centralized parsing and analysis of Python source code for a remotely running process. + /// + /// This class is responsible for maintaining the up-to-date analysis of the active files being worked + /// on inside of a single proejct. + /// + /// This class is built upon the core PythonAnalyzer class which provides basic analysis services. This class + /// maintains the thread safety invarients of working with that class, handles parsing of files as they're + /// updated via interfacing w/ the remote process editor APIs, and supports adding additional files to the + /// analysis. + /// + public sealed class OutOfProcProjectAnalyzer : IDisposable { + private readonly Server _server; + private readonly Dictionary _extensions; + private readonly Action _log; + + private bool _isDisposed; + + private readonly Connection _connection; + + public OutOfProcProjectAnalyzer(Stream writer, Stream reader, Action log) { + _server = new Server(); + _server.OnParseComplete += OnParseComplete; + _server.OnAnalysisComplete += OnAnalysisComplete; + _server.OnLogMessage += Server_OnLogMessage; + _server.OnPublishDiagnostics += OnPublishDiagnostics; + _server.AnalysisQueue.AnalysisComplete += AnalysisQueue_Complete; + _server.AnalysisQueue.AnalysisAborted += AnalysisQueue_Aborted; + + _log = log; + Options = new AP.AnalysisOptions(); + + _extensions = new Dictionary(); + + _connection = new Connection( + writer, + true, + reader, + true, + RequestHandler, + AP.RegisteredTypes, + "Analyzer" + ); + _connection.EventReceived += ConectionReceivedEvent; + if (!string.IsNullOrEmpty(_connection.LogFilename)) { + _log?.Invoke($"Connection log: {_connection.LogFilename}"); + } + } + + private void Server_OnLogMessage(object sender, LS2.LogMessageEventArgs e) { + if (_log != null && Options.traceLevel.HasValue && e.type <= Options.traceLevel.Value) { + _log(e.message); + _connection?.SendEventAsync(new AP.AnalyzerWarningEvent { message = e.message }).DoNotWait(); + } + } + + private void AnalysisQueue_Aborted(object sender, EventArgs e) { + _connection.Dispose(); + } + + private void ConectionReceivedEvent(object sender, EventReceivedEventArgs e) { + switch (e.Event.name) { + case AP.ModulesChangedEvent.Name: OnModulesChanged(this, EventArgs.Empty); break; + case AP.FileChangedEvent.Name: OnFileChanged((AP.FileChangedEvent)e.Event); break; + } + } + + private async Task RequestHandler(RequestArgs requestArgs, Func done) { + Response response; + var command = requestArgs.Command; + var request = requestArgs.Request; + + // These commands send their own responses, and then we return. + switch (command) { + case AP.AddFileRequest.Command: await AnalyzeFileAsync((AP.AddFileRequest)request, done); return; + case AP.AddBulkFileRequest.Command: await AnalyzeFileAsync((AP.AddBulkFileRequest)request, done); return; + } + + // These commands return a response, which we then send. + switch (command) { + case AP.UnloadFileRequest.Command: response = await UnloadFile((AP.UnloadFileRequest)request); break; + case AP.CompletionsRequest.Command: response = await GetCompletions(request); break; + case AP.GetAllMembersRequest.Command: response = await GetAllMembers(request); break; + case AP.GetModulesRequest.Command: response = await GetModules(request); break; + case AP.SignaturesRequest.Command: response = await GetSignatures((AP.SignaturesRequest)request); break; + case AP.QuickInfoRequest.Command: response = await GetQuickInfo((AP.QuickInfoRequest)request); break; + case AP.AnalyzeExpressionRequest.Command: response = await AnalyzeExpression((AP.AnalyzeExpressionRequest)request); break; + case AP.OutliningRegionsRequest.Command: response = GetOutliningRegions((AP.OutliningRegionsRequest)request); break; + case AP.NavigationRequest.Command: response = await GetNavigationsAsync((AP.NavigationRequest)request); break; + case AP.FileUpdateRequest.Command: response = await UpdateContent((AP.FileUpdateRequest)request); break; + case AP.AddImportRequest.Command: response = AddImportRequest((AP.AddImportRequest)request); break; + case AP.IsMissingImportRequest.Command: response = IsMissingImport((AP.IsMissingImportRequest)request); break; + case AP.AvailableImportsRequest.Command: response = AvailableImports((AP.AvailableImportsRequest)request); break; + case AP.FormatCodeRequest.Command: response = FormatCode((AP.FormatCodeRequest)request); break; + case AP.RemoveImportsRequest.Command: response = RemoveImports((AP.RemoveImportsRequest)request); break; + case AP.ExtractMethodRequest.Command: response = ExtractMethod((AP.ExtractMethodRequest)request); break; + case AP.AnalysisStatusRequest.Command: response = AnalysisStatus(); break; + case AP.LocationNameRequest.Command: response = GetLocationName((AP.LocationNameRequest)request); break; + case AP.ProximityExpressionsRequest.Command: response = GetProximityExpressions((AP.ProximityExpressionsRequest)request); break; + case AP.AnalysisClassificationsRequest.Command: response = GetAnalysisClassifications((AP.AnalysisClassificationsRequest)request); break; + case AP.MethodInsertionLocationRequest.Command: response = GetMethodInsertionLocation((AP.MethodInsertionLocationRequest)request); break; + case AP.MethodInfoRequest.Command: response = GetMethodInfo((AP.MethodInfoRequest)request); break; + case AP.FindMethodsRequest.Command: response = FindMethods((AP.FindMethodsRequest)request); break; + case AP.AddReferenceRequest.Command: response = AddReference((AP.AddReferenceRequest)request); break; + case AP.RemoveReferenceRequest.Command: response = RemoveReference((AP.RemoveReferenceRequest)request); break; + case AP.SetSearchPathRequest.Command: response = SetSearchPath((AP.SetSearchPathRequest)request); break; + case AP.ModuleImportsRequest.Command: response = GetModuleImports((AP.ModuleImportsRequest)request); break; + case AP.ValueDescriptionRequest.Command: response = GetValueDescriptions((AP.ValueDescriptionRequest)request); break; + case AP.LoadExtensionRequest.Command: response = LoadExtensionRequest((AP.LoadExtensionRequest)request); break; + case AP.ExtensionRequest.Command: response = ExtensionRequest((AP.ExtensionRequest)request); break; + case AP.ExpressionAtPointRequest.Command: response = ExpressionAtPoint((AP.ExpressionAtPointRequest)request); break; + case AP.InitializeRequest.Command: response = await Initialize((AP.InitializeRequest)request); break; + case AP.SetAnalysisOptionsRequest.Command: response = SetAnalysisOptions((AP.SetAnalysisOptionsRequest)request); break; + case AP.LanguageServerRequest.Command: response = await ProcessLanguageServerRequest((AP.LanguageServerRequest)request); break; + case AP.ExitRequest.Command: throw new OperationCanceledException(); + default: + throw new InvalidOperationException("Unknown command"); + } + + await done(response); + } + + private async Task ProcessLanguageServerRequest(AP.LanguageServerRequest request) { + try { + var body = (Newtonsoft.Json.Linq.JObject)request.body; + object result = null; + + switch (request.name) { + case "textDocument/completion": result = await _server.Completion(body.ToObject(), CancellationToken.None); break; + case "textDocument/hover": result = await _server.Hover(body.ToObject(), CancellationToken.None); break; + case "textDocument/definition": result = await _server.GotoDefinition(body.ToObject(), CancellationToken.None); break; + case "textDocument/references": result = await _server.FindReferences(body.ToObject(), CancellationToken.None); break; + case "textDocument/signatureHelp": result = await _server.SignatureHelp(body.ToObject(), CancellationToken.None); break; + } + + if (result != null) { + return new AP.LanguageServerResponse { body = result }; + } + + return new AP.LanguageServerResponse { error = "Unknown command: " + request.name }; + } catch (Exception ex) { + return new AP.LanguageServerResponse { error = ex.ToString() }; + } + } + + internal void ReportUnhandledException(Exception ex) { + SendUnhandledExceptionAsync(ex).DoNotWait(); + // Allow some time for the other threads to write the event before + // we (probably) come crashing down. + Thread.Sleep(100); + } + + private async Task SendUnhandledExceptionAsync(Exception ex) { + try { + Debug.Fail(ex.ToString()); + await _connection.SendEventAsync( + new AP.UnhandledExceptionEvent(ex) + ).ConfigureAwait(false); + } catch (Exception) { + // We're in pretty bad state, but nothing useful we can do about + // it. + Debug.Fail("Unhandled exception reporting unhandled exception"); + } + } + + private Response IncorrectFileType() { + throw new InvalidOperationException("File was not correct type"); + } + + private Response IncorrectBufferId(Uri documentUri) { + throw new InvalidOperationException($"Buffer was not valid in file {documentUri?.AbsoluteUri ?? "(null)"}"); + } + + private IPythonInterpreterFactory LoadInterpreterFactory(AP.InterpreterInfo info) { + if (string.IsNullOrEmpty(info?.assembly) || string.IsNullOrEmpty(info?.typeName)) { + return null; + } + + var assembly = File.Exists(info.assembly) ? AssemblyName.GetAssemblyName(info.assembly) : new AssemblyName(info.assembly); + var type = Assembly.Load(assembly).GetType(info.typeName, true); + + return (IPythonInterpreterFactory)Activator.CreateInstance( + type, + BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance, + null, + new object[] { info.properties }, + CultureInfo.CurrentCulture + ); + } + + private IAnalysisExtension LoadAnalysisExtension(AP.LoadExtensionRequest info) { + if (string.IsNullOrEmpty(info?.assembly) || string.IsNullOrEmpty(info?.typeName)) { + return null; + } + + var assembly = File.Exists(info.assembly) ? AssemblyName.GetAssemblyName(info.assembly) : new AssemblyName(info.assembly); + var type = Assembly.Load(assembly).GetType(info.typeName, true); + + return (IAnalysisExtension)Activator.CreateInstance( + type, + BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance, + null, + new object[0], + CultureInfo.CurrentCulture + ); + } + + private async Task Initialize(AP.InitializeRequest request) { + try { + await _server.Initialize(new LS2.InitializeParams { + rootUri = request.rootUri, + initializationOptions = new LS2.PythonInitializationOptions { + interpreter = new LS2.PythonInitializationOptions.Interpreter { + assembly = request.interpreter?.assembly, + typeName = request.interpreter?.typeName, + properties = request.interpreter?.properties + }, + displayOptions = new InformationDisplayOptions { + maxDocumentationLineLength = 30, + trimDocumentationLines = true, + maxDocumentationTextLength = 1024, + trimDocumentationText = true, + maxDocumentationLines = 100 + }, + analysisUpdates = true, + traceLogging = request.traceLogging, + }, + capabilities = new LS2.ClientCapabilities { + python = new LS2.PythonClientCapabilities { + manualFileLoad = !request.analyzeAllFiles, + liveLinting = request.liveLinting + }, + textDocument = new LS2.TextDocumentClientCapabilities { + completion = new LS2.TextDocumentClientCapabilities.CompletionCapabilities { + completionItem = new LS2.TextDocumentClientCapabilities.CompletionCapabilities.CompletionItemCapabilities { + documentationFormat = new[] { LS.MarkupKind.PlainText }, + snippetSupport = false + } + }, + signatureHelp = new LS2.TextDocumentClientCapabilities.SignatureHelpCapabilities { + signatureInformation = new LS2.TextDocumentClientCapabilities.SignatureHelpCapabilities.SignatureInformationCapabilities { + documentationFormat = new[] { LS.MarkupKind.PlainText }, + _shortLabel = true + } + }, + hover = new LS2.TextDocumentClientCapabilities.HoverCapabilities { + contentFormat = new[] { LS.MarkupKind.PlainText } + } + } + } + }, CancellationToken.None); + } catch (Exception ex) { + return new AP.InitializeResponse { + error = ex.Message, + fullError = ex.ToString() + }; + } + + return new AP.InitializeResponse(); + } + + private Response LoadExtensionRequest(AP.LoadExtensionRequest request) { + IAnalysisExtension extension, oldExtension; + + if (Project == null) { + return new AP.LoadExtensionResponse { + error = "Uninitialized analyzer", + fullError = $"Uninitialized analyzer{Environment.NewLine}{new StackTrace()}" + }; + } + + try { + extension = LoadAnalysisExtension(request); + extension.Register(Project); + } catch (Exception ex) { + return new AP.LoadExtensionResponse { + error = ex.Message, + fullError = ex.ToString() + }; + } + + lock (_extensions) { + _extensions.TryGetValue(request.extension, out oldExtension); + _extensions[request.extension] = extension; + } + (oldExtension as IDisposable)?.Dispose(); + + return new AP.LoadExtensionResponse(); + } + + private Response ExtensionRequest(AP.ExtensionRequest request) { + IAnalysisExtension extension; + lock (_extensions) { + if (!_extensions.TryGetValue(request.extension, out extension)) { + return new AP.ExtensionResponse { + error = $"Unknown extension: {request.extension}" + }; + } + } + + try { + return new AP.ExtensionResponse { + response = extension.HandleCommand(request.commandId, request.body) + }; + } catch (Exception ex) { + return new AP.ExtensionResponse { + error = ex.ToString() + }; + } + } + + private Response GetValueDescriptions(AP.ValueDescriptionRequest request) { + var entry = GetPythonEntry(request.documentUri); + if (entry == null) { + return IncorrectFileType(); + } + string[] descriptions = Array.Empty(); + if (entry.Analysis != null) { + var values = entry.Analysis.GetValues( + request.expr, + new SourceLocation( + request.line, + request.column + ) + ); + + descriptions = values.Select(x => x.Description).ToArray(); + } + + return new AP.ValueDescriptionResponse() { + descriptions = descriptions + }; + } + + private Response GetModuleImports(AP.ModuleImportsRequest request) { + var res = Analyzer.GetEntriesThatImportModule( + request.moduleName, + request.includeUnresolved + ); + + return new AP.ModuleImportsResponse() { + modules = res.Select(entry => new AP.ModuleInfo() { + filename = entry.FilePath, + moduleName = entry.ModuleName, + documentUri = entry.DocumentUri + }).ToArray() + }; + } + + private Response SetSearchPath(AP.SetSearchPathRequest request) { + Analyzer.SetSearchPaths(request.dir); + + return new Response(); + } + + private Response RemoveReference(AP.RemoveReferenceRequest request) { + var interp = Interpreter as IPythonInterpreterWithProjectReferences; + if (interp != null) { + interp.RemoveReference(AP.ProjectReference.Convert(request.reference)); + } + return new AP.RemoveReferenceResponse(); + } + + private Response AddReference(AP.AddReferenceRequest request) { + var interp = Interpreter as IPythonInterpreterWithProjectReferences; + if (interp != null) { + interp.AddReferenceAsync(AP.ProjectReference.Convert(request.reference)).Wait(); + } + return new AP.AddReferenceResponse(); + } + + private Response FindMethods(AP.FindMethodsRequest request) { + var analysis = GetPythonEntry(request.documentUri); + + List names = new List(); + if (analysis != null) { + int version; + string code; + var ast = analysis.GetVerbatimAstAndCode( + Analyzer.LanguageVersion, + out version, + out code + ); + + if (ast != null) { + foreach (var classDef in FindClassDef(request.className, ast)) { + SuiteStatement suite = classDef.Body as SuiteStatement; + if (suite != null) { + foreach (var methodCandidate in suite.Statements) { + FunctionDefinition funcDef = methodCandidate as FunctionDefinition; + if (funcDef != null) { + if (request.paramCount != null && request.paramCount != funcDef.Parameters.Length) { + continue; + } + + names.Add(funcDef.Name); + } + } + } + } + } + } + + return new AP.FindMethodsResponse() { + names = names.ToArray() + }; + } + + private Response GetMethodInsertionLocation(AP.MethodInsertionLocationRequest request) { + var analysis = GetPythonEntry(request.documentUri); + if (analysis == null) { + return IncorrectFileType(); + } + + int version; + string code; + var ast = analysis.GetVerbatimAstAndCode( + Analyzer.LanguageVersion, + out version, + out code + ); + + if (ast == null) { + return new AP.MethodInsertionLocationResponse(); + } + + foreach (var classDef in FindClassDef(request.className, ast)) { + int end = classDef.Body.EndIndex; + // insert after the newline at the end of the last statement of the class def + if (code[end] == '\r') { + if (end + 1 < code.Length && + code[end + 1] == '\n') { + end += 2; + } else { + end++; + } + } else if (code[end] == '\n') { + end++; + } + + return new AP.MethodInsertionLocationResponse() { + line = ast.IndexToLocation(end).Line, + column = classDef.Body.GetStart(ast).Column, + version = version + }; + } + + throw new InvalidOperationException("Failed to find class definition"); + } + + private static IEnumerable FindClassDef(string name, PythonAst ast) { + var suiteStmt = ast.Body as SuiteStatement; + foreach (var stmt in suiteStmt.Statements) { + var classDef = stmt as ClassDefinition; + if (classDef != null && + (classDef.Name == name || name == null)) { + yield return classDef; + } + } + } + + private Response GetMethodInfo(AP.MethodInfoRequest request) { + var analysis = GetPythonEntry(request.documentUri); + + if (analysis != null) { + int version; + string code; + var ast = analysis.GetVerbatimAstAndCode( + Project.LanguageVersion, + out version, + out code + ); + + if (ast != null) { + foreach (var classDef in FindClassDef(request.className, ast)) { + SuiteStatement suite = classDef.Body as SuiteStatement; + + if (suite != null) { + foreach (var methodCandidate in suite.Statements) { + FunctionDefinition funcDef = methodCandidate as FunctionDefinition; + if (funcDef != null) { + if (funcDef.Name == request.methodName) { + return new AP.MethodInfoResponse() { + start = funcDef.StartIndex, + end = funcDef.EndIndex, + version = version, + found = true + }; + } + } + } + } + } + } + } + + return new AP.MethodInfoResponse() { + found = false + }; + } + + private Response GetAnalysisClassifications(AP.AnalysisClassificationsRequest request) { + var projEntry = GetPythonEntry(request.documentUri); + + if (projEntry == null) { + return IncorrectFileType(); + } + + var bufferVersion = GetPythonBuffer(request.documentUri); + if (bufferVersion.Ast == null) { + return new AP.AnalysisClassificationsResponse(); + } + + var moduleAnalysis = request.colorNames ? projEntry.Analysis : null; + + var walker = new ClassifierWalker(bufferVersion.Ast, moduleAnalysis); + bufferVersion.Ast.Walk(walker); + + return new AP.AnalysisClassificationsResponse() { + version = bufferVersion.Version, + classifications = walker.Spans.Select(s => new AP.AnalysisClassification { + startLine = s.Span.Start.Line, + startColumn = s.Span.Start.Column, + endLine = s.Span.End.Line, + endColumn = s.Span.End.Column, + type = s.Tag + }).ToArray() + }; + } + + private Response GetProximityExpressions(AP.ProximityExpressionsRequest request) { + var projEntry = GetPythonEntry(request.documentUri); + + var res = new AP.ProximityExpressionsResponse(); + + var tree = projEntry?.Tree; + if (tree == null) { + return res; + } + + int startLine = Math.Max(request.line - request.lineCount + 1, 0); + if (startLine <= request.line) { + var walker = new ProximityExpressionWalker(tree, startLine, request.line); + tree.Walk(walker); + res.names = walker.GetExpressions().ToArray(); + } + + return res; + } + + private Response GetLocationName(AP.LocationNameRequest request) { + var projEntry = GetPythonEntry(request.documentUri); + + var res = new AP.LocationNameResponse(); + + var tree = projEntry?.Tree; + if (tree == null) { + return res; + } + + string foundName = FindNodeInTree(tree, tree.Body as SuiteStatement, request.line); + if (foundName != null) { + res.name = projEntry.ModuleName + "." + foundName; + res.lineOffset = request.column; + } else { + res.name = projEntry.ModuleName; + res.lineOffset = request.column; + } + + return res; + } + + private static string FindNodeInTree(PythonAst tree, SuiteStatement statement, int line) { + if (statement == null) { + return null; + } + + foreach (var node in statement.Statements) { + if (node is FunctionDefinition funcDef) { + var span = funcDef.GetSpan(tree); + if (span.Start.Line <= line && line <= span.End.Line) { + var res = FindNodeInTree(tree, funcDef.Body as SuiteStatement, line); + if (res != null) { + return funcDef.Name + "." + res; + } + return funcDef.Name; + } + } else if (node is ClassDefinition classDef) { + var span = classDef.GetSpan(tree); + if (span.Start.Line <= line && line <= span.End.Line) { + var res = FindNodeInTree(tree, classDef.Body as SuiteStatement, line); + if (res != null) { + return classDef.Name + "." + res; + } + return classDef.Name; + } + } + } + return null; + } + + + private Response AnalysisStatus() { + return new AP.AnalysisStatusResponse() { + itemsLeft = _server.EstimateRemainingWork() + }; + } + + private Response ExtractMethod(AP.ExtractMethodRequest request) { + var projectFile = GetPythonEntry(request.documentUri); + if (projectFile == null) { + return IncorrectFileType(); + } + + int version; + string code; + var ast = projectFile.GetVerbatimAstAndCode( + Project.LanguageVersion, + out version, + out code + ); + + return new OutOfProcMethodExtractor( + ast, + code + ).ExtractMethod(request, version); + } + + private Response RemoveImports(AP.RemoveImportsRequest request) { + var projectFile = GetPythonEntry(request.documentUri); + if (projectFile == null) { + return IncorrectFileType(); + } + + int version; + string code; + var ast = projectFile.GetVerbatimAstAndCode( + Project.LanguageVersion, + out version, + out code + ); + if (ast == null) { + return new AP.RemoveImportsResponse(); + } + var remover = new ImportRemover(ast, code, request.allScopes, ast.LocationToIndex(new SourceLocation(request.line, request.column))); + + return new AP.RemoveImportsResponse() { + changes = remover.RemoveImports().Select(AP.ChangeInfo.FromDocumentChange).ToArray(), + version = version + }; + } + + private Response FormatCode(AP.FormatCodeRequest request) { + var projectFile = GetPythonEntry(request.documentUri); + if (projectFile == null) { + return IncorrectFileType(); + } + + int version; + string code; + var ast = projectFile.GetVerbatimAstAndCode( + Project.LanguageVersion, + out version, + out code + ); + if (ast == null) { + return new AP.FormatCodeResponse(); + } + + int startIndex = ast.LocationToIndex(new SourceLocation(request.startLine, request.startColumn)); + int endIndex = ast.LocationToIndex(new SourceLocation(request.endLine, request.endColumn)); + + var walker = new EnclosingNodeWalker(ast, startIndex, endIndex); + ast.Walk(walker); + + if (walker.Target == null || !walker.Target.IsValidSelection) { + return new AP.FormatCodeResponse(); + } + + var body = walker.Target.GetNode(); + + + var whitspaceStart = walker.Target.StartIncludingIndentation; + + int start = ast.LocationToIndex(walker.Target.StartIncludingLeadingWhiteSpace); + int end = ast.LocationToIndex(walker.Target.End); + if (startIndex > start) { + // the user didn't have any comments selected, don't reformat them + body.SetLeadingWhiteSpace(ast, body.GetIndentationLevel(ast)); + + start = ast.LocationToIndex(walker.Target.StartIncludingIndentation); + } + + int length = end - start; + if (end < code.Length) { + if (code[end] == '\r') { + end++; + length++; + if (end < code.Length && + code[end] == '\n') { + end++; + length++; + } + } else if (code[end] == '\n') { + length++; + } + } + + var selectedCode = code.Substring(start, length); + + return new AP.FormatCodeResponse() { + version = version, + changes = selectedCode.ReplaceByLines( + walker.Target.StartIncludingLeadingWhiteSpace, + body.ToCodeString(ast, request.options), + request.newLine + ).Select(AP.ChangeInfo.FromDocumentChange).ToArray() + }; + } + + private Response AvailableImports(AP.AvailableImportsRequest request) { + return new AP.AvailableImportsResponse { + imports = FindNameInAllModules(request.name) + .Select( + x => new AP.ImportInfo { + importName = x.ImportName, + fromName = x.FromName + } + ) + .Distinct() + .ToArray() + }; + } + + private IEnumerable FindNameInAllModules(string name) { + string pkgName; + + // provide module names first + foreach (var keyValue in Analyzer.Modules.GetModuleStates()) { + var modName = keyValue.Key; + var moduleRef = keyValue.Value; + + if (moduleRef.IsValid) { + // include modules which can be imported + if (modName == name) { + yield return new ExportedMemberInfo(null, modName); + } else if (GetPackageNameIfMatch(name, modName, out pkgName)) { + yield return new ExportedMemberInfo(pkgName, name); + } + } + } + + foreach (var modName in Interpreter.GetModuleNames()) { + if (modName == name) { + yield return new ExportedMemberInfo(null, modName); + } else if (GetPackageNameIfMatch(name, modName, out pkgName)) { + yield return new ExportedMemberInfo(pkgName, name); + } + } + + // then include imported module members + foreach (var keyValue in Analyzer.Modules.GetModuleStates()) { + var modName = keyValue.Key; + var moduleRef = keyValue.Value; + + if (moduleRef.IsValid && moduleRef.ModuleContainsMember(Analyzer._defaultContext, name)) { + yield return new ExportedMemberInfo(modName, name); + } + } + } + + private static bool GetPackageNameIfMatch(string name, string fullName, out string packageName) { + var lastDot = fullName.LastIndexOf('.'); + if (lastDot < 0) { + packageName = null; + return false; + } + + packageName = fullName.Remove(lastDot); + return String.Compare(fullName, lastDot + 1, name, 0, name.Length, StringComparison.Ordinal) == 0; + } + + private Response IsMissingImport(AP.IsMissingImportRequest request) { + var entry = GetPythonEntry(request.documentUri); + var analysis = entry?.Analysis; + if (analysis == null) { + return new AP.IsMissingImportResponse(); + } + + var location = new SourceLocation(request.line, request.column); + var nameExpr = GetFirstNameExpression( + analysis.GetAstFromText( + request.text, + location + ).Body + ); + + if (nameExpr != null && !IsImplicitlyDefinedName(nameExpr)) { + var name = nameExpr.Name; + var hasVariables = analysis.GetVariables(name, location).Any(IsDefinition); + var hasValues = analysis.GetValues(name, location).Any(); + + // if we have type information or an assignment to the variable we won't offer + // an import smart tag. + if (!hasValues && !hasVariables) { + return new AP.IsMissingImportResponse() { + isMissing = true + }; + } + } + + return new AP.IsMissingImportResponse(); + } + + private Response AddImportRequest(AP.AddImportRequest request) { + var projectFile = GetPythonEntry(request.documentUri); + if (projectFile == null) { + return IncorrectFileType(); + } + + string name = request.name; + string fromModule = request.fromModule; + + int version; + var curAst = projectFile.GetVerbatimAst(Project.LanguageVersion, out version); + if (curAst == null) { + return new AP.AddImportResponse(); + } + + var suiteBody = curAst.Body as SuiteStatement; + int start = 0; + if (suiteBody != null) { + foreach (var statement in suiteBody.Statements) { + if (IsDocString(statement as ExpressionStatement)) { + // doc string, import comes after this... + start = statement.EndIndex; + continue; + } + + FromImportStatement fromImport; + + if (statement is ImportStatement) { + if (fromModule == "__future__") { + // we need to insert before normal imports + break; + } + + // we insert after this + start = statement.EndIndex; + } else if ((fromImport = (statement as FromImportStatement)) != null) { + // we might update this, we might insert after + if (fromModule != "__future__" && fromImport.Root.MakeString() == fromModule) { + // update the existing from ... import statement to include the new name. + return new AP.AddImportResponse() { + changes = new[] { UpdateFromImport(curAst, fromImport, name) }, + version = version + }; + } + + start = statement.EndIndex; + } + + break; + } + } + + string newText = MakeImportCode(fromModule, name); + if (start == 0) { + // we're adding it at the beginning of the file, we need a new line + // after the import statement + newText += request.newLine; + } else { + // we're adding it after the end of a statement, we need a newline after + // the statement we're appending after. + newText = request.newLine + newText; + } + + return new AP.AddImportResponse() { + changes = new[] { + AP.ChangeInfo.FromDocumentChange(DocumentChange.Insert(newText, curAst.IndexToLocation(start))) + }, + version = version + }; + } + + public static string MakeImportCode(string fromModule, string name) { + if (string.IsNullOrEmpty(fromModule)) { + return string.Format("import {0}", name); + } else { + return string.Format("from {0} import {1}", fromModule, name); + } + } + + private static AP.ChangeInfo UpdateFromImport( + PythonAst curAst, + FromImportStatement fromImport, + string name + ) { + NameExpression[] names = new NameExpression[fromImport.Names.Count + 1]; + NameExpression[] asNames = fromImport.AsNames == null ? null : new NameExpression[fromImport.AsNames.Count + 1]; + NameExpression newName = new NameExpression(name); + for (int i = 0; i < fromImport.Names.Count; i++) { + names[i] = fromImport.Names[i]; + } + names[fromImport.Names.Count] = newName; + + if (asNames != null) { + for (int i = 0; i < fromImport.AsNames.Count; i++) { + asNames[i] = fromImport.AsNames[i]; + } + } + + var newImport = new FromImportStatement((ModuleName)fromImport.Root, names, asNames, fromImport.IsFromFuture, fromImport.ForceAbsolute, -1); + curAst.CopyAttributes(fromImport, newImport); + + var newCode = newImport.ToCodeString(curAst); + + var span = fromImport.GetSpan(curAst); + int leadingWhiteSpaceLength = (fromImport.GetLeadingWhiteSpace(curAst) ?? "").Length; + return AP.ChangeInfo.FromDocumentChange(DocumentChange.Replace( + new SourceSpan( + span.Start.AddColumns(-leadingWhiteSpaceLength), + span.End + ), newCode + )); + } + + private static bool IsDocString(ExpressionStatement exprStmt) { + ConstantExpression constExpr; + return exprStmt != null && + (constExpr = exprStmt.Expression as ConstantExpression) != null && + (constExpr.Value is string || constExpr.Value is AsciiString); + } + + private IPythonProjectEntry GetPythonEntry(Uri documentUri) { + if (documentUri == null) { + return null; + } + return _server.GetEntry(documentUri) as IPythonProjectEntry; + } + + private VersionedAst GetPythonBuffer(Uri documentUri) { + var entry = GetPythonEntry(documentUri); + if (entry == null) { + return default(VersionedAst); + } + + var parse = entry.GetCurrentParse(); + var ast = parse?.Tree; + var cookie = parse?.Cookie; + + if (cookie is VersionCookie vc) { + int i = _server.GetPart(documentUri); + if (vc.Versions.TryGetValue(i, out var bv)) { + return new VersionedAst { Ast = bv.Ast, Version = bv.Version }; + } + } + return new VersionedAst { Ast = ast, Version = 0 }; + } + + private struct VersionedAst { + public PythonAst Ast; + public int Version; + } + + private Response GetOutliningRegions(AP.OutliningRegionsRequest request) { + var bufferVersion = GetPythonBuffer(request.documentUri); + if (bufferVersion.Ast == null) { + return IncorrectBufferId(request.documentUri); + } + + var walker = new OutliningWalker(bufferVersion.Ast); + bufferVersion.Ast.Walk(walker); + + return new AP.OutliningRegionsResponse() { + tags = walker.GetTags().Select(t => new AP.OutliningTag { + startLine = t.Span.Start.Line, + startCol = t.Span.Start.Column, + endLine = t.Span.End.Line, + endCol = t.Span.End.Column + }).ToArray(), + version = bufferVersion.Version + }; + } + + private async Task GetNavigationsAsync(AP.NavigationRequest request) { + var symbols = await _server.HierarchicalDocumentSymbol(new LS2.DocumentSymbolParams { + textDocument = new LS2.TextDocumentIdentifier { uri = request.documentUri } + }, CancellationToken.None); + + var navs = symbols.Select(ToNavigation).ToArray(); + var bufferVersion = GetPythonBuffer(request.documentUri); + return new AP.NavigationResponse() { + version = bufferVersion.Version, + navigations = navs + }; + } + + private AP.Navigation ToNavigation(LS2.DocumentSymbol symbol) => + new AP.Navigation { + name = symbol.name, + startLine = symbol.range.start.line + 1, + startColumn = symbol.range.start.character + 1, + endLine = symbol.range.end.line + 1, + endColumn = symbol.range.end.character + 1, + type = symbol._functionKind, + children = symbol.children.Select(ToNavigation).ToArray() + }; + + private Response ExpressionAtPoint(AP.ExpressionAtPointRequest request) { + var buffer = GetPythonBuffer(request.documentUri); + if (buffer.Ast == null) { + return null; + } + + var res = new AP.ExpressionAtPointResponse(); + if (!GetExpressionAtPoint(buffer.Ast, request.line, request.column, request.purpose, out SourceSpan span, out res.type)) { + return null; + } + res.startLine = span.Start.Line; + res.startColumn = span.Start.Column; + res.endLine = span.End.Line; + res.endColumn = span.End.Column; + res.bufferVersion = buffer.Version; + return res; + } + + private bool GetExpressionAtPoint(PythonAst ast, int line, int column, AP.ExpressionAtPointPurpose purpose, out SourceSpan span, out string type) { + span = default(SourceSpan); + type = null; + + if (ast == null) { + return false; + } + + GetExpressionOptions options; + switch (purpose) { + case AP.ExpressionAtPointPurpose.Evaluate: + options = GetExpressionOptions.Evaluate; + break; + case AP.ExpressionAtPointPurpose.EvaluateMembers: + options = GetExpressionOptions.EvaluateMembers; + break; + case AP.ExpressionAtPointPurpose.Hover: + options = GetExpressionOptions.Hover; + break; + case AP.ExpressionAtPointPurpose.FindDefinition: + options = GetExpressionOptions.FindDefinition; + break; + case AP.ExpressionAtPointPurpose.Rename: + options = GetExpressionOptions.Rename; + break; + default: + options = new GetExpressionOptions(); + break; + } + + var exprFinder = new ExpressionFinder(ast, options); + var expr = exprFinder.GetExpression(new SourceLocation(line, column)); + if (expr == null) { + return false; + } + + span = expr.GetSpan(ast); + type = expr.NodeName; + return true; + } + + private async Task AnalyzeExpression(AP.AnalyzeExpressionRequest request) { + var entry = GetPythonEntry(request.documentUri); + if (entry == null) { + return IncorrectFileType(); + } + + var references = await _server.FindReferences(new LS2.ReferencesParams { + textDocument = request.documentUri, + position = new SourceLocation(request.line, request.column), + context = new LS2.ReferenceContext { + includeDeclaration = true, + _includeValues = true + } + }, CancellationToken.None); + + var privatePrefix = entry.Analysis.GetPrivatePrefix(new SourceLocation(request.line, request.column)); + + return new AP.AnalyzeExpressionResponse { + variables = references.Select(MakeReference).ToArray(), + privatePrefix = privatePrefix + }; + } + + private AP.AnalysisReference MakeReference(LS2.Reference r) { + var range = (SourceSpan)r.range; + + return new AP.AnalysisReference { + documentUri = r.uri, + file = (_server.GetEntry(r.uri, throwIfMissing: false)?.FilePath) ?? r.uri?.LocalPath, + startLine = range.Start.Line, + startColumn = range.Start.Column, + endLine = range.End.Line, + endColumn = range.End.Column, + kind = GetVariableType(r._kind), + version = r._version + }; + } + + private static string GetVariableType(VariableType type) { + switch (type) { + case VariableType.Definition: return "definition"; + case VariableType.Reference: return "reference"; + case VariableType.Value: return "value"; + } + return null; + } + + private static string GetVariableType(LS2.ReferenceKind? type) { + if (!type.HasValue) { + return null; + } + switch (type.Value) { + case LS2.ReferenceKind.Definition: return "definition"; + case LS2.ReferenceKind.Reference: return "reference"; + case LS2.ReferenceKind.Value: return "value"; + } + return null; + } + + private async Task GetQuickInfo(AP.QuickInfoRequest request) { + LS2.Hover hover = await _server.Hover(new LS2.TextDocumentPositionParams { + textDocument = request.documentUri, + position = new SourceLocation(request.line, request.column), + _expr = request.expr, + }, CancellationToken.None); + + return new AP.QuickInfoResponse { + text = hover.contents.value + }; + } + + private async Task GetSignatures(AP.SignaturesRequest request) { + var sigs = await _server.SignatureHelp(new LS2.TextDocumentPositionParams { + textDocument = request.documentUri, + position = new SourceLocation(request.line, request.column), + _expr = request.text + }, CancellationToken.None); + + return new AP.SignaturesResponse { + sigs = sigs?.signatures?.Select( + s => new AP.Signature { + name = s.label, + doc = s.documentation?.value, + parameters = s.parameters.MaybeEnumerate().Select( + p => new AP.Parameter { + name = p.label, + defaultValue = p._defaultValue, + optional = p._isOptional ?? false, + doc = p.documentation?.value, + type = p._type + } + ).ToArray() + } + ).ToArray() + }; + } + + private async Task GetModules(Request request) { + var getModules = (AP.GetModulesRequest)request; + var prefix = getModules.package == null ? null : (string.Join(".", getModules.package)); + + var modules = await _server.Completion(new LS2.CompletionParams { + textDocument = getModules.documentUri, + _expr = prefix, + context = new LS2.CompletionContext { + triggerKind = LS2.CompletionTriggerKind.Invoked, + _filterKind = LS2.CompletionItemKind.Module, + //_includeAllModules = getModules.package == null + } + }, CancellationToken.None); + + return new AP.CompletionsResponse { + completions = await ToCompletions(modules.items, GetMemberOptions.None) + }; + } + + private async Task GetCompletions(Request request) { + var req = (AP.CompletionsRequest)request; + + var members = await _server.Completion(new LS2.CompletionParams { + position = new Position { line = req.line - 1, character = req.column - 1 }, + textDocument = req.documentUri, + context = new LS2.CompletionContext { + _intersection = req.options.HasFlag(GetMemberOptions.IntersectMultipleResults), + //_statementKeywords = req.options.HasFlag(GetMemberOptions.IncludeStatementKeywords), + //_expressionKeywords = req.options.HasFlag(GetMemberOptions.IncludeExpressionKeywords), + //_includeArgumentNames = true + }, + _expr = req.text + }, CancellationToken.None); + + return new AP.CompletionsResponse() { + completions = await ToCompletions(members.items, req.options) + }; + } + + private async Task GetAllMembers(Request request) { + var req = (AP.GetAllMembersRequest)request; + + var members = await _server.WorkspaceSymbols(new LS2.WorkspaceSymbolParams { + query = req.prefix + }, CancellationToken.None).ConfigureAwait(false); + + return new AP.CompletionsResponse() { + completions = await ToCompletions(members) + }; + } + + private async Task ToCompletions(IEnumerable symbols) { + if (symbols == null) { + return null; + } + + var res = new List(); + foreach (var s in symbols) { + var m = new AP.Completion { + name = s.name, + memberType = ToMemberType(s._kind, s.kind) + }; + + if (s.location.uri != null) { + m.detailedValues = new[] { + new AP.CompletionValue { + locations = new [] { + new AP.AnalysisReference { + file = s.location.uri.AbsolutePath, + documentUri = s.location.uri, + startLine = s.location.range.start.line + 1, + startColumn = s.location.range.start.character + 1, + endLine = s.location.range.end.line + 1, + endColumn = s.location.range.end.character + 1, + kind = "definition", + } + } + } + }; + } + + res.Add(m); + } + + return res.ToArray(); + } + + + private async Task ToCompletions(IEnumerable completions, GetMemberOptions options) { + if (completions == null) { + return null; + } + + var res = new List(); + foreach (var c in completions) { + var m = new AP.Completion { + name = c.label, + completion = (c.label == c.insertText) ? null : c.insertText, + doc = c.documentation?.value, + memberType = ToMemberType(c._kind, c.kind) + }; + + if (options.HasFlag(GetMemberOptions.DetailedInformation)) { + var c2 = await _server.CompletionItemResolve(c, CancellationToken.None); + var vars = new List(); + foreach (var v in c2._values.MaybeEnumerate()) { + vars.Add(new AP.CompletionValue { + description = new[] { new AP.DescriptionComponent { kind = null, text = v.description } }, + doc = v.documentation, + locations = v.references?.Where(r => r.uri.IsFile).Select(r => new AP.AnalysisReference { + file = r.uri.AbsolutePath, + documentUri = r.uri, + startLine = r.range.start.line + 1, + startColumn = r.range.start.character + 1, + endLine = r.range.end.line + 1, + endColumn = r.range.end.character + 1, + kind = GetVariableType(r._kind) + }).ToArray() + }); + } + } + + res.Add(m); + } + + return res.ToArray(); + } + + private PythonMemberType ToMemberType(string originalKind, LS2.CompletionItemKind kind) { + PythonMemberType res; + if (!string.IsNullOrEmpty(originalKind) && Enum.TryParse(originalKind, true, out res)) { + return res; + } + + switch (kind) { + case LS2.CompletionItemKind.None: return PythonMemberType.Unknown; + case LS2.CompletionItemKind.Text: return PythonMemberType.Constant; + case LS2.CompletionItemKind.Method: return PythonMemberType.Method; + case LS2.CompletionItemKind.Function: return PythonMemberType.Function; + case LS2.CompletionItemKind.Constructor: return PythonMemberType.Function; + case LS2.CompletionItemKind.Field: return PythonMemberType.Field; + case LS2.CompletionItemKind.Variable: return PythonMemberType.Instance; + case LS2.CompletionItemKind.Class: return PythonMemberType.Class; + case LS2.CompletionItemKind.Interface: return PythonMemberType.Class; + case LS2.CompletionItemKind.Module: return PythonMemberType.Module; + case LS2.CompletionItemKind.Property: return PythonMemberType.Property; + case LS2.CompletionItemKind.Unit: return PythonMemberType.Unknown; + case LS2.CompletionItemKind.Value: return PythonMemberType.Instance; + case LS2.CompletionItemKind.Enum: return PythonMemberType.Enum; + case LS2.CompletionItemKind.Keyword: return PythonMemberType.Keyword; + case LS2.CompletionItemKind.Snippet: return PythonMemberType.CodeSnippet; + case LS2.CompletionItemKind.Color: return PythonMemberType.Instance; + case LS2.CompletionItemKind.File: return PythonMemberType.Module; + case LS2.CompletionItemKind.Reference: return PythonMemberType.Unknown; + case LS2.CompletionItemKind.Folder: return PythonMemberType.Module; + case LS2.CompletionItemKind.EnumMember: return PythonMemberType.EnumInstance; + case LS2.CompletionItemKind.Constant: return PythonMemberType.Constant; + case LS2.CompletionItemKind.Struct: return PythonMemberType.Class; + case LS2.CompletionItemKind.Event: return PythonMemberType.Delegate; + case LS2.CompletionItemKind.Operator: return PythonMemberType.Unknown; + case LS2.CompletionItemKind.TypeParameter: return PythonMemberType.Class; + default: return PythonMemberType.Unknown; + } + } + + private PythonMemberType ToMemberType(string originalKind, LS2.SymbolKind kind) { + PythonMemberType res; + if (!string.IsNullOrEmpty(originalKind) && Enum.TryParse(originalKind, true, out res)) { + return res; + } + + switch (kind) { + case LS2.SymbolKind.None: return PythonMemberType.Unknown; + case LS2.SymbolKind.File: return PythonMemberType.Module; + case LS2.SymbolKind.Module: return PythonMemberType.Module; + case LS2.SymbolKind.Namespace: return PythonMemberType.Namespace; + case LS2.SymbolKind.Package: return PythonMemberType.Module; + case LS2.SymbolKind.Class: return PythonMemberType.Class; + case LS2.SymbolKind.Method: return PythonMemberType.Method; + case LS2.SymbolKind.Property: return PythonMemberType.Property; + case LS2.SymbolKind.Field: return PythonMemberType.Field; + case LS2.SymbolKind.Constructor: return PythonMemberType.Method; + case LS2.SymbolKind.Enum: return PythonMemberType.Enum; + case LS2.SymbolKind.Interface: return PythonMemberType.Class; + case LS2.SymbolKind.Function: return PythonMemberType.Function; + case LS2.SymbolKind.Variable: return PythonMemberType.Field; + case LS2.SymbolKind.Constant: return PythonMemberType.Constant; + case LS2.SymbolKind.String: return PythonMemberType.Constant; + case LS2.SymbolKind.Number: return PythonMemberType.Constant; + case LS2.SymbolKind.Boolean: return PythonMemberType.Constant; + case LS2.SymbolKind.Array: return PythonMemberType.Instance; + case LS2.SymbolKind.Object: return PythonMemberType.Instance; + case LS2.SymbolKind.Key: return PythonMemberType.Unknown; + case LS2.SymbolKind.Null: return PythonMemberType.Unknown; + case LS2.SymbolKind.EnumMember: return PythonMemberType.EnumInstance; + case LS2.SymbolKind.Struct: return PythonMemberType.Class; + case LS2.SymbolKind.Event: return PythonMemberType.Event; + case LS2.SymbolKind.Operator: return PythonMemberType.Method; + case LS2.SymbolKind.TypeParameter: return PythonMemberType.NamedArgument; + default: return PythonMemberType.Unknown; + } + } + + private async Task AnalyzeFileAsync(AP.AddFileRequest request, Func done) { + var uri = request.uri ?? ProjectEntry.MakeDocumentUri(request.path); + var entry = await AddNewFile(uri, request.path, request.addingFromDir); + + await done(new AP.AddFileResponse { documentUri = uri }); + } + + private async Task AnalyzeFileAsync(AP.AddBulkFileRequest request, Func done) { + var entries = new IProjectEntry[request.path.Length]; + var response = new AP.AddBulkFileResponse { + documentUri = new Uri[request.path.Length] + }; + + for (int i = 0; i < request.path.Length; ++i) { + if (!string.IsNullOrEmpty(request.path[i])) { + var documentUri = ProjectEntry.MakeDocumentUri(request.path[i]); + entries[i] = await AddNewFile(documentUri, request.path[i], request.addingFromDir); + response.documentUri[i] = documentUri; + } + } + + await done(response); + } + + private async Task AddNewFile(Uri documentUri, string path, string addingFromDir) { + if (documentUri.IsFile && Path.GetExtension(documentUri.LocalPath).Equals(".xaml", StringComparison.OrdinalIgnoreCase) && Project.Interpreter is IDotNetPythonInterpreter interpreter) { + return interpreter.AddXamlEntry(path, documentUri); + } + + return await _server.LoadFileAsync(documentUri).ConfigureAwait(false); + } + + private async Task UnloadFile(AP.UnloadFileRequest command) { + await _server.UnloadFileAsync(command.documentUri); + return new Response(); + } + + abstract class CodeInfo { + public readonly int Version; + + public CodeInfo(int version) { + Version = version; + } + + public abstract Parser CreateParser(PythonLanguageVersion version, ParserOptions options); + + public abstract TextReader GetReader(); + } + + class StreamCodeInfo : CodeInfo { + private readonly Stream _stream; + private readonly bool _isStubFile; + + public StreamCodeInfo(int version, Stream stream, bool isStubFile) : base(version) { + _stream = stream; + _isStubFile = isStubFile; + } + + public override Parser CreateParser(PythonLanguageVersion version, ParserOptions options) { + if (_isStubFile) { + options = options?.Clone() ?? new ParserOptions(); + options.StubFile = true; + } + return Parser.CreateParser(_stream, version, options); + } + + public override TextReader GetReader() { + return new StreamReader(_stream); + } + } + + class TextCodeInfo : CodeInfo { + private readonly TextReader _text; + private readonly bool _isStubFile; + + public TextCodeInfo(int version, TextReader text, bool isStubFile) : base(version) { + _text = text; + _isStubFile = isStubFile; + } + + public override Parser CreateParser(PythonLanguageVersion version, ParserOptions options) { + if (_isStubFile) { + options = options?.Clone() ?? new ParserOptions(); + options.StubFile = true; + } + return Parser.CreateParser(_text, version, options); + } + + public override TextReader GetReader() { + return _text; + } + } + + private async Task UpdateContent(AP.FileUpdateRequest request) { + int version = -1; + foreach (var fileChange in request.updates) { + var changes = new List(); + if (fileChange.kind == AP.FileUpdateKind.reset) { + changes.Add(new LS2.TextDocumentContentChangedEvent { + text = fileChange.content + }); + version = fileChange.version; + } else if (fileChange.kind == AP.FileUpdateKind.changes) { + changes.AddRange(fileChange.changes.Select(c => new LS2.TextDocumentContentChangedEvent { + range = new SourceSpan( + new SourceLocation(c.startLine, c.startColumn), + new SourceLocation(c.endLine, c.endColumn) + ), + text = c.newText + })); + version = fileChange.version; + } else { + continue; + } + + _server.DidChangeTextDocument(new LS2.DidChangeTextDocumentParams { + textDocument = new LS2.VersionedTextDocumentIdentifier { + uri = request.documentUri, + version = version, + _fromVersion = Math.Max(version - 1, 0) + }, + contentChanges = changes.ToArray() + }, CancellationToken.None).DoNotWait(); + } + +#if DEBUG + var entry = _server.GetEntry(request.documentUri); + int part = _server.GetPart(request.documentUri); + return new AP.FileUpdateResponse { + version = version, + newCode = (entry as IDocument)?.ReadDocument(part, out _)?.ReadToEnd() + }; +#else + return new AP.FileUpdateResponse { + version = version + }; +#endif + } + + public Task ProcessMessages() { + return _connection.ProcessMessages(); + } + + private Response SetAnalysisOptions(AP.SetAnalysisOptionsRequest request) { + Options = request.options ?? new AP.AnalysisOptions(); + + Project.Limits = AnalysisLimitsConverter.FromDictionary(Options.analysisLimits); + _server.ParseQueue.InconsistentIndentation = DiagnosticsErrorSink.GetSeverity(Options.indentationInconsistencySeverity); + _server.ParseQueue.TaskCommentMap = Options.commentTokens; + _server.Analyzer.SetTypeStubPaths(Options.typeStubPaths); + + return new Response(); + } + + + public AP.AnalysisOptions Options { get; set; } + + private void AnalysisQueue_Complete(object sender, EventArgs e) { + _connection?.SendEventAsync(new AP.AnalysisCompleteEvent()).DoNotWait(); + } + + private void OnModulesChanged(object sender, EventArgs args) { + _server.DidChangeConfiguration(new LS2.DidChangeConfigurationParams(), CancellationToken.None).DoNotWait(); + } + + private void OnFileChanged(AP.FileChangedEvent e) { + _server.DidChangeWatchedFiles(new LS2.DidChangeWatchedFilesParams { + changes = e.changes.MaybeEnumerate().Select(c => new LS2.FileEvent { uri = c.documentUri, type = c.kind }).ToArray() + }, CancellationToken.None).DoNotWait(); + } + + private void OnAnalysisComplete(object sender, LS2.AnalysisCompleteEventArgs e) { + _connection.SendEventAsync( + new AP.FileAnalysisCompleteEvent { + documentUri = e.uri, + version = e.version + } + ).DoNotWait(); + } + + private static NameExpression GetFirstNameExpression(Statement stmt) { + return GetFirstNameExpression(Statement.GetExpression(stmt)); + } + + private static NameExpression GetFirstNameExpression(Expression expr) { + NameExpression nameExpr; + CallExpression callExpr; + MemberExpression membExpr; + + if ((nameExpr = expr as NameExpression) != null) { + return nameExpr; + } + if ((callExpr = expr as CallExpression) != null) { + return GetFirstNameExpression(callExpr.Target); + } + if ((membExpr = expr as MemberExpression) != null) { + return GetFirstNameExpression(membExpr.Target); + } + + return null; + } + + private static bool IsDefinition(IAnalysisVariable variable) { + return variable.Type == VariableType.Definition; + } + + private static bool IsImplicitlyDefinedName(NameExpression nameExpr) { + return nameExpr.Name == "__all__" || + nameExpr.Name == "__file__" || + nameExpr.Name == "__doc__" || + nameExpr.Name == "__name__"; + } + + internal IPythonInterpreterFactory InterpreterFactory => Project?.InterpreterFactory; + + internal IPythonInterpreter Interpreter => Project?.Interpreter; + + // Returns the current analyzer or throws InvalidOperationException. + // This should be used in request handlers that should fail when + // analysis is impossible. Callers that explicitly check for null before + // use should use _pyAnalyzer directly. + private PythonAnalyzer Analyzer { + get { + if (Project == null) { + throw new InvalidOperationException("Unable to analyze code"); + } + + return Project; + } + } + + /// + /// Returns the current analyzer or null if unable to analyze code. + /// + /// + /// This is for public consumption only and should not be used within + /// . + /// + public PythonAnalyzer Project => _server.Analyzer; + + private void OnPublishDiagnostics(object sender, LS2.PublishDiagnosticsEventArgs e) { + _connection.SendEventAsync( + new AP.DiagnosticsEvent { + documentUri = e.uri, + version = e._version ?? -1, + diagnostics = e.diagnostics?.ToArray() + } + ).DoNotWait(); + } + + private void OnParseComplete(object sender, LS2.ParseCompleteEventArgs e) { + _connection.SendEventAsync( + new AP.FileParsedEvent { + documentUri = e.uri, + version = e.version + } + ).DoNotWait(); + } + + + #region IDisposable Members + + public void Dispose() { + if (_isDisposed) { + return; + } + + _isDisposed = true; + _server.AnalysisQueue.AnalysisComplete -= AnalysisQueue_Complete; + if (Project != null) { + Project.Interpreter.ModuleNamesChanged -= OnModulesChanged; + Project.Dispose(); + } + + lock (_extensions) { + foreach (var extension in _extensions.Values) { + (extension as IDisposable)?.Dispose(); + } + } + + _server.Dispose(); + + _connection.Dispose(); + } + + #endregion + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutliningWalker.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutliningWalker.cs new file mode 100644 index 000000000..b9da4121d --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutliningWalker.cs @@ -0,0 +1,196 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.PythonTools.Parsing; +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Intellisense { + public class OutliningWalker : PythonWalker { + private readonly PythonAst _ast; + private readonly List _tagSpans; + + public OutliningWalker(PythonAst ast) { + _ast = ast; + _tagSpans = new List(); + } + + // Compound Statements: if, while, for, try, with, func, class, decorated + public override bool Walk(IfStatement node) { + if (node.ElseStatement != null) { + AddTagIfNecessary(node.ElseIndex, node.ElseStatement.EndIndex); + } + + return base.Walk(node); + } + + public override bool Walk(IfStatementTest node) { + AddTagIfNecessary(node.HeaderIndex + 1, node.Body?.EndIndex); + // Only walk body, not the condition. + node.Body?.Walk(this); + return false; + } + + public override bool Walk(WhileStatement node) { + // Walk while statements manually so we don't traverse the test. + // This prevents the test from being collapsed ever. + if (node.Body != null) { + AddTagIfNecessary( + _ast.GetLineEndFromPosition(node.StartIndex), + node.Body.EndIndex + ); + node.Body.Walk(this); + } + if (node.ElseStatement != null) { + AddTagIfNecessary(node.ElseIndex, node.ElseStatement.EndIndex); + node.ElseStatement.Walk(this); + } + return false; + } + + public override bool Walk(ForStatement node) { + // Walk for statements manually so we don't traverse the list. + // This prevents the list and/or left from being collapsed ever. + node.List?.Walk(this); + + if (node.Body != null) { + AddTagIfNecessary(node.HeaderIndex + 1, node.Body.EndIndex); + node.Body.Walk(this); + } + if (node.Else != null) { + AddTagIfNecessary(node.ElseIndex, node.Else.EndIndex); + node.Else.Walk(this); + } + return false; + } + + public override bool Walk(TryStatement node) { + AddTagIfNecessary(node.HeaderIndex, node.Body?.EndIndex); + if (node.Handlers != null) { + foreach (var h in node.Handlers) { + AddTagIfNecessary(h.HeaderIndex, h.EndIndex); + } + } + AddTagIfNecessary(node.FinallyIndex, node.Finally?.EndIndex); + AddTagIfNecessary(node.ElseIndex, node.Else?.EndIndex); + + return base.Walk(node); + } + + public override bool Walk(WithStatement node) { + AddTagIfNecessary(node.HeaderIndex + 1, node.Body?.EndIndex); + return base.Walk(node); + } + + public override bool Walk(FunctionDefinition node) { + // Walk manually so collapsing is not enabled for params. + if (node.Parameters != null) { + AddTagIfNecessary(node.Parameters.FirstOrDefault()?.StartIndex, node.Parameters.LastOrDefault()?.EndIndex); + } + if (node.Body != null) { + AddTagIfNecessary(node.HeaderIndex + 1, node.EndIndex); + node.Body.Walk(this); + } + + return false; + } + + public override bool Walk(ClassDefinition node) { + AddTagIfNecessary(node.HeaderIndex + 1, node.EndIndex); + + return base.Walk(node); + } + + // Not-Compound Statements + public override bool Walk(CallExpression node) { + AddTagIfNecessary(node.Args?.FirstOrDefault()?.StartIndex, node.Args?.LastOrDefault()?.EndIndex); + return base.Walk(node); + } + + public override bool Walk(FromImportStatement node) { + if (node.Names != null && node.Names.Any()) { + int lastName = node.Names.Count - 1; + int? nameEnd = node.Names[lastName]?.EndIndex; + if (node.AsNames != null && node.AsNames.Count >= lastName && node.AsNames[lastName] != null) { + nameEnd = node.AsNames[lastName].EndIndex; + } + AddTagIfNecessary(node.Names[0].StartIndex, nameEnd); + } + return base.Walk(node); + } + + public override bool Walk(ListExpression node) { + AddTagIfNecessary(node.Items?.FirstOrDefault()?.StartIndex, node.Items?.LastOrDefault()?.EndIndex); + return base.Walk(node); + } + + public override bool Walk(TupleExpression node) { + AddTagIfNecessary(node.Items?.FirstOrDefault()?.StartIndex, node.Items?.LastOrDefault()?.EndIndex); + return base.Walk(node); + } + + public override bool Walk(DictionaryExpression node) { + AddTagIfNecessary(node.Items?.FirstOrDefault()?.StartIndex, node.Items?.LastOrDefault()?.EndIndex); + return base.Walk(node); + } + + public override bool Walk(SetExpression node) { + AddTagIfNecessary(node.Items?.FirstOrDefault()?.StartIndex, node.Items?.LastOrDefault()?.EndIndex); + return base.Walk(node); + } + + public override bool Walk(ParenthesisExpression node) { + AddTagIfNecessary(node.Expression?.StartIndex, node.Expression?.EndIndex); + return base.Walk(node); + } + + public override bool Walk(ConstantExpression node) { + AddTagIfNecessary(_ast.GetLineEndFromPosition(node.StartIndex), node.EndIndex); + return base.Walk(node); + } + + private void AddTagIfNecessary(int? startIndex, int? endIndex, int minLinesToCollapse = 3) { + if (!startIndex.HasValue || !endIndex.HasValue) { + return; + } + + if (startIndex < 0 || endIndex < 0) { + return; + } + + var start = _ast.IndexToLocation(startIndex.Value); + var end = _ast.IndexToLocation(endIndex.Value); + var lines = end.Line - start.Line + 1; + + // Collapse if more than 3 lines. + if (lines < minLinesToCollapse) { + return; + } + + var tagSpan = new TaggedSpan(new SourceSpan(start, end), null); + _tagSpans.Add(tagSpan); + } + + public IEnumerable GetTags() { + return _tagSpans + .GroupBy(s => s.Span.Start.Line) + .Select(ss => ss.OrderByDescending(s => s.Span.End.Line).First()) + .ToArray(); + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/ProximityExpressionWalker.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/ProximityExpressionWalker.cs new file mode 100644 index 000000000..a646b61e1 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/ProximityExpressionWalker.cs @@ -0,0 +1,144 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.PythonTools.Parsing; +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Intellisense { + /// + /// Walks the AST, and returns all expressions in the given line range that are eligible to be displayed + /// in the Autos debugger tool window. + /// + /// + /// The following expressions are considered eligible: + /// + /// standalone names: x + /// member access, so long as the target is side-effect free: x.y, 123.real, (x + 1).imag + /// indexing/slicing, so long as target and index are side-effect free: x.y[z], ('abc' + 'd')[1:len(s)-1] + /// function calls, if function is one of the several hardcoded builtin functions (e.g. abs, len, str, repr), + /// and all arguments are side-effect free: len('a' * n) + /// + /// + /// All error-free expressions are considered side-effect-free except those involving function calls, backquotes, yield + /// or yield from. Function calls are considered side-effect-free if they are eligible according to the definition above. + /// + /// + /// For member access and indexing, if an expression is eligible, all its direct nested targets become non-eligible. For example, + /// given a.b.c, neither a nor a.b are eligible. + /// + /// + /// For function calls, the immediate target becomes ineligible. For example, given a.b.c(d), a.b.c is not eligible, + /// but a.b is (and hence a is not, per the earlier rule). + /// + /// + public class ProximityExpressionWalker : PythonWalker { + private readonly PythonAst _ast; + private readonly int _startLine, _endLine; + + // Keys are expressions that have been walked and found to be eligible. + // For a given key, a value is either null, or an expression that was removed from keys when this key was + // added, because it is a subexpression of this key that we wanted to exclude. For example, given A.B.C, + // we will first walk A and add it as a key with null as value; then walk A.B, remove A, and add A.B with + // A as value; then walk A.B.C, remove A.B, and add A.B.C with A.B as value. + // The information about the excluded node is used by PostWalk(CallExpression). + private readonly Dictionary _expressions = new Dictionary(); + + public ProximityExpressionWalker(PythonAst ast, int startLine, int endLine) { + _ast = ast; + _startLine = startLine; + _endLine = endLine; + } + + public override void PostWalk(NameExpression node) { + if (IsInRange(node)) { + if(_ast.LanguageVersion.Is2x()) { + // In 2.7 True and False are constants, we made an exception to not show them Autos window. + if(node.Name == "True" || node.Name == "False") { + return; + } + } + + _expressions.Add(node, null); + } + + } + + public override void PostWalk(MemberExpression node) { + if (IsInRange(node) && IsValidTarget(node.Target)) { + _expressions.Remove(node.Target); + _expressions.Add(node, node.Target); + } + } + + public override void PostWalk(IndexExpression node) { + if (IsInRange(node) && IsValidTarget(node.Target) && IsValidTarget(node.Index)) { + _expressions.Remove(node.Target); + _expressions.Add(node, node.Target); + } + } + + public override void PostWalk(CallExpression node) { + if (IsInRange(node) && node.Target != null) { + // For call nodes, we don't want either the call nor the called function to show up, + // but if it is a method, then we do want to show the object on which it is called. + // For example, given A.B.C(42), we want A.B to show. By the time we get here, we + // already have A.B.C in the list, so we need to remove it and reinstate A.B, which + // will be stored as a value in the dictionary with A.B.C as key. + Expression oldNode; + _expressions.TryGetValue(node.Target, out oldNode); + _expressions.Remove(node.Target); + if (oldNode != null) { + _expressions.Add(oldNode, null); + } + + // Hardcode some commonly used side-effect-free builtins to show even in calls. + var name = node.Target as NameExpression; + if (name != null && DetectSideEffectsWalker.IsSideEffectFreeCall(name.Name) && node.Args.All(arg => IsValidTarget(arg.Expression))) { + _expressions.Add(node, null); + } + } + } + + private bool IsInRange(Expression node) { + var span = node.GetSpan(_ast); + int isct0 = Math.Max(span.Start.Line, _startLine); + int isct1 = Math.Min(span.End.Line, _endLine); + return isct0 <= isct1; + } + + private bool IsValidTarget(Node node) { + if (node == null || node is ConstantExpression || node is NameExpression) { + return true; + } + + var expr = node as Expression; + if (expr != null && _expressions.ContainsKey(expr)) { + return true; + } + + var walker = new DetectSideEffectsWalker(); + node.Walk(walker); + return !walker.HasSideEffects; + } + + public IEnumerable GetExpressions() { + return _expressions.Keys.Select(expr => expr.ToCodeString(_ast, CodeFormattingOptions.Traditional).Trim()).OrderBy(s => s).Distinct(); + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/TaggedSpan.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/TaggedSpan.cs new file mode 100644 index 000000000..95c4b643e --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/TaggedSpan.cs @@ -0,0 +1,27 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.PythonTools.Intellisense { + public sealed class TaggedSpan { + public SourceSpan Span { get; } + public string Tag { get; } + + public TaggedSpan(SourceSpan span, string tag) { + Span = span; + Tag = tag; + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Interpreter/IDotNetPythonInterpreter.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Interpreter/IDotNetPythonInterpreter.cs new file mode 100644 index 000000000..df07e38b5 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Interpreter/IDotNetPythonInterpreter.cs @@ -0,0 +1,32 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using Microsoft.PythonTools.Analysis; + +namespace Microsoft.PythonTools.Interpreter { + public interface IDotNetPythonInterpreter { + /// + /// Gets the IPythonType object for the specified .NET type; + /// + IPythonType GetBuiltinType(Type type); + + /// + /// Adds xaml entry + /// + IProjectEntry AddXamlEntry(string filePath, Uri documentUri); + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Interpreter/ProjectAssemblyReference.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Interpreter/ProjectAssemblyReference.cs new file mode 100644 index 000000000..b2daa0f3e --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Interpreter/ProjectAssemblyReference.cs @@ -0,0 +1,62 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Reflection; + +namespace Microsoft.PythonTools.Interpreter { + public sealed class ProjectAssemblyReference : ProjectReference, IEquatable { + private readonly AssemblyName _asmName; + + public ProjectAssemblyReference(AssemblyName assemblyName, string filename) + : base(filename, ProjectReferenceKind.Assembly) { + _asmName = assemblyName; + } + + public AssemblyName AssemblyName => _asmName; + + public override int GetHashCode() { + return base.GetHashCode() ^ _asmName.GetHashCode(); + } + + public override bool Equals(object obj) { + ProjectAssemblyReference asmRef = obj as ProjectAssemblyReference; + if (asmRef != null) { + return Equals(asmRef); + } + return false; + } + + public override bool Equals(ProjectReference other) { + ProjectAssemblyReference asmRef = other as ProjectAssemblyReference; + if (asmRef != null) { + return Equals(asmRef); + } + return false; + } + + #region IEquatable Members + + public bool Equals(ProjectAssemblyReference other) { + if (base.Equals(other)) { + return other._asmName == this._asmName; + } + return false; + } + + #endregion + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/IronPythonScraper.py b/src/PTVS/Microsoft.PythonTools.Analyzer/IronPythonScraper.py new file mode 100644 index 000000000..f0be30975 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/IronPythonScraper.py @@ -0,0 +1,177 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABILITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +import clr +import PythonScraper + +from System import ParamArrayAttribute, DBNull, Void +from System.Reflection import Missing + +clr.AddReference('IronPython') +from IronPython.Runtime.Operations import PythonOps +from IronPython.Runtime import SiteLocalStorage +from IronPython.Runtime.Operations import InstanceOps + +clr.AddReference('Microsoft.Dynamic') +clr.AddReference('Microsoft.Scripting') +from Microsoft.Scripting import ParamDictionaryAttribute +from Microsoft.Scripting.Generation import CompilerHelpers + +class NonPythonTypeException(Exception): + pass + +def safe_dir(obj): + try: + return frozenset(obj.__dict__) | frozenset(clr.Dir(obj)) + except: + # Some types crash when we access __dict__ and/or dir() + pass + try: + return frozenset(clr.Dir(obj)) + except: + pass + try: + return frozenset(obj.__dict__) + except: + pass + return frozenset() + +def type_to_typelist(type_obj): + if type_obj.IsArray: + return PythonScraper.type_to_typelist(tuple) + elif type_obj == Void: + return PythonScraper.type_to_typelist(type(None)) + elif not PythonOps.IsPythonType(clr.GetPythonType(type_obj)): + raise NonPythonTypeException + + return PythonScraper.type_to_typelist(clr.GetPythonType(type_obj)) + + +def get_default_value(param): + if param.DefaultValue is not DBNull.Value and param.DefaultValue is not Missing.Value: + return repr(param.DefaultValue) + elif param.IsOptional: + missing = CompilerHelpers.GetMissingValue(param.ParameterType) + if missing != Missing.Value: + return repr(missing) + return "" + + +def get_arg_format(param): + if param.IsDefined(ParamArrayAttribute, False): + return "*" + elif param.IsDefined(ParamDictionaryAttribute, False): + return "**" + + +def sanitize_name(param): + for v in param.Name: + if not ((v >= '0' and v <= '9') or (v >= 'A' and v <= 'Z') or (v >= 'a' and v <= 'z') or v == '_'): + break + else: + return param.Name + + letters = [] + for v in param.Name: + if ((v >= '0' and v <= '9') or (v >= 'A' and v <= 'Z') or (v >= 'a' and v <= 'z') or v == '_'): + letters.append(v) + return ''.join(letters) + +def get_parameter_info(param): + parameter_table = { + 'type': type_to_typelist(param.ParameterType), + 'name': sanitize_name(param), + } + + default_value = get_default_value(param) + if default_value is not None: + parameter_table['default_value'] = default_value + + arg_format = get_arg_format(param) + if arg_format is not None: + parameter_table['arg_format'] = arg_format + + return parameter_table + +def get_return_type(target): + if hasattr(target, 'ReturnType'): + return target.ReturnType + # constructor + return target.DeclaringType + +def get_function_overloads(targets): + res = [] + for target in targets: + try: + args = list(target.GetParameters()) + except: + # likely a failure to load an assembly... + continue + if args and args[0].ParameterType.FullName == 'IronPython.Runtime.CodeContext': + del args[0] + if args and args[0].ParameterType.IsSubclassOf(SiteLocalStorage): + del args[0] + + try: + arg_info = [get_parameter_info(arg) for arg in args] + if not target.IsStatic and not target.IsConstructor: + arg_info.insert(0, {'type' : type_to_typelist(target.DeclaringType), 'name': 'self'}) + + res.append({ + 'args' : tuple(arg_info), + 'ret_type' : type_to_typelist(get_return_type(target)) + }) + except NonPythonTypeException: + pass + + + return res + +def get_overloads(func, is_method = False): + if type(func) == type(list.append): + func = PythonOps.GetBuiltinMethodDescriptorTemplate(func) + + targets = func.Targets + + res = get_function_overloads(targets) + + return res + + +def get_descriptor_type(descriptor): + if hasattr(descriptor, 'PropertyType'): + return clr.GetPythonType(descriptor.PropertyType) + elif hasattr(descriptor, 'FieldType'): + return clr.GetPythonType(descriptor.FieldType) + return object + + +def get_new_overloads(type_obj, func): + if func.Targets and func.Targets[0].DeclaringType == clr.GetClrType(InstanceOps): + print('has instance ops ' + str(type_obj)) + clrType = clr.GetClrType(type_obj) + + return get_function_overloads(clrType.GetConstructors()) + + return None + +SPECIAL_MODULES = ('wpf', 'clr') + +def should_include_module(name): + return name not in SPECIAL_MODULES diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Microsoft.PythonTools.Analyzer.csproj b/src/PTVS/Microsoft.PythonTools.Analyzer/Microsoft.PythonTools.Analyzer.csproj new file mode 100644 index 000000000..5ff442e24 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Microsoft.PythonTools.Analyzer.csproj @@ -0,0 +1,73 @@ + + + net472 + Microsoft.PythonTools.Analysis + Microsoft.PythonTools.Analyzer + + + + + Exe + portable + true + + + ..\..\PLS.ruleset + + + ..\..\PLS.ruleset + + + + + + + + + all + runtime; build; native; contentfiles; analyzers + + + + + $(AnalysisReference)/Microsoft.Python.Analysis.Engine.dll + + + PreserveNewest + + + + + PreserveNewest + True + + + PreserveNewest + True + + + PreserveNewest + True + + + PreserveNewest + True + + + + + + + + + + + + + + + + + + + diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/PyLibAnalyzer.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/PyLibAnalyzer.cs new file mode 100644 index 000000000..9e74fc4b7 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/PyLibAnalyzer.cs @@ -0,0 +1,244 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Threading.Tasks; +using Microsoft.PythonTools.Analysis.Infrastructure; +using Microsoft.PythonTools.Intellisense; + +namespace Microsoft.PythonTools.Analysis { + internal class PyLibAnalyzer : IDisposable { + private static void Help() { + var assembly = typeof(PyLibAnalyzer).Assembly; + string version = FileVersionInfo.GetVersionInfo(assembly.Location).FileVersion; + Console.WriteLine("Python Library Analyzer {0} ", version); + Console.WriteLine("Python analysis server."); + Console.WriteLine(); +#if DEBUG + Console.WriteLine(" /unittest - run from tests, Debug.Listeners will be replaced"); +#endif + } + + private static IEnumerable> ParseArguments(IEnumerable args) { + string currentKey = null; + + using (var e = args.GetEnumerator()) { + while (e.MoveNext()) { + if (e.Current.StartsWithOrdinal("/")) { + if (currentKey != null) { + yield return new KeyValuePair(currentKey, null); + } + currentKey = e.Current.Substring(1).Trim(); + } else { + yield return new KeyValuePair(currentKey, e.Current); + currentKey = null; + } + } + + if (currentKey != null) { + yield return new KeyValuePair(currentKey, null); + } + } + } + + /// + /// The exit code returned when database generation fails due to an + /// invalid argument. + /// + public const int InvalidArgumentExitCode = -1; + + /// + /// The exit code returned when database generation fails due to a + /// non-specific error. + /// + public const int InvalidOperationExitCode = -2; + + public static int Main(string[] args) { + PyLibAnalyzer inst; + try { + inst = MakeFromArguments(args); + } catch (ArgumentNullException ex) { + Console.Error.WriteLine("{0} is a required argument", ex.Message); + Help(); + return InvalidArgumentExitCode; + } catch (ArgumentException ex) { + Console.Error.WriteLine("'{0}' is not valid for {1}", ex.Message, ex.ParamName); + Help(); + return InvalidArgumentExitCode; + } catch (InvalidOperationException ex) { + Console.Error.WriteLine(ex.Message); + Help(); + return InvalidOperationExitCode; + } + + using (inst) { + return inst.Run().GetAwaiter().GetResult(); + } + } + + private async Task Run() { +#if DEBUG + // Running with the debugger attached will skip the + // unhandled exception handling to allow easier debugging. + if (Debugger.IsAttached) { + await RunWorker(); + } else { +#endif + try { + await RunWorker(); + } catch (Exception e) { + Console.Error.WriteLine("Error during analysis: {0}{1}", Environment.NewLine, e.ToString()); + return -10; + } +#if DEBUG + } +#endif + + return 0; + } + + private async Task RunWorker() { + var analyzer = new OutOfProcProjectAnalyzer( + Console.OpenStandardOutput(), + Console.OpenStandardInput(), + Console.Error.WriteLine + ); + + await analyzer.ProcessMessages(); + } + + public PyLibAnalyzer() { + } + + public void Dispose() { + } + + private static PyLibAnalyzer MakeFromArguments(IEnumerable args) { + var options = ParseArguments(args) + .ToDictionary(kv => kv.Key, kv => kv.Value, StringComparer.OrdinalIgnoreCase); + +#if DEBUG + if (options.ContainsKey("unittest")) { + AssertListener.Initialize(); + } +#endif + + return new PyLibAnalyzer(); + } + +#if DEBUG + class AssertListener : TraceListener { + private AssertListener() { + } + + public override string Name { + get { return "Microsoft.PythonTools.AssertListener"; } + set { } + } + + public static bool LogObjectDisposedExceptions { get; set; } = true; + + public static void Initialize() { + var listener = new AssertListener(); + if (null == Debug.Listeners[listener.Name]) { + Debug.Listeners.Add(listener); + Debug.Listeners.Remove("Default"); + + AppDomain.CurrentDomain.FirstChanceException += CurrentDomain_FirstChanceException; + } + } + + static void CurrentDomain_FirstChanceException(object sender, FirstChanceExceptionEventArgs e) { + if (e.Exception is NullReferenceException || (e.Exception is ObjectDisposedException && LogObjectDisposedExceptions)) { + // Exclude safe handle messages because they are noisy + if (!e.Exception.Message.Contains("Safe handle has been closed")) { + var log = new EventLog("Application"); + log.Source = "Application Error"; + log.WriteEntry( + "First-chance exception: " + e.Exception.ToString(), + EventLogEntryType.Warning + ); + } + } + } + + public override void Fail(string message) { + Fail(message, null); + } + + public override void Fail(string message, string detailMessage) { + Trace.WriteLine("Debug.Assert failed in analyzer"); + if (!string.IsNullOrEmpty(message)) { + Trace.WriteLine(message); + } else { + Trace.WriteLine("(No message provided)"); + } + if (!string.IsNullOrEmpty(detailMessage)) { + Trace.WriteLine(detailMessage); + } + var trace = new StackTrace(true); + bool seenDebugAssert = false; + foreach (var frame in trace.GetFrames()) { + var mi = frame.GetMethod(); + if (!seenDebugAssert) { + seenDebugAssert = (mi.DeclaringType == typeof(Debug) && + (mi.Name == "Assert" || mi.Name == "Fail")); + } else if (mi.DeclaringType == typeof(System.RuntimeMethodHandle)) { + break; + } else { + var filename = frame.GetFileName(); + Console.WriteLine(string.Format( + " at {0}.{1}({2}) in {3}:line {4}", + mi.DeclaringType.FullName, + mi.Name, + string.Join(", ", mi.GetParameters().Select(p => p.ToString())), + filename ?? "", + frame.GetFileLineNumber() + )); + if (File.Exists(filename)) { + try { + Console.WriteLine( + " " + + File.ReadLines(filename).ElementAt(frame.GetFileLineNumber() - 1).Trim() + ); + } catch { + } + } + } + } + + message = string.IsNullOrEmpty(message) ? "Debug.Assert failed" : message; + if (Debugger.IsAttached) { + Debugger.Break(); + } + + Console.WriteLine(message); + } + + public override void WriteLine(string message) { + } + + public override void Write(string message) { + } + } +#endif + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/PythonScraper.py b/src/PTVS/Microsoft.PythonTools.Analyzer/PythonScraper.py new file mode 100644 index 000000000..8758a556c --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/PythonScraper.py @@ -0,0 +1,999 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABILITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +""" +Generates information for supporting completion and analysis of Python code. + +Outputs a pickled set of dictionaries. The dictionaries are in the format: + +top-level: module_table + +module_table: + { + 'members': member_table, + 'doc': doc_string, + } + +type_table: + { + 'mro' : type_list, + 'bases' : type_list, + 'members' : member_table, + 'doc' : doc_string, + 'is_hidden': bool, + 'builtin': bool + } + +member_table: + { member_name : member_entry } + +member_name: str + +member_entry: + { + 'kind': member_kind + 'value': member_value + 'version': version_spec # optional + } + +member_kind: 'function' | 'funcref' | 'method' | 'property' | 'data' | 'type' | 'multiple' | 'typeref' | 'moduleref' +member_value: builtin_function | getset_descriptor | data | type_table | multiple_value | typeref | moduleref + +moduleref: + { 'module_name' : name } + +typeref: + ( + module name, # use '' to omit + type name, + type_list # index types; optional + ) + +funcref: + { + 'func_name' : fully-qualified function name + } + +multiple_value: + { 'members' : (member_entry, ... ) } + +builtin_function: + { + 'doc': doc string, + 'overloads': overload_table, + 'builtin' : bool, + 'static': bool, + } + +overload_table: + [overload, ...] + +overload: + { + 'args': args_info, + 'ret_type': type_list + } + +args_info: + (arg_info, ...) + +arg_info: + { + 'type': type_list, + 'name': argument name, + 'default_value': repr of default value, + 'arg_format' : ('*' | '**') + } + +getset_descriptor: + { + 'doc': doc string, + 'type': type_list + } + +data: + { 'type': type_list } + +type_list: + [type_ref, ...] +""" + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +try: + import cPickle as pickle +except ImportError: + import pickle # Py3k +import datetime +import os +import subprocess +import sys +import traceback +import types + +# The values in KNOWN_METHOD_TYPES and KNOWN_FUNCTION_TYPES are used when +# detecting the types of members. The names are ('module.name', 'type.name') +# pairs, matching the contents of a typeref. +# +# If the type's name matches an item in KNOWN_METHOD_TYPES, the object is +# treated as a method descriptor. +# +# If the type's name matches an item in KNOWN_FUNCTION_TYPES, the object is +# treated as a method if defined on a type or a function otherwise. +KNOWN_METHOD_TYPES = frozenset( + ('sip', 'methoddescriptor'), +) + +KNOWN_FUNCTION_TYPES = frozenset( + ('numpy', 'ufunc'), +) + + +# Safe member access methods because PyQt4 contains compiled types that crash +# on operations that should be completely safe, such as getattr() and dir(). +# These should be used throughout when accessing members on type objects. +def safe_getattr(obj, attr, default): + try: + return getattr(obj, attr, default) + except: + return default + +def safe_hasattr(obj, attr): + try: + return hasattr(obj, attr) + except: + return False + +def safe_isinstance(obj, types): + try: + return isinstance(obj, types) + except: + return False + +# safe_dir is imported from BuiltinScraper/IronPythonScraper +def safe_repr(obj): + try: + return repr(obj) + except: + return 'invalid object' + +if sys.version_info[0] == 2: + builtin_name = '__builtin__' +else: + builtin_name = 'builtins' + + +TYPE_NAMES = {} + +def types_to_typelist(iterable): + return [type_to_typeref(type) for type in iterable] + +def type_to_typelist(type): + return [type_to_typeref(type)] + +def typename_to_typeref(n1, n2=None): + '''If both n1 and n2 are specified, n1 is the module name and n2 is the type name. + If only n1 is specified, it is the type name. + ''' + if n2 is None: + name = ('', n1) + elif n1 == '__builtin__': + name = (builtin_name, n2) + else: + name = (n1, n2) + return memoize_type_name(name) + +def type_to_typeref(type): + type_name = safe_getattr(type, '__name__', None) + if not type_name: + print('Cannot get type name of ' + safe_repr(type)) + type = object + type_name = 'object' + if safe_hasattr(type, '__module__'): + if safe_getattr(type, '__module__', '') == '__builtin__': + name = (builtin_name, type_name) + else: + name = (type.__module__, type_name) + elif safe_isinstance(type, types.ModuleType): + name = (type_name, '') + else: + name = ('', type_name) + # memoize so when we pickle we can share refs + return memoize_type_name(name) + +def memoize_type_name(name): + key = repr(name) + if key in TYPE_NAMES: + return TYPE_NAMES[key] + TYPE_NAMES[key] = name + return name + +def maybe_to_typelist(type): + if isinstance(type, list) and len(type) > 0 and isinstance(type[0], tuple) and len(type[0]) > 1 and type[0][1]: + return type + elif isinstance(type, tuple) and len(type) > 1 and type[1]: + return [type] + else: + return type_to_typelist(type) + +def generate_overload(ret_type, *args): + '''ret_type is either a type suitable for type_to_typelist, or it is the result of + one of the *_to_typelist() or *_to_typeref() functions. + + Each arg is a tuple of ('name', type or type_ref or type_list, '' or '*' or '**', 'default value string') + The last two elements are optional, but the format is required if the default value + is specified. + ''' + res = { 'ret_type': maybe_to_typelist(ret_type) } + arglist = [] + for arg in args: + arglist.append({ 'name': arg[0], 'type': maybe_to_typelist(arg[1]) }) + if len(arg) >= 3 and arg[2]: + arglist[-1]['arg_format'] = arg[2] + if len(arg) >= 4: + arglist[-1]['default_value'] = arg[3] + res['args'] = tuple(arglist) + return res + +if sys.platform == "cli": + # provides extra type info when generating against IronPython which can be + # used w/ CPython completions + import IronPythonScraper as BuiltinScraper +else: + import BuiltinScraper + +def generate_builtin_function(function, is_method=False): + function_table = {} + + func_doc = safe_getattr(function, '__doc__', None) + if safe_isinstance(func_doc, str): + function_table['doc'] = func_doc + + function_table['overloads'] = BuiltinScraper.get_overloads(function, is_method) + + return function_table + +def generate_getset_descriptor(descriptor): + descriptor_table = {} + + desc_doc = safe_getattr(descriptor, '__doc__', None) + if safe_isinstance(desc_doc, str): + descriptor_table['doc'] = desc_doc + + desc_type = BuiltinScraper.get_descriptor_type(descriptor) + descriptor_table['type'] = type_to_typelist(desc_type) + + return descriptor_table + +NoneType = type(None) +slot_wrapper_type = type(int.__add__) +method_descriptor_type = type(str.center) +member_descriptor_type = type(property.fdel) +try: + getset_descriptor_type = type(file.closed) +except NameError: + getset_descriptor_type = type(Exception.args) # Py3k, no file +class_method_descriptor_type = type(datetime.date.__dict__['today']) +class OldStyleClass: pass +OldStyleClassType = type(OldStyleClass) + +def generate_member_table(obj, is_hidden=False, from_type=False, extra_types=None): + '''Generates a table of members of `obj`. + + `is_hidden` determines whether all the members are hidden from IntelliSense. + + `from_type` determines whether method descriptors are retained (True) or + ignored (False). + + `extra_types` is a sequence of ``(type_name, object)`` pairs to add as types + to this table. These types are always hidden. + ''' + + sentinel = object() + members = [] + for name in BuiltinScraper.safe_dir(obj): + member = safe_getattr(obj, name, sentinel) + if member is not sentinel: + members.append((name, member)) + + dependencies = {} + table = {} + if extra_types: + for name, member in extra_types: + member_kind, member_value = generate_member(member, is_hidden = True, from_type = from_type) + if member_kind == 'typeref': + actual_name = type_to_typeref(member) + if actual_name not in dependencies: + dependencies[actual_name] = member + table[name] = { 'kind': member_kind, 'value': member_value } + + for name, member in members: + member_kind, member_value = generate_member(member, is_hidden, from_type) + if member_kind == 'typeref': + actual_name = type_to_typeref(member) + if actual_name not in dependencies: + dependencies[actual_name] = member + table[name] = { 'kind': member_kind, 'value': member_value } + + if dependencies: + obj_mod, obj_name = type_to_typeref(obj) + def needs_type_info(other_mod, other_name): + if obj_mod != other_mod: + if other_mod == builtin_name: + # Never embed builtins (unless obj_mod is builtins, in + # which case the first comparison failed) + return False + + # Always embed external types + return True + + # We know obj_mod == other_mod at this point + + if not obj_name: + # Writing ourselves in the expected place + return True + elif obj_name.startswith(other_name + '.'): + # Always write references to outer types + return False + elif other_name and other_name.startswith(obj_name + '.'): + # Always write type info for inner types + return True + + # Otherwise, use a typeref + return False + + for (dep_mod, dep_name), dep_obj in dependencies.items(): + if needs_type_info(dep_mod, dep_name): + table[dep_name] = { + 'kind': 'type', + 'value': generate_type(dep_obj, is_hidden = dep_name not in table), + } + + return table + +def generate_member(obj, is_hidden=False, from_type=False): + try: + # Already handling all exceptions here, so don't worry about using the + # 'safe_*' functions. + + if isinstance(obj, (types.BuiltinFunctionType, class_method_descriptor_type)): + return 'function', generate_builtin_function(obj) + elif isinstance(obj, types.FunctionType): + # PyPy - we see plain old Python functions in addition to built-ins + return 'method' if from_type else 'function', generate_builtin_function(obj, from_type) + elif isinstance(obj, (type, OldStyleClassType)): + return 'typeref', type_to_typelist(obj) + elif isinstance(obj, (types.BuiltinMethodType, slot_wrapper_type, method_descriptor_type)): + return 'method', generate_builtin_function(obj, True) + elif isinstance(obj, (getset_descriptor_type, member_descriptor_type)): + return 'property', generate_getset_descriptor(obj) + + # Check whether we recognize the type name as one that does not respond + # correctly to isinstance checks. + type_name = type_to_typeref(type(obj)) + + if type_name in KNOWN_METHOD_TYPES: + return 'method', generate_builtin_function(obj, True) + + if type_name in KNOWN_FUNCTION_TYPES: + return 'method' if from_type else 'function', generate_builtin_function(obj, from_type) + + # Callable objects with a docstring that provides us with at least one + # overload will be treated as functions rather than data. + if safe_hasattr(obj, '__call__'): + try: + info = generate_builtin_function(obj, from_type) + if info and info['overloads']: + return 'method' if from_type else 'function', info + except: + pass + except: + # Some compiled types fail here, so catch everything and treat the + # object as data. + traceback.print_exc() + print('Treating type as data') + + # We don't have any special handling for this object type, so treat it as + # a constant. + return 'data', generate_data(obj) + + +if sys.version > '3.': + str_types = (str, bytes) +else: + str_types = (str, unicode) + + +def generate_type_new(type_obj, obj): + if safe_isinstance(obj, (types.BuiltinFunctionType, class_method_descriptor_type)): + function_info = generate_builtin_function(obj) + + new_overloads = BuiltinScraper.get_new_overloads(type_obj, obj) + if new_overloads is not None: + # replace overloads with better version if available + function_info['overloads'] = new_overloads + return 'function', function_info + + if safe_getattr(obj, '__doc__', '') == 'T.__new__(S, ...) -> a new object with type S, a subtype of T': + doc_str = safe_getattr(type_obj, '__doc__', None) + if not safe_isinstance(doc_str, str_types): + doc_str = '' + return ( + 'function', + { + 'doc': doc_str, + 'overloads' : [{'doc': doc_str, 'args': [{'arg_format': '*', 'name': 'args'}] }] + } + ) + + return generate_member(obj) + +def oldstyle_mro(type_obj, res): + type_bases = safe_getattr(type_obj, '__bases__', None) + + if not type_bases: + return res + + for base in type_bases: + if base not in res: + res.append(type_to_typeref(base)) + + for base in type_bases: + oldstyle_mro(base, res) + return res + +def generate_type(type_obj, is_hidden=False): + type_table = {} + + type_mro = safe_getattr(type_obj, '__mro__', None) + if type_mro: + type_table['mro'] = types_to_typelist(type_mro) + else: + type_table['mro'] = oldstyle_mro(type_obj, []) + + type_bases = safe_getattr(type_obj, '__bases__', None) + if type_bases: + type_table['bases'] = types_to_typelist(type_bases) + + type_doc = safe_getattr(type_obj, '__doc__', None) + if safe_isinstance(type_doc, str): + type_table['doc'] = type_doc + + if is_hidden: + type_table['is_hidden'] = True + + + type_table['members'] = member_table = generate_member_table(type_obj) + + if type_obj is object: + member_table['__new__'] = { + 'kind' : 'function', + 'value': { 'overloads': [generate_overload(object, ('cls', type))] } + } + elif '__new__' not in member_table: + member_table['__new__'] = generate_type_new(type_obj, + safe_getattr(type_obj, '__new__', object.__new__),) + + if ('__getattribute__' in member_table and + type_obj is not object and + safe_isinstance(safe_getattr(type_obj, '__getattribute__', None), slot_wrapper_type)): + # skip __getattribute__ on types other than object if it's just a slot + # wrapper. + del member_table['__getattribute__'] + + return type_table + +def generate_data(data_value): + data_table = {} + + data_type = type(data_value) + data_table['type'] = type_to_typelist(data_type) + + return data_table + +def lookup_module(module_name): + try: + module = __import__(module_name) + except: + return None + if '.' in module_name: + for name in module_name.split('.')[1:]: + module = safe_getattr(module, name, None) + if not module: + module = sys.modules[module_name] + + return module + +def generate_module(module, extra_types=None): + module_table = {} + + module_doc = safe_getattr(module, '__doc__', None) + if safe_isinstance(module_doc, str): + module_table['doc'] = module_doc + + module_table['members'] = generate_member_table(module, extra_types = extra_types) + + return module_table + + +def get_module_members(module): + """returns an iterable which gives the names of the module which should be exposed""" + module_all = safe_getattr(module, '__all__', None) + if module_all: + return frozenset(module_all) + + return BuiltinScraper.safe_dir(module) + +def generate_builtin_module(): + extra_types = {} + extra_types['object'] = type(object) + extra_types['function'] = types.FunctionType + extra_types['builtin_function'] = types.BuiltinFunctionType + extra_types['builtin_method_descriptor'] = types.BuiltinMethodType + extra_types['generator'] = types.GeneratorType + extra_types['NoneType'] = NoneType + extra_types['ellipsis'] = type(Ellipsis) + extra_types['module_type'] = types.ModuleType + if sys.version_info[0] == 2: + extra_types['dict_keys'] = type({}.iterkeys()) + extra_types['dict_values'] = type({}.itervalues()) + extra_types['dict_items'] = type({}.iteritems()) + else: + extra_types['dict_keys'] = type({}.keys()) + extra_types['dict_values'] = type({}.values()) + extra_types['dict_items'] = type({}.items()) + + extra_types['list_iterator'] = type(iter(list())) + extra_types['tuple_iterator'] = type(iter(tuple())) + extra_types['set_iterator'] = type(iter(set())) + extra_types['str_iterator'] = type(iter("")) + if sys.version_info[0] == 2: + extra_types['bytes_iterator'] = type(iter("")) + extra_types['unicode_iterator'] = type(iter(unicode())) + else: + extra_types['bytes_iterator'] = type(iter(bytes())) + extra_types['unicode_iterator'] = type(iter("")) + extra_types['callable_iterator'] = type(iter(lambda: None, None)) + + res = generate_module(lookup_module(builtin_name), extra_types = extra_types.items()) + + if res and 'members' in res and 'object' in res['members']: + assert res['members']['object']['kind'] == 'type', "Unexpected: " + repr(res['members']['object']) + res['members']['object']['value']['doc'] = "The most base type" + + return res + + +def merge_type(baseline_type, new_type): + if 'doc' not in new_type and 'doc' in baseline_type: + new_type['doc'] = baseline_type['doc'] + + merge_member_table(baseline_type['members'], new_type['members']) + + return new_type + +def merge_function(baseline_func, new_func): + new_func['overloads'].extend(baseline_func['overloads']) + return new_func + +def merge_property(baseline_prop, new_prop): + new_prop['type'].extend(baseline_prop['type']) + return new_prop + +def merge_data(baseline_data, new_data): + new_data['type'].extend(baseline_data['type']) + return new_data + +def merge_method(baseline_method, new_method): + if baseline_method.get('overloads') is not None: + if new_method.get('overloads') is None: + new_method['overloads'] = baseline_method['overloads'] + else: + new_method['overloads'].extend(baseline_method['overloads']) + + if 'doc' in baseline_method and 'doc' not in new_method: + new_method['doc'] = baseline_method['doc'] + #print 'new doc string' + + return new_method + +_MERGES = {'type' : merge_type, + 'function': merge_method, + 'property': merge_property, + 'data': merge_data, + 'method': merge_method} + +def merge_member_table(baseline_table, new_table): + for name, member_table in new_table.items(): + base_member_table = baseline_table.get(name, None) + kind = member_table['kind'] + + if base_member_table is not None and base_member_table['kind'] == kind: + merger = _MERGES.get(kind, None) + if merger is not None: + member_table['value'] = merger(base_member_table['value'], member_table['value']) + #else: + # print('unknown kind') + #elif base_member_table is not None: + # print('kinds differ', kind, base_member_table['kind'], name) + +InitMethodEntry = { + 'kind': 'method', + 'value': { + 'doc': 'x.__init__(...) initializes x; see help(type(x)) for signature', + 'overloads': [generate_overload(NoneType, ('self', object), ('args', object, '*'), ('kwargs', object, '**'))] + } +} + +NewMethodEntry = { + 'kind': 'function', + 'value': { + 'doc': 'T.__new__(S, ...) -> a new object with type S, a subtype of T', + 'overloads': [generate_overload(object, ('self', object), ('args', object, '*'), ('kwargs', object, '**'))] + } +} + +ReprMethodEntry = { + 'kind': 'method', + 'value': { + 'doc': 'x.__repr__() <==> repr(x)', + 'overloads': [generate_overload(str, ('self', object))] + } +} + + + +def _sre_post_fixer(mod): + if sys.platform == 'cli': + # IronPython doesn't have a real _sre module + return mod + + mod['members']['compile'] = { + 'kind': 'function', + 'value': { + 'overloads': [generate_overload(typename_to_typeref('_sre', 'SRE_Pattern'), + ('pattern', object), ('flags', object), ('code', object), ('groups', object), + ('groupindex', object), ('indexgroup', object))], + 'builtin' : True, + 'static': True, + } + } + mod['members']['SRE_Match'] = { + 'kind': 'type', + 'value': { + 'bases': [(builtin_name, 'object')], + 'doc': 'SRE_Match(m: Match, pattern: SRE_Pattern, text: str)\r\nRE_Match(m: Match, pattern: SRE_Pattern, text: str, pos: int, endpos: int)\r\n', + 'members': { + '__new__': { + 'kind': 'function', + 'value': { + 'doc': '__new__(cls: type, m: Match, pattern: SRE_Pattern, text: str)\r\n__new__(cls: type, m: Match, pattern: SRE_Pattern, text: str, pos: int, endpos: int)\r\n', + 'overloads': None + } + }, + 'end': { + 'kind': 'method', + 'value': { + 'doc': 'end(self: SRE_Match, group: object) -> int\r\nend(self: SRE_Match) -> int\r\n', + 'overloads': [ + generate_overload(int, ('self', typename_to_typeref('re', 'SRE_Match'))), + generate_overload(int, ('self', typename_to_typeref('re', 'SRE_Match')), ('group', object)) + ], + } + }, + 'endpos': { + 'kind': 'property', + 'value': { + 'doc': 'Get: endpos(self: SRE_Match) -> int\r\n\r\n', + 'type': type_to_typelist(int) + } + }, + 'expand': { + 'kind': 'method', + 'value': { + 'doc': 'expand(self: SRE_Match, template: object) -> str\r\n', + 'overloads': [generate_overload(str, ('self', typename_to_typeref('re', 'SRE_Match')), ('template', object))], + } + }, + 'group': { + 'kind': 'method', + 'value': { + 'doc': 'group(self: SRE_Match) -> str\r\ngroup(self: SRE_Match, index: object) -> str\r\ngroup(self: SRE_Match, index: object, *additional: Array[object]) -> object\r\n', + 'overloads': [ + generate_overload(str, ('self', typename_to_typeref('re', 'SRE_Match'))), + generate_overload(str, ('self', typename_to_typeref('re', 'SRE_Match')), ('index', object)), + generate_overload(object, ('self', typename_to_typeref('re', 'SRE_Match')), ('index', object), ('additional', tuple, '*')) + ], + }, + }, + 'groupdict': { + 'kind': 'method', + 'value': { + 'doc': 'groupdict(self: SRE_Match, value: object) -> dict (of str to object)\r\ngroupdict(self: SRE_Match, value: str) -> dict (of str to str)\r\ngroupdict(self: SRE_Match) -> dict (of str to str)\r\n', + 'overloads': [ + generate_overload(dict, ('self', typename_to_typeref('re', 'SRE_Match')), ('value', types_to_typelist([object, str]))), + generate_overload(dict, ('self', typename_to_typeref('re', 'SRE_Match'))) + ], + } + }, + 'groups': { + 'kind': 'method', + 'value': { + 'doc': 'groups(self: SRE_Match, default: object) -> tuple\r\ngroups(self: SRE_Match) -> tuple (of str)\r\n', + 'overloads': [ + generate_overload(tuple, ('self', typename_to_typeref('re', 'SRE_Match')), ('default', object)), + generate_overload(tuple, ('self', typename_to_typeref('re', 'SRE_Match'))) + ], + } + }, + 'lastgroup': { + 'kind': 'property', + 'value': { + 'doc': 'Get: lastgroup(self: SRE_Match) -> str\r\n\r\n', + 'type': type_to_typelist(str) + } + }, + 'lastindex': { + 'kind': 'property', + 'value': { + 'doc': 'Get: lastindex(self: SRE_Match) -> object\r\n\r\n', + 'type': type_to_typelist(object) + } + }, + 'pos': { + 'kind': 'property', + 'value': { + 'doc': 'Get: pos(self: SRE_Match) -> int\r\n\r\n', + 'type': type_to_typelist(int) + } + }, + 're': { + 'kind': 'property', + 'value': { + 'doc': 'Get: re(self: SRE_Match) -> SRE_Pattern\r\n\r\n', + 'type': [typename_to_typeref('re', 'SRE_Pattern')] + } + }, + 'regs': { + 'kind': 'property', + 'value': { + 'doc': 'Get: regs(self: SRE_Match) -> tuple\r\n\r\n', + 'type': type_to_typelist(tuple) + } + }, + 'span': { + 'kind': 'method', + 'value': { + 'doc': 'span(self: SRE_Match, group: object) -> tuple (of int)\r\nspan(self: SRE_Match) -> tuple (of int)\r\n', + 'overloads': [ + generate_overload(tuple, ('self', typename_to_typeref('re', 'SRE_Match'))), + generate_overload(tuple, ('self', typename_to_typeref('re', 'SRE_Match')), ('group', object)) + ] + } + }, + 'start': { + 'kind': 'method', + 'value': { + 'doc': 'start(self: SRE_Match, group: object) -> int\r\nstart(self: SRE_Match) -> int\r\n', + 'overloads': [ + generate_overload(int, ('self', typename_to_typeref('re', 'SRE_Match'))), + generate_overload(int, ('self', typename_to_typeref('re', 'SRE_Match')), ('group', object)) + ] + } + }, + 'string': { + 'kind': 'property', + 'value': { + 'doc': 'Get: string(self: SRE_Match) -> str\r\n\r\n', + 'type': type_to_typelist(str) + } + } + }, + 'mro': [typename_to_typeref('re', 'SRE_Match'), type_to_typeref(object)] + } + } + mod['members']['SRE_Pattern'] = { + 'kind': 'type', + 'value': {'bases': [type_to_typeref(object)], + 'doc': '', + 'members': { + '__eq__': { + 'kind': 'method', + 'value': { + 'doc': 'x.__eq__(y) <==> x==y', + 'overloads': [generate_overload(bool, ('self', typename_to_typeref('_sre', 'SRE_Pattern')), ('obj', object))], + } + }, + '__ne__': { + 'kind': 'method', + 'value': { + 'doc': '__ne__(x: object, y: object) -> bool\r\n', + 'overloads': [generate_overload(bool, ('x', object), ('y', object))] + } + }, + '__new__': NewMethodEntry, + 'findall': { + 'kind': 'method', + 'value': { + 'doc': 'findall(self: SRE_Pattern, string: object, pos: int, endpos: object) -> object\r\nfindall(self: SRE_Pattern, string: str, pos: int) -> object\r\nfindall(self: SRE_Pattern, string: str) -> object\r\n', + 'overloads': [generate_overload(bool, ('self', typename_to_typeref('_sre', 'SRE_Pattern')), ('string', str), ('pos', int, '', '0'), ('endpos', object, '', 'None'))] + } + }, + 'finditer': { + 'kind': 'method', + 'value': { + 'doc': 'finditer(self: SRE_Pattern, string: object, pos: int, endpos: int) -> object\r\nfinditer(self: SRE_Pattern, string: object, pos: int) -> object\r\nfinditer(self: SRE_Pattern, string: object) -> object\r\n', + 'overloads': [generate_overload(object, ('self', typename_to_typeref('_sre', 'SRE_Pattern')), ('string', str), ('pos', int, '', '0'), ('endpos', int, '', 'None'))] + } + }, + 'flags': { + 'kind': 'property', + 'value': { + 'doc': 'Get: flags(self: SRE_Pattern) -> int\r\n\r\n', + 'type': type_to_typelist(int) + } + }, + 'groupindex': { + 'kind': 'property', + 'value': { + 'doc': 'Get: groupindex(self: SRE_Pattern) -> dict\r\n\r\n', + 'type': type_to_typelist(dict) + } + }, + 'groups': { + 'kind': 'property', + 'value': { + 'doc': 'Get: groups(self: SRE_Pattern) -> int\r\n\r\n', + 'type': type_to_typelist(int) + } + }, + 'match': { + 'kind': 'method', + 'value': { + 'doc': 'match(self: SRE_Pattern, text: object, pos: int, endpos: int) -> RE_Match\r\nmatch(self: SRE_Pattern, text: object, pos: int) -> RE_Match\r\nmatch(self: SRE_Pattern, text: object) -> RE_Match\r\n', + 'overloads': [generate_overload(object, ('self', typename_to_typeref('_sre', 'SRE_Pattern')), ('text', str), ('pos', int, '', '0'), ('endpos', int, '', 'None'))], + } + }, + 'pattern': { + 'kind': 'property', + 'value': { + 'doc': 'Get: pattern(self: SRE_Pattern) -> str\r\n\r\n', + 'type': type_to_typelist(str) + } + }, + 'search': { + 'kind': 'method', + 'value': { + 'doc': 'search(self: SRE_Pattern, text: object, pos: int, endpos: int) -> RE_Match\r\nsearch(self: SRE_Pattern,text: object, pos: int) -> RE_Match\r\nsearch(self: SRE_Pattern, text: object) -> RE_Match\r\n', + 'overloads': [generate_overload(typename_to_typeref('_sre', 'RE_Match'), ('self', typename_to_typeref('_sre', 'SRE_Pattern')), ('text', str), ('pos', int, '', '0'), ('endpos', int, '', 'None'))] + } + }, + 'split': { + 'kind': 'method', + 'value': { + 'doc': 'split(self: SRE_Pattern, string: object, maxsplit: int) -> list (of str)\r\nsplit(self: SRE_Pattern, string: str) -> list (of str)\r\n', + 'overloads': [generate_overload(list, ('self', typename_to_typeref('_sre', 'SRE_Pattern')), ('string', str), ('maxsplit', int, '', 'None'))] + } + }, + 'sub': { + 'kind': 'method', + 'value': { + 'doc': 'sub(self: SRE_Pattern, repl: object, string: object, count: int) -> str\r\nsub(self: SRE_Pattern, repl: object, string: object) -> str\r\n', + 'overloads': [generate_overload(str, ('self', typename_to_typeref('_sre', 'SRE_Pattern')), ('repl', object), ('string', str), ('count', int, '', 'None'))] + } + }, + 'subn': { + 'kind': 'method', + 'value': {'doc': 'subn(self: SRE_Pattern, repl: object, string: object, count: int) -> object\r\nsubn(self: SRE_Pattern, repl: object, string: str) -> object\r\n', + 'overloads': [generate_overload(object, ('self', typename_to_typeref('_sre', 'SRE_Pattern')), ('repl', object), ('string', str), ('count', int, '', 'None'))] + } + }, + }, + 'mro': [typename_to_typeref('_sre', 'SRE_Pattern'), + type_to_typeref(object)] + } + } + + return mod + + +# fixers which run on the newly generated file, not on the baseline file. +post_module_fixers = { + '_sre' : _sre_post_fixer, +} + +def merge_with_baseline(mod_name, baselinepath, final): + baseline_file = os.path.join(baselinepath, mod_name + '.idb') + if os.path.exists(baseline_file): + print(baseline_file) + f = open(baseline_file, 'rb') + baseline = pickle.load(f) + f.close() + + #import pprint + #pp = pprint.PrettyPrinter() + #pp.pprint(baseline['members']) + fixer = post_module_fixers.get(mod_name, None) + if fixer is not None: + final = fixer(final) + + merge_member_table(baseline['members'], final['members']) + + return final + +def write_analysis(out_filename, analysis): + out_file = open(out_filename + '.idb', 'wb') + saved_analysis = pickle.dumps(analysis, 2) + if sys.platform == 'cli': + # work around strings always being unicode on IronPython, we fail to + # write back out here because IronPython doesn't like the non-ascii + # characters in the unicode string + import System + data = System.Array.CreateInstance(System.Byte, len(saved_analysis)) + for i, v in enumerate(saved_analysis): + try: + data[i] = ord(v) + except: + pass + + saved_analysis = data + out_file.write(saved_analysis) + out_file.close() + + # write a list of members which we can load to check for member existance + out_file = open(out_filename + '.idb.$memlist', 'wb') + for member in sorted(analysis['members']): + if sys.version_info >= (3, 0): + out_file.write((member + '\n').encode('utf8')) + else: + out_file.write(member + '\n') + + out_file.close() + +def write_module(mod_name, outpath, analysis): + write_analysis(os.path.join(outpath, mod_name), analysis) + + + +if __name__ == "__main__": + outpath = sys.argv[1] + if len(sys.argv) > 2: + baselinepath = sys.argv[2] + else: + baselinepath = None + + res = generate_builtin_module() + if not res: + raise RuntimeError("Unable to scrape builtins") + res = merge_with_baseline(builtin_name, baselinepath, res) + + write_module(builtin_name, outpath, res) + + for mod_name in sys.builtin_module_names: + if (mod_name == builtin_name or + mod_name == '__main__' or + not BuiltinScraper.should_include_module(mod_name)): + continue + + res = generate_module(lookup_module(mod_name)) + if res is not None: + try: + res = merge_with_baseline(mod_name, baselinepath, res) + + write_module(mod_name, outpath, res) + except ValueError: + pass diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/VersionDiff.py b/src/PTVS/Microsoft.PythonTools.Analyzer/VersionDiff.py new file mode 100644 index 000000000..e2c28f59e --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/VersionDiff.py @@ -0,0 +1,68 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABLITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +# Compares two different completion databases, dumping the results. +# Used for generating fixers for later versions of Python. +# Usage: +# First run: +# python.exe PythonScraper.py OutDir1 +# pythonv2.exe PythonScraper.py OutDir2 +# python.exe VersionDiff.py OutDir1 OutDir2 +# +# This will then dump a list of removed and new members which can be added to PythonScraper.py. +# A person needs to manually merge these in and make sure the version fields are all correct. +# In general this should be low overhead once every couple of years when a new version of Python +# ships. +# +# In theory this could be completely automated but we'd need to automatically run against all Python +# versions to figure out the minimum version. + +import os, pprint, sys +try: + import cPickle +except: + import _pickle as cPickle + +dir1 = sys.argv[1] +dir2 = sys.argv[2] + +files3 = set(os.listdir(dir1)) +files2 = set(os.listdir(dir2)) + +for filename in files3: + if filename.endswith('.idb') and filename in files2: + b2 = cPickle.load(file(dir2 + '\\' + filename, 'rb')) + b3 = cPickle.load(file(dir1 + '\\' + filename, 'rb')) + + removed_three = set(b2['members']) - set(b3['members']) + added_three = set(b3['members']) - set(b2['members']) + + if removed_three or added_three: + print(filename[:-4]) + if removed_three: + print('Removed in ' + dir1, removed_three) + #if added_three: + # print('New in ' + dir1, added_three) + + #for added in added_three: + # sys.stdout.write("mod['members']['%s'] = " % added) + # b3['members'][added]['value']['version'] = '>=3.2' + # pprint.pprint(b3['members'][added]) + # + diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Connection.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Connection.cs new file mode 100644 index 000000000..5914597df --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Connection.cs @@ -0,0 +1,709 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Runtime.ExceptionServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Microsoft.PythonTools.Ipc.Json { + public sealed class Connection : IDisposable { + private readonly object _cacheLock = new object(); + private readonly SemaphoreSlim _writeLock = new SemaphoreSlim(1); + private readonly Dictionary _requestCache; + private readonly Dictionary _types; + private readonly Func, Task> _requestHandler; + private readonly bool _disposeWriter, _disposeReader; + private readonly Stream _writer, _reader; + private readonly TextWriter _basicLog; + private readonly TextWriter _logFile; + private readonly object _logFileLock; + private int _seq; + private static readonly char[] _headerSeparator = new[] { ':' }; + + // Exposed settings for tests + internal static bool AlwaysLog = false; + internal static string LoggingBaseDirectory = Path.GetTempPath(); + + private const string LoggingRegistrySubkey = @"Software\Microsoft\PythonTools\ConnectionLog"; + + private static readonly Encoding TextEncoding = new UTF8Encoding(false); + + /// + /// Creates a new connection object for doing client/server communication. + /// + /// The stream that we write to for sending requests and responses. + /// The writer stream is disposed when this object is disposed. + /// The stream that we read from for reading responses and events. + /// The reader stream is disposed when this object is disposed. + /// The callback that is invoked when a request is received. + /// A set of registered types for receiving requests and events. The key is "request.command_name" or "event.event_name" + /// where command_name and event_name correspond to the fields in the Request and Event objects. The type is the type of object + /// which will be deserialized and instantiated. If a type is not registered a GenericRequest or GenericEvent object will be created + /// which will include the complete body of the request as a dictionary. + /// + /// Name of registry key used to determine if we should log messages and exceptions to disk. + /// Text writer to use for basic logging (message ids only). If null then output will go to . + public Connection( + Stream writer, + bool disposeWriter, + Stream reader, + bool disposeReader, + Func, Task> requestHandler = null, + Dictionary types = null, + string connectionLogKey = null, + TextWriter basicLog = null + ) { + _requestCache = new Dictionary(); + _requestHandler = requestHandler; + _types = types; + _writer = writer; + _disposeWriter = disposeWriter; + _reader = reader; + _disposeReader = disposeReader; + _basicLog = basicLog ?? new DebugTextWriter(); + _logFile = OpenLogFile(connectionLogKey, out var filename); + // FxCop won't let us lock a MarshalByRefObject, so we create + // a plain old object that we can log against. + if (_logFile != null) { + _logFileLock = new object(); + LogFilename = filename; + } + } + + public string LogFilename { get; } + + /// + /// Opens the log file for this connection. The log must be enabled in + /// the registry under HKCU\Software\Microsoft\PythonTools\ConnectionLog + /// with the connectionLogKey value set to a non-zero integer or + /// non-empty string. + /// + private static TextWriter OpenLogFile(string connectionLogKey, out string filename) { + filename = null; + if (!AlwaysLog) { + if (string.IsNullOrEmpty(connectionLogKey)) { + return null; + } + + using (var root = Win32.Registry.CurrentUser.OpenSubKey(LoggingRegistrySubkey, false)) { + var value = root?.GetValue(connectionLogKey); + int? asInt = value as int?; + if (asInt.HasValue) { + if (asInt.GetValueOrDefault() == 0) { + // REG_DWORD but 0 means no logging + return null; + } + } else if (string.IsNullOrEmpty(value as string)) { + // Empty string or no value means no logging + return null; + } + } + } + + var filenameBase = Path.Combine( + LoggingBaseDirectory, + string.Format("PythonTools_{0}_{1}_{2:yyyyMMddHHmmss}", connectionLogKey, Process.GetCurrentProcess().Id.ToString(), DateTime.Now) + ); + + filename = filenameBase + ".log"; + for (int counter = 0; counter < int.MaxValue; ++counter) { + try { + var file = new FileStream(filename, FileMode.CreateNew, FileAccess.Write, FileShare.ReadWrite); + return new StreamWriter(file, TextEncoding); + } catch (IOException) { + } catch (UnauthorizedAccessException) { + } + filename = string.Format("{0}_{1}.log", filenameBase, ++counter); + } + return null; + } + + private void LogToDisk(string message) { + if (_logFile == null || _logFileLock == null) { + return; + } + try { + lock (_logFileLock) { + _logFile.WriteLine(message); + _logFile.Flush(); + } + } catch (IOException) { + } catch (ObjectDisposedException) { + } + } + + private void LogToDisk(Exception ex) { + if (_logFile == null || _logFileLock == null) { + return; + } + // Should have immediately logged a message before, so the exception + // will have context automatically. Stack trace will not be + // interesting because of the async-ness. + LogToDisk(string.Format("{0}: {1}", ex.GetType().Name, ex.Message)); + } + + /// + /// When a fire and forget notification is received from the other side this event is raised. + /// + public event EventHandler EventReceived; + + /// + /// When a fire and forget error notification is received from the other side this event is raised. + /// + public event EventHandler ErrorReceived; + + /// + /// Sends a request from the client to the listening server. + /// + /// All request payloads inherit from Request<TResponse> where the TResponse generic parameter + /// specifies the .NET type used for serializing the response. + /// + /// Request to send to the server. + /// + /// An action that you want to invoke after the SendRequestAsync task + /// has completed, but before the message processing loop reads the + /// next message from the read stream. This is currently used to cancel + /// the message loop immediately on reception of a response. This is + /// necessary when you want to stop processing messages without closing + /// the read stream. + /// + public async Task SendRequestAsync(Request request, CancellationToken cancellationToken = default(CancellationToken), Action onResponse = null) + where T : Response, new() { + int seq = Interlocked.Increment(ref _seq); + + var r = new RequestInfo(request, seq, onResponse); + if (cancellationToken.CanBeCanceled) { + cancellationToken.Register(() => r._task.TrySetCanceled()); + } + + lock (_cacheLock) { + _requestCache[seq] = r; + } + + T res; + try { + _basicLog.WriteLine("Sending request {0}: {1}", seq, request.command); + await SendMessage( + new RequestMessage() { + command = r.Request.command, + arguments = r.Request, + seq = seq, + type = PacketType.Request + }, + cancellationToken + ).ConfigureAwait(false); + + res = await r._task.Task.ConfigureAwait(false); + if (r.success) { + return res; + } + + throw new FailedRequestException(r.message, res); + } finally { + lock (_cacheLock) { + _requestCache.Remove(seq); + } + } + } + + /// + /// Send a fire and forget event to the other side. + /// + /// The event value to be sent. + public async Task SendEventAsync(Event eventValue) { + int seq = Interlocked.Increment(ref _seq); + _basicLog.WriteLine("Sending event {0}: {1}", seq, eventValue.name); + try { + await SendMessage( + new EventMessage() { + @event = eventValue.name, + body = eventValue, + seq = seq, + type = PacketType.Event + }, + CancellationToken.None + ).ConfigureAwait(false); + } catch (ObjectDisposedException) { + } + } + + /// + /// Returns a task which will process incoming messages. This can be started on another thread or + /// in whatever form of synchronization context you like. StartProcessing is a convenience helper + /// for starting this running asynchronously using Task.Run. + /// + /// + public async Task ProcessMessages() { + try { + var reader = new ProtocolReader(_reader); + while (true) { + var packet = await ReadPacketAsJObject(reader); + if (packet == null) { + break; + } + + var type = packet["type"].ToObject(); + switch (type) { + case PacketType.Request: { + var seq = packet["seq"].ToObject(); + if (seq == null) { + throw new InvalidDataException("Request is missing seq attribute"); + } + await ProcessRequest(packet, seq); + } + break; + case PacketType.Response: + ProcessResponse(packet); + break; + case PacketType.Event: + ProcessEvent(packet); + break; + case PacketType.Error: + ProcessError(packet); + break; + default: + throw new InvalidDataException("Bad packet type: " + type ?? ""); + } + } + } catch (InvalidDataException ex) { + // UNDONE: Skipping assert to see if that fixes broken tests + //Debug.Assert(false, "Terminating ProcessMessages loop due to InvalidDataException", ex.Message); + // TODO: unsure that it makes sense to do this, but it maintains existing behavior + await WriteError(ex.Message); + } catch (OperationCanceledException) { + } catch (ObjectDisposedException) { + } + + _basicLog.WriteLine("ProcessMessages ended"); + } + + private void ProcessError(JObject packet) { + var eventBody = packet["body"]; + string message; + try { + message = eventBody["message"].Value(); + } catch (Exception e) { + message = e.Message; + } + try { + ErrorReceived?.Invoke(this, new ErrorReceivedEventArgs(message)); + } catch (Exception e) { + // TODO: Report unhandled exception? + Debug.Fail(e.Message); + } + } + + private void ProcessEvent(JObject packet) { + Type requestType; + var name = packet["event"].ToObject(); + var eventBody = packet["body"]; + Event eventObj; + try { + if (name != null && + _types != null && + _types.TryGetValue("event." + name, out requestType)) { + // We have a strongly typed event type registered, use that. + eventObj = eventBody.ToObject(requestType) as Event; + } else { + // We have no strongly typed event type, so give the user a + // GenericEvent and they can look through the body manually. + eventObj = new GenericEvent() { + body = eventBody.ToObject>() + }; + } + } catch (Exception e) { + // TODO: Notify receiver of invalid message + Debug.Fail(e.Message); + return; + } + try { + EventReceived?.Invoke(this, new EventReceivedEventArgs(name, eventObj)); + } catch (Exception e) { + // TODO: Report unhandled exception? + Debug.Fail(e.Message); + } + } + + private void ProcessResponse(JObject packet) { + var body = packet["body"]; + + var reqSeq = packet["request_seq"].ToObject(); + + _basicLog.WriteLine("Received response {0}", reqSeq); + + RequestInfo r; + lock (_cacheLock) { + // We might not find the entry in the request cache if the CancellationSource + // passed into SendRequestAsync was signaled before the request + // was completed. That's okay, there's no one waiting on the + // response anymore. + if (_requestCache.TryGetValue(reqSeq.Value, out r)) { + r.message = packet["message"]?.ToObject() ?? string.Empty; + r.success = packet["success"]?.ToObject() ?? false; + r.SetResponse(body); + } + } + } + + private async Task ProcessRequest(JObject packet, int? seq) { + var command = packet["command"].ToObject(); + var arguments = packet["arguments"]; + + Request request; + Type requestType; + if (command != null && + _types != null && + _types.TryGetValue("request." + command, out requestType)) { + // We have a strongly typed request type registered, use that... + request = (Request)arguments.ToObject(requestType); + } else { + // There's no strongly typed request type, give the user a generic + // request object and they can look through the dictionary. + request = new GenericRequest() { + body = arguments.ToObject>() + }; + } + + bool success = true; + string message = null; + try { + await _requestHandler( + new RequestArgs(command, request), + result => SendResponseAsync(seq.Value, command, success, message, result, CancellationToken.None) + ); + } catch (OperationCanceledException) { + throw; + } catch (ObjectDisposedException) { + throw; + } catch (Exception e) { + success = false; + message = e.ToString(); + Trace.TraceError(message); + await SendResponseAsync(seq.Value, command, success, message, null, CancellationToken.None).ConfigureAwait(false); + } + } + + public void Dispose() { + if (_disposeWriter) { + _writer.Dispose(); + } + if (_disposeReader) { + _reader.Dispose(); + } + _writeLock.Dispose(); + lock (_cacheLock) { + foreach (var r in _requestCache.Values) { + r.Cancel(); + } + } + try { + _logFile?.Dispose(); + } catch (ObjectDisposedException) { + } + } + + internal static async Task ReadPacketAsJObject(ProtocolReader reader) { + var line = await ReadPacket(reader).ConfigureAwait(false); + if (line == null) { + return null; + } + + string message = ""; + JObject packet = null; + try { + // JObject.Parse is more strict than JsonConvert.DeserializeObject, + // the latter happily deserializes malformed json. + packet = JObject.Parse(line); + } catch (JsonSerializationException ex) { + message = ": " + ex.Message; + } catch (JsonReaderException ex) { + message = ": " + ex.Message; + } + + if (packet == null) { + Debug.WriteLine("Failed to parse {0}{1}", line, message); + throw new InvalidDataException("Failed to parse packet" + message); + } + + return packet; + } + + /// + /// Reads a single message from the protocol buffer. First reads in any headers until a blank + /// line is received. Then reads in the body of the message. The headers must include a Content-Length + /// header specifying the length of the body. + /// + private static async Task ReadPacket(ProtocolReader reader) { + var headers = new Dictionary(StringComparer.OrdinalIgnoreCase); + var lines = new List(); + string line; + while ((line = await reader.ReadHeaderLineAsync().ConfigureAwait(false)) != null) { + lines.Add(line ?? "(null)"); + if (String.IsNullOrEmpty(line)) { + if (headers.Count == 0) { + continue; + } + // end of headers for this request... + break; + } + var split = line.Split(_headerSeparator, 2); + if (split.Length != 2) { + // Probably getting an error message, so read all available text + var error = line; + try { + // Encoding is uncertain since this is malformed + error += TextEncoding.GetString(await reader.ReadToEndAsync()); + } catch (ArgumentException) { + } + throw new InvalidDataException("Malformed header, expected 'name: value'" + Environment.NewLine + error); + } + headers[split[0]] = split[1]; + } + + if (line == null) { + return null; + } + + string contentLengthStr; + int contentLength; + + if (!headers.TryGetValue(Headers.ContentLength, out contentLengthStr)) { + // HACK: Attempting to find problem with message content + Console.Error.WriteLine("Content-Length not specified on request. Lines follow:"); + foreach (var l in lines) { + Console.Error.WriteLine($"> {l}"); + } + Console.Error.Flush(); + throw new InvalidDataException("Content-Length not specified on request"); + } + + if (!Int32.TryParse(contentLengthStr, out contentLength) || contentLength < 0) { + throw new InvalidDataException("Invalid Content-Length: " + contentLengthStr); + } + + var contentBinary = await reader.ReadContentAsync(contentLength); + if (contentBinary.Length == 0 && contentLength > 0) { + // The stream was closed, so let's abort safely + return null; + } + if (contentBinary.Length != contentLength) { + throw new InvalidDataException(string.Format("Content length does not match Content-Length header. Expected {0} bytes but read {1} bytes.", contentLength, contentBinary.Length)); + } + + try { + var text = TextEncoding.GetString(contentBinary); + return text; + } catch (ArgumentException ex) { + throw new InvalidDataException("Content is not valid UTF-8.", ex); + } + } + + private async Task SendResponseAsync( + int sequence, + string command, + bool success, + string message, + Response response, + CancellationToken cancel + ) { + int newSeq = Interlocked.Increment(ref _seq); + _basicLog.WriteLine("Sending response {0}", newSeq); + await SendMessage( + new ResponseMessage() { + request_seq = sequence, + success = success, + message = message, + command = command, + body = response, + seq = newSeq, + type = PacketType.Response + }, + cancel + ).ConfigureAwait(false); + } + + /// + /// Sends a single message across the wire. + /// + /// + /// Base protocol defined at https://github.com/Microsoft/language-server-protocol/blob/master/protocol.md#base-protocol + /// + private async Task SendMessage(ProtocolMessage packet, CancellationToken cancel) { + var str = JsonConvert.SerializeObject(packet, UriJsonConverter.Instance); + + try { + try { + await _writeLock.WaitAsync(cancel).ConfigureAwait(false); + } catch (ArgumentNullException) { + throw new ObjectDisposedException(nameof(_writeLock)); + } catch (ObjectDisposedException) { + throw new ObjectDisposedException(nameof(_writeLock)); + } + try { + LogToDisk(str); + + // The content part is encoded using the charset provided in the Content-Type field. + // It defaults to utf-8, which is the only encoding supported right now. + var contentBytes = TextEncoding.GetBytes(str); + + // The header part is encoded using the 'ascii' encoding. + // This includes the '\r\n' separating the header and content part. + var header = "Content-Length: " + contentBytes.Length + "\r\n\r\n"; + var headerBytes = Encoding.ASCII.GetBytes(header); + + await _writer.WriteAsync(headerBytes, 0, headerBytes.Length).ConfigureAwait(false); + await _writer.WriteAsync(contentBytes, 0, contentBytes.Length).ConfigureAwait(false); + await _writer.FlushAsync().ConfigureAwait(false); + } finally { + _writeLock.Release(); + } + } catch (Exception ex) { + LogToDisk(ex); + throw; + } + } + + /// + /// Writes an error on a malformed request. + /// + private async Task WriteError(string message) { + try { + await SendMessage( + new ErrorMessage() { + seq = Interlocked.Increment(ref _seq), + type = PacketType.Error, + body = new Dictionary() { { "message", message } } + }, + CancellationToken.None + ); + throw new InvalidOperationException(message); + } catch (ObjectDisposedException) { + } catch (IOException) { + } + } + + /// + /// Class used for serializing each packet of information we send across. + /// + /// Each packet consists of a packet type (defined in the PacketType class), a sequence + /// number (requests and responses have linked sequence numbers), and a body which is + /// the rest of the JSON payload. + /// + private class ProtocolMessage { + public string type; + public int seq; + } + + private class RequestMessage : ProtocolMessage { + public string command; + public object arguments; + } + + private class ResponseMessage : ProtocolMessage { + public int request_seq; + public bool success; + public string command; + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public string message; + public object body; + } + + private class EventMessage : ProtocolMessage { + public string @event; + public object body; + } + + private class ErrorMessage : ProtocolMessage { + public object body; + } + + private class PacketType { + public const string Request = "request"; + public const string Response = "response"; + public const string Event = "event"; + public const string Error = "error"; + } + + /// + /// Provides constants for known header types, currently just includs the Content-Length + /// header for specifying the length of the body of the request in bytes. + /// + private class Headers { + public const string ContentLength = "Content-Length"; + } + + /// + /// Base class for tracking state of our requests. This is a non-generic class so we can have + /// a dictionary of these and call the abstract methods which are actually implemented by the + /// generic version. + /// + private abstract class RequestInfo { + private readonly Request _request; + private readonly int _sequence; + public bool success; + public string message; + + internal RequestInfo(Request request, int sequence) { + _request = request; + _sequence = sequence; + } + + public Request Request => _request; + + internal abstract void SetResponse(JToken obj); + internal abstract void Cancel(); + } + + /// + /// Generic version of the request info type which includes the type information for + /// the type of response we should return. This response type is inferred from the + /// TRespones type parameter of the Request<TResponse> generic type which is + /// passed on a SendRequestAsync call. The caller this receives the strongly typed + /// response without any extra need to specify any information beyond the request + /// payload. + /// + private sealed class RequestInfo : RequestInfo where TResponse : Response, new() { + internal readonly TaskCompletionSource _task; + private readonly Action _postResponseAction; + + internal RequestInfo(Request request, int sequence, Action postResponseAction) : base(request, sequence) { + // Make it run continuation asynchronously so that the thread calling SetResponse + // doesn't end up being hijacked to run the code after SendRequest, which would + // prevent it from processing any more messages. + _task = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _postResponseAction = postResponseAction; + } + + internal override void SetResponse(JToken obj) { + var res = obj.ToObject(); + _task.TrySetResult(res); + _postResponseAction?.Invoke(res); + } + + internal override void Cancel() { + _task.TrySetCanceled(); + } + } + } +} + diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/DebugTextWriter.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/DebugTextWriter.cs new file mode 100644 index 000000000..ce8b70be8 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/DebugTextWriter.cs @@ -0,0 +1,38 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Diagnostics; +using System.IO; +using System.Text; + +namespace Microsoft.PythonTools.Ipc.Json { + public class DebugTextWriter : TextWriter { + public override Encoding Encoding => Encoding.UTF8; + public override void Write(char value) { + // Technically this is the only Write/WriteLine overload we need to + // implement. We override the string versions for better performance. + Debug.Write(value); + } + + public override void Write(string value) { + Debug.Write(value); + } + + public override void WriteLine(string value) { + Debug.WriteLine(value); + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/ErrorReceivedEventArgs.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/ErrorReceivedEventArgs.cs new file mode 100644 index 000000000..bb3a9bef9 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/ErrorReceivedEventArgs.cs @@ -0,0 +1,30 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; + +namespace Microsoft.PythonTools.Ipc.Json { + public sealed class ErrorReceivedEventArgs : EventArgs { + private readonly string _message; + + public ErrorReceivedEventArgs(string message) { + _message = message; + } + + public string Message => _message; + } +} + diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Event.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Event.cs new file mode 100644 index 000000000..016a4936f --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Event.cs @@ -0,0 +1,32 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace Microsoft.PythonTools.Ipc.Json { + public class Event { + + [JsonIgnore] + public virtual string name => null; + } + + + public class GenericEvent : Event { + public Dictionary body; + } +} + diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/EventReceivedEventArgs.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/EventReceivedEventArgs.cs new file mode 100644 index 000000000..dc03d6a89 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/EventReceivedEventArgs.cs @@ -0,0 +1,33 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; + +namespace Microsoft.PythonTools.Ipc.Json { + public sealed class EventReceivedEventArgs : EventArgs { + private readonly string _name; + private readonly Event _event; + + public EventReceivedEventArgs(string name, Event event_) { + _name = name; + _event = event_; + } + + public Event Event => _event; + public string Name => _name; + } +} + diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/FailedRequestException.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/FailedRequestException.cs new file mode 100644 index 000000000..41dfa6467 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/FailedRequestException.cs @@ -0,0 +1,38 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Runtime.Serialization; +using System.Security.Permissions; + +namespace Microsoft.PythonTools.Ipc.Json { + [Serializable] + public sealed class FailedRequestException : Exception { + private readonly Response _response; + + public FailedRequestException(string message, Response response) : base(message) { + _response = response; + } + + [SecurityPermission(SecurityAction.Demand, SerializationFormatter = true)] + public override void GetObjectData(SerializationInfo info, StreamingContext context) { + base.GetObjectData(info, context); + info.AddValue("Response", _response); + } + + public Response Response => _response; + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Microsoft.PythonTools.Ipc.Json.csproj b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Microsoft.PythonTools.Ipc.Json.csproj new file mode 100644 index 000000000..3c2837d70 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Microsoft.PythonTools.Ipc.Json.csproj @@ -0,0 +1,15 @@ + + + net472 + Microsoft.PythonTools.Ipc.Json + Microsoft.PythonTools.Ipc.Json + + + + + + + + + + diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Properties/AssemblyInfo.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..fe60dedfe --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Properties/AssemblyInfo.cs @@ -0,0 +1,24 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +[assembly: ComVisible(false)] +[assembly: Guid("e1e1613d-0c96-42f9-9f2d-052c72533297")] + +[assembly: InternalsVisibleTo("IpcJsonTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/ProtocolReader.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/ProtocolReader.cs new file mode 100644 index 000000000..942525613 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/ProtocolReader.cs @@ -0,0 +1,117 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.PythonTools.Ipc.Json { + class ProtocolReader { + private Stream _stream; + private List _buffer = new List(); + + public ProtocolReader(Stream stream) { + if (stream == null) { + throw new ArgumentNullException(nameof(stream)); + } + _stream = stream; + } + + /// + /// Reads an ASCII encoded header line asynchronously from the current stream + /// and returns the data as a string. Line termination chars are '\r\n' and + /// are excluded from the return value. Keeps reading until it finds it, and + /// if it reaches the end of the stream (no more data is read) without finding + /// it then it returns null. + /// + public async Task ReadHeaderLineAsync() { + // Keep reading into the buffer until it contains the '\r\n'. + int searchStartPos = 0; + int newLineIndex; + while ((newLineIndex = IndexOfNewLineInBuffer(searchStartPos)) < 0) { + searchStartPos = Math.Max(0, _buffer.Count - 1); + if (await ReadIntoBuffer() == 0) { + return null; + } + } + + var line = _buffer.Take(newLineIndex).ToArray(); + _buffer.RemoveRange(0, newLineIndex + 2); + return Encoding.ASCII.GetString(line); + } + + /// + /// Reads all bytes from the current position to the end of the + /// stream asynchronously. + /// + public async Task ReadToEndAsync() { + int read; + while ((read = await ReadIntoBuffer()) > 0) { + } + + var all = _buffer.ToArray(); + _buffer.Clear(); + return all; + } + + /// + /// Reads a number of bytes asynchronously from the current stream. + /// + /// Number of bytes to read. + /// + /// May return fewer bytes than requested. + /// + public async Task ReadContentAsync(int byteCount) { + if (byteCount < 0) { + throw new ArgumentOutOfRangeException(nameof(byteCount), byteCount, "Value cannot be negative."); + } + + while (_buffer.Count < byteCount) { + if (await ReadIntoBuffer() == 0) { + break; + } + } + + var actualCount = Math.Min(byteCount, _buffer.Count); + var result = _buffer.Take(actualCount).ToArray(); + _buffer.RemoveRange(0, actualCount); + return result; + } + + private int IndexOfNewLineInBuffer(int searchStartPos) { + for (int i = searchStartPos; i < _buffer.Count - 1; i++) { + if (_buffer[i] == 13 && _buffer[i + 1] == 10) { + return i; + } + } + return -1; + } + + /// + /// Reads bytes from the stream into the buffer, in chunks. + /// + /// Number of bytes that were added to the buffer. + private async Task ReadIntoBuffer() { + var temp = new byte[1024]; + var read = await _stream.ReadAsync(temp, 0, temp.Length); + _buffer.AddRange(temp.Take(read).ToArray()); + return read; + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Request.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Request.cs new file mode 100644 index 000000000..bc38fa1d8 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Request.cs @@ -0,0 +1,37 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Microsoft.PythonTools.Ipc.Json { + public class Request { + [JsonIgnore] + public virtual string command => null; + + public override string ToString() => command; + } + + public class GenericRequest : Request { + public Dictionary body; + } + + public class Request : Request where TResponse : Response, new() { + } +} \ No newline at end of file diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/RequestArgs.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/RequestArgs.cs new file mode 100644 index 000000000..93f1fe538 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/RequestArgs.cs @@ -0,0 +1,31 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + + +namespace Microsoft.PythonTools.Ipc.Json { + public sealed class RequestArgs { + private readonly string _command; + private readonly Request _request; + + public RequestArgs(string command, Request request) { + _command = command; + _request = request; + } + + public Request Request => _request; + public string Command => _command; + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Response.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Response.cs new file mode 100644 index 000000000..1a73e78a4 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Response.cs @@ -0,0 +1,22 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.PythonTools.Ipc.Json { + public class Response { + public Response() { + } + } +} \ No newline at end of file diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/UriJsonConverter.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/UriJsonConverter.cs new file mode 100644 index 000000000..bece70ae5 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/UriJsonConverter.cs @@ -0,0 +1,41 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using Newtonsoft.Json; + +namespace Microsoft.PythonTools.Ipc.Json { + public sealed class UriJsonConverter : JsonConverter { + public static readonly JsonConverter Instance = new UriJsonConverter(); + + public override bool CanConvert(Type objectType) { + return typeof(Uri).IsAssignableFrom(objectType); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) { + var uri = serializer.Deserialize(reader); + return string.IsNullOrEmpty(uri) ? null : new Uri(uri); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) { + if (value is Uri u) { + serializer.Serialize(writer, u.AbsoluteUri); + } else { + serializer.Serialize(writer, null); + } + } + } +} From 07783a3ef9be1de4f87e64b3655b10702704a820 Mon Sep 17 00:00:00 2001 From: Alexander Sher Date: Mon, 10 Dec 2018 22:30:35 -0600 Subject: [PATCH 03/10] CoreProduct buildable --- .../Interpreter/InterpreterConfiguration.cs | 41 +------ .../Impl/Interpreter/InterpreterUIMode.cs | 56 --------- .../PythonInterpreterFactoryExtensions.cs | 112 ------------------ src/Analysis/Engine/Impl/LocationInfo.cs | 2 +- .../Projects/AnalysisCompleteEventArgs.cs | 29 ----- .../Impl/Projects/IPythonProjectProvider.cs | 24 ---- .../Engine/Impl/Projects/ProjectAnalyzer.cs | 53 --------- .../Engine/Impl/Projects/PythonProject.cs | 79 ------------ .../Intellisense/AnalysisProtocol.cs | 2 - 9 files changed, 2 insertions(+), 396 deletions(-) delete mode 100644 src/Analysis/Engine/Impl/Interpreter/InterpreterUIMode.cs delete mode 100644 src/Analysis/Engine/Impl/Interpreter/PythonInterpreterFactoryExtensions.cs delete mode 100644 src/Analysis/Engine/Impl/Projects/AnalysisCompleteEventArgs.cs delete mode 100644 src/Analysis/Engine/Impl/Projects/IPythonProjectProvider.cs delete mode 100644 src/Analysis/Engine/Impl/Projects/ProjectAnalyzer.cs delete mode 100644 src/Analysis/Engine/Impl/Projects/PythonProject.cs diff --git a/src/Analysis/Engine/Impl/Interpreter/InterpreterConfiguration.cs b/src/Analysis/Engine/Impl/Interpreter/InterpreterConfiguration.cs index 4f6af1d68..a2542b99b 100644 --- a/src/Analysis/Engine/Impl/Interpreter/InterpreterConfiguration.cs +++ b/src/Analysis/Engine/Impl/Interpreter/InterpreterConfiguration.cs @@ -48,29 +48,6 @@ public InterpreterConfiguration( SitePackagesPath = sitePackagesPath ?? string.Empty; } - [Obsolete] - public InterpreterConfiguration( - string id, - string description, - string prefixPath = null, - string path = null, - string winPath = "", - string pathVar = "", - InterpreterArchitecture arch = default(InterpreterArchitecture), - Version version = null, - InterpreterUIMode uiMode = InterpreterUIMode.Normal - ) { - Id = id; - _description = description ?? ""; - PrefixPath = prefixPath; - InterpreterPath = path; - WindowsInterpreterPath = string.IsNullOrEmpty(winPath) ? path : winPath; - PathEnvironmentVariable = pathVar; - Architecture = arch ?? InterpreterArchitecture.Unknown; - Version = version ?? new Version(); - UIMode = uiMode; - } - private static string Read(Dictionary d, string k) => d.TryGetValue(k, out var o) ? o as string : null; @@ -155,22 +132,12 @@ public void SwitchToFullDescription() { } } - [Obsolete("Prefix path only applies to Windows.")] - public string PrefixPath { get; } - /// /// Returns the path to the interpreter executable for launching Python /// applications. /// public string InterpreterPath { get; } - - /// - /// Returns the path to the interpreter executable for launching Python - /// applications which are windows applications (pythonw.exe, ipyw.exe). - /// - [Obsolete("Python Language Server is platform-agnostic and does not use Windows-specific settings.")] - public string WindowsInterpreterPath { get; } - + /// /// Gets the environment variable which should be used to set sys.path. /// @@ -198,12 +165,6 @@ public void SwitchToFullDescription() { /// public string SitePackagesPath { get; } - /// - /// The UI behavior of the interpreter. - /// - [Obsolete("Language Server does not support UI features related to the interpreter.")] - public InterpreterUIMode UIMode { get; } - /// /// The fixed search paths of the interpreter. /// diff --git a/src/Analysis/Engine/Impl/Interpreter/InterpreterUIMode.cs b/src/Analysis/Engine/Impl/Interpreter/InterpreterUIMode.cs deleted file mode 100644 index ab7aba8dd..000000000 --- a/src/Analysis/Engine/Impl/Interpreter/InterpreterUIMode.cs +++ /dev/null @@ -1,56 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; - -namespace Microsoft.PythonTools.Interpreter { - /// - /// Specifies the interpreter's behavior in the UI. - /// - [Flags] - [Obsolete("Language Server does not support UI features related to the interpreter.")] - public enum InterpreterUIMode : int { - /// - /// Interpreter can be set or selected as the default, and is visible to - /// the user. - /// - Normal = 0x00, - - /// - /// Interpreter is not displayed in the user interface, but can still be - /// added to a project if the ID is known. - /// - Hidden = 0x01, - - /// - /// Interpreter cannot be selected as the default. Implies - /// . - /// - CannotBeDefault = 0x02, - - /// - /// Interpreter cannot be automatically selected as the default. - /// - CannotBeAutoDefault = 0x04, - - /// - /// Interpreter has no user-modifiable settings. - /// - CannotBeConfigured = 0x08, - - SupportsDatabase = 0x10, - } -} diff --git a/src/Analysis/Engine/Impl/Interpreter/PythonInterpreterFactoryExtensions.cs b/src/Analysis/Engine/Impl/Interpreter/PythonInterpreterFactoryExtensions.cs deleted file mode 100644 index b41fd914f..000000000 --- a/src/Analysis/Engine/Impl/Interpreter/PythonInterpreterFactoryExtensions.cs +++ /dev/null @@ -1,112 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.IO; - -namespace Microsoft.PythonTools.Interpreter { - public static class PythonInterpreterFactoryExtensions { - /// - /// Determines whether two interpreter factories are equivalent. - /// - public static bool IsEqual(this IPythonInterpreterFactory x, IPythonInterpreterFactory y) { - if (x == null || y == null) { - return x == null && y == null; - } - if (x.GetType() != y.GetType()) { - return false; - } - - return x.Configuration.Equals(y.Configuration); - } - - /// - /// Returns true if the factory can be run. This checks whether the - /// configured InterpreterPath value is an actual file. - /// - public static bool IsRunnable(this IPythonInterpreterFactory factory) { - return factory != null && factory.Configuration.IsRunnable(); - } - - /// - /// Returns true if the configuration can be run. This checks whether - /// the configured InterpreterPath value is an actual file. - /// - public static bool IsRunnable(this InterpreterConfiguration config) { - return config != null && - !InterpreterRegistryConstants.IsNoInterpretersFactory(config.Id) && - File.Exists(config.InterpreterPath); - } - - /// - /// Checks whether the factory can be run and throws the appropriate - /// exception if it cannot. - /// - /// - /// factory is null and parameterName is provided. - /// - /// - /// factory is null and parameterName is not provided, or the factory - /// has no configuration. - /// - /// - /// factory is the sentinel used when no environments are installed. - /// - /// - /// factory's InterpreterPath does not exist on disk. - /// - public static void ThrowIfNotRunnable(this IPythonInterpreterFactory factory, string parameterName = null) { - if (factory == null) { - if (string.IsNullOrEmpty(parameterName)) { - throw new NullReferenceException(); - } else { - throw new ArgumentNullException(parameterName); - } - } - factory.Configuration.ThrowIfNotRunnable(); - } - - /// - /// Checks whether the configuration can be run and throws the - /// appropriate exception if it cannot. - /// - /// - /// config is null and parameterName is provided. - /// - /// - /// config is null and parameterName is not provided. - /// - /// - /// config is the sentinel used when no environments are installed. - /// - /// - /// config's InterpreterPath does not exist on disk. - /// - public static void ThrowIfNotRunnable(this InterpreterConfiguration config, string parameterName = null) { - if (config == null) { - if (string.IsNullOrEmpty(parameterName)) { - throw new NullReferenceException(); - } else { - throw new ArgumentNullException(parameterName); - } - } else if (InterpreterRegistryConstants.IsNoInterpretersFactory(config.Id)) { - throw new NoInterpretersException(); - } else if (!File.Exists(config.InterpreterPath)) { - throw new FileNotFoundException(config.InterpreterPath ?? "(null)"); - } - } - } -} diff --git a/src/Analysis/Engine/Impl/LocationInfo.cs b/src/Analysis/Engine/Impl/LocationInfo.cs index c34854ea6..52c25ecad 100644 --- a/src/Analysis/Engine/Impl/LocationInfo.cs +++ b/src/Analysis/Engine/Impl/LocationInfo.cs @@ -18,7 +18,7 @@ using System.Collections.Generic; namespace Microsoft.PythonTools.Analysis { - internal class LocationInfo : ILocationInfo, IEquatable { + public class LocationInfo : ILocationInfo, IEquatable { internal static readonly LocationInfo[] Empty = new LocationInfo[0]; public LocationInfo(string path, Uri documentUri, int line, int column) : diff --git a/src/Analysis/Engine/Impl/Projects/AnalysisCompleteEventArgs.cs b/src/Analysis/Engine/Impl/Projects/AnalysisCompleteEventArgs.cs deleted file mode 100644 index a93c6343f..000000000 --- a/src/Analysis/Engine/Impl/Projects/AnalysisCompleteEventArgs.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; - -namespace Microsoft.PythonTools.Projects { - public sealed class AnalysisCompleteEventArgs : EventArgs { - private readonly string _path; - - public string Path => _path; - - public AnalysisCompleteEventArgs(string path) { - _path = path; - } - } -} diff --git a/src/Analysis/Engine/Impl/Projects/IPythonProjectProvider.cs b/src/Analysis/Engine/Impl/Projects/IPythonProjectProvider.cs deleted file mode 100644 index 0a3145b17..000000000 --- a/src/Analysis/Engine/Impl/Projects/IPythonProjectProvider.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -namespace Microsoft.PythonTools.Projects { - /// - /// Provides access to an abstract Python project. - /// - public interface IPythonProjectProvider { - PythonProject Project { get; } - } -} diff --git a/src/Analysis/Engine/Impl/Projects/ProjectAnalyzer.cs b/src/Analysis/Engine/Impl/Projects/ProjectAnalyzer.cs deleted file mode 100644 index 126bfa37c..000000000 --- a/src/Analysis/Engine/Impl/Projects/ProjectAnalyzer.cs +++ /dev/null @@ -1,53 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Generic; -using System.Threading.Tasks; - -namespace Microsoft.PythonTools.Projects { - public abstract class ProjectAnalyzer { - /// - /// Registers the extension type with the analyzer. The extension must have a - /// public default constructor, as it will be recreated in the out-of-process - /// analyzer. - /// - public abstract Task RegisterExtensionAsync(Type extensionType); - - /// - /// Sends a command to an analysis extension with the specified input and returns - /// the result. - /// - /// The name of the analysis extension, as attributed with - /// AnalysisExtensionNameAttribute. - /// The command that the extension supports and will execute. - /// The input to the command. - /// - public abstract Task SendExtensionCommandAsync(string extensionName, string commandId, string body); - - /// - /// Raised when the analysis is complete for the specified file. - /// - public abstract event EventHandler AnalysisComplete; - - /// - /// Gets the list of files which are being analyzed by this ProjectAnalyzer. - /// - public abstract IEnumerable Files { - get; - } - } -} diff --git a/src/Analysis/Engine/Impl/Projects/PythonProject.cs b/src/Analysis/Engine/Impl/Projects/PythonProject.cs deleted file mode 100644 index 519ddf769..000000000 --- a/src/Analysis/Engine/Impl/Projects/PythonProject.cs +++ /dev/null @@ -1,79 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Threading.Tasks; -using Microsoft.PythonTools.Interpreter; - -namespace Microsoft.PythonTools.Projects { - /// - /// Provides information about a Python project. This is an abstract base class that - /// different project systems can implement. Tools which want to plug in an extend the - /// Python analysis system can work with the PythonProject to get information about - /// the project. - /// - /// This differs from the ProjectAnalyzer class in that it contains more rich information - /// about the configuration of the project related to running and executing. - /// - public abstract class PythonProject { - /// - /// Gets a property for the project. Users can get/set their own properties, also these properties - /// are available: - /// - /// CommandLineArguments -> arguments to be passed to the debugged program. - /// InterpreterPath -> gets a configured directory where the interpreter should be launched from. - /// IsWindowsApplication -> determines whether or not the application is a windows application (for which no console window should be created) - /// - /// - /// - public abstract string GetProperty(string name); - - public abstract string GetUnevaluatedProperty(string name); - - /// - /// Sets a property for the project. See GetProperty for more information on common properties. - /// - /// - /// - public abstract void SetProperty(string name, string value); - - public abstract IPythonInterpreterFactory GetInterpreterFactory(); - - /// - /// Gets the current analyzer for the project, or null if no analyzer is available. - /// - [Obsolete("Use the async version if possible")] - public abstract ProjectAnalyzer Analyzer { get; } - - /// - /// Gets the current analyzer for the project. May wait while creating an analyzer - /// if necessary, where the property would return null. - /// - public abstract Task GetAnalyzerAsync(); - - public abstract event EventHandler ProjectAnalyzerChanged; - - public abstract string ProjectHome { get; } - - /// - /// Attempts to retrieve a PythonProject from the provided object, which - /// should implement . - /// - public static PythonProject FromObject(object source) { - return (source as IPythonProjectProvider)?.Project; - } - } -} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AnalysisProtocol.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AnalysisProtocol.cs index ac2ba6415..ba549ef64 100644 --- a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AnalysisProtocol.cs +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AnalysisProtocol.cs @@ -833,8 +833,6 @@ public sealed class AnalysisOptions { public string[] typeStubPaths; } - - public class AnalysisReference { public string kind; // definition, reference, value public string expr; From 23b3f75573ebe8d0dbc8c4d0ec54e33f8cbd9675 Mon Sep 17 00:00:00 2001 From: Alexander Sher Date: Mon, 17 Dec 2018 23:31:41 -0600 Subject: [PATCH 04/10] Product buildable except IronPython --- .../Definitions/IDotNetPythonInterpreter.cs | 26 ------------------- .../Intellisense/OutOfProcProjectAnalyzer.cs | 2 +- 2 files changed, 1 insertion(+), 27 deletions(-) delete mode 100644 src/Analysis/Engine/Impl/Interpreter/Definitions/IDotNetPythonInterpreter.cs diff --git a/src/Analysis/Engine/Impl/Interpreter/Definitions/IDotNetPythonInterpreter.cs b/src/Analysis/Engine/Impl/Interpreter/Definitions/IDotNetPythonInterpreter.cs deleted file mode 100644 index 6ccd37072..000000000 --- a/src/Analysis/Engine/Impl/Interpreter/Definitions/IDotNetPythonInterpreter.cs +++ /dev/null @@ -1,26 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; - -namespace Microsoft.PythonTools.Interpreter { - public interface IDotNetPythonInterpreter { - /// - /// Gets the IPythonType object for the specifed .NET type; - /// - IPythonType GetBuiltinType(Type type); - } -} diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutOfProcProjectAnalyzer.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutOfProcProjectAnalyzer.cs index 57f0f4792..a3da6977d 100644 --- a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutOfProcProjectAnalyzer.cs +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutOfProcProjectAnalyzer.cs @@ -1539,7 +1539,7 @@ private async Task UpdateContent(AP.FileUpdateRequest request) { int part = _server.GetPart(request.documentUri); return new AP.FileUpdateResponse { version = version, - newCode = (entry as IDocument)?.ReadDocument(part, out _)?.ReadToEnd() + newCode = entry.Document?.ReadDocument(part, out _)?.ReadToEnd() }; #else return new AP.FileUpdateResponse { From 44615ee50f3b460c65e77e9f4b60cf91aacf0f66 Mon Sep 17 00:00:00 2001 From: Alexander Sher Date: Wed, 19 Dec 2018 14:17:45 -0600 Subject: [PATCH 05/10] Move VS-specific file --- .../Interpreter}/IPythonInterpreterWithProjectReferences.cs | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{Analysis/Engine/Impl/Interpreter/Definitions => PTVS/Microsoft.PythonTools.Analyzer/Interpreter}/IPythonInterpreterWithProjectReferences.cs (100%) diff --git a/src/Analysis/Engine/Impl/Interpreter/Definitions/IPythonInterpreterWithProjectReferences.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Interpreter/IPythonInterpreterWithProjectReferences.cs similarity index 100% rename from src/Analysis/Engine/Impl/Interpreter/Definitions/IPythonInterpreterWithProjectReferences.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Interpreter/IPythonInterpreterWithProjectReferences.cs From 954a0ea591708df18706f45ed7ba05922b1890a9 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Thu, 6 Dec 2018 15:25:05 -0800 Subject: [PATCH 06/10] add hack to prevent True/False/None/... from being use-before-def (#464) --- .../Engine/Impl/Analyzer/ExpressionEvaluator.cs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/Analysis/Engine/Impl/Analyzer/ExpressionEvaluator.cs b/src/Analysis/Engine/Impl/Analyzer/ExpressionEvaluator.cs index c45691a34..78508cc01 100644 --- a/src/Analysis/Engine/Impl/Analyzer/ExpressionEvaluator.cs +++ b/src/Analysis/Engine/Impl/Analyzer/ExpressionEvaluator.cs @@ -149,8 +149,19 @@ public IAnalysisSet LookupAnalysisSetByName(Node node, string name, bool addRef refs = createIn.CreateVariable(node, _unit, name, addRef); res = refs.Types; } else { - // ... warn the user - warn = true; + switch (name) { + // "atom" in Python grammar. + case "True": + case "False": + case "None": + case "...": + Debug.Fail($"Known good name '{name}' not found in scope"); + break; + default: + // ... warn the user + warn = true; + break; + } } } } From d0ba35ee899684a8e8697d376829762178653bd6 Mon Sep 17 00:00:00 2001 From: Alexander Sher Date: Fri, 7 Dec 2018 14:13:29 -0800 Subject: [PATCH 07/10] Fix for extra paths inside workspace (#467) Fix #281: Support "go to definition" for namespace packages Fix #466: Fix "go to definition" and resolving imports The fix is to put user search paths in front of workspace directory so that modules inside extra paths can be used as roots for packages --- .../PathResolverSnapshot.cs | 17 ++++-- src/Analysis/Engine/Impl/PythonAnalyzer.cs | 4 +- src/Analysis/Engine/Test/ImportTests.cs | 61 +++++++++++++++++++ src/Analysis/Engine/Test/ServerExtensions.cs | 8 +++ 4 files changed, 83 insertions(+), 7 deletions(-) diff --git a/src/Analysis/Engine/Impl/DependencyResolution/PathResolverSnapshot.cs b/src/Analysis/Engine/Impl/DependencyResolution/PathResolverSnapshot.cs index f3bebb454..c1f69c132 100644 --- a/src/Analysis/Engine/Impl/DependencyResolution/PathResolverSnapshot.cs +++ b/src/Analysis/Engine/Impl/DependencyResolution/PathResolverSnapshot.cs @@ -359,11 +359,11 @@ private void CreateRootsWithDefault(string rootDirectory, string[] userSearchPat .ToArray(); var filteredInterpreterSearchPaths = interpreterSearchPaths.Select(FixPath) - .Except(filteredUserSearchPaths.Prepend(rootDirectory)) + .Except(filteredUserSearchPaths.Append(rootDirectory)) .ToArray(); userRootsCount = filteredUserSearchPaths.Length + 1; - nodes = AddRootsFromSearchPaths(ImmutableArray.Empty.Add(GetOrCreateRoot(rootDirectory)), filteredUserSearchPaths, filteredInterpreterSearchPaths); + nodes = AddRootsFromSearchPaths(rootDirectory, filteredUserSearchPaths, filteredInterpreterSearchPaths); string FixPath(string p) => Path.IsPathRooted(p) ? PathUtils.NormalizePath(p) : PathUtils.NormalizePath(Path.Combine(rootDirectory, p)); } @@ -381,11 +381,18 @@ private void CreateRootsWithoutDefault(string[] userSearchPaths, string[] interp .ToArray(); userRootsCount = filteredUserSearchPaths.Length; - nodes = AddRootsFromSearchPaths(ImmutableArray.Empty, filteredUserSearchPaths, filteredInterpreterSearchPaths); + nodes = AddRootsFromSearchPaths(filteredUserSearchPaths, filteredInterpreterSearchPaths); } - private ImmutableArray AddRootsFromSearchPaths(ImmutableArray roots, string[] userSearchPaths, string[] interpreterSearchPaths) { - return roots + private ImmutableArray AddRootsFromSearchPaths(string rootDirectory, string[] userSearchPaths, string[] interpreterSearchPaths) { + return ImmutableArray.Empty + .AddRange(userSearchPaths.Select(GetOrCreateRoot).ToArray()) + .Add(GetOrCreateRoot(rootDirectory)) + .AddRange(interpreterSearchPaths.Select(GetOrCreateRoot).ToArray()); + } + + private ImmutableArray AddRootsFromSearchPaths(string[] userSearchPaths, string[] interpreterSearchPaths) { + return ImmutableArray.Empty .AddRange(userSearchPaths.Select(GetOrCreateRoot).ToArray()) .AddRange(interpreterSearchPaths.Select(GetOrCreateRoot).ToArray()); } diff --git a/src/Analysis/Engine/Impl/PythonAnalyzer.cs b/src/Analysis/Engine/Impl/PythonAnalyzer.cs index 270d13b93..98fb34a7a 100644 --- a/src/Analysis/Engine/Impl/PythonAnalyzer.cs +++ b/src/Analysis/Engine/Impl/PythonAnalyzer.cs @@ -129,8 +129,8 @@ private async Task LoadKnownTypesAsync(CancellationToken token) { } private void ReloadModulePaths(in IEnumerable rootPaths) { - foreach (var modulePath in rootPaths.Where(Directory.Exists).SelectMany(p => ModulePath.GetModulesInPath(p))) { - _pathResolver.TryAddModulePath(modulePath.SourceFile, out _); + foreach (var modulePath in rootPaths.Where(Directory.Exists).SelectMany(p => PathUtils.EnumerateFiles(p))) { + _pathResolver.TryAddModulePath(modulePath, out _); } } diff --git a/src/Analysis/Engine/Test/ImportTests.cs b/src/Analysis/Engine/Test/ImportTests.cs index a3dc62a6e..4f1ad0acd 100644 --- a/src/Analysis/Engine/Test/ImportTests.cs +++ b/src/Analysis/Engine/Test/ImportTests.cs @@ -98,6 +98,67 @@ import package.sub_package.module2 completionModule2.Should().HaveLabels("Y").And.NotContainLabels("X"); } + [ServerTestMethod(LatestAvailable3X = true, TestSpecificRootUri = true), Priority(0)] + public async Task Completions_ImportResolution_UserSearchPathsInsideWorkspace(Server server) { + var folder1 = TestData.GetTestSpecificPath("folder1"); + var folder2 = TestData.GetTestSpecificPath("folder2"); + var packageInFolder1 = Path.Combine(folder1, "package"); + var packageInFolder2 = Path.Combine(folder2, "package"); + var module1Path = Path.Combine(packageInFolder1, "module1.py"); + var module2Path = Path.Combine(packageInFolder2, "module2.py"); + var module1Content = @"class A(): + @staticmethod + def method1(): + pass"; + var module2Content = @"class B(): + @staticmethod + def method2(): + pass"; + var mainContent = @"from package import module1 as mod1, module2 as mod2 +mod1. +mod2. +mod1.A. +mod2.B."; + + server.Analyzer.SetSearchPaths(new[] { folder1, folder2 }); + + await server.OpenDocumentAndGetUriAsync(module1Path, module1Content); + await server.OpenDocumentAndGetUriAsync(module2Path, module2Content); + var uri = await server.OpenDocumentAndGetUriAsync("main.py", mainContent); + + await server.WaitForCompleteAnalysisAsync(CancellationToken.None); + + var completionMod1 = await server.SendCompletion(uri, 1, 5); + var completionMod2 = await server.SendCompletion(uri, 2, 5); + var completionA = await server.SendCompletion(uri, 3, 7); + var completionB = await server.SendCompletion(uri, 4, 7); + completionMod1.Should().HaveLabels("A").And.NotContainLabels("B"); + completionMod2.Should().HaveLabels("B").And.NotContainLabels("A"); + completionA.Should().HaveLabels("method1"); + completionB.Should().HaveLabels("method2"); + } + + [ServerTestMethod(LatestAvailable3X = true, TestSpecificRootUri = true), Priority(0)] + [Ignore("https://github.com/Microsoft/python-language-server/issues/468")] + public async Task Completions_ImportResolution_ModuleInWorkspaceAndInUserSearchPath(Server server) { + var extraSearchPath = TestData.GetTestSpecificPath(Path.Combine("some", "other")); + var module1Path = TestData.GetTestSpecificPath("module.py"); + var module2Path = Path.Combine(extraSearchPath, "module.py"); + var module1Content = "A = 1"; + var module2Content = "B = 2"; + var mainContent = @"import module as mod; mod."; + + server.Analyzer.SetSearchPaths(new[] { extraSearchPath }); + + await server.OpenDocumentAndGetUriAsync(module1Path, module1Content); + await server.OpenDocumentAndGetUriAsync(module2Path, module2Content); + var uri = await server.OpenDocumentAndGetUriAsync("main.py", mainContent); + + await server.WaitForCompleteAnalysisAsync(CancellationToken.None); + var completion = await server.SendCompletion(uri, 0, 26); + completion.Should().HaveLabels("A").And.NotContainLabels("B"); + } + [Ignore("https://github.com/Microsoft/python-language-server/issues/443")] [ServerTestMethod(LatestAvailable3X = true, TestSpecificRootUri = true), Priority(0)] public async Task Completions_ImportResolution_OneSearchPathInsideAnother(Server server) { diff --git a/src/Analysis/Engine/Test/ServerExtensions.cs b/src/Analysis/Engine/Test/ServerExtensions.cs index b6041b26a..52813934b 100644 --- a/src/Analysis/Engine/Test/ServerExtensions.cs +++ b/src/Analysis/Engine/Test/ServerExtensions.cs @@ -164,6 +164,14 @@ await server.DidOpenTextDocument(new DidOpenTextDocumentParams { }, GetCancellationToken()); } + public static async Task OpenDocumentAndGetAnalysisAsync(this Server server, string relativePath, string content, int failAfter = 30000, string languageId = null) { + var cancellationToken = GetCancellationToken(failAfter); + var uri = TestData.GetTestSpecificUri(relativePath); + await server.SendDidOpenTextDocument(uri, content, languageId); + cancellationToken.ThrowIfCancellationRequested(); + return await server.GetAnalysisAsync(uri, cancellationToken); + } + public static async Task OpenDefaultDocumentAndGetAnalysisAsync(this Server server, string content, int failAfter = 30000, string languageId = null) { var cancellationToken = GetCancellationToken(failAfter); await server.SendDidOpenTextDocument(TestData.GetDefaultModuleUri(), content, languageId); From 1a0f2ba4ee1816aa3741b19261fe93a864401d26 Mon Sep 17 00:00:00 2001 From: Alexander Sher Date: Fri, 21 Dec 2018 20:15:21 -0600 Subject: [PATCH 08/10] Last port --- .../Definitions/PythonMemberType.cs | 1 - src/Analysis/Engine/Impl/ModuleAnalysis.cs | 11 - src/Analysis/Engine/Test/AnalysisTest.cs | 95 +++-- src/LanguageServer/Impl/Program.cs | 1 - src/PLS.sln | 4 +- .../Analyzer.csproj | 125 ------ .../{ => Impl}/AnalyzerScrapers.pyproj | 0 .../{ => Impl}/AnalyzerScrapers.sln | 0 .../{ => Impl}/App.config | 0 .../{ => Impl}/AssemblyInfo.cs | 3 +- .../{ => Impl}/BuiltinScraper.py | 0 .../{ => Impl}/BuiltinScraperTests.py | 0 .../{ => Impl}/ExtensionScraper.py | 0 .../Intellisense/AnalysisLimitsConverter.cs | 0 .../Intellisense/AnalysisProtocol.cs | 0 .../Intellisense/AssignmentWalker.cs | 0 .../Intellisense/ClassifierWalker.cs | 0 .../Intellisense/MethodExtractor.cs | 0 .../Intellisense/OutOfProcMethodExtractor.cs | 0 .../Intellisense/OutOfProcProjectAnalyzer.cs | 2 +- .../Intellisense/OutliningWalker.cs | 0 .../Intellisense/ProximityExpressionWalker.cs | 0 .../{ => Impl}/Intellisense/TaggedSpan.cs | 0 .../Interpreter/IDotNetPythonInterpreter.cs | 0 ...IPythonInterpreterWithProjectReferences.cs | 0 .../Interpreter/ProjectAssemblyReference.cs | 0 .../{ => Impl}/IronPythonScraper.py | 0 .../Microsoft.PythonTools.Analyzer.csproj | 17 +- .../{ => Impl}/PyLibAnalyzer.cs | 0 .../{ => Impl}/PythonScraper.py | 0 .../{ => Impl}/VersionDiff.py | 0 .../{ => Impl}/Connection.cs | 0 .../{ => Impl}/DebugTextWriter.cs | 0 .../{ => Impl}/ErrorReceivedEventArgs.cs | 0 .../{ => Impl}/Event.cs | 0 .../{ => Impl}/EventReceivedEventArgs.cs | 0 .../{ => Impl}/FailedRequestException.cs | 0 .../Microsoft.PythonTools.Ipc.Json.csproj | 6 +- .../{ => Impl}/Properties/AssemblyInfo.cs | 0 .../{ => Impl}/ProtocolReader.cs | 0 .../{ => Impl}/Request.cs | 0 .../{ => Impl}/RequestArgs.cs | 0 .../{ => Impl}/Response.cs | 0 .../{ => Impl}/UriJsonConverter.cs | 0 .../Test/IpcJsonConnectionTests.cs | 396 ++++++++++++++++++ .../Test/IpcJsonPacketReadCSharpTests.cs | 122 ++++++ .../Test/IpcJsonPacketReadPythonTests.cs | 362 ++++++++++++++++ .../Test/IpcJsonTests.csproj | 85 ++++ .../Test/PacketProvider.cs | 162 +++++++ .../Test/Properties/AssemblyInfo.cs | 25 ++ 50 files changed, 1215 insertions(+), 202 deletions(-) delete mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/Analyzer.csproj rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/AnalyzerScrapers.pyproj (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/AnalyzerScrapers.sln (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/App.config (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/AssemblyInfo.cs (89%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/BuiltinScraper.py (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/BuiltinScraperTests.py (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/ExtensionScraper.py (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/Intellisense/AnalysisLimitsConverter.cs (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/Intellisense/AnalysisProtocol.cs (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/Intellisense/AssignmentWalker.cs (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/Intellisense/ClassifierWalker.cs (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/Intellisense/MethodExtractor.cs (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/Intellisense/OutOfProcMethodExtractor.cs (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/Intellisense/OutOfProcProjectAnalyzer.cs (99%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/Intellisense/OutliningWalker.cs (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/Intellisense/ProximityExpressionWalker.cs (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/Intellisense/TaggedSpan.cs (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/Interpreter/IDotNetPythonInterpreter.cs (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/Interpreter/IPythonInterpreterWithProjectReferences.cs (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/Interpreter/ProjectAssemblyReference.cs (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/IronPythonScraper.py (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/Microsoft.PythonTools.Analyzer.csproj (76%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/PyLibAnalyzer.cs (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/PythonScraper.py (100%) rename src/PTVS/Microsoft.PythonTools.Analyzer/{ => Impl}/VersionDiff.py (100%) rename src/PTVS/Microsoft.PythonTools.Ipc.Json/{ => Impl}/Connection.cs (100%) rename src/PTVS/Microsoft.PythonTools.Ipc.Json/{ => Impl}/DebugTextWriter.cs (100%) rename src/PTVS/Microsoft.PythonTools.Ipc.Json/{ => Impl}/ErrorReceivedEventArgs.cs (100%) rename src/PTVS/Microsoft.PythonTools.Ipc.Json/{ => Impl}/Event.cs (100%) rename src/PTVS/Microsoft.PythonTools.Ipc.Json/{ => Impl}/EventReceivedEventArgs.cs (100%) rename src/PTVS/Microsoft.PythonTools.Ipc.Json/{ => Impl}/FailedRequestException.cs (100%) rename src/PTVS/Microsoft.PythonTools.Ipc.Json/{ => Impl}/Microsoft.PythonTools.Ipc.Json.csproj (68%) rename src/PTVS/Microsoft.PythonTools.Ipc.Json/{ => Impl}/Properties/AssemblyInfo.cs (100%) rename src/PTVS/Microsoft.PythonTools.Ipc.Json/{ => Impl}/ProtocolReader.cs (100%) rename src/PTVS/Microsoft.PythonTools.Ipc.Json/{ => Impl}/Request.cs (100%) rename src/PTVS/Microsoft.PythonTools.Ipc.Json/{ => Impl}/RequestArgs.cs (100%) rename src/PTVS/Microsoft.PythonTools.Ipc.Json/{ => Impl}/Response.cs (100%) rename src/PTVS/Microsoft.PythonTools.Ipc.Json/{ => Impl}/UriJsonConverter.cs (100%) create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonConnectionTests.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonPacketReadCSharpTests.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonPacketReadPythonTests.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonTests.csproj create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/PacketProvider.cs create mode 100644 src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/Properties/AssemblyInfo.cs diff --git a/src/Analysis/Engine/Impl/Interpreter/Definitions/PythonMemberType.cs b/src/Analysis/Engine/Impl/Interpreter/Definitions/PythonMemberType.cs index 6b464037a..020ebfcc9 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Definitions/PythonMemberType.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Definitions/PythonMemberType.cs @@ -64,7 +64,6 @@ public enum PythonMemberType { /// An instance of a namespace object that was imported from .NET. /// Namespace, - /// /// A constant defined in source code. /// diff --git a/src/Analysis/Engine/Impl/ModuleAnalysis.cs b/src/Analysis/Engine/Impl/ModuleAnalysis.cs index 3c328585c..702ddb4eb 100644 --- a/src/Analysis/Engine/Impl/ModuleAnalysis.cs +++ b/src/Analysis/Engine/Impl/ModuleAnalysis.cs @@ -857,17 +857,6 @@ private IEnumerable GetKeywordMembers(GetMemberOptions options, I #endregion - /// - /// Gets the available names at the given location. This includes - /// global variables and locals, but not built-in variables. - /// - /// - /// The 0-based absolute index into the file where the available members - /// should be looked up. - /// - /// TODO: Remove; this is only used for tests - internal IEnumerable GetVariablesNoBuiltinsByIndex(int index) => GetVariablesNoBuiltins(_unit.Tree.IndexToLocation(index)); - /// /// Gets the available names at the given location. This includes /// global variables and locals, but not built-in variables. diff --git a/src/Analysis/Engine/Test/AnalysisTest.cs b/src/Analysis/Engine/Test/AnalysisTest.cs index 54d6917ed..b6f0275b2 100644 --- a/src/Analysis/Engine/Test/AnalysisTest.cs +++ b/src/Analysis/Engine/Test/AnalysisTest.cs @@ -29,6 +29,7 @@ using Microsoft.PythonTools.Analysis.Analyzer; using Microsoft.PythonTools.Analysis.FluentAssertions; using Microsoft.PythonTools.Analysis.Values; +using Microsoft.PythonTools.Intellisense; using Microsoft.PythonTools.Interpreter; using Microsoft.PythonTools.Interpreter.Ast; using Microsoft.PythonTools.Parsing; @@ -5576,6 +5577,52 @@ def f(s = 123) -> s: .And.HaveReturnValue().OfTypes(BuiltinTypeId.Int, BuiltinTypeId.NoneType, BuiltinTypeId.Unknown); } } + + + [TestMethod, Priority(0)] + public async Task SysModulesSetSpecialization() { + var code = @"import sys +modules = sys.modules + +modules['name_in_modules'] = None +"; + code += string.Join( + Environment.NewLine, + Enumerable.Range(0, 100).Select(i => $"sys.modules['name{i}'] = None") + ); + + using (var server = await CreateServerAsync(PythonVersions.LatestAvailable2X)) { + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); + analysis.Should().HaveVariable("sys").WithValue() + .And.HaveVariable("modules").WithValue(); + } + + } + + + [DataRow("import abc", 7, "abc", "")] + [DataRow("import abc", 8, "abc", "")] + [DataRow("import abc", 9, "abc", "")] + [DataRow("import abc", 10, "abc", "")] + [DataRow("import deg, abc as A",12, "abc", "")] + [DataRow("from abc import A", 6, "abc", "")] + [DataRow("from .deg import A", 9, "deg", "abc")] + [DataRow("from .hij import A", 9, "abc.hij", "abc.deg")] + [DataRow("from ..hij import A", 10, "hij", "abc.deg")] + [DataRow("from ..hij import A", 10, "abc.hij", "abc.deg.HIJ")] + [DataTestMethod, Priority(0)] + public async Task ModuleNameWalker(string code, int index, string expected, string baseCode) { + using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); + var entry = (IPythonProjectEntry)server.GetEntry(analysis.DocumentUri); + var walker = new ImportedModuleNameWalker(baseCode, string.Empty, index, null); + entry.Tree.Walk(walker); + walker.ImportedModules.Should().NotBeEmpty() + .And.NotContainNulls() + .And.Subject.First().Name.Should().Be(expected); + } + } + /* [TestMethod, Priority(0)] public async Task Super() { @@ -5842,32 +5889,6 @@ def fn(self): ); } - [TestMethod, Priority(0)] - public async Task SysModulesSetSpecialization() { - var code = @"import sys -modules = sys.modules - -modules['name_in_modules'] = None -"; - code += string.Join( - Environment.NewLine, - Enumerable.Range(0, 100).Select(i => string.Format("sys.modules['name{0}'] = None", i)) - ); - - var entry = ProcessTextV2(code); - - var sys = entry.GetValue("sys"); - - var modules = entry.GetValue("modules"); - Assert.IsInstanceOfType(modules, typeof(SysModuleInfo.SysModulesDictionaryInfo)); - - AssertUtil.ContainsExactly( - sys.Modules.Keys, - Enumerable.Range(0, 100).Select(i => string.Format("name{0}", i)) - .Concat(new[] { "name_in_modules" }) - ); - } - [TestMethod, Priority(0)] public async Task SysModulesGetSpecialization() { var code = @"import sys @@ -6030,28 +6051,6 @@ public void NullNamedArgument() { } } - [TestMethod, Priority(0)] - public void ModuleNameWalker() { - foreach (var item in new[] { - new { Code="import abc", Index=7, Expected="abc", Base="" }, - new { Code="import abc", Index=8, Expected="abc", Base="" }, - new { Code="import abc", Index=9, Expected="abc", Base="" }, - new { Code="import abc", Index=10, Expected="abc", Base="" }, - new { Code="import deg, abc as A", Index=12, Expected="abc", Base="" }, - new { Code="from abc import A", Index=6, Expected="abc", Base="" }, - new { Code="from .deg import A", Index=9, Expected="deg", Base="abc" }, - new { Code="from .hij import A", Index=9, Expected="abc.hij", Base="abc.deg" }, - new { Code="from ..hij import A", Index=10, Expected="hij", Base="abc.deg" }, - new { Code="from ..hij import A", Index=10, Expected="abc.hij", Base="abc.deg.HIJ" }, - }) { - var entry = ProcessTextV3(item.Code); - var walker = new ImportedModuleNameWalker(item.Base, string.Empty, item.Index, null); - entry.Modules[entry.DefaultModule].Tree.Walk(walker); - - Assert.AreEqual(item.Expected, walker.ImportedModules.FirstOrDefault()?.Name); - } - } - [TestMethod, Priority(0)] public void CrossModuleFunctionCallMemLeak() { var modA = @"from B import h diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 7b5213588..7485d60bf 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -29,7 +29,6 @@ namespace Microsoft.Python.LanguageServer.Server { internal static class Program { public static void Main(string[] args) { CheckDebugMode(); - Debugger.Launch(); using (CoreShell.Create()) { var services = CoreShell.Current.ServiceManager; diff --git a/src/PLS.sln b/src/PLS.sln index d0f54ef1c..499b9218e 100644 --- a/src/PLS.sln +++ b/src/PLS.sln @@ -15,9 +15,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Analysis.E EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Analysis.Engine.Tests", "Analysis\Engine\Test\Microsoft.Python.Analysis.Engine.Tests.csproj", "{1CFA416B-6932-432F-8C75-34B5615D7664}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.PythonTools.Analyzer", "PTVS\Microsoft.PythonTools.Analyzer\Microsoft.PythonTools.Analyzer.csproj", "{467679B2-D043-4C7D-855C-5A833B635EEE}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.PythonTools.Analyzer", "PTVS\Microsoft.PythonTools.Analyzer\Impl\Microsoft.PythonTools.Analyzer.csproj", "{467679B2-D043-4C7D-855C-5A833B635EEE}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.PythonTools.Ipc.Json", "PTVS\Microsoft.PythonTools.Ipc.Json\Microsoft.PythonTools.Ipc.Json.csproj", "{A5E04CC1-F600-45BA-B493-F7DCBA8C7E1A}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.PythonTools.Ipc.Json", "PTVS\Microsoft.PythonTools.Ipc.Json\Impl\Microsoft.PythonTools.Ipc.Json.csproj", "{A5E04CC1-F600-45BA-B493-F7DCBA8C7E1A}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.LanguageServer.Core", "LanguageServer\Core\Impl\Microsoft.Python.LanguageServer.Core.csproj", "{0EA1F1C3-2733-423C-BEC5-059BD77F0A3F}" EndProject diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Analyzer.csproj b/src/PTVS/Microsoft.PythonTools.Analyzer/Analyzer.csproj deleted file mode 100644 index 412fc0e83..000000000 --- a/src/PTVS/Microsoft.PythonTools.Analyzer/Analyzer.csproj +++ /dev/null @@ -1,125 +0,0 @@ - - - - - - 15.0 - - - - - 14.0 - - - - - 16.0 - - - - - 16.0 - - - - - - Debug - x86 - 8.0.30703 - 2.0 - {82b43b9b-a64c-4715-b499-d71e9ca2bd60};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} - {29A4FA1F-A562-4ED1-86FB-5850EF5DA92C} - Exe - Properties - Microsoft.PythonTools.Analysis - Microsoft.PythonTools.Analyzer - 512 - true - - - x86 - - - ..\Icons\Dev$(VSTarget)\PythonProject.ico - - - - $(PackagesPath)\Newtonsoft.Json\lib\net45\Newtonsoft.Json.dll - True - - - - - - - - - - - - - - - - DebugTimer.cs - - - - - - - - - - - - - - - - - - - {A85D479D-67A9-4BDB-904A-7D86DAF68A6F} - Microsoft.PythonTools.Analysis - - - {e1e1613d-0c96-42f9-9f2d-052c72533297} - Ipc.Json - - - - - PreserveNewest - True - - - PreserveNewest - True - - - PreserveNewest - True - - - PreserveNewest - True - - - - - - - - - - - - \ No newline at end of file diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/AnalyzerScrapers.pyproj b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/AnalyzerScrapers.pyproj similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/AnalyzerScrapers.pyproj rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/AnalyzerScrapers.pyproj diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/AnalyzerScrapers.sln b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/AnalyzerScrapers.sln similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/AnalyzerScrapers.sln rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/AnalyzerScrapers.sln diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/App.config b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/App.config similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/App.config rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/App.config diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/AssemblyInfo.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/AssemblyInfo.cs similarity index 89% rename from src/PTVS/Microsoft.PythonTools.Analyzer/AssemblyInfo.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/AssemblyInfo.cs index 9f4e620c0..2732f17e2 100644 --- a/src/PTVS/Microsoft.PythonTools.Analyzer/AssemblyInfo.cs +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/AssemblyInfo.cs @@ -17,7 +17,6 @@ using System.Reflection; using System.Runtime.InteropServices; -[assembly: AssemblyTitle("Visual Studio - Python background analyzer")] [assembly: AssemblyDescription("Performs analysis of the Python standard library and installed site packages.")] -[assembly: ComVisible(false)] \ No newline at end of file +[assembly: ComVisible(false)] diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/BuiltinScraper.py b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/BuiltinScraper.py similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/BuiltinScraper.py rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/BuiltinScraper.py diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/BuiltinScraperTests.py b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/BuiltinScraperTests.py similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/BuiltinScraperTests.py rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/BuiltinScraperTests.py diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/ExtensionScraper.py b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/ExtensionScraper.py similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/ExtensionScraper.py rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/ExtensionScraper.py diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AnalysisLimitsConverter.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AnalysisLimitsConverter.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AnalysisLimitsConverter.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AnalysisLimitsConverter.cs diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AnalysisProtocol.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AnalysisProtocol.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AnalysisProtocol.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AnalysisProtocol.cs diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AssignmentWalker.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AssignmentWalker.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/AssignmentWalker.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AssignmentWalker.cs diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/ClassifierWalker.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/ClassifierWalker.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/ClassifierWalker.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/ClassifierWalker.cs diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/MethodExtractor.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/MethodExtractor.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/MethodExtractor.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/MethodExtractor.cs diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutOfProcMethodExtractor.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutOfProcMethodExtractor.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutOfProcMethodExtractor.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutOfProcMethodExtractor.cs diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutOfProcProjectAnalyzer.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutOfProcProjectAnalyzer.cs similarity index 99% rename from src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutOfProcProjectAnalyzer.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutOfProcProjectAnalyzer.cs index a3da6977d..cd1b4836e 100644 --- a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutOfProcProjectAnalyzer.cs +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutOfProcProjectAnalyzer.cs @@ -1558,7 +1558,7 @@ private Response SetAnalysisOptions(AP.SetAnalysisOptionsRequest request) { Project.Limits = AnalysisLimitsConverter.FromDictionary(Options.analysisLimits); _server.ParseQueue.InconsistentIndentation = DiagnosticsErrorSink.GetSeverity(Options.indentationInconsistencySeverity); _server.ParseQueue.TaskCommentMap = Options.commentTokens; - _server.Analyzer.SetTypeStubPaths(Options.typeStubPaths); + _server.Analyzer.SetTypeStubPaths(Options.typeStubPaths ?? Enumerable.Empty()); return new Response(); } diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutliningWalker.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutliningWalker.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/OutliningWalker.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutliningWalker.cs diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/ProximityExpressionWalker.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/ProximityExpressionWalker.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/ProximityExpressionWalker.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/ProximityExpressionWalker.cs diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/TaggedSpan.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/TaggedSpan.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/Intellisense/TaggedSpan.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/TaggedSpan.cs diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Interpreter/IDotNetPythonInterpreter.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Interpreter/IDotNetPythonInterpreter.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/Interpreter/IDotNetPythonInterpreter.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Interpreter/IDotNetPythonInterpreter.cs diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Interpreter/IPythonInterpreterWithProjectReferences.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Interpreter/IPythonInterpreterWithProjectReferences.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/Interpreter/IPythonInterpreterWithProjectReferences.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Interpreter/IPythonInterpreterWithProjectReferences.cs diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Interpreter/ProjectAssemblyReference.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Interpreter/ProjectAssemblyReference.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/Interpreter/ProjectAssemblyReference.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Interpreter/ProjectAssemblyReference.cs diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/IronPythonScraper.py b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/IronPythonScraper.py similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/IronPythonScraper.py rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/IronPythonScraper.py diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Microsoft.PythonTools.Analyzer.csproj b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Microsoft.PythonTools.Analyzer.csproj similarity index 76% rename from src/PTVS/Microsoft.PythonTools.Analyzer/Microsoft.PythonTools.Analyzer.csproj rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Microsoft.PythonTools.Analyzer.csproj index 5ff442e24..fc0400806 100644 --- a/src/PTVS/Microsoft.PythonTools.Analyzer/Microsoft.PythonTools.Analyzer.csproj +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Microsoft.PythonTools.Analyzer.csproj @@ -2,9 +2,10 @@ net472 Microsoft.PythonTools.Analysis + Visual Studio - Python background analyzer Microsoft.PythonTools.Analyzer - + Exe @@ -12,10 +13,10 @@ true - ..\..\PLS.ruleset + ..\..\..\PLS.ruleset - ..\..\PLS.ruleset + ..\..\..\PLS.ruleset @@ -61,13 +62,13 @@ - - - + + + - + - + diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/PyLibAnalyzer.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/PyLibAnalyzer.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/PyLibAnalyzer.cs rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/PyLibAnalyzer.cs diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/PythonScraper.py b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/PythonScraper.py similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/PythonScraper.py rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/PythonScraper.py diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/VersionDiff.py b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/VersionDiff.py similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Analyzer/VersionDiff.py rename to src/PTVS/Microsoft.PythonTools.Analyzer/Impl/VersionDiff.py diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Connection.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Connection.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Ipc.Json/Connection.cs rename to src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Connection.cs diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/DebugTextWriter.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/DebugTextWriter.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Ipc.Json/DebugTextWriter.cs rename to src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/DebugTextWriter.cs diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/ErrorReceivedEventArgs.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/ErrorReceivedEventArgs.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Ipc.Json/ErrorReceivedEventArgs.cs rename to src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/ErrorReceivedEventArgs.cs diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Event.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Event.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Ipc.Json/Event.cs rename to src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Event.cs diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/EventReceivedEventArgs.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/EventReceivedEventArgs.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Ipc.Json/EventReceivedEventArgs.cs rename to src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/EventReceivedEventArgs.cs diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/FailedRequestException.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/FailedRequestException.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Ipc.Json/FailedRequestException.cs rename to src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/FailedRequestException.cs diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Microsoft.PythonTools.Ipc.Json.csproj b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Microsoft.PythonTools.Ipc.Json.csproj similarity index 68% rename from src/PTVS/Microsoft.PythonTools.Ipc.Json/Microsoft.PythonTools.Ipc.Json.csproj rename to src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Microsoft.PythonTools.Ipc.Json.csproj index 3c2837d70..08ac149b7 100644 --- a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Microsoft.PythonTools.Ipc.Json.csproj +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Microsoft.PythonTools.Ipc.Json.csproj @@ -5,11 +5,11 @@ Microsoft.PythonTools.Ipc.Json - + - + - + diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Properties/AssemblyInfo.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Properties/AssemblyInfo.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Ipc.Json/Properties/AssemblyInfo.cs rename to src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Properties/AssemblyInfo.cs diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/ProtocolReader.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/ProtocolReader.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Ipc.Json/ProtocolReader.cs rename to src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/ProtocolReader.cs diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Request.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Request.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Ipc.Json/Request.cs rename to src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Request.cs diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/RequestArgs.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/RequestArgs.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Ipc.Json/RequestArgs.cs rename to src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/RequestArgs.cs diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Response.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Response.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Ipc.Json/Response.cs rename to src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/Response.cs diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/UriJsonConverter.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/UriJsonConverter.cs similarity index 100% rename from src/PTVS/Microsoft.PythonTools.Ipc.Json/UriJsonConverter.cs rename to src/PTVS/Microsoft.PythonTools.Ipc.Json/Impl/UriJsonConverter.cs diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonConnectionTests.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonConnectionTests.cs new file mode 100644 index 000000000..32c4e534f --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonConnectionTests.cs @@ -0,0 +1,396 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.PythonTools.Infrastructure; +using Microsoft.PythonTools.Ipc.Json; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace IpcJsonTests { + [TestClass] + public class IpcJsonConnectionTests { + private Connection _client; + private Connection _server; + private readonly AutoResetEvent _connected = new AutoResetEvent(false); + + private static string PythonSocketSendEventPath => Path.Combine(TestData.GetPath("TestData"), "Ipc.Json", "socket_send_event.py"); + private static string PythonSocketHandleRequest => Path.Combine(TestData.GetPath("TestData"), "Ipc.Json", "socket_handle_request.py"); + + // Change this to true if you want to have an easier time debugging the python script + // You'll be expected to start it manually, which you can do by opening + // IpcJsonServers.sln, adjusting the debug script arguments to have the correct port number and pressing F5 + private static readonly bool StartPythonProcessManually = false; + + [ClassInitialize] + public static void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + [TestMethod, Priority(0)] + public async Task DotNetHandleRequest() { + InitConnection(async (requestArgs, done) => { + switch (requestArgs.Command) { + case TestDataProtocol.TestRequest.Command: { + var testRequest = requestArgs.Request as TestDataProtocol.TestRequest; + await done(new TestDataProtocol.TestResponse() { + requestText = testRequest.dataText, + responseText = "test response text", + }); + break; + } + default: + throw new InvalidOperationException(); + } + }); + + _client.EventReceived += (object sender, EventReceivedEventArgs e) => { + Assert.Fail(); + }; + + var response = await _client.SendRequestAsync(new TestDataProtocol.TestRequest() { + dataText = "request data text", + dataTextList = new string[] { "value 1", "value 2" }, + }, CancellationTokens.After5s); + + Assert.AreEqual("request data text", response.requestText); + Assert.AreEqual("test response text", response.responseText); + } + + [TestMethod, Priority(0)] + public async Task DotNetHandleRequestUnicode() { + InitConnection(async (requestArgs, done) => { + switch (requestArgs.Command) { + case TestDataProtocol.TestRequest.Command: { + var testRequest = requestArgs.Request as TestDataProtocol.TestRequest; + await done(new TestDataProtocol.TestResponse() { + requestText = testRequest.dataText, + responseText = "この文は、テストです。私はこれがうまく願っています。", + }); + break; + } + default: + throw new InvalidOperationException(); + } + }); + + _client.EventReceived += (object sender, EventReceivedEventArgs e) => { + Assert.Fail(); + }; + + var response = await _client.SendRequestAsync(new TestDataProtocol.TestRequest() { + dataText = "データテキストを要求する", + dataTextList = new string[] { "value 1", "value 2" }, + }, CancellationTokens.After5s); + + Assert.AreEqual("データテキストを要求する", response.requestText); + Assert.AreEqual("この文は、テストです。私はこれがうまく願っています。", response.responseText); + } + + [TestMethod, Priority(0)] + public async Task PythonHandleRequest() { + using (InitConnection(PythonSocketHandleRequest)) { + _client.EventReceived += (object sender, EventReceivedEventArgs e) => { + Assert.Fail(); + }; + + var response = await _client.SendRequestAsync(new TestDataProtocol.TestRequest() { + dataText = "request data text", + dataTextList = new string[] { "value 1", "value 2" }, + }, StartPythonProcessManually ? CancellationTokens.After60s : CancellationTokens.After5s); + + await _client.SendRequestAsync( + new TestDataProtocol.DisconnectRequest(), + StartPythonProcessManually ? CancellationTokens.After60s : CancellationTokens.After5s + ); + + Assert.AreEqual("request data text", response.requestText); + Assert.AreEqual("test response text", response.responseText); + } + } + + [TestMethod, Priority(0)] + public async Task PythonHandleRequestUnicode() { + using (InitConnection(PythonSocketHandleRequest)) { + _client.EventReceived += (object sender, EventReceivedEventArgs e) => { + Assert.Fail(); + }; + + var response = await _client.SendRequestAsync(new TestDataProtocol.TestRequest() { + dataText = "データテキストを要求する 请输入", + dataTextList = new string[] { "value 1", "value 2" }, + }, StartPythonProcessManually ? CancellationTokens.After60s : CancellationTokens.After5s); + + await _client.SendRequestAsync( + new TestDataProtocol.DisconnectRequest(), + StartPythonProcessManually ? CancellationTokens.After60s : CancellationTokens.After5s + ); + + Assert.AreEqual("データテキストを要求する 请输入", response.requestText); + Assert.AreEqual("test response text", response.responseText); + } + } + + [TestMethod, Priority(0)] + public async Task DotNetSendEvent() { + InitConnection((requestArgs, done) => { + Assert.Fail(); + return Task.CompletedTask; + }); + + var eventReceived = new AutoResetEvent(false); + var eventsReceived = new List(); + _client.EventReceived += (object sender, EventReceivedEventArgs e) => { + eventsReceived.Add(e); + if (eventsReceived.Count == 1) { + eventReceived.Set(); + } + }; + + await _server.SendEventAsync(new TestDataProtocol.TestEvent() { + dataText = "event data text" + }); + + eventReceived.WaitOne(2000); + + Assert.AreEqual(1, eventsReceived.Count); + Assert.AreEqual(TestDataProtocol.TestEvent.Name, eventsReceived[0].Name); + Assert.AreEqual("event data text", ((TestDataProtocol.TestEvent)eventsReceived[0].Event).dataText); + } + + [TestMethod, Priority(0)] + public async Task PythonSendEvent() { + using (InitConnection(PythonSocketSendEventPath)) { + var eventReceived = new AutoResetEvent(false); + var eventsReceived = new List(); + _client.EventReceived += (object sender, EventReceivedEventArgs e) => { + eventsReceived.Add(e); + if (eventsReceived.Count == 1) { + eventReceived.Set(); + } + }; + + eventReceived.WaitOne(2000); + + Assert.AreEqual(1, eventsReceived.Count); + Assert.AreEqual(TestDataProtocol.TestEvent.Name, eventsReceived[0].Name); + Assert.AreEqual("python event data text", ((TestDataProtocol.TestEvent)eventsReceived[0].Event).dataText); + Assert.AreEqual(76, ((TestDataProtocol.TestEvent)eventsReceived[0].Event).dataInt32); + } + } + + private void InitConnection(Func, Task> serverRequestHandler) { + // Client sends requests, receives responses and events + // Server receives requests, sends back responses and events + // Client creates the socket on an available port, + // passes the port number to the server which connects back to it. + var portNum = StartClient(); + + StartServer(portNum, serverRequestHandler); + + _connected.WaitOne(); + } + + private sealed class KillAndDisposeProcess : IDisposable { + public KillAndDisposeProcess(ProcessOutput process) { + Process = process; + } + + public void Dispose() { + Process.Kill(); + Process.Dispose(); + } + + public ProcessOutput Process { get; } + } + + private IDisposable InitConnection(string serverScriptPath) { + // Client sends requests, receives responses and events + // Server receives requests, sends back responses and events + // Client creates the socket on an available port, + // passes the port number to the server which connects back to it. + var portNum = StartClient(); + + if (!StartPythonProcessManually) { + Assert.IsTrue(File.Exists(serverScriptPath), "Python test data script '{0}' was not found.".FormatUI(serverScriptPath)); + + var workingDir = Path.GetDirectoryName(serverScriptPath); + + var searchPaths = new HashSet(); + searchPaths.Add(PtvsdSearchPath); + searchPaths.Add(workingDir); + + var env = new List>(); + env.Add(new KeyValuePair("PYTHONPATH", string.Join(";", searchPaths))); + + var arguments = new List(); + arguments.Add(serverScriptPath); + arguments.Add("-r"); + arguments.Add(portNum.ToString()); + var proc = ProcessOutput.Run( + (PythonPaths.Python27 ?? PythonPaths.Python27_x64).InterpreterPath, + arguments, + workingDir, + env, + false, + null + ); + try { + if (proc.ExitCode.HasValue) { + // Process has already exited + proc.Wait(); + if (proc.StandardErrorLines.Any()) { + Assert.Fail(String.Join(Environment.NewLine, proc.StandardErrorLines)); + } + return null; + } else { + _connected.WaitOne(); + var p = proc; + proc = null; + return new KillAndDisposeProcess(p); + } + } finally { + if (proc != null) { + proc.Dispose(); + } + } + } else { + // Check the port number variable assigned above if you want to + // start the python process manually + Debugger.Break(); + return null; + } + } + + internal static string PtvsdSearchPath { + get { + return Path.GetDirectoryName(Path.GetDirectoryName(PythonToolsInstallPath.GetFile("ptvsd\\__init__.py", typeof(Connection).Assembly))); + } + } + + private void StartServer(int portNum, Func, Task> requestHandler) { + IPEndPoint endPoint = new IPEndPoint(IPAddress.Loopback, portNum); + + var socket = new Socket(endPoint.AddressFamily, SocketType.Stream, ProtocolType.IP); + socket.Connect(endPoint); + + var stream = new NetworkStream(socket, ownsSocket: true); + + _server = new Connection( + stream, + true, + stream, + true, + requestHandler, + TestDataProtocol.RegisteredTypes + ); + Task.Run(_server.ProcessMessages).DoNotWait(); + } + + private int StartClient() { + new SocketPermission(NetworkAccess.Accept, TransportType.All, "", SocketPermission.AllPorts).Demand(); + + // Use an available port + IPEndPoint endPoint = new IPEndPoint(IPAddress.Loopback, 0); + + var socket = new Socket(endPoint.AddressFamily, SocketType.Stream, ProtocolType.IP); + socket.Bind(endPoint); + socket.Listen(10); + + // Find out which port is being used + var portNum = ((IPEndPoint)socket.LocalEndPoint).Port; + + Task.Run(() => { socket.BeginAccept(StartClientAcceptCallback, socket); }); + return portNum; + } + + private void StartClientAcceptCallback(IAsyncResult ar) { + var socket = ((Socket)ar.AsyncState).EndAccept(ar); + var stream = new NetworkStream(socket, ownsSocket: true); + _client = new Connection(stream, true, stream, true, null, TestDataProtocol.RegisteredTypes); + Task.Run(_client.ProcessMessages).DoNotWait(); + _connected.Set(); + } + } + + static class TestDataProtocol { + public static readonly Dictionary RegisteredTypes = CollectCommands(); + + private static Dictionary CollectCommands() { + Dictionary all = new Dictionary(); + foreach (var type in typeof(TestDataProtocol).GetNestedTypes()) { + if (type.IsSubclassOf(typeof(Request))) { + var command = type.GetField("Command"); + if (command != null) { + all["request." + (string)command.GetRawConstantValue()] = type; + } + } else if (type.IsSubclassOf(typeof(Event))) { + var name = type.GetField("Name"); + if (name != null) { + all["event." + (string)name.GetRawConstantValue()] = type; + } + } + } + return all; + } + +#pragma warning disable 0649 // Field 'field' is never assigned to, and will always have its default value 'value' + + public sealed class TestRequest : Request { + public const string Command = "testRequest"; + + public override string command => Command; + + public string dataText; + public string[] dataTextList; + + } + + public sealed class TestResponse : Response { + public string requestText; + public string responseText; + public string[] responseTextList; + } + + public sealed class DisconnectRequest : GenericRequest { + public const string Command = "disconnect"; + + public override string command => Command; + } + + public class TestEvent : Event { + public const string Name = "testEvent"; + public string dataText; + public int dataInt32; + public long dataInt64; + public float dataFloat32; + public double dataFloat64; + + public override string name => Name; + } + +#pragma warning restore 0649 + + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonPacketReadCSharpTests.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonPacketReadCSharpTests.cs new file mode 100644 index 000000000..319c321f5 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonPacketReadCSharpTests.cs @@ -0,0 +1,122 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.IO; +using System.Threading.Tasks; +using Microsoft.PythonTools.Ipc.Json; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace IpcJsonTests { + [TestClass] + public class IpcJsonPacketReadCSharpTests { + [TestMethod, Priority(0)] + public async Task ValidPackets() { + await TestValidPacketAsync(PacketProvider.GetValidPacket1()); + await TestValidPacketAsync(PacketProvider.GetValidPacket2()); + } + + [TestMethod, Priority(0)] + public async Task ValidUnicodePackets() { + await TestValidPacketAsync(PacketProvider.GetValidUnicodePacket1()); + await TestValidPacketAsync(PacketProvider.GetValidUnicodePacket2()); + } + + [TestMethod, Priority(0)] + public async Task TruncatedJson() { + foreach (var packet in PacketProvider.GetTruncatedJsonPackets()) { + await TestInvalidPacketAsync(packet); + } + } + + [TestMethod, Priority(0)] + public async Task IncorrectContentLengthUnderread() { + foreach (var packet in PacketProvider.GetIncorrectContentLengthUnderreadPackets()) { + await TestInvalidPacketAsync(packet); + } + } + + [TestMethod, Priority(0)] + public async Task IncorrectContentLengthOverread() { + foreach (var packet in PacketProvider.GetIncorrectContentLengthOverreadPackets()) { + await TestInvalidPacketAsync(packet); + } + } + + [TestMethod, Priority(0)] + public async Task IncorrectContentLengthOverreadEndOfStream() { + foreach (var packet in PacketProvider.GetIncorrectContentLengthOverreadEndOfStreamPackets()) { + await TestInvalidPacketAsync(packet); + } + } + + [TestMethod, Priority(0)] + public async Task InvalidContentLengthType() { + await TestInvalidPacketAsync(PacketProvider.GetInvalidContentLengthIntegerTooLargePacket()); + await TestInvalidPacketAsync(PacketProvider.GetInvalidContentLengthNegativeIntegerPacket()); + await TestInvalidPacketAsync(PacketProvider.GetInvalidContentLengthNotIntegerPacket()); + } + + [TestMethod, Priority(0)] + public async Task MissingContentLength() { + await TestInvalidPacketAsync(PacketProvider.GetMissingContentLengthPacket()); + } + + [TestMethod, Priority(0)] + public async Task MalformedHeader() { + await TestInvalidPacketAsync(PacketProvider.GetMalformedHeaderPacket()); + } + + [TestMethod, Priority(0)] + public async Task AdditionalHeaders() { + await TestValidPacketAsync(PacketProvider.GetAdditionalHeadersPacket()); + } + + [TestMethod, Priority(0)] + public async Task EmptyStream() { + await TestNoPacketAsync(PacketProvider.GetNoPacket()); + } + + [TestMethod, Priority(0)] + public async Task UnterminatedHeader() { + await TestNoPacketAsync(PacketProvider.GetUnterminatedPacket()); + await TestNoPacketAsync(PacketProvider.GetIncorrectlyTerminatedPacket()); + } + + private static async Task TestValidPacketAsync(Packet packet) { + Assert.IsFalse(packet.BadHeaders || packet.BadContent); + var reader = new ProtocolReader(packet.AsStream()); + var obj = await Connection.ReadPacketAsJObject(reader); + Assert.IsNotNull(obj); + } + + private static async Task TestNoPacketAsync(Packet packet) { + var reader = new ProtocolReader(packet.AsStream()); + var obj = await Connection.ReadPacketAsJObject(reader); + Assert.IsNull(obj); + } + + private static async Task TestInvalidPacketAsync(Packet packet) { + Assert.IsTrue(packet.BadHeaders || packet.BadContent); + var reader = new ProtocolReader(packet.AsStream()); + try { + var obj = await Connection.ReadPacketAsJObject(reader); + Assert.IsNotNull(obj); + Assert.Fail("Failed to raise InvalidDataException."); + } catch (InvalidDataException) { + } + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonPacketReadPythonTests.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonPacketReadPythonTests.cs new file mode 100644 index 000000000..bdc06a4c3 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonPacketReadPythonTests.cs @@ -0,0 +1,362 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.PythonTools.Infrastructure; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; +using TestUtilities.Python; + +namespace IpcJsonTests { + [TestClass] + public class IpcJsonPacketReadPythonTests { + private Stream _clientStream; + private readonly AutoResetEvent _connected = new AutoResetEvent(false); + + private static string PythonParsingTestPath => Path.Combine(TestData.GetPath("TestData"), "Ipc.Json", "parsing_test.py"); + + [ClassInitialize] + public static void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + [TestInitialize] + public void TestInit() { + Version.AssertInstalled(); + } + + internal virtual PythonVersion Version { + get { + return PythonPaths.Python26 ?? PythonPaths.Python26_x64; + } + } + + [TestMethod, Priority(0)] + public async Task ValidPackets() { + await TestValidPacketAsync(PacketProvider.GetValidPacket1()); + await TestValidPacketAsync(PacketProvider.GetValidPacket2()); + } + + [TestMethod, Priority(0)] + public async Task ValidUnicodePackets() { + await TestValidPacketAsync(PacketProvider.GetValidUnicodePacket1()); + await TestValidPacketAsync(PacketProvider.GetValidUnicodePacket2()); + } + + [TestMethod, Priority(0)] + public async Task TruncatedJson() { + foreach (var packet in PacketProvider.GetTruncatedJsonPackets()) { + await TestInvalidPacketAsync(packet); + } + } + + [TestMethod, Priority(0)] + public async Task IncorrectContentLengthUnderread() { + foreach (var packet in PacketProvider.GetIncorrectContentLengthUnderreadPackets()) { + await TestInvalidPacketAsync(packet); + } + } + + [TestMethod, Priority(0)] + public async Task IncorrectContentLengthOverread() { + foreach (var packet in PacketProvider.GetIncorrectContentLengthOverreadPackets()) { + await TestInvalidPacketAsync(packet); + } + } + + [TestMethod, Priority(0)] + public async Task IncorrectContentLengthOverreadEndOfStream() { + foreach (var packet in PacketProvider.GetIncorrectContentLengthOverreadEndOfStreamPackets()) { + await TestInvalidPacketAsync(packet); + } + } + + [TestMethod, Priority(0)] + public async Task InvalidContentLengthType() { + await TestInvalidPacketAsync(PacketProvider.GetInvalidContentLengthIntegerTooLargePacket()); + await TestInvalidPacketAsync(PacketProvider.GetInvalidContentLengthNegativeIntegerPacket()); + await TestInvalidPacketAsync(PacketProvider.GetInvalidContentLengthNotIntegerPacket()); + } + + [TestMethod, Priority(0)] + public async Task MissingContentLength() { + await TestInvalidPacketAsync(PacketProvider.GetMissingContentLengthPacket()); + } + + [TestMethod, Priority(0)] + public async Task MalformedHeader() { + await TestInvalidPacketAsync(PacketProvider.GetMalformedHeaderPacket()); + } + + [TestMethod, Priority(0)] + public async Task AdditionalHeaders() { + await TestValidPacketAsync(PacketProvider.GetAdditionalHeadersPacket()); + } + + [TestMethod, Priority(0)] + public async Task EmptyStream() { + await TestNoPacketAsync(PacketProvider.GetNoPacket()); + } + + [TestMethod, Priority(0)] + public async Task UnterminatedHeader() { + await TestNoPacketAsync(PacketProvider.GetUnterminatedPacket()); + await TestNoPacketAsync(PacketProvider.GetIncorrectlyTerminatedPacket()); + } + + private Task TestValidPacketAsync(Packet packet) { + Assert.IsFalse(packet.BadHeaders || packet.BadContent); + return TestPacketAsync(packet); + } + + private Task TestInvalidPacketAsync(Packet packet) { + Assert.IsTrue(packet.BadHeaders || packet.BadContent); + return TestPacketAsync(packet, + packet.BadHeaders ? "ptvsd.ipcjson.InvalidHeaderError" : "ptvsd.ipcjson.InvalidContentError", + closeStream: packet.ReadPastEndOfStream + ); + } + + private Task TestNoPacketAsync(Packet packet) { + string expectedError = null; + if (packet.BadHeaders) { + expectedError = "ptvsd.ipcjson.InvalidHeaderError"; + } else if (packet.BadContent) { + expectedError = "ptvsd.ipcjson.InvalidContentError"; + } + return TestPacketAsync(packet, expectedError, closeStream: true); + } + + private async Task TestPacketAsync(Packet packet, string expectedError = null, bool closeStream = false) { + using (var proc = InitConnection(PythonParsingTestPath)) { + var bytes = packet.AsBytes(); + await _clientStream.WriteAsync(bytes, 0, bytes.Length); + if (closeStream) { + // We expect the process to be busy reading headers and not exit + var closed = proc.Wait(TimeSpan.FromMilliseconds(500)); + Assert.IsFalse(closed); + + // Close the stream so it gets unblocked + _clientStream.Close(); + } + + if (!proc.Wait(TimeSpan.FromSeconds(2))) { + proc.Kill(); + Assert.Fail("Python process did not exit"); + } + + CheckProcessResult(proc, expectedError); + } + } + + private ProcessOutput InitConnection(string serverScriptPath) { + var portNum = StartClient(); + + Assert.IsTrue(File.Exists(serverScriptPath), "Python test data script '{0}' was not found.".FormatUI(serverScriptPath)); + + var workingDir = Path.GetDirectoryName(serverScriptPath); + + var searchPaths = new HashSet(); + searchPaths.Add(IpcJsonConnectionTests.PtvsdSearchPath); + searchPaths.Add(workingDir); + + var env = new List>(); + env.Add(new KeyValuePair(Version.Configuration.PathEnvironmentVariable, string.Join(";", searchPaths))); + + var arguments = new List(); + arguments.Add(serverScriptPath); + arguments.Add("-r"); + arguments.Add(portNum.ToString()); + var proc = ProcessOutput.Run( + Version.InterpreterPath, + arguments, + workingDir, + env, + false, + null + ); + + if (proc.ExitCode.HasValue) { + // Process has already exited + proc.Wait(); + CheckProcessResult(proc); + } + + _connected.WaitOne(10000); + + return proc; + } + + private void CheckProcessResult(ProcessOutput proc, string expectedError = null) { + Console.WriteLine(String.Join(Environment.NewLine, proc.StandardOutputLines)); + Debug.WriteLine(String.Join(Environment.NewLine, proc.StandardErrorLines)); + if (!string.IsNullOrEmpty(expectedError)) { + var matches = proc.StandardErrorLines.Where(err => err.Contains(expectedError)); + if (matches.Count() == 0) { + Assert.Fail(String.Join(Environment.NewLine, proc.StandardErrorLines)); + } + } else { + if (proc.StandardErrorLines.Any()) { + Assert.Fail(String.Join(Environment.NewLine, proc.StandardErrorLines)); + } + Assert.AreEqual(0, proc.ExitCode); + } + } + + private int StartClient() { + new SocketPermission(NetworkAccess.Accept, TransportType.All, "", SocketPermission.AllPorts).Demand(); + + // Use an available port + IPEndPoint endPoint = new IPEndPoint(IPAddress.Loopback, 0); + + var socket = new Socket(endPoint.AddressFamily, SocketType.Stream, ProtocolType.IP); + socket.Bind(endPoint); + socket.Listen(10); + + // Find out which port is being used + var portNum = ((IPEndPoint)socket.LocalEndPoint).Port; + + Task.Run(() => { socket.BeginAccept(StartClientAcceptCallback, socket); }); + return portNum; + } + + private void StartClientAcceptCallback(IAsyncResult ar) { + var socket = ((Socket)ar.AsyncState).EndAccept(ar); + _clientStream = new NetworkStream(socket, ownsSocket: true); + _connected.Set(); + } + } + + [TestClass] + public class IpcJsonPacketReadPythonTestsIpy : IpcJsonPacketReadPythonTests { + [ClassInitialize] + public static new void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + internal override PythonVersion Version { + get { + return PythonPaths.IronPython27 ?? PythonPaths.IronPython27_x64; + } + } + } + + [TestClass] + public class IpcJsonPacketReadPythonTests27 : IpcJsonPacketReadPythonTests { + [ClassInitialize] + public static new void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + internal override PythonVersion Version { + get { + return PythonPaths.Python27 ?? PythonPaths.Python27_x64; + } + } + } + + [TestClass] + public class IpcJsonPacketReadPythonTests31 : IpcJsonPacketReadPythonTests { + [ClassInitialize] + public static new void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + internal override PythonVersion Version { + get { + return PythonPaths.Python31 ?? PythonPaths.Python31_x64; + } + } + } + + [TestClass] + public class IpcJsonPacketReadPythonTests32 : IpcJsonPacketReadPythonTests { + [ClassInitialize] + public static new void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + internal override PythonVersion Version { + get { + return PythonPaths.Python32 ?? PythonPaths.Python32_x64; + } + } + } + + [TestClass] + public class IpcJsonPacketReadPythonTests33 : IpcJsonPacketReadPythonTests { + [ClassInitialize] + public static new void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + internal override PythonVersion Version { + get { + return PythonPaths.Python33 ?? PythonPaths.Python33_x64; + } + } + } + + [TestClass] + public class IpcJsonPacketReadPythonTests34 : IpcJsonPacketReadPythonTests { + [ClassInitialize] + public static new void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + internal override PythonVersion Version { + get { + return PythonPaths.Python34 ?? PythonPaths.Python34_x64; + } + } + } + + [TestClass] + public class IpcJsonPacketReadPythonTests35 : IpcJsonPacketReadPythonTests { + [ClassInitialize] + public static new void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + internal override PythonVersion Version { + get { + return PythonPaths.Python35 ?? PythonPaths.Python35_x64; + } + } + } + + [TestClass] + public class IpcJsonPacketReadPythonTests36 : IpcJsonPacketReadPythonTests { + [ClassInitialize] + public static new void DoDeployment(TestContext context) { + AssertListener.Initialize(); + } + + internal override PythonVersion Version { + get { + return PythonPaths.Python36 ?? PythonPaths.Python36_x64; + } + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonTests.csproj b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonTests.csproj new file mode 100644 index 000000000..f5801cdd4 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/IpcJsonTests.csproj @@ -0,0 +1,85 @@ + + + + + + 15.0 + + + + + 14.0 + + + + + 16.0 + + + + + 16.0 + + + + + + Debug + AnyCPU + + + 2.0 + {BE9F11BF-EDD4-4138-A8AD-C8126BF2C77F} + Library + Properties + IpcJsonTests + IpcJsonTests + 512 + {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + + + AnyCPU + + + + + + + 3.5 + + + + $(PackagesPath)\Newtonsoft.Json\lib\net45\Newtonsoft.Json.dll + True + + + + + + + + + + + + {D092D54E-FF29-4D32-9AEE-4EF704C92F67} + TestUtilities + + + {e1e1613d-0c96-42f9-9f2d-052c72533297} + Ipc.Json + + + {a731c4c3-3741-4080-a946-c47574c1f3bf} + TestUtilities.Python.Analysis + + + {b3db0521-d9e3-4f48-9e2e-e5ecae886049} + Common + + + + + + + \ No newline at end of file diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/PacketProvider.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/PacketProvider.cs new file mode 100644 index 000000000..90ddb56f5 --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/PacketProvider.cs @@ -0,0 +1,162 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace IpcJsonTests { + class PacketProvider { + private static string validJson1 = @"{'request_seq':1,'success':true,'command':'initialize','message':null,'body':{'failedLoads':[],'error':null},'type':'response','seq':2}".Replace('\'', '"'); + private static string validJson2 = @"{'event':'childFileAnalyzed','body':{'filename':'C:\\Projects\\RemoteDebugApp\\RemoteDebugApp.py','fileId':1,'isTemporaryFile':false,'suppressErrorList':false},'type':'event','seq':5}".Replace('\'', '"'); + private static string validUtf8Json1 = @"{'event':'childFileAnalyzed','body':{'filename':'C:\\Projects\\RemoteDebugApp\\この文は、テストです。私はこれがうまく願っています。.py','fileId':1,'isTemporaryFile':false,'suppressErrorList':false},'type':'event','seq':5}".Replace('\'', '"'); + private static string validUtf8Json2 = @"{'event':'childFileAnalyzed','body':{'filename':'C:\\Projects\\RemoteDebugApp\\é.py','fileId':1,'isTemporaryFile':false,'suppressErrorList':false},'type':'event','seq':5}".Replace('\'', '"'); + + public static Packet GetValidPacket1() => MakePacketFromJson(validJson1); + + public static Packet GetValidPacket2() => MakePacketFromJson(validJson2); + + public static Packet GetValidUnicodePacket1() => MakePacketFromJson(validUtf8Json1); + + public static Packet GetValidUnicodePacket2() => MakePacketFromJson(validUtf8Json2); + + public static Packet GetNoPacket() => MakePacket(new byte[0], new byte[0]); + + public static Packet GetUnterminatedPacket() => MakePacket(Encoding.ASCII.GetBytes("NoTerminator"), new byte[0], badHeaders: true); + + public static Packet GetInvalidContentLengthIntegerTooLargePacket() { + return MakePacket(Encoding.ASCII.GetBytes("Content-Length: 2147483649\r\n\r\n"), MakeBody(validJson1), badHeaders: true); + } + + public static Packet GetInvalidContentLengthNegativeIntegerPacket() { + return MakePacket(Encoding.ASCII.GetBytes("Content-Length: -1\r\n\r\n"), MakeBody(validJson1), badHeaders: true); + } + + public static Packet GetInvalidContentLengthNotIntegerPacket() { + return MakePacket(Encoding.ASCII.GetBytes("Content-Length: BAD\r\n\r\n"), MakeBody(validJson1), badHeaders: true); + } + + public static Packet GetMissingContentLengthPacket() { + return MakePacket(Encoding.ASCII.GetBytes("From: Test\r\n\r\n"), MakeBody(validJson1), badHeaders: true); + } + + public static Packet GetMalformedHeaderPacket() { + return MakePacket(Encoding.ASCII.GetBytes("Content-Length\r\n\r\n"), MakeBody(validJson1), badHeaders: true); + } + + public static Packet GetAdditionalHeadersPacket() { + // Other headers are fine, we only care that Content-Length is there and valid + var body = MakeBody(validJson1); + return MakePacket(Encoding.ASCII.GetBytes(string.Format("From: Test\r\nContent-Length:{0}\r\nTo: You\r\n\r\n", body.Length)), body); + } + + public static Packet GetIncorrectlyTerminatedPacket() { + var body = MakeBody(validJson1); + return MakePacket(Encoding.ASCII.GetBytes(string.Format("Content-Length:{0}\n\n", body.Length)), body, badHeaders: true); + } + + public static IEnumerable GetTruncatedJsonPackets() { + // Valid packet, but the json is invalid because it's truncated + for (int i = 1; i < validJson1.Length; i += 3) { + yield return MakePacketFromJson(validJson1.Substring(0, validJson1.Length - i), badContent: true); + } + } + + public static IEnumerable GetIncorrectContentLengthUnderreadPackets() { + // Full json is in the stream, but the header was corrupted and the + // Content-Length value is SMALLER than it should be, so the packet body + // will miss parts of the json at the end. + for (int i = 1; i < validJson1.Length; i += 3) { + var json = MakeBody(validJson1); + var headers = MakeHeaders(i); + yield return MakePacket(headers, json, badContent: true); + } + } + + public static IEnumerable GetIncorrectContentLengthOverreadPackets() { + // Full json is in the stream, but the header was corrupted and the + // Content-Length value is LARGER than it should be, so the packet body + // will include junk at the end from the next message. + var endJunk = MakeHeaders(5); + for (int i = 1; i < endJunk.Length; i++) { + var json = MakeBody(validJson1); + var headers = MakeHeaders(json.Length + i); + yield return MakePacket(headers, json, endJunk, badContent: true); + } + } + + public static IEnumerable GetIncorrectContentLengthOverreadEndOfStreamPackets() { + // Full json is in the stream, but the header was corrupted and the + // Content-Length value is LARGER than it should be, and there's no + // more data in the stream after this. + for (int i = 1; i < 5; i++) { + var json = MakeBody(validJson1); + var headers = MakeHeaders(json.Length + i); + yield return MakePacket(headers, json, badContent: true, blocked: true); + } + } + + private static Packet MakePacketFromJson(string json, bool badContent = false) { + var encoded = MakeBody(json); + var headers = MakeHeaders(encoded.Length); + + return MakePacket(headers, encoded, badContent: badContent); + } + + private static Packet MakePacket(byte[] headers, byte[] encoded, byte[] endJunk = null, bool badHeaders = false, bool badContent = false, bool blocked = false) { + return new Packet(headers, encoded, endJunk, badHeaders, badContent, blocked); + } + + private static byte[] MakeBody(string json) { + return Encoding.UTF8.GetBytes(json); + } + + private static byte[] MakeHeaders(int contentLength) { + return Encoding.ASCII.GetBytes(string.Format("Content-Length: {0}\r\n\r\n", contentLength)); + } + } + + class Packet { + private List _data = new List(); + public bool BadHeaders { get; } + public bool BadContent { get; } + public bool ReadPastEndOfStream { get; } + + public Packet(byte[] headers, byte[] content, byte[] endJunk = null, bool badHeaders = false, bool badContent = false, bool readPastEndOfStream = false) { + _data.AddRange(headers); + _data.AddRange(content); + if (endJunk != null) { + _data.AddRange(endJunk); + } + BadHeaders = badHeaders; + BadContent = badContent; + ReadPastEndOfStream = readPastEndOfStream; + } + + public Stream AsStream() { + var stream = new MemoryStream(); + var data = AsBytes(); + stream.Write(data, 0, data.Length); + stream.Flush(); + stream.Seek(0, SeekOrigin.Begin); + return stream; + } + + public byte[] AsBytes() { + return _data.ToArray(); + } + } +} diff --git a/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/Properties/AssemblyInfo.cs b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..c29e2e86b --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Ipc.Json/Test/Properties/AssemblyInfo.cs @@ -0,0 +1,25 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("IpcJsonTests")] +[assembly: AssemblyDescription("")] + +[assembly: ComVisible(false)] +[assembly: Guid("5D889F27-FE98-43AC-AE9F-509B47B2484D")] From 0696b81504cdebef02eb6508abe4f005869e4a5f Mon Sep 17 00:00:00 2001 From: Alexander Sher Date: Tue, 8 Jan 2019 15:37:36 -0600 Subject: [PATCH 09/10] - Add nuspec - Fix #501: PTVS-LS Integration: Fix LS hanging during file changes - Fix #502: PTVS-LS Integration: Add required *.py files to the vsix --- .../IGroupableAnalysisProjectEntry.cs | 4 +- .../Engine/Impl/Intellisense/AnalysisQueue.cs | 4 +- src/Analysis/Engine/Impl/ProjectEntry.cs | 129 ++++++++++++------ src/Analysis/Engine/Impl/PythonAnalyzer.cs | 12 +- src/Analysis/Engine/Test/ThreadingTest.cs | 6 +- .../Core/Impl/Implementation/Server.cs | 2 +- .../Microsoft.PythonTools.Analyzer.csproj | 9 +- .../Impl/python-language-server-ptvs.nuspec | 34 +++++ 8 files changed, 142 insertions(+), 58 deletions(-) create mode 100644 src/PTVS/Microsoft.PythonTools.Analyzer/Impl/python-language-server-ptvs.nuspec diff --git a/src/Analysis/Engine/Impl/Definitions/IGroupableAnalysisProjectEntry.cs b/src/Analysis/Engine/Impl/Definitions/IGroupableAnalysisProjectEntry.cs index f0012554c..a8e39eb24 100644 --- a/src/Analysis/Engine/Impl/Definitions/IGroupableAnalysisProjectEntry.cs +++ b/src/Analysis/Engine/Impl/Definitions/IGroupableAnalysisProjectEntry.cs @@ -22,14 +22,14 @@ namespace Microsoft.PythonTools.Analysis { /// more efficient analysis. /// /// To analyze the full group you call Analyze(true) on all the items in the same group (determined - /// by looking at the identity of the AnalysGroup object). Then you call AnalyzeQueuedEntries on the + /// by looking at the identity of the AnalysisGroup object). Then you call AnalyzeQueuedEntries on the /// group. /// public interface IGroupableAnalysisProjectEntry { /// /// Analyzes this project entry optionally just adding it to the queue shared by the project. /// - void Analyze(CancellationToken cancel, bool enqueueOnly); + void PreAnalyze(); IGroupableAnalysisProject AnalysisGroup { get; } } diff --git a/src/Analysis/Engine/Impl/Intellisense/AnalysisQueue.cs b/src/Analysis/Engine/Impl/Intellisense/AnalysisQueue.cs index 143e66ed8..8cc4d0a54 100644 --- a/src/Analysis/Engine/Impl/Intellisense/AnalysisQueue.cs +++ b/src/Analysis/Engine/Impl/Intellisense/AnalysisQueue.cs @@ -137,6 +137,8 @@ public void Enqueue(IAnalyzable item, AnalysisPriority priority) { } private async Task HandleAnalyzable(IAnalyzable item, AnalysisPriority priority, CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); + if (item is IGroupableAnalysisProjectEntry groupable) { var added = _enqueuedGroups.Add(groupable.AnalysisGroup); if (added) { @@ -147,7 +149,7 @@ private async Task HandleAnalyzable(IAnalyzable item, AnalysisPriority priority, } } - groupable.Analyze(cancellationToken, true); + groupable.PreAnalyze(); } else { item.Analyze(cancellationToken); } diff --git a/src/Analysis/Engine/Impl/ProjectEntry.cs b/src/Analysis/Engine/Impl/ProjectEntry.cs index 755726ed5..ec16a777c 100644 --- a/src/Analysis/Engine/Impl/ProjectEntry.cs +++ b/src/Analysis/Engine/Impl/ProjectEntry.cs @@ -21,6 +21,7 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -45,11 +46,10 @@ internal sealed class ProjectEntry : IPythonProjectEntry, IAggregateableProjectE private readonly ConcurrentQueue> _backReferences = new ConcurrentQueue>(); private readonly HashSet _aggregates = new HashSet(); - private TaskCompletionSource _analysisTcs = new TaskCompletionSource(); + private AnalysisCompletionToken _analysisCompletionToken; private AnalysisUnit _unit; private readonly ManualResetEventSlim _pendingParse = new ManualResetEventSlim(true); private long _expectedParseVersion; - private long _expectedAnalysisVersion; internal ProjectEntry( PythonAnalyzer state, @@ -66,6 +66,7 @@ IAnalysisCookie cookie MyScope = new ModuleInfo(ModuleName, this, state.Interpreter.CreateModuleContext()); _unit = new AnalysisUnit(null, MyScope.Scope); + _analysisCompletionToken = AnalysisCompletionToken.Default; _buffers = new SortedDictionary { [0] = new DocumentBuffer() }; if (Cookie is InitialContentCookie c) { @@ -137,10 +138,10 @@ public IPythonParse GetCurrentParse() { } } - internal Task GetAnalysisAsync(int waitingTimeout = -1, CancellationToken cancellationToken = default(CancellationToken)) { + internal Task GetAnalysisAsync(int waitingTimeout = -1, CancellationToken cancellationToken = default) { Task task; lock (this) { - task = _analysisTcs.Task; + task = _analysisCompletionToken.Task; } if (task.IsCompleted || waitingTimeout == -1 && !cancellationToken.CanBeCanceled) { @@ -158,22 +159,15 @@ public IPythonParse GetCurrentParse() { internal void SetCompleteAnalysis() { lock (this) { - if (_expectedAnalysisVersion != Analysis.Version) { - return; - } - _analysisTcs.TrySetResult(Analysis); + _analysisCompletionToken.TrySetAnalysis(Analysis); } RaiseNewAnalysis(); } - internal void ResetCompleteAnalysis() { - TaskCompletionSource analysisTcs = null; + internal void NewAnalysisAwaitableOnParse() { lock (this) { - _expectedAnalysisVersion = AnalysisVersion + 1; - analysisTcs = _analysisTcs; - _analysisTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _analysisCompletionToken = _analysisCompletionToken.NewParse(); } - analysisTcs?.TrySetCanceled(); } public void SetCurrentParse(PythonAst tree, IAnalysisCookie cookie, bool notify = true) { @@ -195,38 +189,58 @@ public void SetCurrentParse(PythonAst tree, IAnalysisCookie cookie, bool notify internal bool IsVisible(ProjectEntry assigningScope) => true; - public void Analyze(CancellationToken cancel) => Analyze(cancel, false); - - public void Analyze(CancellationToken cancel, bool enqueueOnly) { + public void Analyze(CancellationToken cancel) { if (cancel.IsCancellationRequested) { return; } lock (this) { - AnalysisVersion++; + PrepareForAnalysis(); - foreach (var aggregate in _aggregates) { - aggregate?.BumpVersion(); - } + ProjectState.AnalyzeQueuedEntries(cancel); - Parse(enqueueOnly, cancel); + // publish the analysis now that it's complete/running + Analysis = new ModuleAnalysis( + _unit, + ((ModuleScope)_unit.Scope).CloneForPublish(), + DocumentUri, + AnalysisVersion + ); } - if (!enqueueOnly) { - RaiseNewAnalysis(); + RaiseNewAnalysis(); + } + + public void PreAnalyze() { + lock (this) { + PrepareForAnalysis(); + + // publish the analysis now that it's complete/running + Analysis = new ModuleAnalysis( + _unit, + ((ModuleScope)_unit.Scope).CloneForPublish(), + DocumentUri, + AnalysisVersion + ); } } private void RaiseNewAnalysis() => NewAnalysis?.Invoke(this, EventArgs.Empty); - public int AnalysisVersion { get; private set; } + public int AnalysisVersion => _analysisCompletionToken.Version; public bool IsAnalyzed => Analysis != null; - private void Parse(bool enqueueOnly, CancellationToken cancel) { + private void PrepareForAnalysis() { #if DEBUG Debug.Assert(Monitor.IsEntered(this)); #endif + _analysisCompletionToken = _analysisCompletionToken.NewAnalysis(); + + foreach (var aggregate in _aggregates) { + aggregate?.BumpVersion(); + } + var parse = GetCurrentParse(); var tree = parse?.Tree; var cookie = parse?.Cookie; @@ -306,18 +320,6 @@ where lastDot > 0 } _unit.Enqueue(); - - if (!enqueueOnly) { - ProjectState.AnalyzeQueuedEntries(cancel); - } - - // publish the analysis now that it's complete/running - Analysis = new ModuleAnalysis( - _unit, - ((ModuleScope)_unit.Scope).CloneForPublish(), - DocumentUri, - AnalysisVersion - ); } public IGroupableAnalysisProject AnalysisGroup => ProjectState; @@ -348,7 +350,7 @@ where lastDot > 0 public void Dispose() { lock (this) { - AnalysisVersion = -1; + _analysisCompletionToken = AnalysisCompletionToken.Disposed; var state = ProjectState; foreach (var aggregatedInto in _aggregates) { @@ -494,6 +496,55 @@ public void ResetDocument(int version, string content) { public void AddBackReference(ReferenceDict referenceDict) { _backReferences.Enqueue(new WeakReference(referenceDict)); } + + private struct AnalysisCompletionToken { + public static readonly AnalysisCompletionToken Default; + public static readonly AnalysisCompletionToken Disposed; + + static AnalysisCompletionToken() { + var cancelledTcs = CreateTcs(); + cancelledTcs.SetCanceled(); + + Default = new AnalysisCompletionToken(CreateTcs(), -1, false); + Disposed = new AnalysisCompletionToken(cancelledTcs, -1, true); + } + + private readonly bool _isParse; + private readonly TaskCompletionSource _tcs; + + public int Version { get; } + public Task Task => _tcs.Task; + + public AnalysisCompletionToken NewParse() { + if (_isParse) { + return this; + } + + var tcs = CreateTcs(); + _tcs.TrySetCanceled(); + return new AnalysisCompletionToken(tcs, Version + 1, true); + } + + public AnalysisCompletionToken NewAnalysis() { + var tcs = _tcs.Task.IsCompleted ? CreateTcs() : _tcs; + return new AnalysisCompletionToken(tcs, _isParse ? Version : Version + 1, false); + } + + private AnalysisCompletionToken(TaskCompletionSource tcs, int version, bool isParse) { + _tcs = tcs; + Version = version; + _isParse = isParse; + } + + public void TrySetAnalysis(IModuleAnalysis analysis) { + if (Version == analysis.Version) { + _tcs.TrySetResult(analysis); + } + } + + private static TaskCompletionSource CreateTcs() + => new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } } class InitialContentCookie : IAnalysisCookie { diff --git a/src/Analysis/Engine/Impl/PythonAnalyzer.cs b/src/Analysis/Engine/Impl/PythonAnalyzer.cs index 98fb34a7a..982fde416 100644 --- a/src/Analysis/Engine/Impl/PythonAnalyzer.cs +++ b/src/Analysis/Engine/Impl/PythonAnalyzer.cs @@ -201,9 +201,7 @@ public IPythonProjectEntry AddModule(string moduleName, string filePath, Uri doc } return entry; } - - public void RemoveModule(IProjectEntry entry) => RemoveModule(entry, null); - + /// /// Removes the specified project entry from the current analysis. /// @@ -213,9 +211,8 @@ public IPythonProjectEntry AddModule(string moduleName, string filePath, Uri doc /// Action to perform on each module that /// had imported the one being removed. public void RemoveModule(IProjectEntry entry, Action onImporter) { - if (entry == null) { - throw new ArgumentNullException(nameof(entry)); - } + Check.ArgumentNotNull(nameof(entry), entry); + Check.ArgumentNotNull(nameof(onImporter), onImporter); Contract.EndContractBlock(); var pyEntry = entry as IPythonProjectEntry; @@ -237,9 +234,6 @@ public void RemoveModule(IProjectEntry entry, Action onImpo entry.Dispose(); ClearDiagnostics(entry); - if (onImporter == null) { - onImporter = e => e.Analyze(CancellationToken.None, enqueueOnly: true); - } if (!string.IsNullOrEmpty(pyEntry?.ModuleName)) { Modules.TryRemove(pyEntry.ModuleName, out _); diff --git a/src/Analysis/Engine/Test/ThreadingTest.cs b/src/Analysis/Engine/Test/ThreadingTest.cs index 5069dddea..23b954368 100644 --- a/src/Analysis/Engine/Test/ThreadingTest.cs +++ b/src/Analysis/Engine/Test/ThreadingTest.cs @@ -131,11 +131,11 @@ class MyClass: // One analysis before we start foreach (var e in entries) { - e.Analyze(cancel, true); + e.PreAnalyze(); } state.AnalyzeQueuedEntries(cancel); - // Repeatedly re-analyse the code + // Repeatedly re-analyze the code yield return Task.Run(() => { var rnd = new Random(); while (!cancel.IsCancellationRequested) { @@ -146,7 +146,7 @@ class MyClass: .Select(t => t.Item2) .ToList(); foreach (var e in shufEntries) { - e.Analyze(cancel, true); + e.PreAnalyze(); } state.AnalyzeQueuedEntries(cancel); diff --git a/src/LanguageServer/Core/Impl/Implementation/Server.cs b/src/LanguageServer/Core/Impl/Implementation/Server.cs index 8a6a1e6c0..d5fde147d 100644 --- a/src/LanguageServer/Core/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Core/Impl/Implementation/Server.cs @@ -583,7 +583,7 @@ internal Task EnqueueItemAsync(IDocument doc, AnalysisPriority priority = Analys var pending = _pendingAnalysisEnqueue.Incremented(); try { var entry = doc as ProjectEntry; - entry?.ResetCompleteAnalysis(); + entry?.NewAnalysisAwaitableOnParse(); // If we don't need to parse, use null cookie var cookieTask = Task.FromResult(null); diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Microsoft.PythonTools.Analyzer.csproj b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Microsoft.PythonTools.Analyzer.csproj index fc0400806..de8669058 100644 --- a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Microsoft.PythonTools.Analyzer.csproj +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Microsoft.PythonTools.Analyzer.csproj @@ -4,13 +4,16 @@ Microsoft.PythonTools.Analysis Visual Studio - Python background analyzer Microsoft.PythonTools.Analyzer + python-language-server-ptvs.nuspec + version=$(NugetVersion) - Exe - portable - true + Exe + portable + $(OutputPath) + true ..\..\..\PLS.ruleset diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/python-language-server-ptvs.nuspec b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/python-language-server-ptvs.nuspec new file mode 100644 index 000000000..f2b4dbc3a --- /dev/null +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/python-language-server-ptvs.nuspec @@ -0,0 +1,34 @@ + + + + python-language-server-ptvs$os$ + $version$ + Python Language Server for PTVS + Microsoft + Microsoft + Apache-2.0 + https://github.com/Microsoft/PTVS + false + Microsoft Python Language Server for PTVS + Copyright (c) 2018 + Python;VisualStudio + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 78c7ffbd2ab49a2b9f68ed5461ca1b8a01653330 Mon Sep 17 00:00:00 2001 From: Alexander Sher Date: Wed, 9 Jan 2019 11:34:19 -0600 Subject: [PATCH 10/10] Address CR comments --- .../Impl/Intellisense/AnalysisProtocol.cs | 3 - .../Intellisense/OutOfProcProjectAnalyzer.cs | 217 +++++++++--------- .../Impl/python-language-server-ptvs.nuspec | 1 + 3 files changed, 109 insertions(+), 112 deletions(-) diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AnalysisProtocol.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AnalysisProtocol.cs index ba549ef64..46873b01a 100644 --- a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AnalysisProtocol.cs +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/AnalysisProtocol.cs @@ -491,9 +491,6 @@ public sealed class ChangeInfo { public string newText; public int startLine, startColumn; public int endLine, endColumn; -#if DEBUG - public int _startIndex, _endIndex; -#endif public static ChangeInfo FromDocumentChange(DocumentChange c) { return new ChangeInfo { diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutOfProcProjectAnalyzer.cs b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutOfProcProjectAnalyzer.cs index cd1b4836e..f1346fbf9 100644 --- a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutOfProcProjectAnalyzer.cs +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/Intellisense/OutOfProcProjectAnalyzer.cs @@ -38,8 +38,7 @@ namespace Microsoft.PythonTools.Intellisense { using AP = AnalysisProtocol; - using LS = Microsoft.PythonTools.Analysis.LanguageServer; - using LS2 = Microsoft.Python.LanguageServer; + using LS = Microsoft.Python.LanguageServer; /// /// Performs centralized parsing and analysis of Python source code for a remotely running process. @@ -90,7 +89,7 @@ public OutOfProcProjectAnalyzer(Stream writer, Stream reader, Action log } } - private void Server_OnLogMessage(object sender, LS2.LogMessageEventArgs e) { + private void Server_OnLogMessage(object sender, LS.LogMessageEventArgs e) { if (_log != null && Options.traceLevel.HasValue && e.type <= Options.traceLevel.Value) { _log(e.message); _connection?.SendEventAsync(new AP.AnalyzerWarningEvent { message = e.message }).DoNotWait(); @@ -169,11 +168,11 @@ private async Task ProcessLanguageServerRequest(AP.LanguageServerReque object result = null; switch (request.name) { - case "textDocument/completion": result = await _server.Completion(body.ToObject(), CancellationToken.None); break; - case "textDocument/hover": result = await _server.Hover(body.ToObject(), CancellationToken.None); break; - case "textDocument/definition": result = await _server.GotoDefinition(body.ToObject(), CancellationToken.None); break; - case "textDocument/references": result = await _server.FindReferences(body.ToObject(), CancellationToken.None); break; - case "textDocument/signatureHelp": result = await _server.SignatureHelp(body.ToObject(), CancellationToken.None); break; + case "textDocument/completion": result = await _server.Completion(body.ToObject(), CancellationToken.None); break; + case "textDocument/hover": result = await _server.Hover(body.ToObject(), CancellationToken.None); break; + case "textDocument/definition": result = await _server.GotoDefinition(body.ToObject(), CancellationToken.None); break; + case "textDocument/references": result = await _server.FindReferences(body.ToObject(), CancellationToken.None); break; + case "textDocument/signatureHelp": result = await _server.SignatureHelp(body.ToObject(), CancellationToken.None); break; } if (result != null) { @@ -250,10 +249,10 @@ private IAnalysisExtension LoadAnalysisExtension(AP.LoadExtensionRequest info) { private async Task Initialize(AP.InitializeRequest request) { try { - await _server.Initialize(new LS2.InitializeParams { + await _server.Initialize(new LS.InitializeParams { rootUri = request.rootUri, - initializationOptions = new LS2.PythonInitializationOptions { - interpreter = new LS2.PythonInitializationOptions.Interpreter { + initializationOptions = new LS.PythonInitializationOptions { + interpreter = new LS.PythonInitializationOptions.Interpreter { assembly = request.interpreter?.assembly, typeName = request.interpreter?.typeName, properties = request.interpreter?.properties @@ -268,26 +267,26 @@ await _server.Initialize(new LS2.InitializeParams { analysisUpdates = true, traceLogging = request.traceLogging, }, - capabilities = new LS2.ClientCapabilities { - python = new LS2.PythonClientCapabilities { + capabilities = new LS.ClientCapabilities { + python = new LS.PythonClientCapabilities { manualFileLoad = !request.analyzeAllFiles, liveLinting = request.liveLinting }, - textDocument = new LS2.TextDocumentClientCapabilities { - completion = new LS2.TextDocumentClientCapabilities.CompletionCapabilities { - completionItem = new LS2.TextDocumentClientCapabilities.CompletionCapabilities.CompletionItemCapabilities { - documentationFormat = new[] { LS.MarkupKind.PlainText }, + textDocument = new LS.TextDocumentClientCapabilities { + completion = new LS.TextDocumentClientCapabilities.CompletionCapabilities { + completionItem = new LS.TextDocumentClientCapabilities.CompletionCapabilities.CompletionItemCapabilities { + documentationFormat = new[] { MarkupKind.PlainText }, snippetSupport = false } }, - signatureHelp = new LS2.TextDocumentClientCapabilities.SignatureHelpCapabilities { - signatureInformation = new LS2.TextDocumentClientCapabilities.SignatureHelpCapabilities.SignatureInformationCapabilities { - documentationFormat = new[] { LS.MarkupKind.PlainText }, + signatureHelp = new LS.TextDocumentClientCapabilities.SignatureHelpCapabilities { + signatureInformation = new LS.TextDocumentClientCapabilities.SignatureHelpCapabilities.SignatureInformationCapabilities { + documentationFormat = new[] { MarkupKind.PlainText }, _shortLabel = true } }, - hover = new LS2.TextDocumentClientCapabilities.HoverCapabilities { - contentFormat = new[] { LS.MarkupKind.PlainText } + hover = new LS.TextDocumentClientCapabilities.HoverCapabilities { + contentFormat = new[] { MarkupKind.PlainText } } } } @@ -1027,8 +1026,8 @@ private Response GetOutliningRegions(AP.OutliningRegionsRequest request) { } private async Task GetNavigationsAsync(AP.NavigationRequest request) { - var symbols = await _server.HierarchicalDocumentSymbol(new LS2.DocumentSymbolParams { - textDocument = new LS2.TextDocumentIdentifier { uri = request.documentUri } + var symbols = await _server.HierarchicalDocumentSymbol(new LS.DocumentSymbolParams { + textDocument = new LS.TextDocumentIdentifier { uri = request.documentUri } }, CancellationToken.None); var navs = symbols.Select(ToNavigation).ToArray(); @@ -1039,7 +1038,7 @@ private async Task GetNavigationsAsync(AP.NavigationRequest request) { }; } - private AP.Navigation ToNavigation(LS2.DocumentSymbol symbol) => + private AP.Navigation ToNavigation(LS.DocumentSymbol symbol) => new AP.Navigation { name = symbol.name, startLine = symbol.range.start.line + 1, @@ -1115,10 +1114,10 @@ private async Task AnalyzeExpression(AP.AnalyzeExpressionRequest reque return IncorrectFileType(); } - var references = await _server.FindReferences(new LS2.ReferencesParams { + var references = await _server.FindReferences(new LS.ReferencesParams { textDocument = request.documentUri, position = new SourceLocation(request.line, request.column), - context = new LS2.ReferenceContext { + context = new LS.ReferenceContext { includeDeclaration = true, _includeValues = true } @@ -1132,7 +1131,7 @@ private async Task AnalyzeExpression(AP.AnalyzeExpressionRequest reque }; } - private AP.AnalysisReference MakeReference(LS2.Reference r) { + private AP.AnalysisReference MakeReference(LS.Reference r) { var range = (SourceSpan)r.range; return new AP.AnalysisReference { @@ -1156,20 +1155,20 @@ private static string GetVariableType(VariableType type) { return null; } - private static string GetVariableType(LS2.ReferenceKind? type) { + private static string GetVariableType(LS.ReferenceKind? type) { if (!type.HasValue) { return null; } switch (type.Value) { - case LS2.ReferenceKind.Definition: return "definition"; - case LS2.ReferenceKind.Reference: return "reference"; - case LS2.ReferenceKind.Value: return "value"; + case LS.ReferenceKind.Definition: return "definition"; + case LS.ReferenceKind.Reference: return "reference"; + case LS.ReferenceKind.Value: return "value"; } return null; } private async Task GetQuickInfo(AP.QuickInfoRequest request) { - LS2.Hover hover = await _server.Hover(new LS2.TextDocumentPositionParams { + LS.Hover hover = await _server.Hover(new LS.TextDocumentPositionParams { textDocument = request.documentUri, position = new SourceLocation(request.line, request.column), _expr = request.expr, @@ -1181,7 +1180,7 @@ private async Task GetQuickInfo(AP.QuickInfoRequest request) { } private async Task GetSignatures(AP.SignaturesRequest request) { - var sigs = await _server.SignatureHelp(new LS2.TextDocumentPositionParams { + var sigs = await _server.SignatureHelp(new LS.TextDocumentPositionParams { textDocument = request.documentUri, position = new SourceLocation(request.line, request.column), _expr = request.text @@ -1210,12 +1209,12 @@ private async Task GetModules(Request request) { var getModules = (AP.GetModulesRequest)request; var prefix = getModules.package == null ? null : (string.Join(".", getModules.package)); - var modules = await _server.Completion(new LS2.CompletionParams { + var modules = await _server.Completion(new LS.CompletionParams { textDocument = getModules.documentUri, _expr = prefix, - context = new LS2.CompletionContext { - triggerKind = LS2.CompletionTriggerKind.Invoked, - _filterKind = LS2.CompletionItemKind.Module, + context = new LS.CompletionContext { + triggerKind = LS.CompletionTriggerKind.Invoked, + _filterKind = LS.CompletionItemKind.Module, //_includeAllModules = getModules.package == null } }, CancellationToken.None); @@ -1228,10 +1227,10 @@ private async Task GetModules(Request request) { private async Task GetCompletions(Request request) { var req = (AP.CompletionsRequest)request; - var members = await _server.Completion(new LS2.CompletionParams { + var members = await _server.Completion(new LS.CompletionParams { position = new Position { line = req.line - 1, character = req.column - 1 }, textDocument = req.documentUri, - context = new LS2.CompletionContext { + context = new LS.CompletionContext { _intersection = req.options.HasFlag(GetMemberOptions.IntersectMultipleResults), //_statementKeywords = req.options.HasFlag(GetMemberOptions.IncludeStatementKeywords), //_expressionKeywords = req.options.HasFlag(GetMemberOptions.IncludeExpressionKeywords), @@ -1248,7 +1247,7 @@ private async Task GetCompletions(Request request) { private async Task GetAllMembers(Request request) { var req = (AP.GetAllMembersRequest)request; - var members = await _server.WorkspaceSymbols(new LS2.WorkspaceSymbolParams { + var members = await _server.WorkspaceSymbols(new LS.WorkspaceSymbolParams { query = req.prefix }, CancellationToken.None).ConfigureAwait(false); @@ -1257,7 +1256,7 @@ private async Task GetAllMembers(Request request) { }; } - private async Task ToCompletions(IEnumerable symbols) { + private async Task ToCompletions(IEnumerable symbols) { if (symbols == null) { return null; } @@ -1294,7 +1293,7 @@ private async Task GetAllMembers(Request request) { } - private async Task ToCompletions(IEnumerable completions, GetMemberOptions options) { + private async Task ToCompletions(IEnumerable completions, GetMemberOptions options) { if (completions == null) { return null; } @@ -1334,77 +1333,77 @@ private async Task GetAllMembers(Request request) { return res.ToArray(); } - private PythonMemberType ToMemberType(string originalKind, LS2.CompletionItemKind kind) { + private PythonMemberType ToMemberType(string originalKind, LS.CompletionItemKind kind) { PythonMemberType res; if (!string.IsNullOrEmpty(originalKind) && Enum.TryParse(originalKind, true, out res)) { return res; } switch (kind) { - case LS2.CompletionItemKind.None: return PythonMemberType.Unknown; - case LS2.CompletionItemKind.Text: return PythonMemberType.Constant; - case LS2.CompletionItemKind.Method: return PythonMemberType.Method; - case LS2.CompletionItemKind.Function: return PythonMemberType.Function; - case LS2.CompletionItemKind.Constructor: return PythonMemberType.Function; - case LS2.CompletionItemKind.Field: return PythonMemberType.Field; - case LS2.CompletionItemKind.Variable: return PythonMemberType.Instance; - case LS2.CompletionItemKind.Class: return PythonMemberType.Class; - case LS2.CompletionItemKind.Interface: return PythonMemberType.Class; - case LS2.CompletionItemKind.Module: return PythonMemberType.Module; - case LS2.CompletionItemKind.Property: return PythonMemberType.Property; - case LS2.CompletionItemKind.Unit: return PythonMemberType.Unknown; - case LS2.CompletionItemKind.Value: return PythonMemberType.Instance; - case LS2.CompletionItemKind.Enum: return PythonMemberType.Enum; - case LS2.CompletionItemKind.Keyword: return PythonMemberType.Keyword; - case LS2.CompletionItemKind.Snippet: return PythonMemberType.CodeSnippet; - case LS2.CompletionItemKind.Color: return PythonMemberType.Instance; - case LS2.CompletionItemKind.File: return PythonMemberType.Module; - case LS2.CompletionItemKind.Reference: return PythonMemberType.Unknown; - case LS2.CompletionItemKind.Folder: return PythonMemberType.Module; - case LS2.CompletionItemKind.EnumMember: return PythonMemberType.EnumInstance; - case LS2.CompletionItemKind.Constant: return PythonMemberType.Constant; - case LS2.CompletionItemKind.Struct: return PythonMemberType.Class; - case LS2.CompletionItemKind.Event: return PythonMemberType.Delegate; - case LS2.CompletionItemKind.Operator: return PythonMemberType.Unknown; - case LS2.CompletionItemKind.TypeParameter: return PythonMemberType.Class; + case LS.CompletionItemKind.None: return PythonMemberType.Unknown; + case LS.CompletionItemKind.Text: return PythonMemberType.Constant; + case LS.CompletionItemKind.Method: return PythonMemberType.Method; + case LS.CompletionItemKind.Function: return PythonMemberType.Function; + case LS.CompletionItemKind.Constructor: return PythonMemberType.Function; + case LS.CompletionItemKind.Field: return PythonMemberType.Field; + case LS.CompletionItemKind.Variable: return PythonMemberType.Instance; + case LS.CompletionItemKind.Class: return PythonMemberType.Class; + case LS.CompletionItemKind.Interface: return PythonMemberType.Class; + case LS.CompletionItemKind.Module: return PythonMemberType.Module; + case LS.CompletionItemKind.Property: return PythonMemberType.Property; + case LS.CompletionItemKind.Unit: return PythonMemberType.Unknown; + case LS.CompletionItemKind.Value: return PythonMemberType.Instance; + case LS.CompletionItemKind.Enum: return PythonMemberType.Enum; + case LS.CompletionItemKind.Keyword: return PythonMemberType.Keyword; + case LS.CompletionItemKind.Snippet: return PythonMemberType.CodeSnippet; + case LS.CompletionItemKind.Color: return PythonMemberType.Instance; + case LS.CompletionItemKind.File: return PythonMemberType.Module; + case LS.CompletionItemKind.Reference: return PythonMemberType.Unknown; + case LS.CompletionItemKind.Folder: return PythonMemberType.Module; + case LS.CompletionItemKind.EnumMember: return PythonMemberType.EnumInstance; + case LS.CompletionItemKind.Constant: return PythonMemberType.Constant; + case LS.CompletionItemKind.Struct: return PythonMemberType.Class; + case LS.CompletionItemKind.Event: return PythonMemberType.Delegate; + case LS.CompletionItemKind.Operator: return PythonMemberType.Unknown; + case LS.CompletionItemKind.TypeParameter: return PythonMemberType.Class; default: return PythonMemberType.Unknown; } } - private PythonMemberType ToMemberType(string originalKind, LS2.SymbolKind kind) { + private PythonMemberType ToMemberType(string originalKind, LS.SymbolKind kind) { PythonMemberType res; if (!string.IsNullOrEmpty(originalKind) && Enum.TryParse(originalKind, true, out res)) { return res; } switch (kind) { - case LS2.SymbolKind.None: return PythonMemberType.Unknown; - case LS2.SymbolKind.File: return PythonMemberType.Module; - case LS2.SymbolKind.Module: return PythonMemberType.Module; - case LS2.SymbolKind.Namespace: return PythonMemberType.Namespace; - case LS2.SymbolKind.Package: return PythonMemberType.Module; - case LS2.SymbolKind.Class: return PythonMemberType.Class; - case LS2.SymbolKind.Method: return PythonMemberType.Method; - case LS2.SymbolKind.Property: return PythonMemberType.Property; - case LS2.SymbolKind.Field: return PythonMemberType.Field; - case LS2.SymbolKind.Constructor: return PythonMemberType.Method; - case LS2.SymbolKind.Enum: return PythonMemberType.Enum; - case LS2.SymbolKind.Interface: return PythonMemberType.Class; - case LS2.SymbolKind.Function: return PythonMemberType.Function; - case LS2.SymbolKind.Variable: return PythonMemberType.Field; - case LS2.SymbolKind.Constant: return PythonMemberType.Constant; - case LS2.SymbolKind.String: return PythonMemberType.Constant; - case LS2.SymbolKind.Number: return PythonMemberType.Constant; - case LS2.SymbolKind.Boolean: return PythonMemberType.Constant; - case LS2.SymbolKind.Array: return PythonMemberType.Instance; - case LS2.SymbolKind.Object: return PythonMemberType.Instance; - case LS2.SymbolKind.Key: return PythonMemberType.Unknown; - case LS2.SymbolKind.Null: return PythonMemberType.Unknown; - case LS2.SymbolKind.EnumMember: return PythonMemberType.EnumInstance; - case LS2.SymbolKind.Struct: return PythonMemberType.Class; - case LS2.SymbolKind.Event: return PythonMemberType.Event; - case LS2.SymbolKind.Operator: return PythonMemberType.Method; - case LS2.SymbolKind.TypeParameter: return PythonMemberType.NamedArgument; + case LS.SymbolKind.None: return PythonMemberType.Unknown; + case LS.SymbolKind.File: return PythonMemberType.Module; + case LS.SymbolKind.Module: return PythonMemberType.Module; + case LS.SymbolKind.Namespace: return PythonMemberType.Namespace; + case LS.SymbolKind.Package: return PythonMemberType.Module; + case LS.SymbolKind.Class: return PythonMemberType.Class; + case LS.SymbolKind.Method: return PythonMemberType.Method; + case LS.SymbolKind.Property: return PythonMemberType.Property; + case LS.SymbolKind.Field: return PythonMemberType.Field; + case LS.SymbolKind.Constructor: return PythonMemberType.Method; + case LS.SymbolKind.Enum: return PythonMemberType.Enum; + case LS.SymbolKind.Interface: return PythonMemberType.Class; + case LS.SymbolKind.Function: return PythonMemberType.Function; + case LS.SymbolKind.Variable: return PythonMemberType.Field; + case LS.SymbolKind.Constant: return PythonMemberType.Constant; + case LS.SymbolKind.String: return PythonMemberType.Constant; + case LS.SymbolKind.Number: return PythonMemberType.Constant; + case LS.SymbolKind.Boolean: return PythonMemberType.Constant; + case LS.SymbolKind.Array: return PythonMemberType.Instance; + case LS.SymbolKind.Object: return PythonMemberType.Instance; + case LS.SymbolKind.Key: return PythonMemberType.Unknown; + case LS.SymbolKind.Null: return PythonMemberType.Unknown; + case LS.SymbolKind.EnumMember: return PythonMemberType.EnumInstance; + case LS.SymbolKind.Struct: return PythonMemberType.Class; + case LS.SymbolKind.Event: return PythonMemberType.Event; + case LS.SymbolKind.Operator: return PythonMemberType.Method; + case LS.SymbolKind.TypeParameter: return PythonMemberType.NamedArgument; default: return PythonMemberType.Unknown; } } @@ -1505,14 +1504,14 @@ public override TextReader GetReader() { private async Task UpdateContent(AP.FileUpdateRequest request) { int version = -1; foreach (var fileChange in request.updates) { - var changes = new List(); + var changes = new List(); if (fileChange.kind == AP.FileUpdateKind.reset) { - changes.Add(new LS2.TextDocumentContentChangedEvent { + changes.Add(new LS.TextDocumentContentChangedEvent { text = fileChange.content }); version = fileChange.version; } else if (fileChange.kind == AP.FileUpdateKind.changes) { - changes.AddRange(fileChange.changes.Select(c => new LS2.TextDocumentContentChangedEvent { + changes.AddRange(fileChange.changes.Select(c => new LS.TextDocumentContentChangedEvent { range = new SourceSpan( new SourceLocation(c.startLine, c.startColumn), new SourceLocation(c.endLine, c.endColumn) @@ -1524,8 +1523,8 @@ private async Task UpdateContent(AP.FileUpdateRequest request) { continue; } - _server.DidChangeTextDocument(new LS2.DidChangeTextDocumentParams { - textDocument = new LS2.VersionedTextDocumentIdentifier { + _server.DidChangeTextDocument(new LS.DidChangeTextDocumentParams { + textDocument = new LS.VersionedTextDocumentIdentifier { uri = request.documentUri, version = version, _fromVersion = Math.Max(version - 1, 0) @@ -1571,16 +1570,16 @@ private void AnalysisQueue_Complete(object sender, EventArgs e) { } private void OnModulesChanged(object sender, EventArgs args) { - _server.DidChangeConfiguration(new LS2.DidChangeConfigurationParams(), CancellationToken.None).DoNotWait(); + _server.DidChangeConfiguration(new LS.DidChangeConfigurationParams(), CancellationToken.None).DoNotWait(); } private void OnFileChanged(AP.FileChangedEvent e) { - _server.DidChangeWatchedFiles(new LS2.DidChangeWatchedFilesParams { - changes = e.changes.MaybeEnumerate().Select(c => new LS2.FileEvent { uri = c.documentUri, type = c.kind }).ToArray() + _server.DidChangeWatchedFiles(new LS.DidChangeWatchedFilesParams { + changes = e.changes.MaybeEnumerate().Select(c => new LS.FileEvent { uri = c.documentUri, type = c.kind }).ToArray() }, CancellationToken.None).DoNotWait(); } - private void OnAnalysisComplete(object sender, LS2.AnalysisCompleteEventArgs e) { + private void OnAnalysisComplete(object sender, LS.AnalysisCompleteEventArgs e) { _connection.SendEventAsync( new AP.FileAnalysisCompleteEvent { documentUri = e.uri, @@ -1649,7 +1648,7 @@ private PythonAnalyzer Analyzer { /// public PythonAnalyzer Project => _server.Analyzer; - private void OnPublishDiagnostics(object sender, LS2.PublishDiagnosticsEventArgs e) { + private void OnPublishDiagnostics(object sender, LS.PublishDiagnosticsEventArgs e) { _connection.SendEventAsync( new AP.DiagnosticsEvent { documentUri = e.uri, @@ -1659,7 +1658,7 @@ private void OnPublishDiagnostics(object sender, LS2.PublishDiagnosticsEventArgs ).DoNotWait(); } - private void OnParseComplete(object sender, LS2.ParseCompleteEventArgs e) { + private void OnParseComplete(object sender, LS.ParseCompleteEventArgs e) { _connection.SendEventAsync( new AP.FileParsedEvent { documentUri = e.uri, diff --git a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/python-language-server-ptvs.nuspec b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/python-language-server-ptvs.nuspec index f2b4dbc3a..dfe1b8838 100644 --- a/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/python-language-server-ptvs.nuspec +++ b/src/PTVS/Microsoft.PythonTools.Analyzer/Impl/python-language-server-ptvs.nuspec @@ -23,6 +23,7 @@ +