Port import tests from old LS (microsoft#606)

Mikhail Arkhipov · web-flow · commit 050eb345832e · 2019-02-14T12:07:46.000-08:00
* Part 8 * Part 9 * Buildable * Part 10 * Part 11 * Part 12 * Buildable * Part 14 * First passing test * Simplify configuration * Style * Fix test and move code to folders * Builtins import * Fix microsoft#470 * Fluents * Add search path * Import analysis, part I * Simplify builtins handling * Remove IMember * Handle import specific * More tests * Add typeshed * Renames * Make sure lazy modules are loaded * Renames * Move/rename * Rework importing * Derivation rework * Part 2 * Part 3 * Buildable * Module members * Async walk * Imports test pass * Remove lazy types * Fix from import * Stubs * Double overloads * Fix datetime test * Couple more tests + fluents * Few more tests * Additionl test + union type * Built-in scrape tests * Full stdlib scrape test * Complete async AST walker * Conditional defines test + variable loc cleanup * More stub tests Fix stub loading for packages (port from DDG) Split walker into multiple files * Add some (broken mostly) tests from DDG * Move document tests * Function arg eval, part I * Instance/factory * Builds * Test fixes * Fix static and instance call eval * More tests * More ported tests * Specialize builtin functions * Make walkers common and handle nested functions * Moar tests * Parser fixes + more tests * Handle negative numbers * Fix null ref * Basic list support * Few more list tests * Basic iterators * Support __iter__ * Iterators * Fix couple of tests * Add decorator test * Generics, part I * Generics, part 2 * Generics, part 3 * Basic TypeVar test * Typings, part 4 * Fix test * Generics, part 6 * Generics, part 7 * More tests (failing) * Forward ref fixes * Reorg * Improve symbol resolution + test fixes * Test fixes * Dictionary, part I * Part 11 * Fix test * Tests * Tests * More dict work * List ctor * Skip some tests for now * Fix iterators * Tuple slicing * Polish type comparo in return types * Add Mapping and tests * Add Iterable * Fix typo * Add Iterator[T] + test * Simplify typing types * Class reduction * Fix tests * Test fix * Handle 'with' statement * Handle try-except * Class method inheritance + NewType * Container types * Containers test * Tests * Handle generic type alias * Named tuple * Global/non-local * Handle tuples in for Handle custom iterators * Basic generator * Any/AnyStr * Test fixes * Type/Optional/etc handling * Proper doc population * Tests + range * Argument match * Basic argset and diagnostics * Argset tests * Exclude WIP * Exclude WIP * Arg eval * Arg match, part 2 * Tests and generic arg comparisons * Function eval with arguments * Baselines * Fix test * Undo AST formatting change and update baseline * LS cleanup 1 * Fix list ctor argument unpacking * Cleanup 2 * Builds * Partial completions * Partial * Partial * Simple test * Tests * Basic startup * Clean up a bit * Remove debug code * Port formatter tests * Fix tokenizer crash * Async fixes * Hover * Basic hover * Adjust expression options * Basic signature help * Fix class/instance * Update test * Fix builtin creation exception * Fix tests * Actually provide declared module * Completion test (partial) * Undo * Fix null await Fix override completions + test * Exports filtering Prevent augmenting imported types * Filter variables & exports * Ported tests * Test fixes * More ported tests * Fix exception completions * Import completions * Scope completions * With completions * Test fixes * WIP * Test fix * Better arg match * Temp disable WIP * First cut * Fix type leak * WIP * Remove ConfigureAwait and handle canceled and failed in the analysis notifications * WIP * Generic class base * Generic forward reference resolution * Suppress completion in strings + test * Prevent recursion on generic resolution Better match arguments * Handle call expression in generics * Relax condition as it happens in tensorflow * Fix typeshed version search Make writing cached modules async Fix module doc fetching * Hover tests * Fix prom import hover * Hover tests * Synchronize test cache writing * First cut * Test * Fixes * Add tests for os.path Null ref fix * Fix cache check * Improve resolution of builtins and typing in stubs * Merge tests * Add ntst for requests * Handle typeshed better * Fix custom stub handling * Better sync * Move files * Fix parameter locations * Hover improvement * PEP hints * One more test for PEP hints * Better handle hover over import as * Text based generic constraints * Handle with better with generic stubs * Undo debug * Handle non-binary open() Temporary fix 'with' handler since we haven't specialized IO/TextIO/BinaryIO yet. * Output syntax errors * Properly clear * - Fix async issue with analysis completion - Clean up diagnostics service interface - Use real DS in tests * Use proper scope when analyzing module * Severity mapping and reporting * Add publishing test Add NSubstitute Move all services to the same namespace. * Unused var * Test forced publish on close * Fix typo * Update test framework * Import location * Remove incorrect reference * Diagnostic severity mapping test Fix post-mortem earlier PR comment * Minor fixes * Move interface to the main class part * Flicker reduction * - Correct reported unresolved import name - Add tests * PR feedback * Port 2 tests * More tests * More tests * Fix typing and package import test * Last test * Call base instead
diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs
@@ -13,6 +13,7 @@
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
 
+using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Python.Analysis.Core.DependencyResolution;
@@ -50,6 +51,12 @@ public interface IModuleResolution {
         /// </summary>
         IPythonModule GetImportedModule(string name);
 
+        /// <summary>
+        /// Sets user search paths. This changes <see cref="CurrentPathResolver"/>.
+        /// </summary>
+        /// <returns>Added roots.</returns>
+        IEnumerable<string> SetUserSearchPaths(in IEnumerable<string> searchPaths);
+
         Task ReloadAsync(CancellationToken token = default);
     }
 }
diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs
@@ -147,25 +147,25 @@ internal async Task LoadBuiltinTypesAsync(CancellationToken cancellationToken =
             var builtinModuleNamesMember = BuiltinsModule.GetAnyMember("__builtin_module_names__");
             if (builtinModuleNamesMember.TryGetConstant<string>(out var s)) {
                 var builtinModuleNames = s.Split(',').Select(n => n.Trim());
-                _pathResolver.SetBuiltins(builtinModuleNames);
+                PathResolver.SetBuiltins(builtinModuleNames);
             }
         }
 
         public override async Task ReloadAsync(CancellationToken cancellationToken = default) {
             ModuleCache = new ModuleCache(_interpreter, _services);
 
-            _pathResolver = new PathResolver(_interpreter.LanguageVersion);
+            PathResolver = new PathResolver(_interpreter.LanguageVersion);
 
-            var addedRoots = _pathResolver.SetRoot(_root);
+            var addedRoots = PathResolver.SetRoot(_root);
             ReloadModulePaths(addedRoots);
 
             var interpreterPaths = await GetSearchPathsAsync(cancellationToken);
-            addedRoots = _pathResolver.SetInterpreterSearchPaths(interpreterPaths);
+            addedRoots = PathResolver.SetInterpreterSearchPaths(interpreterPaths);
 
             ReloadModulePaths(addedRoots);
             cancellationToken.ThrowIfCancellationRequested();
 
-            addedRoots = _pathResolver.SetUserSearchPaths(_interpreter.Configuration.SearchPaths);
+            addedRoots = SetUserSearchPaths(_interpreter.Configuration.SearchPaths);
             ReloadModulePaths(addedRoots);
         }
 
diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs
@@ -38,7 +38,7 @@ internal abstract class ModuleResolutionBase {
         protected readonly bool _requireInitPy;
         protected string _root;
 
-        protected PathResolver _pathResolver;
+        protected PathResolver PathResolver { get; set; }
 
         protected InterpreterConfiguration Configuration => _interpreter.Configuration;
 
@@ -58,7 +58,7 @@ protected ModuleResolutionBase(string root, IServiceContainer services) {
         /// <summary>
         /// Path resolver providing file resolution in module imports.
         /// </summary>
-        public PathResolverSnapshot CurrentPathResolver => _pathResolver.CurrentSnapshot;
+        public PathResolverSnapshot CurrentPathResolver => PathResolver.CurrentSnapshot;
 
         /// <summary>
         /// Builtins module.
@@ -68,7 +68,7 @@ protected ModuleResolutionBase(string root, IServiceContainer services) {
         public abstract Task ReloadAsync(CancellationToken cancellationToken = default);
         protected abstract Task<IPythonModule> DoImportAsync(string name, CancellationToken cancellationToken = default);
 
- public IReadOnlyCollection<string> GetPackagesFromDirectory(string searchPath, CancellationToken cancellationToken) {
+        public IReadOnlyCollection<string> GetPackagesFromDirectory(string searchPath, CancellationToken cancellationToken) {
             return ModulePath.GetModulesInPath(
                 searchPath,
                 recurse: false,
@@ -77,10 +77,18 @@ public IReadOnlyCollection<string> GetPackagesFromDirectory(string searchPath, C
             ).Select(mp => mp.ModuleName).Where(n => !string.IsNullOrEmpty(n)).TakeWhile(_ => !cancellationToken.IsCancellationRequested).ToList();
         }
 
-        public IPythonModule GetImportedModule(string name)
-            => _modules.TryGetValue(name, out var module) ? module : null;
+        public IPythonModule GetImportedModule(string name) {
+            var module = _interpreter.ModuleResolution.GetSpecializedModule(name);
+            if (module != null) {
+                return module;
+            }
+            return _modules.TryGetValue(name, out module) ? module : null;
+        }
+
+        public IEnumerable<string> SetUserSearchPaths(in IEnumerable<string> searchPaths)
+            => PathResolver.SetUserSearchPaths(searchPaths);
 
-        public void AddModulePath(string path) => _pathResolver.TryAddModulePath(path, out var _);
+        public void AddModulePath(string path) => PathResolver.TryAddModulePath(path, out var _);
 
         public ModulePath FindModule(string filePath) {
             var bestLibraryPath = string.Empty;
@@ -207,7 +215,7 @@ private async Task<TryImportModuleResult> TryImportModuleAsync(string name, Canc
 
         protected void ReloadModulePaths(in IEnumerable<string> rootPaths) {
             foreach (var modulePath in rootPaths.Where(Directory.Exists).SelectMany(p => PathUtils.EnumerateFiles(p))) {
-                _pathResolver.TryAddModulePath(modulePath, out _);
+                PathResolver.TryAddModulePath(modulePath, out _);
             }
         }
 
diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs
@@ -65,12 +65,12 @@ protected override async Task<IPythonModule> DoImportAsync(string name, Cancella
         }
 
         public override Task ReloadAsync(CancellationToken cancellationToken = default) {
-            _pathResolver = new PathResolver(_interpreter.LanguageVersion);
+            PathResolver = new PathResolver(_interpreter.LanguageVersion);
 
-            var addedRoots = _pathResolver.SetRoot(_root);
+            var addedRoots = PathResolver.SetRoot(_root);
             ReloadModulePaths(addedRoots);
 
-            addedRoots = _pathResolver.SetInterpreterSearchPaths(_typeStubPaths);
+            addedRoots = PathResolver.SetInterpreterSearchPaths(_typeStubPaths);
             ReloadModulePaths(addedRoots);
 
             cancellationToken.ThrowIfCancellationRequested();
diff --git a/src/Analysis/Ast/Test/AnalysisTestBase.cs b/src/Analysis/Ast/Test/AnalysisTestBase.cs
@@ -100,6 +100,12 @@ protected async Task<IServiceManager> CreateServicesAsync(string root, Interpret
             return sm;
         }
 
+        protected async Task CreateServicesAsync(InterpreterConfiguration configuration, string modulePath) {
+            modulePath = modulePath ?? TestData.GetDefaultModulePath();
+            var moduleDirectory = Path.GetDirectoryName(modulePath);
+            await CreateServicesAsync(moduleDirectory, configuration);
+        }
+
         protected Task<IDocumentAnalysis> GetAnalysisAsync(string code, PythonLanguageVersion version, string modulePath = null)
             => GetAnalysisAsync(code, PythonVersions.GetRequiredCPythonConfiguration(version), modulePath);
 
@@ -129,7 +135,7 @@ protected async Task<IDocumentAnalysis> GetAnalysisAsync(
             string modulePath = null) {
 
             var moduleUri = modulePath != null ? new Uri(modulePath) : TestData.GetDefaultModuleUri();
-            modulePath = modulePath ?? TestData .GetDefaultModulePath();
+            modulePath = modulePath ?? TestData.GetDefaultModulePath();
             moduleName = moduleName ?? Path.GetFileNameWithoutExtension(modulePath);
 
             IDocument doc;
diff --git a/src/LanguageServer/Impl/Completion/ImportCompletion.cs b/src/LanguageServer/Impl/Completion/ImportCompletion.cs
@@ -150,8 +150,6 @@ private static CompletionResult GetResultFromSearch(IImportSearchResult importSe
                     break;
                 case PackageImport packageImports:
                     return new CompletionResult(packageImports.Modules
-                        .Select(m => mres.GetImportedModule(m.FullName))
-                        .ExcludeDefault()
                         .Select(m => CompletionItemSource.CreateCompletionItem(m.Name, CompletionItemKind.Module))
                         .Prepend(CompletionItemSource.Star));
                 default:
diff --git a/src/LanguageServer/Test/CompletionTests.cs b/src/LanguageServer/Test/CompletionTests.cs
@@ -55,8 +55,8 @@ def method(self):
 ";
             var analysis = await GetAnalysisAsync(code);
             var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion);
-            var comps = (await cs.GetCompletionsAsync(analysis, new SourceLocation(8, 1))).Completions.ToArray();
-            comps.Select(c => c.label).Should().Contain("C", "x", "y", "while", "for", "yield");
+            var comps = await cs.GetCompletionsAsync(analysis, new SourceLocation(8, 1));
+            comps.Should().HaveLabels("C", "x", "y", "while", "for", "yield");
         }
 
         [TestMethod, Priority(0)]
@@ -67,8 +67,8 @@ public async Task StringMembers() {
 ";
             var analysis = await GetAnalysisAsync(code);
             var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion);
-            var comps = (await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 3))).Completions.ToArray();
-            comps.Select(c => c.label).Should().Contain(new[] { @"isupper", @"capitalize", @"split" });
+            var comps = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 3));
+            comps.Should().HaveLabels(@"isupper", @"capitalize", @"split" );
         }
 
         [TestMethod, Priority(0)]
@@ -79,8 +79,8 @@ import datetime
 ";
             var analysis = await GetAnalysisAsync(code);
             var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion);
-            var comps = (await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 19))).Completions.ToArray();
-            comps.Select(c => c.label).Should().Contain(new[] { "now", @"tzinfo", @"ctime" });
+            var comps = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 19));
+            comps.Should().HaveLabels("now", @"tzinfo", @"ctime");
         }
 
         [TestMethod, Priority(0)]
@@ -94,11 +94,11 @@ def method1(self): pass
 ";
             var analysis = await GetAnalysisAsync(code);
             var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion);
-            var comps = (await cs.GetCompletionsAsync(analysis, new SourceLocation(5, 4))).Completions.ToArray();
-            comps.Select(c => c.label).Should().Contain(@"ABCDE");
+            var comps = await cs.GetCompletionsAsync(analysis, new SourceLocation(5, 4));
+            comps.Should().HaveLabels(@"ABCDE");
 
-            comps = (await cs.GetCompletionsAsync(analysis, new SourceLocation(6, 9))).Completions.ToArray();
-            comps.Select(c => c.label).Should().Contain("method1");
+            comps = await cs.GetCompletionsAsync(analysis, new SourceLocation(6, 9));
+            comps.Should().HaveLabels("method1");
         }
 
         [DataRow(PythonLanguageVersion.V36, "value")]
diff --git a/src/LanguageServer/Test/ImportsTests.cs b/src/LanguageServer/Test/ImportsTests.cs
