Fix missing keys issue (#1451)

Author: AlexanderSher

src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs

@@ -60,6 +60,7 @@ protected override IPythonModule CreateModule(string name) {
                     _log?.Log(TraceEventType.Warning, "Unsupported native module in stubs", moduleImport.FullName, moduleImport.ModulePath);
                     return null;
                 }
+
                 return new StubPythonModule(moduleImport.FullName, moduleImport.ModulePath, true, _services);
             }
 

src/Analysis/Ast/Test/ScrapeTests.cs

@@ -262,8 +262,7 @@ await FullStdLibTest(v,
                 "win32ui"
             );
         }
-
-
+        
         private async Task FullStdLibTest(InterpreterConfiguration configuration, params string[] skipModules) {
             configuration.AssertInstalled();
             var moduleUri = TestData.GetDefaultModuleUri();

src/Analysis/Core/Impl/DependencyResolution/ImportRoot.cs

@@ -1,5 +1,4 @@
-
-// Copyright(c) Microsoft Corporation
+// Copyright(c) Microsoft Corporation
 // All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the License); you may not use

src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs

@@ -137,25 +137,15 @@ public static async Task<IList<PythonLibraryPath>> GetSearchPathsAsync(Interpret
                 }
             }
 
-            var standardLibraryPath = GetStandardLibraryPath(fs, config);
+            var ospy = PathUtils.FindFile(fs, config.LibraryPath, "os.py");
+            var standardLibraryPath = !string.IsNullOrEmpty(ospy) ? IOPath.GetDirectoryName(ospy) : string.Empty;
             if (!string.IsNullOrEmpty(standardLibraryPath)) {
                 return GetDefaultSearchPaths(fs, standardLibraryPath);
             }
 
             return Array.Empty<PythonLibraryPath>();
         }
 
-        public static string GetStandardLibraryPath(IFileSystem fs, InterpreterConfiguration config) {
-            var ospy = PathUtils.FindFile(fs, config.LibraryPath, "os.py");
-            return !string.IsNullOrEmpty(ospy) ? IOPath.GetDirectoryName(ospy) : string.Empty;
-        }
-
-        public static string GetSitePackagesPath(IFileSystem fs, InterpreterConfiguration config)
-            => GetSitePackagesPath(GetStandardLibraryPath(fs, config));
-
-        public static string GetSitePackagesPath(string standardLibraryPath)
-            => !string.IsNullOrEmpty(standardLibraryPath) ? IOPath.Combine(standardLibraryPath, "site-packages") : string.Empty;
-
         /// <summary>
         /// Gets the set of search paths by running the interpreter.
         /// </summary>

src/Caching/Impl/ModuleUniqueId.cs

@@ -17,6 +17,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using Microsoft.Python.Analysis.Core.DependencyResolution;
 using Microsoft.Python.Analysis.Core.Interpreter;
 using Microsoft.Python.Analysis.Modules;
 using Microsoft.Python.Analysis.Types;
@@ -39,8 +40,9 @@ public static string GetUniqueId(string moduleName, string filePath, ModuleType
 
             var interpreter = services.GetService<IPythonInterpreter>();
             var fs = services.GetService<IFileSystem>();
-            
-            var modulePathType = GetModulePathType(filePath, interpreter.ModuleResolution.LibraryPaths, fs);
+
+            var moduleResolution = interpreter.ModuleResolution;
+            var modulePathType = GetModulePathType(filePath, moduleResolution.LibraryPaths, fs);
             switch(modulePathType) {
                 case PythonLibraryPathType.Site when cachingLevel < AnalysisCachingLevel.Library:
                     return null;
@@ -82,19 +84,28 @@ public static string GetUniqueId(string moduleName, string filePath, ModuleType
                 return $"{moduleName}({config.Version.Major}.{config.Version.Minor})";
             }
 
-            // If all else fails, hash module data.
-            return $"{moduleName}.{HashModuleContent(Path.GetDirectoryName(filePath), fs)}";
+            var parent = moduleResolution.CurrentPathResolver.GetModuleParentFromModuleName(moduleName);
+            var hash = HashModuleFileSizes(parent);
+            // If all else fails, hash modules file sizes.
+            return $"{moduleName}.{(ulong)hash}";
         }
 
-        private static string HashModuleContent(string moduleFolder, IFileSystem fs) {
-            // Hash file sizes 
-            var total = fs
-                .GetFileSystemEntries(moduleFolder, "*.*", SearchOption.AllDirectories)
-                .Where(fs.FileExists)
-                .Select(fs.FileSize)
-                .Aggregate((hash, e) => unchecked(hash * 31 ^ e.GetHashCode()));
+        private static long HashModuleFileSizes(IImportChildrenSource source) {
+            var hash = 0L;
+            var names = source.GetChildrenNames();
+            foreach (var name in names) {
+                if (source.TryGetChildImport(name, out var child)) {
+                    if (child is ModuleImport moduleImport) {
+                        hash = unchecked(hash * 31 ^ moduleImport.ModuleFileSize);
+                    }
+
+                    if (child is IImportChildrenSource childSource) {
+                        hash = unchecked(hash * 31 ^ HashModuleFileSizes(childSource));
+                    }
+                }
+            }
 
-            return ((uint)total).ToString();
+            return hash;
         }
 
         private static PythonLibraryPathType GetModulePathType(string modulePath, IEnumerable<PythonLibraryPath> libraryPaths, IFileSystem fs) {

src/Caching/Test/LibraryModulesTests.cs

@@ -20,6 +20,7 @@
 using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions;
 using Microsoft.Python.Analysis.Modules;
 using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Parsing;
 using Microsoft.Python.Parsing.Tests;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using TestUtilities;
@@ -120,6 +121,7 @@ public async Task Builtins() {
         public Task Ftplib() => TestModule("ftplib");
 
         [TestMethod, Priority(0)]
+        [Ignore]
         public Task Functools() => TestModule("functools");
 
         [TestMethod, Priority(0)]
@@ -173,6 +175,9 @@ public async Task Builtins() {
         [TestMethod, Priority(0)]
         public Task Multiprocessing() => TestModule("multiprocessing");
 
+        [TestMethod, Priority(0)]
+        public Task Numpy() => TestModule("numpy");
+
         [TestMethod, Priority(0)]
         public Task Os() => TestModule("os");
 
@@ -296,7 +301,7 @@ import requests
         }
 
         private async Task TestModule(string name) {
-            var analysis = await GetAnalysisAsync($"import {name}");
+            var analysis = await GetAnalysisAsync($"import {name}", PythonVersions.Python37_x64);
             var m = analysis.Document.Interpreter.ModuleResolution.GetImportedModule(name);
             var model = ModuleModel.FromAnalysis(m.Analysis, Services, AnalysisCachingLevel.Library);
 

src/Core/Impl/IO/DirectoryInfoProxy.cs

@@ -53,7 +53,7 @@ public IEnumerable<IFileSystemInfo> EnumerateFileSystemInfos(string searchPatter
 
         public IEnumerable<IFileSystemInfo> EnumerateFileSystemInfos(string[] includePatterns, string[] excludePatterns) {
             var matcher = GetMatcher(includePatterns, excludePatterns);
-            PatternMatchingResult matchResult = SafeExecuteMatcher(matcher);
+            var matchResult = SafeExecuteMatcher(matcher);
             return matchResult.Files.Select((filePatternMatch) => {
                 var path = PathUtils.NormalizePath(filePatternMatch.Path);
                 return CreateFileSystemInfoProxy(new FileInfo(path));

src/LanguageServer/Impl/Sources/ReferenceSource.cs

@@ -100,7 +100,7 @@ private async Task<Reference[]> FindAllReferencesAsync(string name, IPythonModul
                 return new List<(Uri, long)>();
             }
 
-            var interpreterPaths = interpreter.ModuleResolution.InterpreterPaths.ToArray();
+            var interpreterPaths = interpreter.ModuleResolution.InterpreterPaths;
             var files = new List<(Uri, long)>();
 
             foreach (var filePath in fs.GetFiles(root, "*.py", SearchOption.AllDirectories).Select(Path.GetFullPath)) {