diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs index bcb56b6e5..c43f3acf1 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs @@ -60,6 +60,7 @@ protected override IPythonModule CreateModule(string name) { _log?.Log(TraceEventType.Warning, "Unsupported native module in stubs", moduleImport.FullName, moduleImport.ModulePath); return null; } + return new StubPythonModule(moduleImport.FullName, moduleImport.ModulePath, true, _services); } diff --git a/src/Analysis/Ast/Test/ScrapeTests.cs b/src/Analysis/Ast/Test/ScrapeTests.cs index ceed5b90b..34d6a8caf 100644 --- a/src/Analysis/Ast/Test/ScrapeTests.cs +++ b/src/Analysis/Ast/Test/ScrapeTests.cs @@ -262,8 +262,7 @@ await FullStdLibTest(v, "win32ui" ); } - - + private async Task FullStdLibTest(InterpreterConfiguration configuration, params string[] skipModules) { configuration.AssertInstalled(); var moduleUri = TestData.GetDefaultModuleUri(); diff --git a/src/Analysis/Core/Impl/DependencyResolution/ImportRoot.cs b/src/Analysis/Core/Impl/DependencyResolution/ImportRoot.cs index 32a88da99..503eb6c6a 100644 --- a/src/Analysis/Core/Impl/DependencyResolution/ImportRoot.cs +++ b/src/Analysis/Core/Impl/DependencyResolution/ImportRoot.cs @@ -1,5 +1,4 @@ - -// Copyright(c) Microsoft Corporation +// Copyright(c) Microsoft Corporation // All rights reserved. // // Licensed under the Apache License, Version 2.0 (the License); you may not use diff --git a/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs b/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs index 2f01a80fe..b7d438a88 100644 --- a/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs +++ b/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs @@ -137,7 +137,8 @@ public static async Task> GetSearchPathsAsync(Interpret } } - var standardLibraryPath = GetStandardLibraryPath(fs, config); + var ospy = PathUtils.FindFile(fs, config.LibraryPath, "os.py"); + var standardLibraryPath = !string.IsNullOrEmpty(ospy) ? IOPath.GetDirectoryName(ospy) : string.Empty; if (!string.IsNullOrEmpty(standardLibraryPath)) { return GetDefaultSearchPaths(fs, standardLibraryPath); } @@ -145,17 +146,6 @@ public static async Task> GetSearchPathsAsync(Interpret return Array.Empty(); } - public static string GetStandardLibraryPath(IFileSystem fs, InterpreterConfiguration config) { - var ospy = PathUtils.FindFile(fs, config.LibraryPath, "os.py"); - return !string.IsNullOrEmpty(ospy) ? IOPath.GetDirectoryName(ospy) : string.Empty; - } - - public static string GetSitePackagesPath(IFileSystem fs, InterpreterConfiguration config) - => GetSitePackagesPath(GetStandardLibraryPath(fs, config)); - - public static string GetSitePackagesPath(string standardLibraryPath) - => !string.IsNullOrEmpty(standardLibraryPath) ? IOPath.Combine(standardLibraryPath, "site-packages") : string.Empty; - /// /// Gets the set of search paths by running the interpreter. /// diff --git a/src/Caching/Impl/ModuleUniqueId.cs b/src/Caching/Impl/ModuleUniqueId.cs index 9cf4fd376..744195d3e 100644 --- a/src/Caching/Impl/ModuleUniqueId.cs +++ b/src/Caching/Impl/ModuleUniqueId.cs @@ -17,6 +17,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; @@ -39,8 +40,9 @@ public static string GetUniqueId(string moduleName, string filePath, ModuleType var interpreter = services.GetService(); var fs = services.GetService(); - - var modulePathType = GetModulePathType(filePath, interpreter.ModuleResolution.LibraryPaths, fs); + + var moduleResolution = interpreter.ModuleResolution; + var modulePathType = GetModulePathType(filePath, moduleResolution.LibraryPaths, fs); switch(modulePathType) { case PythonLibraryPathType.Site when cachingLevel < AnalysisCachingLevel.Library: return null; @@ -82,19 +84,28 @@ public static string GetUniqueId(string moduleName, string filePath, ModuleType return $"{moduleName}({config.Version.Major}.{config.Version.Minor})"; } - // If all else fails, hash module data. - return $"{moduleName}.{HashModuleContent(Path.GetDirectoryName(filePath), fs)}"; + var parent = moduleResolution.CurrentPathResolver.GetModuleParentFromModuleName(moduleName); + var hash = HashModuleFileSizes(parent); + // If all else fails, hash modules file sizes. + return $"{moduleName}.{(ulong)hash}"; } - private static string HashModuleContent(string moduleFolder, IFileSystem fs) { - // Hash file sizes - var total = fs - .GetFileSystemEntries(moduleFolder, "*.*", SearchOption.AllDirectories) - .Where(fs.FileExists) - .Select(fs.FileSize) - .Aggregate((hash, e) => unchecked(hash * 31 ^ e.GetHashCode())); + private static long HashModuleFileSizes(IImportChildrenSource source) { + var hash = 0L; + var names = source.GetChildrenNames(); + foreach (var name in names) { + if (source.TryGetChildImport(name, out var child)) { + if (child is ModuleImport moduleImport) { + hash = unchecked(hash * 31 ^ moduleImport.ModuleFileSize); + } + + if (child is IImportChildrenSource childSource) { + hash = unchecked(hash * 31 ^ HashModuleFileSizes(childSource)); + } + } + } - return ((uint)total).ToString(); + return hash; } private static PythonLibraryPathType GetModulePathType(string modulePath, IEnumerable libraryPaths, IFileSystem fs) { diff --git a/src/Caching/Test/LibraryModulesTests.cs b/src/Caching/Test/LibraryModulesTests.cs index fc1fe5d51..cbe98895d 100644 --- a/src/Caching/Test/LibraryModulesTests.cs +++ b/src/Caching/Test/LibraryModulesTests.cs @@ -20,6 +20,7 @@ using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; @@ -120,6 +121,7 @@ public async Task Builtins() { public Task Ftplib() => TestModule("ftplib"); [TestMethod, Priority(0)] + [Ignore] public Task Functools() => TestModule("functools"); [TestMethod, Priority(0)] @@ -173,6 +175,9 @@ public async Task Builtins() { [TestMethod, Priority(0)] public Task Multiprocessing() => TestModule("multiprocessing"); + [TestMethod, Priority(0)] + public Task Numpy() => TestModule("numpy"); + [TestMethod, Priority(0)] public Task Os() => TestModule("os"); @@ -296,7 +301,7 @@ import requests } private async Task TestModule(string name) { - var analysis = await GetAnalysisAsync($"import {name}"); + var analysis = await GetAnalysisAsync($"import {name}", PythonVersions.Python37_x64); var m = analysis.Document.Interpreter.ModuleResolution.GetImportedModule(name); var model = ModuleModel.FromAnalysis(m.Analysis, Services, AnalysisCachingLevel.Library); diff --git a/src/Core/Impl/IO/DirectoryInfoProxy.cs b/src/Core/Impl/IO/DirectoryInfoProxy.cs index 2e694045f..0104e3675 100644 --- a/src/Core/Impl/IO/DirectoryInfoProxy.cs +++ b/src/Core/Impl/IO/DirectoryInfoProxy.cs @@ -53,7 +53,7 @@ public IEnumerable EnumerateFileSystemInfos(string searchPatter public IEnumerable EnumerateFileSystemInfos(string[] includePatterns, string[] excludePatterns) { var matcher = GetMatcher(includePatterns, excludePatterns); - PatternMatchingResult matchResult = SafeExecuteMatcher(matcher); + var matchResult = SafeExecuteMatcher(matcher); return matchResult.Files.Select((filePatternMatch) => { var path = PathUtils.NormalizePath(filePatternMatch.Path); return CreateFileSystemInfoProxy(new FileInfo(path)); diff --git a/src/LanguageServer/Impl/Sources/ReferenceSource.cs b/src/LanguageServer/Impl/Sources/ReferenceSource.cs index 99f670fe8..3f162a173 100644 --- a/src/LanguageServer/Impl/Sources/ReferenceSource.cs +++ b/src/LanguageServer/Impl/Sources/ReferenceSource.cs @@ -100,7 +100,7 @@ private async Task FindAllReferencesAsync(string name, IPythonModul return new List<(Uri, long)>(); } - var interpreterPaths = interpreter.ModuleResolution.InterpreterPaths.ToArray(); + var interpreterPaths = interpreter.ModuleResolution.InterpreterPaths; var files = new List<(Uri, long)>(); foreach (var filePath in fs.GetFiles(root, "*.py", SearchOption.AllDirectories).Select(Path.GetFullPath)) {