Skip to content
This repository was archived by the owner on Apr 14, 2022. It is now read-only.

Commit bfb0036

Browse files
AlexanderSherAlexanderSher
authored
Merges from db
- Fix the order of builtins initialization during reload (may fix #1156) - Remove some of the dead code - Add file size information to the module import
1 parent ea84477 commit bfb0036

25 files changed

+310
-882
lines changed

‎src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,17 +47,18 @@ private async Task LoadBuiltinTypesAsync(string root, IServiceManager sm, Cancel
4747

4848
sm.AddService(this);
4949
_moduleResolution = new MainModuleResolution(root, sm);
50+
_stubResolution = new TypeshedResolution(sm);
51+
52+
await _moduleResolution.InitializeAsync(cancellationToken);
53+
await _stubResolution.InitializeAsync(cancellationToken);
54+
5055
lock (_lock) {
5156
var builtinModule = _moduleResolution.CreateBuiltinsModule();
5257
_builtinTypes[BuiltinTypeId.NoneType]
5358
= new PythonType("NoneType", new Location(builtinModule), string.Empty, BuiltinTypeId.NoneType);
5459
_builtinTypes[BuiltinTypeId.Unknown]
5560
= UnknownType = new PythonType("Unknown", new Location(builtinModule), string.Empty);
5661
}
57-
await _moduleResolution.InitializeAsync(cancellationToken);
58-
59-
_stubResolution = new TypeshedResolution(sm);
60-
await _stubResolution.InitializeAsync(cancellationToken);
6162

6263
await _moduleResolution.LoadBuiltinTypesAsync(cancellationToken);
6364
}

‎src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -268,7 +268,7 @@ private bool TryAddModulePath(ModuleCreationOptions mco) {
268268
throw new InvalidOperationException("Can't create document with no file path or URI specified");
269269
}
270270

271-
if (!ModuleManagement.TryAddModulePath(filePath, true, out var fullName)) {
271+
if (!ModuleManagement.TryAddModulePath(filePath, 0, true, out var fullName)) {
272272
return false;
273273
}
274274

‎src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
using Microsoft.Python.Analysis.Caching;
2020
using Microsoft.Python.Analysis.Core.Interpreter;
2121
using Microsoft.Python.Analysis.Types;
22+
using Microsoft.Python.Core.Collections;
2223

2324
namespace Microsoft.Python.Analysis.Modules {
2425
/// <summary>
@@ -32,14 +33,12 @@ public interface IModuleManagement: IModuleResolution {
3233
/// <returns></returns>
3334
ModulePath FindModule(string filePath);
3435

35-
IReadOnlyCollection<string> GetPackagesFromDirectory(string searchPath, CancellationToken cancellationToken = default);
36-
3736
/// <summary>
3837
/// Cache of module stubs generated from compiled modules.
3938
/// </summary>
4039
IStubCache StubCache { get; }
4140

42-
bool TryAddModulePath(in string path, in bool allowNonRooted, out string fullName);
41+
bool TryAddModulePath(in string path, in long fileSize, in bool allowNonRooted, out string fullName);
4342

4443
/// <summary>
4544
/// Provides ability to specialize module by replacing module import by
@@ -65,6 +64,6 @@ public interface IModuleManagement: IModuleResolution {
6564
/// <summary>
6665
/// Set of interpreter paths.
6766
/// </summary>
68-
IEnumerable<string> InterpreterPaths { get; }
67+
ImmutableArray<string> InterpreterPaths { get; }
6968
}
7069
}

‎src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
using Microsoft.Python.Analysis.Types;
3030
using Microsoft.Python.Analysis.Values;
3131
using Microsoft.Python.Core;
32+
using Microsoft.Python.Core.Collections;
3233
using Microsoft.Python.Core.Diagnostics;
3334
using Microsoft.Python.Core.IO;
3435
using Microsoft.Python.Core.OS;
@@ -38,7 +39,7 @@ internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManage
3839
private readonly ConcurrentDictionary<string, IPythonModule> _specialized = new ConcurrentDictionary<string, IPythonModule>();
3940
private IRunningDocumentTable _rdt;
4041

41-
private IEnumerable<string> _userPaths = Enumerable.Empty<string>();
42+
private ImmutableArray<string> _userPaths = ImmutableArray<string>.Empty;
4243

4344
public MainModuleResolution(string root, IServiceContainer services)
4445
: base(root, services) { }
@@ -200,22 +201,24 @@ public async Task ReloadAsync(CancellationToken cancellationToken = default) {
200201
}
201202

202203
// Preserve builtins, they don't need to be reloaded since interpreter does not change.
203-
var builtins = Modules[BuiltinModuleName];
204-
Modules.Clear();
205-
Modules[BuiltinModuleName] = builtins;
206-
207-
PathResolver = new PathResolver(_interpreter.LanguageVersion);
208-
209-
var addedRoots = new HashSet<string>();
210-
addedRoots.UnionWith(PathResolver.SetRoot(Root));
204+
if (Modules.TryGetValue(BuiltinModuleName, out var builtins)) {
205+
Modules.Clear();
206+
Modules[BuiltinModuleName] = builtins;
207+
}
211208

212209
await ReloadSearchPaths(cancellationToken);
213210

214-
addedRoots.UnionWith(PathResolver.SetInterpreterSearchPaths(InterpreterPaths));
215-
addedRoots.UnionWith(PathResolver.SetUserSearchPaths(_userPaths));
211+
PathResolver = new PathResolver(_interpreter.LanguageVersion, Root, InterpreterPaths, _userPaths);
212+
213+
var addedRoots = new HashSet<string> { Root };
214+
addedRoots.UnionWith(InterpreterPaths);
215+
addedRoots.UnionWith(_userPaths);
216216
ReloadModulePaths(addedRoots);
217217
}
218218

219+
public bool TryAddModulePath(in string path, in long fileSize, in bool allowNonRooted, out string fullModuleName)
220+
=> PathResolver.TryAddModulePath(path, fileSize, allowNonRooted, out fullModuleName);
221+
219222
// For tests
220223
internal void AddUnimportableModule(string moduleName)
221224
=> Modules[moduleName] = new ModuleRef(new SentinelModule(moduleName, _services));

‎src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs

Lines changed: 5 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
using Microsoft.Python.Analysis.Documents;
2525
using Microsoft.Python.Analysis.Types;
2626
using Microsoft.Python.Core;
27+
using Microsoft.Python.Core.Collections;
2728
using Microsoft.Python.Core.IO;
2829
using Microsoft.Python.Core.Logging;
2930
using Microsoft.Python.Core.Services;
@@ -55,9 +56,8 @@ protected ModuleResolutionBase(string root, IServiceContainer services) {
5556
}
5657

5758
public string Root { get; protected set; }
58-
public IEnumerable<string> InterpreterPaths { get; protected set; } = Enumerable.Empty<string>();
59+
public ImmutableArray<string> InterpreterPaths { get; protected set; } = ImmutableArray<string>.Empty;
5960

60-
public IModuleCache ModuleCache { get; protected set; }
6161
public string BuiltinModuleName => BuiltinTypeId.Unknown.GetModuleName(_interpreter.LanguageVersion);
6262

6363
/// <summary>
@@ -72,15 +72,6 @@ protected ModuleResolutionBase(string root, IServiceContainer services) {
7272

7373
protected abstract IPythonModule CreateModule(string name);
7474

75-
public IReadOnlyCollection<string> GetPackagesFromDirectory(string searchPath, CancellationToken cancellationToken) {
76-
return ModulePath.GetModulesInPath(
77-
searchPath,
78-
recurse: false,
79-
includePackages: true,
80-
requireInitPy: _requireInitPy
81-
).Select(mp => mp.ModuleName).Where(n => !string.IsNullOrEmpty(n)).TakeWhile(_ => !cancellationToken.IsCancellationRequested).ToList();
82-
}
83-
8475
public IStubCache StubCache { get; protected set; }
8576

8677
public IPythonModule GetImportedModule(string name)
@@ -99,10 +90,7 @@ public IPythonModule GetOrLoadModule(string name) {
9990
moduleRef = Modules.GetOrAdd(name, new ModuleRef());
10091
return moduleRef.GetOrCreate(name, this);
10192
}
102-
103-
public bool TryAddModulePath(in string path, in bool allowNonRooted, out string fullModuleName)
104-
=> PathResolver.TryAddModulePath(path, allowNonRooted, out fullModuleName);
105-
93+
10694
public ModulePath FindModule(string filePath) {
10795
var bestLibraryPath = string.Empty;
10896

@@ -117,8 +105,8 @@ public ModulePath FindModule(string filePath) {
117105
}
118106

119107
protected void ReloadModulePaths(in IEnumerable<string> rootPaths) {
120-
foreach (var modulePath in rootPaths.Where(Directory.Exists).SelectMany(p => PathUtils.EnumerateFiles(p))) {
121-
PathResolver.TryAddModulePath(modulePath, false, out _);
108+
foreach (var moduleFile in rootPaths.Where(Directory.Exists).SelectMany(p => PathUtils.EnumerateFiles(_fs, p))) {
109+
PathResolver.TryAddModulePath(moduleFile.FullName, moduleFile.Length, false, out _);
122110
}
123111
}
124112

‎src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs

Lines changed: 10 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,12 @@
2525
using Microsoft.Python.Analysis.Core.Interpreter;
2626
using Microsoft.Python.Analysis.Types;
2727
using Microsoft.Python.Core;
28+
using Microsoft.Python.Core.Collections;
2829
using Microsoft.Python.Core.IO;
2930

3031
namespace Microsoft.Python.Analysis.Modules.Resolution {
3132
internal sealed class TypeshedResolution : ModuleResolutionBase, IModuleResolution {
32-
private readonly IReadOnlyList<string> _typeStubPaths;
33+
private readonly ImmutableArray<string> _typeStubPaths;
3334

3435
public TypeshedResolution(IServiceContainer services) : base(null, services) {
3536
BuiltinsModule = _interpreter.ModuleResolution.BuiltinsModule;
@@ -41,7 +42,7 @@ public TypeshedResolution(IServiceContainer services) : base(null, services) {
4142
_typeStubPaths = GetTypeShedPaths(Root)
4243
.Concat(GetTypeShedPaths(stubs))
4344
.Where(services.GetService<IFileSystem>().DirectoryExists)
44-
.ToArray();
45+
.ToImmutableArray();
4546

4647
_log?.Log(TraceEventType.Verbose, @"Typeshed paths:");
4748
foreach (var p in _typeStubPaths) {
@@ -53,13 +54,13 @@ internal Task InitializeAsync(CancellationToken cancellationToken = default)
5354
=> ReloadAsync(cancellationToken);
5455

5556
protected override IPythonModule CreateModule(string name) {
56-
var mp = FindModuleInSearchPath(_typeStubPaths, null, name);
57-
if (mp != null) {
58-
if (mp.Value.IsCompiled) {
59-
_log?.Log(TraceEventType.Warning, "Unsupported native module in stubs", mp.Value.FullName, mp.Value.SourceFile);
57+
var moduleImport = CurrentPathResolver.GetModuleImportFromModuleName(name);
58+
if (moduleImport != default) {
59+
if (moduleImport.IsCompiled) {
60+
_log?.Log(TraceEventType.Warning, "Unsupported native module in stubs", moduleImport.FullName, moduleImport.ModulePath);
6061
return null;
6162
}
62-
return new StubPythonModule(mp.Value.FullName, mp.Value.SourceFile, true, _services);
63+
return new StubPythonModule(moduleImport.FullName, moduleImport.ModulePath, true, _services);
6364
}
6465

6566
var i = name.IndexOf('.');
@@ -74,14 +75,8 @@ protected override IPythonModule CreateModule(string name) {
7475

7576
public Task ReloadAsync(CancellationToken cancellationToken = default) {
7677
Modules.Clear();
77-
PathResolver = new PathResolver(_interpreter.LanguageVersion);
78-
79-
var addedRoots = PathResolver.SetRoot(Root);
80-
ReloadModulePaths(addedRoots);
81-
82-
addedRoots = PathResolver.SetInterpreterSearchPaths(_typeStubPaths);
83-
ReloadModulePaths(addedRoots);
84-
78+
PathResolver = new PathResolver(_interpreter.LanguageVersion, Root, _typeStubPaths, ImmutableArray<string>.Empty);
79+
ReloadModulePaths(_typeStubPaths.Prepend(Root));
8580
cancellationToken.ThrowIfCancellationRequested();
8681
return Task.CompletedTask;
8782
}
@@ -109,41 +104,5 @@ private IEnumerable<string> GetTypeShedPaths(string typeshedRootPath) {
109104
yield return Path.Combine(thirdParty, subdir);
110105
}
111106
}
112-
113-
private ModulePath? FindModuleInSearchPath(IReadOnlyList<string> searchPaths, IReadOnlyDictionary<string, string> packages, string name) {
114-
if (searchPaths == null || searchPaths.Count == 0) {
115-
return null;
116-
}
117-
118-
var i = name.IndexOf('.');
119-
var firstBit = i < 0 ? name : name.Remove(i);
120-
121-
ModulePath mp;
122-
Func<string, bool> isPackage = IsPackage;
123-
if (firstBit.EndsWithOrdinal("-stubs", ignoreCase: true)) {
124-
isPackage = _fs.DirectoryExists;
125-
}
126-
127-
var requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(Configuration.Version);
128-
if (packages != null && packages.TryGetValue(firstBit, out var searchPath) && !string.IsNullOrEmpty(searchPath)) {
129-
if (ModulePath.FromBasePathAndName_NoThrow(searchPath, name, isPackage, null, requireInitPy, out mp, out _, out _, out _)) {
130-
return mp;
131-
}
132-
}
133-
134-
if (searchPaths.MaybeEnumerate()
135-
.Any(sp => ModulePath.FromBasePathAndName_NoThrow(sp, name, isPackage, null, requireInitPy, out mp, out _, out _, out _))) {
136-
return mp;
137-
}
138-
return null;
139-
}
140-
141-
/// <summary>
142-
/// Determines whether the specified directory is an importable package.
143-
/// </summary>
144-
private bool IsPackage(string directory)
145-
=> ModulePath.PythonVersionRequiresInitPyFiles(Configuration.Version) ?
146-
!string.IsNullOrEmpty(ModulePath.GetPackageInitPy(directory)) :
147-
_fs.DirectoryExists(directory);
148107
}
149108
}

‎src/Analysis/Ast/Test/ScrapeTests.cs

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
using Microsoft.Python.Analysis.Modules.Resolution;
2929
using Microsoft.Python.Analysis.Tests.FluentAssertions;
3030
using Microsoft.Python.Analysis.Types;
31+
using Microsoft.Python.Core;
3132
using Microsoft.Python.Core.IO;
3233
using Microsoft.Python.Parsing;
3334
using Microsoft.Python.Parsing.Ast;
@@ -78,6 +79,7 @@ private async Task CompiledBuiltinScrapeAsync(InterpreterConfiguration configura
7879

7980
var services = await CreateServicesAsync(moduleDirectory, configuration);
8081
var interpreter = services.GetService<IPythonInterpreter>();
82+
var fs = services.GetService<IFileSystem>();
8183

8284
// TODO: this is Windows only
8385
var dllsDir = Path.Combine(Path.GetDirectoryName(interpreter.Configuration.LibraryPath), "DLLs");
@@ -92,7 +94,7 @@ private async Task CompiledBuiltinScrapeAsync(InterpreterConfiguration configura
9294

9395
var modules = new List<IPythonModule>();
9496

95-
foreach (var pyd in PathUtils.EnumerateFiles(dllsDir, "*", recurse: false).Where(ModulePath.IsPythonFile)) {
97+
foreach (var pyd in PathUtils.EnumerateFiles(fs, dllsDir, "*", recurse: false).Select(f => f.FullName).Where(ModulePath.IsPythonFile)) {
9698
var mp = ModulePath.FromFullPath(pyd);
9799
if (mp.IsDebug) {
98100
continue;
@@ -270,7 +272,11 @@ private async Task FullStdLibTest(InterpreterConfiguration configuration, params
270272
var services = await CreateServicesAsync(moduleDirectory, configuration);
271273
var interpreter = services.GetService<IPythonInterpreter>();
272274

273-
var modules = ModulePath.GetModulesInLib(configuration.LibraryPath, configuration.SitePackagesPath).ToList();
275+
var pathResolver = interpreter.ModuleResolution.CurrentPathResolver;
276+
var modules = pathResolver.GetAllModuleNames()
277+
.Select(n => pathResolver.GetModuleImportFromModuleName(n))
278+
.Where(i => i.RootPath.PathEquals(configuration.SitePackagesPath) || i.RootPath.PathEquals(configuration.LibraryPath))
279+
.ToList();
274280

275281
var skip = new HashSet<string>(skipModules);
276282
skip.UnionWith(new[] {
@@ -294,12 +300,12 @@ private async Task FullStdLibTest(InterpreterConfiguration configuration, params
294300
}
295301

296302
var set = modules
297-
.Where(m => !skip.Contains(m.ModuleName))
303+
.Where(m => !skip.Contains(m.FullName))
298304
.GroupBy(m => {
299305
var i = m.FullName.IndexOf('.');
300306
return i <= 0 ? m.FullName : m.FullName.Remove(i);
301307
})
302-
.SelectMany(g => g.Select(m => Tuple.Create(m, m.ModuleName)))
308+
.SelectMany(g => g.Select(m => Tuple.Create(m, m.FullName)))
303309
.ToArray();
304310
set = set.Where(x => x.Item2 != null && x.Item2.Contains("grammar")).ToArray();
305311

@@ -317,22 +323,22 @@ private async Task FullStdLibTest(InterpreterConfiguration configuration, params
317323
var modName = r.Item1;
318324
var mod = interpreter.ModuleResolution.GetOrLoadModule(r.Item2);
319325

320-
anyExtensionSeen |= modName.IsNativeExtension;
326+
anyExtensionSeen |= modName.IsCompiled;
321327
switch (mod) {
322328
case null:
323-
Trace.TraceWarning("failed to import {0} from {1}", modName.ModuleName, modName.SourceFile);
329+
Trace.TraceWarning("failed to import {0} from {1}", modName.FullName, modName.ModulePath);
324330
break;
325331
case CompiledPythonModule compiledPythonModule:
326332
var errors = compiledPythonModule.GetParseErrors().ToArray();
327333
if (errors.Any()) {
328334
anyParseError = true;
329-
Trace.TraceError("Parse errors in {0}", modName.SourceFile);
335+
Trace.TraceError("Parse errors in {0}", modName.ModulePath);
330336
foreach (var e in errors) {
331337
Trace.TraceError(e.Message);
332338
}
333339
} else {
334340
anySuccess = true;
335-
anyExtensionSuccess |= modName.IsNativeExtension;
341+
anyExtensionSuccess |= modName.IsCompiled;
336342
}
337343

338344
break;
@@ -343,13 +349,13 @@ private async Task FullStdLibTest(InterpreterConfiguration configuration, params
343349
if (!mod.FilePath.Contains("site-packages")) {
344350
anyParseError = true;
345351
}
346-
Trace.TraceError("Parse errors in {0}", modName.SourceFile);
352+
Trace.TraceError("Parse errors in {0}", modName.ModulePath);
347353
foreach (var e in filteredErrors) {
348354
Trace.TraceError(e.Message);
349355
}
350356
} else {
351357
anySuccess = true;
352-
anyExtensionSuccess |= modName.IsNativeExtension;
358+
anyExtensionSuccess |= modName.IsCompiled;
353359
}
354360

355361
break;

‎src/Analysis/Core/Impl/DependencyResolution/IImportSearchResult.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
namespace Microsoft.Python.Analysis.Core.DependencyResolution {
1919
public interface IImportSearchResult {}
2020

21-
public interface IImportChildrenSource {
21+
public interface IImportChildrenSource : IImportSearchResult {
2222
ImmutableArray<string> GetChildrenNames();
2323
bool TryGetChildImport(string name, out IImportSearchResult child);
2424
}

‎src/Analysis/Core/Impl/DependencyResolution/ImplicitPackageImport.cs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,10 @@
1313
// See the Apache Version 2.0 License for specific language governing
1414
// permissions and limitations under the License.
1515

16-
using System.Collections.Generic;
1716
using Microsoft.Python.Core.Collections;
1817

1918
namespace Microsoft.Python.Analysis.Core.DependencyResolution {
20-
public class ImplicitPackageImport : IImportSearchResult, IImportChildrenSource {
19+
public class ImplicitPackageImport : IImportChildrenSource {
2120
private readonly IImportChildrenSource _childrenSource;
2221

2322
public string Name { get; }

0 commit comments

Comments
 (0)