diff --git a/build/Common.Build.Core.settings b/build/Common.Build.Core.settings
index cb16b74fa..a38d4dbea 100644
--- a/build/Common.Build.Core.settings
+++ b/build/Common.Build.Core.settings
@@ -16,18 +16,10 @@
10.0
16.0
15.0
- 14.0
- 12.0
- 11.0
- 10.0
-
+
2019
2017
- 2015
- 2013
- 2012
- 2010
-
+
$(BUILD_BUILDNUMBER)
1000.00
+ 1701;1702;1998;$(NoWarn)
+ 7.2
+
+
+
+
+
+ all
+ runtime; build; native; contentfiles; analyzers
+
+
+
+
+ PreserveNewest
+
+
+ PreserveNewest
+
+
+ PreserveNewest
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ True
+ True
+ Resources.resx
+
+
+
+
+ ResXFileCodeGenerator
+ Resources.Designer.cs
+
+
+
+
+
+
+
+
diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs
new file mode 100644
index 000000000..334a68d12
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs
@@ -0,0 +1,157 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Python.Analysis.Specializations;
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Core;
+using Microsoft.Python.Core.IO;
+using Microsoft.Python.Parsing;
+
+namespace Microsoft.Python.Analysis.Modules {
+ ///
+ /// Represents builtins module. Typically generated for a given Python interpreter
+ /// by running special 'scraper' Python script that generates Python code via
+ /// introspection of the compiled built-in language types.
+ ///
+ internal sealed class BuiltinsPythonModule : CompiledPythonModule, IBuiltinsPythonModule {
+ private readonly HashSet _hiddenNames = new HashSet();
+ private IPythonType _boolType;
+
+ public BuiltinsPythonModule(string moduleName, string filePath, IServiceContainer services)
+ : base(moduleName, ModuleType.Builtins, filePath, null, services, ModuleLoadOptions.None) { } // TODO: builtins stub
+
+ public override IMember GetMember(string name) => _hiddenNames.Contains(name) ? null : base.GetMember(name);
+
+ public IMember GetAnyMember(string name) => base.GetMember(name);
+
+ public override IEnumerable GetMemberNames() => base.GetMemberNames().Except(_hiddenNames).ToArray();
+
+ protected override IEnumerable GetScrapeArguments(IPythonInterpreter interpreter)
+ => !InstallPath.TryGetFile("scrape_module.py", out var sb) ? null : new List { "-B", "-E", sb };
+
+ protected override void OnAnalysisComplete() {
+ lock (AnalysisLock) {
+ SpecializeTypes();
+ SpecializeFunctions();
+ }
+ }
+
+ private void SpecializeTypes() {
+ IPythonType noneType = null;
+ var isV3 = Interpreter.LanguageVersion.Is3x();
+
+ foreach (BuiltinTypeId typeId in Enum.GetValues(typeof(BuiltinTypeId))) {
+ var m = GetMember("__{0}__".FormatInvariant(typeId));
+ if (!(m is PythonType biType && biType.IsBuiltin)) {
+ continue;
+ }
+
+ if (biType.IsHidden) {
+ _hiddenNames.Add(biType.Name);
+ }
+
+ _hiddenNames.Add("__{0}__".FormatInvariant(typeId));
+
+ // In V2 Unicode string is 'Unicode' and regular string is 'Str' or 'Bytes'.
+ // In V3 Unicode and regular strings are 'Str' and ASCII/byte string is 'Bytes'.
+ switch (typeId) {
+ case BuiltinTypeId.Bytes: {
+ var id = !isV3 ? BuiltinTypeId.Str : BuiltinTypeId.Bytes;
+ biType.TrySetTypeId(id);
+ biType.AddMember(@"__iter__", BuiltinsSpecializations.__iter__(Interpreter, id), true);
+ break;
+ }
+ case BuiltinTypeId.BytesIterator: {
+ biType.TrySetTypeId(!isV3 ? BuiltinTypeId.StrIterator : BuiltinTypeId.BytesIterator);
+ break;
+ }
+ case BuiltinTypeId.Unicode: {
+ var id = isV3 ? BuiltinTypeId.Str : BuiltinTypeId.Unicode;
+ biType.TrySetTypeId(id);
+ biType.AddMember(@"__iter__", BuiltinsSpecializations.__iter__(Interpreter, id), true);
+ break;
+ }
+ case BuiltinTypeId.UnicodeIterator: {
+ biType.TrySetTypeId(isV3 ? BuiltinTypeId.StrIterator : BuiltinTypeId.UnicodeIterator);
+ break;
+ }
+ case BuiltinTypeId.Str: {
+ biType.AddMember(@"__iter__", BuiltinsSpecializations.__iter__(Interpreter, typeId), true);
+ }
+ break;
+ default:
+ biType.TrySetTypeId(typeId);
+ switch (typeId) {
+ case BuiltinTypeId.Bool:
+ _boolType = _boolType ?? biType;
+ break;
+ case BuiltinTypeId.NoneType:
+ noneType = noneType ?? biType;
+ break;
+ }
+ break;
+ }
+ }
+
+ _hiddenNames.Add("__builtin_module_names__");
+
+ if (_boolType != null) {
+ Analysis.GlobalScope.DeclareVariable("True", _boolType, LocationInfo.Empty);
+ Analysis.GlobalScope.DeclareVariable("False", _boolType, LocationInfo.Empty);
+ }
+
+ if (noneType != null) {
+ Analysis.GlobalScope.DeclareVariable("None", noneType, LocationInfo.Empty);
+ }
+
+ foreach (var n in GetMemberNames()) {
+ var t = GetMember(n).GetPythonType();
+ if (t.TypeId == BuiltinTypeId.Unknown && t.MemberType != PythonMemberType.Unknown) {
+ (t as PythonType)?.TrySetTypeId(BuiltinTypeId.Type);
+ }
+ }
+ }
+
+ private void SpecializeFunctions() {
+ // TODO: deal with commented out functions.
+ SpecializeFunction("abs", BuiltinsSpecializations.Identity);
+ SpecializeFunction("cmp", Interpreter.GetBuiltinType(BuiltinTypeId.Int));
+ SpecializeFunction("dir", BuiltinsSpecializations.ListOfStrings);
+ SpecializeFunction("eval", Interpreter.GetBuiltinType(BuiltinTypeId.Object));
+ SpecializeFunction("globals", BuiltinsSpecializations.DictStringToObject);
+ SpecializeFunction(@"isinstance", _boolType);
+ SpecializeFunction(@"issubclass", _boolType);
+ SpecializeFunction(@"iter", BuiltinsSpecializations.Iterator);
+ SpecializeFunction("locals", BuiltinsSpecializations.DictStringToObject);
+ //SpecializeFunction(_builtinName, "max", ReturnUnionOfInputs);
+ //SpecializeFunction(_builtinName, "min", ReturnUnionOfInputs);
+ SpecializeFunction("next", BuiltinsSpecializations.Next);
+ //SpecializeFunction(_builtinName, "open", SpecialOpen);
+ SpecializeFunction("ord", Interpreter.GetBuiltinType(BuiltinTypeId.Int));
+ SpecializeFunction("pow", BuiltinsSpecializations.Identity);
+ SpecializeFunction("range", BuiltinsSpecializations.Range);
+ SpecializeFunction("type", BuiltinsSpecializations.TypeInfo);
+
+ //SpecializeFunction(_builtinName, "range", RangeConstructor);
+ //SpecializeFunction(_builtinName, "sorted", ReturnsListOfInputIterable);
+ SpecializeFunction("sum", BuiltinsSpecializations.CollectionItem);
+ //SpecializeFunction(_builtinName, "super", SpecialSuper);
+ SpecializeFunction("vars", BuiltinsSpecializations.DictStringToObject);
+ }
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs b/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs
new file mode 100644
index 000000000..aceff65f8
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs
@@ -0,0 +1,47 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Collections.Generic;
+using System.IO;
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Core;
+using Microsoft.Python.Core.IO;
+
+namespace Microsoft.Python.Analysis.Modules {
+ ///
+ /// Represents compiled module that is built into the language.
+ ///
+ internal sealed class CompiledBuiltinPythonModule : CompiledPythonModule {
+ public CompiledBuiltinPythonModule(string moduleName, IPythonModule stub, IServiceContainer services)
+ : base(moduleName, ModuleType.Compiled, MakeFakeFilePath(moduleName, services), stub, services) { }
+
+ protected override IEnumerable GetScrapeArguments(IPythonInterpreter interpreter)
+ => !InstallPath.TryGetFile("scrape_module.py", out var sm)
+ ? null : new List { "-B", "-E", sm, "-u8", Name };
+
+ private static string MakeFakeFilePath(string name, IServiceContainer services) {
+ var interpreterPath = services.GetService().Configuration.InterpreterPath;
+
+ if (string.IsNullOrEmpty(interpreterPath)) {
+ return "python.{0}.exe".FormatInvariant(name);
+ }
+ var ext = Path.GetExtension(interpreterPath);
+ if (ext.Length > 0) { // Typically Windows, make python.exe into python.name.exe
+ return Path.ChangeExtension(interpreterPath, name) + ext;
+ }
+ return $"{interpreterPath}.{name}.exe"; // Fake the extension
+ }
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs b/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs
new file mode 100644
index 000000000..acfd75cba
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs
@@ -0,0 +1,109 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Text;
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Core;
+using Microsoft.Python.Core.IO;
+using Microsoft.Python.Core.OS;
+
+namespace Microsoft.Python.Analysis.Modules {
+ internal class CompiledPythonModule : PythonModule {
+ protected IModuleCache ModuleCache => Interpreter.ModuleResolution.ModuleCache;
+
+ public CompiledPythonModule(string moduleName, ModuleType moduleType, string filePath, IPythonModule stub,
+ IServiceContainer services, ModuleLoadOptions options = ModuleLoadOptions.Analyze)
+ : base(moduleName, filePath, moduleType, options, stub, services) { }
+
+ public override string Documentation
+ => GetMember("__doc__").TryGetConstant(out var s) ? s : string.Empty;
+
+ protected virtual IEnumerable GetScrapeArguments(IPythonInterpreter interpreter) {
+ var args = new List { "-B", "-E" };
+
+ var mp = Interpreter.ModuleResolution.FindModule(FilePath);
+ if (string.IsNullOrEmpty(mp.FullName)) {
+ return null;
+ }
+
+ if (!InstallPath.TryGetFile("scrape_module.py", out var sm)) {
+ return null;
+ }
+
+ args.Add(sm);
+ args.Add("-u8");
+ args.Add(mp.ModuleName);
+ args.Add(mp.LibraryPath);
+
+ return args;
+ }
+
+ protected override string LoadContent(ModuleLoadOptions options) {
+ var code = string.Empty;
+ if ((options & ModuleLoadOptions.Load) == ModuleLoadOptions.Load) {
+ code = ModuleCache.ReadCachedModule(FilePath);
+ if (string.IsNullOrEmpty(code)) {
+ if (!FileSystem.FileExists(Interpreter.Configuration.InterpreterPath)) {
+ return string.Empty;
+ }
+
+ code = ScrapeModule();
+ SaveCachedCode(code);
+ }
+ }
+ return code;
+ }
+
+ protected virtual void SaveCachedCode(string code) => ModuleCache.WriteCachedModule(FilePath, code);
+
+ private string ScrapeModule() {
+ var args = GetScrapeArguments(Interpreter);
+ if (args == null) {
+ return string.Empty;
+ }
+
+ var sb = new StringBuilder();
+ using (var proc = new ProcessHelper(
+ Interpreter.Configuration.InterpreterPath,
+ args,
+ Interpreter.Configuration.LibraryPath
+ )) {
+ proc.StartInfo.StandardOutputEncoding = Encoding.UTF8;
+ proc.OnOutputLine = s => sb.AppendLine(s);
+ proc.OnErrorLine = s => Log?.Log(TraceEventType.Error, "Scrape", s);
+
+ Log?.Log(TraceEventType.Information, "Scrape", proc.FileName, proc.Arguments);
+
+ proc.Start();
+ var exitCode = proc.Wait(60000);
+
+ if (exitCode == null) {
+ proc.Kill();
+ Log?.Log(TraceEventType.Error, "ScrapeTimeout", proc.FileName, proc.Arguments);
+ return string.Empty;
+ }
+
+ if (exitCode != 0) {
+ Log?.Log(TraceEventType.Error, "Scrape", "ExitCode", exitCode);
+ return string.Empty;
+ }
+ }
+
+ return sb.ToString();
+ }
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleCache.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleCache.cs
new file mode 100644
index 000000000..6d735aadf
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleCache.cs
@@ -0,0 +1,28 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Python.Analysis.Documents;
+
+namespace Microsoft.Python.Analysis.Modules {
+ public interface IModuleCache {
+ string SearchPathCachePath { get; }
+ Task ImportFromCacheAsync(string name, CancellationToken cancellationToken);
+ string GetCacheFilePath(string filePath);
+ string ReadCachedModule(string filePath);
+ void WriteCachedModule(string filePath, string code);
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs
new file mode 100644
index 000000000..c896bc270
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs
@@ -0,0 +1,77 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Python.Analysis.Core.DependencyResolution;
+using Microsoft.Python.Analysis.Core.Interpreter;
+using Microsoft.Python.Analysis.Types;
+
+namespace Microsoft.Python.Analysis.Modules {
+ public interface IModuleResolution {
+ string BuiltinModuleName { get; }
+ Task> GetSearchPathsAsync(CancellationToken cancellationToken = default);
+ Task> GetImportableModulesAsync(CancellationToken cancellationToken = default);
+ Task> GetImportableModulesAsync(IEnumerable searchPaths, CancellationToken cancellationToken = default);
+ ModulePath FindModule(string filePath);
+ IReadOnlyCollection GetPackagesFromDirectory(string searchPath, CancellationToken cancellationToken = default);
+
+ ///
+ /// Determines if directory contains Python package.
+ ///
+ bool IsPackage(string directory);
+
+ ///
+ /// Path resolver providing file resolution in module imports.
+ ///
+ PathResolverSnapshot CurrentPathResolver { get; }
+
+ ///
+ /// Returns an IPythonModule for a given module name. Returns null if
+ /// the module does not exist. The import is performed asynchronously.
+ ///
+ Task ImportModuleAsync(string name, CancellationToken cancellationToken = default);
+
+ ///
+ /// Builtins module.
+ ///
+ IBuiltinsPythonModule BuiltinsModule { get; }
+
+ IModuleCache ModuleCache { get; }
+
+ Task ReloadAsync(CancellationToken token = default);
+
+ void AddModulePath(string path);
+
+ ///
+ /// Provides ability to specialize module by replacing module import by
+ /// implementation in code. Real module
+ /// content is loaded and analyzed only for class/functions definitions
+ /// so the original documentation can be extracted.
+ ///
+ /// Module to specialize.
+ /// Specialized module constructor.
+ /// Cancellation token.
+ /// Original (library) module loaded as stub.
+ Task SpecializeModuleAsync(string name, Func specializationConstructor, CancellationToken cancellationToken = default);
+
+ ///
+ /// Returns specialized module, if any.
+ ///
+ IPythonModule GetSpecializedModule(string name);
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs
new file mode 100644
index 000000000..7731b29be
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs
@@ -0,0 +1,56 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using Microsoft.Python.Analysis.Types;
+
+namespace Microsoft.Python.Analysis.Modules {
+ public sealed class ModuleCreationOptions {
+ ///
+ /// The name of the module.
+ ///
+ public string ModuleName { get; set; }
+
+ ///
+ /// Module content. Can be null if file path or URI are provided.
+ ///
+ public string Content { get; set; }
+
+ ///
+ /// The path to the file on disk. Can be null if URI is provided.
+ ///
+ public string FilePath { get; set; }
+
+ ///
+ /// Document URI. Can be null if file path is provided.
+ ///
+ public Uri Uri { get; set; }
+
+ ///
+ /// Module type (user, library, compiled, stub, ...).
+ ///
+ public ModuleType ModuleType { get; set; } = ModuleType.User;
+
+ ///
+ /// Module stub, if any.
+ ///
+ public IPythonModule Stub { get; set; }
+
+ ///
+ /// Module loading options.
+ ///
+ public ModuleLoadOptions LoadOptions { get; set; } = ModuleLoadOptions.Analyze;
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleLoadOptions.cs b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleLoadOptions.cs
new file mode 100644
index 000000000..0fa7c340c
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleLoadOptions.cs
@@ -0,0 +1,47 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+
+namespace Microsoft.Python.Analysis.Modules {
+ [Flags]
+ public enum ModuleLoadOptions {
+ ///
+ /// Do nothing. Typically this is a placeholder or empty module.
+ ///
+ None,
+
+ ///
+ /// Just load the document, do not parse or analyze.
+ ///
+ Load = 1,
+
+ ///
+ /// Load and parse. Do not analyze.
+ ///
+ Ast = Load | 2,
+
+ ///
+ /// Load, parse and analyze.
+ ///
+ Analyze = Ast | 4,
+
+ ///
+ /// The document is opened in the editor.
+ /// This implies Ast and Analysis.
+ ///
+ Open = Analyze | 8
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs
new file mode 100644
index 000000000..b017e0708
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs
@@ -0,0 +1,66 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+
+namespace Microsoft.Python.Analysis.Modules {
+ [Flags]
+ public enum ModuleType {
+ ///
+ /// Module is user file in the workspace.
+ ///
+ User,
+
+ ///
+ /// Module is library module in Python.
+ ///
+ Library,
+
+ ///
+ /// Module is a stub module.
+ ///
+ Stub,
+
+ ///
+ /// Module source was scraped from a compiled module.
+ ///
+ Compiled,
+
+ ///
+ /// Module source was scraped from a built-in compiled module.
+ ///
+ CompiledBuiltin,
+
+ ///
+ /// Module is the Python 'builtins' module.
+ ///
+ Builtins,
+
+ ///
+ /// Module that contains child modules
+ ///
+ Package,
+
+ ///
+ /// Unresolved import.
+ ///
+ Unresolved,
+
+ ///
+ /// Module that is implemented inside the analyzer, such as 'typing'.
+ ///
+ Specialized
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Modules/FallbackBuiltinModule.cs b/src/Analysis/Ast/Impl/Modules/FallbackBuiltinModule.cs
new file mode 100644
index 000000000..6d54c2092
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Modules/FallbackBuiltinModule.cs
@@ -0,0 +1,74 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Parsing;
+
+namespace Microsoft.Python.Analysis.Modules {
+ internal sealed class FallbackBuiltinsModule : PythonModule, IBuiltinsPythonModule {
+ public readonly PythonLanguageVersion LanguageVersion;
+ private readonly Dictionary _cachedInstances;
+
+ public FallbackBuiltinsModule(PythonLanguageVersion version)
+ : base(BuiltinTypeId.Unknown.GetModuleName(version), ModuleType.Builtins, null) {
+ LanguageVersion = version;
+ _cachedInstances = new Dictionary();
+ }
+
+ private IPythonType GetOrCreate(BuiltinTypeId typeId) {
+ if (typeId.IsVirtualId()) {
+ switch (typeId) {
+ case BuiltinTypeId.Str:
+ typeId = BuiltinTypeId.Str;
+ break;
+ case BuiltinTypeId.StrIterator:
+ typeId = BuiltinTypeId.StrIterator;
+ break;
+ default:
+ typeId = BuiltinTypeId.Unknown;
+ break;
+ }
+ }
+
+ lock (_cachedInstances) {
+ if (!_cachedInstances.TryGetValue(typeId, out var value)) {
+ _cachedInstances[typeId] = value = new FallbackBuiltinPythonType(this, typeId);
+ }
+ return value;
+ }
+ }
+
+ public IMember GetAnyMember(string name) {
+ foreach (BuiltinTypeId typeId in Enum.GetValues(typeof(BuiltinTypeId))) {
+ if (typeId.GetTypeName(LanguageVersion) == name) {
+ return GetOrCreate(typeId);
+ }
+ }
+ return GetOrCreate(BuiltinTypeId.Unknown);
+ }
+ }
+
+ class FallbackBuiltinPythonType : PythonType {
+ public FallbackBuiltinPythonType(FallbackBuiltinsModule declaringModule, BuiltinTypeId typeId) :
+ base(typeId.GetModuleName(declaringModule.LanguageVersion), declaringModule, declaringModule.Documentation, null) {
+ TypeId = typeId;
+ }
+
+ public override PythonMemberType MemberType => PythonMemberType.Class;
+ public override BuiltinTypeId TypeId { get; }
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Modules/ModuleCache.cs b/src/Analysis/Ast/Impl/Modules/ModuleCache.cs
new file mode 100644
index 000000000..b9d19afb0
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Modules/ModuleCache.cs
@@ -0,0 +1,159 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Security.Cryptography;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Python.Analysis.Documents;
+using Microsoft.Python.Core;
+using Microsoft.Python.Core.IO;
+using Microsoft.Python.Core.Logging;
+
+namespace Microsoft.Python.Analysis.Modules {
+ internal sealed class ModuleCache : IModuleCache {
+ private readonly IServiceContainer _services;
+ private readonly IPythonInterpreter _interpreter;
+ private readonly IFileSystem _fs;
+ private readonly ILogger _log;
+ private readonly bool _skipCache;
+ private bool _loggedBadDbPath;
+
+ private string ModuleCachePath => _interpreter.Configuration.ModuleCachePath;
+
+ public ModuleCache(IPythonInterpreter interpreter, IServiceContainer services) {
+ _interpreter = interpreter;
+ _services = services;
+ _fs = services.GetService();
+ _log = services.GetService();
+ _skipCache = string.IsNullOrEmpty(_interpreter.Configuration.ModuleCachePath);
+ SearchPathCachePath = Path.Combine(_interpreter.Configuration.ModuleCachePath, "database.path");
+ }
+
+ public string SearchPathCachePath { get; }
+
+ public async Task ImportFromCacheAsync(string name, CancellationToken cancellationToken) {
+ if (string.IsNullOrEmpty(ModuleCachePath)) {
+ return null;
+ }
+
+ var cache = GetCacheFilePath("python.{0}.pyi".FormatInvariant(name));
+ if (!_fs.FileExists(cache)) {
+ cache = GetCacheFilePath("python._{0}.pyi".FormatInvariant(name));
+ if (!_fs.FileExists(cache)) {
+ cache = GetCacheFilePath("{0}.pyi".FormatInvariant(name));
+ if (!_fs.FileExists(cache)) {
+ return null;
+ }
+ }
+ }
+
+ var rdt = _services.GetService();
+ var mco = new ModuleCreationOptions {
+ ModuleName = name,
+ ModuleType = ModuleType.Stub,
+ FilePath = cache,
+ LoadOptions = ModuleLoadOptions.Analyze
+ };
+ var module = rdt.AddModule(mco);
+
+ await module.LoadAndAnalyzeAsync(cancellationToken).ConfigureAwait(false);
+ return module;
+ }
+
+ public string GetCacheFilePath(string filePath) {
+ if (string.IsNullOrEmpty(filePath) || !PathEqualityComparer.IsValidPath(ModuleCachePath)) {
+ if (!_loggedBadDbPath) {
+ _loggedBadDbPath = true;
+ _log?.Log(TraceEventType.Warning, $"Invalid module cache path: {ModuleCachePath}");
+ }
+ return null;
+ }
+
+ var name = PathUtils.GetFileName(filePath);
+ if (!PathEqualityComparer.IsValidPath(name)) {
+ _log?.Log(TraceEventType.Warning, $"Invalid cache name: {name}");
+ return null;
+ }
+ try {
+ var candidate = Path.ChangeExtension(Path.Combine(ModuleCachePath, name), ".pyi");
+ if (_fs.FileExists(candidate)) {
+ return candidate;
+ }
+ } catch (ArgumentException) {
+ return null;
+ }
+
+ var hash = SHA256.Create();
+ var dir = Path.GetDirectoryName(filePath) ?? string.Empty;
+ if (_fs.StringComparison == StringComparison.OrdinalIgnoreCase) {
+ dir = dir.ToLowerInvariant();
+ }
+
+ var dirHash = Convert.ToBase64String(hash.ComputeHash(new UTF8Encoding(false).GetBytes(dir)))
+ .Replace('/', '_').Replace('+', '-');
+
+ return Path.ChangeExtension(Path.Combine(
+ ModuleCachePath,
+ Path.Combine(dirHash, name)
+ ), ".pyi");
+ }
+
+ public string ReadCachedModule(string filePath) {
+ if (_skipCache) {
+ return string.Empty;
+ }
+
+ var path = GetCacheFilePath(filePath);
+ if (string.IsNullOrEmpty(path)) {
+ return string.Empty;
+ }
+
+ var fileIsOkay = false;
+ try {
+ var cacheTime = _fs.GetLastWriteTimeUtc(path);
+ var sourceTime = _fs.GetLastWriteTimeUtc(filePath);
+ if (sourceTime <= cacheTime) {
+ var assemblyTime = _fs.GetLastWriteTimeUtc(GetType().Assembly.Location);
+ if (assemblyTime <= cacheTime) {
+ fileIsOkay = true;
+ }
+ }
+ } catch (Exception ex) when (!ex.IsCriticalException()) {
+ }
+
+ if (fileIsOkay) {
+ try {
+ return _fs.ReadAllText(filePath);
+ } catch (IOException) { } catch (UnauthorizedAccessException) { }
+ }
+
+ _log?.Log(TraceEventType.Verbose, "Invalidate cached module", path);
+ _fs.DeleteFileWithRetries(path);
+ return string.Empty;
+ }
+
+ public void WriteCachedModule(string filePath, string code) {
+ var cache = GetCacheFilePath(filePath);
+ if (!string.IsNullOrEmpty(cache)) {
+ _log?.Log(TraceEventType.Verbose, "Write cached module: ", cache);
+ _fs.WriteTextWithRetry(cache, code);
+ }
+ }
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Modules/ModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/ModuleResolution.cs
new file mode 100644
index 000000000..9128af8e3
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Modules/ModuleResolution.cs
@@ -0,0 +1,475 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Python.Analysis.Core.DependencyResolution;
+using Microsoft.Python.Analysis.Core.Interpreter;
+using Microsoft.Python.Analysis.Documents;
+using Microsoft.Python.Analysis.Specializations;
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Core;
+using Microsoft.Python.Core.IO;
+using Microsoft.Python.Core.Logging;
+
+namespace Microsoft.Python.Analysis.Modules {
+ internal sealed class ModuleResolution : IModuleResolution {
+ private static readonly IReadOnlyDictionary _emptyModuleSet = EmptyDictionary.Instance;
+ private readonly ConcurrentDictionary _modules = new ConcurrentDictionary();
+ private readonly IReadOnlyList _typeStubPaths;
+ private readonly IServiceContainer _services;
+ private readonly IPythonInterpreter _interpreter;
+ private readonly IFileSystem _fs;
+ private readonly ILogger _log;
+ private readonly bool _requireInitPy;
+ private readonly string _root;
+
+ private PathResolver _pathResolver;
+ private IReadOnlyDictionary _searchPathPackages;
+ private IReadOnlyList _searchPaths;
+
+ private InterpreterConfiguration Configuration => _interpreter.Configuration;
+
+ public ModuleResolution(string root, IServiceContainer services) {
+ _root = root;
+ _services = services;
+ _interpreter = services.GetService();
+ _fs = services.GetService();
+ _log = services.GetService();
+
+ _requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(_interpreter.Configuration.Version);
+ // TODO: merge with user-provided stub paths
+ _typeStubPaths = GetTypeShedPaths(_interpreter.Configuration?.TypeshedPath).ToArray();
+ }
+
+ internal async Task LoadBuiltinTypesAsync(CancellationToken cancellationToken = default) {
+ // Add names from search paths
+ await ReloadAsync(cancellationToken);
+
+ // Initialize built-in
+ var moduleName = BuiltinTypeId.Unknown.GetModuleName(_interpreter.LanguageVersion);
+ var modulePath = ModuleCache.GetCacheFilePath(_interpreter.Configuration.InterpreterPath ?? "python.exe");
+
+ var b = new BuiltinsPythonModule(moduleName, modulePath, _services);
+ _modules[BuiltinModuleName] = BuiltinsModule = b;
+ await b.LoadAndAnalyzeAsync(cancellationToken);
+
+ // Add built-in module names
+ var builtinModuleNamesMember = BuiltinsModule.GetAnyMember("__builtin_module_names__");
+ if (builtinModuleNamesMember.TryGetConstant(out var s)) {
+ var builtinModuleNames = s.Split(',').Select(n => n.Trim());
+ _pathResolver.SetBuiltins(builtinModuleNames);
+ }
+ }
+
+ public IModuleCache ModuleCache { get; private set; }
+ public string BuiltinModuleName => BuiltinTypeId.Unknown.GetModuleName(_interpreter.LanguageVersion);
+
+ ///
+ /// Path resolver providing file resolution in module imports.
+ ///
+ public PathResolverSnapshot CurrentPathResolver => _pathResolver.CurrentSnapshot;
+
+ ///
+ /// Builtins module.
+ ///
+ public IBuiltinsPythonModule BuiltinsModule { get; private set; }
+
+ public async Task> GetImportableModulesAsync(CancellationToken cancellationToken) {
+ if (_searchPathPackages != null) {
+ return _searchPathPackages;
+ }
+
+ var packageDict = await GetImportableModulesAsync(Configuration.SearchPaths, cancellationToken).ConfigureAwait(false);
+ if (!packageDict.Any()) {
+ return _emptyModuleSet;
+ }
+
+ _searchPathPackages = packageDict;
+ return packageDict;
+ }
+
+ public async Task> GetSearchPathsAsync(CancellationToken cancellationToken = default) {
+ if (_searchPaths != null) {
+ return _searchPaths;
+ }
+
+ _searchPaths = await GetInterpreterSearchPathsAsync(cancellationToken).ConfigureAwait(false);
+ Debug.Assert(_searchPaths != null, "Should have search paths");
+ _searchPaths = _searchPaths.Concat(Configuration.SearchPaths ?? Array.Empty()).ToArray();
+ _log?.Log(TraceEventType.Information, "SearchPaths", _searchPaths.Cast