diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs index 2b9182661..b13891861 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using Microsoft.Python.Analysis.Dependencies; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { /// @@ -30,6 +31,11 @@ internal interface IAnalyzable { /// void NotifyAnalysisBegins(); + /// + /// Performs standard analysis pass. Does not include any restoration from databases. + /// + ModuleWalker Analyze(PythonAst ast); + /// /// Notifies document that its analysis is now complete. /// diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index caeb42dca..d52d21d29 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -527,11 +527,11 @@ private IDocumentAnalysis RestoreOrAnalyzeModule(IDependencyChainSingleNode _hiddenNames.Contains(name) ? null : base.GetMember(name); public IMember GetAnyMember(string name) => base.GetMember(name); public override IEnumerable GetMemberNames() => base.GetMemberNames().Except(_hiddenNames).ToArray(); + #endregion + + public void Initialize() => ParseAndLogExceptions(CancellationToken.None); protected override string[] GetScrapeArguments(IPythonInterpreter interpreter) => !InstallPath.TryGetFile("scrape_module.py", out var sb) ? null : new[] { "-W", "ignore", "-B", "-E", sb }; + protected override void Parse() { } + + protected override void Analyze(PythonAst ast, int version) { + NotifyAnalysisBegins(); + var walker = Analyze(ast); + var analysis = new DocumentAnalysis(this, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); + NotifyAnalysisComplete(analysis); + } + protected override void OnAnalysisComplete() { SpecializeTypes(); SpecializeFunctions(); diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index d1463ef62..ba9b118bb 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -22,10 +22,11 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Analyzer.Handlers; using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; @@ -305,7 +306,7 @@ public void Invalidate() { Services.GetService().InvalidateAnalysis(this); } - private void Parse() { + protected virtual void Parse() { _parseCts?.Cancel(); _parseCts = new CancellationTokenSource(); @@ -316,7 +317,7 @@ private void Parse() { _parsingTask = Task.Run(() => ParseAndLogExceptions(_linkedParseCts.Token), _linkedParseCts.Token); } - private void ParseAndLogExceptions(CancellationToken cancellationToken) { + protected void ParseAndLogExceptions(CancellationToken cancellationToken) { try { Parse(cancellationToken); } catch (Exception ex) when (!(ex is OperationCanceledException)) { @@ -325,6 +326,15 @@ private void ParseAndLogExceptions(CancellationToken cancellationToken) { } } + protected virtual void Analyze(PythonAst ast, int version) { + if (ContentState < State.Analyzing) { + ContentState = State.Analyzing; + + var analyzer = Services.GetService(); + analyzer.EnqueueDocumentForAnalysis(this, ast, version); + } + } + private void Parse(CancellationToken cancellationToken) { CollectingErrorSink sink = null; int version; @@ -345,7 +355,6 @@ private void Parse(CancellationToken cancellationToken) { } var ast = parser.ParseFile(Uri); - // Log?.Log(TraceEventType.Verbose, $"Parse complete: {Name} ({ModuleType})"); lock (_syncObj) { @@ -370,13 +379,7 @@ private void Parse(CancellationToken cancellationToken) { } NewAst?.Invoke(this, EventArgs.Empty); - - if (ContentState < State.Analyzing) { - ContentState = State.Analyzing; - - var analyzer = Services.GetService(); - analyzer.EnqueueDocumentForAnalysis(this, ast, version); - } + Analyze(ast, version); lock (_syncObj) { _parsingTask = null; @@ -423,6 +426,14 @@ public void NotifyAnalysisBegins() { } } + public ModuleWalker Analyze(PythonAst ast) { + var eval = new ExpressionEval(Services, this, ast); + var walker = new ModuleWalker(eval, SimpleImportedVariableHandler.Instance); + ast.Walk(walker); + walker.Complete(); + return walker; + } + public void NotifyAnalysisComplete(IDocumentAnalysis analysis) { lock (_syncObj) { if (analysis.Version < Analysis.Version) { diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index f138c820b..12aa1e184 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -39,6 +39,7 @@ namespace Microsoft.Python.Analysis.Modules.Resolution { internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManagement { private readonly ConcurrentDictionary _specialized = new ConcurrentDictionary(); private readonly IUIService _ui; + private BuiltinsPythonModule _builtins; private IModuleDatabaseService _dbService; private IRunningDocumentTable _rdt; @@ -53,7 +54,7 @@ public MainModuleResolution(string root, IServiceContainer services, ImmutableAr public string BuiltinModuleName => BuiltinTypeId.Unknown.GetModuleName(Interpreter.LanguageVersion); public ImmutableArray LibraryPaths { get; private set; } = ImmutableArray.Empty; - public IBuiltinsPythonModule BuiltinsModule { get; private set; } + public IBuiltinsPythonModule BuiltinsModule => _builtins; public IEnumerable GetImportedModules(CancellationToken cancellationToken) { foreach (var module in _specialized.Values) { @@ -185,12 +186,8 @@ public IPythonModule GetSpecializedModule(string fullName, bool allowCreation = public bool IsSpecializedModule(string fullName, string modulePath = null) => _specialized.ContainsKey(fullName); - private async Task AddBuiltinTypesToPathResolverAsync(CancellationToken cancellationToken = default) { - var analyzer = Services.GetService(); - await analyzer.GetAnalysisAsync(BuiltinsModule, Timeout.Infinite, cancellationToken); - + private void AddBuiltinTypesToPathResolver() { Check.InvalidOperation(!(BuiltinsModule.Analysis is EmptyAnalysis), "Builtins analysis did not complete correctly."); - // Add built-in module names var builtinModuleNamesMember = BuiltinsModule.GetAnyMember("__builtin_module_names__"); var value = builtinModuleNamesMember is IVariable variable ? variable.Value : builtinModuleNamesMember; @@ -222,16 +219,16 @@ public async Task ReloadAsync(CancellationToken cancellationToken = default) { ReloadModulePaths(addedRoots, cancellationToken); if (!builtinsIsCreated) { - var builtinsModule = CreateBuiltinsModule(Services, Interpreter, StubCache); - BuiltinsModule = builtinsModule; - builtinsRef = new ModuleRef(builtinsModule); + _builtins = CreateBuiltinsModule(Services, Interpreter, StubCache); + builtinsRef = new ModuleRef(_builtins); + _builtins.Initialize(); } Modules[BuiltinModuleName] = builtinsRef; - await AddBuiltinTypesToPathResolverAsync(cancellationToken); + AddBuiltinTypesToPathResolver(); } - private static IBuiltinsPythonModule CreateBuiltinsModule(IServiceContainer services, IPythonInterpreter interpreter, IStubCache stubCache) { + private static BuiltinsPythonModule CreateBuiltinsModule(IServiceContainer services, IPythonInterpreter interpreter, IStubCache stubCache) { var moduleName = BuiltinTypeId.Unknown.GetModuleName(interpreter.LanguageVersion); var modulePath = stubCache.GetCacheFilePath(interpreter.Configuration.InterpreterPath); return new BuiltinsPythonModule(moduleName, modulePath, services);