diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs index 8d4b9751d..2b9182661 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs @@ -13,11 +13,18 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using Microsoft.Python.Analysis.Dependencies; + namespace Microsoft.Python.Analysis.Analyzer { /// /// Represents document that can be analyzed asynchronously. /// internal interface IAnalyzable { + /// + /// Returns object that can calculate dependencies of this entry. + /// + IDependencyProvider DependencyProvider { get; } + /// /// Notifies document that analysis is about to begin. /// diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 8aab07e9c..ddd730e48 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -16,6 +16,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Threading; using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; @@ -32,30 +33,36 @@ namespace Microsoft.Python.Analysis.Analyzer { internal class ModuleWalker : AnalysisWalker { private const string AllVariableName = "__all__"; private readonly IDocumentAnalysis _stubAnalysis; + private readonly CancellationToken _cancellationToken; // A hack to use __all__ export in the most simple case. private int _allReferencesCount; private bool _allIsUsable = true; - public ModuleWalker(IServiceContainer services, IPythonModule module, PythonAst ast) + public ModuleWalker(IServiceContainer services, IPythonModule module, PythonAst ast, CancellationToken cancellationToken) : base(new ExpressionEval(services, module, ast)) { _stubAnalysis = Module.Stub is IDocument doc ? doc.GetAnyAnalysis() : null; + _cancellationToken = cancellationToken; } public override bool Walk(NameExpression node) { if (Eval.CurrentScope == Eval.GlobalScope && node.Name == AllVariableName) { _allReferencesCount++; } + + _cancellationToken.ThrowIfCancellationRequested(); return base.Walk(node); } public override bool Walk(AugmentedAssignStatement node) { HandleAugmentedAllAssign(node); + _cancellationToken.ThrowIfCancellationRequested(); return base.Walk(node); } public override bool Walk(CallExpression node) { HandleAllAppendExtend(node); + _cancellationToken.ThrowIfCancellationRequested(); return base.Walk(node); } @@ -146,6 +153,7 @@ private bool IsHandleableAll(Node node) { public override bool Walk(PythonAst node) { Check.InvalidOperation(() => Ast == node, "walking wrong AST"); + _cancellationToken.ThrowIfCancellationRequested(); // Collect basic information about classes and functions in order // to correctly process forward references. Does not determine @@ -181,16 +189,20 @@ public override bool Walk(PythonAst node) { // Classes and functions are walked by their respective evaluators public override bool Walk(ClassDefinition node) { + _cancellationToken.ThrowIfCancellationRequested(); SymbolTable.Evaluate(node); return false; } public override bool Walk(FunctionDefinition node) { + _cancellationToken.ThrowIfCancellationRequested(); SymbolTable.Evaluate(node); return false; } public void Complete() { + _cancellationToken.ThrowIfCancellationRequested(); + SymbolTable.EvaluateAll(); SymbolTable.ReplacedByStubs.Clear(); MergeStub(); @@ -220,11 +232,13 @@ public void Complete() { /// of the definitions. Stub may contains those so we need to merge it in. /// private void MergeStub() { - if (Module.ModuleType == ModuleType.User) { + _cancellationToken.ThrowIfCancellationRequested(); + + if (Module.ModuleType == ModuleType.User || Module.ModuleType == ModuleType.Stub) { return; } // No stub, no merge. - if (_stubAnalysis == null) { + if (_stubAnalysis.IsEmpty()) { return; } // TODO: figure out why module is getting analyzed before stub. @@ -248,6 +262,18 @@ private void MergeStub() { if (stubType.DeclaringModule is TypingModule && stubType.Name == "Any") { continue; } + + if (sourceVar?.Source == VariableSource.Import && + sourceVar.GetPythonType()?.DeclaringModule.Stub != null) { + // Keep imported types as they are defined in the library. For example, + // 'requests' imports NullHandler as 'from logging import NullHandler'. + // But 'requests' also declares NullHandler in its stub (but not in the main code) + // and that declaration does not have documentation or location. Therefore avoid + // taking types that are stub-only when similar type is imported from another + // module that also has a stub. + continue; + } + TryReplaceMember(v, sourceType, stubType); } @@ -257,7 +283,7 @@ private void MergeStub() { private void TryReplaceMember(IVariable v, IPythonType sourceType, IPythonType stubType) { // If type does not exist in module, but exists in stub, declare it unless it is an import. // If types are the classes, take class from the stub, then add missing members. - // Otherwise, replace type from one from the stub. + // Otherwise, replace type by one from the stub. switch (sourceType) { case null: // Nothing in sources, but there is type in the stub. Declare it. @@ -379,28 +405,26 @@ private static void TransferDocumentationAndLocation(IPythonType s, IPythonType if (s != d && s is PythonType src && d is PythonType dst) { // If type is a class, then doc can either come from class definition node of from __init__. // If class has doc from the class definition, don't stomp on it. - var transferDoc = true; if (src is PythonClassType srcClass && dst is PythonClassType dstClass) { // Higher lever source wins if (srcClass.DocumentationSource == PythonClassType.ClassDocumentationSource.Class || (srcClass.DocumentationSource == PythonClassType.ClassDocumentationSource.Init && dstClass.DocumentationSource == PythonClassType.ClassDocumentationSource.Base)) { dstClass.SetDocumentation(srcClass.Documentation); - transferDoc = false; } - } - - // Sometimes destination (stub type) already has documentation. This happens when stub type - // is used to augment more than one type. For example, in threading module RLock stub class - // replaces both RLock function and _RLock class making 'factory' function RLock to look - // like a class constructor. Effectively a single stub type is used for more than one type - // in the source and two source types may have different documentation. Thus transferring doc - // from one source type affects documentation of another type. It may be better to clone stub - // type and separate instances for separate source type, but for now we'll just avoid stomping - // on the existing documentation. - if (transferDoc && string.IsNullOrEmpty(dst.Documentation)) { - var srcDocumentation = src.Documentation; - if (!string.IsNullOrEmpty(srcDocumentation)) { - dst.SetDocumentation(srcDocumentation); + } else { + // Sometimes destination (stub type) already has documentation. This happens when stub type + // is used to augment more than one type. For example, in threading module RLock stub class + // replaces both RLock function and _RLock class making 'factory' function RLock to look + // like a class constructor. Effectively a single stub type is used for more than one type + // in the source and two source types may have different documentation. Thus transferring doc + // from one source type affects documentation of another type. It may be better to clone stub + // type and separate instances for separate source type, but for now we'll just avoid stomping + // on the existing documentation. + if (string.IsNullOrEmpty(dst.Documentation)) { + var srcDocumentation = src.Documentation; + if (!string.IsNullOrEmpty(srcDocumentation)) { + dst.SetDocumentation(srcDocumentation); + } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index bc9203727..cfa8b64c0 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -238,7 +238,7 @@ internal void RaiseAnalysisComplete(int moduleCount, double msElapsed) private void AnalyzeDocument(in AnalysisModuleKey key, in PythonAnalyzerEntry entry, in ImmutableArray dependencies) { _analysisCompleteEvent.Reset(); ActivityTracker.StartTracking(); - _log?.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name}({entry.Module.ModuleType}) queued"); + _log?.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name} ({entry.Module.ModuleType}) queued. Dependencies: {string.Join(", ", dependencies.Select(d => d.IsTypeshed ? $"{d.Name} (stub)" : d.Name))}"); var graphVersion = _dependencyResolver.ChangeValue(key, entry, entry.IsUserOrBuiltin || key.IsNonUserAsDocument, dependencies); diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs index 1a4772371..f3a8a6046 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs @@ -19,15 +19,13 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Core; using Microsoft.Python.Core.Collections; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { - [DebuggerDisplay("{_module.Name}({_module.ModuleType})")] + [DebuggerDisplay("{_module.Name} : {_module.ModuleType}")] internal sealed class PythonAnalyzerEntry { private readonly object _syncObj = new object(); private TaskCompletionSource _analysisTcs; @@ -248,20 +246,13 @@ public bool Invalidate(IPythonModule module, PythonAst ast, int bufferVersion, i } private HashSet FindDependencies(IPythonModule module, PythonAst ast, int bufferVersion) { - if (_bufferVersion > bufferVersion) { - return new HashSet(); - } - - var walker = new DependencyWalker(module); - ast.Walk(walker); - var dependencies = walker.Dependencies; - dependencies.Remove(new AnalysisModuleKey(module)); + var dependencyProvider = (module as IAnalyzable)?.DependencyProvider; + var dependencies = _bufferVersion <= bufferVersion && module is IAnalyzable analyzable && analyzable.DependencyProvider != null + ? dependencyProvider.GetDependencies() + : new HashSet(); return dependencies; } - private static bool Ignore(IModuleManagement moduleResolution, string fullName, string modulePath) - => moduleResolution.BuiltinModuleName.EqualsOrdinal(fullName) || moduleResolution.IsSpecializedModule(fullName, modulePath); - private void UpdateAnalysisTcs(int analysisVersion) { _analysisVersion = analysisVersion; if (_analysisTcs.Task.Status == TaskStatus.RanToCompletion) { @@ -273,69 +264,6 @@ private void UpdateAnalysisTcs(int analysisVersion) { } } - private class DependencyWalker : PythonWalker { - private readonly IPythonModule _module; - private readonly bool _isTypeshed; - private readonly IModuleManagement _moduleResolution; - private readonly PathResolverSnapshot _pathResolver; - - public HashSet Dependencies { get; } - - public DependencyWalker(IPythonModule module) { - _module = module; - _isTypeshed = module is StubPythonModule stub && stub.IsTypeshed; - _moduleResolution = module.Interpreter.ModuleResolution; - _pathResolver = _isTypeshed - ? module.Interpreter.TypeshedResolution.CurrentPathResolver - : _moduleResolution.CurrentPathResolver; - - Dependencies = new HashSet(); - - if (module.Stub != null) { - Dependencies.Add(new AnalysisModuleKey(module.Stub)); - } - } - - public override bool Walk(ImportStatement import) { - var forceAbsolute = import.ForceAbsolute; - foreach (var moduleName in import.Names) { - var importNames = ImmutableArray.Empty; - foreach (var nameExpression in moduleName.Names) { - importNames = importNames.Add(nameExpression.Name); - var imports = _pathResolver.GetImportsFromAbsoluteName(_module.FilePath, importNames, forceAbsolute); - HandleSearchResults(imports); - } - } - return false; - } - - public override bool Walk(FromImportStatement fromImport) { - var imports = _pathResolver.FindImports(_module.FilePath, fromImport); - HandleSearchResults(imports); - if (imports is IImportChildrenSource childrenSource) { - foreach (var name in fromImport.Names) { - if (childrenSource.TryGetChildImport(name.Name, out var childImport)) { - HandleSearchResults(childImport); - } - } - } - - return false; - } - - private void HandleSearchResults(IImportSearchResult searchResult) { - switch (searchResult) { - case ModuleImport moduleImport when !Ignore(_moduleResolution, moduleImport.FullName, moduleImport.ModulePath): - Dependencies.Add(new AnalysisModuleKey(moduleImport.FullName, moduleImport.ModulePath, _isTypeshed)); - return; - case PossibleModuleImport possibleModuleImport when !Ignore(_moduleResolution, possibleModuleImport.PrecedingModuleFullName, possibleModuleImport.PrecedingModulePath): - Dependencies.Add(new AnalysisModuleKey(possibleModuleImport.PrecedingModuleFullName, possibleModuleImport.PrecedingModulePath, _isTypeshed)); - return; - default: - return; - } - } - } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index eeb5c6b20..9abc1b5a6 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -236,7 +236,7 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { private bool IsAnalyzedLibraryInLoop(IDependencyChainNode node, IDocumentAnalysis currentAnalysis) => !node.HasMissingDependencies && currentAnalysis is LibraryAnalysis && node.IsWalkedWithDependencies && node.IsValidVersion; - private void RunAnalysis(IDependencyChainNode node, Stopwatch stopWatch) + private void RunAnalysis(IDependencyChainNode node, Stopwatch stopWatch) => ExecutionContext.Run(ExecutionContext.Capture(), s => Analyze(node, null, stopWatch), null); private Task StartAnalysis(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwatch stopWatch) @@ -334,83 +334,74 @@ private void AnalyzeEntry(IDependencyChainNode node, Python var analyzable = module as IAnalyzable; analyzable?.NotifyAnalysisBegins(); - var walker = new ModuleWalker(_services, module, ast); - ast.Walk(walker); - + var analysis = DoAnalyzeEntry(node, (IDocument)module, ast, version); _analyzerCancellationToken.ThrowIfCancellationRequested(); - walker.Complete(); - _analyzerCancellationToken.ThrowIfCancellationRequested(); + if (analysis != null) { + analyzable?.NotifyAnalysisComplete(analysis); + entry.TrySetAnalysis(analysis, version); + + if (module.ModuleType == ModuleType.User) { + var linterDiagnostics = _analyzer.LintModule(module); + _diagnosticsService?.Replace(entry.Module.Uri, linterDiagnostics, DiagnosticSource.Linter); + } + } + } + + private IDocumentAnalysis DoAnalyzeEntry(IDependencyChainNode node, IDocument document, PythonAst ast, int version) { + var moduleType = node.Value.Module.ModuleType; bool isCanceled; lock (_syncObj) { isCanceled = _isCanceled; } - if (!isCanceled) { - node?.MarkWalked(); + if (moduleType.CanBeCached() && _moduleDatabaseService?.ModuleExistsInStorage(document.Name, document.FilePath) == true) { + if (!isCanceled && _moduleDatabaseService.TryRestoreGlobalScope(document, out var gs)) { + if (_log != null) { + _log.Log(TraceEventType.Verbose, "Restored from database: ", document.Name); + } + var analysis = new DocumentAnalysis(document, 1, gs, new ExpressionEval(_services, document, document.GetAst()), Array.Empty()); + gs.ReconstructVariables(); + return analysis; + } + return null; } - var analysis = CreateAnalysis(node, (IDocument)module, ast, version, walker, isCanceled); - analyzable?.NotifyAnalysisComplete(analysis); - entry.TrySetAnalysis(analysis, version); - - if (module.ModuleType == ModuleType.User) { - var linterDiagnostics = _analyzer.LintModule(module); - _diagnosticsService?.Replace(entry.Module.Uri, linterDiagnostics, DiagnosticSource.Linter); - } - } - - private void LogCompleted(IDependencyChainNode node, IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) { - if (_log != null) { - var completed = node != null && module.Analysis is LibraryAnalysis ? "completed for library" : "completed"; - var message = node != null - ? $"Analysis of {module.Name} ({module.ModuleType}) on depth {node.VertexDepth} {completed} in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms." - : $"Out of order analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."; - _log.Log(TraceEventType.Verbose, message); - } - } + var walker = new ModuleWalker(_services, document, ast, _analyzerCancellationToken); + ast.Walk(walker); + walker.Complete(); - private void LogCanceled(IPythonModule module) { - if (_log != null) { - _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled."); + lock (_syncObj) { + isCanceled = _isCanceled; } - } - - private void LogException(IPythonModule module, Exception exception) { - if (_log != null) { - _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) failed. {exception}"); + if (!isCanceled) { + node.MarkWalked(); } - if (TestEnvironment.Current != null) { - ExceptionDispatchInfo.Capture(exception).Throw(); - } + return CreateAnalysis(node, document, ast, version, walker, isCanceled); } private IDocumentAnalysis CreateAnalysis(IDependencyChainNode node, IDocument document, PythonAst ast, int version, ModuleWalker walker, bool isCanceled) { var canHaveLibraryAnalysis = false; - var saveAnalysis = false; + // Don't try to drop builtins; it causes issues elsewhere. // We probably want the builtin module's AST and other info for evaluation. switch (document.ModuleType) { case ModuleType.Library: case ModuleType.Compiled: case ModuleType.CompiledBuiltin: - canHaveLibraryAnalysis = true; - saveAnalysis = true; - break; - case ModuleType.Stub: canHaveLibraryAnalysis = true; break; } var createLibraryAnalysis = !isCanceled && - node != null && - !node.HasMissingDependencies && - canHaveLibraryAnalysis && - !document.IsOpen && - node.HasOnlyWalkedDependencies && - node.IsValidVersion; + node != null && + !node.HasMissingDependencies && + canHaveLibraryAnalysis && + !document.IsOpen && + node.HasOnlyWalkedDependencies && + node.IsValidVersion; if (!createLibraryAnalysis) { return new DocumentAnalysis(document, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); @@ -420,15 +411,41 @@ private IDocumentAnalysis CreateAnalysis(IDependencyChainNode(); + dbs?.StoreModuleAnalysisAsync(analysis, CancellationToken.None).DoNotWait(); return analysis; } + + private void LogCompleted(IDependencyChainNode node, IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) { + if (_log != null) { + var completed = node != null && module.Analysis is LibraryAnalysis ? "completed for library" : "completed"; + var message = node != null + ? $"Analysis of {module.Name} ({module.ModuleType}) on depth {node.VertexDepth} {completed} in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms." + : $"Out of order analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."; + _log.Log(TraceEventType.Verbose, message); + } + } + + private void LogCanceled(IPythonModule module) { + if (_log != null) { + _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled."); + } + } + + private void LogException(IPythonModule module, Exception exception) { + if (_log != null) { + _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) failed. {exception}"); + } + + if (TestEnvironment.Current != null) { + ExceptionDispatchInfo.Capture(exception).Throw(); + } + } + private enum State { NotStarted = 0, Started = 1, diff --git a/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs index bd1989aa1..9477a952f 100644 --- a/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs +++ b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs @@ -15,19 +15,36 @@ using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; namespace Microsoft.Python.Analysis.Caching { + /// + /// Represents global scope that has been restored from + /// the database but has not been fully populated yet. + /// Used to attach to analysis so variables can be + /// accessed during classes and methods restoration. + /// + internal interface IRestoredGlobalScope : IGlobalScope { + void ReconstructVariables(); + } + internal interface IModuleDatabaseService { /// - /// Creates module representation from module persistent state. + /// Creates global scope from module persistent state. + /// Global scope is then can be used to construct module analysis. + /// + /// Python module to restore analysis for. + /// Python module global scope. + bool TryRestoreGlobalScope(IPythonModule module, out IRestoredGlobalScope gs); + + /// + /// Retrieves dependencies from the module persistent state. /// - /// Module name. If the name is not qualified - /// the module will ge resolved against active Python version. - /// Module file path. - /// Python module. - /// Module storage state - ModuleStorageState TryCreateModule(string moduleName, string filePath, out IPythonModule module); + /// Python module to restore analysis for. + /// Python module dependency provider. + bool TryRestoreDependencies(IPythonModule module, out IDependencyProvider dp); /// /// Writes module data to the database. @@ -38,5 +55,10 @@ internal interface IModuleDatabaseService { /// Determines if module analysis exists in the storage. /// bool ModuleExistsInStorage(string moduleName, string filePath); + + /// + /// Clear cached data. + /// + void Clear(); } } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs new file mode 100644 index 000000000..9087cd2ee --- /dev/null +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs @@ -0,0 +1,77 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Core.DependencyResolution; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; + +namespace Microsoft.Python.Analysis.Dependencies { + internal sealed class DependencyCollector { + private readonly IPythonModule _module; + private readonly bool _isTypeshed; + private readonly IModuleManagement _moduleResolution; + private readonly PathResolverSnapshot _pathResolver; + + public HashSet Dependencies { get; } = new HashSet(); + + public DependencyCollector(IPythonModule module) { + _module = module; + _isTypeshed = module is StubPythonModule stub && stub.IsTypeshed; + _moduleResolution = module.Interpreter.ModuleResolution; + _pathResolver = _isTypeshed + ? module.Interpreter.TypeshedResolution.CurrentPathResolver + : _moduleResolution.CurrentPathResolver; + + if (module.Stub != null) { + Dependencies.Add(new AnalysisModuleKey(module.Stub)); + } + } + + public void AddImport(IReadOnlyList importNames, bool forceAbsolute) { + var imports = _pathResolver.GetImportsFromAbsoluteName(_module.FilePath, importNames, forceAbsolute); + HandleSearchResults(imports); + } + + public void AddFromImport(IReadOnlyList importNames, int dotCount, bool forceAbsolute) { + var imports = _pathResolver.FindImports(_module.FilePath, importNames, dotCount, forceAbsolute); + HandleSearchResults(imports); + if (imports is IImportChildrenSource childrenSource) { + foreach (var name in importNames) { + if (childrenSource.TryGetChildImport(name, out var childImport)) { + HandleSearchResults(childImport); + } + } + } + } + + private void HandleSearchResults(IImportSearchResult searchResult) { + switch (searchResult) { + case ModuleImport moduleImport when !Ignore(_moduleResolution, moduleImport.FullName, moduleImport.ModulePath): + Dependencies.Add(new AnalysisModuleKey(moduleImport.FullName, moduleImport.ModulePath, _isTypeshed)); + return; + case PossibleModuleImport possibleModuleImport when !Ignore(_moduleResolution, possibleModuleImport.PrecedingModuleFullName, possibleModuleImport.PrecedingModulePath): + Dependencies.Add(new AnalysisModuleKey(possibleModuleImport.PrecedingModuleFullName, possibleModuleImport.PrecedingModulePath, _isTypeshed)); + return; + default: + return; + } + } + private static bool Ignore(IModuleManagement moduleResolution, string fullName, string modulePath) + => moduleResolution.BuiltinModuleName.EqualsOrdinal(fullName) || moduleResolution.IsSpecializedModule(fullName, modulePath); + } +} diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs new file mode 100644 index 000000000..9b2b1502f --- /dev/null +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs @@ -0,0 +1,53 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core.Collections; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Dependencies { + internal sealed class DependencyWalker : PythonWalker { + private readonly DependencyCollector _dependencyCollector; + + public HashSet Dependencies => _dependencyCollector.Dependencies; + + public DependencyWalker(IPythonModule module, PythonAst ast) { + _dependencyCollector = new DependencyCollector(module); + ast.Walk(this); + } + + public override bool Walk(ImportStatement import) { + var forceAbsolute = import.ForceAbsolute; + foreach (var moduleName in import.Names) { + var importNames = ImmutableArray.Empty; + foreach (var nameExpression in moduleName.Names) { + importNames = importNames.Add(nameExpression.Name); + _dependencyCollector.AddImport(importNames, forceAbsolute); + } + } + return false; + } + + public override bool Walk(FromImportStatement fromImport) { + var rootNames = fromImport.Root.Names.Select(n => n.Name).ToArray(); + var dotCount = fromImport.Root is RelativeModuleName relativeName ? relativeName.DotCount : 0; + _dependencyCollector.AddFromImport(rootNames, dotCount, fromImport.ForceAbsolute); + return false; + } + } +} diff --git a/src/Analysis/Ast/Impl/Caching/Definitions/ModuleStorageState.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyProvider.cs similarity index 52% rename from src/Analysis/Ast/Impl/Caching/Definitions/ModuleStorageState.cs rename to src/Analysis/Ast/Impl/Dependencies/IDependencyProvider.cs index bbf14c809..d0a911f5b 100644 --- a/src/Analysis/Ast/Impl/Caching/Definitions/ModuleStorageState.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyProvider.cs @@ -13,30 +13,16 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -namespace Microsoft.Python.Analysis.Caching { +using System.Collections.Generic; +using Microsoft.Python.Analysis.Analyzer; + +namespace Microsoft.Python.Analysis.Dependencies { /// - /// Describes module data stored in a database. + /// Implements provider that can supply list of imports to the dependency analysis. + /// Regular modules provide dependency from the AST, persistent/database modules + /// provide dependencies from their models. /// - public enum ModuleStorageState { - /// - /// Module does not exist in the database. - /// - DoesNotExist, - - /// - /// Partial data. This means module is still being analyzed - /// and the data on the module members is incomplete. - /// - Partial, - - /// - /// Modules exist and the analysis is complete. - /// - Complete, - - /// - /// Storage is corrupted or incompatible. - /// - Corrupted + internal interface IDependencyProvider { + HashSet GetDependencies(); } } diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs index 02fdfbdfd..3816803e3 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Core.Collections; namespace Microsoft.Python.Analysis.Dependencies { diff --git a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs index 1f802edd8..810632abc 100644 --- a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs +++ b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs @@ -241,10 +241,10 @@ private DocumentEntry CreateDocument(ModuleCreationOptions mco) { IDocument document; switch (mco.ModuleType) { case ModuleType.Compiled when TryAddModulePath(mco): - document = new CompiledPythonModule(mco.ModuleName, ModuleType.Compiled, mco.FilePath, mco.Stub, _services); + document = new CompiledPythonModule(mco.ModuleName, ModuleType.Compiled, mco.FilePath, mco.Stub, mco.IsPersistent, _services); break; case ModuleType.CompiledBuiltin: - document = new CompiledBuiltinPythonModule(mco.ModuleName, mco.Stub, _services); + document = new CompiledBuiltinPythonModule(mco.ModuleName, mco.Stub, mco.IsPersistent, _services); break; case ModuleType.User: TryAddModulePath(mco); diff --git a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs index 555a21567..6995f53b3 100644 --- a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs @@ -22,6 +22,8 @@ namespace Microsoft.Python.Analysis { public static class AnalysisExtensions { + public static bool IsEmpty(this IDocumentAnalysis analysis) => analysis == null || analysis is EmptyAnalysis; + public static IScope FindScope(this IDocumentAnalysis analysis, SourceLocation location) => analysis.GlobalScope.FindScope(analysis.Document, location); diff --git a/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs index 7ee3f4a85..d8705f253 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs @@ -61,8 +61,8 @@ public static bool IsPrivateMember(this IPythonClassType cls, string memberName) /// /// Gets specific type for the given generic type parameter, resolving bounds as well /// - public static bool GetSpecificType(this IPythonClassType cls, IGenericTypeParameter param, out IPythonType specificType) { - cls.GenericParameters.TryGetValue(param, out specificType); + public static bool GetSpecificType(this IPythonClassType cls, string paramName, out IPythonType specificType) { + cls.GenericParameters.TryGetValue(paramName, out specificType); // If type has not been found, check if the type parameter has an upper bound and use that if (specificType is IGenericTypeParameter gtp && gtp.Bound != null) { specificType = gtp.Bound; diff --git a/src/Analysis/Ast/Impl/Extensions/PythonFunctionExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonFunctionExtensions.cs index 36f3ac42f..560d46332 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonFunctionExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonFunctionExtensions.cs @@ -47,7 +47,7 @@ public static string GetQualifiedName(this IPythonClassMember cm, string baseNam } return cm.DeclaringModule.ModuleType == ModuleType.Builtins ? string.Join(".", s) - : $"{cm.DeclaringModule.Name}:{string.Join(".", s)}"; + : $"{cm.DeclaringModule.QualifiedName}:{string.Join(".", s)}"; } } } diff --git a/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs index e94bf3161..75a20a039 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs @@ -15,7 +15,6 @@ using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Core; namespace Microsoft.Python.Analysis { public static class PythonTypeExtensions { diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index 095df7033..72988f5a2 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -35,7 +35,7 @@ internal sealed class BuiltinsPythonModule : CompiledPythonModule, IBuiltinsPyth private IPythonType _boolType; public BuiltinsPythonModule(string moduleName, string filePath, IServiceContainer services) - : base(moduleName, ModuleType.Builtins, filePath, null, services) { } // TODO: builtins stub + : base(moduleName, ModuleType.Builtins, filePath, null, false, services) { } // TODO: builtins stub & persistence public override IMember GetMember(string name) => _hiddenNames.Contains(name) ? null : base.GetMember(name); diff --git a/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs b/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs index e020e7b26..500e4dbca 100644 --- a/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs @@ -23,8 +23,8 @@ namespace Microsoft.Python.Analysis.Modules { /// Represents compiled module that is built into the language. /// internal sealed class CompiledBuiltinPythonModule : CompiledPythonModule { - public CompiledBuiltinPythonModule(string moduleName, IPythonModule stub, IServiceContainer services) - : base(moduleName, ModuleType.CompiledBuiltin, MakeFakeFilePath(moduleName, services), stub, services) { } + public CompiledBuiltinPythonModule(string moduleName, IPythonModule stub, bool isPersistent, IServiceContainer services) + : base(moduleName, ModuleType.CompiledBuiltin, MakeFakeFilePath(moduleName, services), stub, isPersistent, services) { } protected override string[] GetScrapeArguments(IPythonInterpreter interpreter) => !InstallPath.TryGetFile("scrape_module.py", out var sm) ? null : new [] { "-W", "ignore", "-B", "-E", sm, "-u8", Name }; diff --git a/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs b/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs index 1335a8ccf..662b25d30 100644 --- a/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs @@ -26,8 +26,8 @@ namespace Microsoft.Python.Analysis.Modules { internal class CompiledPythonModule : PythonModule { protected IStubCache StubCache => Interpreter.ModuleResolution.StubCache; - public CompiledPythonModule(string moduleName, ModuleType moduleType, string filePath, IPythonModule stub, IServiceContainer services) - : base(moduleName, filePath, moduleType, stub, services) { } + public CompiledPythonModule(string moduleName, ModuleType moduleType, string filePath, IPythonModule stub, bool isPersistent, IServiceContainer services) + : base(moduleName, filePath, moduleType, stub, isPersistent, services) { } public override string Documentation => GetMember("__doc__").TryGetConstant(out var s) ? s : string.Empty; diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs index f9c1dc9aa..f3b1058c8 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System; -using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Types; namespace Microsoft.Python.Analysis.Modules { @@ -48,5 +47,10 @@ public sealed class ModuleCreationOptions { /// Module stub, if any. /// public IPythonModule Stub { get; set; } + + /// + /// Indicates if module is restored from database. + /// + public bool IsPersistent { get; set; } } } diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs index a18205271..dba391b96 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs @@ -64,5 +64,6 @@ public enum ModuleType { public static class ModuleTypeExtensions { public static bool IsNonUserFile(this ModuleType type) => type == ModuleType.Library || type == ModuleType.Stub; public static bool IsCompiled(this ModuleType type) => type == ModuleType.Compiled || type == ModuleType.CompiledBuiltin; + public static bool CanBeCached(this ModuleType type) => type == ModuleType.Library || type == ModuleType.Compiled || type == ModuleType.CompiledBuiltin; } } diff --git a/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs b/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs new file mode 100644 index 000000000..ab77d7c4e --- /dev/null +++ b/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs @@ -0,0 +1,53 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Caching; +using Microsoft.Python.Analysis.Dependencies; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; + +namespace Microsoft.Python.Analysis.Modules { + internal sealed class DependencyProvider: IDependencyProvider { + private readonly IPythonModule _module; + private readonly IModuleDatabaseService _dbService; + + public static IDependencyProvider Empty { get; } = new EmptyDependencyProvider(); + + public DependencyProvider(IPythonModule module, IServiceContainer services) { + _dbService = services.GetService(); + _module = module; + } + + #region IDependencyProvider + public HashSet GetDependencies() { + var ast = _module.GetAst(); + + if (_dbService != null && _dbService.TryRestoreDependencies(_module, out var dp)) { + return dp.GetDependencies(); + } + + // TODO: try and handle LoadFunctionDependencyModules functionality here. + var dw = new DependencyWalker(_module, ast); + return dw.Dependencies; + } + #endregion + + private sealed class EmptyDependencyProvider: IDependencyProvider { + public HashSet GetDependencies() => new HashSet(); + } + } +} diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index c9c0b4576..a78501c59 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -22,7 +22,9 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Analyzer; -using Microsoft.Python.Analysis.Core.Interpreter; +using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Caching; +using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Specializations.Typing; @@ -86,12 +88,13 @@ protected PythonModule(string name, ModuleType moduleType, IServiceContainer ser SetDeclaringModule(this); } - protected PythonModule(string moduleName, string filePath, ModuleType moduleType, IPythonModule stub, IServiceContainer services) : + protected PythonModule(string moduleName, string filePath, ModuleType moduleType, IPythonModule stub, bool isPersistent, IServiceContainer services) : this(new ModuleCreationOptions { ModuleName = moduleName, FilePath = filePath, ModuleType = moduleType, - Stub = stub + Stub = stub, + IsPersistent = isPersistent }, services) { } internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer services) @@ -117,12 +120,13 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s ContentState = State.Analyzed; } + IsPersistent = creationOptions.IsPersistent; InitializeContent(creationOptions.Content, 0); } #region IPythonType public string Name { get; } - public string QualifiedName => Name; + public string QualifiedName => ModuleType == ModuleType.Stub ? $"{Name}(stub)" : Name; public BuiltinTypeId TypeId => BuiltinTypeId.Module; public bool IsBuiltin => true; public bool IsAbstract => false; @@ -168,7 +172,7 @@ public virtual IEnumerable GetMemberNames() { if (valueType is PythonModule) { return false; // Do not re-export modules. } - if(valueType is IPythonFunctionType f && f.IsLambda()) { + if (valueType is IPythonFunctionType f && f.IsLambda()) { return false; } if (this is TypingModule) { @@ -178,7 +182,7 @@ public virtual IEnumerable GetMemberNames() { // assigned with types from typing. Example: // from typing import Any # do NOT export Any // x = Union[int, str] # DO export x - if(valueType?.DeclaringModule is TypingModule && v.Name == valueType.Name) { + if (valueType?.DeclaringModule is TypingModule && v.Name == valueType.Name) { return false; } return true; @@ -191,12 +195,10 @@ public virtual IEnumerable GetMemberNames() { public override LocationInfo Definition => new LocationInfo(Uri.ToAbsolutePath(), Uri, 0, 0); #endregion + #region IPythonModule public virtual string FilePath { get; protected set; } public virtual Uri Uri { get; } - - #region IPythonModule public IDocumentAnalysis Analysis { get; private set; } - public IPythonInterpreter Interpreter { get; } /// @@ -216,6 +218,11 @@ public virtual IEnumerable GetMemberNames() { /// wants to see library code and not a stub. /// public IPythonModule PrimaryModule { get; private set; } + + /// + /// Indicates if module is restored from database. + /// + public bool IsPersistent { get; } #endregion #region IDisposable @@ -259,7 +266,7 @@ public string Content { return _buffer.Text; } } - } + } #endregion #region Parsing @@ -399,6 +406,7 @@ public override void Add(string message, SourceSpan span, int errorCode, Severit #endregion #region IAnalyzable + public virtual IDependencyProvider DependencyProvider => new DependencyProvider(this, Services); public void NotifyAnalysisBegins() { lock (_syncObj) { @@ -522,7 +530,11 @@ private void InitializeContent(string content, int version) { private void LoadContent(string content, int version) { if (ContentState < State.Loading) { try { - content = content ?? LoadContent(); + if (IsPersistent) { + content = string.Empty; + } else { + content = content ?? LoadContent(); + } _buffer.Reset(version, content); ContentState = State.Loaded; } catch (IOException) { } catch (UnauthorizedAccessException) { } diff --git a/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs b/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs index e28964258..d81d6a85c 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs @@ -16,9 +16,8 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Python.Analysis.Core.Interpreter; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; @@ -52,6 +51,7 @@ internal sealed class PythonVariableModule : LocatedMember, IPythonModule, IEqua public BuiltinTypeId TypeId => BuiltinTypeId.Module; public Uri Uri => Module?.Uri; public override PythonMemberType MemberType => PythonMemberType.Module; + public bool IsPersistent => Module?.IsPersistent == true; public PythonVariableModule(string name, IPythonInterpreter interpreter) : base(null) { Name = name; @@ -80,5 +80,9 @@ public PythonVariableModule(IPythonModule module): base(module) { public SourceLocation IndexToLocation(int index) => (Module as ILocationConverter)?.IndexToLocation(index) ?? default; public int LocationToIndex(SourceLocation location) => (Module as ILocationConverter)?.LocationToIndex(location) ?? default; #endregion + + #region IDependencyProvider + public IDependencyProvider DependencyProvider => (Module as IAnalyzable)?.DependencyProvider; + #endregion } } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index 24d56166f..26c0d3f83 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -62,9 +62,8 @@ protected override IPythonModule CreateModule(string name) { return null; } - IPythonModule module; if (moduleImport.ModulePath != null) { - module = GetRdt().GetDocument(new Uri(moduleImport.ModulePath)); + var module = GetRdt().GetDocument(new Uri(moduleImport.ModulePath)); if (module != null) { GetRdt().LockDocument(module.Uri); return module; @@ -72,31 +71,31 @@ protected override IPythonModule CreateModule(string name) { } var dbs = GetDbService(); - if (dbs != null && dbs.TryCreateModule(name, moduleImport.ModulePath, out module) != ModuleStorageState.DoesNotExist && module != null) { - SpecializeModule(name, s => module); - return module; - } - - // If there is a stub, make sure it is loaded and attached - // First check stub next to the module. - if (!TryCreateModuleStub(name, moduleImport.ModulePath, out var stub)) { - // If nothing found, try Typeshed. - stub = Interpreter.TypeshedResolution.GetOrLoadModule(moduleImport.IsBuiltin ? name : moduleImport.FullName); - } + moduleImport.IsPersistent = dbs != null && dbs.ModuleExistsInStorage(name, moduleImport.ModulePath); + + IPythonModule stub = null; + if (!moduleImport.IsPersistent) { + // If there is a stub, make sure it is loaded and attached + // First check stub next to the module. + if (!TryCreateModuleStub(name, moduleImport.ModulePath, out stub)) { + // If nothing found, try Typeshed. + stub = Interpreter.TypeshedResolution.GetOrLoadModule(moduleImport.IsBuiltin ? name : moduleImport.FullName); + } - // If stub is created and its path equals to module, return that stub as module - if (stub != null && stub.FilePath.PathEquals(moduleImport.ModulePath)) { - return stub; + // If stub is created and its path equals to module, return that stub as module + if (stub != null && stub.FilePath.PathEquals(moduleImport.ModulePath)) { + return stub; + } } if (moduleImport.IsBuiltin) { Log?.Log(TraceEventType.Verbose, "Create built-in compiled (scraped) module: ", name, Configuration.InterpreterPath); - return new CompiledBuiltinPythonModule(name, stub, Services); + return new CompiledBuiltinPythonModule(name, stub, moduleImport.IsPersistent, Services); } if (moduleImport.IsCompiled) { Log?.Log(TraceEventType.Verbose, "Create compiled (scraped): ", moduleImport.FullName, moduleImport.ModulePath, moduleImport.RootPath); - return new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, Services); + return new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, moduleImport.IsPersistent, Services); } Log?.Log(TraceEventType.Verbose, "Import: ", moduleImport.FullName, moduleImport.ModulePath); @@ -106,7 +105,8 @@ protected override IPythonModule CreateModule(string name) { ModuleName = moduleImport.FullName, ModuleType = moduleImport.IsLibrary ? ModuleType.Library : ModuleType.User, FilePath = moduleImport.ModulePath, - Stub = stub + Stub = stub, + IsPersistent = moduleImport.IsPersistent }; return GetRdt().AddModule(mco); @@ -161,15 +161,8 @@ public IPythonModule GetSpecializedModule(string fullName, bool allowCreation = /// /// Determines of module is specialized or exists in the database. /// - public bool IsSpecializedModule(string fullName, string modulePath = null) { - if (_specialized.ContainsKey(fullName)) { - return true; - } - if (modulePath != null && Path.GetExtension(modulePath) == ".pyi") { - return false; - } - return GetDbService()?.ModuleExistsInStorage(fullName, modulePath) == true; - } + public bool IsSpecializedModule(string fullName, string modulePath = null) + => _specialized.ContainsKey(fullName); internal async Task AddBuiltinTypesToPathResolverAsync(CancellationToken cancellationToken = default) { var analyzer = Services.GetService(); diff --git a/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs b/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs index 0e3e5fbcb..771acd25f 100644 --- a/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs +++ b/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs @@ -13,6 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Core; using Microsoft.Python.Core.IO; @@ -31,11 +32,15 @@ namespace Microsoft.Python.Analysis.Modules { /// internal abstract class SpecializedModule : PythonModule { protected SpecializedModule(string name, string modulePath, IServiceContainer services) - : base(name, modulePath, ModuleType.Specialized, null, services) { } + : base(name, modulePath, ModuleType.Specialized, null, false, services) { } protected override string LoadContent() { // Exceptions are handled in the base return FileSystem.FileExists(FilePath) ? FileSystem.ReadTextWithRetry(FilePath) : string.Empty; } + + #region IAnalyzable + public override IDependencyProvider DependencyProvider => Modules.DependencyProvider.Empty; + #endregion } } diff --git a/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs b/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs index 83fa0ffbe..862daf817 100644 --- a/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs @@ -25,7 +25,7 @@ internal class StubPythonModule : CompiledPythonModule { public bool IsTypeshed { get; } public StubPythonModule(string moduleName, string stubPath, bool isTypeshed, IServiceContainer services) - : base(moduleName, ModuleType.Stub, stubPath, null, services) { + : base(moduleName, ModuleType.Stub, stubPath, null, false, services) { IsTypeshed = isTypeshed; } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBase.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBase.cs index c3f6c573a..7bea95f73 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBase.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBase.cs @@ -37,10 +37,9 @@ internal GenericClassBase(IReadOnlyList typeArgs, IPython #region IPythonClassType public override bool IsGeneric => true; - public override IReadOnlyDictionary GenericParameters - => TypeParameters.ToDictionary(tp => tp, tp => tp as IPythonType ?? UnknownType); - - public override IPythonType CreateSpecificType(IArgumentSet args) + public override IReadOnlyDictionary GenericParameters + => TypeParameters.ToDictionary(tp => tp.Name, tp => tp as IPythonType ?? UnknownType); + public override IPythonType CreateSpecificType(IArgumentSet args) => new GenericClassBase(args.Arguments.Select(a => a.Value).OfType().ToArray(), DeclaringModule.Interpreter); #endregion diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs index e89b342c7..d59dd27ac 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs @@ -33,7 +33,8 @@ public GenericTypeParameter( object covariant, object contravariant, IndexSpan indexSpan) - : base(name, new Location(declaringModule, indexSpan), GetDocumentation(name, constraints, bound, covariant, contravariant)) { + : base(name, new Location(declaringModule, indexSpan), + GetDocumentation(name, constraints, bound, covariant, contravariant, declaringModule)) { Constraints = constraints ?? Array.Empty(); Bound = bound; Covariant = covariant; @@ -106,8 +107,12 @@ private static IPythonType GetBoundType(IArgumentSet argSet) { switch (rawBound) { case IPythonType t: return t; - case IPythonConstant c when c.GetString() != null: - return eval.GetTypeFromString(c.GetString()); + case IPythonConstant c: + var s = c.GetString(); + if (!string.IsNullOrEmpty(s)) { + return eval.GetTypeFromString(s) ?? argSet.Eval.UnknownType; + } + return argSet.Eval.UnknownType; default: return rawBound.GetPythonType(); } @@ -118,7 +123,6 @@ public static IPythonType FromTypeVar(IArgumentSet argSet, IPythonModule declari return declaringModule.Interpreter.UnknownType; } - var args = argSet.Arguments; var constraintArgs = argSet.ListArgument?.Values ?? Array.Empty(); var name = argSet.GetArgumentValue("name")?.GetString(); @@ -135,9 +139,20 @@ public static IPythonType FromTypeVar(IArgumentSet argSet, IPythonModule declari return new GenericTypeParameter(name, declaringModule, constraints, bound, covariant, contravariant, indexSpan); } - private static string GetDocumentation(string name, IReadOnlyList constraints, object bound, object covariant, object contravariant) { + private static string GetDocumentation(string name, IReadOnlyList constraints, IPythonType bound, object covariant, object contravariant, IPythonModule declaringModule) { var constaintStrings = constraints != null ? constraints.Select(c => c.IsUnknown() ? "?" : c.Name) : Enumerable.Empty(); - var boundStrings = bound != null ? Enumerable.Repeat($"bound={bound}", 1) : Enumerable.Empty(); + + var boundStrings = Enumerable.Empty(); + if (bound != null) { + string boundName; + if(bound.DeclaringModule.Equals(declaringModule) || bound.DeclaringModule is IBuiltinsPythonModule) { + boundName = bound.Name; + } else { + boundName = $"{bound.DeclaringModule.Name}.{bound.Name}"; + } + boundStrings = Enumerable.Repeat($"bound={boundName}", 1); + } + var covariantStrings = covariant != null ? Enumerable.Repeat($"covariant={covariant}", 1) : Enumerable.Empty(); var contravariantStrings = contravariant != null ? Enumerable.Repeat($"contravariant={contravariant}", 1) : Enumerable.Empty(); diff --git a/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs b/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs index aa3d89372..839d7adf2 100644 --- a/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs +++ b/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs @@ -75,8 +75,8 @@ public IPythonType CreateSpecificType(IArgumentSet typeArguments) { #region IGenericType public IReadOnlyList Parameters => (InnerType as IGenericType)?.Parameters ?? Array.Empty(); public bool IsGeneric => (InnerType as IPythonClassType)?.IsGeneric == true; - public IReadOnlyDictionary GenericParameters - => (InnerType as IPythonClassType)?.GenericParameters ?? EmptyDictionary.Instance; + public IReadOnlyDictionary GenericParameters + => (InnerType as IPythonClassType)?.GenericParameters ?? EmptyDictionary.Instance; #endregion #region IPythonClassType diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassType.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassType.cs index 697f3e862..2079cb2e2 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassType.cs @@ -41,6 +41,6 @@ public interface IPythonClassType : IPythonClassMember, IGenericType { /// If class is created off generic template, represents /// pairs of the generic parameter / actual supplied type. /// - IReadOnlyDictionary GenericParameters { get; } + IReadOnlyDictionary GenericParameters { get; } } } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs index 2c12d9f9a..168670be1 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs @@ -63,5 +63,10 @@ public interface IPythonModule : IPythonType { /// wants to see library code and not a stub. /// IPythonModule PrimaryModule { get; } + + /// + /// Indicates if module is restored from database. + /// + bool IsPersistent { get; } } } diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs index c74fe841b..9490079e6 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs @@ -30,12 +30,12 @@ internal partial class PythonClassType { private string _qualifiedNameWithParameters; // Qualified name with qualified parameter names for persistence. private Dictionary _specificTypeCache; - private Dictionary _genericParameters; + private Dictionary _genericParameters; private IReadOnlyList _parameters = new List(); #region IGenericType /// - /// List of unfilled generic type parameters. Represented as entries in the GenericParameters dictionary + /// List of unfilled generic type parameters. Represented as entries in the ActualGenericParameters dictionary /// that have both key and value as generic type parameters /// e.g /// {T, T} @@ -72,7 +72,7 @@ public virtual IPythonType CreateSpecificType(IArgumentSet args) { // Storing generic parameters allows methods returning generic types // to know what type parameter returns what specific type - classType.StoreGenericParameters(this, newGenericTypeParameters, genericTypeToSpecificType); + classType.StoreGenericParameters(this, newGenericTypeParameters.Select(p => p.Name).ToArray(), genericTypeToSpecificType); // Set generic name classType.SetNames(); @@ -115,7 +115,7 @@ public virtual IPythonType CreateSpecificType(IArgumentSet args) { // Look through generic type bases and see if any of their required type parameters // have received a specific type, and if so create specific type var st = gt.Parameters - .Select(p => classType.GenericParameters.TryGetValue(p, out var t) ? t : null) + .Select(p => classType.GenericParameters.TryGetValue(p.Name, out var t) ? t : null) .Where(p => !p.IsUnknown()) .ToArray(); if (st.Length > 0) { @@ -127,7 +127,7 @@ public virtual IPythonType CreateSpecificType(IArgumentSet args) { } // Set specific class bases - classType.SetBases(specificBases.Concat(newBases), args.Eval.CurrentScope); + classType.SetBases(specificBases.Concat(newBases), args.Eval?.CurrentScope); // Now that parameters are set, check if class is generic classType.SetGenericParameters(); // Transfer members from generic to specific type. @@ -168,13 +168,13 @@ private IGenericTypeParameter[] GetTypeParameters() { /// Given an argument set, returns a dictionary mapping generic type parameter to the supplied specific /// type from arguments. /// - private IReadOnlyDictionary GetSpecificTypes( + private IReadOnlyDictionary GetSpecificTypes( IArgumentSet args, IReadOnlyList genericTypeParameters, ICollection newBases ) { // For now, map each type parameter to itself, and we can fill in the value as we go - var genericTypeToSpecificType = genericTypeParameters.ToDictionary(gtp => gtp, gtp => gtp as IPythonType); + var genericTypeToSpecificType = genericTypeParameters.ToDictionary(gtp => gtp.Name, gtp => gtp as IPythonType); // Arguments passed are those of __init__ or copy constructor or index expression A[int, str, ...]. // The arguments do not necessarily match all of the declared generic parameters. @@ -190,11 +190,11 @@ ICollection newBases // __init__(self, v: _T), v is annotated as a generic type definition // Check if its generic type name matches any of the generic class parameters i.e. if there is // an argument like 'v: _T' we need to check if class has matching Generic[_T] or A[_T] in bases. - if (genericTypeToSpecificType.ContainsKey(argTypeDefinition)) { + if (genericTypeToSpecificType.ContainsKey(argTypeDefinition.Name)) { // TODO: Check if specific type matches generic type parameter constraints and report mismatches. // Assign specific type. if (arg.Value is IMember m && m.GetPythonType() is IPythonType pt) { - genericTypeToSpecificType[argTypeDefinition] = pt; + genericTypeToSpecificType[argTypeDefinition.Name] = pt; } else { // TODO: report supplied parameter is not a type. } @@ -241,8 +241,8 @@ ICollection newBases var type = member.GetPythonType(); if (!type.IsUnknown()) { var gtd = gtIndex < genericTypeParameters.Count ? genericTypeParameters[gtIndex] : null; - if (gtd != null && genericTypeToSpecificType.TryGetValue(gtd, out var s) && s is IGenericTypeParameter) { - genericTypeToSpecificType[gtd] = type; + if (gtd != null && genericTypeToSpecificType.TryGetValue(gtd.Name, out var s) && s is IGenericTypeParameter) { + genericTypeToSpecificType[gtd.Name] = type; } gtIndex++; } @@ -258,8 +258,8 @@ ICollection newBases /// internal void StoreGenericParameters( IPythonClassType templateClass, - IEnumerable newGenericParameters, - IReadOnlyDictionary genericToSpecificTypes) { + IEnumerable newGenericParameters, + IReadOnlyDictionary genericToSpecificTypes) { // copy original generic parameters over and try to fill them in _genericParameters = templateClass.GenericParameters.ToDictionary(k => k.Key, k => k.Value); @@ -280,7 +280,7 @@ internal void StoreGenericParameters( // class A(Generic[T]): // class B(A[U]) // A has T => U - _genericParameters[gp] = genericToSpecificTypes.TryGetValue(specificType, out var v) ? v : null; + _genericParameters[gp] = genericToSpecificTypes.TryGetValue(specificType.Name, out var v) ? v : null; } } } @@ -302,28 +302,28 @@ internal void StoreGenericParameters( /// Generic type (Generic[T1, T2, ...], A[T1, T2, ..], etc.). /// Argument value passed to the class constructor. /// Dictionary or name (T1) to specific type to populate. - private static void GetSpecificTypeFromArgumentValue(IGenericType gt, object argumentValue, IDictionary specificTypes) { + private void GetSpecificTypeFromArgumentValue(IGenericType gt, object argumentValue, IDictionary specificTypes) { switch (argumentValue) { case IPythonDictionary dict when gt.Parameters.Count == 2: var keyType = dict.Keys.FirstOrDefault()?.GetPythonType(); var valueType = dict.Values.FirstOrDefault()?.GetPythonType(); if (!keyType.IsUnknown()) { - specificTypes[gt.Parameters[0]] = keyType; + specificTypes[gt.Parameters[0].Name] = keyType; } if (!valueType.IsUnknown()) { - specificTypes[gt.Parameters[1]] = valueType; + specificTypes[gt.Parameters[1].Name] = valueType; } break; case IPythonIterable iter when gt.TypeId == BuiltinTypeId.List && gt.Parameters.Count == 1: var itemType = iter.GetIterator().Next.GetPythonType(); if (!itemType.IsUnknown()) { - specificTypes[gt.Parameters[0]] = itemType; + specificTypes[gt.Parameters[0].Name] = itemType; } break; case IPythonCollection coll when gt.TypeId == BuiltinTypeId.Tuple && gt.Parameters.Count >= 1: var itemTypes = coll.Contents.Select(m => m.GetPythonType()).ToArray(); for (var i = 0; i < Math.Min(itemTypes.Length, gt.Parameters.Count); i++) { - specificTypes[gt.Parameters[i]] = itemTypes[i]; + specificTypes[gt.Parameters[i].Name] = itemTypes[i]; } break; } @@ -361,7 +361,7 @@ private void SetClassMembers(IPythonClassType templateClass, IArgumentSet args) specificType = tt.CreateSpecificType(args); break; case IGenericTypeParameter gtd: - GenericParameters.TryGetValue(gtd, out specificType); + GenericParameters.TryGetValue(gtd.Name, out specificType); break; } diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs index a21cd81a5..0f35ff1d9 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs @@ -214,8 +214,8 @@ public IReadOnlyList Mro { /// class B(A[int, str]): ... /// Has the map {T: int, K: str} /// - public virtual IReadOnlyDictionary GenericParameters => - _genericParameters ?? EmptyDictionary.Instance; + public virtual IReadOnlyDictionary GenericParameters => + _genericParameters ?? EmptyDictionary.Instance; #endregion @@ -226,6 +226,13 @@ internal override void SetDocumentation(string documentation) { DocumentationSource = ClassDocumentationSource.Class; } + /// + /// Sets class bases. If scope is provided, detects loops in base classes and removes them. + /// + /// List of base types. + /// Current scope to look up base types. + /// Can be null if class is restored from database, in which case + /// there is no need to try and disambiguate bases. internal void SetBases(IEnumerable bases, IScope currentScope = null) { if (_bases != null) { return; // Already set diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index d5e122867..d50c55fce 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -161,7 +161,7 @@ private IMember CreateSpecificReturnFromClassType(IPythonClassType selfClassType // -> A[_T1, _T2, ...] // Match arguments IReadOnlyList typeArgs = null; - var classGenericParameters = selfClassType?.GenericParameters.Keys.ToArray() ?? Array.Empty(); + var classGenericParameters = selfClassType?.GenericParameters.Keys.ToArray() ?? Array.Empty(); if (classGenericParameters.Length > 0 && selfClassType != null) { // Declaring class is specific and provides definitions of generic parameters typeArgs = classGenericParameters @@ -181,7 +181,7 @@ private IMember CreateSpecificReturnFromClassType(IPythonClassType selfClassType } private IMember CreateSpecificReturnFromTypeVar(IPythonClassType selfClassType, IArgumentSet args, IGenericTypeParameter returnType) { - if (selfClassType.GetSpecificType(returnType, out var specificType)) { + if (selfClassType.GetSpecificType(returnType.Name, out var specificType)) { return new PythonInstance(specificType); } @@ -189,10 +189,10 @@ private IMember CreateSpecificReturnFromTypeVar(IPythonClassType selfClassType, var baseType = selfClassType.Mro .OfType() .Skip(1) - .FirstOrDefault(b => b.GetMember(ClassMember.Name) != null && b.GenericParameters.ContainsKey(returnType)); + .FirstOrDefault(b => b.GetMember(ClassMember.Name) != null && b.GenericParameters.ContainsKey(returnType.Name)); // Try and infer return value from base class - if (baseType != null && baseType.GetSpecificType(returnType, out specificType)) { + if (baseType != null && baseType.GetSpecificType(returnType.Name, out specificType)) { return new PythonInstance(specificType); } diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index 8db2ab7cd..8058a08c8 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -23,7 +23,7 @@ namespace Microsoft.Python.Analysis.Types { [DebuggerDisplay("{" + nameof(Name) + "}")] - internal class PythonType : LocatedMember, IPythonType {//, IEquatable { + internal class PythonType : LocatedMember, IPythonType { private readonly object _lock = new object(); private Dictionary _members; private BuiltinTypeId _typeId; @@ -159,16 +159,14 @@ internal IMember AddMember(string name, IMember member, bool overwrite) { } } - internal void MakeReadOnly() => _readonly = true; + internal void MakeReadOnly() { + lock (_lock) { + _readonly = true; + } + } internal bool IsHidden => ContainsMember("__hidden__"); protected bool ContainsMember(string name) => Members.ContainsKey(name); protected IPythonType UnknownType => DeclaringModule.Interpreter.UnknownType; - - //public bool Equals(IPythonType other) => PythonTypeComparer.Instance.Equals(this, other); - - //public override bool Equals(object obj) - // => obj is IPythonType pt && PythonTypeComparer.Instance.Equals(this, pt); - //public override int GetHashCode() => 0; } } diff --git a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs index 876886d81..77887be1a 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs @@ -138,8 +138,9 @@ public void HaveSameMembersAs(IMember other) { } subjectMemberType.MemberType.Should().Be(otherMemberType.MemberType); + Debug.Assert(subjectMemberType.MemberType == otherMemberType.MemberType); - if(subjectMemberType is IPythonClassType subjectClass) { + if (subjectMemberType is IPythonClassType subjectClass) { var otherClass = otherMemberType as IPythonClassType; otherClass.Should().NotBeNull(); @@ -163,8 +164,10 @@ public void HaveSameMembersAs(IMember other) { case PythonMemberType.Class: // Restored collections (like instance of tuple) turn into classes // rather than into collections with content since we don't track - // collection content in libraries. - subjectMemberType.QualifiedName.Should().Be(otherMemberType.QualifiedName); + // collection content in libraries. We don't compare qualified names + // since original module may be source or a stub and that is not + // preserved during restore. + // subjectMemberType.QualifiedName.Should().Be(otherMemberType.QualifiedName); break; case PythonMemberType.Function: case PythonMemberType.Method: diff --git a/src/Analysis/Ast/Test/TypeshedTests.cs b/src/Analysis/Ast/Test/TypeshedTests.cs index 9b1a72c6f..9ab18657b 100644 --- a/src/Analysis/Ast/Test/TypeshedTests.cs +++ b/src/Analysis/Ast/Test/TypeshedTests.cs @@ -45,7 +45,7 @@ import sys e1, e2, e3 = sys.exc_info() "; var analysis = await GetAnalysisAsync(code); - // sys.exc_info() -> (exception_type, exception_value, traceback) + // def exc_info() -> Tuple[Optional[Type[BaseException]], Optional[BaseException], Optional[TracebackType]]: ... var f = analysis.Should() .HaveVariable("e1").OfType(BuiltinTypeId.Type) .And.HaveVariable("e2").OfType("BaseException") diff --git a/src/Analysis/Ast/Test/TypingTests.cs b/src/Analysis/Ast/Test/TypingTests.cs index 2df9a7df7..31d9fd1de 100644 --- a/src/Analysis/Ast/Test/TypingTests.cs +++ b/src/Analysis/Ast/Test/TypingTests.cs @@ -83,10 +83,25 @@ from typing import TypeVar [TestMethod, Priority(0)] - public async Task KeywordArgMixDocCheck() { + public async Task TypeVarBoundToUnknown() { const string code = @" from typing import TypeVar X = TypeVar('X', bound='hello', covariant=True) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable("X") + .Which.Value.Should().HaveDocumentation("TypeVar('X', bound=Unknown, covariant=True)"); + analysis.Should().HaveGenericVariable("X"); + } + + [TestMethod, Priority(0)] + public async Task TypeVarBoundToStringName() { + const string code = @" +from typing import TypeVar + +X = TypeVar('X', bound='hello', covariant=True) + +class hello: ... "; var analysis = await GetAnalysisAsync(code); analysis.Should().HaveVariable("X") diff --git a/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs b/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs index 132d06ff1..1c7dcfaed 100644 --- a/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs +++ b/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs @@ -13,16 +13,21 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Collections.Generic; using System.Linq; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Core.DependencyResolution { public static class AstUtilities { public static IImportSearchResult FindImports(this PathResolverSnapshot pathResolver, string modulePath, FromImportStatement fromImportStatement) { - var rootNames = fromImportStatement.Root.Names.Select(n => n.Name); - return fromImportStatement.Root is RelativeModuleName relativeName - ? pathResolver.GetImportsFromRelativePath(modulePath, relativeName.DotCount, rootNames) - : pathResolver.GetImportsFromAbsoluteName(modulePath, rootNames, fromImportStatement.ForceAbsolute); + var rootNames = fromImportStatement.Root.Names.Select(n => n.Name).ToArray(); + var dotCount = fromImportStatement.Root is RelativeModuleName relativeName ? relativeName.DotCount : 0; + return pathResolver.FindImports(modulePath, rootNames, dotCount, fromImportStatement.ForceAbsolute); } + + public static IImportSearchResult FindImports(this PathResolverSnapshot pathResolver, string modulePath, IReadOnlyList rootNames, int dotCount, bool forceAbsolute) + => dotCount > 0 + ? pathResolver.GetImportsFromRelativePath(modulePath, dotCount, rootNames) + : pathResolver.GetImportsFromAbsoluteName(modulePath, rootNames, forceAbsolute); } } diff --git a/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs b/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs index b229ab80e..9d95188e3 100644 --- a/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs +++ b/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs @@ -27,6 +27,7 @@ public class ModuleImport : IImportChildrenSource { public bool IsCompiled { get; } public bool IsLibrary { get; } public bool IsBuiltin => IsCompiled && ModulePath == null; + public bool IsPersistent { get; set; } public ModuleImport(IImportChildrenSource childrenSource, string name, string fullName, string rootPath, string modulePath, long moduleFileSize, bool isCompiled, bool isLibrary) { _childrenSource = childrenSource; diff --git a/src/Caching/Impl/Models/ClassModel.cs b/src/Caching/Impl/Models/ClassModel.cs index 2b5aa0a70..527816157 100644 --- a/src/Caching/Impl/Models/ClassModel.cs +++ b/src/Caching/Impl/Models/ClassModel.cs @@ -40,7 +40,7 @@ internal sealed class ClassModel : MemberModel { public ClassModel[] Classes { get; set; } /// - /// FormalGenericParameters of the Generic[...] base class, if any. + /// GenericParameters of the Generic[...] base class, if any. /// public string[] GenericBaseParameters { get; set; } /// @@ -106,7 +106,6 @@ public ClassModel(IPythonClassType cls) { // Only persist documentation from this class, leave bases or __init__ alone. Documentation = (cls as PythonClassType)?.DocumentationSource == PythonClassType.ClassDocumentationSource.Class ? cls.Documentation : null; - var ntBases = cls.Bases.OfType().ToArray(); NamedTupleBases = ntBases.Select(b => new NamedTupleModel(b)).ToArray(); @@ -126,17 +125,16 @@ public ClassModel(IPythonClassType cls) { GenericBaseParameters = GenericBaseParameters ?? Array.Empty(); GenericParameterValues = cls.GenericParameters - .Select(p => new GenericParameterValueModel { Name = p.Key.Name, Type = p.Value.GetPersistentQualifiedName() }) + .Select(p => new GenericParameterValueModel { Name = p.Key, Type = p.Value.GetPersistentQualifiedName() }) .ToArray(); } - protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) { - if (_cls != null) { + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + if(_cls != null) { return _cls; } _cls = new PythonClassType(Name, new Location(mf.Module, IndexSpan.ToSpan())); - - var bases = CreateBases(mf); + var bases = CreateBases(mf, gs); _cls.SetBases(bases); _cls.SetDocumentation(Documentation); @@ -146,37 +144,32 @@ protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringTy _cls, _cls.GenericParameters.Keys.ToArray(), GenericParameterValues.ToDictionary( - k => _cls.GenericParameters.Keys.First(x => x.Name == k.Name), + k => _cls.GenericParameters.Keys.First(x => x == k.Name), v => mf.ConstructType(v.Type) ) ); } - foreach (var f in Methods) { - var m = f.Construct(mf, _cls); - _cls.AddMember(f.Name, m, false); - } - - foreach (var p in Properties) { - var m = p.Construct(mf, _cls); - _cls.AddMember(p.Name, m, false); - } - - foreach (var c in Classes) { - var m = c.Construct(mf, _cls); - _cls.AddMember(c.Name, m, false); + var all = Classes.Concat(Properties).Concat(Methods).Concat(Fields); + foreach (var m in all) { + _cls.AddMember(m.Name, m.Create(mf, _cls, gs), false); } + return _cls; + } - foreach (var vm in Fields) { - var m = vm.Construct(mf, _cls); - _cls.AddMember(vm.Name, m, false); + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + var all = Classes.Concat(Properties).Concat(Methods).Concat(Fields); + foreach (var m in all) { + m.Populate(mf, _cls, gs); } - - return _cls; } - private IPythonType[] CreateBases(ModuleFactory mf) { - var ntBases = NamedTupleBases.Select(ntb => ntb.Construct(mf, _cls)).OfType().ToArray(); + private IEnumerable CreateBases(ModuleFactory mf, IGlobalScope gs) { + var ntBases = NamedTupleBases.Select(ntb => { + var n = ntb.Create(mf, _cls, gs); + ntb.Populate(mf, _cls, gs); + return n; + }).OfType().ToArray(); var is3x = mf.Module.Interpreter.LanguageVersion.Is3x(); var basesNames = Bases.Select(b => is3x && b == "object" ? null : b).ExcludeDefault().ToArray(); @@ -184,11 +177,11 @@ private IPythonType[] CreateBases(ModuleFactory mf) { if (GenericBaseParameters.Length > 0) { // Generic class. Need to reconstruct generic base so code can then - // create specific types off the generic class. + // create specific types off the generic class. var genericBase = bases.OfType().FirstOrDefault(b => b.Name == "Generic"); if (genericBase != null) { - var typeVars = GenericBaseParameters.Select(n => mf.Module.GlobalScope.Variables[n]?.Value).OfType().ToArray(); - Debug.Assert(typeVars.Length > 0, "Class generic type parameters were not defined in the module during restore"); + var typeVars = GenericBaseParameters.Select(n => gs.Variables[n]?.Value).OfType().ToArray(); + //Debug.Assert(typeVars.Length > 0, "Class generic type parameters were not defined in the module during restore"); if (typeVars.Length > 0) { var genericWithParameters = genericBase.CreateSpecificType(new ArgumentSet(typeVars, null, null)); if (genericWithParameters != null) { diff --git a/src/Caching/Impl/Models/FromImportModel.cs b/src/Caching/Impl/Models/FromImportModel.cs new file mode 100644 index 000000000..bda4d9774 --- /dev/null +++ b/src/Caching/Impl/Models/FromImportModel.cs @@ -0,0 +1,25 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.Analysis.Caching.Models { + /// + /// Represents from import statement for dependency resolution. + /// + internal sealed class FromImportModel { + public string[] RootNames { get; set; } + public int DotCount { get; set; } + public bool ForceAbsolute { get; set; } + } +} diff --git a/src/Caching/Impl/Models/FunctionModel.cs b/src/Caching/Impl/Models/FunctionModel.cs index 88da25f78..7c50a97cd 100644 --- a/src/Caching/Impl/Models/FunctionModel.cs +++ b/src/Caching/Impl/Models/FunctionModel.cs @@ -17,6 +17,8 @@ using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; + // ReSharper disable AutoPropertyCanBeMadeGetOnly.Global // ReSharper disable MemberCanBePrivate.Global @@ -33,22 +35,18 @@ public FunctionModel(IPythonFunctionType func) : base(func) { Overloads = func.Overloads.Select(FromOverload).ToArray(); } - protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) { - if (_function != null) { - return _function; - } - _function = new PythonFunctionType(Name, new Location(mf.Module, IndexSpan.ToSpan()), declaringType, Documentation); + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) + => _function ?? (_function = new PythonFunctionType(Name, new Location(mf.Module, IndexSpan.ToSpan()), declaringType, Documentation)); - // Create inner functions and classes first since function - // may be returning one of them. - foreach (var model in Functions) { - var f = model.Construct(mf, _function); - _function.AddMember(Name, f, overwrite: true); - } + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + // Create inner functions and classes first since function may be returning one of them. + var all = Classes.Concat(Functions).ToArray(); - foreach (var cm in Classes) { - var c = cm.Construct(mf, _function); - _function.AddMember(cm.Name, c, overwrite: true); + foreach (var model in all) { + _function.AddMember(Name, model.Create(mf, _function, gs), overwrite: true); + } + foreach (var model in all) { + model.Populate(mf, _function, gs); } foreach (var om in Overloads) { @@ -58,9 +56,8 @@ protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringTy o.SetParameters(om.Parameters.Select(p => ConstructParameter(mf, p)).ToArray()); _function.AddOverload(o); } - - return _function; } + private IParameterInfo ConstructParameter(ModuleFactory mf, ParameterModel pm) => new ParameterInfo(pm.Name, mf.ConstructType(pm.Type), pm.Kind, mf.ConstructMember(pm.DefaultValue)); diff --git a/src/Caching/Impl/Models/ImportModel.cs b/src/Caching/Impl/Models/ImportModel.cs new file mode 100644 index 000000000..e99b58fcf --- /dev/null +++ b/src/Caching/Impl/Models/ImportModel.cs @@ -0,0 +1,24 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.Analysis.Caching.Models { + /// + /// Represents import statement for dependency resolution. + /// + internal sealed class ImportModel { + public string[] ModuleNames { get; set; } + public bool ForceAbsolute { get; set; } + } +} diff --git a/src/Caching/Impl/Models/MemberModel.cs b/src/Caching/Impl/Models/MemberModel.cs index 24d512fcf..e7ce43282 100644 --- a/src/Caching/Impl/Models/MemberModel.cs +++ b/src/Caching/Impl/Models/MemberModel.cs @@ -17,6 +17,7 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; // ReSharper disable MemberCanBeProtected.Global // ReSharper disable UnusedAutoPropertyAccessor.Global @@ -43,13 +44,17 @@ internal abstract class MemberModel { /// public IndexSpanModel IndexSpan { get; set; } - [NonSerialized] - private IMember _member; - - public IMember Construct(ModuleFactory mf, IPythonType declaringType) - => _member ?? (_member = ReConstruct(mf, declaringType)); - protected abstract IMember ReConstruct(ModuleFactory mf, IPythonType declaringType); + /// + /// Create member for declaration but does not construct its parts just yet. + /// Used as a first pass in two-pass handling of forward declarations. + /// + public abstract IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs); + /// + /// Populate member with content, such as create class methods, etc. + /// + public abstract void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs); + public virtual MemberModel GetModel(string name) => GetMemberModels().FirstOrDefault(m => m.Name == name); protected virtual IEnumerable GetMemberModels() => Enumerable.Empty(); } diff --git a/src/Caching/Impl/Models/ModuleModel.cs b/src/Caching/Impl/Models/ModuleModel.cs index dfe859d35..c67837cb3 100644 --- a/src/Caching/Impl/Models/ModuleModel.cs +++ b/src/Caching/Impl/Models/ModuleModel.cs @@ -21,6 +21,7 @@ using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; // ReSharper disable MemberCanBePrivate.Global namespace Microsoft.Python.Analysis.Caching.Models { @@ -49,11 +50,16 @@ internal sealed class ModuleModel : MemberModel { /// public int FileSize { get; set; } + public ImportModel[] Imports { get; set; } + public FromImportModel[] FromImports { get; set; } + public ImportModel[] StubImports { get; set; } + public FromImportModel[] StubFromImports { get; set; } + [NonSerialized] private Dictionary _modelCache; public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceContainer services, AnalysisCachingLevel options) { var uniqueId = analysis.Document.GetUniqueId(services, options); - if(uniqueId == null) { + if (uniqueId == null) { // Caching level setting does not permit this module to be persisted. return null; } @@ -68,9 +74,9 @@ public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceConta // as well as variables that are declarations. var exportedNames = new HashSet(analysis.Document.GetMemberNames()); foreach (var v in analysis.GlobalScope.Variables - .Where(v => exportedNames.Contains(v.Name) || - v.Source == VariableSource.Declaration || - v.Source == VariableSource.Builtin || + .Where(v => exportedNames.Contains(v.Name) || + v.Source == VariableSource.Declaration || + v.Source == VariableSource.Builtin || v.Source == VariableSource.Generic)) { if (v.Value is IGenericTypeParameter && !typeVars.ContainsKey(v.Name)) { @@ -115,10 +121,20 @@ when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals } } + // Take dependencies from imports. If module has stub we should also take + // dependencies from there since persistent state is based on types that + // are combination of stub and the module. Sometimes stub may import more + // and we must make sure dependencies are restored before the module. + var primaryDependencyWalker = new DependencyWalker(analysis.Ast); + var stubDependencyWalker = analysis.Document.Stub != null ? new DependencyWalker(analysis.Document.Stub.Analysis.Ast) : null; + var stubImports = stubDependencyWalker?.Imports ?? Enumerable.Empty(); + var stubFromImports = stubDependencyWalker?.FromImports ?? Enumerable.Empty(); + return new ModuleModel { Id = uniqueId.GetStableHash(), UniqueId = uniqueId, Name = analysis.Document.Name, + QualifiedName = analysis.Document.QualifiedName, Documentation = analysis.Document.Documentation, Functions = functions.Values.ToArray(), Variables = variables.Values.ToArray(), @@ -129,7 +145,11 @@ when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals EndIndex = l.EndIndex, Kind = l.Kind }).ToArray(), - FileSize = analysis.Ast.EndIndex + FileSize = analysis.Ast.EndIndex, + Imports = primaryDependencyWalker.Imports.ToArray(), + FromImports = primaryDependencyWalker.FromImports.ToArray(), + StubImports = stubImports.ToArray(), + StubFromImports = stubFromImports.ToArray() }; } @@ -149,8 +169,6 @@ private static FunctionModel GetFunctionModel(IDocumentAnalysis analysis, IVaria return null; } - protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) => throw new NotImplementedException(); - public override MemberModel GetModel(string name) { if (_modelCache == null) { var models = TypeVars.Concat(NamedTuples).Concat(Classes).Concat(Functions).Concat(Variables); @@ -160,8 +178,38 @@ public override MemberModel GetModel(string name) { _modelCache[m.Name] = m; } } - return _modelCache.TryGetValue(name, out var model) ? model : null; } + + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) => throw new NotImplementedException(); + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) => throw new NotImplementedException(); + + private sealed class DependencyWalker : PythonWalker { + public List Imports { get; } = new List(); + public List FromImports { get; } = new List(); + + public DependencyWalker(PythonAst ast) { + ast.Walk(this); + } + + public override bool Walk(ImportStatement import) { + var model = new ImportModel { + ForceAbsolute = import.ForceAbsolute, + ModuleNames = import.Names.SelectMany(mn => mn.Names).Select(n => n.Name).ToArray() + }; + Imports.Add(model); + return false; + } + + public override bool Walk(FromImportStatement fromImport) { + var model = new FromImportModel { + ForceAbsolute = fromImport.ForceAbsolute, + RootNames = fromImport.Root.Names.Select(n => n.Name).ToArray(), + DotCount = fromImport.Root is RelativeModuleName rn ? rn.DotCount : 0 + }; + FromImports.Add(model); + return false; + } + } } } diff --git a/src/Caching/Impl/Models/NamedTupleModel.cs b/src/Caching/Impl/Models/NamedTupleModel.cs index 5f17059c9..82fed074b 100644 --- a/src/Caching/Impl/Models/NamedTupleModel.cs +++ b/src/Caching/Impl/Models/NamedTupleModel.cs @@ -19,6 +19,7 @@ using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Specializations.Typing.Types; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; namespace Microsoft.Python.Analysis.Caching.Models { @@ -41,7 +42,7 @@ public NamedTupleModel(ITypingNamedTupleType nt) { ItemTypes = nt.ItemTypes.Select(t => t.QualifiedName).ToArray(); } - protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) { + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { if (_namedTuple != null) { return _namedTuple; } @@ -50,5 +51,7 @@ protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringTy _namedTuple = new NamedTupleType(Name, ItemNames, itemTypes, mf.Module, IndexSpan.ToSpan()); return _namedTuple; } + + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { } } } diff --git a/src/Caching/Impl/Models/PropertyModel.cs b/src/Caching/Impl/Models/PropertyModel.cs index a5b33e025..e608584e0 100644 --- a/src/Caching/Impl/Models/PropertyModel.cs +++ b/src/Caching/Impl/Models/PropertyModel.cs @@ -15,6 +15,8 @@ using System; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; + // ReSharper disable AutoPropertyCanBeMadeGetOnly.Global // ReSharper disable MemberCanBePrivate.Global @@ -30,19 +32,16 @@ public PropertyModel(IPythonPropertyType prop) : base(prop) { ReturnType = prop.ReturnType.GetPersistentQualifiedName(); } - protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) { - if (_property != null) { - return _property; - } - _property = new PythonPropertyType(Name, new Location(mf.Module, IndexSpan.ToSpan()), declaringType, (Attributes & FunctionAttributes.Abstract) != 0); + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) + => _property ?? (_property = new PythonPropertyType(Name, new Location(mf.Module, IndexSpan.ToSpan()), declaringType, (Attributes & FunctionAttributes.Abstract) != 0)); + + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { _property.SetDocumentation(Documentation); var o = new PythonFunctionOverload(Name, mf.DefaultLocation); o.SetDocumentation(Documentation); o.SetReturnValue(mf.ConstructMember(ReturnType), true); _property.AddOverload(o); - - return _property; } } } diff --git a/src/Caching/Impl/Models/TypeVarModel.cs b/src/Caching/Impl/Models/TypeVarModel.cs index a3f5c4fc5..e630d801f 100644 --- a/src/Caching/Impl/Models/TypeVarModel.cs +++ b/src/Caching/Impl/Models/TypeVarModel.cs @@ -45,9 +45,11 @@ public static TypeVarModel FromGeneric(IVariable v) { }; } - protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) => new GenericTypeParameter(Name, mf.Module, Constraints.Select(mf.ConstructType).ToArray(), mf.ConstructType(Bound), Covariant, Contravariant, default); + + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { } } } diff --git a/src/Caching/Impl/Models/VariableModel.cs b/src/Caching/Impl/Models/VariableModel.cs index c3e2696cf..892dac8ac 100644 --- a/src/Caching/Impl/Models/VariableModel.cs +++ b/src/Caching/Impl/Models/VariableModel.cs @@ -23,7 +23,7 @@ namespace Microsoft.Python.Analysis.Caching.Models { [Serializable] [DebuggerDisplay("v:{Name} = {Value}")] - internal sealed class VariableModel: MemberModel { + internal sealed class VariableModel : MemberModel { public string Value { get; set; } public static VariableModel FromVariable(IVariable v) => new VariableModel { @@ -49,9 +49,11 @@ internal sealed class VariableModel: MemberModel { Value = t.GetPersistentQualifiedName() }; - protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) { + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { var m = mf.ConstructMember(Value) ?? mf.Module.Interpreter.UnknownType; return new Variable(Name, m, VariableSource.Declaration, new Location(mf.Module, IndexSpan?.ToSpan() ?? default)); } + + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { } } } diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index 97af51827..a58ee0118 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -14,12 +14,15 @@ // permissions and limitations under the License. using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using LiteDB; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; @@ -27,9 +30,12 @@ using Microsoft.Python.Core.Logging; namespace Microsoft.Python.Analysis.Caching { - public sealed class ModuleDatabase : IModuleDatabaseService { + internal sealed class ModuleDatabase : IModuleDatabaseService { private const int _databaseFormatVersion = 1; + private readonly Dictionary _dependencies = new Dictionary(); + private readonly object _lock = new object(); + private readonly IServiceContainer _services; private readonly ILogger _log; private readonly IFileSystem _fs; @@ -39,56 +45,56 @@ public ModuleDatabase(IServiceContainer services) { _services = services; _log = services.GetService(); _fs = services.GetService(); - + var cfs = services.GetService(); _databaseFolder = Path.Combine(cfs.CacheFolder, $"analysis.v{_databaseFormatVersion}"); } /// - /// Retrieves module representation from module index database - /// or null if module does not exist. + /// Retrieves dependencies from the module persistent state. /// - /// Module name. If the name is not qualified - /// the module will ge resolved against active Python version. - /// Module file path. - /// Python module. - /// Module storage state - public ModuleStorageState TryCreateModule(string moduleName, string filePath, out IPythonModule module) { - module = null; + /// Python module to restore analysis for. + /// Python module dependency provider. + public bool TryRestoreDependencies(IPythonModule module, out IDependencyProvider dp) { + dp = null; - if (GetCachingLevel() == AnalysisCachingLevel.None) { - return ModuleStorageState.DoesNotExist; + if (GetCachingLevel() == AnalysisCachingLevel.None || !module.ModuleType.CanBeCached()) { + return false; } - // We don't cache results here. Module resolution service decides when to call in here - // and it is responsible of overall management of the loaded Python modules. - for (var retries = 50; retries > 0; --retries) { - try { - // TODO: make combined db rather than per module? - var dbPath = FindDatabaseFile(moduleName, filePath); - if (string.IsNullOrEmpty(dbPath)) { - return ModuleStorageState.DoesNotExist; - } + lock (_lock) { + if (_dependencies.TryGetValue(module.Name, out dp)) { + return true; + } + if (FindModuleModel(module.Name, module.FilePath, out var model)) { + dp = new DependencyProvider(module, model); + _dependencies[module.Name] = dp; + return true; + } + } + return false; + } - using (var db = new LiteDatabase(dbPath)) { - if (!db.CollectionExists("modules")) { - return ModuleStorageState.Corrupted; - } + /// + /// Creates global scope from module persistent state. + /// Global scope is then can be used to construct module analysis. + /// + /// Python module to restore analysis for. + /// Python module global scope. + public bool TryRestoreGlobalScope(IPythonModule module, out IRestoredGlobalScope gs) { + gs = null; - var modules = db.GetCollection("modules"); - var model = modules.Find(m => m.Name == moduleName).FirstOrDefault(); - if (model == null) { - return ModuleStorageState.DoesNotExist; - } + if (GetCachingLevel() == AnalysisCachingLevel.None || !module.ModuleType.CanBeCached()) { + return false; + } - module = new PythonDbModule(model, filePath, _services); - return ModuleStorageState.Complete; - } - } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) { - Thread.Sleep(10); + lock (_lock) { + if (FindModuleModel(module.Name, module.FilePath, out var model)) { + gs = new RestoredGlobalScope(model, module); } } - return ModuleStorageState.DoesNotExist; + + return gs != null; } /// @@ -101,14 +107,16 @@ public Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationTok /// Determines if module analysis exists in the storage. /// public bool ModuleExistsInStorage(string moduleName, string filePath) { - if(GetCachingLevel() == AnalysisCachingLevel.None) { + if (GetCachingLevel() == AnalysisCachingLevel.None) { return false; } for (var retries = 50; retries > 0; --retries) { try { - var dbPath = FindDatabaseFile(moduleName, filePath); - return !string.IsNullOrEmpty(dbPath); + lock (_lock) { + var dbPath = FindDatabaseFile(moduleName, filePath); + return !string.IsNullOrEmpty(dbPath); + } } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) { Thread.Sleep(10); } @@ -116,9 +124,15 @@ public bool ModuleExistsInStorage(string moduleName, string filePath) { return false; } + public void Clear() { + lock (_lock) { + _dependencies.Clear(); + } + } + private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken cancellationToken = default) { var cachingLevel = GetCachingLevel(); - if(cachingLevel == AnalysisCachingLevel.None) { + if (cachingLevel == AnalysisCachingLevel.None) { return; } @@ -130,24 +144,26 @@ private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken c Exception ex = null; for (var retries = 50; retries > 0; --retries) { - cancellationToken.ThrowIfCancellationRequested(); - try { - if (!_fs.DirectoryExists(_databaseFolder)) { - _fs.CreateDirectory(_databaseFolder); - } - + lock (_lock) { cancellationToken.ThrowIfCancellationRequested(); - using (var db = new LiteDatabase(Path.Combine(_databaseFolder, $"{model.UniqueId}.db"))) { - var modules = db.GetCollection("modules"); - modules.Upsert(model); - return; + try { + if (!_fs.DirectoryExists(_databaseFolder)) { + _fs.CreateDirectory(_databaseFolder); + } + + cancellationToken.ThrowIfCancellationRequested(); + using (var db = new LiteDatabase(Path.Combine(_databaseFolder, $"{model.UniqueId}.db"))) { + var modules = db.GetCollection("modules"); + modules.Upsert(model); + return; + } + } catch (Exception ex1) when (ex1 is IOException || ex1 is UnauthorizedAccessException) { + ex = ex1; + Thread.Sleep(10); + } catch (Exception ex2) { + ex = ex2; + break; } - } catch (Exception ex1) when (ex1 is IOException || ex1 is UnauthorizedAccessException) { - ex = ex1; - Thread.Sleep(10); - } catch (Exception ex2) { - ex = ex2; - break; } } @@ -191,7 +207,52 @@ private string FindDatabaseFile(string moduleName, string filePath) { return _fs.FileExists(dbPath) ? dbPath : null; } + private bool FindModuleModel(string moduleName, string filePath, out ModuleModel model) { + model = null; + + // We don't cache results here. Module resolution service decides when to call in here + // and it is responsible of overall management of the loaded Python modules. + for (var retries = 50; retries > 0; --retries) { + try { + // TODO: make combined db rather than per module? + var dbPath = FindDatabaseFile(moduleName, filePath); + if (string.IsNullOrEmpty(dbPath)) { + return false; + } + + using (var db = new LiteDatabase(dbPath)) { + if (!db.CollectionExists("modules")) { + return false; + } + + var modules = db.GetCollection("modules"); + model = modules.Find(m => m.Name == moduleName).FirstOrDefault(); + return model != null; + } + } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) { + Thread.Sleep(10); + } + } + return false; + } private AnalysisCachingLevel GetCachingLevel() => _services.GetService()?.Options.AnalysisCachingLevel ?? AnalysisCachingLevel.None; + + private sealed class DependencyProvider : IDependencyProvider { + private readonly HashSet _dependencies; + + public DependencyProvider(IPythonModule module, ModuleModel model) { + var dc = new DependencyCollector(module); + foreach (var imp in model.Imports) { + dc.AddImport(imp.ModuleNames, imp.ForceAbsolute); + } + foreach (var fi in model.FromImports) { + dc.AddFromImport(fi.RootNames, fi.DotCount, fi.ForceAbsolute); + } + _dependencies = dc.Dependencies; + } + + public HashSet GetDependencies() => _dependencies; + } } } diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index 10d0f1651..4fad9d9da 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -31,22 +31,26 @@ namespace Microsoft.Python.Analysis.Caching { /// Constructs module from its persistent model. /// internal sealed class ModuleFactory { + /// For use in tests so missing members will assert. + internal static bool EnableMissingMemberAssertions { get; set; } + // TODO: better resolve circular references. - private readonly ReentrancyGuard _typeReentrancy = new ReentrancyGuard(); private readonly ReentrancyGuard _moduleReentrancy = new ReentrancyGuard(); private readonly ModuleModel _model; + private readonly IGlobalScope _gs; public IPythonModule Module { get; } public Location DefaultLocation { get; } - public ModuleFactory(ModuleModel model, IPythonModule module) { + public ModuleFactory(ModuleModel model, IPythonModule module, IGlobalScope gs) { _model = model; - + _gs = gs; Module = module; DefaultLocation = new Location(Module); } - public IPythonType ConstructType(string qualifiedName) => ConstructMember(qualifiedName)?.GetPythonType(); + public IPythonType ConstructType(string qualifiedName) + => ConstructMember(qualifiedName)?.GetPythonType(); public IMember ConstructMember(string qualifiedName) { // Determine module name, member chain and if this is an instance. @@ -101,8 +105,9 @@ private IMember GetMemberFromThisModule(IReadOnlyList memberNames) { return null; } - m = nextModel.Construct(this, declaringType); + m = nextModel.Create(this, declaringType, _gs); Debug.Assert(m != null); + if (m is IGenericType gt && typeArgs.Count > 0) { m = gt.CreateSpecificType(new ArgumentSet(typeArgs, null, null)); } @@ -136,13 +141,13 @@ private IPythonModule GetModule(QualifiedNameParts parts) { // from the stub and the main module was never loaded. This, for example, // happens with io which has member with mmap type coming from mmap // stub rather than the primary mmap module. - var m = Module.Interpreter.ModuleResolution.GetImportedModule(parts.ModuleName); - // Try stub-only case (ex _importlib_modulespec). - m = m ?? Module.Interpreter.TypeshedResolution.GetImportedModule(parts.ModuleName); + var m = parts.IsStub + ? Module.Interpreter.TypeshedResolution.GetImportedModule(parts.ModuleName) + : Module.Interpreter.ModuleResolution.GetImportedModule(parts.ModuleName); + if (m != null) { return parts.ObjectType == ObjectType.VariableModule ? new PythonVariableModule(m) : m; } - return null; } } @@ -175,7 +180,7 @@ private IMember GetMember(IMember root, IEnumerable memberNames) { } if (member == null) { - Debug.Assert(member != null); + Debug.Assert(member != null || EnableMissingMemberAssertions == false); break; } diff --git a/src/Caching/Impl/Properties/AssemblyInfo.cs b/src/Caching/Impl/Properties/AssemblyInfo.cs index 9d32fae0c..d2256d3fe 100644 --- a/src/Caching/Impl/Properties/AssemblyInfo.cs +++ b/src/Caching/Impl/Properties/AssemblyInfo.cs @@ -16,3 +16,4 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Microsoft.Python.Analysis.Caching.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("Microsoft.Python.LanguageServer, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/src/Caching/Impl/PythonDbModule.cs b/src/Caching/Impl/PythonDbModule.cs index addc97053..8da9634ed 100644 --- a/src/Caching/Impl/PythonDbModule.cs +++ b/src/Caching/Impl/PythonDbModule.cs @@ -28,17 +28,17 @@ internal sealed class PythonDbModule : SpecializedModule { public PythonDbModule(ModuleModel model, string filePath, IServiceContainer services) : base(model.Name, filePath, services) { - - var gs = new GlobalScope(model, this); - GlobalScope = gs; - gs.ReconstructVariables(); - Documentation = model.Documentation; - _newLines = model.NewLines.Select(nl => new NewLineLocation(nl.EndIndex, nl.Kind)).ToArray(); _fileSize = model.FileSize; } + public void Construct(ModuleModel model) { + var gs = new RestoredGlobalScope(model, this); + GlobalScope = gs; + gs.ReconstructVariables(); + } + protected override string LoadContent() => string.Empty; public override string Documentation { get; } diff --git a/src/Caching/Impl/QualifiedNameParts.cs b/src/Caching/Impl/QualifiedNameParts.cs index defe456fd..4d1d33539 100644 --- a/src/Caching/Impl/QualifiedNameParts.cs +++ b/src/Caching/Impl/QualifiedNameParts.cs @@ -29,6 +29,8 @@ internal struct QualifiedNameParts { public ObjectType ObjectType; /// Module name. public string ModuleName; + /// Indicates if module is a stub. + public bool IsStub; /// Module member names such as 'A', 'B', 'C' from module:A.B.C. public IReadOnlyList MemberNames; } diff --git a/src/Caching/Impl/GlobalScope.cs b/src/Caching/Impl/RestoredGlobalScope.cs similarity index 67% rename from src/Caching/Impl/GlobalScope.cs rename to src/Caching/Impl/RestoredGlobalScope.cs index 47808dccc..43170d1f4 100644 --- a/src/Caching/Impl/GlobalScope.cs +++ b/src/Caching/Impl/RestoredGlobalScope.cs @@ -22,44 +22,45 @@ using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Caching { - internal sealed class GlobalScope : IGlobalScope { + internal sealed class RestoredGlobalScope : IRestoredGlobalScope { private readonly VariableCollection _scopeVariables = new VariableCollection(); - private ModuleModel _model; + private ModuleModel _model; // Non-readonly b/c of DEBUG conditional. + private ModuleFactory _factory; // Non-readonly b/c of DEBUG conditional. - public GlobalScope(ModuleModel model, IPythonModule module) { - _model = model; - Module = module; + public RestoredGlobalScope(ModuleModel model, IPythonModule module) { + _model = model ?? throw new ArgumentNullException(nameof(model)); + Module = module ?? throw new ArgumentNullException(nameof(module)); Name = model.Name; + _factory = new ModuleFactory(_model, Module, this); + DeclareVariables(); } public void ReconstructVariables() { + var models = _model.TypeVars.Concat(_model.NamedTuples).Concat(_model.Classes).Concat(_model.Functions); + foreach (var m in models.Concat(_model.Variables)) { + m.Populate(_factory, null, this); + } + // TODO: re-declare __doc__, __name__, etc. +#if !DEBUG + _model = null; + _factory = null; +#endif + } + + private void DeclareVariables() { // Member creation may be non-linear. Consider function A returning instance // of a class or type info of a function which hasn't been created yet. // Thus first create members so we can find then, then populate them with content. - var mf = new ModuleFactory(_model, Module); - foreach (var tvm in _model.TypeVars) { - var t = tvm.Construct(mf, null); - _scopeVariables.DeclareVariable(tvm.Name, t, VariableSource.Generic, mf.DefaultLocation); - } - foreach (var ntm in _model.NamedTuples) { - var nt = ntm.Construct(mf, null); - _scopeVariables.DeclareVariable(ntm.Name, nt, VariableSource.Declaration, mf.DefaultLocation); - } - foreach (var cm in _model.Classes) { - var cls = cm.Construct(mf, null); - _scopeVariables.DeclareVariable(cm.Name, cls, VariableSource.Declaration, mf.DefaultLocation); - } - foreach (var fm in _model.Functions) { - var ft = fm.Construct(mf, null); - _scopeVariables.DeclareVariable(fm.Name, ft, VariableSource.Declaration, mf.DefaultLocation); + var mf = new ModuleFactory(_model, Module, this); + var models = _model.TypeVars.Concat(_model.NamedTuples).Concat(_model.Classes).Concat(_model.Functions); + + foreach (var m in models) { + _scopeVariables.DeclareVariable(m.Name, m.Create(mf, null, this), VariableSource.Generic, mf.DefaultLocation); } foreach (var vm in _model.Variables) { - var v = (IVariable)vm.Construct(mf, null); + var v = (IVariable)vm.Create(mf, null, this); _scopeVariables.DeclareVariable(vm.Name, v.Value, VariableSource.Declaration, mf.DefaultLocation); } - - // TODO: re-declare __doc__, __name__, etc. - _model = null; } #region IScope diff --git a/src/Caching/Impl/TypeNames.cs b/src/Caching/Impl/TypeNames.cs index c1e1e7f0e..2d12f482a 100644 --- a/src/Caching/Impl/TypeNames.cs +++ b/src/Caching/Impl/TypeNames.cs @@ -17,7 +17,6 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Python.Analysis.Modules; -using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; @@ -103,6 +102,7 @@ private static void GetModuleNameAndMembers(string qualifiedName, ref QualifiedN default: parts.ModuleName = typeName; parts.MemberNames = Array.Empty(); + DetermineModuleType(ref parts); break; } return; @@ -112,6 +112,15 @@ private static void GetModuleNameAndMembers(string qualifiedName, ref QualifiedN parts.ModuleName = typeName.Substring(0, moduleSeparatorIndex); var memberNamesOffset = parts.ModuleName.Length + 1; parts.MemberNames = GetTypeNames(typeName.Substring(memberNamesOffset), '.'); + + DetermineModuleType(ref parts); + } + + private static void DetermineModuleType(ref QualifiedNameParts parts) { + if (parts.ModuleName.EndsWith("(stub)")) { + parts.ModuleName = parts.ModuleName.Substring(0, parts.ModuleName.Length - 6); + parts.IsStub = true; + } } public static IReadOnlyList GetTypeNames(string qualifiedTypeName, char separator) { diff --git a/src/Caching/Test/AnalysisCachingTestBase.cs b/src/Caching/Test/AnalysisCachingTestBase.cs index 34927e303..4da2db6b3 100644 --- a/src/Caching/Test/AnalysisCachingTestBase.cs +++ b/src/Caching/Test/AnalysisCachingTestBase.cs @@ -16,13 +16,22 @@ using System.IO; using System.Reflection; using System.Text; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions; using Microsoft.Python.Analysis.Tests; +using Microsoft.Python.Analysis.Types; using Newtonsoft.Json; using TestUtilities; namespace Microsoft.Python.Analysis.Caching.Tests { public abstract class AnalysisCachingTestBase: AnalysisTestBase { - protected string ToJson(object model) { + protected AnalysisCachingTestBase() { + ModuleFactory.EnableMissingMemberAssertions = true; + } + + protected static string ToJson(object model) { var sb = new StringBuilder(); using (var sw = new StringWriter(sb)) using (var jw = new JsonTextWriter(sw)) { @@ -45,5 +54,48 @@ protected string GetBaselineFileName(string testName, string suffix = null) => Path.ChangeExtension(suffix == null ? Path.Combine(BaselineFilesFolder, testName) : Path.Combine(BaselineFilesFolder, testName + suffix), "json"); + + internal PythonDbModule CreateDbModule(ModuleModel model, string modulePath) { + var dbModule = new PythonDbModule(model, modulePath, Services); + dbModule.Construct(model); + return dbModule; + } + + internal async Task CompareBaselineAndRestoreAsync(ModuleModel model, IPythonModule m) { + //var json = ToJson(model); + //Baseline.CompareToFile(BaselineFileName, json); + + // In real case dependency analysis will restore model dependencies. + // Here we don't go through the dependency analysis so we have to + // manually restore dependent modules. + foreach (var imp in model.Imports) { + foreach (var name in imp.ModuleNames) { + m.Interpreter.ModuleResolution.GetOrLoadModule(name); + } + } + foreach (var imp in model.FromImports) { + foreach (var name in imp.RootNames) { + m.Interpreter.ModuleResolution.GetOrLoadModule(name); + } + } + + foreach (var imp in model.StubImports) { + foreach (var name in imp.ModuleNames) { + m.Interpreter.TypeshedResolution.GetOrLoadModule(name); + } + } + foreach (var imp in model.StubFromImports) { + foreach (var name in imp.RootNames) { + m.Interpreter.TypeshedResolution.GetOrLoadModule(name); + } + } + + var analyzer = Services.GetService(); + await analyzer.WaitForCompleteAnalysisAsync(); + + using (var dbModule = CreateDbModule(model, m.FilePath)) { + dbModule.Should().HaveSameMembersAs(m); + } + } } } diff --git a/src/Caching/Test/ClassesTests.cs b/src/Caching/Test/ClassesTests.cs index 3aa6c225c..b046213fb 100644 --- a/src/Caching/Test/ClassesTests.cs +++ b/src/Caching/Test/ClassesTests.cs @@ -101,7 +101,7 @@ def func(): //var json = ToJson(model); //Baseline.CompareToFile(BaselineFileName, json); - using (var dbModule = new PythonDbModule(model, analysis.Document.FilePath, Services)) { + using (var dbModule = CreateDbModule(model, analysis.Document.FilePath)) { dbModule.Should().HaveSameMembersAs(analysis.Document); } } @@ -128,7 +128,7 @@ def value(self): //var json = ToJson(model); //Baseline.CompareToFile(BaselineFileName, json); - using (var dbModule = new PythonDbModule(model, analysis.Document.FilePath, Services)) { + using (var dbModule = CreateDbModule(model, analysis.Document.FilePath)) { dbModule.Should().HaveSameMembersAs(analysis.Document); } } diff --git a/src/Caching/Test/Files/ClassOwnDocumentation.json b/src/Caching/Test/Files/ClassOwnDocumentation.json index f6a6c7d35..647537712 100644 --- a/src/Caching/Test/Files/ClassOwnDocumentation.json +++ b/src/Caching/Test/Files/ClassOwnDocumentation.json @@ -191,8 +191,12 @@ } ], "FileSize": 115, + "Imports": [], + "FromImports": [], + "StubImports": [], + "StubFromImports": [], "Id": -2131035837, "Name": "module", - "QualifiedName": null, + "QualifiedName": "module", "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/Files/MemberLocations.json b/src/Caching/Test/Files/MemberLocations.json index ec98ba59b..bc8123835 100644 --- a/src/Caching/Test/Files/MemberLocations.json +++ b/src/Caching/Test/Files/MemberLocations.json @@ -362,8 +362,12 @@ } ], "FileSize": 288, + "Imports": [], + "FromImports": [], + "StubImports": [], + "StubFromImports": [], "Id": -2131035837, "Name": "module", - "QualifiedName": null, + "QualifiedName": "module", "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/Files/NestedClasses.json b/src/Caching/Test/Files/NestedClasses.json index a4de61c16..efb7c122d 100644 --- a/src/Caching/Test/Files/NestedClasses.json +++ b/src/Caching/Test/Files/NestedClasses.json @@ -413,8 +413,12 @@ } ], "FileSize": 353, + "Imports": [], + "FromImports": [], + "StubImports": [], + "StubFromImports": [], "Id": -2131035837, "Name": "module", - "QualifiedName": null, + "QualifiedName": "module", "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/Files/SmokeTest.json b/src/Caching/Test/Files/SmokeTest.json index 5fc51e763..8774ad298 100644 --- a/src/Caching/Test/Files/SmokeTest.json +++ b/src/Caching/Test/Files/SmokeTest.json @@ -310,8 +310,12 @@ } ], "FileSize": 243, + "Imports": [], + "FromImports": [], + "StubImports": [], + "StubFromImports": [], "Id": -2131035837, "Name": "module", - "QualifiedName": null, + "QualifiedName": "module", "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/Files/VersionHandling2.json b/src/Caching/Test/Files/VersionHandling2.json index 02e6c7f13..ee4d02e74 100644 --- a/src/Caching/Test/Files/VersionHandling2.json +++ b/src/Caching/Test/Files/VersionHandling2.json @@ -137,8 +137,12 @@ } ], "FileSize": 91, + "Imports": [], + "FromImports": [], + "StubImports": [], + "StubFromImports": [], "Id": -2131035837, "Name": "module", - "QualifiedName": null, + "QualifiedName": "module", "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/Files/VersionHandling3.json b/src/Caching/Test/Files/VersionHandling3.json index 0ee5f389c..1d92c7ff2 100644 --- a/src/Caching/Test/Files/VersionHandling3.json +++ b/src/Caching/Test/Files/VersionHandling3.json @@ -149,8 +149,12 @@ } ], "FileSize": 91, + "Imports": [], + "FromImports": [], + "StubImports": [], + "StubFromImports": [], "Id": -2131035837, "Name": "module", - "QualifiedName": null, + "QualifiedName": "module", "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/LibraryModulesTests.cs b/src/Caching/Test/LibraryModulesTests.cs index 9f9b710ff..ec4ad2e7d 100644 --- a/src/Caching/Test/LibraryModulesTests.cs +++ b/src/Caching/Test/LibraryModulesTests.cs @@ -20,7 +20,6 @@ using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; @@ -181,7 +180,7 @@ public async Task Builtins() { [TestMethod, Priority(0)] public Task Pickle() => TestModule("pickle"); - + [TestMethod, Priority(0)] public Task Pipes() => TestModule("pipes"); @@ -295,7 +294,7 @@ import requests // Verify this looks like a version. new Version(u.Substring(open + 1, u.IndexOf(')') - open - 1)); - CompareBaselineAndRestore(model, rq); + await CompareBaselineAndRestoreAsync(model, rq); } private async Task TestModule(string name) { @@ -303,16 +302,7 @@ private async Task TestModule(string name) { var m = analysis.Document.Interpreter.ModuleResolution.GetImportedModule(name); var model = ModuleModel.FromAnalysis(m.Analysis, Services, AnalysisCachingLevel.Library); - CompareBaselineAndRestore(model, m); - } - - private void CompareBaselineAndRestore(ModuleModel model, IPythonModule m) { - //var json = ToJson(model); - //Baseline.CompareToFile(BaselineFileName, json); - - using (var dbModule = new PythonDbModule(model, m.FilePath, Services)) { - dbModule.Should().HaveSameMembersAs(m); - } + await CompareBaselineAndRestoreAsync(model, m); } } } diff --git a/src/Caching/Test/ReferencesTests.cs b/src/Caching/Test/ReferencesTests.cs index 867cd95c3..76b9ac803 100644 --- a/src/Caching/Test/ReferencesTests.cs +++ b/src/Caching/Test/ReferencesTests.cs @@ -15,13 +15,12 @@ using System.Threading.Tasks; using FluentAssertions; -using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions; using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions; +using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.VisualStudio.TestTools.UnitTesting; -using NSubstitute; using TestUtilities; -using Microsoft.Python.Analysis.Modules; namespace Microsoft.Python.Analysis.Caching.Tests { [TestClass] @@ -67,6 +66,8 @@ def methodB2(self): Baseline.CompareToFile(BaselineFileName, json); using (var dbModule = new PythonDbModule(model, analysis.Document.FilePath, Services)) { + dbModule.Construct(model); + var sum = dbModule.GetMember("sum") as IPythonFunctionType; sum.Should().NotBeNull(); sum.Definition.Span.Should().Be(4, 5, 4, 8); @@ -103,23 +104,16 @@ import logging var logging = analysis.Document.Interpreter.ModuleResolution.GetImportedModule("logging"); var model = ModuleModel.FromAnalysis(logging.Analysis, Services, AnalysisCachingLevel.Library); - var dbModule = new PythonDbModule(model, logging.FilePath, Services); - analysis.Document.Interpreter.ModuleResolution.SpecializeModule("logging", x => dbModule, replaceExisting: true); - - var moduleName = $"{analysis.Document.Name}_db.py"; - var modulePath = TestData.GetTestSpecificPath(moduleName); - analysis = await GetAnalysisAsync(code, Services, moduleName, modulePath); - - var v = analysis.Should().HaveVariable("logging").Which; - var vm = v.Value.Should().BeOfType().Which; - var m = vm.Module.Should().BeOfType().Which; + await CompareBaselineAndRestoreAsync(model, logging); - var critical = m.GetMember("critical") as IPythonFunctionType; - critical.Should().NotBeNull(); + using (var m = CreateDbModule(model, logging.FilePath)) { + var critical = m.GetMember("critical") as IPythonFunctionType; + critical.Should().NotBeNull(); - var span = critical.Definition.Span; - span.Start.Line.Should().BeGreaterThan(1000); - (span.End.Column - span.Start.Column).Should().Be("critical".Length); + var span = critical.Definition.Span; + span.Start.Line.Should().BeGreaterThan(1000); + (span.End.Column - span.Start.Column).Should().Be("critical".Length); + } } } } diff --git a/src/Core/Impl/OS/ProcessServices.cs b/src/Core/Impl/OS/ProcessServices.cs index 1f74efd40..e9a7c5b72 100644 --- a/src/Core/Impl/OS/ProcessServices.cs +++ b/src/Core/Impl/OS/ProcessServices.cs @@ -39,20 +39,20 @@ public IProcess Start(string path) { public async Task ExecuteAndCaptureOutputAsync(ProcessStartInfo startInfo, CancellationToken cancellationToken = default) { var output = string.Empty; - var process = Start(startInfo); + using (var process = Start(startInfo)) { - if (startInfo.RedirectStandardError && process is PlatformProcess p) { - p.Process.ErrorDataReceived += (s, e) => { }; - p.Process.BeginErrorReadLine(); - } + if (startInfo.RedirectStandardError && process is PlatformProcess p) { + p.Process.ErrorDataReceived += (s, e) => { }; + p.Process.BeginErrorReadLine(); + } - try { - output = await process.StandardOutput.ReadToEndAsync(); - await process.WaitForExitAsync(30000, cancellationToken); - } catch (IOException) { - } catch (OperationCanceledException) { } + try { + output = await process.StandardOutput.ReadToEndAsync(); + await process.WaitForExitAsync(30000, cancellationToken); + } catch (IOException) { } catch (OperationCanceledException) { } - return output; + return output; + } } } } diff --git a/src/Core/Test/PriorityProducerConsumerTest.cs b/src/Core/Test/PriorityProducerConsumerTest.cs index 62c755ca6..a0a5132ac 100644 --- a/src/Core/Test/PriorityProducerConsumerTest.cs +++ b/src/Core/Test/PriorityProducerConsumerTest.cs @@ -24,95 +24,102 @@ namespace Microsoft.Python.Core.Tests { public class PriorityProducerConsumerTest { [TestMethod, Priority(0)] public void PriorityProducerConsumer_NoPending() { - var ppc = new PriorityProducerConsumer(); - ppc.Produce(5); - var consumerTask = ppc.ConsumeAsync(); - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); - Assert.AreEqual(5, consumerTask.Result); + using (var ppc = new PriorityProducerConsumer()) { + ppc.Produce(5); + var consumerTask = ppc.ConsumeAsync(); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); + Assert.AreEqual(5, consumerTask.Result); + } } [TestMethod, Priority(0)] public void PriorityProducerConsumer_NoPending_Priority1() { - var ppc = new PriorityProducerConsumer(2); - ppc.Produce(5); - ppc.Produce(6, 1); - var consumerTask = ppc.ConsumeAsync(); - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); - Assert.AreEqual(5, consumerTask.Result); + using (var ppc = new PriorityProducerConsumer(2)) { + ppc.Produce(5); + ppc.Produce(6, 1); + var consumerTask = ppc.ConsumeAsync(); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); + Assert.AreEqual(5, consumerTask.Result); + } } [TestMethod, Priority(0)] public void PriorityProducerConsumer_NoPending_Priority2() { - var ppc = new PriorityProducerConsumer(2); - ppc.Produce(6, 1); - ppc.Produce(5); - var consumerTask = ppc.ConsumeAsync(); - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); - Assert.AreEqual(5, consumerTask.Result); + using (var ppc = new PriorityProducerConsumer(2)) { + ppc.Produce(6, 1); + ppc.Produce(5); + var consumerTask = ppc.ConsumeAsync(); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); + Assert.AreEqual(5, consumerTask.Result); + } } [TestMethod, Priority(0)] public void PriorityProducerConsumer_NoPending_Duplicates1() { - var ppc = new PriorityProducerConsumer(3, true); - ppc.Produce(5, 2); - ppc.Produce(6, 1); - ppc.Produce(5); - var consumerTask1 = ppc.ConsumeAsync(); - var consumerTask2 = ppc.ConsumeAsync(); - var consumerTask3 = ppc.ConsumeAsync(); - - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status); - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status); - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status); - Assert.AreEqual(5, consumerTask1.Result); - Assert.AreEqual(6, consumerTask2.Result); + using (var ppc = new PriorityProducerConsumer(3, true)) { + ppc.Produce(5, 2); + ppc.Produce(6, 1); + ppc.Produce(5); + var consumerTask1 = ppc.ConsumeAsync(); + var consumerTask2 = ppc.ConsumeAsync(); + var consumerTask3 = ppc.ConsumeAsync(); + + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status); + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status); + Assert.AreEqual(5, consumerTask1.Result); + Assert.AreEqual(6, consumerTask2.Result); + } } [TestMethod, Priority(0)] public void PriorityProducerConsumer_NoPending_Duplicates2() { - var ppc = new PriorityProducerConsumer(3, true); - ppc.Produce(5); - ppc.Produce(6, 1); - ppc.Produce(5, 2); - var consumerTask1 = ppc.ConsumeAsync(); - var consumerTask2 = ppc.ConsumeAsync(); - var consumerTask3 = ppc.ConsumeAsync(); - - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status); - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status); - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status); - Assert.AreEqual(5, consumerTask1.Result); - Assert.AreEqual(6, consumerTask2.Result); + using (var ppc = new PriorityProducerConsumer(3, true)) { + ppc.Produce(5); + ppc.Produce(6, 1); + ppc.Produce(5, 2); + var consumerTask1 = ppc.ConsumeAsync(); + var consumerTask2 = ppc.ConsumeAsync(); + var consumerTask3 = ppc.ConsumeAsync(); + + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status); + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status); + Assert.AreEqual(5, consumerTask1.Result); + Assert.AreEqual(6, consumerTask2.Result); + } } [TestMethod, Priority(0)] public void PriorityProducerConsumer_NoPending_Duplicates3() { - var ppc = new PriorityProducerConsumer(3, true); - ppc.Produce(5, 1); - ppc.Produce(6, 1); - ppc.Produce(5, 1); - var consumerTask1 = ppc.ConsumeAsync(); - var consumerTask2 = ppc.ConsumeAsync(); - var consumerTask3 = ppc.ConsumeAsync(); - - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status); - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status); - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status); - Assert.AreEqual(6, consumerTask1.Result); - Assert.AreEqual(5, consumerTask2.Result); + using (var ppc = new PriorityProducerConsumer(3, true)) { + ppc.Produce(5, 1); + ppc.Produce(6, 1); + ppc.Produce(5, 1); + var consumerTask1 = ppc.ConsumeAsync(); + var consumerTask2 = ppc.ConsumeAsync(); + var consumerTask3 = ppc.ConsumeAsync(); + + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status); + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status); + Assert.AreEqual(6, consumerTask1.Result); + Assert.AreEqual(5, consumerTask2.Result); + } } [TestMethod, Priority(0)] public async Task PriorityProducerConsumer_Pending() { - var ppc = new PriorityProducerConsumer(); - var consumerTask = ppc.ConsumeAsync(); - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status); + using (var ppc = new PriorityProducerConsumer()) { + var consumerTask = ppc.ConsumeAsync(); + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status); - ppc.Produce(5); - await consumerTask; + ppc.Produce(5); + await consumerTask; - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); - Assert.AreEqual(5, consumerTask.Result); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); + Assert.AreEqual(5, consumerTask.Result); + } } [TestMethod, Priority(0)] @@ -131,69 +138,72 @@ public async Task PriorityProducerConsumer_Pending_Dispose() { [TestMethod, Priority(0)] public async Task PriorityProducerConsumer_Pending_Priority1() { - var ppc = new PriorityProducerConsumer(2); - var consumerTask = ppc.ConsumeAsync(); - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status); + using (var ppc = new PriorityProducerConsumer(2)) { + var consumerTask = ppc.ConsumeAsync(); + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status); - ppc.Produce(5); - ppc.Produce(6, 1); - await consumerTask; + ppc.Produce(5); + ppc.Produce(6, 1); + await consumerTask; - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); - Assert.AreEqual(5, consumerTask.Result); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); + Assert.AreEqual(5, consumerTask.Result); + } } [TestMethod, Priority(0)] public async Task PriorityProducerConsumer_Pending_Priority2() { - var ppc = new PriorityProducerConsumer(2); - var consumerTask1 = ppc.ConsumeAsync(); - var consumerTask2 = ppc.ConsumeAsync(); - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask1.Status); - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask2.Status); + using (var ppc = new PriorityProducerConsumer(2)) { + var consumerTask1 = ppc.ConsumeAsync(); + var consumerTask2 = ppc.ConsumeAsync(); + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask1.Status); + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask2.Status); - ppc.Produce(6, 1); - await consumerTask1; + ppc.Produce(6, 1); + await consumerTask1; - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status); - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask2.Status); - Assert.AreEqual(6, consumerTask1.Result); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status); + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask2.Status); + Assert.AreEqual(6, consumerTask1.Result); - ppc.Produce(5); - await consumerTask2; + ppc.Produce(5); + await consumerTask2; - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status); - Assert.AreEqual(5, consumerTask2.Result); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status); + Assert.AreEqual(5, consumerTask2.Result); + } } [TestMethod, Priority(0)] public async Task PriorityProducerConsumer_Pending_Priority3() { - var ppc = new PriorityProducerConsumer(2); - var values = new int[3]; - var tcsConsumer = new TaskCompletionSource(); - var tcsProducer = new TaskCompletionSource(); - var consumerTask = Task.Run(async () => { - for (var i = 0; i < 3; i++) { - var task = ppc.ConsumeAsync(); - tcsConsumer.TrySetResult(true); - values[i] = await task; - await tcsProducer.Task; - } - }); - - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status); - - await tcsConsumer.Task; - ppc.Produce(5, 1); - ppc.Produce(6, 1); - ppc.Produce(7); - tcsProducer.SetResult(false); - - await consumerTask; - - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); - Assert.AreEqual(5, values[0]); - Assert.AreEqual(7, values[1]); - Assert.AreEqual(6, values[2]); + using (var ppc = new PriorityProducerConsumer(2)) { + var values = new int[3]; + var tcsConsumer = new TaskCompletionSource(); + var tcsProducer = new TaskCompletionSource(); + var consumerTask = Task.Run(async () => { + for (var i = 0; i < 3; i++) { + var task = ppc.ConsumeAsync(); + tcsConsumer.TrySetResult(true); + values[i] = await task; + await tcsProducer.Task; + } + }); + + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status); + + await tcsConsumer.Task; + ppc.Produce(5, 1); + ppc.Produce(6, 1); + ppc.Produce(7); + tcsProducer.SetResult(false); + + await consumerTask; + + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); + Assert.AreEqual(5, values[0]); + Assert.AreEqual(7, values[1]); + Assert.AreEqual(6, values[2]); + } } } } diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index d71ae58e8..00071495d 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -52,91 +52,96 @@ public void Cleanup() { [TestMethod, Priority(0)] public async Task AddsRootDirectoryAsync() { - var context = new IndexTestContext(this); - context.FileWithXVarInRootDir(); - context.AddFileToRoot($"{_rootPath}\foo.py", MakeStream("y = 1")); + using (var context = new IndexTestContext(this)) { + context.FileWithXVarInRootDir(); + context.AddFileToRoot($"{_rootPath}\foo.py", MakeStream("y = 1")); - var indexManager = context.GetDefaultIndexManager(); - await indexManager.IndexWorkspace(); + var indexManager = context.GetDefaultIndexManager(); + await indexManager.IndexWorkspace(); - var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); - symbols.Should().HaveCount(2); + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + symbols.Should().HaveCount(2); + } } [TestMethod, Priority(0)] public async Task IgnoresNonPythonFiles() { - var context = new IndexTestContext(this); - - var nonPythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}/bla.txt"); - context.AddFileInfoToRootTestFS(nonPythonTestFileInfo); + using (var context = new IndexTestContext(this)) { + var nonPythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}/bla.txt"); + context.AddFileInfoToRootTestFS(nonPythonTestFileInfo); - IIndexManager indexManager = context.GetDefaultIndexManager(); - await indexManager.IndexWorkspace(); + var indexManager = context.GetDefaultIndexManager(); + await indexManager.IndexWorkspace(); - context.FileSystem.DidNotReceive().FileExists(nonPythonTestFileInfo.FullName); + context.FileSystem.DidNotReceive().FileExists(nonPythonTestFileInfo.FullName); + } } [TestMethod, Priority(0)] public async Task CanOpenFiles() { - string nonRootPath = "C:/nonRoot"; - var context = new IndexTestContext(this); - var pythonTestFileInfo = MakeFileInfoProxy($"{nonRootPath}/bla.py"); - IDocument doc = DocumentWithAst("x = 1"); + var nonRootPath = "C:/nonRoot"; + using (var context = new IndexTestContext(this)) { + var pythonTestFileInfo = MakeFileInfoProxy($"{nonRootPath}/bla.py"); + var doc = DocumentWithAst("x = 1"); - IIndexManager indexManager = context.GetDefaultIndexManager(); - indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); + var indexManager = context.GetDefaultIndexManager(); + indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); - var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); - SymbolsShouldBeOnlyX(symbols); + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + SymbolsShouldBeOnlyX(symbols); + } } [TestMethod, Priority(0)] public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { - var context = new IndexTestContext(this); - var pythonTestFilePath = context.FileWithXVarInRootDir(); + using (var context = new IndexTestContext(this)) { + var pythonTestFilePath = context.FileWithXVarInRootDir(); - var indexManager = context.GetDefaultIndexManager(); - await indexManager.IndexWorkspace(); + var indexManager = context.GetDefaultIndexManager(); + await indexManager.IndexWorkspace(); - indexManager.ReIndexFile(pythonTestFilePath, DocumentWithAst("y = 1")); + indexManager.ReIndexFile(pythonTestFilePath, DocumentWithAst("y = 1")); - var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); - symbols.Should().HaveCount(1); - symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); - symbols.First().Name.Should().BeEquivalentTo("y"); + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("y"); + } } [TestMethod, Priority(0)] public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { - var context = new IndexTestContext(this); - var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); - context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false); + using (var context = new IndexTestContext(this)) { + var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); + context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false); - var indexManager = context.GetDefaultIndexManager(); - indexManager.ProcessNewFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); - indexManager.ProcessClosedFile(pythonTestFileInfo.FullName); + var indexManager = context.GetDefaultIndexManager(); + indexManager.ProcessNewFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); + indexManager.ProcessClosedFile(pythonTestFileInfo.FullName); - await SymbolIndexShouldBeEmpty(indexManager); + await SymbolIndexShouldBeEmpty(indexManager); + } } [TestMethod, Priority(0)] public async Task CloseWorkspaceFilesReUpdatesIndexAsync() { - var context = new IndexTestContext(this); - var pythonTestFilePath = context.FileWithXVarInRootDir(); - context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); - - var indexManager = context.GetDefaultIndexManager(); - await indexManager.IndexWorkspace(); - - indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("r = 1")); - // It Needs to remake the stream for the file, previous one is closed - context.FileSystem.FileExists(pythonTestFilePath).Returns(true); - context.SetFileOpen(pythonTestFilePath, MakeStream("x = 1")); - context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); - indexManager.ProcessClosedFile(pythonTestFilePath); - - var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); - SymbolsShouldBeOnlyX(symbols); + using (var context = new IndexTestContext(this)) { + var pythonTestFilePath = context.FileWithXVarInRootDir(); + context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); + + var indexManager = context.GetDefaultIndexManager(); + await indexManager.IndexWorkspace(); + + indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("r = 1")); + // It Needs to remake the stream for the file, previous one is closed + context.FileSystem.FileExists(pythonTestFilePath).Returns(true); + context.SetFileOpen(pythonTestFilePath, MakeStream("x = 1")); + context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); + indexManager.ProcessClosedFile(pythonTestFilePath); + + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + SymbolsShouldBeOnlyX(symbols); + } } [TestMethod, Priority(0)] @@ -144,92 +149,97 @@ public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() { // If events get to index manager in the order: [open, close, update] // it should not reindex file - var context = new IndexTestContext(this); - var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); + using (var context = new IndexTestContext(this)) { + var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); - var indexManager = context.GetDefaultIndexManager(); - indexManager.ProcessNewFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); - indexManager.ProcessClosedFile(pythonTestFileInfo.FullName); + var indexManager = context.GetDefaultIndexManager(); + indexManager.ProcessNewFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); + indexManager.ProcessClosedFile(pythonTestFileInfo.FullName); - context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false); - indexManager.ReIndexFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); + context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false); + indexManager.ReIndexFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); - await SymbolIndexShouldBeEmpty(indexManager); + await SymbolIndexShouldBeEmpty(indexManager); + } } [TestMethod, Priority(0)] public async Task WorkspaceSymbolsAddsRootDirectory() { - var context = new IndexTestContext(this); + using (var context = new IndexTestContext(this)) { + var pythonTestFilePath = context.FileWithXVarInRootDir(); - var pythonTestFilePath = context.FileWithXVarInRootDir(); + var indexManager = context.GetDefaultIndexManager(); + await indexManager.IndexWorkspace(); - var indexManager = context.GetDefaultIndexManager(); - await indexManager.IndexWorkspace(); - - var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); - SymbolsShouldBeOnlyX(symbols); + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + SymbolsShouldBeOnlyX(symbols); + } } [TestMethod, Priority(0)] public async Task WorkspaceSymbolsLimited() { - var context = new IndexTestContext(this); + using (var context = new IndexTestContext(this)) { + for (var fileNumber = 0; fileNumber < 10; fileNumber++) { + context.AddFileToRoot($"{_rootPath}\bla{fileNumber}.py", MakeStream($"x{fileNumber} = 1")); + } - for (int fileNumber = 0; fileNumber < 10; fileNumber++) { - context.AddFileToRoot($"{_rootPath}\bla{fileNumber}.py", MakeStream($"x{fileNumber} = 1")); - } - var indexManager = context.GetDefaultIndexManager(); - await indexManager.IndexWorkspace(); + var indexManager = context.GetDefaultIndexManager(); + await indexManager.IndexWorkspace(); - const int amountOfSymbols = 3; + const int amountOfSymbols = 3; - var symbols = await indexManager.WorkspaceSymbolsAsync("", amountOfSymbols); - symbols.Should().HaveCount(amountOfSymbols); + var symbols = await indexManager.WorkspaceSymbolsAsync("", amountOfSymbols); + symbols.Should().HaveCount(amountOfSymbols); + } } [TestMethod, Priority(0)] public async Task HierarchicalDocumentSymbolsAsync() { - var context = new IndexTestContext(this); - var pythonTestFilePath = context.FileWithXVarInRootDir(); + using (var context = new IndexTestContext(this)) { + var pythonTestFilePath = context.FileWithXVarInRootDir(); - var indexManager = context.GetDefaultIndexManager(); - await indexManager.IndexWorkspace(); + var indexManager = context.GetDefaultIndexManager(); + await indexManager.IndexWorkspace(); - var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath); - SymbolsShouldBeOnlyX(symbols); + var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath); + SymbolsShouldBeOnlyX(symbols); + } } [TestMethod, Priority(0)] public async Task LatestVersionASTVersionIsIndexed() { - var context = new IndexTestContext(this); - var pythonTestFilePath = context.FileWithXVarInRootDir(); - - var indexManager = context.GetDefaultIndexManager(); - indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("y = 1")); - indexManager.ProcessClosedFile(pythonTestFilePath); - indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("z = 1")); - - var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath); - symbols.Should().HaveCount(1); - symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); - symbols.First().Name.Should().BeEquivalentTo("z"); + using (var context = new IndexTestContext(this)) { + var pythonTestFilePath = context.FileWithXVarInRootDir(); + + var indexManager = context.GetDefaultIndexManager(); + indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("y = 1")); + indexManager.ProcessClosedFile(pythonTestFilePath); + indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("z = 1")); + + var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath); + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("z"); + } } [TestMethod, Priority(0)] public async Task AddFilesToPendingChanges() { - var context = new IndexTestContext(this); - var f1 = context.AddFileToRoot($"{_rootPath}/fileA.py", MakeStream("")); - var f2 = context.AddFileToRoot($"{_rootPath}/fileB.py", MakeStream("")); + using (var context = new IndexTestContext(this)) { + var f1 = context.AddFileToRoot($"{_rootPath}/fileA.py", MakeStream("")); + var f2 = context.AddFileToRoot($"{_rootPath}/fileB.py", MakeStream("")); - var indexManager = context.GetDefaultIndexManager(); - await indexManager.IndexWorkspace(); + var indexManager = context.GetDefaultIndexManager(); + await indexManager.IndexWorkspace(); - indexManager.AddPendingDoc(DocumentWithAst("y = 1", f1)); - indexManager.AddPendingDoc(DocumentWithAst("x = 1", f2)); + indexManager.AddPendingDoc(DocumentWithAst("y = 1", f1)); + indexManager.AddPendingDoc(DocumentWithAst("x = 1", f2)); - context.SetIdleEvent(Raise.Event()); + context.SetIdleEvent(Raise.Event()); - var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); - symbols.Should().HaveCount(2); + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + symbols.Should().HaveCount(2); + } } private static void SymbolsShouldBeOnlyX(IEnumerable symbols) { @@ -249,8 +259,8 @@ private class IndexTestContext : IDisposable { private readonly List _rootFileList = new List(); private readonly IIdleTimeService _idleTimeService = Substitute.For(); private readonly PythonLanguageVersion _pythonLanguageVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); + private readonly IndexManagerTests _tests; private IIndexManager _indexM; - private IndexManagerTests _tests; public IndexTestContext(IndexManagerTests tests) { _tests = tests; @@ -271,9 +281,7 @@ public void AddFileInfoToRootTestFS(FileInfoProxy fileInfo) { FileSystem.FileExists(fileInfo.FullName).Returns(true); } - public string FileWithXVarInRootDir() { - return AddFileToRoot($"{_rootPath}\bla.py", _tests.MakeStream("x = 1")); - } + public string FileWithXVarInRootDir() => AddFileToRoot($"{_rootPath}\bla.py", _tests.MakeStream("x = 1")); public IIndexManager GetDefaultIndexManager() { _indexM = new IndexManager(FileSystem, _pythonLanguageVersion, @@ -301,19 +309,16 @@ public void SetIdleEvent(EventHandler handler) { private void SetupRootDir() { var directoryInfo = Substitute.For(); directoryInfo.Match("", new string[] { }, new string[] { }).ReturnsForAnyArgs(callInfo => { - string path = callInfo.ArgAt(0); + var path = callInfo.ArgAt(0); return _rootFileList - .Where(fsInfo => PathEqualityComparer.Instance.Equals(fsInfo.FullName, path)) - .Count() > 0; + .Count(fsInfo => PathEqualityComparer.Instance.Equals(fsInfo.FullName, path)) > 0; }); // Doesn't work without 'forAnyArgs' directoryInfo.EnumerateFileSystemInfos(new string[] { }, new string[] { }).ReturnsForAnyArgs(_rootFileList); FileSystem.GetDirectoryInfo(_rootPath).Returns(directoryInfo); } - public void Dispose() { - _indexM?.Dispose(); - } + public void Dispose() => _indexM?.Dispose(); public void SetFileOpen(string pythonTestFilePath, Func returnFunc) { FileSystem.FileOpen(pythonTestFilePath, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(returnFunc); @@ -326,7 +331,7 @@ internal void SetFileOpen(string path, Stream stream) { private IDocument DocumentWithAst(string testCode, string filePath = null) { filePath = filePath ?? $"{_rootPath}/{testCode}.py"; - IDocument doc = Substitute.For(); + var doc = Substitute.For(); doc.GetAstAsync().ReturnsForAnyArgs(Task.FromResult(MakeAst(testCode))); doc.Uri.Returns(new Uri(filePath)); return doc; @@ -338,13 +343,11 @@ private async Task SymbolIndexShouldBeEmpty(IIndexManager indexManager) { } public PythonAst MakeAst(string testCode) { - PythonLanguageVersion latestVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); + var latestVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); return Parser.CreateParser(MakeStream(testCode), latestVersion).ParseFile(); } - public Stream MakeStream(string str) { - return new MemoryStream(Encoding.UTF8.GetBytes(str)); - } + public Stream MakeStream(string str) => new MemoryStream(Encoding.UTF8.GetBytes(str)); public FileInfoProxy MakeFileInfoProxy(string filePath) => new FileInfoProxy(new FileInfo(filePath)); diff --git a/src/LanguageServer/Test/IndexParserTests.cs b/src/LanguageServer/Test/IndexParserTests.cs index ecd86654d..32e66ec26 100644 --- a/src/LanguageServer/Test/IndexParserTests.cs +++ b/src/LanguageServer/Test/IndexParserTests.cs @@ -56,15 +56,15 @@ public async Task ParseVariableInFileAsync() { using (var fileStream = MakeStream("x = 1")) { SetFileOpen(_fileSystem, testFilePath, fileStream); - IIndexParser indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion); - var ast = await indexParser.ParseAsync(testFilePath); - - var symbols = GetIndexSymbols(ast); - symbols.Should().HaveCount(1); - symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); - symbols.First().Name.Should().BeEquivalentTo("x"); + using (IIndexParser indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion)) { + var ast = await indexParser.ParseAsync(testFilePath); + + var symbols = GetIndexSymbols(ast); + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("x"); + } } - } private IReadOnlyList GetIndexSymbols(PythonAst ast) { @@ -81,8 +81,9 @@ public async Task ParseFileThatStopsExisting() { _fileSystem.FileExists(testFilePath).Returns(true); SetFileOpen(_fileSystem, testFilePath, _ => throw new FileNotFoundException()); - IIndexParser indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion); - var ast = await indexParser.ParseAsync(testFilePath); + using (var indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion)) { + await indexParser.ParseAsync(testFilePath); + } } [TestMethod, Priority(0)] @@ -94,26 +95,25 @@ public void CancelParsingAsync() { SetFileOpen(_fileSystem, testFilePath, fileStream); } - IIndexParser indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion); - CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); - cancellationTokenSource.Cancel(); + using (var indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion)) + using (var cancellationTokenSource = new CancellationTokenSource()) { + cancellationTokenSource.Cancel(); - Func parse = async () => { - await indexParser.ParseAsync(testFilePath, cancellationTokenSource.Token); - }; - parse.Should().Throw(); + Func parse = async () => { + await indexParser.ParseAsync(testFilePath, cancellationTokenSource.Token); + }; + parse.Should().Throw(); + } } private void SetFileOpen(IFileSystem fileSystem, string path, Stream stream) { fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(stream); } - private void SetFileOpen(IFileSystem fileSystem, string path, Func p) { - fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(p); - } + private void SetFileOpen(IFileSystem fileSystem, string path, Func p) + => fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(p); - private Stream MakeStream(string str) { - return new MemoryStream(Encoding.UTF8.GetBytes(str)); - } + private Stream MakeStream(string str) + => new MemoryStream(Encoding.UTF8.GetBytes(str)); } } diff --git a/src/LanguageServer/Test/SymbolIndexTests.cs b/src/LanguageServer/Test/SymbolIndexTests.cs index 0a91ba2b0..861eb1cec 100644 --- a/src/LanguageServer/Test/SymbolIndexTests.cs +++ b/src/LanguageServer/Test/SymbolIndexTests.cs @@ -27,6 +27,7 @@ using Microsoft.Python.Parsing.Ast; using Microsoft.Python.Parsing.Tests; using Microsoft.Python.Tests.Utilities.FluentAssertions; +using Microsoft.Python.UnitTests.Core.FluentAssertions; using Microsoft.VisualStudio.TestTools.UnitTesting; using NSubstitute; using TestUtilities; @@ -51,14 +52,15 @@ public void TestCleanup() { [TestMethod, Priority(0)] public async Task IndexHierarchicalDocumentAsync() { - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); - index.Add(path, DocumentWithAst("x = 1")); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); + index.Add(path, DocumentWithAst("x = 1")); - var symbols = await index.HierarchicalDocumentSymbolsAsync(path); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - }); + var symbols = await index.HierarchicalDocumentSymbolsAsync(path); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)) + }); + } } private static ISymbolIndex MakeSymbolIndex() { @@ -67,124 +69,130 @@ private static ISymbolIndex MakeSymbolIndex() { [TestMethod, Priority(0)] public async Task IndexHierarchicalDocumentUpdate() { - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); - index.Add(path, DocumentWithAst("x = 1")); + index.Add(path, DocumentWithAst("x = 1")); - var symbols = await index.HierarchicalDocumentSymbolsAsync(path); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - }); + var symbols = await index.HierarchicalDocumentSymbolsAsync(path); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)) + }); - index.Add(path, DocumentWithAst("y = 1")); + index.Add(path, DocumentWithAst("y = 1")); - symbols = await index.HierarchicalDocumentSymbolsAsync(path); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - }); + symbols = await index.HierarchicalDocumentSymbolsAsync(path); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)) + }); + } } [TestMethod, Priority(0)] public async Task IndexHierarchicalDocumentNotFoundAsync() { - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); - var symbols = await index.HierarchicalDocumentSymbolsAsync(path); - symbols.Should().BeEmpty(); + var symbols = await index.HierarchicalDocumentSymbolsAsync(path); + symbols.Should().BeEmpty(); + } } [TestMethod, Priority(0)] public async Task IndexWorkspaceSymbolsFlattenAsync() { - var code = @"class Foo(object): + const string code = @"class Foo(object): def foo(self, x): ..."; - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); + index.Add(path, DocumentWithAst(code)); - index.Add(path, DocumentWithAst(code)); - - var symbols = await index.WorkspaceSymbolsAsync("", maxSymbols); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new FlatSymbol("Foo", SymbolKind.Class, path, new SourceSpan(1, 7, 1, 10)), - new FlatSymbol("foo", SymbolKind.Method, path, new SourceSpan(2, 9, 2, 12), "Foo"), - new FlatSymbol("self", SymbolKind.Variable, path, new SourceSpan(2, 13, 2, 17), "foo"), - new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(2, 19, 2, 20), "foo"), - }); + var symbols = await index.WorkspaceSymbolsAsync("", maxSymbols); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("Foo", SymbolKind.Class, path, new SourceSpan(1, 7, 1, 10)), + new FlatSymbol("foo", SymbolKind.Method, path, new SourceSpan(2, 9, 2, 12), "Foo"), + new FlatSymbol("self", SymbolKind.Variable, path, new SourceSpan(2, 13, 2, 17), "foo"), + new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(2, 19, 2, 20), "foo") + }); + } } [TestMethod, Priority(0)] public async Task IndexWorkspaceSymbolsFilteredAsync() { - var code = @"class Foo(object): + const string code = @"class Foo(object): def foo(self, x): ..."; - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); - index.Add(path, DocumentWithAst(code)); + index.Add(path, DocumentWithAst(code)); - var symbols = await index.WorkspaceSymbolsAsync("x", maxSymbols); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(2, 19, 2, 20), "foo"), - }); + var symbols = await index.WorkspaceSymbolsAsync("x", maxSymbols); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(2, 19, 2, 20), "foo"), + }); + } } [TestMethod, Priority(0)] public async Task IndexWorkspaceSymbolsNotFoundAsync() { - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); - - var symbols = await index.WorkspaceSymbolsAsync("", maxSymbols); - symbols.Should().BeEmpty(); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); + var symbols = await index.WorkspaceSymbolsAsync("", maxSymbols); + symbols.Should().BeEmpty(); + } } [TestMethod, Priority(0)] public async Task IndexWorkspaceSymbolsCaseInsensitiveAsync() { - var code = @"class Foo(object): + const string code = @"class Foo(object): def foo(self, x): ..."; - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); + index.Add(path, DocumentWithAst(code)); - index.Add(path, DocumentWithAst(code)); - - var symbols = await index.WorkspaceSymbolsAsync("foo", maxSymbols); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new FlatSymbol("Foo", SymbolKind.Class, path, new SourceSpan(1, 7, 1, 10)), - new FlatSymbol("foo", SymbolKind.Method, path, new SourceSpan(2, 9, 2, 12), "Foo"), - }); + var symbols = await index.WorkspaceSymbolsAsync("foo", maxSymbols); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("Foo", SymbolKind.Class, path, new SourceSpan(1, 7, 1, 10)), + new FlatSymbol("foo", SymbolKind.Method, path, new SourceSpan(2, 9, 2, 12), "Foo"), + }); + } } [TestMethod, Priority(0)] public void MarkAsPendingWaitsForUpdates() { - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); - - index.Add(path, DocumentWithAst("x = 1")); - index.MarkAsPending(path); - var cts = new CancellationTokenSource(); - var t = index.HierarchicalDocumentSymbolsAsync(path, cts.Token); - t.IsCompleted.Should().BeFalse(); - cts.Cancel(); - Func cancelled = async () => { - await t; - }; - cancelled.Should().Throw(); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); + + index.Add(path, DocumentWithAst("x = 1")); + index.MarkAsPending(path); + using (var cts = new CancellationTokenSource()) { + var t = index.HierarchicalDocumentSymbolsAsync(path, cts.Token); + t.Should().NotBeCompleted(); + cts.Cancel(); + Func cancelled = async () => { + await t; + }; + cancelled.Should().Throw(); + } + } } [TestMethod, Priority(0)] public async Task SymbolsAfterPendingWaitsForUpdateAsync() { - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); - index.Add(path, DocumentWithAst("x = 1")); - index.MarkAsPending(path); - var t = index.WorkspaceSymbolsAsync("", maxSymbols); - index.ReIndex(path, DocumentWithAst("x = 1")); - var symbols = await t; - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(1, 1, 1, 2)), - }); + index.Add(path, DocumentWithAst("x = 1")); + index.MarkAsPending(path); + var t = index.WorkspaceSymbolsAsync("", maxSymbols); + index.ReIndex(path, DocumentWithAst("x = 1")); + var symbols = await t; + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(1, 1, 1, 2)) + }); + } } private PythonAst GetParse(string code, PythonLanguageVersion version = PythonLanguageVersion.V37) @@ -199,14 +207,14 @@ private IReadOnlyList WalkSymbols(string code, PythonLanguag private IDocument DocumentWithAst(string testCode, string filePath = null) { filePath = filePath ?? $"{_rootPath}/{testCode}.py"; - IDocument doc = Substitute.For(); + var doc = Substitute.For(); doc.GetAstAsync().ReturnsForAnyArgs(Task.FromResult(MakeAst(testCode))); doc.Uri.Returns(new Uri(filePath)); return doc; } private PythonAst MakeAst(string testCode) { - PythonLanguageVersion latestVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); + var latestVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); return Parser.CreateParser(MakeStream(testCode), latestVersion).ParseFile(); } diff --git a/src/UnitTests/Core/Impl/Baseline.cs b/src/UnitTests/Core/Impl/Baseline.cs index 181ebf3f0..74e87d235 100644 --- a/src/UnitTests/Core/Impl/Baseline.cs +++ b/src/UnitTests/Core/Impl/Baseline.cs @@ -54,50 +54,51 @@ public static void CompareStringLines(string expected, string actual) { } public static int CompareLines(string expected, string actual, out string expectedLine, out string actualLine, out int index, bool ignoreCase = false) { - var actualReader = new StringReader(actual); - var expectedReader = new StringReader(expected); + using (var actualReader = new StringReader(actual)) + using (var expectedReader = new StringReader(expected)) { - var lineNum = 1; - index = 0; + var lineNum = 1; + index = 0; - for (; ; lineNum++) { - expectedLine = expectedReader.ReadLine(); - actualLine = actualReader.ReadLine(); + for (;; lineNum++) { + expectedLine = expectedReader.ReadLine(); + actualLine = actualReader.ReadLine(); - if (expectedLine == null || actualLine == null) { - break; - } + if (expectedLine == null || actualLine == null) { + break; + } + + var minLength = Math.Min(expectedLine.Length, actualLine.Length); + for (var i = 0; i < minLength; i++) { + var act = actualLine[i]; + var exp = expectedLine[i]; - var minLength = Math.Min(expectedLine.Length, actualLine.Length); - for (var i = 0; i < minLength; i++) { - var act = actualLine[i]; - var exp = expectedLine[i]; + if (ignoreCase) { + act = char.ToLowerInvariant(act); + exp = char.ToLowerInvariant(exp); + } - if (ignoreCase) { - act = char.ToLowerInvariant(act); - exp = char.ToLowerInvariant(exp); + if (act != exp) { + index = i + 1; + return lineNum; + } } - if (act != exp) { - index = i + 1; + if (expectedLine.Length != actualLine.Length) { + index = minLength + 1; return lineNum; } } - if (expectedLine.Length != actualLine.Length) { - index = minLength + 1; - return lineNum; - } - } + if (expectedLine == null && actualLine == null) { + expectedLine = string.Empty; + actualLine = string.Empty; - if (expectedLine == null && actualLine == null) { - expectedLine = string.Empty; - actualLine = string.Empty; + return 0; + } - return 0; + return lineNum; } - - return lineNum; } public static void CompareToFile(string baselineFile, string actual, bool regenerateBaseline = false, bool ignoreCase = false) {