diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs
index 8d4b9751d..2b9182661 100644
--- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs
@@ -13,11 +13,18 @@
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.
+using Microsoft.Python.Analysis.Dependencies;
+
namespace Microsoft.Python.Analysis.Analyzer {
///
/// Represents document that can be analyzed asynchronously.
///
internal interface IAnalyzable {
+ ///
+ /// Returns object that can calculate dependencies of this entry.
+ ///
+ IDependencyProvider DependencyProvider { get; }
+
///
/// Notifies document that analysis is about to begin.
///
diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs
index 8aab07e9c..ddd730e48 100644
--- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs
@@ -16,6 +16,7 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
+using System.Threading;
using Microsoft.Python.Analysis.Analyzer.Evaluation;
using Microsoft.Python.Analysis.Documents;
using Microsoft.Python.Analysis.Modules;
@@ -32,30 +33,36 @@ namespace Microsoft.Python.Analysis.Analyzer {
internal class ModuleWalker : AnalysisWalker {
private const string AllVariableName = "__all__";
private readonly IDocumentAnalysis _stubAnalysis;
+ private readonly CancellationToken _cancellationToken;
// A hack to use __all__ export in the most simple case.
private int _allReferencesCount;
private bool _allIsUsable = true;
- public ModuleWalker(IServiceContainer services, IPythonModule module, PythonAst ast)
+ public ModuleWalker(IServiceContainer services, IPythonModule module, PythonAst ast, CancellationToken cancellationToken)
: base(new ExpressionEval(services, module, ast)) {
_stubAnalysis = Module.Stub is IDocument doc ? doc.GetAnyAnalysis() : null;
+ _cancellationToken = cancellationToken;
}
public override bool Walk(NameExpression node) {
if (Eval.CurrentScope == Eval.GlobalScope && node.Name == AllVariableName) {
_allReferencesCount++;
}
+
+ _cancellationToken.ThrowIfCancellationRequested();
return base.Walk(node);
}
public override bool Walk(AugmentedAssignStatement node) {
HandleAugmentedAllAssign(node);
+ _cancellationToken.ThrowIfCancellationRequested();
return base.Walk(node);
}
public override bool Walk(CallExpression node) {
HandleAllAppendExtend(node);
+ _cancellationToken.ThrowIfCancellationRequested();
return base.Walk(node);
}
@@ -146,6 +153,7 @@ private bool IsHandleableAll(Node node) {
public override bool Walk(PythonAst node) {
Check.InvalidOperation(() => Ast == node, "walking wrong AST");
+ _cancellationToken.ThrowIfCancellationRequested();
// Collect basic information about classes and functions in order
// to correctly process forward references. Does not determine
@@ -181,16 +189,20 @@ public override bool Walk(PythonAst node) {
// Classes and functions are walked by their respective evaluators
public override bool Walk(ClassDefinition node) {
+ _cancellationToken.ThrowIfCancellationRequested();
SymbolTable.Evaluate(node);
return false;
}
public override bool Walk(FunctionDefinition node) {
+ _cancellationToken.ThrowIfCancellationRequested();
SymbolTable.Evaluate(node);
return false;
}
public void Complete() {
+ _cancellationToken.ThrowIfCancellationRequested();
+
SymbolTable.EvaluateAll();
SymbolTable.ReplacedByStubs.Clear();
MergeStub();
@@ -220,11 +232,13 @@ public void Complete() {
/// of the definitions. Stub may contains those so we need to merge it in.
///
private void MergeStub() {
- if (Module.ModuleType == ModuleType.User) {
+ _cancellationToken.ThrowIfCancellationRequested();
+
+ if (Module.ModuleType == ModuleType.User || Module.ModuleType == ModuleType.Stub) {
return;
}
// No stub, no merge.
- if (_stubAnalysis == null) {
+ if (_stubAnalysis.IsEmpty()) {
return;
}
// TODO: figure out why module is getting analyzed before stub.
@@ -248,6 +262,18 @@ private void MergeStub() {
if (stubType.DeclaringModule is TypingModule && stubType.Name == "Any") {
continue;
}
+
+ if (sourceVar?.Source == VariableSource.Import &&
+ sourceVar.GetPythonType()?.DeclaringModule.Stub != null) {
+ // Keep imported types as they are defined in the library. For example,
+ // 'requests' imports NullHandler as 'from logging import NullHandler'.
+ // But 'requests' also declares NullHandler in its stub (but not in the main code)
+ // and that declaration does not have documentation or location. Therefore avoid
+ // taking types that are stub-only when similar type is imported from another
+ // module that also has a stub.
+ continue;
+ }
+
TryReplaceMember(v, sourceType, stubType);
}
@@ -257,7 +283,7 @@ private void MergeStub() {
private void TryReplaceMember(IVariable v, IPythonType sourceType, IPythonType stubType) {
// If type does not exist in module, but exists in stub, declare it unless it is an import.
// If types are the classes, take class from the stub, then add missing members.
- // Otherwise, replace type from one from the stub.
+ // Otherwise, replace type by one from the stub.
switch (sourceType) {
case null:
// Nothing in sources, but there is type in the stub. Declare it.
@@ -379,28 +405,26 @@ private static void TransferDocumentationAndLocation(IPythonType s, IPythonType
if (s != d && s is PythonType src && d is PythonType dst) {
// If type is a class, then doc can either come from class definition node of from __init__.
// If class has doc from the class definition, don't stomp on it.
- var transferDoc = true;
if (src is PythonClassType srcClass && dst is PythonClassType dstClass) {
// Higher lever source wins
if (srcClass.DocumentationSource == PythonClassType.ClassDocumentationSource.Class ||
(srcClass.DocumentationSource == PythonClassType.ClassDocumentationSource.Init && dstClass.DocumentationSource == PythonClassType.ClassDocumentationSource.Base)) {
dstClass.SetDocumentation(srcClass.Documentation);
- transferDoc = false;
}
- }
-
- // Sometimes destination (stub type) already has documentation. This happens when stub type
- // is used to augment more than one type. For example, in threading module RLock stub class
- // replaces both RLock function and _RLock class making 'factory' function RLock to look
- // like a class constructor. Effectively a single stub type is used for more than one type
- // in the source and two source types may have different documentation. Thus transferring doc
- // from one source type affects documentation of another type. It may be better to clone stub
- // type and separate instances for separate source type, but for now we'll just avoid stomping
- // on the existing documentation.
- if (transferDoc && string.IsNullOrEmpty(dst.Documentation)) {
- var srcDocumentation = src.Documentation;
- if (!string.IsNullOrEmpty(srcDocumentation)) {
- dst.SetDocumentation(srcDocumentation);
+ } else {
+ // Sometimes destination (stub type) already has documentation. This happens when stub type
+ // is used to augment more than one type. For example, in threading module RLock stub class
+ // replaces both RLock function and _RLock class making 'factory' function RLock to look
+ // like a class constructor. Effectively a single stub type is used for more than one type
+ // in the source and two source types may have different documentation. Thus transferring doc
+ // from one source type affects documentation of another type. It may be better to clone stub
+ // type and separate instances for separate source type, but for now we'll just avoid stomping
+ // on the existing documentation.
+ if (string.IsNullOrEmpty(dst.Documentation)) {
+ var srcDocumentation = src.Documentation;
+ if (!string.IsNullOrEmpty(srcDocumentation)) {
+ dst.SetDocumentation(srcDocumentation);
+ }
}
}
diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs
index bc9203727..cfa8b64c0 100644
--- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs
@@ -238,7 +238,7 @@ internal void RaiseAnalysisComplete(int moduleCount, double msElapsed)
private void AnalyzeDocument(in AnalysisModuleKey key, in PythonAnalyzerEntry entry, in ImmutableArray dependencies) {
_analysisCompleteEvent.Reset();
ActivityTracker.StartTracking();
- _log?.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name}({entry.Module.ModuleType}) queued");
+ _log?.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name} ({entry.Module.ModuleType}) queued. Dependencies: {string.Join(", ", dependencies.Select(d => d.IsTypeshed ? $"{d.Name} (stub)" : d.Name))}");
var graphVersion = _dependencyResolver.ChangeValue(key, entry, entry.IsUserOrBuiltin || key.IsNonUserAsDocument, dependencies);
diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs
index 1a4772371..f3a8a6046 100644
--- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs
@@ -19,15 +19,13 @@
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
-using Microsoft.Python.Analysis.Core.DependencyResolution;
using Microsoft.Python.Analysis.Modules;
using Microsoft.Python.Analysis.Types;
-using Microsoft.Python.Core;
using Microsoft.Python.Core.Collections;
using Microsoft.Python.Parsing.Ast;
namespace Microsoft.Python.Analysis.Analyzer {
- [DebuggerDisplay("{_module.Name}({_module.ModuleType})")]
+ [DebuggerDisplay("{_module.Name} : {_module.ModuleType}")]
internal sealed class PythonAnalyzerEntry {
private readonly object _syncObj = new object();
private TaskCompletionSource _analysisTcs;
@@ -248,20 +246,13 @@ public bool Invalidate(IPythonModule module, PythonAst ast, int bufferVersion, i
}
private HashSet FindDependencies(IPythonModule module, PythonAst ast, int bufferVersion) {
- if (_bufferVersion > bufferVersion) {
- return new HashSet();
- }
-
- var walker = new DependencyWalker(module);
- ast.Walk(walker);
- var dependencies = walker.Dependencies;
- dependencies.Remove(new AnalysisModuleKey(module));
+ var dependencyProvider = (module as IAnalyzable)?.DependencyProvider;
+ var dependencies = _bufferVersion <= bufferVersion && module is IAnalyzable analyzable && analyzable.DependencyProvider != null
+ ? dependencyProvider.GetDependencies()
+ : new HashSet();
return dependencies;
}
- private static bool Ignore(IModuleManagement moduleResolution, string fullName, string modulePath)
- => moduleResolution.BuiltinModuleName.EqualsOrdinal(fullName) || moduleResolution.IsSpecializedModule(fullName, modulePath);
-
private void UpdateAnalysisTcs(int analysisVersion) {
_analysisVersion = analysisVersion;
if (_analysisTcs.Task.Status == TaskStatus.RanToCompletion) {
@@ -273,69 +264,6 @@ private void UpdateAnalysisTcs(int analysisVersion) {
}
}
- private class DependencyWalker : PythonWalker {
- private readonly IPythonModule _module;
- private readonly bool _isTypeshed;
- private readonly IModuleManagement _moduleResolution;
- private readonly PathResolverSnapshot _pathResolver;
-
- public HashSet Dependencies { get; }
-
- public DependencyWalker(IPythonModule module) {
- _module = module;
- _isTypeshed = module is StubPythonModule stub && stub.IsTypeshed;
- _moduleResolution = module.Interpreter.ModuleResolution;
- _pathResolver = _isTypeshed
- ? module.Interpreter.TypeshedResolution.CurrentPathResolver
- : _moduleResolution.CurrentPathResolver;
-
- Dependencies = new HashSet();
-
- if (module.Stub != null) {
- Dependencies.Add(new AnalysisModuleKey(module.Stub));
- }
- }
-
- public override bool Walk(ImportStatement import) {
- var forceAbsolute = import.ForceAbsolute;
- foreach (var moduleName in import.Names) {
- var importNames = ImmutableArray.Empty;
- foreach (var nameExpression in moduleName.Names) {
- importNames = importNames.Add(nameExpression.Name);
- var imports = _pathResolver.GetImportsFromAbsoluteName(_module.FilePath, importNames, forceAbsolute);
- HandleSearchResults(imports);
- }
- }
- return false;
- }
-
- public override bool Walk(FromImportStatement fromImport) {
- var imports = _pathResolver.FindImports(_module.FilePath, fromImport);
- HandleSearchResults(imports);
- if (imports is IImportChildrenSource childrenSource) {
- foreach (var name in fromImport.Names) {
- if (childrenSource.TryGetChildImport(name.Name, out var childImport)) {
- HandleSearchResults(childImport);
- }
- }
- }
-
- return false;
- }
-
- private void HandleSearchResults(IImportSearchResult searchResult) {
- switch (searchResult) {
- case ModuleImport moduleImport when !Ignore(_moduleResolution, moduleImport.FullName, moduleImport.ModulePath):
- Dependencies.Add(new AnalysisModuleKey(moduleImport.FullName, moduleImport.ModulePath, _isTypeshed));
- return;
- case PossibleModuleImport possibleModuleImport when !Ignore(_moduleResolution, possibleModuleImport.PrecedingModuleFullName, possibleModuleImport.PrecedingModulePath):
- Dependencies.Add(new AnalysisModuleKey(possibleModuleImport.PrecedingModuleFullName, possibleModuleImport.PrecedingModulePath, _isTypeshed));
- return;
- default:
- return;
- }
- }
- }
}
}
diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs
index eeb5c6b20..9abc1b5a6 100644
--- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs
@@ -236,7 +236,7 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) {
private bool IsAnalyzedLibraryInLoop(IDependencyChainNode node, IDocumentAnalysis currentAnalysis)
=> !node.HasMissingDependencies && currentAnalysis is LibraryAnalysis && node.IsWalkedWithDependencies && node.IsValidVersion;
- private void RunAnalysis(IDependencyChainNode node, Stopwatch stopWatch)
+ private void RunAnalysis(IDependencyChainNode node, Stopwatch stopWatch)
=> ExecutionContext.Run(ExecutionContext.Capture(), s => Analyze(node, null, stopWatch), null);
private Task StartAnalysis(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwatch stopWatch)
@@ -334,83 +334,74 @@ private void AnalyzeEntry(IDependencyChainNode node, Python
var analyzable = module as IAnalyzable;
analyzable?.NotifyAnalysisBegins();
- var walker = new ModuleWalker(_services, module, ast);
- ast.Walk(walker);
-
+ var analysis = DoAnalyzeEntry(node, (IDocument)module, ast, version);
_analyzerCancellationToken.ThrowIfCancellationRequested();
- walker.Complete();
- _analyzerCancellationToken.ThrowIfCancellationRequested();
+ if (analysis != null) {
+ analyzable?.NotifyAnalysisComplete(analysis);
+ entry.TrySetAnalysis(analysis, version);
+
+ if (module.ModuleType == ModuleType.User) {
+ var linterDiagnostics = _analyzer.LintModule(module);
+ _diagnosticsService?.Replace(entry.Module.Uri, linterDiagnostics, DiagnosticSource.Linter);
+ }
+ }
+ }
+
+ private IDocumentAnalysis DoAnalyzeEntry(IDependencyChainNode node, IDocument document, PythonAst ast, int version) {
+ var moduleType = node.Value.Module.ModuleType;
bool isCanceled;
lock (_syncObj) {
isCanceled = _isCanceled;
}
- if (!isCanceled) {
- node?.MarkWalked();
+ if (moduleType.CanBeCached() && _moduleDatabaseService?.ModuleExistsInStorage(document.Name, document.FilePath) == true) {
+ if (!isCanceled && _moduleDatabaseService.TryRestoreGlobalScope(document, out var gs)) {
+ if (_log != null) {
+ _log.Log(TraceEventType.Verbose, "Restored from database: ", document.Name);
+ }
+ var analysis = new DocumentAnalysis(document, 1, gs, new ExpressionEval(_services, document, document.GetAst()), Array.Empty());
+ gs.ReconstructVariables();
+ return analysis;
+ }
+ return null;
}
- var analysis = CreateAnalysis(node, (IDocument)module, ast, version, walker, isCanceled);
- analyzable?.NotifyAnalysisComplete(analysis);
- entry.TrySetAnalysis(analysis, version);
-
- if (module.ModuleType == ModuleType.User) {
- var linterDiagnostics = _analyzer.LintModule(module);
- _diagnosticsService?.Replace(entry.Module.Uri, linterDiagnostics, DiagnosticSource.Linter);
- }
- }
-
- private void LogCompleted(IDependencyChainNode node, IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) {
- if (_log != null) {
- var completed = node != null && module.Analysis is LibraryAnalysis ? "completed for library" : "completed";
- var message = node != null
- ? $"Analysis of {module.Name} ({module.ModuleType}) on depth {node.VertexDepth} {completed} in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."
- : $"Out of order analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms.";
- _log.Log(TraceEventType.Verbose, message);
- }
- }
+ var walker = new ModuleWalker(_services, document, ast, _analyzerCancellationToken);
+ ast.Walk(walker);
+ walker.Complete();
- private void LogCanceled(IPythonModule module) {
- if (_log != null) {
- _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled.");
+ lock (_syncObj) {
+ isCanceled = _isCanceled;
}
- }
-
- private void LogException(IPythonModule module, Exception exception) {
- if (_log != null) {
- _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) failed. {exception}");
+ if (!isCanceled) {
+ node.MarkWalked();
}
- if (TestEnvironment.Current != null) {
- ExceptionDispatchInfo.Capture(exception).Throw();
- }
+ return CreateAnalysis(node, document, ast, version, walker, isCanceled);
}
private IDocumentAnalysis CreateAnalysis(IDependencyChainNode node, IDocument document, PythonAst ast, int version, ModuleWalker walker, bool isCanceled) {
var canHaveLibraryAnalysis = false;
- var saveAnalysis = false;
+
// Don't try to drop builtins; it causes issues elsewhere.
// We probably want the builtin module's AST and other info for evaluation.
switch (document.ModuleType) {
case ModuleType.Library:
case ModuleType.Compiled:
case ModuleType.CompiledBuiltin:
- canHaveLibraryAnalysis = true;
- saveAnalysis = true;
- break;
- case ModuleType.Stub:
canHaveLibraryAnalysis = true;
break;
}
var createLibraryAnalysis = !isCanceled &&
- node != null &&
- !node.HasMissingDependencies &&
- canHaveLibraryAnalysis &&
- !document.IsOpen &&
- node.HasOnlyWalkedDependencies &&
- node.IsValidVersion;
+ node != null &&
+ !node.HasMissingDependencies &&
+ canHaveLibraryAnalysis &&
+ !document.IsOpen &&
+ node.HasOnlyWalkedDependencies &&
+ node.IsValidVersion;
if (!createLibraryAnalysis) {
return new DocumentAnalysis(document, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames);
@@ -420,15 +411,41 @@ private IDocumentAnalysis CreateAnalysis(IDependencyChainNode();
+ dbs?.StoreModuleAnalysisAsync(analysis, CancellationToken.None).DoNotWait();
return analysis;
}
+
+ private void LogCompleted(IDependencyChainNode node, IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) {
+ if (_log != null) {
+ var completed = node != null && module.Analysis is LibraryAnalysis ? "completed for library" : "completed";
+ var message = node != null
+ ? $"Analysis of {module.Name} ({module.ModuleType}) on depth {node.VertexDepth} {completed} in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."
+ : $"Out of order analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms.";
+ _log.Log(TraceEventType.Verbose, message);
+ }
+ }
+
+ private void LogCanceled(IPythonModule module) {
+ if (_log != null) {
+ _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled.");
+ }
+ }
+
+ private void LogException(IPythonModule module, Exception exception) {
+ if (_log != null) {
+ _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) failed. {exception}");
+ }
+
+ if (TestEnvironment.Current != null) {
+ ExceptionDispatchInfo.Capture(exception).Throw();
+ }
+ }
+
private enum State {
NotStarted = 0,
Started = 1,
diff --git a/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs
index bd1989aa1..9477a952f 100644
--- a/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs
+++ b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs
@@ -15,19 +15,36 @@
using System.Threading;
using System.Threading.Tasks;
+using Microsoft.Python.Analysis.Dependencies;
using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Analysis.Values;
namespace Microsoft.Python.Analysis.Caching {
+ ///
+ /// Represents global scope that has been restored from
+ /// the database but has not been fully populated yet.
+ /// Used to attach to analysis so variables can be
+ /// accessed during classes and methods restoration.
+ ///
+ internal interface IRestoredGlobalScope : IGlobalScope {
+ void ReconstructVariables();
+ }
+
internal interface IModuleDatabaseService {
///
- /// Creates module representation from module persistent state.
+ /// Creates global scope from module persistent state.
+ /// Global scope is then can be used to construct module analysis.
+ ///
+ /// Python module to restore analysis for.
+ /// Python module global scope.
+ bool TryRestoreGlobalScope(IPythonModule module, out IRestoredGlobalScope gs);
+
+ ///
+ /// Retrieves dependencies from the module persistent state.
///
- /// Module name. If the name is not qualified
- /// the module will ge resolved against active Python version.
- /// Module file path.
- /// Python module.
- /// Module storage state
- ModuleStorageState TryCreateModule(string moduleName, string filePath, out IPythonModule module);
+ /// Python module to restore analysis for.
+ /// Python module dependency provider.
+ bool TryRestoreDependencies(IPythonModule module, out IDependencyProvider dp);
///
/// Writes module data to the database.
@@ -38,5 +55,10 @@ internal interface IModuleDatabaseService {
/// Determines if module analysis exists in the storage.
///
bool ModuleExistsInStorage(string moduleName, string filePath);
+
+ ///
+ /// Clear cached data.
+ ///
+ void Clear();
}
}
diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs
new file mode 100644
index 000000000..9087cd2ee
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs
@@ -0,0 +1,77 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Collections.Generic;
+using Microsoft.Python.Analysis.Analyzer;
+using Microsoft.Python.Analysis.Core.DependencyResolution;
+using Microsoft.Python.Analysis.Modules;
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Core;
+
+namespace Microsoft.Python.Analysis.Dependencies {
+ internal sealed class DependencyCollector {
+ private readonly IPythonModule _module;
+ private readonly bool _isTypeshed;
+ private readonly IModuleManagement _moduleResolution;
+ private readonly PathResolverSnapshot _pathResolver;
+
+ public HashSet Dependencies { get; } = new HashSet();
+
+ public DependencyCollector(IPythonModule module) {
+ _module = module;
+ _isTypeshed = module is StubPythonModule stub && stub.IsTypeshed;
+ _moduleResolution = module.Interpreter.ModuleResolution;
+ _pathResolver = _isTypeshed
+ ? module.Interpreter.TypeshedResolution.CurrentPathResolver
+ : _moduleResolution.CurrentPathResolver;
+
+ if (module.Stub != null) {
+ Dependencies.Add(new AnalysisModuleKey(module.Stub));
+ }
+ }
+
+ public void AddImport(IReadOnlyList importNames, bool forceAbsolute) {
+ var imports = _pathResolver.GetImportsFromAbsoluteName(_module.FilePath, importNames, forceAbsolute);
+ HandleSearchResults(imports);
+ }
+
+ public void AddFromImport(IReadOnlyList importNames, int dotCount, bool forceAbsolute) {
+ var imports = _pathResolver.FindImports(_module.FilePath, importNames, dotCount, forceAbsolute);
+ HandleSearchResults(imports);
+ if (imports is IImportChildrenSource childrenSource) {
+ foreach (var name in importNames) {
+ if (childrenSource.TryGetChildImport(name, out var childImport)) {
+ HandleSearchResults(childImport);
+ }
+ }
+ }
+ }
+
+ private void HandleSearchResults(IImportSearchResult searchResult) {
+ switch (searchResult) {
+ case ModuleImport moduleImport when !Ignore(_moduleResolution, moduleImport.FullName, moduleImport.ModulePath):
+ Dependencies.Add(new AnalysisModuleKey(moduleImport.FullName, moduleImport.ModulePath, _isTypeshed));
+ return;
+ case PossibleModuleImport possibleModuleImport when !Ignore(_moduleResolution, possibleModuleImport.PrecedingModuleFullName, possibleModuleImport.PrecedingModulePath):
+ Dependencies.Add(new AnalysisModuleKey(possibleModuleImport.PrecedingModuleFullName, possibleModuleImport.PrecedingModulePath, _isTypeshed));
+ return;
+ default:
+ return;
+ }
+ }
+ private static bool Ignore(IModuleManagement moduleResolution, string fullName, string modulePath)
+ => moduleResolution.BuiltinModuleName.EqualsOrdinal(fullName) || moduleResolution.IsSpecializedModule(fullName, modulePath);
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs
new file mode 100644
index 000000000..9b2b1502f
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs
@@ -0,0 +1,53 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Python.Analysis.Analyzer;
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Core.Collections;
+using Microsoft.Python.Parsing.Ast;
+
+namespace Microsoft.Python.Analysis.Dependencies {
+ internal sealed class DependencyWalker : PythonWalker {
+ private readonly DependencyCollector _dependencyCollector;
+
+ public HashSet Dependencies => _dependencyCollector.Dependencies;
+
+ public DependencyWalker(IPythonModule module, PythonAst ast) {
+ _dependencyCollector = new DependencyCollector(module);
+ ast.Walk(this);
+ }
+
+ public override bool Walk(ImportStatement import) {
+ var forceAbsolute = import.ForceAbsolute;
+ foreach (var moduleName in import.Names) {
+ var importNames = ImmutableArray.Empty;
+ foreach (var nameExpression in moduleName.Names) {
+ importNames = importNames.Add(nameExpression.Name);
+ _dependencyCollector.AddImport(importNames, forceAbsolute);
+ }
+ }
+ return false;
+ }
+
+ public override bool Walk(FromImportStatement fromImport) {
+ var rootNames = fromImport.Root.Names.Select(n => n.Name).ToArray();
+ var dotCount = fromImport.Root is RelativeModuleName relativeName ? relativeName.DotCount : 0;
+ _dependencyCollector.AddFromImport(rootNames, dotCount, fromImport.ForceAbsolute);
+ return false;
+ }
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Caching/Definitions/ModuleStorageState.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyProvider.cs
similarity index 52%
rename from src/Analysis/Ast/Impl/Caching/Definitions/ModuleStorageState.cs
rename to src/Analysis/Ast/Impl/Dependencies/IDependencyProvider.cs
index bbf14c809..d0a911f5b 100644
--- a/src/Analysis/Ast/Impl/Caching/Definitions/ModuleStorageState.cs
+++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyProvider.cs
@@ -13,30 +13,16 @@
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.
-namespace Microsoft.Python.Analysis.Caching {
+using System.Collections.Generic;
+using Microsoft.Python.Analysis.Analyzer;
+
+namespace Microsoft.Python.Analysis.Dependencies {
///
- /// Describes module data stored in a database.
+ /// Implements provider that can supply list of imports to the dependency analysis.
+ /// Regular modules provide dependency from the AST, persistent/database modules
+ /// provide dependencies from their models.
///
- public enum ModuleStorageState {
- ///
- /// Module does not exist in the database.
- ///
- DoesNotExist,
-
- ///
- /// Partial data. This means module is still being analyzed
- /// and the data on the module members is incomplete.
- ///
- Partial,
-
- ///
- /// Modules exist and the analysis is complete.
- ///
- Complete,
-
- ///
- /// Storage is corrupted or incompatible.
- ///
- Corrupted
+ internal interface IDependencyProvider {
+ HashSet GetDependencies();
}
}
diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs
index 02fdfbdfd..3816803e3 100644
--- a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs
+++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs
@@ -13,7 +13,6 @@
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.
-using Microsoft.Python.Analysis.Analyzer;
using Microsoft.Python.Core.Collections;
namespace Microsoft.Python.Analysis.Dependencies {
diff --git a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs
index 1f802edd8..810632abc 100644
--- a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs
+++ b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs
@@ -241,10 +241,10 @@ private DocumentEntry CreateDocument(ModuleCreationOptions mco) {
IDocument document;
switch (mco.ModuleType) {
case ModuleType.Compiled when TryAddModulePath(mco):
- document = new CompiledPythonModule(mco.ModuleName, ModuleType.Compiled, mco.FilePath, mco.Stub, _services);
+ document = new CompiledPythonModule(mco.ModuleName, ModuleType.Compiled, mco.FilePath, mco.Stub, mco.IsPersistent, _services);
break;
case ModuleType.CompiledBuiltin:
- document = new CompiledBuiltinPythonModule(mco.ModuleName, mco.Stub, _services);
+ document = new CompiledBuiltinPythonModule(mco.ModuleName, mco.Stub, mco.IsPersistent, _services);
break;
case ModuleType.User:
TryAddModulePath(mco);
diff --git a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs
index 555a21567..6995f53b3 100644
--- a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs
+++ b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs
@@ -22,6 +22,8 @@
namespace Microsoft.Python.Analysis {
public static class AnalysisExtensions {
+ public static bool IsEmpty(this IDocumentAnalysis analysis) => analysis == null || analysis is EmptyAnalysis;
+
public static IScope FindScope(this IDocumentAnalysis analysis, SourceLocation location)
=> analysis.GlobalScope.FindScope(analysis.Document, location);
diff --git a/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs
index 7ee3f4a85..d8705f253 100644
--- a/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs
+++ b/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs
@@ -61,8 +61,8 @@ public static bool IsPrivateMember(this IPythonClassType cls, string memberName)
///
/// Gets specific type for the given generic type parameter, resolving bounds as well
///
- public static bool GetSpecificType(this IPythonClassType cls, IGenericTypeParameter param, out IPythonType specificType) {
- cls.GenericParameters.TryGetValue(param, out specificType);
+ public static bool GetSpecificType(this IPythonClassType cls, string paramName, out IPythonType specificType) {
+ cls.GenericParameters.TryGetValue(paramName, out specificType);
// If type has not been found, check if the type parameter has an upper bound and use that
if (specificType is IGenericTypeParameter gtp && gtp.Bound != null) {
specificType = gtp.Bound;
diff --git a/src/Analysis/Ast/Impl/Extensions/PythonFunctionExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonFunctionExtensions.cs
index 36f3ac42f..560d46332 100644
--- a/src/Analysis/Ast/Impl/Extensions/PythonFunctionExtensions.cs
+++ b/src/Analysis/Ast/Impl/Extensions/PythonFunctionExtensions.cs
@@ -47,7 +47,7 @@ public static string GetQualifiedName(this IPythonClassMember cm, string baseNam
}
return cm.DeclaringModule.ModuleType == ModuleType.Builtins
? string.Join(".", s)
- : $"{cm.DeclaringModule.Name}:{string.Join(".", s)}";
+ : $"{cm.DeclaringModule.QualifiedName}:{string.Join(".", s)}";
}
}
}
diff --git a/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs
index e94bf3161..75a20a039 100644
--- a/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs
+++ b/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs
@@ -15,7 +15,6 @@
using Microsoft.Python.Analysis.Specializations.Typing;
using Microsoft.Python.Analysis.Types;
-using Microsoft.Python.Core;
namespace Microsoft.Python.Analysis {
public static class PythonTypeExtensions {
diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs
index 095df7033..72988f5a2 100644
--- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs
+++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs
@@ -35,7 +35,7 @@ internal sealed class BuiltinsPythonModule : CompiledPythonModule, IBuiltinsPyth
private IPythonType _boolType;
public BuiltinsPythonModule(string moduleName, string filePath, IServiceContainer services)
- : base(moduleName, ModuleType.Builtins, filePath, null, services) { } // TODO: builtins stub
+ : base(moduleName, ModuleType.Builtins, filePath, null, false, services) { } // TODO: builtins stub & persistence
public override IMember GetMember(string name) => _hiddenNames.Contains(name) ? null : base.GetMember(name);
diff --git a/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs b/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs
index e020e7b26..500e4dbca 100644
--- a/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs
+++ b/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs
@@ -23,8 +23,8 @@ namespace Microsoft.Python.Analysis.Modules {
/// Represents compiled module that is built into the language.
///
internal sealed class CompiledBuiltinPythonModule : CompiledPythonModule {
- public CompiledBuiltinPythonModule(string moduleName, IPythonModule stub, IServiceContainer services)
- : base(moduleName, ModuleType.CompiledBuiltin, MakeFakeFilePath(moduleName, services), stub, services) { }
+ public CompiledBuiltinPythonModule(string moduleName, IPythonModule stub, bool isPersistent, IServiceContainer services)
+ : base(moduleName, ModuleType.CompiledBuiltin, MakeFakeFilePath(moduleName, services), stub, isPersistent, services) { }
protected override string[] GetScrapeArguments(IPythonInterpreter interpreter)
=> !InstallPath.TryGetFile("scrape_module.py", out var sm) ? null : new [] { "-W", "ignore", "-B", "-E", sm, "-u8", Name };
diff --git a/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs b/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs
index 1335a8ccf..662b25d30 100644
--- a/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs
+++ b/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs
@@ -26,8 +26,8 @@ namespace Microsoft.Python.Analysis.Modules {
internal class CompiledPythonModule : PythonModule {
protected IStubCache StubCache => Interpreter.ModuleResolution.StubCache;
- public CompiledPythonModule(string moduleName, ModuleType moduleType, string filePath, IPythonModule stub, IServiceContainer services)
- : base(moduleName, filePath, moduleType, stub, services) { }
+ public CompiledPythonModule(string moduleName, ModuleType moduleType, string filePath, IPythonModule stub, bool isPersistent, IServiceContainer services)
+ : base(moduleName, filePath, moduleType, stub, isPersistent, services) { }
public override string Documentation
=> GetMember("__doc__").TryGetConstant(out var s) ? s : string.Empty;
diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs
index f9c1dc9aa..f3b1058c8 100644
--- a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs
+++ b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs
@@ -14,7 +14,6 @@
// permissions and limitations under the License.
using System;
-using Microsoft.Python.Analysis.Core.Interpreter;
using Microsoft.Python.Analysis.Types;
namespace Microsoft.Python.Analysis.Modules {
@@ -48,5 +47,10 @@ public sealed class ModuleCreationOptions {
/// Module stub, if any.
///
public IPythonModule Stub { get; set; }
+
+ ///
+ /// Indicates if module is restored from database.
+ ///
+ public bool IsPersistent { get; set; }
}
}
diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs
index a18205271..dba391b96 100644
--- a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs
+++ b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs
@@ -64,5 +64,6 @@ public enum ModuleType {
public static class ModuleTypeExtensions {
public static bool IsNonUserFile(this ModuleType type) => type == ModuleType.Library || type == ModuleType.Stub;
public static bool IsCompiled(this ModuleType type) => type == ModuleType.Compiled || type == ModuleType.CompiledBuiltin;
+ public static bool CanBeCached(this ModuleType type) => type == ModuleType.Library || type == ModuleType.Compiled || type == ModuleType.CompiledBuiltin;
}
}
diff --git a/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs b/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs
new file mode 100644
index 000000000..ab77d7c4e
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs
@@ -0,0 +1,53 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Collections.Generic;
+using Microsoft.Python.Analysis.Analyzer;
+using Microsoft.Python.Analysis.Caching;
+using Microsoft.Python.Analysis.Dependencies;
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Core;
+
+namespace Microsoft.Python.Analysis.Modules {
+ internal sealed class DependencyProvider: IDependencyProvider {
+ private readonly IPythonModule _module;
+ private readonly IModuleDatabaseService _dbService;
+
+ public static IDependencyProvider Empty { get; } = new EmptyDependencyProvider();
+
+ public DependencyProvider(IPythonModule module, IServiceContainer services) {
+ _dbService = services.GetService();
+ _module = module;
+ }
+
+ #region IDependencyProvider
+ public HashSet GetDependencies() {
+ var ast = _module.GetAst();
+
+ if (_dbService != null && _dbService.TryRestoreDependencies(_module, out var dp)) {
+ return dp.GetDependencies();
+ }
+
+ // TODO: try and handle LoadFunctionDependencyModules functionality here.
+ var dw = new DependencyWalker(_module, ast);
+ return dw.Dependencies;
+ }
+ #endregion
+
+ private sealed class EmptyDependencyProvider: IDependencyProvider {
+ public HashSet GetDependencies() => new HashSet();
+ }
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs
index c9c0b4576..a78501c59 100644
--- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs
+++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs
@@ -22,7 +22,9 @@
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Python.Analysis.Analyzer;
-using Microsoft.Python.Analysis.Core.Interpreter;
+using Microsoft.Python.Analysis.Analyzer.Evaluation;
+using Microsoft.Python.Analysis.Caching;
+using Microsoft.Python.Analysis.Dependencies;
using Microsoft.Python.Analysis.Diagnostics;
using Microsoft.Python.Analysis.Documents;
using Microsoft.Python.Analysis.Specializations.Typing;
@@ -86,12 +88,13 @@ protected PythonModule(string name, ModuleType moduleType, IServiceContainer ser
SetDeclaringModule(this);
}
- protected PythonModule(string moduleName, string filePath, ModuleType moduleType, IPythonModule stub, IServiceContainer services) :
+ protected PythonModule(string moduleName, string filePath, ModuleType moduleType, IPythonModule stub, bool isPersistent, IServiceContainer services) :
this(new ModuleCreationOptions {
ModuleName = moduleName,
FilePath = filePath,
ModuleType = moduleType,
- Stub = stub
+ Stub = stub,
+ IsPersistent = isPersistent
}, services) { }
internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer services)
@@ -117,12 +120,13 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s
ContentState = State.Analyzed;
}
+ IsPersistent = creationOptions.IsPersistent;
InitializeContent(creationOptions.Content, 0);
}
#region IPythonType
public string Name { get; }
- public string QualifiedName => Name;
+ public string QualifiedName => ModuleType == ModuleType.Stub ? $"{Name}(stub)" : Name;
public BuiltinTypeId TypeId => BuiltinTypeId.Module;
public bool IsBuiltin => true;
public bool IsAbstract => false;
@@ -168,7 +172,7 @@ public virtual IEnumerable GetMemberNames() {
if (valueType is PythonModule) {
return false; // Do not re-export modules.
}
- if(valueType is IPythonFunctionType f && f.IsLambda()) {
+ if (valueType is IPythonFunctionType f && f.IsLambda()) {
return false;
}
if (this is TypingModule) {
@@ -178,7 +182,7 @@ public virtual IEnumerable GetMemberNames() {
// assigned with types from typing. Example:
// from typing import Any # do NOT export Any
// x = Union[int, str] # DO export x
- if(valueType?.DeclaringModule is TypingModule && v.Name == valueType.Name) {
+ if (valueType?.DeclaringModule is TypingModule && v.Name == valueType.Name) {
return false;
}
return true;
@@ -191,12 +195,10 @@ public virtual IEnumerable GetMemberNames() {
public override LocationInfo Definition => new LocationInfo(Uri.ToAbsolutePath(), Uri, 0, 0);
#endregion
+ #region IPythonModule
public virtual string FilePath { get; protected set; }
public virtual Uri Uri { get; }
-
- #region IPythonModule
public IDocumentAnalysis Analysis { get; private set; }
-
public IPythonInterpreter Interpreter { get; }
///
@@ -216,6 +218,11 @@ public virtual IEnumerable GetMemberNames() {
/// wants to see library code and not a stub.
///
public IPythonModule PrimaryModule { get; private set; }
+
+ ///
+ /// Indicates if module is restored from database.
+ ///
+ public bool IsPersistent { get; }
#endregion
#region IDisposable
@@ -259,7 +266,7 @@ public string Content {
return _buffer.Text;
}
}
- }
+ }
#endregion
#region Parsing
@@ -399,6 +406,7 @@ public override void Add(string message, SourceSpan span, int errorCode, Severit
#endregion
#region IAnalyzable
+ public virtual IDependencyProvider DependencyProvider => new DependencyProvider(this, Services);
public void NotifyAnalysisBegins() {
lock (_syncObj) {
@@ -522,7 +530,11 @@ private void InitializeContent(string content, int version) {
private void LoadContent(string content, int version) {
if (ContentState < State.Loading) {
try {
- content = content ?? LoadContent();
+ if (IsPersistent) {
+ content = string.Empty;
+ } else {
+ content = content ?? LoadContent();
+ }
_buffer.Reset(version, content);
ContentState = State.Loaded;
} catch (IOException) { } catch (UnauthorizedAccessException) { }
diff --git a/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs b/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs
index e28964258..d81d6a85c 100644
--- a/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs
+++ b/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs
@@ -16,9 +16,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.Python.Analysis.Core.Interpreter;
+using Microsoft.Python.Analysis.Analyzer;
+using Microsoft.Python.Analysis.Dependencies;
using Microsoft.Python.Analysis.Types;
using Microsoft.Python.Analysis.Values;
using Microsoft.Python.Core;
@@ -52,6 +51,7 @@ internal sealed class PythonVariableModule : LocatedMember, IPythonModule, IEqua
public BuiltinTypeId TypeId => BuiltinTypeId.Module;
public Uri Uri => Module?.Uri;
public override PythonMemberType MemberType => PythonMemberType.Module;
+ public bool IsPersistent => Module?.IsPersistent == true;
public PythonVariableModule(string name, IPythonInterpreter interpreter) : base(null) {
Name = name;
@@ -80,5 +80,9 @@ public PythonVariableModule(IPythonModule module): base(module) {
public SourceLocation IndexToLocation(int index) => (Module as ILocationConverter)?.IndexToLocation(index) ?? default;
public int LocationToIndex(SourceLocation location) => (Module as ILocationConverter)?.LocationToIndex(location) ?? default;
#endregion
+
+ #region IDependencyProvider
+ public IDependencyProvider DependencyProvider => (Module as IAnalyzable)?.DependencyProvider;
+ #endregion
}
}
diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs
index 24d56166f..26c0d3f83 100644
--- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs
+++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs
@@ -62,9 +62,8 @@ protected override IPythonModule CreateModule(string name) {
return null;
}
- IPythonModule module;
if (moduleImport.ModulePath != null) {
- module = GetRdt().GetDocument(new Uri(moduleImport.ModulePath));
+ var module = GetRdt().GetDocument(new Uri(moduleImport.ModulePath));
if (module != null) {
GetRdt().LockDocument(module.Uri);
return module;
@@ -72,31 +71,31 @@ protected override IPythonModule CreateModule(string name) {
}
var dbs = GetDbService();
- if (dbs != null && dbs.TryCreateModule(name, moduleImport.ModulePath, out module) != ModuleStorageState.DoesNotExist && module != null) {
- SpecializeModule(name, s => module);
- return module;
- }
-
- // If there is a stub, make sure it is loaded and attached
- // First check stub next to the module.
- if (!TryCreateModuleStub(name, moduleImport.ModulePath, out var stub)) {
- // If nothing found, try Typeshed.
- stub = Interpreter.TypeshedResolution.GetOrLoadModule(moduleImport.IsBuiltin ? name : moduleImport.FullName);
- }
+ moduleImport.IsPersistent = dbs != null && dbs.ModuleExistsInStorage(name, moduleImport.ModulePath);
+
+ IPythonModule stub = null;
+ if (!moduleImport.IsPersistent) {
+ // If there is a stub, make sure it is loaded and attached
+ // First check stub next to the module.
+ if (!TryCreateModuleStub(name, moduleImport.ModulePath, out stub)) {
+ // If nothing found, try Typeshed.
+ stub = Interpreter.TypeshedResolution.GetOrLoadModule(moduleImport.IsBuiltin ? name : moduleImport.FullName);
+ }
- // If stub is created and its path equals to module, return that stub as module
- if (stub != null && stub.FilePath.PathEquals(moduleImport.ModulePath)) {
- return stub;
+ // If stub is created and its path equals to module, return that stub as module
+ if (stub != null && stub.FilePath.PathEquals(moduleImport.ModulePath)) {
+ return stub;
+ }
}
if (moduleImport.IsBuiltin) {
Log?.Log(TraceEventType.Verbose, "Create built-in compiled (scraped) module: ", name, Configuration.InterpreterPath);
- return new CompiledBuiltinPythonModule(name, stub, Services);
+ return new CompiledBuiltinPythonModule(name, stub, moduleImport.IsPersistent, Services);
}
if (moduleImport.IsCompiled) {
Log?.Log(TraceEventType.Verbose, "Create compiled (scraped): ", moduleImport.FullName, moduleImport.ModulePath, moduleImport.RootPath);
- return new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, Services);
+ return new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, moduleImport.IsPersistent, Services);
}
Log?.Log(TraceEventType.Verbose, "Import: ", moduleImport.FullName, moduleImport.ModulePath);
@@ -106,7 +105,8 @@ protected override IPythonModule CreateModule(string name) {
ModuleName = moduleImport.FullName,
ModuleType = moduleImport.IsLibrary ? ModuleType.Library : ModuleType.User,
FilePath = moduleImport.ModulePath,
- Stub = stub
+ Stub = stub,
+ IsPersistent = moduleImport.IsPersistent
};
return GetRdt().AddModule(mco);
@@ -161,15 +161,8 @@ public IPythonModule GetSpecializedModule(string fullName, bool allowCreation =
///
/// Determines of module is specialized or exists in the database.
///
- public bool IsSpecializedModule(string fullName, string modulePath = null) {
- if (_specialized.ContainsKey(fullName)) {
- return true;
- }
- if (modulePath != null && Path.GetExtension(modulePath) == ".pyi") {
- return false;
- }
- return GetDbService()?.ModuleExistsInStorage(fullName, modulePath) == true;
- }
+ public bool IsSpecializedModule(string fullName, string modulePath = null)
+ => _specialized.ContainsKey(fullName);
internal async Task AddBuiltinTypesToPathResolverAsync(CancellationToken cancellationToken = default) {
var analyzer = Services.GetService();
diff --git a/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs b/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs
index 0e3e5fbcb..771acd25f 100644
--- a/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs
+++ b/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs
@@ -13,6 +13,7 @@
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.
+using Microsoft.Python.Analysis.Dependencies;
using Microsoft.Python.Analysis.Specializations.Typing;
using Microsoft.Python.Core;
using Microsoft.Python.Core.IO;
@@ -31,11 +32,15 @@ namespace Microsoft.Python.Analysis.Modules {
///
internal abstract class SpecializedModule : PythonModule {
protected SpecializedModule(string name, string modulePath, IServiceContainer services)
- : base(name, modulePath, ModuleType.Specialized, null, services) { }
+ : base(name, modulePath, ModuleType.Specialized, null, false, services) { }
protected override string LoadContent() {
// Exceptions are handled in the base
return FileSystem.FileExists(FilePath) ? FileSystem.ReadTextWithRetry(FilePath) : string.Empty;
}
+
+ #region IAnalyzable
+ public override IDependencyProvider DependencyProvider => Modules.DependencyProvider.Empty;
+ #endregion
}
}
diff --git a/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs b/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs
index 83fa0ffbe..862daf817 100644
--- a/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs
+++ b/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs
@@ -25,7 +25,7 @@ internal class StubPythonModule : CompiledPythonModule {
public bool IsTypeshed { get; }
public StubPythonModule(string moduleName, string stubPath, bool isTypeshed, IServiceContainer services)
- : base(moduleName, ModuleType.Stub, stubPath, null, services) {
+ : base(moduleName, ModuleType.Stub, stubPath, null, false, services) {
IsTypeshed = isTypeshed;
}
diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBase.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBase.cs
index c3f6c573a..7bea95f73 100644
--- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBase.cs
+++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBase.cs
@@ -37,10 +37,9 @@ internal GenericClassBase(IReadOnlyList typeArgs, IPython
#region IPythonClassType
public override bool IsGeneric => true;
- public override IReadOnlyDictionary GenericParameters
- => TypeParameters.ToDictionary(tp => tp, tp => tp as IPythonType ?? UnknownType);
-
- public override IPythonType CreateSpecificType(IArgumentSet args)
+ public override IReadOnlyDictionary GenericParameters
+ => TypeParameters.ToDictionary(tp => tp.Name, tp => tp as IPythonType ?? UnknownType);
+ public override IPythonType CreateSpecificType(IArgumentSet args)
=> new GenericClassBase(args.Arguments.Select(a => a.Value).OfType().ToArray(), DeclaringModule.Interpreter);
#endregion
diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs
index e89b342c7..d59dd27ac 100644
--- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs
+++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs
@@ -33,7 +33,8 @@ public GenericTypeParameter(
object covariant,
object contravariant,
IndexSpan indexSpan)
- : base(name, new Location(declaringModule, indexSpan), GetDocumentation(name, constraints, bound, covariant, contravariant)) {
+ : base(name, new Location(declaringModule, indexSpan),
+ GetDocumentation(name, constraints, bound, covariant, contravariant, declaringModule)) {
Constraints = constraints ?? Array.Empty();
Bound = bound;
Covariant = covariant;
@@ -106,8 +107,12 @@ private static IPythonType GetBoundType(IArgumentSet argSet) {
switch (rawBound) {
case IPythonType t:
return t;
- case IPythonConstant c when c.GetString() != null:
- return eval.GetTypeFromString(c.GetString());
+ case IPythonConstant c:
+ var s = c.GetString();
+ if (!string.IsNullOrEmpty(s)) {
+ return eval.GetTypeFromString(s) ?? argSet.Eval.UnknownType;
+ }
+ return argSet.Eval.UnknownType;
default:
return rawBound.GetPythonType();
}
@@ -118,7 +123,6 @@ public static IPythonType FromTypeVar(IArgumentSet argSet, IPythonModule declari
return declaringModule.Interpreter.UnknownType;
}
- var args = argSet.Arguments;
var constraintArgs = argSet.ListArgument?.Values ?? Array.Empty();
var name = argSet.GetArgumentValue("name")?.GetString();
@@ -135,9 +139,20 @@ public static IPythonType FromTypeVar(IArgumentSet argSet, IPythonModule declari
return new GenericTypeParameter(name, declaringModule, constraints, bound, covariant, contravariant, indexSpan);
}
- private static string GetDocumentation(string name, IReadOnlyList constraints, object bound, object covariant, object contravariant) {
+ private static string GetDocumentation(string name, IReadOnlyList constraints, IPythonType bound, object covariant, object contravariant, IPythonModule declaringModule) {
var constaintStrings = constraints != null ? constraints.Select(c => c.IsUnknown() ? "?" : c.Name) : Enumerable.Empty();
- var boundStrings = bound != null ? Enumerable.Repeat($"bound={bound}", 1) : Enumerable.Empty();
+
+ var boundStrings = Enumerable.Empty();
+ if (bound != null) {
+ string boundName;
+ if(bound.DeclaringModule.Equals(declaringModule) || bound.DeclaringModule is IBuiltinsPythonModule) {
+ boundName = bound.Name;
+ } else {
+ boundName = $"{bound.DeclaringModule.Name}.{bound.Name}";
+ }
+ boundStrings = Enumerable.Repeat($"bound={boundName}", 1);
+ }
+
var covariantStrings = covariant != null ? Enumerable.Repeat($"covariant={covariant}", 1) : Enumerable.Empty();
var contravariantStrings = contravariant != null ? Enumerable.Repeat($"contravariant={contravariant}", 1) : Enumerable.Empty();
diff --git a/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs b/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs
index aa3d89372..839d7adf2 100644
--- a/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs
+++ b/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs
@@ -75,8 +75,8 @@ public IPythonType CreateSpecificType(IArgumentSet typeArguments) {
#region IGenericType
public IReadOnlyList Parameters => (InnerType as IGenericType)?.Parameters ?? Array.Empty();
public bool IsGeneric => (InnerType as IPythonClassType)?.IsGeneric == true;
- public IReadOnlyDictionary GenericParameters
- => (InnerType as IPythonClassType)?.GenericParameters ?? EmptyDictionary.Instance;
+ public IReadOnlyDictionary GenericParameters
+ => (InnerType as IPythonClassType)?.GenericParameters ?? EmptyDictionary.Instance;
#endregion
#region IPythonClassType
diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassType.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassType.cs
index 697f3e862..2079cb2e2 100644
--- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassType.cs
+++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassType.cs
@@ -41,6 +41,6 @@ public interface IPythonClassType : IPythonClassMember, IGenericType {
/// If class is created off generic template, represents
/// pairs of the generic parameter / actual supplied type.
///
- IReadOnlyDictionary GenericParameters { get; }
+ IReadOnlyDictionary GenericParameters { get; }
}
}
diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs
index 2c12d9f9a..168670be1 100644
--- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs
+++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs
@@ -63,5 +63,10 @@ public interface IPythonModule : IPythonType {
/// wants to see library code and not a stub.
///
IPythonModule PrimaryModule { get; }
+
+ ///
+ /// Indicates if module is restored from database.
+ ///
+ bool IsPersistent { get; }
}
}
diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs
index c74fe841b..9490079e6 100644
--- a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs
+++ b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs
@@ -30,12 +30,12 @@ internal partial class PythonClassType {
private string _qualifiedNameWithParameters; // Qualified name with qualified parameter names for persistence.
private Dictionary _specificTypeCache;
- private Dictionary _genericParameters;
+ private Dictionary _genericParameters;
private IReadOnlyList _parameters = new List();
#region IGenericType
///
- /// List of unfilled generic type parameters. Represented as entries in the GenericParameters dictionary
+ /// List of unfilled generic type parameters. Represented as entries in the ActualGenericParameters dictionary
/// that have both key and value as generic type parameters
/// e.g
/// {T, T}
@@ -72,7 +72,7 @@ public virtual IPythonType CreateSpecificType(IArgumentSet args) {
// Storing generic parameters allows methods returning generic types
// to know what type parameter returns what specific type
- classType.StoreGenericParameters(this, newGenericTypeParameters, genericTypeToSpecificType);
+ classType.StoreGenericParameters(this, newGenericTypeParameters.Select(p => p.Name).ToArray(), genericTypeToSpecificType);
// Set generic name
classType.SetNames();
@@ -115,7 +115,7 @@ public virtual IPythonType CreateSpecificType(IArgumentSet args) {
// Look through generic type bases and see if any of their required type parameters
// have received a specific type, and if so create specific type
var st = gt.Parameters
- .Select(p => classType.GenericParameters.TryGetValue(p, out var t) ? t : null)
+ .Select(p => classType.GenericParameters.TryGetValue(p.Name, out var t) ? t : null)
.Where(p => !p.IsUnknown())
.ToArray();
if (st.Length > 0) {
@@ -127,7 +127,7 @@ public virtual IPythonType CreateSpecificType(IArgumentSet args) {
}
// Set specific class bases
- classType.SetBases(specificBases.Concat(newBases), args.Eval.CurrentScope);
+ classType.SetBases(specificBases.Concat(newBases), args.Eval?.CurrentScope);
// Now that parameters are set, check if class is generic
classType.SetGenericParameters();
// Transfer members from generic to specific type.
@@ -168,13 +168,13 @@ private IGenericTypeParameter[] GetTypeParameters() {
/// Given an argument set, returns a dictionary mapping generic type parameter to the supplied specific
/// type from arguments.
///
- private IReadOnlyDictionary GetSpecificTypes(
+ private IReadOnlyDictionary GetSpecificTypes(
IArgumentSet args,
IReadOnlyList genericTypeParameters,
ICollection newBases
) {
// For now, map each type parameter to itself, and we can fill in the value as we go
- var genericTypeToSpecificType = genericTypeParameters.ToDictionary(gtp => gtp, gtp => gtp as IPythonType);
+ var genericTypeToSpecificType = genericTypeParameters.ToDictionary(gtp => gtp.Name, gtp => gtp as IPythonType);
// Arguments passed are those of __init__ or copy constructor or index expression A[int, str, ...].
// The arguments do not necessarily match all of the declared generic parameters.
@@ -190,11 +190,11 @@ ICollection newBases
// __init__(self, v: _T), v is annotated as a generic type definition
// Check if its generic type name matches any of the generic class parameters i.e. if there is
// an argument like 'v: _T' we need to check if class has matching Generic[_T] or A[_T] in bases.
- if (genericTypeToSpecificType.ContainsKey(argTypeDefinition)) {
+ if (genericTypeToSpecificType.ContainsKey(argTypeDefinition.Name)) {
// TODO: Check if specific type matches generic type parameter constraints and report mismatches.
// Assign specific type.
if (arg.Value is IMember m && m.GetPythonType() is IPythonType pt) {
- genericTypeToSpecificType[argTypeDefinition] = pt;
+ genericTypeToSpecificType[argTypeDefinition.Name] = pt;
} else {
// TODO: report supplied parameter is not a type.
}
@@ -241,8 +241,8 @@ ICollection newBases
var type = member.GetPythonType();
if (!type.IsUnknown()) {
var gtd = gtIndex < genericTypeParameters.Count ? genericTypeParameters[gtIndex] : null;
- if (gtd != null && genericTypeToSpecificType.TryGetValue(gtd, out var s) && s is IGenericTypeParameter) {
- genericTypeToSpecificType[gtd] = type;
+ if (gtd != null && genericTypeToSpecificType.TryGetValue(gtd.Name, out var s) && s is IGenericTypeParameter) {
+ genericTypeToSpecificType[gtd.Name] = type;
}
gtIndex++;
}
@@ -258,8 +258,8 @@ ICollection newBases
///
internal void StoreGenericParameters(
IPythonClassType templateClass,
- IEnumerable newGenericParameters,
- IReadOnlyDictionary genericToSpecificTypes) {
+ IEnumerable newGenericParameters,
+ IReadOnlyDictionary genericToSpecificTypes) {
// copy original generic parameters over and try to fill them in
_genericParameters = templateClass.GenericParameters.ToDictionary(k => k.Key, k => k.Value);
@@ -280,7 +280,7 @@ internal void StoreGenericParameters(
// class A(Generic[T]):
// class B(A[U])
// A has T => U
- _genericParameters[gp] = genericToSpecificTypes.TryGetValue(specificType, out var v) ? v : null;
+ _genericParameters[gp] = genericToSpecificTypes.TryGetValue(specificType.Name, out var v) ? v : null;
}
}
}
@@ -302,28 +302,28 @@ internal void StoreGenericParameters(
/// Generic type (Generic[T1, T2, ...], A[T1, T2, ..], etc.).
/// Argument value passed to the class constructor.
/// Dictionary or name (T1) to specific type to populate.
- private static void GetSpecificTypeFromArgumentValue(IGenericType gt, object argumentValue, IDictionary specificTypes) {
+ private void GetSpecificTypeFromArgumentValue(IGenericType gt, object argumentValue, IDictionary specificTypes) {
switch (argumentValue) {
case IPythonDictionary dict when gt.Parameters.Count == 2:
var keyType = dict.Keys.FirstOrDefault()?.GetPythonType();
var valueType = dict.Values.FirstOrDefault()?.GetPythonType();
if (!keyType.IsUnknown()) {
- specificTypes[gt.Parameters[0]] = keyType;
+ specificTypes[gt.Parameters[0].Name] = keyType;
}
if (!valueType.IsUnknown()) {
- specificTypes[gt.Parameters[1]] = valueType;
+ specificTypes[gt.Parameters[1].Name] = valueType;
}
break;
case IPythonIterable iter when gt.TypeId == BuiltinTypeId.List && gt.Parameters.Count == 1:
var itemType = iter.GetIterator().Next.GetPythonType();
if (!itemType.IsUnknown()) {
- specificTypes[gt.Parameters[0]] = itemType;
+ specificTypes[gt.Parameters[0].Name] = itemType;
}
break;
case IPythonCollection coll when gt.TypeId == BuiltinTypeId.Tuple && gt.Parameters.Count >= 1:
var itemTypes = coll.Contents.Select(m => m.GetPythonType()).ToArray();
for (var i = 0; i < Math.Min(itemTypes.Length, gt.Parameters.Count); i++) {
- specificTypes[gt.Parameters[i]] = itemTypes[i];
+ specificTypes[gt.Parameters[i].Name] = itemTypes[i];
}
break;
}
@@ -361,7 +361,7 @@ private void SetClassMembers(IPythonClassType templateClass, IArgumentSet args)
specificType = tt.CreateSpecificType(args);
break;
case IGenericTypeParameter gtd:
- GenericParameters.TryGetValue(gtd, out specificType);
+ GenericParameters.TryGetValue(gtd.Name, out specificType);
break;
}
diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs
index a21cd81a5..0f35ff1d9 100644
--- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs
+++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs
@@ -214,8 +214,8 @@ public IReadOnlyList Mro {
/// class B(A[int, str]): ...
/// Has the map {T: int, K: str}
///
- public virtual IReadOnlyDictionary GenericParameters =>
- _genericParameters ?? EmptyDictionary.Instance;
+ public virtual IReadOnlyDictionary GenericParameters =>
+ _genericParameters ?? EmptyDictionary.Instance;
#endregion
@@ -226,6 +226,13 @@ internal override void SetDocumentation(string documentation) {
DocumentationSource = ClassDocumentationSource.Class;
}
+ ///
+ /// Sets class bases. If scope is provided, detects loops in base classes and removes them.
+ ///
+ /// List of base types.
+ /// Current scope to look up base types.
+ /// Can be null if class is restored from database, in which case
+ /// there is no need to try and disambiguate bases.
internal void SetBases(IEnumerable bases, IScope currentScope = null) {
if (_bases != null) {
return; // Already set
diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs
index d5e122867..d50c55fce 100644
--- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs
+++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs
@@ -161,7 +161,7 @@ private IMember CreateSpecificReturnFromClassType(IPythonClassType selfClassType
// -> A[_T1, _T2, ...]
// Match arguments
IReadOnlyList typeArgs = null;
- var classGenericParameters = selfClassType?.GenericParameters.Keys.ToArray() ?? Array.Empty();
+ var classGenericParameters = selfClassType?.GenericParameters.Keys.ToArray() ?? Array.Empty();
if (classGenericParameters.Length > 0 && selfClassType != null) {
// Declaring class is specific and provides definitions of generic parameters
typeArgs = classGenericParameters
@@ -181,7 +181,7 @@ private IMember CreateSpecificReturnFromClassType(IPythonClassType selfClassType
}
private IMember CreateSpecificReturnFromTypeVar(IPythonClassType selfClassType, IArgumentSet args, IGenericTypeParameter returnType) {
- if (selfClassType.GetSpecificType(returnType, out var specificType)) {
+ if (selfClassType.GetSpecificType(returnType.Name, out var specificType)) {
return new PythonInstance(specificType);
}
@@ -189,10 +189,10 @@ private IMember CreateSpecificReturnFromTypeVar(IPythonClassType selfClassType,
var baseType = selfClassType.Mro
.OfType()
.Skip(1)
- .FirstOrDefault(b => b.GetMember(ClassMember.Name) != null && b.GenericParameters.ContainsKey(returnType));
+ .FirstOrDefault(b => b.GetMember(ClassMember.Name) != null && b.GenericParameters.ContainsKey(returnType.Name));
// Try and infer return value from base class
- if (baseType != null && baseType.GetSpecificType(returnType, out specificType)) {
+ if (baseType != null && baseType.GetSpecificType(returnType.Name, out specificType)) {
return new PythonInstance(specificType);
}
diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs
index 8db2ab7cd..8058a08c8 100644
--- a/src/Analysis/Ast/Impl/Types/PythonType.cs
+++ b/src/Analysis/Ast/Impl/Types/PythonType.cs
@@ -23,7 +23,7 @@
namespace Microsoft.Python.Analysis.Types {
[DebuggerDisplay("{" + nameof(Name) + "}")]
- internal class PythonType : LocatedMember, IPythonType {//, IEquatable {
+ internal class PythonType : LocatedMember, IPythonType {
private readonly object _lock = new object();
private Dictionary _members;
private BuiltinTypeId _typeId;
@@ -159,16 +159,14 @@ internal IMember AddMember(string name, IMember member, bool overwrite) {
}
}
- internal void MakeReadOnly() => _readonly = true;
+ internal void MakeReadOnly() {
+ lock (_lock) {
+ _readonly = true;
+ }
+ }
internal bool IsHidden => ContainsMember("__hidden__");
protected bool ContainsMember(string name) => Members.ContainsKey(name);
protected IPythonType UnknownType => DeclaringModule.Interpreter.UnknownType;
-
- //public bool Equals(IPythonType other) => PythonTypeComparer.Instance.Equals(this, other);
-
- //public override bool Equals(object obj)
- // => obj is IPythonType pt && PythonTypeComparer.Instance.Equals(this, pt);
- //public override int GetHashCode() => 0;
}
}
diff --git a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs
index 876886d81..77887be1a 100644
--- a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs
+++ b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs
@@ -138,8 +138,9 @@ public void HaveSameMembersAs(IMember other) {
}
subjectMemberType.MemberType.Should().Be(otherMemberType.MemberType);
+ Debug.Assert(subjectMemberType.MemberType == otherMemberType.MemberType);
- if(subjectMemberType is IPythonClassType subjectClass) {
+ if (subjectMemberType is IPythonClassType subjectClass) {
var otherClass = otherMemberType as IPythonClassType;
otherClass.Should().NotBeNull();
@@ -163,8 +164,10 @@ public void HaveSameMembersAs(IMember other) {
case PythonMemberType.Class:
// Restored collections (like instance of tuple) turn into classes
// rather than into collections with content since we don't track
- // collection content in libraries.
- subjectMemberType.QualifiedName.Should().Be(otherMemberType.QualifiedName);
+ // collection content in libraries. We don't compare qualified names
+ // since original module may be source or a stub and that is not
+ // preserved during restore.
+ // subjectMemberType.QualifiedName.Should().Be(otherMemberType.QualifiedName);
break;
case PythonMemberType.Function:
case PythonMemberType.Method:
diff --git a/src/Analysis/Ast/Test/TypeshedTests.cs b/src/Analysis/Ast/Test/TypeshedTests.cs
index 9b1a72c6f..9ab18657b 100644
--- a/src/Analysis/Ast/Test/TypeshedTests.cs
+++ b/src/Analysis/Ast/Test/TypeshedTests.cs
@@ -45,7 +45,7 @@ import sys
e1, e2, e3 = sys.exc_info()
";
var analysis = await GetAnalysisAsync(code);
- // sys.exc_info() -> (exception_type, exception_value, traceback)
+ // def exc_info() -> Tuple[Optional[Type[BaseException]], Optional[BaseException], Optional[TracebackType]]: ...
var f = analysis.Should()
.HaveVariable("e1").OfType(BuiltinTypeId.Type)
.And.HaveVariable("e2").OfType("BaseException")
diff --git a/src/Analysis/Ast/Test/TypingTests.cs b/src/Analysis/Ast/Test/TypingTests.cs
index 2df9a7df7..31d9fd1de 100644
--- a/src/Analysis/Ast/Test/TypingTests.cs
+++ b/src/Analysis/Ast/Test/TypingTests.cs
@@ -83,10 +83,25 @@ from typing import TypeVar
[TestMethod, Priority(0)]
- public async Task KeywordArgMixDocCheck() {
+ public async Task TypeVarBoundToUnknown() {
const string code = @"
from typing import TypeVar
X = TypeVar('X', bound='hello', covariant=True)
+";
+ var analysis = await GetAnalysisAsync(code);
+ analysis.Should().HaveVariable("X")
+ .Which.Value.Should().HaveDocumentation("TypeVar('X', bound=Unknown, covariant=True)");
+ analysis.Should().HaveGenericVariable("X");
+ }
+
+ [TestMethod, Priority(0)]
+ public async Task TypeVarBoundToStringName() {
+ const string code = @"
+from typing import TypeVar
+
+X = TypeVar('X', bound='hello', covariant=True)
+
+class hello: ...
";
var analysis = await GetAnalysisAsync(code);
analysis.Should().HaveVariable("X")
diff --git a/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs b/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs
index 132d06ff1..1c7dcfaed 100644
--- a/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs
+++ b/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs
@@ -13,16 +13,21 @@
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.
+using System.Collections.Generic;
using System.Linq;
using Microsoft.Python.Parsing.Ast;
namespace Microsoft.Python.Analysis.Core.DependencyResolution {
public static class AstUtilities {
public static IImportSearchResult FindImports(this PathResolverSnapshot pathResolver, string modulePath, FromImportStatement fromImportStatement) {
- var rootNames = fromImportStatement.Root.Names.Select(n => n.Name);
- return fromImportStatement.Root is RelativeModuleName relativeName
- ? pathResolver.GetImportsFromRelativePath(modulePath, relativeName.DotCount, rootNames)
- : pathResolver.GetImportsFromAbsoluteName(modulePath, rootNames, fromImportStatement.ForceAbsolute);
+ var rootNames = fromImportStatement.Root.Names.Select(n => n.Name).ToArray();
+ var dotCount = fromImportStatement.Root is RelativeModuleName relativeName ? relativeName.DotCount : 0;
+ return pathResolver.FindImports(modulePath, rootNames, dotCount, fromImportStatement.ForceAbsolute);
}
+
+ public static IImportSearchResult FindImports(this PathResolverSnapshot pathResolver, string modulePath, IReadOnlyList rootNames, int dotCount, bool forceAbsolute)
+ => dotCount > 0
+ ? pathResolver.GetImportsFromRelativePath(modulePath, dotCount, rootNames)
+ : pathResolver.GetImportsFromAbsoluteName(modulePath, rootNames, forceAbsolute);
}
}
diff --git a/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs b/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs
index b229ab80e..9d95188e3 100644
--- a/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs
+++ b/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs
@@ -27,6 +27,7 @@ public class ModuleImport : IImportChildrenSource {
public bool IsCompiled { get; }
public bool IsLibrary { get; }
public bool IsBuiltin => IsCompiled && ModulePath == null;
+ public bool IsPersistent { get; set; }
public ModuleImport(IImportChildrenSource childrenSource, string name, string fullName, string rootPath, string modulePath, long moduleFileSize, bool isCompiled, bool isLibrary) {
_childrenSource = childrenSource;
diff --git a/src/Caching/Impl/Models/ClassModel.cs b/src/Caching/Impl/Models/ClassModel.cs
index 2b5aa0a70..527816157 100644
--- a/src/Caching/Impl/Models/ClassModel.cs
+++ b/src/Caching/Impl/Models/ClassModel.cs
@@ -40,7 +40,7 @@ internal sealed class ClassModel : MemberModel {
public ClassModel[] Classes { get; set; }
///
- /// FormalGenericParameters of the Generic[...] base class, if any.
+ /// GenericParameters of the Generic[...] base class, if any.
///
public string[] GenericBaseParameters { get; set; }
///
@@ -106,7 +106,6 @@ public ClassModel(IPythonClassType cls) {
// Only persist documentation from this class, leave bases or __init__ alone.
Documentation = (cls as PythonClassType)?.DocumentationSource == PythonClassType.ClassDocumentationSource.Class ? cls.Documentation : null;
-
var ntBases = cls.Bases.OfType().ToArray();
NamedTupleBases = ntBases.Select(b => new NamedTupleModel(b)).ToArray();
@@ -126,17 +125,16 @@ public ClassModel(IPythonClassType cls) {
GenericBaseParameters = GenericBaseParameters ?? Array.Empty();
GenericParameterValues = cls.GenericParameters
- .Select(p => new GenericParameterValueModel { Name = p.Key.Name, Type = p.Value.GetPersistentQualifiedName() })
+ .Select(p => new GenericParameterValueModel { Name = p.Key, Type = p.Value.GetPersistentQualifiedName() })
.ToArray();
}
- protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) {
- if (_cls != null) {
+ public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) {
+ if(_cls != null) {
return _cls;
}
_cls = new PythonClassType(Name, new Location(mf.Module, IndexSpan.ToSpan()));
-
- var bases = CreateBases(mf);
+ var bases = CreateBases(mf, gs);
_cls.SetBases(bases);
_cls.SetDocumentation(Documentation);
@@ -146,37 +144,32 @@ protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringTy
_cls,
_cls.GenericParameters.Keys.ToArray(),
GenericParameterValues.ToDictionary(
- k => _cls.GenericParameters.Keys.First(x => x.Name == k.Name),
+ k => _cls.GenericParameters.Keys.First(x => x == k.Name),
v => mf.ConstructType(v.Type)
)
);
}
- foreach (var f in Methods) {
- var m = f.Construct(mf, _cls);
- _cls.AddMember(f.Name, m, false);
- }
-
- foreach (var p in Properties) {
- var m = p.Construct(mf, _cls);
- _cls.AddMember(p.Name, m, false);
- }
-
- foreach (var c in Classes) {
- var m = c.Construct(mf, _cls);
- _cls.AddMember(c.Name, m, false);
+ var all = Classes.Concat(Properties).Concat(Methods).Concat(Fields);
+ foreach (var m in all) {
+ _cls.AddMember(m.Name, m.Create(mf, _cls, gs), false);
}
+ return _cls;
+ }
- foreach (var vm in Fields) {
- var m = vm.Construct(mf, _cls);
- _cls.AddMember(vm.Name, m, false);
+ public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) {
+ var all = Classes.Concat(Properties).Concat(Methods).Concat(Fields);
+ foreach (var m in all) {
+ m.Populate(mf, _cls, gs);
}
-
- return _cls;
}
- private IPythonType[] CreateBases(ModuleFactory mf) {
- var ntBases = NamedTupleBases.Select(ntb => ntb.Construct(mf, _cls)).OfType().ToArray();
+ private IEnumerable CreateBases(ModuleFactory mf, IGlobalScope gs) {
+ var ntBases = NamedTupleBases.Select(ntb => {
+ var n = ntb.Create(mf, _cls, gs);
+ ntb.Populate(mf, _cls, gs);
+ return n;
+ }).OfType().ToArray();
var is3x = mf.Module.Interpreter.LanguageVersion.Is3x();
var basesNames = Bases.Select(b => is3x && b == "object" ? null : b).ExcludeDefault().ToArray();
@@ -184,11 +177,11 @@ private IPythonType[] CreateBases(ModuleFactory mf) {
if (GenericBaseParameters.Length > 0) {
// Generic class. Need to reconstruct generic base so code can then
- // create specific types off the generic class.
+ // create specific types off the generic class.
var genericBase = bases.OfType().FirstOrDefault(b => b.Name == "Generic");
if (genericBase != null) {
- var typeVars = GenericBaseParameters.Select(n => mf.Module.GlobalScope.Variables[n]?.Value).OfType().ToArray();
- Debug.Assert(typeVars.Length > 0, "Class generic type parameters were not defined in the module during restore");
+ var typeVars = GenericBaseParameters.Select(n => gs.Variables[n]?.Value).OfType().ToArray();
+ //Debug.Assert(typeVars.Length > 0, "Class generic type parameters were not defined in the module during restore");
if (typeVars.Length > 0) {
var genericWithParameters = genericBase.CreateSpecificType(new ArgumentSet(typeVars, null, null));
if (genericWithParameters != null) {
diff --git a/src/Caching/Impl/Models/FromImportModel.cs b/src/Caching/Impl/Models/FromImportModel.cs
new file mode 100644
index 000000000..bda4d9774
--- /dev/null
+++ b/src/Caching/Impl/Models/FromImportModel.cs
@@ -0,0 +1,25 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+namespace Microsoft.Python.Analysis.Caching.Models {
+ ///
+ /// Represents from import statement for dependency resolution.
+ ///
+ internal sealed class FromImportModel {
+ public string[] RootNames { get; set; }
+ public int DotCount { get; set; }
+ public bool ForceAbsolute { get; set; }
+ }
+}
diff --git a/src/Caching/Impl/Models/FunctionModel.cs b/src/Caching/Impl/Models/FunctionModel.cs
index 88da25f78..7c50a97cd 100644
--- a/src/Caching/Impl/Models/FunctionModel.cs
+++ b/src/Caching/Impl/Models/FunctionModel.cs
@@ -17,6 +17,8 @@
using System.Diagnostics;
using System.Linq;
using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Analysis.Values;
+
// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global
// ReSharper disable MemberCanBePrivate.Global
@@ -33,22 +35,18 @@ public FunctionModel(IPythonFunctionType func) : base(func) {
Overloads = func.Overloads.Select(FromOverload).ToArray();
}
- protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) {
- if (_function != null) {
- return _function;
- }
- _function = new PythonFunctionType(Name, new Location(mf.Module, IndexSpan.ToSpan()), declaringType, Documentation);
+ public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs)
+ => _function ?? (_function = new PythonFunctionType(Name, new Location(mf.Module, IndexSpan.ToSpan()), declaringType, Documentation));
- // Create inner functions and classes first since function
- // may be returning one of them.
- foreach (var model in Functions) {
- var f = model.Construct(mf, _function);
- _function.AddMember(Name, f, overwrite: true);
- }
+ public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) {
+ // Create inner functions and classes first since function may be returning one of them.
+ var all = Classes.Concat(Functions).ToArray();
- foreach (var cm in Classes) {
- var c = cm.Construct(mf, _function);
- _function.AddMember(cm.Name, c, overwrite: true);
+ foreach (var model in all) {
+ _function.AddMember(Name, model.Create(mf, _function, gs), overwrite: true);
+ }
+ foreach (var model in all) {
+ model.Populate(mf, _function, gs);
}
foreach (var om in Overloads) {
@@ -58,9 +56,8 @@ protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringTy
o.SetParameters(om.Parameters.Select(p => ConstructParameter(mf, p)).ToArray());
_function.AddOverload(o);
}
-
- return _function;
}
+
private IParameterInfo ConstructParameter(ModuleFactory mf, ParameterModel pm)
=> new ParameterInfo(pm.Name, mf.ConstructType(pm.Type), pm.Kind, mf.ConstructMember(pm.DefaultValue));
diff --git a/src/Caching/Impl/Models/ImportModel.cs b/src/Caching/Impl/Models/ImportModel.cs
new file mode 100644
index 000000000..e99b58fcf
--- /dev/null
+++ b/src/Caching/Impl/Models/ImportModel.cs
@@ -0,0 +1,24 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+namespace Microsoft.Python.Analysis.Caching.Models {
+ ///
+ /// Represents import statement for dependency resolution.
+ ///
+ internal sealed class ImportModel {
+ public string[] ModuleNames { get; set; }
+ public bool ForceAbsolute { get; set; }
+ }
+}
diff --git a/src/Caching/Impl/Models/MemberModel.cs b/src/Caching/Impl/Models/MemberModel.cs
index 24d512fcf..e7ce43282 100644
--- a/src/Caching/Impl/Models/MemberModel.cs
+++ b/src/Caching/Impl/Models/MemberModel.cs
@@ -17,6 +17,7 @@
using System.Collections.Generic;
using System.Linq;
using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Analysis.Values;
// ReSharper disable MemberCanBeProtected.Global
// ReSharper disable UnusedAutoPropertyAccessor.Global
@@ -43,13 +44,17 @@ internal abstract class MemberModel {
///
public IndexSpanModel IndexSpan { get; set; }
- [NonSerialized]
- private IMember _member;
-
- public IMember Construct(ModuleFactory mf, IPythonType declaringType)
- => _member ?? (_member = ReConstruct(mf, declaringType));
- protected abstract IMember ReConstruct(ModuleFactory mf, IPythonType declaringType);
+ ///
+ /// Create member for declaration but does not construct its parts just yet.
+ /// Used as a first pass in two-pass handling of forward declarations.
+ ///
+ public abstract IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs);
+ ///
+ /// Populate member with content, such as create class methods, etc.
+ ///
+ public abstract void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs);
+
public virtual MemberModel GetModel(string name) => GetMemberModels().FirstOrDefault(m => m.Name == name);
protected virtual IEnumerable GetMemberModels() => Enumerable.Empty();
}
diff --git a/src/Caching/Impl/Models/ModuleModel.cs b/src/Caching/Impl/Models/ModuleModel.cs
index dfe859d35..c67837cb3 100644
--- a/src/Caching/Impl/Models/ModuleModel.cs
+++ b/src/Caching/Impl/Models/ModuleModel.cs
@@ -21,6 +21,7 @@
using Microsoft.Python.Analysis.Types;
using Microsoft.Python.Analysis.Values;
using Microsoft.Python.Core;
+using Microsoft.Python.Parsing.Ast;
// ReSharper disable MemberCanBePrivate.Global
namespace Microsoft.Python.Analysis.Caching.Models {
@@ -49,11 +50,16 @@ internal sealed class ModuleModel : MemberModel {
///
public int FileSize { get; set; }
+ public ImportModel[] Imports { get; set; }
+ public FromImportModel[] FromImports { get; set; }
+ public ImportModel[] StubImports { get; set; }
+ public FromImportModel[] StubFromImports { get; set; }
+
[NonSerialized] private Dictionary _modelCache;
public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceContainer services, AnalysisCachingLevel options) {
var uniqueId = analysis.Document.GetUniqueId(services, options);
- if(uniqueId == null) {
+ if (uniqueId == null) {
// Caching level setting does not permit this module to be persisted.
return null;
}
@@ -68,9 +74,9 @@ public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceConta
// as well as variables that are declarations.
var exportedNames = new HashSet(analysis.Document.GetMemberNames());
foreach (var v in analysis.GlobalScope.Variables
- .Where(v => exportedNames.Contains(v.Name) ||
- v.Source == VariableSource.Declaration ||
- v.Source == VariableSource.Builtin ||
+ .Where(v => exportedNames.Contains(v.Name) ||
+ v.Source == VariableSource.Declaration ||
+ v.Source == VariableSource.Builtin ||
v.Source == VariableSource.Generic)) {
if (v.Value is IGenericTypeParameter && !typeVars.ContainsKey(v.Name)) {
@@ -115,10 +121,20 @@ when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals
}
}
+ // Take dependencies from imports. If module has stub we should also take
+ // dependencies from there since persistent state is based on types that
+ // are combination of stub and the module. Sometimes stub may import more
+ // and we must make sure dependencies are restored before the module.
+ var primaryDependencyWalker = new DependencyWalker(analysis.Ast);
+ var stubDependencyWalker = analysis.Document.Stub != null ? new DependencyWalker(analysis.Document.Stub.Analysis.Ast) : null;
+ var stubImports = stubDependencyWalker?.Imports ?? Enumerable.Empty();
+ var stubFromImports = stubDependencyWalker?.FromImports ?? Enumerable.Empty();
+
return new ModuleModel {
Id = uniqueId.GetStableHash(),
UniqueId = uniqueId,
Name = analysis.Document.Name,
+ QualifiedName = analysis.Document.QualifiedName,
Documentation = analysis.Document.Documentation,
Functions = functions.Values.ToArray(),
Variables = variables.Values.ToArray(),
@@ -129,7 +145,11 @@ when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals
EndIndex = l.EndIndex,
Kind = l.Kind
}).ToArray(),
- FileSize = analysis.Ast.EndIndex
+ FileSize = analysis.Ast.EndIndex,
+ Imports = primaryDependencyWalker.Imports.ToArray(),
+ FromImports = primaryDependencyWalker.FromImports.ToArray(),
+ StubImports = stubImports.ToArray(),
+ StubFromImports = stubFromImports.ToArray()
};
}
@@ -149,8 +169,6 @@ private static FunctionModel GetFunctionModel(IDocumentAnalysis analysis, IVaria
return null;
}
- protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) => throw new NotImplementedException();
-
public override MemberModel GetModel(string name) {
if (_modelCache == null) {
var models = TypeVars.Concat(NamedTuples).Concat(Classes).Concat(Functions).Concat(Variables);
@@ -160,8 +178,38 @@ public override MemberModel GetModel(string name) {
_modelCache[m.Name] = m;
}
}
-
return _modelCache.TryGetValue(name, out var model) ? model : null;
}
+
+ public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) => throw new NotImplementedException();
+ public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) => throw new NotImplementedException();
+
+ private sealed class DependencyWalker : PythonWalker {
+ public List Imports { get; } = new List();
+ public List FromImports { get; } = new List();
+
+ public DependencyWalker(PythonAst ast) {
+ ast.Walk(this);
+ }
+
+ public override bool Walk(ImportStatement import) {
+ var model = new ImportModel {
+ ForceAbsolute = import.ForceAbsolute,
+ ModuleNames = import.Names.SelectMany(mn => mn.Names).Select(n => n.Name).ToArray()
+ };
+ Imports.Add(model);
+ return false;
+ }
+
+ public override bool Walk(FromImportStatement fromImport) {
+ var model = new FromImportModel {
+ ForceAbsolute = fromImport.ForceAbsolute,
+ RootNames = fromImport.Root.Names.Select(n => n.Name).ToArray(),
+ DotCount = fromImport.Root is RelativeModuleName rn ? rn.DotCount : 0
+ };
+ FromImports.Add(model);
+ return false;
+ }
+ }
}
}
diff --git a/src/Caching/Impl/Models/NamedTupleModel.cs b/src/Caching/Impl/Models/NamedTupleModel.cs
index 5f17059c9..82fed074b 100644
--- a/src/Caching/Impl/Models/NamedTupleModel.cs
+++ b/src/Caching/Impl/Models/NamedTupleModel.cs
@@ -19,6 +19,7 @@
using Microsoft.Python.Analysis.Specializations.Typing;
using Microsoft.Python.Analysis.Specializations.Typing.Types;
using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Analysis.Values;
using Microsoft.Python.Core;
namespace Microsoft.Python.Analysis.Caching.Models {
@@ -41,7 +42,7 @@ public NamedTupleModel(ITypingNamedTupleType nt) {
ItemTypes = nt.ItemTypes.Select(t => t.QualifiedName).ToArray();
}
- protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) {
+ public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) {
if (_namedTuple != null) {
return _namedTuple;
}
@@ -50,5 +51,7 @@ protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringTy
_namedTuple = new NamedTupleType(Name, ItemNames, itemTypes, mf.Module, IndexSpan.ToSpan());
return _namedTuple;
}
+
+ public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { }
}
}
diff --git a/src/Caching/Impl/Models/PropertyModel.cs b/src/Caching/Impl/Models/PropertyModel.cs
index a5b33e025..e608584e0 100644
--- a/src/Caching/Impl/Models/PropertyModel.cs
+++ b/src/Caching/Impl/Models/PropertyModel.cs
@@ -15,6 +15,8 @@
using System;
using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Analysis.Values;
+
// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global
// ReSharper disable MemberCanBePrivate.Global
@@ -30,19 +32,16 @@ public PropertyModel(IPythonPropertyType prop) : base(prop) {
ReturnType = prop.ReturnType.GetPersistentQualifiedName();
}
- protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) {
- if (_property != null) {
- return _property;
- }
- _property = new PythonPropertyType(Name, new Location(mf.Module, IndexSpan.ToSpan()), declaringType, (Attributes & FunctionAttributes.Abstract) != 0);
+ public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs)
+ => _property ?? (_property = new PythonPropertyType(Name, new Location(mf.Module, IndexSpan.ToSpan()), declaringType, (Attributes & FunctionAttributes.Abstract) != 0));
+
+ public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) {
_property.SetDocumentation(Documentation);
var o = new PythonFunctionOverload(Name, mf.DefaultLocation);
o.SetDocumentation(Documentation);
o.SetReturnValue(mf.ConstructMember(ReturnType), true);
_property.AddOverload(o);
-
- return _property;
}
}
}
diff --git a/src/Caching/Impl/Models/TypeVarModel.cs b/src/Caching/Impl/Models/TypeVarModel.cs
index a3f5c4fc5..e630d801f 100644
--- a/src/Caching/Impl/Models/TypeVarModel.cs
+++ b/src/Caching/Impl/Models/TypeVarModel.cs
@@ -45,9 +45,11 @@ public static TypeVarModel FromGeneric(IVariable v) {
};
}
- protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType)
+ public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs)
=> new GenericTypeParameter(Name, mf.Module,
Constraints.Select(mf.ConstructType).ToArray(),
mf.ConstructType(Bound), Covariant, Contravariant, default);
+
+ public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { }
}
}
diff --git a/src/Caching/Impl/Models/VariableModel.cs b/src/Caching/Impl/Models/VariableModel.cs
index c3e2696cf..892dac8ac 100644
--- a/src/Caching/Impl/Models/VariableModel.cs
+++ b/src/Caching/Impl/Models/VariableModel.cs
@@ -23,7 +23,7 @@
namespace Microsoft.Python.Analysis.Caching.Models {
[Serializable]
[DebuggerDisplay("v:{Name} = {Value}")]
- internal sealed class VariableModel: MemberModel {
+ internal sealed class VariableModel : MemberModel {
public string Value { get; set; }
public static VariableModel FromVariable(IVariable v) => new VariableModel {
@@ -49,9 +49,11 @@ internal sealed class VariableModel: MemberModel {
Value = t.GetPersistentQualifiedName()
};
- protected override IMember ReConstruct(ModuleFactory mf, IPythonType declaringType) {
+ public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) {
var m = mf.ConstructMember(Value) ?? mf.Module.Interpreter.UnknownType;
return new Variable(Name, m, VariableSource.Declaration, new Location(mf.Module, IndexSpan?.ToSpan() ?? default));
}
+
+ public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { }
}
}
diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs
index 97af51827..a58ee0118 100644
--- a/src/Caching/Impl/ModuleDatabase.cs
+++ b/src/Caching/Impl/ModuleDatabase.cs
@@ -14,12 +14,15 @@
// permissions and limitations under the License.
using System;
+using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using LiteDB;
+using Microsoft.Python.Analysis.Analyzer;
using Microsoft.Python.Analysis.Caching.Models;
+using Microsoft.Python.Analysis.Dependencies;
using Microsoft.Python.Analysis.Modules;
using Microsoft.Python.Analysis.Types;
using Microsoft.Python.Core;
@@ -27,9 +30,12 @@
using Microsoft.Python.Core.Logging;
namespace Microsoft.Python.Analysis.Caching {
- public sealed class ModuleDatabase : IModuleDatabaseService {
+ internal sealed class ModuleDatabase : IModuleDatabaseService {
private const int _databaseFormatVersion = 1;
+ private readonly Dictionary _dependencies = new Dictionary();
+ private readonly object _lock = new object();
+
private readonly IServiceContainer _services;
private readonly ILogger _log;
private readonly IFileSystem _fs;
@@ -39,56 +45,56 @@ public ModuleDatabase(IServiceContainer services) {
_services = services;
_log = services.GetService();
_fs = services.GetService();
-
+
var cfs = services.GetService();
_databaseFolder = Path.Combine(cfs.CacheFolder, $"analysis.v{_databaseFormatVersion}");
}
///
- /// Retrieves module representation from module index database
- /// or null if module does not exist.
+ /// Retrieves dependencies from the module persistent state.
///
- /// Module name. If the name is not qualified
- /// the module will ge resolved against active Python version.
- /// Module file path.
- /// Python module.
- /// Module storage state
- public ModuleStorageState TryCreateModule(string moduleName, string filePath, out IPythonModule module) {
- module = null;
+ /// Python module to restore analysis for.
+ /// Python module dependency provider.
+ public bool TryRestoreDependencies(IPythonModule module, out IDependencyProvider dp) {
+ dp = null;
- if (GetCachingLevel() == AnalysisCachingLevel.None) {
- return ModuleStorageState.DoesNotExist;
+ if (GetCachingLevel() == AnalysisCachingLevel.None || !module.ModuleType.CanBeCached()) {
+ return false;
}
- // We don't cache results here. Module resolution service decides when to call in here
- // and it is responsible of overall management of the loaded Python modules.
- for (var retries = 50; retries > 0; --retries) {
- try {
- // TODO: make combined db rather than per module?
- var dbPath = FindDatabaseFile(moduleName, filePath);
- if (string.IsNullOrEmpty(dbPath)) {
- return ModuleStorageState.DoesNotExist;
- }
+ lock (_lock) {
+ if (_dependencies.TryGetValue(module.Name, out dp)) {
+ return true;
+ }
+ if (FindModuleModel(module.Name, module.FilePath, out var model)) {
+ dp = new DependencyProvider(module, model);
+ _dependencies[module.Name] = dp;
+ return true;
+ }
+ }
+ return false;
+ }
- using (var db = new LiteDatabase(dbPath)) {
- if (!db.CollectionExists("modules")) {
- return ModuleStorageState.Corrupted;
- }
+ ///
+ /// Creates global scope from module persistent state.
+ /// Global scope is then can be used to construct module analysis.
+ ///
+ /// Python module to restore analysis for.
+ /// Python module global scope.
+ public bool TryRestoreGlobalScope(IPythonModule module, out IRestoredGlobalScope gs) {
+ gs = null;
- var modules = db.GetCollection("modules");
- var model = modules.Find(m => m.Name == moduleName).FirstOrDefault();
- if (model == null) {
- return ModuleStorageState.DoesNotExist;
- }
+ if (GetCachingLevel() == AnalysisCachingLevel.None || !module.ModuleType.CanBeCached()) {
+ return false;
+ }
- module = new PythonDbModule(model, filePath, _services);
- return ModuleStorageState.Complete;
- }
- } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) {
- Thread.Sleep(10);
+ lock (_lock) {
+ if (FindModuleModel(module.Name, module.FilePath, out var model)) {
+ gs = new RestoredGlobalScope(model, module);
}
}
- return ModuleStorageState.DoesNotExist;
+
+ return gs != null;
}
///
@@ -101,14 +107,16 @@ public Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationTok
/// Determines if module analysis exists in the storage.
///
public bool ModuleExistsInStorage(string moduleName, string filePath) {
- if(GetCachingLevel() == AnalysisCachingLevel.None) {
+ if (GetCachingLevel() == AnalysisCachingLevel.None) {
return false;
}
for (var retries = 50; retries > 0; --retries) {
try {
- var dbPath = FindDatabaseFile(moduleName, filePath);
- return !string.IsNullOrEmpty(dbPath);
+ lock (_lock) {
+ var dbPath = FindDatabaseFile(moduleName, filePath);
+ return !string.IsNullOrEmpty(dbPath);
+ }
} catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) {
Thread.Sleep(10);
}
@@ -116,9 +124,15 @@ public bool ModuleExistsInStorage(string moduleName, string filePath) {
return false;
}
+ public void Clear() {
+ lock (_lock) {
+ _dependencies.Clear();
+ }
+ }
+
private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken cancellationToken = default) {
var cachingLevel = GetCachingLevel();
- if(cachingLevel == AnalysisCachingLevel.None) {
+ if (cachingLevel == AnalysisCachingLevel.None) {
return;
}
@@ -130,24 +144,26 @@ private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken c
Exception ex = null;
for (var retries = 50; retries > 0; --retries) {
- cancellationToken.ThrowIfCancellationRequested();
- try {
- if (!_fs.DirectoryExists(_databaseFolder)) {
- _fs.CreateDirectory(_databaseFolder);
- }
-
+ lock (_lock) {
cancellationToken.ThrowIfCancellationRequested();
- using (var db = new LiteDatabase(Path.Combine(_databaseFolder, $"{model.UniqueId}.db"))) {
- var modules = db.GetCollection("modules");
- modules.Upsert(model);
- return;
+ try {
+ if (!_fs.DirectoryExists(_databaseFolder)) {
+ _fs.CreateDirectory(_databaseFolder);
+ }
+
+ cancellationToken.ThrowIfCancellationRequested();
+ using (var db = new LiteDatabase(Path.Combine(_databaseFolder, $"{model.UniqueId}.db"))) {
+ var modules = db.GetCollection("modules");
+ modules.Upsert(model);
+ return;
+ }
+ } catch (Exception ex1) when (ex1 is IOException || ex1 is UnauthorizedAccessException) {
+ ex = ex1;
+ Thread.Sleep(10);
+ } catch (Exception ex2) {
+ ex = ex2;
+ break;
}
- } catch (Exception ex1) when (ex1 is IOException || ex1 is UnauthorizedAccessException) {
- ex = ex1;
- Thread.Sleep(10);
- } catch (Exception ex2) {
- ex = ex2;
- break;
}
}
@@ -191,7 +207,52 @@ private string FindDatabaseFile(string moduleName, string filePath) {
return _fs.FileExists(dbPath) ? dbPath : null;
}
+ private bool FindModuleModel(string moduleName, string filePath, out ModuleModel model) {
+ model = null;
+
+ // We don't cache results here. Module resolution service decides when to call in here
+ // and it is responsible of overall management of the loaded Python modules.
+ for (var retries = 50; retries > 0; --retries) {
+ try {
+ // TODO: make combined db rather than per module?
+ var dbPath = FindDatabaseFile(moduleName, filePath);
+ if (string.IsNullOrEmpty(dbPath)) {
+ return false;
+ }
+
+ using (var db = new LiteDatabase(dbPath)) {
+ if (!db.CollectionExists("modules")) {
+ return false;
+ }
+
+ var modules = db.GetCollection("modules");
+ model = modules.Find(m => m.Name == moduleName).FirstOrDefault();
+ return model != null;
+ }
+ } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) {
+ Thread.Sleep(10);
+ }
+ }
+ return false;
+ }
private AnalysisCachingLevel GetCachingLevel()
=> _services.GetService()?.Options.AnalysisCachingLevel ?? AnalysisCachingLevel.None;
+
+ private sealed class DependencyProvider : IDependencyProvider {
+ private readonly HashSet _dependencies;
+
+ public DependencyProvider(IPythonModule module, ModuleModel model) {
+ var dc = new DependencyCollector(module);
+ foreach (var imp in model.Imports) {
+ dc.AddImport(imp.ModuleNames, imp.ForceAbsolute);
+ }
+ foreach (var fi in model.FromImports) {
+ dc.AddFromImport(fi.RootNames, fi.DotCount, fi.ForceAbsolute);
+ }
+ _dependencies = dc.Dependencies;
+ }
+
+ public HashSet GetDependencies() => _dependencies;
+ }
}
}
diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs
index 10d0f1651..4fad9d9da 100644
--- a/src/Caching/Impl/ModuleFactory.cs
+++ b/src/Caching/Impl/ModuleFactory.cs
@@ -31,22 +31,26 @@ namespace Microsoft.Python.Analysis.Caching {
/// Constructs module from its persistent model.
///
internal sealed class ModuleFactory {
+ /// For use in tests so missing members will assert.
+ internal static bool EnableMissingMemberAssertions { get; set; }
+
// TODO: better resolve circular references.
- private readonly ReentrancyGuard _typeReentrancy = new ReentrancyGuard();
private readonly ReentrancyGuard _moduleReentrancy = new ReentrancyGuard();
private readonly ModuleModel _model;
+ private readonly IGlobalScope _gs;
public IPythonModule Module { get; }
public Location DefaultLocation { get; }
- public ModuleFactory(ModuleModel model, IPythonModule module) {
+ public ModuleFactory(ModuleModel model, IPythonModule module, IGlobalScope gs) {
_model = model;
-
+ _gs = gs;
Module = module;
DefaultLocation = new Location(Module);
}
- public IPythonType ConstructType(string qualifiedName) => ConstructMember(qualifiedName)?.GetPythonType();
+ public IPythonType ConstructType(string qualifiedName)
+ => ConstructMember(qualifiedName)?.GetPythonType();
public IMember ConstructMember(string qualifiedName) {
// Determine module name, member chain and if this is an instance.
@@ -101,8 +105,9 @@ private IMember GetMemberFromThisModule(IReadOnlyList memberNames) {
return null;
}
- m = nextModel.Construct(this, declaringType);
+ m = nextModel.Create(this, declaringType, _gs);
Debug.Assert(m != null);
+
if (m is IGenericType gt && typeArgs.Count > 0) {
m = gt.CreateSpecificType(new ArgumentSet(typeArgs, null, null));
}
@@ -136,13 +141,13 @@ private IPythonModule GetModule(QualifiedNameParts parts) {
// from the stub and the main module was never loaded. This, for example,
// happens with io which has member with mmap type coming from mmap
// stub rather than the primary mmap module.
- var m = Module.Interpreter.ModuleResolution.GetImportedModule(parts.ModuleName);
- // Try stub-only case (ex _importlib_modulespec).
- m = m ?? Module.Interpreter.TypeshedResolution.GetImportedModule(parts.ModuleName);
+ var m = parts.IsStub
+ ? Module.Interpreter.TypeshedResolution.GetImportedModule(parts.ModuleName)
+ : Module.Interpreter.ModuleResolution.GetImportedModule(parts.ModuleName);
+
if (m != null) {
return parts.ObjectType == ObjectType.VariableModule ? new PythonVariableModule(m) : m;
}
-
return null;
}
}
@@ -175,7 +180,7 @@ private IMember GetMember(IMember root, IEnumerable memberNames) {
}
if (member == null) {
- Debug.Assert(member != null);
+ Debug.Assert(member != null || EnableMissingMemberAssertions == false);
break;
}
diff --git a/src/Caching/Impl/Properties/AssemblyInfo.cs b/src/Caching/Impl/Properties/AssemblyInfo.cs
index 9d32fae0c..d2256d3fe 100644
--- a/src/Caching/Impl/Properties/AssemblyInfo.cs
+++ b/src/Caching/Impl/Properties/AssemblyInfo.cs
@@ -16,3 +16,4 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("Microsoft.Python.Analysis.Caching.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
+[assembly: InternalsVisibleTo("Microsoft.Python.LanguageServer, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
diff --git a/src/Caching/Impl/PythonDbModule.cs b/src/Caching/Impl/PythonDbModule.cs
index addc97053..8da9634ed 100644
--- a/src/Caching/Impl/PythonDbModule.cs
+++ b/src/Caching/Impl/PythonDbModule.cs
@@ -28,17 +28,17 @@ internal sealed class PythonDbModule : SpecializedModule {
public PythonDbModule(ModuleModel model, string filePath, IServiceContainer services)
: base(model.Name, filePath, services) {
-
- var gs = new GlobalScope(model, this);
- GlobalScope = gs;
- gs.ReconstructVariables();
-
Documentation = model.Documentation;
-
_newLines = model.NewLines.Select(nl => new NewLineLocation(nl.EndIndex, nl.Kind)).ToArray();
_fileSize = model.FileSize;
}
+ public void Construct(ModuleModel model) {
+ var gs = new RestoredGlobalScope(model, this);
+ GlobalScope = gs;
+ gs.ReconstructVariables();
+ }
+
protected override string LoadContent() => string.Empty;
public override string Documentation { get; }
diff --git a/src/Caching/Impl/QualifiedNameParts.cs b/src/Caching/Impl/QualifiedNameParts.cs
index defe456fd..4d1d33539 100644
--- a/src/Caching/Impl/QualifiedNameParts.cs
+++ b/src/Caching/Impl/QualifiedNameParts.cs
@@ -29,6 +29,8 @@ internal struct QualifiedNameParts {
public ObjectType ObjectType;
/// Module name.
public string ModuleName;
+ /// Indicates if module is a stub.
+ public bool IsStub;
/// Module member names such as 'A', 'B', 'C' from module:A.B.C.
public IReadOnlyList MemberNames;
}
diff --git a/src/Caching/Impl/GlobalScope.cs b/src/Caching/Impl/RestoredGlobalScope.cs
similarity index 67%
rename from src/Caching/Impl/GlobalScope.cs
rename to src/Caching/Impl/RestoredGlobalScope.cs
index 47808dccc..43170d1f4 100644
--- a/src/Caching/Impl/GlobalScope.cs
+++ b/src/Caching/Impl/RestoredGlobalScope.cs
@@ -22,44 +22,45 @@
using Microsoft.Python.Parsing.Ast;
namespace Microsoft.Python.Analysis.Caching {
- internal sealed class GlobalScope : IGlobalScope {
+ internal sealed class RestoredGlobalScope : IRestoredGlobalScope {
private readonly VariableCollection _scopeVariables = new VariableCollection();
- private ModuleModel _model;
+ private ModuleModel _model; // Non-readonly b/c of DEBUG conditional.
+ private ModuleFactory _factory; // Non-readonly b/c of DEBUG conditional.
- public GlobalScope(ModuleModel model, IPythonModule module) {
- _model = model;
- Module = module;
+ public RestoredGlobalScope(ModuleModel model, IPythonModule module) {
+ _model = model ?? throw new ArgumentNullException(nameof(model));
+ Module = module ?? throw new ArgumentNullException(nameof(module));
Name = model.Name;
+ _factory = new ModuleFactory(_model, Module, this);
+ DeclareVariables();
}
public void ReconstructVariables() {
+ var models = _model.TypeVars.Concat(_model.NamedTuples).Concat(_model.Classes).Concat(_model.Functions);
+ foreach (var m in models.Concat(_model.Variables)) {
+ m.Populate(_factory, null, this);
+ }
+ // TODO: re-declare __doc__, __name__, etc.
+#if !DEBUG
+ _model = null;
+ _factory = null;
+#endif
+ }
+
+ private void DeclareVariables() {
// Member creation may be non-linear. Consider function A returning instance
// of a class or type info of a function which hasn't been created yet.
// Thus first create members so we can find then, then populate them with content.
- var mf = new ModuleFactory(_model, Module);
- foreach (var tvm in _model.TypeVars) {
- var t = tvm.Construct(mf, null);
- _scopeVariables.DeclareVariable(tvm.Name, t, VariableSource.Generic, mf.DefaultLocation);
- }
- foreach (var ntm in _model.NamedTuples) {
- var nt = ntm.Construct(mf, null);
- _scopeVariables.DeclareVariable(ntm.Name, nt, VariableSource.Declaration, mf.DefaultLocation);
- }
- foreach (var cm in _model.Classes) {
- var cls = cm.Construct(mf, null);
- _scopeVariables.DeclareVariable(cm.Name, cls, VariableSource.Declaration, mf.DefaultLocation);
- }
- foreach (var fm in _model.Functions) {
- var ft = fm.Construct(mf, null);
- _scopeVariables.DeclareVariable(fm.Name, ft, VariableSource.Declaration, mf.DefaultLocation);
+ var mf = new ModuleFactory(_model, Module, this);
+ var models = _model.TypeVars.Concat(_model.NamedTuples).Concat(_model.Classes).Concat(_model.Functions);
+
+ foreach (var m in models) {
+ _scopeVariables.DeclareVariable(m.Name, m.Create(mf, null, this), VariableSource.Generic, mf.DefaultLocation);
}
foreach (var vm in _model.Variables) {
- var v = (IVariable)vm.Construct(mf, null);
+ var v = (IVariable)vm.Create(mf, null, this);
_scopeVariables.DeclareVariable(vm.Name, v.Value, VariableSource.Declaration, mf.DefaultLocation);
}
-
- // TODO: re-declare __doc__, __name__, etc.
- _model = null;
}
#region IScope
diff --git a/src/Caching/Impl/TypeNames.cs b/src/Caching/Impl/TypeNames.cs
index c1e1e7f0e..2d12f482a 100644
--- a/src/Caching/Impl/TypeNames.cs
+++ b/src/Caching/Impl/TypeNames.cs
@@ -17,7 +17,6 @@
using System.Collections.Generic;
using System.Linq;
using Microsoft.Python.Analysis.Modules;
-using Microsoft.Python.Analysis.Specializations.Typing;
using Microsoft.Python.Analysis.Types;
using Microsoft.Python.Analysis.Values;
@@ -103,6 +102,7 @@ private static void GetModuleNameAndMembers(string qualifiedName, ref QualifiedN
default:
parts.ModuleName = typeName;
parts.MemberNames = Array.Empty();
+ DetermineModuleType(ref parts);
break;
}
return;
@@ -112,6 +112,15 @@ private static void GetModuleNameAndMembers(string qualifiedName, ref QualifiedN
parts.ModuleName = typeName.Substring(0, moduleSeparatorIndex);
var memberNamesOffset = parts.ModuleName.Length + 1;
parts.MemberNames = GetTypeNames(typeName.Substring(memberNamesOffset), '.');
+
+ DetermineModuleType(ref parts);
+ }
+
+ private static void DetermineModuleType(ref QualifiedNameParts parts) {
+ if (parts.ModuleName.EndsWith("(stub)")) {
+ parts.ModuleName = parts.ModuleName.Substring(0, parts.ModuleName.Length - 6);
+ parts.IsStub = true;
+ }
}
public static IReadOnlyList GetTypeNames(string qualifiedTypeName, char separator) {
diff --git a/src/Caching/Test/AnalysisCachingTestBase.cs b/src/Caching/Test/AnalysisCachingTestBase.cs
index 34927e303..4da2db6b3 100644
--- a/src/Caching/Test/AnalysisCachingTestBase.cs
+++ b/src/Caching/Test/AnalysisCachingTestBase.cs
@@ -16,13 +16,22 @@
using System.IO;
using System.Reflection;
using System.Text;
+using System.Threading.Tasks;
+using Microsoft.Python.Analysis.Analyzer;
+using Microsoft.Python.Analysis.Caching.Models;
+using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions;
using Microsoft.Python.Analysis.Tests;
+using Microsoft.Python.Analysis.Types;
using Newtonsoft.Json;
using TestUtilities;
namespace Microsoft.Python.Analysis.Caching.Tests {
public abstract class AnalysisCachingTestBase: AnalysisTestBase {
- protected string ToJson(object model) {
+ protected AnalysisCachingTestBase() {
+ ModuleFactory.EnableMissingMemberAssertions = true;
+ }
+
+ protected static string ToJson(object model) {
var sb = new StringBuilder();
using (var sw = new StringWriter(sb))
using (var jw = new JsonTextWriter(sw)) {
@@ -45,5 +54,48 @@ protected string GetBaselineFileName(string testName, string suffix = null)
=> Path.ChangeExtension(suffix == null
? Path.Combine(BaselineFilesFolder, testName)
: Path.Combine(BaselineFilesFolder, testName + suffix), "json");
+
+ internal PythonDbModule CreateDbModule(ModuleModel model, string modulePath) {
+ var dbModule = new PythonDbModule(model, modulePath, Services);
+ dbModule.Construct(model);
+ return dbModule;
+ }
+
+ internal async Task CompareBaselineAndRestoreAsync(ModuleModel model, IPythonModule m) {
+ //var json = ToJson(model);
+ //Baseline.CompareToFile(BaselineFileName, json);
+
+ // In real case dependency analysis will restore model dependencies.
+ // Here we don't go through the dependency analysis so we have to
+ // manually restore dependent modules.
+ foreach (var imp in model.Imports) {
+ foreach (var name in imp.ModuleNames) {
+ m.Interpreter.ModuleResolution.GetOrLoadModule(name);
+ }
+ }
+ foreach (var imp in model.FromImports) {
+ foreach (var name in imp.RootNames) {
+ m.Interpreter.ModuleResolution.GetOrLoadModule(name);
+ }
+ }
+
+ foreach (var imp in model.StubImports) {
+ foreach (var name in imp.ModuleNames) {
+ m.Interpreter.TypeshedResolution.GetOrLoadModule(name);
+ }
+ }
+ foreach (var imp in model.StubFromImports) {
+ foreach (var name in imp.RootNames) {
+ m.Interpreter.TypeshedResolution.GetOrLoadModule(name);
+ }
+ }
+
+ var analyzer = Services.GetService();
+ await analyzer.WaitForCompleteAnalysisAsync();
+
+ using (var dbModule = CreateDbModule(model, m.FilePath)) {
+ dbModule.Should().HaveSameMembersAs(m);
+ }
+ }
}
}
diff --git a/src/Caching/Test/ClassesTests.cs b/src/Caching/Test/ClassesTests.cs
index 3aa6c225c..b046213fb 100644
--- a/src/Caching/Test/ClassesTests.cs
+++ b/src/Caching/Test/ClassesTests.cs
@@ -101,7 +101,7 @@ def func():
//var json = ToJson(model);
//Baseline.CompareToFile(BaselineFileName, json);
- using (var dbModule = new PythonDbModule(model, analysis.Document.FilePath, Services)) {
+ using (var dbModule = CreateDbModule(model, analysis.Document.FilePath)) {
dbModule.Should().HaveSameMembersAs(analysis.Document);
}
}
@@ -128,7 +128,7 @@ def value(self):
//var json = ToJson(model);
//Baseline.CompareToFile(BaselineFileName, json);
- using (var dbModule = new PythonDbModule(model, analysis.Document.FilePath, Services)) {
+ using (var dbModule = CreateDbModule(model, analysis.Document.FilePath)) {
dbModule.Should().HaveSameMembersAs(analysis.Document);
}
}
diff --git a/src/Caching/Test/Files/ClassOwnDocumentation.json b/src/Caching/Test/Files/ClassOwnDocumentation.json
index f6a6c7d35..647537712 100644
--- a/src/Caching/Test/Files/ClassOwnDocumentation.json
+++ b/src/Caching/Test/Files/ClassOwnDocumentation.json
@@ -191,8 +191,12 @@
}
],
"FileSize": 115,
+ "Imports": [],
+ "FromImports": [],
+ "StubImports": [],
+ "StubFromImports": [],
"Id": -2131035837,
"Name": "module",
- "QualifiedName": null,
+ "QualifiedName": "module",
"IndexSpan": null
}
\ No newline at end of file
diff --git a/src/Caching/Test/Files/MemberLocations.json b/src/Caching/Test/Files/MemberLocations.json
index ec98ba59b..bc8123835 100644
--- a/src/Caching/Test/Files/MemberLocations.json
+++ b/src/Caching/Test/Files/MemberLocations.json
@@ -362,8 +362,12 @@
}
],
"FileSize": 288,
+ "Imports": [],
+ "FromImports": [],
+ "StubImports": [],
+ "StubFromImports": [],
"Id": -2131035837,
"Name": "module",
- "QualifiedName": null,
+ "QualifiedName": "module",
"IndexSpan": null
}
\ No newline at end of file
diff --git a/src/Caching/Test/Files/NestedClasses.json b/src/Caching/Test/Files/NestedClasses.json
index a4de61c16..efb7c122d 100644
--- a/src/Caching/Test/Files/NestedClasses.json
+++ b/src/Caching/Test/Files/NestedClasses.json
@@ -413,8 +413,12 @@
}
],
"FileSize": 353,
+ "Imports": [],
+ "FromImports": [],
+ "StubImports": [],
+ "StubFromImports": [],
"Id": -2131035837,
"Name": "module",
- "QualifiedName": null,
+ "QualifiedName": "module",
"IndexSpan": null
}
\ No newline at end of file
diff --git a/src/Caching/Test/Files/SmokeTest.json b/src/Caching/Test/Files/SmokeTest.json
index 5fc51e763..8774ad298 100644
--- a/src/Caching/Test/Files/SmokeTest.json
+++ b/src/Caching/Test/Files/SmokeTest.json
@@ -310,8 +310,12 @@
}
],
"FileSize": 243,
+ "Imports": [],
+ "FromImports": [],
+ "StubImports": [],
+ "StubFromImports": [],
"Id": -2131035837,
"Name": "module",
- "QualifiedName": null,
+ "QualifiedName": "module",
"IndexSpan": null
}
\ No newline at end of file
diff --git a/src/Caching/Test/Files/VersionHandling2.json b/src/Caching/Test/Files/VersionHandling2.json
index 02e6c7f13..ee4d02e74 100644
--- a/src/Caching/Test/Files/VersionHandling2.json
+++ b/src/Caching/Test/Files/VersionHandling2.json
@@ -137,8 +137,12 @@
}
],
"FileSize": 91,
+ "Imports": [],
+ "FromImports": [],
+ "StubImports": [],
+ "StubFromImports": [],
"Id": -2131035837,
"Name": "module",
- "QualifiedName": null,
+ "QualifiedName": "module",
"IndexSpan": null
}
\ No newline at end of file
diff --git a/src/Caching/Test/Files/VersionHandling3.json b/src/Caching/Test/Files/VersionHandling3.json
index 0ee5f389c..1d92c7ff2 100644
--- a/src/Caching/Test/Files/VersionHandling3.json
+++ b/src/Caching/Test/Files/VersionHandling3.json
@@ -149,8 +149,12 @@
}
],
"FileSize": 91,
+ "Imports": [],
+ "FromImports": [],
+ "StubImports": [],
+ "StubFromImports": [],
"Id": -2131035837,
"Name": "module",
- "QualifiedName": null,
+ "QualifiedName": "module",
"IndexSpan": null
}
\ No newline at end of file
diff --git a/src/Caching/Test/LibraryModulesTests.cs b/src/Caching/Test/LibraryModulesTests.cs
index 9f9b710ff..ec4ad2e7d 100644
--- a/src/Caching/Test/LibraryModulesTests.cs
+++ b/src/Caching/Test/LibraryModulesTests.cs
@@ -20,7 +20,6 @@
using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions;
using Microsoft.Python.Analysis.Modules;
using Microsoft.Python.Analysis.Types;
-using Microsoft.Python.Parsing;
using Microsoft.Python.Parsing.Tests;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using TestUtilities;
@@ -181,7 +180,7 @@ public async Task Builtins() {
[TestMethod, Priority(0)]
public Task Pickle() => TestModule("pickle");
-
+
[TestMethod, Priority(0)]
public Task Pipes() => TestModule("pipes");
@@ -295,7 +294,7 @@ import requests
// Verify this looks like a version.
new Version(u.Substring(open + 1, u.IndexOf(')') - open - 1));
- CompareBaselineAndRestore(model, rq);
+ await CompareBaselineAndRestoreAsync(model, rq);
}
private async Task TestModule(string name) {
@@ -303,16 +302,7 @@ private async Task TestModule(string name) {
var m = analysis.Document.Interpreter.ModuleResolution.GetImportedModule(name);
var model = ModuleModel.FromAnalysis(m.Analysis, Services, AnalysisCachingLevel.Library);
- CompareBaselineAndRestore(model, m);
- }
-
- private void CompareBaselineAndRestore(ModuleModel model, IPythonModule m) {
- //var json = ToJson(model);
- //Baseline.CompareToFile(BaselineFileName, json);
-
- using (var dbModule = new PythonDbModule(model, m.FilePath, Services)) {
- dbModule.Should().HaveSameMembersAs(m);
- }
+ await CompareBaselineAndRestoreAsync(model, m);
}
}
}
diff --git a/src/Caching/Test/ReferencesTests.cs b/src/Caching/Test/ReferencesTests.cs
index 867cd95c3..76b9ac803 100644
--- a/src/Caching/Test/ReferencesTests.cs
+++ b/src/Caching/Test/ReferencesTests.cs
@@ -15,13 +15,12 @@
using System.Threading.Tasks;
using FluentAssertions;
-using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions;
using Microsoft.Python.Analysis.Caching.Models;
+using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions;
+using Microsoft.Python.Analysis.Modules;
using Microsoft.Python.Analysis.Types;
using Microsoft.VisualStudio.TestTools.UnitTesting;
-using NSubstitute;
using TestUtilities;
-using Microsoft.Python.Analysis.Modules;
namespace Microsoft.Python.Analysis.Caching.Tests {
[TestClass]
@@ -67,6 +66,8 @@ def methodB2(self):
Baseline.CompareToFile(BaselineFileName, json);
using (var dbModule = new PythonDbModule(model, analysis.Document.FilePath, Services)) {
+ dbModule.Construct(model);
+
var sum = dbModule.GetMember("sum") as IPythonFunctionType;
sum.Should().NotBeNull();
sum.Definition.Span.Should().Be(4, 5, 4, 8);
@@ -103,23 +104,16 @@ import logging
var logging = analysis.Document.Interpreter.ModuleResolution.GetImportedModule("logging");
var model = ModuleModel.FromAnalysis(logging.Analysis, Services, AnalysisCachingLevel.Library);
- var dbModule = new PythonDbModule(model, logging.FilePath, Services);
- analysis.Document.Interpreter.ModuleResolution.SpecializeModule("logging", x => dbModule, replaceExisting: true);
-
- var moduleName = $"{analysis.Document.Name}_db.py";
- var modulePath = TestData.GetTestSpecificPath(moduleName);
- analysis = await GetAnalysisAsync(code, Services, moduleName, modulePath);
-
- var v = analysis.Should().HaveVariable("logging").Which;
- var vm = v.Value.Should().BeOfType().Which;
- var m = vm.Module.Should().BeOfType().Which;
+ await CompareBaselineAndRestoreAsync(model, logging);
- var critical = m.GetMember("critical") as IPythonFunctionType;
- critical.Should().NotBeNull();
+ using (var m = CreateDbModule(model, logging.FilePath)) {
+ var critical = m.GetMember("critical") as IPythonFunctionType;
+ critical.Should().NotBeNull();
- var span = critical.Definition.Span;
- span.Start.Line.Should().BeGreaterThan(1000);
- (span.End.Column - span.Start.Column).Should().Be("critical".Length);
+ var span = critical.Definition.Span;
+ span.Start.Line.Should().BeGreaterThan(1000);
+ (span.End.Column - span.Start.Column).Should().Be("critical".Length);
+ }
}
}
}
diff --git a/src/Core/Impl/OS/ProcessServices.cs b/src/Core/Impl/OS/ProcessServices.cs
index 1f74efd40..e9a7c5b72 100644
--- a/src/Core/Impl/OS/ProcessServices.cs
+++ b/src/Core/Impl/OS/ProcessServices.cs
@@ -39,20 +39,20 @@ public IProcess Start(string path) {
public async Task ExecuteAndCaptureOutputAsync(ProcessStartInfo startInfo, CancellationToken cancellationToken = default) {
var output = string.Empty;
- var process = Start(startInfo);
+ using (var process = Start(startInfo)) {
- if (startInfo.RedirectStandardError && process is PlatformProcess p) {
- p.Process.ErrorDataReceived += (s, e) => { };
- p.Process.BeginErrorReadLine();
- }
+ if (startInfo.RedirectStandardError && process is PlatformProcess p) {
+ p.Process.ErrorDataReceived += (s, e) => { };
+ p.Process.BeginErrorReadLine();
+ }
- try {
- output = await process.StandardOutput.ReadToEndAsync();
- await process.WaitForExitAsync(30000, cancellationToken);
- } catch (IOException) {
- } catch (OperationCanceledException) { }
+ try {
+ output = await process.StandardOutput.ReadToEndAsync();
+ await process.WaitForExitAsync(30000, cancellationToken);
+ } catch (IOException) { } catch (OperationCanceledException) { }
- return output;
+ return output;
+ }
}
}
}
diff --git a/src/Core/Test/PriorityProducerConsumerTest.cs b/src/Core/Test/PriorityProducerConsumerTest.cs
index 62c755ca6..a0a5132ac 100644
--- a/src/Core/Test/PriorityProducerConsumerTest.cs
+++ b/src/Core/Test/PriorityProducerConsumerTest.cs
@@ -24,95 +24,102 @@ namespace Microsoft.Python.Core.Tests {
public class PriorityProducerConsumerTest {
[TestMethod, Priority(0)]
public void PriorityProducerConsumer_NoPending() {
- var ppc = new PriorityProducerConsumer();
- ppc.Produce(5);
- var consumerTask = ppc.ConsumeAsync();
- Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status);
- Assert.AreEqual(5, consumerTask.Result);
+ using (var ppc = new PriorityProducerConsumer()) {
+ ppc.Produce(5);
+ var consumerTask = ppc.ConsumeAsync();
+ Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status);
+ Assert.AreEqual(5, consumerTask.Result);
+ }
}
[TestMethod, Priority(0)]
public void PriorityProducerConsumer_NoPending_Priority1() {
- var ppc = new PriorityProducerConsumer(2);
- ppc.Produce(5);
- ppc.Produce(6, 1);
- var consumerTask = ppc.ConsumeAsync();
- Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status);
- Assert.AreEqual(5, consumerTask.Result);
+ using (var ppc = new PriorityProducerConsumer(2)) {
+ ppc.Produce(5);
+ ppc.Produce(6, 1);
+ var consumerTask = ppc.ConsumeAsync();
+ Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status);
+ Assert.AreEqual(5, consumerTask.Result);
+ }
}
[TestMethod, Priority(0)]
public void PriorityProducerConsumer_NoPending_Priority2() {
- var ppc = new PriorityProducerConsumer(2);
- ppc.Produce(6, 1);
- ppc.Produce(5);
- var consumerTask = ppc.ConsumeAsync();
- Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status);
- Assert.AreEqual(5, consumerTask.Result);
+ using (var ppc = new PriorityProducerConsumer(2)) {
+ ppc.Produce(6, 1);
+ ppc.Produce(5);
+ var consumerTask = ppc.ConsumeAsync();
+ Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status);
+ Assert.AreEqual(5, consumerTask.Result);
+ }
}
[TestMethod, Priority(0)]
public void PriorityProducerConsumer_NoPending_Duplicates1() {
- var ppc = new PriorityProducerConsumer(3, true);
- ppc.Produce(5, 2);
- ppc.Produce(6, 1);
- ppc.Produce(5);
- var consumerTask1 = ppc.ConsumeAsync();
- var consumerTask2 = ppc.ConsumeAsync();
- var consumerTask3 = ppc.ConsumeAsync();
-
- Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status);
- Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status);
- Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status);
- Assert.AreEqual(5, consumerTask1.Result);
- Assert.AreEqual(6, consumerTask2.Result);
+ using (var ppc = new PriorityProducerConsumer(3, true)) {
+ ppc.Produce(5, 2);
+ ppc.Produce(6, 1);
+ ppc.Produce(5);
+ var consumerTask1 = ppc.ConsumeAsync();
+ var consumerTask2 = ppc.ConsumeAsync();
+ var consumerTask3 = ppc.ConsumeAsync();
+
+ Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status);
+ Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status);
+ Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status);
+ Assert.AreEqual(5, consumerTask1.Result);
+ Assert.AreEqual(6, consumerTask2.Result);
+ }
}
[TestMethod, Priority(0)]
public void PriorityProducerConsumer_NoPending_Duplicates2() {
- var ppc = new PriorityProducerConsumer(3, true);
- ppc.Produce(5);
- ppc.Produce(6, 1);
- ppc.Produce(5, 2);
- var consumerTask1 = ppc.ConsumeAsync();
- var consumerTask2 = ppc.ConsumeAsync();
- var consumerTask3 = ppc.ConsumeAsync();
-
- Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status);
- Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status);
- Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status);
- Assert.AreEqual(5, consumerTask1.Result);
- Assert.AreEqual(6, consumerTask2.Result);
+ using (var ppc = new PriorityProducerConsumer(3, true)) {
+ ppc.Produce(5);
+ ppc.Produce(6, 1);
+ ppc.Produce(5, 2);
+ var consumerTask1 = ppc.ConsumeAsync();
+ var consumerTask2 = ppc.ConsumeAsync();
+ var consumerTask3 = ppc.ConsumeAsync();
+
+ Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status);
+ Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status);
+ Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status);
+ Assert.AreEqual(5, consumerTask1.Result);
+ Assert.AreEqual(6, consumerTask2.Result);
+ }
}
[TestMethod, Priority(0)]
public void PriorityProducerConsumer_NoPending_Duplicates3() {
- var ppc = new PriorityProducerConsumer(3, true);
- ppc.Produce(5, 1);
- ppc.Produce(6, 1);
- ppc.Produce(5, 1);
- var consumerTask1 = ppc.ConsumeAsync();
- var consumerTask2 = ppc.ConsumeAsync();
- var consumerTask3 = ppc.ConsumeAsync();
-
- Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status);
- Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status);
- Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status);
- Assert.AreEqual(6, consumerTask1.Result);
- Assert.AreEqual(5, consumerTask2.Result);
+ using (var ppc = new PriorityProducerConsumer(3, true)) {
+ ppc.Produce(5, 1);
+ ppc.Produce(6, 1);
+ ppc.Produce(5, 1);
+ var consumerTask1 = ppc.ConsumeAsync();
+ var consumerTask2 = ppc.ConsumeAsync();
+ var consumerTask3 = ppc.ConsumeAsync();
+
+ Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status);
+ Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status);
+ Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status);
+ Assert.AreEqual(6, consumerTask1.Result);
+ Assert.AreEqual(5, consumerTask2.Result);
+ }
}
[TestMethod, Priority(0)]
public async Task PriorityProducerConsumer_Pending() {
- var ppc = new PriorityProducerConsumer();
- var consumerTask = ppc.ConsumeAsync();
- Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status);
+ using (var ppc = new PriorityProducerConsumer()) {
+ var consumerTask = ppc.ConsumeAsync();
+ Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status);
- ppc.Produce(5);
- await consumerTask;
+ ppc.Produce(5);
+ await consumerTask;
- Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status);
- Assert.AreEqual(5, consumerTask.Result);
+ Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status);
+ Assert.AreEqual(5, consumerTask.Result);
+ }
}
[TestMethod, Priority(0)]
@@ -131,69 +138,72 @@ public async Task PriorityProducerConsumer_Pending_Dispose() {
[TestMethod, Priority(0)]
public async Task PriorityProducerConsumer_Pending_Priority1() {
- var ppc = new PriorityProducerConsumer(2);
- var consumerTask = ppc.ConsumeAsync();
- Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status);
+ using (var ppc = new PriorityProducerConsumer(2)) {
+ var consumerTask = ppc.ConsumeAsync();
+ Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status);
- ppc.Produce(5);
- ppc.Produce(6, 1);
- await consumerTask;
+ ppc.Produce(5);
+ ppc.Produce(6, 1);
+ await consumerTask;
- Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status);
- Assert.AreEqual(5, consumerTask.Result);
+ Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status);
+ Assert.AreEqual(5, consumerTask.Result);
+ }
}
[TestMethod, Priority(0)]
public async Task PriorityProducerConsumer_Pending_Priority2() {
- var ppc = new PriorityProducerConsumer(2);
- var consumerTask1 = ppc.ConsumeAsync();
- var consumerTask2 = ppc.ConsumeAsync();
- Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask1.Status);
- Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask2.Status);
+ using (var ppc = new PriorityProducerConsumer(2)) {
+ var consumerTask1 = ppc.ConsumeAsync();
+ var consumerTask2 = ppc.ConsumeAsync();
+ Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask1.Status);
+ Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask2.Status);
- ppc.Produce(6, 1);
- await consumerTask1;
+ ppc.Produce(6, 1);
+ await consumerTask1;
- Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status);
- Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask2.Status);
- Assert.AreEqual(6, consumerTask1.Result);
+ Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status);
+ Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask2.Status);
+ Assert.AreEqual(6, consumerTask1.Result);
- ppc.Produce(5);
- await consumerTask2;
+ ppc.Produce(5);
+ await consumerTask2;
- Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status);
- Assert.AreEqual(5, consumerTask2.Result);
+ Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status);
+ Assert.AreEqual(5, consumerTask2.Result);
+ }
}
[TestMethod, Priority(0)]
public async Task PriorityProducerConsumer_Pending_Priority3() {
- var ppc = new PriorityProducerConsumer(2);
- var values = new int[3];
- var tcsConsumer = new TaskCompletionSource();
- var tcsProducer = new TaskCompletionSource();
- var consumerTask = Task.Run(async () => {
- for (var i = 0; i < 3; i++) {
- var task = ppc.ConsumeAsync();
- tcsConsumer.TrySetResult(true);
- values[i] = await task;
- await tcsProducer.Task;
- }
- });
-
- Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status);
-
- await tcsConsumer.Task;
- ppc.Produce(5, 1);
- ppc.Produce(6, 1);
- ppc.Produce(7);
- tcsProducer.SetResult(false);
-
- await consumerTask;
-
- Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status);
- Assert.AreEqual(5, values[0]);
- Assert.AreEqual(7, values[1]);
- Assert.AreEqual(6, values[2]);
+ using (var ppc = new PriorityProducerConsumer(2)) {
+ var values = new int[3];
+ var tcsConsumer = new TaskCompletionSource();
+ var tcsProducer = new TaskCompletionSource();
+ var consumerTask = Task.Run(async () => {
+ for (var i = 0; i < 3; i++) {
+ var task = ppc.ConsumeAsync();
+ tcsConsumer.TrySetResult(true);
+ values[i] = await task;
+ await tcsProducer.Task;
+ }
+ });
+
+ Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status);
+
+ await tcsConsumer.Task;
+ ppc.Produce(5, 1);
+ ppc.Produce(6, 1);
+ ppc.Produce(7);
+ tcsProducer.SetResult(false);
+
+ await consumerTask;
+
+ Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status);
+ Assert.AreEqual(5, values[0]);
+ Assert.AreEqual(7, values[1]);
+ Assert.AreEqual(6, values[2]);
+ }
}
}
}
diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs
index d71ae58e8..00071495d 100644
--- a/src/LanguageServer/Test/IndexManagerTests.cs
+++ b/src/LanguageServer/Test/IndexManagerTests.cs
@@ -52,91 +52,96 @@ public void Cleanup() {
[TestMethod, Priority(0)]
public async Task AddsRootDirectoryAsync() {
- var context = new IndexTestContext(this);
- context.FileWithXVarInRootDir();
- context.AddFileToRoot($"{_rootPath}\foo.py", MakeStream("y = 1"));
+ using (var context = new IndexTestContext(this)) {
+ context.FileWithXVarInRootDir();
+ context.AddFileToRoot($"{_rootPath}\foo.py", MakeStream("y = 1"));
- var indexManager = context.GetDefaultIndexManager();
- await indexManager.IndexWorkspace();
+ var indexManager = context.GetDefaultIndexManager();
+ await indexManager.IndexWorkspace();
- var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount);
- symbols.Should().HaveCount(2);
+ var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount);
+ symbols.Should().HaveCount(2);
+ }
}
[TestMethod, Priority(0)]
public async Task IgnoresNonPythonFiles() {
- var context = new IndexTestContext(this);
-
- var nonPythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}/bla.txt");
- context.AddFileInfoToRootTestFS(nonPythonTestFileInfo);
+ using (var context = new IndexTestContext(this)) {
+ var nonPythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}/bla.txt");
+ context.AddFileInfoToRootTestFS(nonPythonTestFileInfo);
- IIndexManager indexManager = context.GetDefaultIndexManager();
- await indexManager.IndexWorkspace();
+ var indexManager = context.GetDefaultIndexManager();
+ await indexManager.IndexWorkspace();
- context.FileSystem.DidNotReceive().FileExists(nonPythonTestFileInfo.FullName);
+ context.FileSystem.DidNotReceive().FileExists(nonPythonTestFileInfo.FullName);
+ }
}
[TestMethod, Priority(0)]
public async Task CanOpenFiles() {
- string nonRootPath = "C:/nonRoot";
- var context = new IndexTestContext(this);
- var pythonTestFileInfo = MakeFileInfoProxy($"{nonRootPath}/bla.py");
- IDocument doc = DocumentWithAst("x = 1");
+ var nonRootPath = "C:/nonRoot";
+ using (var context = new IndexTestContext(this)) {
+ var pythonTestFileInfo = MakeFileInfoProxy($"{nonRootPath}/bla.py");
+ var doc = DocumentWithAst("x = 1");
- IIndexManager indexManager = context.GetDefaultIndexManager();
- indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc);
+ var indexManager = context.GetDefaultIndexManager();
+ indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc);
- var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount);
- SymbolsShouldBeOnlyX(symbols);
+ var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount);
+ SymbolsShouldBeOnlyX(symbols);
+ }
}
[TestMethod, Priority(0)]
public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() {
- var context = new IndexTestContext(this);
- var pythonTestFilePath = context.FileWithXVarInRootDir();
+ using (var context = new IndexTestContext(this)) {
+ var pythonTestFilePath = context.FileWithXVarInRootDir();
- var indexManager = context.GetDefaultIndexManager();
- await indexManager.IndexWorkspace();
+ var indexManager = context.GetDefaultIndexManager();
+ await indexManager.IndexWorkspace();
- indexManager.ReIndexFile(pythonTestFilePath, DocumentWithAst("y = 1"));
+ indexManager.ReIndexFile(pythonTestFilePath, DocumentWithAst("y = 1"));
- var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount);
- symbols.Should().HaveCount(1);
- symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable);
- symbols.First().Name.Should().BeEquivalentTo("y");
+ var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount);
+ symbols.Should().HaveCount(1);
+ symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable);
+ symbols.First().Name.Should().BeEquivalentTo("y");
+ }
}
[TestMethod, Priority(0)]
public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() {
- var context = new IndexTestContext(this);
- var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py");
- context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false);
+ using (var context = new IndexTestContext(this)) {
+ var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py");
+ context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false);
- var indexManager = context.GetDefaultIndexManager();
- indexManager.ProcessNewFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1"));
- indexManager.ProcessClosedFile(pythonTestFileInfo.FullName);
+ var indexManager = context.GetDefaultIndexManager();
+ indexManager.ProcessNewFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1"));
+ indexManager.ProcessClosedFile(pythonTestFileInfo.FullName);
- await SymbolIndexShouldBeEmpty(indexManager);
+ await SymbolIndexShouldBeEmpty(indexManager);
+ }
}
[TestMethod, Priority(0)]
public async Task CloseWorkspaceFilesReUpdatesIndexAsync() {
- var context = new IndexTestContext(this);
- var pythonTestFilePath = context.FileWithXVarInRootDir();
- context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true);
-
- var indexManager = context.GetDefaultIndexManager();
- await indexManager.IndexWorkspace();
-
- indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("r = 1"));
- // It Needs to remake the stream for the file, previous one is closed
- context.FileSystem.FileExists(pythonTestFilePath).Returns(true);
- context.SetFileOpen(pythonTestFilePath, MakeStream("x = 1"));
- context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true);
- indexManager.ProcessClosedFile(pythonTestFilePath);
-
- var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount);
- SymbolsShouldBeOnlyX(symbols);
+ using (var context = new IndexTestContext(this)) {
+ var pythonTestFilePath = context.FileWithXVarInRootDir();
+ context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true);
+
+ var indexManager = context.GetDefaultIndexManager();
+ await indexManager.IndexWorkspace();
+
+ indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("r = 1"));
+ // It Needs to remake the stream for the file, previous one is closed
+ context.FileSystem.FileExists(pythonTestFilePath).Returns(true);
+ context.SetFileOpen(pythonTestFilePath, MakeStream("x = 1"));
+ context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true);
+ indexManager.ProcessClosedFile(pythonTestFilePath);
+
+ var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount);
+ SymbolsShouldBeOnlyX(symbols);
+ }
}
[TestMethod, Priority(0)]
@@ -144,92 +149,97 @@ public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() {
// If events get to index manager in the order: [open, close, update]
// it should not reindex file
- var context = new IndexTestContext(this);
- var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py");
+ using (var context = new IndexTestContext(this)) {
+ var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py");
- var indexManager = context.GetDefaultIndexManager();
- indexManager.ProcessNewFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1"));
- indexManager.ProcessClosedFile(pythonTestFileInfo.FullName);
+ var indexManager = context.GetDefaultIndexManager();
+ indexManager.ProcessNewFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1"));
+ indexManager.ProcessClosedFile(pythonTestFileInfo.FullName);
- context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false);
- indexManager.ReIndexFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1"));
+ context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false);
+ indexManager.ReIndexFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1"));
- await SymbolIndexShouldBeEmpty(indexManager);
+ await SymbolIndexShouldBeEmpty(indexManager);
+ }
}
[TestMethod, Priority(0)]
public async Task WorkspaceSymbolsAddsRootDirectory() {
- var context = new IndexTestContext(this);
+ using (var context = new IndexTestContext(this)) {
+ var pythonTestFilePath = context.FileWithXVarInRootDir();
- var pythonTestFilePath = context.FileWithXVarInRootDir();
+ var indexManager = context.GetDefaultIndexManager();
+ await indexManager.IndexWorkspace();
- var indexManager = context.GetDefaultIndexManager();
- await indexManager.IndexWorkspace();
-
- var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount);
- SymbolsShouldBeOnlyX(symbols);
+ var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount);
+ SymbolsShouldBeOnlyX(symbols);
+ }
}
[TestMethod, Priority(0)]
public async Task WorkspaceSymbolsLimited() {
- var context = new IndexTestContext(this);
+ using (var context = new IndexTestContext(this)) {
+ for (var fileNumber = 0; fileNumber < 10; fileNumber++) {
+ context.AddFileToRoot($"{_rootPath}\bla{fileNumber}.py", MakeStream($"x{fileNumber} = 1"));
+ }
- for (int fileNumber = 0; fileNumber < 10; fileNumber++) {
- context.AddFileToRoot($"{_rootPath}\bla{fileNumber}.py", MakeStream($"x{fileNumber} = 1"));
- }
- var indexManager = context.GetDefaultIndexManager();
- await indexManager.IndexWorkspace();
+ var indexManager = context.GetDefaultIndexManager();
+ await indexManager.IndexWorkspace();
- const int amountOfSymbols = 3;
+ const int amountOfSymbols = 3;
- var symbols = await indexManager.WorkspaceSymbolsAsync("", amountOfSymbols);
- symbols.Should().HaveCount(amountOfSymbols);
+ var symbols = await indexManager.WorkspaceSymbolsAsync("", amountOfSymbols);
+ symbols.Should().HaveCount(amountOfSymbols);
+ }
}
[TestMethod, Priority(0)]
public async Task HierarchicalDocumentSymbolsAsync() {
- var context = new IndexTestContext(this);
- var pythonTestFilePath = context.FileWithXVarInRootDir();
+ using (var context = new IndexTestContext(this)) {
+ var pythonTestFilePath = context.FileWithXVarInRootDir();
- var indexManager = context.GetDefaultIndexManager();
- await indexManager.IndexWorkspace();
+ var indexManager = context.GetDefaultIndexManager();
+ await indexManager.IndexWorkspace();
- var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath);
- SymbolsShouldBeOnlyX(symbols);
+ var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath);
+ SymbolsShouldBeOnlyX(symbols);
+ }
}
[TestMethod, Priority(0)]
public async Task LatestVersionASTVersionIsIndexed() {
- var context = new IndexTestContext(this);
- var pythonTestFilePath = context.FileWithXVarInRootDir();
-
- var indexManager = context.GetDefaultIndexManager();
- indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("y = 1"));
- indexManager.ProcessClosedFile(pythonTestFilePath);
- indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("z = 1"));
-
- var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath);
- symbols.Should().HaveCount(1);
- symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable);
- symbols.First().Name.Should().BeEquivalentTo("z");
+ using (var context = new IndexTestContext(this)) {
+ var pythonTestFilePath = context.FileWithXVarInRootDir();
+
+ var indexManager = context.GetDefaultIndexManager();
+ indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("y = 1"));
+ indexManager.ProcessClosedFile(pythonTestFilePath);
+ indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("z = 1"));
+
+ var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath);
+ symbols.Should().HaveCount(1);
+ symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable);
+ symbols.First().Name.Should().BeEquivalentTo("z");
+ }
}
[TestMethod, Priority(0)]
public async Task AddFilesToPendingChanges() {
- var context = new IndexTestContext(this);
- var f1 = context.AddFileToRoot($"{_rootPath}/fileA.py", MakeStream(""));
- var f2 = context.AddFileToRoot($"{_rootPath}/fileB.py", MakeStream(""));
+ using (var context = new IndexTestContext(this)) {
+ var f1 = context.AddFileToRoot($"{_rootPath}/fileA.py", MakeStream(""));
+ var f2 = context.AddFileToRoot($"{_rootPath}/fileB.py", MakeStream(""));
- var indexManager = context.GetDefaultIndexManager();
- await indexManager.IndexWorkspace();
+ var indexManager = context.GetDefaultIndexManager();
+ await indexManager.IndexWorkspace();
- indexManager.AddPendingDoc(DocumentWithAst("y = 1", f1));
- indexManager.AddPendingDoc(DocumentWithAst("x = 1", f2));
+ indexManager.AddPendingDoc(DocumentWithAst("y = 1", f1));
+ indexManager.AddPendingDoc(DocumentWithAst("x = 1", f2));
- context.SetIdleEvent(Raise.Event());
+ context.SetIdleEvent(Raise.Event());
- var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount);
- symbols.Should().HaveCount(2);
+ var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount);
+ symbols.Should().HaveCount(2);
+ }
}
private static void SymbolsShouldBeOnlyX(IEnumerable symbols) {
@@ -249,8 +259,8 @@ private class IndexTestContext : IDisposable {
private readonly List _rootFileList = new List();
private readonly IIdleTimeService _idleTimeService = Substitute.For();
private readonly PythonLanguageVersion _pythonLanguageVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion();
+ private readonly IndexManagerTests _tests;
private IIndexManager _indexM;
- private IndexManagerTests _tests;
public IndexTestContext(IndexManagerTests tests) {
_tests = tests;
@@ -271,9 +281,7 @@ public void AddFileInfoToRootTestFS(FileInfoProxy fileInfo) {
FileSystem.FileExists(fileInfo.FullName).Returns(true);
}
- public string FileWithXVarInRootDir() {
- return AddFileToRoot($"{_rootPath}\bla.py", _tests.MakeStream("x = 1"));
- }
+ public string FileWithXVarInRootDir() => AddFileToRoot($"{_rootPath}\bla.py", _tests.MakeStream("x = 1"));
public IIndexManager GetDefaultIndexManager() {
_indexM = new IndexManager(FileSystem, _pythonLanguageVersion,
@@ -301,19 +309,16 @@ public void SetIdleEvent(EventHandler handler) {
private void SetupRootDir() {
var directoryInfo = Substitute.For();
directoryInfo.Match("", new string[] { }, new string[] { }).ReturnsForAnyArgs(callInfo => {
- string path = callInfo.ArgAt(0);
+ var path = callInfo.ArgAt(0);
return _rootFileList
- .Where(fsInfo => PathEqualityComparer.Instance.Equals(fsInfo.FullName, path))
- .Count() > 0;
+ .Count(fsInfo => PathEqualityComparer.Instance.Equals(fsInfo.FullName, path)) > 0;
});
// Doesn't work without 'forAnyArgs'
directoryInfo.EnumerateFileSystemInfos(new string[] { }, new string[] { }).ReturnsForAnyArgs(_rootFileList);
FileSystem.GetDirectoryInfo(_rootPath).Returns(directoryInfo);
}
- public void Dispose() {
- _indexM?.Dispose();
- }
+ public void Dispose() => _indexM?.Dispose();
public void SetFileOpen(string pythonTestFilePath, Func