diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index 19c0b0cdb..0a637412c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -16,7 +16,6 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.Reflection; using Microsoft.Python.Analysis.Analyzer.Symbols; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Modules; diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index a5cd285a7..0bc0fbf82 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -197,43 +197,73 @@ private void AnalyzeDocument(AnalysisModuleKey key, PythonAnalyzerEntry entry, I _analysisCompleteEvent.Reset(); _log?.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name}({entry.Module.ModuleType}) queued"); - var snapshot = _dependencyResolver.NotifyChanges(key, entry, dependencies); - + var graphVersion = _dependencyResolver.ChangeValue(key, entry, dependencies); + lock (_syncObj) { - if (_version > snapshot.Version) { + if (_version > graphVersion) { return; } - _version = snapshot.Version; + _version = graphVersion; + _currentSession?.Cancel(); } - if (snapshot.MissingKeys.Count > 0) { - LoadMissingDocuments(entry.Module.Interpreter, snapshot.MissingKeys); - } + UpdateDependentEntriesDepth(entry, dependencies, graphVersion); - if (TryCreateSession(snapshot, entry, cancellationToken, out var session)) { + if (TryCreateSession(graphVersion, entry, cancellationToken, out var session)) { session.Start(true); } } - private bool TryCreateSession(DependencyGraphSnapshot snapshot, PythonAnalyzerEntry entry, CancellationToken cancellationToken, out PythonAnalyzerSession session) { + private void UpdateDependentEntriesDepth(PythonAnalyzerEntry entry, ImmutableArray dependentKeys, int graphVersion) { + if (dependentKeys.Count == 0) { + return; + } + + var dependentEntries = new List(); + lock (_syncObj) { + if (_version > graphVersion) { + return; + } + + foreach (var key in dependentKeys) { + if (_analysisEntries.TryGetValue(key, out var value)) { + dependentEntries.Add(value); + } + } + } + + if (dependentEntries.Count == 0) { + return; + } + + var depth = entry.Depth; + foreach (var dependentEntry in dependentEntries) { + dependentEntry.SetDepth(graphVersion, depth); + } + } + + private bool TryCreateSession(int graphVersion, PythonAnalyzerEntry entry, CancellationToken cancellationToken, out PythonAnalyzerSession session) { var analyzeUserModuleOutOfOrder = false; lock (_syncObj) { if (_currentSession != null) { - if (_currentSession.Version > snapshot.Version || _nextSession != null && _nextSession.Version > snapshot.Version) { + if (_currentSession.Version > graphVersion || _nextSession != null && _nextSession.Version > graphVersion) { session = null; return false; } analyzeUserModuleOutOfOrder = !_currentSession.IsCompleted && entry.IsUserModule && _currentSession.AffectedEntriesCount >= _maxTaskRunning; - if (_version > snapshot.Version && analyzeUserModuleOutOfOrder) { + if (_version > graphVersion && analyzeUserModuleOutOfOrder) { session = CreateSession(null, entry, cancellationToken); return true; } } } - if (!_dependencyResolver.TryCreateWalker(snapshot.Version, out var walker)) { + var snapshot = _dependencyResolver.CurrentGraphSnapshot; + LoadMissingDocuments(entry.Module.Interpreter, snapshot.MissingKeys); + + if (!_dependencyResolver.TryCreateWalker(snapshot, out var walker)) { session = null; return false; } @@ -254,7 +284,6 @@ private bool TryCreateSession(DependencyGraphSnapshot new PythonAnalyzerSession(_services, _progress, _analysisCompleteEvent, _startNextSession, _disposeToken.CancellationToken, cancellationToken, walker, _version, entry); private void LoadMissingDocuments(IPythonInterpreter interpreter, ImmutableArray missingKeys) { - foreach (var (moduleName, _, isTypeshed) in missingKeys) { + if (missingKeys.Count == 0) { + return; + } + + var foundKeys = ImmutableArray.Empty; + foreach (var missingKey in missingKeys) { + lock (_syncObj) { + if (_analysisEntries.TryGetValue(missingKey, out _)) { + continue; + } + } + + var (moduleName, _, isTypeshed) = missingKey; var moduleResolution = isTypeshed ? interpreter.TypeshedResolution : interpreter.ModuleResolution; - moduleResolution.GetOrLoadModule(moduleName); + var module = moduleResolution.GetOrLoadModule(moduleName); + if (module != null && module.ModuleType != ModuleType.Unresolved) { + foundKeys = foundKeys.Add(missingKey); + } + } + + if (foundKeys.Count > 0) { + foreach (var foundKey in foundKeys) { + PythonAnalyzerEntry entry; + lock (_syncObj) { + if (!_analysisEntries.TryGetValue(foundKey, out entry)) { + continue; + } + } + + _dependencyResolver.TryAddValue(foundKey, entry, ImmutableArray.Empty); + } } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs index 0efd483ac..af174eea9 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs @@ -39,6 +39,7 @@ internal sealed class PythonAnalyzerEntry { private HashSet _analysisDependencies; private int _bufferVersion; private int _analysisVersion; + private int _depth; public IPythonModule Module { get { @@ -74,11 +75,21 @@ public int AnalysisVersion { } } + public int Depth { + get { + lock (_syncObj) { + return _depth; + } + } + } + public bool NotAnalyzed => PreviousAnalysis is EmptyAnalysis; public PythonAnalyzerEntry(EmptyAnalysis emptyAnalysis) { _previousAnalysis = emptyAnalysis; + _module = emptyAnalysis.Document; _isUserModule = emptyAnalysis.Document.ModuleType == ModuleType.User; + _depth = _isUserModule ? 0 : -1; _bufferVersion = -1; _analysisVersion = 0; @@ -92,10 +103,24 @@ public bool IsValidVersion(int version, out IPythonModule module, out PythonAst lock (_syncObj) { module = _module; ast = _ast; + if (ast == null || module == null) { + return false; + } + return _previousAnalysis is EmptyAnalysis || _isUserModule || _analysisVersion <= version; } } + public void SetDepth(int version, int depth) { + lock (_syncObj) { + if (_analysisVersion > version) { + return; + } + + _depth = _depth == -1 ? depth : Math.Min(_depth, depth); + } + } + public void TrySetAnalysis(IDocumentAnalysis analysis, int version) { lock (_syncObj) { if (_previousAnalysis is EmptyAnalysis) { @@ -139,7 +164,7 @@ public void TryCancel(OperationCanceledException oce, int version) { public void Invalidate(int analysisVersion) { lock (_syncObj) { - if (_analysisVersion >= analysisVersion) { + if (_analysisVersion >= analysisVersion || !_analysisTcs.Task.IsCompleted) { return; } @@ -176,7 +201,6 @@ public bool Invalidate(ImmutableArray analysisDependencies, int a return false; } - UpdateAnalysisTcs(analysisVersion); if (_analysisDependencies == null) { _analysisDependencies = dependenciesHashSet; } else { @@ -187,6 +211,7 @@ public bool Invalidate(ImmutableArray analysisDependencies, int a } } + UpdateAnalysisTcs(analysisVersion); dependencies = _parserDependencies != null ? ImmutableArray.Create(_parserDependencies.Union(_analysisDependencies).ToArray()) : ImmutableArray.Create(_analysisDependencies); diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 4d1c33d75..ccf3b8f89 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -193,7 +193,7 @@ private void LogResults(double elapsed, int originalRemaining, int remaining, in if (_log == null) { return; } - + if (remaining == 0) { _log.Log(TraceEventType.Verbose, $"Analysis version {version} of {originalRemaining} entries has been completed in {elapsed} ms."); } else if (remaining < originalRemaining) { @@ -208,8 +208,8 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch, Cancell var remaining = 0; var ace = new AsyncCountdownEvent(0); + bool isCanceled; while ((node = await _walker.GetNextAsync(cancellationToken)) != null) { - bool isCanceled; lock (_syncObj) { isCanceled = _isCanceled; } @@ -229,16 +229,18 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch, Cancell await ace.WaitAsync(cancellationToken); - if (_walker.MissingKeys.All(k => k.IsTypeshed)) { - Interlocked.Exchange(ref _runningTasks, 0); - bool isCanceled; - lock (_syncObj) { - isCanceled = _isCanceled; - } + lock (_syncObj) { + isCanceled = _isCanceled; + } + if (_walker.MissingKeys.Count == 0 || _walker.MissingKeys.All(k => k.IsTypeshed)) { + Interlocked.Exchange(ref _runningTasks, 0); + if (!isCanceled) { _analysisCompleteEvent.Set(); } + } else if (!isCanceled && _log != null && _log.LogLevel >= TraceEventType.Verbose) { + _log?.Log(TraceEventType.Verbose, $"Missing keys: {string.Join(", ", _walker.MissingKeys)}"); } return remaining; @@ -259,6 +261,11 @@ private void Analyze(IDependencyChainNode node, AsyncCountd ace?.AddOne(); var entry = node.Value; if (!entry.IsValidVersion(_walker.Version, out module, out var ast)) { + if (ast == null) { + // Entry doesn't have ast yet. There should be at least one more session. + Cancel(); + } + _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled."); node.Skip(); return; @@ -296,6 +303,11 @@ private void Analyze(PythonAnalyzerEntry entry, int version, CancellationToken c var stopWatch = Stopwatch.StartNew(); try { if (!entry.IsValidVersion(version, out var module, out var ast)) { + if (ast == null) { + // Entry doesn't have ast yet. There should be at least one more session. + Cancel(); + } + _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled."); return; } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyGraph.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyGraph.cs index e07eddc38..9dca1472c 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyGraph.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyGraph.cs @@ -45,15 +45,43 @@ internal sealed class DependencyGraph { private readonly Dictionary> _verticesByKey = new Dictionary>(); private readonly List> _verticesByIndex = new List>(); - public DependencyGraphSnapshot Snapshot { get; private set; } + private bool _snapshotIsInvalid; + private DependencyGraphSnapshot _snapshot; + public int Version { get; private set; } + + public DependencyGraphSnapshot Snapshot { + get { + if (_snapshotIsInvalid) { + CreateNewSnapshot(); + } + return _snapshot; + } + } + public DependencyGraph() { - Snapshot = new DependencyGraphSnapshot(0); + _snapshot = new DependencyGraphSnapshot(0); + _snapshotIsInvalid = false; + } + + public DependencyVertex TryAdd(TKey key, TValue value, ImmutableArray incomingKeys) { + var version = ++Version; + _snapshotIsInvalid = true; + + if (_verticesByKey.TryGetValue(key, out _)) { + return null; + } + + var changedVertex = new DependencyVertex(key, value, incomingKeys, version, _verticesByIndex.Count); + _verticesByIndex.Add(changedVertex); + _verticesByKey[key] = changedVertex; + return changedVertex; } public DependencyVertex AddOrUpdate(TKey key, TValue value, ImmutableArray incomingKeys) { - var version = Snapshot.Version + 1; - + var version = ++Version; + _snapshotIsInvalid = true; + DependencyVertex changedVertex; if (_verticesByKey.TryGetValue(key, out var currentVertex)) { changedVertex = new DependencyVertex(currentVertex, value, incomingKeys, version); @@ -64,25 +92,12 @@ public DependencyVertex AddOrUpdate(TKey key, TValue value, Immuta } _verticesByKey[key] = changedVertex; - - - var vertices = _verticesByIndex - .Where(v => !v.IsSealed || v.HasMissingKeys) - .Select(v => GetOrCreateNonSealedVertex(version, v.Index)) - .ToArray(); - - if (vertices.Length == 0) { - Snapshot = new DependencyGraphSnapshot(version); - return changedVertex; - } - - CreateNewSnapshot(vertices, version); - return changedVertex; } public void RemoveKeys(ImmutableArray keys) { - var version = Snapshot.Version + 1; + var version = ++Version; + _snapshotIsInvalid = true; _verticesByIndex.Clear(); foreach (var key in keys) { @@ -98,12 +113,14 @@ public void RemoveKeys(ImmutableArray keys) { _verticesByKey[vertex.Key] = vertex; } - CreateNewSnapshot(_verticesByIndex, version); + CreateNewSnapshot(); } - private void CreateNewSnapshot(IEnumerable> vertices, int version) { + private void CreateNewSnapshot() { + var version = Version; var missingKeysHashSet = new HashSet(); - foreach (var vertex in vertices) { + for (var i = 0; i < _verticesByIndex.Count; i++) { + var vertex = _verticesByIndex[i]; var newIncoming = ImmutableArray.Empty; var oldIncoming = vertex.Incoming; @@ -112,10 +129,17 @@ private void CreateNewSnapshot(IEnumerable> verti newIncoming = newIncoming.Add(dependency.Index); } else { missingKeysHashSet.Add(dependencyKey); + if (vertex.IsSealed) { + vertex = CreateNonSealedVertex(vertex, version, i); + } vertex.SetHasMissingKeys(); } } + if (newIncoming.SequentiallyEquals(oldIncoming)) { + continue; + } + foreach (var index in oldIncoming.Except(newIncoming)) { var incomingVertex = GetOrCreateNonSealedVertex(version, index); incomingVertex.RemoveOutgoing(vertex.Index); @@ -126,6 +150,10 @@ private void CreateNewSnapshot(IEnumerable> verti incomingVertex.AddOutgoing(vertex.Index); } + if (vertex.IsSealed) { + vertex = CreateNonSealedVertex(vertex, version, i); + } + vertex.SetIncoming(newIncoming); } @@ -133,7 +161,8 @@ private void CreateNewSnapshot(IEnumerable> verti vertex.Seal(); } - Snapshot = new DependencyGraphSnapshot(version, + _snapshotIsInvalid = false; + _snapshot = new DependencyGraphSnapshot(version, ImmutableArray>.Create(_verticesByIndex), ImmutableArray.Create(missingKeysHashSet)); } @@ -149,5 +178,14 @@ private DependencyVertex GetOrCreateNonSealedVertex(int version, i _verticesByKey[vertex.Key] = vertex; return vertex; } + + private DependencyVertex CreateNonSealedVertex(DependencyVertex oldVertex, int version, int index) { + var vertex = new DependencyVertex(oldVertex, oldVertex.Value, oldVertex.IncomingKeys, version); + _verticesByIndex[index] = vertex; + _verticesByKey[vertex.Key] = vertex; + return vertex; + } + + } } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs index 6cb88b7f1..be9b86b3d 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs @@ -13,9 +13,11 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Core.Collections; using Microsoft.Python.Core.Threading; @@ -25,21 +27,39 @@ internal sealed class DependencyResolver : IDependencyResolver> _changedVertices = new Dictionary>(); private readonly object _syncObj = new object(); - public DependencyGraphSnapshot NotifyChanges(TKey key, TValue value, params TKey[] incomingKeys) - => NotifyChanges(key, value, ImmutableArray.Create(incomingKeys)); + public DependencyGraphSnapshot CurrentGraphSnapshot { + get { + lock (_syncObj) { + return _vertices.Snapshot; + } + } + } - public DependencyGraphSnapshot NotifyChanges(TKey key, TValue value, ImmutableArray incomingKeys) { + public int ChangeValue(TKey key, TValue value, params TKey[] incomingKeys) + => ChangeValue(key, value, ImmutableArray.Create(incomingKeys)); + + public int ChangeValue(TKey key, TValue value, ImmutableArray incomingKeys) { lock (_syncObj) { var dependencyVertex = _vertices.AddOrUpdate(key, value, incomingKeys); _changedVertices[key] = dependencyVertex; - return _vertices.Snapshot; + return _vertices.Version; } } - public DependencyGraphSnapshot RemoveKeys(params TKey[] keys) - => RemoveKeys(ImmutableArray.Create(keys)); + public int TryAddValue(TKey key, TValue value, ImmutableArray incomingKeys) { + lock (_syncObj) { + var dependencyVertex = _vertices.TryAdd(key, value, incomingKeys); + if (dependencyVertex != null) { + _changedVertices[key] = dependencyVertex; + } - public DependencyGraphSnapshot RemoveKeys(ImmutableArray keys) { + return _vertices.Version; + } + } + + public int RemoveKeys(params TKey[] keys) => RemoveKeys(ImmutableArray.Create(keys)); + + public int RemoveKeys(ImmutableArray keys) { lock (_syncObj) { _vertices.RemoveKeys(keys); var snapshot = _vertices.Snapshot; @@ -52,16 +72,14 @@ public DependencyGraphSnapshot RemoveKeys(ImmutableArray key _changedVertices[vertex.Key] = vertex; } - return snapshot; + return _vertices.Version; } } - public bool TryCreateWalker(int version, out IDependencyChainWalker walker) { - DependencyGraphSnapshot snapshot; + public bool TryCreateWalker(DependencyGraphSnapshot snapshot, out IDependencyChainWalker walker) { ImmutableArray> changedVertices; lock (_syncObj) { - snapshot = _vertices.Snapshot; - if (version != snapshot.Version) { + if (snapshot.Version != _vertices.Version) { walker = default; return false; } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs index 2561c2300..207e08408 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs @@ -33,12 +33,12 @@ internal sealed class DependencyVertex { public ImmutableArray Outgoing { get; private set; } public DependencyVertex(DependencyVertex oldVertex, TValue value, ImmutableArray incomingKeys, int version) { - Key = oldVertex.Key; Value = value; Version = version; - Index = oldVertex.Index; - IncomingKeys = incomingKeys; + + Key = oldVertex.Key; + Index = oldVertex.Index; Incoming = oldVertex.Incoming; Outgoing = oldVertex.Outgoing; } diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs index 84cab4711..2a3583ca4 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs @@ -13,6 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Core.Collections; namespace Microsoft.Python.Analysis.Dependencies { @@ -23,9 +24,13 @@ namespace Microsoft.Python.Analysis.Dependencies { /// concurrently. /// internal interface IDependencyResolver { - DependencyGraphSnapshot NotifyChanges(TKey key, TValue value, ImmutableArray incomingKeys); - DependencyGraphSnapshot RemoveKeys(ImmutableArray keys); + DependencyGraphSnapshot CurrentGraphSnapshot { get; } + + int TryAddValue(TKey key, TValue value, ImmutableArray incomingKeys); + int ChangeValue(TKey key, TValue value, ImmutableArray incomingKeys); + int RemoveKeys(ImmutableArray keys); + IDependencyChainWalker CreateWalker(); - bool TryCreateWalker(int version, out IDependencyChainWalker walker); + bool TryCreateWalker(DependencyGraphSnapshot snapshot, out IDependencyChainWalker walker); } } diff --git a/src/Analysis/Ast/Impl/Documents/Definitions/IRunningDocumentTable.cs b/src/Analysis/Ast/Impl/Documents/Definitions/IRunningDocumentTable.cs index 859266206..0debc17ad 100644 --- a/src/Analysis/Ast/Impl/Documents/Definitions/IRunningDocumentTable.cs +++ b/src/Analysis/Ast/Impl/Documents/Definitions/IRunningDocumentTable.cs @@ -55,12 +55,7 @@ public interface IRunningDocumentTable { /// Fetches document by its URI. Returns null if document is not loaded. /// IDocument GetDocument(Uri uri); - - /// - /// Fetches document by name. Returns null if document is not loaded. - /// - IDocument GetDocument(string name); - + /// /// Increase reference count of the document. /// diff --git a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs index 4c3096ab9..9fc6d389d 100644 --- a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs +++ b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs @@ -32,7 +32,6 @@ namespace Microsoft.Python.Analysis.Documents { /// public sealed class RunningDocumentTable : IRunningDocumentTable, IDisposable { private readonly Dictionary _documentsByUri = new Dictionary(); - private readonly Dictionary _documentsByName = new Dictionary(); private readonly IServiceContainer _services; private readonly ILogger _log; private readonly object _lock = new object(); @@ -65,7 +64,7 @@ public RunningDocumentTable(IServiceContainer services) { /// public IEnumerable GetDocuments() { lock (_lock) { - return _documentsByName.Values.Select(e => e.Document).ToArray(); + return _documentsByUri.Values.Select(e => e.Document).ToArray(); } } @@ -77,9 +76,9 @@ public IEnumerable GetDocuments() { /// Optional file path, if different from the URI. public IDocument OpenDocument(Uri uri, string content, string filePath = null) { bool justOpened; - DocumentEntry entry; + IDocument document; lock (_lock) { - entry = FindDocument(null, uri); + var entry = FindDocument(uri); if (entry == null) { var resolver = _services.GetService().ModuleResolution.CurrentPathResolver; @@ -99,17 +98,23 @@ public IDocument OpenDocument(Uri uri, string content, string filePath = null) { entry = CreateDocument(mco); } justOpened = TryOpenDocument(entry, content); + document = entry.Document; } + if (justOpened) { - Opened?.Invoke(this, new DocumentEventArgs(entry.Document)); + Opened?.Invoke(this, new DocumentEventArgs(document)); + _services.GetService().InvalidateAnalysis(document); } - return entry.Document; + + return document; } /// /// Adds library module to the list of available documents. /// public IDocument AddModule(ModuleCreationOptions mco) { + IDocument document; + lock (_lock) { if (mco.Uri == null) { mco.FilePath = mco.FilePath ?? throw new ArgumentNullException(nameof(mco.FilePath)); @@ -122,10 +127,13 @@ public IDocument AddModule(ModuleCreationOptions mco) { mco.Uri = uri; } - var entry = FindDocument(mco.ModuleName, mco.Uri) ?? CreateDocument(mco); + var entry = FindDocument(mco.Uri) ?? CreateDocument(mco); entry.LockCount++; - return entry.Document; + document = entry.Document; } + + _services.GetService().InvalidateAnalysis(document); + return document; } public IDocument GetDocument(Uri documentUri) { @@ -134,12 +142,6 @@ public IDocument GetDocument(Uri documentUri) { } } - public IDocument GetDocument(string name) { - lock (_lock) { - return _documentsByName.TryGetValue(name, out var entry) ? entry.Document : null; - } - } - public int LockDocument(Uri uri) { lock (_lock) { if (_documentsByUri.TryGetValue(uri, out var entry)) { @@ -175,7 +177,6 @@ public void CloseDocument(Uri documentUri) { if (entry.LockCount == 0) { _documentsByUri.Remove(documentUri); - _documentsByName.Remove(entry.Document.Name); removed = true; entry.Document.Dispose(); } @@ -197,14 +198,11 @@ public void Dispose() { } } - private DocumentEntry FindDocument(string moduleName, Uri uri) { + private DocumentEntry FindDocument(Uri uri) { if (uri != null && _documentsByUri.TryGetValue(uri, out var entry)) { return entry; } - if (!string.IsNullOrEmpty(moduleName) && _documentsByName.TryGetValue(moduleName, out entry)) { - return entry; - } return null; } @@ -230,9 +228,6 @@ private DocumentEntry CreateDocument(ModuleCreationOptions mco) { var entry = new DocumentEntry(document); _documentsByUri[document.Uri] = entry; - _documentsByName[mco.ModuleName] = entry; - - _services.GetService().InvalidateAnalysis(document); return entry; } @@ -242,7 +237,7 @@ private bool TryAddModulePath(ModuleCreationOptions mco) { throw new InvalidOperationException("Can't create document with no file path or URI specified"); } - if (!ModuleManagement.TryAddModulePath(filePath, out var fullName)) { + if (!ModuleManagement.TryAddModulePath(filePath, true, out var fullName)) { return false; } diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs index a463afd1b..06a25fe48 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs @@ -35,7 +35,7 @@ public interface IModuleManagement: IModuleResolution { IModuleCache ModuleCache { get; } - bool TryAddModulePath(in string path, out string fullName); + bool TryAddModulePath(in string path, in bool allowNonRooted, out string fullName); /// /// Sets user search paths. This changes . diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index 05a5156e0..87f17d64c 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -91,12 +91,14 @@ protected override IPythonModule CreateModule(string name) { return null; } - var module = GetRdt().GetDocument(moduleImport.FullName); - if (module != null) { - GetRdt().LockDocument(module.Uri); - return module; + if (moduleImport.ModulePath != null) { + var module = GetRdt().GetDocument(new Uri(moduleImport.ModulePath)); + if (module != null) { + GetRdt().LockDocument(module.Uri); + return module; + } } - + // If there is a stub, make sure it is loaded and attached // First check stub next to the module. if (!TryCreateModuleStub(name, moduleImport.ModulePath, out var stub)) { @@ -111,24 +113,25 @@ protected override IPythonModule CreateModule(string name) { if (moduleImport.IsBuiltin) { _log?.Log(TraceEventType.Verbose, "Create built-in compiled (scraped) module: ", name, Configuration.InterpreterPath); - module = new CompiledBuiltinPythonModule(name, stub, _services); - } else if (moduleImport.IsCompiled) { - _log?.Log(TraceEventType.Verbose, "Create compiled (scraped): ", moduleImport.FullName, moduleImport.ModulePath, moduleImport.RootPath); - module = new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, _services); - } else { - _log?.Log(TraceEventType.Verbose, "Import: ", moduleImport.FullName, moduleImport.ModulePath); - // Module inside workspace == user code. + return new CompiledBuiltinPythonModule(name, stub, _services); + } - var mco = new ModuleCreationOptions { - ModuleName = moduleImport.FullName, - ModuleType = moduleImport.IsLibrary ? ModuleType.Library : ModuleType.User, - FilePath = moduleImport.ModulePath, - Stub = stub - }; - module = GetRdt().AddModule(mco); + if (moduleImport.IsCompiled) { + _log?.Log(TraceEventType.Verbose, "Create compiled (scraped): ", moduleImport.FullName, moduleImport.ModulePath, moduleImport.RootPath); + return new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, _services); } - return module; + _log?.Log(TraceEventType.Verbose, "Import: ", moduleImport.FullName, moduleImport.ModulePath); + // Module inside workspace == user code. + + var mco = new ModuleCreationOptions { + ModuleName = moduleImport.FullName, + ModuleType = moduleImport.IsLibrary ? ModuleType.Library : ModuleType.User, + FilePath = moduleImport.ModulePath, + Stub = stub + }; + + return GetRdt().AddModule(mco); } private async Task> GetInterpreterSearchPathsAsync(CancellationToken cancellationToken = default) { diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs index fb6b1d793..93d7a0a83 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs @@ -97,8 +97,8 @@ public IPythonModule GetOrLoadModule(string name) { return moduleRef.GetOrCreate(name, this); } - public bool TryAddModulePath(in string path, out string fullModuleName) - => PathResolver.TryAddModulePath(path, true, out fullModuleName); + public bool TryAddModulePath(in string path, in bool allowNonRooted, out string fullModuleName) + => PathResolver.TryAddModulePath(path, allowNonRooted, out fullModuleName); public ModulePath FindModule(string filePath) { var bestLibraryPath = string.Empty; diff --git a/src/Analysis/Ast/Test/DependencyResolverTests.cs b/src/Analysis/Ast/Test/DependencyResolverTests.cs index 59aea5eec..c1e22f013 100644 --- a/src/Analysis/Ast/Test/DependencyResolverTests.cs +++ b/src/Analysis/Ast/Test/DependencyResolverTests.cs @@ -48,14 +48,14 @@ public void TestInitialize() [DataRow("A:D|B:E|C:F|D:E|E:F|F:D", "DFEDFE[ABC]")] // ReSharper restore StringLiteralTypo [DataTestMethod] - public void NotifyChanges(string input, string output) { + public void ChangeValue(string input, string output) { var resolver = new DependencyResolver(); var splitInput = input.Split("|"); foreach (var value in splitInput) { var kv = value.Split(":"); var dependencies = kv.Length == 1 ? ImmutableArray.Empty : ImmutableArray.Create(kv[1].Select(c => c.ToString()).ToList()); - resolver.NotifyChanges(value.Split(":")[0], value, dependencies); + resolver.ChangeValue(value.Split(":")[0], value, dependencies); } var walker = resolver.CreateWalker(); @@ -86,11 +86,11 @@ public void NotifyChanges(string input, string output) { } [TestMethod] - public async Task NotifyChanges_RepeatedChange() { + public async Task ChangeValue_RepeatedChange() { var resolver = new DependencyResolver(); - resolver.NotifyChanges("A", "A:B", "B"); - resolver.NotifyChanges("B", "B:C", "C"); - resolver.NotifyChanges("C", "C"); + resolver.ChangeValue("A", "A:B", "B"); + resolver.ChangeValue("B", "B:C", "C"); + resolver.ChangeValue("C", "C"); var walker = resolver.CreateWalker(); var result = new StringBuilder(); @@ -102,7 +102,7 @@ public async Task NotifyChanges_RepeatedChange() { result.ToString().Should().Be("CBA"); - resolver.NotifyChanges("B", "B:C", "C"); + resolver.ChangeValue("B", "B:C", "C"); walker = resolver.CreateWalker(); result = new StringBuilder(); @@ -116,12 +116,12 @@ public async Task NotifyChanges_RepeatedChange() { } [TestMethod] - public async Task NotifyChanges_RepeatedChange2() { + public async Task ChangeValue_RepeatedChange2() { var resolver = new DependencyResolver(); - resolver.NotifyChanges("A", "A:B", "B"); - resolver.NotifyChanges("B", "B"); - resolver.NotifyChanges("C", "C:D", "D"); - resolver.NotifyChanges("D", "D"); + resolver.ChangeValue("A", "A:B", "B"); + resolver.ChangeValue("B", "B"); + resolver.ChangeValue("C", "C:D", "D"); + resolver.ChangeValue("D", "D"); var walker = resolver.CreateWalker(); var result = new StringBuilder(); @@ -133,8 +133,8 @@ public async Task NotifyChanges_RepeatedChange2() { result.ToString().Should().Be("BDAC"); - resolver.NotifyChanges("D", "D"); - resolver.NotifyChanges("B", "B:C", "C"); + resolver.ChangeValue("D", "D"); + resolver.ChangeValue("B", "B:C", "C"); walker = resolver.CreateWalker(); result = new StringBuilder(); @@ -148,11 +148,11 @@ public async Task NotifyChanges_RepeatedChange2() { } [TestMethod] - public async Task NotifyChanges_MissingKeys() { + public async Task ChangeValue_MissingKeys() { var resolver = new DependencyResolver(); - resolver.NotifyChanges("A", "A:B", "B"); - resolver.NotifyChanges("B", "B"); - resolver.NotifyChanges("C", "C:D", "D"); + resolver.ChangeValue("A", "A:B", "B"); + resolver.ChangeValue("B", "B"); + resolver.ChangeValue("C", "C:D", "D"); var walker = resolver.CreateWalker(); var result = new StringBuilder(); @@ -167,7 +167,7 @@ public async Task NotifyChanges_MissingKeys() { walker.MissingKeys.Should().Equal("D"); result.ToString().Should().Be("BC"); - resolver.NotifyChanges("D", "D"); + resolver.ChangeValue("D", "D"); walker = resolver.CreateWalker(); result = new StringBuilder(); result.Append((await walker.GetNextAsync(default)).Value[0]); @@ -178,12 +178,12 @@ public async Task NotifyChanges_MissingKeys() { } [TestMethod] - public async Task NotifyChanges_RemoveKeys() { + public async Task ChangeValue_RemoveKeys() { var resolver = new DependencyResolver(); - resolver.NotifyChanges("A", "A", "B", "C"); - resolver.NotifyChanges("B", "B", "C"); - resolver.NotifyChanges("C", "C", "D"); - resolver.NotifyChanges("D", "D"); + resolver.ChangeValue("A", "A", "B", "C"); + resolver.ChangeValue("B", "B", "C"); + resolver.ChangeValue("C", "C", "D"); + resolver.ChangeValue("D", "D"); var walker = resolver.CreateWalker(); walker.MissingKeys.Should().BeEmpty(); @@ -217,12 +217,12 @@ public async Task NotifyChanges_RemoveKeys() { } [TestMethod] - public async Task NotifyChanges_Skip() { + public async Task ChangeValue_Skip() { var resolver = new DependencyResolver(); - resolver.NotifyChanges("A", "A:B", "B"); - resolver.NotifyChanges("B", "B"); - resolver.NotifyChanges("D", "D"); - resolver.NotifyChanges("C", "C:D", "D"); + resolver.ChangeValue("A", "A:B", "B"); + resolver.ChangeValue("B", "B"); + resolver.ChangeValue("D", "D"); + resolver.ChangeValue("C", "C:D", "D"); var walker = resolver.CreateWalker(); var result = new StringBuilder(); @@ -236,7 +236,7 @@ public async Task NotifyChanges_Skip() { result.ToString().Should().Be("BD"); - resolver.NotifyChanges("D", "D"); + resolver.ChangeValue("D", "D"); walker = resolver.CreateWalker(); result = new StringBuilder(); result.Append((await walker.GetNextAsync(default)).Value[0]); diff --git a/src/Analysis/Ast/Test/LibraryTests.cs b/src/Analysis/Ast/Test/LibraryTests.cs index 13bc48666..eaca5e09f 100644 --- a/src/Analysis/Ast/Test/LibraryTests.cs +++ b/src/Analysis/Ast/Test/LibraryTests.cs @@ -39,9 +39,9 @@ public async Task Random() { var analysis = await GetAnalysisAsync("from random import *", PythonVersions.LatestAvailable3X); foreach (var fnName in new[] { @"seed", @"randrange", @"gauss" }) { - var v = analysis.Should().HaveVariable(fnName).Which; - v.Should().HaveType(BuiltinTypeId.Function); - v.Value.GetPythonType().Documentation.Should().NotBeNullOrEmpty(); + analysis.Should().HaveVariable(fnName).Which + .Should().HaveType(BuiltinTypeId.Function) + .And.Value.GetPythonType().Documentation.Should().NotBeNullOrEmpty(); } } diff --git a/src/Analysis/Ast/Test/ReferencesTests.cs b/src/Analysis/Ast/Test/ReferencesTests.cs index 79d9bc884..936f9dc7b 100644 --- a/src/Analysis/Ast/Test/ReferencesTests.cs +++ b/src/Analysis/Ast/Test/ReferencesTests.cs @@ -329,15 +329,13 @@ from MultiValues import t var parent = t.Parent; parent.Should().NotBeNull(); - parent.References.Should().HaveCount(4); + parent.References.Should().HaveCount(3); parent.References[0].Span.Should().Be(3, 1, 3, 2); parent.References[0].DocumentUri.AbsolutePath.Should().Contain("MultiValues.py"); - parent.References[1].Span.Should().Be(12, 5, 12, 6); - parent.References[1].DocumentUri.AbsolutePath.Should().Contain("MultiValues.py"); - parent.References[2].Span.Should().Be(2, 25, 2, 26); + parent.References[1].Span.Should().Be(2, 25, 2, 26); + parent.References[1].DocumentUri.AbsolutePath.Should().Contain("module.py"); + parent.References[2].Span.Should().Be(3, 5, 3, 6); parent.References[2].DocumentUri.AbsolutePath.Should().Contain("module.py"); - parent.References[3].Span.Should().Be(3, 5, 3, 6); - parent.References[3].DocumentUri.AbsolutePath.Should().Contain("module.py"); } [TestMethod, Priority(0)] @@ -350,13 +348,11 @@ from MultiValues import * var t = analysis.Should().HaveVariable("t").Which; t.Definition.Span.Should().Be(3, 1, 3, 2); t.Definition.DocumentUri.AbsolutePath.Should().Contain("MultiValues.py"); - t.References.Should().HaveCount(3); + t.References.Should().HaveCount(2); t.References[0].Span.Should().Be(3, 1, 3, 2); t.References[0].DocumentUri.AbsolutePath.Should().Contain("MultiValues.py"); - t.References[1].Span.Should().Be(12, 5, 12, 6); - t.References[1].DocumentUri.AbsolutePath.Should().Contain("MultiValues.py"); - t.References[2].Span.Should().Be(3, 5, 3, 6); - t.References[2].DocumentUri.AbsolutePath.Should().Contain("module.py"); + t.References[1].Span.Should().Be(3, 5, 3, 6); + t.References[1].DocumentUri.AbsolutePath.Should().Contain("module.py"); } [TestMethod, Priority(0)] diff --git a/src/Analysis/Core/Impl/DependencyResolution/PathResolverSnapshot.cs b/src/Analysis/Core/Impl/DependencyResolution/PathResolverSnapshot.cs index ae5ff7f5e..a21da51f3 100644 --- a/src/Analysis/Core/Impl/DependencyResolution/PathResolverSnapshot.cs +++ b/src/Analysis/Core/Impl/DependencyResolution/PathResolverSnapshot.cs @@ -101,7 +101,7 @@ private bool TryFindModuleByName(in int rootIndex, in string fullModuleName, out foreach (var nameSpan in fullModuleName.SplitIntoSpans('.')) { var moduleNode = lastEdge.End; var childIndex = moduleNode.GetChildIndex(nameSpan); - if (childIndex == -1 || moduleNode.IsModule) { + if (childIndex == -1 || moduleNode.IsModule && !IsInitPyModule(moduleNode, out _)) { return false; } lastEdge = lastEdge.Append(childIndex); @@ -419,7 +419,7 @@ private Node GetOrCreateRoot(string path) public PathResolverSnapshot AddModulePath(in string modulePath, in bool allowNonRooted, out string fullModuleName) { var isFound = TryFindModule(modulePath, out var lastEdge, out var unmatchedPathSpan); - if (unmatchedPathSpan.Source == default || (!allowNonRooted && lastEdge.IsNonRooted)) { + if (unmatchedPathSpan.Source == default || !allowNonRooted && lastEdge.IsNonRooted) { // Not a module fullModuleName = null; return this; @@ -435,12 +435,11 @@ public PathResolverSnapshot AddModulePath(in string modulePath, in bool allowNon return ReplaceNonRooted(AddToNonRooted(lastEdge, unmatchedPathSpan, out fullModuleName)); } - var newChildNode = CreateNewNodes(lastEdge, unmatchedPathSpan, out fullModuleName); if (unmatchedPathSpan.Length == 0) { - lastEdge = lastEdge.Previous; + } - var newEnd = lastEdge.End.AddChild(newChildNode); + var newEnd = CreateNewNodes(lastEdge, unmatchedPathSpan, out fullModuleName); var newRoot = UpdateNodesFromEnd(lastEdge, newEnd); return ImmutableReplaceRoot(newRoot, lastEdge.FirstEdge.EndIndex); } @@ -530,10 +529,10 @@ private static bool TryFindImport(IEnumerable rootEdges, List full return matchedEdges.Count > 0; } - private static bool TryFindName(in Edge edge, in IEnumerable nameParts, out Edge lastEdge) { + private static bool TryFindName(in Edge edge, in List nameParts, out Edge lastEdge) { lastEdge = edge; foreach (var name in nameParts) { - if (lastEdge.End.IsModule) { + if (lastEdge.End.IsModule && !IsInitPyModule(lastEdge.End, out _)) { return false; } var index = lastEdge.End.GetChildIndex(name); @@ -602,7 +601,8 @@ private static Node CreateNewNodes(in Edge lastEdge, in StringSpan unmatchedPath // Module is added to existing package var name = modulePath.Substring(unmatchedPathStart, unmatchedPathLength); fullModuleName = GetFullModuleName(lastEdge, name); - return Node.CreateModule(name, modulePath, fullModuleName); + var newChildNode = Node.CreateModule(name, modulePath, fullModuleName); + return lastEdge.End.AddChild(newChildNode); } var names = modulePath.Split(Path.DirectorySeparatorChar, unmatchedPathStart, unmatchedPathLength); @@ -613,7 +613,7 @@ private static Node CreateNewNodes(in Edge lastEdge, in StringSpan unmatchedPath newNode = Node.CreatePackage(names[i], GetFullModuleName(lastEdge, names, i), newNode); } - return newNode; + return lastEdge.End.AddChild(newNode); } private static string GetFullModuleName(in Edge lastEdge) { diff --git a/src/Core/Impl/Collections/ImmutableArray.cs b/src/Core/Impl/Collections/ImmutableArray.cs index 058b28d2d..d11b192b6 100644 --- a/src/Core/Impl/Collections/ImmutableArray.cs +++ b/src/Core/Impl/Collections/ImmutableArray.cs @@ -265,6 +265,21 @@ private int GetCapacity(int length) { return capacity; } + public bool SequentiallyEquals(ImmutableArray other) { + if (_count != other._count) { + return false; + } + + var comparer = EqualityComparer.Default; + for (var i = 0; i < _count; i++) { + if (!comparer.Equals(_items[i], other._items[i])) { + return false; + } + } + + return true; + } + public bool Equals(ImmutableArray other) => Equals(_items, other._items) && _count == other._count; diff --git a/src/LanguageServer/Impl/Sources/ReferenceSource.cs b/src/LanguageServer/Impl/Sources/ReferenceSource.cs index ce051de43..fac8c0fe6 100644 --- a/src/LanguageServer/Impl/Sources/ReferenceSource.cs +++ b/src/LanguageServer/Impl/Sources/ReferenceSource.cs @@ -15,13 +15,11 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis; -using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; @@ -30,7 +28,6 @@ using Microsoft.Python.Core.Text; using Microsoft.Python.LanguageServer.Documents; using Microsoft.Python.LanguageServer.Protocol; -using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.LanguageServer.Sources { internal enum ReferenceSearchOptions { @@ -62,15 +59,15 @@ public async Task FindAllReferencesAsync(Uri uri, SourceLocation lo // then the location is invalid and the module is null. Use current module. var declaringModule = rootDefinition.DeclaringModule ?? analysis.Document; if (!string.IsNullOrEmpty(name) && (declaringModule.ModuleType == ModuleType.User || options == ReferenceSearchOptions.All)) { - return await FindAllReferencesAsync(name, rootDefinition, cancellationToken); + return await FindAllReferencesAsync(name, declaringModule, rootDefinition, cancellationToken); } } return Array.Empty(); } - private async Task FindAllReferencesAsync(string name, ILocatedMember rootDefinition, CancellationToken cancellationToken) { + private async Task FindAllReferencesAsync(string name, IPythonModule declaringModule, ILocatedMember rootDefinition, CancellationToken cancellationToken) { var candidateFiles = ScanClosedFiles(name, cancellationToken); - await AnalyzeFiles(candidateFiles, cancellationToken); + await AnalyzeFiles(declaringModule.Interpreter.ModuleResolution, candidateFiles, cancellationToken); return rootDefinition.References .Select(r => new Reference { uri = new Uri(r.FilePath), range = r.Span }) @@ -112,88 +109,23 @@ private IEnumerable ScanClosedFiles(string name, CancellationToken cancella return files; } - private IEnumerable ScanFiles(IDictionary closedFiles, string name, CancellationToken cancellationToken) { - var candidateNames = new HashSet { name }; - var candidateFiles = new HashSet(); - - while (candidateNames.Count > 0) { - var nextCandidateNames = new HashSet(); - - foreach (var kvp in closedFiles.ToArray()) { - cancellationToken.ThrowIfCancellationRequested(); - - var w = new ImportsWalker(candidateNames); - try { - kvp.Value.Walk(w); - } catch (OperationCanceledException) { } - - if (w.IsCandidate) { - candidateFiles.Add(kvp.Key); - nextCandidateNames.Add(Path.GetFileNameWithoutExtension(kvp.Key)); - closedFiles.Remove(kvp.Key); - } - } - candidateNames = nextCandidateNames; - } - return candidateFiles; - } - - private async Task AnalyzeFiles(IEnumerable files, CancellationToken cancellationToken) { - var rdt = _services.GetService(); + private static async Task AnalyzeFiles(IModuleManagement moduleManagement, IEnumerable files, CancellationToken cancellationToken) { var analysisTasks = new List(); foreach (var f in files) { - analysisTasks.Add(GetOrOpenModule(f, rdt).GetAnalysisAsync(cancellationToken: cancellationToken)); + if (moduleManagement.TryAddModulePath(f.ToAbsolutePath(), false, out var fullName)) { + var module = moduleManagement.GetOrLoadModule(fullName); + if (module is IDocument document) { + analysisTasks.Add(document.GetAnalysisAsync(cancellationToken: cancellationToken)); + } + } } await Task.WhenAll(analysisTasks); } - - private static IDocument GetOrOpenModule(Uri uri, IRunningDocumentTable rdt) { - var document = rdt.GetDocument(uri); - if (document != null) { - return document; // Already opened by another analysis. - } - - var filePath = uri.ToAbsolutePath(); - var mco = new ModuleCreationOptions { - ModuleName = Path.GetFileNameWithoutExtension(filePath), - FilePath = filePath, - Uri = uri, - ModuleType = ModuleType.User - }; - - return rdt.AddModule(mco); - } - + private ILocatedMember GetRootDefinition(ILocatedMember lm) { for (; lm.Parent != null; lm = lm.Parent) { } return lm; } - - private class ImportsWalker : PythonWalker { - private readonly HashSet _names; - - public bool IsCandidate { get; private set; } - - public ImportsWalker(HashSet names) { - _names = names; - } - - public override bool Walk(ImportStatement node) { - if (node.Names.ExcludeDefault().Any(n => _names.Any(x => n.MakeString().Contains(x)))) { - IsCandidate = true; - throw new OperationCanceledException(); - } - return false; - } - - public override bool Walk(FromImportStatement node) { - if (_names.Any(x => node.Root.MakeString().Contains(x))) { - IsCandidate = true; - throw new OperationCanceledException(); - } - return false; - } - } } } diff --git a/src/LanguageServer/Test/ImportsTests.cs b/src/LanguageServer/Test/ImportsTests.cs index 11a2e29b7..6c35e91cc 100644 --- a/src/LanguageServer/Test/ImportsTests.cs +++ b/src/LanguageServer/Test/ImportsTests.cs @@ -317,6 +317,102 @@ import package.sub_package.module2 comps.Should().HaveLabels("Y").And.NotContainLabels("X"); } + [TestMethod, Priority(0)] + public async Task InitPyVsModuleNameImport_AbsoluteImport() { + const string appCode = @" +import package.module as module +import package.module.submodule as submodule +module. +submodule."; + + var appUri = TestData.GetTestSpecificUri("app.py"); + var moduleUri = TestData.GetTestSpecificUri("package", "module.py"); + var initPyUri = TestData.GetTestSpecificUri("package", "module", "__init__.py"); + var submoduleUri = TestData.GetTestSpecificUri("package", "module", "submodule.py"); + + var root = TestData.GetTestSpecificRootUri().AbsolutePath; + await CreateServicesAsync(root, PythonVersions.LatestAvailable3X); + var rdt = Services.GetService(); + + rdt.OpenDocument(initPyUri, "Y = 6 * 9"); + rdt.OpenDocument(moduleUri, "X = 42"); + rdt.OpenDocument(submoduleUri, "Z = 0"); + + var doc = rdt.OpenDocument(appUri, appCode); + var analysis = await doc.GetAnalysisAsync(-1); + + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var comps = cs.GetCompletions(analysis, new SourceLocation(4, 8)); + comps.Should().HaveLabels("Y").And.NotContainLabels("X"); + + comps = cs.GetCompletions(analysis, new SourceLocation(5, 11)); + comps.Should().HaveLabels("Z"); + } + + [TestMethod, Priority(0)] + public async Task InitPyVsModuleNameImport_FromAbsoluteImport() { + const string appCode = @" +from package import module +from package.module import submodule +module. +submodule."; + + var appUri = TestData.GetTestSpecificUri("app.py"); + var moduleUri = TestData.GetTestSpecificUri("package", "module.py"); + var initPyUri = TestData.GetTestSpecificUri("package", "module", "__init__.py"); + var submoduleUri = TestData.GetTestSpecificUri("package", "module", "submodule.py"); + + var root = TestData.GetTestSpecificRootUri().AbsolutePath; + await CreateServicesAsync(root, PythonVersions.LatestAvailable3X); + var rdt = Services.GetService(); + + rdt.OpenDocument(initPyUri, "Y = 6 * 9"); + rdt.OpenDocument(moduleUri, "X = 42"); + rdt.OpenDocument(submoduleUri, "Z = 0"); + + var doc = rdt.OpenDocument(appUri, appCode); + var analysis = await doc.GetAnalysisAsync(-1); + + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var comps = cs.GetCompletions(analysis, new SourceLocation(4, 8)); + comps.Should().HaveLabels("Y").And.NotContainLabels("X"); + + comps = cs.GetCompletions(analysis, new SourceLocation(5, 11)); + comps.Should().HaveLabels("Z"); + } + + [TestMethod, Priority(0)] + public async Task InitPyVsModuleNameImport_FromRelativeImport() { + const string appCode = @" +from .sub_package import module +from .sub_package.module import submodule +module. +submodule."; + + var appPath = TestData.GetTestSpecificPath("package", "app.py"); + var modulePath = TestData.GetTestSpecificPath("package", "sub_package", "module.py"); + var initPyPath = TestData.GetTestSpecificPath("package", "sub_package", "module", "__init__.py"); + var submoduleUri = TestData.GetTestSpecificUri("package", "sub_package", "module", "submodule.py"); + + var root = TestData.GetTestSpecificRootUri().AbsolutePath; + await CreateServicesAsync(root, PythonVersions.LatestAvailable3X); + var rdt = Services.GetService(); + + rdt.OpenDocument(new Uri(initPyPath), "Y = 6 * 9"); + rdt.OpenDocument(new Uri(modulePath), "X = 42"); + rdt.OpenDocument(submoduleUri, "Z = 0"); + + var doc = rdt.OpenDocument(new Uri(appPath), appCode); + var analysis = await doc.GetAnalysisAsync(-1); + + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var comps = cs.GetCompletions(analysis, new SourceLocation(4, 8)); + comps.Should().HaveLabels("Y").And.NotContainLabels("X"); + + comps = cs.GetCompletions(analysis, new SourceLocation(5, 11)); + comps.Should().HaveLabels("Z"); + } + [TestMethod, Priority(0)] public async Task LoopImports() { var module1Code = @"