diff --git a/src/Analysis/Ast/Impl/Analyzer/ActivityTracker.cs b/src/Analysis/Ast/Impl/Analyzer/ActivityTracker.cs index 2b5b22c83..937634006 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ActivityTracker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ActivityTracker.cs @@ -19,48 +19,21 @@ namespace Microsoft.Python.Analysis.Analyzer { internal static class ActivityTracker { - private static readonly Dictionary _modules = new Dictionary(); + private static readonly HashSet _modules = new HashSet(); private static readonly object _lock = new object(); private static bool _tracking; private static Stopwatch _sw; - private struct AnalysisState { - public int Count; - public bool IsComplete; - } - public static void OnEnqueueModule(string path) { if (string.IsNullOrEmpty(path)) { return; } lock (_lock) { - if (!_modules.TryGetValue(path, out var st)) { - _modules[path] = default; - } else { - st.IsComplete = false; - } - } - } - - public static void OnModuleAnalysisComplete(string path) { - lock (_lock) { - if (_modules.TryGetValue(path, out var st)) { - st.Count++; - st.IsComplete = true; - } + _modules.Add(path); } } - public static bool IsAnalysisComplete { - get { - lock (_lock) { - return _modules.All(m => m.Value.IsComplete); - } - } - } - - public static void StartTracking() { lock (_lock) { if (!_tracking) { @@ -71,22 +44,15 @@ public static void StartTracking() { } } - public static void EndTracking() { + public static (int modulesCount, double totalMilliseconds) EndTracking() { lock (_lock) { if (_tracking) { _sw?.Stop(); _tracking = false; } - } - } - public static int ModuleCount { - get { - lock (_lock) { - return _modules.Count; - } + return (_modules.Count, _sw?.Elapsed.TotalMilliseconds ?? 0); } } - public static double MillisecondsElapsed => _sw?.Elapsed.TotalMilliseconds ?? 0; } } diff --git a/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs b/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs index 0365fad92..d62f11fe2 100644 --- a/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs +++ b/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs @@ -15,31 +15,48 @@ using System; using System.Diagnostics; +using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; namespace Microsoft.Python.Analysis.Analyzer { [DebuggerDisplay("{Name} : {FilePath}")] - internal struct AnalysisModuleKey : IEquatable { + internal readonly struct AnalysisModuleKey : IEquatable { + private enum KeyType { Default, Typeshed, LibraryAsDocument } + + private readonly KeyType _type; public string Name { get; } public string FilePath { get; } - public bool IsTypeshed { get; } + public bool IsTypeshed => _type == KeyType.Typeshed; + public bool IsLibraryAsDocument => _type == KeyType.LibraryAsDocument; public AnalysisModuleKey(IPythonModule module) { Name = module.Name; FilePath = module.ModuleType == ModuleType.CompiledBuiltin ? null : module.FilePath; - IsTypeshed = module is StubPythonModule stub && stub.IsTypeshed; + _type = module is StubPythonModule stub && stub.IsTypeshed + ? KeyType.Typeshed + : module.ModuleType == ModuleType.Library && module is IDocument document && document.IsOpen + ? KeyType.LibraryAsDocument + : KeyType.Default; } public AnalysisModuleKey(string name, string filePath, bool isTypeshed) { Name = name; FilePath = filePath; - IsTypeshed = isTypeshed; + _type = isTypeshed ? KeyType.Typeshed : KeyType.Default; + } + + private AnalysisModuleKey(string name, string filePath, KeyType type) { + Name = name; + FilePath = filePath; + _type = type; } + public AnalysisModuleKey GetLibraryAsDocumentKey() => new AnalysisModuleKey(Name, FilePath, KeyType.LibraryAsDocument); + public bool Equals(AnalysisModuleKey other) - => Name.EqualsOrdinal(other.Name) && FilePath.PathEquals(other.FilePath) && IsTypeshed == other.IsTypeshed; + => Name.EqualsOrdinal(other.Name) && FilePath.PathEquals(other.FilePath) && _type == other._type; public override bool Equals(object obj) => obj is AnalysisModuleKey other && Equals(other); @@ -47,7 +64,7 @@ public override int GetHashCode() { unchecked { var hashCode = (Name != null ? Name.GetHashCode() : 0); hashCode = (hashCode * 397) ^ (FilePath != null ? FilePath.GetPathHashCode() : 0); - hashCode = (hashCode * 397) ^ IsTypeshed.GetHashCode(); + hashCode = (hashCode * 397) ^ _type.GetHashCode(); return hashCode; } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs index bb1665b8d..8d4b9751d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs @@ -26,6 +26,7 @@ internal interface IAnalyzable { /// /// Notifies document that its analysis is now complete. /// - void NotifyAnalysisComplete(int version, ModuleWalker walker, bool isFinalPass); + /// Document analysis + void NotifyAnalysisComplete(IDocumentAnalysis analysis); } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs index 122992414..970c4bb3f 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs @@ -20,6 +20,7 @@ using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core.Collections; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { public interface IPythonAnalyzer { @@ -27,7 +28,7 @@ public interface IPythonAnalyzer { /// /// Schedules module for analysis. Module will be scheduled if version of AST is greater than the one used to get previous analysis /// - void EnqueueDocumentForAnalysis(IPythonModule module, int version); + void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int version); /// /// Schedules module for analysis for its existing AST, but with new dependencies. diff --git a/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs index b073f20b2..f5d836b0c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs +++ b/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs @@ -30,7 +30,7 @@ public EmptyAnalysis(IServiceContainer services, IDocument document) { Document = document ?? throw new ArgumentNullException(nameof(document)); GlobalScope = new EmptyGlobalScope(document); Ast = AstUtilities.MakeEmptyAst(document.Uri); - ExpressionEvaluator = new ExpressionEval(services, document); + ExpressionEvaluator = new ExpressionEval(services, document, Ast); } public IDocument Document { get; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs index d67c9a012..de3a6903c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs @@ -94,7 +94,7 @@ public IMember GetValueFromClassCtor(IPythonClassType cls, CallExpression expr) var init = cls.GetMember(@"__init__"); if (init != null) { using (OpenScope(cls.DeclaringModule, cls.ClassDefinition, out _)) { - var a = new ArgumentSet(init, 0, new PythonInstance(cls), expr, Module, this); + var a = new ArgumentSet(init, 0, new PythonInstance(cls), expr, this); if (a.Errors.Count > 0) { // AddDiagnostics(Module.Uri, a.Errors); } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs index c5baaa8ec..bd5760289 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs @@ -41,7 +41,7 @@ public void DeclareVariable(string name, IMember value, VariableSource source) => DeclareVariable(name, value, source, default(Location)); public void DeclareVariable(string name, IMember value, VariableSource source, IPythonModule module) - => DeclareVariable(name, value, source, new Location(module, default)); + => DeclareVariable(name, value, source, new Location(module)); public void DeclareVariable(string name, IMember value, VariableSource source, Node location, bool overwrite = false) => DeclareVariable(name, value, source, GetLocationOfName(location), overwrite); diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index 55aa4296e..7f878600d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -38,15 +38,12 @@ internal sealed partial class ExpressionEval : IExpressionEvaluator { private readonly object _lock = new object(); private readonly List _diagnostics = new List(); - public ExpressionEval(IServiceContainer services, IPythonModule module) - : this(services, module, new GlobalScope(module)) { } - - public ExpressionEval(IServiceContainer services, IPythonModule module, GlobalScope gs) { + public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAst ast) { Services = services ?? throw new ArgumentNullException(nameof(services)); Module = module ?? throw new ArgumentNullException(nameof(module)); - Ast = module.GetAst(); + Ast = ast ?? throw new ArgumentNullException(nameof(ast)); - GlobalScope = gs; + GlobalScope = new GlobalScope(module, ast); CurrentScope = GlobalScope; DefaultLocation = new Location(module); //Log = services.GetService(); @@ -60,7 +57,7 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, GlobalSc public IPythonType UnknownType => Interpreter.UnknownType; public Location DefaultLocation { get; } - public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(Module) ?? LocationInfo.Empty; + public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(this) ?? LocationInfo.Empty; public Location GetLocationOfName(Node node) { if (node == null || (Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library)) { @@ -103,7 +100,7 @@ public Location GetLocationOfName(Node node) { public IServiceContainer Services { get; } IScope IExpressionEvaluator.CurrentScope => CurrentScope; IGlobalScope IExpressionEvaluator.GlobalScope => GlobalScope; - public LocationInfo GetLocation(Node node) => node?.GetLocation(Module) ?? LocationInfo.Empty; + public LocationInfo GetLocation(Node node) => node?.GetLocation(this) ?? LocationInfo.Empty; public IEnumerable Diagnostics => _diagnostics; public void ReportDiagnostics(Uri documentUri, DiagnosticsEntry entry) { diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs index cba333e73..6c330bfaa 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs @@ -125,8 +125,8 @@ private void SpecializeFuture(FromImportStatement node) { var printNameExpression = node.Names.FirstOrDefault(n => n?.Name == "print_function"); if (printNameExpression != null) { - var fn = new PythonFunctionType("print", new Location(Module, default), null, string.Empty); - var o = new PythonFunctionOverload(fn.Name, new Location(Module, default)); + var fn = new PythonFunctionType("print", new Location(Module), null, string.Empty); + var o = new PythonFunctionOverload(fn.Name, new Location(Module)); var parameters = new List { new ParameterInfo("*values", Interpreter.GetBuiltinType(BuiltinTypeId.Object), ParameterKind.List, null), new ParameterInfo("sep", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.KeywordOnly, null), diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs index f5fe93800..c6660a218 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs @@ -56,19 +56,22 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa // import_module('fob.oar.baz') var importNames = ImmutableArray.Empty; var variableModule = default(PythonVariableModule); + var importedNamesCount = 0; foreach (var nameExpression in moduleImportExpression.Names) { importNames = importNames.Add(nameExpression.Name); var imports = ModuleResolution.CurrentPathResolver.GetImportsFromAbsoluteName(Module.FilePath, importNames, forceAbsolute); if (!HandleImportSearchResult(imports, variableModule, asNameExpression, moduleImportExpression, out variableModule)) { - return; + break; } + + importedNamesCount++; } // "import fob.oar.baz as baz" is handled as baz = import_module('fob.oar.baz') // "import fob.oar.baz" is handled as fob = import_module('fob') - if (!string.IsNullOrEmpty(asNameExpression?.Name)) { + if (!string.IsNullOrEmpty(asNameExpression?.Name) && importedNamesCount == moduleImportExpression.Names.Count) { Eval.DeclareVariable(asNameExpression.Name, variableModule, VariableSource.Import, asNameExpression); - } else if (importNames.Count > 0 && !string.IsNullOrEmpty(importNames[0]) && _variableModules.TryGetValue(importNames[0], out variableModule)) { + } else if (importedNamesCount > 0 && !string.IsNullOrEmpty(importNames[0]) && _variableModules.TryGetValue(importNames[0], out variableModule)) { var firstName = moduleImportExpression.Names[0]; Eval.DeclareVariable(importNames[0], variableModule, VariableSource.Import, firstName); } @@ -87,7 +90,7 @@ private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonV case RelativeImportBeyondTopLevel importBeyondTopLevel: var message = Resources.ErrorRelativeImportBeyondTopLevel.FormatInvariant(importBeyondTopLevel.RelativeImportName); Eval.ReportDiagnostics(Eval.Module.Uri, - new DiagnosticsEntry(message, location.GetLocation(Eval.Module).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis)); + new DiagnosticsEntry(message, location.GetLocation(Eval).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis)); variableModule = default; return false; case ImportNotFound importNotFound: diff --git a/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs index 42caa54da..bf8f7df93 100644 --- a/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs +++ b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs @@ -29,7 +29,7 @@ namespace Microsoft.Python.Analysis.Analyzer { /// Analysis of a library code. /// internal sealed class LibraryAnalysis : IDocumentAnalysis { - public LibraryAnalysis(IDocument document, int version, IServiceContainer services, GlobalScope globalScope, IReadOnlyList starImportMemberNames) { + public LibraryAnalysis(IDocument document, int version, IGlobalScope globalScope, IExpressionEvaluator eval, IReadOnlyList starImportMemberNames) { Check.ArgumentNotNull(nameof(document), document); Check.ArgumentNotNull(nameof(globalScope), globalScope); @@ -37,14 +37,7 @@ public LibraryAnalysis(IDocument document, int version, IServiceContainer servic Version = version; GlobalScope = globalScope; - var ast = Document.GetAst(); - ast.Reduce(x => x is ImportStatement || x is FromImportStatement); - var c = (IAstNodeContainer)Document; - c.ClearContent(); - c.ClearAst(); - c.AddAstNode(document, ast); - - ExpressionEvaluator = new ExpressionEval(services, document, globalScope); + ExpressionEvaluator = eval; StarImportMemberNames = starImportMemberNames; } diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 2463dfc03..dd8a30c06 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -37,8 +37,8 @@ internal class ModuleWalker : AnalysisWalker { private int _allReferencesCount; private bool _allIsUsable = true; - public ModuleWalker(IServiceContainer services, IPythonModule module) - : base(new ExpressionEval(services, module)) { + public ModuleWalker(IServiceContainer services, IPythonModule module, PythonAst ast) + : base(new ExpressionEval(services, module, ast)) { _stubAnalysis = Module.Stub is IDocument doc ? doc.GetAnyAnalysis() : null; } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 0f3c037fc..322f1c8cb 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -17,6 +17,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Caching; @@ -31,6 +32,7 @@ using Microsoft.Python.Core.Disposables; using Microsoft.Python.Core.Logging; using Microsoft.Python.Core.Services; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { public sealed class PythonAnalyzer : IPythonAnalyzer, IDisposable { @@ -68,9 +70,9 @@ public Task WaitForCompleteAnalysisAsync(CancellationToken cancellationToken = d => _analysisCompleteEvent.WaitAsync(cancellationToken); public async Task GetAnalysisAsync(IPythonModule module, int waitTime, CancellationToken cancellationToken) { - var key = new AnalysisModuleKey(module); PythonAnalyzerEntry entry; lock (_syncObj) { + var key = new AnalysisModuleKey(module); if (!_analysisEntries.TryGetValue(key, out entry)) { var emptyAnalysis = new EmptyAnalysis(_services, (IDocument)module); entry = new PythonAnalyzerEntry(emptyAnalysis); @@ -120,11 +122,13 @@ public void InvalidateAnalysis(IPythonModule module) { } public void RemoveAnalysis(IPythonModule module) { + AnalysisModuleKey key; lock (_syncObj) { - _analysisEntries.Remove(new AnalysisModuleKey(module)); + key = new AnalysisModuleKey(module); + _analysisEntries.Remove(key); } - _dependencyResolver.Remove(new AnalysisModuleKey(module)); + _dependencyResolver.Remove(key); } public void EnqueueDocumentForAnalysis(IPythonModule module, ImmutableArray analysisDependencies) { @@ -144,16 +148,28 @@ public void EnqueueDocumentForAnalysis(IPythonModule module, ImmutableArray= bufferVersion) { return; } + + // It is possible that parsing request for the library has been started when document is open, + // but it is closed at the moment of analysis and then become open again. + // In this case, we still need to analyze the document, but using correct entry. + var libraryAsDocumentKey = key.GetLibraryAsDocumentKey(); + if (entry.PreviousAnalysis is LibraryAnalysis && _analysisEntries.TryGetValue(libraryAsDocumentKey, out var documentEntry)) { + key = libraryAsDocumentKey; + entry = documentEntry; + } + } else { entry = new PythonAnalyzerEntry(new EmptyAnalysis(_services, (IDocument)module)); _analysisEntries[key] = entry; @@ -161,7 +177,7 @@ public void EnqueueDocumentForAnalysis(IPythonModule module, int bufferVersion) } } - if (entry.Invalidate(module, bufferVersion, version, out var dependencies)) { + if (entry.Invalidate(module, ast, bufferVersion, version, out var dependencies)) { AnalyzeDocument(key, entry, dependencies); } } @@ -205,12 +221,12 @@ public IReadOnlyList LoadedModules { internal void RaiseAnalysisComplete(int moduleCount, double msElapsed) => AnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs(moduleCount, msElapsed)); - private void AnalyzeDocument(AnalysisModuleKey key, PythonAnalyzerEntry entry, ImmutableArray dependencies) { + private void AnalyzeDocument(in AnalysisModuleKey key, in PythonAnalyzerEntry entry, in ImmutableArray dependencies) { _analysisCompleteEvent.Reset(); ActivityTracker.StartTracking(); _log?.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name}({entry.Module.ModuleType}) queued"); - var graphVersion = _dependencyResolver.ChangeValue(key, entry, entry.IsUserModule, dependencies); + var graphVersion = _dependencyResolver.ChangeValue(key, entry, entry.IsUserOrBuiltin || key.IsLibraryAsDocument, dependencies); lock (_syncObj) { if (_version > graphVersion) { @@ -226,7 +242,7 @@ private void AnalyzeDocument(AnalysisModuleKey key, PythonAnalyzerEntry entry, I } } - private bool TryCreateSession(int graphVersion, PythonAnalyzerEntry entry, out PythonAnalyzerSession session) { + private bool TryCreateSession(in int graphVersion, in PythonAnalyzerEntry entry, out PythonAnalyzerSession session) { var analyzeUserModuleOutOfOrder = false; lock (_syncObj) { if (_currentSession != null) { @@ -244,11 +260,6 @@ private bool TryCreateSession(int graphVersion, PythonAnalyzerEntry entry, out P } if (!_dependencyResolver.TryCreateWalker(graphVersion, 2, out var walker)) { - if (entry.Module.ModuleType == ModuleType.Builtins) { - session = CreateSession(null, entry); - return true; - } - session = null; return false; } @@ -290,7 +301,7 @@ private void StartNextSession(Task task) { session.Start(false); } - private PythonAnalyzerSession CreateSession(IDependencyChainWalker walker, PythonAnalyzerEntry entry) + private PythonAnalyzerSession CreateSession(in IDependencyChainWalker walker, in PythonAnalyzerEntry entry) => new PythonAnalyzerSession(_services, _progress, _analysisCompleteEvent, _startNextSession, _disposeToken.CancellationToken, walker, _version, entry); private void LoadMissingDocuments(IPythonInterpreter interpreter, ImmutableArray missingKeys) { diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs index 9e41c7ef4..cb00d93c7 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs @@ -20,6 +20,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Core.DependencyResolution; +using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; @@ -32,13 +33,13 @@ internal sealed class PythonAnalyzerEntry { private readonly object _syncObj = new object(); private TaskCompletionSource _analysisTcs; private IPythonModule _module; - private bool _isUserModule; + private ModuleType _moduleType; + private PythonAst _ast; private IDocumentAnalysis _previousAnalysis; private HashSet _parserDependencies; private HashSet _analysisDependencies; private int _bufferVersion; private int _analysisVersion; - private int _depth; public IPythonModule Module { get { @@ -48,10 +49,18 @@ public IPythonModule Module { } } + public bool IsUserOrBuiltin { + get { + lock (_syncObj) { + return _moduleType == ModuleType.User || _moduleType == ModuleType.Builtins; + } + } + } + public bool IsUserModule { get { lock (_syncObj) { - return _isUserModule; + return _moduleType == ModuleType.User; } } } @@ -74,21 +83,12 @@ public int AnalysisVersion { } } - public int Depth { - get { - lock (_syncObj) { - return _depth; - } - } - } - public bool NotAnalyzed => PreviousAnalysis is EmptyAnalysis; public PythonAnalyzerEntry(EmptyAnalysis emptyAnalysis) { _previousAnalysis = emptyAnalysis; _module = emptyAnalysis.Document; - _isUserModule = emptyAnalysis.Document.ModuleType == ModuleType.User; - _depth = _isUserModule ? 0 : -1; + _moduleType = _module.ModuleType; _bufferVersion = -1; _analysisVersion = 0; @@ -101,12 +101,18 @@ public Task GetAnalysisAsync(CancellationToken cancellationTo public bool IsValidVersion(int version, out IPythonModule module, out PythonAst ast) { lock (_syncObj) { module = _module; - ast = module.GetAst(); - if (ast == null || module == null) { + ast = _ast; + + if (module == null) { return false; } - return _previousAnalysis is EmptyAnalysis || _isUserModule || _analysisVersion <= version; + if (ast == null) { + Debug.Assert(!(_previousAnalysis is LibraryAnalysis), $"Library module {module.Name} of type {module.ModuleType} has been analyzed already!"); + return false; + } + + return _previousAnalysis is EmptyAnalysis || _moduleType == ModuleType.User || _analysisVersion <= version; } } @@ -120,6 +126,11 @@ public void TrySetAnalysis(IDocumentAnalysis analysis, int version) { return; } + if (analysis is LibraryAnalysis) { + _ast = null; + _parserDependencies = null; + } + _analysisDependencies = null; UpdateAnalysisTcs(version); _previousAnalysis = analysis; @@ -209,20 +220,21 @@ public bool Invalidate(ImmutableArray analysisDependencies, int a } } - public bool Invalidate(IPythonModule module, int bufferVersion, int analysisVersion, out ImmutableArray dependencies) { + public bool Invalidate(IPythonModule module, PythonAst ast, int bufferVersion, int analysisVersion, out ImmutableArray dependencies) { dependencies = ImmutableArray.Empty; if (_bufferVersion >= bufferVersion) { return false; } - var dependenciesHashSet = FindDependencies(module, bufferVersion); + var dependenciesHashSet = FindDependencies(module, ast, bufferVersion); lock (_syncObj) { if (_analysisVersion >= analysisVersion && _bufferVersion >= bufferVersion) { return false; } + _ast = ast; _module = module; - _isUserModule = module.ModuleType == ModuleType.User; + _moduleType = module.ModuleType; _parserDependencies = dependenciesHashSet; Interlocked.Exchange(ref _bufferVersion, bufferVersion); @@ -234,13 +246,13 @@ public bool Invalidate(IPythonModule module, int bufferVersion, int analysisVers } } - private HashSet FindDependencies(IPythonModule module, int bufferVersion) { + private HashSet FindDependencies(IPythonModule module, PythonAst ast, int bufferVersion) { if (_bufferVersion > bufferVersion) { return new HashSet(); } var walker = new DependencyWalker(module); - module.GetAst().Walk(walker); + ast.Walk(walker); var dependencies = walker.Dependencies; dependencies.Remove(new AnalysisModuleKey(module)); return dependencies; @@ -285,8 +297,14 @@ public DependencyWalker(IPythonModule module) { } public override bool Walk(ImportStatement import) { + var forceAbsolute = import.ForceAbsolute; foreach (var moduleName in import.Names) { - HandleSearchResults(_pathResolver.FindImports(_module.FilePath, moduleName, import.ForceAbsolute)); + var importNames = ImmutableArray.Empty; + foreach (var nameExpression in moduleName.Names) { + importNames = importNames.Add(nameExpression.Name); + var imports = _pathResolver.GetImportsFromAbsoluteName(_module.FilePath, importNames, forceAbsolute); + HandleSearchResults(imports); + } } return false; diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index c0e9215f0..e0781764d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -19,13 +19,16 @@ using System.Runtime; using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; +using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; using Microsoft.Python.Core.Logging; using Microsoft.Python.Core.Services; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { internal sealed class PythonAnalyzerSession { @@ -146,9 +149,9 @@ private async Task StartAsync() { if (!isCanceled) { _progress.ReportRemaining(remaining); if (isFinal) { - ActivityTracker.EndTracking(); - (_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(ActivityTracker.ModuleCount, ActivityTracker.MillisecondsElapsed); - _log?.Log(TraceEventType.Verbose, $"Analysis complete: {ActivityTracker.ModuleCount} modules in { ActivityTracker.MillisecondsElapsed} ms."); + var (modulesCount, totalMilliseconds) = ActivityTracker.EndTracking(); + (_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(modulesCount, totalMilliseconds); + _log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms."); } } } @@ -193,7 +196,7 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { if (isCanceled && !node.Value.NotAnalyzed) { remaining++; - node.Skip(); + node.MoveNext(); continue; } @@ -246,25 +249,23 @@ private void Analyze(IDependencyChainNode node, AsyncCountd } _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled."); - node.Skip(); return; } var startTime = stopWatch.Elapsed; - AnalyzeEntry(entry, module, _walker.Version, node.IsComplete); - node.Commit(); - ActivityTracker.OnModuleAnalysisComplete(node.Value.Module.FilePath); + AnalyzeEntry(node, entry, module, ast, _walker.Version); - LogCompleted(module, stopWatch, startTime); + LogCompleted(node, module, stopWatch, startTime); } catch (OperationCanceledException oce) { node.Value.TryCancel(oce, _walker.Version); - node.Skip(); LogCanceled(node.Value.Module); } catch (Exception exception) { node.Value.TrySetException(exception, _walker.Version); - node.Commit(); + node.MarkWalked(); LogException(node.Value.Module, exception); } finally { + node.MoveNext(); + bool isCanceled; lock (_syncObj) { isCanceled = _isCanceled; @@ -293,9 +294,9 @@ private void AnalyzeEntry() { var startTime = stopWatch?.Elapsed ?? TimeSpan.Zero; - AnalyzeEntry(_entry, module, Version, true); + AnalyzeEntry(null, _entry, module, ast, Version); - LogCompleted(module, stopWatch, startTime); + LogCompleted(null, module, stopWatch, startTime); } catch (OperationCanceledException oce) { _entry.TryCancel(oce, Version); LogCanceled(_entry.Module); @@ -308,7 +309,7 @@ private void AnalyzeEntry() { } } - private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int version, bool isFinalPass) { + private void AnalyzeEntry(IDependencyChainNode node, PythonAnalyzerEntry entry, IPythonModule module, PythonAst ast, int version) { if (entry.PreviousAnalysis is LibraryAnalysis) { _log?.Log(TraceEventType.Verbose, $"Request to re-analyze finalized {module.Name}."); } @@ -317,8 +318,7 @@ private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int v var analyzable = module as IAnalyzable; analyzable?.NotifyAnalysisBegins(); - var ast = module.GetAst(); - var walker = new ModuleWalker(_services, module); + var walker = new ModuleWalker(_services, module, ast); ast.Walk(walker); _analyzerCancellationToken.ThrowIfCancellationRequested(); @@ -326,7 +326,18 @@ private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int v walker.Complete(); _analyzerCancellationToken.ThrowIfCancellationRequested(); - analyzable?.NotifyAnalysisComplete(version, walker, isFinalPass); + bool isCanceled; + lock (_syncObj) { + isCanceled = _isCanceled; + } + + if (!isCanceled) { + node?.MarkWalked(); + } + + var analysis = CreateAnalysis(node, (IDocument)module, ast, version, walker, isCanceled); + + analyzable?.NotifyAnalysisComplete(analysis); entry.TrySetAnalysis(module.Analysis, version); if (module.ModuleType == ModuleType.User) { @@ -335,9 +346,13 @@ private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int v } } - private void LogCompleted(IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) { + private void LogCompleted(IDependencyChainNode node, IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) { if (_log != null) { - _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."); + var completed = node != null && module.Analysis is LibraryAnalysis ? "completed" : "completed for library"; + var message = node != null + ? $"Analysis of {module.Name}({module.ModuleType}) on depth {node.VertexDepth} {completed} in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms." + : $"Out of order analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."; + _log.Log(TraceEventType.Verbose, message); } } @@ -353,6 +368,25 @@ private void LogException(IPythonModule module, Exception exception) { } } + private IDocumentAnalysis CreateAnalysis(IDependencyChainNode node, IDocument document, PythonAst ast, int version, ModuleWalker walker, bool isCanceled) { + var createLibraryAnalysis = !isCanceled && + node != null && + !node.HasMissingDependencies && + document.ModuleType == ModuleType.Library && + !document.IsOpen && + node.HasOnlyWalkedDependencies && + node.IsValidVersion; + + if (!createLibraryAnalysis) { + return new DocumentAnalysis(document, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); + } + + ast.Reduce(x => x is ImportStatement || x is FromImportStatement); + document.SetAst(ast); + var eval = new ExpressionEval(walker.Eval.Services, document, ast); + return new LibraryAnalysis(document, version, walker.GlobalScope, eval, walker.StarImportMemberNames); + } + private enum State { NotStarted = 0, Started = 1, diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs b/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs index e7cd7c3f0..58b06e866 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs @@ -50,9 +50,9 @@ private async Task LoadBuiltinTypesAsync(string root, IServiceManager sm, Cancel lock (_lock) { var builtinModule = _moduleResolution.CreateBuiltinsModule(); _builtinTypes[BuiltinTypeId.NoneType] - = new PythonType("NoneType", new Location(builtinModule, default), string.Empty, BuiltinTypeId.NoneType); + = new PythonType("NoneType", new Location(builtinModule), string.Empty, BuiltinTypeId.NoneType); _builtinTypes[BuiltinTypeId.Unknown] - = UnknownType = new PythonType("Unknown", new Location(builtinModule, default), string.Empty); + = UnknownType = new PythonType("Unknown", new Location(builtinModule), string.Empty); } await _moduleResolution.InitializeAsync(cancellationToken); diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index 55611e6cf..342ffcfd2 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -124,7 +124,7 @@ public override bool Walk(ReturnStatement node) { Eval.ReportDiagnostics(Module.Uri, new Diagnostics.DiagnosticsEntry( Resources.ReturnInInit, - node.GetLocation(Module).Span, + node.GetLocation(Eval).Span, ErrorCodes.ReturnInInit, Severity.Warning, DiagnosticSource.Analysis)); diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs index 1d2e76c6b..1644a1db2 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs @@ -31,10 +31,10 @@ internal sealed class DependencyResolver : IDependencyResolver _version; - public int ChangeValue(TKey key, TValue value, bool isRoot, params TKey[] incomingKeys) + public int ChangeValue(in TKey key, in TValue value, in bool isRoot, params TKey[] incomingKeys) => ChangeValue(key, value, isRoot, ImmutableArray.Create(incomingKeys)); - public int ChangeValue(TKey key, TValue value, bool isRoot, ImmutableArray incomingKeys) { + public int ChangeValue(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys) { lock (_syncObj) { if (!_keys.TryGetValue(key, out var index)) { index = _keys.Count; @@ -47,7 +47,7 @@ public int ChangeValue(TKey key, TValue value, bool isRoot, ImmutableArray } } - public int TryAddValue(TKey key, TValue value, bool isRoot, ImmutableArray incomingKeys) { + public int TryAddValue(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys) { lock (_syncObj) { if (!_keys.TryGetValue(key, out var index)) { index = _keys.Count; @@ -62,7 +62,7 @@ public int TryAddValue(TKey key, TValue value, bool isRoot, ImmutableArray } } - public int Remove(TKey key) { + public int Remove(in TKey key) { lock (_syncObj) { if (!_keys.TryGetValue(key, out var index)) { return _version; @@ -71,14 +71,14 @@ public int Remove(TKey key) { var version = Interlocked.Increment(ref _version); var vertex = _vertices[index]; - _vertices[index] = default; - if (vertex == null) { + if (vertex == default) { return version; } - + + _vertices[index] = default; foreach (var incomingIndex in vertex.Incoming) { var incoming = _vertices[incomingIndex]; - if (incoming != null && incoming.IsSealed) { + if (incoming != default && incoming.IsSealed) { _vertices[incomingIndex] = new DependencyVertex(incoming, version, false); } } @@ -89,7 +89,7 @@ public int Remove(TKey key) { foreach (var outgoingIndex in vertex.Outgoing) { var outgoing = _vertices[outgoingIndex]; - if (outgoing != null && !outgoing.IsNew) { + if (outgoing != default && !outgoing.IsNew) { _vertices[outgoingIndex] = new DependencyVertex(outgoing, version, true); } } @@ -100,7 +100,7 @@ public int Remove(TKey key) { public int RemoveKeys(params TKey[] keys) => RemoveKeys(ImmutableArray.Create(keys)); - public int RemoveKeys(ImmutableArray keys) { + public int RemoveKeys(in ImmutableArray keys) { lock (_syncObj) { foreach (var key in keys) { if (_keys.TryGetValue(key, out var index)) { @@ -138,7 +138,7 @@ public int RemoveKeys(ImmutableArray keys) { } } - private void Update(TKey key, TValue value, bool isRoot, ImmutableArray incomingKeys, int index) { + private void Update(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys, in int index) { var version = Interlocked.Increment(ref _version); var incoming = EnsureKeys(index, incomingKeys, version); @@ -147,7 +147,7 @@ private void Update(TKey key, TValue value, bool isRoot, ImmutableArray in _keys[key] = index; } - private ImmutableArray EnsureKeys(int index, ImmutableArray keys, int version) { + private ImmutableArray EnsureKeys(in int index, in ImmutableArray keys, in int version) { var incoming = ImmutableArray.Empty; foreach (var key in keys) { @@ -175,7 +175,7 @@ public IDependencyChainWalker CreateWalker() { } } - public bool TryCreateWalker(int version, int walkerDepthLimit, out IDependencyChainWalker walker) { + public bool TryCreateWalker(in int version, in int walkerDepthLimit, out IDependencyChainWalker walker) { ImmutableArray> vertices; lock (_syncObj) { @@ -210,8 +210,7 @@ public bool TryCreateWalker(int version, int walkerDepthLimit, out IDependencyCh } // Iterate original graph to get starting vertices - var startingVertices = walkingGraph.Where(v => v.IncomingCount == 0); - if (!TryFindMissingDependencies(vertices, walkingGraph, version)) { + if (!TryFindMissingDependencies(vertices, walkingGraph, version, out var missingKeys)) { walker = default; return false; } @@ -222,26 +221,12 @@ public bool TryCreateWalker(int version, int walkerDepthLimit, out IDependencyCh } var affectedValues = walkingGraph.Select(v => v.DependencyVertex.Value); - var missingKeys = ImmutableArray.Empty; - - lock (_syncObj) { - if (version != _version) { - walker = default; - return false; - } - - foreach (var (key, index) in _keys) { - if (vertices[index] == default/* && depths[index] <= walkerDepthLimit*/) { - missingKeys = missingKeys.Add(key); - } - } - } - - walker = new DependencyChainWalker(startingVertices, affectedValues, depths, missingKeys, totalNodesCount, version); - return true; + var startingVertices = walkingGraph.Where(v => !v.HasIncoming); + walker = new DependencyChainWalker(this, startingVertices, affectedValues, depths, missingKeys, totalNodesCount, version); + return version == _version; } - private bool TryBuildReverseGraph(in ImmutableArray> vertices, int version) { + private bool TryBuildReverseGraph(in ImmutableArray> vertices, in int version) { var reverseGraphIsBuilt = true; foreach (var vertex in vertices) { if (vertex != null && !vertex.IsSealed) { @@ -256,7 +241,7 @@ private bool TryBuildReverseGraph(in ImmutableArray[vertices.Count]; foreach (var vertex in vertices) { - if (vertex == null) { + if (vertex == default) { continue; } @@ -283,7 +268,7 @@ private bool TryBuildReverseGraph(in ImmutableArray> vertices, ImmutableArray depths, int version, out ImmutableArray> analysisGraph) { + private bool TryCreateWalkingGraph(in ImmutableArray> vertices, in ImmutableArray depths, in int version, out ImmutableArray> analysisGraph) { if (version != _version) { analysisGraph = default; return false; @@ -326,7 +311,7 @@ private bool TryCreateWalkingGraph(in ImmutableArray vertex, } } - private bool TryFindMissingDependencies(in ImmutableArray> vertices, in ImmutableArray> walkingGraph, int version) { + private bool TryFindMissingDependencies(in ImmutableArray> vertices, in ImmutableArray> walkingGraph, int version, out ImmutableArray missingKeys) { var haveMissingDependencies = new bool[vertices.Count]; var queue = new Queue>(); + var missingIndicesHashSet = new HashSet(); + // First, go through all the vertices and find those that have missing incoming edges foreach (var vertex in vertices) { if (vertex == default) { continue; @@ -554,11 +541,14 @@ private bool TryFindMissingDependencies(in ImmutableArray.Empty; + + // From them, go through the edges and mark all reachable vertices while (queue.Count > 0) { if (version != _version) { return false; @@ -566,6 +556,10 @@ private bool TryFindMissingDependencies(in ImmutableArray { + private readonly DependencyResolver _dependencyResolver; private readonly ImmutableArray> _startingVertices; private readonly ImmutableArray _depths; private readonly object _syncObj; @@ -604,14 +607,16 @@ public int Remaining { } } - public DependencyChainWalker( + public DependencyChainWalker(in DependencyResolver dependencyResolver, in ImmutableArray> startingVertices, in ImmutableArray affectedValues, in ImmutableArray depths, in ImmutableArray missingKeys, in int totalNodesCount, in int version) { + _syncObj = new object(); + _dependencyResolver = dependencyResolver; _startingVertices = startingVertices; _depths = depths; AffectedValues = affectedValues; @@ -638,18 +643,18 @@ public Task> GetNextAsync(CancellationToken cancell return ppc.ConsumeAsync(cancellationToken); } - public void MarkCompleted(WalkingVertex vertex, bool commitChanges) { + public void MoveNext(WalkingVertex vertex) { var verticesToProduce = new List>(); var isCompleted = false; lock (_syncObj) { _remaining--; foreach (var outgoing in vertex.Outgoing) { - if (outgoing.IncomingCount == 0) { + if (!outgoing.HasIncoming) { continue; } - outgoing.DecrementIncoming(); - if (outgoing.IncomingCount > 0) { + outgoing.DecrementIncoming(vertex.HasOnlyWalkedIncoming); + if (outgoing.HasIncoming) { continue; } @@ -661,10 +666,6 @@ public void MarkCompleted(WalkingVertex vertex, bool commitChanges } } - if (commitChanges && vertex.SecondPass == null) { - vertex.DependencyVertex.MarkWalked(); - } - if (isCompleted) { _ppc.Produce(null); } else { @@ -673,15 +674,24 @@ public void MarkCompleted(WalkingVertex vertex, bool commitChanges } } } + + public bool IsValidVersion { + get { + lock (_dependencyResolver._syncObj) { + return _dependencyResolver._version == Version; + } + } + } } private sealed class DependencyChainNode : IDependencyChainNode { private readonly WalkingVertex _vertex; private DependencyChainWalker _walker; public TValue Value => _vertex.DependencyVertex.Value; - public bool HasMissingDependencies => _vertex.HasMissingDependencies; public int VertexDepth { get; } - public bool IsComplete => _vertex.SecondPass == null && !HasMissingDependencies; + public bool HasMissingDependencies => _vertex.HasMissingDependencies; + public bool HasOnlyWalkedDependencies => _vertex.HasOnlyWalkedIncoming && _vertex.SecondPass == default; + public bool IsValidVersion => _walker.IsValidVersion; public DependencyChainNode(DependencyChainWalker walker, WalkingVertex vertex, int depth) { _walker = walker; @@ -689,8 +699,13 @@ public DependencyChainNode(DependencyChainWalker walker, WalkingVertex Interlocked.Exchange(ref _walker, null)?.MarkCompleted(_vertex, true); - public void Skip() => Interlocked.Exchange(ref _walker, null)?.MarkCompleted(_vertex, false); + public void MarkWalked() { + if (_vertex.SecondPass == null) { + _vertex.DependencyVertex.MarkWalked(); + } + } + + public void MoveNext() => Interlocked.Exchange(ref _walker, null)?.MoveNext(_vertex); } } } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs index 03bffead2..ddbf9160d 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.Diagnostics; using System.Threading; diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs index 299235cc6..a4f394d00 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs @@ -18,12 +18,19 @@ internal interface IDependencyChainNode { int VertexDepth { get; } /// - /// Shows if node has any direct or indirect dependencies that aren't added to the graph + /// Returns true if node has any direct or indirect dependencies that aren't added to the graph, otherwise false /// bool HasMissingDependencies { get; } + /// + /// Returns true if node has only direct and indirect dependencies that have been walked at least once, otherwise false + /// + bool HasOnlyWalkedDependencies { get; } + /// + /// Returns true if node version matches version of the walked graph + /// + bool IsValidVersion { get; } TValue Value { get; } - void Commit(); - void Skip(); - bool IsComplete { get; } + void MarkWalked(); + void MoveNext(); } } diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs index a8328bfb9..02fdfbdfd 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs @@ -24,12 +24,12 @@ namespace Microsoft.Python.Analysis.Dependencies { /// concurrently. /// internal interface IDependencyResolver { - int TryAddValue(TKey key, TValue value, bool isRoot, ImmutableArray incomingKeys); - int ChangeValue(TKey key, TValue value, bool isRoot, ImmutableArray incomingKeys); - int Remove(TKey key); - int RemoveKeys(ImmutableArray keys); + int TryAddValue(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys); + int ChangeValue(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys); + int Remove(in TKey key); + int RemoveKeys(in ImmutableArray keys); IDependencyChainWalker CreateWalker(); - bool TryCreateWalker(int version, int walkerDepthLimit, out IDependencyChainWalker walker); + bool TryCreateWalker(in int version, in int walkerDepthLimit, out IDependencyChainWalker walker); } } diff --git a/src/Analysis/Ast/Impl/Dependencies/WalkingVertex.cs b/src/Analysis/Ast/Impl/Dependencies/WalkingVertex.cs index a2ddb90ca..d3debee4f 100644 --- a/src/Analysis/Ast/Impl/Dependencies/WalkingVertex.cs +++ b/src/Analysis/Ast/Impl/Dependencies/WalkingVertex.cs @@ -21,16 +21,19 @@ namespace Microsoft.Python.Analysis.Dependencies { [DebuggerDisplay("{" + nameof(DebuggerDisplay) + ",nq}")] internal sealed class WalkingVertex { - public static Comparison> FirstPassIncomingComparison { get; } = (v1, v2) => v1.FirstPass.IncomingCount.CompareTo(v2.FirstPass.IncomingCount); + public static Comparison> FirstPassIncomingComparison { get; } = (v1, v2) => v1.FirstPass._incomingCount.CompareTo(v2.FirstPass._incomingCount); private readonly List> _outgoing; private bool _isSealed; + private int _incomingCount; + private int _walkedIncomingCount; public DependencyVertex DependencyVertex { get; } public IReadOnlyList> Outgoing => _outgoing; public int Index { get; set; } public int LoopNumber { get; set; } - public int IncomingCount { get; private set; } + public bool HasIncoming => _incomingCount != 0; + public bool HasOnlyWalkedIncoming => _walkedIncomingCount == 0; public bool HasMissingDependencies { get; private set; } public WalkingVertex FirstPass { get; } @@ -57,7 +60,8 @@ public void AddOutgoing(WalkingVertex outgoingVertex) { CheckNotSealed(); _outgoing.Add(outgoingVertex); - outgoingVertex.IncomingCount++; + outgoingVertex._incomingCount++; + outgoingVertex._walkedIncomingCount++; } public void AddOutgoing(HashSet> loop) { @@ -65,7 +69,8 @@ public void AddOutgoing(HashSet> loop) { _outgoing.AddRange(loop); foreach (var outgoingVertex in loop) { - outgoingVertex.IncomingCount++; + outgoingVertex._incomingCount++; + outgoingVertex._walkedIncomingCount++; } } @@ -74,7 +79,8 @@ public void RemoveOutgoingAt(int index) { var outgoingVertex = _outgoing[index]; _outgoing.RemoveAt(index); - outgoingVertex.IncomingCount--; + outgoingVertex._incomingCount--; + outgoingVertex._walkedIncomingCount--; } public WalkingVertex CreateSecondPassVertex() { @@ -86,9 +92,12 @@ public WalkingVertex CreateSecondPassVertex() { public void Seal() => _isSealed = true; - public void DecrementIncoming() { + public void DecrementIncoming(bool isWalkedIncoming) { CheckSealed(); - IncomingCount--; + _incomingCount--; + if (isWalkedIncoming) { + _walkedIncomingCount--; + } } private void CheckSealed() => Check.InvalidOperation(_isSealed); diff --git a/src/Analysis/Ast/Impl/Documents/Definitions/IDocument.cs b/src/Analysis/Ast/Impl/Documents/Definitions/IDocument.cs index e03fdf3b6..dc1228ddb 100644 --- a/src/Analysis/Ast/Impl/Documents/Definitions/IDocument.cs +++ b/src/Analysis/Ast/Impl/Documents/Definitions/IDocument.cs @@ -51,11 +51,6 @@ public interface IDocument: IPythonModule, IDisposable { /// Task GetAnalysisAsync(int waitTime = 200, CancellationToken cancellationToken = default); - /// - /// Returns last known document AST. The AST may be out of date or null. - /// - PythonAst GetAnyAst(); - /// /// Returns last known document analysis. The analysis may be out of date. /// diff --git a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs index 9fc6d389d..a88533762 100644 --- a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs +++ b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs @@ -103,7 +103,6 @@ public IDocument OpenDocument(Uri uri, string content, string filePath = null) { if (justOpened) { Opened?.Invoke(this, new DocumentEventArgs(document)); - _services.GetService().InvalidateAnalysis(document); } return document; @@ -248,8 +247,8 @@ private bool TryAddModulePath(ModuleCreationOptions mco) { private bool TryOpenDocument(DocumentEntry entry, string content) { if (!entry.Document.IsOpen) { - entry.Document.Reset(content); entry.Document.IsOpen = true; + entry.Document.Reset(content); entry.LockCount++; return true; } diff --git a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs index e5ccb7bfb..508ee9641 100644 --- a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs @@ -13,19 +13,32 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis { public static class NodeExtensions { - public static LocationInfo GetLocation(this Node node, IPythonModule module) { + public static LocationInfo GetLocation(this Node node, IExpressionEvaluator eval) { if (node == null || node.StartIndex >= node.EndIndex) { return LocationInfo.Empty; } - var start = node.GetStart(module.GetAst()); - var end = node.GetEnd(module.GetAst()); - return new LocationInfo(module.FilePath, module.Uri, start.Line, start.Column, end.Line, end.Column); + return GetLocation(node, eval.Ast, eval.Module); + } + + public static LocationInfo GetLocation(this Node node, IDocumentAnalysis analysis) { + if (node == null || node.StartIndex >= node.EndIndex) { + return LocationInfo.Empty; + } + + return GetLocation(node, analysis.Ast, analysis.Document); + } + + private static LocationInfo GetLocation(Node node, PythonAst ast, IPythonFile pythonFile) { + var start = node.GetStart(ast); + var end = node.GetEnd(ast); + return new LocationInfo(pythonFile.FilePath, pythonFile.Uri, start.Line, start.Column, end.Line, end.Column); } public static Expression RemoveParenthesis(this Expression e) { diff --git a/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs index 107c5b41d..81442bc96 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs @@ -33,6 +33,8 @@ public static void AddMemberReference(this IPythonType type, string name, IExpre eval.LookupNameInScopes(name, out _, out var v, LookupOptions.Local); v?.AddReference(location); } + } else if (type is IPythonModule module && module.GlobalScope != null && module.GlobalScope.Variables.TryGetVariable(name, out var variable)) { + variable.AddReference(location); } } diff --git a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs index c33b4620f..b2553b65c 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs @@ -20,10 +20,16 @@ namespace Microsoft.Python.Analysis { public static class PythonModuleExtensions { internal static PythonAst GetAst(this IPythonModule module) - => ((IAstNodeContainer)module).GetAstNode(module); + => (PythonAst)((IAstNodeContainer)module).GetAstNode(module); + + internal static void SetAst(this IPythonModule module, PythonAst ast) { + var contained = (IAstNodeContainer)module; + contained.ClearContent(); + contained.AddAstNode(module, ast); + } internal static T GetAstNode(this IPythonModule module, object o) where T : Node - => ((IAstNodeContainer)module).GetAstNode(o); + => (T)((IAstNodeContainer)module).GetAstNode(o); internal static void AddAstNode(this IPythonModule module, object o, Node n) => ((IAstNodeContainer)module).AddAstNode(o, n); } diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ExpressionWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ExpressionWalker.cs index cc33b194e..5cdae2a1b 100644 --- a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ExpressionWalker.cs +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ExpressionWalker.cs @@ -98,7 +98,7 @@ public override bool Walk(NameExpression node) { // y = x // x = 1 var variableDefinitionSpan = v.Definition.Span; - var nameUseLocation = node.GetLocation(analysis.Document); + var nameUseLocation = node.GetLocation(analysis); // Make sure we are in the same scope in order to avoid // def func(): diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index f792b565f..c86634ddf 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -121,7 +121,7 @@ private void SpecializeTypes() { _hiddenNames.Add("__builtin_module_names__"); - var location = new Location(this, default); + var location = new Location(this); if (_boolType != null) { Analysis.GlobalScope.DeclareVariable("True", _boolType, VariableSource.Builtin, location); Analysis.GlobalScope.DeclareVariable("False", _boolType, VariableSource.Builtin, location); diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs index 0ed36cb97..deeb65c63 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs @@ -24,18 +24,13 @@ internal interface IAstNodeContainer { /// Nodes are not available for library modules as AST is not retained /// in libraries in order to conserve memory. /// - T GetAstNode(object o) where T : Node; + Node GetAstNode(object o); /// /// Associated AST node with the object. /// void AddAstNode(object o, Node n); - /// - /// Removes all AST node associations. - /// - void ClearAst(); - void ClearContent(); } } diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 81664250d..2fbe011e5 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -55,6 +55,7 @@ private enum State { private readonly DocumentBuffer _buffer = new DocumentBuffer(); private readonly DisposeToken _disposeToken = DisposeToken.Create(); + private readonly object _syncObj = new object(); private IReadOnlyList _parseErrors = Array.Empty(); private readonly Dictionary _astMap = new Dictionary(); private readonly IDiagnosticsService _diagnosticsService; @@ -68,7 +69,6 @@ private enum State { protected ILogger Log { get; } protected IFileSystem FileSystem { get; } protected IServiceContainer Services { get; } - private object AnalysisLock { get; } = new object(); private State ContentState { get; set; } = State.None; protected PythonModule(string name, ModuleType moduleType, IServiceContainer services) : base(null) { @@ -237,7 +237,7 @@ protected virtual void Dispose(bool disposing) { /// public string Content { get { - lock (AnalysisLock) { + lock (_syncObj) { return _buffer.Text; } } @@ -252,7 +252,7 @@ public string Content { public async Task GetAstAsync(CancellationToken cancellationToken = default) { Task t = null; while (true) { - lock (AnalysisLock) { + lock (_syncObj) { if (t == _parsingTask) { break; } @@ -270,15 +270,13 @@ public async Task GetAstAsync(CancellationToken cancellationToken = d return this.GetAst(); } - public PythonAst GetAnyAst() => GetAstNode(this); - /// /// Provides collection of parsing errors, if any. /// public IEnumerable GetParseErrors() => _parseErrors.ToArray(); public void Update(IEnumerable changes) { - lock (AnalysisLock) { + lock (_syncObj) { _parseCts?.Cancel(); _parseCts = new CancellationTokenSource(); @@ -295,7 +293,7 @@ public void Update(IEnumerable changes) { } public void Reset(string content) { - lock (AnalysisLock) { + lock (_syncObj) { if (content != Content) { ContentState = State.None; InitializeContent(content, _buffer.Version + 1); @@ -323,7 +321,7 @@ private void Parse(CancellationToken cancellationToken) { //Log?.Log(TraceEventType.Verbose, $"Parse begins: {Name}"); - lock (AnalysisLock) { + lock (_syncObj) { version = _buffer.Version; var options = new ParserOptions { StubFile = FilePath != null && Path.GetExtension(FilePath).Equals(".pyi", FileSystem.StringComparison) @@ -339,7 +337,7 @@ private void Parse(CancellationToken cancellationToken) { //Log?.Log(TraceEventType.Verbose, $"Parse complete: {Name}"); - lock (AnalysisLock) { + lock (_syncObj) { cancellationToken.ThrowIfCancellationRequested(); if (version != _buffer.Version) { throw new OperationCanceledException(); @@ -366,10 +364,10 @@ private void Parse(CancellationToken cancellationToken) { ContentState = State.Analyzing; var analyzer = Services.GetService(); - analyzer.EnqueueDocumentForAnalysis(this, version); + analyzer.EnqueueDocumentForAnalysis(this, ast, version); } - lock (AnalysisLock) { + lock (_syncObj) { _parsingTask = null; } } @@ -386,16 +384,7 @@ public override void Add(string message, SourceSpan span, int errorCode, Severit #region IAnalyzable public void NotifyAnalysisBegins() { - lock (AnalysisLock) { - if (Analysis is LibraryAnalysis) { - var sw = Log != null ? Stopwatch.StartNew() : null; - lock (AnalysisLock) { - _astMap[this] = RecreateAst(); - } - sw?.Stop(); - Log?.Log(TraceEventType.Verbose, $"Reloaded AST of {Name} in {sw?.Elapsed.TotalMilliseconds} ms"); - } - + lock (_syncObj) { if (_updated) { _updated = false; // In all variables find those imported, then traverse imported modules @@ -422,14 +411,14 @@ public void NotifyAnalysisBegins() { } } - public void NotifyAnalysisComplete(int version, ModuleWalker walker, bool isFinalPass) { - lock (AnalysisLock) { - if (version < Analysis.Version) { + public void NotifyAnalysisComplete(IDocumentAnalysis analysis) { + lock (_syncObj) { + if (analysis.Version < Analysis.Version) { return; } - Analysis = CreateAnalysis(version, walker, isFinalPass); - GlobalScope = Analysis.GlobalScope; + Analysis = analysis; + GlobalScope = analysis.GlobalScope; // Derived classes can override OnAnalysisComplete if they want // to perform additional actions on the completed analysis such @@ -444,7 +433,7 @@ public void NotifyAnalysisComplete(int version, ModuleWalker walker, bool isFina // Do not report issues with libraries or stubs if (ModuleType == ModuleType.User) { - _diagnosticsService?.Replace(Uri, Analysis.Diagnostics, DiagnosticSource.Analysis); + _diagnosticsService?.Replace(Uri, analysis.Diagnostics, DiagnosticSource.Analysis); } NewAnalysis?.Invoke(this, EventArgs.Empty); @@ -458,30 +447,24 @@ protected virtual void OnAnalysisComplete() { } #endregion #region IAstNodeContainer - public T GetAstNode(object o) where T : Node { - lock (AnalysisLock) { - return _astMap.TryGetValue(o, out var n) ? (T)n : null; + public Node GetAstNode(object o) { + lock (_syncObj) { + return _astMap.TryGetValue(o, out var n) ? n : null; } } public void AddAstNode(object o, Node n) { - lock (AnalysisLock) { + lock (_syncObj) { Debug.Assert(!_astMap.ContainsKey(o) || _astMap[o] == n); _astMap[o] = n; } } - public void ClearAst() { - lock (AnalysisLock) { - if (ModuleType != ModuleType.User) { - _astMap.Clear(); - } - } - } public void ClearContent() { - lock (AnalysisLock) { + lock (_syncObj) { if (ModuleType != ModuleType.User) { _buffer.Reset(_buffer.Version, string.Empty); + _astMap.Clear(); } } } @@ -509,7 +492,7 @@ protected virtual string LoadContent() { } private void InitializeContent(string content, int version) { - lock (AnalysisLock) { + lock (_syncObj) { LoadContent(content, version); var startParse = ContentState < State.Parsing && (_parsingTask == null || version > 0); @@ -587,21 +570,5 @@ private void RemoveReferencesInModule(IPythonModule module) { } } } - - private IDocumentAnalysis CreateAnalysis(int version, ModuleWalker walker, bool isFinalPass) - => ModuleType == ModuleType.Library && isFinalPass - ? new LibraryAnalysis(this, version, walker.Eval.Services, walker.GlobalScope, walker.StarImportMemberNames) - : (IDocumentAnalysis)new DocumentAnalysis(this, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); - - private PythonAst RecreateAst() { - lock (AnalysisLock) { - ContentState = State.None; - LoadContent(null, _buffer.Version); - var parser = Parser.CreateParser(new StringReader(_buffer.Text), Interpreter.LanguageVersion, ParserOptions.Default); - var ast = parser.ParseFile(Uri); - ContentState = State.Parsed; - return ast; - } - } } } diff --git a/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs b/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs index 1b29df047..5cc5a379b 100644 --- a/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs +++ b/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs @@ -67,7 +67,7 @@ public static IMember Next(IPythonModule module, IPythonFunctionOverload overloa } public static IMember __iter__(IPythonInterpreter interpreter, BuiltinTypeId contentTypeId) { - var location = new Location(interpreter.ModuleResolution.BuiltinsModule, default); + var location = new Location(interpreter.ModuleResolution.BuiltinsModule); var fn = new PythonFunctionType(@"__iter__", location, null, string.Empty); var o = new PythonFunctionOverload(fn.Name, location); o.AddReturnValue(PythonTypeIterator.FromTypeId(interpreter, contentTypeId)); diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs index 960f507b2..9977389eb 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs @@ -44,7 +44,7 @@ public static IPythonModule Create(IServiceContainer services) { #endregion private void SpecializeMembers() { - var location = new Location(this, default); + var location = new Location(this); // TypeVar var fn = PythonFunctionType.Specialize("TypeVar", this, GetMemberDocumentation("TypeVar")); @@ -233,7 +233,7 @@ private IPythonType CreateTypeAlias(IArgumentSet args) { eval.ReportDiagnostics( eval.Module?.Uri, new DiagnosticsEntry(Resources.NewTypeFirstArgNotString.FormatInvariant(firstArgType), - expression?.GetLocation(eval.Module)?.Span ?? default, + expression?.GetLocation(eval)?.Span ?? default, Diagnostics.ErrorCodes.TypingNewTypeArguments, Severity.Error, DiagnosticSource.Analysis) ); diff --git a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs index 1cac04fba..180bbaf63 100644 --- a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs +++ b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs @@ -19,7 +19,6 @@ using System.Linq; using System.Runtime.CompilerServices; using Microsoft.Python.Analysis.Analyzer; -using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Extensions; using Microsoft.Python.Analysis.Values; @@ -82,8 +81,6 @@ public ArgumentSet(IReadOnlyList args, Expression expr, IExpressionEval _evaluated = true; } - public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance instance, CallExpression callExpr, ExpressionEval eval) : - this(fn, overloadIndex, instance, callExpr, eval.Module, eval) { } /// /// Creates set of arguments for a function call based on the call expression @@ -95,9 +92,8 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in /// Function overload to call. /// Type instance the function is bound to. For derived classes it is different from the declared type. /// Call expression that invokes the function. - /// Module that contains the call expression. /// Evaluator that can calculate values of arguments from their respective expressions. - public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance instance, CallExpression callExpr, IPythonModule module, IExpressionEvaluator eval) { + public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance instance, CallExpression callExpr, IExpressionEvaluator eval) { Eval = eval; OverloadIndex = overloadIndex; DeclaringModule = fn.DeclaringModule; @@ -129,7 +125,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in return; } - var callLocation = callExpr.GetLocation(module); + var callLocation = callExpr.GetLocation(eval); // https://www.python.org/dev/peps/pep-3102/#id5 // For each formal parameter, there is a slot which will be used to contain @@ -180,7 +176,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in if (formalParamIndex >= overload.Parameters.Count) { // We ran out of formal parameters and yet haven't seen // any sequence or dictionary ones. This looks like an error. - _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyFunctionArguments, arg.GetLocation(module).Span, + _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyFunctionArguments, arg.GetLocation(eval).Span, ErrorCodes.TooManyFunctionArguments, Severity.Warning, DiagnosticSource.Analysis)); return; } @@ -189,7 +185,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in if (formalParam.Kind == ParameterKind.List) { if (string.IsNullOrEmpty(formalParam.Name)) { // If the next unfilled slot is a vararg slot, and it does not have a name, then it is an error. - _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyPositionalArgumentBeforeStar, arg.GetLocation(module).Span, + _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyPositionalArgumentBeforeStar, arg.GetLocation(eval).Span, ErrorCodes.TooManyPositionalArgumentsBeforeStar, Severity.Warning, DiagnosticSource.Analysis)); return; } @@ -197,7 +193,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in // If the next unfilled slot is a vararg slot then all remaining // non-keyword arguments are placed into the vararg slot. if (_listArgument == null) { - _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyFunctionArguments, arg.GetLocation(module).Span, + _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyFunctionArguments, arg.GetLocation(eval).Span, ErrorCodes.TooManyFunctionArguments, Severity.Warning, DiagnosticSource.Analysis)); return; } @@ -217,7 +213,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in if (formalParam.Kind == ParameterKind.Dictionary) { // Next slot is a dictionary slot, but we have positional arguments still. - _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyPositionalArgumentBeforeStar, arg.GetLocation(module).Span, + _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyPositionalArgumentBeforeStar, arg.GetLocation(eval).Span, ErrorCodes.TooManyPositionalArgumentsBeforeStar, Severity.Warning, DiagnosticSource.Analysis)); return; } @@ -231,7 +227,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in var arg = callExpr.Args[callParamIndex]; if (string.IsNullOrEmpty(arg.Name)) { - _errors.Add(new DiagnosticsEntry(Resources.Analysis_PositionalArgumentAfterKeyword, arg.GetLocation(module).Span, + _errors.Add(new DiagnosticsEntry(Resources.Analysis_PositionalArgumentAfterKeyword, arg.GetLocation(eval).Span, ErrorCodes.PositionalArgumentAfterKeyword, Severity.Warning, DiagnosticSource.Analysis)); return; } @@ -243,13 +239,13 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in // to the dictionary using the keyword name as the dictionary key, // unless there is already an entry with that key, in which case it is an error. if (_dictArgument == null) { - _errors.Add(new DiagnosticsEntry(Resources.Analysis_UnknownParameterName, arg.GetLocation(module).Span, + _errors.Add(new DiagnosticsEntry(Resources.Analysis_UnknownParameterName, arg.GetLocation(eval).Span, ErrorCodes.UnknownParameterName, Severity.Warning, DiagnosticSource.Analysis)); return; } if (_dictArgument.Arguments.ContainsKey(arg.Name)) { - _errors.Add(new DiagnosticsEntry(Resources.Analysis_ParameterAlreadySpecified.FormatUI(arg.Name), arg.GetLocation(module).Span, + _errors.Add(new DiagnosticsEntry(Resources.Analysis_ParameterAlreadySpecified.FormatUI(arg.Name), arg.GetLocation(eval).Span, ErrorCodes.ParameterAlreadySpecified, Severity.Warning, DiagnosticSource.Analysis)); return; } @@ -260,7 +256,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in if (nvp.ValueExpression != null || nvp.Value != null) { // Slot is already filled. - _errors.Add(new DiagnosticsEntry(Resources.Analysis_ParameterAlreadySpecified.FormatUI(arg.Name), arg.GetLocation(module).Span, + _errors.Add(new DiagnosticsEntry(Resources.Analysis_ParameterAlreadySpecified.FormatUI(arg.Name), arg.GetLocation(eval).Span, ErrorCodes.ParameterAlreadySpecified, Severity.Warning, DiagnosticSource.Analysis)); return; } diff --git a/src/Analysis/Ast/Impl/Types/LocatedMember.cs b/src/Analysis/Ast/Impl/Types/LocatedMember.cs index d752e3f4f..36899d865 100644 --- a/src/Analysis/Ast/Impl/Types/LocatedMember.cs +++ b/src/Analysis/Ast/Impl/Types/LocatedMember.cs @@ -23,7 +23,7 @@ namespace Microsoft.Python.Analysis.Types { internal abstract class LocatedMember : ILocatedMember { private HashSet _references; - protected LocatedMember(IPythonModule module) : this(new Location(module, default)) { } + protected LocatedMember(IPythonModule module) : this(new Location(module)) { } protected LocatedMember(Location location) { Location = location; diff --git a/src/Analysis/Ast/Impl/Types/Location.cs b/src/Analysis/Ast/Impl/Types/Location.cs index fd2275707..2485d307d 100644 --- a/src/Analysis/Ast/Impl/Types/Location.cs +++ b/src/Analysis/Ast/Impl/Types/Location.cs @@ -17,7 +17,7 @@ using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Types { - public struct Location { + public readonly struct Location { public Location(IPythonModule module) : this(module, default) { } public Location(IPythonModule module, IndexSpan indexSpan) { diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs index 82c1def63..80670e93c 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs @@ -33,7 +33,7 @@ internal sealed class PythonFunctionType : PythonType, IPythonFunctionType { /// Creates function for specializations /// public static PythonFunctionType Specialize(string name, IPythonModule declaringModule, string documentation) - => new PythonFunctionType(name, new Location(declaringModule, default), documentation, true); + => new PythonFunctionType(name, new Location(declaringModule), documentation, true); private PythonFunctionType(string name, Location location, string documentation, bool isSpecialized = false) : base(name, location, documentation ?? string.Empty, BuiltinTypeId.Function) { diff --git a/src/Analysis/Ast/Impl/Values/GlobalScope.cs b/src/Analysis/Ast/Impl/Values/GlobalScope.cs index 699809d20..44f35bfd1 100644 --- a/src/Analysis/Ast/Impl/Values/GlobalScope.cs +++ b/src/Analysis/Ast/Impl/Values/GlobalScope.cs @@ -19,18 +19,21 @@ namespace Microsoft.Python.Analysis.Values { internal sealed class GlobalScope: Scope, IGlobalScope { - public GlobalScope(IPythonModule module): base(null, null, module) { + private readonly PythonAst _ast; + + public GlobalScope(IPythonModule module, PythonAst ast): base(null, null, module) { + _ast = ast; DeclareBuiltinVariables(); } - public override ScopeStatement Node => Module.GetAst(); + public override ScopeStatement Node => _ast; private void DeclareBuiltinVariables() { if (Module.ModuleType != ModuleType.User) { return; } - var location = new Location(Module, default); + var location = new Location(Module); var boolType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Bool); var strType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Str); diff --git a/src/Analysis/Ast/Impl/Values/Scope.cs b/src/Analysis/Ast/Impl/Values/Scope.cs index adbca97fc..0ffeba6c5 100644 --- a/src/Analysis/Ast/Impl/Values/Scope.cs +++ b/src/Analysis/Ast/Impl/Values/Scope.cs @@ -96,7 +96,7 @@ private void DeclareBuiltinVariables() { return; } - var location = new Location(Module, default); + var location = new Location(Module); var strType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Str); var objType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Object); diff --git a/src/Analysis/Ast/Test/ArgumentSetTests.cs b/src/Analysis/Ast/Test/ArgumentSetTests.cs index 2f4bee636..e0a3eb541 100644 --- a/src/Analysis/Ast/Test/ArgumentSetTests.cs +++ b/src/Analysis/Ast/Test/ArgumentSetTests.cs @@ -370,14 +370,14 @@ private async Task GetArgSetAsync(string code, string funcName = "f var analysis = await GetAnalysisAsync(code); var f = analysis.Should().HaveFunction(funcName).Which; var call = GetCall(analysis.Ast); - return new ArgumentSet(f, 0, null, call, analysis.Document, analysis.ExpressionEvaluator); + return new ArgumentSet(f, 0, null, call, analysis.ExpressionEvaluator); } private async Task GetUnboundArgSetAsync(string code, string funcName = "f") { var analysis = await GetAnalysisAsync(code); var f = analysis.Should().HaveVariable(funcName).Which; var call = GetCall(analysis.Ast); - return new ArgumentSet(f.Value.GetPythonType(), 0, null, call, analysis.Document, analysis.ExpressionEvaluator); + return new ArgumentSet(f.Value.GetPythonType(), 0, null, call, analysis.ExpressionEvaluator); } private async Task GetClassArgSetAsync(string code, string className = "A", string funcName = "f") { @@ -385,7 +385,7 @@ private async Task GetClassArgSetAsync(string code, string classNam var cls = analysis.Should().HaveClass(className).Which; var f = cls.Should().HaveMethod(funcName).Which; var call = GetCall(analysis.Ast); - return new ArgumentSet(f, 0, new PythonInstance(cls), call, analysis.Document, analysis.ExpressionEvaluator); + return new ArgumentSet(f, 0, new PythonInstance(cls), call, analysis.ExpressionEvaluator); } private CallExpression GetCall(PythonAst ast) { diff --git a/src/Analysis/Ast/Test/ClassesTests.cs b/src/Analysis/Ast/Test/ClassesTests.cs index 57400a6a1..19e6639d2 100644 --- a/src/Analysis/Ast/Test/ClassesTests.cs +++ b/src/Analysis/Ast/Test/ClassesTests.cs @@ -89,7 +89,7 @@ public async Task Mro() { var o = interpreter.GetBuiltinType(BuiltinTypeId.Object); var m = new SentinelModule("test", s); - var location = new Location(m, default); + var location = new Location(m); var O = new PythonClassType("O", location); var A = new PythonClassType("A", location); var B = new PythonClassType("B", location); diff --git a/src/Analysis/Ast/Test/DependencyResolverTests.cs b/src/Analysis/Ast/Test/DependencyResolverTests.cs index a9c476b66..8b5adae65 100644 --- a/src/Analysis/Ast/Test/DependencyResolverTests.cs +++ b/src/Analysis/Ast/Test/DependencyResolverTests.cs @@ -44,8 +44,8 @@ public void TestInitialize() [DataRow("A:C|C:B|B:A|D:AF|F:CE|E:BD", "ABCABCDEFDEF", "F")] [DataRow("A:BC|B:AC|C:BA|D:BC", "ACBACBD", "D")] [DataRow("A|B|C|D:AB|E:BC", "[ABC][DE]", "D|E")] - [DataRow("A:CE|B:A|C:B|D:BC|E|F:C", "[CE]ABCAB[DF]", "F")] - [DataRow("A:D|B:E|C:F|D:E|E:F|F:D", "DFEDFE[ABC]", "A")] + [DataRow("A:CE|B:A|C:B|D:BC|E|F:C", "[CE]ABCAB[DF]", "D|F")] + [DataRow("A:D|B:E|C:F|D:E|E:F|F:D", "DFEDFE[ABC]", "A|B|C")] // ReSharper restore StringLiteralTypo [DataTestMethod] public void ChangeValue(string input, string output, string root) { @@ -71,7 +71,8 @@ public void ChangeValue(string input, string output, string root) { foreach (var task in tasks) { result.Append(task.Result.Value[0]); - task.Result.Commit(); + task.Result.MarkWalked(); + task.Result.MoveNext(); } if (tasks.Count > 1) { @@ -87,7 +88,7 @@ public void ChangeValue(string input, string output, string root) { } [TestMethod] - public async Task ChangeValue_RepeatedChange() { + public async Task ChangeValue_ChangeToIdentical() { var resolver = new DependencyResolver(); resolver.ChangeValue("A", "A:B", true, "B"); resolver.ChangeValue("B", "B:C", false, "C"); @@ -98,7 +99,9 @@ public async Task ChangeValue_RepeatedChange() { while (walker.Remaining > 0) { var node = await walker.GetNextAsync(default); result.Append(node.Value[0]); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); } result.ToString().Should().Be("CBA"); @@ -110,14 +113,16 @@ public async Task ChangeValue_RepeatedChange() { while (walker.Remaining > 0) { var node = await walker.GetNextAsync(default); result.Append(node.Value[0]); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); } result.ToString().Should().Be("BA"); } [TestMethod] - public async Task ChangeValue_RepeatedChange2() { + public async Task ChangeValue_TwoChanges() { var resolver = new DependencyResolver(); resolver.ChangeValue("A", "A:B", true, "B"); resolver.ChangeValue("B", "B", true); @@ -129,7 +134,9 @@ public async Task ChangeValue_RepeatedChange2() { while (walker.Remaining > 0) { var node = await walker.GetNextAsync(default); result.Append(node.Value[0]); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); } result.ToString().Should().Be("BDAC"); @@ -142,7 +149,9 @@ public async Task ChangeValue_RepeatedChange2() { while (walker.Remaining > 0) { var node = await walker.GetNextAsync(default); result.Append(node.Value[0]); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); } result.ToString().Should().Be("DCBA"); @@ -159,12 +168,16 @@ public async Task ChangeValue_MissingKeys() { var result = new StringBuilder(); var node = await walker.GetNextAsync(default); result.Append(node.Value[0]); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + node = await walker.GetNextAsync(default); result.Append(node.Value[0]); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + walker.MissingKeys.Should().Equal("D"); result.ToString().Should().Be("BC"); @@ -189,10 +202,13 @@ public async Task ChangeValue_Add() { var node = await walker.GetNextAsync(default); node.Value.Should().Be("A:BD"); node.HasMissingDependencies.Should().BeTrue(); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); walker.Remaining.Should().Be(0); - + + // Add B resolver.ChangeValue("B", "B", false); walker = resolver.CreateWalker(); walker.MissingKeys.Should().Equal("D"); @@ -200,15 +216,20 @@ public async Task ChangeValue_Add() { node = await walker.GetNextAsync(default); node.Value.Should().Be("B"); node.HasMissingDependencies.Should().BeFalse(); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + node = await walker.GetNextAsync(default); node.Value.Should().Be("A:BD"); node.HasMissingDependencies.Should().BeTrue(); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + walker.Remaining.Should().Be(0); + // Add D resolver.ChangeValue("D", "D:C", false, "C"); walker = resolver.CreateWalker(); walker.MissingKeys.Should().BeEmpty(); @@ -216,18 +237,104 @@ public async Task ChangeValue_Add() { node = await walker.GetNextAsync(default); node.Value.Should().Be("C"); node.HasMissingDependencies.Should().BeFalse(); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); node = await walker.GetNextAsync(default); node.Value.Should().Be("D:C"); node.HasMissingDependencies.Should().BeFalse(); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + node = await walker.GetNextAsync(default); node.Value.Should().Be("A:BD"); node.HasMissingDependencies.Should().BeFalse(); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + walker.Remaining.Should().Be(0); + } + + [TestMethod] + public async Task ChangeValue_Add_ParallelWalkers() { + var resolver = new DependencyResolver(); + resolver.ChangeValue("A", "A:BD", true, "B", "D"); + resolver.ChangeValue("B", "B:C", false, "C"); + resolver.ChangeValue("C", "C", false); + + var walker = resolver.CreateWalker(); + walker.MissingKeys.Should().Equal("D"); + + var node = await walker.GetNextAsync(default); + node.Value.Should().Be("C"); + node.HasMissingDependencies.Should().BeFalse(); + node.IsValidVersion.Should().BeTrue(); + + // Add D + resolver.ChangeValue("D", "D:C", false, "C"); + var newWalker = resolver.CreateWalker(); + newWalker.MissingKeys.Should().BeEmpty(); + + // MarkWalked node from old walker + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("B:C"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("A:BD"); + node.HasMissingDependencies.Should().BeTrue(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + walker.Remaining.Should().Be(0); + + // Walk new walker + node = await newWalker.GetNextAsync(default); + node.Value.Should().Be("C"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await newWalker.GetNextAsync(default); + node.Value.Should().Be("B:C"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await newWalker.GetNextAsync(default); + node.Value.Should().Be("D:C"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await newWalker.GetNextAsync(default); + node.Value.Should().Be("A:BD"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + walker.Remaining.Should().Be(0); } @@ -242,15 +349,21 @@ public async Task ChangeValue_Remove() { walker.MissingKeys.Should().BeEmpty(); var node = await walker.GetNextAsync(default); node.Value.Should().Be("C"); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); node = await walker.GetNextAsync(default); node.Value.Should().Be("B:C"); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + node = await walker.GetNextAsync(default); node.Value.Should().Be("A:BC"); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); resolver.Remove("B"); walker = resolver.CreateWalker(); @@ -258,7 +371,106 @@ public async Task ChangeValue_Remove() { node = await walker.GetNextAsync(default); node.Value.Should().Be("A:BC"); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + walker.Remaining.Should().Be(0); + } + + [TestMethod] + public async Task ChangeValue_ChangeChangeRemove() { + var resolver = new DependencyResolver(); + resolver.ChangeValue("A", "A:B", true, "B"); + resolver.ChangeValue("B", "B:C", true, "C"); + resolver.ChangeValue("C", "C:AD", true, "A", "D"); + + var walker = resolver.CreateWalker(); + walker.MissingKeys.Should().Equal("D"); + walker.AffectedValues.Should().Equal("A:B", "B:C", "C:AD"); + walker.Remaining.Should().Be(6); + + //resolver.ChangeValue("D", "D:B", true, "B"); + resolver.ChangeValue("A", "A", true); + resolver.ChangeValue("B", "B", true); + resolver.Remove("B"); + + walker = resolver.CreateWalker(); + walker.MissingKeys.Should().Equal("D"); + walker.Remaining.Should().Be(2); + + var node = await walker.GetNextAsync(default); + node.Value.Should().Be("A"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("C:AD"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + walker.Remaining.Should().Be(0); + } + + [TestMethod] + public async Task ChangeValue_RemoveFromLoop() { + var resolver = new DependencyResolver(); + resolver.ChangeValue("A", "A:B", true, "B"); + resolver.ChangeValue("B", "B:C", false, "C"); + resolver.ChangeValue("C", "C:A", false, "A"); + + var walker = resolver.CreateWalker(); + walker.MissingKeys.Should().BeEmpty(); + + var node = await walker.GetNextAsync(default); + node.Value.Should().Be("A:B"); + node.HasOnlyWalkedDependencies.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("C:A"); + node.HasOnlyWalkedDependencies.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("B:C"); + node.HasOnlyWalkedDependencies.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("A:B"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("C:A"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("B:C"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + walker.Remaining.Should().Be(0); + + resolver.Remove("B"); + walker = resolver.CreateWalker(); + walker.MissingKeys.Should().Equal("B"); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("A:B"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); walker.Remaining.Should().Be(0); } @@ -275,19 +487,27 @@ public async Task ChangeValue_RemoveKeys() { walker.MissingKeys.Should().BeEmpty(); var node = await walker.GetNextAsync(default); node.Value.Should().Be("D"); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + node = await walker.GetNextAsync(default); node.Value.Should().Be("C:D"); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + node = await walker.GetNextAsync(default); node.Value.Should().Be("B:C"); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + node = await walker.GetNextAsync(default); node.Value.Should().Be("A:BC"); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); resolver.RemoveKeys("B", "D"); walker = resolver.CreateWalker(); @@ -295,11 +515,15 @@ public async Task ChangeValue_RemoveKeys() { node = await walker.GetNextAsync(default); node.Value.Should().Be("C:D"); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); node = await walker.GetNextAsync(default); node.Value.Should().Be("A:BC"); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); walker.Remaining.Should().Be(0); } @@ -316,11 +540,11 @@ public async Task ChangeValue_Skip() { var result = new StringBuilder(); var node = await walker.GetNextAsync(default); result.Append(node.Value[0]); - node.Skip(); + node.MoveNext(); node = await walker.GetNextAsync(default); result.Append(node.Value[0]); - node.Skip(); + node.MoveNext(); result.ToString().Should().Be("BD"); diff --git a/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs b/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs index 85a3274c7..132d06ff1 100644 --- a/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs +++ b/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs @@ -18,9 +18,6 @@ namespace Microsoft.Python.Analysis.Core.DependencyResolution { public static class AstUtilities { - public static IImportSearchResult FindImports(this PathResolverSnapshot pathResolver, string modulePath, ModuleName importName, bool forceAbsolute) - => pathResolver.GetImportsFromAbsoluteName(modulePath, importName.Names.Select(n => n.Name), forceAbsolute); - public static IImportSearchResult FindImports(this PathResolverSnapshot pathResolver, string modulePath, FromImportStatement fromImportStatement) { var rootNames = fromImportStatement.Root.Names.Select(n => n.Name); return fromImportStatement.Root is RelativeModuleName relativeName diff --git a/src/LanguageServer/Impl/Implementation/Server.Symbols.cs b/src/LanguageServer/Impl/Implementation/Server.Symbols.cs index 09a000228..f9318f06e 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Symbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Symbols.cs @@ -36,7 +36,7 @@ public async Task HierarchicalDocumentSymbol(DocumentSymbolPar var path = @params.textDocument.uri.AbsolutePath; var symbols = await _indexManager.HierarchicalDocumentSymbolsAsync(path, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); - return symbols.Select(hSym => MakeDocumentSymbol(hSym)).ToArray(); + return symbols.Select(MakeDocumentSymbol).ToArray(); } private static SymbolInformation MakeSymbolInfo(FlatSymbol s) { diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index 277267d09..2096e26fc 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -119,8 +119,7 @@ public void Dispose() { } } - private async Task> IndexAsync(IDocument doc, - CancellationToken indexCt) { + private async Task> IndexAsync(IDocument doc, CancellationToken indexCt) { var ast = await doc.GetAstAsync(indexCt); indexCt.ThrowIfCancellationRequested(); var walker = new SymbolIndexWalker(ast); diff --git a/src/LanguageServer/Impl/LanguageServer.cs b/src/LanguageServer/Impl/LanguageServer.cs index cb172f7c1..f61e951ce 100644 --- a/src/LanguageServer/Impl/LanguageServer.cs +++ b/src/LanguageServer/Impl/LanguageServer.cs @@ -432,7 +432,7 @@ private async Task ConsumerLoop() { return item.Task; } - private struct QueueItem { + private readonly struct QueueItem { private readonly TaskCompletionSource _tcs; public Task Task => _tcs.Task; public bool IsAwaitable { get; } diff --git a/src/LanguageServer/Test/ImportsTests.cs b/src/LanguageServer/Test/ImportsTests.cs index bc9d10265..1df363597 100644 --- a/src/LanguageServer/Test/ImportsTests.cs +++ b/src/LanguageServer/Test/ImportsTests.cs @@ -229,8 +229,8 @@ public async Task UncSearchPaths() { [TestMethod, Priority(0)] public async Task UserSearchPathsInsideWorkspace() { - var folder1 = TestData.GetTestSpecificPath("folder1"); - var folder2 = TestData.GetTestSpecificPath("folder2"); + var folder2 = TestData.GetTestSpecificPath("src"); + var folder1 = TestData.GetTestSpecificPath("src", "virtualenv"); var packageInFolder1 = Path.Combine(folder1, "package"); var packageInFolder2 = Path.Combine(folder2, "package"); var module1Path = Path.Combine(packageInFolder1, "module1.py"); diff --git a/src/LanguageServer/Test/ReferencesTests.cs b/src/LanguageServer/Test/ReferencesTests.cs index cd53d92f4..405808b5c 100644 --- a/src/LanguageServer/Test/ReferencesTests.cs +++ b/src/LanguageServer/Test/ReferencesTests.cs @@ -61,49 +61,79 @@ def func(x): } [TestMethod, Priority(0)] - public async Task TwoOpenFiles() { + public async Task FindAllReferences_MultipleOpenFiles() { const string code1 = @" x = 1 def func(x): return x +class cls: + f = 3 + +c = cls() y = func(x) x = 2 "; - var code2 = $@" -from module1 import x -y = x -"; + var code2 = @" +from module1 import x, c +y = x, +f = c.f"; + + var code3 = @" +from package import module2 as m +from package.module2 import x +a = m.x +b = m.y +c = x"; + var uri1 = await TestData.CreateTestSpecificFileAsync("module1.py", code1); - var uri2 = await TestData.CreateTestSpecificFileAsync("module2.py", code2); + var uri2 = await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "module2.py"), code2); + var uri3 = await TestData.CreateTestSpecificFileAsync("module3.py", code3); - await CreateServicesAsync(PythonVersions.LatestAvailable3X, uri1.AbsolutePath); + await CreateServicesAsync(PythonVersions.LatestAvailable3X); var rdt = Services.GetService(); - rdt.OpenDocument(uri1, code1); - rdt.OpenDocument(uri2, code2); + var doc1 = rdt.OpenDocument(uri1, code1); + var doc2 = rdt.OpenDocument(uri2, code2); + var doc3 = rdt.OpenDocument(uri3, code3); - var doc1 = rdt.GetDocument(uri1); - var analysis = await GetDocumentAnalysisAsync(doc1); + await doc1.GetAnalysisAsync(); + await doc2.GetAnalysisAsync(); + await doc3.GetAnalysisAsync(); var rs = new ReferenceSource(Services); - var refs = await rs.FindAllReferencesAsync(analysis.Document.Uri, new SourceLocation(7, 10), ReferenceSearchOptions.All); + var refs = await rs.FindAllReferencesAsync(uri1, new SourceLocation(11, 10), ReferenceSearchOptions.All); - refs.Should().HaveCount(5); + refs.Should().HaveCount(8); refs[0].range.Should().Be(1, 0, 1, 1); refs[0].uri.Should().Be(uri1); - refs[1].range.Should().Be(6, 9, 6, 10); + refs[1].range.Should().Be(10, 9, 10, 10); refs[1].uri.Should().Be(uri1); - refs[2].range.Should().Be(7, 0, 7, 1); + refs[2].range.Should().Be(11, 0, 11, 1); refs[2].uri.Should().Be(uri1); - refs[3].range.Should().Be(1, 20, 1, 21); - refs[3].uri.Should().Be(uri2); - refs[4].range.Should().Be(2, 4, 2, 5); - refs[4].uri.Should().Be(uri2); + refs[3].range.Should().Be(2, 28, 2, 29); + refs[3].uri.Should().Be(uri3); + refs[4].range.Should().Be(3, 6, 3, 7); + refs[4].uri.Should().Be(uri3); + refs[5].range.Should().Be(5, 4, 5, 5); + refs[5].uri.Should().Be(uri3); + + refs[6].range.Should().Be(1, 20, 1, 21); + refs[6].uri.Should().Be(uri2); + refs[7].range.Should().Be(2, 4, 2, 5); + refs[7].uri.Should().Be(uri2); + + refs = await rs.FindAllReferencesAsync(uri1, new SourceLocation(8, 5), ReferenceSearchOptions.All); + refs.Should().HaveCount(2); + + refs[0].range.Should().Be(7, 4, 7, 5); + refs[0].uri.Should().Be(uri1); + refs[1].range.Should().Be(3, 6, 3, 7); + refs[1].uri.Should().Be(uri2); } [TestMethod, Priority(0)] diff --git a/src/LanguageServer/Test/RenameTests.cs b/src/LanguageServer/Test/RenameTests.cs index a69d5755c..6b6bcb2f5 100644 --- a/src/LanguageServer/Test/RenameTests.cs +++ b/src/LanguageServer/Test/RenameTests.cs @@ -149,6 +149,62 @@ from module import x wse.changes[uri3][1].range.Should().Be(2, 4, 2, 5); } + [TestMethod, Priority(0)] + public async Task Rename_ImportInitPy() { + const string packageInitCode = @"x = 1"; + const string moduleCode = @"import package +y = package.x"; + var initPyUri = TestData.GetTestSpecificUri("package", "__init__.py"); + var moduleUri = TestData.GetTestSpecificUri("module.py"); + + await CreateServicesAsync(PythonVersions.LatestAvailable3X); + var rdt = Services.GetService(); + var initPy = rdt.OpenDocument(initPyUri, packageInitCode); + var module = rdt.OpenDocument(moduleUri, moduleCode); + + await initPy.GetAnalysisAsync(); + await module.GetAnalysisAsync(); + + var rs = new RenameSource(Services); + var wse = await rs.RenameAsync(initPyUri, new SourceLocation(1, 1), "z"); + + wse.changes.Should().HaveCount(2); + + wse.changes[initPyUri].Should().HaveCount(1); + wse.changes[initPyUri][0].range.Should().Be(0, 0, 0, 1); + + wse.changes[moduleUri].Should().HaveCount(1); + wse.changes[moduleUri][0].range.Should().Be(1, 12, 1, 13); + } + + [TestMethod, Priority(0)] + public async Task Rename_ImportSubmodule() { + const string packageInitCode = @"x = 1"; + const string moduleCode = @"import package.submodule +y = package.x"; + var initPyUri = TestData.GetTestSpecificUri("package", "__init__.py"); + var moduleUri = TestData.GetTestSpecificUri("module.py"); + + await CreateServicesAsync(PythonVersions.LatestAvailable3X); + var rdt = Services.GetService(); + var initPy = rdt.OpenDocument(initPyUri, packageInitCode); + var module = rdt.OpenDocument(moduleUri, moduleCode); + + await initPy.GetAnalysisAsync(); + await module.GetAnalysisAsync(); + + var rs = new RenameSource(Services); + var wse = await rs.RenameAsync(initPyUri, new SourceLocation(1, 1), "z"); + + wse.changes.Should().HaveCount(2); + + wse.changes[initPyUri].Should().HaveCount(1); + wse.changes[initPyUri][0].range.Should().Be(0, 0, 0, 1); + + wse.changes[moduleUri].Should().HaveCount(1); + wse.changes[moduleUri][0].range.Should().Be(1, 12, 1, 13); + } + [TestMethod, Priority(0)] public async Task NoRenameInCompiled() { const string code = "from sys import path";